From cc864071cce5518122dfdd1ca74f74530a1c797e Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 31 Mar 2020 14:20:17 -0700
Subject: [PATCH 001/481] rls: Manually implement equality of key builder maps.
 (#3476)

cmp.Equal is not supposed to be used in production code because of its
propensity towards panicking.

This equality will eventually be checked from the LB policy when it gets
a service config update.
---
 balancer/rls/internal/keys/builder.go      |  64 ++++++-
 balancer/rls/internal/keys/builder_test.go | 200 ++++++++++++++++++++-
 2 files changed, 259 insertions(+), 5 deletions(-)

diff --git a/balancer/rls/internal/keys/builder.go b/balancer/rls/internal/keys/builder.go
index a606155f4c9d..abc076ab4da3 100644
--- a/balancer/rls/internal/keys/builder.go
+++ b/balancer/rls/internal/keys/builder.go
@@ -27,7 +27,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/google/go-cmp/cmp"
 	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
 	"google.golang.org/grpc/metadata"
 )
@@ -123,9 +122,25 @@ func (bm BuilderMap) RLSKey(md metadata.MD, path string) KeyMap {
 	return b.keys(md)
 }
 
-// BuilderMapEqual returns true if the provided BuilderMap objects are equal.
-func BuilderMapEqual(a, b BuilderMap) bool {
-	return cmp.Equal(a, b, cmp.AllowUnexported(builder{}, matcher{}))
+// Equal reports whether bm and am represent equivalent BuilderMaps.
+func (bm BuilderMap) Equal(am BuilderMap) bool {
+	if (bm == nil) != (am == nil) {
+		return false
+	}
+	if len(bm) != len(am) {
+		return false
+	}
+
+	for key, bBuilder := range bm {
+		aBuilder, ok := am[key]
+		if !ok {
+			return false
+		}
+		if !bBuilder.Equal(aBuilder) {
+			return false
+		}
+	}
+	return true
 }
 
 // builder provides the actual functionality of building RLS keys. These are
@@ -142,6 +157,28 @@ type builder struct {
 	matchers []matcher
 }
 
+// Equal reports whether b and a represent equivalent key builders.
+func (b builder) Equal(a builder) bool {
+	if (b.matchers == nil) != (a.matchers == nil) {
+		return false
+	}
+	if len(b.matchers) != len(a.matchers) {
+		return false
+	}
+	// Protobuf serialization maintains the order of repeated fields. Matchers
+	// are specified as a repeated field inside the KeyBuilder proto. If the
+	// order changes, it means that the order in the protobuf changed. We report
+	// this case as not being equal even though the builders could possible be
+	// functionally equal.
+	for i, bMatcher := range b.matchers {
+		aMatcher := a.matchers[i]
+		if !bMatcher.Equal(aMatcher) {
+			return false
+		}
+	}
+	return true
+}
+
 // matcher helps extract a key from request headers based on a given name.
 type matcher struct {
 	// The key used in the keyMap sent as part of the RLS request.
@@ -150,6 +187,25 @@ type matcher struct {
 	names []string
 }
 
+// Equal reports if m and are are equivalent matchers.
+func (m matcher) Equal(a matcher) bool {
+	if m.key != a.key {
+		return false
+	}
+	if (m.names == nil) != (a.names == nil) {
+		return false
+	}
+	if len(m.names) != len(a.names) {
+		return false
+	}
+	for i := 0; i < len(m.names); i++ {
+		if m.names[i] != a.names[i] {
+			return false
+		}
+	}
+	return true
+}
+
 func (b builder) keys(md metadata.MD) KeyMap {
 	kvMap := make(map[string]string)
 	for _, m := range b.matchers {
diff --git a/balancer/rls/internal/keys/builder_test.go b/balancer/rls/internal/keys/builder_test.go
index 09a392ea1810..f5e03ac51dd7 100644
--- a/balancer/rls/internal/keys/builder_test.go
+++ b/balancer/rls/internal/keys/builder_test.go
@@ -85,7 +85,7 @@ func TestMakeBuilderMap(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			builderMap, err := MakeBuilderMap(test.cfg)
-			if err != nil || !BuilderMapEqual(builderMap, test.wantBuilderMap) {
+			if err != nil || !builderMap.Equal(test.wantBuilderMap) {
 				t.Errorf("MakeBuilderMap(%+v) returned {%v, %v}, want: {%v, nil}", test.cfg, builderMap, err, test.wantBuilderMap)
 			}
 		})
@@ -330,3 +330,201 @@ func TestMapToString(t *testing.T) {
 		})
 	}
 }
+
+func TestBuilderMapEqual(t *testing.T) {
+	tests := []struct {
+		desc      string
+		a         BuilderMap
+		b         BuilderMap
+		wantEqual bool
+	}{
+		{
+			desc:      "nil builder maps",
+			a:         nil,
+			b:         nil,
+			wantEqual: true,
+		},
+		{
+			desc:      "empty builder maps",
+			a:         make(map[string]builder),
+			b:         make(map[string]builder),
+			wantEqual: true,
+		},
+		{
+			desc:      "nil and non-nil builder maps",
+			a:         nil,
+			b:         map[string]builder{"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "empty and non-empty builder maps",
+			a:         make(map[string]builder),
+			b:         map[string]builder{"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}},
+			wantEqual: false,
+		},
+		{
+			desc: "different number of map keys",
+			a: map[string]builder{
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			b: map[string]builder{
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: false,
+		},
+		{
+			desc: "different map keys",
+			a: map[string]builder{
+				"/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			b: map[string]builder{
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: false,
+		},
+		{
+			desc: "equal keys different values",
+			a: map[string]builder{
+				"/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1", "n2"}}}},
+			},
+			b: map[string]builder{
+				"/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: false,
+		},
+		{
+			desc: "good match",
+			a: map[string]builder{
+				"/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			b: map[string]builder{
+				"/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual {
+				t.Errorf("BuilderMap.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual)
+			}
+		})
+	}
+}
+
+func TestBuilderEqual(t *testing.T) {
+	tests := []struct {
+		desc      string
+		a         builder
+		b         builder
+		wantEqual bool
+	}{
+		{
+			desc:      "nil builders",
+			a:         builder{matchers: nil},
+			b:         builder{matchers: nil},
+			wantEqual: true,
+		},
+		{
+			desc:      "empty builders",
+			a:         builder{matchers: []matcher{}},
+			b:         builder{matchers: []matcher{}},
+			wantEqual: true,
+		},
+		{
+			desc:      "nil and non-nil builders",
+			a:         builder{matchers: nil},
+			b:         builder{matchers: []matcher{}},
+			wantEqual: false,
+		},
+		{
+			desc:      "empty and non-empty builders",
+			a:         builder{matchers: []matcher{}},
+			b:         builder{matchers: []matcher{{key: "foo"}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "different number of matchers",
+			a:         builder{matchers: []matcher{{key: "foo"}, {key: "bar"}}},
+			b:         builder{matchers: []matcher{{key: "foo"}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "equal number but differing matchers",
+			a:         builder{matchers: []matcher{{key: "bar"}}},
+			b:         builder{matchers: []matcher{{key: "foo"}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "good match",
+			a:         builder{matchers: []matcher{{key: "foo"}}},
+			b:         builder{matchers: []matcher{{key: "foo"}}},
+			wantEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			t.Run(test.desc, func(t *testing.T) {
+				if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual {
+					t.Errorf("builder.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual)
+				}
+			})
+		})
+	}
+}
+
+// matcher helps extract a key from request headers based on a given name.
+func TestMatcherEqual(t *testing.T) {
+	tests := []struct {
+		desc      string
+		a         matcher
+		b         matcher
+		wantEqual bool
+	}{
+		{
+			desc:      "different keys",
+			a:         matcher{key: "foo"},
+			b:         matcher{key: "bar"},
+			wantEqual: false,
+		},
+		{
+			desc:      "different number of names",
+			a:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			b:         matcher{key: "foo", names: []string{"v1"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "equal number but differing names",
+			a:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			b:         matcher{key: "foo", names: []string{"v1", "v22"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "same names in different order",
+			a:         matcher{key: "foo", names: []string{"v2", "v1"}},
+			b:         matcher{key: "foo", names: []string{"v1", "v3"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "good match",
+			a:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			b:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			wantEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual {
+				t.Errorf("matcher.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual)
+			}
+		})
+	}
+}

From 508cf42036bc2cb62ad32861a57adaa780556edf Mon Sep 17 00:00:00 2001
From: James Xu <jamesxuhaozhe@gmail.com>
Date: Thu, 2 Apr 2020 12:19:11 +0800
Subject: [PATCH 002/481] cleanup: fix typo in grpclib_remote_balancer.go
 (#3486)

---
 balancer/grpclb/grpclb_remote_balancer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index a46732ac8e92..8f47b019eb2d 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -41,7 +41,7 @@ import (
 	"google.golang.org/grpc/resolver"
 )
 
-// processServerList updates balaner's internal state, create/remove SubConns
+// processServerList updates balancer's internal state, create/remove SubConns
 // and regenerates picker using the received serverList.
 func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
 	if grpclog.V(2) {

From fe1d8e71817f6ab365b9ca133f1b668cbe09a7d3 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 2 Apr 2020 16:55:06 -0700
Subject: [PATCH 003/481] xds: add temporary logging to LRS (#3490)

---
 xds/internal/balancer/lrs/lrs.go         | 14 ++++++++------
 xds/internal/client/client_loadreport.go |  2 ++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/xds/internal/balancer/lrs/lrs.go b/xds/internal/balancer/lrs/lrs.go
index 8aafcc435047..811cd913a2eb 100644
--- a/xds/internal/balancer/lrs/lrs.go
+++ b/xds/internal/balancer/lrs/lrs.go
@@ -313,9 +313,10 @@ func (ls *lrsStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterNa
 			grpclog.Warningf("lrs: failed to create stream: %v", err)
 			continue
 		}
-		if err := stream.Send(&lrspb.LoadStatsRequest{
-			Node: node,
-		}); err != nil {
+		grpclog.Infof("lrs: created LRS stream")
+		req := &lrspb.LoadStatsRequest{Node: node}
+		grpclog.Infof("lrs: sending init LoadStatsRequest: %v", req)
+		if err := stream.Send(req); err != nil {
 			grpclog.Warningf("lrs: failed to send first request: %v", err)
 			continue
 		}
@@ -324,6 +325,7 @@ func (ls *lrsStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterNa
 			grpclog.Warningf("lrs: failed to receive first response: %v", err)
 			continue
 		}
+		grpclog.Infof("lrs: received first LoadStatsResponse: %+v", first)
 		interval, err := ptypes.Duration(first.LoadReportingInterval)
 		if err != nil {
 			grpclog.Warningf("lrs: failed to convert report interval: %v", err)
@@ -366,9 +368,9 @@ func (ls *lrsStore) sendLoads(ctx context.Context, stream lrsgrpc.LoadReportingS
 		case <-ctx.Done():
 			return
 		}
-		if err := stream.Send(&lrspb.LoadStatsRequest{
-			ClusterStats: ls.buildStats(clusterName),
-		}); err != nil {
+		req := &lrspb.LoadStatsRequest{ClusterStats: ls.buildStats(clusterName)}
+		grpclog.Infof("lrs: sending LRS loads: %+v", req)
+		if err := stream.Send(req); err != nil {
 			grpclog.Warningf("lrs: failed to send report: %v", err)
 			return
 		}
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go
index 36b7d5df97a5..42766f997163 100644
--- a/xds/internal/client/client_loadreport.go
+++ b/xds/internal/client/client_loadreport.go
@@ -44,9 +44,11 @@ func (c *Client) ReportLoad(server string, clusterName string, loadStore lrs.Sto
 		cc      *grpc.ClientConn
 		closeCC bool
 	)
+	c.logger.Infof("Starting load report to server: %s", server)
 	if server == "" || server == c.opts.Config.BalancerName {
 		cc = c.cc
 	} else {
+		c.logger.Infof("LRS server is different from xDS server, starting a new ClientConn")
 		dopts := append([]grpc.DialOption{c.opts.Config.Creds}, c.opts.DialOpts...)
 		ccNew, err := grpc.Dial(server, dopts...)
 		if err != nil {

From aedb1362d52a4832ced7751316e356d5577ed0b1 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 3 Apr 2020 09:47:59 -0700
Subject: [PATCH 004/481] balancer: change roundrobin to accept empty address
 list (#3491)

Roundrobin will remove all SubConns. The ClientConn will set SubConn state change to shutdown, and the overall state will turn transient failure.
---
 balancer/base/balancer.go                     |  12 +-
 balancer/roundrobin/roundrobin_test.go        |  25 +---
 .../edsbalancer/eds_impl_priority_test.go     | 125 +++++++++++++++++-
 3 files changed, 136 insertions(+), 26 deletions(-)

diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go
index 0ac26b4805a4..80559b80ace6 100644
--- a/balancer/base/balancer.go
+++ b/balancer/base/balancer.go
@@ -113,10 +113,6 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	if grpclog.V(2) {
 		grpclog.Infoln("base.baseBalancer: got new ClientConn state: ", s)
 	}
-	if len(s.ResolverState.Addresses) == 0 {
-		b.ResolverError(errors.New("produced zero addresses"))
-		return balancer.ErrBadResolverState
-	}
 	// Successful resolution; clear resolver error and ensure we return nil.
 	b.resolverErr = nil
 	// addrsSet is the set converted from addrs, it's used for quick lookup of an address.
@@ -144,6 +140,14 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 			// The entry will be deleted in HandleSubConnStateChange.
 		}
 	}
+	// If resolver state contains no addresses, return an error so ClientConn
+	// will trigger re-resolve. Also records this as an resolver error, so when
+	// the overall state turns transient failure, the error message will have
+	// the zero address information.
+	if len(s.ResolverState.Addresses) == 0 {
+		b.ResolverError(errors.New("produced zero addresses"))
+		return balancer.ErrBadResolverState
+	}
 	return nil
 }
 
diff --git a/balancer/roundrobin/roundrobin_test.go b/balancer/roundrobin/roundrobin_test.go
index 55fa83a0504a..c5d066ed94c8 100644
--- a/balancer/roundrobin/roundrobin_test.go
+++ b/balancer/roundrobin/roundrobin_test.go
@@ -216,31 +216,20 @@ func (s) TestAddressesRemoved(t *testing.T) {
 	}
 
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{}})
-	// Removing addresses results in an error reported to the clientconn, but
-	// the existing connections remain.  RPCs should still succeed.
-	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	if _, err := testc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
-		t.Fatalf("EmptyCall() = _, %v, want _, <nil>", err)
-	}
 
-	// Stop the server to bring the channel state into transient failure.
-	test.cleanup()
-	// Wait for not-ready.
-	for src := cc.GetState(); src == connectivity.Ready; src = cc.GetState() {
-		if !cc.WaitForStateChange(ctx, src) {
-			t.Fatalf("timed out waiting for state change.  got %v; want !%v", src, connectivity.Ready)
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	defer cancel2()
+	// Wait for state to change to transient failure.
+	for src := cc.GetState(); src != connectivity.TransientFailure; src = cc.GetState() {
+		if !cc.WaitForStateChange(ctx2, src) {
+			t.Fatalf("timed out waiting for state change.  got %v; want %v", src, connectivity.TransientFailure)
 		}
 	}
-	// Report an empty server list again; because the state is not ready, the
-	// empty address list error should surface to the user.
-	r.UpdateState(resolver.State{Addresses: []resolver.Address{}})
 
 	const msgWant = "produced zero addresses"
-	if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || !strings.Contains(status.Convert(err).Message(), msgWant) {
+	if _, err := testc.EmptyCall(ctx2, &testpb.Empty{}); err == nil || !strings.Contains(status.Convert(err).Message(), msgWant) {
 		t.Fatalf("EmptyCall() = _, %v, want _, Contains(Message(), %q)", err, msgWant)
 	}
-
 }
 
 func (s) TestCloseWithPendingRPC(t *testing.T) {
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index 3760d438cff5..cdb33300a00f 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -21,6 +21,7 @@ import (
 	"testing"
 	"time"
 
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
@@ -56,7 +57,6 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	p1 := <-cc.newPickerCh
 	want := []balancer.SubConn{sc1}
 	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
-		// t.Fatalf("want %v, got %v", want, err)
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -123,7 +123,6 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	p0 := <-cc.newPickerCh
 	want := []balancer.SubConn{sc0}
 	if err := isRoundRobin(want, subConnFromPicker(p0)); err != nil {
-		// t.Fatalf("want %v, got %v", want, err)
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -426,7 +425,6 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	p0 := <-cc.newPickerCh
 	want := []balancer.SubConn{sc0}
 	if err := isRoundRobin(want, subConnFromPicker(p0)); err != nil {
-		// t.Fatalf("want %v, got %v", want, err)
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -445,7 +443,6 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	p1 := <-cc.newPickerCh
 	want = []balancer.SubConn{sc1}
 	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
-		// t.Fatalf("want %v, got %v", want, err)
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -657,3 +654,123 @@ func (s) TestPriorityType(t *testing.T) {
 		t.Errorf("want p1 to be equal to priority with value 1, got p1==1: %v", got)
 	}
 }
+
+// Test the case where the high priority contains no backends. The low priority
+// will be used.
+func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
+	cc := newTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+
+	// Two localities, with priorities [0, 1], each with one backend.
+	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
+	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+
+	addrs1 := <-cc.newSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.newSubConnCh
+
+	// p0 is ready.
+	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.newPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Remove addresses from priority 0, should use p1.
+	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2.AddLocality(testSubZones[0], 1, 0, nil, nil)
+	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
+	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+
+	// p0 will remove the subconn, and ClientConn will send a sc update to
+	// shutdown.
+	scToRemove := <-cc.removeSubConnCh
+	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+
+	addrs2 := <-cc.newSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.newSubConnCh
+
+	// p1 is ready.
+	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+
+	// Test roundrobin with only p1 subconns.
+	p2 := <-cc.newPickerCh
+	want = []balancer.SubConn{sc2}
+	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+}
+
+// Test the case where the high priority contains no healthy backends. The low
+// priority will be used.
+func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
+	cc := newTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+
+	// Two localities, with priorities [0, 1], each with one backend.
+	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
+	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+
+	addrs1 := <-cc.newSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.newSubConnCh
+
+	// p0 is ready.
+	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.newPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Set priority 0 endpoints to all unhealthy, should use p1.
+	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], &xdsclient.AddLocalityOptions{
+		Health: []corepb.HealthStatus{corepb.HealthStatus_UNHEALTHY},
+	})
+	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
+	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+
+	// p0 will remove the subconn, and ClientConn will send a sc update to
+	// transient failure.
+	scToRemove := <-cc.removeSubConnCh
+	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+
+	addrs2 := <-cc.newSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.newSubConnCh
+
+	// p1 is ready.
+	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+
+	// Test roundrobin with only p1 subconns.
+	p2 := <-cc.newPickerCh
+	want = []balancer.SubConn{sc2}
+	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+}

From 66e9dfe70338d04075ff8617548660f0934e50d8 Mon Sep 17 00:00:00 2001
From: Malte Isberner <2822367+misberner@users.noreply.github.com>
Date: Fri, 3 Apr 2020 21:39:25 +0200
Subject: [PATCH 005/481] transport: fix handling of header metadata in
 serverHandler (#3484)

---
 internal/transport/handler_server.go      |  68 ++++++++++-----
 internal/transport/handler_server_test.go | 102 +++++++++++++++++-----
 2 files changed, 127 insertions(+), 43 deletions(-)

diff --git a/internal/transport/handler_server.go b/internal/transport/handler_server.go
index 49b5d24545c9..fc44e976195d 100644
--- a/internal/transport/handler_server.go
+++ b/internal/transport/handler_server.go
@@ -112,11 +112,10 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats sta
 // at this point to be speaking over HTTP/2, so it's able to speak valid
 // gRPC.
 type serverHandlerTransport struct {
-	rw               http.ResponseWriter
-	req              *http.Request
-	timeoutSet       bool
-	timeout          time.Duration
-	didCommonHeaders bool
+	rw         http.ResponseWriter
+	req        *http.Request
+	timeoutSet bool
+	timeout    time.Duration
 
 	headerMD metadata.MD
 
@@ -186,8 +185,11 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) erro
 	ht.writeStatusMu.Lock()
 	defer ht.writeStatusMu.Unlock()
 
+	headersWritten := s.updateHeaderSent()
 	err := ht.do(func() {
-		ht.writeCommonHeaders(s)
+		if !headersWritten {
+			ht.writePendingHeaders(s)
+		}
 
 		// And flush, in case no header or body has been sent yet.
 		// This forces a separation of headers and trailers if this is the
@@ -238,14 +240,16 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) erro
 	return err
 }
 
+// writePendingHeaders sets common and custom headers on the first
+// write call (Write, WriteHeader, or WriteStatus)
+func (ht *serverHandlerTransport) writePendingHeaders(s *Stream) {
+	ht.writeCommonHeaders(s)
+	ht.writeCustomHeaders(s)
+}
+
 // writeCommonHeaders sets common headers on the first write
 // call (Write, WriteHeader, or WriteStatus).
 func (ht *serverHandlerTransport) writeCommonHeaders(s *Stream) {
-	if ht.didCommonHeaders {
-		return
-	}
-	ht.didCommonHeaders = true
-
 	h := ht.rw.Header()
 	h["Date"] = nil // suppress Date to make tests happy; TODO: restore
 	h.Set("Content-Type", ht.contentType)
@@ -264,9 +268,30 @@ func (ht *serverHandlerTransport) writeCommonHeaders(s *Stream) {
 	}
 }
 
+// writeCustomHeaders sets custom headers set on the stream via SetHeader
+// on the first write call (Write, WriteHeader, or WriteStatus).
+func (ht *serverHandlerTransport) writeCustomHeaders(s *Stream) {
+	h := ht.rw.Header()
+
+	s.hdrMu.Lock()
+	for k, vv := range s.header {
+		if isReservedHeader(k) {
+			continue
+		}
+		for _, v := range vv {
+			h.Add(k, encodeMetadataHeader(k, v))
+		}
+	}
+
+	s.hdrMu.Unlock()
+}
+
 func (ht *serverHandlerTransport) Write(s *Stream, hdr []byte, data []byte, opts *Options) error {
+	headersWritten := s.updateHeaderSent()
 	return ht.do(func() {
-		ht.writeCommonHeaders(s)
+		if !headersWritten {
+			ht.writePendingHeaders(s)
+		}
 		ht.rw.Write(hdr)
 		ht.rw.Write(data)
 		ht.rw.(http.Flusher).Flush()
@@ -274,19 +299,16 @@ func (ht *serverHandlerTransport) Write(s *Stream, hdr []byte, data []byte, opts
 }
 
 func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error {
+	if err := s.SetHeader(md); err != nil {
+		return err
+	}
+
+	headersWritten := s.updateHeaderSent()
 	err := ht.do(func() {
-		ht.writeCommonHeaders(s)
-		h := ht.rw.Header()
-		for k, vv := range md {
-			// Clients don't tolerate reading restricted headers after some non restricted ones were sent.
-			if isReservedHeader(k) {
-				continue
-			}
-			for _, v := range vv {
-				v = encodeMetadataHeader(k, v)
-				h.Add(k, v)
-			}
+		if !headersWritten {
+			ht.writePendingHeaders(s)
 		}
+
 		ht.rw.WriteHeader(200)
 		ht.rw.(http.Flusher).Flush()
 	})
diff --git a/internal/transport/handler_server_test.go b/internal/transport/handler_server_test.go
index 55bc0b88248e..f9efdfb0716d 100644
--- a/internal/transport/handler_server_test.go
+++ b/internal/transport/handler_server_test.go
@@ -269,6 +269,32 @@ func (s) TestHandlerTransport_HandleStreams(t *testing.T) {
 		if want := "/service/foo.bar"; s.method != want {
 			t.Errorf("stream method = %q; want %q", s.method, want)
 		}
+
+		err := s.SetHeader(metadata.Pairs("custom-header", "Custom header value"))
+		if err != nil {
+			t.Error(err)
+		}
+		err = s.SetTrailer(metadata.Pairs("custom-trailer", "Custom trailer value"))
+		if err != nil {
+			t.Error(err)
+		}
+
+		md := metadata.Pairs("custom-header", "Another custom header value")
+		err = s.SendHeader(md)
+		delete(md, "custom-header")
+		if err != nil {
+			t.Error(err)
+		}
+
+		err = s.SetHeader(metadata.Pairs("too-late", "Header value that should be ignored"))
+		if err == nil {
+			t.Error("expected SetHeader call after SendHeader to fail")
+		}
+		err = s.SendHeader(metadata.Pairs("too-late", "This header value should be ignored as well"))
+		if err == nil {
+			t.Error("expected second SendHeader call to fail")
+		}
+
 		st.bodyw.Close() // no body
 		st.ht.WriteStatus(s, status.New(codes.OK, ""))
 	}
@@ -277,14 +303,16 @@ func (s) TestHandlerTransport_HandleStreams(t *testing.T) {
 		func(ctx context.Context, method string) context.Context { return ctx },
 	)
 	wantHeader := http.Header{
-		"Date":         nil,
-		"Content-Type": {"application/grpc"},
-		"Trailer":      {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"},
-		"Grpc-Status":  {"0"},
+		"Date":          {},
+		"Content-Type":  {"application/grpc"},
+		"Trailer":       {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"},
+		"Custom-Header": {"Custom header value", "Another custom header value"},
 	}
-	if !reflect.DeepEqual(st.rw.HeaderMap, wantHeader) {
-		t.Errorf("Header+Trailer Map: %#v; want %#v", st.rw.HeaderMap, wantHeader)
+	wantTrailer := http.Header{
+		"Grpc-Status":    {"0"},
+		"Custom-Trailer": {"Custom trailer value"},
 	}
+	checkHeaderAndTrailer(t, st.rw, wantHeader, wantTrailer)
 }
 
 // Tests that codes.Unimplemented will close the body, per comment in handler_server.go.
@@ -308,16 +336,15 @@ func handleStreamCloseBodyTest(t *testing.T, statusCode codes.Code, msg string)
 		func(ctx context.Context, method string) context.Context { return ctx },
 	)
 	wantHeader := http.Header{
-		"Date":         nil,
+		"Date":         {},
 		"Content-Type": {"application/grpc"},
 		"Trailer":      {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"},
+	}
+	wantTrailer := http.Header{
 		"Grpc-Status":  {fmt.Sprint(uint32(statusCode))},
 		"Grpc-Message": {encodeGrpcMessage(msg)},
 	}
-
-	if !reflect.DeepEqual(st.rw.HeaderMap, wantHeader) {
-		t.Errorf("Header+Trailer mismatch.\n got: %#v\nwant: %#v", st.rw.HeaderMap, wantHeader)
-	}
+	checkHeaderAndTrailer(t, st.rw, wantHeader, wantTrailer)
 }
 
 func (s) TestHandlerTransport_HandleStreams_Timeout(t *testing.T) {
@@ -360,15 +387,15 @@ func (s) TestHandlerTransport_HandleStreams_Timeout(t *testing.T) {
 		func(ctx context.Context, method string) context.Context { return ctx },
 	)
 	wantHeader := http.Header{
-		"Date":         nil,
+		"Date":         {},
 		"Content-Type": {"application/grpc"},
 		"Trailer":      {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"},
+	}
+	wantTrailer := http.Header{
 		"Grpc-Status":  {"4"},
 		"Grpc-Message": {encodeGrpcMessage("too slow")},
 	}
-	if !reflect.DeepEqual(rw.HeaderMap, wantHeader) {
-		t.Errorf("Header+Trailer Map mismatch.\n got: %#v\nwant: %#v", rw.HeaderMap, wantHeader)
-	}
+	checkHeaderAndTrailer(t, rw, wantHeader, wantTrailer)
 }
 
 // TestHandlerTransport_HandleStreams_MultiWriteStatus ensures that
@@ -447,15 +474,50 @@ func (s) TestHandlerTransport_HandleStreams_ErrDetails(t *testing.T) {
 		func(ctx context.Context, method string) context.Context { return ctx },
 	)
 	wantHeader := http.Header{
-		"Date":                    nil,
-		"Content-Type":            {"application/grpc"},
-		"Trailer":                 {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"},
+		"Date":         {},
+		"Content-Type": {"application/grpc"},
+		"Trailer":      {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"},
+	}
+	wantTrailer := http.Header{
 		"Grpc-Status":             {fmt.Sprint(uint32(statusCode))},
 		"Grpc-Message":            {encodeGrpcMessage(msg)},
 		"Grpc-Status-Details-Bin": {encodeBinHeader(stBytes)},
 	}
 
-	if !reflect.DeepEqual(hst.rw.HeaderMap, wantHeader) {
-		t.Errorf("Header+Trailer mismatch.\n got: %#v\nwant: %#v", hst.rw.HeaderMap, wantHeader)
+	checkHeaderAndTrailer(t, hst.rw, wantHeader, wantTrailer)
+}
+
+// checkHeaderAndTrailer checks that the resulting header and trailer matches the expectation.
+func checkHeaderAndTrailer(t *testing.T, rw testHandlerResponseWriter, wantHeader, wantTrailer http.Header) {
+	// For trailer-only responses, the trailer values might be reported as part of the Header. They will however
+	// be present in Trailer in either case. Hence, normalize the header by removing all trailer values.
+	actualHeader := cloneHeader(rw.Result().Header)
+	for _, trailerKey := range actualHeader["Trailer"] {
+		actualHeader.Del(trailerKey)
+	}
+
+	if !reflect.DeepEqual(actualHeader, wantHeader) {
+		t.Errorf("Header mismatch.\n got: %#v\n want: %#v", actualHeader, wantHeader)
+	}
+	if actualTrailer := rw.Result().Trailer; !reflect.DeepEqual(actualTrailer, wantTrailer) {
+		t.Errorf("Trailer mismatch.\n got: %#v\n want: %#v", actualTrailer, wantTrailer)
 	}
 }
+
+// cloneHeader performs a deep clone of an http.Header, since the (http.Header).Clone() method was only added in
+// Go 1.13.
+func cloneHeader(hdr http.Header) http.Header {
+	if hdr == nil {
+		return nil
+	}
+
+	hdrClone := make(http.Header, len(hdr))
+
+	for k, vv := range hdr {
+		vvClone := make([]string, len(vv))
+		copy(vvClone, vv)
+		hdrClone[k] = vvClone
+	}
+
+	return hdrClone
+}

From a9601d93f498eec9b35b3e148162d79d8aa60602 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 3 Apr 2020 13:09:57 -0700
Subject: [PATCH 006/481] xds: update nonce even if the ACK/NACK is not sent on
 wire (#3497)

This can happen when the watch is canceled while the response is on wire.

Also, tag ACK/NACK with the stream so nonce for a new stream doesn't get updated by a ACK from the previous stream.
---
 xds/internal/client/types.go             |   6 +-
 xds/internal/client/v2client.go          |  29 ++++--
 xds/internal/client/v2client_ack_test.go | 111 +++++++++++++++++++++++
 3 files changed, 137 insertions(+), 9 deletions(-)

diff --git a/xds/internal/client/types.go b/xds/internal/client/types.go
index be9a366f6579..84a446d024f3 100644
--- a/xds/internal/client/types.go
+++ b/xds/internal/client/types.go
@@ -72,8 +72,12 @@ func (wi *watchInfo) stopTimer() {
 
 type ackInfo struct {
 	typeURL string
-	version string // Nack if version is an empty string.
+	version string // NACK if version is an empty string.
 	nonce   string
+	// ACK/NACK are tagged with the stream it's for. When the stream is down,
+	// all the ACK/NACK for this stream will be dropped, and the version/nonce
+	// won't be updated.
+	stream adsStream
 }
 
 type ldsUpdate struct {
diff --git a/xds/internal/client/v2client.go b/xds/internal/client/v2client.go
index faaa0e196647..53ac82abee7e 100644
--- a/xds/internal/client/v2client.go
+++ b/xds/internal/client/v2client.go
@@ -255,11 +255,25 @@ func (v2c *v2Client) processWatchInfo(t *watchInfo) (target []string, typeURL, v
 // processAckInfo pulls the fields needed by the ack request from a ackInfo.
 //
 // If no active watch is found for this ack, it returns false for send.
-func (v2c *v2Client) processAckInfo(t *ackInfo) (target []string, typeURL, version, nonce string, send bool) {
+func (v2c *v2Client) processAckInfo(t *ackInfo, stream adsStream) (target []string, typeURL, version, nonce string, send bool) {
+	if t.stream != stream {
+		// If ACK's stream isn't the current sending stream, this means the ACK
+		// was pushed to queue before the old stream broke, and a new stream has
+		// been started since. Return immediately here so we don't update the
+		// nonce for the new stream.
+		return
+	}
 	typeURL = t.typeURL
 
 	v2c.mu.Lock()
 	defer v2c.mu.Unlock()
+
+	// Update the nonce no matter if we are going to send the ACK request on
+	// wire. We may not send the request if the watch is canceled. But the nonce
+	// needs to be updated so the next request will have the right nonce.
+	nonce = t.nonce
+	v2c.nonceMap[typeURL] = nonce
+
 	wi, ok := v2c.watchMap[typeURL]
 	if !ok {
 		// We don't send the request ack if there's no active watch (this can be
@@ -267,13 +281,10 @@ func (v2c *v2Client) processAckInfo(t *ackInfo) (target []string, typeURL, versi
 		// canceled while the ackInfo is in queue), because there's no resource
 		// name. And if we send a request with empty resource name list, the
 		// server may treat it as a wild card and send us everything.
-		return // This returns all zero values, and false for send.
+		return nil, "", "", "", false
 	}
 	send = true
-
 	version = t.version
-	nonce = t.nonce
-	target = wi.target
 	if version == "" {
 		// This is a nack, get the previous acked version.
 		version = v2c.versionMap[typeURL]
@@ -283,8 +294,8 @@ func (v2c *v2Client) processAckInfo(t *ackInfo) (target []string, typeURL, versi
 	} else {
 		v2c.versionMap[typeURL] = version
 	}
-	v2c.nonceMap[typeURL] = nonce
-	return
+	target = wi.target
+	return target, typeURL, version, nonce, send
 }
 
 // send is a separate goroutine for sending watch requests on the xds stream.
@@ -327,7 +338,7 @@ func (v2c *v2Client) send() {
 			case *watchInfo:
 				target, typeURL, version, nonce, send = v2c.processWatchInfo(t)
 			case *ackInfo:
-				target, typeURL, version, nonce, send = v2c.processAckInfo(t)
+				target, typeURL, version, nonce, send = v2c.processAckInfo(t, stream)
 			}
 			if !send {
 				continue
@@ -381,6 +392,7 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 				typeURL: typeURL,
 				version: "",
 				nonce:   resp.GetNonce(),
+				stream:  stream,
 			})
 			v2c.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", typeURL, resp.GetVersionInfo(), resp.GetNonce(), respHandleErr)
 			continue
@@ -389,6 +401,7 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 			typeURL: typeURL,
 			version: resp.GetVersionInfo(),
 			nonce:   resp.GetNonce(),
+			stream:  stream,
 		})
 		v2c.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", typeURL, resp.GetVersionInfo(), resp.GetNonce())
 		success = true
diff --git a/xds/internal/client/v2client_ack_test.go b/xds/internal/client/v2client_ack_test.go
index e2e7f1b4c1fa..1fce1f5e9564 100644
--- a/xds/internal/client/v2client_ack_test.go
+++ b/xds/internal/client/v2client_ack_test.go
@@ -98,6 +98,10 @@ func startXDS(t *testing.T, xdsname string, v2c *v2Client, reqChan *testutils.Ch
 //
 // It also waits and checks that the ack request contains the given version, and
 // the generated nonce.
+//
+// TODO: make this and other helper function either consistently return error,
+// and fatal() in the test code, or all call t.Fatal(), and mark them as
+// helper().
 func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (nonce string) {
 	nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, version)
 	t.Logf("Good %s response pushed to fakeServer...", xdsname)
@@ -263,3 +267,110 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 	versionLDS++
 }
+
+// TestV2ClientAckNewWatchAfterCancel verifies the new request for a new watch
+// after the previous watch is canceled, has the right version.
+func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
+	var versionCDS = 3000
+
+	fakeServer, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	defer v2c.close()
+	t.Log("Started xds v2Client...")
+
+	// Start a CDS watch.
+	callbackCh := testutils.NewChannel()
+	cancel := v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
+		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
+		callbackCh.Send(struct{}{})
+	})
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("FakeServer received %s request...", "CDS")
+
+	// Send a good CDS response, this function waits for the ACK with the right
+	// version.
+	nonce := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+
+	// Cancel the CDS watch, and start a new one. The new watch should have the
+	// version from the response above.
+	cancel()
+	v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
+		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
+		callbackCh.Send(struct{}{})
+	})
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce); err != nil {
+		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+	}
+	versionCDS++
+
+	// Send a bad response with the next version.
+	sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest)
+	versionCDS++
+
+	// send another good response, and check for ack, with the new version.
+	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+	versionCDS++
+}
+
+// TestV2ClientAckCancelResponseRace verifies if the response and ACK request
+// race with cancel (which means the ACK request will not be sent on wire,
+// because there's no active watch), the nonce will still be updated, and the
+// new request with the new watch will have the correct nonce.
+func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
+	var versionCDS = 3000
+
+	fakeServer, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	defer v2c.close()
+	t.Log("Started xds v2Client...")
+
+	// Start a CDS watch.
+	callbackCh := testutils.NewChannel()
+	cancel := v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
+		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
+		callbackCh.Send(struct{}{})
+	})
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
+		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+	}
+	t.Logf("FakeServer received %s request...", "CDS")
+
+	// send another good response, and check for ack, with the new version.
+	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+	versionCDS++
+
+	// Cancel the watch before the next response is sent. This mimics the case
+	// watch is canceled while response is on wire.
+	cancel()
+
+	// Send a good response.
+	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodCDSResponse1, versionCDS)
+	t.Logf("Good %s response pushed to fakeServer...", "CDS")
+
+	// Expect no ACK because watch was canceled.
+	if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout {
+		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
+	}
+
+	// Start a new watch. The new watch should have the nonce from the response
+	// above, and version from the first good response.
+	v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
+		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
+		callbackCh.Send(struct{}{})
+	})
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce); err != nil {
+		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+	}
+
+	// Send a bad response with the next version.
+	sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest)
+	versionCDS++
+
+	// send another good response, and check for ack, with the new version.
+	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+	versionCDS++
+}

From 7cb4db26f12fc431f99b4cc7ac315b1a014b13e6 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 6 Apr 2020 10:09:23 -0700
Subject: [PATCH 007/481] internal: update grpc_lb_v1 proto (#3499)

---
 .../grpclb/grpc_lb_v1/load_balancer.pb.go     | 110 ++++++++----------
 balancer/grpclb/grpclb_remote_balancer.go     |   3 -
 2 files changed, 48 insertions(+), 65 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
index 31b714db4928..b59191ac5825 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
@@ -414,12 +414,6 @@ func (m *FallbackResponse) XXX_DiscardUnknown() {
 var xxx_messageInfo_FallbackResponse proto.InternalMessageInfo
 
 type InitialLoadBalanceResponse struct {
-	// This is an application layer redirect that indicates the client should use
-	// the specified server for load balancing. When this field is non-empty in
-	// the response, the client should open a separate connection to the
-	// load_balancer_delegate and call the BalanceLoad method. Its length should
-	// be less than 64 bytes.
-	LoadBalancerDelegate string `protobuf:"bytes,1,opt,name=load_balancer_delegate,json=loadBalancerDelegate,proto3" json:"load_balancer_delegate,omitempty"`
 	// This interval defines how often the client should send the client stats
 	// to the load balancer. Stats should only be reported when the duration is
 	// positive.
@@ -454,13 +448,6 @@ func (m *InitialLoadBalanceResponse) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_InitialLoadBalanceResponse proto.InternalMessageInfo
 
-func (m *InitialLoadBalanceResponse) GetLoadBalancerDelegate() string {
-	if m != nil {
-		return m.LoadBalancerDelegate
-	}
-	return ""
-}
-
 func (m *InitialLoadBalanceResponse) GetClientStatsReportInterval() *duration.Duration {
 	if m != nil {
 		return m.ClientStatsReportInterval
@@ -604,56 +591,55 @@ func init() {
 func init() { proto.RegisterFile("grpc/lb/v1/load_balancer.proto", fileDescriptor_7cd3f6d792743fdf) }
 
 var fileDescriptor_7cd3f6d792743fdf = []byte{
-	// 785 bytes of a gzipped FileDescriptorProto
+	// 769 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xdd, 0x6e, 0xdb, 0x36,
-	0x14, 0x8e, 0x6a, 0x27, 0x75, 0x8e, 0xb3, 0xc5, 0x61, 0xb7, 0x4e, 0x71, 0xd3, 0x24, 0x13, 0xb0,
-	0x22, 0x18, 0x3a, 0x79, 0xc9, 0x76, 0xb1, 0x01, 0xbb, 0xd8, 0xdc, 0x20, 0x48, 0xd3, 0x5e, 0x04,
-	0x74, 0x80, 0x0e, 0x05, 0x06, 0x8e, 0x92, 0x68, 0x87, 0x08, 0x4d, 0x6a, 0x14, 0xed, 0x62, 0xd7,
-	0x7b, 0x81, 0x3d, 0xc9, 0xb0, 0x57, 0xd8, 0x9b, 0x0d, 0x22, 0x29, 0x4b, 0xb1, 0x63, 0xf4, 0x4a,
-	0xe4, 0x39, 0x1f, 0xbf, 0xf3, 0x7f, 0x04, 0x87, 0x13, 0x9d, 0xa7, 0x03, 0x91, 0x0c, 0xe6, 0xa7,
-	0x03, 0xa1, 0x68, 0x46, 0x12, 0x2a, 0xa8, 0x4c, 0x99, 0x8e, 0x73, 0xad, 0x8c, 0x42, 0x50, 0xea,
-	0x63, 0x91, 0xc4, 0xf3, 0xd3, 0xfe, 0xe1, 0x44, 0xa9, 0x89, 0x60, 0x03, 0xab, 0x49, 0x66, 0xe3,
-	0x41, 0x36, 0xd3, 0xd4, 0x70, 0x25, 0x1d, 0xb6, 0x7f, 0xb4, 0xac, 0x37, 0x7c, 0xca, 0x0a, 0x43,
-	0xa7, 0xb9, 0x03, 0x44, 0xff, 0x05, 0x80, 0xde, 0x2a, 0x9a, 0x0d, 0x9d, 0x0d, 0xcc, 0xfe, 0x98,
-	0xb1, 0xc2, 0xa0, 0x6b, 0xd8, 0xe5, 0x92, 0x1b, 0x4e, 0x05, 0xd1, 0x4e, 0x14, 0x06, 0xc7, 0xc1,
-	0x49, 0xf7, 0xec, 0xab, 0xb8, 0xb6, 0x1e, 0xbf, 0x76, 0x90, 0xd5, 0xf7, 0x97, 0x1b, 0xf8, 0x53,
-	0xff, 0xbe, 0x62, 0xfc, 0x09, 0x76, 0x52, 0xc1, 0x99, 0x34, 0xa4, 0x30, 0xd4, 0x14, 0xe1, 0x23,
-	0x4b, 0xf7, 0x45, 0x93, 0xee, 0x95, 0xd5, 0x8f, 0x4a, 0xf5, 0xe5, 0x06, 0xee, 0xa6, 0xf5, 0x75,
-	0xf8, 0x0c, 0xf6, 0x9b, 0xa9, 0xa8, 0x9c, 0x22, 0xe6, 0xcf, 0x9c, 0x45, 0x03, 0xd8, 0x5f, 0xeb,
-	0x09, 0x42, 0xd0, 0x96, 0x74, 0xca, 0xac, 0xfb, 0xdb, 0xd8, 0x9e, 0xa3, 0xdf, 0xe1, 0x49, 0xc3,
-	0xd6, 0x35, 0xd3, 0x37, 0xea, 0x8e, 0x49, 0xf4, 0x12, 0xd0, 0x3d, 0x23, 0xa6, 0x94, 0xfa, 0x87,
-	0x3d, 0x51, 0x53, 0x3b, 0xf4, 0x33, 0xd8, 0x96, 0xb3, 0x29, 0x49, 0xa9, 0x10, 0x2e, 0x9a, 0x16,
-	0xee, 0xc8, 0xd9, 0xf4, 0x55, 0x79, 0x8f, 0xfe, 0x6d, 0x41, 0xb7, 0x61, 0x02, 0xfd, 0x00, 0xdb,
-	0x8b, 0xcc, 0xfb, 0x4c, 0xf6, 0x63, 0x57, 0x9b, 0xb8, 0xaa, 0x4d, 0x7c, 0x53, 0x21, 0x70, 0x0d,
-	0x46, 0x5f, 0xc3, 0xde, 0xc2, 0x4c, 0x99, 0x3a, 0x6d, 0x58, 0xe6, 0xcd, 0xed, 0x56, 0xe6, 0x46,
-	0x4e, 0x5c, 0x06, 0x50, 0x63, 0xc7, 0x5c, 0xf2, 0xe2, 0x96, 0x65, 0x61, 0xcb, 0x82, 0x7b, 0x15,
-	0xf8, 0xc2, 0xcb, 0xd1, 0x6f, 0xf0, 0xcd, 0x2a, 0x9a, 0x7c, 0xe0, 0xe6, 0x96, 0xf8, 0x4a, 0x8d,
-	0x29, 0x17, 0x2c, 0x23, 0x46, 0x91, 0x82, 0xc9, 0x2c, 0xdc, 0xb2, 0x44, 0x2f, 0x96, 0x89, 0xde,
-	0x71, 0x73, 0xeb, 0x62, 0xbd, 0xb0, 0xf8, 0x1b, 0x35, 0x62, 0x32, 0x43, 0x97, 0xf0, 0xe5, 0x03,
-	0xf4, 0x77, 0x52, 0x7d, 0x90, 0x44, 0xb3, 0x94, 0xf1, 0x39, 0xcb, 0xc2, 0xc7, 0x96, 0xf2, 0xf9,
-	0x32, 0xe5, 0x9b, 0x12, 0x85, 0x3d, 0x08, 0xfd, 0x0a, 0xe1, 0x43, 0x4e, 0x66, 0x5a, 0xe5, 0x61,
-	0xe7, 0xb8, 0x75, 0xd2, 0x3d, 0x3b, 0x5a, 0xd3, 0x46, 0x55, 0x69, 0xf1, 0xe7, 0xe9, 0xb2, 0xc7,
-	0xe7, 0x5a, 0xe5, 0x57, 0xed, 0x4e, 0xbb, 0xb7, 0x79, 0xd5, 0xee, 0x6c, 0xf6, 0xb6, 0xa2, 0xbf,
-	0x1f, 0xc1, 0x93, 0x7b, 0xfd, 0x53, 0xe4, 0x4a, 0x16, 0x0c, 0x8d, 0xa0, 0x57, 0x8f, 0x82, 0x93,
-	0xf9, 0x0a, 0xbe, 0xf8, 0xd8, 0x2c, 0x38, 0xf4, 0xe5, 0x06, 0xde, 0x5d, 0x0c, 0x83, 0x27, 0xfd,
-	0x11, 0xba, 0x05, 0xd3, 0x73, 0xa6, 0x89, 0xe0, 0x85, 0xf1, 0xc3, 0xf0, 0xb4, 0xc9, 0x37, 0xb2,
-	0xea, 0xb7, 0xdc, 0x0e, 0x13, 0x14, 0x8b, 0x1b, 0x7a, 0x03, 0x7b, 0x63, 0x2a, 0x44, 0x42, 0xd3,
-	0xbb, 0xda, 0xa1, 0x96, 0x25, 0x38, 0x68, 0x12, 0x5c, 0x78, 0x50, 0xc3, 0x8d, 0xde, 0x78, 0x49,
-	0x36, 0x3c, 0x80, 0xfe, 0xd2, 0x5c, 0x39, 0x85, 0x1b, 0x2c, 0x04, 0xbd, 0x65, 0x96, 0xe8, 0x9f,
-	0x00, 0xfa, 0xeb, 0x63, 0x45, 0xdf, 0xc3, 0xd3, 0x7b, 0x3b, 0x8b, 0x64, 0x4c, 0xb0, 0x09, 0x35,
-	0xd5, 0x00, 0x7e, 0xd6, 0x98, 0x23, 0x7d, 0xee, 0x75, 0xe8, 0x3d, 0x1c, 0x34, 0x97, 0x03, 0xd1,
-	0x2c, 0x57, 0xda, 0x10, 0x2e, 0x0d, 0xd3, 0x73, 0x2a, 0x7c, 0x7e, 0xf6, 0x57, 0x26, 0xe6, 0xdc,
-	0x6f, 0x3b, 0xbc, 0xdf, 0x58, 0x16, 0xd8, 0x3e, 0x7e, 0xed, 0xdf, 0x46, 0x3f, 0x03, 0xd4, 0xb9,
-	0x44, 0x2f, 0xe1, 0xb1, 0xcb, 0x65, 0x11, 0x06, 0xb6, 0x75, 0xd0, 0x6a, 0xd2, 0x71, 0x05, 0xb9,
-	0x6a, 0x77, 0x5a, 0xbd, 0x76, 0xf4, 0x57, 0x00, 0x5b, 0x4e, 0x83, 0x9e, 0x03, 0xf0, 0x9c, 0xd0,
-	0x2c, 0xd3, 0xac, 0x28, 0x6c, 0x48, 0x3b, 0x78, 0x9b, 0xe7, 0xbf, 0x38, 0x41, 0xb9, 0x6c, 0x4a,
-	0xdb, 0xd6, 0xdf, 0x4d, 0x6c, 0xcf, 0x6b, 0xb6, 0x4a, 0x6b, 0xcd, 0x56, 0x41, 0xd0, 0xb6, 0x7d,
-	0xdd, 0x3e, 0x0e, 0x4e, 0x3a, 0xd8, 0x9e, 0x5d, 0x7f, 0x9e, 0x25, 0xb0, 0xd3, 0x48, 0xb8, 0x46,
-	0x18, 0xba, 0xfe, 0x5c, 0x8a, 0xd1, 0x61, 0x33, 0x8e, 0xd5, 0x3d, 0xd8, 0x3f, 0x5a, 0xab, 0x77,
-	0x95, 0x3b, 0x09, 0xbe, 0x0d, 0x86, 0xef, 0xe0, 0x13, 0xae, 0x1a, 0xc0, 0xe1, 0x5e, 0xd3, 0xe4,
-	0x75, 0x99, 0xf6, 0xeb, 0xe0, 0xfd, 0xa9, 0x2f, 0xc3, 0x44, 0x09, 0x2a, 0x27, 0xb1, 0xd2, 0x93,
-	0x81, 0xfd, 0x65, 0x55, 0x35, 0xb7, 0x37, 0x91, 0xd8, 0x0f, 0x11, 0x09, 0x99, 0x9f, 0x26, 0x5b,
-	0xb6, 0x64, 0xdf, 0xfd, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x09, 0x7b, 0x39, 0x1e, 0xdc, 0x06, 0x00,
+	0x14, 0x8e, 0x62, 0x25, 0x75, 0x8e, 0xb3, 0x45, 0x61, 0xb1, 0xcd, 0x71, 0xd3, 0x24, 0x13, 0xb0,
+	0x22, 0x18, 0x3a, 0x79, 0xc9, 0x6e, 0x36, 0x60, 0x17, 0x9b, 0x5b, 0x04, 0x69, 0xda, 0x8b, 0x80,
+	0x0e, 0xd0, 0xa1, 0xc0, 0xc0, 0x51, 0x12, 0xed, 0x10, 0xa1, 0x49, 0x8d, 0xa2, 0x5d, 0xec, 0x66,
+	0x37, 0x7b, 0x81, 0x3d, 0xca, 0x5e, 0x61, 0x6f, 0x36, 0x88, 0xa4, 0x2c, 0xd5, 0xae, 0xb1, 0x2b,
+	0x91, 0xe7, 0x7c, 0xfc, 0xce, 0xff, 0x11, 0x9c, 0x4c, 0x75, 0x91, 0x0d, 0x45, 0x3a, 0x5c, 0x5c,
+	0x0c, 0x85, 0xa2, 0x39, 0x49, 0xa9, 0xa0, 0x32, 0x63, 0x3a, 0x29, 0xb4, 0x32, 0x0a, 0x41, 0xa5,
+	0x4f, 0x44, 0x9a, 0x2c, 0x2e, 0x06, 0x27, 0x53, 0xa5, 0xa6, 0x82, 0x0d, 0xad, 0x26, 0x9d, 0x4f,
+	0x86, 0xf9, 0x5c, 0x53, 0xc3, 0x95, 0x74, 0xd8, 0xc1, 0xe9, 0xaa, 0xde, 0xf0, 0x19, 0x2b, 0x0d,
+	0x9d, 0x15, 0x0e, 0x10, 0xff, 0x1b, 0x00, 0x7a, 0xa3, 0x68, 0x3e, 0x72, 0x36, 0x30, 0xfb, 0x7d,
+	0xce, 0x4a, 0x83, 0x6e, 0xe1, 0x80, 0x4b, 0x6e, 0x38, 0x15, 0x44, 0x3b, 0x51, 0x3f, 0x38, 0x0b,
+	0xce, 0x7b, 0x97, 0x5f, 0x25, 0x8d, 0xf5, 0xe4, 0x95, 0x83, 0xac, 0xbf, 0xbf, 0xde, 0xc2, 0x9f,
+	0xfa, 0xf7, 0x35, 0xe3, 0x8f, 0xb0, 0x9f, 0x09, 0xce, 0xa4, 0x21, 0xa5, 0xa1, 0xa6, 0xec, 0x6f,
+	0x5b, 0xba, 0x2f, 0xda, 0x74, 0x2f, 0xac, 0x7e, 0x5c, 0xa9, 0xaf, 0xb7, 0x70, 0x2f, 0x6b, 0xae,
+	0xa3, 0x27, 0x70, 0xd4, 0x4e, 0x45, 0xed, 0x14, 0x31, 0x7f, 0x14, 0x2c, 0x1e, 0xc2, 0xd1, 0x46,
+	0x4f, 0x10, 0x82, 0x50, 0xd2, 0x19, 0xb3, 0xee, 0xef, 0x61, 0x7b, 0x8e, 0x7f, 0x83, 0xc7, 0x2d,
+	0x5b, 0xb7, 0x4c, 0xdf, 0xa9, 0x07, 0x26, 0xd1, 0x73, 0x40, 0x1f, 0x18, 0x31, 0x95, 0xd4, 0x3f,
+	0x8c, 0x44, 0x43, 0xed, 0xd0, 0x4f, 0x60, 0x4f, 0xce, 0x67, 0x24, 0xa3, 0x42, 0xb8, 0x68, 0x3a,
+	0xb8, 0x2b, 0xe7, 0xb3, 0x17, 0xd5, 0x3d, 0xfe, 0xa7, 0x03, 0xbd, 0x96, 0x09, 0xf4, 0x3d, 0xec,
+	0x2d, 0x33, 0xef, 0x33, 0x39, 0x48, 0x5c, 0x6d, 0x92, 0xba, 0x36, 0xc9, 0x5d, 0x8d, 0xc0, 0x0d,
+	0x18, 0x7d, 0x0d, 0x87, 0x4b, 0x33, 0x55, 0xea, 0xb4, 0x61, 0xb9, 0x37, 0x77, 0x50, 0x9b, 0x1b,
+	0x3b, 0x71, 0x15, 0x40, 0x83, 0x9d, 0x70, 0xc9, 0xcb, 0x7b, 0x96, 0xf7, 0x3b, 0x16, 0x1c, 0xd5,
+	0xe0, 0x2b, 0x2f, 0x47, 0xbf, 0xc2, 0x37, 0xeb, 0x68, 0xf2, 0x9e, 0x9b, 0x7b, 0xe2, 0x2b, 0x35,
+	0xa1, 0x5c, 0xb0, 0x9c, 0x18, 0x45, 0x4a, 0x26, 0xf3, 0xfe, 0xae, 0x25, 0x7a, 0xb6, 0x4a, 0xf4,
+	0x96, 0x9b, 0x7b, 0x17, 0xeb, 0x95, 0xc5, 0xdf, 0xa9, 0x31, 0x93, 0x39, 0xba, 0x86, 0x2f, 0x3f,
+	0x42, 0xff, 0x20, 0xd5, 0x7b, 0x49, 0x34, 0xcb, 0x18, 0x5f, 0xb0, 0xbc, 0xff, 0xc8, 0x52, 0x3e,
+	0x5d, 0xa5, 0x7c, 0x5d, 0xa1, 0xb0, 0x07, 0xa1, 0x5f, 0xa0, 0xff, 0x31, 0x27, 0x73, 0xad, 0x8a,
+	0x7e, 0xf7, 0xac, 0x73, 0xde, 0xbb, 0x3c, 0xdd, 0xd0, 0x46, 0x75, 0x69, 0xf1, 0x67, 0xd9, 0xaa,
+	0xc7, 0x2f, 0xb5, 0x2a, 0x6e, 0xc2, 0x6e, 0x18, 0xed, 0xdc, 0x84, 0xdd, 0x9d, 0x68, 0x37, 0xfe,
+	0x7b, 0x1b, 0x1e, 0x7f, 0xd0, 0x3f, 0x65, 0xa1, 0x64, 0xc9, 0xd0, 0x18, 0xa2, 0x66, 0x14, 0x9c,
+	0xcc, 0x57, 0xf0, 0xd9, 0xff, 0xcd, 0x82, 0x43, 0x5f, 0x6f, 0xe1, 0x83, 0xe5, 0x30, 0x78, 0xd2,
+	0x1f, 0xa0, 0x57, 0x32, 0xbd, 0x60, 0x9a, 0x08, 0x5e, 0x1a, 0x3f, 0x0c, 0x9f, 0xb7, 0xf9, 0xc6,
+	0x56, 0xfd, 0x86, 0xdb, 0x61, 0x82, 0x72, 0x79, 0x43, 0xaf, 0xe1, 0x70, 0x42, 0x85, 0x48, 0x69,
+	0xf6, 0xd0, 0x38, 0xd4, 0xb1, 0x04, 0xc7, 0x6d, 0x82, 0x2b, 0x0f, 0x6a, 0xb9, 0x11, 0x4d, 0x56,
+	0x64, 0xa3, 0x63, 0x18, 0xac, 0xcc, 0x95, 0x53, 0xb8, 0xc1, 0x42, 0x10, 0xad, 0xb2, 0xc4, 0x7f,
+	0xc2, 0x60, 0x73, 0xa8, 0xe8, 0x1d, 0x1c, 0xb7, 0xa7, 0x9c, 0x68, 0x56, 0x28, 0x6d, 0x08, 0x97,
+	0x86, 0xe9, 0x05, 0x15, 0x3e, 0xd0, 0xa3, 0xb5, 0xd6, 0x7f, 0xe9, 0xd7, 0x16, 0x3e, 0x6a, 0x4d,
+	0x3d, 0xb6, 0x8f, 0x5f, 0xf9, 0xb7, 0x37, 0x61, 0x37, 0x88, 0xb6, 0xe3, 0x9f, 0x00, 0x9a, 0xd4,
+	0xa0, 0xe7, 0xf0, 0xc8, 0xa5, 0xa6, 0xec, 0x07, 0xb6, 0x13, 0xd0, 0x7a, 0x0e, 0x71, 0x0d, 0xb9,
+	0x09, 0xbb, 0x9d, 0x28, 0x8c, 0xff, 0x0a, 0x60, 0xd7, 0x69, 0xd0, 0x53, 0x00, 0x5e, 0x10, 0x9a,
+	0xe7, 0x9a, 0x95, 0xa5, 0xad, 0xea, 0x3e, 0xde, 0xe3, 0xc5, 0xcf, 0x4e, 0x50, 0xed, 0x8e, 0xca,
+	0x03, 0xeb, 0xf5, 0x0e, 0xb6, 0xe7, 0x0d, 0x4b, 0xa2, 0xb3, 0x61, 0x49, 0x20, 0x08, 0x6d, 0x9b,
+	0x86, 0x67, 0xc1, 0x79, 0x17, 0xdb, 0xb3, 0x6b, 0xb7, 0xcb, 0x14, 0xf6, 0x5b, 0x09, 0xd4, 0x08,
+	0x43, 0xcf, 0x9f, 0x2b, 0x31, 0x3a, 0x69, 0xc7, 0xb1, 0xbe, 0xd6, 0x06, 0xa7, 0x1b, 0xf5, 0xae,
+	0x12, 0xe7, 0xc1, 0xb7, 0xc1, 0xe8, 0x2d, 0x7c, 0xc2, 0x55, 0x0b, 0x38, 0x3a, 0x6c, 0x9b, 0xbc,
+	0xad, 0x92, 0x7f, 0x1b, 0xbc, 0xbb, 0xf0, 0xc5, 0x98, 0x2a, 0x41, 0xe5, 0x34, 0x51, 0x7a, 0x3a,
+	0xb4, 0x7f, 0xa0, 0xfa, 0xb7, 0x63, 0x6f, 0x22, 0xb5, 0x1f, 0x22, 0x52, 0xb2, 0xb8, 0x48, 0x77,
+	0x6d, 0xe1, 0xbe, 0xfb, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x47, 0x55, 0xac, 0xab, 0x06, 0x00,
 	0x00,
 }
 
diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index 8f47b019eb2d..c6d555e4d8c8 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -350,9 +350,6 @@ func (ccw *remoteBalancerCCWrapper) callRemoteBalancer() (backoff bool, _ error)
 	if initResp == nil {
 		return true, fmt.Errorf("grpclb: reply from remote balancer did not include initial response")
 	}
-	if initResp.LoadBalancerDelegate != "" {
-		return true, fmt.Errorf("grpclb: Delegation is not supported")
-	}
 
 	ccw.wg.Add(1)
 	go func() {

From c5faf568da50936d0b11fdf4effb22cdb87a061c Mon Sep 17 00:00:00 2001
From: James Protzman <JNProtzman@gmail.com>
Date: Mon, 6 Apr 2020 14:03:17 -0400
Subject: [PATCH 008/481] status: move statusError to internal/status package
 (#3432)

---
 internal/internal.go               |   5 -
 internal/status/status.go          | 166 +++++++++++++++++++++++++++++
 internal/transport/http2_server.go |   7 +-
 status/status.go                   | 119 ++-------------------
 status/status_test.go              |  21 ++--
 5 files changed, 186 insertions(+), 132 deletions(-)
 create mode 100644 internal/status/status.go

diff --git a/internal/internal.go b/internal/internal.go
index 0912f0bf4c31..c6fbe8bb1b27 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -37,11 +37,6 @@ var (
 	// KeepaliveMinPingTime is the minimum ping interval.  This must be 10s by
 	// default, but tests may wish to set it lower for convenience.
 	KeepaliveMinPingTime = 10 * time.Second
-	// StatusRawProto is exported by status/status.go. This func returns a
-	// pointer to the wrapped Status proto for a given status.Status without a
-	// call to proto.Clone(). The returned Status proto should not be mutated by
-	// the caller.
-	StatusRawProto interface{} // func (*status.Status) *spb.Status
 	// NewRequestInfoContext creates a new context based on the argument context attaching
 	// the passed in RequestInfo to the new context.
 	NewRequestInfoContext interface{} // func(context.Context, credentials.RequestInfo) context.Context
diff --git a/internal/status/status.go b/internal/status/status.go
new file mode 100644
index 000000000000..23dae8e5679f
--- /dev/null
+++ b/internal/status/status.go
@@ -0,0 +1,166 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package status implements errors returned by gRPC.  These errors are
+// serialized and transmitted on the wire between server and client, and allow
+// for additional data to be transmitted via the Details field in the status
+// proto.  gRPC service handlers should return an error created by this
+// package, and gRPC clients should expect a corresponding error to be
+// returned from the RPC call.
+//
+// This package upholds the invariants that a non-nil error may not
+// contain an OK code, and an OK code must result in a nil error.
+package status
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	spb "google.golang.org/genproto/googleapis/rpc/status"
+	"google.golang.org/grpc/codes"
+)
+
+// Status represents an RPC status code, message, and details.  It is immutable
+// and should be created with New, Newf, or FromProto.
+type Status struct {
+	s *spb.Status
+}
+
+// New returns a Status representing c and msg.
+func New(c codes.Code, msg string) *Status {
+	return &Status{s: &spb.Status{Code: int32(c), Message: msg}}
+}
+
+// Newf returns New(c, fmt.Sprintf(format, a...)).
+func Newf(c codes.Code, format string, a ...interface{}) *Status {
+	return New(c, fmt.Sprintf(format, a...))
+}
+
+// FromProto returns a Status representing s.
+func FromProto(s *spb.Status) *Status {
+	return &Status{s: proto.Clone(s).(*spb.Status)}
+}
+
+// Err returns an error representing c and msg.  If c is OK, returns nil.
+func Err(c codes.Code, msg string) error {
+	return New(c, msg).Err()
+}
+
+// Errorf returns Error(c, fmt.Sprintf(format, a...)).
+func Errorf(c codes.Code, format string, a ...interface{}) error {
+	return Err(c, fmt.Sprintf(format, a...))
+}
+
+// Code returns the status code contained in s.
+func (s *Status) Code() codes.Code {
+	if s == nil || s.s == nil {
+		return codes.OK
+	}
+	return codes.Code(s.s.Code)
+}
+
+// Message returns the message contained in s.
+func (s *Status) Message() string {
+	if s == nil || s.s == nil {
+		return ""
+	}
+	return s.s.Message
+}
+
+// Proto returns s's status as an spb.Status proto message.
+func (s *Status) Proto() *spb.Status {
+	if s == nil {
+		return nil
+	}
+	return proto.Clone(s.s).(*spb.Status)
+}
+
+// Err returns an immutable error representing s; returns nil if s.Code() is OK.
+func (s *Status) Err() error {
+	if s.Code() == codes.OK {
+		return nil
+	}
+	return (*Error)(s.Proto())
+}
+
+func (s *Status) Error() string {
+	p := s.Proto()
+	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage())
+}
+
+// WithDetails returns a new status with the provided details messages appended to the status.
+// If any errors are encountered, it returns nil and the first error encountered.
+func (s *Status) WithDetails(details ...proto.Message) (*Status, error) {
+	if s.Code() == codes.OK {
+		return nil, errors.New("no error details for status with code OK")
+	}
+	// s.Code() != OK implies that s.Proto() != nil.
+	p := s.Proto()
+	for _, detail := range details {
+		any, err := ptypes.MarshalAny(detail)
+		if err != nil {
+			return nil, err
+		}
+		p.Details = append(p.Details, any)
+	}
+	return &Status{s: p}, nil
+}
+
+// Details returns a slice of details messages attached to the status.
+// If a detail cannot be decoded, the error is returned in place of the detail.
+func (s *Status) Details() []interface{} {
+	if s == nil || s.s == nil {
+		return nil
+	}
+	details := make([]interface{}, 0, len(s.s.Details))
+	for _, any := range s.s.Details {
+		detail := &ptypes.DynamicAny{}
+		if err := ptypes.UnmarshalAny(any, detail); err != nil {
+			details = append(details, err)
+			continue
+		}
+		details = append(details, detail.Message)
+	}
+	return details
+}
+
+// Error is an alias of a status proto. It implements error and Status,
+// and a nil Error should never be returned by this package.
+type Error spb.Status
+
+func (se *Error) Error() string {
+	p := (*spb.Status)(se)
+	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage())
+}
+
+// GRPCStatus returns the Status represented by se.
+func (se *Error) GRPCStatus() *Status {
+	return FromProto((*spb.Status)(se))
+}
+
+// Is implements future error.Is functionality.
+// A Error is equivalent if the code and message are identical.
+func (se *Error) Is(target error) bool {
+	tse, ok := target.(*Error)
+	if !ok {
+		return false
+	}
+	return proto.Equal((*spb.Status)(se), (*spb.Status)(tse))
+}
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index cbcedc8db27a..fa33ffb18856 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -35,11 +35,9 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 
-	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
-	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/keepalive"
@@ -57,9 +55,6 @@ var (
 	// ErrHeaderListSizeLimitViolation indicates that the header list size is larger
 	// than the limit set by peer.
 	ErrHeaderListSizeLimitViolation = errors.New("transport: trying to send header list size larger than the limit set by peer")
-	// statusRawProto is a function to get to the raw status proto wrapped in a
-	// status.Status without a proto.Clone().
-	statusRawProto = internal.StatusRawProto.(func(*status.Status) *spb.Status)
 )
 
 // serverConnectionCounter counts the number of connections a server has seen
@@ -850,7 +845,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error {
 	headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))})
 	headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(st.Message())})
 
-	if p := statusRawProto(st); p != nil && len(p.Details) > 0 {
+	if p := st.Proto(); p != nil && len(p.Details) > 0 {
 		stBytes, err := proto.Marshal(p)
 		if err != nil {
 			// TODO: return error instead, when callers are able to handle it.
diff --git a/status/status.go b/status/status.go
index a1348e9b16bd..01e182c306c2 100644
--- a/status/status.go
+++ b/status/status.go
@@ -29,88 +29,23 @@ package status
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
-	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
 	spb "google.golang.org/genproto/googleapis/rpc/status"
+
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/status"
 )
 
-func init() {
-	internal.StatusRawProto = statusRawProto
-}
-
-func statusRawProto(s *Status) *spb.Status { return s.s }
-
-// statusError is an alias of a status proto.  It implements error and Status,
-// and a nil statusError should never be returned by this package.
-type statusError spb.Status
-
-func (se *statusError) Error() string {
-	p := (*spb.Status)(se)
-	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage())
-}
-
-func (se *statusError) GRPCStatus() *Status {
-	return &Status{s: (*spb.Status)(se)}
-}
-
-// Is implements future error.Is functionality.
-// A statusError is equivalent if the code and message are identical.
-func (se *statusError) Is(target error) bool {
-	tse, ok := target.(*statusError)
-	if !ok {
-		return false
-	}
-
-	return proto.Equal((*spb.Status)(se), (*spb.Status)(tse))
-}
-
-// Status represents an RPC status code, message, and details.  It is immutable
-// and should be created with New, Newf, or FromProto.
-type Status struct {
-	s *spb.Status
-}
-
-// Code returns the status code contained in s.
-func (s *Status) Code() codes.Code {
-	if s == nil || s.s == nil {
-		return codes.OK
-	}
-	return codes.Code(s.s.Code)
-}
-
-// Message returns the message contained in s.
-func (s *Status) Message() string {
-	if s == nil || s.s == nil {
-		return ""
-	}
-	return s.s.Message
-}
-
-// Proto returns s's status as an spb.Status proto message.
-func (s *Status) Proto() *spb.Status {
-	if s == nil {
-		return nil
-	}
-	return proto.Clone(s.s).(*spb.Status)
-}
-
-// Err returns an immutable error representing s; returns nil if s.Code() is
-// OK.
-func (s *Status) Err() error {
-	if s.Code() == codes.OK {
-		return nil
-	}
-	return (*statusError)(s.s)
-}
+// Status references google.golang.org/grpc/internal/status. It represents an
+// RPC status code, message, and details.  It is immutable and should be
+// created with New, Newf, or FromProto.
+// https://godoc.org/google.golang.org/grpc/internal/status
+type Status = status.Status
 
 // New returns a Status representing c and msg.
 func New(c codes.Code, msg string) *Status {
-	return &Status{s: &spb.Status{Code: int32(c), Message: msg}}
+	return status.New(c, msg)
 }
 
 // Newf returns New(c, fmt.Sprintf(format, a...)).
@@ -135,7 +70,7 @@ func ErrorProto(s *spb.Status) error {
 
 // FromProto returns a Status representing s.
 func FromProto(s *spb.Status) *Status {
-	return &Status{s: proto.Clone(s).(*spb.Status)}
+	return status.FromProto(s)
 }
 
 // FromError returns a Status representing err if it was produced from this
@@ -160,42 +95,6 @@ func Convert(err error) *Status {
 	return s
 }
 
-// WithDetails returns a new status with the provided details messages appended to the status.
-// If any errors are encountered, it returns nil and the first error encountered.
-func (s *Status) WithDetails(details ...proto.Message) (*Status, error) {
-	if s.Code() == codes.OK {
-		return nil, errors.New("no error details for status with code OK")
-	}
-	// s.Code() != OK implies that s.Proto() != nil.
-	p := s.Proto()
-	for _, detail := range details {
-		any, err := ptypes.MarshalAny(detail)
-		if err != nil {
-			return nil, err
-		}
-		p.Details = append(p.Details, any)
-	}
-	return &Status{s: p}, nil
-}
-
-// Details returns a slice of details messages attached to the status.
-// If a detail cannot be decoded, the error is returned in place of the detail.
-func (s *Status) Details() []interface{} {
-	if s == nil || s.s == nil {
-		return nil
-	}
-	details := make([]interface{}, 0, len(s.s.Details))
-	for _, any := range s.s.Details {
-		detail := &ptypes.DynamicAny{}
-		if err := ptypes.UnmarshalAny(any, detail); err != nil {
-			details = append(details, err)
-			continue
-		}
-		details = append(details, detail.Message)
-	}
-	return details
-}
-
 // Code returns the Code of the error if it is a Status error, codes.OK if err
 // is nil, or codes.Unknown otherwise.
 func Code(err error) codes.Code {
diff --git a/status/status_test.go b/status/status_test.go
index 35917acbb859..839a3c390ede 100644
--- a/status/status_test.go
+++ b/status/status_test.go
@@ -34,6 +34,7 @@ import (
 	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/status"
 )
 
 type s struct {
@@ -63,7 +64,7 @@ func (s) TestErrorsWithSameParameters(t *testing.T) {
 	e1 := Errorf(codes.AlreadyExists, description)
 	e2 := Errorf(codes.AlreadyExists, description)
 	if e1 == e2 || !errEqual(e1, e2) {
-		t.Fatalf("Errors should be equivalent but unique - e1: %v, %v  e2: %p, %v", e1.(*statusError), e1, e2.(*statusError), e2)
+		t.Fatalf("Errors should be equivalent but unique - e1: %v, %v  e2: %p, %v", e1.(*status.Error), e1, e2.(*status.Error), e2)
 	}
 }
 
@@ -115,7 +116,7 @@ func (s) TestError(t *testing.T) {
 func (s) TestErrorOK(t *testing.T) {
 	err := Error(codes.OK, "foo")
 	if err != nil {
-		t.Fatalf("Error(codes.OK, _) = %p; want nil", err.(*statusError))
+		t.Fatalf("Error(codes.OK, _) = %p; want nil", err.(*status.Error))
 	}
 }
 
@@ -154,13 +155,11 @@ func (c customError) Error() string {
 }
 
 func (c customError) GRPCStatus() *Status {
-	return &Status{
-		s: &spb.Status{
-			Code:    int32(c.Code),
-			Message: c.Message,
-			Details: c.Details,
-		},
-	}
+	return status.FromProto(&spb.Status{
+		Code:    int32(c.Code),
+		Message: c.Message,
+		Details: c.Details,
+	})
 }
 
 func (s) TestFromErrorImplementsInterface(t *testing.T) {
@@ -341,10 +340,10 @@ func str(s *Status) string {
 	if s == nil {
 		return "nil"
 	}
-	if s.s == nil {
+	if s.Proto() == nil {
 		return "<Code=OK>"
 	}
-	return fmt.Sprintf("<Code=%v, Message=%q, Details=%+v>", codes.Code(s.s.GetCode()), s.s.GetMessage(), s.s.GetDetails())
+	return fmt.Sprintf("<Code=%v, Message=%q, Details=%+v>", s.Code(), s.Message(), s.Details())
 }
 
 // mustMarshalAny converts a protobuf message to an any.

From 98e4c7ad3eefd5c1a3cd647c004943ffab4f5722 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 6 Apr 2020 15:12:58 -0700
Subject: [PATCH 009/481] xds: move balancer group to a separate package
 (#3493)

This PR moves balancer group to a separate package and exports the type with some methods. Balancer group will be used by the weighted_target policy.
---
 .../balancergroup.go                          |  78 ++-
 .../balancergroup_test.go                     | 517 +++++++++---------
 .../balancer/balancergroup/testutils_test.go  |  38 ++
 xds/internal/balancer/edsbalancer/eds_impl.go |  25 +-
 .../balancer/edsbalancer/eds_impl_priority.go |   4 +-
 .../edsbalancer/eds_impl_priority_test.go     | 255 ++++-----
 .../balancer/edsbalancer/eds_impl_test.go     | 216 +++-----
 xds/internal/balancer/edsbalancer/eds_test.go |   7 +
 .../balancer/edsbalancer/test_util_test.go    | 350 ------------
 xds/internal/balancer/edsbalancer/util.go     |   2 +
 .../balancer/edsbalancer/util_test.go         |  47 +-
 xds/internal/testutils/balancer.go            | 357 ++++++++++++
 xds/internal/testutils/balancer_test.go       | 134 +++++
 13 files changed, 1078 insertions(+), 952 deletions(-)
 rename xds/internal/balancer/{edsbalancer => balancergroup}/balancergroup.go (89%)
 rename xds/internal/balancer/{edsbalancer => balancergroup}/balancergroup_test.go (57%)
 create mode 100644 xds/internal/balancer/balancergroup/testutils_test.go
 delete mode 100644 xds/internal/balancer/edsbalancer/test_util_test.go
 create mode 100644 xds/internal/testutils/balancer.go
 create mode 100644 xds/internal/testutils/balancer_test.go

diff --git a/xds/internal/balancer/edsbalancer/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
similarity index 89%
rename from xds/internal/balancer/edsbalancer/balancergroup.go
rename to xds/internal/balancer/balancergroup/balancergroup.go
index 28481cd7ccfb..ceb305d1f06c 100644
--- a/xds/internal/balancer/edsbalancer/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 
-package edsbalancer
+// Package balancergroup implements a utility struct to bind multiple balancers
+// into one balancer.
+package balancergroup
 
 import (
 	"fmt"
@@ -52,7 +54,7 @@ type subBalancerWithConfig struct {
 	// Some are forward to balancer group with the sub-balancer ID.
 	balancer.ClientConn
 	id    internal.Locality
-	group *balancerGroup
+	group *BalancerGroup
 
 	mu    sync.Mutex
 	state balancer.State
@@ -154,7 +156,7 @@ func (s *pickerState) String() string {
 	return fmt.Sprintf("weight:%v,picker:%p,state:%v", s.weight, s.picker, s.state)
 }
 
-// balancerGroup takes a list of balancers, and make then into one balancer.
+// BalancerGroup takes a list of balancers, and make them into one balancer.
 //
 // Note that this struct doesn't implement balancer.Balancer, because it's not
 // intended to be used directly as a balancer. It's expected to be used as a
@@ -178,7 +180,7 @@ func (s *pickerState) String() string {
 // balancer group is closed, the sub-balancers are also closed. And it's
 // guaranteed that no updates will be sent to parent ClientConn from a closed
 // balancer group.
-type balancerGroup struct {
+type BalancerGroup struct {
 	cc        balancer.ClientConn
 	logger    *grpclog.PrefixLogger
 	loadStore lrs.Store
@@ -226,26 +228,34 @@ type balancerGroup struct {
 	idToPickerState map[internal.Locality]*pickerState
 }
 
-// defaultSubBalancerCloseTimeout is defined as a variable instead of const for
+// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for
 // testing.
 //
-// TODO: make it a parameter for newBalancerGroup().
-var defaultSubBalancerCloseTimeout = 15 * time.Minute
+// TODO: make it a parameter for New().
+var DefaultSubBalancerCloseTimeout = 15 * time.Minute
 
-func newBalancerGroup(cc balancer.ClientConn, loadStore lrs.Store, logger *grpclog.PrefixLogger) *balancerGroup {
-	return &balancerGroup{
+// New creates a new BalancerGroup. Note that the BalancerGroup
+// needs to be started to work.
+func New(cc balancer.ClientConn, loadStore lrs.Store, logger *grpclog.PrefixLogger) *BalancerGroup {
+	return &BalancerGroup{
 		cc:        cc,
 		logger:    logger,
 		loadStore: loadStore,
 
 		idToBalancerConfig: make(map[internal.Locality]*subBalancerWithConfig),
-		balancerCache:      cache.NewTimeoutCache(defaultSubBalancerCloseTimeout),
+		balancerCache:      cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout),
 		scToSubBalancer:    make(map[balancer.SubConn]*subBalancerWithConfig),
 		idToPickerState:    make(map[internal.Locality]*pickerState),
 	}
 }
 
-func (bg *balancerGroup) start() {
+// Start starts the balancer group, including building all the sub-balancers,
+// and send the existing addresses to them.
+//
+// A BalancerGroup can be closed and started later. When a BalancerGroup is
+// closed, it can still receive address updates, which will be applied when
+// restarted.
+func (bg *BalancerGroup) Start() {
 	bg.incomingMu.Lock()
 	bg.incomingStarted = true
 	bg.incomingMu.Unlock()
@@ -263,12 +273,12 @@ func (bg *balancerGroup) start() {
 	bg.outgoingMu.Unlock()
 }
 
-// add adds a balancer built by builder to the group, with given id and weight.
+// Add adds a balancer built by builder to the group, with given id and weight.
 //
 // weight should never be zero.
-func (bg *balancerGroup) add(id internal.Locality, weight uint32, builder balancer.Builder) {
+func (bg *BalancerGroup) Add(id internal.Locality, weight uint32, builder balancer.Builder) {
 	if weight == 0 {
-		bg.logger.Errorf("balancerGroup.add called with weight 0, locality: %v. Locality is not added to balancer group", id)
+		bg.logger.Errorf("BalancerGroup.add called with weight 0, locality: %v. Locality is not added to balancer group", id)
 		return
 	}
 
@@ -329,7 +339,7 @@ func (bg *balancerGroup) add(id internal.Locality, weight uint32, builder balanc
 	bg.outgoingMu.Unlock()
 }
 
-// remove removes the balancer with id from the group.
+// Remove removes the balancer with id from the group.
 //
 // But doesn't close the balancer. The balancer is kept in a cache, and will be
 // closed after timeout. Cleanup work (closing sub-balancer and removing
@@ -337,7 +347,7 @@ func (bg *balancerGroup) add(id internal.Locality, weight uint32, builder balanc
 //
 // It also removes the picker generated from this balancer from the picker
 // group. It always results in a picker update.
-func (bg *balancerGroup) remove(id internal.Locality) {
+func (bg *BalancerGroup) Remove(id internal.Locality) {
 	bg.outgoingMu.Lock()
 	if sbToRemove, ok := bg.idToBalancerConfig[id]; ok {
 		if bg.outgoingStarted {
@@ -374,7 +384,7 @@ func (bg *balancerGroup) remove(id internal.Locality) {
 
 // bg.remove(id) doesn't do cleanup for the sub-balancer. This function does
 // cleanup after the timeout.
-func (bg *balancerGroup) cleanupSubConns(config *subBalancerWithConfig) {
+func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWithConfig) {
 	bg.incomingMu.Lock()
 	// Remove SubConns. This is only done after the balancer is
 	// actually closed.
@@ -393,16 +403,16 @@ func (bg *balancerGroup) cleanupSubConns(config *subBalancerWithConfig) {
 	bg.incomingMu.Unlock()
 }
 
-// changeWeight changes the weight of the balancer.
+// ChangeWeight changes the weight of the balancer.
 //
 // newWeight should never be zero.
 //
 // NOTE: It always results in a picker update now. This probably isn't
 // necessary. But it seems better to do the update because it's a change in the
 // picker (which is balancer's snapshot).
-func (bg *balancerGroup) changeWeight(id internal.Locality, newWeight uint32) {
+func (bg *BalancerGroup) ChangeWeight(id internal.Locality, newWeight uint32) {
 	if newWeight == 0 {
-		bg.logger.Errorf("balancerGroup.changeWeight called with newWeight 0. Weight is not changed")
+		bg.logger.Errorf("BalancerGroup.changeWeight called with newWeight 0. Weight is not changed")
 		return
 	}
 	bg.incomingMu.Lock()
@@ -425,8 +435,9 @@ func (bg *balancerGroup) changeWeight(id internal.Locality, newWeight uint32) {
 
 // Following are actions from the parent grpc.ClientConn, forward to sub-balancers.
 
-// SubConn state change: find the corresponding balancer and then forward.
-func (bg *balancerGroup) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+// HandleSubConnStateChange handles the state for the subconn. It finds the
+// corresponding balancer and forwards the update.
+func (bg *BalancerGroup) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
 	bg.incomingMu.Lock()
 	config, ok := bg.scToSubBalancer[sc]
 	if !ok {
@@ -444,8 +455,12 @@ func (bg *balancerGroup) handleSubConnStateChange(sc balancer.SubConn, state con
 	bg.outgoingMu.Unlock()
 }
 
-// Address change: forward to balancer.
-func (bg *balancerGroup) handleResolvedAddrs(id internal.Locality, addrs []resolver.Address) {
+// HandleResolvedAddrs handles addresses from resolver. It finds the balancer
+// and forwards the update.
+//
+// TODO: change this to UpdateClientConnState to handle addresses and balancer
+// config.
+func (bg *BalancerGroup) HandleResolvedAddrs(id internal.Locality, addrs []resolver.Address) {
 	bg.outgoingMu.Lock()
 	if config, ok := bg.idToBalancerConfig[id]; ok {
 		config.updateAddrs(addrs)
@@ -467,7 +482,7 @@ func (bg *balancerGroup) handleResolvedAddrs(id internal.Locality, addrs []resol
 // from map. Delete sc from the map only when state changes to Shutdown. Since
 // it's just forwarding the action, there's no need for a removeSubConn()
 // wrapper function.
-func (bg *balancerGroup) newSubConn(config *subBalancerWithConfig, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+func (bg *BalancerGroup) newSubConn(config *subBalancerWithConfig, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
 	// NOTE: if balancer with id was already removed, this should also return
 	// error. But since we call balancer.stopBalancer when removing the balancer, this
 	// shouldn't happen.
@@ -488,7 +503,7 @@ func (bg *balancerGroup) newSubConn(config *subBalancerWithConfig, addrs []resol
 
 // updateBalancerState: create an aggregated picker and an aggregated
 // connectivity state, then forward to ClientConn.
-func (bg *balancerGroup) updateBalancerState(id internal.Locality, state balancer.State) {
+func (bg *BalancerGroup) updateBalancerState(id internal.Locality, state balancer.State) {
 	bg.logger.Infof("Balancer state update from locality %v, new state: %+v", id, state)
 
 	bg.incomingMu.Lock()
@@ -508,7 +523,9 @@ func (bg *balancerGroup) updateBalancerState(id internal.Locality, state balance
 	}
 }
 
-func (bg *balancerGroup) close() {
+// Close closes the balancer. It stops sub-balancers, and removes the subconns.
+// The BalancerGroup can be restarted later.
+func (bg *BalancerGroup) Close() {
 	bg.incomingMu.Lock()
 	if bg.incomingStarted {
 		bg.incomingStarted = false
@@ -568,8 +585,9 @@ func buildPickerAndState(m map[internal.Locality]*pickerState) balancer.State {
 	return balancer.State{ConnectivityState: aggregatedState, Picker: newPickerGroup(readyPickerWithWeights)}
 }
 
-// RandomWRR constructor, to be modified in tests.
-var newRandomWRR = wrr.NewRandom
+// NewRandomWRR is the WRR constructor used to pick sub-pickers from
+// sub-balancers. It's to be modified in tests.
+var NewRandomWRR = wrr.NewRandom
 
 type pickerGroup struct {
 	length int
@@ -584,7 +602,7 @@ type pickerGroup struct {
 // TODO: (bg) confirm this is the expected behavior: non-ready balancers should
 // be ignored when picking. Only ready balancers are picked.
 func newPickerGroup(readyPickerWithWeights []pickerState) *pickerGroup {
-	w := newRandomWRR()
+	w := NewRandomWRR()
 	for _, ps := range readyPickerWithWeights {
 		w.Add(ps.picker, int64(ps.weight))
 	}
diff --git a/xds/internal/balancer/edsbalancer/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
similarity index 57%
rename from xds/internal/balancer/edsbalancer/balancergroup_test.go
rename to xds/internal/balancer/balancergroup/balancergroup_test.go
index c1d438afd123..3707f687d2bc 100644
--- a/xds/internal/balancer/edsbalancer/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package edsbalancer
+package balancergroup
 
 import (
 	"fmt"
@@ -28,6 +28,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 var (
@@ -45,7 +46,7 @@ func init() {
 
 	// Disable caching for all tests. It will be re-enabled in caching specific
 	// tests.
-	defaultSubBalancerCloseTimeout = time.Millisecond
+	DefaultSubBalancerCloseTimeout = time.Millisecond
 }
 
 func subConnFromPicker(p balancer.V2Picker) func() balancer.SubConn {
@@ -57,56 +58,56 @@ func subConnFromPicker(p balancer.V2Picker) func() balancer.SubConn {
 
 // 1 balancer, 1 backend -> 2 backends -> 1 backend.
 func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
 
 	// Add one balancer to group.
-	bg.add(testBalancerIDs[0], 1, rrBuilder)
+	bg.Add(testBalancerIDs[0], 1, rrBuilder)
 	// Send one resolved address.
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
 
 	// Send subconn state change.
-	sc1 := <-cc.newSubConnCh
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
+	sc1 := <-cc.NewSubConnCh
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test pick with one backend.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
 		}
 	}
 
 	// Send two addresses.
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
 	// Expect one new subconn, send state update.
-	sc2 := <-cc.newSubConnCh
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
+	sc2 := <-cc.NewSubConnCh
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin pick.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Remove the first address.
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[1:2])
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testSubConn{})) {
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[1:2])
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
-	bg.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	bg.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with only the second subconn.
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSC, _ := p3.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSC.SubConn, sc2, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSC.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSC, sc2)
 		}
 	}
@@ -114,103 +115,103 @@ func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
 
 // 2 balancers, each with 1 backend.
 func (s) TestBalancerGroup_TwoRR_OneBackend(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
 
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
-	sc1 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
+	sc1 := <-cc.NewSubConnCh
 
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[0:1])
-	sc2 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[0:1])
+	sc2 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
 
 // 2 balancers, each with more than 1 backends.
 func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
 
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	sc1 := <-cc.newSubConnCh
-	sc2 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	sc1 := <-cc.NewSubConnCh
+	sc2 := <-cc.NewSubConnCh
 
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
-	sc3 := <-cc.newSubConnCh
-	sc4 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	sc3 := <-cc.NewSubConnCh
+	sc4 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
-	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc3, connectivity.Ready)
-	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc4, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Turn sc2's connection down, should be RR between balancers.
-	bg.handleSubConnStateChange(sc2, connectivity.TransientFailure)
-	p2 := <-cc.newPickerCh
+	bg.HandleSubConnStateChange(sc2, connectivity.TransientFailure)
+	p2 := <-cc.NewPickerCh
 	// Expect two sc1's in the result, because balancer1 will be picked twice,
 	// but there's only one sc in it.
 	want = []balancer.SubConn{sc1, sc1, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Remove sc3's addresses.
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[3:4])
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc3, cmp.AllowUnexported(testSubConn{})) {
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[3:4])
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc3, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc3, scToRemove)
 	}
-	bg.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
-	p3 := <-cc.newPickerCh
+	bg.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	p3 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc1, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p3)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Turn sc1's connection down.
-	bg.handleSubConnStateChange(sc1, connectivity.TransientFailure)
-	p4 := <-cc.newPickerCh
+	bg.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
+	p4 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p4)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Turn last connection to connecting.
-	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
-	p5 := <-cc.newPickerCh
+	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
+	p5 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := p5.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
 			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
@@ -218,8 +219,8 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	}
 
 	// Turn all connections down.
-	bg.handleSubConnStateChange(sc4, connectivity.TransientFailure)
-	p6 := <-cc.newPickerCh
+	bg.HandleSubConnStateChange(sc4, connectivity.TransientFailure)
+	p6 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := p6.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
 			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
@@ -229,95 +230,95 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 
 // 2 balancers with different weights.
 func (s) TestBalancerGroup_TwoRR_DifferentWeight_MoreBackends(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 2, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	sc1 := <-cc.newSubConnCh
-	sc2 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	sc1 := <-cc.NewSubConnCh
+	sc2 := <-cc.NewSubConnCh
 
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
-	sc3 := <-cc.newSubConnCh
-	sc4 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	sc3 := <-cc.NewSubConnCh
+	sc4 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
-	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc3, connectivity.Ready)
-	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc4, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
 
 // totally 3 balancers, add/remove balancer.
 func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
 
 	// Add three balancers to group and send one resolved address to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
-	sc1 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
+	sc1 := <-cc.NewSubConnCh
 
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[1:2])
-	sc2 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[1:2])
+	sc2 := <-cc.NewSubConnCh
 
-	bg.add(testBalancerIDs[2], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:2])
-	sc3 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[2], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:2])
+	sc3 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
-	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc3, connectivity.Ready)
-
-	p1 := <-cc.newPickerCh
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
+
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2, sc3}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Remove the second balancer, while the others two are ready.
-	bg.remove(testBalancerIDs[1])
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testSubConn{})) {
+	bg.Remove(testBalancerIDs[1])
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
 	}
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc1, sc3}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// move balancer 3 into transient failure.
-	bg.handleSubConnStateChange(sc3, connectivity.TransientFailure)
+	bg.HandleSubConnStateChange(sc3, connectivity.TransientFailure)
 	// Remove the first balancer, while the third is transient failure.
-	bg.remove(testBalancerIDs[0])
-	scToRemove = <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testSubConn{})) {
+	bg.Remove(testBalancerIDs[0])
+	scToRemove = <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := p3.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
 			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
@@ -327,90 +328,90 @@ func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
 
 // 2 balancers, change balancer weight.
 func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 2, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	sc1 := <-cc.newSubConnCh
-	sc2 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	sc1 := <-cc.NewSubConnCh
+	sc2 := <-cc.NewSubConnCh
 
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
-	sc3 := <-cc.newSubConnCh
-	sc4 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	sc3 := <-cc.NewSubConnCh
+	sc4 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
-	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc3, connectivity.Ready)
-	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc4, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
-	bg.changeWeight(testBalancerIDs[0], 3)
+	bg.ChangeWeight(testBalancerIDs[0], 3)
 
 	// Test roundrobin with new weight.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc2, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
 
 func (s) TestBalancerGroup_LoadReport(t *testing.T) {
-	testLoadStore := newTestLoadStore()
+	testLoadStore := testutils.NewTestLoadStore()
 
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, testLoadStore, nil)
-	bg.start()
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, testLoadStore, nil)
+	bg.Start()
 
 	backendToBalancerID := make(map[balancer.SubConn]internal.Locality)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 2, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	sc1 := <-cc.newSubConnCh
-	sc2 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	sc1 := <-cc.NewSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	backendToBalancerID[sc1] = testBalancerIDs[0]
 	backendToBalancerID[sc2] = testBalancerIDs[0]
 
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
-	sc3 := <-cc.newSubConnCh
-	sc4 := <-cc.newSubConnCh
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	sc3 := <-cc.NewSubConnCh
+	sc4 := <-cc.NewSubConnCh
 	backendToBalancerID[sc3] = testBalancerIDs[1]
 	backendToBalancerID[sc4] = testBalancerIDs[1]
 
 	// Send state changes for both subconns.
-	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc1, connectivity.Ready)
-	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc2, connectivity.Ready)
-	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc3, connectivity.Ready)
-	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.handleSubConnStateChange(sc4, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
+	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
+	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	var (
 		wantStart []internal.Locality
 		wantEnd   []internal.Locality
-		wantCost  []testServerLoad
+		wantCost  []testutils.TestServerLoad
 	)
 	for i := 0; i < 10; i++ {
 		scst, _ := p1.Pick(balancer.PickInfo{})
@@ -427,21 +428,21 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 			})
 			wantEnd = append(wantEnd, locality)
 			wantCost = append(wantCost,
-				testServerLoad{name: serverLoadCPUName, d: 10},
-				testServerLoad{name: serverLoadMemoryName, d: 5},
-				testServerLoad{name: "pic", d: 3.14},
-				testServerLoad{name: "piu", d: 3.14})
+				testutils.TestServerLoad{Name: serverLoadCPUName, D: 10},
+				testutils.TestServerLoad{Name: serverLoadMemoryName, D: 5},
+				testutils.TestServerLoad{Name: "pic", D: 3.14},
+				testutils.TestServerLoad{Name: "piu", D: 3.14})
 		}
 	}
 
-	if !cmp.Equal(testLoadStore.callsStarted, wantStart) {
-		t.Fatalf("want started: %v, got: %v", testLoadStore.callsStarted, wantStart)
+	if !cmp.Equal(testLoadStore.CallsStarted, wantStart) {
+		t.Fatalf("want started: %v, got: %v", testLoadStore.CallsStarted, wantStart)
 	}
-	if !cmp.Equal(testLoadStore.callsEnded, wantEnd) {
-		t.Fatalf("want ended: %v, got: %v", testLoadStore.callsEnded, wantEnd)
+	if !cmp.Equal(testLoadStore.CallsEnded, wantEnd) {
+		t.Fatalf("want ended: %v, got: %v", testLoadStore.CallsEnded, wantEnd)
 	}
-	if !cmp.Equal(testLoadStore.callsCost, wantCost, cmp.AllowUnexported(testServerLoad{})) {
-		t.Fatalf("want cost: %v, got: %v", testLoadStore.callsCost, wantCost)
+	if !cmp.Equal(testLoadStore.CallsCost, wantCost, cmp.AllowUnexported(testutils.TestServerLoad{})) {
+		t.Fatalf("want cost: %v, got: %v", testLoadStore.CallsCost, wantCost)
 	}
 }
 
@@ -455,73 +456,73 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 // - b3, weight 1, backends [1,2]
 // Start the balancer group again and check for behavior.
 func (s) TestBalancerGroup_start_close(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 2, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
 
-	bg.start()
+	bg.Start()
 
 	m1 := make(map[resolver.Address]balancer.SubConn)
 	for i := 0; i < 4; i++ {
-		addrs := <-cc.newSubConnAddrsCh
-		sc := <-cc.newSubConnCh
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
 		m1[addrs[0]] = sc
-		bg.handleSubConnStateChange(sc, connectivity.Connecting)
-		bg.handleSubConnStateChange(sc, connectivity.Ready)
+		bg.HandleSubConnStateChange(sc, connectivity.Connecting)
+		bg.HandleSubConnStateChange(sc, connectivity.Ready)
 	}
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{
 		m1[testBackendAddrs[0]], m1[testBackendAddrs[0]],
 		m1[testBackendAddrs[1]], m1[testBackendAddrs[1]],
 		m1[testBackendAddrs[2]], m1[testBackendAddrs[3]],
 	}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
-	bg.close()
+	bg.Close()
 	for i := 0; i < 4; i++ {
-		bg.handleSubConnStateChange(<-cc.removeSubConnCh, connectivity.Shutdown)
+		bg.HandleSubConnStateChange(<-cc.RemoveSubConnCh, connectivity.Shutdown)
 	}
 
 	// Add b3, weight 1, backends [1,2].
-	bg.add(testBalancerIDs[2], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:3])
+	bg.Add(testBalancerIDs[2], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:3])
 
 	// Remove b1.
-	bg.remove(testBalancerIDs[0])
+	bg.Remove(testBalancerIDs[0])
 
 	// Update b2 to weight 3, backends [0,3].
-	bg.changeWeight(testBalancerIDs[1], 3)
-	bg.handleResolvedAddrs(testBalancerIDs[1], append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3]))
+	bg.ChangeWeight(testBalancerIDs[1], 3)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3]))
 
-	bg.start()
+	bg.Start()
 
 	m2 := make(map[resolver.Address]balancer.SubConn)
 	for i := 0; i < 4; i++ {
-		addrs := <-cc.newSubConnAddrsCh
-		sc := <-cc.newSubConnCh
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
 		m2[addrs[0]] = sc
-		bg.handleSubConnStateChange(sc, connectivity.Connecting)
-		bg.handleSubConnStateChange(sc, connectivity.Ready)
+		bg.HandleSubConnStateChange(sc, connectivity.Connecting)
+		bg.HandleSubConnStateChange(sc, connectivity.Ready)
 	}
 
 	// Test roundrobin on the last picker.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{
 		m2[testBackendAddrs[0]], m2[testBackendAddrs[0]], m2[testBackendAddrs[0]],
 		m2[testBackendAddrs[3]], m2[testBackendAddrs[3]], m2[testBackendAddrs[3]],
 		m2[testBackendAddrs[1]], m2[testBackendAddrs[2]],
 	}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
@@ -539,21 +540,21 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 // whenever it gets an address update. It's expected that start() doesn't block
 // because of deadlock.
 func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
 
-	bg.add(testBalancerIDs[0], 2, &testConstBalancerBuilder{})
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	bg.add(testBalancerIDs[1], 1, &testConstBalancerBuilder{})
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.Add(testBalancerIDs[0], 2, &testutils.TestConstBalancerBuilder{})
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.Add(testBalancerIDs[1], 1, &testutils.TestConstBalancerBuilder{})
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
 
-	bg.start()
+	bg.Start()
 }
 
 func replaceDefaultSubBalancerCloseTimeout(n time.Duration) func() {
-	old := defaultSubBalancerCloseTimeout
-	defaultSubBalancerCloseTimeout = n
-	return func() { defaultSubBalancerCloseTimeout = old }
+	old := DefaultSubBalancerCloseTimeout
+	DefaultSubBalancerCloseTimeout = n
+	return func() { DefaultSubBalancerCloseTimeout = old }
 }
 
 // initBalancerGroupForCachingTest creates a balancer group, and initialize it
@@ -562,56 +563,56 @@ func replaceDefaultSubBalancerCloseTimeout(n time.Duration) func() {
 // Two rr balancers are added to bg, each with 2 ready subConns. A sub-balancer
 // is removed later, so the balancer group returned has one sub-balancer in its
 // own map, and one sub-balancer in cache.
-func initBalancerGroupForCachingTest(t *testing.T) (*balancerGroup, *testClientConn, map[resolver.Address]balancer.SubConn) {
-	cc := newTestClientConn(t)
-	bg := newBalancerGroup(cc, nil, nil)
+func initBalancerGroupForCachingTest(t *testing.T) (*BalancerGroup, *testutils.TestClientConn, map[resolver.Address]balancer.SubConn) {
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.add(testBalancerIDs[0], 2, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
 
-	bg.start()
+	bg.Start()
 
 	m1 := make(map[resolver.Address]balancer.SubConn)
 	for i := 0; i < 4; i++ {
-		addrs := <-cc.newSubConnAddrsCh
-		sc := <-cc.newSubConnCh
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
 		m1[addrs[0]] = sc
-		bg.handleSubConnStateChange(sc, connectivity.Connecting)
-		bg.handleSubConnStateChange(sc, connectivity.Ready)
+		bg.HandleSubConnStateChange(sc, connectivity.Connecting)
+		bg.HandleSubConnStateChange(sc, connectivity.Ready)
 	}
 
 	// Test roundrobin on the last picker.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{
 		m1[testBackendAddrs[0]], m1[testBackendAddrs[0]],
 		m1[testBackendAddrs[1]], m1[testBackendAddrs[1]],
 		m1[testBackendAddrs[2]], m1[testBackendAddrs[3]],
 	}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
-	bg.remove(testBalancerIDs[1])
+	bg.Remove(testBalancerIDs[1])
 	// Don't wait for SubConns to be removed after close, because they are only
 	// removed after close timeout.
 	for i := 0; i < 10; i++ {
 		select {
-		case <-cc.removeSubConnCh:
+		case <-cc.RemoveSubConnCh:
 			t.Fatalf("Got request to remove subconn, want no remove subconn (because subconns were still in cache)")
 		default:
 		}
 		time.Sleep(time.Millisecond)
 	}
 	// Test roundrobin on the with only sub-balancer0.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{
 		m1[testBackendAddrs[0]], m1[testBackendAddrs[1]],
 	}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -626,10 +627,10 @@ func (s) TestBalancerGroup_locality_caching(t *testing.T) {
 
 	// Turn down subconn for addr2, shouldn't get picker update because
 	// sub-balancer1 was removed.
-	bg.handleSubConnStateChange(addrToSC[testBackendAddrs[2]], connectivity.TransientFailure)
+	bg.HandleSubConnStateChange(addrToSC[testBackendAddrs[2]], connectivity.TransientFailure)
 	for i := 0; i < 10; i++ {
 		select {
-		case <-cc.newPickerCh:
+		case <-cc.NewPickerCh:
 			t.Fatalf("Got new picker, want no new picker (because the sub-balancer was removed)")
 		default:
 		}
@@ -642,22 +643,22 @@ func (s) TestBalancerGroup_locality_caching(t *testing.T) {
 	// Re-add sub-balancer-1, because subconns were in cache, no new subconns
 	// should be created. But a new picker will still be generated, with subconn
 	// states update to date.
-	bg.add(testBalancerIDs[1], 1, rrBuilder)
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
 
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	want := []balancer.SubConn{
 		addrToSC[testBackendAddrs[0]], addrToSC[testBackendAddrs[0]],
 		addrToSC[testBackendAddrs[1]], addrToSC[testBackendAddrs[1]],
 		// addr2 is down, b2 only has addr3 in READY state.
 		addrToSC[testBackendAddrs[3]], addrToSC[testBackendAddrs[3]],
 	}
-	if err := isRoundRobin(want, subConnFromPicker(p3)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	for i := 0; i < 10; i++ {
 		select {
-		case <-cc.newSubConnAddrsCh:
+		case <-cc.NewSubConnAddrsCh:
 			t.Fatalf("Got new subconn, want no new subconn (because subconns were still in cache)")
 		default:
 		}
@@ -672,7 +673,7 @@ func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) {
 	defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)()
 	bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
 
-	bg.close()
+	bg.Close()
 	// The balancer group is closed. The subconns should be removed immediately.
 	removeTimeout := time.After(time.Millisecond * 500)
 	scToRemove := map[balancer.SubConn]int{
@@ -683,7 +684,7 @@ func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) {
 	}
 	for i := 0; i < len(scToRemove); i++ {
 		select {
-		case sc := <-cc.removeSubConnCh:
+		case sc := <-cc.RemoveSubConnCh:
 			c := scToRemove[sc]
 			if c == 0 {
 				t.Fatalf("Got removeSubConn for %v when there's %d remove expected", sc, c)
@@ -703,14 +704,14 @@ func (s) TestBalancerGroup_locality_caching_not_readd_within_timeout(t *testing.
 
 	// The sub-balancer is not re-added withtin timeout. The subconns should be
 	// removed.
-	removeTimeout := time.After(defaultSubBalancerCloseTimeout)
+	removeTimeout := time.After(DefaultSubBalancerCloseTimeout)
 	scToRemove := map[balancer.SubConn]int{
 		addrToSC[testBackendAddrs[2]]: 1,
 		addrToSC[testBackendAddrs[3]]: 1,
 	}
 	for i := 0; i < len(scToRemove); i++ {
 		select {
-		case sc := <-cc.removeSubConnCh:
+		case sc := <-cc.RemoveSubConnCh:
 			c := scToRemove[sc]
 			if c == 0 {
 				t.Fatalf("Got removeSubConn for %v when there's %d remove expected", sc, c)
@@ -737,7 +738,7 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 	// sub-balancer was still in cache, but cann't be reused. This should cause
 	// old sub-balancer's subconns to be removed immediately, and new subconns
 	// to be created.
-	bg.add(testBalancerIDs[1], 1, &noopBalancerBuilderWrapper{rrBuilder})
+	bg.Add(testBalancerIDs[1], 1, &noopBalancerBuilderWrapper{rrBuilder})
 
 	// The cached sub-balancer should be closed, and the subconns should be
 	// removed immediately.
@@ -748,7 +749,7 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 	}
 	for i := 0; i < len(scToRemove); i++ {
 		select {
-		case sc := <-cc.removeSubConnCh:
+		case sc := <-cc.RemoveSubConnCh:
 			c := scToRemove[sc]
 			if c == 0 {
 				t.Fatalf("Got removeSubConn for %v when there's %d remove expected", sc, c)
@@ -759,7 +760,7 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 		}
 	}
 
-	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[4:6])
+	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[4:6])
 
 	newSCTimeout := time.After(time.Millisecond * 500)
 	scToAdd := map[resolver.Address]int{
@@ -768,29 +769,29 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 	}
 	for i := 0; i < len(scToAdd); i++ {
 		select {
-		case addr := <-cc.newSubConnAddrsCh:
+		case addr := <-cc.NewSubConnAddrsCh:
 			c := scToAdd[addr[0]]
 			if c == 0 {
 				t.Fatalf("Got newSubConn for %v when there's %d new expected", addr, c)
 			}
 			scToAdd[addr[0]] = c - 1
-			sc := <-cc.newSubConnCh
+			sc := <-cc.NewSubConnCh
 			addrToSC[addr[0]] = sc
-			bg.handleSubConnStateChange(sc, connectivity.Connecting)
-			bg.handleSubConnStateChange(sc, connectivity.Ready)
+			bg.HandleSubConnStateChange(sc, connectivity.Connecting)
+			bg.HandleSubConnStateChange(sc, connectivity.Ready)
 		case <-newSCTimeout:
 			t.Fatalf("timeout waiting for subConns (from new sub-balancer) to be newed")
 		}
 	}
 
 	// Test roundrobin on the new picker.
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	want := []balancer.SubConn{
 		addrToSC[testBackendAddrs[0]], addrToSC[testBackendAddrs[0]],
 		addrToSC[testBackendAddrs[1]], addrToSC[testBackendAddrs[1]],
 		addrToSC[testBackendAddrs[4]], addrToSC[testBackendAddrs[5]],
 	}
-	if err := isRoundRobin(want, subConnFromPicker(p3)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
diff --git a/xds/internal/balancer/balancergroup/testutils_test.go b/xds/internal/balancer/balancergroup/testutils_test.go
new file mode 100644
index 000000000000..bca79bf180af
--- /dev/null
+++ b/xds/internal/balancer/balancergroup/testutils_test.go
@@ -0,0 +1,38 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package balancergroup
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func init() {
+	NewRandomWRR = testutils.NewTestWRR
+}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index fcafbbdbdb9f..955421821193 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
@@ -46,7 +47,7 @@ type localityConfig struct {
 // balancerGroupWithConfig contains the localities with the same priority. It
 // manages all localities using a balancerGroup.
 type balancerGroupWithConfig struct {
-	bg      *balancerGroup
+	bg      *balancergroup.BalancerGroup
 	configs map[internal.Locality]*localityConfig
 }
 
@@ -136,9 +137,9 @@ func (edsImpl *edsBalancerImpl) HandleChildPolicy(name string, config json.RawMe
 			// TODO: (eds) add support to balancer group to support smoothly
 			//  switching sub-balancers (keep old balancer around until new
 			//  balancer becomes ready).
-			bgwc.bg.remove(id)
-			bgwc.bg.add(id, config.weight, edsImpl.subBalancerBuilder)
-			bgwc.bg.handleResolvedAddrs(id, config.addrs)
+			bgwc.bg.Remove(id)
+			bgwc.bg.Add(id, config.weight, edsImpl.subBalancerBuilder)
+			bgwc.bg.HandleResolvedAddrs(id, config.addrs)
 		}
 	}
 }
@@ -217,7 +218,7 @@ func (edsImpl *edsBalancerImpl) HandleEDSResponse(edsResp *xdsclient.EDSUpdate)
 			// be started when necessary (e.g. when higher is down, or if it's a
 			// new lowest priority).
 			bgwc = &balancerGroupWithConfig{
-				bg:      newBalancerGroup(edsImpl.ccWrapperWithPriority(priority), edsImpl.loadStore, edsImpl.logger),
+				bg:      balancergroup.New(edsImpl.ccWrapperWithPriority(priority), edsImpl.loadStore, edsImpl.logger),
 				configs: make(map[internal.Locality]*localityConfig),
 			}
 			edsImpl.priorityToLocalities[priority] = bgwc
@@ -233,7 +234,7 @@ func (edsImpl *edsBalancerImpl) HandleEDSResponse(edsResp *xdsclient.EDSUpdate)
 	for p, bgwc := range edsImpl.priorityToLocalities {
 		if _, ok := newLocalitiesWithPriority[p]; !ok {
 			delete(edsImpl.priorityToLocalities, p)
-			bgwc.bg.close()
+			bgwc.bg.Close()
 			delete(edsImpl.priorityToState, p)
 			priorityChanged = true
 			edsImpl.logger.Infof("Priority %v deleted", p)
@@ -284,7 +285,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 		config, ok := bgwc.configs[lid]
 		if !ok {
 			// A new balancer, add it to balancer group and balancer map.
-			bgwc.bg.add(lid, newWeight, edsImpl.subBalancerBuilder)
+			bgwc.bg.Add(lid, newWeight, edsImpl.subBalancerBuilder)
 			config = &localityConfig{
 				weight: newWeight,
 			}
@@ -307,19 +308,19 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 
 		if weightChanged {
 			config.weight = newWeight
-			bgwc.bg.changeWeight(lid, newWeight)
+			bgwc.bg.ChangeWeight(lid, newWeight)
 		}
 
 		if addrsChanged {
 			config.addrs = newAddrs
-			bgwc.bg.handleResolvedAddrs(lid, newAddrs)
+			bgwc.bg.HandleResolvedAddrs(lid, newAddrs)
 		}
 	}
 
 	// Delete localities that are removed in the latest response.
 	for lid := range bgwc.configs {
 		if _, ok := newLocalitiesSet[lid]; !ok {
-			bgwc.bg.remove(lid)
+			bgwc.bg.Remove(lid)
 			delete(bgwc.configs, lid)
 			edsImpl.logger.Infof("Locality %v deleted", lid)
 		}
@@ -343,7 +344,7 @@ func (edsImpl *edsBalancerImpl) HandleSubConnStateChange(sc balancer.SubConn, s
 		return
 	}
 	if bg := bgwc.bg; bg != nil {
-		bg.handleSubConnStateChange(sc, s)
+		bg.HandleSubConnStateChange(sc, s)
 	}
 }
 
@@ -407,7 +408,7 @@ func (edsImpl *edsBalancerImpl) newSubConn(priority priorityType, addrs []resolv
 func (edsImpl *edsBalancerImpl) Close() {
 	for _, bgwc := range edsImpl.priorityToLocalities {
 		if bg := bgwc.bg; bg != nil {
-			bg.close()
+			bg.Close()
 		}
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index 2935e46f4265..a7d65a13854b 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -102,7 +102,7 @@ func (edsImpl *edsBalancerImpl) startPriority(priority priorityType) {
 	// currently avoided by handling balancer update in a goroutine (the run
 	// goroutine in the parent eds balancer). When priority balancer is split
 	// into its own, this asynchronous state handling needs to be copied.
-	p.bg.start()
+	p.bg.Start()
 	// startPriority can be called when
 	// 1. first EDS resp, start p0
 	// 2. a high priority goes Failure, start next
@@ -189,7 +189,7 @@ func (edsImpl *edsBalancerImpl) handlePriorityWithNewStateReady(priority priorit
 		edsImpl.logger.Infof("Switching priority from %v to %v, because latter became Ready", edsImpl.priorityInUse, priority)
 		edsImpl.priorityInUse = priority
 		for i := priority.nextLower(); !i.lowerThan(edsImpl.priorityLowest); i = i.nextLower() {
-			edsImpl.priorityToLocalities[i].bg.close()
+			edsImpl.priorityToLocalities[i].bg.Close()
 		}
 		return true
 	}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index cdb33300a00f..519b8e588ab7 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -26,6 +26,7 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 // When a high priority is ready, adding/removing lower locality doesn't cause
@@ -33,7 +34,7 @@ import (
 //
 // Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0.
 func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -43,20 +44,20 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 
 	// p0 is ready.
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -68,11 +69,11 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	select {
-	case <-cc.newPickerCh:
+	case <-cc.NewPickerCh:
 		t.Fatalf("got unexpected new picker")
-	case <-cc.newSubConnCh:
+	case <-cc.NewSubConnCh:
 		t.Fatalf("got unexpected new SubConn")
-	case <-cc.removeSubConnCh:
+	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
 	case <-time.After(time.Millisecond * 100):
 	}
@@ -84,11 +85,11 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	select {
-	case <-cc.newPickerCh:
+	case <-cc.NewPickerCh:
 		t.Fatalf("got unexpected new picker")
-	case <-cc.newSubConnCh:
+	case <-cc.NewSubConnCh:
 		t.Fatalf("got unexpected new SubConn")
-	case <-cc.removeSubConnCh:
+	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
 	case <-time.After(time.Millisecond * 100):
 	}
@@ -99,7 +100,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 // Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is
 // down, use 2; remove 2, use 1.
 func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -109,38 +110,38 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs0 := <-cc.newSubConnAddrsCh
+	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc0 := <-cc.newSubConnCh
+	sc0 := <-cc.NewSubConnCh
 
 	// p0 is ready.
 	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p0 := <-cc.newPickerCh
+	p0 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc0}
-	if err := isRoundRobin(want, subConnFromPicker(p0)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p0)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Turn down 0, 1 is used.
 	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test pick with 1.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
 		}
 	}
@@ -153,30 +154,30 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	select {
-	case <-cc.newPickerCh:
+	case <-cc.NewPickerCh:
 		t.Fatalf("got unexpected new picker")
-	case <-cc.newSubConnCh:
+	case <-cc.NewSubConnCh:
 		t.Fatalf("got unexpected new SubConn")
-	case <-cc.removeSubConnCh:
+	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
 	case <-time.After(time.Millisecond * 100):
 	}
 
 	// Turn down 1, use 2
 	edsb.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
-	addrs2 := <-cc.newSubConnAddrsCh
+	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test pick with 2.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
 		}
 	}
@@ -188,13 +189,13 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	// p2 SubConns are removed.
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
 	}
 
 	// Should get an update with 1's old picker, to override 2's old picker.
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := p3.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
 			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
@@ -206,7 +207,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 //
 // Init 0 and 1; 0 and 1 both down; add 2, use 2.
 func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -216,24 +217,24 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs0 := <-cc.newSubConnAddrsCh
+	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc0 := <-cc.newSubConnCh
+	sc0 := <-cc.NewSubConnCh
 
 	// Turn down 0, 1 is used.
 	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	// Turn down 1, pick should error.
 	edsb.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
 
 	// Test pick failure.
-	pFail := <-cc.newPickerCh
+	pFail := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := pFail.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
 			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
@@ -247,19 +248,19 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
-	addrs2 := <-cc.newSubConnAddrsCh
+	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test pick with 2.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
 		}
 	}
@@ -271,7 +272,7 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	defer time.Sleep(10 * time.Millisecond)
 
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -282,34 +283,34 @@ func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	clab1.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs0 := <-cc.newSubConnAddrsCh
+	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc0 := <-cc.newSubConnCh
+	sc0 := <-cc.NewSubConnCh
 
 	// Turn down 0, 1 is used.
 	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	// Turn down 1, 2 is used.
 	edsb.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
-	addrs2 := <-cc.newSubConnAddrsCh
+	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test pick with 2.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
 		}
 	}
@@ -321,19 +322,19 @@ func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	//
 	// With localities caching, the lower priorities are closed after a timeout,
 	// in goroutines. The order is no longer guaranteed.
-	scToRemove := []balancer.SubConn{<-cc.removeSubConnCh, <-cc.removeSubConnCh}
-	if !(cmp.Equal(scToRemove[0], sc1, cmp.AllowUnexported(testSubConn{})) &&
-		cmp.Equal(scToRemove[1], sc2, cmp.AllowUnexported(testSubConn{}))) &&
-		!(cmp.Equal(scToRemove[0], sc2, cmp.AllowUnexported(testSubConn{})) &&
-			cmp.Equal(scToRemove[1], sc1, cmp.AllowUnexported(testSubConn{}))) {
+	scToRemove := []balancer.SubConn{<-cc.RemoveSubConnCh, <-cc.RemoveSubConnCh}
+	if !(cmp.Equal(scToRemove[0], sc1, cmp.AllowUnexported(testutils.TestSubConn{})) &&
+		cmp.Equal(scToRemove[1], sc2, cmp.AllowUnexported(testutils.TestSubConn{}))) &&
+		!(cmp.Equal(scToRemove[0], sc2, cmp.AllowUnexported(testutils.TestSubConn{})) &&
+			cmp.Equal(scToRemove[1], sc1, cmp.AllowUnexported(testutils.TestSubConn{}))) {
 		t.Errorf("RemoveSubConn, want [%v, %v], got %v", sc1, sc2, scToRemove)
 	}
 
 	// Test pick with 0.
-	p0 := <-cc.newPickerCh
+	p0 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p0.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc0, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0)
 		}
 	}
@@ -353,7 +354,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 		}
 	}()()
 
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -363,11 +364,11 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs0 := <-cc.newSubConnAddrsCh
+	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc0 := <-cc.newSubConnCh
+	sc0 := <-cc.NewSubConnCh
 
 	// Keep 0 in connecting, 1 will be used after init timeout.
 	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
@@ -375,24 +376,24 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	// Make sure new SubConn is created before timeout.
 	select {
 	case <-time.After(testPriorityInitTimeout * 3 / 4):
-	case <-cc.newSubConnAddrsCh:
+	case <-cc.NewSubConnAddrsCh:
 		t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout")
 	}
 
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test pick with 1.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
 		}
 	}
@@ -403,7 +404,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 //  - start with 2 locality with p0 and p1
 //  - add localities to existing p0 and p1
 func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -413,51 +414,51 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
 
-	addrs0 := <-cc.newSubConnAddrsCh
+	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc0 := <-cc.newSubConnCh
+	sc0 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p0 := <-cc.newPickerCh
+	p0 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc0}
-	if err := isRoundRobin(want, subConnFromPicker(p0)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p0)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Turn down p0 subconns, p1 subconns will be created.
 	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
 
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc1}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	// Reconnect p0 subconns, p1 subconn will be closed.
 	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
 
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
 
 	// Test roundrobin with only p0 subconns.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc0}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -469,18 +470,18 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	clab1.AddLocality(testSubZones[3], 1, 1, testEndpointAddrs[3:4], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs2 := <-cc.newSubConnAddrsCh
+	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with only two p0 subconns.
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc0, sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p3)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -488,17 +489,17 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
 	edsb.HandleSubConnStateChange(sc2, connectivity.TransientFailure)
 
-	sc3 := <-cc.newSubConnCh
+	sc3 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
-	sc4 := <-cc.newSubConnCh
+	sc4 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc4, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
-	p4 := <-cc.newPickerCh
+	p4 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p4)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
@@ -514,7 +515,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 		}
 	}()()
 
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -524,18 +525,18 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
 
-	addrs0 := <-cc.newSubConnAddrsCh
+	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc0 := <-cc.newSubConnCh
+	sc0 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p0 := <-cc.newPickerCh
+	p0 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc0}
-	if err := isRoundRobin(want, subConnFromPicker(p0)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p0)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -544,13 +545,13 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	// p0 subconn should be removed.
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc0, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc0, scToRemove)
 	}
 
 	// Test pick return TransientFailure.
-	pFail := <-cc.newPickerCh
+	pFail := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := pFail.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
 			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
@@ -563,29 +564,29 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[3:4], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
-	addrs01 := <-cc.newSubConnAddrsCh
+	addrs01 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs01[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc01 := <-cc.newSubConnCh
+	sc01 := <-cc.NewSubConnCh
 
 	// Don't send any update to p0, so to not override the old state of p0.
 	// Later, connect to p1 and then remove p1. This will fallback to p0, and
 	// will send p0's old picker if they are not correctly removed.
 
 	// p1 will be used after priority init timeout.
-	addrs11 := <-cc.newSubConnAddrsCh
+	addrs11 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs11[0].Addr, testEndpointAddrs[3]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc11 := <-cc.newSubConnCh
+	sc11 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc11, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc11, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc11}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -595,13 +596,13 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	// p1 subconn should be removed.
-	scToRemove1 := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove1, sc11, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove1 := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove1, sc11, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc11, scToRemove1)
 	}
 
 	// Test pick return TransientFailure.
-	pFail1 := <-cc.newPickerCh
+	pFail1 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if scst, err := pFail1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
 			t.Fatalf("want pick error _, %v, got %v, _ ,%v", balancer.ErrTransientFailure, scst, err)
@@ -614,18 +615,18 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	edsb.HandleSubConnStateChange(sc01, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc01}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
 	select {
-	case <-cc.newPickerCh:
+	case <-cc.NewPickerCh:
 		t.Fatalf("got unexpected new picker")
-	case <-cc.newSubConnCh:
+	case <-cc.NewSubConnCh:
 		t.Fatalf("got unexpected new SubConn")
-	case <-cc.removeSubConnCh:
+	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
 	case <-time.After(time.Millisecond * 100):
 	}
@@ -658,7 +659,7 @@ func (s) TestPriorityType(t *testing.T) {
 // Test the case where the high priority contains no backends. The low priority
 // will be used.
 func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -668,20 +669,20 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 
 	// p0 is ready.
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -693,23 +694,23 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 
 	// p0 will remove the subconn, and ClientConn will send a sc update to
 	// shutdown.
-	scToRemove := <-cc.removeSubConnCh
+	scToRemove := <-cc.RemoveSubConnCh
 	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
-	addrs2 := <-cc.newSubConnAddrsCh
+	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 
 	// p1 is ready.
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
@@ -717,7 +718,7 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 // Test the case where the high priority contains no healthy backends. The low
 // priority will be used.
 func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -727,20 +728,20 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	addrs1 := <-cc.newSubConnAddrsCh
+	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 
 	// p0 is ready.
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -754,23 +755,23 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 
 	// p0 will remove the subconn, and ClientConn will send a sc update to
 	// transient failure.
-	scToRemove := <-cc.removeSubConnCh
+	scToRemove := <-cc.RemoveSubConnCh
 	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
-	addrs2 := <-cc.newSubConnAddrsCh
+	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 
 	// p1 is ready.
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index ec8942437f53..33eabbe4bf09 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -29,7 +29,9 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 var (
@@ -38,10 +40,14 @@ var (
 	testEndpointAddrs []string
 )
 
+const testBackendAddrsCount = 12
+
 func init() {
 	for i := 0; i < testBackendAddrsCount; i++ {
 		testEndpointAddrs = append(testEndpointAddrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
 	}
+	balancergroup.NewRandomWRR = testutils.NewTestWRR
+	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond
 }
 
 // One locality
@@ -50,7 +56,7 @@ func init() {
 //  - replace backend
 //  - change drop rate
 func (s) TestEDS_OneLocality(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -59,15 +65,15 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Pick with only the first backend.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
 		}
 	}
@@ -77,14 +83,14 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with two subconns.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -93,17 +99,17 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
 	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with only the second subconn.
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p3.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
 		}
 	}
@@ -113,20 +119,20 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	clab4.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
 
-	sc3 := <-cc.newSubConnCh
+	sc3 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
-	scToRemove = <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove = <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
 	}
 	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with only the third subconn.
-	p4 := <-cc.newPickerCh
+	p4 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p4.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc3, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc3, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc3)
 		}
 	}
@@ -137,7 +143,7 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
 
 	// Picks with drops.
-	p5 := <-cc.newPickerCh
+	p5 := <-cc.NewPickerCh
 	for i := 0; i < 100; i++ {
 		_, err := p5.Pick(balancer.PickInfo{})
 		// TODO: the dropping algorithm needs a design. When the dropping algorithm
@@ -155,10 +161,10 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
 
 	// Pick without drops.
-	p6 := <-cc.newPickerCh
+	p6 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		gotSCSt, _ := p6.Pick(balancer.PickInfo{})
-		if !cmp.Equal(gotSCSt.SubConn, sc3, cmp.AllowUnexported(testSubConn{})) {
+		if !cmp.Equal(gotSCSt.SubConn, sc3, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc3)
 		}
 	}
@@ -171,7 +177,7 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 //  - address change for the <not-the-first> locality
 //  - update locality weight
 func (s) TestEDS_TwoLocalities(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -179,7 +185,7 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 
@@ -188,14 +194,14 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	// keep localities.
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with two subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -206,14 +212,14 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab2.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
-	sc3 := <-cc.newSubConnCh
+	sc3 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
 
 	// Test roundrobin with three subconns.
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc1, sc2, sc3}
-	if err := isRoundRobin(want, subConnFromPicker(p2)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -223,16 +229,16 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab3.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
-	scToRemove := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
 	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with two subconns (without the first one).
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc2, sc3}
-	if err := isRoundRobin(want, subConnFromPicker(p3)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -242,17 +248,17 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab4.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
 
-	sc4 := <-cc.newSubConnCh
+	sc4 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc4, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test pick with two subconns (without the first one).
-	p4 := <-cc.newPickerCh
+	p4 := <-cc.NewPickerCh
 	// Locality-1 will be picked twice, and locality-2 will be picked twice.
 	// Locality-1 contains only sc2, locality-2 contains sc3 and sc4. So expect
 	// two sc2's and sc3, sc4.
 	want = []balancer.SubConn{sc2, sc2, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p4)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -263,12 +269,12 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
 
 	// Test pick with two subconns different locality weight.
-	p5 := <-cc.newPickerCh
+	p5 := <-cc.NewPickerCh
 	// Locality-1 will be picked four times, and locality-2 will be picked twice
 	// (weight 2 and 1). Locality-1 contains only sc2, locality-2 contains sc3 and
 	// sc4. So expect four sc2's and sc3, sc4.
 	want = []balancer.SubConn{sc2, sc2, sc2, sc2, sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p5)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p5)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -284,17 +290,17 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	// NOTE: this is because we handle locality with weight 0 same as the
 	// locality doesn't exist. If this changes in the future, this removeSubConn
 	// behavior will also change.
-	scToRemove2 := <-cc.removeSubConnCh
-	if !cmp.Equal(scToRemove2, sc2, cmp.AllowUnexported(testSubConn{})) {
+	scToRemove2 := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove2, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove2)
 	}
 
 	// Test pick with two subconns different locality weight.
-	p6 := <-cc.newPickerCh
+	p6 := <-cc.NewPickerCh
 	// Locality-1 will be not be picked, and locality-2 will be picked.
 	// Locality-2 contains sc3 and sc4. So expect sc3, sc4.
 	want = []balancer.SubConn{sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p6)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p6)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
@@ -302,7 +308,7 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 // The EDS balancer gets EDS resp with unhealthy endpoints. Test that only
 // healthy ones are used.
 func (s) TestEDS_EndpointsHealth(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -335,9 +341,9 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 		newSubConnAddrStrs []string
 	)
 	for i := 0; i < 4; i++ {
-		addr := <-cc.newSubConnAddrsCh
+		addr := <-cc.NewSubConnAddrsCh
 		newSubConnAddrStrs = append(newSubConnAddrStrs, addr[0].Addr)
-		sc := <-cc.newSubConnCh
+		sc := <-cc.NewSubConnCh
 		edsb.HandleSubConnStateChange(sc, connectivity.Connecting)
 		edsb.HandleSubConnStateChange(sc, connectivity.Ready)
 		readySCs = append(readySCs, sc)
@@ -361,15 +367,15 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 	// There should be exactly 4 new SubConns. Check to make sure there's no
 	// more subconns being created.
 	select {
-	case <-cc.newSubConnCh:
+	case <-cc.NewSubConnCh:
 		t.Fatalf("Got unexpected new subconn")
 	case <-time.After(time.Microsecond * 100):
 	}
 
 	// Test roundrobin with the subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := readySCs
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
@@ -381,57 +387,11 @@ func (s) TestClose(t *testing.T) {
 	edsb.Close()
 }
 
-func init() {
-	balancer.Register(&testConstBalancerBuilder{})
-}
-
-var errTestConstPicker = fmt.Errorf("const picker error")
-
-type testConstBalancerBuilder struct{}
-
-func (*testConstBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
-	return &testConstBalancer{cc: cc}
-}
-
-func (*testConstBalancerBuilder) Name() string {
-	return "test-const-balancer"
-}
-
-type testConstBalancer struct {
-	cc balancer.ClientConn
-}
-
-func (tb *testConstBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
-	tb.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &testConstPicker{err: errTestConstPicker}})
-}
-
-func (tb *testConstBalancer) HandleResolvedAddrs(a []resolver.Address, err error) {
-	if len(a) == 0 {
-		return
-	}
-	tb.cc.NewSubConn(a, balancer.NewSubConnOptions{})
-}
-
-func (*testConstBalancer) Close() {
-}
-
-type testConstPicker struct {
-	err error
-	sc  balancer.SubConn
-}
-
-func (tcp *testConstPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
-	if tcp.err != nil {
-		return balancer.PickResult{}, tcp.err
-	}
-	return balancer.PickResult{SubConn: tcp.sc}, nil
-}
-
 // Create XDS balancer, and update sub-balancer before handling eds responses.
 // Then switch between round-robin and test-const-balancer after handling first
 // eds response.
 func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -445,15 +405,15 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	for i := 0; i < 2; i++ {
-		sc := <-cc.newSubConnCh
+		sc := <-cc.NewSubConnCh
 		edsb.HandleSubConnStateChange(sc, connectivity.Ready)
 	}
 
-	p0 := <-cc.newPickerCh
+	p0 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		_, err := p0.Pick(balancer.PickInfo{})
-		if err != errTestConstPicker {
-			t.Fatalf("picker.Pick, got err %q, want err %q", err, errTestConstPicker)
+		if err != testutils.ErrTestConstPicker {
+			t.Fatalf("picker.Pick, got err %q, want err %q", err, testutils.ErrTestConstPicker)
 		}
 	}
 
@@ -461,20 +421,20 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	edsb.HandleChildPolicy(roundrobin.Name, nil)
 
 	for i := 0; i < 2; i++ {
-		<-cc.removeSubConnCh
+		<-cc.RemoveSubConnCh
 	}
 
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with two subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2}
-	if err := isRoundRobin(want, subConnFromPicker(p1)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
@@ -482,24 +442,24 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	edsb.HandleChildPolicy("test-const-balancer", nil)
 
 	for i := 0; i < 2; i++ {
-		scToRemove := <-cc.removeSubConnCh
-		if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testSubConn{})) &&
-			!cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testSubConn{})) {
+		scToRemove := <-cc.RemoveSubConnCh
+		if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) &&
+			!cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("RemoveSubConn, want (%v or %v), got %v", sc1, sc2, scToRemove)
 		}
 		edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
 	}
 
 	for i := 0; i < 2; i++ {
-		sc := <-cc.newSubConnCh
+		sc := <-cc.NewSubConnCh
 		edsb.HandleSubConnStateChange(sc, connectivity.Ready)
 	}
 
-	p2 := <-cc.newPickerCh
+	p2 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		_, err := p2.Pick(balancer.PickInfo{})
-		if err != errTestConstPicker {
-			t.Fatalf("picker.Pick, got err %q, want err %q", err, errTestConstPicker)
+		if err != testutils.ErrTestConstPicker {
+			t.Fatalf("picker.Pick, got err %q, want err %q", err, testutils.ErrTestConstPicker)
 		}
 	}
 
@@ -507,19 +467,19 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	edsb.HandleChildPolicy(roundrobin.Name, nil)
 
 	for i := 0; i < 2; i++ {
-		<-cc.removeSubConnCh
+		<-cc.RemoveSubConnCh
 	}
 
-	sc3 := <-cc.newSubConnCh
+	sc3 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
-	sc4 := <-cc.newSubConnCh
+	sc4 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc4, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc4, connectivity.Ready)
 
-	p3 := <-cc.newPickerCh
+	p3 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc3, sc4}
-	if err := isRoundRobin(want, subConnFromPicker(p3)); err != nil {
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
@@ -552,7 +512,7 @@ var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not")
 func (tb *testInlineUpdateBalancer) HandleResolvedAddrs(a []resolver.Address, err error) {
 	tb.cc.UpdateState(balancer.State{
 		ConnectivityState: connectivity.Ready,
-		Picker:            &testConstPicker{err: errTestInlineStateUpdate},
+		Picker:            &testutils.TestConstPicker{Err: errTestInlineStateUpdate},
 	})
 }
 
@@ -563,7 +523,7 @@ func (*testInlineUpdateBalancer) Close() {
 // (e.g., roundrobin handling empty addresses). There could be deadlock caused
 // by acquiring a locked mutex.
 func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) {
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = func(p priorityType, state balancer.State) {
 		// For this test, euqueue needs to happen asynchronously (like in the
@@ -577,7 +537,7 @@ func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) {
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
-	p0 := <-cc.newPickerCh
+	p0 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		_, err := p0.Pick(balancer.PickInfo{})
 		if err != errTestInlineStateUpdate {
@@ -588,8 +548,8 @@ func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) {
 
 func (s) TestDropPicker(t *testing.T) {
 	const pickCount = 12
-	var constPicker = &testConstPicker{
-		sc: testSubConns[0],
+	var constPicker = &testutils.TestConstPicker{
+		SC: testutils.TestSubConns[0],
 	}
 
 	tests := []struct {
@@ -652,9 +612,9 @@ func (s) TestDropPicker(t *testing.T) {
 }
 
 func (s) TestEDS_LoadReport(t *testing.T) {
-	testLoadStore := newTestLoadStore()
+	testLoadStore := testutils.NewTestLoadStore()
 
-	cc := newTestClientConn(t)
+	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, testLoadStore, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
@@ -664,7 +624,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
-	sc1 := <-cc.newSubConnCh
+	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
 	backendToBalancerID[sc1] = internal.Locality{
@@ -676,7 +636,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// keep localities.
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
-	sc2 := <-cc.newSubConnCh
+	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
 	backendToBalancerID[sc2] = internal.Locality{
@@ -684,7 +644,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	}
 
 	// Test roundrobin with two subconns.
-	p1 := <-cc.newPickerCh
+	p1 := <-cc.NewPickerCh
 	var (
 		wantStart []internal.Locality
 		wantEnd   []internal.Locality
@@ -700,10 +660,10 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 		}
 	}
 
-	if !cmp.Equal(testLoadStore.callsStarted, wantStart) {
-		t.Fatalf("want started: %v, got: %v", testLoadStore.callsStarted, wantStart)
+	if !cmp.Equal(testLoadStore.CallsStarted, wantStart) {
+		t.Fatalf("want started: %v, got: %v", testLoadStore.CallsStarted, wantStart)
 	}
-	if !cmp.Equal(testLoadStore.callsEnded, wantEnd) {
-		t.Fatalf("want ended: %v, got: %v", testLoadStore.callsEnded, wantEnd)
+	if !cmp.Equal(testLoadStore.CallsEnded, wantEnd) {
+		t.Fatalf("want ended: %v, got: %v", testLoadStore.CallsEnded, wantEnd)
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 1e795bea4d9d..83dca16efcab 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -55,6 +55,13 @@ func init() {
 	}
 }
 
+func subConnFromPicker(p balancer.V2Picker) func() balancer.SubConn {
+	return func() balancer.SubConn {
+		scst, _ := p.Pick(balancer.PickInfo{})
+		return scst.SubConn
+	}
+}
+
 type s struct {
 	grpctest.Tester
 }
diff --git a/xds/internal/balancer/edsbalancer/test_util_test.go b/xds/internal/balancer/edsbalancer/test_util_test.go
deleted file mode 100644
index 320a710bb54e..000000000000
--- a/xds/internal/balancer/edsbalancer/test_util_test.go
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edsbalancer
-
-import (
-	"context"
-	"fmt"
-	"testing"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/xds/internal"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-)
-
-const testSubConnsCount = 16
-
-var testSubConns []*testSubConn
-
-func init() {
-	for i := 0; i < testSubConnsCount; i++ {
-		testSubConns = append(testSubConns, &testSubConn{
-			id: fmt.Sprintf("sc%d", i),
-		})
-	}
-}
-
-type testSubConn struct {
-	id string
-}
-
-func (tsc *testSubConn) UpdateAddresses([]resolver.Address) {
-	panic("not implemented")
-}
-
-func (tsc *testSubConn) Connect() {
-}
-
-// Implement stringer to get human friendly error message.
-func (tsc *testSubConn) String() string {
-	return tsc.id
-}
-
-type testClientConn struct {
-	t *testing.T // For logging only.
-
-	newSubConnAddrsCh chan []resolver.Address // The last 10 []Address to create subconn.
-	newSubConnCh      chan balancer.SubConn   // The last 10 subconn created.
-	removeSubConnCh   chan balancer.SubConn   // The last 10 subconn removed.
-
-	newPickerCh chan balancer.V2Picker  // The last picker updated.
-	newStateCh  chan connectivity.State // The last state.
-
-	subConnIdx int
-}
-
-func newTestClientConn(t *testing.T) *testClientConn {
-	return &testClientConn{
-		t: t,
-
-		newSubConnAddrsCh: make(chan []resolver.Address, 10),
-		newSubConnCh:      make(chan balancer.SubConn, 10),
-		removeSubConnCh:   make(chan balancer.SubConn, 10),
-
-		newPickerCh: make(chan balancer.V2Picker, 1),
-		newStateCh:  make(chan connectivity.State, 1),
-	}
-}
-
-func (tcc *testClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubConnOptions) (balancer.SubConn, error) {
-	sc := testSubConns[tcc.subConnIdx]
-	tcc.subConnIdx++
-
-	tcc.t.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc)
-	select {
-	case tcc.newSubConnAddrsCh <- a:
-	default:
-	}
-
-	select {
-	case tcc.newSubConnCh <- sc:
-	default:
-	}
-
-	return sc, nil
-}
-
-func (tcc *testClientConn) RemoveSubConn(sc balancer.SubConn) {
-	tcc.t.Logf("testClientCOnn: RemoveSubConn(%p)", sc)
-	select {
-	case tcc.removeSubConnCh <- sc:
-	default:
-	}
-}
-
-func (tcc *testClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) {
-	tcc.t.Fatal("not implemented")
-}
-
-func (tcc *testClientConn) UpdateState(bs balancer.State) {
-	tcc.t.Logf("testClientConn: UpdateState(%v)", bs)
-	select {
-	case <-tcc.newStateCh:
-	default:
-	}
-	tcc.newStateCh <- bs.ConnectivityState
-
-	select {
-	case <-tcc.newPickerCh:
-	default:
-	}
-	tcc.newPickerCh <- bs.Picker
-}
-
-func (tcc *testClientConn) ResolveNow(resolver.ResolveNowOptions) {
-	panic("not implemented")
-}
-
-func (tcc *testClientConn) Target() string {
-	panic("not implemented")
-}
-
-type testServerLoad struct {
-	name string
-	d    float64
-}
-
-type testLoadStore struct {
-	callsStarted []internal.Locality
-	callsEnded   []internal.Locality
-	callsCost    []testServerLoad
-}
-
-func newTestLoadStore() *testLoadStore {
-	return &testLoadStore{}
-}
-
-func (*testLoadStore) CallDropped(category string) {
-	panic("not implemented")
-}
-
-func (tls *testLoadStore) CallStarted(l internal.Locality) {
-	tls.callsStarted = append(tls.callsStarted, l)
-}
-
-func (tls *testLoadStore) CallFinished(l internal.Locality, err error) {
-	tls.callsEnded = append(tls.callsEnded, l)
-}
-
-func (tls *testLoadStore) CallServerLoad(l internal.Locality, name string, d float64) {
-	tls.callsCost = append(tls.callsCost, testServerLoad{name: name, d: d})
-}
-
-func (*testLoadStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) {
-	panic("not implemented")
-}
-
-// isRoundRobin checks whether f's return value is roundrobin of elements from
-// want. But it doesn't check for the order. Note that want can contain
-// duplicate items, which makes it weight-round-robin.
-//
-// Step 1. the return values of f should form a permutation of all elements in
-// want, but not necessary in the same order. E.g. if want is {a,a,b}, the check
-// fails if f returns:
-//  - {a,a,a}: third a is returned before b
-//  - {a,b,b}: second b is returned before the second a
-//
-// If error is found in this step, the returned error contains only the first
-// iteration until where it goes wrong.
-//
-// Step 2. the return values of f should be repetitions of the same permutation.
-// E.g. if want is {a,a,b}, the check failes if f returns:
-//  - {a,b,a,b,a,a}: though it satisfies step 1, the second iteration is not
-//  repeating the first iteration.
-//
-// If error is found in this step, the returned error contains the first
-// iteration + the second iteration until where it goes wrong.
-func isRoundRobin(want []balancer.SubConn, f func() balancer.SubConn) error {
-	wantSet := make(map[balancer.SubConn]int) // SubConn -> count, for weighted RR.
-	for _, sc := range want {
-		wantSet[sc]++
-	}
-
-	// The first iteration: makes sure f's return values form a permutation of
-	// elements in want.
-	//
-	// Also keep the returns values in a slice, so we can compare the order in
-	// the second iteration.
-	gotSliceFirstIteration := make([]balancer.SubConn, 0, len(want))
-	for range want {
-		got := f()
-		gotSliceFirstIteration = append(gotSliceFirstIteration, got)
-		wantSet[got]--
-		if wantSet[got] < 0 {
-			return fmt.Errorf("non-roundrobin want: %v, result: %v", want, gotSliceFirstIteration)
-		}
-	}
-
-	// The second iteration should repeat the first iteration.
-	var gotSliceSecondIteration []balancer.SubConn
-	for i := 0; i < 2; i++ {
-		for _, w := range gotSliceFirstIteration {
-			g := f()
-			gotSliceSecondIteration = append(gotSliceSecondIteration, g)
-			if w != g {
-				return fmt.Errorf("non-roundrobin, first iter: %v, second iter: %v", gotSliceFirstIteration, gotSliceSecondIteration)
-			}
-		}
-	}
-
-	return nil
-}
-
-// testClosure is a test util for TestIsRoundRobin.
-type testClosure struct {
-	r []balancer.SubConn
-	i int
-}
-
-func (tc *testClosure) next() balancer.SubConn {
-	ret := tc.r[tc.i]
-	tc.i = (tc.i + 1) % len(tc.r)
-	return ret
-}
-
-func (s) TestIsRoundRobin(t *testing.T) {
-	var (
-		sc1 = testSubConns[0]
-		sc2 = testSubConns[1]
-		sc3 = testSubConns[2]
-	)
-
-	testCases := []struct {
-		desc string
-		want []balancer.SubConn
-		got  []balancer.SubConn
-		pass bool
-	}{
-		{
-			desc: "0 element",
-			want: []balancer.SubConn{},
-			got:  []balancer.SubConn{},
-			pass: true,
-		},
-		{
-			desc: "1 element RR",
-			want: []balancer.SubConn{sc1},
-			got:  []balancer.SubConn{sc1, sc1, sc1, sc1},
-			pass: true,
-		},
-		{
-			desc: "1 element not RR",
-			want: []balancer.SubConn{sc1},
-			got:  []balancer.SubConn{sc1, sc2, sc1},
-			pass: false,
-		},
-		{
-			desc: "2 elements RR",
-			want: []balancer.SubConn{sc1, sc2},
-			got:  []balancer.SubConn{sc1, sc2, sc1, sc2, sc1, sc2},
-			pass: true,
-		},
-		{
-			desc: "2 elements RR different order from want",
-			want: []balancer.SubConn{sc2, sc1},
-			got:  []balancer.SubConn{sc1, sc2, sc1, sc2, sc1, sc2},
-			pass: true,
-		},
-		{
-			desc: "2 elements RR not RR, mistake in first iter",
-			want: []balancer.SubConn{sc1, sc2},
-			got:  []balancer.SubConn{sc1, sc1, sc1, sc2, sc1, sc2},
-			pass: false,
-		},
-		{
-			desc: "2 elements RR not RR, mistake in second iter",
-			want: []balancer.SubConn{sc1, sc2},
-			got:  []balancer.SubConn{sc1, sc2, sc1, sc1, sc1, sc2},
-			pass: false,
-		},
-		{
-			desc: "2 elements weighted RR",
-			want: []balancer.SubConn{sc1, sc1, sc2},
-			got:  []balancer.SubConn{sc1, sc1, sc2, sc1, sc1, sc2},
-			pass: true,
-		},
-		{
-			desc: "2 elements weighted RR different order",
-			want: []balancer.SubConn{sc1, sc1, sc2},
-			got:  []balancer.SubConn{sc1, sc2, sc1, sc1, sc2, sc1},
-			pass: true,
-		},
-
-		{
-			desc: "3 elements RR",
-			want: []balancer.SubConn{sc1, sc2, sc3},
-			got:  []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc3, sc1, sc2, sc3},
-			pass: true,
-		},
-		{
-			desc: "3 elements RR different order",
-			want: []balancer.SubConn{sc1, sc2, sc3},
-			got:  []balancer.SubConn{sc3, sc2, sc1, sc3, sc2, sc1},
-			pass: true,
-		},
-		{
-			desc: "3 elements weighted RR",
-			want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3},
-			got:  []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc1, sc1, sc2, sc3, sc1, sc2, sc1},
-			pass: true,
-		},
-		{
-			desc: "3 elements weighted RR not RR, mistake in first iter",
-			want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3},
-			got:  []balancer.SubConn{sc1, sc2, sc1, sc1, sc2, sc1, sc1, sc2, sc3, sc1, sc2, sc1},
-			pass: false,
-		},
-		{
-			desc: "3 elements weighted RR not RR, mistake in second iter",
-			want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3},
-			got:  []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc1, sc1, sc1, sc3, sc1, sc2, sc1},
-			pass: false,
-		},
-	}
-	for _, tC := range testCases {
-		t.Run(tC.desc, func(t *testing.T) {
-			err := isRoundRobin(tC.want, (&testClosure{r: tC.got}).next)
-			if err == nil != tC.pass {
-				t.Errorf("want pass %v, want %v, got err %v", tC.pass, tC.want, err)
-			}
-		})
-	}
-}
diff --git a/xds/internal/balancer/edsbalancer/util.go b/xds/internal/balancer/edsbalancer/util.go
index 06322498ee57..132950426466 100644
--- a/xds/internal/balancer/edsbalancer/util.go
+++ b/xds/internal/balancer/edsbalancer/util.go
@@ -21,6 +21,8 @@ import (
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
+var newRandomWRR = wrr.NewRandom
+
 type dropper struct {
 	c xdsclient.OverloadDropConfig
 	w wrr.WRR
diff --git a/xds/internal/balancer/edsbalancer/util_test.go b/xds/internal/balancer/edsbalancer/util_test.go
index f4d5e78229d5..748aeffe2bb9 100644
--- a/xds/internal/balancer/edsbalancer/util_test.go
+++ b/xds/internal/balancer/edsbalancer/util_test.go
@@ -17,57 +17,14 @@
 package edsbalancer
 
 import (
-	"sync"
 	"testing"
 
-	"google.golang.org/grpc/internal/wrr"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils"
 )
 
-// testWRR is a deterministic WRR implementation.
-//
-// The real implementation does random WRR. testWRR makes the balancer behavior
-// deterministic and easier to test.
-//
-// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}.
-type testWRR struct {
-	itemsWithWeight []struct {
-		item   interface{}
-		weight int64
-	}
-	length int
-
-	mu    sync.Mutex
-	idx   int   // The index of the item that will be picked
-	count int64 // The number of times the current item has been picked.
-}
-
-func newTestWRR() wrr.WRR {
-	return &testWRR{}
-}
-
-func (twrr *testWRR) Add(item interface{}, weight int64) {
-	twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct {
-		item   interface{}
-		weight int64
-	}{item: item, weight: weight})
-	twrr.length++
-}
-
-func (twrr *testWRR) Next() interface{} {
-	twrr.mu.Lock()
-	iww := twrr.itemsWithWeight[twrr.idx]
-	twrr.count++
-	if twrr.count >= iww.weight {
-		twrr.idx = (twrr.idx + 1) % twrr.length
-		twrr.count = 0
-	}
-	twrr.mu.Unlock()
-	return iww.item
-}
-
 func init() {
-	newRandomWRR = newTestWRR
+	newRandomWRR = testutils.NewTestWRR
 }
 
 func (s) TestDropper(t *testing.T) {
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
new file mode 100644
index 000000000000..9ffbee237dcf
--- /dev/null
+++ b/xds/internal/testutils/balancer.go
@@ -0,0 +1,357 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+
+	envoy_api_v2_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/wrr"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal"
+)
+
+const testSubConnsCount = 16
+
+// TestSubConns contains a list of SubConns to be used in tests.
+var TestSubConns []*TestSubConn
+
+func init() {
+	for i := 0; i < testSubConnsCount; i++ {
+		TestSubConns = append(TestSubConns, &TestSubConn{
+			id: fmt.Sprintf("sc%d", i),
+		})
+	}
+}
+
+// TestSubConn implements the SubConn interface, to be used in tests.
+type TestSubConn struct {
+	id string
+}
+
+// UpdateAddresses panics.
+func (tsc *TestSubConn) UpdateAddresses([]resolver.Address) { panic("not implemented") }
+
+// Connect is a no-op.
+func (tsc *TestSubConn) Connect() {}
+
+// String implements stringer to print human friendly error message.
+func (tsc *TestSubConn) String() string {
+	return tsc.id
+}
+
+// TestClientConn is a mock balancer.ClientConn used in tests.
+type TestClientConn struct {
+	t *testing.T // For logging only.
+
+	NewSubConnAddrsCh chan []resolver.Address // the last 10 []Address to create subconn.
+	NewSubConnCh      chan balancer.SubConn   // the last 10 subconn created.
+	RemoveSubConnCh   chan balancer.SubConn   // the last 10 subconn removed.
+
+	NewPickerCh chan balancer.V2Picker  // the last picker updated.
+	NewStateCh  chan connectivity.State // the last state.
+
+	subConnIdx int
+}
+
+// NewTestClientConn creates a TestClientConn.
+func NewTestClientConn(t *testing.T) *TestClientConn {
+	return &TestClientConn{
+		t: t,
+
+		NewSubConnAddrsCh: make(chan []resolver.Address, 10),
+		NewSubConnCh:      make(chan balancer.SubConn, 10),
+		RemoveSubConnCh:   make(chan balancer.SubConn, 10),
+
+		NewPickerCh: make(chan balancer.V2Picker, 1),
+		NewStateCh:  make(chan connectivity.State, 1),
+	}
+}
+
+// NewSubConn creates a new SubConn.
+func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	sc := TestSubConns[tcc.subConnIdx]
+	tcc.subConnIdx++
+
+	tcc.t.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc)
+	select {
+	case tcc.NewSubConnAddrsCh <- a:
+	default:
+	}
+
+	select {
+	case tcc.NewSubConnCh <- sc:
+	default:
+	}
+
+	return sc, nil
+}
+
+// RemoveSubConn removes the SubConn.
+func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
+	tcc.t.Logf("testClientCOnn: RemoveSubConn(%p)", sc)
+	select {
+	case tcc.RemoveSubConnCh <- sc:
+	default:
+	}
+}
+
+// UpdateBalancerState implements balancer.Balancer API. It will be removed when
+// switching to the new balancer interface.
+func (tcc *TestClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) {
+	tcc.t.Fatal("not implemented")
+}
+
+// UpdateState updates connectivity state and picker.
+func (tcc *TestClientConn) UpdateState(bs balancer.State) {
+	tcc.t.Logf("testClientConn: UpdateState(%v)", bs)
+	select {
+	case <-tcc.NewStateCh:
+	default:
+	}
+	tcc.NewStateCh <- bs.ConnectivityState
+
+	select {
+	case <-tcc.NewPickerCh:
+	default:
+	}
+	tcc.NewPickerCh <- bs.Picker
+}
+
+// ResolveNow panics.
+func (tcc *TestClientConn) ResolveNow(resolver.ResolveNowOptions) {
+	panic("not implemented")
+}
+
+// Target panics.
+func (tcc *TestClientConn) Target() string {
+	panic("not implemented")
+}
+
+// TestServerLoad is testing Load for testing LRS.
+type TestServerLoad struct {
+	Name string
+	D    float64
+}
+
+// TestLoadStore is a load store to be used in tests.
+type TestLoadStore struct {
+	CallsStarted []internal.Locality
+	CallsEnded   []internal.Locality
+	CallsCost    []TestServerLoad
+}
+
+// NewTestLoadStore creates a new TestLoadStore.
+func NewTestLoadStore() *TestLoadStore {
+	return &TestLoadStore{}
+}
+
+// CallDropped records a call dropped.
+func (*TestLoadStore) CallDropped(category string) {
+	panic("not implemented")
+}
+
+// CallStarted records a call started.
+func (tls *TestLoadStore) CallStarted(l internal.Locality) {
+	tls.CallsStarted = append(tls.CallsStarted, l)
+}
+
+// CallFinished records a call finished.
+func (tls *TestLoadStore) CallFinished(l internal.Locality, err error) {
+	tls.CallsEnded = append(tls.CallsEnded, l)
+}
+
+// CallServerLoad records a call server load.
+func (tls *TestLoadStore) CallServerLoad(l internal.Locality, name string, d float64) {
+	tls.CallsCost = append(tls.CallsCost, TestServerLoad{Name: name, D: d})
+}
+
+// ReportTo panics.
+func (*TestLoadStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *envoy_api_v2_core.Node) {
+	panic("not implemented")
+}
+
+// IsRoundRobin checks whether f's return value is roundrobin of elements from
+// want. But it doesn't check for the order. Note that want can contain
+// duplicate items, which makes it weight-round-robin.
+//
+// Step 1. the return values of f should form a permutation of all elements in
+// want, but not necessary in the same order. E.g. if want is {a,a,b}, the check
+// fails if f returns:
+//  - {a,a,a}: third a is returned before b
+//  - {a,b,b}: second b is returned before the second a
+//
+// If error is found in this step, the returned error contains only the first
+// iteration until where it goes wrong.
+//
+// Step 2. the return values of f should be repetitions of the same permutation.
+// E.g. if want is {a,a,b}, the check failes if f returns:
+//  - {a,b,a,b,a,a}: though it satisfies step 1, the second iteration is not
+//  repeating the first iteration.
+//
+// If error is found in this step, the returned error contains the first
+// iteration + the second iteration until where it goes wrong.
+func IsRoundRobin(want []balancer.SubConn, f func() balancer.SubConn) error {
+	wantSet := make(map[balancer.SubConn]int) // SubConn -> count, for weighted RR.
+	for _, sc := range want {
+		wantSet[sc]++
+	}
+
+	// The first iteration: makes sure f's return values form a permutation of
+	// elements in want.
+	//
+	// Also keep the returns values in a slice, so we can compare the order in
+	// the second iteration.
+	gotSliceFirstIteration := make([]balancer.SubConn, 0, len(want))
+	for range want {
+		got := f()
+		gotSliceFirstIteration = append(gotSliceFirstIteration, got)
+		wantSet[got]--
+		if wantSet[got] < 0 {
+			return fmt.Errorf("non-roundrobin want: %v, result: %v", want, gotSliceFirstIteration)
+		}
+	}
+
+	// The second iteration should repeat the first iteration.
+	var gotSliceSecondIteration []balancer.SubConn
+	for i := 0; i < 2; i++ {
+		for _, w := range gotSliceFirstIteration {
+			g := f()
+			gotSliceSecondIteration = append(gotSliceSecondIteration, g)
+			if w != g {
+				return fmt.Errorf("non-roundrobin, first iter: %v, second iter: %v", gotSliceFirstIteration, gotSliceSecondIteration)
+			}
+		}
+	}
+
+	return nil
+}
+
+// testClosure is a test util for TestIsRoundRobin.
+type testClosure struct {
+	r []balancer.SubConn
+	i int
+}
+
+func (tc *testClosure) next() balancer.SubConn {
+	ret := tc.r[tc.i]
+	tc.i = (tc.i + 1) % len(tc.r)
+	return ret
+}
+
+func init() {
+	balancer.Register(&TestConstBalancerBuilder{})
+}
+
+// ErrTestConstPicker is error returned by test const picker.
+var ErrTestConstPicker = fmt.Errorf("const picker error")
+
+// TestConstBalancerBuilder is a balancer builder for tests.
+type TestConstBalancerBuilder struct{}
+
+// Build builds a test const balancer.
+func (*TestConstBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	return &testConstBalancer{cc: cc}
+}
+
+// Name returns test-const-balancer name.
+func (*TestConstBalancerBuilder) Name() string {
+	return "test-const-balancer"
+}
+
+type testConstBalancer struct {
+	cc balancer.ClientConn
+}
+
+func (tb *testConstBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+	tb.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &TestConstPicker{Err: ErrTestConstPicker}})
+}
+
+func (tb *testConstBalancer) HandleResolvedAddrs(a []resolver.Address, err error) {
+	if len(a) == 0 {
+		return
+	}
+	tb.cc.NewSubConn(a, balancer.NewSubConnOptions{})
+}
+
+func (*testConstBalancer) Close() {
+}
+
+// TestConstPicker is a const picker for tests.
+type TestConstPicker struct {
+	Err error
+	SC  balancer.SubConn
+}
+
+// Pick returns the const SubConn or the error.
+func (tcp *TestConstPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	if tcp.Err != nil {
+		return balancer.PickResult{}, tcp.Err
+	}
+	return balancer.PickResult{SubConn: tcp.SC}, nil
+}
+
+// testWRR is a deterministic WRR implementation.
+//
+// The real implementation does random WRR. testWRR makes the balancer behavior
+// deterministic and easier to test.
+//
+// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}.
+type testWRR struct {
+	itemsWithWeight []struct {
+		item   interface{}
+		weight int64
+	}
+	length int
+
+	mu    sync.Mutex
+	idx   int   // The index of the item that will be picked
+	count int64 // The number of times the current item has been picked.
+}
+
+// NewTestWRR return a WRR for testing. It's deterministic instead random.
+func NewTestWRR() wrr.WRR {
+	return &testWRR{}
+}
+
+func (twrr *testWRR) Add(item interface{}, weight int64) {
+	twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct {
+		item   interface{}
+		weight int64
+	}{item: item, weight: weight})
+	twrr.length++
+}
+
+func (twrr *testWRR) Next() interface{} {
+	twrr.mu.Lock()
+	iww := twrr.itemsWithWeight[twrr.idx]
+	twrr.count++
+	if twrr.count >= iww.weight {
+		twrr.idx = (twrr.idx + 1) % twrr.length
+		twrr.count = 0
+	}
+	twrr.mu.Unlock()
+	return iww.item
+}
diff --git a/xds/internal/testutils/balancer_test.go b/xds/internal/testutils/balancer_test.go
new file mode 100644
index 000000000000..4891eb9cdadf
--- /dev/null
+++ b/xds/internal/testutils/balancer_test.go
@@ -0,0 +1,134 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/balancer"
+)
+
+func TestIsRoundRobin(t *testing.T) {
+	var (
+		sc1 = TestSubConns[0]
+		sc2 = TestSubConns[1]
+		sc3 = TestSubConns[2]
+	)
+
+	testCases := []struct {
+		desc string
+		want []balancer.SubConn
+		got  []balancer.SubConn
+		pass bool
+	}{
+		{
+			desc: "0 element",
+			want: []balancer.SubConn{},
+			got:  []balancer.SubConn{},
+			pass: true,
+		},
+		{
+			desc: "1 element RR",
+			want: []balancer.SubConn{sc1},
+			got:  []balancer.SubConn{sc1, sc1, sc1, sc1},
+			pass: true,
+		},
+		{
+			desc: "1 element not RR",
+			want: []balancer.SubConn{sc1},
+			got:  []balancer.SubConn{sc1, sc2, sc1},
+			pass: false,
+		},
+		{
+			desc: "2 elements RR",
+			want: []balancer.SubConn{sc1, sc2},
+			got:  []balancer.SubConn{sc1, sc2, sc1, sc2, sc1, sc2},
+			pass: true,
+		},
+		{
+			desc: "2 elements RR different order from want",
+			want: []balancer.SubConn{sc2, sc1},
+			got:  []balancer.SubConn{sc1, sc2, sc1, sc2, sc1, sc2},
+			pass: true,
+		},
+		{
+			desc: "2 elements RR not RR, mistake in first iter",
+			want: []balancer.SubConn{sc1, sc2},
+			got:  []balancer.SubConn{sc1, sc1, sc1, sc2, sc1, sc2},
+			pass: false,
+		},
+		{
+			desc: "2 elements RR not RR, mistake in second iter",
+			want: []balancer.SubConn{sc1, sc2},
+			got:  []balancer.SubConn{sc1, sc2, sc1, sc1, sc1, sc2},
+			pass: false,
+		},
+		{
+			desc: "2 elements weighted RR",
+			want: []balancer.SubConn{sc1, sc1, sc2},
+			got:  []balancer.SubConn{sc1, sc1, sc2, sc1, sc1, sc2},
+			pass: true,
+		},
+		{
+			desc: "2 elements weighted RR different order",
+			want: []balancer.SubConn{sc1, sc1, sc2},
+			got:  []balancer.SubConn{sc1, sc2, sc1, sc1, sc2, sc1},
+			pass: true,
+		},
+
+		{
+			desc: "3 elements RR",
+			want: []balancer.SubConn{sc1, sc2, sc3},
+			got:  []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc3, sc1, sc2, sc3},
+			pass: true,
+		},
+		{
+			desc: "3 elements RR different order",
+			want: []balancer.SubConn{sc1, sc2, sc3},
+			got:  []balancer.SubConn{sc3, sc2, sc1, sc3, sc2, sc1},
+			pass: true,
+		},
+		{
+			desc: "3 elements weighted RR",
+			want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3},
+			got:  []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc1, sc1, sc2, sc3, sc1, sc2, sc1},
+			pass: true,
+		},
+		{
+			desc: "3 elements weighted RR not RR, mistake in first iter",
+			want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3},
+			got:  []balancer.SubConn{sc1, sc2, sc1, sc1, sc2, sc1, sc1, sc2, sc3, sc1, sc2, sc1},
+			pass: false,
+		},
+		{
+			desc: "3 elements weighted RR not RR, mistake in second iter",
+			want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3},
+			got:  []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc1, sc1, sc1, sc3, sc1, sc2, sc1},
+			pass: false,
+		},
+	}
+	for _, tC := range testCases {
+		t.Run(tC.desc, func(t *testing.T) {
+			err := IsRoundRobin(tC.want, (&testClosure{r: tC.got}).next)
+			if err == nil != tC.pass {
+				t.Errorf("want pass %v, want %v, got err %v", tC.pass, tC.want, err)
+			}
+		})
+	}
+}

From 3038e58ed29bb7dd09ed357c0024a7bad2d0dafb Mon Sep 17 00:00:00 2001
From: Mya Pitzeruse <j.mya.pitz@gmail.com>
Date: Wed, 8 Apr 2020 12:38:37 -0500
Subject: [PATCH 010/481] examples: add example to show how to use the health
 service (#3381)

---
 examples/features/health/README.md      | 64 ++++++++++++++++++
 examples/features/health/client/main.go | 85 ++++++++++++++++++++++++
 examples/features/health/server/main.go | 87 +++++++++++++++++++++++++
 3 files changed, 236 insertions(+)
 create mode 100644 examples/features/health/README.md
 create mode 100644 examples/features/health/client/main.go
 create mode 100644 examples/features/health/server/main.go

diff --git a/examples/features/health/README.md b/examples/features/health/README.md
new file mode 100644
index 000000000000..f4bb6a1516ba
--- /dev/null
+++ b/examples/features/health/README.md
@@ -0,0 +1,64 @@
+# Health
+
+gRPC provides a health library to communicate a system's health to their clients.
+It works by providing a service definition via the [health/v1](https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto) api.
+
+By using the health library, clients can gracefully avoid using servers as they encounter issues. 
+Most languages provide an implementation out of box, making it interoperable between systems.
+
+## Try it
+
+```
+go run server/main.go -port=50051 -sleep=5s
+go run server/main.go -port=50052 -sleep=10s
+```
+
+```
+go run client/main.go
+```
+
+## Explanation
+
+### Client
+
+Clients have two ways to monitor a servers health.
+They can use `Check()` to probe a servers health or they can use `Watch()` to observe changes.
+
+In most cases, clients do not need to directly check backend servers.
+Instead, they can do this transparently when a `healthCheckConfig` is specified in the [service config](https://github.com/grpc/proposal/blob/master/A17-client-side-health-checking.md#service-config-changes).
+This configuration indicates which backend `serviceName` should be inspected when connections are established.
+An empty string (`""`) typically indicates the overall health of a server should be reported.
+
+```go
+// import grpc/health to enable transparent client side checking 
+import _ "google.golang.org/grpc/health"
+
+// set up appropriate service config
+serviceConfig := grpc.WithDefaultServiceConfig(`{
+  "loadBalancingPolicy": "round_robin",
+  "healthCheckConfig": {
+    "serviceName": ""
+  }
+}`)
+
+conn, err := grpc.Dial(..., serviceConfig)
+```
+
+See [A17 - Client-Side Health Checking](https://github.com/grpc/proposal/blob/master/A17-client-side-health-checking.md) for more details.
+
+### Server
+
+Servers control their serving status.
+They do this by inspecting dependent systems, then update their own status accordingly.
+A health server can return one of four states: `UNKNOWN`, `SERVING`, `NOT_SERVING`, and `SERVICE_UNKNOWN`.
+
+`UNKNOWN` indicates the current state is not yet known.
+This state is often seen at the start up of a server instance.
+
+`SERVING` means that the system is healthy and ready to service requests.
+Conversely, `NOT_SERVING` indicates the system is unable to service requests at the time.
+
+`SERVICE_UNKNOWN` communicates the `serviceName` requested by the client is not known by the server.
+This status is only reported by the `Watch()` call. 
+
+A server may toggle its health using `healthServer.SetServingStatus("serviceName", servingStatus)`.
diff --git a/examples/features/health/client/main.go b/examples/features/health/client/main.go
new file mode 100644
index 000000000000..db4145024fa2
--- /dev/null
+++ b/examples/features/health/client/main.go
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"google.golang.org/grpc"
+	pb "google.golang.org/grpc/examples/features/proto/echo"
+	_ "google.golang.org/grpc/health"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+)
+
+var serviceConfig = `{
+	"loadBalancingPolicy": "round_robin",
+	"healthCheckConfig": {
+		"serviceName": ""
+	}
+}`
+
+func callUnaryEcho(c pb.EchoClient) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	r, err := c.UnaryEcho(ctx, &pb.EchoRequest{})
+	if err != nil {
+		fmt.Println("UnaryEcho: _, ", err)
+	} else {
+		fmt.Println("UnaryEcho: ", r.GetMessage())
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	r, cleanup := manual.GenerateAndRegisterManualResolver()
+	defer cleanup()
+	r.InitialState(resolver.State{
+		Addresses: []resolver.Address{
+			{Addr: "localhost:50051"},
+			{Addr: "localhost:50052"},
+		},
+	})
+
+	address := fmt.Sprintf("%s:///unused", r.Scheme())
+
+	options := []grpc.DialOption{
+		grpc.WithInsecure(),
+		grpc.WithBlock(),
+		grpc.WithDefaultServiceConfig(serviceConfig),
+	}
+
+	conn, err := grpc.Dial(address, options...)
+	if err != nil {
+		log.Fatalf("did not connect %v", err)
+	}
+	defer conn.Close()
+
+	echoClient := pb.NewEchoClient(conn)
+
+	for {
+		callUnaryEcho(echoClient)
+		time.Sleep(time.Second)
+	}
+}
diff --git a/examples/features/health/server/main.go b/examples/features/health/server/main.go
new file mode 100644
index 000000000000..5a9245dfdbcd
--- /dev/null
+++ b/examples/features/health/server/main.go
@@ -0,0 +1,87 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"time"
+
+	"google.golang.org/grpc"
+	pb "google.golang.org/grpc/examples/features/proto/echo"
+	"google.golang.org/grpc/health"
+	healthpb "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+var (
+	port  = flag.Int("port", 50051, "the port to serve on")
+	sleep = flag.Duration("sleep", time.Second*5, "duration between changes in health")
+
+	system = "" // empty string represents the health of the system
+)
+
+type echoServer struct {
+	pb.UnimplementedEchoServer
+}
+
+func (e *echoServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+	return &pb.EchoResponse{
+		Message: fmt.Sprintf("hello from localhost:%d", *port),
+	}, nil
+}
+
+var _ pb.EchoServer = &echoServer{}
+
+func main() {
+	flag.Parse()
+
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+
+	s := grpc.NewServer()
+	healthcheck := health.NewServer()
+	healthpb.RegisterHealthServer(s, healthcheck)
+	pb.RegisterEchoServer(s, &echoServer{})
+
+	go func() {
+		// asynchronously inspect dependencies and toggle serving status as needed
+		next := healthpb.HealthCheckResponse_SERVING
+
+		for {
+			healthcheck.SetServingStatus(system, next)
+
+			if next == healthpb.HealthCheckResponse_SERVING {
+				next = healthpb.HealthCheckResponse_NOT_SERVING
+			} else {
+				next = healthpb.HealthCheckResponse_SERVING
+			}
+
+			time.Sleep(*sleep)
+		}
+	}()
+
+	if err := s.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
+}

From 85e3a6fdd7f598b7dbf7eabcf1043b370829cd74 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 8 Apr 2020 11:05:45 -0700
Subject: [PATCH 011/481] credentials: Update doc strings for
 NewClientTLSFromCert et. al. (#3508)

---
 credentials/tls.go | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/credentials/tls.go b/credentials/tls.go
index 28b4f6232de4..86e956bc8b77 100644
--- a/credentials/tls.go
+++ b/credentials/tls.go
@@ -135,16 +135,26 @@ func NewTLS(c *tls.Config) TransportCredentials {
 	return tc
 }
 
-// NewClientTLSFromCert constructs TLS credentials from the input certificate for client.
+// NewClientTLSFromCert constructs TLS credentials from the provided root
+// certificate authority certificate(s) to validate server connections. If
+// certificates to establish the identity of the client need to be included in
+// the credentials (eg: for mTLS), use NewTLS instead, where a complete
+// tls.Config can be specified.
 // serverNameOverride is for testing only. If set to a non empty string,
-// it will override the virtual host name of authority (e.g. :authority header field) in requests.
+// it will override the virtual host name of authority (e.g. :authority header
+// field) in requests.
 func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials {
 	return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp})
 }
 
-// NewClientTLSFromFile constructs TLS credentials from the input certificate file for client.
+// NewClientTLSFromFile constructs TLS credentials from the provided root
+// certificate authority certificate file(s) to validate server connections. If
+// certificates to establish the identity of the client need to be included in
+// the credentials (eg: for mTLS), use NewTLS instead, where a complete
+// tls.Config can be specified.
 // serverNameOverride is for testing only. If set to a non empty string,
-// it will override the virtual host name of authority (e.g. :authority header field) in requests.
+// it will override the virtual host name of authority (e.g. :authority header
+// field) in requests.
 func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) {
 	b, err := ioutil.ReadFile(certFile)
 	if err != nil {

From 7f19477365da120509204d52ac549ac0dbf772ba Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 8 Apr 2020 13:31:42 -0700
Subject: [PATCH 012/481] Change version to 1.30.0-dev (#3511)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index 18b42cc7854e..c1926ea13edc 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.29.0-dev"
+const Version = "1.30.0-dev"

From 709091fe14006fc0ee8ee57db32439d8ff88c36d Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 8 Apr 2020 13:57:52 -0700
Subject: [PATCH 013/481] service config: move balancer config parsing to
 internal (#3504)

And make it a json unmarshaller, easy to use.
---
 internal/serviceconfig/serviceconfig.go      |  89 ++++++++++++
 internal/serviceconfig/serviceconfig_test.go | 135 +++++++++++++++++++
 service_config.go                            |  44 +-----
 3 files changed, 230 insertions(+), 38 deletions(-)
 create mode 100644 internal/serviceconfig/serviceconfig.go
 create mode 100644 internal/serviceconfig/serviceconfig_test.go

diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go
new file mode 100644
index 000000000000..87f20a3f35b0
--- /dev/null
+++ b/internal/serviceconfig/serviceconfig.go
@@ -0,0 +1,89 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package serviceconfig
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/grpclog"
+	externalserviceconfig "google.golang.org/grpc/serviceconfig"
+)
+
+// BalancerConfig is the balancer config part that service config's
+// loadBalancingConfig fields can be unmarshalled to. It's a json unmarshaller.
+//
+// https://github.com/grpc/grpc-proto/blob/54713b1e8bc6ed2d4f25fb4dff527842150b91b2/grpc/service_config/service_config.proto#L247
+type BalancerConfig struct {
+	Name   string
+	Config externalserviceconfig.LoadBalancingConfig
+}
+
+type intermediateBalancerConfig []map[string]json.RawMessage
+
+// UnmarshalJSON implements json unmarshaller.
+func (bc *BalancerConfig) UnmarshalJSON(b []byte) error {
+	var ir intermediateBalancerConfig
+	err := json.Unmarshal(b, &ir)
+	if err != nil {
+		return err
+	}
+
+	for i, lbcfg := range ir {
+		if len(lbcfg) != 1 {
+			return fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg)
+		}
+		var (
+			name    string
+			jsonCfg json.RawMessage
+		)
+		// Get the key:value pair from the map.
+		for name, jsonCfg = range lbcfg {
+		}
+		builder := balancer.Get(name)
+		if builder == nil {
+			// If the balancer is not registered, move on to the next config.
+			// This is not an error.
+			continue
+		}
+		bc.Name = name
+
+		parser, ok := builder.(balancer.ConfigParser)
+		if !ok {
+			if string(jsonCfg) != "{}" {
+				grpclog.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg))
+			}
+			// Stop at this, though the builder doesn't support parsing config.
+			return nil
+		}
+
+		cfg, err := parser.ParseConfig(jsonCfg)
+		if err != nil {
+			return fmt.Errorf("error parsing loadBalancingConfig for policy %q: %v", name, err)
+		}
+		bc.Config = cfg
+		return nil
+	}
+	// This is reached when the for loop iterates over all entries, but didn't
+	// return. This means we had a loadBalancingConfig slice but did not
+	// encounter a registered policy. The config is considered invalid in this
+	// case.
+	return fmt.Errorf("invalid loadBalancingConfig: no supported policies found")
+}
diff --git a/internal/serviceconfig/serviceconfig_test.go b/internal/serviceconfig/serviceconfig_test.go
new file mode 100644
index 000000000000..b8abaae027ef
--- /dev/null
+++ b/internal/serviceconfig/serviceconfig_test.go
@@ -0,0 +1,135 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package serviceconfig
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	externalserviceconfig "google.golang.org/grpc/serviceconfig"
+)
+
+type testBalancerConfigType struct {
+	externalserviceconfig.LoadBalancingConfig
+}
+
+var testBalancerConfig = testBalancerConfigType{}
+
+const (
+	testBalancerBuilderName          = "test-bb"
+	testBalancerBuilderNotParserName = "test-bb-not-parser"
+
+	testBalancerConfigJSON = `{"test-balancer-config":"true"}`
+)
+
+type testBalancerBuilder struct {
+	balancer.Builder
+}
+
+func (testBalancerBuilder) ParseConfig(js json.RawMessage) (externalserviceconfig.LoadBalancingConfig, error) {
+	if string(js) != testBalancerConfigJSON {
+		return nil, fmt.Errorf("unexpected config json")
+	}
+	return testBalancerConfig, nil
+}
+
+func (testBalancerBuilder) Name() string {
+	return testBalancerBuilderName
+}
+
+type testBalancerBuilderNotParser struct {
+	balancer.Builder
+}
+
+func (testBalancerBuilderNotParser) Name() string {
+	return testBalancerBuilderNotParserName
+}
+
+func init() {
+	balancer.Register(testBalancerBuilder{})
+	balancer.Register(testBalancerBuilderNotParser{})
+}
+
+func TestBalancerConfigUnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		json    string
+		want    BalancerConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			json:    "",
+			wantErr: true,
+		},
+		{
+			// The config should be a slice of maps, but each map should have
+			// exactly one entry.
+			name:    "more than one entry for a map",
+			json:    `[{"balancer1":"1","balancer2":"2"}]`,
+			wantErr: true,
+		},
+		{
+			name:    "no balancer registered",
+			json:    `[{"balancer1":"1"},{"balancer2":"2"}]`,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			json: fmt.Sprintf("[{%q: %v}]", testBalancerBuilderName, testBalancerConfigJSON),
+			want: BalancerConfig{
+				Name:   testBalancerBuilderName,
+				Config: testBalancerConfig,
+			},
+			wantErr: false,
+		},
+		{
+			name: "first balancer not registered",
+			json: fmt.Sprintf(`[{"balancer1":"1"},{%q: %v}]`, testBalancerBuilderName, testBalancerConfigJSON),
+			want: BalancerConfig{
+				Name:   testBalancerBuilderName,
+				Config: testBalancerConfig,
+			},
+			wantErr: false,
+		},
+		{
+			name: "balancer registered but builder not parser",
+			json: fmt.Sprintf("[{%q: %v}]", testBalancerBuilderNotParserName, testBalancerConfigJSON),
+			want: BalancerConfig{
+				Name:   testBalancerBuilderNotParserName,
+				Config: nil,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var bc BalancerConfig
+			if err := bc.UnmarshalJSON([]byte(tt.json)); (err != nil) != tt.wantErr {
+				t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !cmp.Equal(bc, tt.want) {
+				t.Errorf("diff: %v", cmp.Diff(bc, tt.want))
+			}
+		})
+	}
+}
diff --git a/service_config.go b/service_config.go
index 5a80a575a5eb..c8267fc8eb37 100644
--- a/service_config.go
+++ b/service_config.go
@@ -25,10 +25,10 @@ import (
 	"strings"
 	"time"
 
-	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/serviceconfig"
 )
 
@@ -249,12 +249,10 @@ type jsonMC struct {
 	RetryPolicy             *jsonRetryPolicy
 }
 
-type loadBalancingConfig map[string]json.RawMessage
-
 // TODO(lyuxuan): delete this struct after cleaning up old service config implementation.
 type jsonSC struct {
 	LoadBalancingPolicy *string
-	LoadBalancingConfig *[]loadBalancingConfig
+	LoadBalancingConfig *internalserviceconfig.BalancerConfig
 	MethodConfig        *[]jsonMC
 	RetryThrottling     *retryThrottlingPolicy
 	HealthCheckConfig   *healthCheckConfig
@@ -280,40 +278,10 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 		healthCheckConfig: rsc.HealthCheckConfig,
 		rawJSONString:     js,
 	}
-	if rsc.LoadBalancingConfig != nil {
-		for i, lbcfg := range *rsc.LoadBalancingConfig {
-			if len(lbcfg) != 1 {
-				err := fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg)
-				grpclog.Warningf(err.Error())
-				return &serviceconfig.ParseResult{Err: err}
-			}
-			var name string
-			var jsonCfg json.RawMessage
-			for name, jsonCfg = range lbcfg {
-			}
-			builder := balancer.Get(name)
-			if builder == nil {
-				continue
-			}
-			sc.lbConfig = &lbConfig{name: name}
-			if parser, ok := builder.(balancer.ConfigParser); ok {
-				var err error
-				sc.lbConfig.cfg, err = parser.ParseConfig(jsonCfg)
-				if err != nil {
-					return &serviceconfig.ParseResult{Err: fmt.Errorf("error parsing loadBalancingConfig for policy %q: %v", name, err)}
-				}
-			} else if string(jsonCfg) != "{}" {
-				grpclog.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg))
-			}
-			break
-		}
-		if sc.lbConfig == nil {
-			// We had a loadBalancingConfig field but did not encounter a
-			// supported policy.  The config is considered invalid in this
-			// case.
-			err := fmt.Errorf("invalid loadBalancingConfig: no supported policies found")
-			grpclog.Warningf(err.Error())
-			return &serviceconfig.ParseResult{Err: err}
+	if c := rsc.LoadBalancingConfig; c != nil {
+		sc.lbConfig = &lbConfig{
+			name: c.Name,
+			cfg:  c.Config,
 		}
 	}
 

From a9555d046f43df6e7fb6a68ba43266a3fc13789d Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 9 Apr 2020 11:20:37 -0700
Subject: [PATCH 014/481] balancergroup: update methods to V2balancer (#3505)

And make balancergroup stop load reporting is load store is not set
---
 .../balancer/balancergroup/balancergroup.go   |  59 +++---
 .../balancergroup/balancergroup_test.go       | 178 +++++++++---------
 xds/internal/balancer/edsbalancer/eds_impl.go |  10 +-
 3 files changed, 127 insertions(+), 120 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index ceb305d1f06c..6a97561e933a 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -62,7 +62,7 @@ type subBalancerWithConfig struct {
 	// The static part of sub-balancer. Keeps balancerBuilders and addresses.
 	// To be used when restarting sub-balancer.
 	builder balancer.Builder
-	addrs   []resolver.Address
+	ccState balancer.ClientConnState
 	// The dynamic part of sub-balancer. Only used when balancer group is
 	// started. Gets cleared when sub-balancer is closed.
 	balancer balancer.Balancer
@@ -98,13 +98,13 @@ func (sbc *subBalancerWithConfig) startBalancer() {
 	sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name())
 	sbc.balancer = b
 	if ub, ok := b.(balancer.V2Balancer); ok {
-		ub.UpdateClientConnState(balancer.ClientConnState{ResolverState: resolver.State{Addresses: sbc.addrs}})
-	} else {
-		b.HandleResolvedAddrs(sbc.addrs, nil)
+		ub.UpdateClientConnState(sbc.ccState)
+		return
 	}
+	b.HandleResolvedAddrs(sbc.ccState.ResolverState.Addresses, nil)
 }
 
-func (sbc *subBalancerWithConfig) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	b := sbc.balancer
 	if b == nil {
 		// This sub-balancer was closed. This can happen when EDS removes a
@@ -114,14 +114,14 @@ func (sbc *subBalancerWithConfig) handleSubConnStateChange(sc balancer.SubConn,
 		return
 	}
 	if ub, ok := b.(balancer.V2Balancer); ok {
-		ub.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: state})
-	} else {
-		b.HandleSubConnStateChange(sc, state)
+		ub.UpdateSubConnState(sc, state)
+		return
 	}
+	b.HandleSubConnStateChange(sc, state.ConnectivityState)
 }
 
-func (sbc *subBalancerWithConfig) updateAddrs(addrs []resolver.Address) {
-	sbc.addrs = addrs
+func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnState) error {
+	sbc.ccState = s
 	b := sbc.balancer
 	if b == nil {
 		// This sub-balancer was closed. This should never happen because
@@ -132,13 +132,13 @@ func (sbc *subBalancerWithConfig) updateAddrs(addrs []resolver.Address) {
 		// This will be a common case with priority support, because a
 		// sub-balancer (and the whole balancer group) could be closed because
 		// it's the lower priority, but it can still get address updates.
-		return
+		return nil
 	}
 	if ub, ok := b.(balancer.V2Balancer); ok {
-		ub.UpdateClientConnState(balancer.ClientConnState{ResolverState: resolver.State{Addresses: addrs}})
-	} else {
-		b.HandleResolvedAddrs(addrs, nil)
+		return ub.UpdateClientConnState(s)
 	}
+	b.HandleResolvedAddrs(s.ResolverState.Addresses, nil)
+	return nil
 }
 
 func (sbc *subBalancerWithConfig) stopBalancer() {
@@ -213,7 +213,7 @@ type BalancerGroup struct {
 	// incomingMu guards all operations in the direction:
 	// Sub-balancer-->ClientConn. Including NewSubConn, RemoveSubConn, and
 	// updatePicker. It also guards the map from SubConn to balancer ID, so
-	// handleSubConnStateChange needs to hold it shortly to find the
+	// updateSubConnState needs to hold it shortly to find the
 	// sub-balancer to forward the update.
 	//
 	// The corresponding boolean incomingStarted is used to stop further updates
@@ -435,37 +435,35 @@ func (bg *BalancerGroup) ChangeWeight(id internal.Locality, newWeight uint32) {
 
 // Following are actions from the parent grpc.ClientConn, forward to sub-balancers.
 
-// HandleSubConnStateChange handles the state for the subconn. It finds the
+// UpdateSubConnState handles the state for the subconn. It finds the
 // corresponding balancer and forwards the update.
-func (bg *BalancerGroup) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	bg.incomingMu.Lock()
 	config, ok := bg.scToSubBalancer[sc]
 	if !ok {
 		bg.incomingMu.Unlock()
 		return
 	}
-	if state == connectivity.Shutdown {
+	if state.ConnectivityState == connectivity.Shutdown {
 		// Only delete sc from the map when state changed to Shutdown.
 		delete(bg.scToSubBalancer, sc)
 	}
 	bg.incomingMu.Unlock()
 
 	bg.outgoingMu.Lock()
-	config.handleSubConnStateChange(sc, state)
+	config.updateSubConnState(sc, state)
 	bg.outgoingMu.Unlock()
 }
 
-// HandleResolvedAddrs handles addresses from resolver. It finds the balancer
-// and forwards the update.
-//
-// TODO: change this to UpdateClientConnState to handle addresses and balancer
-// config.
-func (bg *BalancerGroup) HandleResolvedAddrs(id internal.Locality, addrs []resolver.Address) {
+// UpdateClientConnState handles ClientState (including balancer config and
+// addresses) from resolver. It finds the balancer and forwards the update.
+func (bg *BalancerGroup) UpdateClientConnState(id internal.Locality, s balancer.ClientConnState) error {
 	bg.outgoingMu.Lock()
+	defer bg.outgoingMu.Unlock()
 	if config, ok := bg.idToBalancerConfig[id]; ok {
-		config.updateAddrs(addrs)
+		return config.updateClientConnState(s)
 	}
-	bg.outgoingMu.Unlock()
+	return nil
 }
 
 // TODO: handleServiceConfig()
@@ -515,7 +513,12 @@ func (bg *BalancerGroup) updateBalancerState(id internal.Locality, state balance
 		bg.logger.Warningf("balancer group: pickerState for %v not found when update picker/state", id)
 		return
 	}
-	pickerSt.picker = newLoadReportPicker(state.Picker, id, bg.loadStore)
+	newPicker := state.Picker
+	if bg.loadStore != nil {
+		// Only wrap the picker to do load reporting if loadStore was set.
+		newPicker = newLoadReportPicker(state.Picker, id, bg.loadStore)
+	}
+	pickerSt.picker = newPicker
 	pickerSt.state = state.ConnectivityState
 	if bg.incomingStarted {
 		bg.logger.Infof("Child pickers with weight: %+v", bg.idToPickerState)
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 3707f687d2bc..55680d8b3a92 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -65,12 +65,12 @@ func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
 	// Add one balancer to group.
 	bg.Add(testBalancerIDs[0], 1, rrBuilder)
 	// Send one resolved address.
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 
 	// Send subconn state change.
 	sc1 := <-cc.NewSubConnCh
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test pick with one backend.
 	p1 := <-cc.NewPickerCh
@@ -82,11 +82,11 @@ func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
 	}
 
 	// Send two addresses.
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	// Expect one new subconn, send state update.
 	sc2 := <-cc.NewSubConnCh
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test roundrobin pick.
 	p2 := <-cc.NewPickerCh
@@ -96,12 +96,12 @@ func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
 	}
 
 	// Remove the first address.
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[1:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}})
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
-	bg.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	bg.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
 
 	// Test pick with only the second subconn.
 	p3 := <-cc.NewPickerCh
@@ -122,18 +122,18 @@ func (s) TestBalancerGroup_TwoRR_OneBackend(t *testing.T) {
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc1 := <-cc.NewSubConnCh
 
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[0:1])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc2 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
@@ -152,24 +152,24 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
@@ -179,7 +179,7 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	}
 
 	// Turn sc2's connection down, should be RR between balancers.
-	bg.HandleSubConnStateChange(sc2, connectivity.TransientFailure)
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
 	p2 := <-cc.NewPickerCh
 	// Expect two sc1's in the result, because balancer1 will be picked twice,
 	// but there's only one sc in it.
@@ -189,12 +189,12 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	}
 
 	// Remove sc3's addresses.
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[3:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[3:4]}})
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc3, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc3, scToRemove)
 	}
-	bg.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	bg.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
 	p3 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc1, sc4}
 	if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil {
@@ -202,7 +202,7 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	}
 
 	// Turn sc1's connection down.
-	bg.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
 	p4 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc4}
 	if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil {
@@ -210,7 +210,7 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	}
 
 	// Turn last connection to connecting.
-	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
 	p5 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := p5.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
@@ -219,7 +219,7 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 	}
 
 	// Turn all connections down.
-	bg.HandleSubConnStateChange(sc4, connectivity.TransientFailure)
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
 	p6 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
 		if _, err := p6.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
@@ -237,24 +237,24 @@ func (s) TestBalancerGroup_TwoRR_DifferentWeight_MoreBackends(t *testing.T) {
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 2, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
@@ -273,24 +273,24 @@ func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
 	// Add three balancers to group and send one resolved address to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc1 := <-cc.NewSubConnCh
 
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[1:2])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}})
 	sc2 := <-cc.NewSubConnCh
 
 	bg.Add(testBalancerIDs[2], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:2])
+	bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}})
 	sc3 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	p1 := <-cc.NewPickerCh
 	want := []balancer.SubConn{sc1, sc2, sc3}
@@ -311,7 +311,7 @@ func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
 	}
 
 	// move balancer 3 into transient failure.
-	bg.HandleSubConnStateChange(sc3, connectivity.TransientFailure)
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
 	// Remove the first balancer, while the third is transient failure.
 	bg.Remove(testBalancerIDs[0])
 	scToRemove = <-cc.RemoveSubConnCh
@@ -335,24 +335,24 @@ func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 2, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
 
 	// Send state changes for both subconns.
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
@@ -383,28 +383,28 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 2, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 	backendToBalancerID[sc1] = testBalancerIDs[0]
 	backendToBalancerID[sc2] = testBalancerIDs[0]
 
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
 	backendToBalancerID[sc3] = testBalancerIDs[1]
 	backendToBalancerID[sc4] = testBalancerIDs[1]
 
 	// Send state changes for both subconns.
-	bg.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc1, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc2, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc3, connectivity.Ready)
-	bg.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	bg.HandleSubConnStateChange(sc4, connectivity.Ready)
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
@@ -462,9 +462,9 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 2, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
 
@@ -473,8 +473,8 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 		addrs := <-cc.NewSubConnAddrsCh
 		sc := <-cc.NewSubConnCh
 		m1[addrs[0]] = sc
-		bg.HandleSubConnStateChange(sc, connectivity.Connecting)
-		bg.HandleSubConnStateChange(sc, connectivity.Ready)
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 	}
 
 	// Test roundrobin on the last picker.
@@ -490,19 +490,19 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 
 	bg.Close()
 	for i := 0; i < 4; i++ {
-		bg.HandleSubConnStateChange(<-cc.RemoveSubConnCh, connectivity.Shutdown)
+		bg.UpdateSubConnState(<-cc.RemoveSubConnCh, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
 	}
 
 	// Add b3, weight 1, backends [1,2].
 	bg.Add(testBalancerIDs[2], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:3])
+	bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:3]}})
 
 	// Remove b1.
 	bg.Remove(testBalancerIDs[0])
 
 	// Update b2 to weight 3, backends [0,3].
 	bg.ChangeWeight(testBalancerIDs[1], 3)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3]))
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3])}})
 
 	bg.Start()
 
@@ -511,8 +511,8 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 		addrs := <-cc.NewSubConnAddrsCh
 		sc := <-cc.NewSubConnCh
 		m2[addrs[0]] = sc
-		bg.HandleSubConnStateChange(sc, connectivity.Connecting)
-		bg.HandleSubConnStateChange(sc, connectivity.Ready)
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 	}
 
 	// Test roundrobin on the last picker.
@@ -544,9 +544,9 @@ func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
 	bg := New(cc, nil, nil)
 
 	bg.Add(testBalancerIDs[0], 2, &testutils.TestConstBalancerBuilder{})
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	bg.Add(testBalancerIDs[1], 1, &testutils.TestConstBalancerBuilder{})
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
 }
@@ -570,9 +570,9 @@ func initBalancerGroupForCachingTest(t *testing.T) (*BalancerGroup, *testutils.T
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
 	bg.Add(testBalancerIDs[0], 2, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	bg.Add(testBalancerIDs[1], 1, rrBuilder)
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
 
@@ -581,8 +581,8 @@ func initBalancerGroupForCachingTest(t *testing.T) (*BalancerGroup, *testutils.T
 		addrs := <-cc.NewSubConnAddrsCh
 		sc := <-cc.NewSubConnCh
 		m1[addrs[0]] = sc
-		bg.HandleSubConnStateChange(sc, connectivity.Connecting)
-		bg.HandleSubConnStateChange(sc, connectivity.Ready)
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 	}
 
 	// Test roundrobin on the last picker.
@@ -627,7 +627,7 @@ func (s) TestBalancerGroup_locality_caching(t *testing.T) {
 
 	// Turn down subconn for addr2, shouldn't get picker update because
 	// sub-balancer1 was removed.
-	bg.HandleSubConnStateChange(addrToSC[testBackendAddrs[2]], connectivity.TransientFailure)
+	bg.UpdateSubConnState(addrToSC[testBackendAddrs[2]], balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
 	for i := 0; i < 10; i++ {
 		select {
 		case <-cc.NewPickerCh:
@@ -760,7 +760,7 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 		}
 	}
 
-	bg.HandleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[4:6])
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[4:6]}})
 
 	newSCTimeout := time.After(time.Millisecond * 500)
 	scToAdd := map[resolver.Address]int{
@@ -777,8 +777,8 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 			scToAdd[addr[0]] = c - 1
 			sc := <-cc.NewSubConnCh
 			addrToSC[addr[0]] = sc
-			bg.HandleSubConnStateChange(sc, connectivity.Connecting)
-			bg.HandleSubConnStateChange(sc, connectivity.Ready)
+			bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+			bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
 		case <-newSCTimeout:
 			t.Fatalf("timeout waiting for subConns (from new sub-balancer) to be newed")
 		}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 955421821193..d8dd8b7a5a96 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -139,7 +139,9 @@ func (edsImpl *edsBalancerImpl) HandleChildPolicy(name string, config json.RawMe
 			//  balancer becomes ready).
 			bgwc.bg.Remove(id)
 			bgwc.bg.Add(id, config.weight, edsImpl.subBalancerBuilder)
-			bgwc.bg.HandleResolvedAddrs(id, config.addrs)
+			bgwc.bg.UpdateClientConnState(id, balancer.ClientConnState{
+				ResolverState: resolver.State{Addresses: config.addrs},
+			})
 		}
 	}
 }
@@ -313,7 +315,9 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 
 		if addrsChanged {
 			config.addrs = newAddrs
-			bgwc.bg.HandleResolvedAddrs(lid, newAddrs)
+			bgwc.bg.UpdateClientConnState(lid, balancer.ClientConnState{
+				ResolverState: resolver.State{Addresses: newAddrs},
+			})
 		}
 	}
 
@@ -344,7 +348,7 @@ func (edsImpl *edsBalancerImpl) HandleSubConnStateChange(sc balancer.SubConn, s
 		return
 	}
 	if bg := bgwc.bg; bg != nil {
-		bg.HandleSubConnStateChange(sc, s)
+		bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: s})
 	}
 }
 

From b02de00073bfccaef63816bfd3ac296e9abc371f Mon Sep 17 00:00:00 2001
From: sethp-nr <30441101+sethp-nr@users.noreply.github.com>
Date: Thu, 9 Apr 2020 13:45:13 -0700
Subject: [PATCH 015/481] client: option to surface connection errors to
 callers (#3430)

This commit allows blocking clients to receive a more informative error
message than "context deadline exceeded", which is especially helpful in
tracking down persistent client misconfiguration (such as an invalid TLS
certificate, an invalid server that's refusing connections, etc.)
---
 clientconn.go      | 12 +++++++++++-
 clientconn_test.go |  7 ++++---
 dialoptions.go     | 34 ++++++++++++++++++++++++----------
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index 0740693b75bc..a07fd3693b83 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -217,7 +217,14 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 	defer func() {
 		select {
 		case <-ctx.Done():
-			conn, err = nil, ctx.Err()
+			switch {
+			case ctx.Err() == err:
+				conn = nil
+			case err == nil || !cc.dopts.returnLastError:
+				conn, err = nil, ctx.Err()
+			default:
+				conn, err = nil, fmt.Errorf("%v: %v", ctx.Err(), err)
+			}
 		default:
 		}
 	}()
@@ -322,6 +329,9 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 			}
 			if !cc.WaitForStateChange(ctx, s) {
 				// ctx got timeout or canceled.
+				if err = cc.blockingpicker.connectionError(); err != nil && cc.dopts.returnLastError {
+					return nil, err
+				}
 				return nil, ctx.Err()
 			}
 		}
diff --git a/clientconn_test.go b/clientconn_test.go
index 23a9c67bce07..dd4a2b427bfa 100644
--- a/clientconn_test.go
+++ b/clientconn_test.go
@@ -221,7 +221,7 @@ func (s) TestDialWaitsForServerSettingsAndFails(t *testing.T) {
 	client, err := DialContext(ctx,
 		lis.Addr().String(),
 		WithInsecure(),
-		WithBlock(),
+		WithReturnConnectionError(),
 		withBackoff(noBackoff{}),
 		withMinConnectDeadline(func() time.Duration { return time.Second / 4 }))
 	lis.Close()
@@ -229,8 +229,9 @@ func (s) TestDialWaitsForServerSettingsAndFails(t *testing.T) {
 		client.Close()
 		t.Fatalf("Unexpected success (err=nil) while dialing")
 	}
-	if err != context.DeadlineExceeded {
-		t.Fatalf("DialContext(_) = %v; want context.DeadlineExceeded", err)
+	expectedMsg := "server handshake"
+	if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) || !strings.Contains(err.Error(), expectedMsg) {
+		t.Fatalf("DialContext(_) = %v; want a message that includes both %q and %q", err, context.DeadlineExceeded.Error(), expectedMsg)
 	}
 	<-done
 	if numConns < 2 {
diff --git a/dialoptions.go b/dialoptions.go
index 35bde1033a7f..d33ec45276f9 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -46,16 +46,17 @@ type dialOptions struct {
 	chainUnaryInts  []UnaryClientInterceptor
 	chainStreamInts []StreamClientInterceptor
 
-	cp          Compressor
-	dc          Decompressor
-	bs          internalbackoff.Strategy
-	block       bool
-	insecure    bool
-	timeout     time.Duration
-	scChan      <-chan ServiceConfig
-	authority   string
-	copts       transport.ConnectOptions
-	callOptions []CallOption
+	cp              Compressor
+	dc              Decompressor
+	bs              internalbackoff.Strategy
+	block           bool
+	returnLastError bool
+	insecure        bool
+	timeout         time.Duration
+	scChan          <-chan ServiceConfig
+	authority       string
+	copts           transport.ConnectOptions
+	callOptions     []CallOption
 	// This is used by v1 balancer dial option WithBalancer to support v1
 	// balancer, and also by WithBalancerName dial option.
 	balancerBuilder             balancer.Builder
@@ -299,6 +300,19 @@ func WithBlock() DialOption {
 	})
 }
 
+// WithReturnConnectionError returns a DialOption which makes the client connection
+// return a string containing both the last connection error that occurred and
+// the context.DeadlineExceeded error.
+// Implies WithBlock()
+//
+// This API is EXPERIMENTAL.
+func WithReturnConnectionError() DialOption {
+	return newFuncDialOption(func(o *dialOptions) {
+		o.block = true
+		o.returnLastError = true
+	})
+}
+
 // WithInsecure returns a DialOption which disables transport security for this
 // ClientConn. Note that transport security is required unless WithInsecure is
 // set.

From 7c3fd13027648f7fe3bfc6e16289c65c09493e69 Mon Sep 17 00:00:00 2001
From: Cesar Ghali <cesarghali@users.noreply.github.com>
Date: Thu, 9 Apr 2020 22:35:28 -0700
Subject: [PATCH 016/481] credentials/alts: Properly release server InBytes
 buffer after the handshake is complete. (#3513)

---
 credentials/alts/internal/conn/record.go | 29 +++++++++++++++---------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/credentials/alts/internal/conn/record.go b/credentials/alts/internal/conn/record.go
index fd5a53d9a7aa..fdf84ce6571a 100644
--- a/credentials/alts/internal/conn/record.go
+++ b/credentials/alts/internal/conn/record.go
@@ -111,25 +111,32 @@ func NewConn(c net.Conn, side core.Side, recordProtocol string, key []byte, prot
 	}
 	overhead := MsgLenFieldSize + msgTypeFieldSize + crypto.EncryptionOverhead()
 	payloadLengthLimit := altsRecordDefaultLength - overhead
+	// We pre-allocate protectedBuf to be at least of size
+	// 2*altsRecordDefaultLength-1 during initialization. We only read from
+	// the network into protectedBuf when protectedBuf does not contain a
+	// complete frame, which is at most altsRecordDefaultLength-1 (bytes).
+	// And we read at most altsRecordDefaultLength (bytes) data into
+	// protectedBuf at one time. Therefore, 2*altsRecordDefaultLength-1 is
+	// large enough to buffer data read from the network. If protected is
+	// not nil, and its size is larger than 2*altsRecordDefaultLength-1, we
+	// allocate protectedBuf to be size of len(protected), then we copy the
+	// protected content to protectedBuf.
+	protectedBufLen := 2*altsRecordDefaultLength - 1
+	if len(protected) > protectedBufLen {
+		protectedBufLen = len(protected)
+	}
+	protectedBuf := make([]byte, 0, protectedBufLen)
 	if protected == nil {
-		// We pre-allocate protected to be of size
-		// 2*altsRecordDefaultLength-1 during initialization. We only
-		// read from the network into protected when protected does not
-		// contain a complete frame, which is at most
-		// altsRecordDefaultLength-1 (bytes). And we read at most
-		// altsRecordDefaultLength (bytes) data into protected at one
-		// time. Therefore, 2*altsRecordDefaultLength-1 is large enough
-		// to buffer data read from the network.
-		protected = make([]byte, 0, 2*altsRecordDefaultLength-1)
+		copy(protectedBuf, protected)
 	}
 
 	altsConn := &conn{
 		Conn:               c,
 		crypto:             crypto,
 		payloadLengthLimit: payloadLengthLimit,
-		protected:          protected,
+		protected:          protectedBuf,
 		writeBuf:           make([]byte, altsWriteBufferInitialSize),
-		nextFrame:          protected,
+		nextFrame:          protectedBuf,
 		overhead:           overhead,
 	}
 	return altsConn, nil

From 27096e8260a4cbde58a8578f3a2fadd723210ba7 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 10 Apr 2020 16:01:05 -0700
Subject: [PATCH 017/481] doc: update README for supported Go versions and
 travis for tests (#3516)

---
 .travis.yml                                        | 14 +++++++-------
 README.md                                          |  2 +-
 go.mod                                             |  2 +-
 go.sum                                             |  4 ++++
 .../grpc_reflection_v1alpha/reflection.pb.go       |  6 ++++--
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a11e8cbca66f..0e24e59f0567 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,19 +2,19 @@ language: go
 
 matrix:
   include:
-  - go: 1.13.x
+  - go: 1.14.x
     env: VET=1 GO111MODULE=on
-  - go: 1.13.x
+  - go: 1.14.x
     env: RACE=1 GO111MODULE=on
-  - go: 1.13.x
+  - go: 1.14.x
     env: RUN386=1
-  - go: 1.13.x
+  - go: 1.14.x
     env: GRPC_GO_RETRY=on
-  - go: 1.13.x
+  - go: 1.14.x
     env: TESTEXTRAS=1
-  - go: 1.12.x
+  - go: 1.13.x
     env: GO111MODULE=on
-  - go: 1.11.x
+  - go: 1.12.x
     env: GO111MODULE=on
   - go: 1.9.x
     env: GAE=1
diff --git a/README.md b/README.md
index 800e7bd4c90c..79cdf7d4488c 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,7 @@ If you are trying to access grpc-go from within China, please see the
 
 Prerequisites
 -------------
-gRPC-Go requires Go 1.9 or later.
+gRPC-Go supports the three latest releases of Go.
 
 Documentation
 -------------
diff --git a/go.mod b/go.mod
index ecef1ab0cd5d..dfceabe15ae7 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/mock v1.1.1
 	github.com/golang/protobuf v1.3.3
-	github.com/google/go-cmp v0.2.0
+	github.com/google/go-cmp v0.4.0
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
diff --git a/go.sum b/go.sum
index 0bf9f0747bd8..be8078eace22 100644
--- a/go.sum
+++ b/go.sum
@@ -23,6 +23,8 @@ github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -50,6 +52,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 89b89622447e..5d8c412f740a 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -105,7 +105,8 @@ func (*ServerReflectionRequest_FileContainingSymbol) isServerReflectionRequest_M
 
 func (*ServerReflectionRequest_FileContainingExtension) isServerReflectionRequest_MessageRequest() {}
 
-func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequest_MessageRequest() {}
+func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequest_MessageRequest() {
+}
 
 func (*ServerReflectionRequest_ListServices) isServerReflectionRequest_MessageRequest() {}
 
@@ -289,7 +290,8 @@ type ServerReflectionResponse_ErrorResponse struct {
 	ErrorResponse *ErrorResponse `protobuf:"bytes,7,opt,name=error_response,json=errorResponse,proto3,oneof"`
 }
 
-func (*ServerReflectionResponse_FileDescriptorResponse) isServerReflectionResponse_MessageResponse() {}
+func (*ServerReflectionResponse_FileDescriptorResponse) isServerReflectionResponse_MessageResponse() {
+}
 
 func (*ServerReflectionResponse_AllExtensionNumbersResponse) isServerReflectionResponse_MessageResponse() {
 }

From a783b25fe5e65c3d4eb0281d835ed1fa88b08365 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 13 Apr 2020 12:37:38 -0700
Subject: [PATCH 018/481] internal: rename proto import to end with pb (#3523)

---
 vet.sh                             | 3 +++
 xds/internal/testutils/balancer.go | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/vet.sh b/vet.sh
index e12024fb8638..5f79fa6e5e84 100755
--- a/vet.sh
+++ b/vet.sh
@@ -88,6 +88,9 @@ git grep -l '"math/rand"' -- "*.go" 2>&1 | not grep -v '^examples\|^stress\|grpc
 # - Ensure all ptypes proto packages are renamed when importing.
 not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go"
 
+# - Ensure all xds proto imports are renamed to *pb or *grpc.
+git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' | not grep -v 'pb "\|grpc "'
+
 # - Check imports that are illegal in appengine (until Go 1.11).
 # TODO: Remove when we drop Go 1.10 support
 go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 9ffbee237dcf..d6cc3f7be440 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -24,7 +24,7 @@ import (
 	"sync"
 	"testing"
 
-	envoy_api_v2_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
@@ -189,7 +189,7 @@ func (tls *TestLoadStore) CallServerLoad(l internal.Locality, name string, d flo
 }
 
 // ReportTo panics.
-func (*TestLoadStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *envoy_api_v2_core.Node) {
+func (*TestLoadStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) {
 	panic("not implemented")
 }
 

From 8e6052a06fef1de81aadcb5ba3feed6faf6d07df Mon Sep 17 00:00:00 2001
From: Tariq Ibrahim <tariq181290@gmail.com>
Date: Mon, 13 Apr 2020 13:55:48 -0700
Subject: [PATCH 019/481] fix typo in flag description (#3520)

---
 examples/route_guide/client/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/route_guide/client/client.go b/examples/route_guide/client/client.go
index db20b1e9dfc6..f7e5c7564b40 100644
--- a/examples/route_guide/client/client.go
+++ b/examples/route_guide/client/client.go
@@ -40,7 +40,7 @@ var (
 	tls                = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP")
 	caFile             = flag.String("ca_file", "", "The file containing the CA root cert file")
 	serverAddr         = flag.String("server_addr", "localhost:10000", "The server address in the format of host:port")
-	serverHostOverride = flag.String("server_host_override", "x.test.youtube.com", "The server name use to verify the hostname returned by TLS handshake")
+	serverHostOverride = flag.String("server_host_override", "x.test.youtube.com", "The server name used to verify the hostname returned by the TLS handshake")
 )
 
 // printFeature gets the feature for the given point.

From 46c755aff11f06d0426e940696712c8e99995968 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zouyee1989@gmail.com>
Date: Wed, 15 Apr 2020 00:04:53 +0800
Subject: [PATCH 020/481] rename to LocalityID (#3522)

---
 .../balancer/balancergroup/balancergroup.go   | 26 ++++++-------
 .../balancergroup/balancergroup_test.go       |  8 ++--
 xds/internal/balancer/edsbalancer/eds_impl.go |  6 +--
 .../balancer/edsbalancer/eds_impl_test.go     | 10 ++---
 xds/internal/balancer/lrs/lrs.go              | 16 ++++----
 xds/internal/balancer/lrs/lrs_test.go         | 38 +++++++++----------
 xds/internal/client/eds.go                    |  4 +-
 xds/internal/client/eds_test.go               |  8 ++--
 xds/internal/internal.go                      | 17 ++++-----
 xds/internal/internal_test.go                 |  2 +-
 xds/internal/testutils/balancer.go            | 10 ++---
 11 files changed, 72 insertions(+), 73 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index 6a97561e933a..d6307867207e 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -53,7 +53,7 @@ type subBalancerWithConfig struct {
 	// of the actions are forwarded to the parent ClientConn with no change.
 	// Some are forward to balancer group with the sub-balancer ID.
 	balancer.ClientConn
-	id    internal.Locality
+	id    internal.LocalityID
 	group *BalancerGroup
 
 	mu    sync.Mutex
@@ -193,7 +193,7 @@ type BalancerGroup struct {
 	// to sub-balancers after they are closed.
 	outgoingMu         sync.Mutex
 	outgoingStarted    bool
-	idToBalancerConfig map[internal.Locality]*subBalancerWithConfig
+	idToBalancerConfig map[internal.LocalityID]*subBalancerWithConfig
 	// Cache for sub-balancers when they are removed.
 	balancerCache *cache.TimeoutCache
 
@@ -225,7 +225,7 @@ type BalancerGroup struct {
 	// started.
 	//
 	// If an ID is not in map, it's either removed or never added.
-	idToPickerState map[internal.Locality]*pickerState
+	idToPickerState map[internal.LocalityID]*pickerState
 }
 
 // DefaultSubBalancerCloseTimeout is defined as a variable instead of const for
@@ -242,10 +242,10 @@ func New(cc balancer.ClientConn, loadStore lrs.Store, logger *grpclog.PrefixLogg
 		logger:    logger,
 		loadStore: loadStore,
 
-		idToBalancerConfig: make(map[internal.Locality]*subBalancerWithConfig),
+		idToBalancerConfig: make(map[internal.LocalityID]*subBalancerWithConfig),
 		balancerCache:      cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout),
 		scToSubBalancer:    make(map[balancer.SubConn]*subBalancerWithConfig),
-		idToPickerState:    make(map[internal.Locality]*pickerState),
+		idToPickerState:    make(map[internal.LocalityID]*pickerState),
 	}
 }
 
@@ -276,7 +276,7 @@ func (bg *BalancerGroup) Start() {
 // Add adds a balancer built by builder to the group, with given id and weight.
 //
 // weight should never be zero.
-func (bg *BalancerGroup) Add(id internal.Locality, weight uint32, builder balancer.Builder) {
+func (bg *BalancerGroup) Add(id internal.LocalityID, weight uint32, builder balancer.Builder) {
 	if weight == 0 {
 		bg.logger.Errorf("BalancerGroup.add called with weight 0, locality: %v. Locality is not added to balancer group", id)
 		return
@@ -347,7 +347,7 @@ func (bg *BalancerGroup) Add(id internal.Locality, weight uint32, builder balanc
 //
 // It also removes the picker generated from this balancer from the picker
 // group. It always results in a picker update.
-func (bg *BalancerGroup) Remove(id internal.Locality) {
+func (bg *BalancerGroup) Remove(id internal.LocalityID) {
 	bg.outgoingMu.Lock()
 	if sbToRemove, ok := bg.idToBalancerConfig[id]; ok {
 		if bg.outgoingStarted {
@@ -410,7 +410,7 @@ func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWithConfig) {
 // NOTE: It always results in a picker update now. This probably isn't
 // necessary. But it seems better to do the update because it's a change in the
 // picker (which is balancer's snapshot).
-func (bg *BalancerGroup) ChangeWeight(id internal.Locality, newWeight uint32) {
+func (bg *BalancerGroup) ChangeWeight(id internal.LocalityID, newWeight uint32) {
 	if newWeight == 0 {
 		bg.logger.Errorf("BalancerGroup.changeWeight called with newWeight 0. Weight is not changed")
 		return
@@ -457,7 +457,7 @@ func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.
 
 // UpdateClientConnState handles ClientState (including balancer config and
 // addresses) from resolver. It finds the balancer and forwards the update.
-func (bg *BalancerGroup) UpdateClientConnState(id internal.Locality, s balancer.ClientConnState) error {
+func (bg *BalancerGroup) UpdateClientConnState(id internal.LocalityID, s balancer.ClientConnState) error {
 	bg.outgoingMu.Lock()
 	defer bg.outgoingMu.Unlock()
 	if config, ok := bg.idToBalancerConfig[id]; ok {
@@ -501,7 +501,7 @@ func (bg *BalancerGroup) newSubConn(config *subBalancerWithConfig, addrs []resol
 
 // updateBalancerState: create an aggregated picker and an aggregated
 // connectivity state, then forward to ClientConn.
-func (bg *BalancerGroup) updateBalancerState(id internal.Locality, state balancer.State) {
+func (bg *BalancerGroup) updateBalancerState(id internal.LocalityID, state balancer.State) {
 	bg.logger.Infof("Balancer state update from locality %v, new state: %+v", id, state)
 
 	bg.incomingMu.Lock()
@@ -561,7 +561,7 @@ func (bg *BalancerGroup) Close() {
 	bg.balancerCache.Clear(true)
 }
 
-func buildPickerAndState(m map[internal.Locality]*pickerState) balancer.State {
+func buildPickerAndState(m map[internal.LocalityID]*pickerState) balancer.State {
 	var readyN, connectingN int
 	readyPickerWithWeights := make([]pickerState, 0, len(m))
 	for _, ps := range m {
@@ -632,11 +632,11 @@ const (
 type loadReportPicker struct {
 	p balancer.V2Picker
 
-	id        internal.Locality
+	id        internal.LocalityID
 	loadStore lrs.Store
 }
 
-func newLoadReportPicker(p balancer.V2Picker, id internal.Locality, loadStore lrs.Store) *loadReportPicker {
+func newLoadReportPicker(p balancer.V2Picker, id internal.LocalityID, loadStore lrs.Store) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
 		id:        id,
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 55680d8b3a92..f2804abf50e1 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -33,7 +33,7 @@ import (
 
 var (
 	rrBuilder        = balancer.Get(roundrobin.Name)
-	testBalancerIDs  = []internal.Locality{{Region: "b1"}, {Region: "b2"}, {Region: "b3"}}
+	testBalancerIDs  = []internal.LocalityID{{Region: "b1"}, {Region: "b2"}, {Region: "b3"}}
 	testBackendAddrs []resolver.Address
 )
 
@@ -378,7 +378,7 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	bg := New(cc, testLoadStore, nil)
 	bg.Start()
 
-	backendToBalancerID := make(map[balancer.SubConn]internal.Locality)
+	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
@@ -409,8 +409,8 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
 	var (
-		wantStart []internal.Locality
-		wantEnd   []internal.Locality
+		wantStart []internal.LocalityID
+		wantEnd   []internal.LocalityID
 		wantCost  []testutils.TestServerLoad
 	)
 	for i := 0; i < 10; i++ {
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index d8dd8b7a5a96..de3ee58d12fd 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -48,7 +48,7 @@ type localityConfig struct {
 // manages all localities using a balancerGroup.
 type balancerGroupWithConfig struct {
 	bg      *balancergroup.BalancerGroup
-	configs map[internal.Locality]*localityConfig
+	configs map[internal.LocalityID]*localityConfig
 }
 
 // edsBalancerImpl does load balancing based on the EDS responses. Note that it
@@ -221,7 +221,7 @@ func (edsImpl *edsBalancerImpl) HandleEDSResponse(edsResp *xdsclient.EDSUpdate)
 			// new lowest priority).
 			bgwc = &balancerGroupWithConfig{
 				bg:      balancergroup.New(edsImpl.ccWrapperWithPriority(priority), edsImpl.loadStore, edsImpl.logger),
-				configs: make(map[internal.Locality]*localityConfig),
+				configs: make(map[internal.LocalityID]*localityConfig),
 			}
 			edsImpl.priorityToLocalities[priority] = bgwc
 			priorityChanged = true
@@ -255,7 +255,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 	// newLocalitiesSet contains all names of localities in the new EDS response
 	// for the same priority. It's used to delete localities that are removed in
 	// the new EDS response.
-	newLocalitiesSet := make(map[internal.Locality]struct{})
+	newLocalitiesSet := make(map[internal.LocalityID]struct{})
 	for _, locality := range newLocalities {
 		// One balancer for each locality.
 
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 33eabbe4bf09..b8a4e186c7be 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -618,7 +618,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	edsb := newEDSBalancerImpl(cc, nil, testLoadStore, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
-	backendToBalancerID := make(map[balancer.SubConn]internal.Locality)
+	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
 	// Two localities, each with one backend.
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
@@ -627,7 +627,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	sc1 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
-	backendToBalancerID[sc1] = internal.Locality{
+	backendToBalancerID[sc1] = internal.LocalityID{
 		SubZone: testSubZones[0],
 	}
 
@@ -639,15 +639,15 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	sc2 := <-cc.NewSubConnCh
 	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
-	backendToBalancerID[sc2] = internal.Locality{
+	backendToBalancerID[sc2] = internal.LocalityID{
 		SubZone: testSubZones[1],
 	}
 
 	// Test roundrobin with two subconns.
 	p1 := <-cc.NewPickerCh
 	var (
-		wantStart []internal.Locality
-		wantEnd   []internal.Locality
+		wantStart []internal.LocalityID
+		wantEnd   []internal.LocalityID
 	)
 
 	for i := 0; i < 10; i++ {
diff --git a/xds/internal/balancer/lrs/lrs.go b/xds/internal/balancer/lrs/lrs.go
index 811cd913a2eb..eaf5997a9dc8 100644
--- a/xds/internal/balancer/lrs/lrs.go
+++ b/xds/internal/balancer/lrs/lrs.go
@@ -40,9 +40,9 @@ const negativeOneUInt64 = ^uint64(0)
 // them to a server when requested.
 type Store interface {
 	CallDropped(category string)
-	CallStarted(l internal.Locality)
-	CallFinished(l internal.Locality, err error)
-	CallServerLoad(l internal.Locality, name string, d float64)
+	CallStarted(l internal.LocalityID)
+	CallFinished(l internal.LocalityID, err error)
+	CallServerLoad(l internal.LocalityID, name string, d float64)
 	// Report the load of clusterName to cc.
 	ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node)
 }
@@ -145,7 +145,7 @@ type lrsStore struct {
 	lastReported time.Time
 
 	drops            sync.Map // map[string]*uint64
-	localityRPCCount sync.Map // map[internal.Locality]*rpcCountData
+	localityRPCCount sync.Map // map[internal.LocalityID]*rpcCountData
 }
 
 // NewStore creates a store for load reports.
@@ -169,7 +169,7 @@ func (ls *lrsStore) CallDropped(category string) {
 	atomic.AddUint64(p.(*uint64), 1)
 }
 
-func (ls *lrsStore) CallStarted(l internal.Locality) {
+func (ls *lrsStore) CallStarted(l internal.LocalityID) {
 	p, ok := ls.localityRPCCount.Load(l)
 	if !ok {
 		tp := newRPCCountData()
@@ -178,7 +178,7 @@ func (ls *lrsStore) CallStarted(l internal.Locality) {
 	p.(*rpcCountData).incrInProgress()
 }
 
-func (ls *lrsStore) CallFinished(l internal.Locality, err error) {
+func (ls *lrsStore) CallFinished(l internal.LocalityID, err error) {
 	p, ok := ls.localityRPCCount.Load(l)
 	if !ok {
 		// The map is never cleared, only values in the map are reset. So the
@@ -193,7 +193,7 @@ func (ls *lrsStore) CallFinished(l internal.Locality, err error) {
 	}
 }
 
-func (ls *lrsStore) CallServerLoad(l internal.Locality, name string, d float64) {
+func (ls *lrsStore) CallServerLoad(l internal.LocalityID, name string, d float64) {
 	p, ok := ls.localityRPCCount.Load(l)
 	if !ok {
 		// The map is never cleared, only values in the map are reset. So the
@@ -222,7 +222,7 @@ func (ls *lrsStore) buildStats(clusterName string) []*endpointpb.ClusterStats {
 		return true
 	})
 	ls.localityRPCCount.Range(func(locality, countP interface{}) bool {
-		tempLocality := locality.(internal.Locality)
+		tempLocality := locality.(internal.LocalityID)
 		tempCount := countP.(*rpcCountData)
 
 		tempSucceeded := tempCount.loadAndClearSucceeded()
diff --git a/xds/internal/balancer/lrs/lrs_test.go b/xds/internal/balancer/lrs/lrs_test.go
index 75220274e26a..09b22ac9fbe6 100644
--- a/xds/internal/balancer/lrs/lrs_test.go
+++ b/xds/internal/balancer/lrs/lrs_test.go
@@ -49,7 +49,7 @@ const (
 
 var (
 	dropCategories = []string{"drop_for_real", "drop_for_fun"}
-	localities     = []internal.Locality{{Region: "a"}, {Region: "b"}}
+	localities     = []internal.LocalityID{{Region: "a"}, {Region: "b"}}
 	errTest        = fmt.Errorf("test error")
 )
 
@@ -182,14 +182,14 @@ func Test_lrsStore_buildStats_drops(t *testing.T) {
 func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 	tests := []struct {
 		name string
-		rpcs []map[internal.Locality]struct {
+		rpcs []map[internal.LocalityID]struct {
 			start, success, failure uint64
 			serverData              map[string]float64 // Will be reported with successful RPCs.
 		}
 	}{
 		{
 			name: "one rpcCount report",
-			rpcs: []map[internal.Locality]struct {
+			rpcs: []map[internal.LocalityID]struct {
 				start, success, failure uint64
 				serverData              map[string]float64
 			}{{
@@ -198,7 +198,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 		},
 		{
 			name: "two localities one rpcCount report",
-			rpcs: []map[internal.Locality]struct {
+			rpcs: []map[internal.LocalityID]struct {
 				start, success, failure uint64
 				serverData              map[string]float64
 			}{{
@@ -208,7 +208,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 		},
 		{
 			name: "three rpcCount reports",
-			rpcs: []map[internal.Locality]struct {
+			rpcs: []map[internal.LocalityID]struct {
 				start, success, failure uint64
 				serverData              map[string]float64
 			}{{
@@ -222,7 +222,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 		},
 		{
 			name: "no empty report",
-			rpcs: []map[internal.Locality]struct {
+			rpcs: []map[internal.LocalityID]struct {
 				start, success, failure uint64
 				serverData              map[string]float64
 			}{{
@@ -235,7 +235,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 		},
 		{
 			name: "two localities one report with server loads",
-			rpcs: []map[internal.Locality]struct {
+			rpcs: []map[internal.LocalityID]struct {
 				start, success, failure uint64
 				serverData              map[string]float64
 			}{{
@@ -245,7 +245,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 		},
 		{
 			name: "three reports with server loads",
-			rpcs: []map[internal.Locality]struct {
+			rpcs: []map[internal.LocalityID]struct {
 				start, success, failure uint64
 				serverData              map[string]float64
 			}{{
@@ -264,7 +264,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 
 			// InProgress count doesn't get cleared at each buildStats, keep
 			// them to carry over.
-			inProgressCounts := make(map[internal.Locality]uint64)
+			inProgressCounts := make(map[internal.LocalityID]uint64)
 
 			for _, counts := range tt.rpcs {
 				var upstreamLocalityStats []*endpointpb.UpstreamLocalityStats
@@ -314,7 +314,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 				for l, count := range counts {
 					for i := 0; i < int(count.success); i++ {
 						wg.Add(1)
-						go func(l internal.Locality, serverData map[string]float64) {
+						go func(l internal.LocalityID, serverData map[string]float64) {
 							ls.CallStarted(l)
 							ls.CallFinished(l, nil)
 							for n, d := range serverData {
@@ -325,7 +325,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 					}
 					for i := 0; i < int(count.failure); i++ {
 						wg.Add(1)
-						go func(l internal.Locality) {
+						go func(l internal.LocalityID) {
 							ls.CallStarted(l)
 							ls.CallFinished(l, errTest)
 							wg.Done()
@@ -333,7 +333,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 					}
 					for i := 0; i < int(count.start-count.success-count.failure); i++ {
 						wg.Add(1)
-						go func(l internal.Locality) {
+						go func(l internal.LocalityID) {
 							ls.CallStarted(l)
 							wg.Done()
 						}(l)
@@ -356,7 +356,7 @@ type lrsServer struct {
 	mu        sync.Mutex
 	dropTotal uint64
 	drops     map[string]uint64
-	rpcs      map[internal.Locality]*rpcCountDataForTest
+	rpcs      map[internal.LocalityID]*rpcCountDataForTest
 }
 
 func (lrss *lrsServer) StreamLoadStats(stream lrsgrpc.LoadReportingService_StreamLoadStatsServer) error {
@@ -390,7 +390,7 @@ func (lrss *lrsServer) StreamLoadStats(stream lrsgrpc.LoadReportingService_Strea
 			lrss.drops[d.Category] += d.DroppedCount
 		}
 		for _, ss := range stats.UpstreamLocalityStats {
-			l := internal.Locality{
+			l := internal.LocalityID{
 				Region:  ss.Locality.Region,
 				Zone:    ss.Locality.Zone,
 				SubZone: ss.Locality.SubZone,
@@ -423,7 +423,7 @@ func setupServer(t *testing.T, reportingInterval *durationpb.Duration) (addr str
 	lrss = &lrsServer{
 		reportingInterval: reportingInterval,
 		drops:             make(map[string]uint64),
-		rpcs:              make(map[internal.Locality]*rpcCountDataForTest),
+		rpcs:              make(map[internal.LocalityID]*rpcCountDataForTest),
 	}
 	lrsgrpc.RegisterLoadReportingServiceServer(svr, lrss)
 	go svr.Serve(lis)
@@ -474,13 +474,13 @@ func Test_lrsStore_ReportTo(t *testing.T) {
 		}
 	}
 
-	rpcs := map[internal.Locality]*rpcCountDataForTest{
+	rpcs := map[internal.LocalityID]*rpcCountDataForTest{
 		localities[0]: newRPCCountDataForTest(3, 1, 4, nil),
 		localities[1]: newRPCCountDataForTest(1, 5, 9, map[string]float64{"pi": 3.14, "e": 2.71}),
 	}
 	for l, count := range rpcs {
 		for i := 0; i < int(count.succeeded); i++ {
-			go func(i int, l internal.Locality, count *rpcCountDataForTest) {
+			go func(i int, l internal.LocalityID, count *rpcCountDataForTest) {
 				ls.CallStarted(l)
 				ls.CallFinished(l, nil)
 				for n, d := range count.serverLoads {
@@ -489,12 +489,12 @@ func Test_lrsStore_ReportTo(t *testing.T) {
 			}(i, l, count)
 		}
 		for i := 0; i < int(count.inProgress); i++ {
-			go func(i int, l internal.Locality) {
+			go func(i int, l internal.LocalityID) {
 				ls.CallStarted(l)
 			}(i, l)
 		}
 		for i := 0; i < int(count.errored); i++ {
-			go func(i int, l internal.Locality) {
+			go func(i int, l internal.LocalityID) {
 				ls.CallStarted(l)
 				ls.CallFinished(l, errTest)
 			}(i, l)
diff --git a/xds/internal/client/eds.go b/xds/internal/client/eds.go
index 5deeddbdcc4b..4c2fbe98c28d 100644
--- a/xds/internal/client/eds.go
+++ b/xds/internal/client/eds.go
@@ -65,7 +65,7 @@ type Endpoint struct {
 // Locality contains information of a locality.
 type Locality struct {
 	Endpoints []Endpoint
-	ID        internal.Locality
+	ID        internal.LocalityID
 	Priority  uint32
 	Weight    uint32
 }
@@ -128,7 +128,7 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (*EDSUpdate, error) {
 		if l == nil {
 			return nil, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
 		}
-		lid := internal.Locality{
+		lid := internal.LocalityID{
 			Region:  l.Region,
 			Zone:    l.Zone,
 			SubZone: l.SubZone,
diff --git a/xds/internal/client/eds_test.go b/xds/internal/client/eds_test.go
index 3ba6dbbae1da..a491f59703c4 100644
--- a/xds/internal/client/eds_test.go
+++ b/xds/internal/client/eds_test.go
@@ -83,7 +83,7 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 							HealthStatus: EndpointHealthStatusUnhealthy,
 							Weight:       271,
 						}},
-						ID:       internal.Locality{SubZone: "locality-1"},
+						ID:       internal.LocalityID{SubZone: "locality-1"},
 						Priority: 1,
 						Weight:   1,
 					},
@@ -93,7 +93,7 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 							HealthStatus: EndpointHealthStatusDraining,
 							Weight:       828,
 						}},
-						ID:       internal.Locality{SubZone: "locality-2"},
+						ID:       internal.LocalityID{SubZone: "locality-2"},
 						Priority: 0,
 						Weight:   1,
 					},
@@ -208,13 +208,13 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 				Localities: []Locality{
 					{
 						Endpoints: []Endpoint{{Address: "addr1:314"}},
-						ID:        internal.Locality{SubZone: "locality-1"},
+						ID:        internal.LocalityID{SubZone: "locality-1"},
 						Priority:  1,
 						Weight:    1,
 					},
 					{
 						Endpoints: []Endpoint{{Address: "addr2:159"}},
-						ID:        internal.Locality{SubZone: "locality-2"},
+						ID:        internal.LocalityID{SubZone: "locality-2"},
 						Priority:  0,
 						Weight:    1,
 					},
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index 9ed520c96ddb..4ab6c9835fad 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -30,7 +30,7 @@ type clientID string
 // created by the resolver and passed to the balancer.
 const XDSClientID = clientID("xdsClientID")
 
-// Locality is xds.Locality without XXX fields, so it can be used as map
+// LocalityID is xds.Locality without XXX fields, so it can be used as map
 // keys.
 //
 // xds.Locality cannot be map keys because one of the XXX fields is a slice.
@@ -38,22 +38,21 @@ const XDSClientID = clientID("xdsClientID")
 // This struct should only be used as map keys. Use the proto message directly
 // in all other places.
 //
-// TODO: rename to LocalityID.
-type Locality struct {
+type LocalityID struct {
 	Region  string
 	Zone    string
 	SubZone string
 }
 
-func (lamk Locality) String() string {
-	return fmt.Sprintf("%s-%s-%s", lamk.Region, lamk.Zone, lamk.SubZone)
+func (l LocalityID) String() string {
+	return fmt.Sprintf("%s-%s-%s", l.Region, l.Zone, l.SubZone)
 }
 
 // ToProto convert Locality to the proto representation.
-func (lamk Locality) ToProto() *corepb.Locality {
+func (l LocalityID) ToProto() *corepb.Locality {
 	return &corepb.Locality{
-		Region:  lamk.Region,
-		Zone:    lamk.Zone,
-		SubZone: lamk.SubZone,
+		Region:  l.Region,
+		Zone:    l.Zone,
+		SubZone: l.SubZone,
 	}
 }
diff --git a/xds/internal/internal_test.go b/xds/internal/internal_test.go
index 1cfaf4291c9d..bdd6e85cba97 100644
--- a/xds/internal/internal_test.go
+++ b/xds/internal/internal_test.go
@@ -49,7 +49,7 @@ func ignore(name string) bool {
 // fields (expect for XXX_) from the proto message.
 func (s) TestLocalityMatchProtoMessage(t *testing.T) {
 	want1 := make(map[string]string)
-	for ty, i := reflect.TypeOf(Locality{}), 0; i < ty.NumField(); i++ {
+	for ty, i := reflect.TypeOf(LocalityID{}), 0; i < ty.NumField(); i++ {
 		f := ty.Field(i)
 		if ignore(f.Name) {
 			continue
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index d6cc3f7be440..604417c8211b 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -158,8 +158,8 @@ type TestServerLoad struct {
 
 // TestLoadStore is a load store to be used in tests.
 type TestLoadStore struct {
-	CallsStarted []internal.Locality
-	CallsEnded   []internal.Locality
+	CallsStarted []internal.LocalityID
+	CallsEnded   []internal.LocalityID
 	CallsCost    []TestServerLoad
 }
 
@@ -174,17 +174,17 @@ func (*TestLoadStore) CallDropped(category string) {
 }
 
 // CallStarted records a call started.
-func (tls *TestLoadStore) CallStarted(l internal.Locality) {
+func (tls *TestLoadStore) CallStarted(l internal.LocalityID) {
 	tls.CallsStarted = append(tls.CallsStarted, l)
 }
 
 // CallFinished records a call finished.
-func (tls *TestLoadStore) CallFinished(l internal.Locality, err error) {
+func (tls *TestLoadStore) CallFinished(l internal.LocalityID, err error) {
 	tls.CallsEnded = append(tls.CallsEnded, l)
 }
 
 // CallServerLoad records a call server load.
-func (tls *TestLoadStore) CallServerLoad(l internal.Locality, name string, d float64) {
+func (tls *TestLoadStore) CallServerLoad(l internal.LocalityID, name string, d float64) {
 	tls.CallsCost = append(tls.CallsCost, TestServerLoad{Name: name, D: d})
 }
 

From 4c4cafbab72d402246dd9653e173777d63db1250 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 14 Apr 2020 14:43:14 -0700
Subject: [PATCH 021/481] internal/transport: Get rid of TargetInfo struct.
 (#3527)

---
 clientconn.go                        | 13 +++----------
 internal/transport/http2_client.go   |  5 +++--
 internal/transport/transport.go      | 12 +++---------
 internal/transport/transport_test.go | 12 +++++-------
 4 files changed, 14 insertions(+), 28 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index a07fd3693b83..3e4b42b3dd47 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -1224,16 +1224,9 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 	onCloseCalled := make(chan struct{})
 	reconnect := grpcsync.NewEvent()
 
-	authority := ac.cc.authority
 	// addr.ServerName takes precedent over ClientConn authority, if present.
-	if addr.ServerName != "" {
-		authority = addr.ServerName
-	}
-
-	target := transport.TargetInfo{
-		Addr:      addr.Addr,
-		Metadata:  addr.Metadata,
-		Authority: authority,
+	if addr.ServerName == "" {
+		addr.ServerName = ac.cc.authority
 	}
 
 	once := sync.Once{}
@@ -1279,7 +1272,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 		copts.ChannelzParentID = ac.channelzID
 	}
 
-	newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, target, copts, onPrefaceReceipt, onGoAway, onClose)
+	newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, addr, copts, onPrefaceReceipt, onGoAway, onClose)
 	if err != nil {
 		// newTr is either nil, or closed.
 		channelz.Warningf(ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v. Err: %v. Reconnecting...", addr, err)
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 1cc586f73e7a..dab4a3c7126e 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -41,6 +41,7 @@ import (
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
 )
@@ -161,7 +162,7 @@ func isTemporary(err error) bool {
 // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2
 // and starts to receive messages on it. Non-nil error returns if construction
 // fails.
-func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (_ *http2Client, err error) {
+func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (_ *http2Client, err error) {
 	scheme := "http"
 	ctx, cancel := context.WithCancel(ctx)
 	defer func() {
@@ -215,7 +216,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne
 	}
 	if transportCreds != nil {
 		scheme = "https"
-		conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.Authority, conn)
+		conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.ServerName, conn)
 		if err != nil {
 			return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err)
 		}
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index a30da9eb324f..1ffd96ff43d1 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/tap"
@@ -568,17 +569,10 @@ type ConnectOptions struct {
 	MaxHeaderListSize *uint32
 }
 
-// TargetInfo contains the information of the target such as network address and metadata.
-type TargetInfo struct {
-	Addr      string
-	Metadata  interface{}
-	Authority string
-}
-
 // NewClientTransport establishes the transport with the required ConnectOptions
 // and returns it to the caller.
-func NewClientTransport(connectCtx, ctx context.Context, target TargetInfo, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (ClientTransport, error) {
-	return newHTTP2Client(connectCtx, ctx, target, opts, onPrefaceReceipt, onGoAway, onClose)
+func NewClientTransport(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (ClientTransport, error) {
+	return newHTTP2Client(connectCtx, ctx, addr, opts, onPrefaceReceipt, onGoAway, onClose)
 }
 
 // Options provides additional hints and information for message
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 4bd312fdbe85..09e310b60e38 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -40,6 +40,7 @@ import (
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/leakcheck"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 )
 
@@ -430,12 +431,9 @@ func setUp(t *testing.T, port int, maxStreams uint32, ht hType) (*server, *http2
 
 func setUpWithOptions(t *testing.T, port int, serverConfig *ServerConfig, ht hType, copts ConnectOptions) (*server, *http2Client, func()) {
 	server := setUpServerOnly(t, port, serverConfig, ht)
-	addr := "localhost:" + server.port
-	target := TargetInfo{
-		Addr: addr,
-	}
+	addr := resolver.Address{Addr: "localhost:" + server.port}
 	connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second))
-	ct, connErr := NewClientTransport(connectCtx, context.Background(), target, copts, func() {}, func(GoAwayReason) {}, func() {})
+	ct, connErr := NewClientTransport(connectCtx, context.Background(), addr, copts, func() {}, func(GoAwayReason) {}, func() {})
 	if connErr != nil {
 		cancel() // Do not cancel in success path.
 		t.Fatalf("failed to create transport: %v", connErr)
@@ -460,7 +458,7 @@ func setUpWithNoPingServer(t *testing.T, copts ConnectOptions, connCh chan net.C
 		connCh <- conn
 	}()
 	connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second))
-	tr, err := NewClientTransport(connectCtx, context.Background(), TargetInfo{Addr: lis.Addr().String()}, copts, func() {}, func(GoAwayReason) {}, func() {})
+	tr, err := NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, copts, func() {}, func(GoAwayReason) {}, func() {})
 	if err != nil {
 		cancel() // Do not cancel in success path.
 		// Server clean-up.
@@ -1281,7 +1279,7 @@ func (s) TestClientWithMisbehavedServer(t *testing.T) {
 	}()
 	connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second))
 	defer cancel()
-	ct, err := NewClientTransport(connectCtx, context.Background(), TargetInfo{Addr: lis.Addr().String()}, ConnectOptions{}, func() {}, func(GoAwayReason) {}, func() {})
+	ct, err := NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, ConnectOptions{}, func() {}, func(GoAwayReason) {}, func() {})
 	if err != nil {
 		t.Fatalf("Error while creating client transport: %v", err)
 	}

From c97e1d3b729b6400bdc83a8c3f192f4d474e8810 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 14 Apr 2020 15:09:23 -0700
Subject: [PATCH 022/481] Revert "credentials/alts: Properly release server
 InBytes buffer after the handshake is complete. (#3513)" (#3528)

This reverts commit 7c3fd13027648f7fe3bfc6e16289c65c09493e69.
---
 credentials/alts/internal/conn/record.go | 29 +++++++++---------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/credentials/alts/internal/conn/record.go b/credentials/alts/internal/conn/record.go
index fdf84ce6571a..fd5a53d9a7aa 100644
--- a/credentials/alts/internal/conn/record.go
+++ b/credentials/alts/internal/conn/record.go
@@ -111,32 +111,25 @@ func NewConn(c net.Conn, side core.Side, recordProtocol string, key []byte, prot
 	}
 	overhead := MsgLenFieldSize + msgTypeFieldSize + crypto.EncryptionOverhead()
 	payloadLengthLimit := altsRecordDefaultLength - overhead
-	// We pre-allocate protectedBuf to be at least of size
-	// 2*altsRecordDefaultLength-1 during initialization. We only read from
-	// the network into protectedBuf when protectedBuf does not contain a
-	// complete frame, which is at most altsRecordDefaultLength-1 (bytes).
-	// And we read at most altsRecordDefaultLength (bytes) data into
-	// protectedBuf at one time. Therefore, 2*altsRecordDefaultLength-1 is
-	// large enough to buffer data read from the network. If protected is
-	// not nil, and its size is larger than 2*altsRecordDefaultLength-1, we
-	// allocate protectedBuf to be size of len(protected), then we copy the
-	// protected content to protectedBuf.
-	protectedBufLen := 2*altsRecordDefaultLength - 1
-	if len(protected) > protectedBufLen {
-		protectedBufLen = len(protected)
-	}
-	protectedBuf := make([]byte, 0, protectedBufLen)
 	if protected == nil {
-		copy(protectedBuf, protected)
+		// We pre-allocate protected to be of size
+		// 2*altsRecordDefaultLength-1 during initialization. We only
+		// read from the network into protected when protected does not
+		// contain a complete frame, which is at most
+		// altsRecordDefaultLength-1 (bytes). And we read at most
+		// altsRecordDefaultLength (bytes) data into protected at one
+		// time. Therefore, 2*altsRecordDefaultLength-1 is large enough
+		// to buffer data read from the network.
+		protected = make([]byte, 0, 2*altsRecordDefaultLength-1)
 	}
 
 	altsConn := &conn{
 		Conn:               c,
 		crypto:             crypto,
 		payloadLengthLimit: payloadLengthLimit,
-		protected:          protectedBuf,
+		protected:          protected,
 		writeBuf:           make([]byte, altsWriteBufferInitialSize),
-		nextFrame:          protectedBuf,
+		nextFrame:          protected,
 		overhead:           overhead,
 	}
 	return altsConn, nil

From c7079afb44cd2f0297aebd643d4f8877babfcdc9 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 15 Apr 2020 09:59:17 -0700
Subject: [PATCH 023/481] lint: fail on missing package comment (#3524)

golint does check for missing package comment, but with low confidence.
golint checks each file, and complains on every file missing package comment, even though another file in the same package has the comment.

This PR adds a golint check with low min_confidence, and filters out false-positives.
---
 benchmark/worker/main.go                      |  2 ++
 examples/features/health/client/main.go       |  1 +
 examples/features/health/server/main.go       |  1 +
 .../helloworld/mock_helloworld/hw_mock.go     |  2 +-
 .../route_guide/mock_routeguide/rg_mock.go    |  2 +-
 internal/serviceconfig/serviceconfig.go       |  1 +
 internal/wrr/wrr.go                           |  2 ++
 interop/client/client.go                      |  1 +
 interop/grpclb_fallback/client.go             |  1 +
 interop/http2/negative_http2_client.go        | 10 +++---
 interop/server/server.go                      |  1 +
 interop/test_utils.go                         |  1 +
 stress/metrics_client/main.go                 |  1 +
 test/servertester.go                          |  1 +
 vet.sh                                        | 31 +++++++++++++++++++
 xds/internal/internal.go                      |  2 ++
 16 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 5933bd3ac7a1..2b760f54f9fa 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -16,6 +16,8 @@
  *
  */
 
+// Binary worker implements the benchmark worker that can turn into a benchmark
+// client or server.
 package main
 
 import (
diff --git a/examples/features/health/client/main.go b/examples/features/health/client/main.go
index db4145024fa2..1a11782acf84 100644
--- a/examples/features/health/client/main.go
+++ b/examples/features/health/client/main.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Binary client is an example client.
 package main
 
 import (
diff --git a/examples/features/health/server/main.go b/examples/features/health/server/main.go
index 5a9245dfdbcd..3f79c8ba3470 100644
--- a/examples/features/health/server/main.go
+++ b/examples/features/health/server/main.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Binary server is an example server.
 package main
 
 import (
diff --git a/examples/helloworld/mock_helloworld/hw_mock.go b/examples/helloworld/mock_helloworld/hw_mock.go
index 2113386076ef..46e97435f6c1 100644
--- a/examples/helloworld/mock_helloworld/hw_mock.go
+++ b/examples/helloworld/mock_helloworld/hw_mock.go
@@ -1,4 +1,4 @@
-// Automatically generated by MockGen. DO NOT EDIT!
+// Code generated by MockGen. DO NOT EDIT.
 // Source: google.golang.org/grpc/examples/helloworld/helloworld (interfaces: GreeterClient)
 
 package mock_helloworld
diff --git a/examples/route_guide/mock_routeguide/rg_mock.go b/examples/route_guide/mock_routeguide/rg_mock.go
index 65687b3f5df2..564f98750d4b 100644
--- a/examples/route_guide/mock_routeguide/rg_mock.go
+++ b/examples/route_guide/mock_routeguide/rg_mock.go
@@ -1,4 +1,4 @@
-// Automatically generated by MockGen. DO NOT EDIT!
+// Code generated by MockGen. DO NOT EDIT.
 // Source: google.golang.org/grpc/examples/route_guide/routeguide (interfaces: RouteGuideClient,RouteGuide_RouteChatClient)
 
 package mock_routeguide
diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go
index 87f20a3f35b0..9b26414d40e2 100644
--- a/internal/serviceconfig/serviceconfig.go
+++ b/internal/serviceconfig/serviceconfig.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Package serviceconfig contains utility functions to parse service config.
 package serviceconfig
 
 import (
diff --git a/internal/wrr/wrr.go b/internal/wrr/wrr.go
index 29af8984b16a..d46bfad86ef4 100644
--- a/internal/wrr/wrr.go
+++ b/internal/wrr/wrr.go
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+// Package wrr contains the interface and common implementations of wrr
+// algorithms.
 package wrr
 
 // WRR defines an interface that implements weighted round robin.
diff --git a/interop/client/client.go b/interop/client/client.go
index e30081b55bed..58b5465ae4d2 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Binary client is an interop client.
 package main
 
 import (
diff --git a/interop/grpclb_fallback/client.go b/interop/grpclb_fallback/client.go
index a552a66d84bc..ce7cc8e66cde 100644
--- a/interop/grpclb_fallback/client.go
+++ b/interop/grpclb_fallback/client.go
@@ -20,6 +20,7 @@
  *
  */
 
+// Binary grpclb_fallback is an interop test client for grpclb fallback.
 package main
 
 import (
diff --git a/interop/http2/negative_http2_client.go b/interop/http2/negative_http2_client.go
index 44a5d0de3b3b..e5d18c3ef0ae 100644
--- a/interop/http2/negative_http2_client.go
+++ b/interop/http2/negative_http2_client.go
@@ -14,13 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
- *
- * Client used to test http2 error edge cases like GOAWAYs and RST_STREAMs
- *
- * Documentation:
- *	 https://github.com/grpc/grpc/blob/master/doc/negative-http2-interop-test-descriptions.md
  */
 
+// Binary http2 is used to test http2 error edge cases like GOAWAYs and
+// RST_STREAMs
+//
+// Documentation:
+// https://github.com/grpc/grpc/blob/master/doc/negative-http2-interop-test-descriptions.md
 package main
 
 import (
diff --git a/interop/server/server.go b/interop/server/server.go
index dd0e8976b7b3..93c90dab5018 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Binary server is an interop server.
 package main
 
 import (
diff --git a/interop/test_utils.go b/interop/test_utils.go
index d8402319b483..08c4c97f43ab 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -18,6 +18,7 @@
 
 //go:generate protoc --go_out=plugins=grpc:. grpc_testing/test.proto
 
+// Package interop contains functions used by interop client/server.
 package interop
 
 import (
diff --git a/stress/metrics_client/main.go b/stress/metrics_client/main.go
index c59768928a7d..655b97a7ae2d 100644
--- a/stress/metrics_client/main.go
+++ b/stress/metrics_client/main.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Binary metrics_client is a client to retrieve metrics from the server.
 package main
 
 import (
diff --git a/test/servertester.go b/test/servertester.go
index ec904b05e0b7..ff4fa0b3c6bc 100644
--- a/test/servertester.go
+++ b/test/servertester.go
@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+// Package test contains tests.
 package test
 
 import (
diff --git a/vet.sh b/vet.sh
index 5f79fa6e5e84..b845102a355e 100755
--- a/vet.sh
+++ b/vet.sh
@@ -163,4 +163,35 @@ naming.Watcher
 resolver.Backend
 resolver.GRPCLB' "${SC_OUT}"
 
+# - special golint on package comments.
+lint_package_comment_per_package() {
+  # Number of files in this go package.
+  fileCount=$(go list -f '{{len .GoFiles}}' $1)
+  if [ ${fileCount} -eq 0 ]; then
+    return 0
+  fi
+  # Number of package errors generated by golint.
+  lintPackageCommentErrorsCount=$(golint --min_confidence 0 $1 | grep -c "should have a package comment")
+  # golint complains about every file that's missing the package comment. If the
+  # number of files for this package is greater than the number of errors, there's
+  # at least one file with package comment, good. Otherwise, fail.
+  if [ ${fileCount} -le ${lintPackageCommentErrorsCount} ]; then
+    echo "Package $1 (with ${fileCount} files) is missing package comment"
+    return 1
+  fi
+}
+lint_package_comment() {
+  set +ex
+
+  count=0
+  for i in $(go list ./...); do
+    lint_package_comment_per_package "$i"
+    ((count += $?))
+  done
+
+  set -ex
+  return $count
+}
+lint_package_comment
+
 echo SUCCESS
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index 4ab6c9835fad..b2c980003dd8 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+// Package internal contains functions/structs shared by xds
+// balancers/resolvers.
 package internal
 
 import (

From 6e001bea42dc0f7b56d23da563c75900bba7c6eb Mon Sep 17 00:00:00 2001
From: Patrice Chalin <chalin@users.noreply.github.com>
Date: Wed, 15 Apr 2020 13:32:57 -0400
Subject: [PATCH 024/481] README: add link to Go releases (#3526)

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 79cdf7d4488c..751aa6d80a92 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,8 @@ If you are trying to access grpc-go from within China, please see the
 
 Prerequisites
 -------------
-gRPC-Go supports the three latest releases of Go.
+gRPC-Go officially supports the
+[three latest major releases of Go](https://golang.org/doc/devel/release.html).
 
 Documentation
 -------------

From ff40ef4227c9e291b74f4cb81cdc74d57337c77a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 15 Apr 2020 14:32:31 -0700
Subject: [PATCH 025/481] xds: refactor xds_client (#3477)

This PR refactors xds_client to support multiples watches. Those watches can be for the same type and same resource_name.

There's upper level `Client` and lower level `v2client`. Before this change, all logic was in `v2client`, and `Client` was a thin wrapper.

This PR moves some of the functionality from `v2client` to `Client`. New layers:

- Upper level `Client`
  - keeps a list of watchers
    - provides method `func WatchXXX() (cancel func())`
    - has `WatchService()` which involves `LDS` and `RDS`
  - handles resources from the xDS responses and dispatch to the watchers
    - including multiple watchers for the same resource_name
  - keeps cache
    - and checks cache for new watches

- Lower level `v2client`
  - is a dumb client that
    - manages ADS stream
    - sends a new xDS request when add/remove watch
    - parses xDS responses
      - It doesn't call watchers, but forwards all parsed results to upper Client
    - handles ACK/NACK
  - supports `addWatch(type, name)` and `removeWatch(type, name)`
    - instead of `func watchCDS() func()`, which is now moved up to upper `Client`

Also includes other changes:
 - Corresponding test changes (some tests for `v2client` were moved to `Client`)
 - Method and type renaming
   - CDS/EDS -> Cluster/Endpoints
 - callback functions all accept updates as non-pointers
---
 .../balancer/cdsbalancer/cdsbalancer.go       |   6 +-
 .../balancer/cdsbalancer/cdsbalancer_test.go  |  14 +-
 xds/internal/balancer/edsbalancer/eds.go      |   8 +-
 xds/internal/balancer/edsbalancer/eds_impl.go |   2 +-
 xds/internal/balancer/edsbalancer/eds_test.go |   8 +-
 .../edsbalancer/xds_client_wrapper.go         |  11 +-
 .../edsbalancer/xds_client_wrapper_test.go    |  27 +-
 xds/internal/client/cds_test.go               | 487 ------------------
 xds/internal/client/client.go                 | 151 +++---
 xds/internal/client/client_callback.go        | 162 ++++++
 .../client/{logging.go => client_logging.go}  |   0
 xds/internal/client/client_test.go            | 233 +++------
 xds/internal/client/client_watchers.go        | 124 +++++
 .../client/client_watchers_cluster.go         |  56 ++
 .../client/client_watchers_cluster_test.go    | 276 ++++++++++
 .../client/client_watchers_endpoints.go       |  93 ++++
 .../client/client_watchers_endpoints_test.go  | 289 +++++++++++
 xds/internal/client/client_watchers_lds.go    |  47 ++
 .../client/client_watchers_lds_test.go        | 230 +++++++++
 xds/internal/client/client_watchers_rds.go    |  47 ++
 .../client/client_watchers_rds_test.go        | 230 +++++++++
 .../client/client_watchers_service.go         | 114 ++++
 .../client/client_watchers_service_test.go    | 339 ++++++++++++
 xds/internal/client/testutil_test.go          | 114 ++--
 xds/internal/client/types.go                  | 104 ----
 xds/internal/client/v2client.go               | 310 ++++-------
 xds/internal/client/v2client_ack_test.go      | 176 ++++---
 .../client/{cds.go => v2client_cds.go}        |  30 +-
 xds/internal/client/v2client_cds_test.go      | 295 +++++++++++
 .../client/{eds.go => v2client_eds.go}        |  89 +---
 .../{eds_test.go => v2client_eds_test.go}     |  67 +--
 ...s_testutil.go => v2client_eds_testutil.go} |   2 +-
 .../client/{lds.go => v2client_lds.go}        |  29 +-
 .../{lds_test.go => v2client_lds_test.go}     |  74 +--
 .../client/{rds.go => v2client_rds.go}        |  49 +-
 .../{rds_test.go => v2client_rds_test.go}     | 227 +-------
 xds/internal/client/v2client_test.go          | 128 ++---
 xds/internal/testutils/fakeclient/client.go   |  12 +-
 38 files changed, 2906 insertions(+), 1754 deletions(-)
 delete mode 100644 xds/internal/client/cds_test.go
 create mode 100644 xds/internal/client/client_callback.go
 rename xds/internal/client/{logging.go => client_logging.go} (100%)
 create mode 100644 xds/internal/client/client_watchers.go
 create mode 100644 xds/internal/client/client_watchers_cluster.go
 create mode 100644 xds/internal/client/client_watchers_cluster_test.go
 create mode 100644 xds/internal/client/client_watchers_endpoints.go
 create mode 100644 xds/internal/client/client_watchers_endpoints_test.go
 create mode 100644 xds/internal/client/client_watchers_lds.go
 create mode 100644 xds/internal/client/client_watchers_lds_test.go
 create mode 100644 xds/internal/client/client_watchers_rds.go
 create mode 100644 xds/internal/client/client_watchers_rds_test.go
 create mode 100644 xds/internal/client/client_watchers_service.go
 create mode 100644 xds/internal/client/client_watchers_service_test.go
 delete mode 100644 xds/internal/client/types.go
 rename xds/internal/client/{cds.go => v2client_cds.go} (77%)
 create mode 100644 xds/internal/client/v2client_cds_test.go
 rename xds/internal/client/{eds.go => v2client_eds.go} (61%)
 rename xds/internal/client/{eds_test.go => v2client_eds_test.go} (78%)
 rename xds/internal/client/{eds_testutil.go => v2client_eds_testutil.go} (97%)
 rename xds/internal/client/{lds.go => v2client_lds.go} (79%)
 rename xds/internal/client/{lds_test.go => v2client_lds_test.go} (70%)
 rename xds/internal/client/{rds.go => v2client_rds.go} (80%)
 rename xds/internal/client/{rds_test.go => v2client_rds_test.go} (58%)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 0a5471236d9d..bb1653868e4c 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -106,7 +106,7 @@ func (cdsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 // xdsClientInterface contains methods from xdsClient.Client which are used by
 // the cdsBalancer. This will be faked out in unittests.
 type xdsClientInterface interface {
-	WatchCluster(string, func(xdsclient.CDSUpdate, error)) func()
+	WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func()
 	Close()
 }
 
@@ -132,7 +132,7 @@ type scUpdate struct {
 // results in creating a new edsBalancer (if one doesn't already exist) and
 // pushing the update to it.
 type watchUpdate struct {
-	cds xdsclient.CDSUpdate
+	cds xdsclient.ClusterUpdate
 	err error
 }
 
@@ -274,7 +274,7 @@ func (b *cdsBalancer) run() {
 
 // handleClusterUpdate is the CDS watch API callback. It simply pushes the
 // received information on to the update channel for run() to pick it up.
-func (b *cdsBalancer) handleClusterUpdate(cu xdsclient.CDSUpdate, err error) {
+func (b *cdsBalancer) handleClusterUpdate(cu xdsclient.ClusterUpdate, err error) {
 	if b.isClosed() {
 		b.logger.Warningf("xds: received cluster update {%+v} after cdsBalancer was closed", cu)
 		return
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index a2cbe2349c4e..2d5d305ab9fa 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -59,7 +59,7 @@ type testClientConn struct {
 
 // cdsWatchInfo wraps the update and the error sent in a CDS watch callback.
 type cdsWatchInfo struct {
-	update xdsclient.CDSUpdate
+	update xdsclient.ClusterUpdate
 	err    error
 }
 
@@ -369,18 +369,18 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 
 	tests := []struct {
 		name      string
-		cdsUpdate xdsclient.CDSUpdate
+		cdsUpdate xdsclient.ClusterUpdate
 		updateErr error
 		wantCCS   balancer.ClientConnState
 	}{
 		{
 			name:      "happy-case-with-lrs",
-			cdsUpdate: xdsclient.CDSUpdate{ServiceName: serviceName, EnableLRS: true},
+			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
 			wantCCS:   edsCCS(serviceName, true, xdsC),
 		},
 		{
 			name:      "happy-case-without-lrs",
-			cdsUpdate: xdsclient.CDSUpdate{ServiceName: serviceName},
+			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
 			wantCCS:   edsCCS(serviceName, false, xdsC),
 		},
 		{
@@ -408,7 +408,7 @@ func (s) TestResolverError(t *testing.T) {
 		cdsB.Close()
 	}()
 
-	cdsUpdate := xdsclient.CDSUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -433,7 +433,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 		cdsB.Close()
 	}()
 
-	cdsUpdate := xdsclient.CDSUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -453,7 +453,7 @@ func (s) TestClose(t *testing.T) {
 	xdsC, cdsB, edsB, cancel := setupWithWatch(t)
 	defer cancel()
 
-	cdsUpdate := xdsclient.CDSUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index d202d1055aff..f6057ad605f6 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -95,7 +95,7 @@ func (b *edsBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadB
 // TODO: none of the methods in this interface needs to be exported.
 type edsBalancerImplInterface interface {
 	// HandleEDSResponse passes the received EDS message from traffic director to eds balancer.
-	HandleEDSResponse(edsResp *xdsclient.EDSUpdate)
+	HandleEDSResponse(edsResp xdsclient.EndpointsUpdate)
 	// HandleChildPolicy updates the eds balancer the intra-cluster load balancing policy to use.
 	HandleChildPolicy(name string, config json.RawMessage)
 	// HandleSubConnStateChange handles state change for SubConn.
@@ -196,9 +196,9 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 
 func (x *edsBalancer) handleXDSClientUpdate(update interface{}) {
 	switch u := update.(type) {
-	// TODO: this func should accept (*xdsclient.EDSUpdate, error), and process
+	// TODO: this func should accept (xdsclient.EndpointsUpdate, error), and process
 	// the error, instead of having a separate loseContact signal.
-	case *xdsclient.EDSUpdate:
+	case xdsclient.EndpointsUpdate:
 		x.edsImpl.HandleEDSResponse(u)
 	case *loseContact:
 		// loseContact can be useful for going into fallback.
@@ -246,7 +246,7 @@ func (x *edsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	return nil
 }
 
-func (x *edsBalancer) handleEDSUpdate(resp *xdsclient.EDSUpdate) error {
+func (x *edsBalancer) handleEDSUpdate(resp xdsclient.EndpointsUpdate) error {
 	// TODO: this function should take (resp, error), and send them together on
 	// the channel. There doesn't need to be a separate `loseContact` function.
 	select {
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index de3ee58d12fd..ef61296c42f8 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -173,7 +173,7 @@ func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropC
 // SubConns. It also handles drops.
 //
 // HandleChildPolicy and HandleEDSResponse must be called by the same goroutine.
-func (edsImpl *edsBalancerImpl) HandleEDSResponse(edsResp *xdsclient.EDSUpdate) {
+func (edsImpl *edsBalancerImpl) HandleEDSResponse(edsResp xdsclient.EndpointsUpdate) {
 	// TODO: Unhandled fields from EDS response:
 	//  - edsResp.GetPolicy().GetOverprovisioningFactor()
 	//  - locality.GetPriority()
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 83dca16efcab..ef2f93cd62c7 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -109,7 +109,7 @@ func (f *fakeEDSBalancer) HandleChildPolicy(name string, config json.RawMessage)
 }
 
 func (f *fakeEDSBalancer) Close()                                              {}
-func (f *fakeEDSBalancer) HandleEDSResponse(edsResp *xdsclient.EDSUpdate)      {}
+func (f *fakeEDSBalancer) HandleEDSResponse(edsResp xdsclient.EndpointsUpdate) {}
 func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {}
 
 func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) error {
@@ -254,7 +254,7 @@ func (s) TestXDSConfigBalancerNameUpdate(t *testing.T) {
 		})
 
 		xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, balancerName)
-		xdsC.InvokeWatchEDSCallback(&xdsclient.EDSUpdate{}, nil)
+		xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	}
 }
 
@@ -335,7 +335,7 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 		},
 	})
 	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
-	xdsC.InvokeWatchEDSCallback(&xdsclient.EDSUpdate{}, nil)
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	edsLB := waitForNewEDSLB(t, edsLBCh)
 	edsLB.waitForChildPolicy(&loadBalancingConfig{
 		Name:   string(fakeBalancerA),
@@ -384,7 +384,7 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 	})
 
 	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
-	xdsC.InvokeWatchEDSCallback(&xdsclient.EDSUpdate{}, nil)
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	edsLB := waitForNewEDSLB(t, edsLBCh)
 
 	fsc := &fakeSubConn{}
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index a4284ac24ebe..ab06c1a28f35 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -32,7 +32,7 @@ import (
 // xdsClientInterface contains only the xds_client methods needed by EDS
 // balancer. It's defined so we can override xdsclientNew function in tests.
 type xdsClientInterface interface {
-	WatchEndpoints(clusterName string, edsCb func(*xdsclient.EDSUpdate, error)) (cancel func())
+	WatchEndpoints(clusterName string, edsCb func(xdsclient.EndpointsUpdate, error)) (cancel func())
 	ReportLoad(server string, clusterName string, loadStore lrs.Store) (cancel func())
 	Close()
 }
@@ -50,7 +50,7 @@ var (
 type xdsclientWrapper struct {
 	logger *grpclog.PrefixLogger
 
-	newEDSUpdate func(*xdsclient.EDSUpdate) error
+	newEDSUpdate func(xdsclient.EndpointsUpdate) error
 	loseContact  func()
 	bbo          balancer.BuildOptions
 	loadStore    lrs.Store
@@ -78,7 +78,7 @@ type xdsclientWrapper struct {
 //
 // The given callbacks won't be called until the underlying xds_client is
 // working and sends updates.
-func newXDSClientWrapper(newEDSUpdate func(*xdsclient.EDSUpdate) error, loseContact func(), bbo balancer.BuildOptions, loadStore lrs.Store, logger *grpclog.PrefixLogger) *xdsclientWrapper {
+func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate) error, loseContact func(), bbo balancer.BuildOptions, loadStore lrs.Store, logger *grpclog.PrefixLogger) *xdsclientWrapper {
 	return &xdsclientWrapper{
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
@@ -184,7 +184,10 @@ func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
 	}
 
 	c.edsServiceName = nameToWatch
-	cancelEDSWatch := c.xdsclient.WatchEndpoints(c.edsServiceName, func(update *xdsclient.EDSUpdate, err error) {
+	if c.cancelEndpointsWatch != nil {
+		c.cancelEndpointsWatch()
+	}
+	cancelEDSWatch := c.xdsclient.WatchEndpoints(c.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
 		if err != nil {
 			// TODO: this should trigger a call to `c.loseContact`, when the
 			// error indicates "lose contact".
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index d3a3d3aeabd2..15124a8c469a 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -21,6 +21,7 @@ package edsbalancer
 import (
 	"errors"
 	"testing"
+	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
@@ -109,9 +110,25 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 				EDSServiceName: test.edsServiceName,
 			}, nil)
 
-			req, err := fakeServer.XDSRequestChan.Receive()
-			if err != nil {
-				t.Fatalf("EDS RPC failed with err: %v", err)
+			var req interface{}
+			for i := 0; i < 2; i++ {
+				// Each new watch will first cancel the previous watch, and then
+				// start a new watch. The cancel will trigger a request as well.
+				// This loop receives the two requests and keeps the last.
+				r, err := fakeServer.XDSRequestChan.TimedReceive(time.Millisecond * 100)
+				if err != nil {
+					t.Fatalf("i: %v, expected xDS request, but got error: %v", i, err)
+				}
+				req = r
+				// If edsServiceName is empty string, the client doesn't cancel
+				// and resend request. The request from channel was from client
+				// init, and we don't expect a second request.
+				if test.edsServiceName == "" {
+					break
+				}
+			}
+			if r, err := fakeServer.XDSRequestChan.TimedReceive(time.Millisecond * 100); err == nil {
+				t.Fatalf("Expected req channel recv timeout, but got request: %v", r)
 			}
 			edsReq := req.(*fakeserver.Request)
 			if edsReq.Err != nil {
@@ -142,7 +159,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 //   edsBalancer with the received error.
 func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 	edsRespChan := testutils.NewChannel()
-	newEDS := func(update *xdsclient.EDSUpdate) error {
+	newEDS := func(update xdsclient.EndpointsUpdate) error {
 		edsRespChan.Send(update)
 		return nil
 	}
@@ -159,7 +176,7 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 	if gotCluster != testEDSClusterName {
 		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
 	}
-	xdsC.InvokeWatchEDSCallback(nil, errors.New("EDS watch callback error"))
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, errors.New("EDS watch callback error"))
 
 	// The callback is called with an error, expect no update from edsRespChan.
 	//
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
deleted file mode 100644
index 502cb60e2509..000000000000
--- a/xds/internal/client/cds_test.go
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"errors"
-	"fmt"
-	"testing"
-	"time"
-
-	discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	"github.com/golang/protobuf/proto"
-	anypb "github.com/golang/protobuf/ptypes/any"
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-)
-
-const (
-	clusterName1 = "foo-cluster"
-	clusterName2 = "bar-cluster"
-	serviceName1 = "foo-service"
-	serviceName2 = "bar-service"
-)
-
-func (v2c *v2Client) cloneCDSCacheForTesting() map[string]CDSUpdate {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	cloneCache := make(map[string]CDSUpdate)
-	for k, v := range v2c.cdsCache {
-		cloneCache[k] = v
-	}
-	return cloneCache
-}
-
-func (s) TestValidateCluster(t *testing.T) {
-	emptyUpdate := CDSUpdate{ServiceName: "", EnableLRS: false}
-	tests := []struct {
-		name       string
-		cluster    *xdspb.Cluster
-		wantUpdate CDSUpdate
-		wantErr    bool
-	}{
-		{
-			name: "non-eds-cluster-type",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_STATIC},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-				},
-				LbPolicy: xdspb.Cluster_LEAST_REQUEST,
-			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
-		},
-		{
-			name: "no-eds-config",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				LbPolicy:             xdspb.Cluster_ROUND_ROBIN,
-			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
-		},
-		{
-			name: "no-ads-config-source",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig:     &xdspb.Cluster_EdsClusterConfig{},
-				LbPolicy:             xdspb.Cluster_ROUND_ROBIN,
-			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
-		},
-		{
-			name: "non-round-robin-lb-policy",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-				},
-				LbPolicy: xdspb.Cluster_LEAST_REQUEST,
-			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
-		},
-		{
-			name: "happy-case-no-service-name-no-lrs",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-				},
-				LbPolicy: xdspb.Cluster_ROUND_ROBIN,
-			},
-			wantUpdate: emptyUpdate,
-		},
-		{
-			name: "happy-case-no-lrs",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-					ServiceName: serviceName1,
-				},
-				LbPolicy: xdspb.Cluster_ROUND_ROBIN,
-			},
-			wantUpdate: CDSUpdate{ServiceName: serviceName1, EnableLRS: false},
-		},
-		{
-			name:       "happiest-case",
-			cluster:    goodCluster1,
-			wantUpdate: CDSUpdate{ServiceName: serviceName1, EnableLRS: true},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, gotErr := validateCluster(test.cluster)
-			if (gotErr != nil) != test.wantErr {
-				t.Errorf("validateCluster(%+v) returned error: %v, wantErr: %v", test.cluster, gotErr, test.wantErr)
-			}
-			if !cmp.Equal(gotUpdate, test.wantUpdate) {
-				t.Errorf("validateCluster(%+v) = %v, want: %v", test.cluster, gotUpdate, test.wantUpdate)
-			}
-		})
-	}
-}
-
-// TestCDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
-// and creates a v2Client using it. Then, it registers a CDS watcher and tests
-// different CDS responses.
-func (s) TestCDSHandleResponse(t *testing.T) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-
-	tests := []struct {
-		name          string
-		cdsResponse   *xdspb.DiscoveryResponse
-		wantErr       bool
-		wantUpdate    *CDSUpdate
-		wantUpdateErr bool
-	}{
-		// Badly marshaled CDS response.
-		{
-			name:          "badly-marshaled-response",
-			cdsResponse:   badlyMarshaledCDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
-			wantUpdateErr: false,
-		},
-		// Response does not contain Cluster proto.
-		{
-			name:          "no-cluster-proto-in-response",
-			cdsResponse:   badResourceTypeInLDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
-			wantUpdateErr: false,
-		},
-		// Response contains no clusters.
-		{
-			name:          "no-cluster",
-			cdsResponse:   &xdspb.DiscoveryResponse{},
-			wantErr:       false,
-			wantUpdate:    &CDSUpdate{},
-			wantUpdateErr: true,
-		},
-		// Response contains one good cluster we are not interested in.
-		{
-			name:          "one-uninteresting-cluster",
-			cdsResponse:   goodCDSResponse2,
-			wantErr:       false,
-			wantUpdate:    &CDSUpdate{},
-			wantUpdateErr: true,
-		},
-		// Response contains one cluster and it is good.
-		{
-			name:          "one-good-cluster",
-			cdsResponse:   goodCDSResponse1,
-			wantErr:       false,
-			wantUpdate:    &CDSUpdate{ServiceName: serviceName1, EnableLRS: true},
-			wantUpdateErr: false,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			testWatchHandle(t, &watchHandleTestcase{
-				responseToHandle: test.cdsResponse,
-				wantHandleErr:    test.wantErr,
-				wantUpdate:       test.wantUpdate,
-				wantUpdateErr:    test.wantUpdateErr,
-
-				cdsWatch:      v2c.watchCDS,
-				watchReqChan:  fakeServer.XDSRequestChan,
-				handleXDSResp: v2c.handleCDSResponse,
-			})
-		})
-	}
-}
-
-// TestCDSHandleResponseWithoutWatch tests the case where the v2Client receives
-// a CDS response without a registered watcher.
-func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) {
-	_, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-
-	if v2c.handleCDSResponse(goodCDSResponse1) == nil {
-		t.Fatal("v2c.handleCDSResponse() succeeded, should have failed")
-	}
-}
-
-// cdsTestOp contains all data related to one particular test operation. Not
-// all fields make sense for all tests.
-type cdsTestOp struct {
-	// target is the resource name to watch for.
-	target string
-	// responseToSend is the xDS response sent to the client
-	responseToSend *fakeserver.Response
-	// wantOpErr specfies whether the main operation should return an error.
-	wantOpErr bool
-	// wantCDSCache is the expected rdsCache at the end of an operation.
-	wantCDSCache map[string]CDSUpdate
-	// wantWatchCallback specifies if the watch callback should be invoked.
-	wantWatchCallback bool
-}
-
-// testCDSCaching is a helper function which starts a fake xDS server, makes a
-// ClientConn to it, creates a v2Client using it.  It then reads a bunch of
-// test operations to be performed from cdsTestOps and returns error, if any,
-// on the provided error channel. This is executed in a separate goroutine.
-func testCDSCaching(t *testing.T, cdsTestOps []cdsTestOp, errCh *testutils.Channel) {
-	t.Helper()
-
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
-
-	callbackCh := make(chan struct{}, 1)
-	for _, cdsTestOp := range cdsTestOps {
-		// Register a watcher if required, and use a channel to signal the
-		// successful invocation of the callback.
-		if cdsTestOp.target != "" {
-			v2c.watchCDS(cdsTestOp.target, func(u CDSUpdate, err error) {
-				t.Logf("Received callback with CDSUpdate {%+v} and error {%v}", u, err)
-				callbackCh <- struct{}{}
-			})
-			t.Logf("Registered a watcher for CDS target: %v...", cdsTestOp.target)
-
-			// Wait till the request makes it to the fakeServer. This ensures that
-			// the watch request has been processed by the v2Client.
-			if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-				errCh.Send(fmt.Errorf("Timeout waiting for CDS request: %v", err))
-				return
-			}
-			t.Log("FakeServer received request...")
-		}
-
-		// Directly push the response through a call to handleCDSResponse,
-		// thereby bypassing the fakeServer.
-		if cdsTestOp.responseToSend != nil {
-			resp := cdsTestOp.responseToSend.Resp.(*discoverypb.DiscoveryResponse)
-			if err := v2c.handleCDSResponse(resp); (err != nil) != cdsTestOp.wantOpErr {
-				errCh.Send(fmt.Errorf("v2c.handleRDSResponse(%+v) returned err: %v", resp, err))
-				return
-			}
-		}
-
-		// If the test needs the callback to be invoked, just verify that
-		// it was invoked. Since we verify the contents of the cache, it's
-		// ok not to verify the contents of the callback.
-		if cdsTestOp.wantWatchCallback {
-			<-callbackCh
-		}
-
-		if !cmp.Equal(v2c.cloneCDSCacheForTesting(), cdsTestOp.wantCDSCache) {
-			errCh.Send(fmt.Errorf("gotCDSCache: %v, wantCDSCache: %v", v2c.rdsCache, cdsTestOp.wantCDSCache))
-			return
-		}
-	}
-	t.Log("Completed all test ops successfully...")
-	errCh.Send(nil)
-}
-
-// TestCDSCaching tests some end-to-end CDS flows using a fake xDS server, and
-// verifies the CDS data cached at the v2Client.
-func (s) TestCDSCaching(t *testing.T) {
-	ops := []cdsTestOp{
-		// Add an CDS watch for a cluster name (clusterName1), which returns one
-		// matching resource in the response.
-		{
-			target:         clusterName1,
-			responseToSend: &fakeserver.Response{Resp: goodCDSResponse1},
-			wantCDSCache: map[string]CDSUpdate{
-				clusterName1: {serviceName1, true},
-			},
-			wantWatchCallback: true,
-		},
-		// Push an CDS response which contains a new resource (apart from the
-		// one received in the previous response). This should be cached.
-		{
-			responseToSend: &fakeserver.Response{Resp: cdsResponseWithMultipleResources},
-			wantCDSCache: map[string]CDSUpdate{
-				clusterName1: {serviceName1, true},
-				clusterName2: {serviceName2, false},
-			},
-			wantWatchCallback: true,
-		},
-		// Switch the watch target to clusterName2, which was already cached.  No
-		// response is received from the server (as expected), but we want the
-		// callback to be invoked with the new serviceName.
-		{
-			target: clusterName2,
-			wantCDSCache: map[string]CDSUpdate{
-				clusterName1: {serviceName1, true},
-				clusterName2: {serviceName2, false},
-			},
-			wantWatchCallback: true,
-		},
-		// Push an empty CDS response. This should clear the cache.
-		{
-			responseToSend:    &fakeserver.Response{Resp: &xdspb.DiscoveryResponse{TypeUrl: cdsURL}},
-			wantOpErr:         false,
-			wantCDSCache:      map[string]CDSUpdate{},
-			wantWatchCallback: true,
-		},
-	}
-	errCh := testutils.NewChannel()
-	go testCDSCaching(t, ops, errCh)
-	waitForNilErr(t, errCh)
-}
-
-// TestCDSWatchExpiryTimer tests the case where the client does not receive an
-// CDS response for the request that it sends out. We want the watch callback
-// to be invoked with an error once the watchExpiryTimer fires.
-func (s) TestCDSWatchExpiryTimer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
-
-	callbackCh := testutils.NewChannel()
-	v2c.watchCDS(clusterName1, func(u CDSUpdate, err error) {
-		t.Logf("Received callback with CDSUpdate {%+v} and error {%v}", u, err)
-		if u.ServiceName != "" {
-			callbackCh.Send(fmt.Errorf("received serviceName %v in cdsCallback, wanted empty string", u.ServiceName))
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("received nil error in cdsCallback"))
-		}
-		callbackCh.Send(nil)
-	})
-
-	// Wait till the request makes it to the fakeServer. This ensures that
-	// the watch request has been processed by the v2Client.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an CDS request")
-	}
-	waitForNilErr(t, callbackCh)
-}
-
-var (
-	badlyMarshaledCDSResponse = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   []byte{1, 2, 3, 4},
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-	goodCluster1 = &xdspb.Cluster{
-		Name:                 clusterName1,
-		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-			EdsConfig: &corepb.ConfigSource{
-				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-					Ads: &corepb.AggregatedConfigSource{},
-				},
-			},
-			ServiceName: serviceName1,
-		},
-		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
-		LrsServer: &corepb.ConfigSource{
-			ConfigSourceSpecifier: &corepb.ConfigSource_Self{
-				Self: &corepb.SelfConfigSource{},
-			},
-		},
-	}
-	marshaledCluster1, _ = proto.Marshal(goodCluster1)
-	goodCluster2         = &xdspb.Cluster{
-		Name:                 clusterName2,
-		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-			EdsConfig: &corepb.ConfigSource{
-				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-					Ads: &corepb.AggregatedConfigSource{},
-				},
-			},
-			ServiceName: serviceName2,
-		},
-		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
-	}
-	marshaledCluster2, _ = proto.Marshal(goodCluster2)
-	goodCDSResponse1     = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   marshaledCluster1,
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-	goodCDSResponse2 = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   marshaledCluster2,
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-	cdsResponseWithMultipleResources = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   marshaledCluster1,
-			},
-			{
-				TypeUrl: cdsURL,
-				Value:   marshaledCluster2,
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-)
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 5aaf5a1af1e1..bca195e8ce69 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -26,9 +26,12 @@ import (
 	"sync"
 	"time"
 
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/backoff"
+	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
@@ -45,6 +48,18 @@ type Options struct {
 	TargetName string
 }
 
+// Interface to be overridden in tests.
+type xdsv2Client interface {
+	addWatch(resourceType, resourceName string)
+	removeWatch(resourceType, resourceName string)
+	close()
+}
+
+// Function to be overridden in tests.
+var newXDSV2Client = func(parent *Client, cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) xdsv2Client {
+	return newV2Client(parent, cc, nodeProto, backoff, logger)
+}
+
 // Client is a full fledged gRPC client which queries a set of discovery APIs
 // (collectively termed as xDS) on a remote management server, to discover
 // various dynamic resources.
@@ -53,16 +68,23 @@ type Options struct {
 // implementations. But the same client can only be shared by the same parent
 // ClientConn.
 type Client struct {
+	done *grpcsync.Event
 	opts Options
 	cc   *grpc.ClientConn // Connection to the xDS server
-	v2c  *v2Client        // Actual xDS client implementation using the v2 API
+	v2c  xdsv2Client      // Actual xDS client implementation using the v2 API
 
 	logger *grpclog.PrefixLogger
 
-	mu              sync.Mutex
-	serviceCallback func(ServiceUpdate, error)
-	ldsCancel       func()
-	rdsCancel       func()
+	updateCh    *buffer.Unbounded // chan *watcherInfoWithUpdate
+	mu          sync.Mutex
+	ldsWatchers map[string]map[*watchInfo]bool
+	ldsCache    map[string]ldsUpdate
+	rdsWatchers map[string]map[*watchInfo]bool
+	rdsCache    map[string]rdsUpdate
+	cdsWatchers map[string]map[*watchInfo]bool
+	cdsCache    map[string]ClusterUpdate
+	edsWatchers map[string]map[*watchInfo]bool
+	edsCache    map[string]EndpointsUpdate
 }
 
 // New returns a new xdsClient configured with opts.
@@ -85,7 +107,20 @@ func New(opts Options) (*Client, error) {
 	}
 	dopts = append(dopts, opts.DialOpts...)
 
-	c := &Client{opts: opts}
+	c := &Client{
+		done: grpcsync.NewEvent(),
+		opts: opts,
+
+		updateCh:    buffer.NewUnbounded(),
+		ldsWatchers: make(map[string]map[*watchInfo]bool),
+		ldsCache:    make(map[string]ldsUpdate),
+		rdsWatchers: make(map[string]map[*watchInfo]bool),
+		rdsCache:    make(map[string]rdsUpdate),
+		cdsWatchers: make(map[string]map[*watchInfo]bool),
+		cdsCache:    make(map[string]ClusterUpdate),
+		edsWatchers: make(map[string]map[*watchInfo]bool),
+		edsCache:    make(map[string]EndpointsUpdate),
+	}
 
 	cc, err := grpc.Dial(opts.Config.BalancerName, dopts...)
 	if err != nil {
@@ -96,92 +131,42 @@ func New(opts Options) (*Client, error) {
 	c.logger = grpclog.NewPrefixLogger(loggingPrefix(c))
 	c.logger.Infof("Created ClientConn to xDS server: %s", opts.Config.BalancerName)
 
-	c.v2c = newV2Client(cc, opts.Config.NodeProto, backoff.DefaultExponential.Backoff, c.logger)
+	c.v2c = newXDSV2Client(c, cc, opts.Config.NodeProto, backoff.DefaultExponential.Backoff, c.logger)
 	c.logger.Infof("Created")
+	go c.run()
 	return c, nil
 }
 
+// run is a goroutine for all the callbacks.
+//
+// Callback can be called in watch(), if an item is found in cache. Without this
+// goroutine, the callback will be called inline, which might cause a deadlock
+// in user's code. Callbacks also cannot be simple `go callback()` because the
+// order matters.
+func (c *Client) run() {
+	for {
+		select {
+		case t := <-c.updateCh.Get():
+			c.updateCh.Load()
+			if c.done.HasFired() {
+				return
+			}
+			c.callCallback(t.(*watcherInfoWithUpdate))
+		case <-c.done.Done():
+			return
+		}
+	}
+}
+
 // Close closes the gRPC connection to the xDS server.
 func (c *Client) Close() {
+	if c.done.HasFired() {
+		return
+	}
+	c.done.Fire()
 	// TODO: Should we invoke the registered callbacks here with an error that
 	// the client is closed?
 	c.v2c.close()
 	c.cc.Close()
 	c.logger.Infof("Shutdown")
 }
-
-// ServiceUpdate contains update about the service.
-type ServiceUpdate struct {
-	Cluster string
-}
-
-// handleLDSUpdate is the LDS watcher callback we registered with the v2Client.
-func (c *Client) handleLDSUpdate(u ldsUpdate, err error) {
-	c.logger.Infof("xds: client received LDS update: %+v, err: %v", u, err)
-	if err != nil {
-		c.mu.Lock()
-		if c.serviceCallback != nil {
-			c.serviceCallback(ServiceUpdate{}, err)
-		}
-		c.mu.Unlock()
-		return
-	}
-
-	c.mu.Lock()
-	c.rdsCancel = c.v2c.watchRDS(u.routeName, c.handleRDSUpdate)
-	c.mu.Unlock()
-}
-
-// handleRDSUpdate is the RDS watcher callback we registered with the v2Client.
-func (c *Client) handleRDSUpdate(u rdsUpdate, err error) {
-	c.logger.Infof("xds: client received RDS update: %+v, err: %v", u, err)
-	if err != nil {
-		c.mu.Lock()
-		if c.serviceCallback != nil {
-			c.serviceCallback(ServiceUpdate{}, err)
-		}
-		c.mu.Unlock()
-		return
-	}
-
-	c.mu.Lock()
-	if c.serviceCallback != nil {
-		c.serviceCallback(ServiceUpdate{Cluster: u.clusterName}, nil)
-	}
-	c.mu.Unlock()
-}
-
-// WatchService uses LDS and RDS protocols to discover information about the
-// provided serviceName.
-func (c *Client) WatchService(serviceName string, callback func(ServiceUpdate, error)) (cancel func()) {
-	// TODO: Error out early if the client is closed. Ideally, this should
-	// never be called after the client is closed though.
-	c.mu.Lock()
-	c.serviceCallback = callback
-	c.ldsCancel = c.v2c.watchLDS(serviceName, c.handleLDSUpdate)
-	c.mu.Unlock()
-
-	return func() {
-		c.mu.Lock()
-		c.serviceCallback = nil
-		if c.ldsCancel != nil {
-			c.ldsCancel()
-		}
-		if c.rdsCancel != nil {
-			c.rdsCancel()
-		}
-		c.mu.Unlock()
-	}
-}
-
-// WatchCluster uses CDS to discover information about the provided
-// clusterName.
-func (c *Client) WatchCluster(clusterName string, cdsCb func(CDSUpdate, error)) (cancel func()) {
-	return c.v2c.watchCDS(clusterName, cdsCb)
-}
-
-// WatchEndpoints uses EDS to discover information about the endpoints in the
-// provided clusterName.
-func (c *Client) WatchEndpoints(clusterName string, edsCb func(*EDSUpdate, error)) (cancel func()) {
-	return c.v2c.watchEDS(clusterName, edsCb)
-}
diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go
new file mode 100644
index 000000000000..3cd456ac5252
--- /dev/null
+++ b/xds/internal/client/client_callback.go
@@ -0,0 +1,162 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+type watcherInfoWithUpdate struct {
+	wi     *watchInfo
+	update interface{}
+	err    error
+}
+
+func (c *Client) scheduleCallback(wi *watchInfo, update interface{}, err error) {
+	c.updateCh.Put(&watcherInfoWithUpdate{
+		wi:     wi,
+		update: update,
+		err:    err,
+	})
+}
+
+func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
+	c.mu.Lock()
+	// Use a closure to capture the callback and type assertion, to save one
+	// more switch case.
+	//
+	// The callback must be called without c.mu. Otherwise if the callback calls
+	// another watch() inline, it will cause a deadlock. This leaves a small
+	// window that a watcher's callback could be called after the watcher is
+	// canceled, and the user needs to take care of it.
+	var ccb func()
+	switch wiu.wi.typeURL {
+	case ldsURL:
+		if s, ok := c.ldsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
+			ccb = func() { wiu.wi.ldsCallback(wiu.update.(ldsUpdate), wiu.err) }
+		}
+	case rdsURL:
+		if s, ok := c.rdsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
+			ccb = func() { wiu.wi.rdsCallback(wiu.update.(rdsUpdate), wiu.err) }
+		}
+	case cdsURL:
+		if s, ok := c.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
+			ccb = func() { wiu.wi.cdsCallback(wiu.update.(ClusterUpdate), wiu.err) }
+		}
+	case edsURL:
+		if s, ok := c.edsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
+			ccb = func() { wiu.wi.edsCallback(wiu.update.(EndpointsUpdate), wiu.err) }
+		}
+	}
+	c.mu.Unlock()
+
+	if ccb != nil {
+		ccb()
+	}
+}
+
+// newLDSUpdate is called by the underlying xdsv2Client when it receives an xDS
+// response.
+//
+// A response can contain multiple resources. They will be parsed and put in a
+// map from resource name to the resource content.
+func (c *Client) newLDSUpdate(d map[string]ldsUpdate) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for name, update := range d {
+		if s, ok := c.ldsWatchers[name]; ok {
+			for wi := range s {
+				c.scheduleCallback(wi, update, nil)
+			}
+			// Sync cache.
+			c.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, update)
+			c.ldsCache[name] = update
+		}
+	}
+	// TODO: handle removing resources, which means if a resource exists in the
+	// previous update, but not in the new update. This needs the balancers and
+	// resolvers to handle errors correctly.
+
+	// TODO: remove item from cache and remove corresponding RDS cached data.
+}
+
+// newRDSUpdate is called by the underlying xdsv2Client when it receives an xDS
+// response.
+//
+// A response can contain multiple resources. They will be parsed and put in a
+// map from resource name to the resource content.
+func (c *Client) newRDSUpdate(d map[string]rdsUpdate) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for name, update := range d {
+		if s, ok := c.rdsWatchers[name]; ok {
+			for wi := range s {
+				c.scheduleCallback(wi, update, nil)
+			}
+			// Sync cache.
+			c.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, update)
+			c.rdsCache[name] = update
+		}
+	}
+}
+
+// newCDSUpdate is called by the underlying xdsv2Client when it receives an xDS
+// response.
+//
+// A response can contain multiple resources. They will be parsed and put in a
+// map from resource name to the resource content.
+func (c *Client) newCDSUpdate(d map[string]ClusterUpdate) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for name, update := range d {
+		if s, ok := c.cdsWatchers[name]; ok {
+			for wi := range s {
+				c.scheduleCallback(wi, update, nil)
+			}
+			// Sync cache.
+			c.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, update)
+			c.cdsCache[name] = update
+		}
+	}
+	// TODO: handle removing resources, which means if a resource exists in the
+	// previous update, but not in the new update. This needs the balancers and
+	// resolvers to handle errors correctly.
+
+	// TODO: remove item from cache and remove corresponding EDS cached data.
+}
+
+// newEDSUpdate is called by the underlying xdsv2Client when it receives an xDS
+// response.
+//
+// A response can contain multiple resources. They will be parsed and put in a
+// map from resource name to the resource content.
+func (c *Client) newEDSUpdate(d map[string]EndpointsUpdate) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for name, update := range d {
+		if s, ok := c.edsWatchers[name]; ok {
+			for wi := range s {
+				c.scheduleCallback(wi, update, nil)
+			}
+			// Sync cache.
+			c.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, update)
+			c.edsCache[name] = update
+		}
+	}
+}
diff --git a/xds/internal/client/logging.go b/xds/internal/client/client_logging.go
similarity index 100%
rename from xds/internal/client/logging.go
rename to xds/internal/client/client_logging.go
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 9d99bc68a244..0775032711c6 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -19,12 +19,11 @@
 package client
 
 import (
-	"errors"
-	"fmt"
 	"testing"
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/testutils"
@@ -41,6 +40,16 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
+const (
+	testXDSServer   = "xds-server"
+	chanRecvTimeout = 100 * time.Millisecond
+
+	testLDSName = "test-lds"
+	testRDSName = "test-rds"
+	testCDSName = "test-cds"
+	testEDSName = "test-eds"
+)
+
 func clientOpts(balancerName string) Options {
 	return Options{
 		Config: bootstrap.Config{
@@ -48,9 +57,6 @@ func clientOpts(balancerName string) Options {
 			Creds:        grpc.WithInsecure(),
 			NodeProto:    &corepb.Node{},
 		},
-		// WithTimeout is deprecated. But we are OK to call it here from the
-		// test, so we clearly know that the dial failed.
-		DialOpts: []grpc.DialOption{grpc.WithTimeout(5 * time.Second), grpc.WithBlock()},
 	}
 }
 
@@ -117,185 +123,88 @@ func (s) TestNew(t *testing.T) {
 	}
 }
 
-// TestWatchService tests the happy case of registering a watcher for
-// service updates and receiving a good update.
-func (s) TestWatchService(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
-	defer cleanup()
-
-	xdsClient, err := New(clientOpts(fakeServer.Address))
-	if err != nil {
-		t.Fatalf("New returned error: %v", err)
-	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
-
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if err != nil {
-			callbackCh.Send(fmt.Errorf("xdsClient.WatchService returned error: %v", err))
-			return
-		}
-		if su.Cluster != goodClusterName1 {
-			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want clusterName: %+v", su.Cluster, goodClusterName1))
-			return
-		}
-		callbackCh.Send(nil)
-	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
-
-	// Make the fakeServer send LDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
-	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
+type testXDSV2Client struct {
+	r updateHandler
 
-	// Make the fakeServer send RDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an RDS request")
-	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodRDSResponse1}
-	waitForNilErr(t, callbackCh)
+	addWatches    map[string]chan string
+	removeWatches map[string]chan string
 }
 
-// TestWatchServiceWithNoResponseFromServer tests the case where the
-// xDS server does not respond to the requests being sent out as part of
-// registering a service update watcher. The underlying v2Client will timeout
-// and will send us an error.
-func (s) TestWatchServiceWithNoResponseFromServer(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
+func overrideNewXDSV2Client() (<-chan *testXDSV2Client, func()) {
+	oldNewXDSV2Client := newXDSV2Client
+	ch := make(chan *testXDSV2Client, 1)
+	newXDSV2Client = func(parent *Client, cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) xdsv2Client {
+		ret := newTestXDSV2Client(parent)
+		ch <- ret
+		return ret
 	}
-	defer cleanup()
+	return ch, func() { newXDSV2Client = oldNewXDSV2Client }
+}
 
-	xdsClient, err := New(clientOpts(fakeServer.Address))
-	if err != nil {
-		t.Fatalf("New returned error: %v", err)
+func newTestXDSV2Client(r updateHandler) *testXDSV2Client {
+	addWatches := make(map[string]chan string)
+	addWatches[ldsURL] = make(chan string, 10)
+	addWatches[rdsURL] = make(chan string, 10)
+	addWatches[cdsURL] = make(chan string, 10)
+	addWatches[edsURL] = make(chan string, 10)
+	removeWatches := make(map[string]chan string)
+	removeWatches[ldsURL] = make(chan string, 10)
+	removeWatches[rdsURL] = make(chan string, 10)
+	removeWatches[cdsURL] = make(chan string, 10)
+	removeWatches[edsURL] = make(chan string, 10)
+	return &testXDSV2Client{
+		r:             r,
+		addWatches:    addWatches,
+		removeWatches: removeWatches,
 	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
-
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
+}
 
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.Cluster != "" {
-			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want empty clusterName", su.Cluster))
-			return
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("xdsClient.WatchService returned error non-nil error"))
-			return
-		}
-		callbackCh.Send(nil)
-	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
+func (c *testXDSV2Client) addWatch(resourceType, resourceName string) {
+	c.addWatches[resourceType] <- resourceName
+}
 
-	// Wait for one request from the client, but send no reponses.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
-	}
-	waitForNilErr(t, callbackCh)
+func (c *testXDSV2Client) removeWatch(resourceType, resourceName string) {
+	c.removeWatches[resourceType] <- resourceName
 }
 
-// TestWatchServiceEmptyRDS tests the case where the underlying
-// v2Client receives an empty RDS response.
-func (s) TestWatchServiceEmptyRDS(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
+func (c *testXDSV2Client) close() {}
+
+// TestWatchCallAnotherWatch covers the case where watch() is called inline by a
+// callback. It makes sure it doesn't cause a deadlock.
+func (s) TestWatchCallAnotherWatch(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
 	defer cleanup()
 
-	xdsClient, err := New(clientOpts(fakeServer.Address))
+	c, err := New(clientOpts(testXDSServer))
 	if err != nil {
-		t.Fatalf("New returned error: %v", err)
+		t.Fatalf("failed to create client: %v", err)
 	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
+	defer c.Close()
 
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
+	v2Client := <-v2ClientCh
 
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.Cluster != "" {
-			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want empty clusterName", su.Cluster))
-			return
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("xdsClient.WatchService returned error non-nil error"))
-			return
-		}
-		callbackCh.Send(nil)
+	clusterUpdateCh := testutils.NewChannel()
+	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
+		// Calls another watch inline, to ensure there's deadlock.
+		c.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
 	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
-
-	// Make the fakeServer send LDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
-	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
-
-	// Make the fakeServer send an empty RDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an RDS request")
-	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: noVirtualHostsInRDSResponse}
-	waitForNilErr(t, callbackCh)
-}
-
-// TestWatchServiceWithClientClose tests the case where xDS responses are
-// received after the client is closed, and we make sure that the registered
-// watcher callback is not invoked.
-func (s) TestWatchServiceWithClientClose(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
-	defer cleanup()
-
-	xdsClient, err := New(clientOpts(fakeServer.Address))
-	if err != nil {
-		t.Fatalf("New returned error: %v", err)
-	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
 
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		callbackCh.Send(errors.New("watcher callback invoked after client close"))
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
 	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
 
-	// Make the fakeServer send LDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
+	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
 
-	xdsClient.Close()
-	t.Log("Closing the xdsClient...")
+	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate2,
+	})
 
-	// Push an RDS response from the fakeserver
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodRDSResponse1}
-	if cbErr, err := callbackCh.Receive(); err != testutils.ErrRecvTimeout {
-		t.Fatal(cbErr)
+	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
-
 }
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
new file mode 100644
index 000000000000..bb1e15db709a
--- /dev/null
+++ b/xds/internal/client/client_watchers.go
@@ -0,0 +1,124 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"time"
+)
+
+// The value chosen here is based on the default value of the
+// initial_fetch_timeout field in corepb.ConfigSource proto.
+var defaultWatchExpiryTimeout = 15 * time.Second
+
+const (
+	ldsURL = "type.googleapis.com/envoy.api.v2.Listener"
+	rdsURL = "type.googleapis.com/envoy.api.v2.RouteConfiguration"
+	cdsURL = "type.googleapis.com/envoy.api.v2.Cluster"
+	edsURL = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
+)
+
+// watchInfo holds all the information from a watch() call.
+type watchInfo struct {
+	typeURL string
+	target  string
+
+	ldsCallback ldsCallbackFunc
+	rdsCallback rdsCallbackFunc
+	cdsCallback func(ClusterUpdate, error)
+	edsCallback func(EndpointsUpdate, error)
+	expiryTimer *time.Timer
+}
+
+func (c *Client) watch(wi *watchInfo) (cancel func()) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.logger.Debugf("new watch for type %v, resource name %v", wi.typeURL, wi.target)
+	var watchers map[string]map[*watchInfo]bool
+	switch wi.typeURL {
+	case ldsURL:
+		watchers = c.ldsWatchers
+	case rdsURL:
+		watchers = c.rdsWatchers
+	case cdsURL:
+		watchers = c.cdsWatchers
+	case edsURL:
+		watchers = c.edsWatchers
+	}
+
+	resourceName := wi.target
+	s, ok := watchers[wi.target]
+	if !ok {
+		// If this is a new watcher, will ask lower level to send a new request with
+		// the resource name.
+		//
+		// If this type+name is already being watched, will not notify the
+		// underlying xdsv2Client.
+		c.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.typeURL, wi.target)
+		s = make(map[*watchInfo]bool)
+		watchers[resourceName] = s
+		c.v2c.addWatch(wi.typeURL, resourceName)
+	}
+	// No matter what, add the new watcher to the set, so it's callback will be
+	// call for new responses.
+	s[wi] = true
+
+	// If the resource is in cache, call the callback with the value.
+	switch wi.typeURL {
+	case ldsURL:
+		if v, ok := c.ldsCache[resourceName]; ok {
+			c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, v)
+			c.scheduleCallback(wi, v, nil)
+		}
+	case rdsURL:
+		if v, ok := c.rdsCache[resourceName]; ok {
+			c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, v)
+			c.scheduleCallback(wi, v, nil)
+		}
+	case cdsURL:
+		if v, ok := c.cdsCache[resourceName]; ok {
+			c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, v)
+			c.scheduleCallback(wi, v, nil)
+		}
+	case edsURL:
+		if v, ok := c.edsCache[resourceName]; ok {
+			c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, v)
+			c.scheduleCallback(wi, v, nil)
+		}
+	}
+
+	return func() {
+		c.logger.Debugf("watch for type %v, resource name %v canceled", wi.typeURL, wi.target)
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		if s := watchers[resourceName]; s != nil {
+			wi.expiryTimer.Stop()
+			// Remove this watcher, so it's callback will not be called in the
+			// future.
+			delete(s, wi)
+			if len(s) == 0 {
+				c.logger.Debugf("last watch for type %v, resource name %v canceled, will send a new xDS request", wi.typeURL, wi.target)
+				// If this was the last watcher, also tell xdsv2Client to stop
+				// watching this resource.
+				delete(watchers, resourceName)
+				c.v2c.removeWatch(wi.typeURL, resourceName)
+				// TODO: remove item from cache.
+			}
+		}
+	}
+}
diff --git a/xds/internal/client/client_watchers_cluster.go b/xds/internal/client/client_watchers_cluster.go
new file mode 100644
index 000000000000..90d905f7a246
--- /dev/null
+++ b/xds/internal/client/client_watchers_cluster.go
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"time"
+)
+
+// ClusterUpdate contains information from a received CDS response, which is of
+// interest to the registered CDS watcher.
+type ClusterUpdate struct {
+	// ServiceName is the service name corresponding to the clusterName which
+	// is being watched for through CDS.
+	ServiceName string
+	// EnableLRS indicates whether or not load should be reported through LRS.
+	EnableLRS bool
+}
+
+// WatchCluster uses CDS to discover information about the provided
+// clusterName.
+//
+// WatchCluster can be called multiple times, with same or different
+// clusterNames. Each call will start an independent watcher for the resource.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
+	wi := &watchInfo{
+		typeURL:     cdsURL,
+		target:      clusterName,
+		cdsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		c.scheduleCallback(wi, ClusterUpdate{}, fmt.Errorf("xds: CDS target %s not found, watcher timeout", clusterName))
+	})
+	return c.watch(wi)
+}
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
new file mode 100644
index 000000000000..3621d8dfcd19
--- /dev/null
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -0,0 +1,276 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type clusterUpdateErr struct {
+	u   ClusterUpdate
+	err error
+}
+
+// TestClusterWatch covers the cases:
+// - an update is received after a watch()
+// - an update for another resource name (which doesn't trigger callback)
+// - an upate is received after cancel()
+func (s) TestClusterWatch(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	clusterUpdateCh := testutils.NewChannel()
+	cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	// This is calling v2Client.r to send the update, but r is set to Client, so
+	// this is same as calling Client to update. The one thing this covers is
+	// that `NewXDSV2Client` is called with the right parent.
+	//
+	// TODO: in a future cleanup, this (and the same thing in other tests) can
+	// be changed call Client directly.
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another update for a different resource name.
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		"randomName": {},
+	})
+
+	if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+
+	// Cancel watch, and send update again.
+	cancelWatch()
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestClusterTwoWatchSameResourceName covers the case where an update is received
+// after two watch() for the same resource name.
+func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var clusterUpdateChs []*testutils.Channel
+	const count = 2
+
+	var cancelLastWatch func()
+
+	for i := 0; i < count; i++ {
+		clusterUpdateCh := testutils.NewChannel()
+		clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh)
+		cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
+		})
+	}
+
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	// Cancel the last watch, and send update again.
+	cancelLastWatch()
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	for i := 0; i < count-1; i++ {
+		if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := clusterUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestClusterThreeWatchDifferentResourceName covers the case where an update is
+// received after three watch() for different resource names.
+func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var clusterUpdateChs []*testutils.Channel
+	const count = 2
+
+	// Two watches for the same name.
+	for i := 0; i < count; i++ {
+		clusterUpdateCh := testutils.NewChannel()
+		clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh)
+		c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
+			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
+		})
+	}
+
+	// Third watch for a different name.
+	clusterUpdateCh2 := testutils.NewChannel()
+	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
+		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName + "1": wantUpdate1,
+		testCDSName + "2": wantUpdate2,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
+			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
+
+// TestClusterWatchAfterCache covers the case where watch is called after the update
+// is in cache.
+func (s) TestClusterWatchAfterCache(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	clusterUpdateCh := testutils.NewChannel()
+	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another watch for the resource in cache.
+	clusterUpdateCh2 := testutils.NewChannel()
+	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
+	})
+
+	// New watch should receives the update.
+	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Old watch should see nothing.
+	if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestClusterWatchExpiryTimer tests the case where the client does not receive
+// an CDS response for the request that it sends out. We want the watch callback
+// to be invoked with an error once the watchExpiryTimer fires.
+func (s) TestClusterWatchExpiryTimer(t *testing.T) {
+	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
+	defaultWatchExpiryTimeout = 500 * time.Millisecond
+	defer func() {
+		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
+	}()
+
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	<-v2ClientCh
+
+	clusterUpdateCh := testutils.NewChannel()
+	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
+		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
+	})
+
+	u, err := clusterUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	if err != nil {
+		t.Fatalf("failed to get clusterUpdate: %v", err)
+	}
+	uu := u.(clusterUpdateErr)
+	if uu.u != (ClusterUpdate{}) {
+		t.Errorf("unexpected clusterUpdate: %v, want %v", uu.u, ClusterUpdate{})
+	}
+	if uu.err == nil {
+		t.Errorf("unexpected clusterError: <nil>, want error watcher timeout")
+	}
+}
diff --git a/xds/internal/client/client_watchers_endpoints.go b/xds/internal/client/client_watchers_endpoints.go
new file mode 100644
index 000000000000..db28865aebe3
--- /dev/null
+++ b/xds/internal/client/client_watchers_endpoints.go
@@ -0,0 +1,93 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc/xds/internal"
+)
+
+// OverloadDropConfig contains the config to drop overloads.
+type OverloadDropConfig struct {
+	Category    string
+	Numerator   uint32
+	Denominator uint32
+}
+
+// EndpointHealthStatus represents the health status of an endpoint.
+type EndpointHealthStatus int32
+
+const (
+	// EndpointHealthStatusUnknown represents HealthStatus UNKNOWN.
+	EndpointHealthStatusUnknown EndpointHealthStatus = iota
+	// EndpointHealthStatusHealthy represents HealthStatus HEALTHY.
+	EndpointHealthStatusHealthy
+	// EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY.
+	EndpointHealthStatusUnhealthy
+	// EndpointHealthStatusDraining represents HealthStatus DRAINING.
+	EndpointHealthStatusDraining
+	// EndpointHealthStatusTimeout represents HealthStatus TIMEOUT.
+	EndpointHealthStatusTimeout
+	// EndpointHealthStatusDegraded represents HealthStatus DEGRADED.
+	EndpointHealthStatusDegraded
+)
+
+// Endpoint contains information of an endpoint.
+type Endpoint struct {
+	Address      string
+	HealthStatus EndpointHealthStatus
+	Weight       uint32
+}
+
+// Locality contains information of a locality.
+type Locality struct {
+	Endpoints []Endpoint
+	ID        internal.LocalityID
+	Priority  uint32
+	Weight    uint32
+}
+
+// EndpointsUpdate contains an EDS update.
+type EndpointsUpdate struct {
+	Drops      []OverloadDropConfig
+	Localities []Locality
+}
+
+// WatchEndpoints uses EDS to discover endpoints in the provided clusterName.
+//
+// WatchEndpoints can be called multiple times, with same or different
+// clusterNames. Each call will start an independent watcher for the resource.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
+	wi := &watchInfo{
+		typeURL:     edsURL,
+		target:      clusterName,
+		edsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		c.scheduleCallback(wi, EndpointsUpdate{}, fmt.Errorf("xds: EDS target %s not found, watcher timeout", clusterName))
+	})
+	return c.watch(wi)
+}
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
new file mode 100644
index 000000000000..46e62bf57ec7
--- /dev/null
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -0,0 +1,289 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+var (
+	testLocalities = []Locality{
+		{
+			Endpoints: []Endpoint{{Address: "addr1:314"}},
+			ID:        internal.LocalityID{SubZone: "locality-1"},
+			Priority:  1,
+			Weight:    1,
+		},
+		{
+			Endpoints: []Endpoint{{Address: "addr2:159"}},
+			ID:        internal.LocalityID{SubZone: "locality-2"},
+			Priority:  0,
+			Weight:    1,
+		},
+	}
+)
+
+type endpointsUpdateErr struct {
+	u   EndpointsUpdate
+	err error
+}
+
+// TestEndpointsWatch covers the cases:
+// - an update is received after a watch()
+// - an update for another resource name (which doesn't trigger callback)
+// - an upate is received after cancel()
+func (s) TestEndpointsWatch(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	endpointsUpdateCh := testutils.NewChannel()
+	cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another update for a different resource name.
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		"randomName": {},
+	})
+
+	if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+
+	// Cancel watch, and send update again.
+	cancelWatch()
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestEndpointsTwoWatchSameResourceName covers the case where an update is received
+// after two watch() for the same resource name.
+func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var endpointsUpdateChs []*testutils.Channel
+	const count = 2
+
+	var cancelLastWatch func()
+
+	for i := 0; i < count; i++ {
+		endpointsUpdateCh := testutils.NewChannel()
+		endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh)
+		cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
+		})
+	}
+
+	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	// Cancel the last watch, and send update again.
+	cancelLastWatch()
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	for i := 0; i < count-1; i++ {
+		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := endpointsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestEndpointsThreeWatchDifferentResourceName covers the case where an update is
+// received after three watch() for different resource names.
+func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var endpointsUpdateChs []*testutils.Channel
+	const count = 2
+
+	// Two watches for the same name.
+	for i := 0; i < count; i++ {
+		endpointsUpdateCh := testutils.NewChannel()
+		endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh)
+		c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
+			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
+		})
+	}
+
+	// Third watch for a different name.
+	endpointsUpdateCh2 := testutils.NewChannel()
+	c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
+		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate1 := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
+	wantUpdate2 := EndpointsUpdate{Localities: []Locality{testLocalities[1]}}
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		testCDSName + "1": wantUpdate1,
+		testCDSName + "2": wantUpdate2,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
+
+// TestEndpointsWatchAfterCache covers the case where watch is called after the update
+// is in cache.
+func (s) TestEndpointsWatchAfterCache(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	endpointsUpdateCh := testutils.NewChannel()
+	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
+	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another watch for the resource in cache.
+	endpointsUpdateCh2 := testutils.NewChannel()
+	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
+	})
+
+	// New watch should receives the update.
+	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Old watch should see nothing.
+	if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestEndpointsWatchExpiryTimer tests the case where the client does not receive
+// an CDS response for the request that it sends out. We want the watch callback
+// to be invoked with an error once the watchExpiryTimer fires.
+func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
+	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
+	defaultWatchExpiryTimeout = 500 * time.Millisecond
+	defer func() {
+		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
+	}()
+
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	<-v2ClientCh
+
+	endpointsUpdateCh := testutils.NewChannel()
+	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
+	})
+
+	u, err := endpointsUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	if err != nil {
+		t.Fatalf("failed to get endpointsUpdate: %v", err)
+	}
+	uu := u.(endpointsUpdateErr)
+	if !cmp.Equal(uu.u, EndpointsUpdate{}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		t.Errorf("unexpected endpointsUpdate: %v, want %v", uu.u, EndpointsUpdate{})
+	}
+	if uu.err == nil {
+		t.Errorf("unexpected endpointsError: <nil>, want error watcher timeout")
+	}
+}
diff --git a/xds/internal/client/client_watchers_lds.go b/xds/internal/client/client_watchers_lds.go
new file mode 100644
index 000000000000..9728cf8023d3
--- /dev/null
+++ b/xds/internal/client/client_watchers_lds.go
@@ -0,0 +1,47 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"time"
+)
+
+type ldsUpdate struct {
+	routeName string
+}
+type ldsCallbackFunc func(ldsUpdate, error)
+
+// watchLDS starts a listener watcher for the service..
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) watchLDS(serviceName string, cb ldsCallbackFunc) (cancel func()) {
+	wi := &watchInfo{
+		typeURL:     ldsURL,
+		target:      serviceName,
+		ldsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		c.scheduleCallback(wi, ldsUpdate{}, fmt.Errorf("xds: LDS target %s not found, watcher timeout", serviceName))
+	})
+	return c.watch(wi)
+}
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
new file mode 100644
index 000000000000..114726c029a5
--- /dev/null
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -0,0 +1,230 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type ldsUpdateErr struct {
+	u   ldsUpdate
+	err error
+}
+
+// TestLDSWatch covers the cases:
+// - an update is received after a watch()
+// - an update for another resource name (which doesn't trigger callback)
+// - an upate is received after cancel()
+func (s) TestLDSWatch(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	ldsUpdateCh := testutils.NewChannel()
+	cancelWatch := c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ldsUpdate{routeName: testRDSName}
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: wantUpdate,
+	})
+
+	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another update for a different resource name.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		"randomName": {},
+	})
+
+	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+
+	// Cancel watch, and send update again.
+	cancelWatch()
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: wantUpdate,
+	})
+
+	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestLDSTwoWatchSameResourceName covers the case where an update is received
+// after two watch() for the same resource name.
+func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var ldsUpdateChs []*testutils.Channel
+	const count = 2
+
+	var cancelLastWatch func()
+
+	for i := 0; i < count; i++ {
+		ldsUpdateCh := testutils.NewChannel()
+		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
+		cancelLastWatch = c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
+		})
+	}
+
+	wantUpdate := ldsUpdate{routeName: testRDSName}
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: wantUpdate,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+			t.Errorf("i=%v, unexpected ldsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	// Cancel the last watch, and send update again.
+	cancelLastWatch()
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: wantUpdate,
+	})
+
+	for i := 0; i < count-1; i++ {
+		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+			t.Errorf("i=%v, unexpected ldsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := ldsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestLDSThreeWatchDifferentResourceName covers the case where an update is
+// received after three watch() for different resource names.
+func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var ldsUpdateChs []*testutils.Channel
+	const count = 2
+
+	// Two watches for the same name.
+	for i := 0; i < count; i++ {
+		ldsUpdateCh := testutils.NewChannel()
+		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
+		c.watchLDS(testLDSName+"1", func(update ldsUpdate, err error) {
+			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
+		})
+	}
+
+	// Third watch for a different name.
+	ldsUpdateCh2 := testutils.NewChannel()
+	c.watchLDS(testLDSName+"2", func(update ldsUpdate, err error) {
+		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate1 := ldsUpdate{routeName: testRDSName + "1"}
+	wantUpdate2 := ldsUpdate{routeName: testRDSName + "2"}
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName + "1": wantUpdate1,
+		testLDSName + "2": wantUpdate2,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
+			t.Errorf("i=%v, unexpected ldsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
+
+// TestLDSWatchAfterCache covers the case where watch is called after the update
+// is in cache.
+func (s) TestLDSWatchAfterCache(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	ldsUpdateCh := testutils.NewChannel()
+	c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ldsUpdate{routeName: testRDSName}
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: wantUpdate,
+	})
+
+	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another watch for the resource in cache.
+	ldsUpdateCh2 := testutils.NewChannel()
+	c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
+	})
+
+	// New watch should receives the update.
+	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Old watch should see nothing.
+	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
new file mode 100644
index 000000000000..e7f1cc321f36
--- /dev/null
+++ b/xds/internal/client/client_watchers_rds.go
@@ -0,0 +1,47 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"time"
+)
+
+type rdsUpdate struct {
+	clusterName string
+}
+type rdsCallbackFunc func(rdsUpdate, error)
+
+// watchRDS starts a listener watcher for the service..
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) watchRDS(routeName string, cb rdsCallbackFunc) (cancel func()) {
+	wi := &watchInfo{
+		typeURL:     rdsURL,
+		target:      routeName,
+		rdsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		c.scheduleCallback(wi, rdsUpdate{}, fmt.Errorf("xds: RDS target %s not found, watcher timeout", routeName))
+	})
+	return c.watch(wi)
+}
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
new file mode 100644
index 000000000000..0b2f91455cbe
--- /dev/null
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -0,0 +1,230 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type rdsUpdateErr struct {
+	u   rdsUpdate
+	err error
+}
+
+// TestRDSWatch covers the cases:
+// - an update is received after a watch()
+// - an update for another resource name (which doesn't trigger callback)
+// - an upate is received after cancel()
+func (s) TestRDSWatch(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	rdsUpdateCh := testutils.NewChannel()
+	cancelWatch := c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := rdsUpdate{clusterName: testCDSName}
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: wantUpdate,
+	})
+
+	if u, err := rdsUpdateCh.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another update for a different resource name.
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		"randomName": {},
+	})
+
+	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+
+	// Cancel watch, and send update again.
+	cancelWatch()
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: wantUpdate,
+	})
+
+	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestRDSTwoWatchSameResourceName covers the case where an update is received
+// after two watch() for the same resource name.
+func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var rdsUpdateChs []*testutils.Channel
+	const count = 2
+
+	var cancelLastWatch func()
+
+	for i := 0; i < count; i++ {
+		rdsUpdateCh := testutils.NewChannel()
+		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
+		cancelLastWatch = c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
+		})
+	}
+
+	wantUpdate := rdsUpdate{clusterName: testCDSName}
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: wantUpdate,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	// Cancel the last watch, and send update again.
+	cancelLastWatch()
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: wantUpdate,
+	})
+
+	for i := 0; i < count-1; i++ {
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := rdsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestRDSThreeWatchDifferentResourceName covers the case where an update is
+// received after three watch() for different resource names.
+func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	var rdsUpdateChs []*testutils.Channel
+	const count = 2
+
+	// Two watches for the same name.
+	for i := 0; i < count; i++ {
+		rdsUpdateCh := testutils.NewChannel()
+		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
+		c.watchRDS(testRDSName+"1", func(update rdsUpdate, err error) {
+			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
+		})
+	}
+
+	// Third watch for a different name.
+	rdsUpdateCh2 := testutils.NewChannel()
+	c.watchRDS(testRDSName+"2", func(update rdsUpdate, err error) {
+		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate1 := rdsUpdate{clusterName: testCDSName + "1"}
+	wantUpdate2 := rdsUpdate{clusterName: testCDSName + "2"}
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName + "1": wantUpdate1,
+		testRDSName + "2": wantUpdate2,
+	})
+
+	for i := 0; i < count; i++ {
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || u != (rdsUpdateErr{wantUpdate1, nil}) {
+			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
+		}
+	}
+
+	if u, err := rdsUpdateCh2.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
+
+// TestRDSWatchAfterCache covers the case where watch is called after the update
+// is in cache.
+func (s) TestRDSWatchAfterCache(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	rdsUpdateCh := testutils.NewChannel()
+	c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := rdsUpdate{clusterName: testCDSName}
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: wantUpdate,
+	})
+
+	if u, err := rdsUpdateCh.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another watch for the resource in cache.
+	rdsUpdateCh2 := testutils.NewChannel()
+	c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
+	})
+
+	// New watch should receives the update.
+	if u, err := rdsUpdateCh2.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Old watch should see nothing.
+	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
new file mode 100644
index 000000000000..529de4d41677
--- /dev/null
+++ b/xds/internal/client/client_watchers_service.go
@@ -0,0 +1,114 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"sync"
+)
+
+// ServiceUpdate contains update about the service.
+type ServiceUpdate struct {
+	Cluster string
+}
+
+// WatchService uses LDS and RDS to discover information about the provided
+// serviceName.
+//
+// WatchService can only be called once. The second call will not start a
+// watcher and the callback will get an error. It's this case because an xDS
+// client is expected to be used only by one ClientConn.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error)) (cancel func()) {
+	c.mu.Lock()
+	if len(c.ldsWatchers) != 0 {
+		go cb(ServiceUpdate{}, fmt.Errorf("unexpected WatchService when there's another service being watched"))
+		c.mu.Unlock()
+		return func() {}
+	}
+	c.mu.Unlock()
+
+	w := &serviceUpdateWatcher{c: c, serviceCb: cb}
+	w.ldsCancel = c.watchLDS(serviceName, w.handleLDSResp)
+
+	return w.close
+}
+
+// serviceUpdateWatcher handles LDS and RDS response, and calls the service
+// callback at the right time.
+type serviceUpdateWatcher struct {
+	c         *Client
+	ldsCancel func()
+	serviceCb func(ServiceUpdate, error)
+
+	mu        sync.Mutex
+	closed    bool
+	rdsCancel func()
+}
+
+func (w *serviceUpdateWatcher) handleLDSResp(update ldsUpdate, err error) {
+	w.c.logger.Infof("xds: client received LDS update: %+v, err: %v", update, err)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return
+	}
+	// TODO: this error case returns early, without canceling the existing RDS
+	// watch. If we decided to stop the RDS watch when LDS errors, move this
+	// after rdsCancel(). We may also need to check the error type and do
+	// different things based on that (e.g. cancel RDS watch only on
+	// resourceRemovedError, but not on connectionError).
+	if err != nil {
+		w.serviceCb(ServiceUpdate{}, err)
+		return
+	}
+
+	if w.rdsCancel != nil {
+		w.rdsCancel()
+	}
+	w.rdsCancel = w.c.watchRDS(update.routeName, w.handleRDSResp)
+}
+
+func (w *serviceUpdateWatcher) handleRDSResp(update rdsUpdate, err error) {
+	w.c.logger.Infof("xds: client received RDS update: %+v, err: %v", update, err)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return
+	}
+	if err != nil {
+		w.serviceCb(ServiceUpdate{}, err)
+		return
+	}
+	w.serviceCb(ServiceUpdate{Cluster: update.clusterName}, nil)
+}
+
+func (w *serviceUpdateWatcher) close() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.closed = true
+	w.ldsCancel()
+	if w.rdsCancel != nil {
+		w.rdsCancel()
+		w.rdsCancel = nil
+	}
+}
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
new file mode 100644
index 000000000000..004d97c9a164
--- /dev/null
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -0,0 +1,339 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+)
+
+type serviceUpdateErr struct {
+	u   ServiceUpdate
+	err error
+}
+
+// TestServiceWatch covers the cases:
+// - an update is received after a watch()
+// - an update for another resource name (which doesn't trigger callback)
+// - an upate is received after cancel()
+func (s) TestServiceWatch(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+
+	<-v2Client.addWatches[ldsURL]
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	<-v2Client.addWatches[rdsURL]
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {clusterName: testCDSName},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
+
+// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS
+// response, the second LDS response trigger an new RDS watch, and an update of
+// the old RDS watch doesn't trigger update to service callback.
+func (s) TestServiceWatchLDSUpdate(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+
+	<-v2Client.addWatches[ldsURL]
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	<-v2Client.addWatches[rdsURL]
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {clusterName: testCDSName},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another LDS update with a different RDS_name.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName + "2"},
+	})
+	<-v2Client.addWatches[rdsURL]
+
+	// Another update for the old name.
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {clusterName: testCDSName},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+
+	wantUpdate2 := ServiceUpdate{Cluster: testCDSName + "2"}
+	// RDS update for the new name.
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName + "2": {clusterName: testCDSName + "2"},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
+
+// TestServiceWatchSecond covers the case where a second WatchService() gets an
+// error (because only one is allowed). But the first watch still receives
+// updates.
+func (s) TestServiceWatchSecond(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+
+	<-v2Client.addWatches[ldsURL]
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	<-v2Client.addWatches[rdsURL]
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {clusterName: testCDSName},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	serviceUpdateCh2 := testutils.NewChannel()
+	// Call WatchService() again, with the same or different name.
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh2.Send(serviceUpdateErr{u: update, err: err})
+	})
+
+	u, err := serviceUpdateCh2.Receive()
+	if err != nil {
+		t.Fatalf("failed to get serviceUpdate: %v", err)
+	}
+	uu := u.(serviceUpdateErr)
+	if uu.u != (ServiceUpdate{}) {
+		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
+	}
+	if uu.err == nil {
+		t.Errorf("unexpected serviceError: <nil>, want error watcher timeout")
+	}
+
+	// Send update again, first callback should be called, second should
+	// timeout.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {clusterName: testCDSName},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	if u, err := serviceUpdateCh2.Receive(); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
+	}
+}
+
+// TestServiceWatchWithNoResponseFromServer tests the case where the xDS server
+// does not respond to the requests being sent out as part of registering a
+// service update watcher. The callback will get an error.
+func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
+	fakeServer, cleanup, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("Failed to start fake xDS server: %v", err)
+	}
+	defer cleanup()
+
+	xdsClient, err := New(clientOpts(fakeServer.Address))
+	if err != nil {
+		t.Fatalf("New returned error: %v", err)
+	}
+	defer xdsClient.Close()
+	t.Log("Created an xdsClient...")
+
+	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
+	defaultWatchExpiryTimeout = 500 * time.Millisecond
+	defer func() {
+		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
+	}()
+
+	callbackCh := testutils.NewChannel()
+	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
+		if su.Cluster != "" {
+			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want empty clusterName", su.Cluster))
+			return
+		}
+		if err == nil {
+			callbackCh.Send(errors.New("xdsClient.WatchService returned error non-nil error"))
+			return
+		}
+		callbackCh.Send(nil)
+	})
+	defer cancelWatch()
+	t.Log("Registered a watcher for service updates...")
+
+	// Wait for one request from the client, but send no reponses.
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout expired when expecting an LDS request")
+	}
+	waitForNilErr(t, callbackCh)
+}
+
+// TestServiceWatchEmptyRDS tests the case where the underlying v2Client
+// receives an empty RDS response. The callback will get an error.
+func (s) TestServiceWatchEmptyRDS(t *testing.T) {
+	fakeServer, cleanup, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("Failed to start fake xDS server: %v", err)
+	}
+	defer cleanup()
+
+	xdsClient, err := New(clientOpts(fakeServer.Address))
+	if err != nil {
+		t.Fatalf("New returned error: %v", err)
+	}
+	defer xdsClient.Close()
+	t.Log("Created an xdsClient...")
+
+	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
+	defaultWatchExpiryTimeout = 500 * time.Millisecond
+	defer func() {
+		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
+	}()
+
+	callbackCh := testutils.NewChannel()
+	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
+		if su.Cluster != "" {
+			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want empty clusterName", su.Cluster))
+			return
+		}
+		if err == nil {
+			callbackCh.Send(errors.New("xdsClient.WatchService returned error non-nil error"))
+			return
+		}
+		callbackCh.Send(nil)
+	})
+	defer cancelWatch()
+	t.Log("Registered a watcher for service updates...")
+
+	// Make the fakeServer send LDS response.
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout expired when expecting an LDS request")
+	}
+	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
+
+	// Make the fakeServer send an empty RDS response.
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout expired when expecting an RDS request")
+	}
+	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: noVirtualHostsInRDSResponse}
+	waitForNilErr(t, callbackCh)
+}
+
+// TestServiceWatchWithClientClose tests the case where xDS responses are
+// received after the client is closed, and we make sure that the registered
+// watcher callback is not invoked.
+func (s) TestServiceWatchWithClientClose(t *testing.T) {
+	fakeServer, cleanup, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("Failed to start fake xDS server: %v", err)
+	}
+	defer cleanup()
+
+	xdsClient, err := New(clientOpts(fakeServer.Address))
+	if err != nil {
+		t.Fatalf("New returned error: %v", err)
+	}
+	defer xdsClient.Close()
+	t.Log("Created an xdsClient...")
+
+	callbackCh := testutils.NewChannel()
+	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
+		callbackCh.Send(errors.New("watcher callback invoked after client close"))
+	})
+	defer cancelWatch()
+	t.Log("Registered a watcher for service updates...")
+
+	// Make the fakeServer send LDS response.
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout expired when expecting an LDS request")
+	}
+	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
+
+	xdsClient.Close()
+	t.Log("Closing the xdsClient...")
+
+	// Push an RDS response from the fakeserver
+	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodRDSResponse1}
+	if cbErr, err := callbackCh.Receive(); err != testutils.ErrRecvTimeout {
+		t.Fatal(cbErr)
+	}
+}
diff --git a/xds/internal/client/testutil_test.go b/xds/internal/client/testutil_test.go
index 436c4be65560..a63fbdb38a91 100644
--- a/xds/internal/client/testutil_test.go
+++ b/xds/internal/client/testutil_test.go
@@ -20,6 +20,7 @@ package client
 import (
 	"reflect"
 	"testing"
+	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	"github.com/google/go-cmp/cmp"
@@ -29,19 +30,53 @@ import (
 )
 
 type watchHandleTestcase struct {
+	typeURL      string
+	resourceName string
+
 	responseToHandle *xdspb.DiscoveryResponse
 	wantHandleErr    bool
 	wantUpdate       interface{}
 	wantUpdateErr    bool
+}
+
+type testUpdateReceiver struct {
+	f func(typeURL string, d map[string]interface{})
+}
+
+func (t *testUpdateReceiver) newLDSUpdate(d map[string]ldsUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(ldsURL, dd)
+}
+
+func (t *testUpdateReceiver) newRDSUpdate(d map[string]rdsUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(rdsURL, dd)
+}
 
-	// Only one of the following should be non-nil. The one corresponding with
-	// typeURL will be called.
-	ldsWatch      func(target string, ldsCb ldsCallbackFunc) (cancel func())
-	rdsWatch      func(routeName string, rdsCb rdsCallbackFunc) (cancel func())
-	cdsWatch      func(clusterName string, cdsCb cdsCallbackFunc) (cancel func())
-	edsWatch      func(clusterName string, edsCb edsCallbackFunc) (cancel func())
-	watchReqChan  *testutils.Channel // The request sent for watch will be sent to this channel.
-	handleXDSResp func(response *xdspb.DiscoveryResponse) error
+func (t *testUpdateReceiver) newCDSUpdate(d map[string]ClusterUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(cdsURL, dd)
+}
+
+func (t *testUpdateReceiver) newEDSUpdate(d map[string]EndpointsUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(edsURL, dd)
+}
+
+func (t *testUpdateReceiver) newUpdate(typeURL string, d map[string]interface{}) {
+	t.f(typeURL, d)
 }
 
 // testWatchHandle is called to test response handling for each xDS.
@@ -51,44 +86,38 @@ type watchHandleTestcase struct {
 // handleXDSResp with responseToHandle (if it's set). It then compares the
 // update received by watch callback with the expected results.
 func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
+	fakeServer, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+
 	type updateErr struct {
 		u   interface{}
 		err error
 	}
 	gotUpdateCh := testutils.NewChannel()
 
-	var cancelWatch func()
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(typeURL string, d map[string]interface{}) {
+			if typeURL == test.typeURL {
+				if u, ok := d[test.resourceName]; ok {
+					gotUpdateCh.Send(updateErr{u, nil})
+				}
+			}
+		},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	defer v2c.close()
+
+	// RDS needs an existin LDS watch for the hostname.
+	if test.typeURL == rdsURL {
+		doLDS(t, v2c, fakeServer)
+	}
+
 	// Register the watcher, this will also trigger the v2Client to send the xDS
 	// request.
-	switch {
-	case test.ldsWatch != nil:
-		cancelWatch = test.ldsWatch(goodLDSTarget1, func(u ldsUpdate, err error) {
-			t.Logf("in v2c.watchLDS callback, ldsUpdate: %+v, err: %v", u, err)
-			gotUpdateCh.Send(updateErr{u, err})
-		})
-	case test.rdsWatch != nil:
-		cancelWatch = test.rdsWatch(goodRouteName1, func(u rdsUpdate, err error) {
-			t.Logf("in v2c.watchRDS callback, rdsUpdate: %+v, err: %v", u, err)
-			gotUpdateCh.Send(updateErr{u, err})
-		})
-	case test.cdsWatch != nil:
-		cancelWatch = test.cdsWatch(clusterName1, func(u CDSUpdate, err error) {
-			t.Logf("in v2c.watchCDS callback, cdsUpdate: %+v, err: %v", u, err)
-			gotUpdateCh.Send(updateErr{u, err})
-		})
-	case test.edsWatch != nil:
-		cancelWatch = test.edsWatch(goodEDSName, func(u *EDSUpdate, err error) {
-			t.Logf("in v2c.watchEDS callback, edsUpdate: %+v, err: %v", u, err)
-			gotUpdateCh.Send(updateErr{*u, err})
-		})
-	default:
-		t.Fatalf("no watch() is set")
-	}
-	defer cancelWatch()
+	v2c.addWatch(test.typeURL, test.resourceName)
 
 	// Wait till the request makes it to the fakeServer. This ensures that
 	// the watch request has been processed by the v2Client.
-	if _, err := test.watchReqChan.Receive(); err != nil {
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout waiting for an xDS request: %v", err)
 	}
 
@@ -98,7 +127,18 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	//
 	// Also note that this won't trigger ACK, so there's no need to clear the
 	// request channel afterwards.
-	if err := test.handleXDSResp(test.responseToHandle); (err != nil) != test.wantHandleErr {
+	var handleXDSResp func(response *xdspb.DiscoveryResponse) error
+	switch test.typeURL {
+	case ldsURL:
+		handleXDSResp = v2c.handleLDSResponse
+	case rdsURL:
+		handleXDSResp = v2c.handleRDSResponse
+	case cdsURL:
+		handleXDSResp = v2c.handleCDSResponse
+	case edsURL:
+		handleXDSResp = v2c.handleEDSResponse
+	}
+	if err := handleXDSResp(test.responseToHandle); (err != nil) != test.wantHandleErr {
 		t.Fatalf("v2c.handleRDSResponse() returned err: %v, wantErr: %v", err, test.wantHandleErr)
 	}
 
@@ -121,7 +161,7 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 		t.Fatal("Timeout expecting xDS update")
 	}
 	gotUpdate := uErr.(updateErr).u
-	opt := cmp.AllowUnexported(rdsUpdate{}, ldsUpdate{}, CDSUpdate{}, EDSUpdate{})
+	opt := cmp.AllowUnexported(rdsUpdate{}, ldsUpdate{}, ClusterUpdate{}, EndpointsUpdate{})
 	if diff := cmp.Diff(gotUpdate, wantUpdate, opt); diff != "" {
 		t.Fatalf("got update : %+v, want %+v, diff: %s", gotUpdate, wantUpdate, diff)
 	}
diff --git a/xds/internal/client/types.go b/xds/internal/client/types.go
deleted file mode 100644
index 84a446d024f3..000000000000
--- a/xds/internal/client/types.go
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"time"
-
-	adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
-)
-
-type adsStream adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient
-
-const (
-	ldsURL = "type.googleapis.com/envoy.api.v2.Listener"
-	rdsURL = "type.googleapis.com/envoy.api.v2.RouteConfiguration"
-	cdsURL = "type.googleapis.com/envoy.api.v2.Cluster"
-	edsURL = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
-)
-
-// watchState is an enum to represent the state of a watch call.
-type watchState int
-
-const (
-	watchEnqueued watchState = iota
-	watchCancelled
-	watchStarted
-)
-
-// watchInfo holds all the information about a watch call.
-type watchInfo struct {
-	typeURL string
-	target  []string
-	state   watchState
-
-	ldsCallback ldsCallbackFunc
-	rdsCallback rdsCallbackFunc
-	cdsCallback cdsCallbackFunc
-	edsCallback edsCallbackFunc
-	expiryTimer *time.Timer
-}
-
-// cancel marks the state as cancelled, and also stops the expiry timer.
-func (wi *watchInfo) cancel() {
-	wi.state = watchCancelled
-	if wi.expiryTimer != nil {
-		wi.expiryTimer.Stop()
-	}
-}
-
-// stopTimer stops the expiry timer without cancelling the watch.
-func (wi *watchInfo) stopTimer() {
-	if wi.expiryTimer != nil {
-		wi.expiryTimer.Stop()
-	}
-}
-
-type ackInfo struct {
-	typeURL string
-	version string // NACK if version is an empty string.
-	nonce   string
-	// ACK/NACK are tagged with the stream it's for. When the stream is down,
-	// all the ACK/NACK for this stream will be dropped, and the version/nonce
-	// won't be updated.
-	stream adsStream
-}
-
-type ldsUpdate struct {
-	routeName string
-}
-type ldsCallbackFunc func(ldsUpdate, error)
-
-type rdsUpdate struct {
-	clusterName string
-}
-type rdsCallbackFunc func(rdsUpdate, error)
-
-// CDSUpdate contains information from a received CDS response, which is of
-// interest to the registered CDS watcher.
-type CDSUpdate struct {
-	// ServiceName is the service name corresponding to the clusterName which
-	// is being watched for through CDS.
-	ServiceName string
-	// EnableLRS indicates whether or not load should be reported through LRS.
-	EnableLRS bool
-}
-type cdsCallbackFunc func(CDSUpdate, error)
-
-type edsCallbackFunc func(*EDSUpdate, error)
diff --git a/xds/internal/client/v2client.go b/xds/internal/client/v2client.go
index 53ac82abee7e..28536f26f286 100644
--- a/xds/internal/client/v2client.go
+++ b/xds/internal/client/v2client.go
@@ -20,7 +20,6 @@ package client
 
 import (
 	"context"
-	"fmt"
 	"sync"
 	"time"
 
@@ -33,19 +32,37 @@ import (
 	adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
 )
 
-// The value chosen here is based on the default value of the
-// initial_fetch_timeout field in corepb.ConfigSource proto.
-var defaultWatchExpiryTimeout = 15 * time.Second
+type adsStream adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient
+
+var _ xdsv2Client = &v2Client{}
+
+// updateHandler handles the update (parsed from xds responses). It's
+// implemented by the upper level Client.
+//
+// It's an interface to be overridden in test.
+type updateHandler interface {
+	newLDSUpdate(d map[string]ldsUpdate)
+	newRDSUpdate(d map[string]rdsUpdate)
+	newCDSUpdate(d map[string]ClusterUpdate)
+	newEDSUpdate(d map[string]EndpointsUpdate)
+}
 
 // v2Client performs the actual xDS RPCs using the xDS v2 API. It creates a
 // single ADS stream on which the different types of xDS requests and responses
 // are multiplexed.
+//
+// This client's main purpose is to make the RPC, build/parse proto messages,
+// and do ACK/NACK. It's a naive implementation that sends whatever the upper
+// layer tells it to send. It will call the callback with everything in every
+// response. It doesn't keep a cache of responses, or check for duplicates.
+//
 // The reason for splitting this out from the top level xdsClient object is
 // because there is already an xDS v3Aplha API in development. If and when we
 // want to switch to that, this separation will ease that process.
 type v2Client struct {
 	ctx       context.Context
 	cancelCtx context.CancelFunc
+	parent    updateHandler
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
 	cc        *grpc.ClientConn
@@ -54,8 +71,11 @@ type v2Client struct {
 
 	logger *grpclog.PrefixLogger
 
+	// streamCh is the channel where new ADS streams are pushed to (when the old
+	// stream is broken). It's monitored by the sending goroutine, so requests
+	// are sent to the most up-to-date stream.
 	streamCh chan adsStream
-	// sendCh in the channel onto which watchInfo objects are pushed by the
+	// sendCh is the channel onto which watchAction objects are pushed by the
 	// watch API, and it is read and acted upon by the send() goroutine.
 	sendCh *buffer.Unbounded
 
@@ -66,7 +86,7 @@ type v2Client struct {
 	// messages. When the user of this client object cancels a watch call,
 	// these are set to nil. All accesses to the map protected and any value
 	// inside the map should be protected with the above mutex.
-	watchMap map[string]*watchInfo
+	watchMap map[string]map[string]bool
 	// versionMap contains the version that was acked (the version in the ack
 	// request that was sent on wire). The key is typeURL, the value is the
 	// version string, becaues the versions for different resource types should
@@ -74,33 +94,19 @@ type v2Client struct {
 	versionMap map[string]string
 	// nonceMap contains the nonce from the most recent received response.
 	nonceMap map[string]string
-	// rdsCache maintains a mapping of {routeConfigName --> clusterName} from
-	// validated route configurations received in RDS responses. We cache all
-	// valid route configurations, whether or not we are interested in them
-	// when we received them (because we could become interested in them in the
-	// future and the server wont send us those resources again).
-	// Protected by the above mutex.
+	// hostname is the LDS resource_name to watch. It is set to the first LDS
+	// resource_name to watch, and removed when the LDS watch is canceled.
 	//
-	// TODO: remove RDS cache. The updated spec says client can ignore
-	// unrequested resources.
-	// https://github.com/envoyproxy/envoy/blob/master/api/xds_protocol.rst#resource-hints
-	rdsCache map[string]string
-	// rdsCache maintains a mapping of {clusterName --> CDSUpdate} from
-	// validated cluster configurations received in CDS responses. We cache all
-	// valid cluster configurations, whether or not we are interested in them
-	// when we received them (because we could become interested in them in the
-	// future and the server wont send us those resources again). This is only
-	// to support legacy management servers that do not honor the
-	// resource_names field. As per the latest spec, the server should resend
-	// the response when the request changes, even if it had sent the same
-	// resource earlier (when not asked for). Protected by the above mutex.
-	cdsCache map[string]CDSUpdate
+	// It's from the dial target of the parent ClientConn. RDS resource
+	// processing needs this to do the host matching.
+	hostname string
 }
 
 // newV2Client creates a new v2Client initialized with the passed arguments.
-func newV2Client(cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) *v2Client {
+func newV2Client(parent updateHandler, cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) *v2Client {
 	v2c := &v2Client{
 		cc:        cc,
+		parent:    parent,
 		nodeProto: nodeProto,
 		backoff:   backoff,
 
@@ -109,11 +115,9 @@ func newV2Client(cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int)
 		streamCh: make(chan adsStream, 1),
 		sendCh:   buffer.NewUnbounded(),
 
-		watchMap:   make(map[string]*watchInfo),
+		watchMap:   make(map[string]map[string]bool),
 		versionMap: make(map[string]string),
 		nonceMap:   make(map[string]string),
-		rdsCache:   make(map[string]string),
-		cdsCache:   make(map[string]CDSUpdate),
 	}
 	v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background())
 
@@ -214,8 +218,8 @@ func (v2c *v2Client) sendExisting(stream adsStream) bool {
 	v2c.versionMap = make(map[string]string)
 	v2c.nonceMap = make(map[string]string)
 
-	for typeURL, wi := range v2c.watchMap {
-		if !v2c.sendRequest(stream, wi.target, typeURL, "", "") {
+	for typeURL, s := range v2c.watchMap {
+		if !v2c.sendRequest(stream, mapToSlice(s), typeURL, "", "") {
 			return false
 		}
 	}
@@ -223,45 +227,79 @@ func (v2c *v2Client) sendExisting(stream adsStream) bool {
 	return true
 }
 
-// processWatchInfo pulls the fields needed by the request from a watchInfo.
-//
-// It also calls callback with cached response, and updates the watch map in
-// v2c.
+type watchAction struct {
+	typeURL  string
+	remove   bool // Whether this is to remove watch for the resource.
+	resource string
+}
+
+// processWatchInfo pulls the fields needed by the request from a watchAction.
 //
-// If the watch was already canceled, it returns false for send
-func (v2c *v2Client) processWatchInfo(t *watchInfo) (target []string, typeURL, version, nonce string, send bool) {
+// It also updates the watch map in v2c.
+func (v2c *v2Client) processWatchInfo(t *watchAction) (target []string, typeURL, version, nonce string, send bool) {
 	v2c.mu.Lock()
 	defer v2c.mu.Unlock()
-	if t.state == watchCancelled {
-		return // This returns all zero values, and false for send.
+
+	var current map[string]bool
+	current, ok := v2c.watchMap[t.typeURL]
+	if !ok {
+		current = make(map[string]bool)
+		v2c.watchMap[t.typeURL] = current
 	}
-	t.state = watchStarted
-	send = true
 
-	typeURL = t.typeURL
-	target = t.target
-	v2c.checkCacheAndUpdateWatchMap(t)
-	// TODO: if watch is called again with the same resource names,
-	// there's no need to send another request.
+	if t.remove {
+		delete(current, t.resource)
+		if len(current) == 0 {
+			delete(v2c.watchMap, t.typeURL)
+		}
+	} else {
+		current[t.resource] = true
+	}
 
+	// Special handling for LDS, because RDS needs the LDS resource_name for
+	// response host matching.
+	if t.typeURL == ldsURL {
+		// Set hostname to the first LDS resource_name, and reset it when the
+		// last LDS watch is removed. The upper level Client isn't expected to
+		// watchLDS more than once.
+		if l := len(current); l == 1 {
+			v2c.hostname = t.resource
+		} else if l == 0 {
+			v2c.hostname = ""
+		}
+	}
+
+	send = true
+	typeURL = t.typeURL
+	target = mapToSlice(current)
 	// We don't reset version or nonce when a new watch is started. The version
 	// and nonce from previous response are carried by the request unless the
 	// stream is recreated.
 	version = v2c.versionMap[typeURL]
 	nonce = v2c.nonceMap[typeURL]
-	return
+	return target, typeURL, version, nonce, send
+}
+
+type ackAction struct {
+	typeURL string
+	version string // NACK if version is an empty string.
+	nonce   string
+	// ACK/NACK are tagged with the stream it's for. When the stream is down,
+	// all the ACK/NACK for this stream will be dropped, and the version/nonce
+	// won't be updated.
+	stream adsStream
 }
 
-// processAckInfo pulls the fields needed by the ack request from a ackInfo.
+// processAckInfo pulls the fields needed by the ack request from a ackAction.
 //
 // If no active watch is found for this ack, it returns false for send.
-func (v2c *v2Client) processAckInfo(t *ackInfo, stream adsStream) (target []string, typeURL, version, nonce string, send bool) {
+func (v2c *v2Client) processAckInfo(t *ackAction, stream adsStream) (target []string, typeURL, version, nonce string, send bool) {
 	if t.stream != stream {
 		// If ACK's stream isn't the current sending stream, this means the ACK
 		// was pushed to queue before the old stream broke, and a new stream has
 		// been started since. Return immediately here so we don't update the
 		// nonce for the new stream.
-		return
+		return nil, "", "", "", false
 	}
 	typeURL = t.typeURL
 
@@ -274,16 +312,18 @@ func (v2c *v2Client) processAckInfo(t *ackInfo, stream adsStream) (target []stri
 	nonce = t.nonce
 	v2c.nonceMap[typeURL] = nonce
 
-	wi, ok := v2c.watchMap[typeURL]
-	if !ok {
+	s, ok := v2c.watchMap[typeURL]
+	if !ok || len(s) == 0 {
 		// We don't send the request ack if there's no active watch (this can be
 		// either the server sends responses before any request, or the watch is
-		// canceled while the ackInfo is in queue), because there's no resource
+		// canceled while the ackAction is in queue), because there's no resource
 		// name. And if we send a request with empty resource name list, the
 		// server may treat it as a wild card and send us everything.
 		return nil, "", "", "", false
 	}
 	send = true
+	target = mapToSlice(s)
+
 	version = t.version
 	if version == "" {
 		// This is a nack, get the previous acked version.
@@ -294,7 +334,6 @@ func (v2c *v2Client) processAckInfo(t *ackInfo, stream adsStream) (target []stri
 	} else {
 		v2c.versionMap[typeURL] = version
 	}
-	target = wi.target
 	return target, typeURL, version, nonce, send
 }
 
@@ -303,7 +342,7 @@ func (v2c *v2Client) processAckInfo(t *ackInfo, stream adsStream) (target []stri
 // It watches the stream channel for new streams, and the request channel for
 // new requests to send on the stream.
 //
-// For each new request (watchInfo), it's
+// For each new request (watchAction), it's
 //  - processed and added to the watch map
 //    - so resend will pick them up when there are new streams)
 //  - sent on the current stream if there's one
@@ -320,8 +359,7 @@ func (v2c *v2Client) send() {
 		select {
 		case <-v2c.ctx.Done():
 			return
-		case newStream := <-v2c.streamCh:
-			stream = newStream
+		case stream = <-v2c.streamCh:
 			if !v2c.sendExisting(stream) {
 				// send failed, clear the current stream.
 				stream = nil
@@ -335,9 +373,9 @@ func (v2c *v2Client) send() {
 				send                    bool
 			)
 			switch t := u.(type) {
-			case *watchInfo:
+			case *watchAction:
 				target, typeURL, version, nonce, send = v2c.processWatchInfo(t)
-			case *ackInfo:
+			case *ackAction:
 				target, typeURL, version, nonce, send = v2c.processAckInfo(t, stream)
 			}
 			if !send {
@@ -388,7 +426,7 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 
 		typeURL := resp.GetTypeUrl()
 		if respHandleErr != nil {
-			v2c.sendCh.Put(&ackInfo{
+			v2c.sendCh.Put(&ackAction{
 				typeURL: typeURL,
 				version: "",
 				nonce:   resp.GetNonce(),
@@ -397,7 +435,7 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 			v2c.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", typeURL, resp.GetVersionInfo(), resp.GetNonce(), respHandleErr)
 			continue
 		}
-		v2c.sendCh.Put(&ackInfo{
+		v2c.sendCh.Put(&ackAction{
 			typeURL: typeURL,
 			version: resp.GetVersionInfo(),
 			nonce:   resp.GetNonce(),
@@ -408,145 +446,25 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 	}
 }
 
-// watchLDS registers an LDS watcher for the provided target. Updates
-// corresponding to received LDS responses will be pushed to the provided
-// callback. The caller can cancel the watch by invoking the returned cancel
-// function.
-// The provided callback should not block or perform any expensive operations
-// or call other methods of the v2Client object.
-func (v2c *v2Client) watchLDS(target string, ldsCb ldsCallbackFunc) (cancel func()) {
-	return v2c.watch(&watchInfo{
-		typeURL:     ldsURL,
-		target:      []string{target},
-		ldsCallback: ldsCb,
-	})
-}
-
-// watchRDS registers an RDS watcher for the provided routeName. Updates
-// corresponding to received RDS responses will be pushed to the provided
-// callback. The caller can cancel the watch by invoking the returned cancel
-// function.
-// The provided callback should not block or perform any expensive operations
-// or call other methods of the v2Client object.
-func (v2c *v2Client) watchRDS(routeName string, rdsCb rdsCallbackFunc) (cancel func()) {
-	return v2c.watch(&watchInfo{
-		typeURL:     rdsURL,
-		target:      []string{routeName},
-		rdsCallback: rdsCb,
-	})
-	// TODO: Once a registered RDS watch is cancelled, we should send an RDS
-	// request with no resources. This will let the server know that we are no
-	// longer interested in this resource.
-}
-
-// watchCDS registers an CDS watcher for the provided clusterName. Updates
-// corresponding to received CDS responses will be pushed to the provided
-// callback. The caller can cancel the watch by invoking the returned cancel
-// function.
-// The provided callback should not block or perform any expensive operations
-// or call other methods of the v2Client object.
-func (v2c *v2Client) watchCDS(clusterName string, cdsCb cdsCallbackFunc) (cancel func()) {
-	return v2c.watch(&watchInfo{
-		typeURL:     cdsURL,
-		target:      []string{clusterName},
-		cdsCallback: cdsCb,
+func (v2c *v2Client) addWatch(resourceType, resourceName string) {
+	v2c.sendCh.Put(&watchAction{
+		typeURL:  resourceType,
+		remove:   false,
+		resource: resourceName,
 	})
 }
 
-// watchEDS registers an EDS watcher for the provided clusterName. Updates
-// corresponding to received EDS responses will be pushed to the provided
-// callback. The caller can cancel the watch by invoking the returned cancel
-// function.
-// The provided callback should not block or perform any expensive operations
-// or call other methods of the v2Client object.
-func (v2c *v2Client) watchEDS(clusterName string, edsCb edsCallbackFunc) (cancel func()) {
-	return v2c.watch(&watchInfo{
-		typeURL:     edsURL,
-		target:      []string{clusterName},
-		edsCallback: edsCb,
+func (v2c *v2Client) removeWatch(resourceType, resourceName string) {
+	v2c.sendCh.Put(&watchAction{
+		typeURL:  resourceType,
+		remove:   true,
+		resource: resourceName,
 	})
-	// TODO: Once a registered EDS watch is cancelled, we should send an EDS
-	// request with no resources. This will let the server know that we are no
-	// longer interested in this resource.
 }
 
-func (v2c *v2Client) watch(wi *watchInfo) (cancel func()) {
-	v2c.sendCh.Put(wi)
-	v2c.logger.Infof("Sending ADS request for new watch of type: %v, resource names: %v", wi.typeURL, wi.target)
-	return func() {
-		v2c.mu.Lock()
-		defer v2c.mu.Unlock()
-		if wi.state == watchEnqueued {
-			wi.state = watchCancelled
-			return
-		}
-		v2c.watchMap[wi.typeURL].cancel()
-		delete(v2c.watchMap, wi.typeURL)
-		// TODO: should we reset ack version string when cancelling the watch?
-	}
-}
-
-// checkCacheAndUpdateWatchMap is called when a new watch call is handled in
-// send(). If an existing watcher is found, its expiry timer is stopped. If the
-// watchInfo to be added to the watchMap is found in the cache, the watcher
-// callback is immediately invoked.
-//
-// Caller should hold v2c.mu
-func (v2c *v2Client) checkCacheAndUpdateWatchMap(wi *watchInfo) {
-	if existing := v2c.watchMap[wi.typeURL]; existing != nil {
-		existing.cancel()
-	}
-
-	v2c.watchMap[wi.typeURL] = wi
-	switch wi.typeURL {
-	// We need to grab the lock inside of the expiryTimer's afterFunc because
-	// we need to access the watchInfo, which is stored in the watchMap.
-	case ldsURL:
-		wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-			v2c.mu.Lock()
-			wi.ldsCallback(ldsUpdate{}, fmt.Errorf("xds: LDS target %s not found, watcher timeout", wi.target))
-			v2c.mu.Unlock()
-		})
-	case rdsURL:
-		routeName := wi.target[0]
-		if cluster := v2c.rdsCache[routeName]; cluster != "" {
-			var err error
-			if v2c.watchMap[ldsURL] == nil {
-				cluster = ""
-				err = fmt.Errorf("xds: no LDS watcher found when handling RDS watch for route {%v} from cache", routeName)
-			}
-			v2c.logger.Infof("Resource with name %v, type %v found in cache", routeName, wi.typeURL)
-			wi.rdsCallback(rdsUpdate{clusterName: cluster}, err)
-			return
-		}
-		// Add the watch expiry timer only for new watches we don't find in
-		// the cache, and return from here.
-		wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-			v2c.mu.Lock()
-			wi.rdsCallback(rdsUpdate{}, fmt.Errorf("xds: RDS target %s not found, watcher timeout", wi.target))
-			v2c.mu.Unlock()
-		})
-	case cdsURL:
-		clusterName := wi.target[0]
-		if update, ok := v2c.cdsCache[clusterName]; ok {
-			var err error
-			if v2c.watchMap[cdsURL] == nil {
-				err = fmt.Errorf("xds: no CDS watcher found when handling CDS watch for cluster {%v} from cache", clusterName)
-			}
-			v2c.logger.Infof("Resource with name %v, type %v found in cache", clusterName, wi.typeURL)
-			wi.cdsCallback(update, err)
-			return
-		}
-		wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-			v2c.mu.Lock()
-			wi.cdsCallback(CDSUpdate{}, fmt.Errorf("xds: CDS target %s not found, watcher timeout", wi.target))
-			v2c.mu.Unlock()
-		})
-	case edsURL:
-		wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-			v2c.mu.Lock()
-			wi.edsCallback(nil, fmt.Errorf("xds: EDS target %s not found, watcher timeout", wi.target))
-			v2c.mu.Unlock()
-		})
+func mapToSlice(m map[string]bool) (ret []string) {
+	for i := range m {
+		ret = append(ret, i)
 	}
+	return
 }
diff --git a/xds/internal/client/v2client_ack_test.go b/xds/internal/client/v2client_ack_test.go
index 1fce1f5e9564..c4f49d4b0303 100644
--- a/xds/internal/client/v2client_ack_test.go
+++ b/xds/internal/client/v2client_ack_test.go
@@ -27,10 +27,43 @@ import (
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
 
+func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *v2Client, cbLDS, cbRDS, cbCDS, cbEDS *testutils.Channel, cleanup func()) {
+	cbLDS = testutils.NewChannel()
+	cbRDS = testutils.NewChannel()
+	cbCDS = testutils.NewChannel()
+	cbEDS = testutils.NewChannel()
+	v2c = newV2Client(&testUpdateReceiver{
+		f: func(typeURL string, d map[string]interface{}) {
+			t.Logf("Received %s callback with {%+v}", typeURL, d)
+			switch typeURL {
+			case ldsURL:
+				if _, ok := d[goodLDSTarget1]; ok {
+					cbLDS.Send(struct{}{})
+				}
+			case rdsURL:
+				if _, ok := d[goodRouteName1]; ok {
+					cbRDS.Send(struct{}{})
+				}
+			case cdsURL:
+				if _, ok := d[goodClusterName1]; ok {
+					cbCDS.Send(struct{}{})
+				}
+			case edsURL:
+				if _, ok := d[goodEDSName]; ok {
+					cbEDS.Send(struct{}{})
+				}
+			}
+		},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	t.Log("Started xds v2Client...")
+	return v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2c.close
+}
+
 // compareXDSRequest reads requests from channel, compare it with want.
 func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, version, nonce string) error {
 	val, err := ch.Receive()
@@ -61,36 +94,30 @@ func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *
 
 // startXDS calls watch to send the first request. It then sends a good response
 // and checks for ack.
-func startXDS(t *testing.T, xdsname string, v2c *v2Client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) *testutils.Channel {
-	callbackCh := testutils.NewChannel()
+func startXDS(t *testing.T, xdsname string, v2c *v2Client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
+	var (
+		nameToWatch, typeURLToWatch string
+	)
 	switch xdsname {
 	case "LDS":
-		v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-			t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", xdsname, u, err)
-			callbackCh.Send(struct{}{})
-		})
+		typeURLToWatch = ldsURL
+		nameToWatch = goodLDSTarget1
 	case "RDS":
-		v2c.watchRDS(goodRouteName1, func(u rdsUpdate, err error) {
-			t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", xdsname, u, err)
-			callbackCh.Send(struct{}{})
-		})
+		typeURLToWatch = rdsURL
+		nameToWatch = goodRouteName1
 	case "CDS":
-		v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
-			t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", xdsname, u, err)
-			callbackCh.Send(struct{}{})
-		})
+		typeURLToWatch = cdsURL
+		nameToWatch = goodClusterName1
 	case "EDS":
-		v2c.watchEDS(goodEDSName, func(u *EDSUpdate, err error) {
-			t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", xdsname, u, err)
-			callbackCh.Send(struct{}{})
-		})
+		typeURLToWatch = edsURL
+		nameToWatch = goodEDSName
 	}
+	v2c.addWatch(typeURLToWatch, nameToWatch)
 
 	if err := compareXDSRequest(reqChan, req, preVersion, preNonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("FakeServer received %s request...", xdsname)
-	return callbackCh
 }
 
 // sendGoodResp sends the good response, with the given version, and a random
@@ -107,12 +134,12 @@ func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, v
 	t.Logf("Good %s response pushed to fakeServer...", xdsname)
 
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(version), nonce); err != nil {
-		t.Errorf("Failed to receive %s request: %v", xdsname, err)
+		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("Good %s response acked", xdsname)
 
 	if _, err := callbackCh.Receive(); err != nil {
-		t.Errorf("Timeout when expecting %s update", xdsname)
+		t.Fatalf("Timeout when expecting %s update", xdsname)
 	}
 	t.Logf("Good %s response callback executed", xdsname)
 	return
@@ -140,7 +167,7 @@ func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, ve
 	}, version)
 	t.Logf("Bad %s response pushed to fakeServer...", xdsname)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(version-1), nonce); err != nil {
-		t.Errorf("Failed to receive %s request: %v", xdsname, err)
+		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("Bad %s response nacked", xdsname)
 }
@@ -159,21 +186,21 @@ func (s) TestV2ClientAck(t *testing.T) {
 
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
+
+	v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2cCleanup := startXDSV2Client(t, cc)
+	defer v2cCleanup()
 
 	// Start the watch, send a good response, and check for ack.
-	cbLDS := startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
 	sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 	versionLDS++
-	cbRDS := startXDS(t, "RDS", v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
+	startXDS(t, "RDS", v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
 	sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS)
 	versionRDS++
-	cbCDS := startXDS(t, "CDS", v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
+	startXDS(t, "CDS", v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
 	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	versionCDS++
-	cbEDS := startXDS(t, "EDS", v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
+	startXDS(t, "EDS", v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
 	sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS)
 	versionEDS++
 
@@ -205,12 +232,12 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
+
+	v2c, cbLDS, _, _, _, v2cCleanup := startXDSV2Client(t, cc)
+	defer v2cCleanup()
 
 	// Start the watch, send a good response, and check for ack.
-	cbLDS := startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
 
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
@@ -237,17 +264,17 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
+
+	v2c, cbLDS, _, _, _, v2cCleanup := startXDSV2Client(t, cc)
+	defer v2cCleanup()
 
 	// Start the watch, send a good response, and check for ack.
-	cbLDS := startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
 	nonce := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 
 	// Start a new watch. The version in the new request should be the version
 	// from the previous response, thus versionLDS before ++.
-	cbLDS = startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
+	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
 	versionLDS++
 
 	// This is an invalid response after the new watch.
@@ -275,16 +302,12 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
+
+	v2c, _, _, cbCDS, _, v2cCleanup := startXDSV2Client(t, cc)
+	defer v2cCleanup()
 
 	// Start a CDS watch.
-	callbackCh := testutils.NewChannel()
-	cancel := v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
-		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
-		callbackCh.Send(struct{}{})
-	})
+	v2c.addWatch(cdsURL, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
 		t.Fatal(err)
 	}
@@ -292,15 +315,19 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 
 	// Send a good CDS response, this function waits for the ACK with the right
 	// version.
-	nonce := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+	nonce := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 
 	// Cancel the CDS watch, and start a new one. The new watch should have the
 	// version from the response above.
-	cancel()
-	v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
-		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
-		callbackCh.Send(struct{}{})
-	})
+	v2c.removeWatch(cdsURL, goodClusterName1)
+	// Wait for a request with no resource names, because the only watch was
+	// removed.
+	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: cdsURL}
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
+		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+	}
+	v2c.addWatch(cdsURL, goodClusterName1)
+	// Wait for a request with correct resource names and version.
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
@@ -311,7 +338,7 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	versionCDS++
 }
 
@@ -324,44 +351,51 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
+
+	v2c, _, _, cbCDS, _, v2cCleanup := startXDSV2Client(t, cc)
+	defer v2cCleanup()
 
 	// Start a CDS watch.
-	callbackCh := testutils.NewChannel()
-	cancel := v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
-		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
-		callbackCh.Send(struct{}{})
-	})
+	v2c.addWatch(cdsURL, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
 	t.Logf("FakeServer received %s request...", "CDS")
 
-	// send another good response, and check for ack, with the new version.
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
-	versionCDS++
+	// send a good response, and check for ack, with the new version.
+	nonce := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 
 	// Cancel the watch before the next response is sent. This mimics the case
 	// watch is canceled while response is on wire.
-	cancel()
+	v2c.removeWatch(cdsURL, goodClusterName1)
+	// Wait for a request with no resource names, because the only watch was
+	// removed.
+	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: cdsURL}
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
+		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+	}
+	versionCDS++
+
+	if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout {
+		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
+	}
 
 	// Send a good response.
-	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodCDSResponse1, versionCDS)
+	nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodCDSResponse1, versionCDS)
 	t.Logf("Good %s response pushed to fakeServer...", "CDS")
 
 	// Expect no ACK because watch was canceled.
 	if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout {
 		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
 	}
+	// Still expected an callback update, because response was good.
+	if _, err := cbCDS.Receive(); err != nil {
+		t.Fatalf("Timeout when expecting %s update", "CDS")
+	}
 
 	// Start a new watch. The new watch should have the nonce from the response
 	// above, and version from the first good response.
-	v2c.watchCDS(goodClusterName1, func(u CDSUpdate, err error) {
-		t.Logf("Received %s callback with ldsUpdate {%+v} and error {%v}", "CDS", u, err)
-		callbackCh.Send(struct{}{})
-	})
+	v2c.addWatch(cdsURL, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
@@ -371,6 +405,6 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, callbackCh)
+	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	versionCDS++
 }
diff --git a/xds/internal/client/cds.go b/xds/internal/client/v2client_cds.go
similarity index 77%
rename from xds/internal/client/cds.go
rename to xds/internal/client/v2client_cds.go
index aa1ffd89a70a..6fc9ae86d098 100644
--- a/xds/internal/client/cds.go
+++ b/xds/internal/client/v2client_cds.go
@@ -28,16 +28,7 @@ import (
 // handleCDSResponse processes an CDS response received from the xDS server. On
 // receipt of a good response, it also invokes the registered watcher callback.
 func (v2c *v2Client) handleCDSResponse(resp *xdspb.DiscoveryResponse) error {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	wi := v2c.watchMap[cdsURL]
-	if wi == nil {
-		return fmt.Errorf("xds: no CDS watcher found when handling CDS response: %+v", resp)
-	}
-
-	var returnUpdate CDSUpdate
-	localCache := make(map[string]CDSUpdate)
+	returnUpdate := make(map[string]ClusterUpdate)
 	for _, r := range resp.GetResources() {
 		var resource ptypes.DynamicAny
 		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
@@ -58,25 +49,16 @@ func (v2c *v2Client) handleCDSResponse(resp *xdspb.DiscoveryResponse) error {
 		if update.ServiceName == "" {
 			update.ServiceName = cluster.GetName()
 		}
-		localCache[cluster.GetName()] = update
 		v2c.logger.Debugf("Resource with name %v, type %T, value %+v added to cache", cluster.GetName(), update, update)
-		if cluster.GetName() == wi.target[0] {
-			returnUpdate = update
-		}
+		returnUpdate[cluster.GetName()] = update
 	}
-	v2c.cdsCache = localCache
 
-	var err error
-	if returnUpdate.ServiceName == "" {
-		err = fmt.Errorf("xds: CDS target %s not found in received response %+v", wi.target, resp)
-	}
-	wi.stopTimer()
-	wi.cdsCallback(returnUpdate, err)
+	v2c.parent.newCDSUpdate(returnUpdate)
 	return nil
 }
 
-func validateCluster(cluster *xdspb.Cluster) (CDSUpdate, error) {
-	emptyUpdate := CDSUpdate{ServiceName: "", EnableLRS: false}
+func validateCluster(cluster *xdspb.Cluster) (ClusterUpdate, error) {
+	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
 	switch {
 	case cluster.GetType() != xdspb.Cluster_EDS:
 		return emptyUpdate, fmt.Errorf("xds: unexpected cluster type %v in response: %+v", cluster.GetType(), cluster)
@@ -86,7 +68,7 @@ func validateCluster(cluster *xdspb.Cluster) (CDSUpdate, error) {
 		return emptyUpdate, fmt.Errorf("xds: unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
 
-	return CDSUpdate{
+	return ClusterUpdate{
 		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
 		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
 	}, nil
diff --git a/xds/internal/client/v2client_cds_test.go b/xds/internal/client/v2client_cds_test.go
new file mode 100644
index 000000000000..b93beede4485
--- /dev/null
+++ b/xds/internal/client/v2client_cds_test.go
@@ -0,0 +1,295 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+	"time"
+
+	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	"github.com/google/go-cmp/cmp"
+)
+
+const (
+	serviceName1 = "foo-service"
+	serviceName2 = "bar-service"
+)
+
+func (s) TestValidateCluster(t *testing.T) {
+	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
+	tests := []struct {
+		name       string
+		cluster    *xdspb.Cluster
+		wantUpdate ClusterUpdate
+		wantErr    bool
+	}{
+		{
+			name: "non-eds-cluster-type",
+			cluster: &xdspb.Cluster{
+				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_STATIC},
+				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+					EdsConfig: &corepb.ConfigSource{
+						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+							Ads: &corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: xdspb.Cluster_LEAST_REQUEST,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "no-eds-config",
+			cluster: &xdspb.Cluster{
+				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+				LbPolicy:             xdspb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "no-ads-config-source",
+			cluster: &xdspb.Cluster{
+				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+				EdsClusterConfig:     &xdspb.Cluster_EdsClusterConfig{},
+				LbPolicy:             xdspb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "non-round-robin-lb-policy",
+			cluster: &xdspb.Cluster{
+				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+					EdsConfig: &corepb.ConfigSource{
+						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+							Ads: &corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: xdspb.Cluster_LEAST_REQUEST,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "happy-case-no-service-name-no-lrs",
+			cluster: &xdspb.Cluster{
+				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+					EdsConfig: &corepb.ConfigSource{
+						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+							Ads: &corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: emptyUpdate,
+		},
+		{
+			name: "happy-case-no-lrs",
+			cluster: &xdspb.Cluster{
+				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+					EdsConfig: &corepb.ConfigSource{
+						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+							Ads: &corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName1,
+				},
+				LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName1, EnableLRS: false},
+		},
+		{
+			name:       "happiest-case",
+			cluster:    goodCluster1,
+			wantUpdate: ClusterUpdate{ServiceName: serviceName1, EnableLRS: true},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotUpdate, gotErr := validateCluster(test.cluster)
+			if (gotErr != nil) != test.wantErr {
+				t.Errorf("validateCluster(%+v) returned error: %v, wantErr: %v", test.cluster, gotErr, test.wantErr)
+			}
+			if !cmp.Equal(gotUpdate, test.wantUpdate) {
+				t.Errorf("validateCluster(%+v) = %v, want: %v", test.cluster, gotUpdate, test.wantUpdate)
+			}
+		})
+	}
+}
+
+// TestCDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
+// and creates a v2Client using it. Then, it registers a CDS watcher and tests
+// different CDS responses.
+func (s) TestCDSHandleResponse(t *testing.T) {
+	tests := []struct {
+		name          string
+		cdsResponse   *xdspb.DiscoveryResponse
+		wantErr       bool
+		wantUpdate    *ClusterUpdate
+		wantUpdateErr bool
+	}{
+		// Badly marshaled CDS response.
+		{
+			name:          "badly-marshaled-response",
+			cdsResponse:   badlyMarshaledCDSResponse,
+			wantErr:       true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response does not contain Cluster proto.
+		{
+			name:          "no-cluster-proto-in-response",
+			cdsResponse:   badResourceTypeInLDSResponse,
+			wantErr:       true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response contains no clusters.
+		{
+			name:          "no-cluster",
+			cdsResponse:   &xdspb.DiscoveryResponse{},
+			wantErr:       false,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response contains one good cluster we are not interested in.
+		{
+			name:          "one-uninteresting-cluster",
+			cdsResponse:   goodCDSResponse2,
+			wantErr:       false,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response contains one cluster and it is good.
+		{
+			name:          "one-good-cluster",
+			cdsResponse:   goodCDSResponse1,
+			wantErr:       false,
+			wantUpdate:    &ClusterUpdate{ServiceName: serviceName1, EnableLRS: true},
+			wantUpdateErr: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testWatchHandle(t, &watchHandleTestcase{
+				typeURL:      cdsURL,
+				resourceName: goodClusterName1,
+
+				responseToHandle: test.cdsResponse,
+				wantHandleErr:    test.wantErr,
+				wantUpdate:       test.wantUpdate,
+				wantUpdateErr:    test.wantUpdateErr,
+			})
+		})
+	}
+}
+
+// TestCDSHandleResponseWithoutWatch tests the case where the v2Client receives
+// a CDS response without a registered watcher.
+func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) {
+	_, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	defer v2c.close()
+
+	if v2c.handleCDSResponse(badResourceTypeInLDSResponse) == nil {
+		t.Fatal("v2c.handleCDSResponse() succeeded, should have failed")
+	}
+
+	if v2c.handleCDSResponse(goodCDSResponse1) != nil {
+		t.Fatal("v2c.handleCDSResponse() succeeded, should have failed")
+	}
+}
+
+var (
+	badlyMarshaledCDSResponse = &xdspb.DiscoveryResponse{
+		Resources: []*anypb.Any{
+			{
+				TypeUrl: cdsURL,
+				Value:   []byte{1, 2, 3, 4},
+			},
+		},
+		TypeUrl: cdsURL,
+	}
+	goodCluster1 = &xdspb.Cluster{
+		Name:                 goodClusterName1,
+		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+			EdsConfig: &corepb.ConfigSource{
+				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+					Ads: &corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: serviceName1,
+		},
+		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+		LrsServer: &corepb.ConfigSource{
+			ConfigSourceSpecifier: &corepb.ConfigSource_Self{
+				Self: &corepb.SelfConfigSource{},
+			},
+		},
+	}
+	marshaledCluster1, _ = proto.Marshal(goodCluster1)
+	goodCluster2         = &xdspb.Cluster{
+		Name:                 goodClusterName2,
+		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+			EdsConfig: &corepb.ConfigSource{
+				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+					Ads: &corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: serviceName2,
+		},
+		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+	}
+	marshaledCluster2, _ = proto.Marshal(goodCluster2)
+	goodCDSResponse1     = &xdspb.DiscoveryResponse{
+		Resources: []*anypb.Any{
+			{
+				TypeUrl: cdsURL,
+				Value:   marshaledCluster1,
+			},
+		},
+		TypeUrl: cdsURL,
+	}
+	goodCDSResponse2 = &xdspb.DiscoveryResponse{
+		Resources: []*anypb.Any{
+			{
+				TypeUrl: cdsURL,
+				Value:   marshaledCluster2,
+			},
+		},
+		TypeUrl: cdsURL,
+	}
+)
diff --git a/xds/internal/client/eds.go b/xds/internal/client/v2client_eds.go
similarity index 61%
rename from xds/internal/client/eds.go
rename to xds/internal/client/v2client_eds.go
index 4c2fbe98c28d..0055de52b832 100644
--- a/xds/internal/client/eds.go
+++ b/xds/internal/client/v2client_eds.go
@@ -30,52 +30,6 @@ import (
 	"google.golang.org/grpc/xds/internal"
 )
 
-// OverloadDropConfig contains the config to drop overloads.
-type OverloadDropConfig struct {
-	Category    string
-	Numerator   uint32
-	Denominator uint32
-}
-
-// EndpointHealthStatus represents the health status of an endpoint.
-type EndpointHealthStatus int32
-
-const (
-	// EndpointHealthStatusUnknown represents HealthStatus UNKNOWN.
-	EndpointHealthStatusUnknown EndpointHealthStatus = iota
-	// EndpointHealthStatusHealthy represents HealthStatus HEALTHY.
-	EndpointHealthStatusHealthy
-	// EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY.
-	EndpointHealthStatusUnhealthy
-	// EndpointHealthStatusDraining represents HealthStatus DRAINING.
-	EndpointHealthStatusDraining
-	// EndpointHealthStatusTimeout represents HealthStatus TIMEOUT.
-	EndpointHealthStatusTimeout
-	// EndpointHealthStatusDegraded represents HealthStatus DEGRADED.
-	EndpointHealthStatusDegraded
-)
-
-// Endpoint contains information of an endpoint.
-type Endpoint struct {
-	Address      string
-	HealthStatus EndpointHealthStatus
-	Weight       uint32
-}
-
-// Locality contains information of a locality.
-type Locality struct {
-	Endpoints []Endpoint
-	ID        internal.LocalityID
-	Priority  uint32
-	Weight    uint32
-}
-
-// EDSUpdate contains an EDS update.
-type EDSUpdate struct {
-	Drops      []OverloadDropConfig
-	Localities []Locality
-}
-
 func parseAddress(socketAddress *corepb.SocketAddress) string {
 	return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue())))
 }
@@ -113,12 +67,12 @@ func parseEndpoints(lbEndpoints []*endpointpb.LbEndpoint) []Endpoint {
 	return endpoints
 }
 
-// ParseEDSRespProto turns EDS response proto message to EDSUpdate.
+// ParseEDSRespProto turns EDS response proto message to EndpointsUpdate.
 //
 // This is temporarily exported to be used in eds balancer, before it switches
 // to use xds client. TODO: unexport.
-func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (*EDSUpdate, error) {
-	ret := &EDSUpdate{}
+func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (EndpointsUpdate, error) {
+	ret := EndpointsUpdate{}
 	for _, dropPolicy := range m.GetPolicy().GetDropOverloads() {
 		ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy))
 	}
@@ -126,7 +80,7 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (*EDSUpdate, error) {
 	for _, locality := range m.Endpoints {
 		l := locality.GetLocality()
 		if l == nil {
-			return nil, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
+			return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
 		}
 		lid := internal.LocalityID{
 			Region:  l.Region,
@@ -144,7 +98,7 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (*EDSUpdate, error) {
 	}
 	for i := 0; i < len(priorities); i++ {
 		if _, ok := priorities[uint32(i)]; !ok {
-			return nil, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities)
+			return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities)
 		}
 	}
 	return ret, nil
@@ -153,9 +107,9 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (*EDSUpdate, error) {
 // ParseEDSRespProtoForTesting parses EDS response, and panic if parsing fails.
 // This is used by EDS balancer tests.
 //
-// TODO: delete this. The EDS balancer tests should build an EDSUpdate directly,
+// TODO: delete this. The EDS balancer tests should build an EndpointsUpdate directly,
 //  instead of building and parsing a proto message.
-func ParseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) *EDSUpdate {
+func ParseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) EndpointsUpdate {
 	u, err := ParseEDSRespProto(m)
 	if err != nil {
 		panic(err.Error())
@@ -164,15 +118,7 @@ func ParseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) *EDSUpdate {
 }
 
 func (v2c *v2Client) handleEDSResponse(resp *xdspb.DiscoveryResponse) error {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	wi := v2c.watchMap[edsURL]
-	if wi == nil {
-		return fmt.Errorf("xds: no EDS watcher found when handling EDS response: %+v", resp)
-	}
-
-	var returnUpdate *EDSUpdate
+	returnUpdate := make(map[string]EndpointsUpdate)
 	for _, r := range resp.GetResources() {
 		var resource ptypes.DynamicAny
 		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
@@ -184,29 +130,14 @@ func (v2c *v2Client) handleEDSResponse(resp *xdspb.DiscoveryResponse) error {
 		}
 		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
 
-		if cla.GetClusterName() != wi.target[0] {
-			// We won't validate the remaining resources. If one of the
-			// uninteresting ones is invalid, we will still ACK the response.
-			continue
-		}
-
 		u, err := ParseEDSRespProto(cla)
 		if err != nil {
 			return err
 		}
 
-		returnUpdate = u
-		// Break from the loop because the request resource is found. But
-		// this also means we won't validate the remaining resources. If one
-		// of the uninteresting ones is invalid, we will still ACK the
-		// response.
-		break
-	}
-
-	if returnUpdate != nil {
-		wi.stopTimer()
-		wi.edsCallback(returnUpdate, nil)
+		returnUpdate[cla.GetClusterName()] = u
 	}
 
+	v2c.parent.newEDSUpdate(returnUpdate)
 	return nil
 }
diff --git a/xds/internal/client/eds_test.go b/xds/internal/client/v2client_eds_test.go
similarity index 78%
rename from xds/internal/client/eds_test.go
rename to xds/internal/client/v2client_eds_test.go
index a491f59703c4..009fbcb28190 100644
--- a/xds/internal/client/eds_test.go
+++ b/xds/internal/client/v2client_eds_test.go
@@ -18,8 +18,6 @@
 package client
 
 import (
-	"errors"
-	"fmt"
 	"testing"
 	"time"
 
@@ -29,14 +27,13 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 func (s) TestEDSParseRespProto(t *testing.T) {
 	tests := []struct {
 		name    string
 		m       *xdspb.ClusterLoadAssignment
-		want    *EDSUpdate
+		want    EndpointsUpdate
 		wantErr bool
 	}{
 		{
@@ -47,7 +44,7 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 				clab0.AddLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
 				return clab0.Build()
 			}(),
-			want:    nil,
+			want:    EndpointsUpdate{},
 			wantErr: true,
 		},
 		{
@@ -57,7 +54,7 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 				clab0.AddLocality("", 1, 0, []string{"addr1:314"}, nil)
 				return clab0.Build()
 			}(),
-			want:    nil,
+			want:    EndpointsUpdate{},
 			wantErr: true,
 		},
 		{
@@ -74,7 +71,7 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 				})
 				return clab0.Build()
 			}(),
-			want: &EDSUpdate{
+			want: EndpointsUpdate{
 				Drops: nil,
 				Localities: []Locality{
 					{
@@ -162,17 +159,11 @@ var (
 )
 
 func (s) TestEDSHandleResponse(t *testing.T) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-
 	tests := []struct {
 		name          string
 		edsResponse   *xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *EDSUpdate
+		wantUpdate    *EndpointsUpdate
 		wantUpdateErr bool
 	}{
 		// Any in resource is badly marshaled.
@@ -204,7 +195,7 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 			name:        "one-good-assignment",
 			edsResponse: goodEDSResponse1,
 			wantErr:     false,
-			wantUpdate: &EDSUpdate{
+			wantUpdate: &EndpointsUpdate{
 				Localities: []Locality{
 					{
 						Endpoints: []Endpoint{{Address: "addr1:314"}},
@@ -226,14 +217,12 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
+				typeURL:          edsURL,
+				resourceName:     goodEDSName,
 				responseToHandle: test.edsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
 				wantUpdateErr:    test.wantUpdateErr,
-
-				edsWatch:      v2c.watchEDS,
-				watchReqChan:  fakeServer.XDSRequestChan,
-				handleXDSResp: v2c.handleEDSResponse,
 			})
 		})
 	}
@@ -245,44 +234,16 @@ func (s) TestEDSHandleResponseWithoutWatch(t *testing.T) {
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 
-	if v2c.handleEDSResponse(goodEDSResponse1) == nil {
+	if v2c.handleEDSResponse(badResourceTypeInEDSResponse) == nil {
 		t.Fatal("v2c.handleEDSResponse() succeeded, should have failed")
 	}
-}
-
-func (s) TestEDSWatchExpiryTimer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
 
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
-
-	callbackCh := testutils.NewChannel()
-	v2c.watchEDS(goodRouteName1, func(u *EDSUpdate, err error) {
-		t.Logf("Received callback with edsUpdate {%+v} and error {%v}", u, err)
-		if u != nil {
-			callbackCh.Send(fmt.Errorf("received EDSUpdate %v in edsCallback, wanted nil", u))
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("received nil error in edsCallback"))
-		}
-		callbackCh.Send(nil)
-	})
-
-	// Wait till the request makes it to the fakeServer. This ensures that
-	// the watch request has been processed by the v2Client.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an CDS request")
+	if v2c.handleEDSResponse(goodEDSResponse1) != nil {
+		t.Fatal("v2c.handleEDSResponse() succeeded, should have failed")
 	}
-	waitForNilErr(t, callbackCh)
 }
diff --git a/xds/internal/client/eds_testutil.go b/xds/internal/client/v2client_eds_testutil.go
similarity index 97%
rename from xds/internal/client/eds_testutil.go
rename to xds/internal/client/v2client_eds_testutil.go
index 503a9073c6c1..2d187a510365 100644
--- a/xds/internal/client/eds_testutil.go
+++ b/xds/internal/client/v2client_eds_testutil.go
@@ -17,7 +17,7 @@
 
 // All structs/functions in this file should be unexported. They are used in EDS
 // balancer tests now, to generate test inputs. Eventually, EDS balancer tests
-// should generate EDSUpdate directly, instead of generating and parsing the
+// should generate EndpointsUpdate directly, instead of generating and parsing the
 // proto message.
 // TODO: unexported everything in this file.
 
diff --git a/xds/internal/client/lds.go b/xds/internal/client/v2client_lds.go
similarity index 79%
rename from xds/internal/client/lds.go
rename to xds/internal/client/v2client_lds.go
index a67cde205a93..49f4a46a28cc 100644
--- a/xds/internal/client/lds.go
+++ b/xds/internal/client/v2client_lds.go
@@ -29,15 +29,7 @@ import (
 // handleLDSResponse processes an LDS response received from the xDS server. On
 // receipt of a good response, it also invokes the registered watcher callback.
 func (v2c *v2Client) handleLDSResponse(resp *xdspb.DiscoveryResponse) error {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	wi := v2c.watchMap[ldsURL]
-	if wi == nil {
-		return fmt.Errorf("xds: no LDS watcher found when handling LDS response: %+v", resp)
-	}
-
-	routeName := ""
+	returnUpdate := make(map[string]ldsUpdate)
 	for _, r := range resp.GetResources() {
 		var resource ptypes.DynamicAny
 		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
@@ -48,27 +40,14 @@ func (v2c *v2Client) handleLDSResponse(resp *xdspb.DiscoveryResponse) error {
 			return fmt.Errorf("xds: unexpected resource type: %T in LDS response", resource.Message)
 		}
 		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
-		if lis.GetName() != wi.target[0] {
-			// We ignore listeners we are not watching for because LDS is
-			// special in the sense that there is only one resource we are
-			// interested in, and this resource does not change over the
-			// lifetime of the v2Client. So, we don't have to cache other
-			// listeners which we are not interested in.
-			continue
-		}
-		var err error
-		routeName, err = v2c.getRouteConfigNameFromListener(lis)
+		routeName, err := v2c.getRouteConfigNameFromListener(lis)
 		if err != nil {
 			return err
 		}
+		returnUpdate[lis.GetName()] = ldsUpdate{routeName: routeName}
 	}
 
-	var err error
-	if routeName == "" {
-		err = fmt.Errorf("xds: LDS target %s not found in received response %+v", wi.target, resp)
-	}
-	wi.stopTimer()
-	wi.ldsCallback(ldsUpdate{routeName: routeName}, err)
+	v2c.parent.newLDSUpdate(returnUpdate)
 	return nil
 }
 
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/v2client_lds_test.go
similarity index 70%
rename from xds/internal/client/lds_test.go
rename to xds/internal/client/v2client_lds_test.go
index 7457a968c983..a61b18bbbae0 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/v2client_lds_test.go
@@ -19,13 +19,10 @@
 package client
 
 import (
-	"errors"
-	"fmt"
 	"testing"
 	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 func (s) TestLDSGetRouteConfig(t *testing.T) {
@@ -74,7 +71,7 @@ func (s) TestLDSGetRouteConfig(t *testing.T) {
 	}
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	v2c := newV2Client(nil, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -93,12 +90,6 @@ func (s) TestLDSGetRouteConfig(t *testing.T) {
 // and creates a v2Client using it. Then, it registers a watchLDS and tests
 // different LDS responses.
 func (s) TestLDSHandleResponse(t *testing.T) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-
 	tests := []struct {
 		name          string
 		ldsResponse   *xdspb.DiscoveryResponse
@@ -150,12 +141,13 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 			wantUpdateErr: false,
 		},
 		// Response contains two good listeners (one interesting and one
-		// uninteresting), and one badly marshaled listener.
+		// uninteresting), and one badly marshaled listener. This will cause a
+		// nack because the uninteresting listener will still be parsed.
 		{
 			name:          "good-bad-ugly-listeners",
 			ldsResponse:   goodBadUglyLDSResponse,
-			wantErr:       false,
-			wantUpdate:    &ldsUpdate{routeName: goodRouteName1},
+			wantErr:       true,
+			wantUpdate:    nil,
 			wantUpdateErr: false,
 		},
 		// Response contains one listener, but we are not interested in it.
@@ -163,8 +155,8 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 			name:          "one-uninteresting-listener",
 			ldsResponse:   goodLDSResponse2,
 			wantErr:       false,
-			wantUpdate:    &ldsUpdate{routeName: ""},
-			wantUpdateErr: true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
 		},
 		// Response constains no resources. This is the case where the server
 		// does not know about the target we are interested in.
@@ -172,22 +164,20 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 			name:          "empty-response",
 			ldsResponse:   emptyLDSResponse,
 			wantErr:       false,
-			wantUpdate:    &ldsUpdate{routeName: ""},
-			wantUpdateErr: true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
+				typeURL:          ldsURL,
+				resourceName:     goodLDSTarget1,
 				responseToHandle: test.ldsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
 				wantUpdateErr:    test.wantUpdateErr,
-
-				ldsWatch:      v2c.watchLDS,
-				watchReqChan:  fakeServer.XDSRequestChan,
-				handleXDSResp: v2c.handleLDSResponse,
 			})
 		})
 	}
@@ -199,46 +189,16 @@ func (s) TestLDSHandleResponseWithoutWatch(t *testing.T) {
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 
-	if v2c.handleLDSResponse(goodLDSResponse1) == nil {
+	if v2c.handleLDSResponse(badResourceTypeInLDSResponse) == nil {
 		t.Fatal("v2c.handleLDSResponse() succeeded, should have failed")
 	}
-}
-
-// TestLDSWatchExpiryTimer tests the case where the client does not receive an
-// LDS response for the request that it sends out. We want the watch callback
-// to be invoked with an error once the watchExpiryTimer fires.
-func (s) TestLDSWatchExpiryTimer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
 
-	callbackCh := testutils.NewChannel()
-	v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-		t.Logf("in v2c.watchLDS callback, ldsUpdate: %+v, err: %v", u, err)
-		if u.routeName != "" {
-			callbackCh.Send(fmt.Errorf("received routeName %v in ldsCallback, wanted empty string", u.routeName))
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("received nil error in ldsCallback"))
-		}
-		callbackCh.Send(nil)
-	})
-
-	// Wait till the request makes it to the fakeServer. This ensures that
-	// the watch request has been processed by the v2Client.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
+	if v2c.handleLDSResponse(goodLDSResponse1) != nil {
+		t.Fatal("v2c.handleLDSResponse() succeeded, should have failed")
 	}
-	waitForNilErr(t, callbackCh)
 }
diff --git a/xds/internal/client/rds.go b/xds/internal/client/v2client_rds.go
similarity index 80%
rename from xds/internal/client/rds.go
rename to xds/internal/client/v2client_rds.go
index 0594a9cd2676..4141d31d4596 100644
--- a/xds/internal/client/rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -32,20 +32,10 @@ import (
 // the registered watcher callback.
 func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
+	hostname := v2c.hostname
+	v2c.mu.Unlock()
 
-	if v2c.watchMap[ldsURL] == nil {
-		return fmt.Errorf("xds: unexpected RDS response when no LDS watcher is registered: %+v", resp)
-	}
-	target := v2c.watchMap[ldsURL].target[0]
-
-	wi := v2c.watchMap[rdsURL]
-	if wi == nil {
-		return fmt.Errorf("xds: no RDS watcher found when handling RDS response: %+v", resp)
-	}
-
-	returnCluster := ""
-	localCache := make(map[string]string)
+	returnUpdate := make(map[string]rdsUpdate)
 	for _, r := range resp.GetResources() {
 		var resource ptypes.DynamicAny
 		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
@@ -55,40 +45,19 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 		if !ok {
 			return fmt.Errorf("xds: unexpected resource type: %T in RDS response", resource.Message)
 		}
-		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v", rc.GetName(), rc, rc)
-		cluster := getClusterFromRouteConfiguration(rc, target)
+		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, v2c.hostname)
+
+		// Use the hostname (resourceName for LDS) to find the routes.
+		cluster := getClusterFromRouteConfiguration(rc, hostname)
 		if cluster == "" {
 			return fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v", rc)
 		}
 
 		// If we get here, it means that this resource was a good one.
-		localCache[rc.GetName()] = cluster
-		v2c.logger.Debugf("Resource with name %v, type %T, value %+v added to cache", rc.GetName(), cluster, cluster)
-
-		// TODO: remove cache, and only process resources that are interesting.
-		if rc.GetName() == wi.target[0] {
-			returnCluster = cluster
-		}
+		returnUpdate[rc.GetName()] = rdsUpdate{clusterName: cluster}
 	}
 
-	// Update the cache in the v2Client only after we have confirmed that all
-	// resources in the received response were good.
-	for k, v := range localCache {
-		// TODO: Need to handle deletion of entries from the cache based on LDS
-		// watch calls. Not handling it does not affect correctness, but leads
-		// to unnecessary memory consumption.
-		v2c.rdsCache[k] = v
-	}
-
-	if returnCluster != "" {
-		// We stop the expiry timer and invoke the callback only when we have
-		// received the resource that we are watching for. Since RDS is an
-		// incremental protocol, the fact that we did not receive the resource
-		// that we are watching for in this response does not mean that the
-		// server does not know about it.
-		wi.stopTimer()
-		wi.rdsCallback(rdsUpdate{clusterName: returnCluster}, nil)
-	}
+	v2c.parent.newRDSUpdate(returnUpdate)
 	return nil
 }
 
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/v2client_rds_test.go
similarity index 58%
rename from xds/internal/client/rds_test.go
rename to xds/internal/client/v2client_rds_test.go
index 0ffe23c003a6..6c10b22d2dfe 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -19,31 +19,16 @@
 package client
 
 import (
-	"errors"
-	"fmt"
 	"testing"
 	"time"
 
-	discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
 
-func (v2c *v2Client) cloneRDSCacheForTesting() map[string]string {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	cloneCache := make(map[string]string)
-	for k, v := range v2c.rdsCache {
-		cloneCache[k] = v
-	}
-	return cloneCache
-}
-
 func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 	tests := []struct {
 		name        string
@@ -169,38 +154,16 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 // pre-requirement for RDS, and RDS handle would fail without an existing LDS
 // watch.
 func doLDS(t *testing.T, v2c *v2Client, fakeServer *fakeserver.Server) {
-	// Register an LDS watcher, and wait till the request is sent out, the
-	// response is received and the callback is invoked.
-	cbCh := testutils.NewChannel()
-	v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-		t.Logf("v2c.watchLDS callback, ldsUpdate: %+v, err: %v", u, err)
-		cbCh.Send(err)
-	})
-
+	v2c.addWatch(ldsURL, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout waiting for LDS request: %v", err)
 	}
-
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
-	waitForNilErr(t, cbCh)
-
-	// Read the LDS ack, to clear RequestChan for following tests.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout waiting for LDS ACK: %v", err)
-	}
 }
 
 // TestRDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
 // and creates a v2Client using it. Then, it registers an LDS and RDS watcher
 // and tests different RDS responses.
 func (s) TestRDSHandleResponse(t *testing.T) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	doLDS(t, v2c, fakeServer)
-
 	tests := []struct {
 		name          string
 		rdsResponse   *xdspb.DiscoveryResponse
@@ -251,18 +214,15 @@ func (s) TestRDSHandleResponse(t *testing.T) {
 			wantUpdateErr: false,
 		},
 	}
-
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
+				typeURL:          rdsURL,
+				resourceName:     goodRouteName1,
 				responseToHandle: test.rdsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
 				wantUpdateErr:    test.wantUpdateErr,
-
-				rdsWatch:      v2c.watchRDS,
-				watchReqChan:  fakeServer.XDSRequestChan,
-				handleXDSResp: v2c.handleRDSResponse,
 			})
 		})
 	}
@@ -274,7 +234,9 @@ func (s) TestRDSHandleResponseWithoutLDSWatch(t *testing.T) {
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 
 	if v2c.handleRDSResponse(goodRDSResponse1) == nil {
@@ -288,181 +250,22 @@ func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 	doLDS(t, v2c, fakeServer)
 
-	if v2c.handleRDSResponse(goodRDSResponse1) == nil {
+	if v2c.handleRDSResponse(badResourceTypeInRDSResponse) == nil {
 		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
 	}
-}
-
-// rdsTestOp contains all data related to one particular test operation. Not
-// all fields make sense for all tests.
-type rdsTestOp struct {
-	// target is the resource name to watch for.
-	target string
-	// responseToSend is the xDS response sent to the client
-	responseToSend *fakeserver.Response
-	// wantOpErr specfies whether the main operation should return an error.
-	wantOpErr bool
-	// wantRDSCache is the expected rdsCache at the end of an operation.
-	wantRDSCache map[string]string
-	// wantWatchCallback specifies if the watch callback should be invoked.
-	wantWatchCallback bool
-}
-
-// testRDSCaching is a helper function which starts a fake xDS server, makes a
-// ClientConn to it, creates a v2Client using it, registers an LDS watcher and
-// pushes a good LDS response. It then reads a bunch of test operations to be
-// performed from rdsTestOps and returns error, if any, on the provided error
-// channel. This is executed in a separate goroutine.
-func testRDSCaching(t *testing.T, rdsTestOps []rdsTestOp, errCh *testutils.Channel) {
-	t.Helper()
-
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
-	doLDS(t, v2c, fakeServer)
-
-	callbackCh := make(chan struct{}, 1)
-	for _, rdsTestOp := range rdsTestOps {
-		// Register a watcher if required, and use a channel to signal the
-		// successful invocation of the callback.
-		if rdsTestOp.target != "" {
-			v2c.watchRDS(rdsTestOp.target, func(u rdsUpdate, err error) {
-				t.Logf("Received callback with rdsUpdate {%+v} and error {%v}", u, err)
-				callbackCh <- struct{}{}
-			})
-			t.Logf("Registered a watcher for RDS target: %v...", rdsTestOp.target)
-
-			// Wait till the request makes it to the fakeServer. This ensures that
-			// the watch request has been processed by the v2Client.
-			if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-				errCh.Send(fmt.Errorf("Timeout waiting for RDS request: %v", err))
-			}
-			t.Log("FakeServer received request...")
-		}
-
-		// Directly push the response through a call to handleRDSResponse,
-		// thereby bypassing the fakeServer.
-		if rdsTestOp.responseToSend != nil {
-			resp := rdsTestOp.responseToSend.Resp.(*discoverypb.DiscoveryResponse)
-			if err := v2c.handleRDSResponse(resp); (err != nil) != rdsTestOp.wantOpErr {
-				errCh.Send(fmt.Errorf("v2c.handleRDSResponse(%+v) returned err: %v", resp, err))
-				return
-			}
-		}
-
-		// If the test needs the callback to be invoked, just verify that
-		// it was invoked. Since we verify the contents of the cache, it's
-		// ok not to verify the contents of the callback.
-		if rdsTestOp.wantWatchCallback {
-			<-callbackCh
-		}
-
-		if !cmp.Equal(v2c.cloneRDSCacheForTesting(), rdsTestOp.wantRDSCache) {
-			errCh.Send(fmt.Errorf("gotRDSCache: %v, wantRDSCache: %v", v2c.rdsCache, rdsTestOp.wantRDSCache))
-			return
-		}
-	}
-	t.Log("Completed all test ops successfully...")
-	errCh.Send(nil)
-}
-
-// TestRDSCaching tests some end-to-end RDS flows using a fake xDS server, and
-// verifies the RDS data cached at the v2Client.
-func (s) TestRDSCaching(t *testing.T) {
-	ops := []rdsTestOp{
-		// Add an RDS watch for a resource name (goodRouteName1), which returns one
-		// matching resource in the response.
-		{
-			target:            goodRouteName1,
-			responseToSend:    &fakeserver.Response{Resp: goodRDSResponse1},
-			wantRDSCache:      map[string]string{goodRouteName1: goodClusterName1},
-			wantWatchCallback: true,
-		},
-		// Push an RDS response with a new resource. This resource is considered
-		// good because its domain field matches our LDS watch target, but the
-		// routeConfigName does not match our RDS watch (so the watch callback will
-		// not be invoked). But this should still be cached.
-		{
-			responseToSend: &fakeserver.Response{Resp: goodRDSResponse2},
-			wantRDSCache: map[string]string{
-				goodRouteName1: goodClusterName1,
-				goodRouteName2: goodClusterName2,
-			},
-		},
-		// Push an uninteresting RDS response. This should cause handleRDSResponse
-		// to return an error. But the watch callback should not be invoked, and
-		// the cache should not be updated.
-		{
-			responseToSend: &fakeserver.Response{Resp: uninterestingRDSResponse},
-			wantOpErr:      true,
-			wantRDSCache: map[string]string{
-				goodRouteName1: goodClusterName1,
-				goodRouteName2: goodClusterName2,
-			},
-		},
-		// Switch the watch target to goodRouteName2, which was already cached.  No
-		// response is received from the server (as expected), but we want the
-		// callback to be invoked with the new clusterName.
-		{
-			target: goodRouteName2,
-			wantRDSCache: map[string]string{
-				goodRouteName1: goodClusterName1,
-				goodRouteName2: goodClusterName2,
-			},
-			wantWatchCallback: true,
-		},
-	}
-	errCh := testutils.NewChannel()
-	go testRDSCaching(t, ops, errCh)
-	waitForNilErr(t, errCh)
-}
-
-// TestRDSWatchExpiryTimer tests the case where the client does not receive an
-// RDS response for the request that it sends out. We want the watch callback
-// to be invoked with an error once the watchExpiryTimer fires.
-func (s) TestRDSWatchExpiryTimer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
-	doLDS(t, v2c, fakeServer)
-
-	callbackCh := testutils.NewChannel()
-	v2c.watchRDS(goodRouteName1, func(u rdsUpdate, err error) {
-		t.Logf("Received callback with rdsUpdate {%+v} and error {%v}", u, err)
-		if u.clusterName != "" {
-			callbackCh.Send(fmt.Errorf("received clusterName %v in rdsCallback, wanted empty string", u.clusterName))
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("received nil error in rdsCallback"))
-		}
-		callbackCh.Send(nil)
-	})
-
-	// Wait till the request makes it to the fakeServer. This ensures that
-	// the watch request has been processed by the v2Client.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an RDS request")
+	if v2c.handleRDSResponse(goodRDSResponse1) != nil {
+		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
 	}
-	waitForNilErr(t, callbackCh)
 }
 
-func TestMatchTypeForDomain(t *testing.T) {
+func (s) TestMatchTypeForDomain(t *testing.T) {
 	tests := []struct {
 		d    string
 		want domainMatchType
@@ -481,7 +284,7 @@ func TestMatchTypeForDomain(t *testing.T) {
 	}
 }
 
-func TestMatch(t *testing.T) {
+func (s) TestMatch(t *testing.T) {
 	tests := []struct {
 		name        string
 		domain      string
@@ -508,7 +311,7 @@ func TestMatch(t *testing.T) {
 	}
 }
 
-func TestFindBestMatchingVirtualHost(t *testing.T) {
+func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
 	var (
 		oneExactMatch = &routepb.VirtualHost{
 			Name:    "one-exact-match",
diff --git a/xds/internal/client/v2client_test.go b/xds/internal/client/v2client_test.go
index 5eedda9e39c7..e52648cfeb1f 100644
--- a/xds/internal/client/v2client_test.go
+++ b/xds/internal/client/v2client_test.go
@@ -343,25 +343,7 @@ var (
 		},
 	}
 	marshaledGoodRouteConfig2, _ = proto.Marshal(goodRouteConfig2)
-	uninterestingRouteConfig     = &xdspb.RouteConfiguration{
-		Name: uninterestingRouteName,
-		VirtualHosts: []*routepb.VirtualHost{
-			{
-				Domains: []string{uninterestingDomain},
-				Routes: []*routepb.Route{
-					{
-						Action: &routepb.Route_Route{
-							Route: &routepb.RouteAction{
-								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: uninterestingClusterName},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-	marshaledUninterestingRouteConfig, _ = proto.Marshal(uninterestingRouteConfig)
-	goodRDSResponse1                     = &xdspb.DiscoveryResponse{
+	goodRDSResponse1             = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
 				TypeUrl: rdsURL,
@@ -379,15 +361,6 @@ var (
 		},
 		TypeUrl: rdsURL,
 	}
-	uninterestingRDSResponse = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: rdsURL,
-				Value:   marshaledUninterestingRouteConfig,
-			},
-		},
-		TypeUrl: rdsURL,
-	}
 )
 
 // TestV2ClientBackoffAfterRecvError verifies if the v2Client backoffs when it
@@ -404,14 +377,15 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 		return 0
 	}
 
-	v2c := newV2Client(cc, goodNodeProto, clientBackoff, nil)
+	callbackCh := make(chan struct{})
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) { close(callbackCh) },
+	}, cc, goodNodeProto, clientBackoff, nil)
 	defer v2c.close()
 	t.Log("Started xds v2Client...")
 
-	callbackCh := make(chan struct{})
-	v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-		close(callbackCh)
-	})
+	// v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {})
+	v2c.addWatch(ldsURL, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
@@ -439,15 +413,21 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	callbackCh := testutils.NewChannel()
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(typeURL string, d map[string]interface{}) {
+			if typeURL == ldsURL {
+				if u, ok := d[goodLDSTarget1]; ok {
+					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
+					callbackCh.Send(struct{}{})
+				}
+			}
+		},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 	t.Log("Started xds v2Client...")
 
-	callbackCh := testutils.NewChannel()
-	v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-		t.Logf("Received LDS callback with ldsUpdate {%+v} and error {%v}", u, err)
-		callbackCh.Send(struct{}{})
-	})
+	v2c.addWatch(ldsURL, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
@@ -478,43 +458,11 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	}
 }
 
-// TestV2ClientCancelWatch verifies that the registered watch callback is not
-// invoked if a response is received after the watcher is cancelled.
-func (s) TestV2ClientCancelWatch(t *testing.T) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	t.Log("Started xds v2Client...")
-
-	callbackCh := testutils.NewChannel()
-	cancelFunc := v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-		t.Logf("Received LDS callback with ldsUpdate {%+v} and error {%v}", u, err)
-		callbackCh.Send(struct{}{})
-	})
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
-	}
-	t.Log("FakeServer received request...")
-
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
-	t.Log("Good LDS response pushed to fakeServer...")
-
-	if _, err := callbackCh.Receive(); err != nil {
-		t.Fatal("Timeout when expecting LDS update")
-	}
-
-	cancelFunc()
-
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
-	t.Log("Another good LDS response pushed to fakeServer...")
-
-	if _, err := callbackCh.Receive(); err != testutils.ErrRecvTimeout {
-		t.Fatalf("Watch callback invoked after the watcher was cancelled")
-	}
-}
-
+// TestV2ClientWatchWithoutStream verifies the case where a watch is started
+// when the xds stream is not created. The watcher should not receive any update
+// (because there won't be any xds response, and timeout is done at a upper
+// level). And when the stream is re-created, the watcher should get future
+// updates.
 func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
 	defaultWatchExpiryTimeout = 500 * time.Millisecond
@@ -538,25 +486,27 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	}
 	defer cc.Close()
 
-	v2c := newV2Client(cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	callbackCh := testutils.NewChannel()
+	v2c := newV2Client(&testUpdateReceiver{
+		f: func(typeURL string, d map[string]interface{}) {
+			if typeURL == ldsURL {
+				if u, ok := d[goodLDSTarget1]; ok {
+					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
+					callbackCh.Send(u)
+				}
+			}
+		},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	defer v2c.close()
 	t.Log("Started xds v2Client...")
 
-	callbackCh := testutils.NewChannel()
 	// This watch is started when the xds-ClientConn is in Transient Failure,
 	// and no xds stream is created.
-	v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {
-		t.Logf("Received LDS callback with ldsUpdate {%+v} and error {%v}", u, err)
-		if err != nil {
-			callbackCh.Send(err)
-		}
-		callbackCh.Send(u)
-	})
+	v2c.addWatch(ldsURL, goodLDSTarget1)
+
 	// The watcher should receive an update, with a timeout error in it.
-	if v, err := callbackCh.TimedReceive(time.Second); err != nil {
-		t.Fatal("Timeout when expecting LDS update")
-	} else if _, ok := v.(error); !ok {
-		t.Fatalf("Expect an error from watcher, got %v", v)
+	if v, err := callbackCh.TimedReceive(100 * time.Millisecond); err == nil {
+		t.Fatalf("Expect an timeout error from watcher, got %v", v)
 	}
 
 	// Send the real server address to the ClientConn, the stream should be
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index bb145b06ab28..2b290101e9ce 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -42,8 +42,8 @@ type Client struct {
 
 	mu        sync.Mutex
 	serviceCb func(xdsclient.ServiceUpdate, error)
-	cdsCb     func(xdsclient.CDSUpdate, error)
-	edsCb     func(*xdsclient.EDSUpdate, error)
+	cdsCb     func(xdsclient.ClusterUpdate, error)
+	edsCb     func(xdsclient.EndpointsUpdate, error)
 }
 
 // WatchService registers a LDS/RDS watch.
@@ -77,7 +77,7 @@ func (xdsC *Client) InvokeWatchServiceCallback(cluster string, err error) {
 }
 
 // WatchCluster registers a CDS watch.
-func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsclient.CDSUpdate, error)) func() {
+func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsclient.ClusterUpdate, error)) func() {
 	xdsC.mu.Lock()
 	defer xdsC.mu.Unlock()
 
@@ -99,7 +99,7 @@ func (xdsC *Client) WaitForWatchCluster() (string, error) {
 }
 
 // InvokeWatchClusterCallback invokes the registered cdsWatch callback.
-func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.CDSUpdate, err error) {
+func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.ClusterUpdate, err error) {
 	xdsC.mu.Lock()
 	defer xdsC.mu.Unlock()
 
@@ -114,7 +114,7 @@ func (xdsC *Client) WaitForCancelClusterWatch() error {
 }
 
 // WatchEndpoints registers an EDS watch for provided clusterName.
-func (xdsC *Client) WatchEndpoints(clusterName string, callback func(*xdsclient.EDSUpdate, error)) (cancel func()) {
+func (xdsC *Client) WatchEndpoints(clusterName string, callback func(xdsclient.EndpointsUpdate, error)) (cancel func()) {
 	xdsC.mu.Lock()
 	defer xdsC.mu.Unlock()
 
@@ -136,7 +136,7 @@ func (xdsC *Client) WaitForWatchEDS() (string, error) {
 }
 
 // InvokeWatchEDSCallback invokes the registered edsWatch callback.
-func (xdsC *Client) InvokeWatchEDSCallback(update *xdsclient.EDSUpdate, err error) {
+func (xdsC *Client) InvokeWatchEDSCallback(update xdsclient.EndpointsUpdate, err error) {
 	xdsC.mu.Lock()
 	defer xdsC.mu.Unlock()
 

From f9a1aeb4f25273448942c02780bd80c25bde2308 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 15 Apr 2020 16:50:46 -0700
Subject: [PATCH 026/481] xds: accept either "" or "/" as the prefix for the
 default route (#3535)

---
 xds/internal/client/v2client_rds.go      |  4 ++--
 xds/internal/client/v2client_rds_test.go | 17 ++++++++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
index 4141d31d4596..14ae6cdfc37f 100644
--- a/xds/internal/client/v2client_rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -95,8 +95,8 @@ func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string)
 		return ""
 	}
 	dr := vh.Routes[len(vh.Routes)-1]
-	if match := dr.GetMatch(); match == nil || match.GetPrefix() != "" {
-		// The matched virtual host is invalid.
+	if match := dr.GetMatch(); match == nil || (match.GetPrefix() != "" && match.GetPrefix() != "/") {
+		// The matched virtual host is invalid. Match is not "" or "/".
 		return ""
 	}
 	if route := dr.GetRoute(); route != nil {
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
index 6c10b22d2dfe..8bbabb9828fc 100644
--- a/xds/internal/client/v2client_rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -133,10 +133,25 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 			wantCluster: "",
 		},
 		{
-			name:        "good-route-config",
+			name:        "good-route-config-with-empty-string-route",
 			rc:          goodRouteConfig1,
 			wantCluster: goodClusterName1,
 		},
+		{
+			// default route's match is not empty string, but "/".
+			name: "good-route-config-with-slash-string-route",
+			rc: &xdspb.RouteConfiguration{
+				Name: goodRouteName1,
+				VirtualHosts: []*routepb.VirtualHost{{
+					Domains: []string{goodLDSTarget1},
+					Routes: []*routepb.Route{{
+						Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"}},
+						Action: &routepb.Route_Route{
+							Route: &routepb.RouteAction{
+								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
+							}}}}}}},
+			wantCluster: goodClusterName1,
+		},
 	}
 
 	for _, test := range tests {

From 759569bb9c835b555736bfb3e2c24f1bc8a6b4ae Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 16 Apr 2020 14:45:21 -0700
Subject: [PATCH 027/481] balancer: support hierarchical paths in addresses
 (#3494)

---
 internal/hierarchy/hierarchy.go      |  99 ++++++++++++++
 internal/hierarchy/hierarchy_test.go | 197 +++++++++++++++++++++++++++
 2 files changed, 296 insertions(+)
 create mode 100644 internal/hierarchy/hierarchy.go
 create mode 100644 internal/hierarchy/hierarchy_test.go

diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go
new file mode 100644
index 000000000000..17185d95d38e
--- /dev/null
+++ b/internal/hierarchy/hierarchy.go
@@ -0,0 +1,99 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package hierarchy contains functions to set and get hierarchy string from
+// addresses.
+//
+// This package is experimental.
+package hierarchy
+
+import (
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/resolver"
+)
+
+type pathKeyType string
+
+const pathKey = pathKeyType("grpc.internal.address.hierarchical_path")
+
+// Get returns the hierarchical path of addr.
+func Get(addr resolver.Address) []string {
+	attrs := addr.Attributes
+	if attrs == nil {
+		return nil
+	}
+	path, ok := attrs.Value(pathKey).([]string)
+	if !ok {
+		return nil
+	}
+	return path
+}
+
+// Set overrides the hierarchical path in addr with path.
+func Set(addr resolver.Address, path []string) resolver.Address {
+	if addr.Attributes == nil {
+		addr.Attributes = attributes.New(pathKey, path)
+		return addr
+	}
+	addr.Attributes = addr.Attributes.WithValues(pathKey, path)
+	return addr
+}
+
+// Group splits a slice of addresses into groups based on
+// the first hierarchy path. The first hierarchy path will be removed from the
+// result.
+//
+// Input:
+// [
+//   {addr0, path: [p0, wt0]}
+//   {addr1, path: [p0, wt1]}
+//   {addr2, path: [p1, wt2]}
+//   {addr3, path: [p1, wt3]}
+// ]
+//
+// Addresses will be split into p0/p1, and the p0/p1 will be removed from the
+// path.
+//
+// Output:
+// {
+//   p0: [
+//     {addr0, path: [wt0]},
+//     {addr1, path: [wt1]},
+//   ],
+//   p1: [
+//     {addr2, path: [wt2]},
+//     {addr3, path: [wt3]},
+//   ],
+// }
+//
+// If hierarchical path is not set, or has no path in it, the address is
+// dropped.
+func Group(addrs []resolver.Address) map[string][]resolver.Address {
+	ret := make(map[string][]resolver.Address)
+	for _, addr := range addrs {
+		oldPath := Get(addr)
+		if len(oldPath) == 0 {
+			continue
+		}
+		curPath := oldPath[0]
+		newPath := oldPath[1:]
+		newAddr := Set(addr, newPath)
+		ret[curPath] = append(ret[curPath], newAddr)
+	}
+	return ret
+}
diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go
new file mode 100644
index 000000000000..fc62f82b0850
--- /dev/null
+++ b/internal/hierarchy/hierarchy_test.go
@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package hierarchy
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/resolver"
+)
+
+func TestGet(t *testing.T) {
+	tests := []struct {
+		name string
+		addr resolver.Address
+		want []string
+	}{
+		{
+			name: "not set",
+			addr: resolver.Address{},
+			want: nil,
+		},
+		{
+			name: "set",
+			addr: resolver.Address{
+				Attributes: attributes.New(pathKey, []string{"a", "b"}),
+			},
+			want: []string{"a", "b"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Get(tt.addr); !cmp.Equal(got, tt.want) {
+				t.Errorf("Get() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSet(t *testing.T) {
+	tests := []struct {
+		name string
+		addr resolver.Address
+		path []string
+	}{
+		{
+			name: "before is not set",
+			addr: resolver.Address{},
+			path: []string{"a", "b"},
+		},
+		{
+			name: "before is set",
+			addr: resolver.Address{
+				Attributes: attributes.New(pathKey, []string{"before", "a", "b"}),
+			},
+			path: []string{"a", "b"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			newAddr := Set(tt.addr, tt.path)
+			newPath := Get(newAddr)
+			if !cmp.Equal(newPath, tt.path) {
+				t.Errorf("path after Set() = %v, want %v", newPath, tt.path)
+			}
+		})
+	}
+}
+
+func TestGroup(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []resolver.Address
+		want  map[string][]resolver.Address
+	}{
+		{
+			name: "all with hierarchy",
+			addrs: []resolver.Address{
+				{Addr: "a0", Attributes: attributes.New(pathKey, []string{"a"})},
+				{Addr: "a1", Attributes: attributes.New(pathKey, []string{"a"})},
+				{Addr: "b0", Attributes: attributes.New(pathKey, []string{"b"})},
+				{Addr: "b1", Attributes: attributes.New(pathKey, []string{"b"})},
+			},
+			want: map[string][]resolver.Address{
+				"a": {
+					{Addr: "a0", Attributes: attributes.New(pathKey, []string{})},
+					{Addr: "a1", Attributes: attributes.New(pathKey, []string{})},
+				},
+				"b": {
+					{Addr: "b0", Attributes: attributes.New(pathKey, []string{})},
+					{Addr: "b1", Attributes: attributes.New(pathKey, []string{})},
+				},
+			},
+		},
+		{
+			// Addresses without hierarchy are ignored.
+			name: "without hierarchy",
+			addrs: []resolver.Address{
+				{Addr: "a0", Attributes: attributes.New(pathKey, []string{"a"})},
+				{Addr: "a1", Attributes: attributes.New(pathKey, []string{"a"})},
+				{Addr: "b0", Attributes: nil},
+				{Addr: "b1", Attributes: nil},
+			},
+			want: map[string][]resolver.Address{
+				"a": {
+					{Addr: "a0", Attributes: attributes.New(pathKey, []string{})},
+					{Addr: "a1", Attributes: attributes.New(pathKey, []string{})},
+				},
+			},
+		},
+		{
+			// If hierarchy is set to a wrong type (which should never happen),
+			// the address is ignored.
+			name: "wrong type",
+			addrs: []resolver.Address{
+				{Addr: "a0", Attributes: attributes.New(pathKey, []string{"a"})},
+				{Addr: "a1", Attributes: attributes.New(pathKey, []string{"a"})},
+				{Addr: "b0", Attributes: attributes.New(pathKey, "b")},
+				{Addr: "b1", Attributes: attributes.New(pathKey, 314)},
+			},
+			want: map[string][]resolver.Address{
+				"a": {
+					{Addr: "a0", Attributes: attributes.New(pathKey, []string{})},
+					{Addr: "a1", Attributes: attributes.New(pathKey, []string{})},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Group(tt.addrs); !cmp.Equal(got, tt.want, cmp.AllowUnexported(attributes.Attributes{})) {
+				t.Errorf("Group() = %v, want %v", got, tt.want)
+				t.Errorf("diff: %v", cmp.Diff(got, tt.want, cmp.AllowUnexported(attributes.Attributes{})))
+			}
+		})
+	}
+}
+
+func TestGroupE2E(t *testing.T) {
+	hierarchy := map[string]map[string][]string{
+		"p0": {
+			"wt0": {"addr0", "addr1"},
+			"wt1": {"addr2", "addr3"},
+		},
+		"p1": {
+			"wt10": {"addr10", "addr11"},
+			"wt11": {"addr12", "addr13"},
+		},
+	}
+
+	var addrsWithHierarchy []resolver.Address
+	for p, wts := range hierarchy {
+		path1 := []string{p}
+		for wt, addrs := range wts {
+			path2 := append([]string(nil), path1...)
+			path2 = append(path2, wt)
+			for _, addr := range addrs {
+				a := resolver.Address{
+					Addr:       addr,
+					Attributes: attributes.New(pathKey, path2),
+				}
+				addrsWithHierarchy = append(addrsWithHierarchy, a)
+			}
+		}
+	}
+
+	gotHierarchy := make(map[string]map[string][]string)
+	for p1, wts := range Group(addrsWithHierarchy) {
+		gotHierarchy[p1] = make(map[string][]string)
+		for p2, addrs := range Group(wts) {
+			for _, addr := range addrs {
+				gotHierarchy[p1][p2] = append(gotHierarchy[p1][p2], addr.Addr)
+			}
+		}
+	}
+
+	if !cmp.Equal(gotHierarchy, hierarchy) {
+		t.Errorf("diff: %v", cmp.Diff(gotHierarchy, hierarchy))
+	}
+}

From f9ac13d4698ab7feb6e4f86165a2d9e74430e70b Mon Sep 17 00:00:00 2001
From: Cesar Ghali <cesarghali@users.noreply.github.com>
Date: Thu, 16 Apr 2020 15:09:15 -0700
Subject: [PATCH 028/481] credentials/alts: Properly release server InBytes
 buffer after the handshake is complete. (#3529)

---
 credentials/alts/internal/conn/record.go      | 10 ++-
 credentials/alts/internal/conn/record_test.go | 65 +++++++++++++++----
 2 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/credentials/alts/internal/conn/record.go b/credentials/alts/internal/conn/record.go
index fd5a53d9a7aa..8a872c3cffd0 100644
--- a/credentials/alts/internal/conn/record.go
+++ b/credentials/alts/internal/conn/record.go
@@ -111,6 +111,7 @@ func NewConn(c net.Conn, side core.Side, recordProtocol string, key []byte, prot
 	}
 	overhead := MsgLenFieldSize + msgTypeFieldSize + crypto.EncryptionOverhead()
 	payloadLengthLimit := altsRecordDefaultLength - overhead
+	var protectedBuf []byte
 	if protected == nil {
 		// We pre-allocate protected to be of size
 		// 2*altsRecordDefaultLength-1 during initialization. We only
@@ -120,16 +121,19 @@ func NewConn(c net.Conn, side core.Side, recordProtocol string, key []byte, prot
 		// altsRecordDefaultLength (bytes) data into protected at one
 		// time. Therefore, 2*altsRecordDefaultLength-1 is large enough
 		// to buffer data read from the network.
-		protected = make([]byte, 0, 2*altsRecordDefaultLength-1)
+		protectedBuf = make([]byte, 0, 2*altsRecordDefaultLength-1)
+	} else {
+		protectedBuf = make([]byte, len(protected))
+		copy(protectedBuf, protected)
 	}
 
 	altsConn := &conn{
 		Conn:               c,
 		crypto:             crypto,
 		payloadLengthLimit: payloadLengthLimit,
-		protected:          protected,
+		protected:          protectedBuf,
 		writeBuf:           make([]byte, altsWriteBufferInitialSize),
-		nextFrame:          protected,
+		nextFrame:          protectedBuf,
 		overhead:           overhead,
 	}
 	return altsConn, nil
diff --git a/credentials/alts/internal/conn/record_test.go b/credentials/alts/internal/conn/record_test.go
index af3bc9b604b7..59d4f41e9e1c 100644
--- a/credentials/alts/internal/conn/record_test.go
+++ b/credentials/alts/internal/conn/record_test.go
@@ -77,7 +77,7 @@ func (c *testConn) Close() error {
 	return nil
 }
 
-func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, np string) *conn {
+func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, np string, protected []byte) *conn {
 	key := []byte{
 		// 16 arbitrary bytes.
 		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49}
@@ -85,23 +85,23 @@ func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, np string) *co
 		in:  in,
 		out: out,
 	}
-	c, err := NewConn(&tc, side, np, key, nil)
+	c, err := NewConn(&tc, side, np, key, protected)
 	if err != nil {
 		panic(fmt.Sprintf("Unexpected error creating test ALTS record connection: %v", err))
 	}
 	return c.(*conn)
 }
 
-func newConnPair(np string) (client, server *conn) {
+func newConnPair(np string, clientProtected []byte, serverProtected []byte) (client, server *conn) {
 	clientBuf := new(bytes.Buffer)
 	serverBuf := new(bytes.Buffer)
-	clientConn := newTestALTSRecordConn(clientBuf, serverBuf, core.ClientSide, np)
-	serverConn := newTestALTSRecordConn(serverBuf, clientBuf, core.ServerSide, np)
+	clientConn := newTestALTSRecordConn(clientBuf, serverBuf, core.ClientSide, np, clientProtected)
+	serverConn := newTestALTSRecordConn(serverBuf, clientBuf, core.ServerSide, np, serverProtected)
 	return clientConn, serverConn
 }
 
 func testPingPong(t *testing.T, np string) {
-	clientConn, serverConn := newConnPair(np)
+	clientConn, serverConn := newConnPair(np, nil, nil)
 	clientMsg := []byte("Client Message")
 	if n, err := clientConn.Write(clientMsg); n != len(clientMsg) || err != nil {
 		t.Fatalf("Client Write() = %v, %v; want %v, <nil>", n, err, len(clientMsg))
@@ -134,7 +134,7 @@ func (s) TestPingPong(t *testing.T) {
 }
 
 func testSmallReadBuffer(t *testing.T, np string) {
-	clientConn, serverConn := newConnPair(np)
+	clientConn, serverConn := newConnPair(np, nil, nil)
 	msg := []byte("Very Important Message")
 	if n, err := clientConn.Write(msg); err != nil {
 		t.Fatalf("Write() = %v, %v; want %v, <nil>", n, err, len(msg))
@@ -161,7 +161,7 @@ func (s) TestSmallReadBuffer(t *testing.T) {
 }
 
 func testLargeMsg(t *testing.T, np string) {
-	clientConn, serverConn := newConnPair(np)
+	clientConn, serverConn := newConnPair(np, nil, nil)
 	// msgLen is such that the length in the framing is larger than the
 	// default size of one frame.
 	msgLen := altsRecordDefaultLength - msgTypeFieldSize - clientConn.crypto.EncryptionOverhead() + 1
@@ -193,7 +193,7 @@ func testIncorrectMsgType(t *testing.T, np string) {
 	binary.LittleEndian.PutUint32(framedMsg[MsgLenFieldSize:], wrongMsgType)
 
 	in := bytes.NewBuffer(framedMsg)
-	c := newTestALTSRecordConn(in, nil, core.ClientSide, np)
+	c := newTestALTSRecordConn(in, nil, core.ClientSide, np, nil)
 	b := make([]byte, 1)
 	if n, err := c.Read(b); n != 0 || err == nil {
 		t.Fatalf("Read() = <nil>, want %v", fmt.Errorf("received frame with incorrect message type %v", wrongMsgType))
@@ -208,8 +208,8 @@ func (s) TestIncorrectMsgType(t *testing.T) {
 
 func testFrameTooLarge(t *testing.T, np string) {
 	buf := new(bytes.Buffer)
-	clientConn := newTestALTSRecordConn(nil, buf, core.ClientSide, np)
-	serverConn := newTestALTSRecordConn(buf, nil, core.ServerSide, np)
+	clientConn := newTestALTSRecordConn(nil, buf, core.ClientSide, np, nil)
+	serverConn := newTestALTSRecordConn(buf, nil, core.ServerSide, np, nil)
 	// payloadLen is such that the length in the framing is larger than
 	// allowed in one frame.
 	payloadLen := altsRecordLengthLimit - msgTypeFieldSize - clientConn.crypto.EncryptionOverhead() + 1
@@ -242,7 +242,7 @@ func (s) TestFrameTooLarge(t *testing.T) {
 func testWriteLargeData(t *testing.T, np string) {
 	// Test sending and receiving messages larger than the maximum write
 	// buffer size.
-	clientConn, serverConn := newConnPair(np)
+	clientConn, serverConn := newConnPair(np, nil, nil)
 	// Message size is intentionally chosen to not be multiple of
 	// payloadLengthLimtit.
 	msgSize := altsWriteBufferMaxSize + (100 * 1024)
@@ -281,3 +281,44 @@ func (s) TestWriteLargeData(t *testing.T) {
 		testWriteLargeData(t, np)
 	}
 }
+
+func testProtectedBuffer(t *testing.T, np string) {
+	key := []byte{
+		// 16 arbitrary bytes.
+		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49}
+
+	// Encrypt a message to be passed to NewConn as a client-side protected
+	// buffer.
+	newCrypto := protocols[np]
+	if newCrypto == nil {
+		t.Fatalf("Unknown next protocol %q", np)
+	}
+	crypto, err := newCrypto(core.ClientSide, key)
+	if err != nil {
+		t.Fatalf("Failed to create a crypter for protocol %q: %v", np, err)
+	}
+	msg := []byte("Client Protected Message")
+	encryptedMsg, err := crypto.Encrypt(nil, msg)
+	if err != nil {
+		t.Fatalf("Failed to encrypt the client protected message: %v", err)
+	}
+	protectedMsg := make([]byte, 8)                                          // 8 bytes = 4 length + 4 type
+	binary.LittleEndian.PutUint32(protectedMsg, uint32(len(encryptedMsg))+4) // 4 bytes for the type
+	binary.LittleEndian.PutUint32(protectedMsg[4:], altsRecordMsgType)
+	protectedMsg = append(protectedMsg, encryptedMsg...)
+
+	_, serverConn := newConnPair(np, nil, protectedMsg)
+	rcvClientMsg := make([]byte, len(msg))
+	if n, err := serverConn.Read(rcvClientMsg); n != len(rcvClientMsg) || err != nil {
+		t.Fatalf("Server Read() = %v, %v; want %v, <nil>", n, err, len(rcvClientMsg))
+	}
+	if !reflect.DeepEqual(msg, rcvClientMsg) {
+		t.Fatalf("Client protected/Server Read() = %v, want %v", rcvClientMsg, msg)
+	}
+}
+
+func (s) TestProtectedBuffer(t *testing.T) {
+	for _, np := range nextProtocols {
+		testProtectedBuffer(t, np)
+	}
+}

From d15f1a4aa159e36c58e43e21e3feea418ccbc468 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 16 Apr 2020 16:30:23 -0700
Subject: [PATCH 029/481] test: Move creds related to tests to creds_test.go
 (#3542)

---
 test/creds_test.go   | 410 ++++++++++++++++++++++++++++++++++++-
 test/end2end_test.go | 466 +------------------------------------------
 2 files changed, 408 insertions(+), 468 deletions(-)

diff --git a/test/creds_test.go b/test/creds_test.go
index 275165c8cd75..65d99c7555d2 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -18,15 +18,24 @@
 
 package test
 
-// TODO(https://github.com/grpc/grpc-go/issues/2330): move all creds related
-// tests to this file.
-
 import (
 	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
 	"testing"
+	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/tap"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/testdata"
 )
@@ -125,3 +134,398 @@ func (s) TestCredsBundlePerRPCCredentials(t *testing.T) {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
+
+type clientTimeoutCreds struct {
+	credentials.TransportCredentials
+	timeoutReturned bool
+}
+
+func (c *clientTimeoutCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	if !c.timeoutReturned {
+		c.timeoutReturned = true
+		return nil, nil, context.DeadlineExceeded
+	}
+	return rawConn, nil, nil
+}
+
+func (c *clientTimeoutCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+
+func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials {
+	return nil
+}
+
+func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
+	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty", balancer: "v1"})
+	te.userAgent = testAppUA
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{}))
+	tc := testpb.NewTestServiceClient(cc)
+	// This unary call should succeed, because ClientHandshake will succeed for the second time.
+	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want <nil>", err)
+	}
+}
+
+type methodTestCreds struct{}
+
+func (m *methodTestCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	return nil, status.Errorf(codes.Unknown, ri.Method)
+}
+
+func (m *methodTestCreds) RequireTransportSecurity() bool { return false }
+
+func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
+	const wantMethod = "/grpc.testing.TestService/EmptyCall"
+	te := newTest(t, env{name: "context-request-info", network: "tcp", balancer: "v1"})
+	te.userAgent = testAppUA
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{}))
+	tc := testpb.NewTestServiceClient(cc)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMethod {
+		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod)
+	}
+}
+
+const clientAlwaysFailCredErrorMsg = "clientAlwaysFailCred always fails"
+
+type clientAlwaysFailCred struct {
+	credentials.TransportCredentials
+}
+
+func (c clientAlwaysFailCred) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return nil, nil, errors.New(clientAlwaysFailCredErrorMsg)
+}
+func (c clientAlwaysFailCred) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c clientAlwaysFailCred) Clone() credentials.TransportCredentials {
+	return nil
+}
+
+func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
+	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	cc, err := grpc.DialContext(ctx, te.srvAddr, opts...)
+	if err != nil {
+		t.Fatalf("Dial(_) = %v, want %v", err, nil)
+	}
+	defer cc.Close()
+
+	tc := testpb.NewTestServiceClient(cc)
+	for i := 0; i < 1000; i++ {
+		// This loop runs for at most 1 second. The first several RPCs will fail
+		// with Unavailable because the connection hasn't started. When the
+		// first connection failed with creds error, the next RPC should also
+		// fail with the expected error.
+		if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg)
+}
+
+func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) {
+	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
+	dctx, dcancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer dcancel()
+	cc, err := grpc.DialContext(dctx, te.srvAddr, opts...)
+	if err != nil {
+		t.Fatalf("Dial(_) = %v, want %v", err, nil)
+	}
+	defer cc.Close()
+
+	tc := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
+		return
+	}
+	te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg)
+}
+
+var (
+	// test authdata
+	authdata = map[string]string{
+		"test-key":      "test-value",
+		"test-key2-bin": string([]byte{1, 2, 3}),
+	}
+)
+
+type testPerRPCCredentials struct{}
+
+func (cr testPerRPCCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
+	return authdata, nil
+}
+
+func (cr testPerRPCCredentials) RequireTransportSecurity() bool {
+	return false
+}
+
+func authHandle(ctx context.Context, info *tap.Info) (context.Context, error) {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return ctx, fmt.Errorf("didn't find metadata in context")
+	}
+	for k, vwant := range authdata {
+		vgot, ok := md[k]
+		if !ok {
+			return ctx, fmt.Errorf("didn't find authdata key %v in context", k)
+		}
+		if vgot[0] != vwant {
+			return ctx, fmt.Errorf("for key %v, got value %v, want %v", k, vgot, vwant)
+		}
+	}
+	return ctx, nil
+}
+
+func (s) TestPerRPCCredentialsViaDialOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPerRPCCredentialsViaDialOptions(t, e)
+	}
+}
+
+func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.tapHandle = authHandle
+	te.perRPCCreds = testPerRPCCredentials{}
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testpb.NewTestServiceClient(cc)
+	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestPerRPCCredentialsViaCallOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPerRPCCredentialsViaCallOptions(t, e)
+	}
+}
+
+func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.tapHandle = authHandle
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testpb.NewTestServiceClient(cc)
+	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPerRPCCredentialsViaDialOptionsAndCallOptions(t, e)
+	}
+}
+
+func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.perRPCCreds = testPerRPCCredentials{}
+	// When credentials are provided via both dial options and call options,
+	// we apply both sets.
+	te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) {
+		md, ok := metadata.FromIncomingContext(ctx)
+		if !ok {
+			return ctx, fmt.Errorf("couldn't find metadata in context")
+		}
+		for k, vwant := range authdata {
+			vgot, ok := md[k]
+			if !ok {
+				return ctx, fmt.Errorf("couldn't find metadata for key %v", k)
+			}
+			if len(vgot) != 2 {
+				return ctx, fmt.Errorf("len of value for key %v was %v, want 2", k, len(vgot))
+			}
+			if vgot[0] != vwant || vgot[1] != vwant {
+				return ctx, fmt.Errorf("value for %v was %v, want [%v, %v]", k, vgot, vwant, vwant)
+			}
+		}
+		return ctx, nil
+	}
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testpb.NewTestServiceClient(cc)
+	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+const testAuthority = "test.auth.ori.ty"
+
+type authorityCheckCreds struct {
+	credentials.TransportCredentials
+	got string
+}
+
+func (c *authorityCheckCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	c.got = authority
+	return rawConn, nil, nil
+}
+func (c *authorityCheckCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c *authorityCheckCreds) Clone() credentials.TransportCredentials {
+	return c
+}
+
+// This test makes sure that the authority client handshake gets is the endpoint
+// in dial target, not the resolved ip address.
+func (s) TestCredsHandshakeAuthority(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cred := &authorityCheckCreds{}
+	s := grpc.NewServer()
+	go s.Serve(lis)
+	defer s.Stop()
+
+	r, rcleanup := manual.GenerateAndRegisterManualResolver()
+	defer rcleanup()
+
+	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred))
+	if err != nil {
+		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	for {
+		s := cc.GetState()
+		if s == connectivity.Ready {
+			break
+		}
+		if !cc.WaitForStateChange(ctx, s) {
+			t.Fatalf("ClientConn is not ready after 100 ms")
+		}
+	}
+
+	if cred.got != testAuthority {
+		t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority)
+	}
+}
+
+// This test makes sure that the authority client handshake gets is the endpoint
+// of the ServerName of the address when it is set.
+func (s) TestCredsHandshakeServerNameAuthority(t *testing.T) {
+	const testServerName = "test.server.name"
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cred := &authorityCheckCreds{}
+	s := grpc.NewServer()
+	go s.Serve(lis)
+	defer s.Stop()
+
+	r, rcleanup := manual.GenerateAndRegisterManualResolver()
+	defer rcleanup()
+
+	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred))
+	if err != nil {
+		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String(), ServerName: testServerName}}})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	for {
+		s := cc.GetState()
+		if s == connectivity.Ready {
+			break
+		}
+		if !cc.WaitForStateChange(ctx, s) {
+			t.Fatalf("ClientConn is not ready after 100 ms")
+		}
+	}
+
+	if cred.got != testServerName {
+		t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority)
+	}
+}
+
+type serverDispatchCred struct {
+	rawConnCh chan net.Conn
+}
+
+func (c *serverDispatchCred) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return rawConn, nil, nil
+}
+func (c *serverDispatchCred) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	select {
+	case c.rawConnCh <- rawConn:
+	default:
+	}
+	return nil, nil, credentials.ErrConnDispatched
+}
+func (c *serverDispatchCred) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c *serverDispatchCred) Clone() credentials.TransportCredentials {
+	return nil
+}
+func (c *serverDispatchCred) OverrideServerName(s string) error {
+	return nil
+}
+func (c *serverDispatchCred) getRawConn() net.Conn {
+	return <-c.rawConnCh
+}
+
+func (s) TestServerCredsDispatch(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cred := &serverDispatchCred{
+		rawConnCh: make(chan net.Conn, 1),
+	}
+	s := grpc.NewServer(grpc.Creds(cred))
+	go s.Serve(lis)
+	defer s.Stop()
+
+	cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(cred))
+	if err != nil {
+		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+
+	rawConn := cred.getRawConn()
+	// Give grpc a chance to see the error and potentially close the connection.
+	// And check that connection is not closed after that.
+	time.Sleep(100 * time.Millisecond)
+	// Check rawConn is not closed.
+	if n, err := rawConn.Write([]byte{0}); n <= 0 || err != nil {
+		t.Errorf("Read() = %v, %v; want n>0, <nil>", n, err)
+	}
+}
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 19ba39c9f60b..a32e93410a98 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -619,15 +619,12 @@ func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network,
 	if err != nil {
 		te.t.Fatalf("Failed to listen: %v", err)
 	}
-	switch te.e.security {
-	case "tls":
+	if te.e.security == "tls" {
 		creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
 		if err != nil {
 			te.t.Fatalf("Failed to generate credentials %v", err)
 		}
 		sopts = append(sopts, grpc.Creds(creds))
-	case "clientTimeoutCreds":
-		sopts = append(sopts, grpc.Creds(&clientTimeoutCreds{}))
 	}
 	sopts = append(sopts, te.customServerOptions...)
 	s := grpc.NewServer(sopts...)
@@ -803,8 +800,6 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string)
 			te.t.Fatalf("Failed to load credentials: %v", err)
 		}
 		opts = append(opts, grpc.WithTransportCredentials(creds))
-	case "clientTimeoutCreds":
-		opts = append(opts, grpc.WithTransportCredentials(&clientTimeoutCreds{}))
 	case "empty":
 		// Don't add any transport creds option.
 	default:
@@ -4720,250 +4715,6 @@ func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) {
 	}
 }
 
-type clientTimeoutCreds struct {
-	timeoutReturned bool
-}
-
-func (c *clientTimeoutCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	if !c.timeoutReturned {
-		c.timeoutReturned = true
-		return nil, nil, context.DeadlineExceeded
-	}
-	return rawConn, nil, nil
-}
-func (c *clientTimeoutCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return rawConn, nil, nil
-}
-func (c *clientTimeoutCreds) Info() credentials.ProtocolInfo {
-	return credentials.ProtocolInfo{}
-}
-func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials {
-	return nil
-}
-func (c *clientTimeoutCreds) OverrideServerName(s string) error {
-	return nil
-}
-
-func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
-	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "clientTimeoutCreds", balancer: "v1"})
-	te.userAgent = testAppUA
-	te.startServer(&testServer{security: te.e.security})
-	defer te.tearDown()
-
-	cc := te.clientConn()
-	tc := testpb.NewTestServiceClient(cc)
-	// This unary call should succeed, because ClientHandshake will succeed for the second time.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
-		te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want <nil>", err)
-	}
-}
-
-type serverDispatchCred struct {
-	rawConnCh chan net.Conn
-}
-
-func newServerDispatchCred() *serverDispatchCred {
-	return &serverDispatchCred{
-		rawConnCh: make(chan net.Conn, 1),
-	}
-}
-func (c *serverDispatchCred) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return rawConn, nil, nil
-}
-func (c *serverDispatchCred) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	select {
-	case c.rawConnCh <- rawConn:
-	default:
-	}
-	return nil, nil, credentials.ErrConnDispatched
-}
-func (c *serverDispatchCred) Info() credentials.ProtocolInfo {
-	return credentials.ProtocolInfo{}
-}
-func (c *serverDispatchCred) Clone() credentials.TransportCredentials {
-	return nil
-}
-func (c *serverDispatchCred) OverrideServerName(s string) error {
-	return nil
-}
-func (c *serverDispatchCred) getRawConn() net.Conn {
-	return <-c.rawConnCh
-}
-
-func (s) TestServerCredsDispatch(t *testing.T) {
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("Failed to listen: %v", err)
-	}
-	cred := newServerDispatchCred()
-	s := grpc.NewServer(grpc.Creds(cred))
-	go s.Serve(lis)
-	defer s.Stop()
-
-	cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(cred))
-	if err != nil {
-		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
-	}
-	defer cc.Close()
-
-	rawConn := cred.getRawConn()
-	// Give grpc a chance to see the error and potentially close the connection.
-	// And check that connection is not closed after that.
-	time.Sleep(100 * time.Millisecond)
-	// Check rawConn is not closed.
-	if n, err := rawConn.Write([]byte{0}); n <= 0 || err != nil {
-		t.Errorf("Read() = %v, %v; want n>0, <nil>", n, err)
-	}
-}
-
-type authorityCheckCreds struct {
-	got string
-}
-
-func (c *authorityCheckCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return rawConn, nil, nil
-}
-func (c *authorityCheckCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	c.got = authority
-	return rawConn, nil, nil
-}
-func (c *authorityCheckCreds) Info() credentials.ProtocolInfo {
-	return credentials.ProtocolInfo{}
-}
-func (c *authorityCheckCreds) Clone() credentials.TransportCredentials {
-	return c
-}
-func (c *authorityCheckCreds) OverrideServerName(s string) error {
-	return nil
-}
-
-// This test makes sure that the authority client handshake gets is the endpoint
-// in dial target, not the resolved ip address.
-func (s) TestCredsHandshakeAuthority(t *testing.T) {
-	const testAuthority = "test.auth.ori.ty"
-
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("Failed to listen: %v", err)
-	}
-	cred := &authorityCheckCreds{}
-	s := grpc.NewServer()
-	go s.Serve(lis)
-	defer s.Stop()
-
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
-
-	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred))
-	if err != nil {
-		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
-	}
-	defer cc.Close()
-	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
-
-	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
-	defer cancel()
-	for {
-		s := cc.GetState()
-		if s == connectivity.Ready {
-			break
-		}
-		if !cc.WaitForStateChange(ctx, s) {
-			// ctx got timeout or canceled.
-			t.Fatalf("ClientConn is not ready after 100 ms")
-		}
-	}
-
-	if cred.got != testAuthority {
-		t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority)
-	}
-}
-
-// This test makes sure that the authority client handshake gets is the endpoint
-// of the ServerName of the address when it is set.
-func (s) TestCredsHandshakeServerNameAuthority(t *testing.T) {
-	const testAuthority = "test.auth.ori.ty"
-	const testServerName = "test.server.name"
-
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("Failed to listen: %v", err)
-	}
-	cred := &authorityCheckCreds{}
-	s := grpc.NewServer()
-	go s.Serve(lis)
-	defer s.Stop()
-
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
-
-	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred))
-	if err != nil {
-		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
-	}
-	defer cc.Close()
-	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String(), ServerName: testServerName}}})
-
-	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
-	defer cancel()
-	for {
-		s := cc.GetState()
-		if s == connectivity.Ready {
-			break
-		}
-		if !cc.WaitForStateChange(ctx, s) {
-			// ctx got timeout or canceled.
-			t.Fatalf("ClientConn is not ready after 100 ms")
-		}
-	}
-
-	if cred.got != testServerName {
-		t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority)
-	}
-}
-
-type clientFailCreds struct{}
-
-func (c *clientFailCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return rawConn, nil, nil
-}
-func (c *clientFailCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return nil, nil, fmt.Errorf("client handshake fails with fatal error")
-}
-func (c *clientFailCreds) Info() credentials.ProtocolInfo {
-	return credentials.ProtocolInfo{}
-}
-func (c *clientFailCreds) Clone() credentials.TransportCredentials {
-	return c
-}
-func (c *clientFailCreds) OverrideServerName(s string) error {
-	return nil
-}
-
-// This test makes sure that failfast RPCs fail if client handshake fails with
-// fatal errors.
-func (s) TestFailfastRPCFailOnFatalHandshakeError(t *testing.T) {
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("Failed to listen: %v", err)
-	}
-	defer lis.Close()
-
-	cc, err := grpc.Dial("passthrough:///"+lis.Addr().String(), grpc.WithTransportCredentials(&clientFailCreds{}))
-	if err != nil {
-		t.Fatalf("grpc.Dial(_) = %v", err)
-	}
-	defer cc.Close()
-
-	tc := testpb.NewTestServiceClient(cc)
-	// This unary call should fail, but not timeout.
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(false)); status.Code(err) != codes.Unavailable {
-		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want <Unavailable>", err)
-	}
-}
-
 func (s) TestFlowControlLogicalRace(t *testing.T) {
 	// Test for a regression of https://github.com/grpc/grpc-go/issues/632,
 	// and other flow control bugs.
@@ -5633,120 +5384,6 @@ func testConfigurableWindowSize(t *testing.T, e env, wc windowSizeConfig) {
 	}
 }
 
-var (
-	// test authdata
-	authdata = map[string]string{
-		"test-key":      "test-value",
-		"test-key2-bin": string([]byte{1, 2, 3}),
-	}
-)
-
-type testPerRPCCredentials struct{}
-
-func (cr testPerRPCCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
-	return authdata, nil
-}
-
-func (cr testPerRPCCredentials) RequireTransportSecurity() bool {
-	return false
-}
-
-func authHandle(ctx context.Context, info *tap.Info) (context.Context, error) {
-	md, ok := metadata.FromIncomingContext(ctx)
-	if !ok {
-		return ctx, fmt.Errorf("didn't find metadata in context")
-	}
-	for k, vwant := range authdata {
-		vgot, ok := md[k]
-		if !ok {
-			return ctx, fmt.Errorf("didn't find authdata key %v in context", k)
-		}
-		if vgot[0] != vwant {
-			return ctx, fmt.Errorf("for key %v, got value %v, want %v", k, vgot, vwant)
-		}
-	}
-	return ctx, nil
-}
-
-func (s) TestPerRPCCredentialsViaDialOptions(t *testing.T) {
-	for _, e := range listTestEnv() {
-		testPerRPCCredentialsViaDialOptions(t, e)
-	}
-}
-
-func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) {
-	te := newTest(t, e)
-	te.tapHandle = authHandle
-	te.perRPCCreds = testPerRPCCredentials{}
-	te.startServer(&testServer{security: e.security})
-	defer te.tearDown()
-
-	cc := te.clientConn()
-	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
-		t.Fatalf("Test failed. Reason: %v", err)
-	}
-}
-
-func (s) TestPerRPCCredentialsViaCallOptions(t *testing.T) {
-	for _, e := range listTestEnv() {
-		testPerRPCCredentialsViaCallOptions(t, e)
-	}
-}
-
-func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) {
-	te := newTest(t, e)
-	te.tapHandle = authHandle
-	te.startServer(&testServer{security: e.security})
-	defer te.tearDown()
-
-	cc := te.clientConn()
-	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
-		t.Fatalf("Test failed. Reason: %v", err)
-	}
-}
-
-func (s) TestPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T) {
-	for _, e := range listTestEnv() {
-		testPerRPCCredentialsViaDialOptionsAndCallOptions(t, e)
-	}
-}
-
-func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) {
-	te := newTest(t, e)
-	te.perRPCCreds = testPerRPCCredentials{}
-	// When credentials are provided via both dial options and call options,
-	// we apply both sets.
-	te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) {
-		md, ok := metadata.FromIncomingContext(ctx)
-		if !ok {
-			return ctx, fmt.Errorf("couldn't find metadata in context")
-		}
-		for k, vwant := range authdata {
-			vgot, ok := md[k]
-			if !ok {
-				return ctx, fmt.Errorf("couldn't find metadata for key %v", k)
-			}
-			if len(vgot) != 2 {
-				return ctx, fmt.Errorf("len of value for key %v was %v, want 2", k, len(vgot))
-			}
-			if vgot[0] != vwant || vgot[1] != vwant {
-				return ctx, fmt.Errorf("value for %v was %v, want [%v, %v]", k, vgot, vwant, vwant)
-			}
-		}
-		return ctx, nil
-	}
-	te.startServer(&testServer{security: e.security})
-	defer te.tearDown()
-
-	cc := te.clientConn()
-	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
-		t.Fatalf("Test failed. Reason: %v", err)
-	}
-}
-
 func (s) TestWaitForReadyConnection(t *testing.T) {
 	for _, e := range listTestEnv() {
 		testWaitForReadyConnection(t, e)
@@ -6637,79 +6274,6 @@ func testClientDoesntDeadlockWhileWritingErrornousLargeMessages(t *testing.T, e
 	wg.Wait()
 }
 
-const clientAlwaysFailCredErrorMsg = "clientAlwaysFailCred always fails"
-
-var errClientAlwaysFailCred = errors.New(clientAlwaysFailCredErrorMsg)
-
-type clientAlwaysFailCred struct{}
-
-func (c clientAlwaysFailCred) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return nil, nil, errClientAlwaysFailCred
-}
-func (c clientAlwaysFailCred) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return rawConn, nil, nil
-}
-func (c clientAlwaysFailCred) Info() credentials.ProtocolInfo {
-	return credentials.ProtocolInfo{}
-}
-func (c clientAlwaysFailCred) Clone() credentials.TransportCredentials {
-	return nil
-}
-func (c clientAlwaysFailCred) OverrideServerName(s string) error {
-	return nil
-}
-
-func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
-	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "clientAlwaysFailCred", balancer: "round_robin"})
-	te.startServer(&testServer{security: te.e.security})
-	defer te.tearDown()
-
-	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	cc, err := grpc.DialContext(ctx, te.srvAddr, opts...)
-	if err != nil {
-		t.Fatalf("Dial(_) = %v, want %v", err, nil)
-	}
-	defer cc.Close()
-
-	tc := testpb.NewTestServiceClient(cc)
-	for i := 0; i < 1000; i++ {
-		// This loop runs for at most 1 second. The first several RPCs will fail
-		// with Unavailable because the connection hasn't started. When the
-		// first connection failed with creds error, the next RPC should also
-		// fail with the expected error.
-		if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
-			return
-		}
-		time.Sleep(time.Millisecond)
-	}
-	te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg)
-}
-
-func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) {
-	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "clientAlwaysFailCred", balancer: "round_robin"})
-	te.startServer(&testServer{security: te.e.security})
-	defer te.tearDown()
-
-	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
-	dctx, dcancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer dcancel()
-	cc, err := grpc.DialContext(dctx, te.srvAddr, opts...)
-	if err != nil {
-		t.Fatalf("Dial(_) = %v, want %v", err, nil)
-	}
-	defer cc.Close()
-
-	tc := testpb.NewTestServiceClient(cc)
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
-	defer cancel()
-	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
-		return
-	}
-	te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg)
-}
-
 func (s) TestRPCTimeout(t *testing.T) {
 	for _, e := range listTestEnv() {
 		testRPCTimeout(t, e)
@@ -6747,7 +6311,6 @@ func testRPCTimeout(t *testing.T, e env) {
 }
 
 func (s) TestDisabledIOBuffers(t *testing.T) {
-
 	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(60000))
 	if err != nil {
 		t.Fatalf("Failed to create payload: %v", err)
@@ -7439,33 +7002,6 @@ func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult {
 	return g
 }
 
-type methodTestCreds struct{}
-
-func (m methodTestCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
-	ri, _ := credentials.RequestInfoFromContext(ctx)
-	return nil, status.Errorf(codes.Unknown, ri.Method)
-}
-
-func (m methodTestCreds) RequireTransportSecurity() bool {
-	return false
-}
-
-func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
-	const wantMethod = "/grpc.testing.TestService/EmptyCall"
-	ss := &stubServer{}
-	if err := ss.Start(nil, grpc.WithPerRPCCredentials(methodTestCreds{})); err != nil {
-		t.Fatalf("Error starting endpoint server: %v", err)
-	}
-	defer ss.Stop()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancel()
-
-	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMethod {
-		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod)
-	}
-}
-
 func (s) TestClientCancellationPropagatesUnary(t *testing.T) {
 	wg := &sync.WaitGroup{}
 	called, done := make(chan struct{}), make(chan struct{})

From aff515dc67c304a402dca48a1c7cbfd9fef570a3 Mon Sep 17 00:00:00 2001
From: Timur Naurazbaev <nautim@ya.ru>
Date: Fri, 17 Apr 2020 18:49:40 +0300
Subject: [PATCH 030/481] env: check GRPC_GO_IGNORE_TXT_ERRORS env var (#3532)

---
 internal/envconfig/envconfig.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/envconfig/envconfig.go b/internal/envconfig/envconfig.go
index ae6c8972fd79..73931a94bcad 100644
--- a/internal/envconfig/envconfig.go
+++ b/internal/envconfig/envconfig.go
@@ -34,5 +34,5 @@ var (
 	// Retry is set if retry is explicitly enabled via "GRPC_GO_RETRY=on".
 	Retry = strings.EqualFold(os.Getenv(retryStr), "on")
 	// TXTErrIgnore is set if TXT errors should be ignored ("GRPC_GO_IGNORE_TXT_ERRORS" is not "false").
-	TXTErrIgnore = !strings.EqualFold(os.Getenv(retryStr), "false")
+	TXTErrIgnore = !strings.EqualFold(os.Getenv(txtErrIgnoreStr), "false")
 )

From 6774920a778abf4fe0454ba184cf0d4dcb359524 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Sat, 18 Apr 2020 02:59:34 +0800
Subject: [PATCH 031/481] xds: Modify return value of
 getClusterFromRouteConfiguration (#3534)

---
 xds/internal/client/v2client_rds.go      | 20 ++++++++++----------
 xds/internal/client/v2client_rds_test.go | 13 ++++++++++++-
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
index 14ae6cdfc37f..82140c9c43d0 100644
--- a/xds/internal/client/v2client_rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -48,9 +48,9 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, v2c.hostname)
 
 		// Use the hostname (resourceName for LDS) to find the routes.
-		cluster := getClusterFromRouteConfiguration(rc, hostname)
+		cluster, err := getClusterFromRouteConfiguration(rc, hostname)
 		if cluster == "" {
-			return fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v", rc)
+			return fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
 		}
 
 		// If we get here, it means that this resource was a good one.
@@ -62,7 +62,8 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 }
 
 // getClusterFromRouteConfiguration checks if the provided RouteConfiguration
-// meets the expected criteria. If so, it returns a non-empty clusterName.
+// meets the expected criteria. If so, it returns a non-empty clusterName with
+// nil error.
 //
 // A RouteConfiguration resource is considered valid when only if it contains a
 // VirtualHost whose domain field matches the server name from the URI passed
@@ -75,8 +76,7 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 // only look at the last route in the list (the default route), whose match
 // field must be empty and whose route field must be set.  Inside that route
 // message, the cluster field will contain the clusterName we are looking for.
-func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string) string {
-	// TODO: return error for better error logging and nack.
+func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string) (string, error) {
 	//
 	// Currently this returns "" on error, and the caller will return an error.
 	// But the error doesn't contain details of why the response is invalid
@@ -87,22 +87,22 @@ func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string)
 	vh := findBestMatchingVirtualHost(host, rc.GetVirtualHosts())
 	if vh == nil {
 		// No matching virtual host found.
-		return ""
+		return "", fmt.Errorf("no matching virtual host found")
 	}
 	if len(vh.Routes) == 0 {
 		// The matched virtual host has no routes, this is invalid because there
 		// should be at least one default route.
-		return ""
+		return "", fmt.Errorf("matched virtual host has no routes")
 	}
 	dr := vh.Routes[len(vh.Routes)-1]
 	if match := dr.GetMatch(); match == nil || (match.GetPrefix() != "" && match.GetPrefix() != "/") {
 		// The matched virtual host is invalid. Match is not "" or "/".
-		return ""
+		return "", fmt.Errorf("matched virtual host is invalid")
 	}
 	if route := dr.GetRoute(); route != nil {
-		return route.GetCluster()
+		return route.GetCluster(), nil
 	}
-	return ""
+	return "", fmt.Errorf("matched route is nil")
 }
 
 type domainMatchType int
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
index 8bbabb9828fc..f9936d793796 100644
--- a/xds/internal/client/v2client_rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -34,16 +34,19 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 		name        string
 		rc          *xdspb.RouteConfiguration
 		wantCluster string
+		wantError   bool
 	}{
 		{
 			name:        "no-virtual-hosts-in-rc",
 			rc:          emptyRouteConfig,
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name:        "no-domains-in-rc",
 			rc:          noDomainsInRouteConfig,
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name: "non-matching-domain-in-rc",
@@ -53,6 +56,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 				},
 			},
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name: "no-routes-in-rc",
@@ -62,6 +66,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 				},
 			},
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name: "default-route-match-field-is-nil",
@@ -82,6 +87,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 				},
 			},
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name: "default-route-match-field-is-non-nil",
@@ -99,6 +105,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 				},
 			},
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name: "default-route-routeaction-field-is-nil",
@@ -111,6 +118,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 				},
 			},
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name: "default-route-cluster-field-is-empty",
@@ -131,11 +139,13 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 				},
 			},
 			wantCluster: "",
+			wantError:   true,
 		},
 		{
 			name:        "good-route-config-with-empty-string-route",
 			rc:          goodRouteConfig1,
 			wantCluster: goodClusterName1,
+			wantError:   false,
 		},
 		{
 			// default route's match is not empty string, but "/".
@@ -156,7 +166,8 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if gotCluster := getClusterFromRouteConfiguration(test.rc, goodLDSTarget1); gotCluster != test.wantCluster {
+			gotCluster, gotError := getClusterFromRouteConfiguration(test.rc, goodLDSTarget1)
+			if gotCluster != test.wantCluster || (gotError != nil) != test.wantError {
 				t.Errorf("getClusterFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, goodLDSTarget1, gotCluster, test.wantCluster)
 			}
 		})

From da1517890dd2065ca3448fe9c479334b4f8890e4 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Sat, 18 Apr 2020 03:02:55 +0800
Subject: [PATCH 032/481] internal: move parseMethodName func to
 internal/grpcutil (#3525)

---
 internal/binarylog/binarylog.go               |  3 +-
 internal/binarylog/util_test.go               | 59 -------------------
 .../{binarylog/util.go => grpcutil/method.go} |  7 +--
 internal/grpcutil/method_test.go              | 46 +++++++++++++++
 4 files changed, 51 insertions(+), 64 deletions(-)
 delete mode 100644 internal/binarylog/util_test.go
 rename internal/{binarylog/util.go => grpcutil/method.go} (82%)
 create mode 100644 internal/grpcutil/method_test.go

diff --git a/internal/binarylog/binarylog.go b/internal/binarylog/binarylog.go
index 8b1051674917..b7a3dd8f918d 100644
--- a/internal/binarylog/binarylog.go
+++ b/internal/binarylog/binarylog.go
@@ -25,6 +25,7 @@ import (
 	"os"
 
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/grpcutil"
 )
 
 // Logger is the global binary logger. It can be used to get binary logger for
@@ -146,7 +147,7 @@ func (l *logger) setBlacklist(method string) error {
 // Each methodLogger returned by this method is a new instance. This is to
 // generate sequence id within the call.
 func (l *logger) getMethodLogger(methodName string) *MethodLogger {
-	s, m, err := parseMethodName(methodName)
+	s, m, err := grpcutil.ParseMethod(methodName)
 	if err != nil {
 		grpclog.Infof("binarylogging: failed to parse %q: %v", methodName, err)
 		return nil
diff --git a/internal/binarylog/util_test.go b/internal/binarylog/util_test.go
deleted file mode 100644
index 6e2d932195b9..000000000000
--- a/internal/binarylog/util_test.go
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- *
- * Copyright 2018 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package binarylog
-
-import "testing"
-
-func (s) TestParseMethodName(t *testing.T) {
-	testCases := []struct {
-		methodName      string
-		service, method string
-	}{
-		{methodName: "/s/m", service: "s", method: "m"},
-		{methodName: "/p.s/m", service: "p.s", method: "m"},
-		{methodName: "/p/s/m", service: "p/s", method: "m"},
-	}
-	for _, tc := range testCases {
-		s, m, err := parseMethodName(tc.methodName)
-		if err != nil {
-			t.Errorf("Parsing %q got error %v, want nil", tc.methodName, err)
-			continue
-		}
-		if s != tc.service || m != tc.method {
-			t.Errorf("Parseing %q got service %q, method %q, want service %q, method %q",
-				tc.methodName, s, m, tc.service, tc.method,
-			)
-		}
-	}
-}
-
-func (s) TestParseMethodNameInvalid(t *testing.T) {
-	testCases := []string{
-		"/",
-		"/sm",
-		"",
-		"sm",
-	}
-	for _, tc := range testCases {
-		_, _, err := parseMethodName(tc)
-		if err == nil {
-			t.Errorf("Parsing %q got nil error, want non-nil error", tc)
-		}
-	}
-}
diff --git a/internal/binarylog/util.go b/internal/grpcutil/method.go
similarity index 82%
rename from internal/binarylog/util.go
rename to internal/grpcutil/method.go
index 15dc7803d8bf..2c2ff7732a8f 100644
--- a/internal/binarylog/util.go
+++ b/internal/grpcutil/method.go
@@ -16,18 +16,17 @@
  *
  */
 
-package binarylog
+package grpcutil
 
 import (
 	"errors"
 	"strings"
 )
 
-// parseMethodName splits service and method from the input. It expects format
+// ParseMethod splits service and method from the input. It expects format
 // "/service/method".
 //
-// TODO: move to internal/grpcutil.
-func parseMethodName(methodName string) (service, method string, _ error) {
+func ParseMethod(methodName string) (service, method string, _ error) {
 	if !strings.HasPrefix(methodName, "/") {
 		return "", "", errors.New("invalid method name: should start with /")
 	}
diff --git a/internal/grpcutil/method_test.go b/internal/grpcutil/method_test.go
new file mode 100644
index 000000000000..9880e1c43293
--- /dev/null
+++ b/internal/grpcutil/method_test.go
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpcutil
+
+import (
+	"testing"
+)
+
+func TestParseMethod(t *testing.T) {
+	testCases := []struct {
+		methodName  string
+		wantService string
+		wantMethod  string
+		wantError   bool
+	}{
+		{methodName: "/s/m", wantService: "s", wantMethod: "m", wantError: false},
+		{methodName: "/p.s/m", wantService: "p.s", wantMethod: "m", wantError: false},
+		{methodName: "/p/s/m", wantService: "p/s", wantMethod: "m", wantError: false},
+		{methodName: "/", wantError: true},
+		{methodName: "/sm", wantError: true},
+		{methodName: "", wantError: true},
+		{methodName: "sm", wantError: true},
+	}
+	for _, tc := range testCases {
+		s, m, err := ParseMethod(tc.methodName)
+		if (err != nil) != tc.wantError || s != tc.wantService || m != tc.wantMethod {
+			t.Errorf("ParseMethod(%s) = (%s, %s, %v), want (%s, %s, %v)", tc.methodName, s, m, err, tc.wantService, tc.wantMethod, tc.wantError)
+		}
+	}
+}

From f67e7c03dcbcfc81906f302a87ab0d400738877d Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 17 Apr 2020 15:47:55 -0700
Subject: [PATCH 033/481] Make vet happy on non_linux platforms. (#3549)

---
 internal/syscall/syscall_nonlinux.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go
index d3fd9dab3331..ae0a9117e7b9 100644
--- a/internal/syscall/syscall_nonlinux.go
+++ b/internal/syscall/syscall_nonlinux.go
@@ -18,6 +18,8 @@
  *
  */
 
+// Package syscall provides functionalities that grpc uses to get low-level
+// operating system stats/info.
 package syscall
 
 import (

From 03def2a1f3d4fd22fb566d561195c31330984c29 Mon Sep 17 00:00:00 2001
From: Patrice Chalin <chalin@users.noreply.github.com>
Date: Tue, 21 Apr 2020 13:55:54 -0400
Subject: [PATCH 034/481] examples: add go.mod to make examples a separate
 module (#3546)

---
 examples/examples_test.sh | 10 +++---
 examples/go.mod           | 11 ++++++
 examples/go.sum           | 73 +++++++++++++++++++++++++++++++++++++++
 go.mod                    |  1 -
 4 files changed, 90 insertions(+), 5 deletions(-)
 create mode 100644 examples/go.mod
 create mode 100644 examples/go.sum

diff --git a/examples/examples_test.sh b/examples/examples_test.sh
index dcafe401c97e..a382b18c8276 100755
--- a/examples/examples_test.sh
+++ b/examples/examples_test.sh
@@ -90,18 +90,20 @@ declare -A EXPECTED_CLIENT_OUTPUT=(
     ["features/name_resolving"]="calling helloworld.Greeter/SayHello to \"example:///resolver.example.grpc.io\""
 )
 
+cd ./examples
+
 for example in ${EXAMPLES[@]}; do
     echo "$(tput setaf 4) testing: ${example} $(tput sgr 0)"
 
     # Build server
-    if ! go build -o /dev/null ./examples/${example}/*server/*.go; then
+    if ! go build -o /dev/null ./${example}/*server/*.go; then
         fail "failed to build server"
     else
         pass "successfully built server"
     fi
 
     # Build client
-    if ! go build -o /dev/null ./examples/${example}/*client/*.go; then
+    if ! go build -o /dev/null ./${example}/*client/*.go; then
         fail "failed to build client"
     else
         pass "successfully built client"
@@ -109,10 +111,10 @@ for example in ${EXAMPLES[@]}; do
 
     # Start server
     SERVER_LOG="$(mktemp)"
-    go run ./examples/$example/*server/*.go &> $SERVER_LOG  &
+    go run ./$example/*server/*.go &> $SERVER_LOG  &
 
     CLIENT_LOG="$(mktemp)"
-    if ! timeout 20 go run examples/${example}/*client/*.go &> $CLIENT_LOG; then
+    if ! timeout 20 go run ${example}/*client/*.go &> $CLIENT_LOG; then
         fail "client failed to communicate with server
         got server log:
         $(cat $SERVER_LOG)
diff --git a/examples/go.mod b/examples/go.mod
new file mode 100644
index 000000000000..8be62bc7b4a3
--- /dev/null
+++ b/examples/go.mod
@@ -0,0 +1,11 @@
+module google.golang.org/grpc/examples
+
+go 1.11
+
+require (
+	github.com/golang/mock v1.1.1
+	github.com/golang/protobuf v1.4.0
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
+	google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55
+	google.golang.org/grpc v1.30.0-dev.1
+)
diff --git a/examples/go.sum b/examples/go.sum
new file mode 100644
index 000000000000..0d649b1f5ae2
--- /dev/null
+++ b/examples/go.sum
@@ -0,0 +1,73 @@
+cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/go.mod b/go.mod
index dfceabe15ae7..31f2b01f64e8 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,6 @@ require (
 	github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f
 	github.com/envoyproxy/go-control-plane v0.9.4
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/mock v1.1.1
 	github.com/golang/protobuf v1.3.3
 	github.com/google/go-cmp v0.4.0
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a

From 843b06d54916514c9f1a9a4639a15182048449a8 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 22 Apr 2020 12:00:57 -0700
Subject: [PATCH 035/481] benchmark: Fix segfault in benchmark code. (#3555)

If a benchmark run did not specify req/resp payload curve files, and the
result from that run is used on the `benchresult` binary, it will
segfault because there is no check for the presence of the payload curve
option before printing the set of features used. Added a sanity check
there.
---
 benchmark/stats/stats.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/benchmark/stats/stats.go b/benchmark/stats/stats.go
index f2f6a08878cb..6275c3c3a71c 100644
--- a/benchmark/stats/stats.go
+++ b/benchmark/stats/stats.go
@@ -180,9 +180,13 @@ func (f Features) partialString(b *bytes.Buffer, wantFeatures []bool, sep, delim
 			case RespSizeBytesIndex:
 				b.WriteString(fmt.Sprintf("RespSize%v%vB%v", sep, f.RespSizeBytes, delim))
 			case ReqPayloadCurveIndex:
-				b.WriteString(fmt.Sprintf("ReqPayloadCurve%vSHA-256:%v%v", sep, f.ReqPayloadCurve.Hash(), delim))
+				if f.ReqPayloadCurve != nil {
+					b.WriteString(fmt.Sprintf("ReqPayloadCurve%vSHA-256:%v%v", sep, f.ReqPayloadCurve.Hash(), delim))
+				}
 			case RespPayloadCurveIndex:
-				b.WriteString(fmt.Sprintf("RespPayloadCurve%vSHA-256:%v%v", sep, f.RespPayloadCurve.Hash(), delim))
+				if f.RespPayloadCurve != nil {
+					b.WriteString(fmt.Sprintf("RespPayloadCurve%vSHA-256:%v%v", sep, f.RespPayloadCurve.Hash(), delim))
+				}
 			case CompModesIndex:
 				b.WriteString(fmt.Sprintf("Compressor%v%v%v", sep, f.ModeCompressor, delim))
 			case EnableChannelzIndex:

From f313ade84ce22334b1411af48361ab7fcf1c91df Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Wed, 22 Apr 2020 13:44:18 -0700
Subject: [PATCH 036/481] advancedtls: add fine-grained verification levels in
 XXXOptions (#3454)

---
 security/advancedtls/advancedtls.go           | 310 ++++++++++--------
 .../advancedtls_integration_test.go           | 168 +++++++---
 security/advancedtls/advancedtls_test.go      | 131 ++++++--
 3 files changed, 413 insertions(+), 196 deletions(-)

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index eec489c0b084..529ecc76f285 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -17,8 +17,8 @@
  */
 
 // Package advancedtls is a utility library containing functions to construct
-// credentials.TransportCredentials that can perform credential reloading and custom
-// server authorization.
+// credentials.TransportCredentials that can perform credential reloading and
+// custom verification check.
 package advancedtls
 
 import (
@@ -33,98 +33,155 @@ import (
 	"google.golang.org/grpc/credentials"
 )
 
-// VerificationFuncParams contains the parameters available to users when implementing CustomVerificationFunc.
+// VerificationFuncParams contains parameters available to users when
+// implementing CustomVerificationFunc.
+// The fields in this struct are read-only.
 type VerificationFuncParams struct {
-	ServerName     string
-	RawCerts       [][]byte
+	// The target server name that the client connects to when establishing the
+	// connection. This field is only meaningful for client side. On server side,
+	// this field would be an empty string.
+	ServerName string
+	// The raw certificates sent from peer.
+	RawCerts [][]byte
+	// The verification chain obtained by checking peer RawCerts against the
+	// trust certificate bundle(s), if applicable.
 	VerifiedChains [][]*x509.Certificate
 }
 
-// VerificationResults contains the information about results of CustomVerificationFunc.
-// VerificationResults is an empty struct for now. It may be extended in the future to include more information.
+// VerificationResults contains the information about results of
+// CustomVerificationFunc.
+// VerificationResults is an empty struct for now. It may be extended in the
+// future to include more information.
 type VerificationResults struct{}
 
-// CustomVerificationFunc is the function defined by users to perform custom server authorization.
-// CustomVerificationFunc returns nil if the authorization fails; otherwise returns an empty struct.
+// CustomVerificationFunc is the function defined by users to perform custom
+// verification check.
+// CustomVerificationFunc returns nil if the authorization fails; otherwise
+// returns an empty struct.
 type CustomVerificationFunc func(params *VerificationFuncParams) (*VerificationResults, error)
 
-// GetRootCAsParams contains the parameters available to users when implementing GetRootCAs.
+// GetRootCAsParams contains the parameters available to users when
+// implementing GetRootCAs.
 type GetRootCAsParams struct {
 	RawConn  net.Conn
 	RawCerts [][]byte
 }
 
 // GetRootCAsResults contains the results of GetRootCAs.
-// If users want to reload the root trust certificate, it is required to return the proper TrustCerts in GetRootCAs.
+// If users want to reload the root trust certificate, it is required to return
+// the proper TrustCerts in GetRootCAs.
 type GetRootCAsResults struct {
 	TrustCerts *x509.CertPool
 }
 
-// RootCertificateOptions contains a field and a function for obtaining root trust certificates.
-// It is used by both ClientOptions and ServerOptions. Note that RootCertificateOptions is required
-// to be correctly set on client side; on server side, it is only required when mutual TLS is
-// enabled(RequireClientCert in ServerOptions is true).
+// RootCertificateOptions contains a field and a function for obtaining root
+// trust certificates.
+// It is used by both ClientOptions and ServerOptions.
 type RootCertificateOptions struct {
-	// If field RootCACerts is set, field GetRootCAs will be ignored. RootCACerts will be used
-	// every time when verifying the peer certificates, without performing root certificate reloading.
+	// If field RootCACerts is set, field GetRootCAs will be ignored. RootCACerts
+	// will be used every time when verifying the peer certificates, without
+	// performing root certificate reloading.
 	RootCACerts *x509.CertPool
-	// If GetRootCAs is set and RootCACerts is nil, GetRootCAs will be invoked every time
-	// asked to check certificates sent from the server when a new connection is established.
+	// If GetRootCAs is set and RootCACerts is nil, GetRootCAs will be invoked
+	// every time asked to check certificates sent from the server when a new
+	// connection is established.
 	// This is known as root CA certificate reloading.
 	GetRootCAs func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 }
 
-// ClientOptions contains all the fields and functions needed to be filled by the client.
+// VerificationType is the enum type that represents different levels of
+// verification users could set, both on client side and on server side.
+type VerificationType int
+
+const (
+	// CertAndHostVerification indicates doing both certificate signature check
+	// and hostname check.
+	CertAndHostVerification VerificationType = iota
+	// CertVerification indicates doing certificate signature check only. Setting
+	// this field without proper custom verification check would leave the
+	// application susceptible to the MITM attack.
+	CertVerification
+	// SkipVerification indicates skipping both certificate signature check and
+	// hostname check. If setting this field, proper custom verification needs to
+	// be implemented in order to complete the authentication. Setting this field
+	// with a nil custom verification would raise an error.
+	SkipVerification
+)
+
+// ClientOptions contains all the fields and functions needed to be filled by
+// the client.
 // General rules for certificate setting on client side:
-// Certificates or GetClientCertificate indicates the certificates sent from the client to the
-// server to prove client's identities. The rules for setting these two fields are:
+// Certificates or GetClientCertificate indicates the certificates sent from
+// the client to the server to prove client's identities. The rules for setting
+// these two fields are:
 // If requiring mutual authentication on server side:
-//     Either Certificates or GetClientCertificate must be set; the other will be ignored
+//     Either Certificates or GetClientCertificate must be set; the other will
+//     be ignored.
 // Otherwise:
-//     Nothing needed(the two fields will be ignored)
+//     Nothing needed(the two fields will be ignored).
 type ClientOptions struct {
-	// If field Certificates is set, field GetClientCertificate will be ignored. The client will use
-	// Certificates every time when asked for a certificate, without performing certificate reloading.
+	// If field Certificates is set, field GetClientCertificate will be ignored.
+	// The client will use Certificates every time when asked for a certificate,
+	// without performing certificate reloading.
 	Certificates []tls.Certificate
-	// If GetClientCertificate is set and Certificates is nil, the client will invoke this
-	// function every time asked to present certificates to the server when a new connection is
-	// established. This is known as peer certificate reloading.
+	// If GetClientCertificate is set and Certificates is nil, the client will
+	// invoke this function every time asked to present certificates to the
+	// server when a new connection is established. This is known as peer
+	// certificate reloading.
 	GetClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
-	// VerifyPeer is a custom server authorization checking after certificate signature check.
-	// If this is set, we will replace the hostname check with this customized authorization check.
-	// If this is nil, we fall back to typical hostname check.
+	// VerifyPeer is a custom verification check after certificate signature
+	// check.
+	// If this is set, we will perform this customized check after doing the
+	// normal check(s) indicated by setting VType.
 	VerifyPeer CustomVerificationFunc
 	// ServerNameOverride is for testing only. If set to a non-empty string,
-	// it will override the virtual host name of authority (e.g. :authority header field) in requests.
+	// it will override the virtual host name of authority (e.g. :authority
+	// header field) in requests.
 	ServerNameOverride string
+	// RootCertificateOptions is REQUIRED to be correctly set on client side.
 	RootCertificateOptions
+	// VType is the verification type on the client side.
+	VType VerificationType
 }
 
-// ServerOptions contains all the fields and functions needed to be filled by the client.
+// ServerOptions contains all the fields and functions needed to be filled by
+// the client.
 // General rules for certificate setting on server side:
-// Certificates or GetClientCertificate indicates the certificates sent from the server to
-// the client to prove server's identities. The rules for setting these two fields are:
-// Either Certificates or GetCertificate must be set; the other will be ignored
+// Certificates or GetClientCertificate indicates the certificates sent from
+// the server to the client to prove server's identities. The rules for setting
+// these two fields are:
+// Either Certificates or GetCertificate must be set; the other will be ignored.
 type ServerOptions struct {
-	// If field Certificates is set, field GetClientCertificate will be ignored. The server will use
-	// Certificates every time when asked for a certificate, without performing certificate reloading.
+	// If field Certificates is set, field GetClientCertificate will be ignored.
+	// The server will use Certificates every time when asked for a certificate,
+	// without performing certificate reloading.
 	Certificates []tls.Certificate
-	// If GetClientCertificate is set and Certificates is nil, the server will invoke this
-	// function every time asked to present certificates to the client when a new connection is
-	// established. This is known as peer certificate reloading.
+	// If GetClientCertificate is set and Certificates is nil, the server will
+	// invoke this function every time asked to present certificates to the
+	// client when a new connection is established. This is known as peer
+	// certificate reloading.
 	GetCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error)
+	// VerifyPeer is a custom verification check after certificate signature
+	// check.
+	// If this is set, we will perform this customized check after doing the
+	// normal check(s) indicated by setting VType.
+	VerifyPeer CustomVerificationFunc
+	// RootCertificateOptions is only required when mutual TLS is
+	// enabled(RequireClientCert is true).
 	RootCertificateOptions
 	// If the server want the client to send certificates.
 	RequireClientCert bool
+	// VType is the verification type on the server side.
+	VType VerificationType
 }
 
 func (o *ClientOptions) config() (*tls.Config, error) {
-	if o.RootCACerts == nil && o.GetRootCAs == nil && o.VerifyPeer == nil {
+	if o.VType == SkipVerification && o.VerifyPeer == nil {
 		return nil, fmt.Errorf(
-			"client needs to provide root CA certs, or a custom verification function")
+			"client needs to provide custom verification mechanism if choose to skip default verification")
 	}
-	// We have to set InsecureSkipVerify to true to skip the default checks and use the
-	// verification function we built from buildVerifyFunc.
+	// We have to set InsecureSkipVerify to true to skip the default checks and
+	// use the verification function we built from buildVerifyFunc.
 	config := &tls.Config{
 		ServerName:           o.ServerNameOverride,
 		Certificates:         o.Certificates,
@@ -139,21 +196,16 @@ func (o *ServerOptions) config() (*tls.Config, error) {
 	if o.Certificates == nil && o.GetCertificate == nil {
 		return nil, fmt.Errorf("either Certificates or GetCertificate must be specified")
 	}
-	if o.RequireClientCert && o.GetRootCAs == nil && o.RootCACerts == nil {
-		return nil, fmt.Errorf("server needs to provide root CA certs if requiring client cert")
+	if o.RequireClientCert && o.VType == SkipVerification && o.VerifyPeer == nil {
+		return nil, fmt.Errorf(
+			"server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)")
 	}
 	clientAuth := tls.NoClientCert
 	if o.RequireClientCert {
-		// We fall back to normal config settings if users don't need to reload root certificates.
-		// If using RequireAndVerifyClientCert, the underlying stack would use the default
-		// checking and ignore the verification function we built from buildVerifyFunc.
-		// If using RequireAnyClientCert, the code would skip all the checks and use the
-		// function from buildVerifyFunc.
-		if o.RootCACerts != nil {
-			clientAuth = tls.RequireAndVerifyClientCert
-		} else {
-			clientAuth = tls.RequireAnyClientCert
-		}
+		// We have to set clientAuth to RequireAnyClientCert to force underlying
+		// TLS package to use the verification function we built from
+		// buildVerifyFunc.
+		clientAuth = tls.RequireAnyClientCert
 	}
 	config := &tls.Config{
 		ClientAuth:     clientAuth,
@@ -166,12 +218,14 @@ func (o *ServerOptions) config() (*tls.Config, error) {
 	return config, nil
 }
 
-// advancedTLSCreds is the credentials required for authenticating a connection using TLS.
+// advancedTLSCreds is the credentials required for authenticating a connection
+// using TLS.
 type advancedTLSCreds struct {
 	config     *tls.Config
 	verifyFunc CustomVerificationFunc
 	getRootCAs func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 	isClient   bool
+	vType      VerificationType
 }
 
 func (c advancedTLSCreds) Info() credentials.ProtocolInfo {
@@ -218,10 +272,7 @@ func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string
 
 func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
 	cfg := cloneTLSConfig(c.config)
-	// We build server side verification function only when root cert reloading is needed.
-	if c.getRootCAs != nil {
-		cfg.VerifyPeerCertificate = buildVerifyFunc(c, "", rawConn)
-	}
+	cfg.VerifyPeerCertificate = buildVerifyFunc(c, "", rawConn)
 	conn := tls.Server(rawConn, cfg)
 	if err := conn.Handshake(); err != nil {
 		conn.Close()
@@ -250,91 +301,82 @@ func (c *advancedTLSCreds) OverrideServerName(serverNameOverride string) error {
 	return nil
 }
 
-// The function buildVerifyFunc is used when users want root cert reloading, and possibly custom
-// server authorization check.
-// We have to build our own verification function here because current tls module:
-// 1. does not have a good support on root cert reloading
-// 2. will ignore basic certificate check when setting InsecureSkipVerify to true
+// The function buildVerifyFunc is used when users want root cert reloading,
+// and possibly custom verification check.
+// We have to build our own verification function here because current
+// tls module:
+//   1. does not have a good support on root cert reloading.
+//   2. will ignore basic certificate check when setting InsecureSkipVerify
+//   to true.
 func buildVerifyFunc(c *advancedTLSCreds,
 	serverName string,
 	rawConn net.Conn) func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
 	return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
-		// If users didn't specify either rootCAs or getRootCAs on client side,
-		// as we see some use cases such as https://github.com/grpc/grpc/pull/20530,
-		// instead of failing, we just don't validate the server cert and let
-		// application decide via VerifyPeer
-		if c.isClient && c.config.RootCAs == nil && c.getRootCAs == nil {
-			if c.verifyFunc != nil {
-				_, err := c.verifyFunc(&VerificationFuncParams{
-					ServerName:     serverName,
-					RawCerts:       rawCerts,
-					VerifiedChains: verifiedChains,
+		chains := verifiedChains
+		if c.vType == CertAndHostVerification || c.vType == CertVerification {
+			// perform possible trust credential reloading and certificate check
+			rootCAs := c.config.RootCAs
+			if !c.isClient {
+				rootCAs = c.config.ClientCAs
+			}
+			// Reload root CA certs.
+			if rootCAs == nil && c.getRootCAs != nil {
+				results, err := c.getRootCAs(&GetRootCAsParams{
+					RawConn:  rawConn,
+					RawCerts: rawCerts,
 				})
-				return err
+				if err != nil {
+					return err
+				}
+				rootCAs = results.TrustCerts
 			}
-		}
-		var rootCAs *x509.CertPool
-		if c.isClient {
-			rootCAs = c.config.RootCAs
-		} else {
-			rootCAs = c.config.ClientCAs
-		}
-		// reload root CA certs
-		if rootCAs == nil && c.getRootCAs != nil {
-			results, err := c.getRootCAs(&GetRootCAsParams{
-				RawConn:  rawConn,
-				RawCerts: rawCerts,
-			})
-			if err != nil {
-				return err
+			// Verify peers' certificates against RootCAs and get verifiedChains.
+			certs := make([]*x509.Certificate, len(rawCerts))
+			for i, asn1Data := range rawCerts {
+				cert, err := x509.ParseCertificate(asn1Data)
+				if err != nil {
+					return err
+				}
+				certs[i] = cert
 			}
-			rootCAs = results.TrustCerts
-		}
-		// verify peers' certificates against RootCAs and get verifiedChains
-		certs := make([]*x509.Certificate, len(rawCerts))
-		for i, asn1Data := range rawCerts {
-			cert, err := x509.ParseCertificate(asn1Data)
+			keyUsages := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
+			if !c.isClient {
+				keyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
+			}
+			opts := x509.VerifyOptions{
+				Roots:         rootCAs,
+				CurrentTime:   time.Now(),
+				Intermediates: x509.NewCertPool(),
+				KeyUsages:     keyUsages,
+			}
+			for _, cert := range certs[1:] {
+				opts.Intermediates.AddCert(cert)
+			}
+			// Perform default hostname check if specified.
+			if c.isClient && c.vType == CertAndHostVerification && serverName != "" {
+				opts.DNSName = serverName
+			}
+			var err error
+			chains, err = certs[0].Verify(opts)
 			if err != nil {
 				return err
 			}
-			certs[i] = cert
-		}
-		opts := x509.VerifyOptions{
-			Roots:         rootCAs,
-			CurrentTime:   time.Now(),
-			Intermediates: x509.NewCertPool(),
-		}
-		if !c.isClient {
-			opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
-		} else {
-			opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
-		}
-		for _, cert := range certs[1:] {
-			opts.Intermediates.AddCert(cert)
-		}
-		// We use default hostname check if users don't specify verifyFunc function
-		if c.isClient && c.verifyFunc == nil && serverName != "" {
-			opts.DNSName = serverName
 		}
-		verifiedChains, err := certs[0].Verify(opts)
-		if err != nil {
+		// Perform custom verification check if specified.
+		if c.verifyFunc != nil {
+			_, err := c.verifyFunc(&VerificationFuncParams{
+				ServerName:     serverName,
+				RawCerts:       rawCerts,
+				VerifiedChains: chains,
+			})
 			return err
 		}
-		if c.isClient && c.verifyFunc != nil {
-			if c.verifyFunc != nil {
-				_, err := c.verifyFunc(&VerificationFuncParams{
-					ServerName:     serverName,
-					RawCerts:       rawCerts,
-					VerifiedChains: verifiedChains,
-				})
-				return err
-			}
-		}
 		return nil
 	}
 }
 
-// NewClientCreds uses ClientOptions to construct a TransportCredentials based on TLS.
+// NewClientCreds uses ClientOptions to construct a TransportCredentials based
+// on TLS.
 func NewClientCreds(o *ClientOptions) (credentials.TransportCredentials, error) {
 	conf, err := o.config()
 	if err != nil {
@@ -345,12 +387,14 @@ func NewClientCreds(o *ClientOptions) (credentials.TransportCredentials, error)
 		isClient:   true,
 		getRootCAs: o.GetRootCAs,
 		verifyFunc: o.VerifyPeer,
+		vType:      o.VType,
 	}
 	tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos)
 	return tc, nil
 }
 
-// NewServerCreds uses ServerOptions to construct a TransportCredentials based on TLS.
+// NewServerCreds uses ServerOptions to construct a TransportCredentials based
+// on TLS.
 func NewServerCreds(o *ServerOptions) (credentials.TransportCredentials, error) {
 	conf, err := o.config()
 	if err != nil {
@@ -360,6 +404,8 @@ func NewServerCreds(o *ServerOptions) (credentials.TransportCredentials, error)
 		config:     conf,
 		isClient:   false,
 		getRootCAs: o.GetRootCAs,
+		verifyFunc: o.VerifyPeer,
+		vType:      o.VType,
 	}
 	tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos)
 	return tc, nil
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 0a7efe435a56..c259e1146552 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -39,9 +39,11 @@ var (
 	port    = ":50051"
 )
 
-// stageInfo contains a stage number indicating the current phase of each integration test, and a mutex.
-// Based on the stage number of current test, we will use different certificates and server authorization
-// functions to check if our tests behave as expected.
+// stageInfo contains a stage number indicating the current phase of each
+// integration test, and a mutex.
+// Based on the stage number of current test, we will use different
+// certificates and custom verification functions to check if our tests behave
+// as expected.
 type stageInfo struct {
 	mutex sync.Mutex
 	stage int
@@ -67,13 +69,17 @@ func (s *stageInfo) reset() {
 
 // certStore contains all the certificates used in the integration tests.
 type certStore struct {
-	// clientPeer1 is the certificate sent by client to prove its identity. It is trusted by serverTrust1.
+	// clientPeer1 is the certificate sent by client to prove its identity.
+	// It is trusted by serverTrust1.
 	clientPeer1 tls.Certificate
-	// clientPeer2 is the certificate sent by client to prove its identity. It is trusted by serverTrust2.
+	// clientPeer2 is the certificate sent by client to prove its identity.
+	// It is trusted by serverTrust2.
 	clientPeer2 tls.Certificate
-	// serverPeer1 is the certificate sent by server to prove its identity. It is trusted by clientTrust1.
+	// serverPeer1 is the certificate sent by server to prove its identity.
+	// It is trusted by clientTrust1.
 	serverPeer1 tls.Certificate
-	// serverPeer2 is the certificate sent by server to prove its identity. It is trusted by clientTrust2.
+	// serverPeer2 is the certificate sent by server to prove its identity.
+	// It is trusted by clientTrust2.
 	serverPeer2  tls.Certificate
 	clientTrust1 *x509.CertPool
 	clientTrust2 *x509.CertPool
@@ -81,7 +87,8 @@ type certStore struct {
 	serverTrust2 *x509.CertPool
 }
 
-// loadCerts function is used to load test certificates at the beginning of each integration test.
+// loadCerts function is used to load test certificates at the beginning of
+// each integration test.
 func (cs *certStore) loadCerts() error {
 	var err error
 	cs.clientPeer1, err = tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"),
@@ -144,7 +151,8 @@ func callAndVerify(msg string, client pb.GreeterClient, shouldFail bool) error {
 func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) {
 	var conn *grpc.ClientConn
 	var err error
-	// If we want the test to fail, we establish a non-blocking connection to avoid it hangs and killed by the context.
+	// If we want the test to fail, we establish a non-blocking connection to
+	// avoid it hangs and killed by the context.
 	if shouldFail {
 		conn, err = grpc.DialContext(connCtx, address, grpc.WithTransportCredentials(creds))
 		if err != nil {
@@ -166,10 +174,13 @@ func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds cred
 
 // The advanced TLS features are tested in different stages.
 // At stage 0, we establish a good connection between client and server.
-// At stage 1, we change one factor(it could be we change the server's certificate, or server authorization function, etc),
-// and test if the following connections would be dropped.
-// At stage 2, we re-establish the connection by changing the counterpart of the factor we modified in stage 1.
-// (could be change the client's trust certificate, or change server authorization function, etc)
+// At stage 1, we change one factor(it could be we change the server's
+// certificate, or custom verification function, etc), and test if the
+// following connections would be dropped.
+// At stage 2, we re-establish the connection by changing the counterpart of
+// the factor we modified in stage 1.
+// (could be change the client's trust certificate, or change custom
+// verification function, etc)
 func TestEnd2End(t *testing.T) {
 	cs := &certStore{}
 	err := cs.loadCerts()
@@ -184,17 +195,25 @@ func TestEnd2End(t *testing.T) {
 		clientRoot       *x509.CertPool
 		clientGetRoot    func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		clientVerifyFunc CustomVerificationFunc
+		clientVType      VerificationType
 		serverCert       []tls.Certificate
 		serverGetCert    func(*tls.ClientHelloInfo) (*tls.Certificate, error)
 		serverRoot       *x509.CertPool
 		serverGetRoot    func(params *GetRootCAsParams) (*GetRootCAsResults, error)
+		serverVerifyFunc CustomVerificationFunc
+		serverVType      VerificationType
 	}{
 		// Test Scenarios:
-		// At initialization(stage = 0), client will be initialized with cert clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1 is trusted by serverTrust1, and serverPeer1 by clientTrust1.
-		// At stage 1, client changes clientPeer1 to clientPeer2. Since clientPeer2 is not trusted by serverTrust1, following rpc calls are expected
-		// to fail, while the previous rpc calls are still good because those are already authenticated.
-		// At stage 2, the server changes serverTrust1 to serverTrust2, and we should see it again accepts the connection, since clientPeer2 is trusted
+		// At initialization(stage = 0), client will be initialized with cert
+		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
+		// The mutual authentication works at the beginning, since clientPeer1 is
+		// trusted by serverTrust1, and serverPeer1 by clientTrust1.
+		// At stage 1, client changes clientPeer1 to clientPeer2. Since clientPeer2
+		// is not trusted by serverTrust1, following rpc calls are expected to
+		// fail, while the previous rpc calls are still good because those are
+		// already authenticated.
+		// At stage 2, the server changes serverTrust1 to serverTrust2, and we
+		// should see it again accepts the connection, since clientPeer2 is trusted
 		// by serverTrust2.
 		{
 			desc:       "TestClientPeerCertReloadServerTrustCertReload",
@@ -212,6 +231,7 @@ func TestEnd2End(t *testing.T) {
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
+			clientVType:   CertVerification,
 			serverCert:    []tls.Certificate{cs.serverPeer1},
 			serverGetCert: nil,
 			serverRoot:    nil,
@@ -223,13 +243,22 @@ func TestEnd2End(t *testing.T) {
 					return &GetRootCAsResults{TrustCerts: cs.serverTrust2}, nil
 				}
 			},
+			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
+				return &VerificationResults{}, nil
+			},
+			serverVType: CertVerification,
 		},
 		// Test Scenarios:
-		// At initialization(stage = 0), client will be initialized with cert clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1 is trusted by serverTrust1, and serverPeer1 by clientTrust1.
-		// At stage 1, server changes serverPeer1 to serverPeer2. Since serverPeer2 is not trusted by clientTrust1, following rpc calls are expected
-		// to fail, while the previous rpc calls are still good because those are already authenticated.
-		// At stage 2, the client changes clientTrust1 to clientTrust2, and we should see it again accepts the connection, since serverPeer2 is trusted
+		// At initialization(stage = 0), client will be initialized with cert
+		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
+		// The mutual authentication works at the beginning, since clientPeer1 is
+		// trusted by serverTrust1, and serverPeer1 by clientTrust1.
+		// At stage 1, server changes serverPeer1 to serverPeer2. Since serverPeer2
+		// is not trusted by clientTrust1, following rpc calls are expected to
+		// fail, while the previous rpc calls are still good because those are
+		// already authenticated.
+		// At stage 2, the client changes clientTrust1 to clientTrust2, and we
+		// should see it again accepts the connection, since serverPeer2 is trusted
 		// by clientTrust2.
 		{
 			desc:          "TestServerPeerCertReloadClientTrustCertReload",
@@ -247,7 +276,8 @@ func TestEnd2End(t *testing.T) {
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
-			serverCert: nil,
+			clientVType: CertVerification,
+			serverCert:  nil,
 			serverGetCert: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
@@ -258,17 +288,26 @@ func TestEnd2End(t *testing.T) {
 			},
 			serverRoot:    cs.serverTrust1,
 			serverGetRoot: nil,
+			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
+				return &VerificationResults{}, nil
+			},
+			serverVType: CertVerification,
 		},
 		// Test Scenarios:
-		// At initialization(stage = 0), client will be initialized with cert clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1 trusted by serverTrust1, serverPeer1 by clientTrust1, and also the
-		// custom server authorization check allows the CommonName on serverPeer1.
-		// At stage 1, server changes serverPeer1 to serverPeer2, and client changes clientTrust1 to clientTrust2. Although serverPeer2 is trusted by
-		// clientTrust2, our authorization check only accepts serverPeer1, and hence the following calls should fail. Previous connections should
+		// At initialization(stage = 0), client will be initialized with cert
+		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
+		// The mutual authentication works at the beginning, since clientPeer1
+		// trusted by serverTrust1, serverPeer1 by clientTrust1, and also the
+		// custom verification check allows the CommonName on serverPeer1.
+		// At stage 1, server changes serverPeer1 to serverPeer2, and client
+		// changes clientTrust1 to clientTrust2. Although serverPeer2 is trusted by
+		// clientTrust2, our authorization check only accepts serverPeer1, and
+		// hence the following calls should fail. Previous connections should
 		// not be affected.
-		// At stage 2, the client changes authorization check to only accept serverPeer2. Now we should see the connection becomes normal again.
+		// At stage 2, the client changes authorization check to only accept
+		// serverPeer2. Now we should see the connection becomes normal again.
 		{
-			desc:          "TestClientCustomServerAuthz",
+			desc:          "TestClientCustomVerification",
 			clientCert:    []tls.Certificate{cs.clientPeer1},
 			clientGetCert: nil,
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
@@ -306,7 +345,8 @@ func TestEnd2End(t *testing.T) {
 				}
 				return nil, fmt.Errorf("custom authz check fails")
 			},
-			serverCert: nil,
+			clientVType: CertVerification,
+			serverCert:  nil,
 			serverGetCert: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
@@ -317,6 +357,47 @@ func TestEnd2End(t *testing.T) {
 			},
 			serverRoot:    cs.serverTrust1,
 			serverGetRoot: nil,
+			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
+				return &VerificationResults{}, nil
+			},
+			serverVType: CertVerification,
+		},
+		// Test Scenarios:
+		// At initialization(stage = 0), client will be initialized with cert
+		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
+		// The mutual authentication works at the beginning, since clientPeer1
+		// trusted by serverTrust1, serverPeer1 by clientTrust1, and also the
+		// custom verification check on server side allows all connections.
+		// At stage 1, server disallows the the connections by setting custom
+		// verification check. The following calls should fail. Previous
+		// connections should not be affected.
+		// At stage 2, server allows all the connections again and the
+		// authentications should go back to normal.
+		{
+			desc:          "TestServerCustomVerification",
+			clientCert:    []tls.Certificate{cs.clientPeer1},
+			clientGetCert: nil,
+			clientGetRoot: nil,
+			clientRoot:    cs.clientTrust1,
+			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
+				return &VerificationResults{}, nil
+			},
+			clientVType:   CertVerification,
+			serverCert:    []tls.Certificate{cs.serverPeer1},
+			serverGetCert: nil,
+			serverRoot:    cs.serverTrust1,
+			serverGetRoot: nil,
+			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
+				switch stage.read() {
+				case 0, 2:
+					return &VerificationResults{}, nil
+				case 1:
+					return nil, fmt.Errorf("custom authz check fails")
+				default:
+					return nil, fmt.Errorf("custom authz check fails")
+				}
+			},
+			serverVType: CertVerification,
 		},
 	} {
 		test := test
@@ -330,6 +411,8 @@ func TestEnd2End(t *testing.T) {
 					GetRootCAs:  test.serverGetRoot,
 				},
 				RequireClientCert: true,
+				VerifyPeer:        test.serverVerifyFunc,
+				VType:             test.serverVType,
 			}
 			serverTLSCreds, err := NewServerCreds(serverOptions)
 			if err != nil {
@@ -356,49 +439,50 @@ func TestEnd2End(t *testing.T) {
 					RootCACerts: test.clientRoot,
 					GetRootCAs:  test.clientGetRoot,
 				},
+				VType: test.clientVType,
 			}
 			clientTLSCreds, err := NewClientCreds(clientOptions)
 			if err != nil {
 				t.Fatalf("clientTLSCreds failed to create")
 			}
-			// ------------------------Scenario 1-----------------------------------------
+			// ------------------------Scenario 1------------------------------------
 			// stage = 0, initial connection should succeed
 			ctx1, cancel1 := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel1()
 			conn, greetClient, err := callAndVerifyWithClientConn(ctx1, "rpc call 1", clientTLSCreds, false)
-			defer conn.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
-			// ---------------------------------------------------------------------------
+			defer conn.Close()
+			// ----------------------------------------------------------------------
 			stage.increase()
-			// ------------------------Scenario 2-----------------------------------------
+			// ------------------------Scenario 2------------------------------------
 			// stage = 1, previous connection should still succeed
 			err = callAndVerify("rpc call 2", greetClient, false)
 			if err != nil {
 				t.Fatal(err)
 			}
-			// ------------------------Scenario 3-----------------------------------------
+			// ------------------------Scenario 3------------------------------------
 			// stage = 1, new connection should fail
 			ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel2()
 			conn2, greetClient, err := callAndVerifyWithClientConn(ctx2, "rpc call 3", clientTLSCreds, true)
-			defer conn2.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
-			//// ---------------------------------------------------------------------------
+			defer conn2.Close()
+			//// --------------------------------------------------------------------
 			stage.increase()
-			// ------------------------Scenario 4-----------------------------------------
+			// ------------------------Scenario 4------------------------------------
 			// stage = 2,  new connection should succeed
 			ctx3, cancel3 := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel3()
 			conn3, greetClient, err := callAndVerifyWithClientConn(ctx3, "rpc call 4", clientTLSCreds, false)
-			defer conn3.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
-			// ---------------------------------------------------------------------------
+			defer conn3.Close()
+			// ----------------------------------------------------------------------
 			stage.reset()
 		})
 	}
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 509befa0d8fe..6c1909c44863 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -77,6 +77,7 @@ func TestClientServerHandshake(t *testing.T) {
 		clientRoot                 *x509.CertPool
 		clientGetRoot              func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		clientVerifyFunc           CustomVerificationFunc
+		clientVType                VerificationType
 		clientExpectCreateError    bool
 		clientExpectHandshakeError bool
 		serverMutualTLS            bool
@@ -84,13 +85,17 @@ func TestClientServerHandshake(t *testing.T) {
 		serverGetCert              func(*tls.ClientHelloInfo) (*tls.Certificate, error)
 		serverRoot                 *x509.CertPool
 		serverGetRoot              func(params *GetRootCAsParams) (*GetRootCAsResults, error)
+		serverVerifyFunc           CustomVerificationFunc
+		serverVType                VerificationType
 		serverExpectError          bool
 	}{
 		// Client: nil setting
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: server side failure
-		// Reason: if either clientCert or clientGetClientCert is not set and
-		// verifyFunc is not set, we will fail directly
+		// Reason: if clientRoot, clientGetRoot and verifyFunc is not set, client
+		// side doesn't provide any verification mechanism. We don't allow this
+		// even setting vType to SkipVerification. Clients should at least provide
+		// their own verification logic.
 		{
 			"Client_no_trust_cert_Server_peer_cert",
 			nil,
@@ -98,6 +103,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			SkipVerification,
 			true,
 			false,
 			false,
@@ -105,6 +111,8 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertAndHostVerification,
 			true,
 		},
 		// Client: nil setting except verifyFuncGood
@@ -119,6 +127,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			verifyFuncGood,
+			SkipVerification,
 			false,
 			false,
 			false,
@@ -126,13 +135,16 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertAndHostVerification,
 			false,
 		},
 		// Client: only set clientRoot
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: server side failure and client handshake failure
-		// Reason: not setting advanced TLS features will fall back to normal check, and will hence fail
-		// on default host name check. All the default hostname checks will fail in this test suites.
+		// Reason: client side sets vType to CertAndHostVerification, and will do
+		// default hostname check. All the default hostname checks will fail in
+		// this test suites.
 		{
 			"Client_root_cert_Server_peer_cert",
 			nil,
@@ -140,6 +152,7 @@ func TestClientServerHandshake(t *testing.T) {
 			clientTrustPool,
 			nil,
 			nil,
+			CertAndHostVerification,
 			false,
 			true,
 			false,
@@ -147,13 +160,16 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertAndHostVerification,
 			true,
 		},
 		// Client: only set clientGetRoot
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: server side failure and client handshake failure
-		// Reason: setting root reloading function without custom verifyFunc will also fail,
-		// since it will also fall back to default host name check
+		// Reason: client side sets vType to CertAndHostVerification, and will do
+		// default hostname check. All the default hostname checks will fail in
+		// this test suites.
 		{
 			"Client_reload_root_Server_peer_cert",
 			nil,
@@ -161,6 +177,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			nil,
+			CertAndHostVerification,
 			false,
 			true,
 			false,
@@ -168,6 +185,8 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertAndHostVerification,
 			true,
 		},
 		// Client: set clientGetRoot and clientVerifyFunc
@@ -180,6 +199,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			false,
@@ -187,6 +207,8 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertAndHostVerification,
 			false,
 		},
 		// Client: set clientGetRoot and bad clientVerifyFunc function
@@ -200,6 +222,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncBad,
+			CertVerification,
 			false,
 			true,
 			false,
@@ -207,6 +230,8 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertVerification,
 			true,
 		},
 		// Client: set clientGetRoot and clientVerifyFunc
@@ -220,6 +245,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			false,
@@ -227,46 +253,57 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			nil,
+			nil,
+			CertVerification,
 			true,
 		},
-		// Client: set clientGetRoot and clientVerifyFunc
-		// Server: only set serverCert with mutual TLS on
-		// Expected Behavior: server side failure
-		// Reason: server side must either set serverRoot or serverGetRoot when using mutual TLS
+		// Client: set clientGetRoot, clientVerifyFunc and clientCert
+		// Server: set serverRoot and serverCert with mutual TLS on
+		// Expected Behavior: success
 		{
-			"Client_reload_root_verifyFuncGood_Server_peer_cert_no_root_cert_mutualTLS",
-			nil,
+			"Client_peer_cert_reload_root_verifyFuncGood_Server_peer_cert_root_cert_mutualTLS",
+			[]tls.Certificate{clientPeerCert},
 			nil,
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			true,
 			[]tls.Certificate{serverPeerCert},
 			nil,
+			serverTrustPool,
 			nil,
 			nil,
-			true,
+			CertVerification,
+			false,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
-		// Server: set serverRoot and serverCert with mutual TLS on
-		// Expected Behavior: success
+		// Server: set serverCert, but not setting any of serverRoot, serverGetRoot
+		// or serverVerifyFunc, with mutual TLS on
+		// Expected Behavior: server side failure
+		// Reason: server side needs to provide any verification mechanism when
+		// mTLS in on, even setting vType to SkipVerification. Servers should at
+		// least provide their own verification logic.
 		{
-			"Client_peer_cert_reload_root_verifyFuncGood_Server_peer_cert_root_cert_mutualTLS",
+			"Client_peer_cert_reload_root_verifyFuncGood_Server_no_verification_mutualTLS",
 			[]tls.Certificate{clientPeerCert},
 			nil,
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
-			false,
+			true,
 			true,
 			[]tls.Certificate{serverPeerCert},
 			nil,
-			serverTrustPool,
 			nil,
-			false,
+			nil,
+			nil,
+			SkipVerification,
+			true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverGetRoot and serverCert with mutual TLS on
@@ -278,6 +315,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			true,
@@ -285,10 +323,13 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			getRootCAsForServer,
+			nil,
+			CertVerification,
 			false,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
-		// Server: set serverGetRoot returning error and serverCert with mutual TLS on
+		// Server: set serverGetRoot returning error and serverCert with mutual
+		// TLS on
 		// Expected Behavior: server side failure
 		// Reason: server side reloading returns failure
 		{
@@ -298,6 +339,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			true,
@@ -305,6 +347,8 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			nil,
 			getRootCAsForServerBad,
+			nil,
+			CertVerification,
 			true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientGetClientCert
@@ -319,6 +363,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			true,
@@ -328,9 +373,12 @@ func TestClientServerHandshake(t *testing.T) {
 			},
 			nil,
 			getRootCAsForServer,
+			verifyFuncGood,
+			CertVerification,
 			false,
 		},
-		// Client: set everything but with the wrong peer cert not trusted by server
+		// Client: set everything but with the wrong peer cert not trusted by
+		// server
 		// Server: set serverGetRoot and serverGetCert with mutual TLS on
 		// Expected Behavior: server side returns failure because of
 		// certificate mismatch
@@ -343,6 +391,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			true,
@@ -352,6 +401,8 @@ func TestClientServerHandshake(t *testing.T) {
 			},
 			nil,
 			getRootCAsForServer,
+			verifyFuncGood,
+			CertVerification,
 			true,
 		},
 		// Client: set everything but with the wrong trust cert not trusting server
@@ -367,6 +418,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForServer,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			true,
 			true,
@@ -376,10 +428,13 @@ func TestClientServerHandshake(t *testing.T) {
 			},
 			nil,
 			getRootCAsForServer,
+			verifyFuncGood,
+			CertVerification,
 			true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
-		// Server: set everything but with the wrong peer cert not trusted by client
+		// Server: set everything but with the wrong peer cert not trusted by
+		// client
 		// Expected Behavior: server side and client side return failure due to
 		// certificate mismatch and handshake failure
 		{
@@ -391,6 +446,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			false,
 			true,
@@ -400,6 +456,8 @@ func TestClientServerHandshake(t *testing.T) {
 			},
 			nil,
 			getRootCAsForServer,
+			verifyFuncGood,
+			CertVerification,
 			true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
@@ -415,6 +473,7 @@ func TestClientServerHandshake(t *testing.T) {
 			nil,
 			getRootCAsForClient,
 			verifyFuncGood,
+			CertVerification,
 			false,
 			true,
 			true,
@@ -424,6 +483,31 @@ func TestClientServerHandshake(t *testing.T) {
 			},
 			nil,
 			getRootCAsForClient,
+			verifyFuncGood,
+			CertVerification,
+			true,
+		},
+		// Client: set clientGetRoot, clientVerifyFunc and clientCert
+		// Server: set serverGetRoot and serverCert, but with bad verifyFunc
+		// Expected Behavior: server side and client side return failure due to
+		// server custom check fails
+		{
+			"Client_peer_cert_reload_root_verifyFuncGood_Server_bad_custom_verification_mutualTLS",
+			[]tls.Certificate{clientPeerCert},
+			nil,
+			nil,
+			getRootCAsForClient,
+			verifyFuncGood,
+			CertVerification,
+			false,
+			true,
+			true,
+			[]tls.Certificate{serverPeerCert},
+			nil,
+			nil,
+			getRootCAsForServer,
+			verifyFuncBad,
+			CertVerification,
 			true,
 		},
 	} {
@@ -443,6 +527,8 @@ func TestClientServerHandshake(t *testing.T) {
 					GetRootCAs:  test.serverGetRoot,
 				},
 				RequireClientCert: test.serverMutualTLS,
+				VerifyPeer:        test.serverVerifyFunc,
+				VType:             test.serverVType,
 			}
 			go func(done chan credentials.AuthInfo, lis net.Listener, serverOptions *ServerOptions) {
 				serverRawConn, err := lis.Accept()
@@ -480,6 +566,7 @@ func TestClientServerHandshake(t *testing.T) {
 					RootCACerts: test.clientRoot,
 					GetRootCAs:  test.clientGetRoot,
 				},
+				VType: test.clientVType,
 			}
 			clientTLS, newClientErr := NewClientCreds(clientOptions)
 			if newClientErr != nil && test.clientExpectCreateError {

From 18b6aa77284bb59a288579d722a5c47f5f0fe990 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Thu, 23 Apr 2020 06:55:30 +0800
Subject: [PATCH 037/481] xds: unexport edsBalancerImplInterface's function
 (#3550)

Signed-off-by: Zou Nengren <zouyee1989@gmail.com>
---
 xds/internal/balancer/edsbalancer/eds.go      |  28 ++--
 xds/internal/balancer/edsbalancer/eds_impl.go |  16 +-
 .../edsbalancer/eds_impl_priority_test.go     | 144 +++++++++---------
 .../balancer/edsbalancer/eds_impl_test.go     | 116 +++++++-------
 xds/internal/balancer/edsbalancer/eds_test.go |   8 +-
 5 files changed, 155 insertions(+), 157 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index f6057ad605f6..d1578c32771d 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -91,19 +91,17 @@ func (b *edsBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadB
 // implement to communicate with edsBalancer.
 //
 // It's implemented by the real eds balancer and a fake testing eds balancer.
-//
-// TODO: none of the methods in this interface needs to be exported.
 type edsBalancerImplInterface interface {
-	// HandleEDSResponse passes the received EDS message from traffic director to eds balancer.
-	HandleEDSResponse(edsResp xdsclient.EndpointsUpdate)
-	// HandleChildPolicy updates the eds balancer the intra-cluster load balancing policy to use.
-	HandleChildPolicy(name string, config json.RawMessage)
-	// HandleSubConnStateChange handles state change for SubConn.
-	HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State)
+	// handleEDSResponse passes the received EDS message from traffic director to eds balancer.
+	handleEDSResponse(edsResp xdsclient.EndpointsUpdate)
+	// handleChildPolicy updates the eds balancer the intra-cluster load balancing policy to use.
+	handleChildPolicy(name string, config json.RawMessage)
+	// handleSubConnStateChange handles state change for SubConn.
+	handleSubConnStateChange(sc balancer.SubConn, state connectivity.State)
 	// updateState handle a balancer state update from the priority.
 	updateState(priority priorityType, s balancer.State)
-	// Close closes the eds balancer.
-	Close()
+	// close closes the eds balancer.
+	close()
 }
 
 var _ balancer.V2Balancer = (*edsBalancer)(nil) // Assert that we implement V2Balancer
@@ -149,7 +147,7 @@ func (x *edsBalancer) run() {
 				x.client.close()
 			}
 			if x.edsImpl != nil {
-				x.edsImpl.Close()
+				x.edsImpl.close()
 			}
 			return
 		}
@@ -160,7 +158,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 	switch u := update.(type) {
 	case *subConnStateUpdate:
 		if x.edsImpl != nil {
-			x.edsImpl.HandleSubConnStateChange(u.sc, u.state.ConnectivityState)
+			x.edsImpl.handleSubConnStateChange(u.sc, u.state.ConnectivityState)
 		}
 	case *balancer.ClientConnState:
 		x.logger.Infof("Receive update from resolver, balancer config: %+v", u.BalancerConfig)
@@ -181,9 +179,9 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 		// different one.
 		if x.edsImpl != nil && !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy) {
 			if cfg.ChildPolicy != nil {
-				x.edsImpl.HandleChildPolicy(cfg.ChildPolicy.Name, cfg.ChildPolicy.Config)
+				x.edsImpl.handleChildPolicy(cfg.ChildPolicy.Name, cfg.ChildPolicy.Config)
 			} else {
-				x.edsImpl.HandleChildPolicy(roundrobin.Name, nil)
+				x.edsImpl.handleChildPolicy(roundrobin.Name, nil)
 			}
 		}
 
@@ -199,7 +197,7 @@ func (x *edsBalancer) handleXDSClientUpdate(update interface{}) {
 	// TODO: this func should accept (xdsclient.EndpointsUpdate, error), and process
 	// the error, instead of having a separate loseContact signal.
 	case xdsclient.EndpointsUpdate:
-		x.edsImpl.HandleEDSResponse(u)
+		x.edsImpl.handleEDSResponse(u)
 	case *loseContact:
 		// loseContact can be useful for going into fallback.
 	default:
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index ef61296c42f8..7dfa6f835051 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -114,12 +114,12 @@ func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType,
 	return edsImpl
 }
 
-// HandleChildPolicy updates the child balancers handling endpoints. Child
+// handleChildPolicy updates the child balancers handling endpoints. Child
 // policy is roundrobin by default. If the specified balancer is not installed,
 // the old child balancer will be used.
 //
 // HandleChildPolicy and HandleEDSResponse must be called by the same goroutine.
-func (edsImpl *edsBalancerImpl) HandleChildPolicy(name string, config json.RawMessage) {
+func (edsImpl *edsBalancerImpl) handleChildPolicy(name string, config json.RawMessage) {
 	if edsImpl.subBalancerBuilder.Name() == name {
 		return
 	}
@@ -169,11 +169,11 @@ func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropC
 	edsImpl.pickerMu.Unlock()
 }
 
-// HandleEDSResponse handles the EDS response and creates/deletes localities and
+// handleEDSResponse handles the EDS response and creates/deletes localities and
 // SubConns. It also handles drops.
 //
 // HandleChildPolicy and HandleEDSResponse must be called by the same goroutine.
-func (edsImpl *edsBalancerImpl) HandleEDSResponse(edsResp xdsclient.EndpointsUpdate) {
+func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {
 	// TODO: Unhandled fields from EDS response:
 	//  - edsResp.GetPolicy().GetOverprovisioningFactor()
 	//  - locality.GetPriority()
@@ -331,8 +331,8 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 	}
 }
 
-// HandleSubConnStateChange handles the state change and update pickers accordingly.
-func (edsImpl *edsBalancerImpl) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
+// handleSubConnStateChange handles the state change and update pickers accordingly.
+func (edsImpl *edsBalancerImpl) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
 	edsImpl.subConnMu.Lock()
 	var bgwc *balancerGroupWithConfig
 	if p, ok := edsImpl.subConnToPriority[sc]; ok {
@@ -408,8 +408,8 @@ func (edsImpl *edsBalancerImpl) newSubConn(priority priorityType, addrs []resolv
 	return sc, nil
 }
 
-// Close closes the balancer.
-func (edsImpl *edsBalancerImpl) Close() {
+// close closes the balancer.
+func (edsImpl *edsBalancerImpl) close() {
 	for _, bgwc := range edsImpl.priorityToLocalities {
 		if bg := bgwc.bg; bg != nil {
 			bg.Close()
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index 519b8e588ab7..c98f7205fc87 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -42,7 +42,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
@@ -51,8 +51,8 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	sc1 := <-cc.NewSubConnCh
 
 	// p0 is ready.
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p1 := <-cc.NewPickerCh
@@ -66,7 +66,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	select {
 	case <-cc.NewPickerCh:
@@ -82,7 +82,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	select {
 	case <-cc.NewPickerCh:
@@ -108,7 +108,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -117,8 +117,8 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	sc0 := <-cc.NewSubConnCh
 
 	// p0 is ready.
-	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc0, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc0, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p0 := <-cc.NewPickerCh
@@ -128,14 +128,14 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	}
 
 	// Turn down 0, 1 is used.
-	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure)
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc1 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test pick with 1.
 	p1 := <-cc.NewPickerCh
@@ -151,7 +151,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	select {
 	case <-cc.NewPickerCh:
@@ -164,14 +164,14 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	}
 
 	// Turn down 1, use 2
-	edsb.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc1, connectivity.TransientFailure)
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test pick with 2.
 	p2 := <-cc.NewPickerCh
@@ -186,7 +186,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	// p2 SubConns are removed.
 	scToRemove := <-cc.RemoveSubConnCh
@@ -215,7 +215,7 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -224,14 +224,14 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	sc0 := <-cc.NewSubConnCh
 
 	// Turn down 0, 1 is used.
-	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure)
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc1 := <-cc.NewSubConnCh
 	// Turn down 1, pick should error.
-	edsb.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc1, connectivity.TransientFailure)
 
 	// Test pick failure.
 	pFail := <-cc.NewPickerCh
@@ -246,15 +246,15 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test pick with 2.
 	p2 := <-cc.NewPickerCh
@@ -281,7 +281,7 @@ func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab1.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -290,21 +290,21 @@ func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	sc0 := <-cc.NewSubConnCh
 
 	// Turn down 0, 1 is used.
-	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure)
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc1 := <-cc.NewSubConnCh
 	// Turn down 1, 2 is used.
-	edsb.HandleSubConnStateChange(sc1, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc1, connectivity.TransientFailure)
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test pick with 2.
 	p2 := <-cc.NewPickerCh
@@ -316,7 +316,7 @@ func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	}
 
 	// When 0 becomes ready, 0 should be used, 1 and 2 should all be closed.
-	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc0, connectivity.Ready)
 
 	// sc1 and sc2 should be removed.
 	//
@@ -362,7 +362,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -371,7 +371,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	sc0 := <-cc.NewSubConnCh
 
 	// Keep 0 in connecting, 1 will be used after init timeout.
-	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc0, connectivity.Connecting)
 
 	// Make sure new SubConn is created before timeout.
 	select {
@@ -386,8 +386,8 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	}
 	sc1 := <-cc.NewSubConnCh
 
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test pick with 1.
 	p1 := <-cc.NewPickerCh
@@ -412,15 +412,15 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	clab0 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc0 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc0, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc0, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p0 := <-cc.NewPickerCh
@@ -430,15 +430,15 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	}
 
 	// Turn down p0 subconns, p1 subconns will be created.
-	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure)
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc1 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
 	p1 := <-cc.NewPickerCh
@@ -448,7 +448,7 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	}
 
 	// Reconnect p0 subconns, p1 subconn will be closed.
-	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc0, connectivity.Ready)
 
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
@@ -468,15 +468,15 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab1.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
 	clab1.AddLocality(testSubZones[3], 1, 1, testEndpointAddrs[3:4], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with only two p0 subconns.
 	p3 := <-cc.NewPickerCh
@@ -486,15 +486,15 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	}
 
 	// Turn down p0 subconns, p1 subconns will be created.
-	edsb.HandleSubConnStateChange(sc0, connectivity.TransientFailure)
-	edsb.HandleSubConnStateChange(sc2, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure)
+	edsb.handleSubConnStateChange(sc2, connectivity.TransientFailure)
 
 	sc3 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc3, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc3, connectivity.Ready)
 	sc4 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc4, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc4, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
 	p4 := <-cc.NewPickerCh
@@ -523,15 +523,15 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	clab0 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc0 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc0, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc0, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc0, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc0, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p0 := <-cc.NewPickerCh
@@ -542,7 +542,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 
 	// Remove all priorities.
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	// p0 subconn should be removed.
 	scToRemove := <-cc.RemoveSubConnCh
@@ -562,7 +562,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[3:4], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	addrs01 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs01[0].Addr, testEndpointAddrs[2]; got != want {
@@ -580,8 +580,8 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	sc11 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc11, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc11, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc11, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc11, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
 	p1 := <-cc.NewPickerCh
@@ -593,7 +593,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	// Remove p1 from EDS, to fallback to p0.
 	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	// p1 subconn should be removed.
 	scToRemove1 := <-cc.RemoveSubConnCh
@@ -611,8 +611,8 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 
 	// Send an ready update for the p0 sc that was received when re-adding
 	// localities to EDS.
-	edsb.HandleSubConnStateChange(sc01, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc01, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc01, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc01, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p2 := <-cc.NewPickerCh
@@ -667,7 +667,7 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
@@ -676,8 +676,8 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	sc1 := <-cc.NewSubConnCh
 
 	// p0 is ready.
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p1 := <-cc.NewPickerCh
@@ -690,12 +690,12 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, nil, nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	// p0 will remove the subconn, and ClientConn will send a sc update to
 	// shutdown.
 	scToRemove := <-cc.RemoveSubConnCh
-	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
@@ -704,8 +704,8 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	sc2 := <-cc.NewSubConnCh
 
 	// p1 is ready.
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
 	p2 := <-cc.NewPickerCh
@@ -726,7 +726,7 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
@@ -735,8 +735,8 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	sc1 := <-cc.NewSubConnCh
 
 	// p0 is ready.
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Test roundrobin with only p0 subconns.
 	p1 := <-cc.NewPickerCh
@@ -751,12 +751,12 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 		Health: []corepb.HealthStatus{corepb.HealthStatus_UNHEALTHY},
 	})
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	// p0 will remove the subconn, and ClientConn will send a sc update to
 	// transient failure.
 	scToRemove := <-cc.RemoveSubConnCh
-	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
@@ -765,8 +765,8 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	sc2 := <-cc.NewSubConnCh
 
 	// p1 is ready.
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with only p1 subconns.
 	p2 := <-cc.NewPickerCh
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index b8a4e186c7be..550807c3f241 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -63,11 +63,11 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	// One locality with one backend.
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	sc1 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Pick with only the first backend.
 	p1 := <-cc.NewPickerCh
@@ -81,11 +81,11 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	// The same locality, add one more backend.
 	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with two subconns.
 	p2 := <-cc.NewPickerCh
@@ -97,13 +97,13 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	// The same locality, delete first backend.
 	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
-	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with only the second subconn.
 	p3 := <-cc.NewPickerCh
@@ -117,16 +117,16 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	// The same locality, replace backend.
 	clab4 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab4.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
 
 	sc3 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc3, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc3, connectivity.Ready)
 	scToRemove = <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
 	}
-	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with only the third subconn.
 	p4 := <-cc.NewPickerCh
@@ -140,7 +140,7 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	// The same locality, different drop rate, dropping 50%.
 	clab5 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], []uint32{50})
 	clab5.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
 
 	// Picks with drops.
 	p5 := <-cc.NewPickerCh
@@ -158,7 +158,7 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	// The same locality, remove drops.
 	clab6 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab6.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
 
 	// Pick without drops.
 	p6 := <-cc.NewPickerCh
@@ -184,19 +184,19 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	// Two localities, each with one backend.
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 	sc1 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 
 	// Add the second locality later to make sure sc2 belongs to the second
 	// locality. Otherwise the test is flaky because of a map is used in EDS to
 	// keep localities.
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with two subconns.
 	p1 := <-cc.NewPickerCh
@@ -210,11 +210,11 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
 
 	sc3 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc3, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc3, connectivity.Ready)
 
 	// Test roundrobin with three subconns.
 	p2 := <-cc.NewPickerCh
@@ -227,13 +227,13 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	clab3.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
 
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
 	}
-	edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+	edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
 
 	// Test pick with two subconns (without the first one).
 	p3 := <-cc.NewPickerCh
@@ -246,11 +246,11 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab4 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab4.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	clab4.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
 
 	sc4 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc4, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc4, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc4, connectivity.Ready)
 
 	// Test pick with two subconns (without the first one).
 	p4 := <-cc.NewPickerCh
@@ -266,7 +266,7 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab5 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab5.AddLocality(testSubZones[1], 2, 0, testEndpointAddrs[1:2], nil)
 	clab5.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
 
 	// Test pick with two subconns different locality weight.
 	p5 := <-cc.NewPickerCh
@@ -282,7 +282,7 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	clab6 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab6.AddLocality(testSubZones[1], 0, 0, testEndpointAddrs[1:2], nil)
 	clab6.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
 
 	// Changing weight of locality[1] to 0 caused it to be removed. It's subconn
 	// should also be removed.
@@ -334,7 +334,7 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 			corepb.HealthStatus_DEGRADED,
 		},
 	})
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	var (
 		readySCs           []balancer.SubConn
@@ -344,8 +344,8 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 		addr := <-cc.NewSubConnAddrsCh
 		newSubConnAddrStrs = append(newSubConnAddrStrs, addr[0].Addr)
 		sc := <-cc.NewSubConnCh
-		edsb.HandleSubConnStateChange(sc, connectivity.Connecting)
-		edsb.HandleSubConnStateChange(sc, connectivity.Ready)
+		edsb.handleSubConnStateChange(sc, connectivity.Connecting)
+		edsb.handleSubConnStateChange(sc, connectivity.Ready)
 		readySCs = append(readySCs, sc)
 	}
 
@@ -384,7 +384,7 @@ func (s) TestClose(t *testing.T) {
 	edsb := newEDSBalancerImpl(nil, nil, nil, nil)
 	// This is what could happen when switching between fallback and eds. This
 	// make sure it doesn't panic.
-	edsb.Close()
+	edsb.close()
 }
 
 // Create XDS balancer, and update sub-balancer before handling eds responses.
@@ -396,17 +396,17 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	t.Logf("update sub-balancer to test-const-balancer")
-	edsb.HandleChildPolicy("test-const-balancer", nil)
+	edsb.handleChildPolicy("test-const-balancer", nil)
 
 	// Two localities, each with one backend.
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	for i := 0; i < 2; i++ {
 		sc := <-cc.NewSubConnCh
-		edsb.HandleSubConnStateChange(sc, connectivity.Ready)
+		edsb.handleSubConnStateChange(sc, connectivity.Ready)
 	}
 
 	p0 := <-cc.NewPickerCh
@@ -418,18 +418,18 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	}
 
 	t.Logf("update sub-balancer to round-robin")
-	edsb.HandleChildPolicy(roundrobin.Name, nil)
+	edsb.handleChildPolicy(roundrobin.Name, nil)
 
 	for i := 0; i < 2; i++ {
 		<-cc.RemoveSubConnCh
 	}
 
 	sc1 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 
 	// Test roundrobin with two subconns.
 	p1 := <-cc.NewPickerCh
@@ -439,7 +439,7 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	}
 
 	t.Logf("update sub-balancer to test-const-balancer")
-	edsb.HandleChildPolicy("test-const-balancer", nil)
+	edsb.handleChildPolicy("test-const-balancer", nil)
 
 	for i := 0; i < 2; i++ {
 		scToRemove := <-cc.RemoveSubConnCh
@@ -447,12 +447,12 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 			!cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 			t.Fatalf("RemoveSubConn, want (%v or %v), got %v", sc1, sc2, scToRemove)
 		}
-		edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown)
+		edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
 	}
 
 	for i := 0; i < 2; i++ {
 		sc := <-cc.NewSubConnCh
-		edsb.HandleSubConnStateChange(sc, connectivity.Ready)
+		edsb.handleSubConnStateChange(sc, connectivity.Ready)
 	}
 
 	p2 := <-cc.NewPickerCh
@@ -464,18 +464,18 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	}
 
 	t.Logf("update sub-balancer to round-robin")
-	edsb.HandleChildPolicy(roundrobin.Name, nil)
+	edsb.handleChildPolicy(roundrobin.Name, nil)
 
 	for i := 0; i < 2; i++ {
 		<-cc.RemoveSubConnCh
 	}
 
 	sc3 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc3, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc3, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc3, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc3, connectivity.Ready)
 	sc4 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc4, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc4, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc4, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc4, connectivity.Ready)
 
 	p3 := <-cc.NewPickerCh
 	want = []balancer.SubConn{sc3, sc4}
@@ -531,11 +531,11 @@ func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) {
 		go edsb.updateState(p, state)
 	}
 
-	edsb.HandleChildPolicy("test-inline-update-balancer", nil)
+	edsb.handleChildPolicy("test-inline-update-balancer", nil)
 
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 
 	p0 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
@@ -623,10 +623,10 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// Two localities, each with one backend.
 	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 	sc1 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc1, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc1, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
 	backendToBalancerID[sc1] = internal.LocalityID{
 		SubZone: testSubZones[0],
 	}
@@ -635,10 +635,10 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// locality. Otherwise the test is flaky because of a map is used in EDS to
 	// keep localities.
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.HandleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
 	sc2 := <-cc.NewSubConnCh
-	edsb.HandleSubConnStateChange(sc2, connectivity.Connecting)
-	edsb.HandleSubConnStateChange(sc2, connectivity.Ready)
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
 	backendToBalancerID[sc2] = internal.LocalityID{
 		SubZone: testSubZones[1],
 	}
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index ef2f93cd62c7..a9e9e663bdef 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -100,16 +100,16 @@ type fakeEDSBalancer struct {
 	loadStore          lrs.Store
 }
 
-func (f *fakeEDSBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+func (f *fakeEDSBalancer) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
 	f.subconnStateChange.Send(&scStateChange{sc: sc, state: state})
 }
 
-func (f *fakeEDSBalancer) HandleChildPolicy(name string, config json.RawMessage) {
+func (f *fakeEDSBalancer) handleChildPolicy(name string, config json.RawMessage) {
 	f.childPolicy.Send(&loadBalancingConfig{Name: name, Config: config})
 }
 
-func (f *fakeEDSBalancer) Close()                                              {}
-func (f *fakeEDSBalancer) HandleEDSResponse(edsResp xdsclient.EndpointsUpdate) {}
+func (f *fakeEDSBalancer) close()                                              {}
+func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {}
 func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {}
 
 func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) error {

From 8f94cb18c0d4848d172b048c73ded5dcbb1c55b4 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 23 Apr 2020 08:42:39 -0700
Subject: [PATCH 038/481] status: remove Error method accidentally added to
 *Status (#3561)

---
 internal/status/status.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/internal/status/status.go b/internal/status/status.go
index 23dae8e5679f..681260692e36 100644
--- a/internal/status/status.go
+++ b/internal/status/status.go
@@ -100,11 +100,6 @@ func (s *Status) Err() error {
 	return (*Error)(s.Proto())
 }
 
-func (s *Status) Error() string {
-	p := s.Proto()
-	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage())
-}
-
 // WithDetails returns a new status with the provided details messages appended to the status.
 // If any errors are encountered, it returns nil and the first error encountered.
 func (s *Status) WithDetails(details ...proto.Message) (*Status, error) {

From 6a3c03883db90c9bc0b9bd477f26ca93d8841e55 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 23 Apr 2020 11:03:42 -0700
Subject: [PATCH 039/481] Pass address attributes from balancer to creds
 handshaker. (#3548)

---
 balancer/balancer.go                 |   3 +
 credentials/credentials.go           |  48 ++++++++--
 internal/balancer/stub/stub.go       | 101 ++++++++++++++++++++
 internal/internal.go                 |   3 +
 internal/transport/http2_client.go   |   6 ++
 internal/transport/transport_test.go |  54 +++++++++++
 test/balancer_test.go                | 132 ++++++++++++++++++++++++++-
 vet.sh                               |   3 +-
 8 files changed, 338 insertions(+), 12 deletions(-)
 create mode 100644 internal/balancer/stub/stub.go

diff --git a/balancer/balancer.go b/balancer/balancer.go
index 9258858ed755..080fe0d189e4 100644
--- a/balancer/balancer.go
+++ b/balancer/balancer.go
@@ -111,6 +111,9 @@ type NewSubConnOptions struct {
 	// CredsBundle is the credentials bundle that will be used in the created
 	// SubConn. If it's nil, the original creds from grpc DialOptions will be
 	// used.
+	//
+	// Deprecated: Use the Attributes field in resolver.Address to pass
+	// arbitrary data to the credential handshaker.
 	CredsBundle credentials.Bundle
 	// HealthCheckEnabled indicates whether health check service should be
 	// enabled on this SubConn
diff --git a/credentials/credentials.go b/credentials/credentials.go
index e438fda226fe..53addd8c71e9 100644
--- a/credentials/credentials.go
+++ b/credentials/credentials.go
@@ -29,6 +29,7 @@ import (
 	"net"
 
 	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/internal"
 )
 
@@ -124,15 +125,18 @@ var ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gR
 // TransportCredentials defines the common interface for all the live gRPC wire
 // protocols and supported transport security protocols (e.g., TLS, SSL).
 type TransportCredentials interface {
-	// ClientHandshake does the authentication handshake specified by the corresponding
-	// authentication protocol on rawConn for clients. It returns the authenticated
-	// connection and the corresponding auth information about the connection.
-	// The auth information should embed CommonAuthInfo to return additional information about
-	// the credentials. Implementations must use the provided context to implement timely cancellation.
-	// gRPC will try to reconnect if the error returned is a temporary error
-	// (io.EOF, context.DeadlineExceeded or err.Temporary() == true).
-	// If the returned error is a wrapper error, implementations should make sure that
+	// ClientHandshake does the authentication handshake specified by the
+	// corresponding authentication protocol on rawConn for clients. It returns
+	// the authenticated connection and the corresponding auth information
+	// about the connection.  The auth information should embed CommonAuthInfo
+	// to return additional information about the credentials. Implementations
+	// must use the provided context to implement timely cancellation.  gRPC
+	// will try to reconnect if the error returned is a temporary error
+	// (io.EOF, context.DeadlineExceeded or err.Temporary() == true).  If the
+	// returned error is a wrapper error, implementations should make sure that
 	// the error implements Temporary() to have the correct retry behaviors.
+	// Additionally, ClientHandshakeInfo data will be available via the context
+	// passed to this call.
 	//
 	// If the returned net.Conn is closed, it MUST close the net.Conn provided.
 	ClientHandshake(context.Context, string, net.Conn) (net.Conn, AuthInfo, error)
@@ -193,6 +197,31 @@ func RequestInfoFromContext(ctx context.Context) (ri RequestInfo, ok bool) {
 	return
 }
 
+// ClientHandshakeInfo holds data to be passed to ClientHandshake. This makes
+// it possible to pass arbitrary data to the handshaker from gRPC, resolver,
+// balancer etc. Individual credential implementations control the actual
+// format of the data that they are willing to receive.
+//
+// This API is experimental.
+type ClientHandshakeInfo struct {
+	// Attributes contains the attributes for the address. It could be provided
+	// by the gRPC, resolver, balancer etc.
+	Attributes *attributes.Attributes
+}
+
+// clientHandshakeInfoKey is a struct used as the key to store
+// ClientHandshakeInfo in a context.
+type clientHandshakeInfoKey struct{}
+
+// ClientHandshakeInfoFromContext returns the ClientHandshakeInfo struct stored
+// in ctx.
+//
+// This API is experimental.
+func ClientHandshakeInfoFromContext(ctx context.Context) ClientHandshakeInfo {
+	chi, _ := ctx.Value(clientHandshakeInfoKey{}).(ClientHandshakeInfo)
+	return chi
+}
+
 // CheckSecurityLevel checks if a connection's security level is greater than or equal to the specified one.
 // It returns success if 1) the condition is satisified or 2) AuthInfo struct does not implement GetCommonAuthInfo() method
 // or 3) CommonAuthInfo.SecurityLevel has an invalid zero value. For 2) and 3), it is for the purpose of backward-compatibility.
@@ -223,6 +252,9 @@ func init() {
 	internal.NewRequestInfoContext = func(ctx context.Context, ri RequestInfo) context.Context {
 		return context.WithValue(ctx, requestInfoKey{}, ri)
 	}
+	internal.NewClientHandshakeInfoContext = func(ctx context.Context, chi ClientHandshakeInfo) context.Context {
+		return context.WithValue(ctx, clientHandshakeInfoKey{}, chi)
+	}
 }
 
 // ChannelzSecurityInfo defines the interface that security protocols should implement
diff --git a/internal/balancer/stub/stub.go b/internal/balancer/stub/stub.go
new file mode 100644
index 000000000000..2ddaf5f804d2
--- /dev/null
+++ b/internal/balancer/stub/stub.go
@@ -0,0 +1,101 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package stub implements a balancer for testing purposes.
+package stub
+
+import "google.golang.org/grpc/balancer"
+
+// BalancerFuncs contains all balancer.Balancer functions with a preceding
+// *BalancerData parameter for passing additional instance information.  Any
+// nil functions will never be called.
+type BalancerFuncs struct {
+	// Init is called after ClientConn and BuildOptions are set in
+	// BalancerData.  It may be used to initialize BalancerData.Data.
+	Init func(*BalancerData)
+
+	UpdateClientConnState func(*BalancerData, balancer.ClientConnState) error
+	ResolverError         func(*BalancerData, error)
+	UpdateSubConnState    func(*BalancerData, balancer.SubConn, balancer.SubConnState)
+	Close                 func(*BalancerData)
+}
+
+// BalancerData contains data relevant to a stub balancer.
+type BalancerData struct {
+	// ClientConn is set by the builder.
+	ClientConn balancer.ClientConn
+	// BuildOptions is set by the builder.
+	BuildOptions balancer.BuildOptions
+	// Data may be used to store arbitrary user data.
+	Data interface{}
+}
+
+type bal struct {
+	// TODO: Remove this once the legacy balancer API is removed. See
+	// https://github.com/grpc/grpc-go/pull/3431.
+	balancer.Balancer
+
+	bf BalancerFuncs
+	bd *BalancerData
+}
+
+func (b *bal) UpdateClientConnState(c balancer.ClientConnState) error {
+	if b.bf.UpdateClientConnState != nil {
+		return b.bf.UpdateClientConnState(b.bd, c)
+	}
+	return nil
+}
+
+func (b *bal) ResolverError(e error) {
+	if b.bf.ResolverError != nil {
+		b.bf.ResolverError(b.bd, e)
+	}
+}
+
+func (b *bal) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) {
+	if b.bf.UpdateSubConnState != nil {
+		b.bf.UpdateSubConnState(b.bd, sc, scs)
+	}
+}
+
+func (b *bal) Close() {
+	if b.bf.Close != nil {
+		b.bf.Close(b.bd)
+	}
+}
+
+type bb struct {
+	name string
+	bf   BalancerFuncs
+}
+
+func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	b := &bal{bf: bb.bf, bd: &BalancerData{ClientConn: cc, BuildOptions: opts}}
+	if b.bf.Init != nil {
+		b.bf.Init(b.bd)
+	}
+	return b
+}
+
+func (bb bb) Name() string { return bb.name }
+
+// Register registers a stub balancer builder which will call the provided
+// functions.  The name used should be unique.
+func Register(name string, bf BalancerFuncs) {
+	balancer.Register(bb{name: name, bf: bf})
+}
diff --git a/internal/internal.go b/internal/internal.go
index c6fbe8bb1b27..883345eee939 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -40,6 +40,9 @@ var (
 	// NewRequestInfoContext creates a new context based on the argument context attaching
 	// the passed in RequestInfo to the new context.
 	NewRequestInfoContext interface{} // func(context.Context, credentials.RequestInfo) context.Context
+	// NewClientHandshakeInfoContext returns a copy of the input context with
+	// the passed in ClientHandshakeInfo struct added to it.
+	NewClientHandshakeInfoContext interface{} // func(context.Context, credentials.ClientHandshakeInfo) context.Context
 	// ParseServiceConfigForTesting is for creating a fake
 	// ClientConn for resolver testing only
 	ParseServiceConfigForTesting interface{} // func(string) *serviceconfig.ParseResult
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index dab4a3c7126e..d1eb17e068fe 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -215,6 +215,12 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		}
 	}
 	if transportCreds != nil {
+		// gRPC, resolver, balancer etc. can specify arbitrary data in the
+		// Attributes field of resolver.Address, which is shoved into connectCtx
+		// and passed to the credential handshaker. This makes it possible for
+		// address specific arbitrary data to reach the credential handshaker.
+		contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
+		connectCtx = contextWithHandshakeInfo(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 		scheme = "https"
 		conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.ServerName, conn)
 		if err != nil {
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 09e310b60e38..391ad9925c1a 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -34,9 +34,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
+	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/leakcheck"
 	"google.golang.org/grpc/internal/testutils"
@@ -1816,3 +1819,54 @@ func (s) TestHeaderTblSize(t *testing.T) {
 		t.Fatalf("expected len(limits) = 2 within 10s, got != 2")
 	}
 }
+
+// attrTransportCreds is a transport credential implementation which stores
+// Attributes from the ClientHandshakeInfo struct passed in the context locally
+// for the test to inspect.
+type attrTransportCreds struct {
+	credentials.TransportCredentials
+	attr *attributes.Attributes
+}
+
+func (ac *attrTransportCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	ai := credentials.ClientHandshakeInfoFromContext(ctx)
+	ac.attr = ai.Attributes
+	return rawConn, nil, nil
+}
+func (ac *attrTransportCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (ac *attrTransportCreds) Clone() credentials.TransportCredentials {
+	return nil
+}
+
+// TestClientHandshakeInfo adds attributes to the resolver.Address passes to
+// NewClientTransport and verifies that these attributes are received by the
+// transport credential handshaker.
+func (s) TestClientHandshakeInfo(t *testing.T) {
+	server := setUpServerOnly(t, 0, &ServerConfig{}, pingpong)
+	defer server.stop()
+
+	const (
+		testAttrKey = "foo"
+		testAttrVal = "bar"
+	)
+	addr := resolver.Address{
+		Addr:       "localhost:" + server.port,
+		Attributes: attributes.New(testAttrKey, testAttrVal),
+	}
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second))
+	defer cancel()
+	creds := &attrTransportCreds{}
+
+	tr, err := NewClientTransport(ctx, context.Background(), addr, ConnectOptions{TransportCredentials: creds}, func() {}, func(GoAwayReason) {}, func() {})
+	if err != nil {
+		t.Fatalf("NewClientTransport(): %v", err)
+	}
+	defer tr.Close()
+
+	wantAttr := attributes.New(testAttrKey, testAttrVal)
+	if gotAttr := creds.attr; !cmp.Equal(gotAttr, wantAttr, cmp.AllowUnexported(attributes.Attributes{})) {
+		t.Fatalf("received attributes %v in creds, want %v", gotAttr, wantAttr)
+	}
+}
diff --git a/test/balancer_test.go b/test/balancer_test.go
index afa9ec1b7ea2..7db9635a8971 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -20,20 +20,27 @@ package test
 
 import (
 	"context"
+	"fmt"
+	"net"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/balancerload"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/testdata"
 )
@@ -43,7 +50,7 @@ const testBalancerName = "testbalancer"
 // testBalancer creates one subconn with the first address from resolved
 // addresses.
 //
-// It's used to test options for NewSubConn are applies correctly.
+// It's used to test whether options for NewSubConn are applied correctly.
 type testBalancer struct {
 	cc balancer.ClientConn
 	sc balancer.SubConn
@@ -96,8 +103,7 @@ func (b *testBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectiv
 	}
 }
 
-func (b *testBalancer) Close() {
-}
+func (b *testBalancer) Close() {}
 
 type picker struct {
 	err error
@@ -346,3 +352,123 @@ func (s) TestNonGRPCLBBalancerGetsNoGRPCLBAddress(t *testing.T) {
 		t.Fatalf("With both backend and grpclb addresses, balancer got addresses %v, want %v", got, nonGRPCLBAddresses)
 	}
 }
+
+type aiPicker struct {
+	result balancer.PickResult
+	err    error
+}
+
+func (aip *aiPicker) Pick(_ balancer.PickInfo) (balancer.PickResult, error) {
+	return aip.result, aip.err
+}
+
+// attrTransportCreds is a transport credential implementation which stores
+// Attributes from the ClientHandshakeInfo struct passed in the context locally
+// for the test to inspect.
+type attrTransportCreds struct {
+	credentials.TransportCredentials
+	attr *attributes.Attributes
+}
+
+func (ac *attrTransportCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	ai := credentials.ClientHandshakeInfoFromContext(ctx)
+	ac.attr = ai.Attributes
+	return rawConn, nil, nil
+}
+func (ac *attrTransportCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (ac *attrTransportCreds) Clone() credentials.TransportCredentials {
+	return nil
+}
+
+// TestAddressAttributesInNewSubConn verifies that the Attributes passed from a
+// balancer in the resolver.Address that is passes to NewSubConn reaches all the
+// way to the ClientHandshake method of the credentials configured on the parent
+// channel.
+func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
+	const (
+		testAttrKey      = "foo"
+		testAttrVal      = "bar"
+		attrBalancerName = "attribute-balancer"
+	)
+
+	// Register a stub balancer which adds attributes to the first address that
+	// it receives and then calls NewSubConn on it.
+	bf := stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			addrs := ccs.ResolverState.Addresses
+			if len(addrs) == 0 {
+				return nil
+			}
+
+			// Only use the first address.
+			attr := attributes.New(testAttrKey, testAttrVal)
+			addrs[0].Attributes = attr
+			sc, err := bd.ClientConn.NewSubConn([]resolver.Address{addrs[0]}, balancer.NewSubConnOptions{})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+		UpdateSubConnState: func(bd *stub.BalancerData, sc balancer.SubConn, state balancer.SubConnState) {
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}})
+		},
+	}
+	stub.Register(attrBalancerName, bf)
+	t.Logf("Registered balancer %s...", attrBalancerName)
+
+	r, cleanup := manual.GenerateAndRegisterManualResolver()
+	defer cleanup()
+	t.Logf("Registered manual resolver with scheme %s...", r.Scheme())
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := grpc.NewServer()
+	testpb.RegisterTestServiceServer(s, &testServer{})
+	go s.Serve(lis)
+	defer s.Stop()
+	t.Logf("Started gRPC server at %s...", lis.Addr().String())
+
+	creds := &attrTransportCreds{}
+	dopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(creds),
+		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, attrBalancerName)),
+	}
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cc.Close()
+	tc := testpb.NewTestServiceClient(cc)
+	t.Log("Created a ClientConn...")
+
+	// The first RPC should fail because there's no address.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded {
+		t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err)
+	}
+	t.Log("Made an RPC which was expected to fail...")
+
+	state := resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}
+	r.UpdateState(state)
+	t.Logf("Pushing resolver state update: %v through the manual resolver", state)
+
+	// The second RPC should succeed.
+	ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() = _, %v, want _, <nil>", err)
+	}
+	t.Log("Made an RPC which succeeded...")
+
+	wantAttr := attributes.New(testAttrKey, testAttrVal)
+	if gotAttr := creds.attr; !cmp.Equal(gotAttr, wantAttr, cmp.AllowUnexported(attributes.Attributes{})) {
+		t.Fatalf("received attributes %v in creds, want %v", gotAttr, wantAttr)
+	}
+}
diff --git a/vet.sh b/vet.sh
index b845102a355e..9d20e46f0c36 100755
--- a/vet.sh
+++ b/vet.sh
@@ -126,7 +126,8 @@ staticcheck -go 1.9 -checks 'inherit,-ST1015' ./... > "${SC_OUT}" || true
 # Error if anything other than deprecation warnings are printed.
 not grep -v "is deprecated:.*SA1019" "${SC_OUT}"
 # Only ignore the following deprecated types/fields/functions.
-not grep -Fv '.HandleResolvedAddrs
+not grep -Fv '.CredsBundle
+.HandleResolvedAddrs
 .HandleSubConnStateChange
 .HeaderMap
 .NewAddress

From 29f40a46f63bf26ce539a8a77b081b9012ea5fb6 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Fri, 24 Apr 2020 05:58:34 +0800
Subject: [PATCH 040/481] make helper function return error (#3553)

---
 xds/internal/client/v2client_ack_test.go | 105 +++++++++++++++--------
 1 file changed, 71 insertions(+), 34 deletions(-)

diff --git a/xds/internal/client/v2client_ack_test.go b/xds/internal/client/v2client_ack_test.go
index c4f49d4b0303..cbeb862eefff 100644
--- a/xds/internal/client/v2client_ack_test.go
+++ b/xds/internal/client/v2client_ack_test.go
@@ -125,31 +125,27 @@ func startXDS(t *testing.T, xdsname string, v2c *v2Client, reqChan *testutils.Ch
 //
 // It also waits and checks that the ack request contains the given version, and
 // the generated nonce.
-//
-// TODO: make this and other helper function either consistently return error,
-// and fatal() in the test code, or all call t.Fatal(), and mark them as
-// helper().
-func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (nonce string) {
-	nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, version)
+func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
+	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, version)
 	t.Logf("Good %s response pushed to fakeServer...", xdsname)
 
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(version), nonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
+		return "", fmt.Errorf("failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("Good %s response acked", xdsname)
 
 	if _, err := callbackCh.Receive(); err != nil {
-		t.Fatalf("Timeout when expecting %s update", xdsname)
+		return "", fmt.Errorf("timeout when expecting %s update", xdsname)
 	}
 	t.Logf("Good %s response callback executed", xdsname)
-	return
+	return nonce, nil
 }
 
 // sendBadResp sends a bad response with the given version. This response will
 // be nacked, so we expect a request with the previous version (version-1).
 //
 // But the nonce in request should be the new nonce.
-func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, wantReq *xdspb.DiscoveryRequest) {
+func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, wantReq *xdspb.DiscoveryRequest) error {
 	var typeURL string
 	switch xdsname {
 	case "LDS":
@@ -167,9 +163,10 @@ func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, ve
 	}, version)
 	t.Logf("Bad %s response pushed to fakeServer...", xdsname)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(version-1), nonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
+		return fmt.Errorf("failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("Bad %s response nacked", xdsname)
+	return nil
 }
 
 // TestV2ClientAck verifies that valid responses are acked, and invalid ones
@@ -192,36 +189,60 @@ func (s) TestV2ClientAck(t *testing.T) {
 
 	// Start the watch, send a good response, and check for ack.
 	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
-	sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	if _, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+		t.Fatal(err)
+	}
 	versionLDS++
 	startXDS(t, "RDS", v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
-	sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS)
+	if _, err := sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
+		t.Fatal(err)
+	}
 	versionRDS++
 	startXDS(t, "CDS", v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
 	startXDS(t, "EDS", v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
-	sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS)
+	if _, err := sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
+		t.Fatal(err)
+	}
 	versionEDS++
 
 	// Send a bad response, and check for nack.
-	sendBadResp(t, "LDS", fakeServer, versionLDS, goodLDSRequest)
+	if err := sendBadResp(t, "LDS", fakeServer, versionLDS, goodLDSRequest); err != nil {
+		t.Fatal(err)
+	}
 	versionLDS++
-	sendBadResp(t, "RDS", fakeServer, versionRDS, goodRDSRequest)
+	if err := sendBadResp(t, "RDS", fakeServer, versionRDS, goodRDSRequest); err != nil {
+		t.Fatal(err)
+	}
 	versionRDS++
-	sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest)
+	if err := sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
-	sendBadResp(t, "EDS", fakeServer, versionEDS, goodEDSRequest)
+	if err := sendBadResp(t, "EDS", fakeServer, versionEDS, goodEDSRequest); err != nil {
+		t.Fatal(err)
+	}
 	versionEDS++
 
 	// send another good response, and check for ack, with the new version.
-	sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	if _, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+		t.Fatal(err)
+	}
 	versionLDS++
-	sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS)
+	if _, err := sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
+		t.Fatal(err)
+	}
 	versionRDS++
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
-	sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS)
+	if _, err := sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
+		t.Fatal(err)
+	}
 	versionEDS++
 }
 
@@ -270,8 +291,10 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 
 	// Start the watch, send a good response, and check for ack.
 	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
-	nonce := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
-
+	nonce, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Start a new watch. The version in the new request should be the version
 	// from the previous response, thus versionLDS before ++.
 	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
@@ -291,7 +314,9 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	t.Logf("Bad response nacked")
 	versionLDS++
 
-	sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	if _, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+		t.Fatal(err)
+	}
 	versionLDS++
 }
 
@@ -315,8 +340,10 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 
 	// Send a good CDS response, this function waits for the ACK with the right
 	// version.
-	nonce := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
-
+	nonce, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Cancel the CDS watch, and start a new one. The new watch should have the
 	// version from the response above.
 	v2c.removeWatch(cdsURL, goodClusterName1)
@@ -334,11 +361,15 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	versionCDS++
 
 	// Send a bad response with the next version.
-	sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest)
+	if err := sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
 }
 
@@ -363,8 +394,10 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	t.Logf("FakeServer received %s request...", "CDS")
 
 	// send a good response, and check for ack, with the new version.
-	nonce := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
-
+	nonce, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Cancel the watch before the next response is sent. This mimics the case
 	// watch is canceled while response is on wire.
 	v2c.removeWatch(cdsURL, goodClusterName1)
@@ -401,10 +434,14 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	}
 
 	// Send a bad response with the next version.
-	sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest)
+	if err := sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+		t.Fatal(err)
+	}
 	versionCDS++
 }

From a0cdc21e61ccf7c10243874491f30a6b3f9a96cd Mon Sep 17 00:00:00 2001
From: Adhityaa Chandrasekar <adtac@adtac.in>
Date: Thu, 23 Apr 2020 18:50:02 -0400
Subject: [PATCH 041/481] server.go: use worker goroutines for fewer stack
 allocations (#3204)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Currently (go1.13.4), the default stack size for newly spawned
goroutines is 2048 bytes. This is insufficient when processing gRPC
requests as the we often require more than 4 KiB stacks. This causes the
Go runtime to call runtime.morestack at least twice per RPC, which
causes performance to suffer needlessly as stack reallocations require
all sorts of internal work such as changing pointers to point to new
addresses.

Since this stack growth is guaranteed to happen at least twice per RPC,
reusing goroutines gives us two wins:

  1. The stack is already grown to 8 KiB after the first RPC, so
     subsequent RPCs do not call runtime.morestack.
  2. We eliminate the need to spawn a new goroutine for each request
     (even though they're relatively inexpensive).

Performance improves across the board. The improvement is especially
visible in small, unary requests as the overhead of stack reallocation
is higher, percentage-wise. QPS is up anywhere between 3% and 5%
depending on the number of concurrent RPC requests in flight. Latency is
down ~3%. There is even a 1% decrease in memory footprint in some cases,
though that is an unintended, but happy coincidence.

unary-networkMode_none-bufConn_false-keepalive_false-benchTime_1m0s-trace_false-latency_0s-kbps_0-MTU_0-maxConcurrentCalls_8-reqSize_1B-respSize_1B-compressor_off-channelz_false-preloader_false
               Title       Before        After Percentage
            TotalOps      2613512      2701705     3.37%
             SendOps            0            0      NaN%
             RecvOps            0            0      NaN%
            Bytes/op      8657.00      8654.17    -0.03%
           Allocs/op       173.37       173.28     0.00%
             ReqT/op    348468.27    360227.33     3.37%
            RespT/op    348468.27    360227.33     3.37%
            50th-Lat    174.601µs    167.378µs    -4.14%
            90th-Lat    233.132µs    229.087µs    -1.74%
            99th-Lat     438.98µs    441.857µs     0.66%
             Avg-Lat    183.263µs     177.26µs    -3.28%
---
 server.go | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 96 insertions(+), 4 deletions(-)

diff --git a/server.go b/server.go
index edfcdcaee9e3..bac3c04cbce4 100644
--- a/server.go
+++ b/server.go
@@ -42,6 +42,7 @@ import (
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/binarylog"
 	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/keepalive"
@@ -87,6 +88,12 @@ type service struct {
 	mdata  interface{}
 }
 
+type serverWorkerData struct {
+	st     transport.ServerTransport
+	wg     *sync.WaitGroup
+	stream *transport.Stream
+}
+
 // Server is a gRPC server to serve RPC requests.
 type Server struct {
 	opts serverOptions
@@ -107,6 +114,8 @@ type Server struct {
 
 	channelzID int64 // channelz unique identification number
 	czData     *channelzData
+
+	serverWorkerChannels []chan *serverWorkerData
 }
 
 type serverOptions struct {
@@ -133,6 +142,7 @@ type serverOptions struct {
 	connectionTimeout     time.Duration
 	maxHeaderListSize     *uint32
 	headerTableSize       *uint32
+	numServerWorkers      uint32
 }
 
 var defaultServerOptions = serverOptions{
@@ -410,6 +420,66 @@ func HeaderTableSize(s uint32) ServerOption {
 	})
 }
 
+// NumStreamWorkers returns a ServerOption that sets the number of worker
+// goroutines that should be used to process incoming streams. Setting this to
+// zero (default) will disable workers and spawn a new goroutine for each
+// stream.
+//
+// This API is EXPERIMENTAL.
+func NumStreamWorkers(numServerWorkers uint32) ServerOption {
+	// TODO: If/when this API gets stabilized (i.e. stream workers become the
+	// only way streams are processed), change the behavior of the zero value to
+	// a sane default. Preliminary experiments suggest that a value equal to the
+	// number of CPUs available is most performant; requires thorough testing.
+	return newFuncServerOption(func(o *serverOptions) {
+		o.numServerWorkers = numServerWorkers
+	})
+}
+
+// serverWorkerResetThreshold defines how often the stack must be reset. Every
+// N requests, by spawning a new goroutine in its place, a worker can reset its
+// stack so that large stacks don't live in memory forever. 2^16 should allow
+// each goroutine stack to live for at least a few seconds in a typical
+// workload (assuming a QPS of a few thousand requests/sec).
+const serverWorkerResetThreshold = 1 << 16
+
+// serverWorkers blocks on a *transport.Stream channel forever and waits for
+// data to be fed by serveStreams. This allows different requests to be
+// processed by the same goroutine, removing the need for expensive stack
+// re-allocations (see the runtime.morestack problem [1]).
+//
+// [1] https://github.com/golang/go/issues/18138
+func (s *Server) serverWorker(ch chan *serverWorkerData) {
+	// To make sure all server workers don't reset at the same time, choose a
+	// random number of iterations before resetting.
+	threshold := serverWorkerResetThreshold + grpcrand.Intn(serverWorkerResetThreshold)
+	for completed := 0; completed < threshold; completed++ {
+		data, ok := <-ch
+		if !ok {
+			return
+		}
+		s.handleStream(data.st, data.stream, s.traceInfo(data.st, data.stream))
+		data.wg.Done()
+	}
+	go s.serverWorker(ch)
+}
+
+// initServerWorkers creates worker goroutines and channels to process incoming
+// connections to reduce the time spent overall on runtime.morestack.
+func (s *Server) initServerWorkers() {
+	s.serverWorkerChannels = make([]chan *serverWorkerData, s.opts.numServerWorkers)
+	for i := uint32(0); i < s.opts.numServerWorkers; i++ {
+		s.serverWorkerChannels[i] = make(chan *serverWorkerData)
+		go s.serverWorker(s.serverWorkerChannels[i])
+	}
+}
+
+func (s *Server) stopServerWorkers() {
+	for i := uint32(0); i < s.opts.numServerWorkers; i++ {
+		close(s.serverWorkerChannels[i])
+	}
+}
+
 // NewServer creates a gRPC server which has no service registered and has not
 // started to accept requests yet.
 func NewServer(opt ...ServerOption) *Server {
@@ -434,6 +504,10 @@ func NewServer(opt ...ServerOption) *Server {
 		s.events = trace.NewEventLog("grpc.Server", fmt.Sprintf("%s:%d", file, line))
 	}
 
+	if s.opts.numServerWorkers > 0 {
+		s.initServerWorkers()
+	}
+
 	if channelz.IsOn() {
 		s.channelzID = channelz.RegisterServer(&channelzServer{s}, "")
 	}
@@ -739,12 +813,27 @@ func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) tr
 func (s *Server) serveStreams(st transport.ServerTransport) {
 	defer st.Close()
 	var wg sync.WaitGroup
+
+	var roundRobinCounter uint32
 	st.HandleStreams(func(stream *transport.Stream) {
 		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			s.handleStream(st, stream, s.traceInfo(st, stream))
-		}()
+		if s.opts.numServerWorkers > 0 {
+			data := &serverWorkerData{st: st, wg: &wg, stream: stream}
+			select {
+			case s.serverWorkerChannels[atomic.AddUint32(&roundRobinCounter, 1)%s.opts.numServerWorkers] <- data:
+			default:
+				// If all stream workers are busy, fallback to the default code path.
+				go func() {
+					s.handleStream(st, stream, s.traceInfo(st, stream))
+					wg.Done()
+				}()
+			}
+		} else {
+			go func() {
+				defer wg.Done()
+				s.handleStream(st, stream, s.traceInfo(st, stream))
+			}()
+		}
 	}, func(ctx context.Context, method string) context.Context {
 		if !EnableTracing {
 			return ctx
@@ -1507,6 +1596,9 @@ func (s *Server) Stop() {
 	for c := range st {
 		c.Close()
 	}
+	if s.opts.numServerWorkers > 0 {
+		s.stopServerWorkers()
+	}
 
 	s.mu.Lock()
 	if s.events != nil {

From c10d2c6f855eb43236351962b0e3114e1bf703cb Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 23 Apr 2020 16:41:27 -0700
Subject: [PATCH 042/481] rls: Update rls protobufs. (#3562)

The `Target` field in the response has been deprecated in favor or
`Targets` which specifies a list of possible targets. Pulled in the
latest proto and updated code referencing it.
---
 balancer/rls/internal/client.go               |  4 +-
 balancer/rls/internal/client_test.go          | 24 +++----
 .../internal/proto/grpc_lookup_v1/rls.pb.go   | 70 ++++++++++++-------
 .../proto/grpc_lookup_v1/rls_config.pb.go     |  4 +-
 4 files changed, 59 insertions(+), 43 deletions(-)

diff --git a/balancer/rls/internal/client.go b/balancer/rls/internal/client.go
index 79acc9da1f47..b6fe22572949 100644
--- a/balancer/rls/internal/client.go
+++ b/balancer/rls/internal/client.go
@@ -62,7 +62,7 @@ func newRLSClient(cc *grpc.ClientConn, dialTarget string, rpcTimeout time.Durati
 	}
 }
 
-type lookupCallback func(target, headerData string, err error)
+type lookupCallback func(targets []string, headerData string, err error)
 
 // lookup starts a RouteLookup RPC in a separate goroutine and returns the
 // results (and error, if any) in the provided callback.
@@ -75,7 +75,7 @@ func (c *rlsClient) lookup(path string, keyMap map[string]string, cb lookupCallb
 			TargetType: grpcTargetType,
 			KeyMap:     keyMap,
 		})
-		cb(resp.GetTarget(), resp.GetHeaderData(), err)
+		cb(resp.GetTargets(), resp.GetHeaderData(), err)
 		cancel()
 	}()
 }
diff --git a/balancer/rls/internal/client_test.go b/balancer/rls/internal/client_test.go
index 7e6342b41f1f..386267b9033d 100644
--- a/balancer/rls/internal/client_test.go
+++ b/balancer/rls/internal/client_test.go
@@ -69,13 +69,13 @@ func TestLookupFailure(t *testing.T) {
 	rlsClient := newRLSClient(cc, defaultDialTarget, defaultRPCTimeout)
 
 	errCh := make(chan error)
-	rlsClient.lookup("", nil, func(target, headerData string, err error) {
+	rlsClient.lookup("", nil, func(targets []string, headerData string, err error) {
 		if err == nil {
 			errCh <- errors.New("rlsClient.lookup() succeeded, should have failed")
 			return
 		}
-		if target != "" || headerData != "" {
-			errCh <- fmt.Errorf("rlsClient.lookup() = (%s, %s), should be empty strings", target, headerData)
+		if len(targets) != 0 || headerData != "" {
+			errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (nil, \"\")", targets, headerData)
 			return
 		}
 		errCh <- nil
@@ -104,7 +104,7 @@ func TestLookupDeadlineExceeded(t *testing.T) {
 	rlsClient := newRLSClient(cc, defaultDialTarget, 100*time.Millisecond)
 
 	errCh := make(chan error)
-	rlsClient.lookup("", nil, func(target, headerData string, err error) {
+	rlsClient.lookup("", nil, func(_ []string, _ string, err error) {
 		if st, ok := status.FromError(err); !ok || st.Code() != codes.DeadlineExceeded {
 			errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded)
 			return
@@ -130,9 +130,8 @@ func TestLookupSuccess(t *testing.T) {
 	defer cleanup()
 
 	const (
-		rlsReqPath    = "/service/method"
-		rlsRespTarget = "us_east_1.firestore.googleapis.com"
-		rlsHeaderData = "headerData"
+		rlsReqPath     = "/service/method"
+		wantHeaderData = "headerData"
 	)
 
 	rlsReqKeyMap := map[string]string{
@@ -145,17 +144,18 @@ func TestLookupSuccess(t *testing.T) {
 		TargetType: "grpc",
 		KeyMap:     rlsReqKeyMap,
 	}
+	wantRespTargets := []string{"us_east_1.firestore.googleapis.com"}
 
 	rlsClient := newRLSClient(cc, defaultDialTarget, defaultRPCTimeout)
 
 	errCh := make(chan error)
-	rlsClient.lookup(rlsReqPath, rlsReqKeyMap, func(t, hd string, err error) {
+	rlsClient.lookup(rlsReqPath, rlsReqKeyMap, func(targets []string, hd string, err error) {
 		if err != nil {
 			errCh <- fmt.Errorf("rlsClient.Lookup() failed: %v", err)
 			return
 		}
-		if t != rlsRespTarget || hd != rlsHeaderData {
-			errCh <- fmt.Errorf("rlsClient.lookup() = (%s, %s), want (%s, %s)", t, hd, rlsRespTarget, rlsHeaderData)
+		if !cmp.Equal(targets, wantRespTargets) || hd != wantHeaderData {
+			errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, hd, wantRespTargets, wantHeaderData)
 			return
 		}
 		errCh <- nil
@@ -180,8 +180,8 @@ func TestLookupSuccess(t *testing.T) {
 	// request.
 	server.ResponseChan <- fakeserver.Response{
 		Resp: &rlspb.RouteLookupResponse{
-			Target:     rlsRespTarget,
-			HeaderData: rlsHeaderData,
+			Targets:    wantRespTargets,
+			HeaderData: wantHeaderData,
 		},
 	}
 
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index ef64866c509b..ad9fa4c48fd5 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -97,7 +97,14 @@ func (m *RouteLookupRequest) GetKeyMap() map[string]string {
 type RouteLookupResponse struct {
 	// Actual addressable entity to use for routing decision, using syntax
 	// requested by the request target_type.
-	Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"`
+	// This field is deprecated in favor of the new "targets" field, below.
+	Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Deprecated: Do not use.
+	// Prioritized list (best one first) of addressable entities to use
+	// for routing, using syntax requested by the request target_type.
+	// The targets will be tried in order until a healthy one is found.
+	// If present, it should be used by proxy/gRPC client code instead of
+	// "target" (which is deprecated).
+	Targets []string `protobuf:"bytes,3,rep,name=targets,proto3" json:"targets,omitempty"`
 	// Optional header value to pass along to AFE in the X-Google-RLS-Data header.
 	// Cached with "target" and sent with all requests that match the request key.
 	// Allows the RLS to pass its work product to the eventual target.
@@ -132,6 +139,7 @@ func (m *RouteLookupResponse) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_RouteLookupResponse proto.InternalMessageInfo
 
+// Deprecated: Do not use.
 func (m *RouteLookupResponse) GetTarget() string {
 	if m != nil {
 		return m.Target
@@ -139,6 +147,13 @@ func (m *RouteLookupResponse) GetTarget() string {
 	return ""
 }
 
+func (m *RouteLookupResponse) GetTargets() []string {
+	if m != nil {
+		return m.Targets
+	}
+	return nil
+}
+
 func (m *RouteLookupResponse) GetHeaderData() string {
 	if m != nil {
 		return m.HeaderData
@@ -155,37 +170,38 @@ func init() {
 func init() { proto.RegisterFile("grpc/rls/grpc_lookup_v1/rls.proto", fileDescriptor_5fe9649e373b9d12) }
 
 var fileDescriptor_5fe9649e373b9d12 = []byte{
-	// 325 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x4b, 0xc3, 0x30,
-	0x14, 0xc6, 0xed, 0x36, 0xa7, 0xbe, 0x82, 0x68, 0x14, 0x29, 0xbb, 0x38, 0xeb, 0x65, 0x07, 0xc9,
-	0xd8, 0xbc, 0xa8, 0xc7, 0xa1, 0x78, 0xd0, 0xc9, 0xa8, 0x1e, 0xc4, 0x4b, 0x89, 0xdb, 0x23, 0x1b,
-	0xad, 0x4d, 0x4c, 0xd3, 0x42, 0xff, 0x60, 0xff, 0x0f, 0x49, 0x52, 0x61, 0x9d, 0xa0, 0xb7, 0xf7,
-	0xfd, 0xde, 0x23, 0xf9, 0xbe, 0xe4, 0xc1, 0x19, 0x57, 0x72, 0x3e, 0x54, 0x69, 0x3e, 0x34, 0x45,
-	0x9c, 0x0a, 0x91, 0x14, 0x32, 0x2e, 0x47, 0x06, 0x51, 0xa9, 0x84, 0x16, 0x64, 0xdf, 0x74, 0xa8,
-	0xeb, 0xd0, 0x72, 0x14, 0x7e, 0x79, 0x40, 0x22, 0x51, 0x68, 0x7c, 0xb4, 0x28, 0xc2, 0xcf, 0x02,
-	0x73, 0x4d, 0x4e, 0xa0, 0x9b, 0xa3, 0x2a, 0x51, 0x05, 0x5e, 0xdf, 0x1b, 0xec, 0x45, 0xb5, 0x22,
-	0x04, 0x3a, 0x92, 0xe9, 0x65, 0xd0, 0xb2, 0xd4, 0xd6, 0xe4, 0x14, 0x7c, 0xcd, 0x14, 0x47, 0x1d,
-	0xeb, 0x4a, 0x62, 0xd0, 0xb6, 0x2d, 0x70, 0xe8, 0xa5, 0x92, 0x48, 0xee, 0x61, 0x27, 0xc1, 0x2a,
-	0xfe, 0x60, 0x32, 0xe8, 0xf4, 0xdb, 0x03, 0x7f, 0x4c, 0x69, 0xd3, 0x05, 0xfd, 0xed, 0x80, 0x3e,
-	0x60, 0x35, 0x65, 0xf2, 0x2e, 0xd3, 0xaa, 0x8a, 0xba, 0x89, 0x15, 0xbd, 0x6b, 0xf0, 0xd7, 0x30,
-	0x39, 0x80, 0x76, 0x82, 0x55, 0xed, 0xd0, 0x94, 0xe4, 0x18, 0xb6, 0x4b, 0x96, 0x16, 0x58, 0xfb,
-	0x73, 0xe2, 0xa6, 0x75, 0xe5, 0x85, 0x4f, 0x70, 0xd4, 0xb8, 0x24, 0x97, 0x22, 0xcb, 0xd1, 0xe4,
-	0x74, 0x46, 0x7f, 0x72, 0x3a, 0x65, 0x32, 0x2d, 0x91, 0x2d, 0x50, 0xc5, 0x0b, 0xa6, 0x59, 0x7d,
-	0x1c, 0x38, 0x74, 0xcb, 0x34, 0x1b, 0x67, 0x8d, 0x67, 0x7b, 0x46, 0x55, 0xae, 0xe6, 0x48, 0x5e,
-	0xc1, 0x5f, 0xa3, 0x24, 0xfc, 0x3f, 0x67, 0xef, 0xfc, 0xcf, 0x19, 0x67, 0x33, 0xdc, 0x9a, 0x4c,
-	0xe1, 0x70, 0x25, 0x36, 0x46, 0x27, 0xbb, 0x51, 0x9a, 0xcf, 0xcc, 0xb7, 0xce, 0xbc, 0xb7, 0x0b,
-	0x2e, 0x04, 0x4f, 0x91, 0x72, 0x91, 0xb2, 0x8c, 0x53, 0xa1, 0xb8, 0x5d, 0x82, 0xa1, 0x9b, 0xde,
-	0x58, 0x88, 0xf7, 0xae, 0xdd, 0x86, 0xcb, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0xba, 0x10, 0x2d,
-	0xb5, 0x32, 0x02, 0x00, 0x00,
+	// 345 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x4b, 0xeb, 0x40,
+	0x10, 0xc6, 0x5f, 0x9a, 0xbe, 0xf4, 0x75, 0x02, 0x8f, 0xf7, 0x56, 0x91, 0xd0, 0x8b, 0x35, 0x5e,
+	0x7a, 0x90, 0x2d, 0xad, 0x17, 0xf5, 0x58, 0x14, 0x0f, 0x5a, 0x28, 0xd1, 0x83, 0x78, 0x09, 0x6b,
+	0x3b, 0xa4, 0x25, 0x6b, 0x76, 0xdd, 0xdd, 0x04, 0xf2, 0x07, 0xfb, 0x7f, 0x48, 0x76, 0x23, 0xb4,
+	0x15, 0xf4, 0xf6, 0x7d, 0xbf, 0x19, 0xb2, 0xdf, 0x4c, 0x06, 0x4e, 0x32, 0x25, 0x97, 0x63, 0xc5,
+	0xf5, 0xb8, 0x11, 0x29, 0x17, 0x22, 0x2f, 0x65, 0x5a, 0x4d, 0x1a, 0x44, 0xa5, 0x12, 0x46, 0x90,
+	0xbf, 0x4d, 0x85, 0xba, 0x0a, 0xad, 0x26, 0xf1, 0xbb, 0x07, 0x24, 0x11, 0xa5, 0xc1, 0x7b, 0x8b,
+	0x12, 0x7c, 0x2b, 0x51, 0x1b, 0x72, 0x04, 0x81, 0x46, 0x55, 0xa1, 0x8a, 0xbc, 0xa1, 0x37, 0xea,
+	0x27, 0xad, 0x23, 0x04, 0xba, 0x92, 0x99, 0x75, 0xd4, 0xb1, 0xd4, 0x6a, 0x72, 0x0c, 0xa1, 0x61,
+	0x2a, 0x43, 0x93, 0x9a, 0x5a, 0x62, 0xe4, 0xdb, 0x12, 0x38, 0xf4, 0x58, 0x4b, 0x24, 0xb7, 0xd0,
+	0xcb, 0xb1, 0x4e, 0x5f, 0x99, 0x8c, 0xba, 0x43, 0x7f, 0x14, 0x4e, 0x29, 0xdd, 0x4d, 0x41, 0xbf,
+	0x26, 0xa0, 0x77, 0x58, 0xcf, 0x99, 0xbc, 0x29, 0x8c, 0xaa, 0x93, 0x20, 0xb7, 0x66, 0x70, 0x09,
+	0xe1, 0x16, 0x26, 0xff, 0xc0, 0xcf, 0xb1, 0x6e, 0x13, 0x36, 0x92, 0x1c, 0xc2, 0xef, 0x8a, 0xf1,
+	0x12, 0xdb, 0x7c, 0xce, 0x5c, 0x75, 0x2e, 0xbc, 0x98, 0xc3, 0xc1, 0xce, 0x23, 0x5a, 0x8a, 0x42,
+	0x23, 0x19, 0x40, 0xe0, 0x82, 0xba, 0xaf, 0xcc, 0x3a, 0x91, 0x97, 0xb4, 0x84, 0x44, 0xd0, 0x73,
+	0x4a, 0x47, 0xfe, 0xd0, 0x1f, 0xf5, 0x93, 0x4f, 0xdb, 0x4c, 0xbc, 0x46, 0xb6, 0x42, 0x95, 0xae,
+	0x98, 0x61, 0xed, 0x63, 0xe0, 0xd0, 0x35, 0x33, 0x6c, 0x5a, 0xec, 0x2c, 0xf5, 0x01, 0x55, 0xb5,
+	0x59, 0x22, 0x79, 0x82, 0x70, 0x8b, 0x92, 0xf8, 0xe7, 0x2d, 0x0c, 0x4e, 0xbf, 0xed, 0x71, 0x43,
+	0xc4, 0xbf, 0x66, 0x73, 0xf8, 0xbf, 0x11, 0x7b, 0xad, 0xb3, 0x3f, 0x09, 0xd7, 0x8b, 0xe6, 0xa7,
+	0x2f, 0xbc, 0xe7, 0xb3, 0x4c, 0x88, 0x8c, 0x23, 0xcd, 0x04, 0x67, 0x45, 0x46, 0x85, 0xca, 0xec,
+	0x89, 0x8c, 0x5d, 0xf7, 0xde, 0xb9, 0xbc, 0x04, 0xf6, 0x56, 0xce, 0x3f, 0x02, 0x00, 0x00, 0xff,
+	0xff, 0xb4, 0xe3, 0xe2, 0xd0, 0x50, 0x02, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ context.Context
-var _ grpc.ClientConn
+var _ grpc.ClientConnInterface
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
+const _ = grpc.SupportPackageIsVersion6
 
 // RouteLookupServiceClient is the client API for RouteLookupService service.
 //
@@ -196,10 +212,10 @@ type RouteLookupServiceClient interface {
 }
 
 type routeLookupServiceClient struct {
-	cc *grpc.ClientConn
+	cc grpc.ClientConnInterface
 }
 
-func NewRouteLookupServiceClient(cc *grpc.ClientConn) RouteLookupServiceClient {
+func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
 	return &routeLookupServiceClient{cc}
 }
 
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
index 5e2942a7c14a..88479f41e397 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
@@ -135,7 +135,7 @@ type GrpcKeyBuilder struct {
 	Names []*GrpcKeyBuilder_Name `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"`
 	// Extract keys from all listed headers.
 	// For gRPC, it is an error to specify "required_match" on the NameMatcher
-	// protos, and we ignore it if set.
+	// protos.
 	Headers              []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
 	XXX_unrecognized     []byte         `json:"-"`
@@ -379,7 +379,7 @@ type RouteLookupConfig struct {
 	// Defaults to 10 seconds if not specified.
 	LookupServiceTimeout *duration.Duration `protobuf:"bytes,4,opt,name=lookup_service_timeout,json=lookupServiceTimeout,proto3" json:"lookup_service_timeout,omitempty"`
 	// How long are responses valid for (like HTTP Cache-Control).
-	// If omitted (i.e. 0), responses are considered not to be cacheable.
+	// If omitted or zero, the longest valid cache time is used.
 	// This value is clamped to 5 minutes to avoid unflushable bad responses.
 	MaxAge *duration.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"`
 	// After a response has been in the client cache for this amount of time

From 7d6aa9ed5e020314d49a1331f21e62207b3845fd Mon Sep 17 00:00:00 2001
From: Eric Gribkoff <ericgribkoff@google.com>
Date: Thu, 23 Apr 2020 17:54:09 -0700
Subject: [PATCH 043/481] kokoro: use xds-test-server image (#3564)

---
 test/kokoro/xds.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index e82d3b781d6e..788c7c9f7e3a 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -23,6 +23,8 @@ GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
     --test_case=all \
     --project_id=grpc-testing \
+    --source_image=projects/grpc-testing/global/images/xds-test-server \
+    --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \
     --gcp_suffix=$(date '+%s') \
     --verbose \
     --client_cmd="grpc-go/interop/xds/client/client \

From 15653fec60abffdd4ed23ec7b488cbd6e1a34160 Mon Sep 17 00:00:00 2001
From: Damien Neil <neild@users.noreply.github.com>
Date: Fri, 24 Apr 2020 08:54:03 -0700
Subject: [PATCH 044/481] all: refactor .pb.go generation (#3451)

Replace various //go:generate lines and regenerate.sh scripts with a
single, top-level regenerate.sh that regenerates all .pb.go files.

Placing generation in a single script ensures that all files are
generated with similar parameters. The new regenerate.sh uses the
protoc-gen-go version defined in test/tools/go.mod and automatically
handles new .proto files as they are added.

Do some minor refactoring on .proto files: Every file now has a
go_package option (which will be required by a future version of the
code generator), and file imports are all relative to the repository
root.
---
 balancer/grpclb/grpclb.go                     |   2 -
 balancer/grpclb/regenerate.sh                 |  33 --
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  56 +--
 balancer/rls/internal/proto/regenerate.sh     |  33 --
 benchmark/benchmark.go                        |   2 -
 benchmark/grpc_testing/control.pb.go          | 207 +++++------
 benchmark/grpc_testing/control.proto          |   6 +-
 benchmark/grpc_testing/messages.pb.go         | 123 +++----
 benchmark/grpc_testing/messages.proto         |   2 +
 benchmark/grpc_testing/payloads.pb.go         |  55 +--
 benchmark/grpc_testing/payloads.proto         |   2 +
 benchmark/grpc_testing/services.pb.go         |  53 +--
 benchmark/grpc_testing/services.proto         |   6 +-
 benchmark/grpc_testing/stats.pb.go            |  66 ++--
 benchmark/grpc_testing/stats.proto            |   2 +
 binarylog/grpc_binarylog_v1/binarylog.pb.go   | 320 +++++++-----------
 channelz/service/regenerate.sh                |  33 --
 channelz/service/service.go                   |   2 -
 credentials/alts/internal/common.go           |   2 -
 credentials/alts/internal/regenerate.sh       |  35 --
 doc.go                                        |   2 +
 examples/features/proto/doc.go                |  22 --
 examples/features/proto/echo/echo.pb.go       |  44 +--
 examples/features/proto/echo/echo.proto       |   4 +-
 examples/helloworld/greeter_server/main.go    |   2 -
 .../helloworld/helloworld/helloworld.pb.go    |  42 +--
 .../helloworld/helloworld/helloworld.proto    |   1 +
 .../route_guide/routeguide/route_guide.pb.go  |  78 +++--
 .../route_guide/routeguide/route_guide.proto  |   1 +
 examples/route_guide/server/server.go         |   2 -
 health/regenerate.sh                          |  33 --
 health/server.go                              |   2 -
 internal/binarylog/regenerate.sh              |  33 --
 .../proto/grpc_service_config/example_test.go |   2 -
 .../proto/grpc_service_config/regenerate.sh   |  35 --
 interop/grpc_testing/test.pb.go               | 166 ++++-----
 interop/grpc_testing/test.proto               |   2 +
 interop/test_utils.go                         |   2 -
 profiling/proto/service.pb.go                 |  74 ++--
 profiling/service/service.go                  |   2 -
 .../grpc_reflection_v1alpha/reflection.pb.go  | 114 ++++---
 .../grpc_reflection_v1alpha/reflection.proto  |   2 +
 reflection/grpc_testing/proto2.pb.go          |  27 +-
 reflection/grpc_testing/proto2.proto          |   2 +
 reflection/grpc_testing/proto2_ext.pb.go      |  44 +--
 reflection/grpc_testing/proto2_ext.proto      |   6 +-
 reflection/grpc_testing/proto2_ext2.pb.go     |  40 ++-
 reflection/grpc_testing/proto2_ext2.proto     |   4 +-
 reflection/grpc_testing/test.pb.go            |  52 +--
 reflection/grpc_testing/test.proto            |   2 +
 reflection/grpc_testingv3/testv3.proto        |   2 +
 reflection/serverreflection.go                |   2 -
 reflection/serverreflection_test.go           |  19 +-
 regenerate.sh                                 |  76 +++++
 stats/grpc_testing/test.pb.go                 |  42 +--
 stats/grpc_testing/test.proto                 |   2 +
 stats/stats.go                                |   2 -
 stress/client/main.go                         |   2 -
 stress/grpc_testing/metrics.pb.go             |  48 +--
 stress/grpc_testing/metrics.proto             |   2 +
 test/codec_perf/perf.pb.go                    |  24 +-
 test/codec_perf/perf.proto                    |   2 +
 test/end2end_test.go                          |   3 -
 test/grpc_testing/test.pb.go                  | 108 +++---
 test/grpc_testing/test.proto                  |   2 +
 vet.sh                                        |   6 +-
 66 files changed, 1009 insertions(+), 1215 deletions(-)
 delete mode 100755 balancer/grpclb/regenerate.sh
 delete mode 100755 balancer/rls/internal/proto/regenerate.sh
 delete mode 100755 channelz/service/regenerate.sh
 delete mode 100755 credentials/alts/internal/regenerate.sh
 delete mode 100644 examples/features/proto/doc.go
 delete mode 100755 health/regenerate.sh
 delete mode 100755 internal/binarylog/regenerate.sh
 delete mode 100755 internal/proto/grpc_service_config/regenerate.sh
 create mode 100755 regenerate.sh

diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go
index 219ca7235b48..55ccd065a2ba 100644
--- a/balancer/grpclb/grpclb.go
+++ b/balancer/grpclb/grpclb.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate ./regenerate.sh
-
 // Package grpclb defines a grpclb balancer.
 //
 // To install grpclb balancer, import this package as:
diff --git a/balancer/grpclb/regenerate.sh b/balancer/grpclb/regenerate.sh
deleted file mode 100755
index b8978e11b38e..000000000000
--- a/balancer/grpclb/regenerate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2018 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/lb/v1
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/lb/v1/load_balancer.proto > grpc/lb/v1/load_balancer.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/lb/v1/*.proto
-popd
-rm -f grpc_lb_v1/*.pb.go
-cp "$TMP"/grpc/lb/v1/*.pb.go grpc_lb_v1/
-
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index ad9fa4c48fd5..57cb1c93ba50 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc/rls/grpc_lookup_v1/rls.proto
+// source: grpc/lookup/v1/rls.proto
 
 package grpc_lookup_v1
 
@@ -45,7 +45,7 @@ func (m *RouteLookupRequest) Reset()         { *m = RouteLookupRequest{} }
 func (m *RouteLookupRequest) String() string { return proto.CompactTextString(m) }
 func (*RouteLookupRequest) ProtoMessage()    {}
 func (*RouteLookupRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe9649e373b9d12, []int{0}
+	return fileDescriptor_3bab962d3362f3ca, []int{0}
 }
 
 func (m *RouteLookupRequest) XXX_Unmarshal(b []byte) error {
@@ -118,7 +118,7 @@ func (m *RouteLookupResponse) Reset()         { *m = RouteLookupResponse{} }
 func (m *RouteLookupResponse) String() string { return proto.CompactTextString(m) }
 func (*RouteLookupResponse) ProtoMessage()    {}
 func (*RouteLookupResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe9649e373b9d12, []int{1}
+	return fileDescriptor_3bab962d3362f3ca, []int{1}
 }
 
 func (m *RouteLookupResponse) XXX_Unmarshal(b []byte) error {
@@ -167,32 +167,32 @@ func init() {
 	proto.RegisterType((*RouteLookupResponse)(nil), "grpc.lookup.v1.RouteLookupResponse")
 }
 
-func init() { proto.RegisterFile("grpc/rls/grpc_lookup_v1/rls.proto", fileDescriptor_5fe9649e373b9d12) }
+func init() { proto.RegisterFile("grpc/lookup/v1/rls.proto", fileDescriptor_3bab962d3362f3ca) }
 
-var fileDescriptor_5fe9649e373b9d12 = []byte{
-	// 345 bytes of a gzipped FileDescriptorProto
+var fileDescriptor_3bab962d3362f3ca = []byte{
+	// 343 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x4b, 0xeb, 0x40,
-	0x10, 0xc6, 0x5f, 0x9a, 0xbe, 0xf4, 0x75, 0x02, 0x8f, 0xf7, 0x56, 0x91, 0xd0, 0x8b, 0x35, 0x5e,
-	0x7a, 0x90, 0x2d, 0xad, 0x17, 0xf5, 0x58, 0x14, 0x0f, 0x5a, 0x28, 0xd1, 0x83, 0x78, 0x09, 0x6b,
-	0x3b, 0xa4, 0x25, 0x6b, 0x76, 0xdd, 0xdd, 0x04, 0xf2, 0x07, 0xfb, 0x7f, 0x48, 0x76, 0x23, 0xb4,
-	0x15, 0xf4, 0xf6, 0x7d, 0xbf, 0x19, 0xb2, 0xdf, 0x4c, 0x06, 0x4e, 0x32, 0x25, 0x97, 0x63, 0xc5,
-	0xf5, 0xb8, 0x11, 0x29, 0x17, 0x22, 0x2f, 0x65, 0x5a, 0x4d, 0x1a, 0x44, 0xa5, 0x12, 0x46, 0x90,
-	0xbf, 0x4d, 0x85, 0xba, 0x0a, 0xad, 0x26, 0xf1, 0xbb, 0x07, 0x24, 0x11, 0xa5, 0xc1, 0x7b, 0x8b,
-	0x12, 0x7c, 0x2b, 0x51, 0x1b, 0x72, 0x04, 0x81, 0x46, 0x55, 0xa1, 0x8a, 0xbc, 0xa1, 0x37, 0xea,
-	0x27, 0xad, 0x23, 0x04, 0xba, 0x92, 0x99, 0x75, 0xd4, 0xb1, 0xd4, 0x6a, 0x72, 0x0c, 0xa1, 0x61,
-	0x2a, 0x43, 0x93, 0x9a, 0x5a, 0x62, 0xe4, 0xdb, 0x12, 0x38, 0xf4, 0x58, 0x4b, 0x24, 0xb7, 0xd0,
-	0xcb, 0xb1, 0x4e, 0x5f, 0x99, 0x8c, 0xba, 0x43, 0x7f, 0x14, 0x4e, 0x29, 0xdd, 0x4d, 0x41, 0xbf,
-	0x26, 0xa0, 0x77, 0x58, 0xcf, 0x99, 0xbc, 0x29, 0x8c, 0xaa, 0x93, 0x20, 0xb7, 0x66, 0x70, 0x09,
-	0xe1, 0x16, 0x26, 0xff, 0xc0, 0xcf, 0xb1, 0x6e, 0x13, 0x36, 0x92, 0x1c, 0xc2, 0xef, 0x8a, 0xf1,
-	0x12, 0xdb, 0x7c, 0xce, 0x5c, 0x75, 0x2e, 0xbc, 0x98, 0xc3, 0xc1, 0xce, 0x23, 0x5a, 0x8a, 0x42,
-	0x23, 0x19, 0x40, 0xe0, 0x82, 0xba, 0xaf, 0xcc, 0x3a, 0x91, 0x97, 0xb4, 0x84, 0x44, 0xd0, 0x73,
-	0x4a, 0x47, 0xfe, 0xd0, 0x1f, 0xf5, 0x93, 0x4f, 0xdb, 0x4c, 0xbc, 0x46, 0xb6, 0x42, 0x95, 0xae,
-	0x98, 0x61, 0xed, 0x63, 0xe0, 0xd0, 0x35, 0x33, 0x6c, 0x5a, 0xec, 0x2c, 0xf5, 0x01, 0x55, 0xb5,
-	0x59, 0x22, 0x79, 0x82, 0x70, 0x8b, 0x92, 0xf8, 0xe7, 0x2d, 0x0c, 0x4e, 0xbf, 0xed, 0x71, 0x43,
-	0xc4, 0xbf, 0x66, 0x73, 0xf8, 0xbf, 0x11, 0x7b, 0xad, 0xb3, 0x3f, 0x09, 0xd7, 0x8b, 0xe6, 0xa7,
-	0x2f, 0xbc, 0xe7, 0xb3, 0x4c, 0x88, 0x8c, 0x23, 0xcd, 0x04, 0x67, 0x45, 0x46, 0x85, 0xca, 0xec,
-	0x89, 0x8c, 0x5d, 0xf7, 0xde, 0xb9, 0xbc, 0x04, 0xf6, 0x56, 0xce, 0x3f, 0x02, 0x00, 0x00, 0xff,
-	0xff, 0xb4, 0xe3, 0xe2, 0xd0, 0x50, 0x02, 0x00, 0x00,
+	0x10, 0xc6, 0x5f, 0x9a, 0xbe, 0xf4, 0x75, 0x02, 0x8f, 0xf7, 0x56, 0x91, 0xa5, 0x17, 0x4b, 0xbc,
+	0xf4, 0x20, 0x1b, 0x5a, 0x2f, 0xea, 0xb1, 0x28, 0x1e, 0xb4, 0x50, 0xa2, 0x07, 0xf1, 0x12, 0xd6,
+	0x76, 0x48, 0x4b, 0xd6, 0xec, 0xba, 0xbb, 0x09, 0xe4, 0x0f, 0xf6, 0xff, 0x90, 0x64, 0x23, 0xb4,
+	0x0a, 0x7a, 0xfb, 0xbe, 0xdf, 0x2c, 0x99, 0x6f, 0x26, 0x03, 0x34, 0xd3, 0x6a, 0x15, 0x0b, 0x29,
+	0xf3, 0x52, 0xc5, 0xd5, 0x34, 0xd6, 0xc2, 0x30, 0xa5, 0xa5, 0x95, 0xe4, 0x6f, 0x53, 0x61, 0xae,
+	0xc2, 0xaa, 0x69, 0xf4, 0xe6, 0x01, 0x49, 0x64, 0x69, 0xf1, 0xae, 0x45, 0x09, 0xbe, 0x96, 0x68,
+	0x2c, 0x39, 0x82, 0xc0, 0xa0, 0xae, 0x50, 0x53, 0x6f, 0xec, 0x4d, 0x86, 0x49, 0xe7, 0x08, 0x81,
+	0xbe, 0xe2, 0x76, 0x43, 0x7b, 0x2d, 0x6d, 0x35, 0x39, 0x86, 0xd0, 0x72, 0x9d, 0xa1, 0x4d, 0x6d,
+	0xad, 0x90, 0xfa, 0x6d, 0x09, 0x1c, 0x7a, 0xa8, 0x15, 0x92, 0x1b, 0x18, 0xe4, 0x58, 0xa7, 0x2f,
+	0x5c, 0xd1, 0xfe, 0xd8, 0x9f, 0x84, 0x33, 0xc6, 0xf6, 0x53, 0xb0, 0xaf, 0x09, 0xd8, 0x2d, 0xd6,
+	0x0b, 0xae, 0xae, 0x0b, 0xab, 0xeb, 0x24, 0xc8, 0x5b, 0x33, 0xba, 0x80, 0x70, 0x07, 0x93, 0x7f,
+	0xe0, 0xe7, 0x58, 0x77, 0x09, 0x1b, 0x49, 0x0e, 0xe1, 0x77, 0xc5, 0x45, 0x89, 0x5d, 0x3e, 0x67,
+	0x2e, 0x7b, 0xe7, 0x5e, 0x24, 0xe0, 0x60, 0xaf, 0x89, 0x51, 0xb2, 0x30, 0x48, 0x46, 0x10, 0xb8,
+	0xa0, 0xee, 0x2b, 0xf3, 0x1e, 0xf5, 0x92, 0x8e, 0x10, 0x0a, 0x03, 0xa7, 0x0c, 0xf5, 0xc7, 0xfe,
+	0x64, 0x98, 0x7c, 0xd8, 0x66, 0xe2, 0x0d, 0xf2, 0x35, 0xea, 0x74, 0xcd, 0x2d, 0xef, 0x9a, 0x81,
+	0x43, 0x57, 0xdc, 0xf2, 0x59, 0xb1, 0xb7, 0xd4, 0x7b, 0xd4, 0xd5, 0x76, 0x85, 0xe4, 0x11, 0xc2,
+	0x1d, 0x4a, 0xa2, 0x9f, 0xb7, 0x30, 0x3a, 0xf9, 0xf6, 0x8d, 0x1b, 0x22, 0xfa, 0x35, 0x5f, 0xc0,
+	0xff, 0xad, 0xfc, 0xf4, 0x74, 0xfe, 0x27, 0x11, 0x66, 0xd9, 0xfc, 0xf4, 0xa5, 0xf7, 0x74, 0x9a,
+	0x49, 0x99, 0x09, 0x64, 0x99, 0x14, 0xbc, 0xc8, 0x98, 0xd4, 0x59, 0xbc, 0x7b, 0x22, 0x8d, 0x4e,
+	0x9d, 0x4e, 0xab, 0xe9, 0x73, 0xd0, 0xde, 0xca, 0xd9, 0x7b, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf5,
+	0x11, 0x43, 0x0e, 0x47, 0x02, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -274,5 +274,5 @@ var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/rls/grpc_lookup_v1/rls.proto",
+	Metadata: "grpc/lookup/v1/rls.proto",
 }
diff --git a/balancer/rls/internal/proto/regenerate.sh b/balancer/rls/internal/proto/regenerate.sh
deleted file mode 100755
index 1a16886a07ed..000000000000
--- a/balancer/rls/internal/proto/regenerate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2020 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/rls/grpc_lookup_v1
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/lookup/v1/rls.proto > grpc/rls/grpc_lookup_v1/rls.proto
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/lookup/v1/rls_config.proto > grpc/rls/grpc_lookup_v1/rls_config.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/rls/grpc_lookup_v1/*.proto
-popd
-rm -f ./grpc_lookup_v1/*.pb.go
-cp "$TMP"/grpc/rls/grpc_lookup_v1/*.pb.go ../../../rls/internal/proto/grpc_lookup_v1/
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index b5e9837210c9..8d9b20f28b1b 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc -I grpc_testing --go_out=plugins=grpc:grpc_testing grpc_testing/control.proto grpc_testing/messages.proto grpc_testing/payloads.proto grpc_testing/services.proto grpc_testing/stats.proto
-
 /*
 Package benchmark implements the building blocks to setup end-to-end gRPC benchmarks.
 */
diff --git a/benchmark/grpc_testing/control.pb.go b/benchmark/grpc_testing/control.pb.go
index c4af112a07b4..7a109d8e9cb8 100644
--- a/benchmark/grpc_testing/control.pb.go
+++ b/benchmark/grpc_testing/control.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: control.proto
+// source: benchmark/grpc_testing/control.proto
 
 package grpc_testing
 
@@ -42,7 +42,7 @@ func (x ClientType) String() string {
 }
 
 func (ClientType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{0}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{0}
 }
 
 type ServerType int32
@@ -70,7 +70,7 @@ func (x ServerType) String() string {
 }
 
 func (ServerType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{1}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{1}
 }
 
 type RpcType int32
@@ -95,7 +95,7 @@ func (x RpcType) String() string {
 }
 
 func (RpcType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{2}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{2}
 }
 
 // Parameters of poisson process distribution, which is a good representation
@@ -112,7 +112,7 @@ func (m *PoissonParams) Reset()         { *m = PoissonParams{} }
 func (m *PoissonParams) String() string { return proto.CompactTextString(m) }
 func (*PoissonParams) ProtoMessage()    {}
 func (*PoissonParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{0}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{0}
 }
 
 func (m *PoissonParams) XXX_Unmarshal(b []byte) error {
@@ -152,7 +152,7 @@ func (m *UniformParams) Reset()         { *m = UniformParams{} }
 func (m *UniformParams) String() string { return proto.CompactTextString(m) }
 func (*UniformParams) ProtoMessage()    {}
 func (*UniformParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{1}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{1}
 }
 
 func (m *UniformParams) XXX_Unmarshal(b []byte) error {
@@ -198,7 +198,7 @@ func (m *DeterministicParams) Reset()         { *m = DeterministicParams{} }
 func (m *DeterministicParams) String() string { return proto.CompactTextString(m) }
 func (*DeterministicParams) ProtoMessage()    {}
 func (*DeterministicParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{2}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{2}
 }
 
 func (m *DeterministicParams) XXX_Unmarshal(b []byte) error {
@@ -238,7 +238,7 @@ func (m *ParetoParams) Reset()         { *m = ParetoParams{} }
 func (m *ParetoParams) String() string { return proto.CompactTextString(m) }
 func (*ParetoParams) ProtoMessage()    {}
 func (*ParetoParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{3}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{3}
 }
 
 func (m *ParetoParams) XXX_Unmarshal(b []byte) error {
@@ -285,7 +285,7 @@ func (m *ClosedLoopParams) Reset()         { *m = ClosedLoopParams{} }
 func (m *ClosedLoopParams) String() string { return proto.CompactTextString(m) }
 func (*ClosedLoopParams) ProtoMessage()    {}
 func (*ClosedLoopParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{4}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{4}
 }
 
 func (m *ClosedLoopParams) XXX_Unmarshal(b []byte) error {
@@ -323,7 +323,7 @@ func (m *LoadParams) Reset()         { *m = LoadParams{} }
 func (m *LoadParams) String() string { return proto.CompactTextString(m) }
 func (*LoadParams) ProtoMessage()    {}
 func (*LoadParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{5}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{5}
 }
 
 func (m *LoadParams) XXX_Unmarshal(b []byte) error {
@@ -444,7 +444,7 @@ func (m *SecurityParams) Reset()         { *m = SecurityParams{} }
 func (m *SecurityParams) String() string { return proto.CompactTextString(m) }
 func (*SecurityParams) ProtoMessage()    {}
 func (*SecurityParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{6}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{6}
 }
 
 func (m *SecurityParams) XXX_Unmarshal(b []byte) error {
@@ -509,7 +509,7 @@ func (m *ClientConfig) Reset()         { *m = ClientConfig{} }
 func (m *ClientConfig) String() string { return proto.CompactTextString(m) }
 func (*ClientConfig) ProtoMessage()    {}
 func (*ClientConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{7}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{7}
 }
 
 func (m *ClientConfig) XXX_Unmarshal(b []byte) error {
@@ -625,7 +625,7 @@ func (m *ClientStatus) Reset()         { *m = ClientStatus{} }
 func (m *ClientStatus) String() string { return proto.CompactTextString(m) }
 func (*ClientStatus) ProtoMessage()    {}
 func (*ClientStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{8}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{8}
 }
 
 func (m *ClientStatus) XXX_Unmarshal(b []byte) error {
@@ -666,7 +666,7 @@ func (m *Mark) Reset()         { *m = Mark{} }
 func (m *Mark) String() string { return proto.CompactTextString(m) }
 func (*Mark) ProtoMessage()    {}
 func (*Mark) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{9}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{9}
 }
 
 func (m *Mark) XXX_Unmarshal(b []byte) error {
@@ -708,7 +708,7 @@ func (m *ClientArgs) Reset()         { *m = ClientArgs{} }
 func (m *ClientArgs) String() string { return proto.CompactTextString(m) }
 func (*ClientArgs) ProtoMessage()    {}
 func (*ClientArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{10}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{10}
 }
 
 func (m *ClientArgs) XXX_Unmarshal(b []byte) error {
@@ -796,7 +796,7 @@ func (m *ServerConfig) Reset()         { *m = ServerConfig{} }
 func (m *ServerConfig) String() string { return proto.CompactTextString(m) }
 func (*ServerConfig) ProtoMessage()    {}
 func (*ServerConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{11}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{11}
 }
 
 func (m *ServerConfig) XXX_Unmarshal(b []byte) error {
@@ -880,7 +880,7 @@ func (m *ServerArgs) Reset()         { *m = ServerArgs{} }
 func (m *ServerArgs) String() string { return proto.CompactTextString(m) }
 func (*ServerArgs) ProtoMessage()    {}
 func (*ServerArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{12}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{12}
 }
 
 func (m *ServerArgs) XXX_Unmarshal(b []byte) error {
@@ -961,7 +961,7 @@ func (m *ServerStatus) Reset()         { *m = ServerStatus{} }
 func (m *ServerStatus) String() string { return proto.CompactTextString(m) }
 func (*ServerStatus) ProtoMessage()    {}
 func (*ServerStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{13}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{13}
 }
 
 func (m *ServerStatus) XXX_Unmarshal(b []byte) error {
@@ -1013,7 +1013,7 @@ func (m *CoreRequest) Reset()         { *m = CoreRequest{} }
 func (m *CoreRequest) String() string { return proto.CompactTextString(m) }
 func (*CoreRequest) ProtoMessage()    {}
 func (*CoreRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{14}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{14}
 }
 
 func (m *CoreRequest) XXX_Unmarshal(b []byte) error {
@@ -1046,7 +1046,7 @@ func (m *CoreResponse) Reset()         { *m = CoreResponse{} }
 func (m *CoreResponse) String() string { return proto.CompactTextString(m) }
 func (*CoreResponse) ProtoMessage()    {}
 func (*CoreResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{15}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{15}
 }
 
 func (m *CoreResponse) XXX_Unmarshal(b []byte) error {
@@ -1084,7 +1084,7 @@ func (m *Void) Reset()         { *m = Void{} }
 func (m *Void) String() string { return proto.CompactTextString(m) }
 func (*Void) ProtoMessage()    {}
 func (*Void) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{16}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{16}
 }
 
 func (m *Void) XXX_Unmarshal(b []byte) error {
@@ -1132,7 +1132,7 @@ func (m *Scenario) Reset()         { *m = Scenario{} }
 func (m *Scenario) String() string { return proto.CompactTextString(m) }
 func (*Scenario) ProtoMessage()    {}
 func (*Scenario) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{17}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{17}
 }
 
 func (m *Scenario) XXX_Unmarshal(b []byte) error {
@@ -1221,7 +1221,7 @@ func (m *Scenarios) Reset()         { *m = Scenarios{} }
 func (m *Scenarios) String() string { return proto.CompactTextString(m) }
 func (*Scenarios) ProtoMessage()    {}
 func (*Scenarios) Descriptor() ([]byte, []int) {
-	return fileDescriptor_0c5120591600887d, []int{18}
+	return fileDescriptor_6f4e2bf9f888bddb, []int{18}
 }
 
 func (m *Scenarios) XXX_Unmarshal(b []byte) error {
@@ -1274,82 +1274,87 @@ func init() {
 	proto.RegisterType((*Scenarios)(nil), "grpc.testing.Scenarios")
 }
 
-func init() { proto.RegisterFile("control.proto", fileDescriptor_0c5120591600887d) }
-
-var fileDescriptor_0c5120591600887d = []byte{
-	// 1179 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x6f, 0x6f, 0xdb, 0xb6,
-	0x13, 0xb6, 0x1d, 0xdb, 0xb1, 0x4e, 0xb6, 0xe3, 0x1f, 0x7f, 0xe9, 0xa0, 0xa6, 0x69, 0x97, 0x6a,
-	0x1b, 0x16, 0x64, 0x40, 0x5a, 0x78, 0x05, 0xba, 0x62, 0x2f, 0x02, 0xc7, 0x33, 0xea, 0x00, 0x69,
-	0x96, 0xd1, 0x69, 0x87, 0xbe, 0x12, 0x18, 0x99, 0xb1, 0x85, 0xc8, 0xa2, 0x46, 0x52, 0x09, 0xf2,
-	0x15, 0xf6, 0x99, 0xf6, 0x39, 0xf6, 0x35, 0xf6, 0x15, 0x06, 0xfe, 0x91, 0x23, 0xb9, 0x06, 0x9a,
-	0x6d, 0xef, 0xc4, 0xbb, 0xe7, 0xe1, 0x91, 0xf7, 0xdc, 0x1d, 0x05, 0x9d, 0x90, 0x25, 0x92, 0xb3,
-	0xf8, 0x30, 0xe5, 0x4c, 0x32, 0xd4, 0x9e, 0xf1, 0x34, 0x3c, 0x94, 0x54, 0xc8, 0x28, 0x99, 0xed,
-	0x74, 0x53, 0x72, 0x17, 0x33, 0x32, 0x15, 0xc6, 0xbb, 0xe3, 0x0a, 0x49, 0xa4, 0x5d, 0xf8, 0x7d,
-	0xe8, 0x9c, 0xb3, 0x48, 0x08, 0x96, 0x9c, 0x13, 0x4e, 0x16, 0x02, 0x3d, 0x87, 0x36, 0xbb, 0xba,
-	0xa2, 0x9c, 0x4e, 0x03, 0x45, 0xf2, 0xaa, 0x7b, 0xd5, 0xfd, 0x2a, 0x76, 0xad, 0xed, 0x94, 0x91,
-	0xa9, 0x4f, 0xa0, 0xf3, 0x3e, 0x89, 0xae, 0x18, 0x5f, 0x58, 0xce, 0xb7, 0xb0, 0x15, 0x25, 0x92,
-	0x72, 0xc2, 0x79, 0x74, 0x43, 0xe2, 0x20, 0x66, 0x96, 0xd6, 0x2d, 0x9a, 0x4f, 0xd9, 0x27, 0xc0,
-	0x79, 0xe4, 0xd5, 0x3e, 0x05, 0x8e, 0x23, 0xff, 0x07, 0xf8, 0xff, 0x4f, 0x54, 0x52, 0xbe, 0x88,
-	0x92, 0x48, 0xc8, 0x28, 0x7c, 0xf8, 0xe1, 0x7e, 0x81, 0xf6, 0x39, 0xe1, 0x54, 0x32, 0x4b, 0xf9,
-	0x0e, 0xfe, 0x57, 0x0a, 0x79, 0x49, 0x04, 0xb5, 0xbc, 0x5e, 0xd1, 0x71, 0x4c, 0x04, 0x45, 0xdb,
-	0xd0, 0x20, 0x71, 0x3a, 0x27, 0xf6, 0x54, 0x66, 0xe1, 0x23, 0xe8, 0x0d, 0x63, 0x26, 0x54, 0x00,
-	0x96, 0x9a, 0x6d, 0xfd, 0x3f, 0x6a, 0x00, 0x2a, 0x9e, 0x8d, 0x32, 0x00, 0x37, 0xd4, 0x90, 0x20,
-	0x66, 0x2c, 0xd5, 0xfb, 0xbb, 0xfd, 0x67, 0x87, 0x45, 0x1d, 0x0e, 0x57, 0xf7, 0x18, 0x57, 0x30,
-	0x84, 0x4b, 0x1b, 0x7a, 0x0d, 0x9b, 0xa9, 0x51, 0x42, 0x47, 0x77, 0xfb, 0x4f, 0xca, 0xf4, 0x92,
-	0x4c, 0xe3, 0x0a, 0xce, 0xd1, 0x8a, 0x98, 0x19, 0x39, 0xbc, 0x8d, 0x75, 0xc4, 0x92, 0x56, 0x8a,
-	0x68, 0xd1, 0xe8, 0x47, 0x68, 0x4e, 0x75, 0x92, 0xbd, 0xba, 0xe6, 0x3d, 0x2f, 0xf3, 0xd6, 0x08,
-	0x30, 0xae, 0x60, 0x4b, 0x41, 0xaf, 0xa0, 0x99, 0xea, 0x3c, 0x7b, 0x0d, 0x4d, 0xde, 0x59, 0x39,
-	0x6d, 0x41, 0x03, 0xc5, 0x32, 0xd8, 0xe3, 0x26, 0xd4, 0x95, 0x70, 0xfe, 0x25, 0x74, 0x27, 0x34,
-	0xcc, 0x78, 0x24, 0xef, 0x6c, 0x06, 0x9f, 0x81, 0x9b, 0x09, 0x1a, 0x28, 0x7e, 0x10, 0x12, 0x9d,
-	0xc1, 0x16, 0x76, 0x32, 0x41, 0x2f, 0xa8, 0x90, 0x43, 0x82, 0x5e, 0xc2, 0xb6, 0xa0, 0xfc, 0x86,
-	0xf2, 0x60, 0xce, 0x84, 0x0c, 0xd8, 0x0d, 0xe5, 0x3c, 0x9a, 0x52, 0x9d, 0x2b, 0x07, 0x23, 0xe3,
-	0x1b, 0x33, 0x21, 0x7f, 0xb6, 0x1e, 0xff, 0xf7, 0x06, 0xb4, 0x87, 0x71, 0x44, 0x13, 0x39, 0x64,
-	0xc9, 0x55, 0x34, 0x43, 0xdf, 0x40, 0xd7, 0x6e, 0x21, 0x09, 0x9f, 0x51, 0x29, 0xbc, 0xea, 0xde,
-	0xc6, 0xbe, 0x83, 0x3b, 0xc6, 0x7a, 0x61, 0x8c, 0xe8, 0x8d, 0xd2, 0x52, 0xd1, 0x02, 0x79, 0x97,
-	0x9a, 0x00, 0xdd, 0xbe, 0xb7, 0xaa, 0xa5, 0x02, 0x5c, 0xdc, 0xa5, 0x54, 0x69, 0x98, 0x7f, 0xa3,
-	0x11, 0x6c, 0x09, 0x7b, 0xad, 0x20, 0xd5, 0xf7, 0xb2, 0x92, 0xec, 0x96, 0xe9, 0xe5, 0xbb, 0xe3,
-	0xae, 0x28, 0xe7, 0xe2, 0x08, 0x76, 0x59, 0x26, 0x85, 0x24, 0xc9, 0x34, 0x4a, 0x66, 0x01, 0x4f,
-	0x43, 0x11, 0xa4, 0x94, 0x07, 0xe1, 0x9c, 0x24, 0x09, 0x8d, 0xb5, 0x5c, 0x0d, 0xfc, 0xb8, 0x80,
-	0xc1, 0x69, 0x28, 0xce, 0x29, 0x1f, 0x1a, 0x80, 0xea, 0x33, 0x7b, 0x05, 0x4b, 0x11, 0x5a, 0xa5,
-	0x06, 0xee, 0x1a, 0xb3, 0xc5, 0x09, 0x95, 0x55, 0x22, 0xee, 0x92, 0x30, 0xc8, 0x6f, 0x3c, 0xe7,
-	0x94, 0x4c, 0x85, 0xb7, 0xa9, 0xd1, 0x48, 0xfb, 0xec, 0x5d, 0x8d, 0x07, 0xbd, 0x84, 0x16, 0x4f,
-	0x43, 0x93, 0x9a, 0x96, 0x4e, 0xcd, 0xa3, 0xf2, 0xdd, 0x70, 0x1a, 0xea, 0xbc, 0x6c, 0x72, 0xf3,
-	0xa1, 0xf2, 0xa9, 0x34, 0xcf, 0x13, 0x02, 0x3a, 0x21, 0x2b, 0xf9, 0xbc, 0x6f, 0x25, 0x0c, 0xf1,
-	0x7d, 0x5b, 0x1d, 0x43, 0x3e, 0xbc, 0x82, 0x50, 0x6b, 0xe8, 0xb9, 0x6b, 0x5b, 0xc3, 0x60, 0x8c,
-	0xcc, 0xb8, 0x93, 0x16, 0x97, 0x68, 0x0c, 0xbd, 0x79, 0x24, 0x24, 0x9b, 0x71, 0xb2, 0xc8, 0xcf,
-	0xd0, 0xd6, 0xbb, 0x3c, 0x2d, 0xef, 0x32, 0xce, 0x51, 0xf6, 0x20, 0x5b, 0xf3, 0xb2, 0x01, 0x3d,
-	0x01, 0x27, 0x64, 0x9c, 0x06, 0x71, 0x24, 0xa4, 0xd7, 0xd9, 0xdb, 0xd8, 0x6f, 0xe0, 0x96, 0x32,
-	0x9c, 0x46, 0x42, 0xa2, 0xa7, 0x00, 0xd6, 0xb9, 0x88, 0xa4, 0xd7, 0xd5, 0xf9, 0x73, 0x8c, 0x77,
-	0x11, 0x49, 0xff, 0x28, 0xaf, 0xc5, 0x89, 0x24, 0x32, 0x13, 0xe8, 0x05, 0x34, 0xf4, 0x18, 0xb6,
-	0xa3, 0xe2, 0xf1, 0xba, 0xf2, 0x52, 0x50, 0x81, 0x0d, 0xce, 0xdf, 0x85, 0xfa, 0x3b, 0xc2, 0xaf,
-	0xd5, 0x88, 0xe2, 0x54, 0x50, 0x69, 0x3b, 0xc4, 0x2c, 0xfc, 0x0c, 0xc0, 0x70, 0x06, 0x7c, 0x26,
-	0x50, 0x1f, 0x1a, 0x82, 0xca, 0x2c, 0x9f, 0x43, 0x3b, 0xeb, 0x36, 0x37, 0xd9, 0x19, 0x57, 0xb0,
-	0x81, 0xa2, 0x7d, 0xa8, 0x2f, 0x08, 0xbf, 0xb6, 0xb3, 0x07, 0x95, 0x29, 0x2a, 0xf2, 0xb8, 0x82,
-	0x35, 0xe2, 0xd8, 0x81, 0x4d, 0xc2, 0x67, 0xaa, 0x00, 0xfc, 0x3f, 0x6b, 0xd0, 0x9e, 0xe8, 0xe6,
-	0xb1, 0xc9, 0x7e, 0x03, 0x6e, 0xde, 0x62, 0xaa, 0x40, 0xaa, 0xeb, 0x7a, 0xc7, 0x10, 0x4c, 0xef,
-	0x88, 0xe5, 0xf7, 0xba, 0xde, 0xa9, 0xfd, 0x8b, 0xde, 0x41, 0x50, 0x4f, 0x19, 0x97, 0xb6, 0x47,
-	0xf4, 0xf7, 0x7d, 0x95, 0xe7, 0x67, 0x5b, 0x53, 0xe5, 0xf6, 0x54, 0xb6, 0xca, 0xcb, 0x6a, 0xb6,
-	0x56, 0xd4, 0x5c, 0x53, 0x97, 0xce, 0x3f, 0xae, 0xcb, 0x52, 0x35, 0x41, 0xb9, 0x9a, 0x94, 0x9e,
-	0xe6, 0x40, 0x0f, 0xd0, 0xb3, 0x28, 0xc0, 0x7f, 0xd4, 0x33, 0xca, 0xe5, 0x7c, 0x50, 0x95, 0xde,
-	0x43, 0xf3, 0x2a, 0x5d, 0x66, 0xbf, 0x56, 0xc8, 0xfe, 0x36, 0x34, 0xd4, 0xbd, 0xcc, 0x28, 0x6c,
-	0x60, 0xb3, 0xf0, 0x3b, 0xe0, 0x0e, 0x19, 0xa7, 0x98, 0xfe, 0x96, 0x51, 0x21, 0xfd, 0xaf, 0xa1,
-	0x6d, 0x96, 0x22, 0x65, 0x89, 0x79, 0x89, 0x0d, 0xa9, 0x5a, 0x24, 0x35, 0xa1, 0xfe, 0x81, 0x45,
-	0x53, 0xff, 0xaf, 0x1a, 0xb4, 0x26, 0x21, 0x4d, 0x08, 0x8f, 0x98, 0x8a, 0x99, 0x90, 0x85, 0x29,
-	0x36, 0x07, 0xeb, 0x6f, 0x74, 0x04, 0x9d, 0x7c, 0x00, 0x1a, 0x7d, 0x6a, 0x9f, 0xeb, 0x04, 0xdc,
-	0x0e, 0x8b, 0x6f, 0xc5, 0x97, 0xe0, 0x26, 0xd9, 0xc2, 0x8e, 0xc5, 0xfc, 0xe8, 0x90, 0x64, 0x0b,
-	0xc3, 0x51, 0x33, 0xda, 0x3e, 0x1b, 0x79, 0x84, 0xfa, 0xe7, 0xb4, 0xc1, 0x6d, 0x51, 0x6c, 0x15,
-	0x1b, 0xc1, 0xd8, 0xf2, 0xf9, 0xac, 0x22, 0x18, 0x8e, 0x50, 0xcf, 0xd5, 0x2d, 0xe1, 0x8b, 0x2c,
-	0x0d, 0x04, 0x0d, 0x59, 0x32, 0x15, 0x5e, 0x53, 0x63, 0x3a, 0xc6, 0x3a, 0x31, 0x46, 0xf5, 0x83,
-	0x73, 0x49, 0x93, 0x70, 0xae, 0xb4, 0x5c, 0x22, 0x4d, 0x65, 0xf7, 0x96, 0x8e, 0x1c, 0xfc, 0x1a,
-	0x3c, 0x91, 0x92, 0xdb, 0x24, 0x88, 0x59, 0x48, 0xe2, 0xe0, 0x96, 0xf1, 0x6b, 0x7d, 0x83, 0x2c,
-	0xc9, 0xab, 0xfc, 0x91, 0xf6, 0x9f, 0x2a, 0xf7, 0xaf, 0xda, 0x3b, 0x54, 0x4e, 0x7f, 0x00, 0x4e,
-	0x9e, 0x70, 0x81, 0x5e, 0x81, 0x23, 0xf2, 0x85, 0x7e, 0x43, 0xdd, 0xfe, 0x17, 0x2b, 0xf7, 0xb6,
-	0x6e, 0x7c, 0x0f, 0x3c, 0x78, 0x91, 0xcf, 0x28, 0xdd, 0xee, 0x5b, 0xe0, 0x4e, 0x3e, 0x9e, 0x0d,
-	0x83, 0xe1, 0xe9, 0xc9, 0xe8, 0xec, 0xa2, 0x57, 0x41, 0x3d, 0x68, 0x0f, 0x8a, 0x96, 0xea, 0xc1,
-	0x49, 0xde, 0x04, 0x25, 0xc2, 0x64, 0x84, 0x3f, 0x8c, 0x70, 0x91, 0x60, 0x2d, 0x55, 0xe4, 0xc1,
-	0xb6, 0xb1, 0xbc, 0x1d, 0x9d, 0x8d, 0xf0, 0xc9, 0xd2, 0x53, 0x3b, 0xf8, 0x0a, 0x36, 0xed, 0xbb,
-	0x84, 0x1c, 0x68, 0xbc, 0x3f, 0x1b, 0xe0, 0x8f, 0xbd, 0x0a, 0xea, 0x80, 0x33, 0xb9, 0xc0, 0xa3,
-	0xc1, 0xbb, 0x93, 0xb3, 0xb7, 0xbd, 0xea, 0x65, 0x53, 0xff, 0x12, 0x7f, 0xff, 0x77, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0x75, 0x59, 0xf4, 0x03, 0x4e, 0x0b, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("benchmark/grpc_testing/control.proto", fileDescriptor_6f4e2bf9f888bddb)
+}
+
+var fileDescriptor_6f4e2bf9f888bddb = []byte{
+	// 1219 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xed, 0x6e, 0xdb, 0x36,
+	0x17, 0xb6, 0x5d, 0xdb, 0xb1, 0x8e, 0x3f, 0xe2, 0x97, 0x6f, 0x3a, 0xa8, 0xe9, 0xc7, 0x5a, 0xad,
+	0xc5, 0x82, 0x0e, 0xb3, 0x0b, 0xaf, 0x40, 0x57, 0xec, 0x47, 0xe1, 0x78, 0x46, 0x1d, 0x20, 0xcd,
+	0x32, 0x3a, 0xed, 0xd0, 0xfd, 0x11, 0x18, 0x99, 0x91, 0x85, 0xca, 0xa2, 0x46, 0x52, 0x2d, 0x72,
+	0x0b, 0xbb, 0xa6, 0x5d, 0xc7, 0x6e, 0x63, 0xb7, 0x30, 0xf0, 0x43, 0x8e, 0xe4, 0xba, 0x68, 0xb6,
+	0xfd, 0x13, 0xcf, 0x79, 0x1e, 0x1e, 0xf2, 0x3c, 0xe7, 0x1c, 0x0a, 0x1e, 0x9e, 0xd3, 0x24, 0x58,
+	0xae, 0x08, 0x7f, 0x37, 0x0c, 0x79, 0x1a, 0xf8, 0x92, 0x0a, 0x19, 0x25, 0xe1, 0x30, 0x60, 0x89,
+	0xe4, 0x2c, 0x1e, 0xa4, 0x9c, 0x49, 0x86, 0x3a, 0xca, 0x37, 0xb0, 0xbe, 0xfd, 0x47, 0x9f, 0xe0,
+	0xa4, 0xe4, 0x32, 0x66, 0x64, 0x21, 0x0c, 0x69, 0xdf, 0xfb, 0x04, 0x4c, 0x48, 0x22, 0x2d, 0xc6,
+	0x1b, 0x41, 0xf7, 0x94, 0x45, 0x42, 0xb0, 0xe4, 0x94, 0x70, 0xb2, 0x12, 0xe8, 0x01, 0x74, 0xd8,
+	0xc5, 0x05, 0xe5, 0x74, 0xe1, 0xab, 0xbd, 0xdc, 0xea, 0xfd, 0xea, 0x41, 0x15, 0xb7, 0xad, 0xed,
+	0x98, 0x91, 0x85, 0x47, 0xa0, 0xfb, 0x3a, 0x89, 0x2e, 0x18, 0x5f, 0x59, 0xce, 0xd7, 0xb0, 0x1b,
+	0x25, 0x92, 0x72, 0xc2, 0x79, 0xf4, 0x9e, 0xc4, 0x7e, 0xcc, 0x2c, 0xad, 0x57, 0x34, 0x1f, 0xb3,
+	0x8f, 0x80, 0xcb, 0xc8, 0xad, 0x7d, 0x0c, 0x9c, 0x45, 0xde, 0xf7, 0xf0, 0xff, 0x1f, 0xa9, 0xa4,
+	0x7c, 0x15, 0x25, 0x91, 0x90, 0x51, 0x70, 0xfd, 0xc3, 0xfd, 0x0c, 0x9d, 0x53, 0xc2, 0xa9, 0x64,
+	0x96, 0xf2, 0x0d, 0xfc, 0xaf, 0x14, 0xf2, 0x9c, 0x08, 0x6a, 0x79, 0xfd, 0xa2, 0xe3, 0x90, 0x08,
+	0x8a, 0xf6, 0xa0, 0x41, 0xe2, 0x74, 0x49, 0xec, 0xa9, 0xcc, 0xc2, 0x43, 0xd0, 0x9f, 0xc4, 0x4c,
+	0xa8, 0x00, 0x2c, 0x35, 0xdb, 0x7a, 0x7f, 0xd4, 0x00, 0x54, 0x3c, 0x1b, 0x65, 0x0c, 0xed, 0x40,
+	0x43, 0xfc, 0x98, 0xb1, 0x54, 0xef, 0xdf, 0x1e, 0xdd, 0x1b, 0x14, 0x55, 0x1b, 0x6c, 0xee, 0x31,
+	0xab, 0x60, 0x08, 0xd6, 0x36, 0xf4, 0x0c, 0x76, 0x52, 0xa3, 0x84, 0x8e, 0xde, 0x1e, 0xdd, 0x2e,
+	0xd3, 0x4b, 0x32, 0xcd, 0x2a, 0x38, 0x47, 0x2b, 0x62, 0x66, 0xe4, 0x70, 0x6f, 0x6c, 0x23, 0x96,
+	0xb4, 0x52, 0x44, 0x8b, 0x46, 0x3f, 0x40, 0x73, 0xa1, 0x93, 0xec, 0xd6, 0x35, 0xef, 0x41, 0x99,
+	0xb7, 0x45, 0x80, 0x59, 0x05, 0x5b, 0x0a, 0x7a, 0x0a, 0xcd, 0x54, 0xe7, 0xd9, 0x6d, 0x68, 0xf2,
+	0xfe, 0xc6, 0x69, 0x0b, 0x1a, 0x28, 0x96, 0xc1, 0x1e, 0x36, 0xa1, 0xae, 0x84, 0xf3, 0xce, 0xa1,
+	0x37, 0xa7, 0x41, 0xc6, 0x23, 0x79, 0x69, 0x33, 0x78, 0x0f, 0xda, 0x99, 0xa0, 0xba, 0x46, 0xfd,
+	0x80, 0xe8, 0x0c, 0xb6, 0xb0, 0x93, 0x09, 0x7a, 0x46, 0x85, 0x9c, 0x10, 0xf4, 0x04, 0xf6, 0x04,
+	0xe5, 0xef, 0x29, 0xf7, 0x97, 0x4c, 0x48, 0x9f, 0xbd, 0xa7, 0x9c, 0x47, 0x0b, 0xaa, 0x73, 0xe5,
+	0x60, 0x64, 0x7c, 0x33, 0x26, 0xe4, 0x4f, 0xd6, 0xe3, 0xfd, 0xde, 0x80, 0xce, 0x24, 0x8e, 0x68,
+	0x22, 0x27, 0x2c, 0xb9, 0x88, 0x42, 0xf4, 0x08, 0x7a, 0x76, 0x0b, 0x49, 0x78, 0x48, 0xa5, 0x70,
+	0xab, 0xf7, 0x6f, 0x1c, 0x38, 0xb8, 0x6b, 0xac, 0x67, 0xc6, 0x88, 0x9e, 0x2b, 0x2d, 0x15, 0xcd,
+	0x97, 0x97, 0xa9, 0x09, 0xd0, 0x1b, 0xb9, 0x9b, 0x5a, 0x2a, 0xc0, 0xd9, 0x65, 0x4a, 0x95, 0x86,
+	0xf9, 0x37, 0x9a, 0xc2, 0xae, 0xb0, 0xd7, 0xf2, 0x53, 0x7d, 0x2f, 0x2b, 0xc9, 0x9d, 0x32, 0xbd,
+	0x7c, 0x77, 0xdc, 0x13, 0xe5, 0x5c, 0xbc, 0x80, 0x3b, 0x2c, 0x93, 0x42, 0x92, 0x64, 0x11, 0x25,
+	0xa1, 0xcf, 0xd3, 0x40, 0xf8, 0x29, 0xe5, 0x7e, 0xb0, 0x24, 0x49, 0x42, 0x63, 0x2d, 0x57, 0x03,
+	0xdf, 0x2a, 0x60, 0x70, 0x1a, 0x88, 0x53, 0xca, 0x27, 0x06, 0xa0, 0xfa, 0xcc, 0x5e, 0xc1, 0x52,
+	0x84, 0x56, 0xa9, 0x81, 0x7b, 0xc6, 0x6c, 0x71, 0x42, 0x65, 0x95, 0x88, 0xcb, 0x24, 0xf0, 0xf3,
+	0x1b, 0x2f, 0x39, 0x25, 0x0b, 0xe1, 0xee, 0x68, 0x34, 0xd2, 0x3e, 0x7b, 0x57, 0xe3, 0x41, 0x4f,
+	0xa0, 0xa5, 0x67, 0x89, 0x4a, 0x4d, 0x4b, 0xa7, 0xe6, 0x66, 0xf9, 0x6e, 0x38, 0x0d, 0x74, 0x5e,
+	0x76, 0xb8, 0xf9, 0x50, 0xf9, 0x54, 0x9a, 0xe7, 0x09, 0x01, 0x9d, 0x90, 0x8d, 0x7c, 0x5e, 0xb5,
+	0x12, 0x86, 0xf8, 0xaa, 0xad, 0x0e, 0xa1, 0x67, 0x67, 0x9a, 0x1f, 0x68, 0x0d, 0xdd, 0xf6, 0xd6,
+	0xd6, 0x30, 0x18, 0x23, 0x33, 0xee, 0xa6, 0xc5, 0x25, 0x9a, 0x41, 0x7f, 0x19, 0x09, 0xc9, 0x42,
+	0x4e, 0x56, 0xf9, 0x19, 0x3a, 0x7a, 0x97, 0xbb, 0xe5, 0x5d, 0x66, 0x39, 0xca, 0x1e, 0x64, 0x77,
+	0x59, 0x36, 0xa0, 0xdb, 0xe0, 0x04, 0x8c, 0x53, 0x3f, 0x8e, 0x84, 0x74, 0xbb, 0xf7, 0x6f, 0x1c,
+	0x34, 0x70, 0x4b, 0x19, 0x8e, 0x23, 0x21, 0xd1, 0x5d, 0x00, 0xeb, 0x5c, 0x45, 0xd2, 0xed, 0xe9,
+	0xfc, 0x39, 0xc6, 0xbb, 0x8a, 0xa4, 0xf7, 0x22, 0xaf, 0xc5, 0xb9, 0x24, 0x32, 0x13, 0x68, 0x08,
+	0x0d, 0x3d, 0x86, 0xed, 0xa8, 0xb8, 0xb5, 0xad, 0xbc, 0x14, 0x54, 0x60, 0x83, 0xf3, 0xee, 0x40,
+	0xfd, 0x15, 0xe1, 0xef, 0xd4, 0x88, 0xe2, 0x54, 0x50, 0x69, 0x3b, 0xc4, 0x2c, 0xbc, 0x0c, 0xc0,
+	0x70, 0xc6, 0x3c, 0x14, 0x68, 0x04, 0x0d, 0x41, 0x65, 0x96, 0xcf, 0xa1, 0xfd, 0x6d, 0x9b, 0x9b,
+	0xec, 0xcc, 0x2a, 0xd8, 0x40, 0xd1, 0x01, 0xd4, 0xd5, 0x4b, 0x61, 0x67, 0x0f, 0x2a, 0x53, 0x54,
+	0xe4, 0x59, 0x05, 0x6b, 0xc4, 0xa1, 0x03, 0x3b, 0x84, 0x87, 0xaa, 0x00, 0xbc, 0x3f, 0x6b, 0xd0,
+	0x99, 0xeb, 0xe6, 0xb1, 0xc9, 0x7e, 0x0e, 0xed, 0xbc, 0xc5, 0x54, 0x81, 0x54, 0xb7, 0xf5, 0x8e,
+	0x21, 0x98, 0xde, 0x11, 0xeb, 0xef, 0x6d, 0xbd, 0x53, 0xfb, 0x17, 0xbd, 0x83, 0xa0, 0x9e, 0x32,
+	0x2e, 0x6d, 0x8f, 0xe8, 0xef, 0xab, 0x2a, 0xcf, 0xcf, 0xb6, 0xa5, 0xca, 0xed, 0xa9, 0x6c, 0x95,
+	0x97, 0xd5, 0x6c, 0x6d, 0xa8, 0xb9, 0xa5, 0x2e, 0x9d, 0x7f, 0x5c, 0x97, 0xa5, 0x6a, 0x82, 0x72,
+	0x35, 0x29, 0x3d, 0xcd, 0x81, 0xae, 0xa1, 0x67, 0x51, 0x80, 0xff, 0xa8, 0x67, 0x94, 0xcb, 0x79,
+	0xad, 0x2a, 0xbd, 0x82, 0xe6, 0x55, 0xba, 0xce, 0x7e, 0xad, 0x90, 0xfd, 0x3d, 0x68, 0xa8, 0x7b,
+	0x99, 0x51, 0xd8, 0xc0, 0x66, 0xe1, 0x75, 0xa1, 0x3d, 0x61, 0x9c, 0x62, 0xfa, 0x5b, 0x46, 0x85,
+	0xf4, 0x1e, 0x42, 0xc7, 0x2c, 0x45, 0xca, 0x12, 0xf3, 0x12, 0x1b, 0x52, 0xb5, 0x48, 0x6a, 0x42,
+	0xfd, 0x0d, 0x8b, 0x16, 0xde, 0x5f, 0x35, 0x68, 0xcd, 0x03, 0x9a, 0x10, 0x1e, 0x31, 0x15, 0x33,
+	0x21, 0x2b, 0x53, 0x6c, 0x0e, 0xd6, 0xdf, 0xe8, 0x05, 0x74, 0xf3, 0x01, 0x68, 0xf4, 0xa9, 0x7d,
+	0xae, 0x13, 0x70, 0x27, 0x28, 0xbe, 0x15, 0x5f, 0x42, 0x3b, 0xc9, 0x56, 0x76, 0x2c, 0xe6, 0x47,
+	0x87, 0x24, 0x5b, 0x19, 0x8e, 0x9a, 0xd1, 0xf6, 0xd9, 0xc8, 0x23, 0xd4, 0x3f, 0xa7, 0x0d, 0xee,
+	0x88, 0x62, 0xab, 0xd8, 0x08, 0xc6, 0x96, 0xcf, 0x67, 0x15, 0xc1, 0x70, 0x84, 0x7a, 0xae, 0x3e,
+	0x10, 0xbe, 0xca, 0x52, 0x5f, 0xd0, 0x80, 0x25, 0x0b, 0xe1, 0x36, 0x35, 0xa6, 0x6b, 0xac, 0x73,
+	0x63, 0x54, 0x3f, 0x38, 0xeb, 0xff, 0xbc, 0x35, 0xd2, 0x54, 0x76, 0x7f, 0xed, 0xc8, 0xc1, 0xcf,
+	0xc0, 0x15, 0x29, 0xf9, 0x90, 0xf8, 0x31, 0x0b, 0x48, 0xec, 0x7f, 0x60, 0xfc, 0x9d, 0xbe, 0x41,
+	0x96, 0xe4, 0x55, 0x7e, 0x53, 0xfb, 0x8f, 0x95, 0xfb, 0x17, 0xed, 0x9d, 0x28, 0xa7, 0x37, 0x06,
+	0x27, 0x4f, 0xb8, 0x40, 0x4f, 0xc1, 0x11, 0xf9, 0x42, 0xbf, 0xa1, 0xed, 0xd1, 0x17, 0x1b, 0xf7,
+	0xb6, 0x6e, 0x7c, 0x05, 0x7c, 0x3c, 0xcc, 0x67, 0x94, 0x6e, 0xf7, 0x5d, 0x68, 0xcf, 0xdf, 0x9e,
+	0x4c, 0xfc, 0xc9, 0xf1, 0xd1, 0xf4, 0xe4, 0xac, 0x5f, 0x41, 0x7d, 0xe8, 0x8c, 0x8b, 0x96, 0xea,
+	0xe3, 0xa3, 0xbc, 0x09, 0x4a, 0x84, 0xf9, 0x14, 0xbf, 0x99, 0xe2, 0x22, 0xc1, 0x5a, 0xaa, 0xc8,
+	0x85, 0x3d, 0x63, 0x79, 0x39, 0x3d, 0x99, 0xe2, 0xa3, 0xb5, 0xa7, 0xf6, 0xf8, 0x2b, 0xd8, 0xb1,
+	0xef, 0x12, 0x72, 0xa0, 0xf1, 0xfa, 0x64, 0x8c, 0xdf, 0xf6, 0x2b, 0xa8, 0x0b, 0xce, 0xfc, 0x0c,
+	0x4f, 0xc7, 0xaf, 0x8e, 0x4e, 0x5e, 0xf6, 0xab, 0x87, 0xc3, 0x5f, 0xbf, 0x0d, 0x19, 0x0b, 0x63,
+	0x3a, 0x08, 0x59, 0x4c, 0x92, 0x70, 0xc0, 0x78, 0xa8, 0xff, 0x9c, 0x87, 0xdb, 0x7f, 0xa4, 0xcf,
+	0x9b, 0xfa, 0x1f, 0xfa, 0xbb, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x89, 0xaf, 0xc1, 0xc4,
+	0x0b, 0x00, 0x00,
 }
diff --git a/benchmark/grpc_testing/control.proto b/benchmark/grpc_testing/control.proto
index 9379ef49a2e3..e9ee3484d19f 100644
--- a/benchmark/grpc_testing/control.proto
+++ b/benchmark/grpc_testing/control.proto
@@ -14,8 +14,10 @@
 
 syntax = "proto3";
 
-import "payloads.proto";
-import "stats.proto";
+option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
+
+import "benchmark/grpc_testing/payloads.proto";
+import "benchmark/grpc_testing/stats.proto";
 
 package grpc.testing;
 
diff --git a/benchmark/grpc_testing/messages.pb.go b/benchmark/grpc_testing/messages.pb.go
index e2c694507b05..0fddbc137fe2 100644
--- a/benchmark/grpc_testing/messages.pb.go
+++ b/benchmark/grpc_testing/messages.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: messages.proto
+// source: benchmark/grpc_testing/messages.proto
 
 package grpc_testing
 
@@ -49,7 +49,7 @@ func (x PayloadType) String() string {
 }
 
 func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{0}
+	return fileDescriptor_7e3146bec7fa4346, []int{0}
 }
 
 // Compression algorithms
@@ -79,7 +79,7 @@ func (x CompressionType) String() string {
 }
 
 func (CompressionType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{1}
+	return fileDescriptor_7e3146bec7fa4346, []int{1}
 }
 
 // A block of data, to simply increase gRPC message size.
@@ -97,7 +97,7 @@ func (m *Payload) Reset()         { *m = Payload{} }
 func (m *Payload) String() string { return proto.CompactTextString(m) }
 func (*Payload) ProtoMessage()    {}
 func (*Payload) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{0}
+	return fileDescriptor_7e3146bec7fa4346, []int{0}
 }
 
 func (m *Payload) XXX_Unmarshal(b []byte) error {
@@ -146,7 +146,7 @@ func (m *EchoStatus) Reset()         { *m = EchoStatus{} }
 func (m *EchoStatus) String() string { return proto.CompactTextString(m) }
 func (*EchoStatus) ProtoMessage()    {}
 func (*EchoStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{1}
+	return fileDescriptor_7e3146bec7fa4346, []int{1}
 }
 
 func (m *EchoStatus) XXX_Unmarshal(b []byte) error {
@@ -208,7 +208,7 @@ func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
 func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
 func (*SimpleRequest) ProtoMessage()    {}
 func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{2}
+	return fileDescriptor_7e3146bec7fa4346, []int{2}
 }
 
 func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
@@ -296,7 +296,7 @@ func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
 func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
 func (*SimpleResponse) ProtoMessage()    {}
 func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{3}
+	return fileDescriptor_7e3146bec7fa4346, []int{3}
 }
 
 func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
@@ -351,7 +351,7 @@ func (m *StreamingInputCallRequest) Reset()         { *m = StreamingInputCallReq
 func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) }
 func (*StreamingInputCallRequest) ProtoMessage()    {}
 func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{4}
+	return fileDescriptor_7e3146bec7fa4346, []int{4}
 }
 
 func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error {
@@ -392,7 +392,7 @@ func (m *StreamingInputCallResponse) Reset()         { *m = StreamingInputCallRe
 func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) }
 func (*StreamingInputCallResponse) ProtoMessage()    {}
 func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{5}
+	return fileDescriptor_7e3146bec7fa4346, []int{5}
 }
 
 func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error {
@@ -437,7 +437,7 @@ func (m *ResponseParameters) Reset()         { *m = ResponseParameters{} }
 func (m *ResponseParameters) String() string { return proto.CompactTextString(m) }
 func (*ResponseParameters) ProtoMessage()    {}
 func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{6}
+	return fileDescriptor_7e3146bec7fa4346, []int{6}
 }
 
 func (m *ResponseParameters) XXX_Unmarshal(b []byte) error {
@@ -496,7 +496,7 @@ func (m *StreamingOutputCallRequest) Reset()         { *m = StreamingOutputCallR
 func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) }
 func (*StreamingOutputCallRequest) ProtoMessage()    {}
 func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{7}
+	return fileDescriptor_7e3146bec7fa4346, []int{7}
 }
 
 func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error {
@@ -565,7 +565,7 @@ func (m *StreamingOutputCallResponse) Reset()         { *m = StreamingOutputCall
 func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) }
 func (*StreamingOutputCallResponse) ProtoMessage()    {}
 func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{8}
+	return fileDescriptor_7e3146bec7fa4346, []int{8}
 }
 
 func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error {
@@ -606,7 +606,7 @@ func (m *ReconnectParams) Reset()         { *m = ReconnectParams{} }
 func (m *ReconnectParams) String() string { return proto.CompactTextString(m) }
 func (*ReconnectParams) ProtoMessage()    {}
 func (*ReconnectParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{9}
+	return fileDescriptor_7e3146bec7fa4346, []int{9}
 }
 
 func (m *ReconnectParams) XXX_Unmarshal(b []byte) error {
@@ -649,7 +649,7 @@ func (m *ReconnectInfo) Reset()         { *m = ReconnectInfo{} }
 func (m *ReconnectInfo) String() string { return proto.CompactTextString(m) }
 func (*ReconnectInfo) ProtoMessage()    {}
 func (*ReconnectInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dc296cbfe5ffcd5, []int{10}
+	return fileDescriptor_7e3146bec7fa4346, []int{10}
 }
 
 func (m *ReconnectInfo) XXX_Unmarshal(b []byte) error {
@@ -700,49 +700,54 @@ func init() {
 	proto.RegisterType((*ReconnectInfo)(nil), "grpc.testing.ReconnectInfo")
 }
 
-func init() { proto.RegisterFile("messages.proto", fileDescriptor_4dc296cbfe5ffcd5) }
-
-var fileDescriptor_4dc296cbfe5ffcd5 = []byte{
-	// 652 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x55, 0x4d, 0x6f, 0xd3, 0x40,
-	0x10, 0xc5, 0xf9, 0xee, 0x24, 0x4d, 0xa3, 0x85, 0x82, 0x5b, 0x54, 0x11, 0x99, 0x4b, 0x54, 0x89,
-	0x20, 0x05, 0x09, 0x24, 0x0e, 0xa0, 0xb4, 0x4d, 0x51, 0x50, 0x9a, 0x84, 0x75, 0x7b, 0xe1, 0x62,
-	0x6d, 0x9c, 0x8d, 0x6b, 0x11, 0x7b, 0x8d, 0x77, 0x8d, 0x9a, 0x1e, 0xb8, 0xf3, 0x83, 0xb9, 0xa3,
-	0x5d, 0x7f, 0xc4, 0x69, 0x7b, 0x68, 0xe1, 0xc2, 0x6d, 0xf7, 0xed, 0x9b, 0x97, 0x79, 0x33, 0xcf,
-	0x0a, 0x34, 0x3d, 0xca, 0x39, 0x71, 0x28, 0xef, 0x06, 0x21, 0x13, 0x0c, 0x35, 0x9c, 0x30, 0xb0,
-	0xbb, 0x82, 0x72, 0xe1, 0xfa, 0x8e, 0x31, 0x82, 0xea, 0x94, 0xac, 0x96, 0x8c, 0xcc, 0xd1, 0x2b,
-	0x28, 0x89, 0x55, 0x40, 0x75, 0xad, 0xad, 0x75, 0x9a, 0xbd, 0xbd, 0x6e, 0x9e, 0xd7, 0x4d, 0x48,
-	0xe7, 0xab, 0x80, 0x62, 0x45, 0x43, 0x08, 0x4a, 0x33, 0x36, 0x5f, 0xe9, 0x85, 0xb6, 0xd6, 0x69,
-	0x60, 0x75, 0x36, 0xde, 0x03, 0x0c, 0xec, 0x4b, 0x66, 0x0a, 0x22, 0x22, 0x2e, 0x19, 0x36, 0x9b,
-	0xc7, 0x82, 0x65, 0xac, 0xce, 0x48, 0x87, 0x6a, 0xd2, 0x8f, 0x2a, 0xdc, 0xc2, 0xe9, 0xd5, 0xf8,
-	0x55, 0x84, 0x6d, 0xd3, 0xf5, 0x82, 0x25, 0xc5, 0xf4, 0x7b, 0x44, 0xb9, 0x40, 0x1f, 0x60, 0x3b,
-	0xa4, 0x3c, 0x60, 0x3e, 0xa7, 0xd6, 0xfd, 0x3a, 0x6b, 0xa4, 0x7c, 0x79, 0x43, 0x2f, 0x73, 0xf5,
-	0xdc, 0xbd, 0x8e, 0x7f, 0xb1, 0xbc, 0x26, 0x99, 0xee, 0x35, 0x45, 0xaf, 0xa1, 0x1a, 0xc4, 0x0a,
-	0x7a, 0xb1, 0xad, 0x75, 0xea, 0xbd, 0xdd, 0x3b, 0xe5, 0x71, 0xca, 0x92, 0xaa, 0x0b, 0x77, 0xb9,
-	0xb4, 0x22, 0x4e, 0x43, 0x9f, 0x78, 0x54, 0x2f, 0xb5, 0xb5, 0x4e, 0x0d, 0x37, 0x24, 0x78, 0x91,
-	0x60, 0xa8, 0x03, 0x2d, 0x45, 0x62, 0x24, 0x12, 0x97, 0x16, 0xb7, 0x59, 0x40, 0xf5, 0xb2, 0xe2,
-	0x35, 0x25, 0x3e, 0x91, 0xb0, 0x29, 0x51, 0x34, 0x85, 0x27, 0x59, 0x93, 0x36, 0xf3, 0x82, 0x90,
-	0x72, 0xee, 0x32, 0x5f, 0xaf, 0x28, 0xaf, 0x07, 0x9b, 0xcd, 0x1c, 0xaf, 0x09, 0xca, 0xef, 0xe3,
-	0xb4, 0x34, 0xf7, 0x80, 0xfa, 0xb0, 0xb3, 0xb6, 0xad, 0x36, 0xa1, 0x57, 0x95, 0x33, 0x7d, 0x53,
-	0x6c, 0xbd, 0x29, 0xdc, 0xcc, 0x46, 0xa2, 0xee, 0xc6, 0x4f, 0x68, 0xa6, 0xab, 0x88, 0xf1, 0xfc,
-	0x98, 0xb4, 0x7b, 0x8d, 0x69, 0x1f, 0x6a, 0xd9, 0x84, 0xe2, 0x4d, 0x67, 0x77, 0xf4, 0x02, 0xea,
-	0xf9, 0xc1, 0x14, 0xd5, 0x33, 0xb0, 0x6c, 0x28, 0xc6, 0x08, 0xf6, 0x4c, 0x11, 0x52, 0xe2, 0xb9,
-	0xbe, 0x33, 0xf4, 0x83, 0x48, 0x1c, 0x93, 0xe5, 0x32, 0x8d, 0xc5, 0x43, 0x5b, 0x31, 0xce, 0x61,
-	0xff, 0x2e, 0xb5, 0xc4, 0xd9, 0x5b, 0x78, 0x46, 0x1c, 0x27, 0xa4, 0x0e, 0x11, 0x74, 0x6e, 0x25,
-	0x35, 0x71, 0x5e, 0xe2, 0xe0, 0xee, 0xae, 0x9f, 0x13, 0x69, 0x19, 0x1c, 0x63, 0x08, 0x28, 0xd5,
-	0x98, 0x92, 0x90, 0x78, 0x54, 0xd0, 0x50, 0x65, 0x3e, 0x57, 0xaa, 0xce, 0xd2, 0xae, 0xeb, 0x0b,
-	0x1a, 0xfe, 0x20, 0x32, 0x35, 0x49, 0x0a, 0x21, 0x85, 0x2e, 0xb8, 0xf1, 0xbb, 0x90, 0xeb, 0x70,
-	0x12, 0x89, 0x1b, 0x86, 0xff, 0xf5, 0x3b, 0xf8, 0x02, 0x59, 0x4e, 0xac, 0x20, 0x6b, 0x55, 0x2f,
-	0xb4, 0x8b, 0x9d, 0x7a, 0xaf, 0xbd, 0xa9, 0x72, 0xdb, 0x12, 0x46, 0xe1, 0x6d, 0x9b, 0x0f, 0xfe,
-	0x6a, 0xfe, 0xcb, 0x98, 0x8f, 0xe1, 0xf9, 0x9d, 0x63, 0xff, 0xcb, 0xcc, 0x1b, 0x9f, 0x61, 0x07,
-	0x53, 0x9b, 0xf9, 0x3e, 0xb5, 0x85, 0x1a, 0x16, 0x47, 0xef, 0x40, 0xf7, 0xc8, 0x95, 0x15, 0xa6,
-	0xb0, 0x35, 0x23, 0xf6, 0x37, 0xb6, 0x58, 0x58, 0x1e, 0x4f, 0xe3, 0xe5, 0x91, 0xab, 0xac, 0xea,
-	0x28, 0x7e, 0x3d, 0xe3, 0xc6, 0x29, 0x6c, 0x67, 0xe8, 0xd0, 0x5f, 0x30, 0xf4, 0x14, 0x2a, 0x01,
-	0xe1, 0x9c, 0xc6, 0xcd, 0xd4, 0x70, 0x72, 0x43, 0x07, 0x00, 0x39, 0x4d, 0xb9, 0xd4, 0x32, 0xde,
-	0x9a, 0xa5, 0x3a, 0x87, 0x1f, 0xa1, 0x9e, 0x4b, 0x06, 0x6a, 0x41, 0xe3, 0x78, 0x72, 0x36, 0xc5,
-	0x03, 0xd3, 0xec, 0x1f, 0x8d, 0x06, 0xad, 0x47, 0x08, 0x41, 0xf3, 0x62, 0xbc, 0x81, 0x69, 0x08,
-	0xa0, 0x82, 0xfb, 0xe3, 0x93, 0xc9, 0x59, 0xab, 0x70, 0xd8, 0x83, 0x9d, 0x1b, 0xfb, 0x40, 0x35,
-	0x28, 0x8d, 0x27, 0x63, 0x59, 0x5c, 0x83, 0xd2, 0xa7, 0xaf, 0xc3, 0x69, 0x4b, 0x43, 0x75, 0xa8,
-	0x9e, 0x0c, 0x4e, 0x47, 0xfd, 0xf3, 0x41, 0xab, 0x30, 0xab, 0xa8, 0xbf, 0x9a, 0x37, 0x7f, 0x02,
-	0x00, 0x00, 0xff, 0xff, 0xc2, 0x6a, 0xce, 0x1e, 0x7c, 0x06, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("benchmark/grpc_testing/messages.proto", fileDescriptor_7e3146bec7fa4346)
+}
+
+var fileDescriptor_7e3146bec7fa4346 = []byte{
+	// 690 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x55, 0x4b, 0x6f, 0xd3, 0x4c,
+	0x14, 0xfd, 0x9c, 0x77, 0x6f, 0xd2, 0x34, 0x9a, 0x8f, 0x82, 0x5b, 0x54, 0x11, 0x19, 0x21, 0x45,
+	0x95, 0x9a, 0x48, 0x41, 0x02, 0x89, 0x05, 0x28, 0x6d, 0x53, 0x14, 0x94, 0x26, 0xc1, 0x6e, 0x37,
+	0xdd, 0x58, 0x13, 0x67, 0xe2, 0x5a, 0xb5, 0x3d, 0x66, 0x66, 0x82, 0x9a, 0x2e, 0xd8, 0xf3, 0x83,
+	0xd9, 0x23, 0x8f, 0x1f, 0x71, 0xda, 0x2c, 0x5a, 0xd8, 0xb0, 0x9b, 0x39, 0xf7, 0xdc, 0xe3, 0xfb,
+	0x38, 0xd6, 0xc0, 0x9b, 0x29, 0xf1, 0xad, 0x6b, 0x0f, 0xb3, 0x9b, 0x8e, 0xcd, 0x02, 0xcb, 0x14,
+	0x84, 0x0b, 0xc7, 0xb7, 0x3b, 0x1e, 0xe1, 0x1c, 0xdb, 0x84, 0xb7, 0x03, 0x46, 0x05, 0x45, 0xb5,
+	0x30, 0xd8, 0x8e, 0x83, 0xda, 0x10, 0xca, 0x13, 0xbc, 0x74, 0x29, 0x9e, 0xa1, 0x23, 0x28, 0x88,
+	0x65, 0x40, 0x54, 0xa5, 0xa9, 0xb4, 0xea, 0xdd, 0xbd, 0x76, 0x96, 0xd7, 0x8e, 0x49, 0x17, 0xcb,
+	0x80, 0xe8, 0x92, 0x86, 0x10, 0x14, 0xa6, 0x74, 0xb6, 0x54, 0x73, 0x4d, 0xa5, 0x55, 0xd3, 0xe5,
+	0x59, 0xfb, 0x00, 0xd0, 0xb7, 0xae, 0xa9, 0x21, 0xb0, 0x58, 0xf0, 0x90, 0x61, 0xd1, 0x59, 0x24,
+	0x58, 0xd4, 0xe5, 0x19, 0xa9, 0x50, 0x8e, 0xeb, 0x91, 0x89, 0x5b, 0x7a, 0x72, 0xd5, 0x7e, 0xe6,
+	0x61, 0xdb, 0x70, 0xbc, 0xc0, 0x25, 0x3a, 0xf9, 0xb6, 0x20, 0x5c, 0xa0, 0x8f, 0xb0, 0xcd, 0x08,
+	0x0f, 0xa8, 0xcf, 0x89, 0xf9, 0xb8, 0xca, 0x6a, 0x09, 0x3f, 0xbc, 0xa1, 0xd7, 0x99, 0x7c, 0xee,
+	0xdc, 0x45, 0x5f, 0x2c, 0xae, 0x48, 0x86, 0x73, 0x47, 0x50, 0x07, 0xca, 0x41, 0xa4, 0xa0, 0xe6,
+	0x9b, 0x4a, 0xab, 0xda, 0xdd, 0xdd, 0x28, 0xaf, 0x27, 0xac, 0x50, 0x75, 0xee, 0xb8, 0xae, 0xb9,
+	0xe0, 0x84, 0xf9, 0xd8, 0x23, 0x6a, 0xa1, 0xa9, 0xb4, 0x2a, 0x7a, 0x2d, 0x04, 0x2f, 0x63, 0x0c,
+	0xb5, 0xa0, 0x21, 0x49, 0x14, 0x2f, 0xc4, 0xb5, 0xc9, 0x2d, 0x1a, 0x10, 0xb5, 0x28, 0x79, 0xf5,
+	0x10, 0x1f, 0x87, 0xb0, 0x11, 0xa2, 0x68, 0x02, 0xcf, 0xd2, 0x22, 0x2d, 0xea, 0x05, 0x8c, 0x70,
+	0xee, 0x50, 0x5f, 0x2d, 0xc9, 0x5e, 0x0f, 0xd6, 0x8b, 0x39, 0x59, 0x11, 0x64, 0xbf, 0xff, 0x27,
+	0xa9, 0x99, 0x00, 0xea, 0xc1, 0xce, 0xaa, 0x6d, 0xb9, 0x09, 0xb5, 0x2c, 0x3b, 0x53, 0xd7, 0xc5,
+	0x56, 0x9b, 0xd2, 0xeb, 0xe9, 0x48, 0xe4, 0x5d, 0xfb, 0x01, 0xf5, 0x64, 0x15, 0x11, 0x9e, 0x1d,
+	0x93, 0xf2, 0xa8, 0x31, 0xed, 0x43, 0x25, 0x9d, 0x50, 0xb4, 0xe9, 0xf4, 0x8e, 0x5e, 0x41, 0x35,
+	0x3b, 0x98, 0xbc, 0x0c, 0x03, 0x4d, 0x87, 0xa2, 0x0d, 0x61, 0xcf, 0x10, 0x8c, 0x60, 0xcf, 0xf1,
+	0xed, 0x81, 0x1f, 0x2c, 0xc4, 0x09, 0x76, 0xdd, 0xc4, 0x16, 0x4f, 0x2d, 0x45, 0xbb, 0x80, 0xfd,
+	0x4d, 0x6a, 0x71, 0x67, 0xef, 0xe0, 0x05, 0xb6, 0x6d, 0x46, 0x6c, 0x2c, 0xc8, 0xcc, 0x8c, 0x73,
+	0x22, 0xbf, 0x44, 0xc6, 0xdd, 0x5d, 0x85, 0x63, 0xe9, 0xd0, 0x38, 0xda, 0x00, 0x50, 0xa2, 0x31,
+	0xc1, 0x0c, 0x7b, 0x44, 0x10, 0x26, 0x3d, 0x9f, 0x49, 0x95, 0xe7, 0xb0, 0x5d, 0xc7, 0x17, 0x84,
+	0x7d, 0xc7, 0xa1, 0x6b, 0x62, 0x17, 0x42, 0x02, 0x5d, 0x72, 0xed, 0x57, 0x2e, 0x53, 0xe1, 0x78,
+	0x21, 0xee, 0x35, 0xfc, 0xb7, 0xff, 0xc1, 0x57, 0x48, 0x7d, 0x62, 0x06, 0x69, 0xa9, 0x6a, 0xae,
+	0x99, 0x6f, 0x55, 0xbb, 0xcd, 0x75, 0x95, 0x87, 0x2d, 0xe9, 0x88, 0x3d, 0x6c, 0xf3, 0xc9, 0x7f,
+	0xcd, 0x3f, 0x69, 0xf3, 0x11, 0xbc, 0xdc, 0x38, 0xf6, 0x3f, 0xf4, 0xbc, 0xf6, 0x05, 0x76, 0x74,
+	0x62, 0x51, 0xdf, 0x27, 0x96, 0x90, 0xc3, 0xe2, 0xe8, 0x3d, 0xa8, 0x1e, 0xbe, 0x35, 0x59, 0x02,
+	0x9b, 0x53, 0x6c, 0xdd, 0xd0, 0xf9, 0xdc, 0xf4, 0x78, 0x62, 0x2f, 0x0f, 0xdf, 0xa6, 0x59, 0xc7,
+	0x51, 0xf4, 0x9c, 0x6b, 0x67, 0xb0, 0x9d, 0xa2, 0x03, 0x7f, 0x4e, 0xd1, 0x73, 0x28, 0x05, 0x98,
+	0x73, 0x12, 0x15, 0x53, 0xd1, 0xe3, 0x1b, 0x3a, 0x00, 0xc8, 0x68, 0x86, 0x4b, 0x2d, 0xea, 0x5b,
+	0xd3, 0x44, 0xe7, 0xf0, 0x13, 0x54, 0x33, 0xce, 0x40, 0x0d, 0xa8, 0x9d, 0x8c, 0xcf, 0x27, 0x7a,
+	0xdf, 0x30, 0x7a, 0xc7, 0xc3, 0x7e, 0xe3, 0x3f, 0x84, 0xa0, 0x7e, 0x39, 0x5a, 0xc3, 0x14, 0x04,
+	0x50, 0xd2, 0x7b, 0xa3, 0xd3, 0xf1, 0x79, 0x23, 0x77, 0xd8, 0x85, 0x9d, 0x7b, 0xfb, 0x40, 0x15,
+	0x28, 0x8c, 0xc6, 0xa3, 0x30, 0xb9, 0x02, 0x85, 0xcf, 0x57, 0x83, 0x49, 0x43, 0x41, 0x55, 0x28,
+	0x9f, 0xf6, 0xcf, 0x86, 0xbd, 0x8b, 0x7e, 0x23, 0x77, 0xdc, 0xb9, 0x3a, 0xb2, 0x29, 0xb5, 0x5d,
+	0xd2, 0xb6, 0xa9, 0x8b, 0x7d, 0xbb, 0x4d, 0x99, 0x2d, 0x1f, 0xa5, 0xce, 0xe6, 0x37, 0x6a, 0x5a,
+	0x92, 0x6f, 0xd3, 0xdb, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xea, 0xde, 0x4b, 0x12, 0xc4, 0x06,
+	0x00, 0x00,
 }
diff --git a/benchmark/grpc_testing/messages.proto b/benchmark/grpc_testing/messages.proto
index bd83f095fb15..c48cdae9a29e 100644
--- a/benchmark/grpc_testing/messages.proto
+++ b/benchmark/grpc_testing/messages.proto
@@ -16,6 +16,8 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
+
 package grpc.testing;
 
 // The type of payload that should be returned.
diff --git a/benchmark/grpc_testing/payloads.pb.go b/benchmark/grpc_testing/payloads.pb.go
index 5e58aacea16e..c96b7b6c4213 100644
--- a/benchmark/grpc_testing/payloads.pb.go
+++ b/benchmark/grpc_testing/payloads.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: payloads.proto
+// source: benchmark/grpc_testing/payloads.proto
 
 package grpc_testing
 
@@ -32,7 +32,7 @@ func (m *ByteBufferParams) Reset()         { *m = ByteBufferParams{} }
 func (m *ByteBufferParams) String() string { return proto.CompactTextString(m) }
 func (*ByteBufferParams) ProtoMessage()    {}
 func (*ByteBufferParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_3a075f58f70088c8, []int{0}
+	return fileDescriptor_69438956f5d73a41, []int{0}
 }
 
 func (m *ByteBufferParams) XXX_Unmarshal(b []byte) error {
@@ -79,7 +79,7 @@ func (m *SimpleProtoParams) Reset()         { *m = SimpleProtoParams{} }
 func (m *SimpleProtoParams) String() string { return proto.CompactTextString(m) }
 func (*SimpleProtoParams) ProtoMessage()    {}
 func (*SimpleProtoParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_3a075f58f70088c8, []int{1}
+	return fileDescriptor_69438956f5d73a41, []int{1}
 }
 
 func (m *SimpleProtoParams) XXX_Unmarshal(b []byte) error {
@@ -124,7 +124,7 @@ func (m *ComplexProtoParams) Reset()         { *m = ComplexProtoParams{} }
 func (m *ComplexProtoParams) String() string { return proto.CompactTextString(m) }
 func (*ComplexProtoParams) ProtoMessage()    {}
 func (*ComplexProtoParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_3a075f58f70088c8, []int{2}
+	return fileDescriptor_69438956f5d73a41, []int{2}
 }
 
 func (m *ComplexProtoParams) XXX_Unmarshal(b []byte) error {
@@ -160,7 +160,7 @@ func (m *PayloadConfig) Reset()         { *m = PayloadConfig{} }
 func (m *PayloadConfig) String() string { return proto.CompactTextString(m) }
 func (*PayloadConfig) ProtoMessage()    {}
 func (*PayloadConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_3a075f58f70088c8, []int{3}
+	return fileDescriptor_69438956f5d73a41, []int{3}
 }
 
 func (m *PayloadConfig) XXX_Unmarshal(b []byte) error {
@@ -247,24 +247,29 @@ func init() {
 	proto.RegisterType((*PayloadConfig)(nil), "grpc.testing.PayloadConfig")
 }
 
-func init() { proto.RegisterFile("payloads.proto", fileDescriptor_3a075f58f70088c8) }
-
-var fileDescriptor_3a075f58f70088c8 = []byte{
-	// 254 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2b, 0x48, 0xac, 0xcc,
-	0xc9, 0x4f, 0x4c, 0x29, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x49, 0x2f, 0x2a, 0x48,
-	0xd6, 0x2b, 0x49, 0x2d, 0x2e, 0xc9, 0xcc, 0x4b, 0x57, 0xf2, 0xe2, 0x12, 0x70, 0xaa, 0x2c, 0x49,
-	0x75, 0x2a, 0x4d, 0x4b, 0x4b, 0x2d, 0x0a, 0x48, 0x2c, 0x4a, 0xcc, 0x2d, 0x16, 0x92, 0xe4, 0xe2,
-	0x28, 0x4a, 0x2d, 0x8c, 0x2f, 0xce, 0xac, 0x4a, 0x95, 0x60, 0x54, 0x60, 0xd4, 0x60, 0x0d, 0x62,
-	0x2f, 0x4a, 0x2d, 0x0c, 0xce, 0xac, 0x4a, 0x15, 0x92, 0xe6, 0xe2, 0x2c, 0x4a, 0x2d, 0x2e, 0x80,
-	0xc8, 0x31, 0x81, 0xe5, 0x38, 0x40, 0x02, 0x20, 0x49, 0x25, 0x6f, 0x2e, 0xc1, 0xe0, 0xcc, 0xdc,
-	0x82, 0x9c, 0xd4, 0x00, 0x90, 0x45, 0x14, 0x1a, 0x26, 0xc2, 0x25, 0xe4, 0x9c, 0x0f, 0x32, 0xac,
-	0x02, 0xc9, 0x34, 0xa5, 0x6f, 0x8c, 0x5c, 0xbc, 0x01, 0x10, 0xff, 0x38, 0xe7, 0xe7, 0xa5, 0x65,
-	0xa6, 0x0b, 0xb9, 0x73, 0xf1, 0x25, 0x55, 0x96, 0xa4, 0x26, 0x95, 0xa6, 0xc5, 0x17, 0x80, 0xd5,
-	0x80, 0x6d, 0xe1, 0x36, 0x92, 0xd3, 0x43, 0xf6, 0xa7, 0x1e, 0xba, 0x27, 0x3d, 0x18, 0x82, 0x78,
-	0xa1, 0xfa, 0xa0, 0x0e, 0x75, 0xe3, 0xe2, 0x2d, 0x06, 0xbb, 0x1e, 0x66, 0x0e, 0x13, 0xd8, 0x1c,
-	0x79, 0x54, 0x73, 0x30, 0x3c, 0xe8, 0xc1, 0x10, 0xc4, 0x03, 0xd1, 0x07, 0x35, 0xc7, 0x93, 0x8b,
-	0x2f, 0x19, 0xe2, 0x70, 0x98, 0x41, 0xcc, 0x60, 0x83, 0x14, 0x50, 0x0d, 0xc2, 0xf4, 0x1c, 0xc8,
-	0x49, 0x50, 0x9d, 0x10, 0x01, 0x27, 0x4e, 0x2e, 0x76, 0x68, 0xe4, 0x25, 0xb1, 0x81, 0x23, 0xcf,
-	0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0xb0, 0x8c, 0x18, 0x4e, 0xce, 0x01, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("benchmark/grpc_testing/payloads.proto", fileDescriptor_69438956f5d73a41)
+}
+
+var fileDescriptor_69438956f5d73a41 = []byte{
+	// 289 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x91, 0xc1, 0x4b, 0xc3, 0x30,
+	0x18, 0xc5, 0xd7, 0x89, 0x6e, 0xfb, 0x5c, 0x87, 0x16, 0x0f, 0x8a, 0xa0, 0xa3, 0x20, 0x78, 0x31,
+	0x05, 0xfd, 0x0f, 0x3a, 0xd0, 0xa9, 0x97, 0xd2, 0xdd, 0xbc, 0x94, 0xb4, 0x7e, 0x8d, 0xc5, 0xb6,
+	0xc9, 0x92, 0x0e, 0xec, 0xfe, 0x70, 0xcf, 0x92, 0x34, 0x83, 0xcd, 0x79, 0xf3, 0x9a, 0x97, 0xf7,
+	0xfb, 0xde, 0xe3, 0xc1, 0x4d, 0x8a, 0x75, 0xf6, 0x51, 0x51, 0xf9, 0x19, 0x30, 0x29, 0xb2, 0xa4,
+	0x41, 0xd5, 0x14, 0x35, 0x0b, 0x04, 0x6d, 0x4b, 0x4e, 0xdf, 0x15, 0x11, 0x92, 0x37, 0xdc, 0x1b,
+	0x6b, 0x91, 0x58, 0xd1, 0x7f, 0x81, 0x93, 0xb0, 0x6d, 0x30, 0x5c, 0xe5, 0x39, 0xca, 0x88, 0x4a,
+	0x5a, 0x29, 0xef, 0x02, 0x86, 0x12, 0x97, 0x89, 0x2a, 0xd6, 0x78, 0xee, 0x4c, 0x9d, 0xdb, 0xc3,
+	0x78, 0x20, 0x71, 0xb9, 0x28, 0xd6, 0xe8, 0x5d, 0xc2, 0x48, 0xa2, 0x12, 0x9d, 0xd6, 0x37, 0xda,
+	0x50, 0x3f, 0x68, 0xd1, 0x7f, 0x85, 0xd3, 0x45, 0x51, 0x89, 0x12, 0x23, 0x7d, 0xe8, 0x9f, 0xb0,
+	0x33, 0xf0, 0x66, 0x5c, 0xc3, 0xbe, 0xb6, 0x68, 0xfe, 0xb7, 0x03, 0x6e, 0xd4, 0xf5, 0x99, 0xf1,
+	0x3a, 0x2f, 0x98, 0xf7, 0x04, 0x93, 0xb4, 0x6d, 0x30, 0x5d, 0xe5, 0x89, 0x30, 0x7f, 0xcc, 0x95,
+	0xe3, 0xfb, 0x2b, 0xb2, 0xdd, 0x93, 0xfc, 0x2e, 0x39, 0xef, 0xc5, 0xae, 0xf5, 0xd9, 0xa0, 0x8f,
+	0xe0, 0x2a, 0x93, 0x7e, 0xc3, 0xe9, 0x1b, 0xce, 0xf5, 0x2e, 0x67, 0xaf, 0xe0, 0xbc, 0x17, 0x8f,
+	0x3b, 0x9f, 0xe5, 0x3c, 0xc3, 0x24, 0xeb, 0x82, 0x6f, 0x40, 0x07, 0x06, 0x34, 0xdd, 0x05, 0xed,
+	0x97, 0xd3, 0x91, 0xac, 0xb3, 0x7b, 0x08, 0x47, 0x30, 0xb0, 0xe3, 0x85, 0xc1, 0xdb, 0x1d, 0xe3,
+	0x9c, 0x95, 0x48, 0x18, 0x2f, 0x69, 0xcd, 0x08, 0x97, 0xcc, 0xcc, 0x1c, 0xfc, 0xbd, 0x7a, 0x7a,
+	0x64, 0xd6, 0x7e, 0xf8, 0x09, 0x00, 0x00, 0xff, 0xff, 0x15, 0x79, 0xad, 0xbf, 0x16, 0x02, 0x00,
+	0x00,
 }
diff --git a/benchmark/grpc_testing/payloads.proto b/benchmark/grpc_testing/payloads.proto
index 5d4871f5f9fd..862fb71bc135 100644
--- a/benchmark/grpc_testing/payloads.proto
+++ b/benchmark/grpc_testing/payloads.proto
@@ -14,6 +14,8 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
+
 package grpc.testing;
 
 message ByteBufferParams {
diff --git a/benchmark/grpc_testing/services.pb.go b/benchmark/grpc_testing/services.pb.go
index 38f02540194f..7c71cf193004 100644
--- a/benchmark/grpc_testing/services.pb.go
+++ b/benchmark/grpc_testing/services.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: services.proto
+// source: benchmark/grpc_testing/services.proto
 
 package grpc_testing
 
@@ -24,27 +24,32 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
-func init() { proto.RegisterFile("services.proto", fileDescriptor_8e16ccb8c5307b32) }
-
-var fileDescriptor_8e16ccb8c5307b32 = []byte{
-	// 271 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x92, 0xc1, 0x4a, 0xc3, 0x40,
-	0x10, 0x86, 0x69, 0x0f, 0x42, 0x16, 0x53, 0x64, 0x4f, 0xba, 0xfa, 0x00, 0x9e, 0x82, 0x54, 0x5f,
-	0xc0, 0x06, 0x3d, 0x0a, 0x36, 0x54, 0x0f, 0x9e, 0xd6, 0x74, 0x88, 0x4b, 0x93, 0x99, 0x38, 0x33,
-	0x11, 0x7c, 0x02, 0x1f, 0xc1, 0xd7, 0x15, 0xb3, 0x56, 0x6a, 0xc8, 0xcd, 0x1e, 0xe7, 0xff, 0x86,
-	0x8f, 0xfd, 0x77, 0xd7, 0xcc, 0x04, 0xf8, 0x2d, 0x94, 0x20, 0x59, 0xcb, 0xa4, 0x64, 0x0f, 0x2b,
-	0x6e, 0xcb, 0x4c, 0x41, 0x34, 0x60, 0xe5, 0x66, 0x0d, 0x88, 0xf8, 0x6a, 0x4b, 0x5d, 0x5a, 0x12,
-	0x2a, 0x53, 0x1d, 0xc7, 0xf9, 0xc7, 0xd4, 0x1c, 0x2d, 0x00, 0xcb, 0x97, 0xc6, 0xf3, 0xa6, 0x88,
-	0x22, 0x7b, 0x6b, 0x92, 0x15, 0x7a, 0x7e, 0xcf, 0x7d, 0x5d, 0xdb, 0xd3, 0x6c, 0xd7, 0x97, 0x15,
-	0xa1, 0x69, 0x6b, 0x58, 0xc2, 0x6b, 0x07, 0xa2, 0xee, 0x6c, 0x1c, 0x4a, 0x4b, 0x28, 0x60, 0xef,
-	0x4c, 0x5a, 0x28, 0x83, 0x6f, 0x02, 0x56, 0xff, 0x74, 0x9d, 0x4f, 0x2e, 0x26, 0xf6, 0xc9, 0xb8,
-	0x15, 0x96, 0x84, 0xa2, 0xec, 0x03, 0xc2, 0x7a, 0x9f, 0xf2, 0xf9, 0xe7, 0xd4, 0xa4, 0x8f, 0xc4,
-	0x1b, 0xe0, 0xed, 0x35, 0xdc, 0x98, 0x64, 0xd9, 0xe1, 0xf7, 0x04, 0x6c, 0x8f, 0x07, 0x82, 0x3e,
-	0xbd, 0xe6, 0x4a, 0x9c, 0x1b, 0x23, 0x85, 0x7a, 0xed, 0xa4, 0x3f, 0x75, 0xd4, 0xe4, 0x75, 0x00,
-	0xd4, 0xa1, 0x26, 0xa6, 0x63, 0x9a, 0x48, 0x76, 0x34, 0x0b, 0x93, 0xe4, 0xc4, 0x90, 0x53, 0x87,
-	0x6a, 0x4f, 0x06, 0xcb, 0xc4, 0xbf, 0x4d, 0xdd, 0x18, 0xfa, 0x79, 0x90, 0x2b, 0x63, 0xee, 0xbb,
-	0xa0, 0xb1, 0xa6, 0xb5, 0x7f, 0x37, 0x1f, 0x28, 0xac, 0xdd, 0x48, 0xf6, 0x7c, 0xd0, 0x7f, 0x95,
-	0xcb, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x9a, 0xb4, 0x19, 0x36, 0x69, 0x02, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("benchmark/grpc_testing/services.proto", fileDescriptor_e86b6b5d31c265e4)
+}
+
+var fileDescriptor_e86b6b5d31c265e4 = []byte{
+	// 309 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x92, 0xc1, 0x4a, 0x3b, 0x31,
+	0x10, 0xc6, 0x69, 0x0f, 0x7f, 0x68, 0xf8, 0x17, 0x24, 0x27, 0x8d, 0x1e, 0x15, 0xbc, 0xb8, 0x2b,
+	0xd5, 0x17, 0xb0, 0x8b, 0x1e, 0x05, 0xbb, 0x54, 0x41, 0x0f, 0x92, 0x6e, 0x87, 0x18, 0x9a, 0x9d,
+	0x59, 0x27, 0xb3, 0x82, 0x4f, 0xe0, 0x23, 0xf8, 0xba, 0xd2, 0xdd, 0x56, 0x6a, 0xd9, 0x9e, 0xf4,
+	0x98, 0xf9, 0xbe, 0xf9, 0x4d, 0x26, 0xf9, 0xd4, 0xc9, 0x0c, 0xb0, 0x78, 0x29, 0x2d, 0x2f, 0x52,
+	0xc7, 0x55, 0xf1, 0x2c, 0x10, 0xc5, 0xa3, 0x4b, 0x23, 0xf0, 0x9b, 0x2f, 0x20, 0x26, 0x15, 0x93,
+	0x90, 0xfe, 0xbf, 0x14, 0x93, 0x95, 0x68, 0x76, 0x35, 0x95, 0x10, 0xa3, 0x75, 0xeb, 0x26, 0x73,
+	0xbc, 0xc3, 0x56, 0x10, 0x0a, 0x53, 0x68, 0x5d, 0xa3, 0x8f, 0xbe, 0xda, 0x1b, 0xaf, 0x8d, 0x79,
+	0x3b, 0x56, 0xdf, 0xa8, 0xc1, 0x14, 0x2d, 0xbf, 0x67, 0x36, 0x04, 0x7d, 0x98, 0x6c, 0x4e, 0x4f,
+	0x72, 0x5f, 0x56, 0x01, 0x26, 0xf0, 0x5a, 0x43, 0x14, 0x73, 0xd4, 0x2d, 0xc6, 0x8a, 0x30, 0x82,
+	0xbe, 0x55, 0xc3, 0x5c, 0x18, 0x6c, 0xe9, 0xd1, 0xfd, 0x92, 0x75, 0xda, 0x3b, 0xef, 0xe9, 0x27,
+	0x65, 0xa6, 0x58, 0x10, 0x46, 0x61, 0xeb, 0x11, 0xe6, 0x7f, 0x09, 0x1f, 0x7d, 0xf6, 0xd5, 0xf0,
+	0x81, 0x78, 0x01, 0xbc, 0x7e, 0x86, 0x6b, 0x35, 0x98, 0xd4, 0xb8, 0x3c, 0x01, 0xeb, 0xfd, 0x2d,
+	0x40, 0x53, 0xbd, 0x62, 0x17, 0x8d, 0xe9, 0x52, 0x72, 0xb1, 0x52, 0xc7, 0xe6, 0xd6, 0x2d, 0x26,
+	0x0b, 0x1e, 0x50, 0xb6, 0x31, 0x6d, 0xb5, 0x0b, 0xd3, 0x2a, 0x1b, 0x98, 0xb1, 0x1a, 0x64, 0xc4,
+	0x90, 0x51, 0x8d, 0xa2, 0x0f, 0xb6, 0xcc, 0xc4, 0xdf, 0x9b, 0x9a, 0x2e, 0x69, 0xf5, 0x21, 0x97,
+	0x4a, 0xdd, 0xd5, 0x5e, 0xda, 0x35, 0xb5, 0xfe, 0xe9, 0xbc, 0x27, 0x3f, 0x37, 0x1d, 0xb5, 0x71,
+	0xfa, 0x78, 0xe6, 0x88, 0x5c, 0x80, 0xc4, 0x51, 0xb0, 0xe8, 0x12, 0x62, 0xd7, 0x64, 0x2a, 0xed,
+	0x8e, 0xd8, 0xec, 0x5f, 0x93, 0xad, 0x8b, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x77, 0x96, 0x9e,
+	0xdb, 0xdf, 0x02, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -273,7 +278,7 @@ var _BenchmarkService_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "services.proto",
+	Metadata: "benchmark/grpc_testing/services.proto",
 }
 
 // WorkerServiceClient is the client API for WorkerService service.
@@ -546,5 +551,5 @@ var _WorkerService_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "services.proto",
+	Metadata: "benchmark/grpc_testing/services.proto",
 }
diff --git a/benchmark/grpc_testing/services.proto b/benchmark/grpc_testing/services.proto
index 6ae2f9d1118a..9028c9cfe113 100644
--- a/benchmark/grpc_testing/services.proto
+++ b/benchmark/grpc_testing/services.proto
@@ -16,8 +16,10 @@
 // of unary/streaming requests/responses.
 syntax = "proto3";
 
-import "messages.proto";
-import "control.proto";
+option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
+
+import "benchmark/grpc_testing/messages.proto";
+import "benchmark/grpc_testing/control.proto";
 
 package grpc.testing;
 
diff --git a/benchmark/grpc_testing/stats.pb.go b/benchmark/grpc_testing/stats.pb.go
index 2e6bcc70655e..ebe310c41dcc 100644
--- a/benchmark/grpc_testing/stats.pb.go
+++ b/benchmark/grpc_testing/stats.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: stats.proto
+// source: benchmark/grpc_testing/stats.proto
 
 package grpc_testing
 
@@ -37,7 +37,7 @@ func (m *ServerStats) Reset()         { *m = ServerStats{} }
 func (m *ServerStats) String() string { return proto.CompactTextString(m) }
 func (*ServerStats) ProtoMessage()    {}
 func (*ServerStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b4756a0aec8b9d44, []int{0}
+	return fileDescriptor_a0658d5f374c86d6, []int{0}
 }
 
 func (m *ServerStats) XXX_Unmarshal(b []byte) error {
@@ -92,7 +92,7 @@ func (m *HistogramParams) Reset()         { *m = HistogramParams{} }
 func (m *HistogramParams) String() string { return proto.CompactTextString(m) }
 func (*HistogramParams) ProtoMessage()    {}
 func (*HistogramParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b4756a0aec8b9d44, []int{1}
+	return fileDescriptor_a0658d5f374c86d6, []int{1}
 }
 
 func (m *HistogramParams) XXX_Unmarshal(b []byte) error {
@@ -144,7 +144,7 @@ func (m *HistogramData) Reset()         { *m = HistogramData{} }
 func (m *HistogramData) String() string { return proto.CompactTextString(m) }
 func (*HistogramData) ProtoMessage()    {}
 func (*HistogramData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b4756a0aec8b9d44, []int{2}
+	return fileDescriptor_a0658d5f374c86d6, []int{2}
 }
 
 func (m *HistogramData) XXX_Unmarshal(b []byte) error {
@@ -223,7 +223,7 @@ func (m *ClientStats) Reset()         { *m = ClientStats{} }
 func (m *ClientStats) String() string { return proto.CompactTextString(m) }
 func (*ClientStats) ProtoMessage()    {}
 func (*ClientStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b4756a0aec8b9d44, []int{3}
+	return fileDescriptor_a0658d5f374c86d6, []int{3}
 }
 
 func (m *ClientStats) XXX_Unmarshal(b []byte) error {
@@ -279,30 +279,34 @@ func init() {
 	proto.RegisterType((*ClientStats)(nil), "grpc.testing.ClientStats")
 }
 
-func init() { proto.RegisterFile("stats.proto", fileDescriptor_b4756a0aec8b9d44) }
-
-var fileDescriptor_b4756a0aec8b9d44 = []byte{
-	// 341 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x4a, 0xeb, 0x40,
-	0x14, 0x86, 0x49, 0xd3, 0xf6, 0xb6, 0x27, 0xed, 0xbd, 0x97, 0x41, 0x24, 0x52, 0xd0, 0x1a, 0x5c,
-	0x74, 0x95, 0x85, 0xae, 0x5c, 0xab, 0xe0, 0xce, 0xd2, 0xe8, 0x3a, 0x4c, 0xe3, 0x69, 0x19, 0xcc,
-	0xcc, 0xc4, 0x39, 0x33, 0x12, 0x1f, 0x49, 0x7c, 0x49, 0xc9, 0x24, 0x68, 0x55, 0xd0, 0x5d, 0xe6,
-	0xfb, 0x7e, 0xe6, 0xe4, 0xe4, 0x0f, 0x44, 0x64, 0xb9, 0xa5, 0xb4, 0x32, 0xda, 0x6a, 0x36, 0xd9,
-	0x9a, 0xaa, 0x48, 0x2d, 0x92, 0x15, 0x6a, 0x9b, 0x28, 0x88, 0x32, 0x34, 0x4f, 0x68, 0xb2, 0x26,
-	0xc2, 0x8e, 0x61, 0x62, 0x85, 0xc4, 0x1c, 0x4b, 0x5e, 0x11, 0xde, 0xc7, 0xc1, 0x3c, 0x58, 0x04,
-	0xab, 0xa8, 0x61, 0x57, 0x2d, 0x62, 0x33, 0x18, 0xfb, 0x88, 0x23, 0x34, 0x71, 0xcf, 0xfb, 0x51,
-	0x03, 0xee, 0x08, 0x0d, 0x3b, 0x02, 0x9f, 0xcd, 0xe9, 0x99, 0x2c, 0xca, 0x38, 0xf4, 0x1a, 0x1a,
-	0x94, 0x79, 0x92, 0xdc, 0xc2, 0xbf, 0x6b, 0x41, 0x56, 0x6f, 0x0d, 0x97, 0x4b, 0x6e, 0xb8, 0x24,
-	0x76, 0x08, 0x60, 0x90, 0x74, 0xe9, 0xac, 0xd0, 0xaa, 0x9b, 0xb8, 0x43, 0x9a, 0x77, 0x92, 0xbc,
-	0xce, 0x2b, 0x4d, 0x24, 0xd6, 0x25, 0x76, 0x33, 0x23, 0xc9, 0xeb, 0x65, 0x87, 0x92, 0xd7, 0x00,
-	0xa6, 0xef, 0xd7, 0x5e, 0x72, 0xcb, 0xd9, 0x3e, 0x0c, 0xd7, 0xae, 0x78, 0x40, 0x1b, 0x07, 0xf3,
-	0x70, 0x31, 0x5d, 0x75, 0x27, 0x76, 0x00, 0x23, 0x29, 0x54, 0x4e, 0x88, 0xaa, 0xbb, 0xe8, 0x8f,
-	0x14, 0x2a, 0x43, 0x54, 0x5e, 0xf1, 0xba, 0x55, 0x61, 0xa7, 0x78, 0xed, 0xd5, 0x7f, 0x08, 0xc9,
-	0xc9, 0xb8, 0xef, 0x69, 0xf3, 0xc8, 0x4e, 0xe0, 0x2f, 0x39, 0x99, 0xeb, 0x4d, 0x4e, 0x8f, 0x8e,
-	0x1b, 0xa4, 0x78, 0xe0, 0xe5, 0x84, 0x9c, 0xbc, 0xd9, 0x64, 0x2d, 0x63, 0x7b, 0x30, 0x28, 0xb4,
-	0x53, 0x36, 0x1e, 0x7a, 0xd9, 0x1e, 0x92, 0x97, 0x00, 0xa2, 0x8b, 0x52, 0xa0, 0xb2, 0xed, 0x47,
-	0x3f, 0x87, 0x71, 0xc9, 0x2d, 0xaa, 0x42, 0x20, 0xf9, 0xfd, 0xa3, 0xd3, 0x59, 0xba, 0xdb, 0x52,
-	0xfa, 0x69, 0xb7, 0xd5, 0x47, 0xfa, 0x5b, 0x5f, 0xbd, 0x5f, 0xfa, 0x0a, 0x7f, 0xee, 0xab, 0xff,
-	0xb5, 0xaf, 0xf5, 0xd0, 0xff, 0x34, 0x67, 0x6f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xea, 0x75, 0x34,
-	0x90, 0x43, 0x02, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("benchmark/grpc_testing/stats.proto", fileDescriptor_a0658d5f374c86d6)
+}
+
+var fileDescriptor_a0658d5f374c86d6 = []byte{
+	// 376 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xcf, 0x0b, 0xd3, 0x30,
+	0x14, 0xc7, 0xe9, 0xba, 0xcd, 0xed, 0x75, 0x53, 0x09, 0x22, 0x95, 0x81, 0xce, 0xe2, 0x61, 0x17,
+	0x5b, 0xd0, 0x93, 0x57, 0x7f, 0x80, 0x37, 0xc7, 0xaa, 0x17, 0x2f, 0x25, 0xad, 0x6f, 0x31, 0xac,
+	0x49, 0x6a, 0x5e, 0x22, 0xf5, 0x4f, 0x12, 0xff, 0x49, 0x69, 0x5a, 0x74, 0xfe, 0x40, 0x6f, 0xcd,
+	0xe7, 0xf3, 0x25, 0x69, 0xf2, 0x7d, 0x90, 0xd5, 0xa8, 0x9b, 0x8f, 0x8a, 0xdb, 0x4b, 0x21, 0x6c,
+	0xd7, 0x54, 0x0e, 0xc9, 0x49, 0x2d, 0x0a, 0x72, 0xdc, 0x51, 0xde, 0x59, 0xe3, 0x0c, 0xdb, 0x0c,
+	0x26, 0x9f, 0x4c, 0xa6, 0x21, 0x29, 0xd1, 0x7e, 0x46, 0x5b, 0x0e, 0x11, 0xf6, 0x10, 0x36, 0x4e,
+	0x2a, 0xac, 0xb0, 0xe5, 0x1d, 0xe1, 0x87, 0x34, 0xda, 0x47, 0x87, 0xe8, 0x94, 0x0c, 0xec, 0xd5,
+	0x88, 0xd8, 0x0e, 0xd6, 0x21, 0xe2, 0x09, 0x6d, 0x3a, 0x0b, 0x7e, 0x35, 0x80, 0x77, 0x84, 0x96,
+	0x3d, 0x80, 0x90, 0xad, 0xe8, 0x0b, 0x39, 0x54, 0x69, 0x1c, 0x34, 0x0c, 0xa8, 0x0c, 0x24, 0x7b,
+	0x0b, 0xb7, 0x5e, 0x4b, 0x72, 0x46, 0x58, 0xae, 0x8e, 0xdc, 0x72, 0x45, 0xec, 0x3e, 0x80, 0x45,
+	0x32, 0xad, 0x77, 0xd2, 0xe8, 0xe9, 0xc4, 0x2b, 0x32, 0xfc, 0x93, 0xe2, 0x7d, 0xd5, 0x19, 0x22,
+	0x59, 0xb7, 0x38, 0x9d, 0x99, 0x28, 0xde, 0x1f, 0x27, 0x94, 0x7d, 0x8b, 0x60, 0xfb, 0x63, 0xdb,
+	0x97, 0xdc, 0x71, 0x76, 0x17, 0x96, 0xb5, 0x6f, 0x2e, 0xe8, 0xd2, 0x68, 0x1f, 0x1f, 0xb6, 0xa7,
+	0x69, 0xc5, 0xee, 0xc1, 0x4a, 0x49, 0x5d, 0x11, 0xa2, 0x9e, 0x36, 0xba, 0xa1, 0xa4, 0x2e, 0x11,
+	0x75, 0x50, 0xbc, 0x1f, 0x55, 0x3c, 0x29, 0xde, 0x07, 0x75, 0x1b, 0x62, 0xf2, 0x2a, 0x9d, 0x07,
+	0x3a, 0x7c, 0xb2, 0x47, 0x70, 0x93, 0xbc, 0xaa, 0xcc, 0xb9, 0xa2, 0x4f, 0x9e, 0x5b, 0xa4, 0x74,
+	0x11, 0xe4, 0x86, 0xbc, 0x7a, 0x73, 0x2e, 0x47, 0xc6, 0xee, 0xc0, 0xa2, 0x31, 0x5e, 0xbb, 0x74,
+	0x19, 0xe4, 0xb8, 0xc8, 0xbe, 0x46, 0x90, 0xbc, 0x68, 0x25, 0x6a, 0x37, 0x3e, 0xfa, 0x33, 0x58,
+	0xb7, 0xdc, 0xa1, 0x6e, 0x24, 0x52, 0xb8, 0x7f, 0xf2, 0x64, 0x97, 0x5f, 0xb7, 0x94, 0xff, 0x72,
+	0xb7, 0xd3, 0xcf, 0xf4, 0x1f, 0x7d, 0xcd, 0xfe, 0xd3, 0x57, 0xfc, 0xef, 0xbe, 0xe6, 0xbf, 0xf7,
+	0xf5, 0xbc, 0x78, 0xff, 0x58, 0x18, 0x23, 0x5a, 0xcc, 0x85, 0x69, 0xb9, 0x16, 0xb9, 0xb1, 0x22,
+	0xcc, 0x56, 0xf1, 0xf7, 0x51, 0xab, 0x97, 0x61, 0xca, 0x9e, 0x7e, 0x0f, 0x00, 0x00, 0xff, 0xff,
+	0xcc, 0xda, 0x99, 0x65, 0x8b, 0x02, 0x00, 0x00,
 }
diff --git a/benchmark/grpc_testing/stats.proto b/benchmark/grpc_testing/stats.proto
index baf3610f3351..1517e7f6d2ef 100644
--- a/benchmark/grpc_testing/stats.proto
+++ b/benchmark/grpc_testing/stats.proto
@@ -14,6 +14,8 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
+
 package grpc.testing;
 
 message ServerStats {
diff --git a/binarylog/grpc_binarylog_v1/binarylog.pb.go b/binarylog/grpc_binarylog_v1/binarylog.pb.go
index f393bb66187e..f826ec76984d 100644
--- a/binarylog/grpc_binarylog_v1/binarylog.pb.go
+++ b/binarylog/grpc_binarylog_v1/binarylog.pb.go
@@ -1,13 +1,15 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc/binarylog/grpc_binarylog_v1/binarylog.proto
+// source: grpc/binlog/v1/binarylog.proto
 
-package grpc_binarylog_v1 // import "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
+package grpc_binarylog_v1
 
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import duration "github.com/golang/protobuf/ptypes/duration"
-import timestamp "github.com/golang/protobuf/ptypes/timestamp"
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	duration "github.com/golang/protobuf/ptypes/duration"
+	timestamp "github.com/golang/protobuf/ptypes/timestamp"
+	math "math"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -18,7 +20,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
 // Enumerates the type of event
 // Note the terminology is different from the RPC semantics
@@ -64,6 +66,7 @@ var GrpcLogEntry_EventType_name = map[int32]string{
 	6: "EVENT_TYPE_SERVER_TRAILER",
 	7: "EVENT_TYPE_CANCEL",
 }
+
 var GrpcLogEntry_EventType_value = map[string]int32{
 	"EVENT_TYPE_UNKNOWN":           0,
 	"EVENT_TYPE_CLIENT_HEADER":     1,
@@ -78,8 +81,9 @@ var GrpcLogEntry_EventType_value = map[string]int32{
 func (x GrpcLogEntry_EventType) String() string {
 	return proto.EnumName(GrpcLogEntry_EventType_name, int32(x))
 }
+
 func (GrpcLogEntry_EventType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{0, 0}
+	return fileDescriptor_b7972e58de45083a, []int{0, 0}
 }
 
 // Enumerates the entity that generates the log entry
@@ -96,6 +100,7 @@ var GrpcLogEntry_Logger_name = map[int32]string{
 	1: "LOGGER_CLIENT",
 	2: "LOGGER_SERVER",
 }
+
 var GrpcLogEntry_Logger_value = map[string]int32{
 	"LOGGER_UNKNOWN": 0,
 	"LOGGER_CLIENT":  1,
@@ -105,8 +110,9 @@ var GrpcLogEntry_Logger_value = map[string]int32{
 func (x GrpcLogEntry_Logger) String() string {
 	return proto.EnumName(GrpcLogEntry_Logger_name, int32(x))
 }
+
 func (GrpcLogEntry_Logger) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{0, 1}
+	return fileDescriptor_b7972e58de45083a, []int{0, 1}
 }
 
 type Address_Type int32
@@ -128,6 +134,7 @@ var Address_Type_name = map[int32]string{
 	2: "TYPE_IPV6",
 	3: "TYPE_UNIX",
 }
+
 var Address_Type_value = map[string]int32{
 	"TYPE_UNKNOWN": 0,
 	"TYPE_IPV4":    1,
@@ -138,8 +145,9 @@ var Address_Type_value = map[string]int32{
 func (x Address_Type) String() string {
 	return proto.EnumName(Address_Type_name, int32(x))
 }
+
 func (Address_Type) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{7, 0}
+	return fileDescriptor_b7972e58de45083a, []int{7, 0}
 }
 
 // Log entry we store in binary logs
@@ -185,16 +193,17 @@ func (m *GrpcLogEntry) Reset()         { *m = GrpcLogEntry{} }
 func (m *GrpcLogEntry) String() string { return proto.CompactTextString(m) }
 func (*GrpcLogEntry) ProtoMessage()    {}
 func (*GrpcLogEntry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{0}
+	return fileDescriptor_b7972e58de45083a, []int{0}
 }
+
 func (m *GrpcLogEntry) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_GrpcLogEntry.Unmarshal(m, b)
 }
 func (m *GrpcLogEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_GrpcLogEntry.Marshal(b, m, deterministic)
 }
-func (dst *GrpcLogEntry) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GrpcLogEntry.Merge(dst, src)
+func (m *GrpcLogEntry) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GrpcLogEntry.Merge(m, src)
 }
 func (m *GrpcLogEntry) XXX_Size() int {
 	return xxx_messageInfo_GrpcLogEntry.Size(m)
@@ -317,9 +326,9 @@ func (m *GrpcLogEntry) GetPeer() *Address {
 	return nil
 }
 
-// XXX_OneofFuncs is for the internal use of the proto package.
-func (*GrpcLogEntry) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
-	return _GrpcLogEntry_OneofMarshaler, _GrpcLogEntry_OneofUnmarshaler, _GrpcLogEntry_OneofSizer, []interface{}{
+// XXX_OneofWrappers is for the internal use of the proto package.
+func (*GrpcLogEntry) XXX_OneofWrappers() []interface{} {
+	return []interface{}{
 		(*GrpcLogEntry_ClientHeader)(nil),
 		(*GrpcLogEntry_ServerHeader)(nil),
 		(*GrpcLogEntry_Message)(nil),
@@ -327,108 +336,6 @@ func (*GrpcLogEntry) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer)
 	}
 }
 
-func _GrpcLogEntry_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
-	m := msg.(*GrpcLogEntry)
-	// payload
-	switch x := m.Payload.(type) {
-	case *GrpcLogEntry_ClientHeader:
-		b.EncodeVarint(6<<3 | proto.WireBytes)
-		if err := b.EncodeMessage(x.ClientHeader); err != nil {
-			return err
-		}
-	case *GrpcLogEntry_ServerHeader:
-		b.EncodeVarint(7<<3 | proto.WireBytes)
-		if err := b.EncodeMessage(x.ServerHeader); err != nil {
-			return err
-		}
-	case *GrpcLogEntry_Message:
-		b.EncodeVarint(8<<3 | proto.WireBytes)
-		if err := b.EncodeMessage(x.Message); err != nil {
-			return err
-		}
-	case *GrpcLogEntry_Trailer:
-		b.EncodeVarint(9<<3 | proto.WireBytes)
-		if err := b.EncodeMessage(x.Trailer); err != nil {
-			return err
-		}
-	case nil:
-	default:
-		return fmt.Errorf("GrpcLogEntry.Payload has unexpected type %T", x)
-	}
-	return nil
-}
-
-func _GrpcLogEntry_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
-	m := msg.(*GrpcLogEntry)
-	switch tag {
-	case 6: // payload.client_header
-		if wire != proto.WireBytes {
-			return true, proto.ErrInternalBadWireType
-		}
-		msg := new(ClientHeader)
-		err := b.DecodeMessage(msg)
-		m.Payload = &GrpcLogEntry_ClientHeader{msg}
-		return true, err
-	case 7: // payload.server_header
-		if wire != proto.WireBytes {
-			return true, proto.ErrInternalBadWireType
-		}
-		msg := new(ServerHeader)
-		err := b.DecodeMessage(msg)
-		m.Payload = &GrpcLogEntry_ServerHeader{msg}
-		return true, err
-	case 8: // payload.message
-		if wire != proto.WireBytes {
-			return true, proto.ErrInternalBadWireType
-		}
-		msg := new(Message)
-		err := b.DecodeMessage(msg)
-		m.Payload = &GrpcLogEntry_Message{msg}
-		return true, err
-	case 9: // payload.trailer
-		if wire != proto.WireBytes {
-			return true, proto.ErrInternalBadWireType
-		}
-		msg := new(Trailer)
-		err := b.DecodeMessage(msg)
-		m.Payload = &GrpcLogEntry_Trailer{msg}
-		return true, err
-	default:
-		return false, nil
-	}
-}
-
-func _GrpcLogEntry_OneofSizer(msg proto.Message) (n int) {
-	m := msg.(*GrpcLogEntry)
-	// payload
-	switch x := m.Payload.(type) {
-	case *GrpcLogEntry_ClientHeader:
-		s := proto.Size(x.ClientHeader)
-		n += 1 // tag and wire
-		n += proto.SizeVarint(uint64(s))
-		n += s
-	case *GrpcLogEntry_ServerHeader:
-		s := proto.Size(x.ServerHeader)
-		n += 1 // tag and wire
-		n += proto.SizeVarint(uint64(s))
-		n += s
-	case *GrpcLogEntry_Message:
-		s := proto.Size(x.Message)
-		n += 1 // tag and wire
-		n += proto.SizeVarint(uint64(s))
-		n += s
-	case *GrpcLogEntry_Trailer:
-		s := proto.Size(x.Trailer)
-		n += 1 // tag and wire
-		n += proto.SizeVarint(uint64(s))
-		n += s
-	case nil:
-	default:
-		panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
-	}
-	return n
-}
-
 type ClientHeader struct {
 	// This contains only the metadata from the application.
 	Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
@@ -453,16 +360,17 @@ func (m *ClientHeader) Reset()         { *m = ClientHeader{} }
 func (m *ClientHeader) String() string { return proto.CompactTextString(m) }
 func (*ClientHeader) ProtoMessage()    {}
 func (*ClientHeader) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{1}
+	return fileDescriptor_b7972e58de45083a, []int{1}
 }
+
 func (m *ClientHeader) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ClientHeader.Unmarshal(m, b)
 }
 func (m *ClientHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_ClientHeader.Marshal(b, m, deterministic)
 }
-func (dst *ClientHeader) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientHeader.Merge(dst, src)
+func (m *ClientHeader) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ClientHeader.Merge(m, src)
 }
 func (m *ClientHeader) XXX_Size() int {
 	return xxx_messageInfo_ClientHeader.Size(m)
@@ -513,16 +421,17 @@ func (m *ServerHeader) Reset()         { *m = ServerHeader{} }
 func (m *ServerHeader) String() string { return proto.CompactTextString(m) }
 func (*ServerHeader) ProtoMessage()    {}
 func (*ServerHeader) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{2}
+	return fileDescriptor_b7972e58de45083a, []int{2}
 }
+
 func (m *ServerHeader) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ServerHeader.Unmarshal(m, b)
 }
 func (m *ServerHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_ServerHeader.Marshal(b, m, deterministic)
 }
-func (dst *ServerHeader) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerHeader.Merge(dst, src)
+func (m *ServerHeader) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ServerHeader.Merge(m, src)
 }
 func (m *ServerHeader) XXX_Size() int {
 	return xxx_messageInfo_ServerHeader.Size(m)
@@ -560,16 +469,17 @@ func (m *Trailer) Reset()         { *m = Trailer{} }
 func (m *Trailer) String() string { return proto.CompactTextString(m) }
 func (*Trailer) ProtoMessage()    {}
 func (*Trailer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{3}
+	return fileDescriptor_b7972e58de45083a, []int{3}
 }
+
 func (m *Trailer) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Trailer.Unmarshal(m, b)
 }
 func (m *Trailer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Trailer.Marshal(b, m, deterministic)
 }
-func (dst *Trailer) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Trailer.Merge(dst, src)
+func (m *Trailer) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Trailer.Merge(m, src)
 }
 func (m *Trailer) XXX_Size() int {
 	return xxx_messageInfo_Trailer.Size(m)
@@ -624,16 +534,17 @@ func (m *Message) Reset()         { *m = Message{} }
 func (m *Message) String() string { return proto.CompactTextString(m) }
 func (*Message) ProtoMessage()    {}
 func (*Message) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{4}
+	return fileDescriptor_b7972e58de45083a, []int{4}
 }
+
 func (m *Message) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Message.Unmarshal(m, b)
 }
 func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Message.Marshal(b, m, deterministic)
 }
-func (dst *Message) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Message.Merge(dst, src)
+func (m *Message) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Message.Merge(m, src)
 }
 func (m *Message) XXX_Size() int {
 	return xxx_messageInfo_Message.Size(m)
@@ -690,16 +601,17 @@ func (m *Metadata) Reset()         { *m = Metadata{} }
 func (m *Metadata) String() string { return proto.CompactTextString(m) }
 func (*Metadata) ProtoMessage()    {}
 func (*Metadata) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{5}
+	return fileDescriptor_b7972e58de45083a, []int{5}
 }
+
 func (m *Metadata) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Metadata.Unmarshal(m, b)
 }
 func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Metadata.Marshal(b, m, deterministic)
 }
-func (dst *Metadata) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Metadata.Merge(dst, src)
+func (m *Metadata) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Metadata.Merge(m, src)
 }
 func (m *Metadata) XXX_Size() int {
 	return xxx_messageInfo_Metadata.Size(m)
@@ -730,16 +642,17 @@ func (m *MetadataEntry) Reset()         { *m = MetadataEntry{} }
 func (m *MetadataEntry) String() string { return proto.CompactTextString(m) }
 func (*MetadataEntry) ProtoMessage()    {}
 func (*MetadataEntry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{6}
+	return fileDescriptor_b7972e58de45083a, []int{6}
 }
+
 func (m *MetadataEntry) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_MetadataEntry.Unmarshal(m, b)
 }
 func (m *MetadataEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_MetadataEntry.Marshal(b, m, deterministic)
 }
-func (dst *MetadataEntry) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MetadataEntry.Merge(dst, src)
+func (m *MetadataEntry) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MetadataEntry.Merge(m, src)
 }
 func (m *MetadataEntry) XXX_Size() int {
 	return xxx_messageInfo_MetadataEntry.Size(m)
@@ -779,16 +692,17 @@ func (m *Address) Reset()         { *m = Address{} }
 func (m *Address) String() string { return proto.CompactTextString(m) }
 func (*Address) ProtoMessage()    {}
 func (*Address) Descriptor() ([]byte, []int) {
-	return fileDescriptor_binarylog_264c8c9c551ce911, []int{7}
+	return fileDescriptor_b7972e58de45083a, []int{7}
 }
+
 func (m *Address) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Address.Unmarshal(m, b)
 }
 func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Address.Marshal(b, m, deterministic)
 }
-func (dst *Address) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Address.Merge(dst, src)
+func (m *Address) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Address.Merge(m, src)
 }
 func (m *Address) XXX_Size() int {
 	return xxx_messageInfo_Address.Size(m)
@@ -821,6 +735,9 @@ func (m *Address) GetIpPort() uint32 {
 }
 
 func init() {
+	proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_EventType", GrpcLogEntry_EventType_name, GrpcLogEntry_EventType_value)
+	proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_Logger", GrpcLogEntry_Logger_name, GrpcLogEntry_Logger_value)
+	proto.RegisterEnum("grpc.binarylog.v1.Address_Type", Address_Type_name, Address_Type_value)
 	proto.RegisterType((*GrpcLogEntry)(nil), "grpc.binarylog.v1.GrpcLogEntry")
 	proto.RegisterType((*ClientHeader)(nil), "grpc.binarylog.v1.ClientHeader")
 	proto.RegisterType((*ServerHeader)(nil), "grpc.binarylog.v1.ServerHeader")
@@ -829,72 +746,67 @@ func init() {
 	proto.RegisterType((*Metadata)(nil), "grpc.binarylog.v1.Metadata")
 	proto.RegisterType((*MetadataEntry)(nil), "grpc.binarylog.v1.MetadataEntry")
 	proto.RegisterType((*Address)(nil), "grpc.binarylog.v1.Address")
-	proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_EventType", GrpcLogEntry_EventType_name, GrpcLogEntry_EventType_value)
-	proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_Logger", GrpcLogEntry_Logger_name, GrpcLogEntry_Logger_value)
-	proto.RegisterEnum("grpc.binarylog.v1.Address_Type", Address_Type_name, Address_Type_value)
 }
 
-func init() {
-	proto.RegisterFile("grpc/binarylog/grpc_binarylog_v1/binarylog.proto", fileDescriptor_binarylog_264c8c9c551ce911)
-}
+func init() { proto.RegisterFile("grpc/binlog/v1/binarylog.proto", fileDescriptor_b7972e58de45083a) }
 
-var fileDescriptor_binarylog_264c8c9c551ce911 = []byte{
-	// 900 bytes of a gzipped FileDescriptorProto
+var fileDescriptor_b7972e58de45083a = []byte{
+	// 904 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x51, 0x6f, 0xe3, 0x44,
-	0x10, 0x3e, 0x37, 0x69, 0xdc, 0x4c, 0x92, 0xca, 0x5d, 0x95, 0x3b, 0x5f, 0x29, 0x34, 0xb2, 0x04,
-	0x0a, 0x42, 0x72, 0xb9, 0x94, 0xeb, 0xf1, 0x02, 0x52, 0x92, 0xfa, 0xd2, 0x88, 0x5c, 0x1a, 0x6d,
-	0x72, 0x3d, 0x40, 0x48, 0xd6, 0x36, 0x5e, 0x1c, 0x0b, 0xc7, 0x6b, 0xd6, 0x9b, 0xa0, 0xfc, 0x2c,
-	0xde, 0x90, 0xee, 0x77, 0xf1, 0x8e, 0xbc, 0x6b, 0x27, 0xa6, 0x69, 0x0f, 0x09, 0xde, 0x3c, 0xdf,
-	0x7c, 0xf3, 0xcd, 0xee, 0x78, 0x66, 0x16, 0xbe, 0xf2, 0x79, 0x3c, 0x3b, 0xbf, 0x0b, 0x22, 0xc2,
-	0xd7, 0x21, 0xf3, 0xcf, 0x53, 0xd3, 0xdd, 0x98, 0xee, 0xea, 0xc5, 0xd6, 0x67, 0xc7, 0x9c, 0x09,
-	0x86, 0x8e, 0x52, 0x8a, 0xbd, 0x45, 0x57, 0x2f, 0x4e, 0x3e, 0xf5, 0x19, 0xf3, 0x43, 0x7a, 0x2e,
-	0x09, 0x77, 0xcb, 0x5f, 0xce, 0xbd, 0x25, 0x27, 0x22, 0x60, 0x91, 0x0a, 0x39, 0x39, 0xbb, 0xef,
-	0x17, 0xc1, 0x82, 0x26, 0x82, 0x2c, 0x62, 0x45, 0xb0, 0xde, 0xeb, 0x50, 0xef, 0xf3, 0x78, 0x36,
-	0x64, 0xbe, 0x13, 0x09, 0xbe, 0x46, 0xdf, 0x40, 0x75, 0xc3, 0x31, 0xb5, 0xa6, 0xd6, 0xaa, 0xb5,
-	0x4f, 0x6c, 0xa5, 0x62, 0xe7, 0x2a, 0xf6, 0x34, 0x67, 0xe0, 0x2d, 0x19, 0x3d, 0x03, 0x7d, 0x46,
-	0xc2, 0xd0, 0x0d, 0x3c, 0x73, 0xaf, 0xa9, 0xb5, 0xca, 0xb8, 0x92, 0x9a, 0x03, 0x0f, 0xbd, 0x84,
-	0x67, 0x09, 0xfd, 0x6d, 0x49, 0xa3, 0x19, 0x75, 0x03, 0xcf, 0xfd, 0x3d, 0x10, 0xf3, 0x20, 0x72,
-	0x53, 0xa7, 0x59, 0x92, 0xc4, 0xe3, 0xdc, 0x3d, 0xf0, 0xde, 0x49, 0x67, 0x8f, 0x84, 0x21, 0xfa,
-	0x16, 0xca, 0x62, 0x1d, 0x53, 0xb3, 0xdc, 0xd4, 0x5a, 0x87, 0xed, 0x2f, 0xec, 0x9d, 0xdb, 0xdb,
-	0xc5, 0x83, 0xdb, 0xce, 0x8a, 0x46, 0x62, 0xba, 0x8e, 0x29, 0x96, 0x61, 0xe8, 0x3b, 0xa8, 0x84,
-	0xcc, 0xf7, 0x29, 0x37, 0xf7, 0xa5, 0xc0, 0xe7, 0xff, 0x26, 0x30, 0x94, 0x6c, 0x9c, 0x45, 0xa1,
-	0xd7, 0xd0, 0x98, 0x85, 0x01, 0x8d, 0x84, 0x3b, 0xa7, 0xc4, 0xa3, 0xdc, 0xac, 0xc8, 0x62, 0x9c,
-	0x3d, 0x20, 0xd3, 0x93, 0xbc, 0x6b, 0x49, 0xbb, 0x7e, 0x82, 0xeb, 0xb3, 0x82, 0x9d, 0xea, 0x24,
-	0x94, 0xaf, 0x28, 0xcf, 0x75, 0xf4, 0x47, 0x75, 0x26, 0x92, 0xb7, 0xd5, 0x49, 0x0a, 0x36, 0xba,
-	0x04, 0x7d, 0x41, 0x93, 0x84, 0xf8, 0xd4, 0x3c, 0xc8, 0x7f, 0xcb, 0x8e, 0xc2, 0x1b, 0xc5, 0xb8,
-	0x7e, 0x82, 0x73, 0x72, 0x1a, 0x27, 0x38, 0x09, 0x42, 0xca, 0xcd, 0xea, 0xa3, 0x71, 0x53, 0xc5,
-	0x48, 0xe3, 0x32, 0x32, 0xfa, 0x12, 0x8e, 0x62, 0xb2, 0x0e, 0x19, 0xf1, 0x5c, 0xc1, 0x97, 0xd1,
-	0x8c, 0x08, 0xea, 0x99, 0xd0, 0xd4, 0x5a, 0x07, 0xd8, 0xc8, 0x1c, 0xd3, 0x1c, 0x47, 0x36, 0x94,
-	0x63, 0x4a, 0xb9, 0x59, 0x7b, 0x34, 0x43, 0xc7, 0xf3, 0x38, 0x4d, 0x12, 0x2c, 0x79, 0xd6, 0x5f,
-	0x1a, 0x54, 0x37, 0x3f, 0x0c, 0x3d, 0x05, 0xe4, 0xdc, 0x3a, 0xa3, 0xa9, 0x3b, 0xfd, 0x71, 0xec,
-	0xb8, 0x6f, 0x47, 0xdf, 0x8f, 0x6e, 0xde, 0x8d, 0x8c, 0x27, 0xe8, 0x14, 0xcc, 0x02, 0xde, 0x1b,
-	0x0e, 0xd2, 0xef, 0x6b, 0xa7, 0x73, 0xe5, 0x60, 0x43, 0xbb, 0xe7, 0x9d, 0x38, 0xf8, 0xd6, 0xc1,
-	0xb9, 0x77, 0x0f, 0x7d, 0x02, 0xcf, 0x77, 0x63, 0xdf, 0x38, 0x93, 0x49, 0xa7, 0xef, 0x18, 0xa5,
-	0x7b, 0xee, 0x2c, 0x38, 0x77, 0x97, 0x51, 0x13, 0x4e, 0x1f, 0xc8, 0xdc, 0x19, 0xbe, 0x76, 0x7b,
-	0xc3, 0x9b, 0x89, 0x63, 0xec, 0x3f, 0x2c, 0x30, 0xc5, 0x9d, 0xc1, 0xd0, 0xc1, 0x46, 0x05, 0x7d,
-	0x04, 0x47, 0x45, 0x81, 0xce, 0xa8, 0xe7, 0x0c, 0x0d, 0xdd, 0xea, 0x42, 0x45, 0xb5, 0x19, 0x42,
-	0x70, 0x38, 0xbc, 0xe9, 0xf7, 0x1d, 0x5c, 0xb8, 0xef, 0x11, 0x34, 0x32, 0x4c, 0x65, 0x34, 0xb4,
-	0x02, 0xa4, 0x52, 0x18, 0x7b, 0xdd, 0x2a, 0xe8, 0x59, 0xfd, 0xad, 0xf7, 0x1a, 0xd4, 0x8b, 0xcd,
-	0x87, 0x5e, 0xc1, 0xc1, 0x82, 0x0a, 0xe2, 0x11, 0x41, 0xb2, 0xe1, 0xfd, 0xf8, 0xc1, 0x2e, 0x51,
-	0x14, 0xbc, 0x21, 0xa3, 0x33, 0xa8, 0x2d, 0xa8, 0x98, 0x33, 0xcf, 0x8d, 0xc8, 0x82, 0xca, 0x01,
-	0xae, 0x62, 0x50, 0xd0, 0x88, 0x2c, 0x28, 0x3a, 0x85, 0x2a, 0x59, 0x8a, 0x39, 0xe3, 0x81, 0x58,
-	0xcb, 0xb1, 0xad, 0xe2, 0x2d, 0x80, 0x2e, 0x40, 0x4f, 0x17, 0x01, 0x5b, 0x0a, 0x39, 0xae, 0xb5,
-	0xf6, 0xf3, 0x9d, 0x9d, 0x71, 0x95, 0x6d, 0x26, 0x9c, 0x33, 0xad, 0x3e, 0xd4, 0x8b, 0x1d, 0xff,
-	0x9f, 0x0f, 0x6f, 0xfd, 0xa1, 0x81, 0x9e, 0x75, 0xf0, 0xff, 0xaa, 0x40, 0x22, 0x88, 0x58, 0x26,
-	0xee, 0x8c, 0x79, 0xaa, 0x02, 0x0d, 0x0c, 0x0a, 0xea, 0x31, 0x8f, 0xa2, 0xcf, 0xe0, 0x30, 0x23,
-	0xe4, 0x73, 0xa8, 0xca, 0xd0, 0x50, 0x68, 0x36, 0x7a, 0x05, 0x9a, 0x47, 0x05, 0x09, 0xc2, 0x44,
-	0x56, 0xa4, 0x9e, 0xd3, 0xae, 0x14, 0x68, 0xbd, 0x04, 0x3d, 0x8f, 0x78, 0x0a, 0x95, 0x90, 0x46,
-	0xbe, 0x98, 0xcb, 0x03, 0x37, 0x70, 0x66, 0x21, 0x04, 0x65, 0x79, 0x8d, 0x3d, 0x19, 0x2f, 0xbf,
-	0xad, 0x2e, 0x1c, 0xe4, 0x67, 0x47, 0x97, 0xb0, 0x4f, 0xd3, 0xcd, 0x65, 0x6a, 0xcd, 0x52, 0xab,
-	0xd6, 0x6e, 0x7e, 0xe0, 0x9e, 0x72, 0xc3, 0x61, 0x45, 0xb7, 0x5e, 0x41, 0xe3, 0x1f, 0x38, 0x32,
-	0xa0, 0xf4, 0x2b, 0x5d, 0xcb, 0xec, 0x55, 0x9c, 0x7e, 0xa2, 0x63, 0xd8, 0x5f, 0x91, 0x70, 0x49,
-	0xb3, 0xdc, 0xca, 0xb0, 0xfe, 0xd4, 0x40, 0xcf, 0xe6, 0x18, 0x5d, 0x64, 0xdb, 0x59, 0x93, 0xcb,
-	0xf5, 0xec, 0xf1, 0x89, 0xb7, 0x0b, 0x3b, 0xd9, 0x04, 0x9d, 0x28, 0x34, 0xeb, 0xb0, 0xdc, 0x4c,
-	0x1f, 0x8f, 0x20, 0x76, 0x63, 0xc6, 0x85, 0xac, 0x6a, 0x03, 0x57, 0x82, 0x78, 0xcc, 0xb8, 0xb0,
-	0x1c, 0x28, 0xcb, 0x1d, 0x61, 0x40, 0xfd, 0xde, 0x76, 0x68, 0x40, 0x55, 0x22, 0x83, 0xf1, 0xed,
-	0xd7, 0x86, 0x56, 0x34, 0x2f, 0x8d, 0xbd, 0x8d, 0xf9, 0x76, 0x34, 0xf8, 0xc1, 0x28, 0x75, 0x7f,
-	0x86, 0xe3, 0x80, 0xed, 0x1e, 0xb2, 0x7b, 0xd8, 0x95, 0xd6, 0x90, 0xf9, 0xe3, 0xb4, 0x51, 0xc7,
-	0xda, 0x4f, 0xed, 0xac, 0x71, 0x7d, 0x16, 0x92, 0xc8, 0xb7, 0x19, 0x57, 0x4f, 0xf3, 0x87, 0x5e,
-	0xea, 0xbb, 0x8a, 0xec, 0xf2, 0x8b, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe7, 0xf6, 0x4b, 0x50,
-	0xd4, 0x07, 0x00, 0x00,
+	0x10, 0xae, 0xdb, 0x34, 0x6e, 0x26, 0x49, 0xe5, 0xae, 0xca, 0x9d, 0xaf, 0x94, 0x6b, 0x64, 0x09,
+	0x14, 0x84, 0xe4, 0xa8, 0x29, 0xd7, 0xe3, 0x05, 0xa4, 0x24, 0xf5, 0xa5, 0x11, 0xb9, 0x34, 0xda,
+	0xe4, 0x7a, 0x80, 0x90, 0xac, 0x6d, 0xbc, 0x38, 0x16, 0x8e, 0xd7, 0xac, 0x37, 0x41, 0xf9, 0x59,
+	0xbc, 0x21, 0xdd, 0xef, 0xe2, 0x1d, 0x79, 0xd7, 0x4e, 0x4d, 0xd3, 0x82, 0xc4, 0xbd, 0xed, 0x7c,
+	0xf3, 0xcd, 0x37, 0xbb, 0xe3, 0x99, 0x31, 0xbc, 0xf4, 0x79, 0x3c, 0x6b, 0xdd, 0x05, 0x51, 0xc8,
+	0xfc, 0xd6, 0xea, 0x3c, 0x3d, 0x11, 0xbe, 0x0e, 0x99, 0x6f, 0xc7, 0x9c, 0x09, 0x86, 0x8e, 0x52,
+	0xbf, 0x7d, 0x8f, 0xae, 0xce, 0x4f, 0x5e, 0xfa, 0x8c, 0xf9, 0x21, 0x6d, 0x49, 0xc2, 0xdd, 0xf2,
+	0x97, 0x96, 0xb7, 0xe4, 0x44, 0x04, 0x2c, 0x52, 0x21, 0x27, 0x67, 0x0f, 0xfd, 0x22, 0x58, 0xd0,
+	0x44, 0x90, 0x45, 0xac, 0x08, 0xd6, 0x07, 0x1d, 0x6a, 0x7d, 0x1e, 0xcf, 0x86, 0xcc, 0x77, 0x22,
+	0xc1, 0xd7, 0xe8, 0x1b, 0xa8, 0x6c, 0x38, 0xa6, 0xd6, 0xd0, 0x9a, 0xd5, 0xf6, 0x89, 0xad, 0x54,
+	0xec, 0x5c, 0xc5, 0x9e, 0xe6, 0x0c, 0x7c, 0x4f, 0x46, 0xcf, 0x41, 0x9f, 0x91, 0x30, 0x74, 0x03,
+	0xcf, 0xdc, 0x6d, 0x68, 0xcd, 0x12, 0x2e, 0xa7, 0xe6, 0xc0, 0x43, 0xaf, 0xe0, 0x79, 0x42, 0x7f,
+	0x5b, 0xd2, 0x68, 0x46, 0xdd, 0xc0, 0x73, 0x7f, 0x0f, 0xc4, 0x3c, 0x88, 0xdc, 0xd4, 0x69, 0xee,
+	0x49, 0xe2, 0x71, 0xee, 0x1e, 0x78, 0xef, 0xa5, 0xb3, 0x47, 0xc2, 0x10, 0x7d, 0x0b, 0x25, 0xb1,
+	0x8e, 0xa9, 0x59, 0x6a, 0x68, 0xcd, 0xc3, 0xf6, 0x97, 0xf6, 0xd6, 0xeb, 0xed, 0xe2, 0xc5, 0x6d,
+	0x67, 0x45, 0x23, 0x31, 0x5d, 0xc7, 0x14, 0xcb, 0x30, 0xf4, 0x1d, 0x94, 0x43, 0xe6, 0xfb, 0x94,
+	0x9b, 0xfb, 0x52, 0xe0, 0x8b, 0xff, 0x12, 0x18, 0x4a, 0x36, 0xce, 0xa2, 0xd0, 0x1b, 0xa8, 0xcf,
+	0xc2, 0x80, 0x46, 0xc2, 0x9d, 0x53, 0xe2, 0x51, 0x6e, 0x96, 0x65, 0x31, 0xce, 0x1e, 0x91, 0xe9,
+	0x49, 0xde, 0xb5, 0xa4, 0x5d, 0xef, 0xe0, 0xda, 0xac, 0x60, 0xa7, 0x3a, 0x09, 0xe5, 0x2b, 0xca,
+	0x73, 0x1d, 0xfd, 0x49, 0x9d, 0x89, 0xe4, 0xdd, 0xeb, 0x24, 0x05, 0x1b, 0x5d, 0x82, 0xbe, 0xa0,
+	0x49, 0x42, 0x7c, 0x6a, 0x1e, 0xe4, 0x9f, 0x65, 0x4b, 0xe1, 0xad, 0x62, 0x5c, 0xef, 0xe0, 0x9c,
+	0x9c, 0xc6, 0x09, 0x4e, 0x82, 0x90, 0x72, 0xb3, 0xf2, 0x64, 0xdc, 0x54, 0x31, 0xd2, 0xb8, 0x8c,
+	0x8c, 0xbe, 0x82, 0xa3, 0x98, 0xac, 0x43, 0x46, 0x3c, 0x57, 0xf0, 0x65, 0x34, 0x23, 0x82, 0x7a,
+	0x26, 0x34, 0xb4, 0xe6, 0x01, 0x36, 0x32, 0xc7, 0x34, 0xc7, 0x91, 0x0d, 0xa5, 0x98, 0x52, 0x6e,
+	0x56, 0x9f, 0xcc, 0xd0, 0xf1, 0x3c, 0x4e, 0x93, 0x04, 0x4b, 0x9e, 0xf5, 0x97, 0x06, 0x95, 0xcd,
+	0x07, 0x43, 0xcf, 0x00, 0x39, 0xb7, 0xce, 0x68, 0xea, 0x4e, 0x7f, 0x1c, 0x3b, 0xee, 0xbb, 0xd1,
+	0xf7, 0xa3, 0x9b, 0xf7, 0x23, 0x63, 0x07, 0x9d, 0x82, 0x59, 0xc0, 0x7b, 0xc3, 0x41, 0x7a, 0xbe,
+	0x76, 0x3a, 0x57, 0x0e, 0x36, 0xb4, 0x07, 0xde, 0x89, 0x83, 0x6f, 0x1d, 0x9c, 0x7b, 0x77, 0xd1,
+	0x67, 0xf0, 0x62, 0x3b, 0xf6, 0xad, 0x33, 0x99, 0x74, 0xfa, 0x8e, 0xb1, 0xf7, 0xc0, 0x9d, 0x05,
+	0xe7, 0xee, 0x12, 0x6a, 0xc0, 0xe9, 0x23, 0x99, 0x3b, 0xc3, 0x37, 0x6e, 0x6f, 0x78, 0x33, 0x71,
+	0x8c, 0xfd, 0xc7, 0x05, 0xa6, 0xb8, 0x33, 0x18, 0x3a, 0xd8, 0x28, 0xa3, 0x4f, 0xe0, 0xa8, 0x28,
+	0xd0, 0x19, 0xf5, 0x9c, 0xa1, 0xa1, 0x5b, 0x5d, 0x28, 0xab, 0x36, 0x43, 0x08, 0x0e, 0x87, 0x37,
+	0xfd, 0xbe, 0x83, 0x0b, 0xef, 0x3d, 0x82, 0x7a, 0x86, 0xa9, 0x8c, 0x86, 0x56, 0x80, 0x54, 0x0a,
+	0x63, 0xb7, 0x5b, 0x01, 0x3d, 0xab, 0xbf, 0xf5, 0x41, 0x83, 0x5a, 0xb1, 0xf9, 0xd0, 0x6b, 0x38,
+	0x58, 0x50, 0x41, 0x3c, 0x22, 0x48, 0x36, 0xbc, 0x9f, 0x3e, 0xda, 0x25, 0x8a, 0x82, 0x37, 0x64,
+	0x74, 0x06, 0xd5, 0x05, 0x15, 0x73, 0xe6, 0xb9, 0x11, 0x59, 0x50, 0x39, 0xc0, 0x15, 0x0c, 0x0a,
+	0x1a, 0x91, 0x05, 0x45, 0xa7, 0x50, 0x21, 0x4b, 0x31, 0x67, 0x3c, 0x10, 0x6b, 0x39, 0xb6, 0x15,
+	0x7c, 0x0f, 0xa0, 0x0b, 0xd0, 0xd3, 0x45, 0xc0, 0x96, 0x42, 0x8e, 0x6b, 0xb5, 0xfd, 0x62, 0x6b,
+	0x67, 0x5c, 0x65, 0x9b, 0x09, 0xe7, 0x4c, 0xab, 0x0f, 0xb5, 0x62, 0xc7, 0xff, 0xef, 0xcb, 0x5b,
+	0x7f, 0x68, 0xa0, 0x67, 0x1d, 0xfc, 0x51, 0x15, 0x48, 0x04, 0x11, 0xcb, 0xc4, 0x9d, 0x31, 0x4f,
+	0x55, 0xa0, 0x8e, 0x41, 0x41, 0x3d, 0xe6, 0x51, 0xf4, 0x39, 0x1c, 0x66, 0x84, 0x7c, 0x0e, 0x55,
+	0x19, 0xea, 0x0a, 0xcd, 0x46, 0xaf, 0x40, 0xf3, 0xa8, 0x20, 0x41, 0x98, 0xc8, 0x8a, 0xd4, 0x72,
+	0xda, 0x95, 0x02, 0xad, 0x57, 0xa0, 0xe7, 0x11, 0xcf, 0xa0, 0x1c, 0xd2, 0xc8, 0x17, 0x73, 0x79,
+	0xe1, 0x3a, 0xce, 0x2c, 0x84, 0xa0, 0x24, 0x9f, 0xb1, 0x2b, 0xe3, 0xe5, 0xd9, 0xea, 0xc2, 0x41,
+	0x7e, 0x77, 0x74, 0x09, 0xfb, 0x34, 0xdd, 0x5c, 0xa6, 0xd6, 0xd8, 0x6b, 0x56, 0xdb, 0x8d, 0x7f,
+	0x79, 0xa7, 0xdc, 0x70, 0x58, 0xd1, 0xad, 0xd7, 0x50, 0xff, 0x07, 0x8e, 0x0c, 0xd8, 0xfb, 0x95,
+	0xae, 0x65, 0xf6, 0x0a, 0x4e, 0x8f, 0xe8, 0x18, 0xf6, 0x57, 0x24, 0x5c, 0xd2, 0x2c, 0xb7, 0x32,
+	0xac, 0x3f, 0x35, 0xd0, 0xb3, 0x39, 0x46, 0x17, 0xd9, 0x76, 0xd6, 0xe4, 0x72, 0x3d, 0x7b, 0x7a,
+	0xe2, 0xed, 0xc2, 0x4e, 0x36, 0x41, 0x27, 0x0a, 0xcd, 0x3a, 0x2c, 0x37, 0xd3, 0x9f, 0x47, 0x10,
+	0xbb, 0x31, 0xe3, 0x42, 0x56, 0xb5, 0x8e, 0xcb, 0x41, 0x3c, 0x66, 0x5c, 0x58, 0x0e, 0x94, 0xe4,
+	0x8e, 0x30, 0xa0, 0xf6, 0x60, 0x3b, 0xd4, 0xa1, 0x22, 0x91, 0xc1, 0xf8, 0xf6, 0x6b, 0x43, 0x2b,
+	0x9a, 0x97, 0xc6, 0xee, 0xc6, 0x7c, 0x37, 0x1a, 0xfc, 0x60, 0xec, 0x75, 0x7f, 0x86, 0xe3, 0x80,
+	0x6d, 0x5f, 0xb2, 0x7b, 0xd8, 0x95, 0xd6, 0x90, 0xf9, 0xe3, 0xb4, 0x51, 0xc7, 0xda, 0x4f, 0xed,
+	0xac, 0x71, 0x7d, 0x16, 0x92, 0xc8, 0xb7, 0x19, 0xf7, 0x5b, 0xf9, 0x7f, 0x59, 0x85, 0x49, 0xd3,
+	0xdd, 0x98, 0xee, 0xea, 0xfc, 0xae, 0x2c, 0xbb, 0xfc, 0xe2, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff,
+	0x10, 0x93, 0x68, 0x41, 0xc2, 0x07, 0x00, 0x00,
 }
diff --git a/channelz/service/regenerate.sh b/channelz/service/regenerate.sh
deleted file mode 100755
index 9c255b12cab1..000000000000
--- a/channelz/service/regenerate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2018 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/channelz/v1
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/channelz/v1/channelz.proto > grpc/channelz/v1/channelz.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/channelz/v1/*.proto
-popd
-rm -f ../grpc_channelz_v1/*.pb.go
-cp "$TMP"/grpc/channelz/v1/*.pb.go ../grpc_channelz_v1/
-
diff --git a/channelz/service/service.go b/channelz/service/service.go
index c8174e5e09d9..b7650b3b29fa 100644
--- a/channelz/service/service.go
+++ b/channelz/service/service.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate ./regenerate.sh
-
 // Package service provides an implementation for channelz service server.
 package service
 
diff --git a/credentials/alts/internal/common.go b/credentials/alts/internal/common.go
index 33fba81239ad..3896e8cf2b52 100644
--- a/credentials/alts/internal/common.go
+++ b/credentials/alts/internal/common.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate ./regenerate.sh
-
 // Package internal contains common core functionality for ALTS.
 package internal
 
diff --git a/credentials/alts/internal/regenerate.sh b/credentials/alts/internal/regenerate.sh
deleted file mode 100755
index a79c4201b36b..000000000000
--- a/credentials/alts/internal/regenerate.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Copyright 2018 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/gcp
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/gcp/altscontext.proto > grpc/gcp/altscontext.proto
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/gcp/handshaker.proto > grpc/gcp/handshaker.proto
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/gcp/transport_security_common.proto > grpc/gcp/transport_security_common.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/gcp/*.proto
-popd
-rm -f proto/grpc_gcp/*.pb.go
-cp "$TMP"/grpc/gcp/*.pb.go proto/grpc_gcp/
-
diff --git a/doc.go b/doc.go
index 187adbb117f2..0022859ad746 100644
--- a/doc.go
+++ b/doc.go
@@ -16,6 +16,8 @@
  *
  */
 
+//go:generate ./regenerate.sh
+
 /*
 Package grpc implements an RPC system called gRPC.
 
diff --git a/examples/features/proto/doc.go b/examples/features/proto/doc.go
deleted file mode 100644
index 1400e63baa36..000000000000
--- a/examples/features/proto/doc.go
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- *
- * Copyright 2018 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-//go:generate protoc -I ./echo --go_out=plugins=grpc,paths=source_relative:./echo ./echo/echo.proto
-
-// Package proto is for go generate.
-package proto
diff --git a/examples/features/proto/echo/echo.pb.go b/examples/features/proto/echo/echo.pb.go
index ccf98a685799..520356d5affe 100644
--- a/examples/features/proto/echo/echo.pb.go
+++ b/examples/features/proto/echo/echo.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: echo.proto
+// source: examples/features/proto/echo/echo.proto
 
 package echo
 
@@ -36,7 +36,7 @@ func (m *EchoRequest) Reset()         { *m = EchoRequest{} }
 func (m *EchoRequest) String() string { return proto.CompactTextString(m) }
 func (*EchoRequest) ProtoMessage()    {}
 func (*EchoRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_08134aea513e0001, []int{0}
+	return fileDescriptor_2fd1d686b7b805dc, []int{0}
 }
 
 func (m *EchoRequest) XXX_Unmarshal(b []byte) error {
@@ -76,7 +76,7 @@ func (m *EchoResponse) Reset()         { *m = EchoResponse{} }
 func (m *EchoResponse) String() string { return proto.CompactTextString(m) }
 func (*EchoResponse) ProtoMessage()    {}
 func (*EchoResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_08134aea513e0001, []int{1}
+	return fileDescriptor_2fd1d686b7b805dc, []int{1}
 }
 
 func (m *EchoResponse) XXX_Unmarshal(b []byte) error {
@@ -109,25 +109,27 @@ func init() {
 	proto.RegisterType((*EchoResponse)(nil), "grpc.examples.echo.EchoResponse")
 }
 
-func init() { proto.RegisterFile("echo.proto", fileDescriptor_08134aea513e0001) }
+func init() {
+	proto.RegisterFile("examples/features/proto/echo/echo.proto", fileDescriptor_2fd1d686b7b805dc)
+}
 
-var fileDescriptor_08134aea513e0001 = []byte{
-	// 234 bytes of a gzipped FileDescriptorProto
+var fileDescriptor_2fd1d686b7b805dc = []byte{
+	// 236 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xb1, 0x4b, 0x03, 0x31,
-	0x14, 0x87, 0x3d, 0x11, 0xa5, 0x4f, 0xa7, 0xb8, 0x94, 0x2e, 0x96, 0x5b, 0xbc, 0x29, 0x29, 0x16,
-	0xff, 0x81, 0x8a, 0xbb, 0xb4, 0xb8, 0x88, 0x4b, 0x3c, 0x7f, 0xa6, 0x81, 0x5c, 0xde, 0xf9, 0x92,
-	0x8a, 0xfe, 0xed, 0x2e, 0x92, 0x2b, 0x05, 0x41, 0xba, 0xd5, 0x2d, 0x8f, 0x7c, 0xef, 0xfb, 0x96,
-	0x47, 0x84, 0x76, 0xcd, 0xba, 0x17, 0xce, 0xac, 0x94, 0x93, 0xbe, 0xd5, 0xf8, 0xb4, 0x5d, 0x1f,
-	0x90, 0x74, 0xf9, 0xa9, 0xaf, 0xe9, 0xfc, 0xbe, 0x5d, 0xf3, 0x12, 0xef, 0x1b, 0xa4, 0xac, 0xc6,
-	0x74, 0xd6, 0x21, 0x25, 0xeb, 0x30, 0xae, 0xa6, 0x55, 0x33, 0x5a, 0xee, 0xc6, 0xba, 0xa1, 0x8b,
-	0x2d, 0x98, 0x7a, 0x8e, 0x09, 0xfb, 0xc9, 0x9b, 0xef, 0x63, 0x3a, 0x29, 0xa8, 0x7a, 0xa0, 0xd1,
-	0x63, 0xb4, 0xf2, 0x35, 0x0c, 0x57, 0xfa, 0x6f, 0x5d, 0xff, 0x4a, 0x4f, 0xa6, 0xfb, 0x81, 0x6d,
-	0xb2, 0x3e, 0x52, 0xcf, 0x74, 0xb9, 0x82, 0x7c, 0x40, 0x56, 0x59, 0x60, 0x3b, 0x1f, 0xdd, 0xc1,
-	0xdc, 0xb3, 0xaa, 0xd8, 0xef, 0x82, 0x47, 0xcc, 0x87, 0xb7, 0x37, 0x95, 0x02, 0x4d, 0x16, 0xfe,
-	0xd5, 0x0b, 0xda, 0xec, 0x39, 0xda, 0xf0, 0x1f, 0x91, 0x59, 0xb5, 0xb8, 0x7d, 0x9a, 0x3b, 0x66,
-	0x17, 0xa0, 0x1d, 0x07, 0x1b, 0x9d, 0x66, 0x71, 0xa6, 0xac, 0x9a, 0xdd, 0xaa, 0x79, 0x83, 0xcd,
-	0x1b, 0x41, 0x32, 0xc3, 0x59, 0x98, 0x62, 0x7a, 0x39, 0x1d, 0xde, 0xf3, 0x9f, 0x00, 0x00, 0x00,
-	0xff, 0xff, 0x23, 0x14, 0x26, 0x96, 0x30, 0x02, 0x00, 0x00,
+	0x14, 0xc6, 0x8d, 0x88, 0xd2, 0xe8, 0x14, 0x97, 0xd2, 0xc5, 0x72, 0x4b, 0x6f, 0x4a, 0x8a, 0xc5,
+	0x7f, 0xa0, 0xe2, 0x2e, 0x2d, 0x2e, 0xe2, 0x12, 0xcf, 0xcf, 0x34, 0x90, 0xcb, 0x3b, 0x5f, 0x52,
+	0xd1, 0xbf, 0xdd, 0x45, 0x72, 0x47, 0x41, 0x90, 0x3a, 0xd5, 0x25, 0xe4, 0xe3, 0xfd, 0xde, 0xf7,
+	0x5b, 0x9e, 0x9c, 0xe1, 0xc3, 0xb6, 0x5d, 0x40, 0x32, 0xaf, 0xb0, 0x79, 0xcb, 0x48, 0xa6, 0x63,
+	0xca, 0x64, 0xd0, 0x6c, 0x86, 0x47, 0xf7, 0x59, 0x29, 0xc7, 0x5d, 0xa3, 0x77, 0xb4, 0x2e, 0x93,
+	0x6a, 0x26, 0xcf, 0xef, 0x9a, 0x0d, 0xad, 0xf0, 0xb6, 0x45, 0xca, 0x6a, 0x2c, 0xcf, 0x5a, 0xa4,
+	0x64, 0x1d, 0xc6, 0x62, 0x2a, 0xea, 0xd1, 0x6a, 0x17, 0xab, 0x5a, 0x5e, 0x0c, 0x60, 0xea, 0x28,
+	0x26, 0xec, 0x27, 0xaf, 0xbf, 0x8e, 0xe5, 0x49, 0x41, 0xd5, 0xbd, 0x1c, 0x3d, 0x44, 0xcb, 0x9f,
+	0x7d, 0xb8, 0xd2, 0xbf, 0xed, 0xfa, 0x87, 0x7a, 0x32, 0xdd, 0x0f, 0x0c, 0xca, 0xea, 0x48, 0x3d,
+	0xc9, 0xcb, 0x35, 0xf8, 0x1d, 0xbc, 0xce, 0x0c, 0xdb, 0xfa, 0xe8, 0x0e, 0xd6, 0x3d, 0x17, 0xa5,
+	0xfd, 0x36, 0x78, 0xc4, 0x7c, 0xf8, 0xf6, 0x5a, 0x28, 0xc8, 0xc9, 0xd2, 0xbf, 0x78, 0x46, 0x93,
+	0x3d, 0x45, 0x1b, 0xfe, 0x43, 0x32, 0x17, 0xcb, 0x9b, 0xc7, 0x85, 0x23, 0x72, 0x01, 0xda, 0x51,
+	0xb0, 0xd1, 0x69, 0x62, 0x67, 0xca, 0xaa, 0xf9, 0xeb, 0x4c, 0x9e, 0x4f, 0xfb, 0xff, 0xe2, 0x3b,
+	0x00, 0x00, 0xff, 0xff, 0xf7, 0x79, 0x87, 0xf0, 0x4d, 0x02, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -417,5 +419,5 @@ var _Echo_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "echo.proto",
+	Metadata: "examples/features/proto/echo/echo.proto",
 }
diff --git a/examples/features/proto/echo/echo.proto b/examples/features/proto/echo/echo.proto
index 93f9c941b997..2dde5633e2dd 100644
--- a/examples/features/proto/echo/echo.proto
+++ b/examples/features/proto/echo/echo.proto
@@ -18,10 +18,10 @@
 
 syntax = "proto3";
 
-package grpc.examples.echo;
-
 option go_package = "google.golang.org/grpc/examples/features/proto/echo";
 
+package grpc.examples.echo;
+
 // EchoRequest is the request for echo.
 message EchoRequest {
   string message = 1;
diff --git a/examples/helloworld/greeter_server/main.go b/examples/helloworld/greeter_server/main.go
index eb2fa0ef8e28..15604f9fc1f4 100644
--- a/examples/helloworld/greeter_server/main.go
+++ b/examples/helloworld/greeter_server/main.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc -I ../helloworld --go_out=plugins=grpc:../helloworld ../helloworld/helloworld.proto
-
 // Package main implements a server for Greeter service.
 package main
 
diff --git a/examples/helloworld/helloworld/helloworld.pb.go b/examples/helloworld/helloworld/helloworld.pb.go
index b12e92f23470..9fde58df0a6e 100644
--- a/examples/helloworld/helloworld/helloworld.pb.go
+++ b/examples/helloworld/helloworld/helloworld.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: helloworld.proto
+// source: examples/helloworld/helloworld/helloworld.proto
 
 package helloworld
 
@@ -36,7 +36,7 @@ func (m *HelloRequest) Reset()         { *m = HelloRequest{} }
 func (m *HelloRequest) String() string { return proto.CompactTextString(m) }
 func (*HelloRequest) ProtoMessage()    {}
 func (*HelloRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_17b8c58d586b62f2, []int{0}
+	return fileDescriptor_b83ea99a5323a2c7, []int{0}
 }
 
 func (m *HelloRequest) XXX_Unmarshal(b []byte) error {
@@ -76,7 +76,7 @@ func (m *HelloReply) Reset()         { *m = HelloReply{} }
 func (m *HelloReply) String() string { return proto.CompactTextString(m) }
 func (*HelloReply) ProtoMessage()    {}
 func (*HelloReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_17b8c58d586b62f2, []int{1}
+	return fileDescriptor_b83ea99a5323a2c7, []int{1}
 }
 
 func (m *HelloReply) XXX_Unmarshal(b []byte) error {
@@ -109,21 +109,25 @@ func init() {
 	proto.RegisterType((*HelloReply)(nil), "helloworld.HelloReply")
 }
 
-func init() { proto.RegisterFile("helloworld.proto", fileDescriptor_17b8c58d586b62f2) }
-
-var fileDescriptor_17b8c58d586b62f2 = []byte{
-	// 175 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xc8, 0x48, 0xcd, 0xc9,
-	0xc9, 0x2f, 0xcf, 0x2f, 0xca, 0x49, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x42, 0x88,
-	0x28, 0x29, 0x71, 0xf1, 0x78, 0x80, 0x78, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x42, 0x42,
-	0x5c, 0x2c, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x60, 0xb6, 0x92,
-	0x1a, 0x17, 0x17, 0x54, 0x4d, 0x41, 0x4e, 0xa5, 0x90, 0x04, 0x17, 0x7b, 0x6e, 0x6a, 0x71, 0x71,
-	0x62, 0x3a, 0x4c, 0x11, 0x8c, 0x6b, 0xe4, 0xc9, 0xc5, 0xee, 0x5e, 0x94, 0x9a, 0x5a, 0x92, 0x5a,
-	0x24, 0x64, 0xc7, 0xc5, 0x11, 0x9c, 0x58, 0x09, 0xd6, 0x25, 0x24, 0xa1, 0x87, 0xe4, 0x02, 0x64,
-	0xcb, 0xa4, 0xc4, 0xb0, 0xc8, 0x14, 0xe4, 0x54, 0x2a, 0x31, 0x38, 0x19, 0x70, 0x49, 0x67, 0xe6,
-	0xeb, 0xa5, 0x17, 0x15, 0x24, 0xeb, 0xa5, 0x56, 0x24, 0xe6, 0x16, 0xe4, 0xa4, 0x16, 0x23, 0xa9,
-	0x75, 0xe2, 0x07, 0x2b, 0x0e, 0x07, 0xb1, 0x03, 0x40, 0x5e, 0x0a, 0x60, 0x4c, 0x62, 0x03, 0xfb,
-	0xcd, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x0f, 0xb7, 0xcd, 0xf2, 0xef, 0x00, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("examples/helloworld/helloworld/helloworld.proto", fileDescriptor_b83ea99a5323a2c7)
+}
+
+var fileDescriptor_b83ea99a5323a2c7 = []byte{
+	// 205 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x4f, 0xad, 0x48, 0xcc,
+	0x2d, 0xc8, 0x49, 0x2d, 0xd6, 0xcf, 0x48, 0xcd, 0xc9, 0xc9, 0x2f, 0xcf, 0x2f, 0xca, 0x49, 0xc1,
+	0xce, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x42, 0x88, 0x28, 0x29, 0x71, 0xf1, 0x78,
+	0x80, 0x78, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x42, 0x42, 0x5c, 0x2c, 0x79, 0x89, 0xb9,
+	0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x60, 0xb6, 0x92, 0x1a, 0x17, 0x17, 0x54, 0x4d,
+	0x41, 0x4e, 0xa5, 0x90, 0x04, 0x17, 0x7b, 0x6e, 0x6a, 0x71, 0x71, 0x62, 0x3a, 0x4c, 0x11, 0x8c,
+	0x6b, 0xe4, 0xc9, 0xc5, 0xee, 0x5e, 0x94, 0x9a, 0x5a, 0x92, 0x5a, 0x24, 0x64, 0xc7, 0xc5, 0x11,
+	0x9c, 0x58, 0x09, 0xd6, 0x25, 0x24, 0xa1, 0x87, 0xe4, 0x02, 0x64, 0xcb, 0xa4, 0xc4, 0xb0, 0xc8,
+	0x14, 0xe4, 0x54, 0x2a, 0x31, 0x38, 0xa5, 0x73, 0x49, 0x67, 0xe6, 0xeb, 0xa5, 0x17, 0x15, 0x24,
+	0xeb, 0xc1, 0x7c, 0x87, 0xa4, 0xd6, 0x89, 0x1f, 0xac, 0x38, 0x1c, 0xc4, 0x0e, 0x00, 0x79, 0x29,
+	0x80, 0x31, 0xca, 0x34, 0x3d, 0x3f, 0x3f, 0x3d, 0x27, 0x55, 0x2f, 0x3d, 0x3f, 0x27, 0x31, 0x2f,
+	0x5d, 0x2f, 0xbf, 0x28, 0x5d, 0x1f, 0xa4, 0x9d, 0x40, 0xe0, 0x24, 0xb1, 0x81, 0x83, 0xc4, 0x18,
+	0x10, 0x00, 0x00, 0xff, 0xff, 0x46, 0xfe, 0x45, 0x5c, 0x45, 0x01, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -205,5 +209,5 @@ var _Greeter_serviceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "helloworld.proto",
+	Metadata: "examples/helloworld/helloworld/helloworld.proto",
 }
diff --git a/examples/helloworld/helloworld/helloworld.proto b/examples/helloworld/helloworld/helloworld.proto
index d79a6a0d1f57..692ef9deda07 100644
--- a/examples/helloworld/helloworld/helloworld.proto
+++ b/examples/helloworld/helloworld/helloworld.proto
@@ -14,6 +14,7 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/examples/helloworld/helloworld";
 option java_multiple_files = true;
 option java_package = "io.grpc.examples.helloworld";
 option java_outer_classname = "HelloWorldProto";
diff --git a/examples/route_guide/routeguide/route_guide.pb.go b/examples/route_guide/routeguide/route_guide.pb.go
index 9536e3b5babb..b0edc2c25558 100644
--- a/examples/route_guide/routeguide/route_guide.pb.go
+++ b/examples/route_guide/routeguide/route_guide.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: route_guide.proto
+// source: examples/route_guide/routeguide/route_guide.proto
 
 package routeguide
 
@@ -40,7 +40,7 @@ func (m *Point) Reset()         { *m = Point{} }
 func (m *Point) String() string { return proto.CompactTextString(m) }
 func (*Point) ProtoMessage()    {}
 func (*Point) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7d679f20da65b7b, []int{0}
+	return fileDescriptor_af806a20656386f8, []int{0}
 }
 
 func (m *Point) XXX_Unmarshal(b []byte) error {
@@ -91,7 +91,7 @@ func (m *Rectangle) Reset()         { *m = Rectangle{} }
 func (m *Rectangle) String() string { return proto.CompactTextString(m) }
 func (*Rectangle) ProtoMessage()    {}
 func (*Rectangle) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7d679f20da65b7b, []int{1}
+	return fileDescriptor_af806a20656386f8, []int{1}
 }
 
 func (m *Rectangle) XXX_Unmarshal(b []byte) error {
@@ -143,7 +143,7 @@ func (m *Feature) Reset()         { *m = Feature{} }
 func (m *Feature) String() string { return proto.CompactTextString(m) }
 func (*Feature) ProtoMessage()    {}
 func (*Feature) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7d679f20da65b7b, []int{2}
+	return fileDescriptor_af806a20656386f8, []int{2}
 }
 
 func (m *Feature) XXX_Unmarshal(b []byte) error {
@@ -193,7 +193,7 @@ func (m *RouteNote) Reset()         { *m = RouteNote{} }
 func (m *RouteNote) String() string { return proto.CompactTextString(m) }
 func (*RouteNote) ProtoMessage()    {}
 func (*RouteNote) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7d679f20da65b7b, []int{3}
+	return fileDescriptor_af806a20656386f8, []int{3}
 }
 
 func (m *RouteNote) XXX_Unmarshal(b []byte) error {
@@ -251,7 +251,7 @@ func (m *RouteSummary) Reset()         { *m = RouteSummary{} }
 func (m *RouteSummary) String() string { return proto.CompactTextString(m) }
 func (*RouteSummary) ProtoMessage()    {}
 func (*RouteSummary) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7d679f20da65b7b, []int{4}
+	return fileDescriptor_af806a20656386f8, []int{4}
 }
 
 func (m *RouteSummary) XXX_Unmarshal(b []byte) error {
@@ -308,36 +308,40 @@ func init() {
 	proto.RegisterType((*RouteSummary)(nil), "routeguide.RouteSummary")
 }
 
-func init() { proto.RegisterFile("route_guide.proto", fileDescriptor_b7d679f20da65b7b) }
-
-var fileDescriptor_b7d679f20da65b7b = []byte{
-	// 404 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x53, 0xdd, 0xca, 0xd3, 0x40,
-	0x10, 0xfd, 0x36, 0x7e, 0x9f, 0x6d, 0x26, 0x11, 0xe9, 0x88, 0x10, 0xa2, 0xa0, 0x8d, 0x37, 0xbd,
-	0x31, 0x94, 0x0a, 0x5e, 0x56, 0x6c, 0xc1, 0xde, 0x14, 0xa9, 0xb1, 0xf7, 0x65, 0x4d, 0xc6, 0x74,
-	0x61, 0x93, 0x0d, 0xc9, 0x06, 0xf4, 0x01, 0x7c, 0x02, 0x5f, 0x58, 0xb2, 0x49, 0xda, 0x54, 0x5b,
-	0xbc, 0xdb, 0x39, 0x73, 0xce, 0xfc, 0x9c, 0x61, 0x61, 0x52, 0xaa, 0x5a, 0xd3, 0x21, 0xad, 0x45,
-	0x42, 0x61, 0x51, 0x2a, 0xad, 0x10, 0x0c, 0x64, 0x90, 0xe0, 0x23, 0x3c, 0xec, 0x94, 0xc8, 0x35,
-	0xfa, 0x30, 0x96, 0x5c, 0x0b, 0x5d, 0x27, 0xe4, 0xb1, 0xd7, 0x6c, 0xf6, 0x10, 0x9d, 0x62, 0x7c,
-	0x09, 0xb6, 0x54, 0x79, 0xda, 0x26, 0x2d, 0x93, 0x3c, 0x03, 0xc1, 0x17, 0xb0, 0x23, 0x8a, 0x35,
-	0xcf, 0x53, 0x49, 0x38, 0x05, 0x4b, 0x2a, 0x53, 0xc0, 0x59, 0x4c, 0xc2, 0x73, 0xa3, 0xd0, 0x74,
-	0x89, 0x2c, 0xa9, 0x1a, 0xca, 0x51, 0x98, 0x32, 0xd7, 0x29, 0x47, 0x11, 0x6c, 0x61, 0xf4, 0x89,
-	0xb8, 0xae, 0x4b, 0x42, 0x84, 0xfb, 0x9c, 0x67, 0xed, 0x4c, 0x76, 0x64, 0xde, 0xf8, 0x16, 0xc6,
-	0x52, 0xc5, 0x5c, 0x0b, 0x95, 0xdf, 0xae, 0x73, 0xa2, 0x04, 0x7b, 0xb0, 0xa3, 0x26, 0xfb, 0x59,
-	0xe9, 0x4b, 0x2d, 0xfb, 0xaf, 0x16, 0x3d, 0x18, 0x65, 0x54, 0x55, 0x3c, 0x6d, 0x17, 0xb7, 0xa3,
-	0x3e, 0x0c, 0x7e, 0x33, 0x70, 0x4d, 0xd9, 0xaf, 0x75, 0x96, 0xf1, 0xf2, 0x27, 0xbe, 0x02, 0xa7,
-	0x68, 0xd4, 0x87, 0x58, 0xd5, 0xb9, 0xee, 0x4c, 0x04, 0x03, 0xad, 0x1b, 0x04, 0xdf, 0xc0, 0x93,
-	0xef, 0xed, 0x56, 0x1d, 0xa5, 0xb5, 0xd2, 0xed, 0xc0, 0x96, 0xe4, 0xc3, 0x38, 0x11, 0x95, 0xe6,
-	0x79, 0x4c, 0xde, 0xa3, 0xf6, 0x0e, 0x7d, 0x8c, 0x53, 0x70, 0x49, 0xf2, 0xa2, 0xa2, 0xe4, 0xa0,
-	0x45, 0x46, 0xde, 0xbd, 0xc9, 0x3b, 0x1d, 0xb6, 0x17, 0x19, 0x2d, 0x7e, 0x59, 0x00, 0x66, 0xaa,
-	0x4d, 0xb3, 0x0e, 0xbe, 0x07, 0xd8, 0x90, 0xee, 0xbd, 0xfc, 0x77, 0x53, 0xff, 0xd9, 0x10, 0xea,
-	0x78, 0xc1, 0x1d, 0x2e, 0xc1, 0xdd, 0x8a, 0xaa, 0x17, 0x56, 0xf8, 0x7c, 0x48, 0x3b, 0x5d, 0xfb,
-	0x86, 0x7a, 0xce, 0x70, 0x09, 0x4e, 0x44, 0xb1, 0x2a, 0x13, 0x33, 0xcb, 0xb5, 0xc6, 0xde, 0x45,
-	0xc5, 0x81, 0x8f, 0xc1, 0xdd, 0x8c, 0xe1, 0x87, 0xee, 0x64, 0xeb, 0x23, 0xd7, 0x7f, 0x35, 0xef,
-	0x2f, 0xe9, 0x5f, 0x87, 0x1b, 0xf9, 0x9c, 0xad, 0xe6, 0xf0, 0x42, 0xa8, 0x30, 0x2d, 0x8b, 0x38,
-	0xa4, 0x1f, 0x3c, 0x2b, 0x24, 0x55, 0x03, 0xfa, 0xea, 0xe9, 0xd9, 0xa3, 0x5d, 0xf3, 0x27, 0x76,
-	0xec, 0xdb, 0x63, 0xf3, 0x39, 0xde, 0xfd, 0x09, 0x00, 0x00, 0xff, 0xff, 0xc8, 0xe4, 0xef, 0xe6,
-	0x31, 0x03, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("examples/route_guide/routeguide/route_guide.proto", fileDescriptor_af806a20656386f8)
+}
+
+var fileDescriptor_af806a20656386f8 = []byte{
+	// 434 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x53, 0xcd, 0x6e, 0xd3, 0x40,
+	0x10, 0xae, 0x43, 0x4b, 0xe3, 0x49, 0x10, 0x62, 0x10, 0x92, 0x15, 0x90, 0xa0, 0xe6, 0xd2, 0x0b,
+	0x4e, 0x29, 0x52, 0x8f, 0x45, 0xb4, 0x12, 0xbd, 0x54, 0x28, 0x98, 0x9e, 0xb8, 0x44, 0x8b, 0x3d,
+	0x6c, 0x56, 0x5a, 0x7b, 0x2c, 0x7b, 0x2d, 0xc1, 0x03, 0xf0, 0x04, 0xbc, 0x30, 0xda, 0x5d, 0x3b,
+	0x76, 0x69, 0xaa, 0xde, 0x66, 0xbe, 0xf9, 0xbe, 0xf9, 0xd5, 0xc0, 0x7b, 0xfa, 0x25, 0x8a, 0x4a,
+	0x53, 0xb3, 0xac, 0xb9, 0x35, 0xb4, 0x96, 0xad, 0xca, 0xc9, 0xdb, 0x23, 0xd3, 0xc3, 0x49, 0x55,
+	0xb3, 0x61, 0x84, 0x21, 0x1a, 0x7f, 0x82, 0x83, 0x15, 0xab, 0xd2, 0xe0, 0x02, 0xa6, 0x5a, 0x18,
+	0x65, 0xda, 0x9c, 0xa2, 0xe0, 0x4d, 0x70, 0x7c, 0x90, 0x6e, 0x7d, 0x7c, 0x05, 0xa1, 0xe6, 0x52,
+	0xfa, 0xe0, 0xc4, 0x05, 0x07, 0x20, 0xfe, 0x0a, 0x61, 0x4a, 0x99, 0x11, 0xa5, 0xd4, 0x84, 0x47,
+	0x30, 0xd1, 0xec, 0x12, 0xcc, 0x4e, 0x9f, 0x25, 0x43, 0xa1, 0xc4, 0x55, 0x49, 0x27, 0x9a, 0x2d,
+	0x65, 0xa3, 0x5c, 0x9a, 0xdd, 0x94, 0x8d, 0x8a, 0xaf, 0xe1, 0xf0, 0x33, 0x09, 0xd3, 0xd6, 0x84,
+	0x08, 0xfb, 0xa5, 0x28, 0x7c, 0x4f, 0x61, 0xea, 0x6c, 0x7c, 0x07, 0x53, 0xcd, 0x99, 0x30, 0x8a,
+	0xcb, 0xfb, 0xf3, 0x6c, 0x29, 0xf1, 0x0d, 0x84, 0xa9, 0x8d, 0x7e, 0x61, 0x73, 0x5b, 0x1b, 0x3c,
+	0xa8, 0xc5, 0x08, 0x0e, 0x0b, 0x6a, 0x1a, 0x21, 0xfd, 0xe0, 0x61, 0xda, 0xbb, 0xf1, 0xdf, 0x00,
+	0xe6, 0x2e, 0xed, 0xb7, 0xb6, 0x28, 0x44, 0xfd, 0x1b, 0x5f, 0xc3, 0xac, 0xb2, 0xea, 0x75, 0xc6,
+	0x6d, 0x69, 0xba, 0x25, 0x82, 0x83, 0x2e, 0x2d, 0x82, 0x6f, 0xe1, 0xc9, 0x4f, 0x3f, 0x55, 0x47,
+	0xf1, 0xab, 0x9c, 0x77, 0xa0, 0x27, 0x2d, 0x60, 0x9a, 0xab, 0xc6, 0x88, 0x32, 0xa3, 0xe8, 0x91,
+	0xbf, 0x43, 0xef, 0xe3, 0x11, 0xcc, 0x49, 0x8b, 0xaa, 0xa1, 0x7c, 0x6d, 0x54, 0x41, 0xd1, 0xbe,
+	0x8b, 0xcf, 0x3a, 0xec, 0x46, 0x15, 0x74, 0xfa, 0x67, 0x02, 0xe0, 0xba, 0xba, 0xb2, 0xe3, 0xe0,
+	0x19, 0xc0, 0x15, 0x99, 0x7e, 0x97, 0x77, 0x27, 0x5d, 0x3c, 0x1f, 0x43, 0x1d, 0x2f, 0xde, 0xc3,
+	0x73, 0x98, 0x5f, 0xab, 0xa6, 0x17, 0x36, 0xf8, 0x62, 0x4c, 0xdb, 0x5e, 0xfb, 0x1e, 0xf5, 0x49,
+	0x80, 0xe7, 0x30, 0x4b, 0x29, 0xe3, 0x3a, 0x77, 0xbd, 0xec, 0x2a, 0x1c, 0xdd, 0xca, 0x38, 0xda,
+	0x63, 0xbc, 0x77, 0x1c, 0xe0, 0xc7, 0xee, 0x64, 0x97, 0x1b, 0x61, 0xfe, 0x2b, 0xde, 0x5f, 0x72,
+	0xb1, 0x1b, 0xb6, 0xf2, 0x93, 0xe0, 0x62, 0x03, 0x2f, 0x15, 0x27, 0xb2, 0xae, 0xb2, 0xa4, 0x7f,
+	0x90, 0x11, 0xfd, 0xe2, 0xe9, 0xb0, 0xa3, 0x95, 0xfd, 0x89, 0x55, 0xf0, 0xfd, 0x4c, 0x32, 0x4b,
+	0x4d, 0x89, 0x64, 0x2d, 0x4a, 0x99, 0x70, 0x2d, 0x97, 0x56, 0xbe, 0x7c, 0xe0, 0xbf, 0x7e, 0x3c,
+	0x76, 0x4f, 0xf5, 0xe1, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x28, 0xef, 0x54, 0xdd, 0x89, 0x03,
+	0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -659,5 +663,5 @@ var _RouteGuide_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "route_guide.proto",
+	Metadata: "examples/route_guide/routeguide/route_guide.proto",
 }
diff --git a/examples/route_guide/routeguide/route_guide.proto b/examples/route_guide/routeguide/route_guide.proto
index fe21e437adbc..966c434a89e5 100644
--- a/examples/route_guide/routeguide/route_guide.proto
+++ b/examples/route_guide/routeguide/route_guide.proto
@@ -14,6 +14,7 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/examples/route_guide/routeguide";
 option java_multiple_files = true;
 option java_package = "io.grpc.examples.routeguide";
 option java_outer_classname = "RouteGuideProto";
diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go
index 9f523e21f4e6..aa157f0be099 100644
--- a/examples/route_guide/server/server.go
+++ b/examples/route_guide/server/server.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc -I ../routeguide --go_out=plugins=grpc:../routeguide ../routeguide/route_guide.proto
-
 // Package main implements a simple gRPC server that demonstrates how to use gRPC-Go libraries
 // to perform unary, client streaming, server streaming and full duplex RPCs.
 //
diff --git a/health/regenerate.sh b/health/regenerate.sh
deleted file mode 100755
index b11eccb295b2..000000000000
--- a/health/regenerate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2018 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/health/v1
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/health/v1/health.proto > grpc/health/v1/health.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/health/v1/*.proto
-popd
-rm -f grpc_health_v1/*.pb.go
-cp "$TMP"/grpc/health/v1/*.pb.go grpc_health_v1/
-
diff --git a/health/server.go b/health/server.go
index 2262607f8828..6b65124242b8 100644
--- a/health/server.go
+++ b/health/server.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate ./regenerate.sh
-
 // Package health provides a service that exposes server's health and it must be
 // imported to enable support for client-side health checks.
 package health
diff --git a/internal/binarylog/regenerate.sh b/internal/binarylog/regenerate.sh
deleted file mode 100755
index 113d40cbe16c..000000000000
--- a/internal/binarylog/regenerate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2018 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/binarylog/grpc_binarylog_v1
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/binlog/v1/binarylog.proto > grpc/binarylog/grpc_binarylog_v1/binarylog.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/binarylog/grpc_binarylog_v1/*.proto
-popd
-rm -f ./grpc_binarylog_v1/*.pb.go
-cp "$TMP"/grpc/binarylog/grpc_binarylog_v1/*.pb.go ../../binarylog/grpc_binarylog_v1/
-
diff --git a/internal/proto/grpc_service_config/example_test.go b/internal/proto/grpc_service_config/example_test.go
index 6a5e79621e1a..b707d8b05e39 100644
--- a/internal/proto/grpc_service_config/example_test.go
+++ b/internal/proto/grpc_service_config/example_test.go
@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-//go:generate ./regenerate.sh
-
 package grpc_service_config_test
 
 import (
diff --git a/internal/proto/grpc_service_config/regenerate.sh b/internal/proto/grpc_service_config/regenerate.sh
deleted file mode 100755
index b67a25c36461..000000000000
--- a/internal/proto/grpc_service_config/regenerate.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Copyright 2019 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eux -o pipefail
-
-TMP=$(mktemp -d)
-
-function finish {
-  rm -rf "$TMP"
-}
-trap finish EXIT
-
-pushd "$TMP"
-mkdir -p grpc/service_config
-curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/service_config/service_config.proto > grpc/service_config/service_config.proto
-mkdir -p google/rpc
-curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > google/rpc/code.proto
-
-protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/service_config/*.proto
-popd
-rm -f ./*.pb.go
-cp "$TMP"/grpc/service_config/*.pb.go ./
-
diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 86a985775295..8bbff7a69dff 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc_testing/test.proto
+// source: interop/grpc_testing/test.proto
 
 package grpc_testing
 
@@ -53,7 +53,7 @@ func (x PayloadType) String() string {
 }
 
 func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{0}
+	return fileDescriptor_534063719f48d90d, []int{0}
 }
 
 // The type of route that a client took to reach a server w.r.t. gRPCLB.
@@ -90,7 +90,7 @@ func (x GrpclbRouteType) String() string {
 }
 
 func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{1}
+	return fileDescriptor_534063719f48d90d, []int{1}
 }
 
 type Empty struct {
@@ -103,7 +103,7 @@ func (m *Empty) Reset()         { *m = Empty{} }
 func (m *Empty) String() string { return proto.CompactTextString(m) }
 func (*Empty) ProtoMessage()    {}
 func (*Empty) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{0}
+	return fileDescriptor_534063719f48d90d, []int{0}
 }
 
 func (m *Empty) XXX_Unmarshal(b []byte) error {
@@ -139,7 +139,7 @@ func (m *Payload) Reset()         { *m = Payload{} }
 func (m *Payload) String() string { return proto.CompactTextString(m) }
 func (*Payload) ProtoMessage()    {}
 func (*Payload) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{1}
+	return fileDescriptor_534063719f48d90d, []int{1}
 }
 
 func (m *Payload) XXX_Unmarshal(b []byte) error {
@@ -188,7 +188,7 @@ func (m *EchoStatus) Reset()         { *m = EchoStatus{} }
 func (m *EchoStatus) String() string { return proto.CompactTextString(m) }
 func (*EchoStatus) ProtoMessage()    {}
 func (*EchoStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{2}
+	return fileDescriptor_534063719f48d90d, []int{2}
 }
 
 func (m *EchoStatus) XXX_Unmarshal(b []byte) error {
@@ -252,7 +252,7 @@ func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
 func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
 func (*SimpleRequest) ProtoMessage()    {}
 func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{3}
+	return fileDescriptor_534063719f48d90d, []int{3}
 }
 
 func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
@@ -354,7 +354,7 @@ func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
 func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
 func (*SimpleResponse) ProtoMessage()    {}
 func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{4}
+	return fileDescriptor_534063719f48d90d, []int{4}
 }
 
 func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
@@ -430,7 +430,7 @@ func (m *StreamingInputCallRequest) Reset()         { *m = StreamingInputCallReq
 func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) }
 func (*StreamingInputCallRequest) ProtoMessage()    {}
 func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{5}
+	return fileDescriptor_534063719f48d90d, []int{5}
 }
 
 func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error {
@@ -471,7 +471,7 @@ func (m *StreamingInputCallResponse) Reset()         { *m = StreamingInputCallRe
 func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) }
 func (*StreamingInputCallResponse) ProtoMessage()    {}
 func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{6}
+	return fileDescriptor_534063719f48d90d, []int{6}
 }
 
 func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error {
@@ -516,7 +516,7 @@ func (m *ResponseParameters) Reset()         { *m = ResponseParameters{} }
 func (m *ResponseParameters) String() string { return proto.CompactTextString(m) }
 func (*ResponseParameters) ProtoMessage()    {}
 func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{7}
+	return fileDescriptor_534063719f48d90d, []int{7}
 }
 
 func (m *ResponseParameters) XXX_Unmarshal(b []byte) error {
@@ -573,7 +573,7 @@ func (m *StreamingOutputCallRequest) Reset()         { *m = StreamingOutputCallR
 func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) }
 func (*StreamingOutputCallRequest) ProtoMessage()    {}
 func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{8}
+	return fileDescriptor_534063719f48d90d, []int{8}
 }
 
 func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error {
@@ -635,7 +635,7 @@ func (m *StreamingOutputCallResponse) Reset()         { *m = StreamingOutputCall
 func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) }
 func (*StreamingOutputCallResponse) ProtoMessage()    {}
 func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{9}
+	return fileDescriptor_534063719f48d90d, []int{9}
 }
 
 func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error {
@@ -677,7 +677,7 @@ func (m *LoadBalancerStatsRequest) Reset()         { *m = LoadBalancerStatsReque
 func (m *LoadBalancerStatsRequest) String() string { return proto.CompactTextString(m) }
 func (*LoadBalancerStatsRequest) ProtoMessage()    {}
 func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{10}
+	return fileDescriptor_534063719f48d90d, []int{10}
 }
 
 func (m *LoadBalancerStatsRequest) XXX_Unmarshal(b []byte) error {
@@ -726,7 +726,7 @@ func (m *LoadBalancerStatsResponse) Reset()         { *m = LoadBalancerStatsResp
 func (m *LoadBalancerStatsResponse) String() string { return proto.CompactTextString(m) }
 func (*LoadBalancerStatsResponse) ProtoMessage()    {}
 func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{11}
+	return fileDescriptor_534063719f48d90d, []int{11}
 }
 
 func (m *LoadBalancerStatsResponse) XXX_Unmarshal(b []byte) error {
@@ -779,72 +779,74 @@ func init() {
 	proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry")
 }
 
-func init() { proto.RegisterFile("grpc_testing/test.proto", fileDescriptor_e1cda82041fed8bf) }
+func init() { proto.RegisterFile("interop/grpc_testing/test.proto", fileDescriptor_534063719f48d90d) }
 
-var fileDescriptor_e1cda82041fed8bf = []byte{
-	// 989 bytes of a gzipped FileDescriptorProto
+var fileDescriptor_534063719f48d90d = []byte{
+	// 1019 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0xfd, 0x6e, 0x1b, 0x45,
-	0x10, 0xef, 0x39, 0x71, 0x1c, 0x8f, 0x5d, 0xc7, 0xdd, 0xb4, 0xf4, 0xec, 0x50, 0x30, 0x07, 0xa2,
-	0x47, 0x25, 0x5c, 0xe4, 0x8a, 0x0f, 0x55, 0x2a, 0xc8, 0x76, 0x9c, 0x10, 0xd5, 0xb5, 0xcd, 0xd9,
-	0x06, 0xf5, 0xaf, 0xd3, 0xe6, 0x3c, 0x71, 0x4e, 0xdc, 0x17, 0x7b, 0x7b, 0x11, 0xee, 0x3f, 0x48,
-	0x3c, 0x02, 0xaf, 0xc0, 0x63, 0xf0, 0x22, 0x3c, 0x0e, 0xda, 0xbd, 0x3b, 0x7f, 0x47, 0x4d, 0xa8,
-	0xe0, 0x2f, 0xef, 0xce, 0xc7, 0x6f, 0x66, 0x7e, 0x33, 0x3b, 0x67, 0x78, 0x38, 0x65, 0x81, 0x65,
-	0x72, 0x0c, 0xb9, 0xed, 0x4d, 0x9f, 0x8a, 0xdf, 0x7a, 0xc0, 0x7c, 0xee, 0x93, 0xa2, 0x50, 0xd4,
-	0x13, 0x85, 0x96, 0x83, 0x6c, 0xc7, 0x0d, 0xf8, 0x4c, 0xeb, 0x42, 0x6e, 0x40, 0x67, 0x8e, 0x4f,
-	0x27, 0xe4, 0x73, 0xd8, 0xe5, 0xb3, 0x00, 0x55, 0xa5, 0xa6, 0xe8, 0xa5, 0x46, 0xa5, 0xbe, 0xec,
-	0x50, 0x4f, 0x8c, 0x46, 0xb3, 0x00, 0x0d, 0x69, 0x46, 0x08, 0xec, 0x9e, 0xfb, 0x93, 0x99, 0x9a,
-	0xa9, 0x29, 0x7a, 0xd1, 0x90, 0x67, 0xed, 0x39, 0x40, 0xc7, 0xba, 0xf4, 0x87, 0x9c, 0xf2, 0x28,
-	0x14, 0x16, 0x96, 0x3f, 0x89, 0x01, 0xb3, 0x86, 0x3c, 0x13, 0x15, 0x72, 0x2e, 0x86, 0x21, 0x9d,
-	0xa2, 0x74, 0xcc, 0x1b, 0xe9, 0x55, 0xfb, 0x63, 0x07, 0xee, 0x0e, 0x6d, 0x37, 0x70, 0xd0, 0xc0,
-	0x5f, 0x22, 0x0c, 0x39, 0xf9, 0x16, 0xee, 0x32, 0x0c, 0x03, 0xdf, 0x0b, 0xd1, 0xbc, 0x59, 0x66,
-	0xc5, 0xd4, 0x5e, 0xdc, 0xc8, 0xc7, 0x4b, 0xfe, 0xa1, 0xfd, 0x26, 0x8e, 0x98, 0x5d, 0x18, 0x0d,
-	0xed, 0x37, 0x48, 0x9e, 0x42, 0x2e, 0x88, 0x11, 0xd4, 0x9d, 0x9a, 0xa2, 0x17, 0x1a, 0x0f, 0xb6,
-	0xc2, 0x1b, 0xa9, 0x95, 0x40, 0xbd, 0xb0, 0x1d, 0xc7, 0x8c, 0x42, 0x64, 0x1e, 0x75, 0x51, 0xdd,
-	0xad, 0x29, 0xfa, 0xbe, 0x51, 0x14, 0xc2, 0x71, 0x22, 0x23, 0x3a, 0x94, 0xa5, 0x91, 0x4f, 0x23,
-	0x7e, 0x69, 0x86, 0x96, 0x1f, 0xa0, 0x9a, 0x95, 0x76, 0x25, 0x21, 0xef, 0x0b, 0xf1, 0x50, 0x48,
-	0x49, 0x13, 0x0e, 0x16, 0x49, 0x4a, 0xde, 0xd4, 0x9c, 0xcc, 0x43, 0x5d, 0xcd, 0x63, 0xc1, 0xab,
-	0x51, 0x9a, 0x17, 0x10, 0xf3, 0xfc, 0x09, 0x48, 0x50, 0x33, 0x44, 0x76, 0x85, 0xcc, 0xb4, 0x27,
-	0x6a, 0x7e, 0x91, 0xd2, 0x50, 0x0a, 0xcf, 0x26, 0xe4, 0x19, 0xbc, 0x27, 0xad, 0x04, 0xaa, 0x73,
-	0x6e, 0x32, 0x3f, 0xe2, 0x09, 0xad, 0x20, 0xad, 0x0f, 0x85, 0xf6, 0x54, 0x2a, 0x0d, 0xa1, 0x13,
-	0x14, 0x6a, 0xbf, 0x67, 0xa0, 0x94, 0x36, 0x25, 0x8e, 0xb9, 0x4c, 0x98, 0x72, 0x23, 0xc2, 0xaa,
-	0xb0, 0x3f, 0xe7, 0x2a, 0xee, 0xf9, 0xfc, 0x4e, 0x3e, 0x84, 0xc2, 0x32, 0x45, 0x3b, 0x52, 0x0d,
-	0xfe, 0x82, 0x9e, 0x23, 0xc8, 0x2f, 0xca, 0xda, 0x8d, 0xbd, 0xc3, 0xb4, 0xa4, 0x33, 0xb8, 0xb7,
-	0x59, 0x4d, 0x56, 0x0e, 0xc9, 0xa3, 0xd5, 0xa4, 0xd6, 0xea, 0x32, 0x0e, 0xa6, 0xab, 0x02, 0x91,
-	0xe4, 0xa5, 0x1f, 0x72, 0x99, 0xe4, 0x5e, 0x1c, 0x26, 0xbd, 0x6b, 0x5d, 0xa8, 0x0c, 0x39, 0x43,
-	0xea, 0xda, 0xde, 0xf4, 0xcc, 0x0b, 0x22, 0xde, 0xa6, 0x8e, 0x93, 0x0e, 0xe9, 0x6d, 0xe9, 0xd0,
-	0x46, 0x50, 0xdd, 0x86, 0x96, 0xb0, 0xfb, 0x15, 0x3c, 0xa4, 0xd3, 0x29, 0xc3, 0x29, 0xe5, 0x38,
-	0x31, 0x13, 0x9f, 0x78, 0x7a, 0xe3, 0x67, 0xf4, 0x60, 0xa1, 0x4e, 0xa0, 0xc5, 0x18, 0x6b, 0x67,
-	0x40, 0x52, 0x8c, 0x01, 0x65, 0xd4, 0x45, 0x8e, 0x4c, 0xbe, 0xc0, 0x25, 0x57, 0x79, 0x16, 0x94,
-	0xdb, 0x1e, 0x47, 0x76, 0x45, 0xc5, 0x0c, 0x27, 0x6f, 0x02, 0x52, 0xd1, 0x38, 0xd4, 0xfe, 0xcc,
-	0x2c, 0x65, 0xd8, 0x8f, 0xf8, 0x5a, 0xc1, 0xef, 0xfa, 0x2a, 0x7f, 0x80, 0xc3, 0xb9, 0x7f, 0x30,
-	0x4f, 0x55, 0xcd, 0xd4, 0x76, 0xf4, 0x42, 0xa3, 0xb6, 0x8a, 0xb2, 0x59, 0x92, 0x41, 0xd8, 0x66,
-	0x99, 0xb7, 0x7e, 0xc3, 0xef, 0xfe, 0xe8, 0xb4, 0x1e, 0x1c, 0x6d, 0x25, 0xe9, 0x5f, 0xbe, 0x12,
-	0xed, 0x47, 0x50, 0xbb, 0x3e, 0x9d, 0xb4, 0xa8, 0x43, 0x3d, 0x0b, 0x99, 0x88, 0x12, 0xa6, 0x94,
-	0x57, 0x60, 0xdf, 0x8b, 0x5c, 0x93, 0x05, 0x56, 0x98, 0xb4, 0x32, 0xe7, 0x45, 0xae, 0x11, 0x58,
-	0xa1, 0xe8, 0x26, 0xb7, 0x5d, 0xf4, 0x23, 0x6e, 0x86, 0x68, 0xa5, 0xdd, 0x4c, 0x44, 0x43, 0xb4,
-	0xb4, 0xbf, 0x15, 0xa8, 0x6c, 0x01, 0x4e, 0xd2, 0x7c, 0x0d, 0x45, 0x81, 0x6a, 0x9e, 0xcf, 0xcc,
-	0x00, 0x91, 0xa9, 0x8a, 0xec, 0xc2, 0xd7, 0xab, 0xb9, 0x5e, 0xeb, 0x5e, 0x17, 0x29, 0xb4, 0x66,
-	0x03, 0x44, 0xd6, 0xf1, 0x38, 0x9b, 0x19, 0xc0, 0xe6, 0x02, 0xf2, 0x11, 0x14, 0x45, 0xd2, 0x17,
-	0xd4, 0x76, 0x22, 0x86, 0xe9, 0xa0, 0x15, 0xbc, 0xc8, 0x3d, 0x49, 0x44, 0xd5, 0x17, 0x70, 0xb0,
-	0x86, 0x40, 0xca, 0xb0, 0xf3, 0x33, 0xce, 0x64, 0x95, 0x79, 0x43, 0x1c, 0xc9, 0x7d, 0xc8, 0x5e,
-	0x51, 0x27, 0x4a, 0xb7, 0x77, 0x7c, 0x79, 0x9e, 0xf9, 0x46, 0x79, 0xf2, 0x1d, 0x14, 0x96, 0xc6,
-	0x8c, 0x94, 0xa1, 0xd8, 0xee, 0xbf, 0x1a, 0x18, 0x9d, 0xe1, 0xb0, 0xd9, 0xea, 0x76, 0xca, 0x77,
-	0x08, 0x81, 0xd2, 0xb8, 0xb7, 0x22, 0x53, 0x08, 0xc0, 0x9e, 0xd1, 0xec, 0x1d, 0xf7, 0x5f, 0x95,
-	0x33, 0x4f, 0x7c, 0x38, 0x58, 0x5b, 0x0c, 0xe4, 0x11, 0x54, 0x4e, 0x8d, 0x41, 0xbb, 0xdb, 0x32,
-	0x8d, 0xfe, 0x78, 0xd4, 0x31, 0x47, 0xaf, 0x07, 0x1d, 0x73, 0xdc, 0x7b, 0xd9, 0xeb, 0xff, 0xd4,
-	0x2b, 0xdf, 0x21, 0x1f, 0x40, 0x75, 0x53, 0x7d, 0xd2, 0xec, 0x76, 0x5b, 0xcd, 0xf6, 0xcb, 0xb2,
-	0xb2, 0xdd, 0x5d, 0xe8, 0x3a, 0xbd, 0xe3, 0x72, 0xa6, 0xf1, 0xd7, 0x2e, 0x14, 0x46, 0x18, 0x72,
-	0xb1, 0x94, 0x6d, 0x0b, 0xc9, 0x97, 0x90, 0x97, 0x9f, 0x61, 0x31, 0x3a, 0xe4, 0x70, 0x6d, 0xf6,
-	0x84, 0xa2, 0xba, 0x4d, 0x48, 0x4e, 0x20, 0x3f, 0xf6, 0x28, 0x8b, 0xdd, 0x8e, 0x56, 0x2d, 0x56,
-	0x3e, 0xa1, 0xd5, 0xf7, 0xb7, 0x2b, 0x93, 0xee, 0x3b, 0x70, 0xb8, 0x65, 0x86, 0x89, 0xbe, 0xe6,
-	0x74, 0xed, 0x2e, 0xa8, 0x7e, 0x76, 0x03, 0xcb, 0x38, 0xd6, 0x17, 0x0a, 0xb1, 0x81, 0x6c, 0x2e,
-	0x3e, 0xf2, 0xf8, 0x1a, 0x88, 0xf5, 0x45, 0x5b, 0xd5, 0xdf, 0x6e, 0x18, 0x87, 0xd2, 0x45, 0xa8,
-	0xd2, 0x49, 0xe4, 0x38, 0xc7, 0x51, 0xe0, 0xe0, 0xaf, 0xff, 0x59, 0x4d, 0xba, 0x22, 0xab, 0x2a,
-	0x7d, 0x4f, 0x9d, 0x8b, 0xff, 0x21, 0x54, 0x63, 0x0c, 0xf7, 0xc7, 0x9e, 0xec, 0xa0, 0x8b, 0x1e,
-	0xc7, 0x49, 0x3a, 0x45, 0x2f, 0xe0, 0xde, 0x8a, 0xfc, 0x76, 0xd3, 0xd4, 0xf8, 0x6d, 0xcb, 0xe6,
-	0x49, 0xa1, 0x2d, 0x28, 0x9d, 0x22, 0x6f, 0x3b, 0x36, 0x7a, 0x5c, 0x2a, 0xc8, 0xa7, 0x6f, 0xdd,
-	0x0d, 0x71, 0x6d, 0x8f, 0x6f, 0xb8, 0x43, 0xb4, 0x3b, 0xe7, 0x7b, 0xf2, 0x1f, 0xea, 0xb3, 0x7f,
-	0x02, 0x00, 0x00, 0xff, 0xff, 0x13, 0xfb, 0x86, 0xc3, 0xbc, 0x0a, 0x00, 0x00,
+	0x10, 0xcf, 0x39, 0x71, 0x1c, 0x8f, 0x5d, 0xc7, 0xdd, 0xb4, 0x70, 0x71, 0x28, 0x35, 0x07, 0xa2,
+	0xa6, 0xa8, 0x0e, 0x72, 0xc5, 0x87, 0x2a, 0x15, 0x14, 0x27, 0x4e, 0x88, 0xea, 0xda, 0xe6, 0x1c,
+	0x83, 0xca, 0x3f, 0xa7, 0xcd, 0x79, 0x72, 0x39, 0x71, 0x5f, 0xec, 0xed, 0x45, 0xb8, 0xff, 0x20,
+	0xf1, 0x08, 0xbc, 0x02, 0x8f, 0xc1, 0x8b, 0xf0, 0x38, 0x68, 0xf7, 0xee, 0xfc, 0x79, 0x51, 0x13,
+	0x2a, 0xfa, 0x97, 0x77, 0xe7, 0xe3, 0x37, 0x33, 0xbf, 0x99, 0x1d, 0x1f, 0x3c, 0xb4, 0x3d, 0x8e,
+	0xcc, 0x0f, 0xf6, 0x2d, 0x16, 0x98, 0x06, 0xc7, 0x90, 0xdb, 0x9e, 0xb5, 0x2f, 0x7e, 0x9b, 0x01,
+	0xf3, 0xb9, 0x4f, 0xca, 0x42, 0xd1, 0x4c, 0x14, 0x5a, 0x01, 0xf2, 0x1d, 0x37, 0xe0, 0x13, 0xad,
+	0x0b, 0x85, 0x01, 0x9d, 0x38, 0x3e, 0x1d, 0x93, 0x27, 0xb0, 0xc1, 0x27, 0x01, 0xaa, 0x4a, 0x5d,
+	0x69, 0x54, 0x5a, 0xbb, 0xcd, 0x79, 0x87, 0x66, 0x62, 0x74, 0x36, 0x09, 0x50, 0x97, 0x66, 0x84,
+	0xc0, 0xc6, 0xb9, 0x3f, 0x9e, 0xa8, 0xb9, 0xba, 0xd2, 0x28, 0xeb, 0xf2, 0xac, 0x3d, 0x03, 0xe8,
+	0x98, 0x97, 0xfe, 0x90, 0x53, 0x1e, 0x85, 0xc2, 0xc2, 0xf4, 0xc7, 0x31, 0x60, 0x5e, 0x97, 0x67,
+	0xa2, 0x42, 0xc1, 0xc5, 0x30, 0xa4, 0x16, 0x4a, 0xc7, 0xa2, 0x9e, 0x5e, 0xb5, 0x3f, 0xd7, 0xe1,
+	0xce, 0xd0, 0x76, 0x03, 0x07, 0x75, 0xfc, 0x35, 0xc2, 0x90, 0x93, 0x6f, 0xe1, 0x0e, 0xc3, 0x30,
+	0xf0, 0xbd, 0x10, 0x8d, 0x9b, 0x65, 0x56, 0x4e, 0xed, 0xc5, 0x8d, 0x7c, 0x3c, 0xe7, 0x1f, 0xda,
+	0xaf, 0xe3, 0x88, 0xf9, 0x99, 0xd1, 0xd0, 0x7e, 0x8d, 0x64, 0x1f, 0x0a, 0x41, 0x8c, 0xa0, 0xae,
+	0xd7, 0x95, 0x46, 0xa9, 0x75, 0x3f, 0x13, 0x5e, 0x4f, 0xad, 0x04, 0xea, 0x85, 0xed, 0x38, 0x46,
+	0x14, 0x22, 0xf3, 0xa8, 0x8b, 0xea, 0x46, 0x5d, 0x69, 0x6c, 0xe9, 0x65, 0x21, 0x1c, 0x25, 0x32,
+	0xd2, 0x80, 0xaa, 0x34, 0xf2, 0x69, 0xc4, 0x2f, 0x8d, 0xd0, 0xf4, 0x03, 0x54, 0xf3, 0xd2, 0xae,
+	0x22, 0xe4, 0x7d, 0x21, 0x1e, 0x0a, 0x29, 0x39, 0x80, 0xed, 0x59, 0x92, 0x92, 0x37, 0xb5, 0x20,
+	0xf3, 0x50, 0x17, 0xf3, 0x98, 0xf1, 0xaa, 0x57, 0xa6, 0x05, 0xc4, 0x3c, 0x7f, 0x02, 0x12, 0xd4,
+	0x08, 0x91, 0x5d, 0x21, 0x33, 0xec, 0xb1, 0x5a, 0x9c, 0xa5, 0x34, 0x94, 0xc2, 0xd3, 0x31, 0x79,
+	0x0a, 0xef, 0x49, 0x2b, 0x81, 0xea, 0x9c, 0x1b, 0xcc, 0x8f, 0x78, 0x42, 0x2b, 0x48, 0xeb, 0x1d,
+	0xa1, 0x3d, 0x91, 0x4a, 0x5d, 0xe8, 0x04, 0x85, 0xda, 0x1f, 0x39, 0xa8, 0xa4, 0x4d, 0x89, 0x63,
+	0xce, 0x13, 0xa6, 0xdc, 0x88, 0xb0, 0x1a, 0x6c, 0x4d, 0xb9, 0x8a, 0x7b, 0x3e, 0xbd, 0x93, 0x87,
+	0x50, 0x9a, 0xa7, 0x68, 0x5d, 0xaa, 0xc1, 0x9f, 0xd1, 0xb3, 0x07, 0xc5, 0x59, 0x59, 0x1b, 0xb1,
+	0x77, 0x98, 0x96, 0x74, 0x0a, 0x77, 0x57, 0xab, 0xc9, 0xcb, 0x21, 0x79, 0xb0, 0x98, 0xd4, 0x52,
+	0x5d, 0xfa, 0xb6, 0xb5, 0x28, 0x10, 0x49, 0x5e, 0xfa, 0x21, 0x97, 0x49, 0x6e, 0xc6, 0x61, 0xd2,
+	0xbb, 0xd6, 0x85, 0xdd, 0x21, 0x67, 0x48, 0x5d, 0xdb, 0xb3, 0x4e, 0xbd, 0x20, 0xe2, 0x87, 0xd4,
+	0x71, 0xd2, 0x21, 0xbd, 0x2d, 0x1d, 0xda, 0x19, 0xd4, 0xb2, 0xd0, 0x12, 0x76, 0xbf, 0x82, 0xf7,
+	0xa9, 0x65, 0x31, 0xb4, 0x28, 0xc7, 0xb1, 0x91, 0xf8, 0xc4, 0xd3, 0x1b, 0x3f, 0xa3, 0xfb, 0x33,
+	0x75, 0x02, 0x2d, 0xc6, 0x58, 0x3b, 0x05, 0x92, 0x62, 0x0c, 0x28, 0xa3, 0x2e, 0x72, 0x64, 0xf2,
+	0x05, 0xce, 0xb9, 0xca, 0xb3, 0xa0, 0x5c, 0xee, 0x8a, 0x2b, 0x2a, 0x66, 0x38, 0x79, 0x13, 0x90,
+	0x8a, 0x46, 0xa1, 0xf6, 0x57, 0x6e, 0x2e, 0xc3, 0x7e, 0xc4, 0x97, 0x0a, 0x7e, 0xdb, 0x57, 0xf9,
+	0x03, 0xec, 0x4c, 0xfd, 0x83, 0x69, 0xaa, 0x6a, 0xae, 0xbe, 0xde, 0x28, 0xb5, 0xea, 0x8b, 0x28,
+	0xab, 0x25, 0xe9, 0x84, 0xad, 0x96, 0x79, 0xeb, 0x37, 0xfc, 0xf6, 0x8f, 0x4e, 0xeb, 0xc1, 0x5e,
+	0x26, 0x49, 0xff, 0xf1, 0x95, 0x68, 0x3f, 0x82, 0xda, 0xf5, 0xe9, 0xb8, 0x4d, 0x1d, 0xea, 0x99,
+	0xc8, 0x44, 0x94, 0x30, 0xa5, 0x7c, 0x17, 0xb6, 0xbc, 0xc8, 0x35, 0x58, 0x60, 0x86, 0x49, 0x2b,
+	0x0b, 0x5e, 0xe4, 0xea, 0x81, 0x19, 0x8a, 0x6e, 0x72, 0xdb, 0x45, 0x3f, 0xe2, 0x46, 0x88, 0x66,
+	0xda, 0xcd, 0x44, 0x34, 0x44, 0x53, 0xfb, 0x47, 0x81, 0xdd, 0x0c, 0xe0, 0x24, 0xcd, 0x57, 0x50,
+	0x16, 0xa8, 0xc6, 0xf9, 0xc4, 0x08, 0x10, 0x99, 0xaa, 0xc8, 0x2e, 0x7c, 0xbd, 0x98, 0xeb, 0xb5,
+	0xee, 0x4d, 0x91, 0x42, 0x7b, 0x32, 0x40, 0x64, 0x1d, 0x8f, 0xb3, 0x89, 0x0e, 0x6c, 0x2a, 0x20,
+	0x1f, 0x41, 0x59, 0x24, 0x7d, 0x41, 0x6d, 0x27, 0x62, 0x98, 0x0e, 0x5a, 0xc9, 0x8b, 0xdc, 0xe3,
+	0x44, 0x54, 0x7b, 0x0e, 0xdb, 0x4b, 0x08, 0xa4, 0x0a, 0xeb, 0xbf, 0xe0, 0x44, 0x56, 0x59, 0xd4,
+	0xc5, 0x91, 0xdc, 0x83, 0xfc, 0x15, 0x75, 0xa2, 0x74, 0x7b, 0xc7, 0x97, 0x67, 0xb9, 0x6f, 0x94,
+	0xc7, 0xdf, 0x41, 0x69, 0x6e, 0xcc, 0x48, 0x15, 0xca, 0x87, 0xfd, 0x97, 0x03, 0xbd, 0x33, 0x1c,
+	0x1e, 0xb4, 0xbb, 0x9d, 0xea, 0x1a, 0x21, 0x50, 0x19, 0xf5, 0x16, 0x64, 0x0a, 0x01, 0xd8, 0xd4,
+	0x0f, 0x7a, 0x47, 0xfd, 0x97, 0xd5, 0xdc, 0x63, 0x1f, 0xb6, 0x97, 0x16, 0x03, 0x79, 0x00, 0xbb,
+	0x27, 0xfa, 0xe0, 0xb0, 0xdb, 0x36, 0xf4, 0xfe, 0xe8, 0xac, 0x63, 0x9c, 0xbd, 0x1a, 0x74, 0x8c,
+	0x51, 0xef, 0x45, 0xaf, 0xff, 0x53, 0xaf, 0xba, 0x46, 0x3e, 0x84, 0xda, 0xaa, 0xfa, 0xf8, 0xa0,
+	0xdb, 0x6d, 0x1f, 0x1c, 0xbe, 0xa8, 0x2a, 0xd9, 0xee, 0x42, 0xd7, 0xe9, 0x1d, 0x55, 0x73, 0xad,
+	0xbf, 0x37, 0xa0, 0x74, 0x86, 0x21, 0x17, 0x4b, 0xd9, 0x36, 0x91, 0x7c, 0x09, 0x45, 0xf9, 0x37,
+	0x2c, 0x46, 0x87, 0xec, 0x2c, 0xcd, 0x9e, 0x50, 0xd4, 0xb2, 0x84, 0xe4, 0x18, 0x8a, 0x23, 0x8f,
+	0xb2, 0xd8, 0x6d, 0x6f, 0xd1, 0x62, 0xe1, 0x2f, 0xb4, 0xf6, 0x41, 0xb6, 0x32, 0xe9, 0xbe, 0x03,
+	0x3b, 0x19, 0x33, 0x4c, 0x1a, 0x4b, 0x4e, 0xd7, 0xee, 0x82, 0xda, 0x67, 0x37, 0xb0, 0x8c, 0x63,
+	0x7d, 0xa1, 0x10, 0x1b, 0xc8, 0xea, 0xe2, 0x23, 0x8f, 0xae, 0x81, 0x58, 0x5e, 0xb4, 0xb5, 0xc6,
+	0x9b, 0x0d, 0xe3, 0x50, 0x0d, 0x11, 0xaa, 0x72, 0x1c, 0x39, 0xce, 0x51, 0x14, 0x38, 0xf8, 0xdb,
+	0xff, 0x56, 0x53, 0x43, 0x91, 0x55, 0x55, 0xbe, 0xa7, 0xce, 0xc5, 0x3b, 0x08, 0xd5, 0x1a, 0xc1,
+	0xbd, 0x91, 0x27, 0x3b, 0xe8, 0xa2, 0xc7, 0x71, 0x9c, 0x4e, 0xd1, 0x73, 0xb8, 0xbb, 0x20, 0xbf,
+	0xdd, 0x34, 0xb5, 0x7e, 0xcf, 0xd8, 0x3c, 0x29, 0xb4, 0x09, 0x95, 0x13, 0xe4, 0x87, 0x8e, 0x8d,
+	0x1e, 0x97, 0x0a, 0xf2, 0xe9, 0x1b, 0x77, 0x43, 0x5c, 0xdb, 0xa3, 0x1b, 0xee, 0x10, 0x6d, 0xad,
+	0xfd, 0xe4, 0xe7, 0xcf, 0x2d, 0xdf, 0xb7, 0x1c, 0x6c, 0x5a, 0xbe, 0x43, 0x3d, 0xab, 0xe9, 0x33,
+	0x4b, 0x7e, 0xc7, 0xee, 0x67, 0x7d, 0xd4, 0x9e, 0x6f, 0xca, 0x0f, 0xda, 0xa7, 0xff, 0x06, 0x00,
+	0x00, 0xff, 0xff, 0xaa, 0x43, 0x4c, 0xeb, 0xf3, 0x0a, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -1258,7 +1260,7 @@ var _TestService_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "grpc_testing/test.proto",
+	Metadata: "interop/grpc_testing/test.proto",
 }
 
 // UnimplementedServiceClient is the client API for UnimplementedService service.
@@ -1332,7 +1334,7 @@ var _UnimplementedService_serviceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc_testing/test.proto",
+	Metadata: "interop/grpc_testing/test.proto",
 }
 
 // LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
@@ -1406,5 +1408,5 @@ var _LoadBalancerStatsService_serviceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc_testing/test.proto",
+	Metadata: "interop/grpc_testing/test.proto",
 }
diff --git a/interop/grpc_testing/test.proto b/interop/grpc_testing/test.proto
index b9eaa64baab6..f827a3ef2d64 100644
--- a/interop/grpc_testing/test.proto
+++ b/interop/grpc_testing/test.proto
@@ -16,6 +16,8 @@
 // of unary/streaming requests/responses.
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/interop/grpc_testing";
+
 package grpc.testing;
 
 message Empty {}
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 08c4c97f43ab..63ba6a5ebd0a 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc --go_out=plugins=grpc:. grpc_testing/test.proto
-
 // Package interop contains functions used by interop client/server.
 package interop
 
diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go
index 6f249372ee42..90f02824ef69 100644
--- a/profiling/proto/service.pb.go
+++ b/profiling/proto/service.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: service.proto
+// source: profiling/proto/service.proto
 
 package proto
 
@@ -39,7 +39,7 @@ func (m *EnableRequest) Reset()         { *m = EnableRequest{} }
 func (m *EnableRequest) String() string { return proto.CompactTextString(m) }
 func (*EnableRequest) ProtoMessage()    {}
 func (*EnableRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0b84a42fa06f626, []int{0}
+	return fileDescriptor_e1ab2aa17b47c6fb, []int{0}
 }
 
 func (m *EnableRequest) XXX_Unmarshal(b []byte) error {
@@ -78,7 +78,7 @@ func (m *EnableResponse) Reset()         { *m = EnableResponse{} }
 func (m *EnableResponse) String() string { return proto.CompactTextString(m) }
 func (*EnableResponse) ProtoMessage()    {}
 func (*EnableResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0b84a42fa06f626, []int{1}
+	return fileDescriptor_e1ab2aa17b47c6fb, []int{1}
 }
 
 func (m *EnableResponse) XXX_Unmarshal(b []byte) error {
@@ -111,7 +111,7 @@ func (m *GetStreamStatsRequest) Reset()         { *m = GetStreamStatsRequest{} }
 func (m *GetStreamStatsRequest) String() string { return proto.CompactTextString(m) }
 func (*GetStreamStatsRequest) ProtoMessage()    {}
 func (*GetStreamStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0b84a42fa06f626, []int{2}
+	return fileDescriptor_e1ab2aa17b47c6fb, []int{2}
 }
 
 func (m *GetStreamStatsRequest) XXX_Unmarshal(b []byte) error {
@@ -145,7 +145,7 @@ func (m *GetStreamStatsResponse) Reset()         { *m = GetStreamStatsResponse{}
 func (m *GetStreamStatsResponse) String() string { return proto.CompactTextString(m) }
 func (*GetStreamStatsResponse) ProtoMessage()    {}
 func (*GetStreamStatsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0b84a42fa06f626, []int{3}
+	return fileDescriptor_e1ab2aa17b47c6fb, []int{3}
 }
 
 func (m *GetStreamStatsResponse) XXX_Unmarshal(b []byte) error {
@@ -200,7 +200,7 @@ func (m *Timer) Reset()         { *m = Timer{} }
 func (m *Timer) String() string { return proto.CompactTextString(m) }
 func (*Timer) ProtoMessage()    {}
 func (*Timer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0b84a42fa06f626, []int{4}
+	return fileDescriptor_e1ab2aa17b47c6fb, []int{4}
 }
 
 func (m *Timer) XXX_Unmarshal(b []byte) error {
@@ -285,7 +285,7 @@ func (m *Stat) Reset()         { *m = Stat{} }
 func (m *Stat) String() string { return proto.CompactTextString(m) }
 func (*Stat) ProtoMessage()    {}
 func (*Stat) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0b84a42fa06f626, []int{5}
+	return fileDescriptor_e1ab2aa17b47c6fb, []int{5}
 }
 
 func (m *Stat) XXX_Unmarshal(b []byte) error {
@@ -336,35 +336,35 @@ func init() {
 	proto.RegisterType((*Stat)(nil), "grpc.go.profiling.v1alpha.Stat")
 }
 
-func init() { proto.RegisterFile("service.proto", fileDescriptor_a0b84a42fa06f626) }
-
-var fileDescriptor_a0b84a42fa06f626 = []byte{
-	// 388 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0x4f, 0xcf, 0xd2, 0x40,
-	0x10, 0xc6, 0x53, 0x68, 0x6b, 0x19, 0xfe, 0xc4, 0xac, 0x51, 0x0a, 0xc6, 0xd8, 0xf4, 0x60, 0xca,
-	0xa5, 0x08, 0x5e, 0x3c, 0x93, 0x18, 0xe3, 0xc5, 0x98, 0xc5, 0x93, 0xd1, 0x90, 0xa5, 0x1d, 0xd7,
-	0x26, 0xa5, 0x5b, 0xbb, 0x0b, 0x9f, 0xc7, 0xaf, 0xe6, 0x37, 0x31, 0x3b, 0x05, 0x0c, 0x6f, 0x78,
-	0xc9, 0xfb, 0x9e, 0x60, 0x66, 0x9e, 0xdf, 0xd3, 0x67, 0x32, 0x0b, 0x43, 0x8d, 0xcd, 0xa1, 0xc8,
-	0x30, 0xad, 0x1b, 0x65, 0x14, 0x9b, 0xc8, 0xa6, 0xce, 0x52, 0xa9, 0x6c, 0xf9, 0xb3, 0x28, 0x8b,
-	0x4a, 0xa6, 0x87, 0x85, 0x28, 0xeb, 0x5f, 0x22, 0x9e, 0xc1, 0xf0, 0x43, 0x25, 0xb6, 0x25, 0x72,
-	0xfc, 0xbd, 0x47, 0x6d, 0x58, 0x08, 0x4f, 0x90, 0x1a, 0x79, 0xe8, 0x44, 0x4e, 0x12, 0xf0, 0x53,
-	0x19, 0x3f, 0x85, 0xd1, 0x49, 0xaa, 0x6b, 0x55, 0x69, 0x8c, 0xc7, 0xf0, 0xfc, 0x23, 0x9a, 0xb5,
-	0x69, 0x50, 0xec, 0xd6, 0x46, 0x18, 0x7d, 0x34, 0x89, 0xbf, 0xc3, 0x8b, 0xbb, 0x83, 0x16, 0x61,
-	0x2b, 0x18, 0x68, 0x6a, 0x6f, 0xb4, 0xed, 0x87, 0x4e, 0xd4, 0x4d, 0xfa, 0xcb, 0xd7, 0xe9, 0xbd,
-	0x09, 0x53, 0xcb, 0xf3, 0xbe, 0xfe, 0xef, 0x15, 0xff, 0x71, 0xc0, 0xfb, 0x5a, 0xec, 0xb0, 0x61,
-	0x0c, 0x5c, 0x23, 0xa4, 0xa6, 0xa4, 0x3d, 0x4e, 0xff, 0xd9, 0x4b, 0xe8, 0x6d, 0x51, 0x16, 0xd5,
-	0x46, 0x63, 0x16, 0x76, 0x22, 0x27, 0xe9, 0xf2, 0x80, 0x1a, 0x6b, 0xcc, 0xd8, 0x2b, 0x80, 0x76,
-	0x58, 0xd9, 0x69, 0x37, 0x72, 0x12, 0x8f, 0xb7, 0xf2, 0xcf, 0x1a, 0x33, 0x36, 0xb6, 0xcb, 0xe7,
-	0x44, 0xba, 0x44, 0xfa, 0x58, 0xe5, 0x96, 0x9b, 0x40, 0x60, 0x07, 0x44, 0x79, 0x44, 0x59, 0x21,
-	0x31, 0xcf, 0xc0, 0x93, 0x6a, 0x53, 0xe4, 0xa1, 0x4f, 0x84, 0x2b, 0xd5, 0xa7, 0x3c, 0xae, 0xc1,
-	0xb5, 0x59, 0xaf, 0x06, 0x7c, 0x0f, 0xbe, 0xb1, 0xe9, 0x75, 0xd8, 0xa1, 0xe5, 0xa3, 0x1b, 0xcb,
-	0xd3, 0x9a, 0xfc, 0xa8, 0x67, 0x53, 0x08, 0x76, 0x68, 0x44, 0x2e, 0x8c, 0xa0, 0xec, 0x03, 0x7e,
-	0xae, 0x97, 0x7f, 0x1d, 0xe8, 0x7d, 0x39, 0xf1, 0xec, 0x07, 0xf8, 0xed, 0xad, 0x58, 0x72, 0xc3,
-	0xfd, 0xe2, 0xf2, 0xd3, 0xd9, 0x03, 0x94, 0xc7, 0x2b, 0xee, 0x61, 0x74, 0x79, 0x5f, 0xf6, 0xf6,
-	0x06, 0x7c, 0xf5, 0x8d, 0x4c, 0x17, 0x8f, 0x20, 0xda, 0xcf, 0xae, 0x92, 0x6f, 0x6f, 0xa4, 0x52,
-	0xb2, 0xc4, 0x54, 0xaa, 0x52, 0x54, 0x32, 0x55, 0x8d, 0x9c, 0x5b, 0x97, 0xf9, 0xd9, 0x62, 0x4e,
-	0x2f, 0x7e, 0xeb, 0xd3, 0xcf, 0xbb, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x13, 0x63, 0x69, 0xce,
-	0x09, 0x03, 0x00, 0x00,
+func init() { proto.RegisterFile("profiling/proto/service.proto", fileDescriptor_e1ab2aa17b47c6fb) }
+
+var fileDescriptor_e1ab2aa17b47c6fb = []byte{
+	// 392 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0x41, 0xab, 0xd3, 0x40,
+	0x10, 0xc7, 0xc9, 0x6b, 0x12, 0xd3, 0x79, 0xcf, 0x87, 0xac, 0xe8, 0xcb, 0xab, 0x14, 0x43, 0x0e,
+	0x92, 0x5e, 0x36, 0xb6, 0x5e, 0x3c, 0x17, 0x44, 0xbc, 0x88, 0x6c, 0x3d, 0x89, 0x52, 0xb6, 0xc9,
+	0xb8, 0x06, 0xd2, 0x6c, 0xcc, 0x6e, 0xfb, 0x79, 0xfc, 0x6a, 0x7e, 0x13, 0xd9, 0x49, 0x5b, 0x69,
+	0xa9, 0xc5, 0x77, 0x4a, 0x66, 0xe6, 0xff, 0x9b, 0xfd, 0x0f, 0x33, 0x30, 0x6e, 0x3b, 0xfd, 0xbd,
+	0xaa, 0xab, 0x46, 0xe5, 0x6d, 0xa7, 0xad, 0xce, 0x0d, 0x76, 0xdb, 0xaa, 0x40, 0x4e, 0x11, 0xbb,
+	0x57, 0x5d, 0x5b, 0x70, 0xa5, 0xf9, 0x41, 0xc6, 0xb7, 0x53, 0x59, 0xb7, 0x3f, 0x64, 0x3a, 0x81,
+	0xc7, 0xef, 0x1a, 0xb9, 0xaa, 0x51, 0xe0, 0xcf, 0x0d, 0x1a, 0xcb, 0x62, 0x78, 0x84, 0x94, 0x28,
+	0x63, 0x2f, 0xf1, 0xb2, 0x48, 0xec, 0xc3, 0xf4, 0x09, 0xdc, 0xee, 0xa5, 0xa6, 0xd5, 0x8d, 0xc1,
+	0xf4, 0x0e, 0x9e, 0xbd, 0x47, 0xbb, 0xb0, 0x1d, 0xca, 0xf5, 0xc2, 0x4a, 0x6b, 0x76, 0x4d, 0xd2,
+	0xaf, 0xf0, 0xfc, 0xb4, 0xd0, 0x23, 0x6c, 0x0e, 0x37, 0x86, 0xd2, 0x4b, 0xe3, 0xf2, 0xb1, 0x97,
+	0x0c, 0xb2, 0xeb, 0xd9, 0x4b, 0xfe, 0x4f, 0x87, 0xdc, 0xf1, 0xe2, 0xda, 0xfc, 0xed, 0x95, 0xfe,
+	0xf2, 0x20, 0xf8, 0x5c, 0xad, 0xb1, 0x63, 0x0c, 0x7c, 0x2b, 0x95, 0x21, 0xa7, 0x43, 0x41, 0xff,
+	0xec, 0x05, 0x0c, 0x57, 0xa8, 0xaa, 0x66, 0x69, 0xb0, 0x88, 0xaf, 0x12, 0x2f, 0x1b, 0x88, 0x88,
+	0x12, 0x0b, 0x2c, 0xd8, 0x18, 0xa0, 0x2f, 0x36, 0xae, 0x3a, 0x48, 0xbc, 0x2c, 0x10, 0xbd, 0xfc,
+	0xa3, 0xc1, 0x82, 0xdd, 0xb9, 0xe1, 0x4b, 0x22, 0x7d, 0x22, 0x43, 0x6c, 0x4a, 0xc7, 0xdd, 0x43,
+	0xe4, 0x0a, 0x44, 0x05, 0x44, 0x39, 0x21, 0x31, 0x4f, 0x21, 0x50, 0x7a, 0x59, 0x95, 0x71, 0x48,
+	0x84, 0xaf, 0xf4, 0x87, 0x32, 0x6d, 0xc1, 0x77, 0x5e, 0xcf, 0x1a, 0x7c, 0x0b, 0xa1, 0x75, 0xee,
+	0x4d, 0x7c, 0x45, 0xc3, 0x27, 0x17, 0x86, 0xa7, 0x31, 0xc5, 0x4e, 0xcf, 0x46, 0x10, 0xad, 0xd1,
+	0xca, 0x52, 0x5a, 0x49, 0xde, 0x6f, 0xc4, 0x21, 0x9e, 0xfd, 0xf6, 0x60, 0xf8, 0x69, 0xcf, 0xb3,
+	0x6f, 0x10, 0xf6, 0xbb, 0x62, 0xd9, 0x85, 0xee, 0x47, 0x9b, 0x1f, 0x4d, 0xfe, 0x43, 0xb9, 0xdb,
+	0xe2, 0x06, 0x6e, 0x8f, 0xf7, 0xcb, 0x5e, 0x5f, 0x80, 0xcf, 0xde, 0xc8, 0x68, 0xfa, 0x00, 0xa2,
+	0x7f, 0x76, 0x9e, 0x7d, 0x79, 0xa5, 0xb4, 0x56, 0x35, 0x72, 0xa5, 0x6b, 0xd9, 0x28, 0xae, 0x3b,
+	0x95, 0xbb, 0x2e, 0xf9, 0xc9, 0xfd, 0xaf, 0x42, 0xfa, 0xbc, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff,
+	0x5d, 0x47, 0x09, 0xa9, 0x19, 0x03, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -486,5 +486,5 @@ var _Profiling_serviceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "service.proto",
+	Metadata: "profiling/proto/service.proto",
 }
diff --git a/profiling/service/service.go b/profiling/service/service.go
index 9942207a5fba..a36b8bdda883 100644
--- a/profiling/service/service.go
+++ b/profiling/service/service.go
@@ -24,8 +24,6 @@
 // This package and all its methods are EXPERIMENTAL.
 package service
 
-//go:generate protoc --go_out=plugins=grpc,paths=source_relative:../proto -I../proto ../proto/service.proto
-
 import (
 	"context"
 	"errors"
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 5d8c412f740a..900bd6c05c78 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc_reflection_v1alpha/reflection.proto
+// source: reflection/grpc_reflection_v1alpha/reflection.proto
 
 package grpc_reflection_v1alpha
 
@@ -47,7 +47,7 @@ func (m *ServerReflectionRequest) Reset()         { *m = ServerReflectionRequest
 func (m *ServerReflectionRequest) String() string { return proto.CompactTextString(m) }
 func (*ServerReflectionRequest) ProtoMessage()    {}
 func (*ServerReflectionRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{0}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{0}
 }
 
 func (m *ServerReflectionRequest) XXX_Unmarshal(b []byte) error {
@@ -178,7 +178,7 @@ func (m *ExtensionRequest) Reset()         { *m = ExtensionRequest{} }
 func (m *ExtensionRequest) String() string { return proto.CompactTextString(m) }
 func (*ExtensionRequest) ProtoMessage()    {}
 func (*ExtensionRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{1}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{1}
 }
 
 func (m *ExtensionRequest) XXX_Unmarshal(b []byte) error {
@@ -235,7 +235,7 @@ func (m *ServerReflectionResponse) Reset()         { *m = ServerReflectionRespon
 func (m *ServerReflectionResponse) String() string { return proto.CompactTextString(m) }
 func (*ServerReflectionResponse) ProtoMessage()    {}
 func (*ServerReflectionResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{2}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{2}
 }
 
 func (m *ServerReflectionResponse) XXX_Unmarshal(b []byte) error {
@@ -362,7 +362,7 @@ func (m *FileDescriptorResponse) Reset()         { *m = FileDescriptorResponse{}
 func (m *FileDescriptorResponse) String() string { return proto.CompactTextString(m) }
 func (*FileDescriptorResponse) ProtoMessage()    {}
 func (*FileDescriptorResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{3}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{3}
 }
 
 func (m *FileDescriptorResponse) XXX_Unmarshal(b []byte) error {
@@ -406,7 +406,7 @@ func (m *ExtensionNumberResponse) Reset()         { *m = ExtensionNumberResponse
 func (m *ExtensionNumberResponse) String() string { return proto.CompactTextString(m) }
 func (*ExtensionNumberResponse) ProtoMessage()    {}
 func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{4}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{4}
 }
 
 func (m *ExtensionNumberResponse) XXX_Unmarshal(b []byte) error {
@@ -455,7 +455,7 @@ func (m *ListServiceResponse) Reset()         { *m = ListServiceResponse{} }
 func (m *ListServiceResponse) String() string { return proto.CompactTextString(m) }
 func (*ListServiceResponse) ProtoMessage()    {}
 func (*ListServiceResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{5}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{5}
 }
 
 func (m *ListServiceResponse) XXX_Unmarshal(b []byte) error {
@@ -498,7 +498,7 @@ func (m *ServiceResponse) Reset()         { *m = ServiceResponse{} }
 func (m *ServiceResponse) String() string { return proto.CompactTextString(m) }
 func (*ServiceResponse) ProtoMessage()    {}
 func (*ServiceResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{6}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{6}
 }
 
 func (m *ServiceResponse) XXX_Unmarshal(b []byte) error {
@@ -540,7 +540,7 @@ func (m *ErrorResponse) Reset()         { *m = ErrorResponse{} }
 func (m *ErrorResponse) String() string { return proto.CompactTextString(m) }
 func (*ErrorResponse) ProtoMessage()    {}
 func (*ErrorResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_42a8ac412db3cb03, []int{7}
+	return fileDescriptor_e8cf9f2921ad6c95, []int{7}
 }
 
 func (m *ErrorResponse) XXX_Unmarshal(b []byte) error {
@@ -587,52 +587,54 @@ func init() {
 }
 
 func init() {
-	proto.RegisterFile("grpc_reflection_v1alpha/reflection.proto", fileDescriptor_42a8ac412db3cb03)
-}
-
-var fileDescriptor_42a8ac412db3cb03 = []byte{
-	// 656 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x51, 0x73, 0xd2, 0x40,
-	0x10, 0x6e, 0x5a, 0x68, 0x87, 0x85, 0x02, 0x5e, 0x2b, 0xa4, 0x3a, 0x75, 0x98, 0x68, 0x35, 0x75,
-	0x1c, 0xda, 0xe2, 0x8c, 0x3f, 0x80, 0xaa, 0x83, 0x33, 0xb5, 0x75, 0x0e, 0x5f, 0x1c, 0x1f, 0x6e,
-	0x02, 0x2c, 0x34, 0x1a, 0x72, 0xf1, 0x2e, 0x45, 0x79, 0xf2, 0x47, 0xf8, 0xa3, 0xfc, 0x4b, 0x3e,
-	0x3a, 0x77, 0x09, 0x21, 0xa4, 0x44, 0xa7, 0x4f, 0x30, 0xdf, 0xee, 0xde, 0xb7, 0xbb, 0xdf, 0xb7,
-	0x01, 0x7b, 0x22, 0x82, 0x21, 0x13, 0x38, 0xf6, 0x70, 0x18, 0xba, 0xdc, 0x67, 0xb3, 0x33, 0xc7,
-	0x0b, 0xae, 0x9d, 0x93, 0x25, 0xd4, 0x0e, 0x04, 0x0f, 0x39, 0x69, 0xaa, 0xcc, 0x76, 0x0a, 0x8e,
-	0x33, 0xad, 0x3f, 0x9b, 0xd0, 0xec, 0xa3, 0x98, 0xa1, 0xa0, 0x49, 0x90, 0xe2, 0xb7, 0x1b, 0x94,
-	0x21, 0x21, 0x50, 0xb8, 0xe6, 0x32, 0x34, 0x8d, 0x96, 0x61, 0x97, 0xa8, 0xfe, 0x4f, 0x9e, 0x43,
-	0x7d, 0xec, 0x7a, 0xc8, 0x06, 0x73, 0xa6, 0x7e, 0x7d, 0x67, 0x8a, 0xe6, 0x96, 0x8a, 0xf7, 0x36,
-	0x68, 0x55, 0x21, 0xdd, 0xf9, 0xdb, 0x18, 0x27, 0xaf, 0xa0, 0xa1, 0x73, 0x87, 0xdc, 0x0f, 0x1d,
-	0xd7, 0x77, 0xfd, 0x09, 0x93, 0xf3, 0xe9, 0x80, 0x7b, 0x66, 0x21, 0xae, 0xd8, 0x57, 0xf1, 0xf3,
-	0x24, 0xdc, 0xd7, 0x51, 0x32, 0x81, 0x83, 0x6c, 0x1d, 0xfe, 0x08, 0xd1, 0x97, 0x2e, 0xf7, 0xcd,
-	0x62, 0xcb, 0xb0, 0xcb, 0x9d, 0xe3, 0x76, 0xce, 0x40, 0xed, 0x37, 0x8b, 0xcc, 0x78, 0x8a, 0xde,
-	0x06, 0x6d, 0xae, 0xb2, 0x24, 0x19, 0xa4, 0x0b, 0x87, 0x8e, 0xe7, 0x2d, 0x1f, 0x67, 0xfe, 0xcd,
-	0x74, 0x80, 0x42, 0x32, 0x3e, 0x66, 0xe1, 0x3c, 0x40, 0x73, 0x3b, 0xee, 0xf3, 0xc0, 0xf1, 0xbc,
-	0xa4, 0xec, 0x32, 0x4a, 0xba, 0x1a, 0x7f, 0x9c, 0x07, 0x48, 0x8e, 0x60, 0xd7, 0x73, 0x65, 0xc8,
-	0x24, 0x8a, 0x99, 0x3b, 0x44, 0x69, 0xee, 0xc4, 0x35, 0x15, 0x05, 0xf7, 0x63, 0xb4, 0x7b, 0x0f,
-	0x6a, 0x53, 0x94, 0xd2, 0x99, 0x20, 0x13, 0x51, 0x63, 0xd6, 0x18, 0xea, 0xd9, 0x66, 0xc9, 0x33,
-	0xa8, 0xa5, 0xa6, 0xd6, 0x3d, 0x44, 0xdb, 0xaf, 0x2e, 0x61, 0x4d, 0x7b, 0x0c, 0xf5, 0x6c, 0xdb,
-	0xe6, 0x66, 0xcb, 0xb0, 0x8b, 0xb4, 0x86, 0xab, 0x8d, 0x5a, 0xbf, 0x0b, 0x60, 0xde, 0x96, 0x58,
-	0x06, 0xdc, 0x97, 0x48, 0x0e, 0x01, 0x66, 0x8e, 0xe7, 0x8e, 0x58, 0x4a, 0xe9, 0x92, 0x46, 0x7a,
-	0x4a, 0xee, 0xcf, 0x50, 0xe7, 0xc2, 0x9d, 0xb8, 0xbe, 0xe3, 0x2d, 0xfa, 0xd6, 0x34, 0xe5, 0xce,
-	0x69, 0xae, 0x02, 0x39, 0x76, 0xa2, 0xb5, 0xc5, 0x4b, 0x8b, 0x61, 0xbf, 0x82, 0xa9, 0x75, 0x1e,
-	0xa1, 0x1c, 0x0a, 0x37, 0x08, 0xb9, 0x60, 0x22, 0xee, 0x4b, 0x3b, 0xa4, 0xdc, 0x39, 0xc9, 0x25,
-	0x51, 0x26, 0x7b, 0x9d, 0xd4, 0x2d, 0xc6, 0xe9, 0x6d, 0x50, 0x6d, 0xb9, 0xdb, 0x11, 0xf2, 0x1d,
-	0x1e, 0xad, 0xd7, 0x3a, 0xa1, 0x2c, 0xfe, 0x67, 0xae, 0x8c, 0x01, 0x52, 0x9c, 0x0f, 0xd7, 0xd8,
-	0x23, 0x21, 0x1e, 0x41, 0x63, 0xc5, 0x20, 0x4b, 0xc2, 0x6d, 0x4d, 0xf8, 0x22, 0x97, 0xf0, 0x62,
-	0x69, 0xa0, 0x14, 0xd9, 0x7e, 0xda, 0x57, 0x09, 0xcb, 0x15, 0x54, 0x51, 0x88, 0xf4, 0x06, 0x77,
-	0xf4, 0xeb, 0x4f, 0xf3, 0xc7, 0x51, 0xe9, 0xa9, 0x77, 0x77, 0x31, 0x0d, 0x74, 0x09, 0xd4, 0x97,
-	0x86, 0x8d, 0x30, 0xeb, 0x02, 0x1a, 0xeb, 0xf7, 0x4e, 0x3a, 0x70, 0x3f, 0x2b, 0xa5, 0xfe, 0xf0,
-	0x98, 0x46, 0x6b, 0xcb, 0xae, 0xd0, 0xbd, 0x55, 0x51, 0x3e, 0xa8, 0x90, 0xf5, 0x05, 0x9a, 0x39,
-	0x2b, 0x25, 0x4f, 0xa0, 0x3a, 0x70, 0x24, 0xea, 0x03, 0x60, 0xfa, 0x1b, 0x13, 0x39, 0xb3, 0xa2,
-	0x50, 0xe5, 0xff, 0x4b, 0xf5, 0x7d, 0x59, 0x7f, 0x03, 0x5b, 0xeb, 0x6e, 0xe0, 0x13, 0xec, 0xad,
-	0xd9, 0x26, 0xe9, 0xc2, 0x4e, 0x2c, 0x8b, 0x6e, 0xb4, 0xdc, 0xb1, 0xff, 0xe9, 0xea, 0x54, 0x29,
-	0x5d, 0x14, 0x5a, 0x47, 0x50, 0xcb, 0x3e, 0x4b, 0xa0, 0x90, 0x6a, 0x5a, 0xff, 0xb7, 0xfa, 0xb0,
-	0xbb, 0xb2, 0x71, 0x75, 0x79, 0x91, 0x62, 0x43, 0x3e, 0x8a, 0x52, 0x8b, 0xb4, 0xa4, 0x91, 0x73,
-	0x3e, 0x42, 0xf2, 0x18, 0x22, 0x41, 0x58, 0xac, 0x82, 0x3e, 0xbb, 0x12, 0xad, 0x68, 0xf0, 0x7d,
-	0x84, 0x75, 0x7e, 0x19, 0x50, 0xcf, 0x9e, 0x1b, 0xf9, 0x09, 0xfb, 0x59, 0xec, 0x9d, 0x3f, 0xe6,
-	0xe4, 0xce, 0x17, 0xfb, 0xe0, 0xec, 0x0e, 0x15, 0xd1, 0x54, 0xb6, 0x71, 0x6a, 0x0c, 0xb6, 0xb5,
-	0xf4, 0x2f, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x85, 0x02, 0x09, 0x9d, 0x9f, 0x06, 0x00, 0x00,
+	proto.RegisterFile("reflection/grpc_reflection_v1alpha/reflection.proto", fileDescriptor_e8cf9f2921ad6c95)
+}
+
+var fileDescriptor_e8cf9f2921ad6c95 = []byte{
+	// 686 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x41, 0x6f, 0xd3, 0x4c,
+	0x10, 0xad, 0xdb, 0xa4, 0x55, 0x26, 0x69, 0x92, 0x6f, 0xdb, 0xaf, 0x71, 0x41, 0x45, 0x91, 0xa1,
+	0x90, 0x22, 0x94, 0xb4, 0xa9, 0x84, 0x84, 0xb8, 0xa5, 0x80, 0x82, 0x54, 0x5a, 0xe4, 0x70, 0x01,
+	0x0e, 0x2b, 0x27, 0x99, 0xb8, 0x06, 0xc7, 0x6b, 0x76, 0xdd, 0x40, 0x4e, 0xfc, 0x08, 0x7e, 0x14,
+	0x7f, 0x89, 0x23, 0xda, 0xb5, 0x63, 0x3b, 0x6e, 0x4c, 0xd5, 0x53, 0x9c, 0x37, 0x33, 0xfb, 0x66,
+	0xf6, 0xbd, 0xb1, 0xe1, 0x94, 0xe3, 0xc4, 0xc5, 0x51, 0xe0, 0x30, 0xaf, 0x63, 0x73, 0x7f, 0x44,
+	0x93, 0xff, 0x74, 0x76, 0x62, 0xb9, 0xfe, 0x95, 0xd5, 0x49, 0xa0, 0xb6, 0xcf, 0x59, 0xc0, 0x48,
+	0x43, 0x66, 0xb6, 0x53, 0x70, 0x94, 0x69, 0xfc, 0x59, 0x87, 0xc6, 0x00, 0xf9, 0x0c, 0xb9, 0x19,
+	0x07, 0x4d, 0xfc, 0x76, 0x8d, 0x22, 0x20, 0x04, 0x0a, 0x57, 0x4c, 0x04, 0xba, 0xd6, 0xd4, 0x5a,
+	0x25, 0x53, 0x3d, 0x93, 0xa7, 0x50, 0x9f, 0x38, 0x2e, 0xd2, 0xe1, 0x9c, 0xca, 0x5f, 0xcf, 0x9a,
+	0xa2, 0xbe, 0x21, 0xe3, 0xfd, 0x35, 0xb3, 0x2a, 0x91, 0xde, 0xfc, 0x4d, 0x84, 0x93, 0xe7, 0xb0,
+	0xa7, 0x72, 0x47, 0xcc, 0x0b, 0x2c, 0xc7, 0x73, 0x3c, 0x9b, 0x8a, 0xf9, 0x74, 0xc8, 0x5c, 0xbd,
+	0x10, 0x55, 0xec, 0xca, 0xf8, 0x59, 0x1c, 0x1e, 0xa8, 0x28, 0xb1, 0x61, 0x3f, 0x5b, 0x87, 0x3f,
+	0x02, 0xf4, 0x84, 0xc3, 0x3c, 0xbd, 0xd8, 0xd4, 0x5a, 0xe5, 0xee, 0x51, 0x3b, 0x67, 0xa0, 0xf6,
+	0xeb, 0x45, 0x66, 0x34, 0x45, 0x7f, 0xcd, 0x6c, 0x2c, 0xb3, 0xc4, 0x19, 0xa4, 0x07, 0x07, 0x96,
+	0xeb, 0x26, 0x87, 0x53, 0xef, 0x7a, 0x3a, 0x44, 0x2e, 0x28, 0x9b, 0xd0, 0x60, 0xee, 0xa3, 0xbe,
+	0x19, 0xf5, 0xb9, 0x6f, 0xb9, 0x6e, 0x5c, 0x76, 0x11, 0x26, 0x5d, 0x4e, 0x3e, 0xcc, 0x7d, 0x24,
+	0x87, 0xb0, 0xed, 0x3a, 0x22, 0xa0, 0x02, 0xf9, 0xcc, 0x19, 0xa1, 0xd0, 0xb7, 0xa2, 0x9a, 0x8a,
+	0x84, 0x07, 0x11, 0xda, 0xfb, 0x0f, 0x6a, 0x53, 0x14, 0xc2, 0xb2, 0x91, 0xf2, 0xb0, 0x31, 0x63,
+	0x02, 0xf5, 0x6c, 0xb3, 0xe4, 0x09, 0xd4, 0x52, 0x53, 0xab, 0x1e, 0xc2, 0xdb, 0xaf, 0x26, 0xb0,
+	0xa2, 0x3d, 0x82, 0x7a, 0xb6, 0x6d, 0x7d, 0xbd, 0xa9, 0xb5, 0x8a, 0x66, 0x0d, 0x97, 0x1b, 0x35,
+	0x7e, 0x17, 0x40, 0xbf, 0x29, 0xb1, 0xf0, 0x99, 0x27, 0x90, 0x1c, 0x00, 0xcc, 0x2c, 0xd7, 0x19,
+	0xd3, 0x94, 0xd2, 0x25, 0x85, 0xf4, 0xa5, 0xdc, 0x9f, 0xa1, 0xce, 0xb8, 0x63, 0x3b, 0x9e, 0xe5,
+	0x2e, 0xfa, 0x56, 0x34, 0xe5, 0xee, 0x71, 0xae, 0x02, 0x39, 0x76, 0x32, 0x6b, 0x8b, 0x93, 0x16,
+	0xc3, 0x7e, 0x05, 0x5d, 0xe9, 0x3c, 0x46, 0x31, 0xe2, 0x8e, 0x1f, 0x30, 0x4e, 0x79, 0xd4, 0x97,
+	0x72, 0x48, 0xb9, 0xdb, 0xc9, 0x25, 0x91, 0x26, 0x7b, 0x15, 0xd7, 0x2d, 0xc6, 0xe9, 0xaf, 0x99,
+	0xca, 0x72, 0x37, 0x23, 0xe4, 0x3b, 0x3c, 0x58, 0xad, 0x75, 0x4c, 0x59, 0xbc, 0x65, 0xae, 0x8c,
+	0x01, 0x52, 0x9c, 0xf7, 0x57, 0xd8, 0x23, 0x26, 0x1e, 0xc3, 0xde, 0x92, 0x41, 0x12, 0xc2, 0x4d,
+	0x45, 0xf8, 0x2c, 0x97, 0xf0, 0x3c, 0x31, 0x50, 0x8a, 0x6c, 0x37, 0xed, 0xab, 0x98, 0xe5, 0x12,
+	0xaa, 0xc8, 0x79, 0xfa, 0x06, 0xb7, 0xd4, 0xe9, 0x8f, 0xf3, 0xc7, 0x91, 0xe9, 0xa9, 0x73, 0xb7,
+	0x31, 0x0d, 0xf4, 0x08, 0xd4, 0x13, 0xc3, 0x86, 0x98, 0x71, 0x0e, 0x7b, 0xab, 0xef, 0x9d, 0x74,
+	0xe1, 0xff, 0xac, 0x94, 0xea, 0xc5, 0xa3, 0x6b, 0xcd, 0x8d, 0x56, 0xc5, 0xdc, 0x59, 0x16, 0xe5,
+	0xbd, 0x0c, 0x19, 0x5f, 0xa0, 0x91, 0x73, 0xa5, 0xe4, 0x11, 0x54, 0x87, 0x96, 0x40, 0xb5, 0x00,
+	0x54, 0xbd, 0x63, 0x42, 0x67, 0x56, 0x24, 0x2a, 0xfd, 0x7f, 0x21, 0xdf, 0x2f, 0xab, 0x77, 0x60,
+	0x63, 0xd5, 0x0e, 0x7c, 0x84, 0x9d, 0x15, 0xb7, 0x49, 0x7a, 0xb0, 0x15, 0xc9, 0xa2, 0x1a, 0x2d,
+	0x77, 0x5b, 0xff, 0x74, 0x75, 0xaa, 0xd4, 0x5c, 0x14, 0x1a, 0x87, 0x50, 0xcb, 0x1e, 0x4b, 0xa0,
+	0x90, 0x6a, 0x5a, 0x3d, 0x1b, 0x03, 0xd8, 0x5e, 0xba, 0x71, 0xb9, 0x79, 0xa1, 0x62, 0x23, 0x36,
+	0x0e, 0x53, 0x8b, 0x66, 0x49, 0x21, 0x67, 0x6c, 0x8c, 0xe4, 0x21, 0x84, 0x82, 0xd0, 0x48, 0x05,
+	0xb5, 0x76, 0x25, 0xb3, 0xa2, 0xc0, 0x77, 0x21, 0xd6, 0xfd, 0xa5, 0x41, 0x3d, 0xbb, 0x6e, 0xe4,
+	0x27, 0xec, 0x66, 0xb1, 0xb7, 0xde, 0x84, 0x91, 0x3b, 0x6f, 0xec, 0xbd, 0x93, 0x3b, 0x54, 0x84,
+	0x53, 0xb5, 0xb4, 0x63, 0xad, 0xf7, 0xf2, 0xd3, 0x0b, 0x9b, 0x31, 0xdb, 0xc5, 0xb6, 0xcd, 0x5c,
+	0xcb, 0xb3, 0xdb, 0x8c, 0xdb, 0xea, 0x53, 0xd5, 0xb9, 0xfd, 0xd3, 0x35, 0xdc, 0x54, 0xbe, 0x39,
+	0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x74, 0x3a, 0x67, 0xe7, 0x06, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -748,5 +750,5 @@ var _ServerReflection_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "grpc_reflection_v1alpha/reflection.proto",
+	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
 }
diff --git a/reflection/grpc_reflection_v1alpha/reflection.proto b/reflection/grpc_reflection_v1alpha/reflection.proto
index 99b00df0a294..ee2b82c0a5b3 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.proto
+++ b/reflection/grpc_reflection_v1alpha/reflection.proto
@@ -16,6 +16,8 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/reflection/grpc_reflection_v1alpha";
+
 package grpc.reflection.v1alpha;
 
 service ServerReflection {
diff --git a/reflection/grpc_testing/proto2.pb.go b/reflection/grpc_testing/proto2.pb.go
index 319ea0deaea7..d34adb82a85e 100644
--- a/reflection/grpc_testing/proto2.pb.go
+++ b/reflection/grpc_testing/proto2.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: proto2.proto
+// source: reflection/grpc_testing/proto2.proto
 
 package grpc_testing
 
@@ -32,7 +32,7 @@ func (m *ToBeExtended) Reset()         { *m = ToBeExtended{} }
 func (m *ToBeExtended) String() string { return proto.CompactTextString(m) }
 func (*ToBeExtended) ProtoMessage()    {}
 func (*ToBeExtended) Descriptor() ([]byte, []int) {
-	return fileDescriptor_1f509089572db8e7, []int{0}
+	return fileDescriptor_dddbb2c1ebdcf2b8, []int{0}
 }
 
 var extRange_ToBeExtended = []proto.ExtensionRange{
@@ -72,14 +72,19 @@ func init() {
 	proto.RegisterType((*ToBeExtended)(nil), "grpc.testing.ToBeExtended")
 }
 
-func init() { proto.RegisterFile("proto2.proto", fileDescriptor_1f509089572db8e7) }
+func init() {
+	proto.RegisterFile("reflection/grpc_testing/proto2.proto", fileDescriptor_dddbb2c1ebdcf2b8)
+}
 
-var fileDescriptor_1f509089572db8e7 = []byte{
-	// 86 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x28, 0xca, 0x2f,
-	0xc9, 0x37, 0xd2, 0x03, 0x53, 0x42, 0x3c, 0xe9, 0x45, 0x05, 0xc9, 0x7a, 0x25, 0xa9, 0xc5, 0x25,
-	0x99, 0x79, 0xe9, 0x4a, 0x6a, 0x5c, 0x3c, 0x21, 0xf9, 0x4e, 0xa9, 0xae, 0x15, 0x25, 0xa9, 0x79,
-	0x29, 0xa9, 0x29, 0x42, 0x02, 0x5c, 0xcc, 0x69, 0xf9, 0xf9, 0x12, 0x8c, 0x0a, 0x4c, 0x1a, 0xac,
-	0x41, 0x20, 0xa6, 0x16, 0x0b, 0x07, 0x97, 0x80, 0x3c, 0x20, 0x00, 0x00, 0xff, 0xff, 0x74, 0x86,
-	0x9c, 0x08, 0x44, 0x00, 0x00, 0x00,
+var fileDescriptor_dddbb2c1ebdcf2b8 = []byte{
+	// 130 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x29, 0x4a, 0x4d, 0xcb,
+	0x49, 0x4d, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x4f, 0x2f, 0x2a, 0x48, 0x8e, 0x2f, 0x49, 0x2d, 0x2e,
+	0xc9, 0xcc, 0x4b, 0xd7, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x37, 0xd2, 0x03, 0x53, 0x42, 0x3c, 0x20,
+	0x29, 0x3d, 0xa8, 0x94, 0x92, 0x1a, 0x17, 0x4f, 0x48, 0xbe, 0x53, 0xaa, 0x6b, 0x45, 0x49, 0x6a,
+	0x5e, 0x4a, 0x6a, 0x8a, 0x90, 0x00, 0x17, 0x73, 0x5a, 0x7e, 0xbe, 0x04, 0xa3, 0x02, 0x93, 0x06,
+	0x6b, 0x10, 0x88, 0xa9, 0xc5, 0xc2, 0xc1, 0x25, 0x20, 0xef, 0x64, 0x10, 0xa5, 0x97, 0x9e, 0x9f,
+	0x9f, 0x9e, 0x93, 0xaa, 0x97, 0x9e, 0x9f, 0x93, 0x98, 0x97, 0xae, 0x97, 0x5f, 0x94, 0x0e, 0xb6,
+	0x44, 0x1f, 0x87, 0xa5, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x05, 0x50, 0x64, 0x8e, 0x00,
+	0x00, 0x00,
 }
diff --git a/reflection/grpc_testing/proto2.proto b/reflection/grpc_testing/proto2.proto
index a675d143d713..c90d8098f868 100644
--- a/reflection/grpc_testing/proto2.proto
+++ b/reflection/grpc_testing/proto2.proto
@@ -14,6 +14,8 @@
 
 syntax = "proto2";
 
+option go_package = "google.golang.org/grpc/reflection/grpc_testing";
+
 package grpc.testing;
 
 message ToBeExtended {
diff --git a/reflection/grpc_testing/proto2_ext.pb.go b/reflection/grpc_testing/proto2_ext.pb.go
index f2bfe5f3793c..631b5146d56c 100644
--- a/reflection/grpc_testing/proto2_ext.pb.go
+++ b/reflection/grpc_testing/proto2_ext.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: proto2_ext.proto
+// source: reflection/grpc_testing/proto2_ext.proto
 
 package grpc_testing
 
@@ -31,7 +31,7 @@ func (m *Extension) Reset()         { *m = Extension{} }
 func (m *Extension) String() string { return proto.CompactTextString(m) }
 func (*Extension) ProtoMessage()    {}
 func (*Extension) Descriptor() ([]byte, []int) {
-	return fileDescriptor_85b2817ade17959b, []int{0}
+	return fileDescriptor_071dc827b8673a0c, []int{0}
 }
 
 func (m *Extension) XXX_Unmarshal(b []byte) error {
@@ -65,7 +65,7 @@ var E_Foo = &proto.ExtensionDesc{
 	Field:         13,
 	Name:          "grpc.testing.foo",
 	Tag:           "varint,13,opt,name=foo",
-	Filename:      "proto2_ext.proto",
+	Filename:      "reflection/grpc_testing/proto2_ext.proto",
 }
 
 var E_Bar = &proto.ExtensionDesc{
@@ -74,7 +74,7 @@ var E_Bar = &proto.ExtensionDesc{
 	Field:         17,
 	Name:          "grpc.testing.bar",
 	Tag:           "bytes,17,opt,name=bar",
-	Filename:      "proto2_ext.proto",
+	Filename:      "reflection/grpc_testing/proto2_ext.proto",
 }
 
 var E_Baz = &proto.ExtensionDesc{
@@ -83,7 +83,7 @@ var E_Baz = &proto.ExtensionDesc{
 	Field:         19,
 	Name:          "grpc.testing.baz",
 	Tag:           "bytes,19,opt,name=baz",
-	Filename:      "proto2_ext.proto",
+	Filename:      "reflection/grpc_testing/proto2_ext.proto",
 }
 
 func init() {
@@ -93,20 +93,24 @@ func init() {
 	proto.RegisterExtension(E_Baz)
 }
 
-func init() { proto.RegisterFile("proto2_ext.proto", fileDescriptor_85b2817ade17959b) }
+func init() {
+	proto.RegisterFile("reflection/grpc_testing/proto2_ext.proto", fileDescriptor_071dc827b8673a0c)
+}
 
-var fileDescriptor_85b2817ade17959b = []byte{
-	// 179 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x28, 0x28, 0xca, 0x2f,
-	0xc9, 0x37, 0x8a, 0x4f, 0xad, 0x28, 0xd1, 0x03, 0x33, 0x85, 0x78, 0xd2, 0x8b, 0x0a, 0x92, 0xf5,
-	0x4a, 0x52, 0x8b, 0x4b, 0x32, 0xf3, 0xd2, 0xa5, 0x78, 0x20, 0xf2, 0x10, 0x39, 0x29, 0x2e, 0x90,
-	0x30, 0x84, 0xad, 0xa4, 0xca, 0xc5, 0xe9, 0x5a, 0x51, 0x92, 0x9a, 0x57, 0x9c, 0x99, 0x9f, 0x27,
-	0x24, 0xc1, 0xc5, 0x5e, 0x9e, 0x91, 0x58, 0x52, 0x95, 0x59, 0x22, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1,
-	0x1a, 0x04, 0xe3, 0x5a, 0xe9, 0x70, 0x31, 0xa7, 0xe5, 0xe7, 0x0b, 0x49, 0xe9, 0x21, 0x1b, 0xab,
-	0x17, 0x92, 0xef, 0x94, 0x0a, 0xd6, 0x9d, 0x92, 0x9a, 0x22, 0xc1, 0x0b, 0xd6, 0x01, 0x52, 0x66,
-	0xe5, 0xca, 0xc5, 0x9c, 0x94, 0x58, 0x84, 0x57, 0xb5, 0xa0, 0x02, 0xa3, 0x06, 0xb7, 0x91, 0x38,
-	0xaa, 0x0a, 0xb8, 0x4b, 0x82, 0x40, 0xfa, 0xad, 0x3c, 0x41, 0xc6, 0x54, 0xe1, 0x35, 0x46, 0x18,
-	0x6c, 0x8c, 0x34, 0xaa, 0x8a, 0xe0, 0xd4, 0xc4, 0xa2, 0xe4, 0x8c, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4,
-	0xe2, 0x12, 0x90, 0x51, 0x55, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x71, 0x6b, 0x94, 0x9f, 0x21,
-	0x01, 0x00, 0x00,
+var fileDescriptor_071dc827b8673a0c = []byte{
+	// 224 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0xc1, 0x4b, 0x85, 0x40,
+	0x10, 0xc6, 0x91, 0x47, 0x44, 0x5b, 0x1d, 0xb2, 0x43, 0x62, 0x97, 0x87, 0x14, 0x78, 0x88, 0x35,
+	0x3c, 0x7a, 0x14, 0x3c, 0x74, 0xb5, 0x4e, 0x5d, 0x64, 0xd3, 0x71, 0x5d, 0x90, 0x1d, 0x5b, 0x27,
+	0x12, 0xff, 0xfa, 0xd8, 0xb5, 0xe2, 0x79, 0xd0, 0xd3, 0xee, 0x30, 0xbf, 0xdf, 0xc7, 0xf0, 0xb1,
+	0xd8, 0x40, 0xdb, 0x43, 0x4d, 0x0a, 0x75, 0x22, 0xcd, 0x50, 0x57, 0x04, 0x23, 0x29, 0x2d, 0x93,
+	0xc1, 0x20, 0x61, 0x5a, 0xc1, 0x44, 0xdc, 0x7d, 0xfd, 0x2b, 0xbb, 0xe6, 0xbf, 0xeb, 0xf0, 0x61,
+	0xdf, 0x5b, 0x9c, 0x30, 0xda, 0xa2, 0xec, 0xbb, 0x30, 0xd1, 0x23, 0xbb, 0x28, 0x26, 0x02, 0x3d,
+	0x2a, 0xd4, 0x7e, 0xc0, 0xce, 0xbf, 0x3b, 0x41, 0xb3, 0xa2, 0xc0, 0x3b, 0x7a, 0xf1, 0x59, 0xf9,
+	0x37, 0x66, 0x4f, 0xec, 0xd0, 0x22, 0xfa, 0x21, 0x3f, 0x3d, 0x83, 0xbf, 0x61, 0x0e, 0xce, 0x6e,
+	0xa0, 0x09, 0xae, 0x9d, 0x61, 0xb1, 0xac, 0x60, 0x87, 0x0f, 0x61, 0x76, 0xe9, 0x9b, 0xa3, 0x17,
+	0x5f, 0xa6, 0x77, 0x6b, 0xe2, 0xff, 0x92, 0xd2, 0xfa, 0xd9, 0x8b, 0x8d, 0x99, 0x77, 0x63, 0x6e,
+	0x5d, 0xcc, 0xfd, 0x9a, 0x78, 0x05, 0x61, 0xea, 0xae, 0x84, 0xcf, 0x2f, 0x18, 0xc9, 0x46, 0xcd,
+	0xf9, 0xf3, 0x3b, 0x97, 0x88, 0xb2, 0x07, 0x2e, 0xb1, 0x17, 0x5a, 0x72, 0x34, 0xd2, 0x75, 0x92,
+	0x6c, 0x74, 0xf4, 0x13, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x14, 0xcf, 0xc9, 0x9b, 0x01, 0x00, 0x00,
 }
diff --git a/reflection/grpc_testing/proto2_ext.proto b/reflection/grpc_testing/proto2_ext.proto
index a4942e481b43..4ba2dc6bca5d 100644
--- a/reflection/grpc_testing/proto2_ext.proto
+++ b/reflection/grpc_testing/proto2_ext.proto
@@ -14,10 +14,12 @@
 
 syntax = "proto2";
 
+option go_package = "google.golang.org/grpc/reflection/grpc_testing";
+
 package grpc.testing;
 
-import "proto2.proto";
-import "test.proto";
+import "reflection/grpc_testing/proto2.proto";
+import "reflection/grpc_testing/test.proto";
 
 extend ToBeExtended {
   optional int32 foo = 13;
diff --git a/reflection/grpc_testing/proto2_ext2.pb.go b/reflection/grpc_testing/proto2_ext2.pb.go
index 6601cfc44b19..ad2fef3addd3 100644
--- a/reflection/grpc_testing/proto2_ext2.pb.go
+++ b/reflection/grpc_testing/proto2_ext2.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: proto2_ext2.proto
+// source: reflection/grpc_testing/proto2_ext2.proto
 
 package grpc_testing
 
@@ -31,7 +31,7 @@ func (m *AnotherExtension) Reset()         { *m = AnotherExtension{} }
 func (m *AnotherExtension) String() string { return proto.CompactTextString(m) }
 func (*AnotherExtension) ProtoMessage()    {}
 func (*AnotherExtension) Descriptor() ([]byte, []int) {
-	return fileDescriptor_21d110045b8a354c, []int{0}
+	return fileDescriptor_ead6f7bd8a66fb18, []int{0}
 }
 
 func (m *AnotherExtension) XXX_Unmarshal(b []byte) error {
@@ -65,7 +65,7 @@ var E_Frob = &proto.ExtensionDesc{
 	Field:         23,
 	Name:          "grpc.testing.frob",
 	Tag:           "bytes,23,opt,name=frob",
-	Filename:      "proto2_ext2.proto",
+	Filename:      "reflection/grpc_testing/proto2_ext2.proto",
 }
 
 var E_Nitz = &proto.ExtensionDesc{
@@ -74,7 +74,7 @@ var E_Nitz = &proto.ExtensionDesc{
 	Field:         29,
 	Name:          "grpc.testing.nitz",
 	Tag:           "bytes,29,opt,name=nitz",
-	Filename:      "proto2_ext2.proto",
+	Filename:      "reflection/grpc_testing/proto2_ext2.proto",
 }
 
 func init() {
@@ -83,19 +83,23 @@ func init() {
 	proto.RegisterExtension(E_Nitz)
 }
 
-func init() { proto.RegisterFile("proto2_ext2.proto", fileDescriptor_21d110045b8a354c) }
+func init() {
+	proto.RegisterFile("reflection/grpc_testing/proto2_ext2.proto", fileDescriptor_ead6f7bd8a66fb18)
+}
 
-var fileDescriptor_21d110045b8a354c = []byte{
-	// 165 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2c, 0x28, 0xca, 0x2f,
-	0xc9, 0x37, 0x8a, 0x4f, 0xad, 0x28, 0x31, 0xd2, 0x03, 0xb3, 0x85, 0x78, 0xd2, 0x8b, 0x0a, 0x92,
-	0xf5, 0x4a, 0x52, 0x8b, 0x4b, 0x32, 0xf3, 0xd2, 0xa5, 0x78, 0x20, 0x0a, 0x20, 0x72, 0x4a, 0x36,
-	0x5c, 0x02, 0x8e, 0x79, 0xf9, 0x25, 0x19, 0xa9, 0x45, 0xae, 0x15, 0x25, 0xa9, 0x79, 0xc5, 0x99,
-	0xf9, 0x79, 0x42, 0x1a, 0x5c, 0xfc, 0xe5, 0x19, 0x89, 0x25, 0xc9, 0x19, 0x89, 0xb9, 0x89, 0xc9,
-	0x89, 0x39, 0x39, 0x99, 0x25, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0xac, 0x41, 0xe8, 0xc2, 0x56, 0x7a,
-	0x5c, 0x2c, 0x69, 0x45, 0xf9, 0x49, 0x42, 0x52, 0x7a, 0xc8, 0x56, 0xe8, 0x85, 0xe4, 0x3b, 0xa5,
-	0x82, 0x8d, 0x4b, 0x49, 0x4d, 0x91, 0x10, 0x57, 0x60, 0xd4, 0xe0, 0x0c, 0x02, 0xab, 0xb3, 0xf2,
-	0xe3, 0x62, 0xc9, 0xcb, 0x2c, 0xa9, 0xc2, 0xab, 0x5e, 0x56, 0x81, 0x51, 0x83, 0xdb, 0x48, 0x0e,
-	0x55, 0x05, 0xba, 0x1b, 0x83, 0xc0, 0xe6, 0x00, 0x02, 0x00, 0x00, 0xff, 0xff, 0xf0, 0x7e, 0x0d,
-	0x26, 0xed, 0x00, 0x00, 0x00,
+var fileDescriptor_ead6f7bd8a66fb18 = []byte{
+	// 208 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x8f, 0x31, 0x4b, 0xc7, 0x30,
+	0x10, 0xc5, 0x09, 0xfc, 0x1d, 0x8c, 0x82, 0xd2, 0xc5, 0x52, 0x50, 0x8a, 0x38, 0xc4, 0xe5, 0x2a,
+	0x1d, 0x8b, 0x8b, 0x05, 0x57, 0x87, 0xe2, 0xe4, 0x52, 0x62, 0x7a, 0x4d, 0x03, 0x31, 0x57, 0xd2,
+	0x03, 0x8b, 0x9f, 0x5e, 0x8c, 0x0e, 0x5a, 0xb0, 0xdb, 0x71, 0xef, 0xf7, 0xde, 0xdd, 0x93, 0xb7,
+	0x11, 0x47, 0x8f, 0x86, 0x1d, 0x85, 0xca, 0xc6, 0xd9, 0xf4, 0x8c, 0x0b, 0xbb, 0x60, 0xab, 0x39,
+	0x12, 0x53, 0xdd, 0xe3, 0xca, 0x35, 0xa4, 0x39, 0x3b, 0xfd, 0xd2, 0xe1, 0x47, 0x2f, 0x6e, 0xf6,
+	0x8d, 0xdf, 0x9e, 0xeb, 0x7b, 0x79, 0xfe, 0x10, 0x88, 0x27, 0x8c, 0x8f, 0x2b, 0x63, 0x58, 0x1c,
+	0x85, 0x4c, 0xc9, 0xb3, 0xf7, 0x49, 0xb3, 0x99, 0xf4, 0x9b, 0x36, 0xda, 0x7b, 0xc7, 0xb9, 0x28,
+	0x85, 0x3a, 0xea, 0xb6, 0xeb, 0x06, 0xe4, 0x61, 0x8c, 0xf4, 0x9a, 0x15, 0xf0, 0xfb, 0x34, 0x3c,
+	0x53, 0x8b, 0x29, 0x6e, 0xc0, 0x21, 0xbf, 0x28, 0x85, 0x3a, 0xee, 0x12, 0xd7, 0x3c, 0xc9, 0x43,
+	0x70, 0xfc, 0xb1, 0xcb, 0x5f, 0x96, 0x42, 0x9d, 0xd4, 0x57, 0x7f, 0x89, 0xed, 0x8f, 0x5d, 0xca,
+	0x69, 0xef, 0x5e, 0xc0, 0x12, 0x59, 0x8f, 0x60, 0xc9, 0xeb, 0x60, 0x81, 0xa2, 0x4d, 0x65, 0xab,
+	0x7f, 0xca, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0x18, 0x74, 0xc1, 0x53, 0x4f, 0x01, 0x00, 0x00,
 }
diff --git a/reflection/grpc_testing/proto2_ext2.proto b/reflection/grpc_testing/proto2_ext2.proto
index d91ba0061914..931154667ede 100644
--- a/reflection/grpc_testing/proto2_ext2.proto
+++ b/reflection/grpc_testing/proto2_ext2.proto
@@ -14,9 +14,11 @@
 
 syntax = "proto2";
 
+option go_package = "google.golang.org/grpc/reflection/grpc_testing";
+
 package grpc.testing;
 
-import "proto2.proto";
+import "reflection/grpc_testing/proto2.proto";
 
 extend ToBeExtended {
   optional string frob = 23;
diff --git a/reflection/grpc_testing/test.pb.go b/reflection/grpc_testing/test.pb.go
index 6cfb1e3f518b..72841c8ce216 100644
--- a/reflection/grpc_testing/test.pb.go
+++ b/reflection/grpc_testing/test.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: test.proto
+// source: reflection/grpc_testing/test.proto
 
 package grpc_testing
 
@@ -35,7 +35,7 @@ func (m *SearchResponse) Reset()         { *m = SearchResponse{} }
 func (m *SearchResponse) String() string { return proto.CompactTextString(m) }
 func (*SearchResponse) ProtoMessage()    {}
 func (*SearchResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_c161fcfdc0c3ff1e, []int{0}
+	return fileDescriptor_b179ea967ba71047, []int{0}
 }
 
 func (m *SearchResponse) XXX_Unmarshal(b []byte) error {
@@ -76,7 +76,7 @@ func (m *SearchResponse_Result) Reset()         { *m = SearchResponse_Result{} }
 func (m *SearchResponse_Result) String() string { return proto.CompactTextString(m) }
 func (*SearchResponse_Result) ProtoMessage()    {}
 func (*SearchResponse_Result) Descriptor() ([]byte, []int) {
-	return fileDescriptor_c161fcfdc0c3ff1e, []int{0, 0}
+	return fileDescriptor_b179ea967ba71047, []int{0, 0}
 }
 
 func (m *SearchResponse_Result) XXX_Unmarshal(b []byte) error {
@@ -129,7 +129,7 @@ func (m *SearchRequest) Reset()         { *m = SearchRequest{} }
 func (m *SearchRequest) String() string { return proto.CompactTextString(m) }
 func (*SearchRequest) ProtoMessage()    {}
 func (*SearchRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_c161fcfdc0c3ff1e, []int{1}
+	return fileDescriptor_b179ea967ba71047, []int{1}
 }
 
 func (m *SearchRequest) XXX_Unmarshal(b []byte) error {
@@ -163,25 +163,29 @@ func init() {
 	proto.RegisterType((*SearchRequest)(nil), "grpc.testing.SearchRequest")
 }
 
-func init() { proto.RegisterFile("test.proto", fileDescriptor_c161fcfdc0c3ff1e) }
-
-var fileDescriptor_c161fcfdc0c3ff1e = []byte{
-	// 231 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x91, 0xbd, 0x4a, 0xc5, 0x40,
-	0x10, 0x85, 0x59, 0x83, 0xd1, 0x3b, 0xfe, 0x32, 0x58, 0x84, 0x68, 0x11, 0xae, 0x08, 0xa9, 0x16,
-	0xb9, 0xd6, 0x56, 0xb6, 0x16, 0xb2, 0x79, 0x82, 0x6b, 0x18, 0xe2, 0x42, 0x4c, 0x36, 0x33, 0x13,
-	0xc1, 0x87, 0xb1, 0xf5, 0x39, 0x25, 0x59, 0x23, 0x0a, 0x62, 0x63, 0xb7, 0xe7, 0xe3, 0xcc, 0xb7,
-	0xbb, 0x0c, 0x80, 0x92, 0xa8, 0x0d, 0xdc, 0x6b, 0x8f, 0x87, 0x0d, 0x87, 0xda, 0x4e, 0xc0, 0x77,
-	0xcd, 0xfa, 0xcd, 0xc0, 0x71, 0x45, 0x5b, 0xae, 0x9f, 0x1c, 0x49, 0xe8, 0x3b, 0x21, 0xbc, 0x85,
-	0x3d, 0x26, 0x19, 0x5b, 0x95, 0xcc, 0x14, 0x49, 0x79, 0xb0, 0xb9, 0xb4, 0xdf, 0x47, 0xec, 0xcf,
-	0xba, 0x75, 0x73, 0xd7, 0x2d, 0x33, 0xf9, 0x3d, 0xa4, 0x11, 0xe1, 0x29, 0x24, 0x23, 0xb7, 0x99,
-	0x29, 0x4c, 0xb9, 0x72, 0xd3, 0x11, 0xcf, 0x60, 0x57, 0xbd, 0xb6, 0x94, 0xed, 0xcc, 0x2c, 0x06,
-	0xcc, 0x61, 0x5f, 0x3a, 0x1f, 0x02, 0xa9, 0x64, 0x49, 0x91, 0x94, 0x2b, 0xf7, 0x95, 0xd7, 0x57,
-	0x70, 0xb4, 0xdc, 0x37, 0x8c, 0x24, 0x3a, 0x29, 0x86, 0x91, 0xf8, 0xf5, 0x53, 0x1b, 0xc3, 0xe6,
-	0xdd, 0x2c, 0xbd, 0x8a, 0xf8, 0xc5, 0xd7, 0x84, 0x77, 0x90, 0x46, 0x80, 0xe7, 0xbf, 0x3f, 0x7f,
-	0xd6, 0xe5, 0x17, 0x7f, 0xfd, 0x0d, 0x1f, 0xe0, 0xa4, 0x52, 0xa6, 0xed, 0xb3, 0xef, 0x9a, 0x7f,
-	0xdb, 0x4a, 0x73, 0x6d, 0x1e, 0xd3, 0x79, 0x09, 0x37, 0x1f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x20,
-	0xd6, 0x09, 0xb8, 0x92, 0x01, 0x00, 0x00,
+func init() {
+	proto.RegisterFile("reflection/grpc_testing/test.proto", fileDescriptor_b179ea967ba71047)
+}
+
+var fileDescriptor_b179ea967ba71047 = []byte{
+	// 267 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x51, 0x3f, 0x4b, 0xfd, 0x30,
+	0x14, 0x25, 0xbf, 0xf2, 0xab, 0xbe, 0xeb, 0x5f, 0x82, 0x43, 0xa9, 0x0e, 0xa5, 0x22, 0x74, 0x4a,
+	0x1f, 0xcf, 0xd9, 0x45, 0x57, 0x07, 0x69, 0x37, 0x17, 0xa9, 0xe5, 0x1a, 0x03, 0x31, 0xe9, 0xbb,
+	0xb9, 0x15, 0xfc, 0x30, 0xae, 0x7e, 0x4e, 0x69, 0xfb, 0x2a, 0x0a, 0xea, 0xe2, 0x94, 0x9c, 0x73,
+	0xcf, 0x39, 0xc9, 0xe5, 0x40, 0x4e, 0xf8, 0x60, 0xb1, 0x65, 0xe3, 0x5d, 0xa9, 0xa9, 0x6b, 0xef,
+	0x18, 0x03, 0x1b, 0xa7, 0xcb, 0xe1, 0x54, 0x1d, 0x79, 0xf6, 0x72, 0x77, 0x18, 0xa8, 0xcd, 0x20,
+	0x7f, 0x15, 0xb0, 0x5f, 0x63, 0x43, 0xed, 0x63, 0x85, 0xa1, 0xf3, 0x2e, 0xa0, 0xbc, 0x80, 0x2d,
+	0xc2, 0xd0, 0x5b, 0x0e, 0x89, 0xc8, 0xa2, 0x62, 0x67, 0x75, 0xaa, 0x3e, 0x5b, 0xd4, 0x57, 0xb9,
+	0xaa, 0x46, 0x6d, 0x35, 0x7b, 0xd2, 0x6b, 0x88, 0x27, 0x4a, 0x1e, 0x42, 0xd4, 0x93, 0x4d, 0x44,
+	0x26, 0x8a, 0x45, 0x35, 0x5c, 0xe5, 0x11, 0xfc, 0x67, 0xc3, 0x16, 0x93, 0x7f, 0x23, 0x37, 0x01,
+	0x99, 0xc2, 0x76, 0x70, 0xa6, 0xeb, 0x90, 0x43, 0x12, 0x65, 0x51, 0xb1, 0xa8, 0x3e, 0x70, 0x7e,
+	0x06, 0x7b, 0xf3, 0x7b, 0xeb, 0x1e, 0x03, 0x0f, 0x11, 0xeb, 0x1e, 0xe9, 0x65, 0x13, 0x3b, 0x81,
+	0xd5, 0x9b, 0x98, 0x75, 0x35, 0xd2, 0xb3, 0x69, 0x51, 0x5e, 0x41, 0x3c, 0x11, 0xf2, 0xf8, 0xfb,
+	0xef, 0x8f, 0x71, 0xe9, 0xc9, 0x6f, 0xbb, 0xc9, 0x1b, 0x38, 0xa8, 0x99, 0xb0, 0x79, 0x32, 0x4e,
+	0xff, 0x39, 0xad, 0x10, 0x4b, 0x71, 0xb9, 0xbc, 0x55, 0xda, 0x7b, 0x6d, 0x51, 0x69, 0x6f, 0x1b,
+	0xa7, 0x95, 0x27, 0x3d, 0x56, 0x55, 0xfe, 0x50, 0xdd, 0x7d, 0x3c, 0xd6, 0x76, 0xfe, 0x1e, 0x00,
+	0x00, 0xff, 0xff, 0x38, 0x42, 0x3b, 0xd2, 0xdc, 0x01, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -330,5 +334,5 @@ var _SearchService_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "test.proto",
+	Metadata: "reflection/grpc_testing/test.proto",
 }
diff --git a/reflection/grpc_testing/test.proto b/reflection/grpc_testing/test.proto
index cae3f01a0438..c4b18de649e1 100644
--- a/reflection/grpc_testing/test.proto
+++ b/reflection/grpc_testing/test.proto
@@ -14,6 +14,8 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/reflection/grpc_testing";
+
 package grpc.testing;
 
 message SearchResponse {
diff --git a/reflection/grpc_testingv3/testv3.proto b/reflection/grpc_testingv3/testv3.proto
index ee4966bf72cf..38a615a90d91 100644
--- a/reflection/grpc_testingv3/testv3.proto
+++ b/reflection/grpc_testingv3/testv3.proto
@@ -1,5 +1,7 @@
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/reflection/grpc_testingv3";
+
 package grpc.testingv3;
 
 message SearchResponseV3 {
diff --git a/reflection/serverreflection.go b/reflection/serverreflection.go
index dd22a2da7849..de3e132f0451 100644
--- a/reflection/serverreflection.go
+++ b/reflection/serverreflection.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc --go_out=plugins=grpc:. grpc_reflection_v1alpha/reflection.proto
-
 /*
 Package reflection implements server reflection service.
 
diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go
index d21b7a6eacc5..4cf6717feafa 100644
--- a/reflection/serverreflection_test.go
+++ b/reflection/serverreflection_test.go
@@ -16,11 +16,6 @@
  *
  */
 
-//go:generate protoc -I grpc_testing --go_out=plugins=grpc:grpc_testing/ grpc_testing/proto2.proto grpc_testing/proto2_ext.proto grpc_testing/proto2_ext2.proto grpc_testing/test.proto
-
-// Note: grpc_testingv3/testv3.pb.go is not re-generated because it was
-// intentionally generated by an older version of protoc-gen-go.
-
 package reflection
 
 import (
@@ -81,11 +76,11 @@ func loadFileDesc(filename string) (*dpb.FileDescriptorProto, []byte) {
 }
 
 func init() {
-	fdTest, fdTestByte = loadFileDesc("test.proto")
+	fdTest, fdTestByte = loadFileDesc("reflection/grpc_testing/test.proto")
 	fdTestv3, fdTestv3Byte = loadFileDesc("testv3.proto")
-	fdProto2, fdProto2Byte = loadFileDesc("proto2.proto")
-	fdProto2Ext, fdProto2ExtByte = loadFileDesc("proto2_ext.proto")
-	fdProto2Ext2, fdProto2Ext2Byte = loadFileDesc("proto2_ext2.proto")
+	fdProto2, fdProto2Byte = loadFileDesc("reflection/grpc_testing/proto2.proto")
+	fdProto2Ext, fdProto2ExtByte = loadFileDesc("reflection/grpc_testing/proto2_ext.proto")
+	fdProto2Ext2, fdProto2Ext2Byte = loadFileDesc("reflection/grpc_testing/proto2_ext2.proto")
 }
 
 func (x) TestFileDescForType(t *testing.T) {
@@ -237,9 +232,9 @@ func testFileByFilename(t *testing.T, stream rpb.ServerReflection_ServerReflecti
 		filename string
 		want     []byte
 	}{
-		{"test.proto", fdTestByte},
-		{"proto2.proto", fdProto2Byte},
-		{"proto2_ext.proto", fdProto2ExtByte},
+		{"reflection/grpc_testing/test.proto", fdTestByte},
+		{"reflection/grpc_testing/proto2.proto", fdProto2Byte},
+		{"reflection/grpc_testing/proto2_ext.proto", fdProto2ExtByte},
 	} {
 		if err := stream.Send(&rpb.ServerReflectionRequest{
 			MessageRequest: &rpb.ServerReflectionRequest_FileByFilename{
diff --git a/regenerate.sh b/regenerate.sh
new file mode 100755
index 000000000000..17f3c85138e5
--- /dev/null
+++ b/regenerate.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+# Copyright 2020 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu -o pipefail
+
+WORKDIR=$(mktemp -d)
+
+function finish {
+  rm -rf "$WORKDIR"
+}
+trap finish EXIT
+
+export GOBIN=${WORKDIR}/bin
+export PATH=${GOBIN}:${PATH}
+mkdir -p ${GOBIN}
+
+echo "go install github.com/golang/protobuf/protoc-gen-go"
+(cd test/tools && go install github.com/golang/protobuf/protoc-gen-go)
+
+echo "git clone https://github.com/grpc/grpc-proto"
+git clone --quiet https://github.com/grpc/grpc-proto ${WORKDIR}/grpc-proto
+
+mkdir -p ${WORKDIR}/googleapis/google/rpc
+echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
+curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > ${WORKDIR}/googleapis/google/rpc/code.proto
+
+mkdir -p ${WORKDIR}/out
+
+SOURCES=(
+  ${WORKDIR}/googleapis/google/rpc/code.proto
+  ${WORKDIR}/grpc-proto/grpc/binlog/v1/binarylog.proto
+  ${WORKDIR}/grpc-proto/grpc/channelz/v1/channelz.proto
+  ${WORKDIR}/grpc-proto/grpc/gcp/altscontext.proto
+  ${WORKDIR}/grpc-proto/grpc/gcp/handshaker.proto
+  ${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto
+  ${WORKDIR}/grpc-proto/grpc/health/v1/health.proto
+  ${WORKDIR}/grpc-proto/grpc/lb/v1/load_balancer.proto
+  ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
+  ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
+  $(git ls-files --exclude-standard --cached --others "*.proto")
+)
+OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config
+for src in ${SOURCES[@]}; do
+  echo "protoc ${src}"
+  protoc --go_opt=plugins=grpc --go_out=${OPTS}:${WORKDIR}/out \
+    -I"." \
+    -I${WORKDIR}/grpc-proto \
+    -I${WORKDIR}/googleapis \
+    ${src}
+done
+
+# The go_package option in grpc/lookup/v1/rls.proto doesn't match the
+# current location. Move it into the right place.
+mkdir -p ${WORKDIR}/out/google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1
+mv ${WORKDIR}/out/google.golang.org/grpc/lookup/grpc_lookup_v1/* ${WORKDIR}/out/google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1
+
+# grpc_testingv3/testv3.pb.go is not re-generated because it was
+# intentionally generated by an older version of protoc-gen-go.
+rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/testv3.pb.go
+
+# grpc/service_config/service_config.proto does not have a go_package option.
+cp ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config
+
+cp -R ${WORKDIR}/out/google.golang.org/grpc/ .
diff --git a/stats/grpc_testing/test.pb.go b/stats/grpc_testing/test.pb.go
index e040ab050cbd..9ce8e6819ef9 100644
--- a/stats/grpc_testing/test.pb.go
+++ b/stats/grpc_testing/test.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc_testing/test.proto
+// source: stats/grpc_testing/test.proto
 
 package grpc_testing
 
@@ -35,7 +35,7 @@ func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
 func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
 func (*SimpleRequest) ProtoMessage()    {}
 func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{0}
+	return fileDescriptor_d7a50f7a8e9e8e09, []int{0}
 }
 
 func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
@@ -74,7 +74,7 @@ func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
 func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
 func (*SimpleResponse) ProtoMessage()    {}
 func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{1}
+	return fileDescriptor_d7a50f7a8e9e8e09, []int{1}
 }
 
 func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
@@ -107,23 +107,25 @@ func init() {
 	proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse")
 }
 
-func init() { proto.RegisterFile("grpc_testing/test.proto", fileDescriptor_e1cda82041fed8bf) }
+func init() { proto.RegisterFile("stats/grpc_testing/test.proto", fileDescriptor_d7a50f7a8e9e8e09) }
 
-var fileDescriptor_e1cda82041fed8bf = []byte{
-	// 202 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4f, 0x2f, 0x2a, 0x48,
-	0x8e, 0x2f, 0x49, 0x2d, 0x2e, 0xc9, 0xcc, 0x4b, 0xd7, 0x07, 0xd1, 0x7a, 0x05, 0x45, 0xf9, 0x25,
-	0xf9, 0x42, 0x3c, 0x20, 0x09, 0x3d, 0xa8, 0x84, 0x92, 0x3c, 0x17, 0x6f, 0x70, 0x66, 0x6e, 0x41,
-	0x4e, 0x6a, 0x50, 0x6a, 0x61, 0x69, 0x6a, 0x71, 0x89, 0x10, 0x1f, 0x17, 0x53, 0x66, 0x8a, 0x04,
-	0x93, 0x02, 0xa3, 0x06, 0x6b, 0x10, 0x53, 0x66, 0x8a, 0x92, 0x02, 0x17, 0x1f, 0x4c, 0x41, 0x71,
-	0x41, 0x7e, 0x5e, 0x71, 0x2a, 0x54, 0x05, 0x33, 0x4c, 0x85, 0xd1, 0x09, 0x26, 0x2e, 0xee, 0x90,
-	0xd4, 0xe2, 0x92, 0xe0, 0xd4, 0xa2, 0xb2, 0xcc, 0xe4, 0x54, 0x21, 0x37, 0x2e, 0xce, 0xd0, 0xbc,
-	0xc4, 0xa2, 0x4a, 0xe7, 0xc4, 0x9c, 0x1c, 0x21, 0x69, 0x3d, 0x64, 0xeb, 0xf4, 0x50, 0xec, 0x92,
-	0x92, 0xc1, 0x2e, 0x09, 0xb5, 0xc7, 0x9f, 0x8b, 0xcf, 0xad, 0x34, 0x27, 0xc7, 0xa5, 0xb4, 0x20,
-	0x27, 0xb5, 0x82, 0x42, 0xc3, 0x34, 0x18, 0x0d, 0x18, 0x85, 0xfc, 0xb9, 0x04, 0x9c, 0x73, 0x32,
-	0x53, 0xf3, 0x4a, 0x82, 0x4b, 0x8a, 0x52, 0x13, 0x73, 0x29, 0x36, 0x12, 0x64, 0x20, 0xc8, 0xd3,
-	0xa9, 0x45, 0x54, 0x31, 0xd0, 0x80, 0x31, 0x89, 0x0d, 0x1c, 0x45, 0xc6, 0x80, 0x00, 0x00, 0x00,
-	0xff, 0xff, 0x4c, 0x43, 0x27, 0x67, 0xbd, 0x01, 0x00, 0x00,
+var fileDescriptor_d7a50f7a8e9e8e09 = []byte{
+	// 233 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x91, 0xcf, 0x4a, 0x03, 0x31,
+	0x10, 0xc6, 0x49, 0x44, 0xc1, 0x51, 0x17, 0xc9, 0x49, 0xfc, 0x83, 0xa5, 0xa7, 0x8a, 0x90, 0x2d,
+	0xfa, 0x06, 0x56, 0x7a, 0x2d, 0x74, 0xf5, 0xe2, 0x45, 0x62, 0x3b, 0x84, 0xc0, 0x34, 0x89, 0xc9,
+	0xac, 0xe8, 0x1b, 0xfa, 0x58, 0x92, 0xb2, 0x0b, 0x8a, 0xde, 0xd6, 0xd3, 0x1c, 0xbe, 0x8f, 0xdf,
+	0xc7, 0xf0, 0x83, 0x8b, 0xcc, 0x86, 0x73, 0x6d, 0x53, 0x5c, 0x3d, 0x33, 0x66, 0x76, 0xde, 0xd6,
+	0xe5, 0xea, 0x98, 0x02, 0x07, 0x75, 0x58, 0x02, 0xdd, 0x05, 0xe3, 0x4b, 0x38, 0x6a, 0xdc, 0x26,
+	0x12, 0x2e, 0xf1, 0xb5, 0xc5, 0xcc, 0xaa, 0x02, 0xe9, 0xd6, 0x27, 0x72, 0x24, 0x26, 0xbb, 0x4b,
+	0xe9, 0xd6, 0xe3, 0x11, 0x54, 0x7d, 0x21, 0xc7, 0xe0, 0x33, 0x76, 0x8d, 0x9d, 0xbe, 0x71, 0xf3,
+	0x29, 0xe1, 0xe0, 0x01, 0x33, 0x37, 0x98, 0xde, 0xdc, 0x0a, 0xd5, 0x1c, 0xf6, 0x1f, 0xbd, 0x49,
+	0x1f, 0x33, 0x43, 0xa4, 0xce, 0xf4, 0xf7, 0x39, 0xfd, 0x63, 0xeb, 0xf4, 0xfc, 0xef, 0xb0, 0xdb,
+	0x59, 0x40, 0x35, 0x6f, 0x89, 0xee, 0xdb, 0x48, 0xf8, 0x3e, 0x10, 0x36, 0x11, 0x53, 0xa1, 0x16,
+	0x70, 0x3c, 0x23, 0x87, 0x9e, 0x1b, 0x4e, 0x68, 0x36, 0x83, 0x91, 0x05, 0x58, 0x9e, 0xc6, 0xf4,
+	0x2f, 0xc0, 0xa9, 0xb8, 0xbb, 0x7e, 0xba, 0xb2, 0x21, 0x58, 0x42, 0x6d, 0x03, 0x19, 0x6f, 0x75,
+	0x48, 0x76, 0x2b, 0xb2, 0xfe, 0xed, 0xf4, 0x65, 0x6f, 0xeb, 0xf3, 0xf6, 0x2b, 0x00, 0x00, 0xff,
+	0xff, 0x7c, 0x26, 0xce, 0x3c, 0xf0, 0x01, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -419,5 +421,5 @@ var _TestService_serviceDesc = grpc.ServiceDesc{
 			ServerStreams: true,
 		},
 	},
-	Metadata: "grpc_testing/test.proto",
+	Metadata: "stats/grpc_testing/test.proto",
 }
diff --git a/stats/grpc_testing/test.proto b/stats/grpc_testing/test.proto
index b49a0d5a7c75..2716363260a6 100644
--- a/stats/grpc_testing/test.proto
+++ b/stats/grpc_testing/test.proto
@@ -14,6 +14,8 @@
 
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/stats/grpc_testing";
+
 package grpc.testing;
 
 message SimpleRequest {
diff --git a/stats/stats.go b/stats/stats.go
index a7970c79abe5..63e476ee7ff8 100644
--- a/stats/stats.go
+++ b/stats/stats.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc --go_out=plugins=grpc:. grpc_testing/test.proto
-
 // Package stats is for collecting and reporting various network and RPC stats.
 // This package is for monitoring purpose only. All fields are read-only.
 // All APIs are experimental.
diff --git a/stress/client/main.go b/stress/client/main.go
index 40aa99fec725..86d7539ae7f1 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -16,8 +16,6 @@
  *
  */
 
-//go:generate protoc -I ../grpc_testing --go_out=plugins=grpc:../grpc_testing ../grpc_testing/metrics.proto
-
 // client starts an interop client to do stress test and a metrics server to report qps.
 package main
 
diff --git a/stress/grpc_testing/metrics.pb.go b/stress/grpc_testing/metrics.pb.go
index bd5c1942c53b..16dfe4f1e85e 100644
--- a/stress/grpc_testing/metrics.pb.go
+++ b/stress/grpc_testing/metrics.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: metrics.proto
+// source: stress/grpc_testing/metrics.proto
 
 package grpc_testing
 
@@ -41,7 +41,7 @@ func (m *GaugeResponse) Reset()         { *m = GaugeResponse{} }
 func (m *GaugeResponse) String() string { return proto.CompactTextString(m) }
 func (*GaugeResponse) ProtoMessage()    {}
 func (*GaugeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6039342a2ba47b72, []int{0}
+	return fileDescriptor_028251bc41da09ab, []int{0}
 }
 
 func (m *GaugeResponse) XXX_Unmarshal(b []byte) error {
@@ -140,7 +140,7 @@ func (m *GaugeRequest) Reset()         { *m = GaugeRequest{} }
 func (m *GaugeRequest) String() string { return proto.CompactTextString(m) }
 func (*GaugeRequest) ProtoMessage()    {}
 func (*GaugeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6039342a2ba47b72, []int{1}
+	return fileDescriptor_028251bc41da09ab, []int{1}
 }
 
 func (m *GaugeRequest) XXX_Unmarshal(b []byte) error {
@@ -178,7 +178,7 @@ func (m *EmptyMessage) Reset()         { *m = EmptyMessage{} }
 func (m *EmptyMessage) String() string { return proto.CompactTextString(m) }
 func (*EmptyMessage) ProtoMessage()    {}
 func (*EmptyMessage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6039342a2ba47b72, []int{2}
+	return fileDescriptor_028251bc41da09ab, []int{2}
 }
 
 func (m *EmptyMessage) XXX_Unmarshal(b []byte) error {
@@ -205,26 +205,28 @@ func init() {
 	proto.RegisterType((*EmptyMessage)(nil), "grpc.testing.EmptyMessage")
 }
 
-func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) }
+func init() { proto.RegisterFile("stress/grpc_testing/metrics.proto", fileDescriptor_028251bc41da09ab) }
 
-var fileDescriptor_6039342a2ba47b72 = []byte{
-	// 256 bytes of a gzipped FileDescriptorProto
+var fileDescriptor_028251bc41da09ab = []byte{
+	// 288 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x3f, 0x4f, 0xc3, 0x30,
-	0x10, 0xc5, 0x6b, 0x5a, 0xfe, 0xf4, 0x70, 0x3b, 0x78, 0xaa, 0xca, 0x40, 0x14, 0x96, 0x4c, 0x11,
-	0x82, 0x4f, 0x00, 0x08, 0xa5, 0x0c, 0x5d, 0x82, 0xc4, 0x8a, 0xd2, 0x70, 0xb2, 0x22, 0x39, 0x71,
-	0xf0, 0x5d, 0x2a, 0xf1, 0x49, 0x58, 0xf9, 0xa8, 0xc8, 0x4e, 0x55, 0xa5, 0x08, 0x75, 0xb3, 0x7e,
-	0xf7, 0xfc, 0xfc, 0x9e, 0x0f, 0x66, 0x35, 0xb2, 0xab, 0x4a, 0x4a, 0x5b, 0x67, 0xd9, 0x2a, 0xa9,
-	0x5d, 0x5b, 0xa6, 0x8c, 0xc4, 0x55, 0xa3, 0xe3, 0x6f, 0x01, 0xb3, 0xac, 0xe8, 0x34, 0xe6, 0x48,
-	0xad, 0x6d, 0x08, 0x95, 0x82, 0x49, 0x53, 0xd4, 0xb8, 0x10, 0x91, 0x48, 0xa6, 0x79, 0x38, 0xab,
-	0x6b, 0x00, 0x63, 0x1b, 0xfd, 0xbe, 0x2d, 0x4c, 0x87, 0x8b, 0x93, 0x48, 0x24, 0xe3, 0xd5, 0x28,
-	0x9f, 0x7a, 0xf6, 0xe6, 0x91, 0xba, 0x01, 0xf9, 0x61, 0xbb, 0x8d, 0xc1, 0x9d, 0x64, 0x1c, 0x89,
-	0x44, 0xac, 0x46, 0xf9, 0x65, 0x4f, 0xf7, 0x22, 0x62, 0x57, 0xed, 0x7d, 0x26, 0xfe, 0x05, 0x2f,
-	0xea, 0x69, 0x10, 0x3d, 0x9e, 0xc3, 0x69, 0x98, 0xc6, 0x31, 0xc8, 0x5d, 0xb0, 0xcf, 0x0e, 0x89,
-	0xff, 0xcb, 0x15, 0xcf, 0x41, 0x3e, 0xd7, 0x2d, 0x7f, 0xad, 0x91, 0xa8, 0xd0, 0x78, 0xf7, 0x23,
-	0x60, 0xbe, 0xee, 0xdb, 0xbe, 0xa2, 0xdb, 0x56, 0x25, 0xaa, 0x17, 0x90, 0x19, 0xf2, 0x83, 0x31,
-	0xc1, 0x8c, 0xd4, 0x32, 0x1d, 0xf6, 0x4f, 0x87, 0xd7, 0x97, 0x57, 0x87, 0xb3, 0x83, 0x7f, 0xb9,
-	0x15, 0xea, 0x09, 0x2e, 0x32, 0xe4, 0x40, 0xff, 0xda, 0x0c, 0x93, 0x1e, 0xb5, 0xd9, 0x9c, 0x85,
-	0x2d, 0xdc, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x5e, 0x7d, 0xb2, 0xc9, 0x96, 0x01, 0x00, 0x00,
+	0x10, 0xc5, 0x6b, 0x5a, 0xfe, 0xf4, 0x08, 0x1d, 0x3c, 0x55, 0x65, 0x20, 0x84, 0x25, 0x42, 0xc8,
+	0x41, 0xf0, 0x09, 0x28, 0x42, 0x29, 0x43, 0x97, 0x20, 0x31, 0xb0, 0x54, 0x69, 0x38, 0x59, 0x91,
+	0x9c, 0x38, 0xf8, 0x9c, 0x4a, 0x7c, 0x12, 0x56, 0x3e, 0x2a, 0x8a, 0x13, 0x55, 0x29, 0xaa, 0xba,
+	0x59, 0xbf, 0xf7, 0xfc, 0x7c, 0xe7, 0x07, 0xd7, 0x64, 0x0d, 0x12, 0x45, 0xd2, 0x54, 0xd9, 0xca,
+	0x22, 0xd9, 0xbc, 0x94, 0x51, 0x81, 0xd6, 0xe4, 0x19, 0x89, 0xca, 0x68, 0xab, 0xb9, 0xd7, 0x68,
+	0xa2, 0xd3, 0x82, 0x1f, 0x06, 0x17, 0x71, 0x5a, 0x4b, 0x4c, 0x90, 0x2a, 0x5d, 0x12, 0x72, 0x0e,
+	0xa3, 0x32, 0x2d, 0x70, 0xca, 0x7c, 0x16, 0x8e, 0x13, 0x77, 0xe6, 0x57, 0x00, 0x4a, 0x97, 0x72,
+	0xb5, 0x49, 0x55, 0x8d, 0xd3, 0x23, 0x9f, 0x85, 0xc3, 0xc5, 0x20, 0x19, 0x37, 0xec, 0xbd, 0x41,
+	0xfc, 0x06, 0xbc, 0x4f, 0x5d, 0xaf, 0x15, 0x76, 0x96, 0xa1, 0xcf, 0x42, 0xb6, 0x18, 0x24, 0xe7,
+	0x2d, 0xdd, 0x9a, 0xc8, 0x9a, 0x7c, 0x9b, 0x33, 0x6a, 0x5e, 0x68, 0x4c, 0x2d, 0x75, 0xa6, 0xf9,
+	0x29, 0x1c, 0x3b, 0x35, 0x08, 0xc0, 0xeb, 0x06, 0xfb, 0xaa, 0x91, 0xec, 0xbe, 0xb9, 0x82, 0x09,
+	0x78, 0x2f, 0x45, 0x65, 0xbf, 0x97, 0x48, 0x94, 0x4a, 0x7c, 0xf8, 0x65, 0x30, 0x59, 0xb6, 0xdb,
+	0xbe, 0xa1, 0xd9, 0xe4, 0x19, 0xf2, 0x57, 0xf0, 0x62, 0xb4, 0x4f, 0x4a, 0xb9, 0x30, 0xe2, 0x33,
+	0xd1, 0xdf, 0x5f, 0xf4, 0xaf, 0xcf, 0x2e, 0x77, 0xb5, 0x9d, 0x7f, 0xb9, 0x67, 0xfc, 0x19, 0xce,
+	0x62, 0xb4, 0x8e, 0xfe, 0x8f, 0xe9, 0x4f, 0x7a, 0x30, 0x66, 0x7e, 0xf7, 0x71, 0x2b, 0xb5, 0x96,
+	0x0a, 0x85, 0xd4, 0x2a, 0x2d, 0xa5, 0xd0, 0x46, 0xba, 0xba, 0xa2, 0x3d, 0xd5, 0xad, 0x4f, 0x5c,
+	0x67, 0x8f, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x8c, 0x62, 0x73, 0xd8, 0x01, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -374,5 +376,5 @@ var _MetricsService_serviceDesc = grpc.ServiceDesc{
 			ServerStreams: true,
 		},
 	},
-	Metadata: "metrics.proto",
+	Metadata: "stress/grpc_testing/metrics.proto",
 }
diff --git a/stress/grpc_testing/metrics.proto b/stress/grpc_testing/metrics.proto
index 695040064317..0197d4d70e53 100644
--- a/stress/grpc_testing/metrics.proto
+++ b/stress/grpc_testing/metrics.proto
@@ -20,6 +20,8 @@
 // service.
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/stress/grpc_testing";
+
 package grpc.testing;
 
 // Response message containing the gauge name and value
diff --git a/test/codec_perf/perf.pb.go b/test/codec_perf/perf.pb.go
index 43737fdd6672..b98764ade4ec 100644
--- a/test/codec_perf/perf.pb.go
+++ b/test/codec_perf/perf.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: codec_perf/perf.proto
+// source: test/codec_perf/perf.proto
 
 package codec_perf
 
@@ -33,7 +33,7 @@ func (m *Buffer) Reset()         { *m = Buffer{} }
 func (m *Buffer) String() string { return proto.CompactTextString(m) }
 func (*Buffer) ProtoMessage()    {}
 func (*Buffer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_afad72ea7772fe3a, []int{0}
+	return fileDescriptor_a913550de912e506, []int{0}
 }
 
 func (m *Buffer) XXX_Unmarshal(b []byte) error {
@@ -65,14 +65,16 @@ func init() {
 	proto.RegisterType((*Buffer)(nil), "codec.perf.Buffer")
 }
 
-func init() { proto.RegisterFile("codec_perf/perf.proto", fileDescriptor_afad72ea7772fe3a) }
+func init() { proto.RegisterFile("test/codec_perf/perf.proto", fileDescriptor_a913550de912e506) }
 
-var fileDescriptor_afad72ea7772fe3a = []byte{
-	// 83 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4d, 0xce, 0x4f, 0x49,
-	0x4d, 0x8e, 0x2f, 0x48, 0x2d, 0x4a, 0xd3, 0x07, 0x11, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42,
-	0x5c, 0x60, 0x61, 0x3d, 0x90, 0x88, 0x92, 0x0c, 0x17, 0x9b, 0x53, 0x69, 0x5a, 0x5a, 0x6a, 0x91,
-	0x90, 0x10, 0x17, 0x4b, 0x52, 0x7e, 0x4a, 0xa5, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x4f, 0x10, 0x98,
-	0x9d, 0xc4, 0x06, 0xd6, 0x60, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x5f, 0x4f, 0x3c, 0x49,
-	0x00, 0x00, 0x00,
+var fileDescriptor_a913550de912e506 = []byte{
+	// 118 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2a, 0x49, 0x2d, 0x2e,
+	0xd1, 0x4f, 0xce, 0x4f, 0x49, 0x4d, 0x8e, 0x2f, 0x48, 0x2d, 0x4a, 0xd3, 0x07, 0x11, 0x7a, 0x05,
+	0x45, 0xf9, 0x25, 0xf9, 0x42, 0x5c, 0x60, 0x61, 0x3d, 0x90, 0x88, 0x92, 0x0c, 0x17, 0x9b, 0x53,
+	0x69, 0x5a, 0x5a, 0x6a, 0x91, 0x90, 0x10, 0x17, 0x4b, 0x52, 0x7e, 0x4a, 0xa5, 0x04, 0xa3, 0x02,
+	0xa3, 0x06, 0x4f, 0x10, 0x98, 0xed, 0xa4, 0x11, 0xa5, 0x96, 0x9e, 0x9f, 0x9f, 0x9e, 0x93, 0xaa,
+	0x97, 0x9e, 0x9f, 0x93, 0x98, 0x97, 0xae, 0x97, 0x5f, 0x94, 0xae, 0x9f, 0x5e, 0x54, 0x90, 0xac,
+	0x8f, 0x66, 0x7c, 0x12, 0x1b, 0xd8, 0x68, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xdb,
+	0x49, 0x8b, 0x78, 0x00, 0x00, 0x00,
 }
diff --git a/test/codec_perf/perf.proto b/test/codec_perf/perf.proto
index 594c6f011f92..7a678d1ca91a 100644
--- a/test/codec_perf/perf.proto
+++ b/test/codec_perf/perf.proto
@@ -16,6 +16,8 @@
 // reasons of import cycles.
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/test/codec_perf";
+
 package codec.perf;
 
 // Buffer is a message that contains a body of bytes that is used to exercise
diff --git a/test/end2end_test.go b/test/end2end_test.go
index a32e93410a98..fb95f4a2fee3 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -16,9 +16,6 @@
  *
  */
 
-//go:generate protoc --go_out=plugins=grpc:. codec_perf/perf.proto
-//go:generate protoc --go_out=plugins=grpc:. grpc_testing/test.proto
-
 package test
 
 import (
diff --git a/test/grpc_testing/test.pb.go b/test/grpc_testing/test.pb.go
index 78e9ddf54822..2b4e59cd20b6 100644
--- a/test/grpc_testing/test.pb.go
+++ b/test/grpc_testing/test.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc_testing/test.proto
+// source: test/grpc_testing/test.proto
 
 package grpc_testing
 
@@ -53,7 +53,7 @@ func (x PayloadType) String() string {
 }
 
 func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{0}
+	return fileDescriptor_139516ae706ad4b7, []int{0}
 }
 
 type Empty struct {
@@ -66,7 +66,7 @@ func (m *Empty) Reset()         { *m = Empty{} }
 func (m *Empty) String() string { return proto.CompactTextString(m) }
 func (*Empty) ProtoMessage()    {}
 func (*Empty) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{0}
+	return fileDescriptor_139516ae706ad4b7, []int{0}
 }
 
 func (m *Empty) XXX_Unmarshal(b []byte) error {
@@ -102,7 +102,7 @@ func (m *Payload) Reset()         { *m = Payload{} }
 func (m *Payload) String() string { return proto.CompactTextString(m) }
 func (*Payload) ProtoMessage()    {}
 func (*Payload) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{1}
+	return fileDescriptor_139516ae706ad4b7, []int{1}
 }
 
 func (m *Payload) XXX_Unmarshal(b []byte) error {
@@ -160,7 +160,7 @@ func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
 func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
 func (*SimpleRequest) ProtoMessage()    {}
 func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{2}
+	return fileDescriptor_139516ae706ad4b7, []int{2}
 }
 
 func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
@@ -234,7 +234,7 @@ func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
 func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
 func (*SimpleResponse) ProtoMessage()    {}
 func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{3}
+	return fileDescriptor_139516ae706ad4b7, []int{3}
 }
 
 func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
@@ -289,7 +289,7 @@ func (m *StreamingInputCallRequest) Reset()         { *m = StreamingInputCallReq
 func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) }
 func (*StreamingInputCallRequest) ProtoMessage()    {}
 func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{4}
+	return fileDescriptor_139516ae706ad4b7, []int{4}
 }
 
 func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error {
@@ -330,7 +330,7 @@ func (m *StreamingInputCallResponse) Reset()         { *m = StreamingInputCallRe
 func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) }
 func (*StreamingInputCallResponse) ProtoMessage()    {}
 func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{5}
+	return fileDescriptor_139516ae706ad4b7, []int{5}
 }
 
 func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error {
@@ -375,7 +375,7 @@ func (m *ResponseParameters) Reset()         { *m = ResponseParameters{} }
 func (m *ResponseParameters) String() string { return proto.CompactTextString(m) }
 func (*ResponseParameters) ProtoMessage()    {}
 func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{6}
+	return fileDescriptor_139516ae706ad4b7, []int{6}
 }
 
 func (m *ResponseParameters) XXX_Unmarshal(b []byte) error {
@@ -430,7 +430,7 @@ func (m *StreamingOutputCallRequest) Reset()         { *m = StreamingOutputCallR
 func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) }
 func (*StreamingOutputCallRequest) ProtoMessage()    {}
 func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{7}
+	return fileDescriptor_139516ae706ad4b7, []int{7}
 }
 
 func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error {
@@ -485,7 +485,7 @@ func (m *StreamingOutputCallResponse) Reset()         { *m = StreamingOutputCall
 func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) }
 func (*StreamingOutputCallResponse) ProtoMessage()    {}
 func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1cda82041fed8bf, []int{8}
+	return fileDescriptor_139516ae706ad4b7, []int{8}
 }
 
 func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error {
@@ -526,47 +526,49 @@ func init() {
 	proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse")
 }
 
-func init() { proto.RegisterFile("grpc_testing/test.proto", fileDescriptor_e1cda82041fed8bf) }
-
-var fileDescriptor_e1cda82041fed8bf = []byte{
-	// 587 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xdb, 0x6e, 0xd3, 0x40,
-	0x10, 0x65, 0xdb, 0xf4, 0x36, 0x49, 0xad, 0x68, 0xab, 0xaa, 0xae, 0x8b, 0x84, 0x65, 0x1e, 0x30,
-	0x48, 0xa4, 0x28, 0x08, 0x1e, 0x41, 0xa5, 0x17, 0x51, 0x29, 0x4d, 0x82, 0x9d, 0x3c, 0x47, 0xdb,
-	0x64, 0x6b, 0x2c, 0x39, 0xf6, 0xb2, 0x5e, 0x57, 0xa4, 0x0f, 0xfc, 0x18, 0x3f, 0xc3, 0x47, 0xf0,
-	0x01, 0x68, 0xd7, 0x76, 0xe2, 0x24, 0xae, 0x48, 0x41, 0xf0, 0x14, 0x7b, 0xe6, 0xcc, 0x99, 0x73,
-	0x3c, 0xb3, 0x1b, 0x38, 0xf0, 0x38, 0x1b, 0x0e, 0x04, 0x8d, 0x85, 0x1f, 0x7a, 0xc7, 0xf2, 0xb7,
-	0xc1, 0x78, 0x24, 0x22, 0x5c, 0x93, 0x89, 0x46, 0x96, 0xb0, 0xb6, 0x60, 0xe3, 0x7c, 0xcc, 0xc4,
-	0xc4, 0x6a, 0xc1, 0x56, 0x97, 0x4c, 0x82, 0x88, 0x8c, 0xf0, 0x4b, 0xa8, 0x88, 0x09, 0xa3, 0x3a,
-	0x32, 0x91, 0xad, 0x35, 0x0f, 0x1b, 0xc5, 0x82, 0x46, 0x06, 0xea, 0x4d, 0x18, 0x75, 0x14, 0x0c,
-	0x63, 0xa8, 0x5c, 0x47, 0xa3, 0x89, 0xbe, 0x66, 0x22, 0xbb, 0xe6, 0xa8, 0x67, 0xeb, 0x27, 0x82,
-	0x5d, 0xd7, 0x1f, 0xb3, 0x80, 0x3a, 0xf4, 0x4b, 0x42, 0x63, 0x81, 0xdf, 0xc1, 0x2e, 0xa7, 0x31,
-	0x8b, 0xc2, 0x98, 0x0e, 0x56, 0x63, 0xaf, 0xe5, 0x78, 0xf9, 0x86, 0x9f, 0x16, 0xea, 0x63, 0xff,
-	0x8e, 0xaa, 0x76, 0x1b, 0x33, 0x90, 0xeb, 0xdf, 0x51, 0x7c, 0x0c, 0x5b, 0x2c, 0x65, 0xd0, 0xd7,
-	0x4d, 0x64, 0x57, 0x9b, 0xfb, 0xa5, 0xf4, 0x4e, 0x8e, 0x92, 0xac, 0x37, 0x7e, 0x10, 0x0c, 0x92,
-	0x98, 0xf2, 0x90, 0x8c, 0xa9, 0x5e, 0x31, 0x91, 0xbd, 0xed, 0xd4, 0x64, 0xb0, 0x9f, 0xc5, 0xb0,
-	0x0d, 0x75, 0x05, 0x8a, 0x48, 0x22, 0x3e, 0x0f, 0xe2, 0x61, 0xc4, 0xa8, 0xbe, 0xa1, 0x70, 0x9a,
-	0x8c, 0x77, 0x64, 0xd8, 0x95, 0x51, 0xeb, 0x1b, 0x68, 0xb9, 0xeb, 0x54, 0x55, 0x51, 0x11, 0x5a,
-	0x49, 0x91, 0x01, 0xdb, 0x53, 0x31, 0xd2, 0xe2, 0x8e, 0x33, 0x7d, 0xc7, 0x4f, 0xa0, 0x5a, 0xd4,
-	0xb0, 0xae, 0xd2, 0x10, 0xcd, 0xfa, 0xb7, 0xe0, 0xd0, 0x15, 0x9c, 0x92, 0xb1, 0x1f, 0x7a, 0x97,
-	0x21, 0x4b, 0xc4, 0x29, 0x09, 0x82, 0x7c, 0x02, 0x0f, 0x95, 0x62, 0xf5, 0xc0, 0x28, 0x63, 0xcb,
-	0x9c, 0xbd, 0x85, 0x03, 0xe2, 0x79, 0x9c, 0x7a, 0x44, 0xd0, 0xd1, 0x20, 0xab, 0x49, 0x47, 0x83,
-	0xd4, 0x68, 0xf6, 0x67, 0xe9, 0x8c, 0x5a, 0xce, 0xc8, 0xba, 0x04, 0x9c, 0x73, 0x74, 0x09, 0x27,
-	0x63, 0x2a, 0x28, 0x8f, 0xe5, 0x12, 0x15, 0x4a, 0xd5, 0xb3, 0xb4, 0xeb, 0x87, 0x82, 0xf2, 0x5b,
-	0x22, 0x07, 0x94, 0x0d, 0x1c, 0xf2, 0x50, 0x3f, 0xb6, 0x7e, 0xa0, 0x82, 0xc2, 0x4e, 0x22, 0x16,
-	0x0c, 0xff, 0xed, 0xca, 0x7d, 0x82, 0xbd, 0x69, 0x3d, 0x9b, 0x4a, 0xd5, 0xd7, 0xcc, 0x75, 0xbb,
-	0xda, 0x34, 0xe7, 0x59, 0x96, 0x2d, 0x39, 0x98, 0x2f, 0xdb, 0x7c, 0xe8, 0x82, 0x5a, 0x6d, 0x38,
-	0x2a, 0x75, 0xf8, 0x87, 0xeb, 0xf5, 0xe2, 0x3d, 0x54, 0x0b, 0x86, 0x71, 0x1d, 0x6a, 0xa7, 0x9d,
-	0xab, 0xae, 0x73, 0xee, 0xba, 0x27, 0x1f, 0x5a, 0xe7, 0xf5, 0x47, 0x18, 0x83, 0xd6, 0x6f, 0xcf,
-	0xc5, 0x10, 0x06, 0xd8, 0x74, 0x4e, 0xda, 0x67, 0x9d, 0xab, 0xfa, 0x5a, 0xf3, 0x7b, 0x05, 0xaa,
-	0x3d, 0x1a, 0x0b, 0x97, 0xf2, 0x5b, 0x7f, 0x48, 0xf1, 0x1b, 0xd8, 0x51, 0x17, 0x88, 0x94, 0x85,
-	0xf7, 0xe6, 0xbb, 0xab, 0x84, 0x51, 0x16, 0xc4, 0x17, 0xb0, 0xd3, 0x0f, 0x09, 0x4f, 0xcb, 0x8e,
-	0xe6, 0x11, 0x73, 0x17, 0x87, 0xf1, 0xb8, 0x3c, 0x99, 0x7d, 0x80, 0x00, 0xf6, 0x4a, 0xbe, 0x0f,
-	0xb6, 0x17, 0x8a, 0xee, 0x5d, 0x12, 0xe3, 0xf9, 0x0a, 0xc8, 0xb4, 0xd7, 0x2b, 0x84, 0x7d, 0xc0,
-	0xcb, 0x27, 0x02, 0x3f, 0xbb, 0x87, 0x62, 0xf1, 0x04, 0x1a, 0xf6, 0xef, 0x81, 0x69, 0x2b, 0x5b,
-	0xb6, 0xd2, 0x2e, 0x92, 0x20, 0x38, 0x4b, 0x58, 0x40, 0xbf, 0xfe, 0x33, 0x4f, 0x36, 0x52, 0xae,
-	0xb4, 0x8f, 0x24, 0xb8, 0xf9, 0x0f, 0xad, 0xae, 0x37, 0xd5, 0x7f, 0xd0, 0xeb, 0x5f, 0x01, 0x00,
-	0x00, 0xff, 0xff, 0x07, 0xc7, 0x76, 0x69, 0x9e, 0x06, 0x00, 0x00,
+func init() { proto.RegisterFile("test/grpc_testing/test.proto", fileDescriptor_139516ae706ad4b7) }
+
+var fileDescriptor_139516ae706ad4b7 = []byte{
+	// 615 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xdd, 0x6e, 0xd3, 0x4c,
+	0x10, 0xfd, 0xb6, 0x4d, 0xff, 0x26, 0x69, 0x14, 0x6d, 0x55, 0x7d, 0x69, 0x5a, 0x89, 0xc8, 0x5c,
+	0x60, 0x2a, 0x91, 0xa0, 0x20, 0xb8, 0x04, 0xf5, 0x57, 0x54, 0x6a, 0x9b, 0x62, 0x37, 0x37, 0xdc,
+	0x44, 0xdb, 0x76, 0x6a, 0x2c, 0x6d, 0xbc, 0xcb, 0x7a, 0x5d, 0x91, 0x5e, 0xf0, 0x62, 0xbc, 0x0c,
+	0x0f, 0xc1, 0x03, 0xa0, 0xdd, 0xd8, 0xa9, 0xd3, 0xb8, 0x22, 0x05, 0xc1, 0x55, 0x36, 0x33, 0x67,
+	0xce, 0x9c, 0xe3, 0x19, 0x7b, 0x61, 0x4b, 0x63, 0xac, 0xdb, 0x81, 0x92, 0x97, 0x7d, 0x73, 0x0a,
+	0xa3, 0xa0, 0x6d, 0x7e, 0x5b, 0x52, 0x09, 0x2d, 0x68, 0xc5, 0x24, 0x5a, 0x69, 0xc2, 0x59, 0x82,
+	0x85, 0x83, 0x81, 0xd4, 0x43, 0xe7, 0x18, 0x96, 0xce, 0xd8, 0x90, 0x0b, 0x76, 0x45, 0x5f, 0x40,
+	0x49, 0x0f, 0x25, 0xd6, 0x49, 0x93, 0xb8, 0xd5, 0xce, 0x46, 0x2b, 0x5f, 0xd0, 0x4a, 0x41, 0xe7,
+	0x43, 0x89, 0x9e, 0x85, 0x51, 0x0a, 0xa5, 0x0b, 0x71, 0x35, 0xac, 0xcf, 0x35, 0x89, 0x5b, 0xf1,
+	0xec, 0xd9, 0xf9, 0x41, 0x60, 0xd5, 0x0f, 0x07, 0x92, 0xa3, 0x87, 0x9f, 0x13, 0x8c, 0x35, 0x7d,
+	0x0b, 0xab, 0x0a, 0x63, 0x29, 0xa2, 0x18, 0xfb, 0xb3, 0xb1, 0x57, 0x32, 0xbc, 0xf9, 0x47, 0x9f,
+	0xe6, 0xea, 0xe3, 0xf0, 0x16, 0x6d, 0xbb, 0x85, 0x3b, 0x90, 0x1f, 0xde, 0x22, 0x6d, 0xc3, 0x92,
+	0x1c, 0x31, 0xd4, 0xe7, 0x9b, 0xc4, 0x2d, 0x77, 0xd6, 0x0b, 0xe9, 0xbd, 0x0c, 0x65, 0x58, 0xaf,
+	0x43, 0xce, 0xfb, 0x49, 0x8c, 0x2a, 0x62, 0x03, 0xac, 0x97, 0x9a, 0xc4, 0x5d, 0xf6, 0x2a, 0x26,
+	0xd8, 0x4b, 0x63, 0xd4, 0x85, 0x9a, 0x05, 0x09, 0x96, 0xe8, 0x4f, 0xfd, 0xf8, 0x52, 0x48, 0xac,
+	0x2f, 0x58, 0x5c, 0xd5, 0xc4, 0xbb, 0x26, 0xec, 0x9b, 0xa8, 0xf3, 0x15, 0xaa, 0x99, 0xeb, 0x91,
+	0xaa, 0xbc, 0x22, 0x32, 0x93, 0xa2, 0x06, 0x2c, 0x8f, 0xc5, 0x18, 0x8b, 0x2b, 0xde, 0xf8, 0x3f,
+	0x7d, 0x02, 0xe5, 0xbc, 0x86, 0x79, 0x9b, 0x06, 0x71, 0xd7, 0xff, 0x18, 0x36, 0x7c, 0xad, 0x90,
+	0x0d, 0xc2, 0x28, 0x38, 0x8a, 0x64, 0xa2, 0xf7, 0x18, 0xe7, 0xd9, 0x04, 0x1e, 0x2b, 0xc5, 0x39,
+	0x87, 0x46, 0x11, 0x5b, 0xea, 0xec, 0x0d, 0xfc, 0xcf, 0x82, 0x40, 0x61, 0xc0, 0x34, 0x5e, 0xf5,
+	0xd3, 0x9a, 0xd1, 0x68, 0x88, 0x1d, 0xcd, 0xfa, 0x5d, 0x3a, 0xa5, 0x36, 0x33, 0x72, 0x8e, 0x80,
+	0x66, 0x1c, 0x67, 0x4c, 0xb1, 0x01, 0x6a, 0x54, 0xb1, 0x59, 0xa2, 0x5c, 0xa9, 0x3d, 0x1b, 0xbb,
+	0x61, 0xa4, 0x51, 0xdd, 0x30, 0x33, 0xa0, 0x74, 0xe0, 0x90, 0x85, 0x7a, 0xb1, 0xf3, 0x9d, 0xe4,
+	0x14, 0x76, 0x13, 0x7d, 0xcf, 0xf0, 0x9f, 0xae, 0xdc, 0x07, 0x58, 0x1b, 0xd7, 0xcb, 0xb1, 0xd4,
+	0xfa, 0x5c, 0x73, 0xde, 0x2d, 0x77, 0x9a, 0x93, 0x2c, 0xd3, 0x96, 0x3c, 0xaa, 0xa6, 0x6d, 0x3e,
+	0x76, 0x41, 0x9d, 0x53, 0xd8, 0x2c, 0x74, 0xf8, 0x9b, 0xeb, 0xb5, 0xfd, 0x0e, 0xca, 0x39, 0xc3,
+	0xb4, 0x06, 0x95, 0xbd, 0xee, 0xc9, 0x99, 0x77, 0xe0, 0xfb, 0x3b, 0xbb, 0xc7, 0x07, 0xb5, 0xff,
+	0x28, 0x85, 0x6a, 0xef, 0x74, 0x22, 0x46, 0x28, 0xc0, 0xa2, 0xb7, 0x73, 0xba, 0xdf, 0x3d, 0xa9,
+	0xcd, 0x75, 0xbe, 0x95, 0xa0, 0x7c, 0x8e, 0xb1, 0xf6, 0x51, 0xdd, 0x84, 0x97, 0x48, 0x5f, 0xc3,
+	0x8a, 0xfd, 0x80, 0x18, 0x59, 0x74, 0x6d, 0xb2, 0xbb, 0x4d, 0x34, 0x8a, 0x82, 0xf4, 0x10, 0x56,
+	0x7a, 0x11, 0x53, 0xa3, 0xb2, 0xcd, 0x49, 0xc4, 0xc4, 0x87, 0xa3, 0xb1, 0x55, 0x9c, 0x4c, 0x1f,
+	0x00, 0x87, 0xb5, 0x82, 0xe7, 0x43, 0xdd, 0x7b, 0x45, 0x0f, 0x2e, 0x49, 0xe3, 0xf9, 0x0c, 0xc8,
+	0x51, 0xaf, 0x97, 0x84, 0x86, 0x40, 0xa7, 0xdf, 0x08, 0xfa, 0xec, 0x01, 0x8a, 0xfb, 0x6f, 0x60,
+	0xc3, 0xfd, 0x35, 0x70, 0xd4, 0xca, 0x35, 0xad, 0xaa, 0x87, 0x09, 0xe7, 0xfb, 0x89, 0xe4, 0xf8,
+	0xe5, 0xaf, 0x79, 0x72, 0x89, 0x75, 0x55, 0x7d, 0xcf, 0xf8, 0xf5, 0x3f, 0x68, 0xb5, 0xbb, 0xfd,
+	0xd1, 0x0d, 0x84, 0x08, 0x38, 0xb6, 0x02, 0xc1, 0x59, 0x14, 0xb4, 0x84, 0x0a, 0xec, 0x4d, 0xd5,
+	0x9e, 0xba, 0xb3, 0x2e, 0x16, 0xed, 0x7d, 0xf5, 0xea, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e,
+	0x50, 0x51, 0x5b, 0xcf, 0x06, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -980,5 +982,5 @@ var _TestService_serviceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "grpc_testing/test.proto",
+	Metadata: "test/grpc_testing/test.proto",
 }
diff --git a/test/grpc_testing/test.proto b/test/grpc_testing/test.proto
index 6f62f3a7a6fa..0c6650401d59 100644
--- a/test/grpc_testing/test.proto
+++ b/test/grpc_testing/test.proto
@@ -16,6 +16,8 @@
 // of unary/streaming requests/responses.
 syntax = "proto3";
 
+option go_package = "google.golang.org/grpc/test/grpc_testing";
+
 package grpc.testing;
 
 message Empty {}
diff --git a/vet.sh b/vet.sh
index 9d20e46f0c36..84b18859eb9b 100755
--- a/vet.sh
+++ b/vet.sh
@@ -39,8 +39,7 @@ if [[ "$1" = "-install" ]]; then
       golang.org/x/lint/golint \
       golang.org/x/tools/cmd/goimports \
       honnef.co/go/tools/cmd/staticcheck \
-      github.com/client9/misspell/cmd/misspell \
-      github.com/golang/protobuf/protoc-gen-go
+      github.com/client9/misspell/cmd/misspell
     popd
   else
     # Ye olde `go get` incantation.
@@ -50,8 +49,7 @@ if [[ "$1" = "-install" ]]; then
       golang.org/x/lint/golint \
       golang.org/x/tools/cmd/goimports \
       honnef.co/go/tools/cmd/staticcheck \
-      github.com/client9/misspell/cmd/misspell \
-      github.com/golang/protobuf/protoc-gen-go
+      github.com/client9/misspell/cmd/misspell
   fi
   if [[ -z "${VET_SKIP_PROTO}" ]]; then
     if [[ "${TRAVIS}" = "true" ]]; then

From a3cc4f613d6d53278c2b5decf5d0987d0d3462cd Mon Sep 17 00:00:00 2001
From: Eric Gribkoff <ericgribkoff@google.com>
Date: Fri, 24 Apr 2020 12:11:27 -0700
Subject: [PATCH 045/481] interop: add --fail_on_failed_rpc xds client flag
 (#3567)

---
 interop/xds/client/client.go | 18 +++++++++++-------
 test/kokoro/xds.sh           |  3 ++-
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 0164cb4bfeaf..77b4563b7bef 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -47,12 +47,13 @@ type statsWatcher struct {
 }
 
 var (
-	numChannels   = flag.Int("num_channels", 1, "Num of channels")
-	printResponse = flag.Bool("print_response", false, "Write RPC response to stdout")
-	qps           = flag.Int("qps", 1, "QPS per channel")
-	rpcTimeout    = flag.Duration("rpc_timeout", 10*time.Second, "Per RPC timeout")
-	server        = flag.String("server", "localhost:8080", "Address of server to connect to")
-	statsPort     = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service")
+	failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail")
+	numChannels     = flag.Int("num_channels", 1, "Num of channels")
+	printResponse   = flag.Bool("print_response", false, "Write RPC response to stdout")
+	qps             = flag.Int("qps", 1, "QPS per channel")
+	rpcTimeout      = flag.Duration("rpc_timeout", 10*time.Second, "Per RPC timeout")
+	server          = flag.String("server", "localhost:8080", "Address of server to connect to")
+	statsPort       = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service")
 
 	mu               sync.Mutex
 	currentRequestID int32
@@ -123,7 +124,7 @@ func main() {
 
 	clients := make([]testpb.TestServiceClient, *numChannels)
 	for i := 0; i < *numChannels; i++ {
-		conn, err := grpc.DialContext(context.Background(), *server, grpc.WithInsecure(), grpc.WithBlock())
+		conn, err := grpc.DialContext(context.Background(), *server, grpc.WithInsecure())
 		if err != nil {
 			grpclog.Fatalf("Fail to dial: %v", err)
 		}
@@ -161,6 +162,9 @@ func sendRPCs(clients []testpb.TestServiceClient, ticker *time.Ticker) {
 				watcher.c <- r
 			}
 
+			if err != nil && *failOnFailedRPC {
+				grpclog.Fatalf("RPC failed: %v", err)
+			}
 			if success && *printResponse {
 				fmt.Printf("Greeting: Hello world, this is %s, from %v\n", r.GetHostname(), p.Addr)
 			}
diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index 788c7c9f7e3a..06d826381084 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -30,4 +30,5 @@ GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
     --client_cmd="grpc-go/interop/xds/client/client \
       --server=xds-experimental:///{server_uri} \
       --stats_port={stats_port} \
-      --qps={qps}"
+      --qps={qps} \
+      {fail_on_failed_rpc}"

From b2df44eac8b1ffedf1ec721a1b7d706293e7abd7 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 27 Apr 2020 20:55:58 -0700
Subject: [PATCH 046/481] Pass weights to wrr balancer through attributes.
 (#3530)

---
 attributes/attributes.go                      |  6 ++
 .../weightedroundrobin/weightedroundrobin.go  | 30 ++++++-
 .../weightedwoundrobin_test.go                | 82 +++++++++++++++++++
 xds/internal/balancer/edsbalancer/eds_impl.go | 13 ++-
 4 files changed, 126 insertions(+), 5 deletions(-)
 create mode 100644 balancer/weightedroundrobin/weightedwoundrobin_test.go

diff --git a/attributes/attributes.go b/attributes/attributes.go
index 68ffc6201375..ee5c51e6cdb0 100644
--- a/attributes/attributes.go
+++ b/attributes/attributes.go
@@ -50,6 +50,9 @@ func New(kvs ...interface{}) *Attributes {
 // times, the last value overwrites all previous values for that key.  To
 // remove an existing key, use a nil value.
 func (a *Attributes) WithValues(kvs ...interface{}) *Attributes {
+	if a == nil {
+		return New(kvs...)
+	}
 	if len(kvs)%2 != 0 {
 		panic(fmt.Sprintf("attributes.New called with unexpected input: len(kvs) = %v", len(kvs)))
 	}
@@ -66,5 +69,8 @@ func (a *Attributes) WithValues(kvs ...interface{}) *Attributes {
 // Value returns the value associated with these attributes for key, or nil if
 // no value is associated with key.
 func (a *Attributes) Value(key interface{}) interface{} {
+	if a == nil {
+		return nil
+	}
 	return a.m[key]
 }
diff --git a/balancer/weightedroundrobin/weightedroundrobin.go b/balancer/weightedroundrobin/weightedroundrobin.go
index 739e64d6229b..d232491aef28 100644
--- a/balancer/weightedroundrobin/weightedroundrobin.go
+++ b/balancer/weightedroundrobin/weightedroundrobin.go
@@ -19,11 +19,37 @@
 // Package weightedroundrobin defines a weighted roundrobin balancer.
 package weightedroundrobin
 
+import (
+	"google.golang.org/grpc/resolver"
+)
+
 // Name is the name of weighted_round_robin balancer.
 const Name = "weighted_round_robin"
 
-// AddrInfo will be stored inside Address metadata in order to use weighted roundrobin
-// balancer.
+// attributeKey is the type used as the key to store AddrInfo in the Attributes
+// field of resolver.Address.
+type attributeKey struct{}
+
+// AddrInfo will be stored inside Address metadata in order to use weighted
+// roundrobin balancer.
 type AddrInfo struct {
 	Weight uint32
 }
+
+// SetAddrInfo returns a copy of addr in which the Attributes field is updated
+// with addrInfo.
+//
+// This is an EXPERIMENTAL API.
+func SetAddrInfo(addr resolver.Address, addrInfo AddrInfo) resolver.Address {
+	addr.Attributes = addr.Attributes.WithValues(attributeKey{}, addrInfo)
+	return addr
+}
+
+// GetAddrInfo returns the AddrInfo stored in the Attributes fields of addr.
+//
+// This is an EXPERIMENTAL API.
+func GetAddrInfo(addr resolver.Address) AddrInfo {
+	v := addr.Attributes.Value(attributeKey{})
+	ai, _ := v.(AddrInfo)
+	return ai
+}
diff --git a/balancer/weightedroundrobin/weightedwoundrobin_test.go b/balancer/weightedroundrobin/weightedwoundrobin_test.go
new file mode 100644
index 000000000000..aa46c449a13d
--- /dev/null
+++ b/balancer/weightedroundrobin/weightedwoundrobin_test.go
@@ -0,0 +1,82 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package weightedroundrobin
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/resolver"
+)
+
+func TestAddrInfoToAndFromAttributes(t *testing.T) {
+	tests := []struct {
+		desc            string
+		inputAddrInfo   AddrInfo
+		inputAttributes *attributes.Attributes
+		wantAddrInfo    AddrInfo
+	}{
+		{
+			desc:            "empty attributes",
+			inputAddrInfo:   AddrInfo{Weight: 100},
+			inputAttributes: nil,
+			wantAddrInfo:    AddrInfo{Weight: 100},
+		},
+		{
+			desc:            "non-empty attributes",
+			inputAddrInfo:   AddrInfo{Weight: 100},
+			inputAttributes: attributes.New("foo", "bar"),
+			wantAddrInfo:    AddrInfo{Weight: 100},
+		},
+		{
+			desc:            "addrInfo not present in empty attributes",
+			inputAddrInfo:   AddrInfo{},
+			inputAttributes: nil,
+			wantAddrInfo:    AddrInfo{},
+		},
+		{
+			desc:            "addrInfo not present in non-empty attributes",
+			inputAddrInfo:   AddrInfo{},
+			inputAttributes: attributes.New("foo", "bar"),
+			wantAddrInfo:    AddrInfo{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			addr := resolver.Address{Attributes: test.inputAttributes}
+			addr = SetAddrInfo(addr, test.inputAddrInfo)
+			gotAddrInfo := GetAddrInfo(addr)
+			if !cmp.Equal(gotAddrInfo, test.wantAddrInfo) {
+				t.Errorf("gotAddrInfo: %v, wantAddrInfo: %v", gotAddrInfo, test.wantAddrInfo)
+			}
+
+		})
+	}
+}
+
+func TestGetAddInfoEmpty(t *testing.T) {
+	addr := resolver.Address{Attributes: attributes.New()}
+	gotAddrInfo := GetAddrInfo(addr)
+	wantAddrInfo := AddrInfo{}
+	if !cmp.Equal(gotAddrInfo, wantAddrInfo) {
+		t.Errorf("gotAddrInfo: %v, wantAddrInfo: %v", gotAddrInfo, wantAddrInfo)
+	}
+}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 7dfa6f835051..82d39e252e63 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -277,9 +277,16 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 				Addr: lbEndpoint.Address,
 			}
 			if edsImpl.subBalancerBuilder.Name() == weightedroundrobin.Name && lbEndpoint.Weight != 0 {
-				address.Metadata = &weightedroundrobin.AddrInfo{
-					Weight: lbEndpoint.Weight,
-				}
+				ai := weightedroundrobin.AddrInfo{Weight: lbEndpoint.Weight}
+				address = weightedroundrobin.SetAddrInfo(address, ai)
+				// Metadata field in resolver.Address is deprecated. The
+				// attributes field should be used to specify arbitrary
+				// attributes about the address. We still need to populate the
+				// Metadata field here to allow users of this field to migrate
+				// to the new one.
+				// TODO(easwars): Remove this once all users have migrated.
+				// See https://github.com/grpc/grpc-go/issues/3563.
+				address.Metadata = &ai
 			}
 			newAddrs = append(newAddrs, address)
 		}

From b0ac601168814cb8f146f6b8e373ee529d88142b Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 28 Apr 2020 10:47:24 -0700
Subject: [PATCH 047/481] rls: LB policy with only control channel handling
 (#3496)

---
 balancer/rls/internal/balancer.go             | 211 ++++++++++++++++
 balancer/rls/internal/balancer_test.go        | 228 ++++++++++++++++++
 balancer/rls/internal/builder.go              |  25 +-
 balancer/rls/internal/client.go               |   2 -
 balancer/rls/internal/client_test.go          |  90 +++----
 balancer/rls/internal/config.go               |  34 +++
 balancer/rls/internal/config_test.go          |  32 +--
 balancer/rls/internal/picker_test.go          |  25 +-
 .../testutils/fakeserver/fakeserver.go        |  46 ++--
 internal/testutils/channel.go                 |  68 ++++++
 10 files changed, 654 insertions(+), 107 deletions(-)
 create mode 100644 balancer/rls/internal/balancer.go
 create mode 100644 balancer/rls/internal/balancer_test.go
 create mode 100644 internal/testutils/channel.go

diff --git a/balancer/rls/internal/balancer.go b/balancer/rls/internal/balancer.go
new file mode 100644
index 000000000000..7c4a4817466a
--- /dev/null
+++ b/balancer/rls/internal/balancer.go
@@ -0,0 +1,211 @@
+// +build go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"sync"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/resolver"
+)
+
+var (
+	_ balancer.Balancer   = (*rlsBalancer)(nil)
+	_ balancer.V2Balancer = (*rlsBalancer)(nil)
+
+	// For overriding in tests.
+	newRLSClientFunc = newRLSClient
+)
+
+// rlsBalancer implements the RLS LB policy.
+type rlsBalancer struct {
+	done *grpcsync.Event
+	cc   balancer.ClientConn
+	opts balancer.BuildOptions
+
+	// Mutex protects all the state maintained by the LB policy.
+	// TODO(easwars): Once we add the cache, we will also have another lock for
+	// the cache alone.
+	mu    sync.Mutex
+	lbCfg *lbConfig        // Most recently received service config.
+	rlsCC *grpc.ClientConn // ClientConn to the RLS server.
+	rlsC  *rlsClient       // RLS client wrapper.
+
+	ccUpdateCh chan *balancer.ClientConnState
+}
+
+// run is a long running goroutine which handles all the updates that the
+// balancer wishes to handle. The appropriate updateHandler will push the update
+// on to a channel that this goroutine will select on, thereby the handling of
+// the update will happen asynchronously.
+func (lb *rlsBalancer) run() {
+	for {
+		// TODO(easwars): Handle other updates like subConn state changes, RLS
+		// responses from the server etc.
+		select {
+		case u := <-lb.ccUpdateCh:
+			lb.handleClientConnUpdate(u)
+		case <-lb.done.Done():
+			return
+		}
+	}
+}
+
+// handleClientConnUpdate handles updates to the service config.
+// If the RLS server name or the RLS RPC timeout changes, it updates the control
+// channel accordingly.
+// TODO(easwars): Handle updates to other fields in the service config.
+func (lb *rlsBalancer) handleClientConnUpdate(ccs *balancer.ClientConnState) {
+	grpclog.Infof("rls: service config: %+v", ccs.BalancerConfig)
+	lb.mu.Lock()
+	defer lb.mu.Unlock()
+
+	if lb.done.HasFired() {
+		grpclog.Warning("rls: received service config after balancer close")
+		return
+	}
+
+	newCfg := ccs.BalancerConfig.(*lbConfig)
+	if lb.lbCfg.Equal(newCfg) {
+		grpclog.Info("rls: new service config matches existing config")
+		return
+	}
+
+	lb.updateControlChannel(newCfg)
+	lb.lbCfg = newCfg
+}
+
+// UpdateClientConnState pushes the received ClientConnState update on the
+// update channel which will be processed asynchronously by the run goroutine.
+// Implements balancer.V2Balancer interface.
+func (lb *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error {
+	select {
+	case lb.ccUpdateCh <- &ccs:
+	case <-lb.done.Done():
+	}
+	return nil
+}
+
+// ResolverErr implements balancer.V2Balancer interface.
+func (lb *rlsBalancer) ResolverError(error) {
+	// ResolverError is called by gRPC when the name resolver reports an error.
+	// TODO(easwars): How do we handle this?
+	grpclog.Fatal("rls: ResolverError is not yet unimplemented")
+}
+
+// UpdateSubConnState implements balancer.V2Balancer interface.
+func (lb *rlsBalancer) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConnState) {
+	grpclog.Fatal("rls: UpdateSubConnState is not yet implemented")
+}
+
+// Cleans up the resources allocated by the LB policy including the clientConn
+// to the RLS server.
+// Implements balancer.Balancer and balancer.V2Balancer interfaces.
+func (lb *rlsBalancer) Close() {
+	lb.mu.Lock()
+	defer lb.mu.Unlock()
+
+	lb.done.Fire()
+	if lb.rlsCC != nil {
+		lb.rlsCC.Close()
+	}
+}
+
+// HandleSubConnStateChange implements balancer.Balancer interface.
+func (lb *rlsBalancer) HandleSubConnStateChange(_ balancer.SubConn, _ connectivity.State) {
+	grpclog.Fatal("UpdateSubConnState should be called instead of HandleSubConnStateChange")
+}
+
+// HandleResolvedAddrs implements balancer.Balancer interface.
+func (lb *rlsBalancer) HandleResolvedAddrs(_ []resolver.Address, _ error) {
+	grpclog.Fatal("UpdateClientConnState should be called instead of HandleResolvedAddrs")
+}
+
+// updateControlChannel updates the RLS client if required.
+// Caller must hold lb.mu.
+func (lb *rlsBalancer) updateControlChannel(newCfg *lbConfig) {
+	oldCfg := lb.lbCfg
+	if newCfg.lookupService == oldCfg.lookupService && newCfg.lookupServiceTimeout == oldCfg.lookupServiceTimeout {
+		return
+	}
+
+	// Use RPC timeout from new config, if different from existing one.
+	timeout := oldCfg.lookupServiceTimeout
+	if timeout != newCfg.lookupServiceTimeout {
+		timeout = newCfg.lookupServiceTimeout
+	}
+
+	if newCfg.lookupService == oldCfg.lookupService {
+		// This is the case where only the timeout has changed. We will continue
+		// to use the existing clientConn. but will create a new rlsClient with
+		// the new timeout.
+		lb.rlsC = newRLSClientFunc(lb.rlsCC, lb.opts.Target.Endpoint, timeout)
+		return
+	}
+
+	// This is the case where the RLS server name has changed. We need to create
+	// a new clientConn and close the old one.
+	var dopts []grpc.DialOption
+	if dialer := lb.opts.Dialer; dialer != nil {
+		dopts = append(dopts, grpc.WithContextDialer(dialer))
+	}
+	dopts = append(dopts, dialCreds(lb.opts))
+
+	cc, err := grpc.Dial(newCfg.lookupService, dopts...)
+	if err != nil {
+		grpclog.Errorf("rls: dialRLS(%s, %v): %v", newCfg.lookupService, lb.opts, err)
+		// An error from a non-blocking dial indicates something serious. We
+		// should continue to use the old control channel if one exists, and
+		// return so that the rest of the config updates can be processes.
+		return
+	}
+	if lb.rlsCC != nil {
+		lb.rlsCC.Close()
+	}
+	lb.rlsCC = cc
+	lb.rlsC = newRLSClientFunc(cc, lb.opts.Target.Endpoint, timeout)
+}
+
+func dialCreds(opts balancer.BuildOptions) grpc.DialOption {
+	// The control channel should use the same authority as that of the parent
+	// channel. This ensures that the identify of the RLS server and that of the
+	// backend is the same, so if the RLS config is injected by an attacker, it
+	// cannot cause leakage of private information contained in headers set by
+	// the application.
+	server := opts.Target.Authority
+	switch {
+	case opts.DialCreds != nil:
+		if err := opts.DialCreds.OverrideServerName(server); err != nil {
+			grpclog.Warningf("rls: OverrideServerName(%s) = (%v), using Insecure", server, err)
+			return grpc.WithInsecure()
+		}
+		return grpc.WithTransportCredentials(opts.DialCreds)
+	case opts.CredsBundle != nil:
+		return grpc.WithTransportCredentials(opts.CredsBundle.TransportCredentials())
+	default:
+		grpclog.Warning("rls: no credentials available, using Insecure")
+		return grpc.WithInsecure()
+	}
+}
diff --git a/balancer/rls/internal/balancer_test.go b/balancer/rls/internal/balancer_test.go
new file mode 100644
index 000000000000..990372d0e98f
--- /dev/null
+++ b/balancer/rls/internal/balancer_test.go
@@ -0,0 +1,228 @@
+// +build go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"net"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/rls/internal/testutils/fakeserver"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/testdata"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type listenerWrapper struct {
+	net.Listener
+	connCh *testutils.Channel
+}
+
+// Accept waits for and returns the next connection to the listener.
+func (l *listenerWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	l.connCh.Send(c)
+	return c, nil
+}
+
+func setupwithListener(t *testing.T, opts ...grpc.ServerOption) (*fakeserver.Server, *listenerWrapper, func()) {
+	t.Helper()
+
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen(tcp, localhost:0): %v", err)
+	}
+	lw := &listenerWrapper{
+		Listener: l,
+		connCh:   testutils.NewChannel(),
+	}
+
+	server, cleanup, err := fakeserver.Start(lw, opts...)
+	if err != nil {
+		t.Fatalf("fakeserver.Start(): %v", err)
+	}
+	t.Logf("Fake RLS server started at %s ...", server.Address)
+
+	return server, lw, cleanup
+}
+
+type testBalancerCC struct {
+	balancer.ClientConn
+}
+
+// TestUpdateControlChannelFirstConfig tests the scenario where the LB policy
+// receives its first service config and verifies that a control channel to the
+// RLS server specified in the serviceConfig is established.
+func (s) TestUpdateControlChannelFirstConfig(t *testing.T) {
+	server, lis, cleanup := setupwithListener(t)
+	defer cleanup()
+
+	bb := balancer.Get(rlsBalancerName)
+	if bb == nil {
+		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
+	}
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}).(balancer.V2Balancer)
+	defer rlsB.Close()
+	t.Log("Built RLS LB policy ...")
+
+	lbCfg := &lbConfig{lookupService: server.Address}
+	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
+	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
+
+	if _, err := lis.connCh.Receive(); err != nil {
+		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
+	}
+
+	// TODO: Verify channel connectivity state once control channel connectivity
+	// state monitoring is in place.
+
+	// TODO: Verify RLS RPC can be made once we integrate with the picker.
+}
+
+// TestUpdateControlChannelSwitch tests the scenario where a control channel
+// exists and the LB policy receives a new serviceConfig with a different RLS
+// server name. Verifies that the new control channel is created and the old one
+// is closed (the leakchecker takes care of this).
+func (s) TestUpdateControlChannelSwitch(t *testing.T) {
+	server1, lis1, cleanup1 := setupwithListener(t)
+	defer cleanup1()
+
+	server2, lis2, cleanup2 := setupwithListener(t)
+	defer cleanup2()
+
+	bb := balancer.Get(rlsBalancerName)
+	if bb == nil {
+		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
+	}
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}).(balancer.V2Balancer)
+	defer rlsB.Close()
+	t.Log("Built RLS LB policy ...")
+
+	lbCfg := &lbConfig{lookupService: server1.Address}
+	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
+	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
+
+	if _, err := lis1.connCh.Receive(); err != nil {
+		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
+	}
+
+	lbCfg = &lbConfig{lookupService: server2.Address}
+	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
+	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
+
+	if _, err := lis2.connCh.Receive(); err != nil {
+		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
+	}
+
+	// TODO: Verify channel connectivity state once control channel connectivity
+	// state monitoring is in place.
+
+	// TODO: Verify RLS RPC can be made once we integrate with the picker.
+}
+
+// TestUpdateControlChannelTimeout tests the scenario where the LB policy
+// receives a service config update with a different lookupServiceTimeout, but
+// the lookupService itself remains unchanged. It verifies that the LB policy
+// does not create a new control channel in this case.
+func (s) TestUpdateControlChannelTimeout(t *testing.T) {
+	server, lis, cleanup := setupwithListener(t)
+	defer cleanup()
+
+	bb := balancer.Get(rlsBalancerName)
+	if bb == nil {
+		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
+	}
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}).(balancer.V2Balancer)
+	defer rlsB.Close()
+	t.Log("Built RLS LB policy ...")
+
+	lbCfg := &lbConfig{lookupService: server.Address, lookupServiceTimeout: 1 * time.Second}
+	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
+	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
+	if _, err := lis.connCh.Receive(); err != nil {
+		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
+	}
+
+	lbCfg = &lbConfig{lookupService: server.Address, lookupServiceTimeout: 2 * time.Second}
+	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
+	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
+	if _, err := lis.connCh.Receive(); err != testutils.ErrRecvTimeout {
+		t.Fatal("LB policy created new control channel when only lookupServiceTimeout changed")
+	}
+
+	// TODO: Verify channel connectivity state once control channel connectivity
+	// state monitoring is in place.
+
+	// TODO: Verify RLS RPC can be made once we integrate with the picker.
+}
+
+// TestUpdateControlChannelWithCreds tests the scenario where the control
+// channel is to established with credentials from the parent channel.
+func (s) TestUpdateControlChannelWithCreds(t *testing.T) {
+	sCreds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	if err != nil {
+		t.Fatalf("credentials.NewServerTLSFromFile(server1.pem, server1.key) = %v", err)
+	}
+	cCreds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "")
+	if err != nil {
+		t.Fatalf("credentials.NewClientTLSFromFile(ca.pem) = %v", err)
+	}
+
+	server, lis, cleanup := setupwithListener(t, grpc.Creds(sCreds))
+	defer cleanup()
+
+	bb := balancer.Get(rlsBalancerName)
+	if bb == nil {
+		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
+	}
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{
+		DialCreds: cCreds,
+	}).(balancer.V2Balancer)
+	defer rlsB.Close()
+	t.Log("Built RLS LB policy ...")
+
+	lbCfg := &lbConfig{lookupService: server.Address}
+	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
+	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
+
+	if _, err := lis.connCh.Receive(); err != nil {
+		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
+	}
+
+	// TODO: Verify channel connectivity state once control channel connectivity
+	// state monitoring is in place.
+
+	// TODO: Verify RLS RPC can be made once we integrate with the picker.
+}
diff --git a/balancer/rls/internal/builder.go b/balancer/rls/internal/builder.go
index ddb5e3cf0bb2..c38babff4d3d 100644
--- a/balancer/rls/internal/builder.go
+++ b/balancer/rls/internal/builder.go
@@ -21,16 +21,35 @@
 // Package rls implements the RLS LB policy.
 package rls
 
+import (
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/grpcsync"
+)
+
 const rlsBalancerName = "rls"
 
+func init() {
+	balancer.Register(&rlsBB{})
+}
+
 // rlsBB helps build RLS load balancers and parse the service config to be
 // passed to the RLS load balancer.
-type rlsBB struct {
-	// TODO(easwars): Implement the Build() method and register the builder.
-}
+type rlsBB struct{}
 
 // Name returns the name of the RLS LB policy and helps implement the
 // balancer.Balancer interface.
 func (*rlsBB) Name() string {
 	return rlsBalancerName
 }
+
+func (*rlsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	lb := &rlsBalancer{
+		done:       grpcsync.NewEvent(),
+		cc:         cc,
+		opts:       opts,
+		lbCfg:      &lbConfig{},
+		ccUpdateCh: make(chan *balancer.ClientConnState),
+	}
+	go lb.run()
+	return lb
+}
diff --git a/balancer/rls/internal/client.go b/balancer/rls/internal/client.go
index b6fe22572949..0e8a1c932f11 100644
--- a/balancer/rls/internal/client.go
+++ b/balancer/rls/internal/client.go
@@ -43,7 +43,6 @@ const grpcTargetType = "grpc"
 // throttling and asks this client to make an RPC call only after checking with
 // the throttler.
 type rlsClient struct {
-	cc   *grpc.ClientConn
 	stub rlspb.RouteLookupServiceClient
 	// origDialTarget is the original dial target of the user and sent in each
 	// RouteLookup RPC made to the RLS server.
@@ -55,7 +54,6 @@ type rlsClient struct {
 
 func newRLSClient(cc *grpc.ClientConn, dialTarget string, rpcTimeout time.Duration) *rlsClient {
 	return &rlsClient{
-		cc:             cc,
 		stub:           rlspb.NewRouteLookupServiceClient(cc),
 		origDialTarget: dialTarget,
 		rpcTimeout:     rpcTimeout,
diff --git a/balancer/rls/internal/client_test.go b/balancer/rls/internal/client_test.go
index 386267b9033d..1a1a75d1be98 100644
--- a/balancer/rls/internal/client_test.go
+++ b/balancer/rls/internal/client_test.go
@@ -1,3 +1,5 @@
+// +build go1.10
+
 /*
  *
  * Copyright 2020 gRPC authors.
@@ -30,25 +32,26 @@ import (
 	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
 	"google.golang.org/grpc/balancer/rls/internal/testutils/fakeserver"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/status"
 )
 
 const (
-	defaultDialTarget  = "dummy"
-	defaultRPCTimeout  = 5 * time.Second
-	defaultTestTimeout = 1 * time.Second
+	defaultDialTarget = "dummy"
+	defaultRPCTimeout = 5 * time.Second
 )
 
 func setup(t *testing.T) (*fakeserver.Server, *grpc.ClientConn, func()) {
 	t.Helper()
 
-	server, sCleanup, err := fakeserver.Start()
+	server, sCleanup, err := fakeserver.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start fake RLS server: %v", err)
 	}
 
 	cc, cCleanup, err := server.ClientConn()
 	if err != nil {
+		sCleanup()
 		t.Fatalf("Failed to get a ClientConn to the RLS server: %v", err)
 	}
 
@@ -59,7 +62,7 @@ func setup(t *testing.T) (*fakeserver.Server, *grpc.ClientConn, func()) {
 }
 
 // TestLookupFailure verifies the case where the RLS server returns an error.
-func TestLookupFailure(t *testing.T) {
+func (s) TestLookupFailure(t *testing.T) {
 	server, cc, cleanup := setup(t)
 	defer cleanup()
 
@@ -68,64 +71,50 @@ func TestLookupFailure(t *testing.T) {
 
 	rlsClient := newRLSClient(cc, defaultDialTarget, defaultRPCTimeout)
 
-	errCh := make(chan error)
+	errCh := testutils.NewChannel()
 	rlsClient.lookup("", nil, func(targets []string, headerData string, err error) {
 		if err == nil {
-			errCh <- errors.New("rlsClient.lookup() succeeded, should have failed")
+			errCh.Send(errors.New("rlsClient.lookup() succeeded, should have failed"))
 			return
 		}
 		if len(targets) != 0 || headerData != "" {
-			errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (nil, \"\")", targets, headerData)
+			errCh.Send(fmt.Errorf("rlsClient.lookup() = (%v, %s), want (nil, \"\")", targets, headerData))
 			return
 		}
-		errCh <- nil
+		errCh.Send(nil)
 	})
 
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		t.Fatal("Timeout when expecting a routeLookup callback")
-	case err := <-errCh:
-		timer.Stop()
-		if err != nil {
-			t.Fatal(err)
-		}
+	if e, err := errCh.Receive(); err != nil || e != nil {
+		t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err)
 	}
 }
 
 // TestLookupDeadlineExceeded tests the case where the RPC deadline associated
 // with the lookup expires.
-func TestLookupDeadlineExceeded(t *testing.T) {
+func (s) TestLookupDeadlineExceeded(t *testing.T) {
 	_, cc, cleanup := setup(t)
 	defer cleanup()
 
 	// Give the Lookup RPC a small deadline, but don't setup the fake server to
-	// return anything. So the Lookup call will block and eventuall expire.
+	// return anything. So the Lookup call will block and eventually expire.
 	rlsClient := newRLSClient(cc, defaultDialTarget, 100*time.Millisecond)
 
-	errCh := make(chan error)
+	errCh := testutils.NewChannel()
 	rlsClient.lookup("", nil, func(_ []string, _ string, err error) {
 		if st, ok := status.FromError(err); !ok || st.Code() != codes.DeadlineExceeded {
-			errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded)
+			errCh.Send(fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded))
 			return
 		}
-		errCh <- nil
+		errCh.Send(nil)
 	})
 
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		t.Fatal("Timeout when expecting a routeLookup callback")
-	case err := <-errCh:
-		timer.Stop()
-		if err != nil {
-			t.Fatal(err)
-		}
+	if e, err := errCh.Receive(); err != nil || e != nil {
+		t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err)
 	}
 }
 
 // TestLookupSuccess verifies the successful Lookup API case.
-func TestLookupSuccess(t *testing.T) {
+func (s) TestLookupSuccess(t *testing.T) {
 	server, cc, cleanup := setup(t)
 	defer cleanup()
 
@@ -148,33 +137,29 @@ func TestLookupSuccess(t *testing.T) {
 
 	rlsClient := newRLSClient(cc, defaultDialTarget, defaultRPCTimeout)
 
-	errCh := make(chan error)
+	errCh := testutils.NewChannel()
 	rlsClient.lookup(rlsReqPath, rlsReqKeyMap, func(targets []string, hd string, err error) {
 		if err != nil {
-			errCh <- fmt.Errorf("rlsClient.Lookup() failed: %v", err)
+			errCh.Send(fmt.Errorf("rlsClient.Lookup() failed: %v", err))
 			return
 		}
 		if !cmp.Equal(targets, wantRespTargets) || hd != wantHeaderData {
-			errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, hd, wantRespTargets, wantHeaderData)
+			errCh.Send(fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, hd, wantRespTargets, wantHeaderData))
 			return
 		}
-		errCh <- nil
+		errCh.Send(nil)
 	})
 
 	// Make sure that the fake server received the expected RouteLookupRequest
 	// proto.
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case gotLookupRequest := <-server.RequestChan:
-		if !timer.Stop() {
-			<-timer.C
-		}
-		if diff := cmp.Diff(wantLookupRequest, gotLookupRequest, cmp.Comparer(proto.Equal)); diff != "" {
-			t.Fatalf("RouteLookupRequest diff (-want, +got):\n%s", diff)
-		}
-	case <-timer.C:
+	req, err := server.RequestChan.Receive()
+	if err != nil {
 		t.Fatalf("Timed out wile waiting for a RouteLookupRequest")
 	}
+	gotLookupRequest := req.(*rlspb.RouteLookupRequest)
+	if diff := cmp.Diff(wantLookupRequest, gotLookupRequest, cmp.Comparer(proto.Equal)); diff != "" {
+		t.Fatalf("RouteLookupRequest diff (-want, +got):\n%s", diff)
+	}
 
 	// We setup the fake server to return this response when it receives a
 	// request.
@@ -185,14 +170,7 @@ func TestLookupSuccess(t *testing.T) {
 		},
 	}
 
-	timer = time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		t.Fatal("Timeout when expecting a routeLookup callback")
-	case err := <-errCh:
-		timer.Stop()
-		if err != nil {
-			t.Fatal(err)
-		}
+	if e, err := errCh.Receive(); err != nil || e != nil {
+		t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err)
 	}
 }
diff --git a/balancer/rls/internal/config.go b/balancer/rls/internal/config.go
index e1e36e445fa6..816ab093a650 100644
--- a/balancer/rls/internal/config.go
+++ b/balancer/rls/internal/config.go
@@ -71,6 +71,40 @@ type lbConfig struct {
 	cpConfig             map[string]json.RawMessage
 }
 
+func (lbCfg *lbConfig) Equal(other *lbConfig) bool {
+	return lbCfg.kbMap.Equal(other.kbMap) &&
+		lbCfg.lookupService == other.lookupService &&
+		lbCfg.lookupServiceTimeout == other.lookupServiceTimeout &&
+		lbCfg.maxAge == other.maxAge &&
+		lbCfg.staleAge == other.staleAge &&
+		lbCfg.cacheSizeBytes == other.cacheSizeBytes &&
+		lbCfg.rpStrategy == other.rpStrategy &&
+		lbCfg.defaultTarget == other.defaultTarget &&
+		lbCfg.cpName == other.cpName &&
+		lbCfg.cpTargetField == other.cpTargetField &&
+		cpConfigEqual(lbCfg.cpConfig, other.cpConfig)
+}
+
+func cpConfigEqual(am, bm map[string]json.RawMessage) bool {
+	if (bm == nil) != (am == nil) {
+		return false
+	}
+	if len(bm) != len(am) {
+		return false
+	}
+
+	for k, jsonA := range am {
+		jsonB, ok := bm[k]
+		if !ok {
+			return false
+		}
+		if !bytes.Equal(jsonA, jsonB) {
+			return false
+		}
+	}
+	return true
+}
+
 // This struct resembles the JSON respresentation of the loadBalancing config
 // and makes it easier to unmarshal.
 type lbConfigJSON struct {
diff --git a/balancer/rls/internal/config_test.go b/balancer/rls/internal/config_test.go
index 6285f0cdadb8..9200a29d8a3c 100644
--- a/balancer/rls/internal/config_test.go
+++ b/balancer/rls/internal/config_test.go
@@ -49,20 +49,22 @@ func init() {
 	balancer.Register(&dummyBB{})
 }
 
-func (lbCfg *lbConfig) Equal(other *lbConfig) bool {
-	// This only ignores the keyBuilderMap field because its internals are not
-	// exported, and hence not possible to specify in the want section of the
-	// test.
-	return lbCfg.lookupService == other.lookupService &&
-		lbCfg.lookupServiceTimeout == other.lookupServiceTimeout &&
-		lbCfg.maxAge == other.maxAge &&
-		lbCfg.staleAge == other.staleAge &&
-		lbCfg.cacheSizeBytes == other.cacheSizeBytes &&
-		lbCfg.rpStrategy == other.rpStrategy &&
-		lbCfg.defaultTarget == other.defaultTarget &&
-		lbCfg.cpName == other.cpName &&
-		lbCfg.cpTargetField == other.cpTargetField &&
-		cmp.Equal(lbCfg.cpConfig, other.cpConfig)
+// testEqual reports whether the lbCfgs a and b are equal. This is to be used
+// only from tests. This ignores the keyBuilderMap field because its internals
+// are not exported, and hence not possible to specify in the want section of
+// the test. This is fine because we already have tests to make sure that the
+// keyBuilder is parsed properly from the service config.
+func testEqual(a, b *lbConfig) bool {
+	return a.lookupService == b.lookupService &&
+		a.lookupServiceTimeout == b.lookupServiceTimeout &&
+		a.maxAge == b.maxAge &&
+		a.staleAge == b.staleAge &&
+		a.cacheSizeBytes == b.cacheSizeBytes &&
+		a.rpStrategy == b.rpStrategy &&
+		a.defaultTarget == b.defaultTarget &&
+		a.cpName == b.cpName &&
+		a.cpTargetField == b.cpTargetField &&
+		cmp.Equal(a.cpConfig, b.cpConfig)
 }
 
 func TestParseConfig(t *testing.T) {
@@ -152,7 +154,7 @@ func TestParseConfig(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			lbCfg, err := builder.ParseConfig(test.input)
-			if err != nil || !cmp.Equal(lbCfg, test.wantCfg) {
+			if err != nil || !testEqual(lbCfg.(*lbConfig), test.wantCfg) {
 				t.Errorf("ParseConfig(%s) = {%+v, %v}, want {%+v, nil}", string(test.input), lbCfg, err, test.wantCfg)
 			}
 		})
diff --git a/balancer/rls/internal/picker_test.go b/balancer/rls/internal/picker_test.go
index a58db3b332fa..b14a9fad340a 100644
--- a/balancer/rls/internal/picker_test.go
+++ b/balancer/rls/internal/picker_test.go
@@ -28,13 +28,13 @@ import (
 	"testing"
 	"time"
 
-	"google.golang.org/grpc/internal/grpcrand"
-
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/rls/internal/cache"
 	"google.golang.org/grpc/balancer/rls/internal/keys"
 	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -502,7 +502,7 @@ func TestPick(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			rlsCh := make(chan error, 1)
+			rlsCh := testutils.NewChannel()
 			randID := grpcrand.Intn(math.MaxInt32)
 			// We instantiate a fakeChildPicker which will return a fakeSubConn
 			// with configured id. Either the childPicker or the defaultPicker
@@ -525,18 +525,18 @@ func TestPick(t *testing.T) {
 				shouldThrottle: func() bool { return test.throttle },
 				startRLS: func(path string, km keys.KeyMap) {
 					if !test.newRLSRequest {
-						rlsCh <- errors.New("RLS request attempted when none was expected")
+						rlsCh.Send(errors.New("RLS request attempted when none was expected"))
 						return
 					}
 					if path != rpcPath {
-						rlsCh <- fmt.Errorf("RLS request initiated for rpcPath %s, want %s", path, rpcPath)
+						rlsCh.Send(fmt.Errorf("RLS request initiated for rpcPath %s, want %s", path, rpcPath))
 						return
 					}
 					if km.Str != wantKeyMapStr {
-						rlsCh <- fmt.Errorf("RLS request initiated with keys %v, want %v", km.Str, wantKeyMapStr)
+						rlsCh.Send(fmt.Errorf("RLS request initiated with keys %v, want %v", km.Str, wantKeyMapStr))
 						return
 					}
-					rlsCh <- nil
+					rlsCh.Send(nil)
 				},
 				defaultPick: func(info balancer.PickInfo) (balancer.PickResult, error) {
 					if !test.useDefaultPick {
@@ -569,15 +569,8 @@ func TestPick(t *testing.T) {
 			// If the test specified that a new RLS request should be made,
 			// verify it.
 			if test.newRLSRequest {
-				timer := time.NewTimer(defaultTestTimeout)
-				select {
-				case err := <-rlsCh:
-					timer.Stop()
-					if err != nil {
-						t.Fatal(err)
-					}
-				case <-timer.C:
-					t.Fatal("Timeout waiting for RLS request to be sent out")
+				if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil {
+					t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err)
 				}
 			}
 		})
diff --git a/balancer/rls/internal/testutils/fakeserver/fakeserver.go b/balancer/rls/internal/testutils/fakeserver/fakeserver.go
index 1cdf81550243..93947da4ccef 100644
--- a/balancer/rls/internal/testutils/fakeserver/fakeserver.go
+++ b/balancer/rls/internal/testutils/fakeserver/fakeserver.go
@@ -22,6 +22,7 @@ package fakeserver
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"time"
@@ -29,9 +30,14 @@ import (
 	"google.golang.org/grpc"
 	rlsgrpc "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
 	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/internal/testutils"
 )
 
-const defaultDialTimeout = 5 * time.Second
+const (
+	defaultDialTimeout       = 5 * time.Second
+	defaultRPCTimeout        = 5 * time.Second
+	defaultChannelBufferSize = 50
+)
 
 // Response wraps the response protobuf (xds/LRS) and error that the Server
 // should send out to the client through a call to stream.Send()
@@ -43,29 +49,31 @@ type Response struct {
 // Server is a fake implementation of RLS. It exposes channels to send/receive
 // RLS requests and responses.
 type Server struct {
-	RequestChan  chan *rlspb.RouteLookupRequest
+	RequestChan  *testutils.Channel
 	ResponseChan chan Response
 	Address      string
 }
 
-// Start makes a new Server and gets it to start listening on a local port for
-// gRPC requests. The returned cancel function should be invoked by the caller
-// upon completion of the test.
-func Start() (*Server, func(), error) {
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err)
+// Start makes a new Server which uses the provided net.Listener. If lis is nil,
+// it creates a new net.Listener on a local port. The returned cancel function
+// should be invoked by the caller upon completion of the test.
+func Start(lis net.Listener, opts ...grpc.ServerOption) (*Server, func(), error) {
+	if lis == nil {
+		var err error
+		lis, err = net.Listen("tcp", "localhost:0")
+		if err != nil {
+			return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err)
+		}
 	}
-
 	s := &Server{
 		// Give the channels a buffer size of 1 so that we can setup
 		// expectations for one lookup call, without blocking.
-		RequestChan:  make(chan *rlspb.RouteLookupRequest, 1),
+		RequestChan:  testutils.NewChannelWithSize(defaultChannelBufferSize),
 		ResponseChan: make(chan Response, 1),
 		Address:      lis.Addr().String(),
 	}
 
-	server := grpc.NewServer()
+	server := grpc.NewServer(opts...)
 	rlsgrpc.RegisterRouteLookupServiceServer(server, s)
 	go server.Serve(lis)
 
@@ -74,9 +82,17 @@ func Start() (*Server, func(), error) {
 
 // RouteLookup implements the RouteLookupService.
 func (s *Server) RouteLookup(ctx context.Context, req *rlspb.RouteLookupRequest) (*rlspb.RouteLookupResponse, error) {
-	s.RequestChan <- req
-	resp := <-s.ResponseChan
-	return resp.Resp, resp.Err
+	s.RequestChan.Send(req)
+
+	// The leakchecker fails if we don't exit out of here in a reasonable time.
+	timer := time.NewTimer(defaultRPCTimeout)
+	select {
+	case <-timer.C:
+		return nil, errors.New("default RPC timeout exceeded")
+	case resp := <-s.ResponseChan:
+		timer.Stop()
+		return resp.Resp, resp.Err
+	}
 }
 
 // ClientConn returns a grpc.ClientConn connected to the fakeServer.
diff --git a/internal/testutils/channel.go b/internal/testutils/channel.go
new file mode 100644
index 000000000000..35f67ea285c1
--- /dev/null
+++ b/internal/testutils/channel.go
@@ -0,0 +1,68 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package testutils
+
+import (
+	"errors"
+	"time"
+)
+
+// ErrRecvTimeout is an error to indicate that a receive operation on the
+// channel timed out.
+var ErrRecvTimeout = errors.New("timed out when waiting for value on channel")
+
+const (
+	// DefaultChanRecvTimeout is the default timeout for receive operations on the
+	// underlying channel.
+	DefaultChanRecvTimeout = 1 * time.Second
+	// DefaultChanBufferSize is the default buffer size of the underlying channel.
+	DefaultChanBufferSize = 1
+)
+
+// Channel wraps a generic channel and provides a timed receive operation.
+type Channel struct {
+	ch chan interface{}
+}
+
+// Send sends value on the underlying channel.
+func (cwt *Channel) Send(value interface{}) {
+	cwt.ch <- value
+}
+
+// Receive returns the value received on the underlying channel, or
+// ErrRecvTimeout if DefaultChanRecvTimeout amount of time elapses.
+func (cwt *Channel) Receive() (interface{}, error) {
+	timer := time.NewTimer(DefaultChanRecvTimeout)
+	select {
+	case <-timer.C:
+		return nil, ErrRecvTimeout
+	case got := <-cwt.ch:
+		timer.Stop()
+		return got, nil
+	}
+}
+
+// NewChannel returns a new Channel.
+func NewChannel() *Channel {
+	return NewChannelWithSize(DefaultChanBufferSize)
+}
+
+// NewChannelWithSize returns a new Channel with a buffer of bufSize.
+func NewChannelWithSize(bufSize int) *Channel {
+	return &Channel{ch: make(chan interface{}, bufSize)}
+}

From 4eb418e5b2e10614c9638bc7748fb524636e5c4a Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 28 Apr 2020 14:52:49 -0700
Subject: [PATCH 048/481] balancer: move Balancer and Picker to V2; delete
 legacy API (#3431)

---
 balancer.go                                   | 391 ---------
 balancer/balancer.go                          | 169 ++--
 balancer/base/balancer.go                     | 109 +--
 balancer/base/base.go                         |  37 +-
 balancer/grpclb/grpclb.go                     |  15 +-
 balancer/grpclb/grpclb_remote_balancer.go     |   2 +-
 balancer/rls/internal/cache/cache.go          |   4 +-
 balancer/rls/internal/picker.go               |   5 +-
 balancer/roundrobin/roundrobin.go             |   6 +-
 balancer_conn_wrappers.go                     |  37 +-
 balancer_conn_wrappers_test.go                |  39 +-
 balancer_switching_test.go                    |  30 +-
 balancer_test.go                              | 789 ------------------
 balancer_v1_wrapper.go                        | 334 --------
 clientconn.go                                 |  23 +-
 clientconn_state_transition_test.go           |   6 +-
 clientconn_test.go                            | 127 ---
 dialoptions.go                                |  16 +-
 examples/go.sum                               |   2 +
 internal/status/status.go                     |  14 +-
 naming/dns_resolver.go                        | 293 -------
 naming/dns_resolver_test.go                   | 341 --------
 naming/naming.go                              |  68 --
 picker_wrapper.go                             |  71 +-
 picker_wrapper_test.go                        |   8 +-
 pickfirst.go                                  |  36 +-
 pickfirst_test.go                             |  14 +-
 resolver_conn_wrapper_test.go                 |   7 +-
 service_config.go                             |   2 +-
 test/balancer_test.go                         | 256 +++++-
 test/channelz_test.go                         |   3 +-
 test/creds_test.go                            |  10 +-
 test/end2end_test.go                          |  58 +-
 vet.sh                                        |  11 +-
 .../balancer/balancergroup/balancergroup.go   |  31 +-
 .../balancergroup/balancergroup_test.go       |   2 +-
 .../balancer/cdsbalancer/cdsbalancer.go       |  15 +-
 .../balancer/cdsbalancer/cdsbalancer_test.go  |   4 +-
 xds/internal/balancer/edsbalancer/eds.go      |  11 -
 xds/internal/balancer/edsbalancer/eds_impl.go |   8 +-
 .../balancer/edsbalancer/eds_impl_priority.go |   4 +-
 .../balancer/edsbalancer/eds_impl_test.go     |  10 +-
 xds/internal/balancer/edsbalancer/eds_test.go |  10 +-
 xds/internal/testutils/balancer.go            |  19 +-
 44 files changed, 540 insertions(+), 2907 deletions(-)
 delete mode 100644 balancer.go
 delete mode 100644 balancer_test.go
 delete mode 100644 balancer_v1_wrapper.go
 delete mode 100644 naming/dns_resolver.go
 delete mode 100644 naming/dns_resolver_test.go
 delete mode 100644 naming/naming.go

diff --git a/balancer.go b/balancer.go
deleted file mode 100644
index a8eb0f476091..000000000000
--- a/balancer.go
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
- *
- * Copyright 2016 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package grpc
-
-import (
-	"context"
-	"net"
-	"sync"
-
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
-	"google.golang.org/grpc/naming"
-	"google.golang.org/grpc/status"
-)
-
-// Address represents a server the client connects to.
-//
-// Deprecated: please use package balancer.
-type Address struct {
-	// Addr is the server address on which a connection will be established.
-	Addr string
-	// Metadata is the information associated with Addr, which may be used
-	// to make load balancing decision.
-	Metadata interface{}
-}
-
-// BalancerConfig specifies the configurations for Balancer.
-//
-// Deprecated: please use package balancer.  May be removed in a future 1.x release.
-type BalancerConfig struct {
-	// DialCreds is the transport credential the Balancer implementation can
-	// use to dial to a remote load balancer server. The Balancer implementations
-	// can ignore this if it does not need to talk to another party securely.
-	DialCreds credentials.TransportCredentials
-	// Dialer is the custom dialer the Balancer implementation can use to dial
-	// to a remote load balancer server. The Balancer implementations
-	// can ignore this if it doesn't need to talk to remote balancer.
-	Dialer func(context.Context, string) (net.Conn, error)
-}
-
-// BalancerGetOptions configures a Get call.
-//
-// Deprecated: please use package balancer.  May be removed in a future 1.x release.
-type BalancerGetOptions struct {
-	// BlockingWait specifies whether Get should block when there is no
-	// connected address.
-	BlockingWait bool
-}
-
-// Balancer chooses network addresses for RPCs.
-//
-// Deprecated: please use package balancer.  May be removed in a future 1.x release.
-type Balancer interface {
-	// Start does the initialization work to bootstrap a Balancer. For example,
-	// this function may start the name resolution and watch the updates. It will
-	// be called when dialing.
-	Start(target string, config BalancerConfig) error
-	// Up informs the Balancer that gRPC has a connection to the server at
-	// addr. It returns down which is called once the connection to addr gets
-	// lost or closed.
-	// TODO: It is not clear how to construct and take advantage of the meaningful error
-	// parameter for down. Need realistic demands to guide.
-	Up(addr Address) (down func(error))
-	// Get gets the address of a server for the RPC corresponding to ctx.
-	// i) If it returns a connected address, gRPC internals issues the RPC on the
-	// connection to this address;
-	// ii) If it returns an address on which the connection is under construction
-	// (initiated by Notify(...)) but not connected, gRPC internals
-	//  * fails RPC if the RPC is fail-fast and connection is in the TransientFailure or
-	//  Shutdown state;
-	//  or
-	//  * issues RPC on the connection otherwise.
-	// iii) If it returns an address on which the connection does not exist, gRPC
-	// internals treats it as an error and will fail the corresponding RPC.
-	//
-	// Therefore, the following is the recommended rule when writing a custom Balancer.
-	// If opts.BlockingWait is true, it should return a connected address or
-	// block if there is no connected address. It should respect the timeout or
-	// cancellation of ctx when blocking. If opts.BlockingWait is false (for fail-fast
-	// RPCs), it should return an address it has notified via Notify(...) immediately
-	// instead of blocking.
-	//
-	// The function returns put which is called once the rpc has completed or failed.
-	// put can collect and report RPC stats to a remote load balancer.
-	//
-	// This function should only return the errors Balancer cannot recover by itself.
-	// gRPC internals will fail the RPC if an error is returned.
-	Get(ctx context.Context, opts BalancerGetOptions) (addr Address, put func(), err error)
-	// Notify returns a channel that is used by gRPC internals to watch the addresses
-	// gRPC needs to connect. The addresses might be from a name resolver or remote
-	// load balancer. gRPC internals will compare it with the existing connected
-	// addresses. If the address Balancer notified is not in the existing connected
-	// addresses, gRPC starts to connect the address. If an address in the existing
-	// connected addresses is not in the notification list, the corresponding connection
-	// is shutdown gracefully. Otherwise, there are no operations to take. Note that
-	// the Address slice must be the full list of the Addresses which should be connected.
-	// It is NOT delta.
-	Notify() <-chan []Address
-	// Close shuts down the balancer.
-	Close() error
-}
-
-// RoundRobin returns a Balancer that selects addresses round-robin. It uses r to watch
-// the name resolution updates and updates the addresses available correspondingly.
-//
-// Deprecated: please use package balancer/roundrobin. May be removed in a future 1.x release.
-func RoundRobin(r naming.Resolver) Balancer {
-	return &roundRobin{r: r}
-}
-
-type addrInfo struct {
-	addr      Address
-	connected bool
-}
-
-type roundRobin struct {
-	r      naming.Resolver
-	w      naming.Watcher
-	addrs  []*addrInfo // all the addresses the client should potentially connect
-	mu     sync.Mutex
-	addrCh chan []Address // the channel to notify gRPC internals the list of addresses the client should connect to.
-	next   int            // index of the next address to return for Get()
-	waitCh chan struct{}  // the channel to block when there is no connected address available
-	done   bool           // The Balancer is closed.
-}
-
-func (rr *roundRobin) watchAddrUpdates() error {
-	updates, err := rr.w.Next()
-	if err != nil {
-		grpclog.Warningf("grpc: the naming watcher stops working due to %v.", err)
-		return err
-	}
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
-	for _, update := range updates {
-		addr := Address{
-			Addr:     update.Addr,
-			Metadata: update.Metadata,
-		}
-		switch update.Op {
-		case naming.Add:
-			var exist bool
-			for _, v := range rr.addrs {
-				if addr == v.addr {
-					exist = true
-					grpclog.Infoln("grpc: The name resolver wanted to add an existing address: ", addr)
-					break
-				}
-			}
-			if exist {
-				continue
-			}
-			rr.addrs = append(rr.addrs, &addrInfo{addr: addr})
-		case naming.Delete:
-			for i, v := range rr.addrs {
-				if addr == v.addr {
-					copy(rr.addrs[i:], rr.addrs[i+1:])
-					rr.addrs = rr.addrs[:len(rr.addrs)-1]
-					break
-				}
-			}
-		default:
-			grpclog.Errorln("Unknown update.Op ", update.Op)
-		}
-	}
-	// Make a copy of rr.addrs and write it onto rr.addrCh so that gRPC internals gets notified.
-	open := make([]Address, len(rr.addrs))
-	for i, v := range rr.addrs {
-		open[i] = v.addr
-	}
-	if rr.done {
-		return ErrClientConnClosing
-	}
-	select {
-	case <-rr.addrCh:
-	default:
-	}
-	rr.addrCh <- open
-	return nil
-}
-
-func (rr *roundRobin) Start(target string, config BalancerConfig) error {
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
-	if rr.done {
-		return ErrClientConnClosing
-	}
-	if rr.r == nil {
-		// If there is no name resolver installed, it is not needed to
-		// do name resolution. In this case, target is added into rr.addrs
-		// as the only address available and rr.addrCh stays nil.
-		rr.addrs = append(rr.addrs, &addrInfo{addr: Address{Addr: target}})
-		return nil
-	}
-	w, err := rr.r.Resolve(target)
-	if err != nil {
-		return err
-	}
-	rr.w = w
-	rr.addrCh = make(chan []Address, 1)
-	go func() {
-		for {
-			if err := rr.watchAddrUpdates(); err != nil {
-				return
-			}
-		}
-	}()
-	return nil
-}
-
-// Up sets the connected state of addr and sends notification if there are pending
-// Get() calls.
-func (rr *roundRobin) Up(addr Address) func(error) {
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
-	var cnt int
-	for _, a := range rr.addrs {
-		if a.addr == addr {
-			if a.connected {
-				return nil
-			}
-			a.connected = true
-		}
-		if a.connected {
-			cnt++
-		}
-	}
-	// addr is only one which is connected. Notify the Get() callers who are blocking.
-	if cnt == 1 && rr.waitCh != nil {
-		close(rr.waitCh)
-		rr.waitCh = nil
-	}
-	return func(err error) {
-		rr.down(addr, err)
-	}
-}
-
-// down unsets the connected state of addr.
-func (rr *roundRobin) down(addr Address, err error) {
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
-	for _, a := range rr.addrs {
-		if addr == a.addr {
-			a.connected = false
-			break
-		}
-	}
-}
-
-// Get returns the next addr in the rotation.
-func (rr *roundRobin) Get(ctx context.Context, opts BalancerGetOptions) (addr Address, put func(), err error) {
-	var ch chan struct{}
-	rr.mu.Lock()
-	if rr.done {
-		rr.mu.Unlock()
-		err = ErrClientConnClosing
-		return
-	}
-
-	if len(rr.addrs) > 0 {
-		if rr.next >= len(rr.addrs) {
-			rr.next = 0
-		}
-		next := rr.next
-		for {
-			a := rr.addrs[next]
-			next = (next + 1) % len(rr.addrs)
-			if a.connected {
-				addr = a.addr
-				rr.next = next
-				rr.mu.Unlock()
-				return
-			}
-			if next == rr.next {
-				// Has iterated all the possible address but none is connected.
-				break
-			}
-		}
-	}
-	if !opts.BlockingWait {
-		if len(rr.addrs) == 0 {
-			rr.mu.Unlock()
-			err = status.Errorf(codes.Unavailable, "there is no address available")
-			return
-		}
-		// Returns the next addr on rr.addrs for failfast RPCs.
-		addr = rr.addrs[rr.next].addr
-		rr.next++
-		rr.mu.Unlock()
-		return
-	}
-	// Wait on rr.waitCh for non-failfast RPCs.
-	if rr.waitCh == nil {
-		ch = make(chan struct{})
-		rr.waitCh = ch
-	} else {
-		ch = rr.waitCh
-	}
-	rr.mu.Unlock()
-	for {
-		select {
-		case <-ctx.Done():
-			err = ctx.Err()
-			return
-		case <-ch:
-			rr.mu.Lock()
-			if rr.done {
-				rr.mu.Unlock()
-				err = ErrClientConnClosing
-				return
-			}
-
-			if len(rr.addrs) > 0 {
-				if rr.next >= len(rr.addrs) {
-					rr.next = 0
-				}
-				next := rr.next
-				for {
-					a := rr.addrs[next]
-					next = (next + 1) % len(rr.addrs)
-					if a.connected {
-						addr = a.addr
-						rr.next = next
-						rr.mu.Unlock()
-						return
-					}
-					if next == rr.next {
-						// Has iterated all the possible address but none is connected.
-						break
-					}
-				}
-			}
-			// The newly added addr got removed by Down() again.
-			if rr.waitCh == nil {
-				ch = make(chan struct{})
-				rr.waitCh = ch
-			} else {
-				ch = rr.waitCh
-			}
-			rr.mu.Unlock()
-		}
-	}
-}
-
-func (rr *roundRobin) Notify() <-chan []Address {
-	return rr.addrCh
-}
-
-func (rr *roundRobin) Close() error {
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
-	if rr.done {
-		return errBalancerClosed
-	}
-	rr.done = true
-	if rr.w != nil {
-		rr.w.Close()
-	}
-	if rr.waitCh != nil {
-		close(rr.waitCh)
-		rr.waitCh = nil
-	}
-	if rr.addrCh != nil {
-		close(rr.addrCh)
-	}
-	return nil
-}
-
-// pickFirst is used to test multi-addresses in one addrConn in which all addresses share the same addrConn.
-// It is a wrapper around roundRobin balancer. The logic of all methods works fine because balancer.Get()
-// returns the only address Up by resetTransport().
-type pickFirst struct {
-	*roundRobin
-}
diff --git a/balancer/balancer.go b/balancer/balancer.go
index 080fe0d189e4..e75b28436047 100644
--- a/balancer/balancer.go
+++ b/balancer/balancer.go
@@ -126,7 +126,7 @@ type State struct {
 	// determine the state of the ClientConn.
 	ConnectivityState connectivity.State
 	// Picker is used to choose connections (SubConns) for RPCs.
-	Picker V2Picker
+	Picker Picker
 }
 
 // ClientConn represents a gRPC ClientConn.
@@ -144,20 +144,11 @@ type ClientConn interface {
 	// The SubConn will be shutdown.
 	RemoveSubConn(SubConn)
 
-	// UpdateBalancerState is called by balancer to notify gRPC that some internal
-	// state in balancer has changed.
-	//
-	// gRPC will update the connectivity state of the ClientConn, and will call pick
-	// on the new picker to pick new SubConn.
-	//
-	// Deprecated: use UpdateState instead
-	UpdateBalancerState(s connectivity.State, p Picker)
-
 	// UpdateState notifies gRPC that the balancer's internal state has
 	// changed.
 	//
-	// gRPC will update the connectivity state of the ClientConn, and will call pick
-	// on the new picker to pick new SubConns.
+	// gRPC will update the connectivity state of the ClientConn, and will call
+	// Pick on the new Picker to pick new SubConns.
 	UpdateState(State)
 
 	// ResolveNow is called by balancer to notify gRPC to do a name resolving.
@@ -235,55 +226,16 @@ type DoneInfo struct {
 
 var (
 	// ErrNoSubConnAvailable indicates no SubConn is available for pick().
-	// gRPC will block the RPC until a new picker is available via UpdateBalancerState().
+	// gRPC will block the RPC until a new picker is available via UpdateState().
 	ErrNoSubConnAvailable = errors.New("no SubConn is available")
 	// ErrTransientFailure indicates all SubConns are in TransientFailure.
 	// WaitForReady RPCs will block, non-WaitForReady RPCs will fail.
-	ErrTransientFailure = TransientFailureError(errors.New("all SubConns are in TransientFailure"))
-)
-
-// Picker is used by gRPC to pick a SubConn to send an RPC.
-// Balancer is expected to generate a new picker from its snapshot every time its
-// internal state has changed.
-//
-// The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState().
-//
-// Deprecated: use V2Picker instead
-type Picker interface {
-	// Pick returns the SubConn to be used to send the RPC.
-	// The returned SubConn must be one returned by NewSubConn().
-	//
-	// This functions is expected to return:
-	// - a SubConn that is known to be READY;
-	// - ErrNoSubConnAvailable if no SubConn is available, but progress is being
-	//   made (for example, some SubConn is in CONNECTING mode);
-	// - other errors if no active connecting is happening (for example, all SubConn
-	//   are in TRANSIENT_FAILURE mode).
-	//
-	// If a SubConn is returned:
-	// - If it is READY, gRPC will send the RPC on it;
-	// - If it is not ready, or becomes not ready after it's returned, gRPC will
-	//   block until UpdateBalancerState() is called and will call pick on the
-	//   new picker. The done function returned from Pick(), if not nil, will be
-	//   called with nil error, no bytes sent and no bytes received.
-	//
-	// If the returned error is not nil:
-	// - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState()
-	// - If the error is ErrTransientFailure or implements IsTransientFailure()
-	//   bool, returning true:
-	//   - If the RPC is wait-for-ready, gRPC will block until UpdateBalancerState()
-	//     is called to pick again;
-	//   - Otherwise, RPC will fail with unavailable error.
-	// - Else (error is other non-nil error):
-	//   - The RPC will fail with the error's status code, or Unknown if it is
-	//     not a status error.
 	//
-	// The returned done() function will be called once the rpc has finished,
-	// with the final status of that RPC.  If the SubConn returned is not a
-	// valid SubConn type, done may not be called.  done may be nil if balancer
-	// doesn't care about the RPC status.
-	Pick(ctx context.Context, info PickInfo) (conn SubConn, done func(DoneInfo), err error)
-}
+	// Deprecated: return an appropriate error based on the last resolution or
+	// connection attempt instead.  The behavior is the same for any non-gRPC
+	// status error.
+	ErrTransientFailure = errors.New("all SubConns are in TransientFailure")
+)
 
 // PickResult contains information related to a connection chosen for an RPC.
 type PickResult struct {
@@ -300,24 +252,19 @@ type PickResult struct {
 	Done func(DoneInfo)
 }
 
-type transientFailureError struct {
-	error
-}
-
-func (e *transientFailureError) IsTransientFailure() bool { return true }
-
-// TransientFailureError wraps err in an error implementing
-// IsTransientFailure() bool, returning true.
-func TransientFailureError(err error) error {
-	return &transientFailureError{error: err}
-}
+// TransientFailureError returns e.  It exists for backward compatibility and
+// will be deleted soon.
+//
+// Deprecated: no longer necessary, picker errors are treated this way by
+// default.
+func TransientFailureError(e error) error { return e }
 
-// V2Picker is used by gRPC to pick a SubConn to send an RPC.
+// Picker is used by gRPC to pick a SubConn to send an RPC.
 // Balancer is expected to generate a new picker from its snapshot every time its
 // internal state has changed.
 //
-// The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState().
-type V2Picker interface {
+// The pickers used by gRPC can be updated by ClientConn.UpdateState().
+type Picker interface {
 	// Pick returns the connection to use for this RPC and related information.
 	//
 	// Pick should not block.  If the balancer needs to do I/O or any blocking
@@ -330,14 +277,13 @@ type V2Picker interface {
 	// - If the error is ErrNoSubConnAvailable, gRPC will block until a new
 	//   Picker is provided by the balancer (using ClientConn.UpdateState).
 	//
-	// - If the error implements IsTransientFailure() bool, returning true,
-	//   wait for ready RPCs will wait, but non-wait for ready RPCs will be
-	//   terminated with this error's Error() string and status code
-	//   Unavailable.
+	// - If the error is a status error (implemented by the grpc/status
+	//   package), gRPC will terminate the RPC with the code and message
+	//   provided.
 	//
-	// - Any other errors terminate all RPCs with the code and message
-	//   provided.  If the error is not a status error, it will be converted by
-	//   gRPC to a status error with code Unknown.
+	// - For all other errors, wait for ready RPCs will wait, but non-wait for
+	//   ready RPCs will be terminated with this error's Error() string and
+	//   status code Unavailable.
 	Pick(info PickInfo) (PickResult, error)
 }
 
@@ -346,34 +292,36 @@ type V2Picker interface {
 //
 // It also generates and updates the Picker used by gRPC to pick SubConns for RPCs.
 //
-// HandleSubConnectionStateChange, HandleResolvedAddrs and Close are guaranteed
-// to be called synchronously from the same goroutine.
-// There's no guarantee on picker.Pick, it may be called anytime.
+// UpdateClientConnState, ResolverError, UpdateSubConnState, and Close are
+// guaranteed to be called synchronously from the same goroutine.  There's no
+// guarantee on picker.Pick, it may be called anytime.
 type Balancer interface {
-	// HandleSubConnStateChange is called by gRPC when the connectivity state
-	// of sc has changed.
-	// Balancer is expected to aggregate all the state of SubConn and report
-	// that back to gRPC.
-	// Balancer should also generate and update Pickers when its internal state has
-	// been changed by the new state.
-	//
-	// Deprecated: if V2Balancer is implemented by the Balancer,
-	// UpdateSubConnState will be called instead.
-	HandleSubConnStateChange(sc SubConn, state connectivity.State)
-	// HandleResolvedAddrs is called by gRPC to send updated resolved addresses to
-	// balancers.
-	// Balancer can create new SubConn or remove SubConn with the addresses.
-	// An empty address slice and a non-nil error will be passed if the resolver returns
-	// non-nil error to gRPC.
-	//
-	// Deprecated: if V2Balancer is implemented by the Balancer,
-	// UpdateClientConnState will be called instead.
-	HandleResolvedAddrs([]resolver.Address, error)
+	// UpdateClientConnState is called by gRPC when the state of the ClientConn
+	// changes.  If the error returned is ErrBadResolverState, the ClientConn
+	// will begin calling ResolveNow on the active name resolver with
+	// exponential backoff until a subsequent call to UpdateClientConnState
+	// returns a nil error.  Any other errors are currently ignored.
+	UpdateClientConnState(ClientConnState) error
+	// ResolverError is called by gRPC when the name resolver reports an error.
+	ResolverError(error)
+	// UpdateSubConnState is called by gRPC when the state of a SubConn
+	// changes.
+	UpdateSubConnState(SubConn, SubConnState)
 	// Close closes the balancer. The balancer is not required to call
 	// ClientConn.RemoveSubConn for its existing SubConns.
 	Close()
 }
 
+// V2Balancer is temporarily defined for backward compatibility reasons.
+//
+// Deprecated: use Balancer directly instead.
+type V2Balancer = Balancer
+
+// V2Picker is temporarily defined for backward compatibility reasons.
+//
+// Deprecated: use Picker directly instead.
+type V2Picker = Picker
+
 // SubConnState describes the state of a SubConn.
 type SubConnState struct {
 	// ConnectivityState is the connectivity state of the SubConn.
@@ -396,27 +344,6 @@ type ClientConnState struct {
 // problem with the provided name resolver data.
 var ErrBadResolverState = errors.New("bad resolver state")
 
-// V2Balancer is defined for documentation purposes.  If a Balancer also
-// implements V2Balancer, its UpdateClientConnState method will be called
-// instead of HandleResolvedAddrs and its UpdateSubConnState will be called
-// instead of HandleSubConnStateChange.
-type V2Balancer interface {
-	// UpdateClientConnState is called by gRPC when the state of the ClientConn
-	// changes.  If the error returned is ErrBadResolverState, the ClientConn
-	// will begin calling ResolveNow on the active name resolver with
-	// exponential backoff until a subsequent call to UpdateClientConnState
-	// returns a nil error.  Any other errors are currently ignored.
-	UpdateClientConnState(ClientConnState) error
-	// ResolverError is called by gRPC when the name resolver reports an error.
-	ResolverError(error)
-	// UpdateSubConnState is called by gRPC when the state of a SubConn
-	// changes.
-	UpdateSubConnState(SubConn, SubConnState)
-	// Close closes the balancer. The balancer is not required to call
-	// ClientConn.RemoveSubConn for its existing SubConns.
-	Close()
-}
-
 // ConnectivityStateEvaluator takes the connectivity states of multiple SubConns
 // and returns one aggregated connectivity state.
 //
diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go
index 80559b80ace6..d62b4b6069a8 100644
--- a/balancer/base/balancer.go
+++ b/balancer/base/balancer.go
@@ -19,7 +19,6 @@
 package base
 
 import (
-	"context"
 	"errors"
 	"fmt"
 
@@ -30,17 +29,15 @@ import (
 )
 
 type baseBuilder struct {
-	name            string
-	pickerBuilder   PickerBuilder
-	v2PickerBuilder V2PickerBuilder
-	config          Config
+	name          string
+	pickerBuilder PickerBuilder
+	config        Config
 }
 
 func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer {
 	bal := &baseBalancer{
-		cc:              cc,
-		pickerBuilder:   bb.pickerBuilder,
-		v2PickerBuilder: bb.v2PickerBuilder,
+		cc:            cc,
+		pickerBuilder: bb.pickerBuilder,
 
 		subConns: make(map[resolver.Address]balancer.SubConn),
 		scStates: make(map[balancer.SubConn]connectivity.State),
@@ -50,11 +47,7 @@ func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions)
 	// Initialize picker to a picker that always returns
 	// ErrNoSubConnAvailable, because when state of a SubConn changes, we
 	// may call UpdateState with this picker.
-	if bb.pickerBuilder != nil {
-		bal.picker = NewErrPicker(balancer.ErrNoSubConnAvailable)
-	} else {
-		bal.v2Picker = NewErrPickerV2(balancer.ErrNoSubConnAvailable)
-	}
+	bal.picker = NewErrPicker(balancer.ErrNoSubConnAvailable)
 	return bal
 }
 
@@ -62,12 +55,9 @@ func (bb *baseBuilder) Name() string {
 	return bb.name
 }
 
-var _ balancer.V2Balancer = (*baseBalancer)(nil) // Assert that we implement V2Balancer
-
 type baseBalancer struct {
-	cc              balancer.ClientConn
-	pickerBuilder   PickerBuilder
-	v2PickerBuilder V2PickerBuilder
+	cc            balancer.ClientConn
+	pickerBuilder PickerBuilder
 
 	csEvltr *balancer.ConnectivityStateEvaluator
 	state   connectivity.State
@@ -75,40 +65,31 @@ type baseBalancer struct {
 	subConns map[resolver.Address]balancer.SubConn
 	scStates map[balancer.SubConn]connectivity.State
 	picker   balancer.Picker
-	v2Picker balancer.V2Picker
 	config   Config
 
 	resolverErr error // the last error reported by the resolver; cleared on successful resolution
 	connErr     error // the last connection error; cleared upon leaving TransientFailure
 }
 
-func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
-	panic("not implemented")
-}
-
 func (b *baseBalancer) ResolverError(err error) {
 	b.resolverErr = err
 	if len(b.subConns) == 0 {
 		b.state = connectivity.TransientFailure
 	}
+
 	if b.state != connectivity.TransientFailure {
 		// The picker will not change since the balancer does not currently
 		// report an error.
 		return
 	}
 	b.regeneratePicker()
-	if b.picker != nil {
-		b.cc.UpdateBalancerState(b.state, b.picker)
-	} else {
-		b.cc.UpdateState(balancer.State{
-			ConnectivityState: b.state,
-			Picker:            b.v2Picker,
-		})
-	}
+	b.cc.UpdateState(balancer.State{
+		ConnectivityState: b.state,
+		Picker:            b.picker,
+	})
 }
 
 func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	// TODO: handle s.ResolverState.Err (log if not nil) once implemented.
 	// TODO: handle s.ResolverState.ServiceConfig?
 	if grpclog.V(2) {
 		grpclog.Infoln("base.baseBalancer: got new ClientConn state: ", s)
@@ -137,7 +118,7 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 			b.cc.RemoveSubConn(sc)
 			delete(b.subConns, a)
 			// Keep the state of this sc in b.scStates until sc's state becomes Shutdown.
-			// The entry will be deleted in HandleSubConnStateChange.
+			// The entry will be deleted in UpdateSubConnState.
 		}
 	}
 	// If resolver state contains no addresses, return an error so ClientConn
@@ -171,38 +152,18 @@ func (b *baseBalancer) mergeErrors() error {
 //  - built by the pickerBuilder with all READY SubConns otherwise.
 func (b *baseBalancer) regeneratePicker() {
 	if b.state == connectivity.TransientFailure {
-		if b.pickerBuilder != nil {
-			b.picker = NewErrPicker(balancer.ErrTransientFailure)
-		} else {
-			b.v2Picker = NewErrPickerV2(balancer.TransientFailureError(b.mergeErrors()))
-		}
+		b.picker = NewErrPicker(b.mergeErrors())
 		return
 	}
-	if b.pickerBuilder != nil {
-		readySCs := make(map[resolver.Address]balancer.SubConn)
+	readySCs := make(map[balancer.SubConn]SubConnInfo)
 
-		// Filter out all ready SCs from full subConn map.
-		for addr, sc := range b.subConns {
-			if st, ok := b.scStates[sc]; ok && st == connectivity.Ready {
-				readySCs[addr] = sc
-			}
-		}
-		b.picker = b.pickerBuilder.Build(readySCs)
-	} else {
-		readySCs := make(map[balancer.SubConn]SubConnInfo)
-
-		// Filter out all ready SCs from full subConn map.
-		for addr, sc := range b.subConns {
-			if st, ok := b.scStates[sc]; ok && st == connectivity.Ready {
-				readySCs[sc] = SubConnInfo{Address: addr}
-			}
+	// Filter out all ready SCs from full subConn map.
+	for addr, sc := range b.subConns {
+		if st, ok := b.scStates[sc]; ok && st == connectivity.Ready {
+			readySCs[sc] = SubConnInfo{Address: addr}
 		}
-		b.v2Picker = b.v2PickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs})
 	}
-}
-
-func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	panic("not implemented")
+	b.picker = b.pickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs})
 }
 
 func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
@@ -247,11 +208,7 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su
 		b.regeneratePicker()
 	}
 
-	if b.picker != nil {
-		b.cc.UpdateBalancerState(b.state, b.picker)
-	} else {
-		b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.v2Picker})
-	}
+	b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker})
 }
 
 // Close is a nop because base balancer doesn't have internal state to clean up,
@@ -259,28 +216,20 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su
 func (b *baseBalancer) Close() {
 }
 
-// NewErrPicker returns a picker that always returns err on Pick().
+// NewErrPicker returns a Picker that always returns err on Pick().
 func NewErrPicker(err error) balancer.Picker {
 	return &errPicker{err: err}
 }
 
-type errPicker struct {
-	err error // Pick() always returns this err.
-}
+// NewErrPickerV2 is temporarily defined for backward compatibility reasons.
+//
+// Deprecated: use NewErrPicker instead.
+var NewErrPickerV2 = NewErrPicker
 
-func (p *errPicker) Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error) {
-	return nil, nil, p.err
-}
-
-// NewErrPickerV2 returns a V2Picker that always returns err on Pick().
-func NewErrPickerV2(err error) balancer.V2Picker {
-	return &errPickerV2{err: err}
-}
-
-type errPickerV2 struct {
+type errPicker struct {
 	err error // Pick() always returns this err.
 }
 
-func (p *errPickerV2) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+func (p *errPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 	return balancer.PickResult{}, p.err
 }
diff --git a/balancer/base/base.go b/balancer/base/base.go
index 4192918b9e28..c4fc89111bf1 100644
--- a/balancer/base/base.go
+++ b/balancer/base/base.go
@@ -37,15 +37,8 @@ import (
 
 // PickerBuilder creates balancer.Picker.
 type PickerBuilder interface {
-	// Build takes a slice of ready SubConns, and returns a picker that will be
-	// used by gRPC to pick a SubConn.
-	Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker
-}
-
-// V2PickerBuilder creates balancer.V2Picker.
-type V2PickerBuilder interface {
 	// Build returns a picker that will be used by gRPC to pick a SubConn.
-	Build(info PickerBuildInfo) balancer.V2Picker
+	Build(info PickerBuildInfo) balancer.Picker
 }
 
 // PickerBuildInfo contains information needed by the picker builder to
@@ -62,20 +55,14 @@ type SubConnInfo struct {
 	Address resolver.Address // the address used to create this SubConn
 }
 
-// NewBalancerBuilder returns a balancer builder. The balancers
-// built by this builder will use the picker builder to build pickers.
-func NewBalancerBuilder(name string, pb PickerBuilder) balancer.Builder {
-	return NewBalancerBuilderWithConfig(name, pb, Config{})
-}
-
 // Config contains the config info about the base balancer builder.
 type Config struct {
 	// HealthCheck indicates whether health checking should be enabled for this specific balancer.
 	HealthCheck bool
 }
 
-// NewBalancerBuilderWithConfig returns a base balancer builder configured by the provided config.
-func NewBalancerBuilderWithConfig(name string, pb PickerBuilder, config Config) balancer.Builder {
+// NewBalancerBuilder returns a base balancer builder configured by the provided config.
+func NewBalancerBuilder(name string, pb PickerBuilder, config Config) balancer.Builder {
 	return &baseBuilder{
 		name:          name,
 		pickerBuilder: pb,
@@ -83,11 +70,13 @@ func NewBalancerBuilderWithConfig(name string, pb PickerBuilder, config Config)
 	}
 }
 
-// NewBalancerBuilderV2 returns a base balancer builder configured by the provided config.
-func NewBalancerBuilderV2(name string, pb V2PickerBuilder, config Config) balancer.Builder {
-	return &baseBuilder{
-		name:            name,
-		v2PickerBuilder: pb,
-		config:          config,
-	}
-}
+// NewBalancerBuilderV2 is temporarily defined for backward compatibility
+// reasons.
+//
+// Deprecated: use NewBalancerBuilder instead.
+var NewBalancerBuilderV2 = NewBalancerBuilder
+
+// V2PickerBuilder is temporarily defined for backward compatibility reasons.
+//
+// Deprecated: use PickerBuilder instead.
+type V2PickerBuilder = PickerBuilder
diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go
index 55ccd065a2ba..193873b0fa13 100644
--- a/balancer/grpclb/grpclb.go
+++ b/balancer/grpclb/grpclb.go
@@ -159,8 +159,6 @@ func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) bal
 	return lb
 }
 
-var _ balancer.V2Balancer = (*lbBalancer)(nil) // Assert that we implement V2Balancer
-
 type lbBalancer struct {
 	cc     *lbCacheClientConn
 	target string
@@ -210,7 +208,7 @@ type lbBalancer struct {
 	state    connectivity.State
 	subConns map[resolver.Address]balancer.SubConn   // Used to new/remove SubConn.
 	scStates map[balancer.SubConn]connectivity.State // Used to filter READY SubConns.
-	picker   balancer.V2Picker
+	picker   balancer.Picker
 	// Support fallback to resolved backend addresses if there's no response
 	// from remote balancer within fallbackTimeout.
 	remoteBalancerConnected bool
@@ -308,10 +306,6 @@ func (lb *lbBalancer) aggregateSubConnStates() connectivity.State {
 	return connectivity.TransientFailure
 }
 
-func (lb *lbBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	panic("not used")
-}
-
 func (lb *lbBalancer) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) {
 	s := scs.ConnectivityState
 	if grpclog.V(2) {
@@ -389,13 +383,6 @@ func (lb *lbBalancer) fallbackToBackendsAfter(fallbackTimeout time.Duration) {
 	lb.mu.Unlock()
 }
 
-// HandleResolvedAddrs sends the updated remoteLB addresses to remoteLB
-// clientConn. The remoteLB clientConn will handle creating/removing remoteLB
-// connections.
-func (lb *lbBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
-	panic("not used")
-}
-
 func (lb *lbBalancer) handleServiceConfig(gc *grpclbServiceConfig) {
 	lb.mu.Lock()
 	defer lb.mu.Unlock()
diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index c6d555e4d8c8..302d71316d59 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -192,7 +192,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback
 			lb.cc.RemoveSubConn(sc)
 			delete(lb.subConns, a)
 			// Keep the state of this sc in b.scStates until sc's state becomes Shutdown.
-			// The entry will be deleted in HandleSubConnStateChange.
+			// The entry will be deleted in UpdateSubConnState.
 		}
 	}
 
diff --git a/balancer/rls/internal/cache/cache.go b/balancer/rls/internal/cache/cache.go
index c945d272ab15..dd03695e0e9d 100644
--- a/balancer/rls/internal/cache/cache.go
+++ b/balancer/rls/internal/cache/cache.go
@@ -85,10 +85,10 @@ type Entry struct {
 	// X-Google-RLS-Data header for matching RPCs.
 	HeaderData string
 	// ChildPicker is a very thin wrapper around the child policy wrapper.
-	// The type is declared as a V2Picker interface since the users of
+	// The type is declared as a Picker interface since the users of
 	// the cache only care about the picker provided by the child policy, and
 	// this makes it easy for testing.
-	ChildPicker balancer.V2Picker
+	ChildPicker balancer.Picker
 
 	// size stores the size of this cache entry. Uses only a subset of the
 	// fields. See `entrySize` for this is computed.
diff --git a/balancer/rls/internal/picker.go b/balancer/rls/internal/picker.go
index e823cf581b47..698185b1595b 100644
--- a/balancer/rls/internal/picker.go
+++ b/balancer/rls/internal/picker.go
@@ -31,10 +31,7 @@ import (
 	"google.golang.org/grpc/metadata"
 )
 
-var errRLSThrottled = balancer.TransientFailureError(errors.New("RLS call throttled at client side"))
-
-// Compile time assert to ensure we implement V2Picker.
-var _ balancer.V2Picker = (*rlsPicker)(nil)
+var errRLSThrottled = errors.New("RLS call throttled at client side")
 
 // RLS rlsPicker selects the subConn to be used for a particular RPC. It does
 // not manage subConns directly and usually deletegates to pickers provided by
diff --git a/balancer/roundrobin/roundrobin.go b/balancer/roundrobin/roundrobin.go
index d4d645501c14..a02b372cf204 100644
--- a/balancer/roundrobin/roundrobin.go
+++ b/balancer/roundrobin/roundrobin.go
@@ -35,7 +35,7 @@ const Name = "round_robin"
 
 // newBuilder creates a new roundrobin balancer builder.
 func newBuilder() balancer.Builder {
-	return base.NewBalancerBuilderV2(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true})
+	return base.NewBalancerBuilder(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true})
 }
 
 func init() {
@@ -44,10 +44,10 @@ func init() {
 
 type rrPickerBuilder struct{}
 
-func (*rrPickerBuilder) Build(info base.PickerBuildInfo) balancer.V2Picker {
+func (*rrPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker {
 	grpclog.Infof("roundrobinPicker: newPicker called with info: %v", info)
 	if len(info.ReadySCs) == 0 {
-		return base.NewErrPickerV2(balancer.ErrNoSubConnAvailable)
+		return base.NewErrPicker(balancer.ErrNoSubConnAvailable)
 	}
 	var scs []balancer.SubConn
 	for sc := range info.ReadySCs {
diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go
index f8667a23f2c8..807d1919777e 100644
--- a/balancer_conn_wrappers.go
+++ b/balancer_conn_wrappers.go
@@ -74,11 +74,7 @@ func (ccb *ccBalancerWrapper) watcher() {
 			}
 			ccb.balancerMu.Lock()
 			su := t.(*scStateUpdate)
-			if ub, ok := ccb.balancer.(balancer.V2Balancer); ok {
-				ub.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state, ConnectionError: su.err})
-			} else {
-				ccb.balancer.HandleSubConnStateChange(su.sc, su.state)
-			}
+			ccb.balancer.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state, ConnectionError: su.err})
 			ccb.balancerMu.Unlock()
 		case <-ccb.done.Done():
 		}
@@ -123,19 +119,13 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co
 func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error {
 	ccb.balancerMu.Lock()
 	defer ccb.balancerMu.Unlock()
-	if ub, ok := ccb.balancer.(balancer.V2Balancer); ok {
-		return ub.UpdateClientConnState(*ccs)
-	}
-	ccb.balancer.HandleResolvedAddrs(ccs.ResolverState.Addresses, nil)
-	return nil
+	return ccb.balancer.UpdateClientConnState(*ccs)
 }
 
 func (ccb *ccBalancerWrapper) resolverError(err error) {
-	if ub, ok := ccb.balancer.(balancer.V2Balancer); ok {
-		ccb.balancerMu.Lock()
-		ub.ResolverError(err)
-		ccb.balancerMu.Unlock()
-	}
+	ccb.balancerMu.Lock()
+	ccb.balancer.ResolverError(err)
+	ccb.balancerMu.Unlock()
 }
 
 func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
@@ -173,21 +163,6 @@ func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) {
 	ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain)
 }
 
-func (ccb *ccBalancerWrapper) UpdateBalancerState(s connectivity.State, p balancer.Picker) {
-	ccb.mu.Lock()
-	defer ccb.mu.Unlock()
-	if ccb.subConns == nil {
-		return
-	}
-	// Update picker before updating state.  Even though the ordering here does
-	// not matter, it can lead to multiple calls of Pick in the common start-up
-	// case where we wait for ready and then perform an RPC.  If the picker is
-	// updated later, we could call the "connecting" picker when the state is
-	// updated, and then call the "ready" picker after the picker gets updated.
-	ccb.cc.blockingpicker.updatePicker(p)
-	ccb.cc.csMgr.updateState(s)
-}
-
 func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) {
 	ccb.mu.Lock()
 	defer ccb.mu.Unlock()
@@ -199,7 +174,7 @@ func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) {
 	// case where we wait for ready and then perform an RPC.  If the picker is
 	// updated later, we could call the "connecting" picker when the state is
 	// updated, and then call the "ready" picker after the picker gets updated.
-	ccb.cc.blockingpicker.updatePickerV2(s.Picker)
+	ccb.cc.blockingpicker.updatePicker(s.Picker)
 	ccb.cc.csMgr.updateState(s.ConnectivityState)
 }
 
diff --git a/balancer_conn_wrappers_test.go b/balancer_conn_wrappers_test.go
index 33a439f806d1..935d11d1d391 100644
--- a/balancer_conn_wrappers_test.go
+++ b/balancer_conn_wrappers_test.go
@@ -25,50 +25,19 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
-	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 )
 
-var _ balancer.V2Balancer = &funcBalancer{}
-
-type funcBalancer struct {
-	updateClientConnState func(s balancer.ClientConnState) error
-}
-
-func (*funcBalancer) HandleSubConnStateChange(balancer.SubConn, connectivity.State) {
-	panic("unimplemented") // v1 API
-}
-func (*funcBalancer) HandleResolvedAddrs([]resolver.Address, error) {
-	panic("unimplemented") // v1 API
-}
-func (b *funcBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	return b.updateClientConnState(s)
-}
-func (*funcBalancer) ResolverError(error) {}
-func (*funcBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {
-	panic("unimplemented") // we never have sub-conns
-}
-func (*funcBalancer) Close() {}
-
-type funcBalancerBuilder struct {
-	name     string
-	instance *funcBalancer
-}
-
-func (b *funcBalancerBuilder) Build(balancer.ClientConn, balancer.BuildOptions) balancer.Balancer {
-	return b.instance
-}
-func (b *funcBalancerBuilder) Name() string { return b.name }
-
 // TestBalancerErrorResolverPolling injects balancer errors and verifies
 // ResolveNow is called on the resolver with the appropriate backoff strategy
 // being consulted between ResolveNow calls.
 func (s) TestBalancerErrorResolverPolling(t *testing.T) {
 	// The test balancer will return ErrBadResolverState iff the
 	// ClientConnState contains no addresses.
-	fb := &funcBalancer{
-		updateClientConnState: func(s balancer.ClientConnState) error {
+	bf := stub.BalancerFuncs{
+		UpdateClientConnState: func(_ *stub.BalancerData, s balancer.ClientConnState) error {
 			if len(s.ResolverState.Addresses) == 0 {
 				return balancer.ErrBadResolverState
 			}
@@ -76,7 +45,7 @@ func (s) TestBalancerErrorResolverPolling(t *testing.T) {
 		},
 	}
 	const balName = "BalancerErrorResolverPolling"
-	balancer.Register(&funcBalancerBuilder{name: balName, instance: fb})
+	stub.Register(balName, bf)
 
 	testResolverErrorPolling(t,
 		func(r *manual.Resolver) {
diff --git a/balancer_switching_test.go b/balancer_switching_test.go
index 1ccbaba6cdc0..f47754bfdfeb 100644
--- a/balancer_switching_test.go
+++ b/balancer_switching_test.go
@@ -27,7 +27,6 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
-	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
@@ -48,9 +47,13 @@ func (b *magicalLB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) ba
 	return b
 }
 
-func (b *magicalLB) HandleSubConnStateChange(balancer.SubConn, connectivity.State) {}
+func (b *magicalLB) ResolverError(error) {}
 
-func (b *magicalLB) HandleResolvedAddrs([]resolver.Address, error) {}
+func (b *magicalLB) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {}
+
+func (b *magicalLB) UpdateClientConnState(balancer.ClientConnState) error {
+	return nil
+}
 
 func (b *magicalLB) Close() {}
 
@@ -58,6 +61,21 @@ func init() {
 	balancer.Register(&magicalLB{})
 }
 
+func startServers(t *testing.T, numServers int, maxStreams uint32) ([]*server, func()) {
+	var servers []*server
+	for i := 0; i < numServers; i++ {
+		s := newTestServer()
+		servers = append(servers, s)
+		go s.start(t, 0, maxStreams)
+		s.wait(t, 2*time.Second)
+	}
+	return servers, func() {
+		for i := 0; i < numServers; i++ {
+			servers[i].stop()
+		}
+	}
+}
+
 func checkPickFirst(cc *ClientConn, servers []*server) error {
 	var (
 		req   = "port"
@@ -133,7 +151,7 @@ func (s) TestSwitchBalancer(t *testing.T) {
 	defer rcleanup()
 
 	const numServers = 2
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -165,7 +183,7 @@ func (s) TestBalancerDialOption(t *testing.T) {
 	defer rcleanup()
 
 	const numServers = 2
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}), WithBalancerName(roundrobin.Name))
@@ -481,7 +499,7 @@ func (s) TestSwitchBalancerGRPCLBWithGRPCLBNotRegistered(t *testing.T) {
 	defer rcleanup()
 
 	const numServers = 3
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
diff --git a/balancer_test.go b/balancer_test.go
deleted file mode 100644
index 524fd43a6f52..000000000000
--- a/balancer_test.go
+++ /dev/null
@@ -1,789 +0,0 @@
-/*
- *
- * Copyright 2016 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package grpc
-
-import (
-	"context"
-	"fmt"
-	"math"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/naming"
-	"google.golang.org/grpc/status"
-)
-
-func pickFirstBalancerV1(r naming.Resolver) Balancer {
-	return &pickFirst{&roundRobin{r: r}}
-}
-
-type testWatcher struct {
-	// the channel to receives name resolution updates
-	update chan *naming.Update
-	// the side channel to get to know how many updates in a batch
-	side chan int
-	// the channel to notify update injector that the update reading is done
-	readDone chan int
-}
-
-func (w *testWatcher) Next() (updates []*naming.Update, err error) {
-	n := <-w.side
-	if n == 0 {
-		return nil, fmt.Errorf("w.side is closed")
-	}
-	for i := 0; i < n; i++ {
-		u := <-w.update
-		if u != nil {
-			updates = append(updates, u)
-		}
-	}
-	w.readDone <- 0
-	return
-}
-
-func (w *testWatcher) Close() {
-	close(w.side)
-}
-
-// Inject naming resolution updates to the testWatcher.
-func (w *testWatcher) inject(updates []*naming.Update) {
-	w.side <- len(updates)
-	for _, u := range updates {
-		w.update <- u
-	}
-	<-w.readDone
-}
-
-type testNameResolver struct {
-	w    *testWatcher
-	addr string
-}
-
-func (r *testNameResolver) Resolve(target string) (naming.Watcher, error) {
-	r.w = &testWatcher{
-		update:   make(chan *naming.Update, 1),
-		side:     make(chan int, 1),
-		readDone: make(chan int),
-	}
-	r.w.side <- 1
-	r.w.update <- &naming.Update{
-		Op:   naming.Add,
-		Addr: r.addr,
-	}
-	go func() {
-		<-r.w.readDone
-	}()
-	return r.w, nil
-}
-
-func startServers(t *testing.T, numServers int, maxStreams uint32) ([]*server, *testNameResolver, func()) {
-	var servers []*server
-	for i := 0; i < numServers; i++ {
-		s := newTestServer()
-		servers = append(servers, s)
-		go s.start(t, 0, maxStreams)
-		s.wait(t, 2*time.Second)
-	}
-	// Point to server[0]
-	addr := "localhost:" + servers[0].port
-	return servers, &testNameResolver{
-			addr: addr,
-		}, func() {
-			for i := 0; i < numServers; i++ {
-				servers[i].stop()
-			}
-		}
-}
-
-func (s) TestNameDiscovery(t *testing.T) {
-	// Start 2 servers on 2 ports.
-	numServers := 2
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	req := "port"
-	var reply string
-	if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port {
-		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want %s", err, servers[0].port)
-	}
-	// Inject the name resolution change to remove servers[0] and add servers[1].
-	var updates []*naming.Update
-	updates = append(updates, &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	})
-	updates = append(updates, &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	})
-	r.w.inject(updates)
-	// Loop until the rpcs in flight talks to servers[1].
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-}
-
-func (s) TestEmptyAddrs(t *testing.T) {
-	servers, r, cleanup := startServers(t, 1, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	var reply string
-	if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse {
-		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, reply = %q, want %q, <nil>", err, reply, expectedResponse)
-	}
-	// Inject name resolution change to remove the server so that there is no address
-	// available after that.
-	u := &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	// Loop until the above updates apply.
-	for {
-		time.Sleep(10 * time.Millisecond)
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
-		if err := cc.Invoke(ctx, "/foo/bar", &expectedRequest, &reply); err != nil {
-			cancel()
-			break
-		}
-		cancel()
-	}
-}
-
-func (s) TestRoundRobin(t *testing.T) {
-	// Start 3 servers on 3 ports.
-	numServers := 3
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Add servers[1] to the service discovery.
-	u := &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	req := "port"
-	var reply string
-	// Loop until servers[1] is up
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	// Add server2[2] to the service discovery.
-	u = &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[2].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	// Loop until both servers[2] are up.
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[2].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	// Check the incoming RPCs served in a round-robin manner.
-	for i := 0; i < 10; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[i%numServers].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", i, err, servers[i%numServers].port)
-		}
-	}
-}
-
-func (s) TestCloseWithPendingRPC(t *testing.T) {
-	servers, r, cleanup := startServers(t, 1, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	var reply string
-	if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err != nil {
-		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want %s", err, servers[0].port)
-	}
-	// Remove the server.
-	updates := []*naming.Update{{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	}}
-	r.w.inject(updates)
-	// Loop until the above update applies.
-	for {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
-		if err := cc.Invoke(ctx, "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); status.Code(err) == codes.DeadlineExceeded {
-			cancel()
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-		cancel()
-	}
-	// Issue 2 RPCs which should be completed with error status once cc is closed.
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		var reply string
-		if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err == nil {
-			t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want not nil", err)
-		}
-	}()
-	go func() {
-		defer wg.Done()
-		var reply string
-		time.Sleep(5 * time.Millisecond)
-		if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err == nil {
-			t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want not nil", err)
-		}
-	}()
-	time.Sleep(5 * time.Millisecond)
-	cc.Close()
-	wg.Wait()
-}
-
-func (s) TestGetOnWaitChannel(t *testing.T) {
-	servers, r, cleanup := startServers(t, 1, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Remove all servers so that all upcoming RPCs will block on waitCh.
-	updates := []*naming.Update{{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	}}
-	r.w.inject(updates)
-	for {
-		var reply string
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
-		if err := cc.Invoke(ctx, "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); status.Code(err) == codes.DeadlineExceeded {
-			cancel()
-			break
-		}
-		cancel()
-		time.Sleep(10 * time.Millisecond)
-	}
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		var reply string
-		if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err != nil {
-			t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want <nil>", err)
-		}
-	}()
-	// Add a connected server to get the above RPC through.
-	updates = []*naming.Update{{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[0].port,
-	}}
-	r.w.inject(updates)
-	// Wait until the above RPC succeeds.
-	wg.Wait()
-}
-
-func (s) TestOneServerDown(t *testing.T) {
-	// Start 2 servers.
-	numServers := 2
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Add servers[1] to the service discovery.
-	var updates []*naming.Update
-	updates = append(updates, &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	})
-	r.w.inject(updates)
-	req := "port"
-	var reply string
-	// Loop until servers[1] is up
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	var wg sync.WaitGroup
-	numRPC := 100
-	sleepDuration := 10 * time.Millisecond
-	wg.Add(1)
-	go func() {
-		time.Sleep(sleepDuration)
-		// After sleepDuration, kill server[0].
-		servers[0].stop()
-		wg.Done()
-	}()
-
-	// All non-failfast RPCs should not block because there's at least one connection available.
-	for i := 0; i < numRPC; i++ {
-		wg.Add(1)
-		go func() {
-			time.Sleep(sleepDuration)
-			// After sleepDuration, invoke RPC.
-			// server[0] is killed around the same time to make it racy between balancer and gRPC internals.
-			cc.Invoke(context.Background(), "/foo/bar", &req, &reply, WaitForReady(true))
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-}
-
-func (s) TestOneAddressRemoval(t *testing.T) {
-	// Start 2 servers.
-	numServers := 2
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Add servers[1] to the service discovery.
-	var updates []*naming.Update
-	updates = append(updates, &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	})
-	r.w.inject(updates)
-	req := "port"
-	var reply string
-	// Loop until servers[1] is up
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	var wg sync.WaitGroup
-	numRPC := 100
-	sleepDuration := 10 * time.Millisecond
-	wg.Add(1)
-	go func() {
-		time.Sleep(sleepDuration)
-		// After sleepDuration, delete server[0].
-		var updates []*naming.Update
-		updates = append(updates, &naming.Update{
-			Op:   naming.Delete,
-			Addr: "localhost:" + servers[0].port,
-		})
-		r.w.inject(updates)
-		wg.Done()
-	}()
-
-	// All non-failfast RPCs should not fail because there's at least one connection available.
-	for i := 0; i < numRPC; i++ {
-		wg.Add(1)
-		go func() {
-			var reply string
-			time.Sleep(sleepDuration)
-			// After sleepDuration, invoke RPC.
-			// server[0] is removed around the same time to make it racy between balancer and gRPC internals.
-			if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err != nil {
-				t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want nil", err)
-			}
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-}
-
-func checkServerUp(t *testing.T, currentServer *server) {
-	req := "port"
-	port := currentServer.port
-	cc, err := Dial("passthrough:///localhost:"+port, WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	var reply string
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-}
-
-func (s) TestPickFirstEmptyAddrs(t *testing.T) {
-	servers, r, cleanup := startServers(t, 1, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(pickFirstBalancerV1(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	var reply string
-	if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse {
-		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, reply = %q, want %q, <nil>", err, reply, expectedResponse)
-	}
-	// Inject name resolution change to remove the server so that there is no address
-	// available after that.
-	u := &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	// Loop until the above updates apply.
-	for {
-		time.Sleep(10 * time.Millisecond)
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
-		if err := cc.Invoke(ctx, "/foo/bar", &expectedRequest, &reply); err != nil {
-			cancel()
-			break
-		}
-		cancel()
-	}
-}
-
-func (s) TestPickFirstCloseWithPendingRPC(t *testing.T) {
-	servers, r, cleanup := startServers(t, 1, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(pickFirstBalancerV1(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	var reply string
-	if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err != nil {
-		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want %s", err, servers[0].port)
-	}
-	// Remove the server.
-	updates := []*naming.Update{{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	}}
-	r.w.inject(updates)
-	// Loop until the above update applies.
-	for {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
-		if err := cc.Invoke(ctx, "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); status.Code(err) == codes.DeadlineExceeded {
-			cancel()
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-		cancel()
-	}
-	// Issue 2 RPCs which should be completed with error status once cc is closed.
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		var reply string
-		if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err == nil {
-			t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want not nil", err)
-		}
-	}()
-	go func() {
-		defer wg.Done()
-		var reply string
-		time.Sleep(5 * time.Millisecond)
-		if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err == nil {
-			t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want not nil", err)
-		}
-	}()
-	time.Sleep(5 * time.Millisecond)
-	cc.Close()
-	wg.Wait()
-}
-
-func (s) TestPickFirstOrderAllServerUp(t *testing.T) {
-	// Start 3 servers on 3 ports.
-	numServers := 3
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(pickFirstBalancerV1(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Add servers[1] and [2] to the service discovery.
-	u := &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	}
-	r.w.inject([]*naming.Update{u})
-
-	u = &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[2].port,
-	}
-	r.w.inject([]*naming.Update{u})
-
-	// Loop until all 3 servers are up
-	checkServerUp(t, servers[0])
-	checkServerUp(t, servers[1])
-	checkServerUp(t, servers[2])
-
-	// Check the incoming RPCs served in server[0]
-	req := "port"
-	var reply string
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 0, err, servers[0].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// Delete server[0] in the balancer, the incoming RPCs served in server[1]
-	// For test addrconn, close server[0] instead
-	u = &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[0].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	// Loop until it changes to server[1]
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[1].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 1, err, servers[1].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// Add server[0] back to the balancer, the incoming RPCs served in server[1]
-	// Add is append operation, the order of Notify now is {server[1].port server[2].port server[0].port}
-	u = &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[0].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[1].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 1, err, servers[1].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// Delete server[1] in the balancer, the incoming RPCs served in server[2]
-	u = &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[1].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[2].port {
-			break
-		}
-		time.Sleep(1 * time.Second)
-	}
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[2].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 2, err, servers[2].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// Delete server[2] in the balancer, the incoming RPCs served in server[0]
-	u = &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[2].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port {
-			break
-		}
-		time.Sleep(1 * time.Second)
-	}
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 0, err, servers[0].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-}
-
-func (s) TestPickFirstOrderOneServerDown(t *testing.T) {
-	// Start 3 servers on 3 ports.
-	numServers := 3
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(pickFirstBalancerV1(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Add servers[1] and [2] to the service discovery.
-	u := &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	}
-	r.w.inject([]*naming.Update{u})
-
-	u = &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[2].port,
-	}
-	r.w.inject([]*naming.Update{u})
-
-	// Loop until all 3 servers are up
-	checkServerUp(t, servers[0])
-	checkServerUp(t, servers[1])
-	checkServerUp(t, servers[2])
-
-	// Check the incoming RPCs served in server[0]
-	req := "port"
-	var reply string
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 0, err, servers[0].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// server[0] down, incoming RPCs served in server[1], but the order of Notify still remains
-	// {server[0] server[1] server[2]}
-	servers[0].stop()
-	// Loop until it changes to server[1]
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[1].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 1, err, servers[1].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// up the server[0] back, the incoming RPCs served in server[1]
-	p, _ := strconv.Atoi(servers[0].port)
-	servers[0] = newTestServer()
-	go servers[0].start(t, p, math.MaxUint32)
-	defer servers[0].stop()
-	servers[0].wait(t, 2*time.Second)
-	checkServerUp(t, servers[0])
-
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[1].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 1, err, servers[1].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	// Delete server[1] in the balancer, the incoming RPCs served in server[0]
-	u = &naming.Update{
-		Op:   naming.Delete,
-		Addr: "localhost:" + servers[1].port,
-	}
-	r.w.inject([]*naming.Update{u})
-	for {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port {
-			break
-		}
-		time.Sleep(1 * time.Second)
-	}
-	for i := 0; i < 20; i++ {
-		if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port {
-			t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 0, err, servers[0].port)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-}
-
-func (s) TestPickFirstOneAddressRemoval(t *testing.T) {
-	// Start 2 servers.
-	numServers := 2
-	servers, r, cleanup := startServers(t, numServers, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///localhost:"+servers[0].port, WithBalancer(pickFirstBalancerV1(r)), WithBlock(), WithInsecure(), WithCodec(testCodec{}))
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	defer cc.Close()
-	// Add servers[1] to the service discovery.
-	var updates []*naming.Update
-	updates = append(updates, &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	})
-	r.w.inject(updates)
-
-	// Create a new cc to Loop until servers[1] is up
-	checkServerUp(t, servers[0])
-	checkServerUp(t, servers[1])
-
-	var wg sync.WaitGroup
-	numRPC := 100
-	sleepDuration := 10 * time.Millisecond
-	wg.Add(1)
-	go func() {
-		time.Sleep(sleepDuration)
-		// After sleepDuration, delete server[0].
-		var updates []*naming.Update
-		updates = append(updates, &naming.Update{
-			Op:   naming.Delete,
-			Addr: "localhost:" + servers[0].port,
-		})
-		r.w.inject(updates)
-		wg.Done()
-	}()
-
-	// All non-failfast RPCs should not fail because there's at least one connection available.
-	for i := 0; i < numRPC; i++ {
-		wg.Add(1)
-		go func() {
-			var reply string
-			time.Sleep(sleepDuration)
-			// After sleepDuration, invoke RPC.
-			// server[0] is removed around the same time to make it racy between balancer and gRPC internals.
-			if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply, WaitForReady(true)); err != nil {
-				t.Errorf("grpc.Invoke(_, _, _, _, _) = %v, want nil", err)
-			}
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-}
diff --git a/balancer_v1_wrapper.go b/balancer_v1_wrapper.go
deleted file mode 100644
index db04b08b8429..000000000000
--- a/balancer_v1_wrapper.go
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package grpc
-
-import (
-	"sync"
-
-	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/grpclog"
-	"google.golang.org/grpc/resolver"
-)
-
-type balancerWrapperBuilder struct {
-	b Balancer // The v1 balancer.
-}
-
-func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
-	bwb.b.Start(opts.Target.Endpoint, BalancerConfig{
-		DialCreds: opts.DialCreds,
-		Dialer:    opts.Dialer,
-	})
-	_, pickfirst := bwb.b.(*pickFirst)
-	bw := &balancerWrapper{
-		balancer:   bwb.b,
-		pickfirst:  pickfirst,
-		cc:         cc,
-		targetAddr: opts.Target.Endpoint,
-		startCh:    make(chan struct{}),
-		conns:      make(map[resolver.Address]balancer.SubConn),
-		connSt:     make(map[balancer.SubConn]*scState),
-		csEvltr:    &balancer.ConnectivityStateEvaluator{},
-		state:      connectivity.Idle,
-	}
-	cc.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: bw})
-	go bw.lbWatcher()
-	return bw
-}
-
-func (bwb *balancerWrapperBuilder) Name() string {
-	return "wrapper"
-}
-
-type scState struct {
-	addr Address // The v1 address type.
-	s    connectivity.State
-	down func(error)
-}
-
-type balancerWrapper struct {
-	balancer  Balancer // The v1 balancer.
-	pickfirst bool
-
-	cc         balancer.ClientConn
-	targetAddr string // Target without the scheme.
-
-	mu     sync.Mutex
-	conns  map[resolver.Address]balancer.SubConn
-	connSt map[balancer.SubConn]*scState
-	// This channel is closed when handling the first resolver result.
-	// lbWatcher blocks until this is closed, to avoid race between
-	// - NewSubConn is created, cc wants to notify balancer of state changes;
-	// - Build hasn't return, cc doesn't have access to balancer.
-	startCh chan struct{}
-
-	// To aggregate the connectivity state.
-	csEvltr *balancer.ConnectivityStateEvaluator
-	state   connectivity.State
-}
-
-// lbWatcher watches the Notify channel of the balancer and manages
-// connections accordingly.
-func (bw *balancerWrapper) lbWatcher() {
-	<-bw.startCh
-	notifyCh := bw.balancer.Notify()
-	if notifyCh == nil {
-		// There's no resolver in the balancer. Connect directly.
-		a := resolver.Address{
-			Addr: bw.targetAddr,
-			Type: resolver.Backend,
-		}
-		sc, err := bw.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{})
-		if err != nil {
-			grpclog.Warningf("Error creating connection to %v. Err: %v", a, err)
-		} else {
-			bw.mu.Lock()
-			bw.conns[a] = sc
-			bw.connSt[sc] = &scState{
-				addr: Address{Addr: bw.targetAddr},
-				s:    connectivity.Idle,
-			}
-			bw.mu.Unlock()
-			sc.Connect()
-		}
-		return
-	}
-
-	for addrs := range notifyCh {
-		grpclog.Infof("balancerWrapper: got update addr from Notify: %v", addrs)
-		if bw.pickfirst {
-			var (
-				oldA  resolver.Address
-				oldSC balancer.SubConn
-			)
-			bw.mu.Lock()
-			for oldA, oldSC = range bw.conns {
-				break
-			}
-			bw.mu.Unlock()
-			if len(addrs) <= 0 {
-				if oldSC != nil {
-					// Teardown old sc.
-					bw.mu.Lock()
-					delete(bw.conns, oldA)
-					delete(bw.connSt, oldSC)
-					bw.mu.Unlock()
-					bw.cc.RemoveSubConn(oldSC)
-				}
-				continue
-			}
-
-			var newAddrs []resolver.Address
-			for _, a := range addrs {
-				newAddr := resolver.Address{
-					Addr:       a.Addr,
-					Type:       resolver.Backend, // All addresses from balancer are all backends.
-					ServerName: "",
-					Metadata:   a.Metadata,
-				}
-				newAddrs = append(newAddrs, newAddr)
-			}
-			if oldSC == nil {
-				// Create new sc.
-				sc, err := bw.cc.NewSubConn(newAddrs, balancer.NewSubConnOptions{})
-				if err != nil {
-					grpclog.Warningf("Error creating connection to %v. Err: %v", newAddrs, err)
-				} else {
-					bw.mu.Lock()
-					// For pickfirst, there should be only one SubConn, so the
-					// address doesn't matter. All states updating (up and down)
-					// and picking should all happen on that only SubConn.
-					bw.conns[resolver.Address{}] = sc
-					bw.connSt[sc] = &scState{
-						addr: addrs[0], // Use the first address.
-						s:    connectivity.Idle,
-					}
-					bw.mu.Unlock()
-					sc.Connect()
-				}
-			} else {
-				bw.mu.Lock()
-				bw.connSt[oldSC].addr = addrs[0]
-				bw.mu.Unlock()
-				oldSC.UpdateAddresses(newAddrs)
-			}
-		} else {
-			var (
-				add []resolver.Address // Addresses need to setup connections.
-				del []balancer.SubConn // Connections need to tear down.
-			)
-			resAddrs := make(map[resolver.Address]Address)
-			for _, a := range addrs {
-				resAddrs[resolver.Address{
-					Addr:       a.Addr,
-					Type:       resolver.Backend, // All addresses from balancer are all backends.
-					ServerName: "",
-					Metadata:   a.Metadata,
-				}] = a
-			}
-			bw.mu.Lock()
-			for a := range resAddrs {
-				if _, ok := bw.conns[a]; !ok {
-					add = append(add, a)
-				}
-			}
-			for a, c := range bw.conns {
-				if _, ok := resAddrs[a]; !ok {
-					del = append(del, c)
-					delete(bw.conns, a)
-					// Keep the state of this sc in bw.connSt until its state becomes Shutdown.
-				}
-			}
-			bw.mu.Unlock()
-			for _, a := range add {
-				sc, err := bw.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{})
-				if err != nil {
-					grpclog.Warningf("Error creating connection to %v. Err: %v", a, err)
-				} else {
-					bw.mu.Lock()
-					bw.conns[a] = sc
-					bw.connSt[sc] = &scState{
-						addr: resAddrs[a],
-						s:    connectivity.Idle,
-					}
-					bw.mu.Unlock()
-					sc.Connect()
-				}
-			}
-			for _, c := range del {
-				bw.cc.RemoveSubConn(c)
-			}
-		}
-	}
-}
-
-func (bw *balancerWrapper) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	bw.mu.Lock()
-	defer bw.mu.Unlock()
-	scSt, ok := bw.connSt[sc]
-	if !ok {
-		return
-	}
-	if s == connectivity.Idle {
-		sc.Connect()
-	}
-	oldS := scSt.s
-	scSt.s = s
-	if oldS != connectivity.Ready && s == connectivity.Ready {
-		scSt.down = bw.balancer.Up(scSt.addr)
-	} else if oldS == connectivity.Ready && s != connectivity.Ready {
-		if scSt.down != nil {
-			scSt.down(errConnClosing)
-		}
-	}
-	sa := bw.csEvltr.RecordTransition(oldS, s)
-	if bw.state != sa {
-		bw.state = sa
-	}
-	bw.cc.UpdateState(balancer.State{ConnectivityState: bw.state, Picker: bw})
-	if s == connectivity.Shutdown {
-		// Remove state for this sc.
-		delete(bw.connSt, sc)
-	}
-}
-
-func (bw *balancerWrapper) HandleResolvedAddrs([]resolver.Address, error) {
-	bw.mu.Lock()
-	defer bw.mu.Unlock()
-	select {
-	case <-bw.startCh:
-	default:
-		close(bw.startCh)
-	}
-	// There should be a resolver inside the balancer.
-	// All updates here, if any, are ignored.
-}
-
-func (bw *balancerWrapper) Close() {
-	bw.mu.Lock()
-	defer bw.mu.Unlock()
-	select {
-	case <-bw.startCh:
-	default:
-		close(bw.startCh)
-	}
-	bw.balancer.Close()
-}
-
-// The picker is the balancerWrapper itself.
-// It either blocks or returns error, consistent with v1 balancer Get().
-func (bw *balancerWrapper) Pick(info balancer.PickInfo) (result balancer.PickResult, err error) {
-	failfast := true // Default failfast is true.
-	if ss, ok := rpcInfoFromContext(info.Ctx); ok {
-		failfast = ss.failfast
-	}
-	a, p, err := bw.balancer.Get(info.Ctx, BalancerGetOptions{BlockingWait: !failfast})
-	if err != nil {
-		return balancer.PickResult{}, toRPCErr(err)
-	}
-	if p != nil {
-		result.Done = func(balancer.DoneInfo) { p() }
-		defer func() {
-			if err != nil {
-				p()
-			}
-		}()
-	}
-
-	bw.mu.Lock()
-	defer bw.mu.Unlock()
-	if bw.pickfirst {
-		// Get the first sc in conns.
-		for _, result.SubConn = range bw.conns {
-			return result, nil
-		}
-		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
-	}
-	var ok1 bool
-	result.SubConn, ok1 = bw.conns[resolver.Address{
-		Addr:       a.Addr,
-		Type:       resolver.Backend,
-		ServerName: "",
-		Metadata:   a.Metadata,
-	}]
-	s, ok2 := bw.connSt[result.SubConn]
-	if !ok1 || !ok2 {
-		// This can only happen due to a race where Get() returned an address
-		// that was subsequently removed by Notify.  In this case we should
-		// retry always.
-		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
-	}
-	switch s.s {
-	case connectivity.Ready, connectivity.Idle:
-		return result, nil
-	case connectivity.Shutdown, connectivity.TransientFailure:
-		// If the returned sc has been shut down or is in transient failure,
-		// return error, and this RPC will fail or wait for another picker (if
-		// non-failfast).
-		return balancer.PickResult{}, balancer.ErrTransientFailure
-	default:
-		// For other states (connecting or unknown), the v1 balancer would
-		// traditionally wait until ready and then issue the RPC.  Returning
-		// ErrNoSubConnAvailable will be a slight improvement in that it will
-		// allow the balancer to choose another address in case others are
-		// connected.
-		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
-	}
-}
diff --git a/clientconn.go b/clientconn.go
index 3e4b42b3dd47..ef327e8af4f7 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -68,8 +68,6 @@ var (
 	errConnDrain = errors.New("grpc: the connection is drained")
 	// errConnClosing indicates that the connection is closing.
 	errConnClosing = errors.New("grpc: the connection is closing")
-	// errBalancerClosed indicates that the balancer is closed.
-	errBalancerClosed = errors.New("grpc: balancer is closed")
 	// invalidDefaultServiceConfigErrPrefix is used to prefix the json parsing error for the default
 	// service config.
 	invalidDefaultServiceConfigErrPrefix = "grpc: the provided default service config is invalid"
@@ -318,7 +316,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 			if s == connectivity.Ready {
 				break
 			} else if cc.dopts.copts.FailOnNonTempDialError && s == connectivity.TransientFailure {
-				if err = cc.blockingpicker.connectionError(); err != nil {
+				if err = cc.connectionError(); err != nil {
 					terr, ok := err.(interface {
 						Temporary() bool
 					})
@@ -329,7 +327,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 			}
 			if !cc.WaitForStateChange(ctx, s) {
 				// ctx got timeout or canceled.
-				if err = cc.blockingpicker.connectionError(); err != nil && cc.dopts.returnLastError {
+				if err = cc.connectionError(); err != nil && cc.dopts.returnLastError {
 					return nil, err
 				}
 				return nil, ctx.Err()
@@ -500,6 +498,9 @@ type ClientConn struct {
 
 	channelzID int64 // channelz unique identification number
 	czData     *channelzData
+
+	lceMu               sync.Mutex // protects lastConnectionError
+	lastConnectionError error
 }
 
 // WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or
@@ -1209,7 +1210,7 @@ func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.T
 		if firstConnErr == nil {
 			firstConnErr = err
 		}
-		ac.cc.blockingpicker.updateConnectionError(err)
+		ac.cc.updateConnectionError(err)
 	}
 
 	// Couldn't connect to any address.
@@ -1535,3 +1536,15 @@ func (cc *ClientConn) getResolver(scheme string) resolver.Builder {
 	}
 	return resolver.Get(scheme)
 }
+
+func (cc *ClientConn) updateConnectionError(err error) {
+	cc.lceMu.Lock()
+	cc.lastConnectionError = err
+	cc.lceMu.Unlock()
+}
+
+func (cc *ClientConn) connectionError() error {
+	cc.lceMu.Lock()
+	defer cc.lceMu.Unlock()
+	return cc.lastConnectionError
+}
diff --git a/clientconn_state_transition_test.go b/clientconn_state_transition_test.go
index 0e9b1e752156..0c58131a1c6f 100644
--- a/clientconn_state_transition_test.go
+++ b/clientconn_state_transition_test.go
@@ -449,9 +449,9 @@ type stateRecordingBalancer struct {
 	balancer.Balancer
 }
 
-func (b *stateRecordingBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	b.notifier <- s
-	b.Balancer.HandleSubConnStateChange(sc, s)
+func (b *stateRecordingBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	b.notifier <- s.ConnectivityState
+	b.Balancer.UpdateSubConnState(sc, s)
 }
 
 func (b *stateRecordingBalancer) ResetNotifier(r chan<- connectivity.State) {
diff --git a/clientconn_test.go b/clientconn_test.go
index dd4a2b427bfa..524b9736c1a0 100644
--- a/clientconn_test.go
+++ b/clientconn_test.go
@@ -36,21 +36,11 @@ import (
 	internalbackoff "google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/keepalive"
-	"google.golang.org/grpc/naming"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 	"google.golang.org/grpc/testdata"
 )
 
-func assertState(wantState connectivity.State, cc *ClientConn) (connectivity.State, bool) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-	var state connectivity.State
-	for state = cc.GetState(); state != wantState && cc.WaitForStateChange(ctx, state); state = cc.GetState() {
-	}
-	return state, state == wantState
-}
-
 func (s) TestDialWithTimeout(t *testing.T) {
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -361,42 +351,6 @@ func (s) TestBackoffWhenNoServerPrefaceReceived(t *testing.T) {
 
 }
 
-func (s) TestConnectivityStates(t *testing.T) {
-	servers, resolver, cleanup := startServers(t, 2, math.MaxUint32)
-	defer cleanup()
-	cc, err := Dial("passthrough:///foo.bar.com", WithBalancer(RoundRobin(resolver)), WithInsecure())
-	if err != nil {
-		t.Fatalf("Dial(\"foo.bar.com\", WithBalancer(_)) = _, %v, want _ <nil>", err)
-	}
-	defer cc.Close()
-	wantState := connectivity.Ready
-	if state, ok := assertState(wantState, cc); !ok {
-		t.Fatalf("asserState(%s) = %s, false, want %s, true", wantState, state, wantState)
-	}
-	// Send an update to delete the server connection (tearDown addrConn).
-	update := []*naming.Update{
-		{
-			Op:   naming.Delete,
-			Addr: "localhost:" + servers[0].port,
-		},
-	}
-	resolver.w.inject(update)
-	wantState = connectivity.TransientFailure
-	if state, ok := assertState(wantState, cc); !ok {
-		t.Fatalf("asserState(%s) = %s, false, want %s, true", wantState, state, wantState)
-	}
-	update[0] = &naming.Update{
-		Op:   naming.Add,
-		Addr: "localhost:" + servers[1].port,
-	}
-	resolver.w.inject(update)
-	wantState = connectivity.Ready
-	if state, ok := assertState(wantState, cc); !ok {
-		t.Fatalf("asserState(%s) = %s, false, want %s, true", wantState, state, wantState)
-	}
-
-}
-
 func (s) TestWithTimeout(t *testing.T) {
 	conn, err := Dial("passthrough:///Non-Existent.Server:80", WithTimeout(time.Millisecond), WithBlock(), WithInsecure())
 	if err == nil {
@@ -592,43 +546,6 @@ func (s) TestDialContextFailFast(t *testing.T) {
 	}
 }
 
-// blockingBalancer mimics the behavior of balancers whose initialization takes a long time.
-// In this test, reading from blockingBalancer.Notify() blocks forever.
-type blockingBalancer struct {
-	ch chan []Address
-}
-
-func newBlockingBalancer() Balancer {
-	return &blockingBalancer{ch: make(chan []Address)}
-}
-func (b *blockingBalancer) Start(target string, config BalancerConfig) error {
-	return nil
-}
-func (b *blockingBalancer) Up(addr Address) func(error) {
-	return nil
-}
-func (b *blockingBalancer) Get(ctx context.Context, opts BalancerGetOptions) (addr Address, put func(), err error) {
-	return Address{}, nil, nil
-}
-func (b *blockingBalancer) Notify() <-chan []Address {
-	return b.ch
-}
-func (b *blockingBalancer) Close() error {
-	close(b.ch)
-	return nil
-}
-
-func (s) TestDialWithBlockingBalancer(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	dialDone := make(chan struct{})
-	go func() {
-		DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithInsecure(), WithBalancer(newBlockingBalancer()))
-		close(dialDone)
-	}()
-	cancel()
-	<-dialDone
-}
-
 // securePerRPCCredentials always requires transport security.
 type securePerRPCCredentials struct{}
 
@@ -724,50 +641,6 @@ func (s) TestConnectParamsWithMinConnectTimeout(t *testing.T) {
 	}
 }
 
-// emptyBalancer returns an empty set of servers.
-type emptyBalancer struct {
-	ch chan []Address
-}
-
-func newEmptyBalancer() Balancer {
-	return &emptyBalancer{ch: make(chan []Address, 1)}
-}
-func (b *emptyBalancer) Start(_ string, _ BalancerConfig) error {
-	b.ch <- nil
-	return nil
-}
-func (b *emptyBalancer) Up(_ Address) func(error) {
-	return nil
-}
-func (b *emptyBalancer) Get(_ context.Context, _ BalancerGetOptions) (Address, func(), error) {
-	return Address{}, nil, nil
-}
-func (b *emptyBalancer) Notify() <-chan []Address {
-	return b.ch
-}
-func (b *emptyBalancer) Close() error {
-	close(b.ch)
-	return nil
-}
-
-func (s) TestNonblockingDialWithEmptyBalancer(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	dialDone := make(chan error)
-	go func() {
-		dialDone <- func() error {
-			conn, err := DialContext(ctx, "Non-Existent.Server:80", WithInsecure(), WithBalancer(newEmptyBalancer()))
-			if err != nil {
-				return err
-			}
-			return conn.Close()
-		}()
-	}()
-	if err := <-dialDone; err != nil {
-		t.Fatalf("unexpected error dialing connection: %s", err)
-	}
-}
-
 func (s) TestResolverServiceConfigBeforeAddressNotPanic(t *testing.T) {
 	r, rcleanup := manual.GenerateAndRegisterManualResolver()
 	defer rcleanup()
diff --git a/dialoptions.go b/dialoptions.go
index d33ec45276f9..b5c810927b10 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -57,8 +57,7 @@ type dialOptions struct {
 	authority       string
 	copts           transport.ConnectOptions
 	callOptions     []CallOption
-	// This is used by v1 balancer dial option WithBalancer to support v1
-	// balancer, and also by WithBalancerName dial option.
+	// This is used by WithBalancerName dial option.
 	balancerBuilder             balancer.Builder
 	channelzParentID            int64
 	disableServiceConfig        bool
@@ -200,19 +199,6 @@ func WithDecompressor(dc Decompressor) DialOption {
 	})
 }
 
-// WithBalancer returns a DialOption which sets a load balancer with the v1 API.
-// Name resolver will be ignored if this DialOption is specified.
-//
-// Deprecated: use the new balancer APIs in balancer package and
-// WithBalancerName.  Will be removed in a future 1.x release.
-func WithBalancer(b Balancer) DialOption {
-	return newFuncDialOption(func(o *dialOptions) {
-		o.balancerBuilder = &balancerWrapperBuilder{
-			b: b,
-		}
-	})
-}
-
 // WithBalancerName sets the balancer that the ClientConn will be initialized
 // with. Balancer registered with balancerName will be used. This function
 // panics if no balancer was registered by balancerName.
diff --git a/examples/go.sum b/examples/go.sum
index 0d649b1f5ae2..7aa49f559848 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -63,6 +63,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.30.0-dev.1 h1:UPWdABFs9zu2kdq7GrCUcfnVgCT65hSpvHmy0RiKn0M=
+google.golang.org/grpc v1.30.0-dev.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
diff --git a/internal/status/status.go b/internal/status/status.go
index 681260692e36..981c265b4c1b 100644
--- a/internal/status/status.go
+++ b/internal/status/status.go
@@ -137,25 +137,25 @@ func (s *Status) Details() []interface{} {
 }
 
 // Error is an alias of a status proto. It implements error and Status,
-// and a nil Error should never be returned by this package.
+// and a nil *Error should never be returned by this package.
 type Error spb.Status
 
-func (se *Error) Error() string {
-	p := (*spb.Status)(se)
+func (e *Error) Error() string {
+	p := (*spb.Status)(e)
 	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage())
 }
 
 // GRPCStatus returns the Status represented by se.
-func (se *Error) GRPCStatus() *Status {
-	return FromProto((*spb.Status)(se))
+func (e *Error) GRPCStatus() *Status {
+	return FromProto((*spb.Status)(e))
 }
 
 // Is implements future error.Is functionality.
 // A Error is equivalent if the code and message are identical.
-func (se *Error) Is(target error) bool {
+func (e *Error) Is(target error) bool {
 	tse, ok := target.(*Error)
 	if !ok {
 		return false
 	}
-	return proto.Equal((*spb.Status)(se), (*spb.Status)(tse))
+	return proto.Equal((*spb.Status)(e), (*spb.Status)(tse))
 }
diff --git a/naming/dns_resolver.go b/naming/dns_resolver.go
deleted file mode 100644
index c9f79dc53362..000000000000
--- a/naming/dns_resolver.go
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package naming
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net"
-	"strconv"
-	"time"
-
-	"google.golang.org/grpc/grpclog"
-)
-
-const (
-	defaultPort = "443"
-	defaultFreq = time.Minute * 30
-)
-
-var (
-	errMissingAddr  = errors.New("missing address")
-	errWatcherClose = errors.New("watcher has been closed")
-
-	lookupHost = net.DefaultResolver.LookupHost
-	lookupSRV  = net.DefaultResolver.LookupSRV
-)
-
-// NewDNSResolverWithFreq creates a DNS Resolver that can resolve DNS names, and
-// create watchers that poll the DNS server using the frequency set by freq.
-func NewDNSResolverWithFreq(freq time.Duration) (Resolver, error) {
-	return &dnsResolver{freq: freq}, nil
-}
-
-// NewDNSResolver creates a DNS Resolver that can resolve DNS names, and create
-// watchers that poll the DNS server using the default frequency defined by defaultFreq.
-func NewDNSResolver() (Resolver, error) {
-	return NewDNSResolverWithFreq(defaultFreq)
-}
-
-// dnsResolver handles name resolution for names following the DNS scheme
-type dnsResolver struct {
-	// frequency of polling the DNS server that the watchers created by this resolver will use.
-	freq time.Duration
-}
-
-// formatIP returns ok = false if addr is not a valid textual representation of an IP address.
-// If addr is an IPv4 address, return the addr and ok = true.
-// If addr is an IPv6 address, return the addr enclosed in square brackets and ok = true.
-func formatIP(addr string) (addrIP string, ok bool) {
-	ip := net.ParseIP(addr)
-	if ip == nil {
-		return "", false
-	}
-	if ip.To4() != nil {
-		return addr, true
-	}
-	return "[" + addr + "]", true
-}
-
-// parseTarget takes the user input target string, returns formatted host and port info.
-// If target doesn't specify a port, set the port to be the defaultPort.
-// If target is in IPv6 format and host-name is enclosed in square brackets, brackets
-// are stripped when setting the host.
-// examples:
-// target: "www.google.com" returns host: "www.google.com", port: "443"
-// target: "ipv4-host:80" returns host: "ipv4-host", port: "80"
-// target: "[ipv6-host]" returns host: "ipv6-host", port: "443"
-// target: ":80" returns host: "localhost", port: "80"
-// target: ":" returns host: "localhost", port: "443"
-func parseTarget(target string) (host, port string, err error) {
-	if target == "" {
-		return "", "", errMissingAddr
-	}
-
-	if ip := net.ParseIP(target); ip != nil {
-		// target is an IPv4 or IPv6(without brackets) address
-		return target, defaultPort, nil
-	}
-	if host, port, err := net.SplitHostPort(target); err == nil {
-		// target has port, i.e ipv4-host:port, [ipv6-host]:port, host-name:port
-		if host == "" {
-			// Keep consistent with net.Dial(): If the host is empty, as in ":80", the local system is assumed.
-			host = "localhost"
-		}
-		if port == "" {
-			// If the port field is empty(target ends with colon), e.g. "[::1]:", defaultPort is used.
-			port = defaultPort
-		}
-		return host, port, nil
-	}
-	if host, port, err := net.SplitHostPort(target + ":" + defaultPort); err == nil {
-		// target doesn't have port
-		return host, port, nil
-	}
-	return "", "", fmt.Errorf("invalid target address %v", target)
-}
-
-// Resolve creates a watcher that watches the name resolution of the target.
-func (r *dnsResolver) Resolve(target string) (Watcher, error) {
-	host, port, err := parseTarget(target)
-	if err != nil {
-		return nil, err
-	}
-
-	if net.ParseIP(host) != nil {
-		ipWatcher := &ipWatcher{
-			updateChan: make(chan *Update, 1),
-		}
-		host, _ = formatIP(host)
-		ipWatcher.updateChan <- &Update{Op: Add, Addr: host + ":" + port}
-		return ipWatcher, nil
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	return &dnsWatcher{
-		r:      r,
-		host:   host,
-		port:   port,
-		ctx:    ctx,
-		cancel: cancel,
-		t:      time.NewTimer(0),
-	}, nil
-}
-
-// dnsWatcher watches for the name resolution update for a specific target
-type dnsWatcher struct {
-	r    *dnsResolver
-	host string
-	port string
-	// The latest resolved address set
-	curAddrs map[string]*Update
-	ctx      context.Context
-	cancel   context.CancelFunc
-	t        *time.Timer
-}
-
-// ipWatcher watches for the name resolution update for an IP address.
-type ipWatcher struct {
-	updateChan chan *Update
-}
-
-// Next returns the address resolution Update for the target. For IP address,
-// the resolution is itself, thus polling name server is unnecessary. Therefore,
-// Next() will return an Update the first time it is called, and will be blocked
-// for all following calls as no Update exists until watcher is closed.
-func (i *ipWatcher) Next() ([]*Update, error) {
-	u, ok := <-i.updateChan
-	if !ok {
-		return nil, errWatcherClose
-	}
-	return []*Update{u}, nil
-}
-
-// Close closes the ipWatcher.
-func (i *ipWatcher) Close() {
-	close(i.updateChan)
-}
-
-// AddressType indicates the address type returned by name resolution.
-type AddressType uint8
-
-const (
-	// Backend indicates the server is a backend server.
-	Backend AddressType = iota
-	// GRPCLB indicates the server is a grpclb load balancer.
-	GRPCLB
-)
-
-// AddrMetadataGRPCLB contains the information the name resolver for grpclb should provide. The
-// name resolver used by the grpclb balancer is required to provide this type of metadata in
-// its address updates.
-type AddrMetadataGRPCLB struct {
-	// AddrType is the type of server (grpc load balancer or backend).
-	AddrType AddressType
-	// ServerName is the name of the grpc load balancer. Used for authentication.
-	ServerName string
-}
-
-// compileUpdate compares the old resolved addresses and newly resolved addresses,
-// and generates an update list
-func (w *dnsWatcher) compileUpdate(newAddrs map[string]*Update) []*Update {
-	var res []*Update
-	for a, u := range w.curAddrs {
-		if _, ok := newAddrs[a]; !ok {
-			u.Op = Delete
-			res = append(res, u)
-		}
-	}
-	for a, u := range newAddrs {
-		if _, ok := w.curAddrs[a]; !ok {
-			res = append(res, u)
-		}
-	}
-	return res
-}
-
-func (w *dnsWatcher) lookupSRV() map[string]*Update {
-	newAddrs := make(map[string]*Update)
-	_, srvs, err := lookupSRV(w.ctx, "grpclb", "tcp", w.host)
-	if err != nil {
-		grpclog.Infof("grpc: failed dns SRV record lookup due to %v.\n", err)
-		return nil
-	}
-	for _, s := range srvs {
-		lbAddrs, err := lookupHost(w.ctx, s.Target)
-		if err != nil {
-			grpclog.Warningf("grpc: failed load balancer address dns lookup due to %v.\n", err)
-			continue
-		}
-		for _, a := range lbAddrs {
-			a, ok := formatIP(a)
-			if !ok {
-				grpclog.Errorf("grpc: failed IP parsing due to %v.\n", err)
-				continue
-			}
-			addr := a + ":" + strconv.Itoa(int(s.Port))
-			newAddrs[addr] = &Update{Addr: addr,
-				Metadata: AddrMetadataGRPCLB{AddrType: GRPCLB, ServerName: s.Target}}
-		}
-	}
-	return newAddrs
-}
-
-func (w *dnsWatcher) lookupHost() map[string]*Update {
-	newAddrs := make(map[string]*Update)
-	addrs, err := lookupHost(w.ctx, w.host)
-	if err != nil {
-		grpclog.Warningf("grpc: failed dns A record lookup due to %v.\n", err)
-		return nil
-	}
-	for _, a := range addrs {
-		a, ok := formatIP(a)
-		if !ok {
-			grpclog.Errorf("grpc: failed IP parsing due to %v.\n", err)
-			continue
-		}
-		addr := a + ":" + w.port
-		newAddrs[addr] = &Update{Addr: addr}
-	}
-	return newAddrs
-}
-
-func (w *dnsWatcher) lookup() []*Update {
-	newAddrs := w.lookupSRV()
-	if newAddrs == nil {
-		// If failed to get any balancer address (either no corresponding SRV for the
-		// target, or caused by failure during resolution/parsing of the balancer target),
-		// return any A record info available.
-		newAddrs = w.lookupHost()
-	}
-	result := w.compileUpdate(newAddrs)
-	w.curAddrs = newAddrs
-	return result
-}
-
-// Next returns the resolved address update(delta) for the target. If there's no
-// change, it will sleep for 30 mins and try to resolve again after that.
-func (w *dnsWatcher) Next() ([]*Update, error) {
-	for {
-		select {
-		case <-w.ctx.Done():
-			return nil, errWatcherClose
-		case <-w.t.C:
-		}
-		result := w.lookup()
-		// Next lookup should happen after an interval defined by w.r.freq.
-		w.t.Reset(w.r.freq)
-		if len(result) > 0 {
-			return result, nil
-		}
-	}
-}
-
-func (w *dnsWatcher) Close() {
-	w.cancel()
-}
diff --git a/naming/dns_resolver_test.go b/naming/dns_resolver_test.go
deleted file mode 100644
index a7eff2d4038f..000000000000
--- a/naming/dns_resolver_test.go
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package naming
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"reflect"
-	"sync"
-	"testing"
-	"time"
-
-	"google.golang.org/grpc/internal/grpctest"
-)
-
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
-func newUpdateWithMD(op Operation, addr, lb string) *Update {
-	return &Update{
-		Op:       op,
-		Addr:     addr,
-		Metadata: AddrMetadataGRPCLB{AddrType: GRPCLB, ServerName: lb},
-	}
-}
-
-func toMap(u []*Update) map[string]*Update {
-	m := make(map[string]*Update)
-	for _, v := range u {
-		m[v.Addr] = v
-	}
-	return m
-}
-
-func (s) TestCompileUpdate(t *testing.T) {
-	tests := []struct {
-		oldAddrs []string
-		newAddrs []string
-		want     []*Update
-	}{
-		{
-			[]string{},
-			[]string{"1.0.0.1"},
-			[]*Update{{Op: Add, Addr: "1.0.0.1"}},
-		},
-		{
-			[]string{"1.0.0.1"},
-			[]string{"1.0.0.1"},
-			[]*Update{},
-		},
-		{
-			[]string{"1.0.0.0"},
-			[]string{"1.0.0.1"},
-			[]*Update{{Op: Delete, Addr: "1.0.0.0"}, {Op: Add, Addr: "1.0.0.1"}},
-		},
-		{
-			[]string{"1.0.0.1"},
-			[]string{"1.0.0.0"},
-			[]*Update{{Op: Add, Addr: "1.0.0.0"}, {Op: Delete, Addr: "1.0.0.1"}},
-		},
-		{
-			[]string{"1.0.0.1"},
-			[]string{"1.0.0.1", "1.0.0.2", "1.0.0.3"},
-			[]*Update{{Op: Add, Addr: "1.0.0.2"}, {Op: Add, Addr: "1.0.0.3"}},
-		},
-		{
-			[]string{"1.0.0.1", "1.0.0.2", "1.0.0.3"},
-			[]string{"1.0.0.0"},
-			[]*Update{{Op: Add, Addr: "1.0.0.0"}, {Op: Delete, Addr: "1.0.0.1"}, {Op: Delete, Addr: "1.0.0.2"}, {Op: Delete, Addr: "1.0.0.3"}},
-		},
-		{
-			[]string{"1.0.0.1", "1.0.0.3", "1.0.0.5"},
-			[]string{"1.0.0.2", "1.0.0.3", "1.0.0.6"},
-			[]*Update{{Op: Delete, Addr: "1.0.0.1"}, {Op: Add, Addr: "1.0.0.2"}, {Op: Delete, Addr: "1.0.0.5"}, {Op: Add, Addr: "1.0.0.6"}},
-		},
-		{
-			[]string{"1.0.0.1", "1.0.0.1", "1.0.0.2"},
-			[]string{"1.0.0.1"},
-			[]*Update{{Op: Delete, Addr: "1.0.0.2"}},
-		},
-	}
-
-	var w dnsWatcher
-	for _, c := range tests {
-		w.curAddrs = make(map[string]*Update)
-		newUpdates := make(map[string]*Update)
-		for _, a := range c.oldAddrs {
-			w.curAddrs[a] = &Update{Addr: a}
-		}
-		for _, a := range c.newAddrs {
-			newUpdates[a] = &Update{Addr: a}
-		}
-		r := w.compileUpdate(newUpdates)
-		if !reflect.DeepEqual(toMap(c.want), toMap(r)) {
-			t.Errorf("w(%+v).compileUpdate(%+v) = %+v, want %+v", c.oldAddrs, c.newAddrs, updatesToSlice(r), updatesToSlice(c.want))
-		}
-	}
-}
-
-func (s) TestResolveFunc(t *testing.T) {
-	tests := []struct {
-		addr string
-		want error
-	}{
-		// TODO(yuxuanli): More false cases?
-		{"www.google.com", nil},
-		{"foo.bar:12345", nil},
-		{"127.0.0.1", nil},
-		{"127.0.0.1:12345", nil},
-		{"[::1]:80", nil},
-		{"[2001:db8:a0b:12f0::1]:21", nil},
-		{":80", nil},
-		{"127.0.0...1:12345", nil},
-		{"[fe80::1%lo0]:80", nil},
-		{"golang.org:http", nil},
-		{"[2001:db8::1]:http", nil},
-		{":", nil},
-		{"", errMissingAddr},
-		{"[2001:db8:a0b:12f0::1", fmt.Errorf("invalid target address %v", "[2001:db8:a0b:12f0::1")},
-	}
-
-	r, err := NewDNSResolver()
-	if err != nil {
-		t.Errorf("%v", err)
-	}
-	for _, v := range tests {
-		_, err := r.Resolve(v.addr)
-		if !reflect.DeepEqual(err, v.want) {
-			t.Errorf("Resolve(%q) = %v, want %v", v.addr, err, v.want)
-		}
-	}
-}
-
-var hostLookupTbl = map[string][]string{
-	"foo.bar.com":      {"1.2.3.4", "5.6.7.8"},
-	"ipv4.single.fake": {"1.2.3.4"},
-	"ipv4.multi.fake":  {"1.2.3.4", "5.6.7.8", "9.10.11.12"},
-	"ipv6.single.fake": {"2607:f8b0:400a:801::1001"},
-	"ipv6.multi.fake":  {"2607:f8b0:400a:801::1001", "2607:f8b0:400a:801::1002", "2607:f8b0:400a:801::1003"},
-}
-
-func hostLookup(host string) ([]string, error) {
-	if addrs, ok := hostLookupTbl[host]; ok {
-		return addrs, nil
-	}
-	return nil, fmt.Errorf("failed to lookup host:%s resolution in hostLookupTbl", host)
-}
-
-var srvLookupTbl = map[string][]*net.SRV{
-	"_grpclb._tcp.srv.ipv4.single.fake": {&net.SRV{Target: "ipv4.single.fake", Port: 1234}},
-	"_grpclb._tcp.srv.ipv4.multi.fake":  {&net.SRV{Target: "ipv4.multi.fake", Port: 1234}},
-	"_grpclb._tcp.srv.ipv6.single.fake": {&net.SRV{Target: "ipv6.single.fake", Port: 1234}},
-	"_grpclb._tcp.srv.ipv6.multi.fake":  {&net.SRV{Target: "ipv6.multi.fake", Port: 1234}},
-}
-
-func srvLookup(service, proto, name string) (string, []*net.SRV, error) {
-	cname := "_" + service + "._" + proto + "." + name
-	if srvs, ok := srvLookupTbl[cname]; ok {
-		return cname, srvs, nil
-	}
-	return "", nil, fmt.Errorf("failed to lookup srv record for %s in srvLookupTbl", cname)
-}
-
-func updatesToSlice(updates []*Update) []Update {
-	res := make([]Update, len(updates))
-	for i, u := range updates {
-		res[i] = *u
-	}
-	return res
-}
-
-func testResolver(t *testing.T, freq time.Duration, slp time.Duration) {
-	tests := []struct {
-		target string
-		want   []*Update
-	}{
-		{
-			"foo.bar.com",
-			[]*Update{{Op: Add, Addr: "1.2.3.4" + colonDefaultPort}, {Op: Add, Addr: "5.6.7.8" + colonDefaultPort}},
-		},
-		{
-			"foo.bar.com:1234",
-			[]*Update{{Op: Add, Addr: "1.2.3.4:1234"}, {Op: Add, Addr: "5.6.7.8:1234"}},
-		},
-		{
-			"srv.ipv4.single.fake",
-			[]*Update{newUpdateWithMD(Add, "1.2.3.4:1234", "ipv4.single.fake")},
-		},
-		{
-			"srv.ipv4.multi.fake",
-			[]*Update{
-				newUpdateWithMD(Add, "1.2.3.4:1234", "ipv4.multi.fake"),
-				newUpdateWithMD(Add, "5.6.7.8:1234", "ipv4.multi.fake"),
-				newUpdateWithMD(Add, "9.10.11.12:1234", "ipv4.multi.fake")},
-		},
-		{
-			"srv.ipv6.single.fake",
-			[]*Update{newUpdateWithMD(Add, "[2607:f8b0:400a:801::1001]:1234", "ipv6.single.fake")},
-		},
-		{
-			"srv.ipv6.multi.fake",
-			[]*Update{
-				newUpdateWithMD(Add, "[2607:f8b0:400a:801::1001]:1234", "ipv6.multi.fake"),
-				newUpdateWithMD(Add, "[2607:f8b0:400a:801::1002]:1234", "ipv6.multi.fake"),
-				newUpdateWithMD(Add, "[2607:f8b0:400a:801::1003]:1234", "ipv6.multi.fake"),
-			},
-		},
-	}
-
-	for _, a := range tests {
-		r, err := NewDNSResolverWithFreq(freq)
-		if err != nil {
-			t.Fatalf("%v\n", err)
-		}
-		w, err := r.Resolve(a.target)
-		if err != nil {
-			t.Fatalf("%v\n", err)
-		}
-		updates, err := w.Next()
-		if err != nil {
-			t.Fatalf("%v\n", err)
-		}
-		if !reflect.DeepEqual(toMap(a.want), toMap(updates)) {
-			t.Errorf("Resolve(%q) = %+v, want %+v\n", a.target, updatesToSlice(updates), updatesToSlice(a.want))
-		}
-		var wg sync.WaitGroup
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for {
-				_, err := w.Next()
-				if err != nil {
-					return
-				}
-				t.Error("Execution shouldn't reach here, since w.Next() should be blocked until close happen.")
-			}
-		}()
-		// Sleep for sometime to let watcher do more than one lookup
-		time.Sleep(slp)
-		w.Close()
-		wg.Wait()
-	}
-}
-
-func replaceNetFunc() func() {
-	oldLookupHost := lookupHost
-	oldLookupSRV := lookupSRV
-	lookupHost = func(ctx context.Context, host string) ([]string, error) {
-		return hostLookup(host)
-	}
-	lookupSRV = func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) {
-		return srvLookup(service, proto, name)
-	}
-	return func() {
-		lookupHost = oldLookupHost
-		lookupSRV = oldLookupSRV
-	}
-}
-
-func (s) TestResolve(t *testing.T) {
-	defer replaceNetFunc()()
-	testResolver(t, time.Millisecond*5, time.Millisecond*10)
-}
-
-const colonDefaultPort = ":" + defaultPort
-
-func (s) TestIPWatcher(t *testing.T) {
-	tests := []struct {
-		target string
-		want   []*Update
-	}{
-		{"127.0.0.1", []*Update{{Op: Add, Addr: "127.0.0.1" + colonDefaultPort}}},
-		{"127.0.0.1:12345", []*Update{{Op: Add, Addr: "127.0.0.1:12345"}}},
-		{"::1", []*Update{{Op: Add, Addr: "[::1]" + colonDefaultPort}}},
-		{"[::1]:12345", []*Update{{Op: Add, Addr: "[::1]:12345"}}},
-		{"[::1]:", []*Update{{Op: Add, Addr: "[::1]:443"}}},
-		{"2001:db8:85a3::8a2e:370:7334", []*Update{{Op: Add, Addr: "[2001:db8:85a3::8a2e:370:7334]" + colonDefaultPort}}},
-		{"[2001:db8:85a3::8a2e:370:7334]", []*Update{{Op: Add, Addr: "[2001:db8:85a3::8a2e:370:7334]" + colonDefaultPort}}},
-		{"[2001:db8:85a3::8a2e:370:7334]:12345", []*Update{{Op: Add, Addr: "[2001:db8:85a3::8a2e:370:7334]:12345"}}},
-		{"[2001:db8::1]:http", []*Update{{Op: Add, Addr: "[2001:db8::1]:http"}}},
-		// TODO(yuxuanli): zone support?
-	}
-
-	for _, v := range tests {
-		r, err := NewDNSResolverWithFreq(time.Millisecond * 5)
-		if err != nil {
-			t.Fatalf("%v\n", err)
-		}
-		w, err := r.Resolve(v.target)
-		if err != nil {
-			t.Fatalf("%v\n", err)
-		}
-		var updates []*Update
-		var wg sync.WaitGroup
-		wg.Add(1)
-		count := 0
-		go func() {
-			defer wg.Done()
-			for {
-				u, err := w.Next()
-				if err != nil {
-					return
-				}
-				updates = u
-				count++
-			}
-		}()
-		// Sleep for sometime to let watcher do more than one lookup
-		time.Sleep(time.Millisecond * 10)
-		w.Close()
-		wg.Wait()
-		if !reflect.DeepEqual(v.want, updates) {
-			t.Errorf("Resolve(%q) = %v, want %+v\n", v.target, updatesToSlice(updates), updatesToSlice(v.want))
-		}
-		if count != 1 {
-			t.Errorf("IPWatcher Next() should return only once, not %d times\n", count)
-		}
-	}
-}
diff --git a/naming/naming.go b/naming/naming.go
deleted file mode 100644
index f4c1c8b68947..000000000000
--- a/naming/naming.go
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- *
- * Copyright 2014 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Package naming defines the naming API and related data structures for gRPC.
-//
-// This package is deprecated: please use package resolver instead.
-package naming
-
-// Operation defines the corresponding operations for a name resolution change.
-//
-// Deprecated: please use package resolver.
-type Operation uint8
-
-const (
-	// Add indicates a new address is added.
-	Add Operation = iota
-	// Delete indicates an existing address is deleted.
-	Delete
-)
-
-// Update defines a name resolution update. Notice that it is not valid having both
-// empty string Addr and nil Metadata in an Update.
-//
-// Deprecated: please use package resolver.
-type Update struct {
-	// Op indicates the operation of the update.
-	Op Operation
-	// Addr is the updated address. It is empty string if there is no address update.
-	Addr string
-	// Metadata is the updated metadata. It is nil if there is no metadata update.
-	// Metadata is not required for a custom naming implementation.
-	Metadata interface{}
-}
-
-// Resolver creates a Watcher for a target to track its resolution changes.
-//
-// Deprecated: please use package resolver.
-type Resolver interface {
-	// Resolve creates a Watcher for target.
-	Resolve(target string) (Watcher, error)
-}
-
-// Watcher watches for the updates on the specified target.
-//
-// Deprecated: please use package resolver.
-type Watcher interface {
-	// Next blocks until an update or error happens. It may return one or more
-	// updates. The first call should get the full set of the results. It should
-	// return an error if and only if Watcher cannot recover.
-	Next() ([]*Update, error)
-	// Close closes the Watcher.
-	Close()
-}
diff --git a/picker_wrapper.go b/picker_wrapper.go
index 00447894f07b..7f3edaaedc60 100644
--- a/picker_wrapper.go
+++ b/picker_wrapper.go
@@ -20,7 +20,6 @@ package grpc
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"sync"
 
@@ -32,68 +31,21 @@ import (
 	"google.golang.org/grpc/status"
 )
 
-// v2PickerWrapper wraps a balancer.Picker while providing the
-// balancer.V2Picker API.  It requires a pickerWrapper to generate errors
-// including the latest connectionError.  To be deleted when balancer.Picker is
-// updated to the balancer.V2Picker API.
-type v2PickerWrapper struct {
-	picker  balancer.Picker
-	connErr *connErr
-}
-
-func (v *v2PickerWrapper) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
-	sc, done, err := v.picker.Pick(info.Ctx, info)
-	if err != nil {
-		if err == balancer.ErrTransientFailure {
-			return balancer.PickResult{}, balancer.TransientFailureError(fmt.Errorf("%v, latest connection error: %v", err, v.connErr.connectionError()))
-		}
-		return balancer.PickResult{}, err
-	}
-	return balancer.PickResult{SubConn: sc, Done: done}, nil
-}
-
 // pickerWrapper is a wrapper of balancer.Picker. It blocks on certain pick
 // actions and unblock when there's a picker update.
 type pickerWrapper struct {
 	mu         sync.Mutex
 	done       bool
 	blockingCh chan struct{}
-	picker     balancer.V2Picker
-
-	// The latest connection error.  TODO: remove when V1 picker is deprecated;
-	// balancer should be responsible for providing the error.
-	*connErr
-}
-
-type connErr struct {
-	mu  sync.Mutex
-	err error
-}
-
-func (c *connErr) updateConnectionError(err error) {
-	c.mu.Lock()
-	c.err = err
-	c.mu.Unlock()
-}
-
-func (c *connErr) connectionError() error {
-	c.mu.Lock()
-	err := c.err
-	c.mu.Unlock()
-	return err
+	picker     balancer.Picker
 }
 
 func newPickerWrapper() *pickerWrapper {
-	return &pickerWrapper{blockingCh: make(chan struct{}), connErr: &connErr{}}
+	return &pickerWrapper{blockingCh: make(chan struct{})}
 }
 
 // updatePicker is called by UpdateBalancerState. It unblocks all blocked pick.
 func (pw *pickerWrapper) updatePicker(p balancer.Picker) {
-	pw.updatePickerV2(&v2PickerWrapper{picker: p, connErr: pw.connErr})
-}
-
-// updatePicker is called by UpdateBalancerState. It unblocks all blocked pick.
-func (pw *pickerWrapper) updatePickerV2(p balancer.V2Picker) {
 	pw.mu.Lock()
 	if pw.done {
 		pw.mu.Unlock()
@@ -154,8 +106,6 @@ func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.
 				var errStr string
 				if lastPickErr != nil {
 					errStr = "latest balancer error: " + lastPickErr.Error()
-				} else if connectionErr := pw.connectionError(); connectionErr != nil {
-					errStr = "latest connection error: " + connectionErr.Error()
 				} else {
 					errStr = ctx.Err().Error()
 				}
@@ -180,18 +130,17 @@ func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.
 			if err == balancer.ErrNoSubConnAvailable {
 				continue
 			}
-			if tfe, ok := err.(interface{ IsTransientFailure() bool }); ok && tfe.IsTransientFailure() {
-				if !failfast {
-					lastPickErr = err
-					continue
-				}
-				return nil, nil, status.Error(codes.Unavailable, err.Error())
-			}
 			if _, ok := status.FromError(err); ok {
+				// Status error: end the RPC unconditionally with this status.
 				return nil, nil, err
 			}
-			// err is some other error.
-			return nil, nil, status.Error(codes.Unknown, err.Error())
+			// For all other errors, wait for ready RPCs should block and other
+			// RPCs should fail with unavailable.
+			if !failfast {
+				lastPickErr = err
+				continue
+			}
+			return nil, nil, status.Error(codes.Unavailable, err.Error())
 		}
 
 		acw, ok := pickResult.SubConn.(*acBalancerWrapper)
diff --git a/picker_wrapper_test.go b/picker_wrapper_test.go
index 50916a2dfde0..5f786b28580e 100644
--- a/picker_wrapper_test.go
+++ b/picker_wrapper_test.go
@@ -55,14 +55,14 @@ type testingPicker struct {
 	maxCalled int64
 }
 
-func (p *testingPicker) Pick(ctx context.Context, info balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error) {
+func (p *testingPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 	if atomic.AddInt64(&p.maxCalled, -1) < 0 {
-		return nil, nil, fmt.Errorf("pick called to many times (> goroutineCount)")
+		return balancer.PickResult{}, fmt.Errorf("pick called to many times (> goroutineCount)")
 	}
 	if p.err != nil {
-		return nil, nil, p.err
+		return balancer.PickResult{}, p.err
 	}
-	return p.sc, nil, nil
+	return balancer.PickResult{SubConn: p.sc}, nil
 }
 
 func (s) TestBlockingPickTimeout(t *testing.T) {
diff --git a/pickfirst.go b/pickfirst.go
index c43dac9ad842..4b7340ad3ecc 100644
--- a/pickfirst.go
+++ b/pickfirst.go
@@ -20,13 +20,11 @@ package grpc
 
 import (
 	"errors"
+	"fmt"
 
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/grpclog"
-	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/status"
 )
 
 // PickFirstBalancerName is the name of the pick_first balancer.
@@ -52,27 +50,13 @@ type pickfirstBalancer struct {
 	sc    balancer.SubConn
 }
 
-var _ balancer.V2Balancer = &pickfirstBalancer{} // Assert we implement v2
-
-func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
-	if err != nil {
-		b.ResolverError(err)
-		return
-	}
-	b.UpdateClientConnState(balancer.ClientConnState{ResolverState: resolver.State{Addresses: addrs}}) // Ignore error
-}
-
-func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	b.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: s})
-}
-
 func (b *pickfirstBalancer) ResolverError(err error) {
 	switch b.state {
 	case connectivity.TransientFailure, connectivity.Idle, connectivity.Connecting:
 		// Set a failing picker if we don't have a good picker.
 		b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure,
-			Picker: &picker{err: status.Errorf(codes.Unavailable, "name resolver error: %v", err)}},
-		)
+			Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)},
+		})
 	}
 	if grpclog.V(2) {
 		grpclog.Infof("pickfirstBalancer: ResolverError called with error %v", err)
@@ -93,8 +77,8 @@ func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) e
 			}
 			b.state = connectivity.TransientFailure
 			b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure,
-				Picker: &picker{err: status.Errorf(codes.Unavailable, "error creating connection: %v", err)}},
-			)
+				Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)},
+			})
 			return balancer.ErrBadResolverState
 		}
 		b.state = connectivity.Idle
@@ -109,7 +93,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) e
 
 func (b *pickfirstBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
 	if grpclog.V(2) {
-		grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s)
+		grpclog.Infof("pickfirstBalancer: UpdateSubConnState: %p, %v", sc, s)
 	}
 	if b.sc != sc {
 		if grpclog.V(2) {
@@ -129,15 +113,9 @@ func (b *pickfirstBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.S
 	case connectivity.Connecting:
 		b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable}})
 	case connectivity.TransientFailure:
-		err := balancer.ErrTransientFailure
-		// TODO: this can be unconditional after the V1 API is removed, as
-		// SubConnState will always contain a connection error.
-		if s.ConnectionError != nil {
-			err = balancer.TransientFailureError(s.ConnectionError)
-		}
 		b.cc.UpdateState(balancer.State{
 			ConnectivityState: s.ConnectivityState,
-			Picker:            &picker{err: err},
+			Picker:            &picker{err: s.ConnectionError},
 		})
 	}
 }
diff --git a/pickfirst_test.go b/pickfirst_test.go
index 475eedfdf46c..a69cec1c51de 100644
--- a/pickfirst_test.go
+++ b/pickfirst_test.go
@@ -43,7 +43,7 @@ func (s) TestOneBackendPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 1
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -76,7 +76,7 @@ func (s) TestBackendsPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 2
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -109,7 +109,7 @@ func (s) TestNewAddressWhileBlockingPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 1
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -145,7 +145,7 @@ func (s) TestCloseWithPendingRPCPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 1
-	_, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	_, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -181,7 +181,7 @@ func (s) TestOneServerDownPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 2
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -222,7 +222,7 @@ func (s) TestAllServersDownPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 2
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
@@ -265,7 +265,7 @@ func (s) TestAddressesRemovedPickfirst(t *testing.T) {
 	defer rcleanup()
 
 	numServers := 3
-	servers, _, scleanup := startServers(t, numServers, math.MaxInt32)
+	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
 	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
diff --git a/resolver_conn_wrapper_test.go b/resolver_conn_wrapper_test.go
index 605043f5f5dc..9f22c8b90f6c 100644
--- a/resolver_conn_wrapper_test.go
+++ b/resolver_conn_wrapper_test.go
@@ -29,6 +29,7 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 	"google.golang.org/grpc/serviceconfig"
@@ -130,12 +131,12 @@ const happyBalancerName = "happy balancer"
 func init() {
 	// Register a balancer that never returns an error from
 	// UpdateClientConnState, and doesn't do anything else either.
-	fb := &funcBalancer{
-		updateClientConnState: func(s balancer.ClientConnState) error {
+	bf := stub.BalancerFuncs{
+		UpdateClientConnState: func(*stub.BalancerData, balancer.ClientConnState) error {
 			return nil
 		},
 	}
-	balancer.Register(&funcBalancerBuilder{name: happyBalancerName, instance: fb})
+	stub.Register(happyBalancerName, bf)
 }
 
 // TestResolverErrorPolling injects resolver errors and verifies ResolveNow is
diff --git a/service_config.go b/service_config.go
index c8267fc8eb37..37d4a58f1222 100644
--- a/service_config.go
+++ b/service_config.go
@@ -79,7 +79,7 @@ type ServiceConfig struct {
 	serviceconfig.Config
 
 	// LB is the load balancer the service providers recommends. The balancer
-	// specified via grpc.WithBalancer will override this.  This is deprecated;
+	// specified via grpc.WithBalancerName will override this.  This is deprecated;
 	// lbConfigs is preferred.  If lbConfig and LB are both present, lbConfig
 	// will be used.
 	LB *string
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 7db9635a8971..3cd4e0e91fb7 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -20,6 +20,7 @@ package test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"reflect"
@@ -30,12 +31,14 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/balancerload"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
@@ -69,37 +72,43 @@ func (*testBalancer) Name() string {
 	return testBalancerName
 }
 
-func (b *testBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
+func (*testBalancer) ResolverError(err error) {
+	panic("not implemented")
+}
+
+func (b *testBalancer) UpdateClientConnState(state balancer.ClientConnState) error {
 	// Only create a subconn at the first time.
-	if err == nil && b.sc == nil {
-		b.sc, err = b.cc.NewSubConn(addrs, b.newSubConnOptions)
+	if b.sc == nil {
+		var err error
+		b.sc, err = b.cc.NewSubConn(state.ResolverState.Addresses, b.newSubConnOptions)
 		if err != nil {
 			grpclog.Errorf("testBalancer: failed to NewSubConn: %v", err)
-			return
+			return nil
 		}
 		b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &picker{sc: b.sc, bal: b}})
 		b.sc.Connect()
 	}
+	return nil
 }
 
-func (b *testBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	grpclog.Infof("testBalancer: HandleSubConnStateChange: %p, %v", sc, s)
+func (b *testBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	grpclog.Infof("testBalancer: UpdateSubConnState: %p, %v", sc, s)
 	if b.sc != sc {
 		grpclog.Infof("testBalancer: ignored state change because sc is not recognized")
 		return
 	}
-	if s == connectivity.Shutdown {
+	if s.ConnectivityState == connectivity.Shutdown {
 		b.sc = nil
 		return
 	}
 
-	switch s {
+	switch s.ConnectivityState {
 	case connectivity.Ready, connectivity.Idle:
-		b.cc.UpdateState(balancer.State{ConnectivityState: s, Picker: &picker{sc: sc, bal: b}})
+		b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{sc: sc, bal: b}})
 	case connectivity.Connecting:
-		b.cc.UpdateState(balancer.State{ConnectivityState: s, Picker: &picker{err: balancer.ErrNoSubConnAvailable, bal: b}})
+		b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable, bal: b}})
 	case connectivity.TransientFailure:
-		b.cc.UpdateState(balancer.State{ConnectivityState: s, Picker: &picker{err: balancer.ErrTransientFailure, bal: b}})
+		b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrTransientFailure, bal: b}})
 	}
 }
 
@@ -285,6 +294,10 @@ func newTestBalancerKeepAddresses() *testBalancerKeepAddresses {
 	}
 }
 
+func (testBalancerKeepAddresses) ResolverError(err error) {
+	panic("not implemented")
+}
+
 func (b *testBalancerKeepAddresses) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer {
 	return b
 }
@@ -293,11 +306,12 @@ func (*testBalancerKeepAddresses) Name() string {
 	return testBalancerKeepAddressesName
 }
 
-func (b *testBalancerKeepAddresses) HandleResolvedAddrs(addrs []resolver.Address, err error) {
-	b.addrsChan <- addrs
+func (b *testBalancerKeepAddresses) UpdateClientConnState(state balancer.ClientConnState) error {
+	b.addrsChan <- state.ResolverState.Addresses
+	return nil
 }
 
-func (testBalancerKeepAddresses) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
+func (testBalancerKeepAddresses) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
 	panic("not used")
 }
 
@@ -472,3 +486,217 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 		t.Fatalf("received attributes %v in creds, want %v", gotAttr, wantAttr)
 	}
 }
+
+// TestServersSwap creates two servers and verifies the client switches between
+// them when the name resolver reports the first and then the second.
+func (s) TestServersSwap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// Initialize servers
+	reg := func(username string) (addr string, cleanup func()) {
+		lis, err := net.Listen("tcp", "localhost:0")
+		if err != nil {
+			t.Fatalf("Error while listening. Err: %v", err)
+		}
+		s := grpc.NewServer()
+		ts := &funcServer{
+			unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return &testpb.SimpleResponse{Username: username}, nil
+			},
+		}
+		testpb.RegisterTestServiceServer(s, ts)
+		go s.Serve(lis)
+		return lis.Addr().String(), s.Stop
+	}
+	const one = "1"
+	addr1, cleanup := reg(one)
+	defer cleanup()
+	const two = "2"
+	addr2, cleanup := reg(two)
+	defer cleanup()
+
+	// Initialize client
+	r, cleanup := manual.GenerateAndRegisterManualResolver()
+	defer cleanup()
+	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: addr1}}})
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer cc.Close()
+	client := testpb.NewTestServiceClient(cc)
+
+	// Confirm we are connected to the first server
+	if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one {
+		t.Fatalf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one)
+	}
+
+	// Update resolver to report only the second server
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: addr2}}})
+
+	// Loop until new RPCs talk to server two.
+	for i := 0; i < 2000; i++ {
+		if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+			t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err)
+		} else if res.Username == two {
+			break // pass
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// TestEmptyAddrs verifies client behavior when a working connection is
+// removed.  In pick first and round-robin, both will continue using the old
+// connections.
+func (s) TestEmptyAddrs(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// Initialize server
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	s := grpc.NewServer()
+	defer s.Stop()
+	const one = "1"
+	ts := &funcServer{
+		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{Username: one}, nil
+		},
+	}
+	testpb.RegisterTestServiceServer(s, ts)
+	go s.Serve(lis)
+
+	// Initialize pickfirst client
+	pfr, cleanup := manual.GenerateAndRegisterManualResolver()
+	defer cleanup()
+	pfrnCalled := grpcsync.NewEvent()
+	pfr.ResolveNowCallback = func(resolver.ResolveNowOptions) {
+		pfrnCalled.Fire()
+	}
+	pfr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
+
+	pfcc, err := grpc.DialContext(ctx, pfr.Scheme()+":///", grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer pfcc.Close()
+	pfclient := testpb.NewTestServiceClient(pfcc)
+
+	// Confirm we are connected to the server
+	if res, err := pfclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one {
+		t.Fatalf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one)
+	}
+
+	// Remove all addresses.
+	pfr.UpdateState(resolver.State{})
+	// Wait for a ResolveNow call on the pick first client's resolver.
+	<-pfrnCalled.Done()
+
+	// Initialize roundrobin client
+	rrr, cleanup := manual.GenerateAndRegisterManualResolver()
+	defer cleanup()
+	rrrnCalled := grpcsync.NewEvent()
+	rrr.ResolveNowCallback = func(resolver.ResolveNowOptions) {
+		rrrnCalled.Fire()
+	}
+	rrr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
+
+	rrcc, err := grpc.DialContext(ctx, rrr.Scheme()+":///", grpc.WithInsecure(),
+		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, roundrobin.Name)))
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer rrcc.Close()
+	rrclient := testpb.NewTestServiceClient(rrcc)
+
+	// Confirm we are connected to the server
+	if res, err := rrclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one {
+		t.Fatalf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one)
+	}
+
+	// Remove all addresses.
+	rrr.UpdateState(resolver.State{})
+	// Wait for a ResolveNow call on the round robin client's resolver.
+	<-rrrnCalled.Done()
+
+	// Confirm several new RPCs succeed on pick first.
+	for i := 0; i < 10; i++ {
+		if _, err := pfclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+			t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err)
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	// Confirm several new RPCs succeed on round robin.
+	for i := 0; i < 10; i++ {
+		if _, err := pfclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+			t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err)
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+func (s) TestWaitForReady(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// Initialize server
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	s := grpc.NewServer()
+	defer s.Stop()
+	const one = "1"
+	ts := &funcServer{
+		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{Username: one}, nil
+		},
+	}
+	testpb.RegisterTestServiceServer(s, ts)
+	go s.Serve(lis)
+
+	// Initialize client
+	r, cleanup := manual.GenerateAndRegisterManualResolver()
+	defer cleanup()
+
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer cc.Close()
+	client := testpb.NewTestServiceClient(cc)
+
+	// Report an error so non-WFR RPCs will give up early.
+	r.CC.ReportError(errors.New("fake resolver error"))
+
+	// Ensure the client is not connected to anything and fails non-WFR RPCs.
+	if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable {
+		t.Fatalf("UnaryCall(_) = %v, %v; want _, Code()=%v", res, err, codes.Unavailable)
+	}
+
+	errChan := make(chan error, 1)
+	go func() {
+		if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.WaitForReady(true)); err != nil || res.Username != one {
+			errChan <- fmt.Errorf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one)
+		}
+		close(errChan)
+	}()
+
+	select {
+	case err := <-errChan:
+		t.Errorf("unexpected receive from errChan before addresses provided")
+		t.Fatal(err.Error())
+	case <-time.After(5 * time.Millisecond):
+	}
+
+	// Resolve the server.  The WFR RPC should unblock and use it.
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
+
+	if err := <-errChan; err != nil {
+		t.Fatal(err.Error())
+	}
+}
diff --git a/test/channelz_test.go b/test/channelz_test.go
index 75846640e227..c69e0cec2e6e 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -1880,7 +1880,8 @@ func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) {
 	czCleanup := channelz.NewChannelzStorage()
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
-	// avoid newTest using WithBalancer, which would override service config's change of balancer below.
+	// avoid newTest using WithBalancerName, which would override service
+	// config's change of balancer below.
 	e.balancer = ""
 	te := newTest(t, e)
 	channelz.SetMaxTraceEntry(1)
diff --git a/test/creds_test.go b/test/creds_test.go
index 65d99c7555d2..7b607ef03716 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -75,7 +75,7 @@ func (c *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) {
 }
 
 func (s) TestCredsBundleBoth(t *testing.T) {
-	te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: "v1", security: "empty"})
+	te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"})
 	te.tapHandle = authHandle
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t}),
@@ -98,7 +98,7 @@ func (s) TestCredsBundleBoth(t *testing.T) {
 }
 
 func (s) TestCredsBundleTransportCredentials(t *testing.T) {
-	te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: "v1", security: "empty"})
+	te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"})
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundleTLSOnly}),
 	}
@@ -120,7 +120,7 @@ func (s) TestCredsBundleTransportCredentials(t *testing.T) {
 }
 
 func (s) TestCredsBundlePerRPCCredentials(t *testing.T) {
-	te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: "v1", security: "empty"})
+	te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"})
 	te.tapHandle = authHandle
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundlePerRPCOnly}),
@@ -157,7 +157,7 @@ func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials {
 }
 
 func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
-	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty", balancer: "v1"})
+	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty"})
 	te.userAgent = testAppUA
 	te.startServer(&testServer{security: te.e.security})
 	defer te.tearDown()
@@ -181,7 +181,7 @@ func (m *methodTestCreds) RequireTransportSecurity() bool { return false }
 
 func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
 	const wantMethod = "/grpc.testing.TestService/EmptyCall"
-	te := newTest(t, env{name: "context-request-info", network: "tcp", balancer: "v1"})
+	te := newTest(t, env{name: "context-request-info", network: "tcp"})
 	te.userAgent = testAppUA
 	te.startServer(&testServer{security: te.e.security})
 	defer te.tearDown()
diff --git a/test/end2end_test.go b/test/end2end_test.go
index fb95f4a2fee3..f3a60de5a96f 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -47,7 +47,6 @@ import (
 	"golang.org/x/net/http2/hpack"
 	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
@@ -397,7 +396,7 @@ type env struct {
 	network      string // The type of network such as tcp, unix, etc.
 	security     string // The security protocol such as TLS, SSH, etc.
 	httpHandler  bool   // whether to use the http.Handler ServerTransport; requires TLS
-	balancer     string // One of "round_robin", "pick_first", "v1", or "".
+	balancer     string // One of "round_robin", "pick_first", or "".
 	customDialer func(string, string, time.Duration) (net.Conn, error)
 }
 
@@ -416,8 +415,8 @@ func (e env) dialer(addr string, timeout time.Duration) (net.Conn, error) {
 }
 
 var (
-	tcpClearEnv   = env{name: "tcp-clear-v1-balancer", network: "tcp", balancer: "v1"}
-	tcpTLSEnv     = env{name: "tcp-tls-v1-balancer", network: "tcp", security: "tls", balancer: "v1"}
+	tcpClearEnv   = env{name: "tcp-clear-v1-balancer", network: "tcp"}
+	tcpTLSEnv     = env{name: "tcp-tls-v1-balancer", network: "tcp", security: "tls"}
 	tcpClearRREnv = env{name: "tcp-clear", network: "tcp", balancer: "round_robin"}
 	tcpTLSRREnv   = env{name: "tcp-tls", network: "tcp", security: "tls", balancer: "round_robin"}
 	handlerEnv    = env{name: "handler-tls", network: "tcp", security: "tls", httpHandler: true, balancer: "round_robin"}
@@ -568,6 +567,7 @@ func newTest(t *testing.T, e env) *test {
 }
 
 func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network, address string) (net.Listener, error)) net.Listener {
+	te.t.Helper()
 	te.t.Logf("Running test in %s environment...", te.e.name)
 	sopts := []grpc.ServerOption{grpc.MaxConcurrentStreams(te.maxStream)}
 	if te.maxServerMsgSize != nil {
@@ -699,6 +699,7 @@ func (te *test) startServerWithConnControl(ts testpb.TestServiceServer) *listene
 // startServer starts a gRPC server exposing the provided TestService
 // implementation. Callers should defer a call to te.tearDown to clean up
 func (te *test) startServer(ts testpb.TestServiceServer) {
+	te.t.Helper()
 	te.listenAndServe(ts, net.Listen)
 }
 
@@ -809,11 +810,8 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string)
 	} else {
 		scheme = te.resolverScheme + ":///"
 	}
-	switch te.e.balancer {
-	case "v1":
-		opts = append(opts, grpc.WithBalancer(grpc.RoundRobin(nil)))
-	case "round_robin":
-		opts = append(opts, grpc.WithBalancerName(roundrobin.Name))
+	if te.e.balancer != "" {
+		opts = append(opts, grpc.WithBalancerName(te.e.balancer))
 	}
 	if te.clientInitialWindowSize > 0 {
 		opts = append(opts, grpc.WithInitialWindowSize(te.clientInitialWindowSize))
@@ -4712,6 +4710,48 @@ func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) {
 	}
 }
 
+type clientFailCreds struct{}
+
+func (c *clientFailCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return rawConn, nil, nil
+}
+func (c *clientFailCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return nil, nil, fmt.Errorf("client handshake fails with fatal error")
+}
+func (c *clientFailCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c *clientFailCreds) Clone() credentials.TransportCredentials {
+	return c
+}
+func (c *clientFailCreds) OverrideServerName(s string) error {
+	return nil
+}
+
+// This test makes sure that failfast RPCs fail if client handshake fails with
+// fatal errors.
+func (s) TestFailfastRPCFailOnFatalHandshakeError(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Failed to listen: %v", err)
+	}
+	defer lis.Close()
+
+	cc, err := grpc.Dial("passthrough:///"+lis.Addr().String(), grpc.WithTransportCredentials(&clientFailCreds{}))
+	if err != nil {
+		t.Fatalf("grpc.Dial(_) = %v", err)
+	}
+	defer cc.Close()
+
+	tc := testpb.NewTestServiceClient(cc)
+	// This unary call should fail, but not timeout.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(false)); status.Code(err) != codes.Unavailable {
+		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want <Unavailable>", err)
+	}
+}
+
 func (s) TestFlowControlLogicalRace(t *testing.T) {
 	// Test for a regression of https://github.com/grpc/grpc-go/issues/632,
 	// and other flow control bugs.
diff --git a/vet.sh b/vet.sh
index 84b18859eb9b..f0a67298a5b0 100755
--- a/vet.sh
+++ b/vet.sh
@@ -125,14 +125,12 @@ staticcheck -go 1.9 -checks 'inherit,-ST1015' ./... > "${SC_OUT}" || true
 not grep -v "is deprecated:.*SA1019" "${SC_OUT}"
 # Only ignore the following deprecated types/fields/functions.
 not grep -Fv '.CredsBundle
-.HandleResolvedAddrs
-.HandleSubConnStateChange
 .HeaderMap
+.Metadata is deprecated: use Attributes
 .NewAddress
 .NewServiceConfig
-.Metadata is deprecated: use Attributes
 .Type is deprecated: use Attributes
-.UpdateBalancerState
+balancer.ErrTransientFailure
 balancer.Picker
 grpc.CallCustomCodec
 grpc.Code
@@ -144,9 +142,7 @@ grpc.NewGZIPCompressor
 grpc.NewGZIPDecompressor
 grpc.RPCCompressor
 grpc.RPCDecompressor
-grpc.RoundRobin
 grpc.ServiceConfig
-grpc.WithBalancer
 grpc.WithBalancerName
 grpc.WithCompressor
 grpc.WithDecompressor
@@ -156,9 +152,6 @@ grpc.WithServiceConfig
 grpc.WithTimeout
 http.CloseNotifier
 info.SecurityVersion
-naming.Resolver
-naming.Update
-naming.Watcher
 resolver.Backend
 resolver.GRPCLB' "${SC_OUT}"
 
diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index d6307867207e..d6428afb96a5 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -68,9 +68,6 @@ type subBalancerWithConfig struct {
 	balancer balancer.Balancer
 }
 
-func (sbc *subBalancerWithConfig) UpdateBalancerState(state connectivity.State, picker balancer.Picker) {
-}
-
 // UpdateState overrides balancer.ClientConn, to keep state and picker.
 func (sbc *subBalancerWithConfig) UpdateState(state balancer.State) {
 	sbc.mu.Lock()
@@ -97,11 +94,7 @@ func (sbc *subBalancerWithConfig) startBalancer() {
 	b := sbc.builder.Build(sbc, balancer.BuildOptions{})
 	sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name())
 	sbc.balancer = b
-	if ub, ok := b.(balancer.V2Balancer); ok {
-		ub.UpdateClientConnState(sbc.ccState)
-		return
-	}
-	b.HandleResolvedAddrs(sbc.ccState.ResolverState.Addresses, nil)
+	b.UpdateClientConnState(sbc.ccState)
 }
 
 func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
@@ -113,11 +106,7 @@ func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state
 		// happen.
 		return
 	}
-	if ub, ok := b.(balancer.V2Balancer); ok {
-		ub.UpdateSubConnState(sc, state)
-		return
-	}
-	b.HandleSubConnStateChange(sc, state.ConnectivityState)
+	b.UpdateSubConnState(sc, state)
 }
 
 func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnState) error {
@@ -134,11 +123,7 @@ func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnSta
 		// it's the lower priority, but it can still get address updates.
 		return nil
 	}
-	if ub, ok := b.(balancer.V2Balancer); ok {
-		return ub.UpdateClientConnState(s)
-	}
-	b.HandleResolvedAddrs(s.ResolverState.Addresses, nil)
-	return nil
+	return b.UpdateClientConnState(s)
 }
 
 func (sbc *subBalancerWithConfig) stopBalancer() {
@@ -148,7 +133,7 @@ func (sbc *subBalancerWithConfig) stopBalancer() {
 
 type pickerState struct {
 	weight uint32
-	picker balancer.V2Picker
+	picker balancer.Picker
 	state  connectivity.State
 }
 
@@ -583,7 +568,7 @@ func buildPickerAndState(m map[internal.LocalityID]*pickerState) balancer.State
 		aggregatedState = connectivity.TransientFailure
 	}
 	if aggregatedState == connectivity.TransientFailure {
-		return balancer.State{ConnectivityState: aggregatedState, Picker: base.NewErrPickerV2(balancer.ErrTransientFailure)}
+		return balancer.State{ConnectivityState: aggregatedState, Picker: base.NewErrPicker(balancer.ErrTransientFailure)}
 	}
 	return balancer.State{ConnectivityState: aggregatedState, Picker: newPickerGroup(readyPickerWithWeights)}
 }
@@ -620,7 +605,7 @@ func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error)
 	if pg.length <= 0 {
 		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
 	}
-	p := pg.w.Next().(balancer.V2Picker)
+	p := pg.w.Next().(balancer.Picker)
 	return p.Pick(info)
 }
 
@@ -630,13 +615,13 @@ const (
 )
 
 type loadReportPicker struct {
-	p balancer.V2Picker
+	p balancer.Picker
 
 	id        internal.LocalityID
 	loadStore lrs.Store
 }
 
-func newLoadReportPicker(p balancer.V2Picker, id internal.LocalityID, loadStore lrs.Store) *loadReportPicker {
+func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore lrs.Store) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
 		id:        id,
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index f2804abf50e1..8c211c041612 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -49,7 +49,7 @@ func init() {
 	DefaultSubBalancerCloseTimeout = time.Millisecond
 }
 
-func subConnFromPicker(p balancer.V2Picker) func() balancer.SubConn {
+func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
 	return func() balancer.SubConn {
 		scst, _ := p.Pick(balancer.PickInfo{})
 		return scst.SubConn
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index bb1653868e4c..31ab7d697b34 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -25,7 +25,6 @@ import (
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/resolver"
@@ -46,7 +45,7 @@ var (
 
 	// newEDSBalancer is a helper function to build a new edsBalancer and will be
 	// overridden in unittests.
-	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.V2Balancer, error) {
+	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
 		builder := balancer.Get(edsName)
 		if builder == nil {
 			return nil, fmt.Errorf("xds: no balancer builder with name %v", edsName)
@@ -54,7 +53,7 @@ var (
 		// We directly pass the parent clientConn to the
 		// underlying edsBalancer because the cdsBalancer does
 		// not deal with subConns.
-		return builder.Build(cc, opts).(balancer.V2Balancer), nil
+		return builder.Build(cc, opts), nil
 	}
 )
 
@@ -151,7 +150,7 @@ type cdsBalancer struct {
 	updateCh       *buffer.Unbounded
 	client         xdsClientInterface
 	cancelWatch    func()
-	edsLB          balancer.V2Balancer
+	edsLB          balancer.Balancer
 	clusterToWatch string
 
 	logger *grpclog.PrefixLogger
@@ -354,11 +353,3 @@ func (b *cdsBalancer) isClosed() bool {
 	b.mu.Unlock()
 	return closed
 }
-
-func (b *cdsBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
-	b.logger.Errorf("UpdateSubConnState should be called instead of HandleSubConnStateChange")
-}
-
-func (b *cdsBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
-	b.logger.Errorf("UpdateClientConnState should be called instead of HandleResolvedAddrs")
-}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 2d5d305ab9fa..683c40b89cb7 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -219,11 +219,11 @@ func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.Clie
 func setup() (*cdsBalancer, *testEDSBalancer, func()) {
 	builder := cdsBB{}
 	tcc := &testClientConn{}
-	cdsB := builder.Build(tcc, balancer.BuildOptions{}).(balancer.V2Balancer)
+	cdsB := builder.Build(tcc, balancer.BuildOptions{})
 
 	edsB := newTestEDSBalancer()
 	oldEDSBalancerBuilder := newEDSBalancer
-	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.V2Balancer, error) {
+	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
 		return edsB, nil
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index d1578c32771d..02ac314cd71b 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -31,7 +31,6 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -104,8 +103,6 @@ type edsBalancerImplInterface interface {
 	close()
 }
 
-var _ balancer.V2Balancer = (*edsBalancer)(nil) // Assert that we implement V2Balancer
-
 // edsBalancer manages xdsClient and the actual EDS balancer implementation that
 // does load balancing.
 //
@@ -210,14 +207,6 @@ type subConnStateUpdate struct {
 	state balancer.SubConnState
 }
 
-func (x *edsBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
-	x.logger.Errorf("UpdateSubConnState should be called instead of HandleSubConnStateChange")
-}
-
-func (x *edsBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
-	x.logger.Errorf("UpdateClientConnState should be called instead of HandleResolvedAddrs")
-}
-
 func (x *edsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	update := &subConnStateUpdate{
 		sc:    sc,
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 82d39e252e63..0aeb6cc328ad 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -396,10 +396,6 @@ type edsBalancerWrapperCC struct {
 func (ebwcc *edsBalancerWrapperCC) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
 	return ebwcc.parent.newSubConn(ebwcc.priority, addrs, opts)
 }
-
-func (ebwcc *edsBalancerWrapperCC) UpdateBalancerState(state connectivity.State, picker balancer.Picker) {
-}
-
 func (ebwcc *edsBalancerWrapperCC) UpdateState(state balancer.State) {
 	ebwcc.parent.enqueueChildBalancerStateUpdate(ebwcc.priority, state)
 }
@@ -426,11 +422,11 @@ func (edsImpl *edsBalancerImpl) close() {
 
 type dropPicker struct {
 	drops     []*dropper
-	p         balancer.V2Picker
+	p         balancer.Picker
 	loadStore lrs.Store
 }
 
-func newDropPicker(p balancer.V2Picker, drops []*dropper, loadStore lrs.Store) *dropPicker {
+func newDropPicker(p balancer.Picker, drops []*dropper, loadStore lrs.Store) *dropPicker {
 	return &dropPicker{
 		drops:     drops,
 		p:         p,
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index a7d65a13854b..a279ddc64374 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -46,7 +46,7 @@ func (edsImpl *edsBalancerImpl) handlePriorityChange() {
 	// Everything was removed by EDS.
 	if !edsImpl.priorityLowest.isSet() {
 		edsImpl.priorityInUse = newPriorityTypeUnset()
-		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPickerV2(balancer.ErrTransientFailure)})
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(balancer.ErrTransientFailure)})
 		return
 	}
 
@@ -73,7 +73,7 @@ func (edsImpl *edsBalancerImpl) handlePriorityChange() {
 			// We don't have an old state to send to parent, but we also don't
 			// want parent to keep using picker from old_priorityInUse. Send an
 			// update to trigger block picks until a new picker is ready.
-			edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: base.NewErrPickerV2(balancer.ErrNoSubConnAvailable)})
+			edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable)})
 		}
 		return
 	}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 550807c3f241..1e449430697f 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -27,7 +27,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -504,16 +503,21 @@ type testInlineUpdateBalancer struct {
 	cc balancer.ClientConn
 }
 
-func (tb *testInlineUpdateBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+func (tb *testInlineUpdateBalancer) ResolverError(error) {
+	panic("not implemented")
+}
+
+func (tb *testInlineUpdateBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {
 }
 
 var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not")
 
-func (tb *testInlineUpdateBalancer) HandleResolvedAddrs(a []resolver.Address, err error) {
+func (tb *testInlineUpdateBalancer) UpdateClientConnState(balancer.ClientConnState) error {
 	tb.cc.UpdateState(balancer.State{
 		ConnectivityState: connectivity.Ready,
 		Picker:            &testutils.TestConstPicker{Err: errTestInlineStateUpdate},
 	})
+	return nil
 }
 
 func (*testInlineUpdateBalancer) Close() {
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index a9e9e663bdef..5f8b04d6820f 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -55,7 +55,7 @@ func init() {
 	}
 }
 
-func subConnFromPicker(p balancer.V2Picker) func() balancer.SubConn {
+func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
 	return func() balancer.SubConn {
 		scst, _ := p.Pick(balancer.PickInfo{})
 		return scst.SubConn
@@ -288,11 +288,15 @@ type fakeBalancer struct {
 	cc balancer.ClientConn
 }
 
-func (b *fakeBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
+func (b *fakeBalancer) ResolverError(error) {
 	panic("implement me")
 }
 
-func (b *fakeBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+func (b *fakeBalancer) UpdateClientConnState(balancer.ClientConnState) error {
+	panic("implement me")
+}
+
+func (b *fakeBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {
 	panic("implement me")
 }
 
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 604417c8211b..ec3c4b5c6e91 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -70,7 +70,7 @@ type TestClientConn struct {
 	NewSubConnCh      chan balancer.SubConn   // the last 10 subconn created.
 	RemoveSubConnCh   chan balancer.SubConn   // the last 10 subconn removed.
 
-	NewPickerCh chan balancer.V2Picker  // the last picker updated.
+	NewPickerCh chan balancer.Picker    // the last picker updated.
 	NewStateCh  chan connectivity.State // the last state.
 
 	subConnIdx int
@@ -85,7 +85,7 @@ func NewTestClientConn(t *testing.T) *TestClientConn {
 		NewSubConnCh:      make(chan balancer.SubConn, 10),
 		RemoveSubConnCh:   make(chan balancer.SubConn, 10),
 
-		NewPickerCh: make(chan balancer.V2Picker, 1),
+		NewPickerCh: make(chan balancer.Picker, 1),
 		NewStateCh:  make(chan connectivity.State, 1),
 	}
 }
@@ -285,15 +285,20 @@ type testConstBalancer struct {
 	cc balancer.ClientConn
 }
 
-func (tb *testConstBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+func (tb *testConstBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	tb.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &TestConstPicker{Err: ErrTestConstPicker}})
 }
 
-func (tb *testConstBalancer) HandleResolvedAddrs(a []resolver.Address, err error) {
-	if len(a) == 0 {
-		return
+func (tb *testConstBalancer) ResolverError(error) {
+	panic("not implemented")
+}
+
+func (tb *testConstBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	if len(s.ResolverState.Addresses) == 0 {
+		return nil
 	}
-	tb.cc.NewSubConn(a, balancer.NewSubConnOptions{})
+	tb.cc.NewSubConn(s.ResolverState.Addresses, balancer.NewSubConnOptions{})
+	return nil
 }
 
 func (*testConstBalancer) Close() {

From d70354e6e20fea44145c7c996163804436c6a24c Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Tue, 28 Apr 2020 16:00:26 -0700
Subject: [PATCH 049/481] advancedtls: add field names for unit tests (#3570)

* advancedtls: add field names for unit tests
---
 .../advancedtls_integration_test.go           |  48 +-
 security/advancedtls/advancedtls_test.go      | 448 ++++++------------
 2 files changed, 171 insertions(+), 325 deletions(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index c259e1146552..84ca953f00ba 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -216,8 +216,7 @@ func TestEnd2End(t *testing.T) {
 		// should see it again accepts the connection, since clientPeer2 is trusted
 		// by serverTrust2.
 		{
-			desc:       "TestClientPeerCertReloadServerTrustCertReload",
-			clientCert: nil,
+			desc: "TestClientPeerCertReloadServerTrustCertReload",
 			clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
@@ -226,15 +225,12 @@ func TestEnd2End(t *testing.T) {
 					return &cs.clientPeer2, nil
 				}
 			},
-			clientGetRoot: nil,
-			clientRoot:    cs.clientTrust1,
+			clientRoot: cs.clientTrust1,
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
-			clientVType:   CertVerification,
-			serverCert:    []tls.Certificate{cs.serverPeer1},
-			serverGetCert: nil,
-			serverRoot:    nil,
+			clientVType: CertVerification,
+			serverCert:  []tls.Certificate{cs.serverPeer1},
 			serverGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
 				case 0, 1:
@@ -261,9 +257,8 @@ func TestEnd2End(t *testing.T) {
 		// should see it again accepts the connection, since serverPeer2 is trusted
 		// by clientTrust2.
 		{
-			desc:          "TestServerPeerCertReloadClientTrustCertReload",
-			clientCert:    []tls.Certificate{cs.clientPeer1},
-			clientGetCert: nil,
+			desc:       "TestServerPeerCertReloadClientTrustCertReload",
+			clientCert: []tls.Certificate{cs.clientPeer1},
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
 				case 0, 1:
@@ -272,12 +267,10 @@ func TestEnd2End(t *testing.T) {
 					return &GetRootCAsResults{TrustCerts: cs.clientTrust2}, nil
 				}
 			},
-			clientRoot: nil,
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
 			clientVType: CertVerification,
-			serverCert:  nil,
 			serverGetCert: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
@@ -286,8 +279,7 @@ func TestEnd2End(t *testing.T) {
 					return &cs.serverPeer2, nil
 				}
 			},
-			serverRoot:    cs.serverTrust1,
-			serverGetRoot: nil,
+			serverRoot: cs.serverTrust1,
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
@@ -307,9 +299,8 @@ func TestEnd2End(t *testing.T) {
 		// At stage 2, the client changes authorization check to only accept
 		// serverPeer2. Now we should see the connection becomes normal again.
 		{
-			desc:          "TestClientCustomVerification",
-			clientCert:    []tls.Certificate{cs.clientPeer1},
-			clientGetCert: nil,
+			desc:       "TestClientCustomVerification",
+			clientCert: []tls.Certificate{cs.clientPeer1},
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
 				case 0:
@@ -318,7 +309,6 @@ func TestEnd2End(t *testing.T) {
 					return &GetRootCAsResults{TrustCerts: cs.clientTrust2}, nil
 				}
 			},
-			clientRoot: nil,
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				if len(params.RawCerts) == 0 {
 					return nil, fmt.Errorf("no peer certs")
@@ -346,7 +336,6 @@ func TestEnd2End(t *testing.T) {
 				return nil, fmt.Errorf("custom authz check fails")
 			},
 			clientVType: CertVerification,
-			serverCert:  nil,
 			serverGetCert: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
@@ -355,8 +344,7 @@ func TestEnd2End(t *testing.T) {
 					return &cs.serverPeer2, nil
 				}
 			},
-			serverRoot:    cs.serverTrust1,
-			serverGetRoot: nil,
+			serverRoot: cs.serverTrust1,
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
@@ -374,19 +362,15 @@ func TestEnd2End(t *testing.T) {
 		// At stage 2, server allows all the connections again and the
 		// authentications should go back to normal.
 		{
-			desc:          "TestServerCustomVerification",
-			clientCert:    []tls.Certificate{cs.clientPeer1},
-			clientGetCert: nil,
-			clientGetRoot: nil,
-			clientRoot:    cs.clientTrust1,
+			desc:       "TestServerCustomVerification",
+			clientCert: []tls.Certificate{cs.clientPeer1},
+			clientRoot: cs.clientTrust1,
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
-			clientVType:   CertVerification,
-			serverCert:    []tls.Certificate{cs.serverPeer1},
-			serverGetCert: nil,
-			serverRoot:    cs.serverTrust1,
-			serverGetRoot: nil,
+			clientVType: CertVerification,
+			serverCert:  []tls.Certificate{cs.serverPeer1},
+			serverRoot:  cs.serverTrust1,
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				switch stage.read() {
 				case 0, 2:
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 6c1909c44863..8dea2e493dbb 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -73,7 +73,7 @@ func TestClientServerHandshake(t *testing.T) {
 	for _, test := range []struct {
 		desc                       string
 		clientCert                 []tls.Certificate
-		clientGetClientCert        func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+		clientGetCert              func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
 		clientRoot                 *x509.CertPool
 		clientGetRoot              func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		clientVerifyFunc           CustomVerificationFunc
@@ -97,47 +97,24 @@ func TestClientServerHandshake(t *testing.T) {
 		// even setting vType to SkipVerification. Clients should at least provide
 		// their own verification logic.
 		{
-			"Client_no_trust_cert_Server_peer_cert",
-			nil,
-			nil,
-			nil,
-			nil,
-			nil,
-			SkipVerification,
-			true,
-			false,
-			false,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			CertAndHostVerification,
-			true,
+			desc:                    "Client has no trust cert; server sends peer cert",
+			clientVType:             SkipVerification,
+			clientExpectCreateError: true,
+			serverCert:              []tls.Certificate{serverPeerCert},
+			serverVType:             CertAndHostVerification,
+			serverExpectError:       true,
 		},
 		// Client: nil setting except verifyFuncGood
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: success
 		// Reason: we will use verifyFuncGood to verify the server,
-		// if either clientCert or clientGetClientCert is not set
+		// if either clientCert or clientGetCert is not set
 		{
-			"Client_no_trust_cert_verifyFuncGood_Server_peer_cert",
-			nil,
-			nil,
-			nil,
-			nil,
-			verifyFuncGood,
-			SkipVerification,
-			false,
-			false,
-			false,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			CertAndHostVerification,
-			false,
+			desc:             "Client has no trust cert with verifyFuncGood; server sends peer cert",
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      SkipVerification,
+			serverCert:       []tls.Certificate{serverPeerCert},
+			serverVType:      CertAndHostVerification,
 		},
 		// Client: only set clientRoot
 		// Server: only set serverCert with mutual TLS off
@@ -146,23 +123,13 @@ func TestClientServerHandshake(t *testing.T) {
 		// default hostname check. All the default hostname checks will fail in
 		// this test suites.
 		{
-			"Client_root_cert_Server_peer_cert",
-			nil,
-			nil,
-			clientTrustPool,
-			nil,
-			nil,
-			CertAndHostVerification,
-			false,
-			true,
-			false,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			CertAndHostVerification,
-			true,
+			desc:                       "Client has root cert; server sends peer cert",
+			clientRoot:                 clientTrustPool,
+			clientVType:                CertAndHostVerification,
+			clientExpectHandshakeError: true,
+			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverVType:                CertAndHostVerification,
+			serverExpectError:          true,
 		},
 		// Client: only set clientGetRoot
 		// Server: only set serverCert with mutual TLS off
@@ -171,113 +138,64 @@ func TestClientServerHandshake(t *testing.T) {
 		// default hostname check. All the default hostname checks will fail in
 		// this test suites.
 		{
-			"Client_reload_root_Server_peer_cert",
-			nil,
-			nil,
-			nil,
-			getRootCAsForClient,
-			nil,
-			CertAndHostVerification,
-			false,
-			true,
-			false,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			CertAndHostVerification,
-			true,
+			desc:                       "Client sets reload root function; server sends peer cert",
+			clientGetRoot:              getRootCAsForClient,
+			clientVType:                CertAndHostVerification,
+			clientExpectHandshakeError: true,
+			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverVType:                CertAndHostVerification,
+			serverExpectError:          true,
 		},
 		// Client: set clientGetRoot and clientVerifyFunc
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: success
 		{
-			"Client_reload_root_verifyFuncGood_Server_peer_cert",
-			nil,
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			false,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			CertAndHostVerification,
-			false,
+			desc:             "Client sets reload root function with verifyFuncGood; server sends peer cert",
+			clientGetRoot:    getRootCAsForClient,
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      CertVerification,
+			serverCert:       []tls.Certificate{serverPeerCert},
+			serverVType:      CertAndHostVerification,
 		},
 		// Client: set clientGetRoot and bad clientVerifyFunc function
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: server side failure and client handshake failure
 		// Reason: custom verification function is bad
 		{
-			"Client_reload_root_verifyFuncBad_Server_peer_cert",
-			nil,
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncBad,
-			CertVerification,
-			false,
-			true,
-			false,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			CertVerification,
-			true,
+			desc:                       "Client sets reload root function with verifyFuncBad; server sends peer cert",
+			clientGetRoot:              getRootCAsForClient,
+			clientVerifyFunc:           verifyFuncBad,
+			clientVType:                CertVerification,
+			clientExpectHandshakeError: true,
+			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverVType:                CertVerification,
+			serverExpectError:          true,
 		},
 		// Client: set clientGetRoot and clientVerifyFunc
 		// Server: nil setting
 		// Expected Behavior: server side failure
 		// Reason: server side must either set serverCert or serverGetCert
 		{
-			"Client_reload_root_verifyFuncGood_Server_nil",
-			nil,
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			false,
-			nil,
-			nil,
-			nil,
-			nil,
-			nil,
-			CertVerification,
-			true,
+			desc:              "Client sets reload root function with verifyFuncGood; server sets nil",
+			clientGetRoot:     getRootCAsForClient,
+			clientVerifyFunc:  verifyFuncGood,
+			clientVType:       CertVerification,
+			serverVType:       CertVerification,
+			serverExpectError: true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverRoot and serverCert with mutual TLS on
 		// Expected Behavior: success
 		{
-			"Client_peer_cert_reload_root_verifyFuncGood_Server_peer_cert_root_cert_mutualTLS",
-			[]tls.Certificate{clientPeerCert},
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			true,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			serverTrustPool,
-			nil,
-			nil,
-			CertVerification,
-			false,
+			desc:             "Client sets peer cert, reload root function with verifyFuncGood; server sets peer cert and root cert; mutualTLS",
+			clientCert:       []tls.Certificate{clientPeerCert},
+			clientGetRoot:    getRootCAsForClient,
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      CertVerification,
+			serverMutualTLS:  true,
+			serverCert:       []tls.Certificate{serverPeerCert},
+			serverRoot:       serverTrustPool,
+			serverVType:      CertVerification,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverCert, but not setting any of serverRoot, serverGetRoot
@@ -287,45 +205,30 @@ func TestClientServerHandshake(t *testing.T) {
 		// mTLS in on, even setting vType to SkipVerification. Servers should at
 		// least provide their own verification logic.
 		{
-			"Client_peer_cert_reload_root_verifyFuncGood_Server_no_verification_mutualTLS",
-			[]tls.Certificate{clientPeerCert},
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			true,
-			true,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			nil,
-			nil,
-			SkipVerification,
-			true,
+			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; server sets no verification; mutualTLS",
+			clientCert:                 []tls.Certificate{clientPeerCert},
+			clientGetRoot:              getRootCAsForClient,
+			clientVerifyFunc:           verifyFuncGood,
+			clientVType:                CertVerification,
+			clientExpectHandshakeError: true,
+			serverMutualTLS:            true,
+			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverVType:                SkipVerification,
+			serverExpectError:          true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverGetRoot and serverCert with mutual TLS on
 		// Expected Behavior: success
 		{
-			"Client_peer_cert_reload_root_verifyFuncGood_Server_peer_cert_reload_root_mutualTLS",
-			[]tls.Certificate{clientPeerCert},
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			true,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			getRootCAsForServer,
-			nil,
-			CertVerification,
-			false,
+			desc:             "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS",
+			clientCert:       []tls.Certificate{clientPeerCert},
+			clientGetRoot:    getRootCAsForClient,
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      CertVerification,
+			serverMutualTLS:  true,
+			serverCert:       []tls.Certificate{serverPeerCert},
+			serverGetRoot:    getRootCAsForServer,
+			serverVType:      CertVerification,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverGetRoot returning error and serverCert with mutual
@@ -333,49 +236,35 @@ func TestClientServerHandshake(t *testing.T) {
 		// Expected Behavior: server side failure
 		// Reason: server side reloading returns failure
 		{
-			"Client_peer_cert_reload_root_verifyFuncGood_Server_peer_cert_bad_reload_root_mutualTLS",
-			[]tls.Certificate{clientPeerCert},
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			true,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			getRootCAsForServerBad,
-			nil,
-			CertVerification,
-			true,
+			desc:              "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, bad reload root function; mutualTLS",
+			clientCert:        []tls.Certificate{clientPeerCert},
+			clientGetRoot:     getRootCAsForClient,
+			clientVerifyFunc:  verifyFuncGood,
+			clientVType:       CertVerification,
+			serverMutualTLS:   true,
+			serverCert:        []tls.Certificate{serverPeerCert},
+			serverGetRoot:     getRootCAsForServerBad,
+			serverVType:       CertVerification,
+			serverExpectError: true,
 		},
-		// Client: set clientGetRoot, clientVerifyFunc and clientGetClientCert
+		// Client: set clientGetRoot, clientVerifyFunc and clientGetCert
 		// Server: set serverGetRoot and serverGetCert with mutual TLS on
 		// Expected Behavior: success
 		{
-			"Client_reload_both_certs_verifyFuncGood_Server_reload_both_certs_mutualTLS",
-			nil,
-			func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			desc: "Client sets reload peer/root function with verifyFuncGood; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
+			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				return &clientPeerCert, nil
 			},
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			true,
-			nil,
-			func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			clientGetRoot:    getRootCAsForClient,
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      CertVerification,
+			serverMutualTLS:  true,
+			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
-			nil,
-			getRootCAsForServer,
-			verifyFuncGood,
-			CertVerification,
-			false,
+			serverGetRoot:    getRootCAsForServer,
+			serverVerifyFunc: verifyFuncGood,
+			serverVType:      CertVerification,
 		},
 		// Client: set everything but with the wrong peer cert not trusted by
 		// server
@@ -383,54 +272,43 @@ func TestClientServerHandshake(t *testing.T) {
 		// Expected Behavior: server side returns failure because of
 		// certificate mismatch
 		{
-			"Client_wrong_peer_cert_Server_reload_both_certs_mutualTLS",
-			nil,
-			func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			desc: "Client sends wrong peer cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
+			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			true,
-			nil,
-			func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			clientGetRoot:    getRootCAsForClient,
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      CertVerification,
+			serverMutualTLS:  true,
+			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
-			nil,
-			getRootCAsForServer,
-			verifyFuncGood,
-			CertVerification,
-			true,
+			serverGetRoot:     getRootCAsForServer,
+			serverVerifyFunc:  verifyFuncGood,
+			serverVType:       CertVerification,
+			serverExpectError: true,
 		},
 		// Client: set everything but with the wrong trust cert not trusting server
 		// Server: set serverGetRoot and serverGetCert with mutual TLS on
 		// Expected Behavior: server side and client side return failure due to
 		// certificate mismatch and handshake failure
 		{
-			"Client_wrong_trust_cert_Server_reload_both_certs_mutualTLS",
-			nil,
-			func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			desc: "Client has wrong trust cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
+			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				return &clientPeerCert, nil
 			},
-			nil,
-			getRootCAsForServer,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			true,
-			true,
-			nil,
-			func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			clientGetRoot:              getRootCAsForServer,
+			clientVerifyFunc:           verifyFuncGood,
+			clientVType:                CertVerification,
+			clientExpectHandshakeError: true,
+			serverMutualTLS:            true,
+			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
-			nil,
-			getRootCAsForServer,
-			verifyFuncGood,
-			CertVerification,
-			true,
+			serverGetRoot:     getRootCAsForServer,
+			serverVerifyFunc:  verifyFuncGood,
+			serverVType:       CertVerification,
+			serverExpectError: true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set everything but with the wrong peer cert not trusted by
@@ -438,77 +316,61 @@ func TestClientServerHandshake(t *testing.T) {
 		// Expected Behavior: server side and client side return failure due to
 		// certificate mismatch and handshake failure
 		{
-			"Client_reload_both_certs_verifyFuncGood_Server_wrong_peer_cert",
-			nil,
-			func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			desc: "Client sets reload peer/root function with verifyFuncGood; Server sends wrong peer cert; mutualTLS",
+			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				return &clientPeerCert, nil
 			},
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			false,
-			true,
-			nil,
-			func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			clientGetRoot:    getRootCAsForClient,
+			clientVerifyFunc: verifyFuncGood,
+			clientVType:      CertVerification,
+			serverMutualTLS:  true,
+			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &clientPeerCert, nil
 			},
-			nil,
-			getRootCAsForServer,
-			verifyFuncGood,
-			CertVerification,
-			true,
+			serverGetRoot:     getRootCAsForServer,
+			serverVerifyFunc:  verifyFuncGood,
+			serverVType:       CertVerification,
+			serverExpectError: true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set everything but with the wrong trust cert not trusting client
 		// Expected Behavior: server side and client side return failure due to
 		// certificate mismatch and handshake failure
 		{
-			"Client_reload_both_certs_verifyFuncGood_Server_wrong_trust_cert",
-			nil,
-			func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			desc: "Client sets reload peer/root function with verifyFuncGood; Server has wrong trust cert; mutualTLS",
+			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				return &clientPeerCert, nil
 			},
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			true,
-			true,
-			nil,
-			func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			clientGetRoot:              getRootCAsForClient,
+			clientVerifyFunc:           verifyFuncGood,
+			clientVType:                CertVerification,
+			clientExpectHandshakeError: true,
+			serverMutualTLS:            true,
+			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			true,
+			serverGetRoot:     getRootCAsForClient,
+			serverVerifyFunc:  verifyFuncGood,
+			serverVType:       CertVerification,
+			serverExpectError: true,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverGetRoot and serverCert, but with bad verifyFunc
 		// Expected Behavior: server side and client side return failure due to
 		// server custom check fails
 		{
-			"Client_peer_cert_reload_root_verifyFuncGood_Server_bad_custom_verification_mutualTLS",
-			[]tls.Certificate{clientPeerCert},
-			nil,
-			nil,
-			getRootCAsForClient,
-			verifyFuncGood,
-			CertVerification,
-			false,
-			true,
-			true,
-			[]tls.Certificate{serverPeerCert},
-			nil,
-			nil,
-			getRootCAsForServer,
-			verifyFuncBad,
-			CertVerification,
-			true,
+			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; Server sets bad custom check; mutualTLS",
+			clientCert:                 []tls.Certificate{clientPeerCert},
+			clientGetRoot:              getRootCAsForClient,
+			clientVerifyFunc:           verifyFuncGood,
+			clientVType:                CertVerification,
+			clientExpectHandshakeError: true,
+			serverMutualTLS:            true,
+			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverGetRoot:              getRootCAsForServer,
+			serverVerifyFunc:           verifyFuncBad,
+			serverVType:                CertVerification,
+			serverExpectError:          true,
 		},
 	} {
 		test := test
@@ -560,7 +422,7 @@ func TestClientServerHandshake(t *testing.T) {
 			defer conn.Close()
 			clientOptions := &ClientOptions{
 				Certificates:         test.clientCert,
-				GetClientCertificate: test.clientGetClientCert,
+				GetClientCertificate: test.clientGetCert,
 				VerifyPeer:           test.clientVerifyFunc,
 				RootCertificateOptions: RootCertificateOptions{
 					RootCACerts: test.clientRoot,

From accb8795961cc3bef08879b130dc16c04e849799 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 28 Apr 2020 16:47:39 -0700
Subject: [PATCH 050/481] cleanup: remove lingering V2 references (#3576)

---
 balancer/rls/internal/balancer.go      | 17 ++---------------
 balancer/rls/internal/balancer_test.go |  8 ++++----
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/balancer/rls/internal/balancer.go b/balancer/rls/internal/balancer.go
index 7c4a4817466a..2b8162485047 100644
--- a/balancer/rls/internal/balancer.go
+++ b/balancer/rls/internal/balancer.go
@@ -25,15 +25,12 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
-	"google.golang.org/grpc/resolver"
 )
 
 var (
-	_ balancer.Balancer   = (*rlsBalancer)(nil)
-	_ balancer.V2Balancer = (*rlsBalancer)(nil)
+	_ balancer.Balancer = (*rlsBalancer)(nil)
 
 	// For overriding in tests.
 	newRLSClientFunc = newRLSClient
@@ -122,7 +119,7 @@ func (lb *rlsBalancer) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConn
 
 // Cleans up the resources allocated by the LB policy including the clientConn
 // to the RLS server.
-// Implements balancer.Balancer and balancer.V2Balancer interfaces.
+// Implements balancer.Balancer.
 func (lb *rlsBalancer) Close() {
 	lb.mu.Lock()
 	defer lb.mu.Unlock()
@@ -133,16 +130,6 @@ func (lb *rlsBalancer) Close() {
 	}
 }
 
-// HandleSubConnStateChange implements balancer.Balancer interface.
-func (lb *rlsBalancer) HandleSubConnStateChange(_ balancer.SubConn, _ connectivity.State) {
-	grpclog.Fatal("UpdateSubConnState should be called instead of HandleSubConnStateChange")
-}
-
-// HandleResolvedAddrs implements balancer.Balancer interface.
-func (lb *rlsBalancer) HandleResolvedAddrs(_ []resolver.Address, _ error) {
-	grpclog.Fatal("UpdateClientConnState should be called instead of HandleResolvedAddrs")
-}
-
 // updateControlChannel updates the RLS client if required.
 // Caller must hold lb.mu.
 func (lb *rlsBalancer) updateControlChannel(newCfg *lbConfig) {
diff --git a/balancer/rls/internal/balancer_test.go b/balancer/rls/internal/balancer_test.go
index 990372d0e98f..9df769c2a8c3 100644
--- a/balancer/rls/internal/balancer_test.go
+++ b/balancer/rls/internal/balancer_test.go
@@ -93,7 +93,7 @@ func (s) TestUpdateControlChannelFirstConfig(t *testing.T) {
 	if bb == nil {
 		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
 	}
-	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}).(balancer.V2Balancer)
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{})
 	defer rlsB.Close()
 	t.Log("Built RLS LB policy ...")
 
@@ -126,7 +126,7 @@ func (s) TestUpdateControlChannelSwitch(t *testing.T) {
 	if bb == nil {
 		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
 	}
-	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}).(balancer.V2Balancer)
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{})
 	defer rlsB.Close()
 	t.Log("Built RLS LB policy ...")
 
@@ -164,7 +164,7 @@ func (s) TestUpdateControlChannelTimeout(t *testing.T) {
 	if bb == nil {
 		t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName)
 	}
-	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}).(balancer.V2Balancer)
+	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{})
 	defer rlsB.Close()
 	t.Log("Built RLS LB policy ...")
 
@@ -209,7 +209,7 @@ func (s) TestUpdateControlChannelWithCreds(t *testing.T) {
 	}
 	rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{
 		DialCreds: cCreds,
-	}).(balancer.V2Balancer)
+	})
 	defer rlsB.Close()
 	t.Log("Built RLS LB policy ...")
 

From 80ff2c9d786682aacc48ab66df1babf5a587859a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 28 Apr 2020 18:27:18 -0700
Subject: [PATCH 051/481] internal: code regen move file to correct location
 (#3574)

---
 .travis.yml   | 1 -
 regenerate.sh | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0e24e59f0567..afd7280a8724 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,7 +25,6 @@ before_install:
   - if [[ "${GO111MODULE}" = "on" ]]; then mkdir "${HOME}/go"; export GOPATH="${HOME}/go"; fi
   - if [[ -n "${RUN386}" ]]; then export GOARCH=386; fi
   - if [[ "${TRAVIS_EVENT_TYPE}" = "cron" && -z "${RUN386}" ]]; then RACE=1; fi
-  - if [[ "${TRAVIS_EVENT_TYPE}" != "cron" ]]; then export VET_SKIP_PROTO=1; fi
 
 install:
   - try3() { eval "$*" || eval "$*" || eval "$*"; }
diff --git a/regenerate.sh b/regenerate.sh
index 17f3c85138e5..9dc72e16af33 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -73,4 +73,4 @@ rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/testv3.pb.go
 # grpc/service_config/service_config.proto does not have a go_package option.
 cp ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config
 
-cp -R ${WORKDIR}/out/google.golang.org/grpc/ .
+cp -R ${WORKDIR}/out/google.golang.org/grpc/* .

From 695da821e98cb7a10c41e3f5008aa994560e551a Mon Sep 17 00:00:00 2001
From: Eric Gribkoff <ericgribkoff@google.com>
Date: Tue, 28 Apr 2020 19:30:16 -0700
Subject: [PATCH 052/481] interop: increase xds test client rpc timeout (#3579)

---
 interop/xds/client/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 77b4563b7bef..de3e1ebf8093 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -51,7 +51,7 @@ var (
 	numChannels     = flag.Int("num_channels", 1, "Num of channels")
 	printResponse   = flag.Bool("print_response", false, "Write RPC response to stdout")
 	qps             = flag.Int("qps", 1, "QPS per channel")
-	rpcTimeout      = flag.Duration("rpc_timeout", 10*time.Second, "Per RPC timeout")
+	rpcTimeout      = flag.Duration("rpc_timeout", 20*time.Second, "Per RPC timeout")
 	server          = flag.String("server", "localhost:8080", "Address of server to connect to")
 	statsPort       = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service")
 

From 4b0710563d84d873df236c61166090f0f588d50b Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 29 Apr 2020 09:40:14 -0700
Subject: [PATCH 053/481] internal: revert change to make travis always build
 proto (#3580)

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index afd7280a8724..0e24e59f0567 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,6 +25,7 @@ before_install:
   - if [[ "${GO111MODULE}" = "on" ]]; then mkdir "${HOME}/go"; export GOPATH="${HOME}/go"; fi
   - if [[ -n "${RUN386}" ]]; then export GOARCH=386; fi
   - if [[ "${TRAVIS_EVENT_TYPE}" = "cron" && -z "${RUN386}" ]]; then RACE=1; fi
+  - if [[ "${TRAVIS_EVENT_TYPE}" != "cron" ]]; then export VET_SKIP_PROTO=1; fi
 
 install:
   - try3() { eval "$*" || eval "$*" || eval "$*"; }

From eedec2c1c3fe8a87d6c966dba773c3c6bb984b12 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Wed, 29 Apr 2020 11:00:02 -0700
Subject: [PATCH 054/481] advancedtls: add leaf cert in verify params (#3571)

---
 security/advancedtls/advancedtls.go      |  7 +++
 security/advancedtls/advancedtls_test.go | 58 ++++++++++++++++--------
 2 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index 529ecc76f285..db78a566262e 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -46,6 +46,10 @@ type VerificationFuncParams struct {
 	// The verification chain obtained by checking peer RawCerts against the
 	// trust certificate bundle(s), if applicable.
 	VerifiedChains [][]*x509.Certificate
+	// The leaf certificate sent from peer, if choosing to verify the peer
+	// certificate(s) and that verification passed. This field would be nil if
+	// either user chose not to verify or the verification failed.
+	Leaf *x509.Certificate
 }
 
 // VerificationResults contains the information about results of
@@ -313,6 +317,7 @@ func buildVerifyFunc(c *advancedTLSCreds,
 	rawConn net.Conn) func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
 	return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
 		chains := verifiedChains
+		var leafCert *x509.Certificate
 		if c.vType == CertAndHostVerification || c.vType == CertVerification {
 			// perform possible trust credential reloading and certificate check
 			rootCAs := c.config.RootCAs
@@ -361,6 +366,7 @@ func buildVerifyFunc(c *advancedTLSCreds,
 			if err != nil {
 				return err
 			}
+			leafCert = certs[0]
 		}
 		// Perform custom verification check if specified.
 		if c.verifyFunc != nil {
@@ -368,6 +374,7 @@ func buildVerifyFunc(c *advancedTLSCreds,
 				ServerName:     serverName,
 				RawCerts:       rawCerts,
 				VerifiedChains: chains,
+				Leaf:           leafCert,
 			})
 			return err
 		}
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 8dea2e493dbb..ab6ba59068ac 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -23,6 +23,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -43,7 +44,15 @@ func TestClientServerHandshake(t *testing.T) {
 	getRootCAsForClient := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 		return &GetRootCAsResults{TrustCerts: clientTrustPool}, nil
 	}
-	verifyFuncGood := func(params *VerificationFuncParams) (*VerificationResults, error) {
+	clientVerifyFuncGood := func(params *VerificationFuncParams) (*VerificationResults, error) {
+		if params.ServerName == "" {
+			return nil, errors.New("client side server name should have a value")
+		}
+		// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
+		if len(params.VerifiedChains) > 0 && (params.Leaf == nil || params.Leaf.Subject.CommonName != "foo.bar.com") {
+			return nil, errors.New("client side params parsing error")
+		}
+
 		return &VerificationResults{}, nil
 	}
 	verifyFuncBad := func(params *VerificationFuncParams) (*VerificationResults, error) {
@@ -62,6 +71,17 @@ func TestClientServerHandshake(t *testing.T) {
 	getRootCAsForServer := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 		return &GetRootCAsResults{TrustCerts: serverTrustPool}, nil
 	}
+	serverVerifyFunc := func(params *VerificationFuncParams) (*VerificationResults, error) {
+		if params.ServerName != "" {
+			return nil, errors.New("server side server name should not have a value")
+		}
+		// "foo.bar.hoo.com" is the common name on client certificate client_cert_1.pem.
+		if len(params.VerifiedChains) > 0 && (params.Leaf == nil || params.Leaf.Subject.CommonName != "foo.bar.hoo.com") {
+			return nil, errors.New("server side params parsing error")
+		}
+
+		return &VerificationResults{}, nil
+	}
 	serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"),
 		testdata.Path("server_key_1.pem"))
 	if err != nil {
@@ -111,7 +131,7 @@ func TestClientServerHandshake(t *testing.T) {
 		// if either clientCert or clientGetCert is not set
 		{
 			desc:             "Client has no trust cert with verifyFuncGood; server sends peer cert",
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      SkipVerification,
 			serverCert:       []tls.Certificate{serverPeerCert},
 			serverVType:      CertAndHostVerification,
@@ -152,7 +172,7 @@ func TestClientServerHandshake(t *testing.T) {
 		{
 			desc:             "Client sets reload root function with verifyFuncGood; server sends peer cert",
 			clientGetRoot:    getRootCAsForClient,
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverCert:       []tls.Certificate{serverPeerCert},
 			serverVType:      CertAndHostVerification,
@@ -178,7 +198,7 @@ func TestClientServerHandshake(t *testing.T) {
 		{
 			desc:              "Client sets reload root function with verifyFuncGood; server sets nil",
 			clientGetRoot:     getRootCAsForClient,
-			clientVerifyFunc:  verifyFuncGood,
+			clientVerifyFunc:  clientVerifyFuncGood,
 			clientVType:       CertVerification,
 			serverVType:       CertVerification,
 			serverExpectError: true,
@@ -190,7 +210,7 @@ func TestClientServerHandshake(t *testing.T) {
 			desc:             "Client sets peer cert, reload root function with verifyFuncGood; server sets peer cert and root cert; mutualTLS",
 			clientCert:       []tls.Certificate{clientPeerCert},
 			clientGetRoot:    getRootCAsForClient,
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverCert:       []tls.Certificate{serverPeerCert},
@@ -208,7 +228,7 @@ func TestClientServerHandshake(t *testing.T) {
 			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; server sets no verification; mutualTLS",
 			clientCert:                 []tls.Certificate{clientPeerCert},
 			clientGetRoot:              getRootCAsForClient,
-			clientVerifyFunc:           verifyFuncGood,
+			clientVerifyFunc:           clientVerifyFuncGood,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
@@ -223,7 +243,7 @@ func TestClientServerHandshake(t *testing.T) {
 			desc:             "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS",
 			clientCert:       []tls.Certificate{clientPeerCert},
 			clientGetRoot:    getRootCAsForClient,
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverCert:       []tls.Certificate{serverPeerCert},
@@ -239,7 +259,7 @@ func TestClientServerHandshake(t *testing.T) {
 			desc:              "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, bad reload root function; mutualTLS",
 			clientCert:        []tls.Certificate{clientPeerCert},
 			clientGetRoot:     getRootCAsForClient,
-			clientVerifyFunc:  verifyFuncGood,
+			clientVerifyFunc:  clientVerifyFuncGood,
 			clientVType:       CertVerification,
 			serverMutualTLS:   true,
 			serverCert:        []tls.Certificate{serverPeerCert},
@@ -256,14 +276,14 @@ func TestClientServerHandshake(t *testing.T) {
 				return &clientPeerCert, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
 			serverGetRoot:    getRootCAsForServer,
-			serverVerifyFunc: verifyFuncGood,
+			serverVerifyFunc: serverVerifyFunc,
 			serverVType:      CertVerification,
 		},
 		// Client: set everything but with the wrong peer cert not trusted by
@@ -277,14 +297,14 @@ func TestClientServerHandshake(t *testing.T) {
 				return &serverPeerCert, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &serverPeerCert, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
-			serverVerifyFunc:  verifyFuncGood,
+			serverVerifyFunc:  serverVerifyFunc,
 			serverVType:       CertVerification,
 			serverExpectError: true,
 		},
@@ -298,7 +318,7 @@ func TestClientServerHandshake(t *testing.T) {
 				return &clientPeerCert, nil
 			},
 			clientGetRoot:              getRootCAsForServer,
-			clientVerifyFunc:           verifyFuncGood,
+			clientVerifyFunc:           clientVerifyFuncGood,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
@@ -306,7 +326,7 @@ func TestClientServerHandshake(t *testing.T) {
 				return &serverPeerCert, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
-			serverVerifyFunc:  verifyFuncGood,
+			serverVerifyFunc:  serverVerifyFunc,
 			serverVType:       CertVerification,
 			serverExpectError: true,
 		},
@@ -321,14 +341,14 @@ func TestClientServerHandshake(t *testing.T) {
 				return &clientPeerCert, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
-			clientVerifyFunc: verifyFuncGood,
+			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 				return &clientPeerCert, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
-			serverVerifyFunc:  verifyFuncGood,
+			serverVerifyFunc:  serverVerifyFunc,
 			serverVType:       CertVerification,
 			serverExpectError: true,
 		},
@@ -342,7 +362,7 @@ func TestClientServerHandshake(t *testing.T) {
 				return &clientPeerCert, nil
 			},
 			clientGetRoot:              getRootCAsForClient,
-			clientVerifyFunc:           verifyFuncGood,
+			clientVerifyFunc:           clientVerifyFuncGood,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
@@ -350,7 +370,7 @@ func TestClientServerHandshake(t *testing.T) {
 				return &serverPeerCert, nil
 			},
 			serverGetRoot:     getRootCAsForClient,
-			serverVerifyFunc:  verifyFuncGood,
+			serverVerifyFunc:  serverVerifyFunc,
 			serverVType:       CertVerification,
 			serverExpectError: true,
 		},
@@ -362,7 +382,7 @@ func TestClientServerHandshake(t *testing.T) {
 			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; Server sets bad custom check; mutualTLS",
 			clientCert:                 []tls.Certificate{clientPeerCert},
 			clientGetRoot:              getRootCAsForClient,
-			clientVerifyFunc:           verifyFuncGood,
+			clientVerifyFunc:           clientVerifyFuncGood,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,

From fe604e1fdf1925823924fbb3d5eae5834571a287 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 29 Apr 2020 12:56:03 -0700
Subject: [PATCH 055/481] xds: add weighted_target balancer (#3541)

---
 .../balancer/balancergroup/balancergroup.go   |  28 ++-
 .../balancer/weightedtarget/logging.go        |  29 +++
 .../balancer/weightedtarget/weightedtarget.go | 151 ++++++++++++
 .../weightedtarget/weightedtarget_config.go   |  56 +++++
 .../weightedtarget_config_test.go             | 104 ++++++++
 .../weightedtarget/weightedtarget_test.go     | 225 ++++++++++++++++++
 6 files changed, 589 insertions(+), 4 deletions(-)
 create mode 100644 xds/internal/balancer/weightedtarget/logging.go
 create mode 100644 xds/internal/balancer/weightedtarget/weightedtarget.go
 create mode 100644 xds/internal/balancer/weightedtarget/weightedtarget_config.go
 create mode 100644 xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
 create mode 100644 xds/internal/balancer/weightedtarget/weightedtarget_test.go

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index d6428afb96a5..44468c104736 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -126,6 +126,22 @@ func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnSta
 	return b.UpdateClientConnState(s)
 }
 
+func (sbc *subBalancerWithConfig) resolverError(err error) {
+	b := sbc.balancer
+	if b == nil {
+		// This sub-balancer was closed. This should never happen because
+		// sub-balancers are closed when the locality is removed from EDS, or
+		// the balancer group is closed. There should be no further address
+		// updates when either of this happened.
+		//
+		// This will be a common case with priority support, because a
+		// sub-balancer (and the whole balancer group) could be closed because
+		// it's the lower priority, but it can still get address updates.
+		return
+	}
+	b.ResolverError(err)
+}
+
 func (sbc *subBalancerWithConfig) stopBalancer() {
 	sbc.balancer.Close()
 	sbc.balancer = nil
@@ -451,10 +467,14 @@ func (bg *BalancerGroup) UpdateClientConnState(id internal.LocalityID, s balance
 	return nil
 }
 
-// TODO: handleServiceConfig()
-//
-// For BNS address for slicer, comes from endpoint.Metadata. It will be sent
-// from parent to sub-balancers as service config.
+// ResolverError forwards resolver errors to all sub-balancers.
+func (bg *BalancerGroup) ResolverError(err error) {
+	bg.outgoingMu.Lock()
+	for _, config := range bg.idToBalancerConfig {
+		config.resolverError(err)
+	}
+	bg.outgoingMu.Unlock()
+}
 
 // Following are actions from sub-balancers, forward to ClientConn.
 
diff --git a/xds/internal/balancer/weightedtarget/logging.go b/xds/internal/balancer/weightedtarget/logging.go
new file mode 100644
index 000000000000..5f81da926fb8
--- /dev/null
+++ b/xds/internal/balancer/weightedtarget/logging.go
@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package weightedtarget
+
+import (
+	"fmt"
+)
+
+const prefix = "[weighted-target-lb %p] "
+
+func loggingPrefix(p *weightedTargetBalancer) string {
+	return fmt.Sprintf(prefix, p)
+}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
new file mode 100644
index 000000000000..f90f90f8b66d
--- /dev/null
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -0,0 +1,151 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package weightedtarget implements the weighted_target balancer.
+package weightedtarget
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+)
+
+const weightedTargetName = "weighted_target_experimental"
+
+func init() {
+	balancer.Register(&weightedTargetBB{})
+}
+
+type weightedTargetBB struct{}
+
+func (wt *weightedTargetBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+	b := &weightedTargetBalancer{}
+	b.logger = grpclog.NewPrefixLogger(loggingPrefix(b))
+	b.bg = balancergroup.New(cc, nil, b.logger)
+	b.bg.Start()
+	b.logger.Infof("Created")
+	return b
+}
+
+func (wt *weightedTargetBB) Name() string {
+	return weightedTargetName
+}
+
+func (wt *weightedTargetBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type weightedTargetBalancer struct {
+	logger *grpclog.PrefixLogger
+
+	// TODO: Make this package not dependent on any xds specific code.
+	// BalancerGroup uses xdsinternal.LocalityID as the key in the map of child
+	// policies that it maintains and reports load using LRS. Once these two
+	// dependencies are removed from the balancerGroup, this package will not
+	// have any dependencies on xds code.
+	bg *balancergroup.BalancerGroup
+
+	targets map[string]target
+}
+
+// TODO: remove this and use strings directly as keys for balancer group.
+func makeLocalityFromName(name string) internal.LocalityID {
+	return internal.LocalityID{Region: name}
+}
+
+// UpdateClientConnState takes the new targets in balancer group,
+// creates/deletes sub-balancers and sends them update. Addresses are split into
+// groups based on hierarchy path.
+func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
+
+	// Remove sub-balancers that are not in the new config.
+	for name := range w.targets {
+		if _, ok := newConfig.Targets[name]; !ok {
+			w.bg.Remove(makeLocalityFromName(name))
+		}
+	}
+
+	// For sub-balancers in the new config
+	// - if it's new. add to balancer group,
+	// - if it's old, but has a new weight, update weight in balancer group.
+	//
+	// For all sub-balancers, forward the address/balancer config update.
+	for name, newT := range newConfig.Targets {
+		l := makeLocalityFromName(name)
+
+		oldT, ok := w.targets[name]
+		if !ok {
+			// If this is a new sub-balancer, add it.
+			w.bg.Add(l, newT.Weight, balancer.Get(newT.ChildPolicy.Name))
+		} else if newT.Weight != oldT.Weight {
+			// If this is an existing sub-balancer, update weight if necessary.
+			w.bg.ChangeWeight(l, newT.Weight)
+		}
+
+		// Forwards all the update:
+		// - Addresses are from the map after splitting with hierarchy path,
+		// - Top level service config and attributes are the same,
+		// - Balancer config comes from the targets map.
+		//
+		// TODO: handle error? How to aggregate errors and return?
+		_ = w.bg.UpdateClientConnState(l, balancer.ClientConnState{
+			ResolverState: resolver.State{
+				Addresses:     addressesSplit[name],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			},
+			BalancerConfig: newT.ChildPolicy.Config,
+		})
+	}
+
+	w.targets = newConfig.Targets
+	return nil
+}
+
+func (w *weightedTargetBalancer) ResolverError(err error) {
+	w.bg.ResolverError(err)
+}
+
+func (w *weightedTargetBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	w.bg.UpdateSubConnState(sc, state)
+}
+
+func (w *weightedTargetBalancer) Close() {
+	w.bg.Close()
+}
+
+func (w *weightedTargetBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
+	w.logger.Errorf("UpdateSubConnState should be called instead of HandleSubConnStateChange")
+}
+
+func (w *weightedTargetBalancer) HandleResolvedAddrs([]resolver.Address, error) {
+	w.logger.Errorf("UpdateClientConnState should be called instead of HandleResolvedAddrs")
+}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config.go b/xds/internal/balancer/weightedtarget/weightedtarget_config.go
new file mode 100644
index 000000000000..747ce918bc68
--- /dev/null
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_config.go
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package weightedtarget
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type target struct {
+	// Weight is the weight of the child policy.
+	Weight uint32
+	// ChildPolicy is the child policy and it's config.
+	ChildPolicy *internalserviceconfig.BalancerConfig
+}
+
+// lbConfig is the balancer config for weighted_target. The proto representation
+// is:
+//
+// message WeightedTargetConfig {
+//   message Target {
+//     uint32 weight = 1;
+//     repeated LoadBalancingConfig child_policy = 2;
+//   }
+//   map<string, Target> targets = 1;
+// }
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	Targets map[string]target
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	var cfg lbConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
new file mode 100644
index 000000000000..2208117f60e1
--- /dev/null
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package weightedtarget
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"
+)
+
+const (
+	testJSONConfig = `{
+  "targets": {
+	"cluster_1" : {
+	  "weight":75,
+	  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+	},
+	"cluster_2" : {
+	  "weight":25,
+	  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+	}
+  }
+}`
+
+	cdsName = "cds_experimental"
+)
+
+var (
+	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
+	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
+	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
+	cdsConfigJSON2  = `{"cluster":"cluster_2"}`
+	cdsConfig2, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON2))
+)
+
+func Test_parseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				Targets: map[string]target{
+					"cluster_1": {
+						Weight: 75,
+						ChildPolicy: &internalserviceconfig.BalancerConfig{
+							Name:   cdsName,
+							Config: cdsConfig1,
+						},
+					},
+					"cluster_2": {
+						Weight: 25,
+						ChildPolicy: &internalserviceconfig.BalancerConfig{
+							Name:   cdsName,
+							Config: cdsConfig2,
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.want) {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want))
+			}
+		})
+	}
+}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
new file mode 100644
index 000000000000..7f9e566ca5b5
--- /dev/null
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
@@ -0,0 +1,225 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package weightedtarget
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type testConfigBalancerBuilder struct {
+	balancer.Builder
+}
+
+func newTestConfigBalancerBuilder() *testConfigBalancerBuilder {
+	return &testConfigBalancerBuilder{
+		Builder: balancer.Get(roundrobin.Name),
+	}
+}
+
+func (t *testConfigBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	rr := t.Builder.Build(cc, opts)
+	return &testConfigBalancer{
+		Balancer: rr,
+	}
+}
+
+const testConfigBalancerName = "test_config_balancer"
+
+func (t *testConfigBalancerBuilder) Name() string {
+	return testConfigBalancerName
+}
+
+type stringBalancerConfig struct {
+	serviceconfig.LoadBalancingConfig
+	s string
+}
+
+func (t *testConfigBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	// Return string without quotes.
+	return stringBalancerConfig{s: string(c[1 : len(c)-1])}, nil
+}
+
+// testConfigBalancer is a roundrobin balancer, but it takes the balancer config
+// string and append it to the backend addresses.
+type testConfigBalancer struct {
+	balancer.Balancer
+}
+
+func (b *testConfigBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	c, ok := s.BalancerConfig.(stringBalancerConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type %T", s.BalancerConfig)
+	}
+	oneMoreAddr := resolver.Address{Addr: c.s}
+	s.BalancerConfig = nil
+	s.ResolverState.Addresses = append(s.ResolverState.Addresses, oneMoreAddr)
+	return b.Balancer.UpdateClientConnState(s)
+}
+
+func (b *testConfigBalancer) Close() {
+	b.Balancer.Close()
+}
+
+var (
+	wtbBuilder          balancer.Builder
+	wtbParser           balancer.ConfigParser
+	testBackendAddrStrs []string
+)
+
+const testBackendAddrsCount = 12
+
+func init() {
+	balancer.Register(newTestConfigBalancerBuilder())
+	for i := 0; i < testBackendAddrsCount; i++ {
+		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
+	}
+	wtbBuilder = balancer.Get(weightedTargetName)
+	wtbParser = wtbBuilder.(balancer.ConfigParser)
+
+	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond
+}
+
+// TestWeightedTarget covers the cases that a sub-balancer is added and a
+// sub-balancer is removed. It verifies that the addresses and balancer configs
+// are forwarded to the right sub-balancer.
+//
+// This test is intended to test the glue code in weighted_target. Most of the
+// functionality tests are covered by the balancer group tests.
+func TestWeightedTarget(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	wtb := wtbBuilder.Build(cc, balancer.BuildOptions{})
+
+	// Start with "cluster_1: round_robin".
+	config1, err := wtbParser.ParseConfig([]byte(`{"targets":{"cluster_1":{"weight":1,"childPolicy":[{"round_robin":""}]}}}`))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddr1 := resolver.Address{Addr: testBackendAddrStrs[0], Attributes: nil}
+	if err := wtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddr1, []string{"cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	addr1 := <-cc.NewSubConnAddrsCh
+	if want := []resolver.Address{
+		hierarchy.Set(wantAddr1, []string{}),
+	}; !cmp.Equal(addr1, want, cmp.AllowUnexported(attributes.Attributes{})) {
+		t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr1, want, cmp.AllowUnexported(attributes.Attributes{})))
+	}
+
+	// Send subconn state change.
+	sc1 := <-cc.NewSubConnCh
+	wtb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	wtb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with one backend.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+	}
+
+	// Remove cluster_1, and add "cluster_2: test_config_balancer".
+	wantAddr3Str := testBackendAddrStrs[2]
+	config2, err := wtbParser.ParseConfig([]byte(
+		fmt.Sprintf(`{"targets":{"cluster_2":{"weight":1,"childPolicy":[{%q:%q}]}}}`, testConfigBalancerName, wantAddr3Str),
+	))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and one address with hierarchy path "cluster_2".
+	wantAddr2 := resolver.Address{Addr: testBackendAddrStrs[1], Attributes: nil}
+	if err := wtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddr2, []string{"cluster_2"}),
+		}},
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect the address sent in the address list. The hierarchy path should be
+	// cleared.
+	addr2 := <-cc.NewSubConnAddrsCh
+	if want := []resolver.Address{
+		hierarchy.Set(wantAddr2, []string{}),
+	}; !cmp.Equal(addr2, want, cmp.AllowUnexported(attributes.Attributes{})) {
+		t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr2, want, cmp.AllowUnexported(attributes.Attributes{})))
+	}
+	// Expect the other address sent as balancer config. This address doesn't
+	// have hierarchy path.
+	wantAddr3 := resolver.Address{Addr: wantAddr3Str, Attributes: nil}
+	addr3 := <-cc.NewSubConnAddrsCh
+	if want := []resolver.Address{wantAddr3}; !cmp.Equal(addr3, want, cmp.AllowUnexported(attributes.Attributes{})) {
+		t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr3, want, cmp.AllowUnexported(attributes.Attributes{})))
+	}
+
+	// The subconn for cluster_1 should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
+	}
+	wtb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
+
+	sc2 := <-cc.NewSubConnCh
+	wtb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	wtb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	sc3 := <-cc.NewSubConnCh
+	wtb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	wtb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin pick with backends in cluster_2.
+	p2 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc2, sc3}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+}
+
+func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
+	return func() balancer.SubConn {
+		scst, _ := p.Pick(balancer.PickInfo{})
+		return scst.SubConn
+	}
+}

From 661d4c963003fe720d65e1a6e0e00c5a86029336 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 29 Apr 2020 15:00:09 -0700
Subject: [PATCH 056/481] internal: update service_config.pb.go (#3584)

---
 .../grpc_service_config/service_config.pb.go  | 674 +++++++++++++++---
 1 file changed, 592 insertions(+), 82 deletions(-)

diff --git a/internal/proto/grpc_service_config/service_config.pb.go b/internal/proto/grpc_service_config/service_config.pb.go
index 6b91aec4d992..c0436b62a64b 100644
--- a/internal/proto/grpc_service_config/service_config.pb.go
+++ b/internal/proto/grpc_service_config/service_config.pb.go
@@ -63,7 +63,7 @@ func (x ServiceConfig_LoadBalancingPolicy) String() string {
 }
 
 func (ServiceConfig_LoadBalancingPolicy) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{7, 0}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{11, 0}
 }
 
 // Configuration for a method.
@@ -535,6 +535,194 @@ func (m *RoundRobinConfig) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_RoundRobinConfig proto.InternalMessageInfo
 
+// Configuration for priority LB policy.
+type PriorityLoadBalancingPolicyConfig struct {
+	Children map[string]*PriorityLoadBalancingPolicyConfig_Child `protobuf:"bytes,1,rep,name=children,proto3" json:"children,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// A list of child names in decreasing priority order
+	// (i.e., first element is the highest priority).
+	Priorities           []string `protobuf:"bytes,2,rep,name=priorities,proto3" json:"priorities,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *PriorityLoadBalancingPolicyConfig) Reset()         { *m = PriorityLoadBalancingPolicyConfig{} }
+func (m *PriorityLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
+func (*PriorityLoadBalancingPolicyConfig) ProtoMessage()    {}
+func (*PriorityLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{3}
+}
+
+func (m *PriorityLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Unmarshal(m, b)
+}
+func (m *PriorityLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
+}
+func (m *PriorityLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Merge(m, src)
+}
+func (m *PriorityLoadBalancingPolicyConfig) XXX_Size() int {
+	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Size(m)
+}
+func (m *PriorityLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
+	xxx_messageInfo_PriorityLoadBalancingPolicyConfig.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PriorityLoadBalancingPolicyConfig proto.InternalMessageInfo
+
+func (m *PriorityLoadBalancingPolicyConfig) GetChildren() map[string]*PriorityLoadBalancingPolicyConfig_Child {
+	if m != nil {
+		return m.Children
+	}
+	return nil
+}
+
+func (m *PriorityLoadBalancingPolicyConfig) GetPriorities() []string {
+	if m != nil {
+		return m.Priorities
+	}
+	return nil
+}
+
+// A map of name to child policy configuration.
+// The names are used to allow the priority policy to update
+// existing child policies instead of creating new ones every
+// time it receives a config update.
+type PriorityLoadBalancingPolicyConfig_Child struct {
+	Config               []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=config,proto3" json:"config,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
+	XXX_unrecognized     []byte                 `json:"-"`
+	XXX_sizecache        int32                  `json:"-"`
+}
+
+func (m *PriorityLoadBalancingPolicyConfig_Child) Reset() {
+	*m = PriorityLoadBalancingPolicyConfig_Child{}
+}
+func (m *PriorityLoadBalancingPolicyConfig_Child) String() string { return proto.CompactTextString(m) }
+func (*PriorityLoadBalancingPolicyConfig_Child) ProtoMessage()    {}
+func (*PriorityLoadBalancingPolicyConfig_Child) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{3, 0}
+}
+
+func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Unmarshal(m, b)
+}
+func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Marshal(b, m, deterministic)
+}
+func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Merge(m, src)
+}
+func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Size() int {
+	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Size(m)
+}
+func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_DiscardUnknown() {
+	xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child proto.InternalMessageInfo
+
+func (m *PriorityLoadBalancingPolicyConfig_Child) GetConfig() []*LoadBalancingConfig {
+	if m != nil {
+		return m.Config
+	}
+	return nil
+}
+
+// Configuration for weighted_target LB policy.
+type WeightedTargetLoadBalancingPolicyConfig struct {
+	Targets              map[string]*WeightedTargetLoadBalancingPolicyConfig_Target `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}                                                   `json:"-"`
+	XXX_unrecognized     []byte                                                     `json:"-"`
+	XXX_sizecache        int32                                                      `json:"-"`
+}
+
+func (m *WeightedTargetLoadBalancingPolicyConfig) Reset() {
+	*m = WeightedTargetLoadBalancingPolicyConfig{}
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
+func (*WeightedTargetLoadBalancingPolicyConfig) ProtoMessage()    {}
+func (*WeightedTargetLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{4}
+}
+
+func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Unmarshal(m, b)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Merge(m, src)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Size() int {
+	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Size(m)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
+	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig proto.InternalMessageInfo
+
+func (m *WeightedTargetLoadBalancingPolicyConfig) GetTargets() map[string]*WeightedTargetLoadBalancingPolicyConfig_Target {
+	if m != nil {
+		return m.Targets
+	}
+	return nil
+}
+
+type WeightedTargetLoadBalancingPolicyConfig_Target struct {
+	Weight               uint32                 `protobuf:"varint,1,opt,name=weight,proto3" json:"weight,omitempty"`
+	ChildPolicy          []*LoadBalancingConfig `protobuf:"bytes,2,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
+	XXX_unrecognized     []byte                 `json:"-"`
+	XXX_sizecache        int32                  `json:"-"`
+}
+
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) Reset() {
+	*m = WeightedTargetLoadBalancingPolicyConfig_Target{}
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) String() string {
+	return proto.CompactTextString(m)
+}
+func (*WeightedTargetLoadBalancingPolicyConfig_Target) ProtoMessage() {}
+func (*WeightedTargetLoadBalancingPolicyConfig_Target) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{4, 0}
+}
+
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Unmarshal(m, b)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Marshal(b, m, deterministic)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Merge(m, src)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Size() int {
+	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Size(m)
+}
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_DiscardUnknown() {
+	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target proto.InternalMessageInfo
+
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) GetWeight() uint32 {
+	if m != nil {
+		return m.Weight
+	}
+	return 0
+}
+
+func (m *WeightedTargetLoadBalancingPolicyConfig_Target) GetChildPolicy() []*LoadBalancingConfig {
+	if m != nil {
+		return m.ChildPolicy
+	}
+	return nil
+}
+
 // Configuration for grpclb LB policy.
 type GrpcLbConfig struct {
 	// Optional.  What LB policy to use for routing between the backend
@@ -556,7 +744,7 @@ func (m *GrpcLbConfig) Reset()         { *m = GrpcLbConfig{} }
 func (m *GrpcLbConfig) String() string { return proto.CompactTextString(m) }
 func (*GrpcLbConfig) ProtoMessage()    {}
 func (*GrpcLbConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{3}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{5}
 }
 
 func (m *GrpcLbConfig) XXX_Unmarshal(b []byte) error {
@@ -603,7 +791,7 @@ func (m *CdsConfig) Reset()         { *m = CdsConfig{} }
 func (m *CdsConfig) String() string { return proto.CompactTextString(m) }
 func (*CdsConfig) ProtoMessage()    {}
 func (*CdsConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{4}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{6}
 }
 
 func (m *CdsConfig) XXX_Unmarshal(b []byte) error {
@@ -662,7 +850,7 @@ func (m *XdsConfig) Reset()         { *m = XdsConfig{} }
 func (m *XdsConfig) String() string { return proto.CompactTextString(m) }
 func (*XdsConfig) ProtoMessage()    {}
 func (*XdsConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{5}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{7}
 }
 
 func (m *XdsConfig) XXX_Unmarshal(b []byte) error {
@@ -719,6 +907,230 @@ func (m *XdsConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
 	return nil
 }
 
+// Configuration for eds LB policy.
+type EdsLoadBalancingPolicyConfig struct {
+	// Cluster name.  Required.
+	Cluster string `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"`
+	// EDS service name, as returned in CDS.
+	// May be unset if not specified in CDS.
+	EdsServiceName string `protobuf:"bytes,2,opt,name=eds_service_name,json=edsServiceName,proto3" json:"eds_service_name,omitempty"`
+	// Server to send load reports to.
+	// If unset, no load reporting is done.
+	// If set to empty string, load reporting will be sent to the same
+	// server as we are getting xds data from.
+	LrsLoadReportingServerName *wrappers.StringValue `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
+	// Locality-picking policy.
+	// This policy's config is expected to be in the format used
+	// by the weighted_target policy.  Note that the config should include
+	// an empty value for the "targets" field; that empty value will be
+	// replaced by one that is dynamically generated based on the EDS data.
+	// Optional; defaults to "weighted_target".
+	LocalityPickingPolicy []*LoadBalancingConfig `protobuf:"bytes,4,rep,name=locality_picking_policy,json=localityPickingPolicy,proto3" json:"locality_picking_policy,omitempty"`
+	// Endpoint-picking policy.
+	// This will be configured as the policy for each child in the
+	// locality-policy's config.
+	// Optional; defaults to "round_robin".
+	EndpointPickingPolicy []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=endpoint_picking_policy,json=endpointPickingPolicy,proto3" json:"endpoint_picking_policy,omitempty"`
+	XXX_NoUnkeyedLiteral  struct{}               `json:"-"`
+	XXX_unrecognized      []byte                 `json:"-"`
+	XXX_sizecache         int32                  `json:"-"`
+}
+
+func (m *EdsLoadBalancingPolicyConfig) Reset()         { *m = EdsLoadBalancingPolicyConfig{} }
+func (m *EdsLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
+func (*EdsLoadBalancingPolicyConfig) ProtoMessage()    {}
+func (*EdsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{8}
+}
+
+func (m *EdsLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Unmarshal(m, b)
+}
+func (m *EdsLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
+}
+func (m *EdsLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_EdsLoadBalancingPolicyConfig.Merge(m, src)
+}
+func (m *EdsLoadBalancingPolicyConfig) XXX_Size() int {
+	return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Size(m)
+}
+func (m *EdsLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
+	xxx_messageInfo_EdsLoadBalancingPolicyConfig.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_EdsLoadBalancingPolicyConfig proto.InternalMessageInfo
+
+func (m *EdsLoadBalancingPolicyConfig) GetCluster() string {
+	if m != nil {
+		return m.Cluster
+	}
+	return ""
+}
+
+func (m *EdsLoadBalancingPolicyConfig) GetEdsServiceName() string {
+	if m != nil {
+		return m.EdsServiceName
+	}
+	return ""
+}
+
+func (m *EdsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
+	if m != nil {
+		return m.LrsLoadReportingServerName
+	}
+	return nil
+}
+
+func (m *EdsLoadBalancingPolicyConfig) GetLocalityPickingPolicy() []*LoadBalancingConfig {
+	if m != nil {
+		return m.LocalityPickingPolicy
+	}
+	return nil
+}
+
+func (m *EdsLoadBalancingPolicyConfig) GetEndpointPickingPolicy() []*LoadBalancingConfig {
+	if m != nil {
+		return m.EndpointPickingPolicy
+	}
+	return nil
+}
+
+// Configuration for lrs LB policy.
+type LrsLoadBalancingPolicyConfig struct {
+	// Cluster name.  Required.
+	ClusterName string `protobuf:"bytes,1,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"`
+	// EDS service name, as returned in CDS.
+	// May be unset if not specified in CDS.
+	EdsServiceName string `protobuf:"bytes,2,opt,name=eds_service_name,json=edsServiceName,proto3" json:"eds_service_name,omitempty"`
+	// Server to send load reports to.  Required.
+	// If set to empty string, load reporting will be sent to the same
+	// server as we are getting xds data from.
+	LrsLoadReportingServerName string                                 `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
+	Locality                   *LrsLoadBalancingPolicyConfig_Locality `protobuf:"bytes,4,opt,name=locality,proto3" json:"locality,omitempty"`
+	// Endpoint-picking policy.
+	ChildPolicy          []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
+	XXX_unrecognized     []byte                 `json:"-"`
+	XXX_sizecache        int32                  `json:"-"`
+}
+
+func (m *LrsLoadBalancingPolicyConfig) Reset()         { *m = LrsLoadBalancingPolicyConfig{} }
+func (m *LrsLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
+func (*LrsLoadBalancingPolicyConfig) ProtoMessage()    {}
+func (*LrsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{9}
+}
+
+func (m *LrsLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Unmarshal(m, b)
+}
+func (m *LrsLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
+}
+func (m *LrsLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_LrsLoadBalancingPolicyConfig.Merge(m, src)
+}
+func (m *LrsLoadBalancingPolicyConfig) XXX_Size() int {
+	return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Size(m)
+}
+func (m *LrsLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
+	xxx_messageInfo_LrsLoadBalancingPolicyConfig.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_LrsLoadBalancingPolicyConfig proto.InternalMessageInfo
+
+func (m *LrsLoadBalancingPolicyConfig) GetClusterName() string {
+	if m != nil {
+		return m.ClusterName
+	}
+	return ""
+}
+
+func (m *LrsLoadBalancingPolicyConfig) GetEdsServiceName() string {
+	if m != nil {
+		return m.EdsServiceName
+	}
+	return ""
+}
+
+func (m *LrsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() string {
+	if m != nil {
+		return m.LrsLoadReportingServerName
+	}
+	return ""
+}
+
+func (m *LrsLoadBalancingPolicyConfig) GetLocality() *LrsLoadBalancingPolicyConfig_Locality {
+	if m != nil {
+		return m.Locality
+	}
+	return nil
+}
+
+func (m *LrsLoadBalancingPolicyConfig) GetChildPolicy() []*LoadBalancingConfig {
+	if m != nil {
+		return m.ChildPolicy
+	}
+	return nil
+}
+
+// The locality for which this policy will report load.  Required.
+type LrsLoadBalancingPolicyConfig_Locality struct {
+	Region               string   `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"`
+	Zone                 string   `protobuf:"bytes,2,opt,name=zone,proto3" json:"zone,omitempty"`
+	Subzone              string   `protobuf:"bytes,3,opt,name=subzone,proto3" json:"subzone,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *LrsLoadBalancingPolicyConfig_Locality) Reset()         { *m = LrsLoadBalancingPolicyConfig_Locality{} }
+func (m *LrsLoadBalancingPolicyConfig_Locality) String() string { return proto.CompactTextString(m) }
+func (*LrsLoadBalancingPolicyConfig_Locality) ProtoMessage()    {}
+func (*LrsLoadBalancingPolicyConfig_Locality) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e32d3cb2c41c77ce, []int{9, 0}
+}
+
+func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Unmarshal(m, b)
+}
+func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Marshal(b, m, deterministic)
+}
+func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Merge(m, src)
+}
+func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Size() int {
+	return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Size(m)
+}
+func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_DiscardUnknown() {
+	xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality proto.InternalMessageInfo
+
+func (m *LrsLoadBalancingPolicyConfig_Locality) GetRegion() string {
+	if m != nil {
+		return m.Region
+	}
+	return ""
+}
+
+func (m *LrsLoadBalancingPolicyConfig_Locality) GetZone() string {
+	if m != nil {
+		return m.Zone
+	}
+	return ""
+}
+
+func (m *LrsLoadBalancingPolicyConfig_Locality) GetSubzone() string {
+	if m != nil {
+		return m.Subzone
+	}
+	return ""
+}
+
 // Selects LB policy and provides corresponding configuration.
 //
 // In general, all instances of this field should be repeated. Clients will
@@ -737,7 +1149,11 @@ type LoadBalancingConfig struct {
 	//	*LoadBalancingConfig_PickFirst
 	//	*LoadBalancingConfig_RoundRobin
 	//	*LoadBalancingConfig_Grpclb
+	//	*LoadBalancingConfig_Priority
+	//	*LoadBalancingConfig_WeightedTarget
 	//	*LoadBalancingConfig_Cds
+	//	*LoadBalancingConfig_Eds
+	//	*LoadBalancingConfig_Lrs
 	//	*LoadBalancingConfig_Xds
 	//	*LoadBalancingConfig_XdsExperimental
 	Policy               isLoadBalancingConfig_Policy `protobuf_oneof:"policy"`
@@ -750,7 +1166,7 @@ func (m *LoadBalancingConfig) Reset()         { *m = LoadBalancingConfig{} }
 func (m *LoadBalancingConfig) String() string { return proto.CompactTextString(m) }
 func (*LoadBalancingConfig) ProtoMessage()    {}
 func (*LoadBalancingConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{6}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{10}
 }
 
 func (m *LoadBalancingConfig) XXX_Unmarshal(b []byte) error {
@@ -787,10 +1203,26 @@ type LoadBalancingConfig_Grpclb struct {
 	Grpclb *GrpcLbConfig `protobuf:"bytes,3,opt,name=grpclb,proto3,oneof"`
 }
 
+type LoadBalancingConfig_Priority struct {
+	Priority *PriorityLoadBalancingPolicyConfig `protobuf:"bytes,9,opt,name=priority,proto3,oneof"`
+}
+
+type LoadBalancingConfig_WeightedTarget struct {
+	WeightedTarget *WeightedTargetLoadBalancingPolicyConfig `protobuf:"bytes,10,opt,name=weighted_target,json=weightedTarget,proto3,oneof"`
+}
+
 type LoadBalancingConfig_Cds struct {
 	Cds *CdsConfig `protobuf:"bytes,6,opt,name=cds,proto3,oneof"`
 }
 
+type LoadBalancingConfig_Eds struct {
+	Eds *EdsLoadBalancingPolicyConfig `protobuf:"bytes,7,opt,name=eds,proto3,oneof"`
+}
+
+type LoadBalancingConfig_Lrs struct {
+	Lrs *LrsLoadBalancingPolicyConfig `protobuf:"bytes,8,opt,name=lrs,proto3,oneof"`
+}
+
 type LoadBalancingConfig_Xds struct {
 	Xds *XdsConfig `protobuf:"bytes,2,opt,name=xds,proto3,oneof"`
 }
@@ -805,8 +1237,16 @@ func (*LoadBalancingConfig_RoundRobin) isLoadBalancingConfig_Policy() {}
 
 func (*LoadBalancingConfig_Grpclb) isLoadBalancingConfig_Policy() {}
 
+func (*LoadBalancingConfig_Priority) isLoadBalancingConfig_Policy() {}
+
+func (*LoadBalancingConfig_WeightedTarget) isLoadBalancingConfig_Policy() {}
+
 func (*LoadBalancingConfig_Cds) isLoadBalancingConfig_Policy() {}
 
+func (*LoadBalancingConfig_Eds) isLoadBalancingConfig_Policy() {}
+
+func (*LoadBalancingConfig_Lrs) isLoadBalancingConfig_Policy() {}
+
 func (*LoadBalancingConfig_Xds) isLoadBalancingConfig_Policy() {}
 
 func (*LoadBalancingConfig_XdsExperimental) isLoadBalancingConfig_Policy() {}
@@ -839,6 +1279,20 @@ func (m *LoadBalancingConfig) GetGrpclb() *GrpcLbConfig {
 	return nil
 }
 
+func (m *LoadBalancingConfig) GetPriority() *PriorityLoadBalancingPolicyConfig {
+	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Priority); ok {
+		return x.Priority
+	}
+	return nil
+}
+
+func (m *LoadBalancingConfig) GetWeightedTarget() *WeightedTargetLoadBalancingPolicyConfig {
+	if x, ok := m.GetPolicy().(*LoadBalancingConfig_WeightedTarget); ok {
+		return x.WeightedTarget
+	}
+	return nil
+}
+
 func (m *LoadBalancingConfig) GetCds() *CdsConfig {
 	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Cds); ok {
 		return x.Cds
@@ -846,6 +1300,21 @@ func (m *LoadBalancingConfig) GetCds() *CdsConfig {
 	return nil
 }
 
+func (m *LoadBalancingConfig) GetEds() *EdsLoadBalancingPolicyConfig {
+	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Eds); ok {
+		return x.Eds
+	}
+	return nil
+}
+
+func (m *LoadBalancingConfig) GetLrs() *LrsLoadBalancingPolicyConfig {
+	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Lrs); ok {
+		return x.Lrs
+	}
+	return nil
+}
+
+// Deprecated: Do not use.
 func (m *LoadBalancingConfig) GetXds() *XdsConfig {
 	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Xds); ok {
 		return x.Xds
@@ -853,6 +1322,7 @@ func (m *LoadBalancingConfig) GetXds() *XdsConfig {
 	return nil
 }
 
+// Deprecated: Do not use.
 func (m *LoadBalancingConfig) GetXdsExperimental() *XdsConfig {
 	if x, ok := m.GetPolicy().(*LoadBalancingConfig_XdsExperimental); ok {
 		return x.XdsExperimental
@@ -866,7 +1336,11 @@ func (*LoadBalancingConfig) XXX_OneofWrappers() []interface{} {
 		(*LoadBalancingConfig_PickFirst)(nil),
 		(*LoadBalancingConfig_RoundRobin)(nil),
 		(*LoadBalancingConfig_Grpclb)(nil),
+		(*LoadBalancingConfig_Priority)(nil),
+		(*LoadBalancingConfig_WeightedTarget)(nil),
 		(*LoadBalancingConfig_Cds)(nil),
+		(*LoadBalancingConfig_Eds)(nil),
+		(*LoadBalancingConfig_Lrs)(nil),
 		(*LoadBalancingConfig_Xds)(nil),
 		(*LoadBalancingConfig_XdsExperimental)(nil),
 	}
@@ -893,7 +1367,7 @@ func (m *ServiceConfig) Reset()         { *m = ServiceConfig{} }
 func (m *ServiceConfig) String() string { return proto.CompactTextString(m) }
 func (*ServiceConfig) ProtoMessage()    {}
 func (*ServiceConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{7}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{11}
 }
 
 func (m *ServiceConfig) XXX_Unmarshal(b []byte) error {
@@ -984,7 +1458,7 @@ func (m *ServiceConfig_RetryThrottlingPolicy) Reset()         { *m = ServiceConf
 func (m *ServiceConfig_RetryThrottlingPolicy) String() string { return proto.CompactTextString(m) }
 func (*ServiceConfig_RetryThrottlingPolicy) ProtoMessage()    {}
 func (*ServiceConfig_RetryThrottlingPolicy) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{7, 0}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{11, 0}
 }
 
 func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Unmarshal(b []byte) error {
@@ -1031,7 +1505,7 @@ func (m *ServiceConfig_HealthCheckConfig) Reset()         { *m = ServiceConfig_H
 func (m *ServiceConfig_HealthCheckConfig) String() string { return proto.CompactTextString(m) }
 func (*ServiceConfig_HealthCheckConfig) ProtoMessage()    {}
 func (*ServiceConfig_HealthCheckConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{7, 1}
+	return fileDescriptor_e32d3cb2c41c77ce, []int{11, 1}
 }
 
 func (m *ServiceConfig_HealthCheckConfig) XXX_Unmarshal(b []byte) error {
@@ -1067,9 +1541,18 @@ func init() {
 	proto.RegisterType((*MethodConfig_HedgingPolicy)(nil), "grpc.service_config.MethodConfig.HedgingPolicy")
 	proto.RegisterType((*PickFirstConfig)(nil), "grpc.service_config.PickFirstConfig")
 	proto.RegisterType((*RoundRobinConfig)(nil), "grpc.service_config.RoundRobinConfig")
+	proto.RegisterType((*PriorityLoadBalancingPolicyConfig)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig")
+	proto.RegisterMapType((map[string]*PriorityLoadBalancingPolicyConfig_Child)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig.ChildrenEntry")
+	proto.RegisterType((*PriorityLoadBalancingPolicyConfig_Child)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig.Child")
+	proto.RegisterType((*WeightedTargetLoadBalancingPolicyConfig)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig")
+	proto.RegisterMapType((map[string]*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.TargetsEntry")
+	proto.RegisterType((*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.Target")
 	proto.RegisterType((*GrpcLbConfig)(nil), "grpc.service_config.GrpcLbConfig")
 	proto.RegisterType((*CdsConfig)(nil), "grpc.service_config.CdsConfig")
 	proto.RegisterType((*XdsConfig)(nil), "grpc.service_config.XdsConfig")
+	proto.RegisterType((*EdsLoadBalancingPolicyConfig)(nil), "grpc.service_config.EdsLoadBalancingPolicyConfig")
+	proto.RegisterType((*LrsLoadBalancingPolicyConfig)(nil), "grpc.service_config.LrsLoadBalancingPolicyConfig")
+	proto.RegisterType((*LrsLoadBalancingPolicyConfig_Locality)(nil), "grpc.service_config.LrsLoadBalancingPolicyConfig.Locality")
 	proto.RegisterType((*LoadBalancingConfig)(nil), "grpc.service_config.LoadBalancingConfig")
 	proto.RegisterType((*ServiceConfig)(nil), "grpc.service_config.ServiceConfig")
 	proto.RegisterType((*ServiceConfig_RetryThrottlingPolicy)(nil), "grpc.service_config.ServiceConfig.RetryThrottlingPolicy")
@@ -1081,78 +1564,105 @@ func init() {
 }
 
 var fileDescriptor_e32d3cb2c41c77ce = []byte{
-	// 1161 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xdb, 0x6e, 0x23, 0x45,
-	0x10, 0x8d, 0xed, 0x6c, 0xb2, 0x2e, 0x5f, 0xe2, 0x74, 0x48, 0xd6, 0x6b, 0xc1, 0x92, 0x44, 0x2c,
-	0xf8, 0x25, 0x8e, 0xe4, 0x20, 0x58, 0x2d, 0x12, 0x17, 0x27, 0x31, 0x8e, 0x48, 0xb2, 0xa1, 0x93,
-	0xb0, 0x41, 0x42, 0x6a, 0xda, 0x33, 0x6d, 0x7b, 0x94, 0x99, 0xe9, 0xa1, 0xa7, 0xcd, 0x3a, 0x2f,
-	0xfc, 0x0d, 0xe2, 0x1b, 0xf8, 0x13, 0x9e, 0x10, 0x9f, 0x82, 0xfa, 0x32, 0x8e, 0xc7, 0xf6, 0xe2,
-	0xac, 0x78, 0x9c, 0xaa, 0x3a, 0xa7, 0xab, 0xab, 0x4e, 0x55, 0x0f, 0xd4, 0xfb, 0x22, 0x72, 0xf6,
-	0x63, 0x26, 0x7e, 0xf5, 0x1c, 0x46, 0x1c, 0x1e, 0xf6, 0xbc, 0xfe, 0xd4, 0x67, 0x23, 0x12, 0x5c,
-	0x72, 0xb4, 0xa1, 0x22, 0x1b, 0x69, 0x57, 0xed, 0x59, 0x9f, 0xf3, 0xbe, 0xcf, 0xf6, 0x75, 0x48,
-	0x77, 0xd8, 0xdb, 0x77, 0x87, 0x82, 0x4a, 0x8f, 0x87, 0x06, 0x34, 0xeb, 0x7f, 0x23, 0x68, 0x14,
-	0x31, 0x11, 0x5b, 0xff, 0xa6, 0xf5, 0xab, 0x24, 0x1c, 0xee, 0x32, 0x63, 0xde, 0xfd, 0xfb, 0x31,
-	0x14, 0xcf, 0x98, 0x1c, 0x70, 0xf7, 0x50, 0x9f, 0x83, 0x5e, 0xc2, 0x72, 0x48, 0x03, 0x56, 0xcd,
-	0x6c, 0xe7, 0xea, 0x85, 0xe6, 0xc7, 0x8d, 0x39, 0xb9, 0x34, 0x26, 0x01, 0x8d, 0x73, 0x1a, 0x30,
-	0xac, 0x31, 0xe8, 0x6b, 0x28, 0xbf, 0xa1, 0x9e, 0x24, 0x3d, 0x2e, 0x88, 0x60, 0xd4, 0xbd, 0xab,
-	0x66, 0xb7, 0x33, 0xf5, 0x42, 0xb3, 0xd6, 0x30, 0x87, 0x37, 0x92, 0xe4, 0x1a, 0x2d, 0xce, 0xfd,
-	0x1f, 0xa8, 0x3f, 0x64, 0xb8, 0xa8, 0x10, 0x6d, 0x2e, 0xb0, 0x8a, 0x47, 0x07, 0xb0, 0x2a, 0xbd,
-	0x80, 0xf1, 0xa1, 0xac, 0xe6, 0x34, 0xf4, 0xe9, 0x0c, 0xf4, 0xc8, 0xde, 0x1b, 0x27, 0x91, 0xe8,
-	0x35, 0x3c, 0x0d, 0xe8, 0x88, 0x08, 0xf6, 0xcb, 0x90, 0xc5, 0x92, 0x04, 0x2c, 0x8e, 0x69, 0x9f,
-	0x91, 0xee, 0x9d, 0x64, 0x71, 0x75, 0x59, 0xd3, 0xbc, 0x3f, 0x43, 0x73, 0x7d, 0x12, 0xca, 0x83,
-	0xa6, 0xc9, 0x61, 0x2b, 0xa0, 0x23, 0x6c, 0xd0, 0x67, 0x06, 0xdc, 0x52, 0x58, 0xf4, 0x23, 0xd4,
-	0x0c, 0x71, 0x1c, 0xf1, 0x30, 0x66, 0x53, 0xcc, 0x8f, 0x1e, 0xc0, 0xfc, 0x44, 0x33, 0x1b, 0x78,
-	0x8a, 0x1a, 0x43, 0x51, 0x30, 0x29, 0xee, 0x48, 0xc4, 0x7d, 0xcf, 0xb9, 0xab, 0xae, 0x68, 0xb2,
-	0xbd, 0xc5, 0xe5, 0xc6, 0x0a, 0x75, 0xa1, 0x41, 0x9d, 0x25, 0x5c, 0x10, 0xf7, 0x9f, 0xe8, 0x06,
-	0xca, 0x03, 0xe6, 0xf6, 0xbd, 0xb0, 0x9f, 0xb0, 0xae, 0x6a, 0xd6, 0xfd, 0xc5, 0xac, 0x1d, 0x83,
-	0x1b, 0xf3, 0x96, 0x06, 0x93, 0x86, 0xda, 0x0b, 0x58, 0x56, 0x6d, 0x46, 0x55, 0x58, 0xb5, 0x2c,
-	0xd5, 0xcc, 0x76, 0xa6, 0x9e, 0xc7, 0xc9, 0x27, 0xda, 0x82, 0x95, 0x40, 0x13, 0xea, 0x96, 0xe7,
-	0xb1, 0xfd, 0xaa, 0xfd, 0x91, 0x85, 0xc2, 0x44, 0xca, 0x68, 0x07, 0x8a, 0xaa, 0xa4, 0x54, 0x4a,
-	0x16, 0x44, 0x32, 0xd6, 0x34, 0x25, 0x5c, 0x08, 0xe8, 0xe8, 0x1b, 0x6b, 0x42, 0x2d, 0x58, 0xf3,
-	0x42, 0x4f, 0x7a, 0xd4, 0x27, 0x5d, 0xea, 0xdc, 0xf2, 0x5e, 0xcf, 0xca, 0xe8, 0x3f, 0xb4, 0x50,
-	0xb6, 0x88, 0x96, 0x01, 0xa0, 0x97, 0xa0, 0x28, 0xc7, 0xf8, 0x85, 0x5a, 0x82, 0x80, 0x8e, 0x12,
-	0xec, 0x1e, 0x20, 0x8b, 0x23, 0xc1, 0xd0, 0x97, 0x5e, 0xe4, 0x7b, 0x4c, 0x68, 0x1d, 0x65, 0xf1,
-	0xba, 0xf5, 0x9c, 0x8d, 0x1d, 0xa8, 0x0d, 0x5b, 0xba, 0x09, 0xb4, 0xeb, 0x33, 0x12, 0x4b, 0x2a,
-	0x87, 0x31, 0x51, 0x03, 0xa6, 0x04, 0x92, 0xab, 0x97, 0x9b, 0x95, 0xe4, 0x54, 0xd5, 0x83, 0x43,
-	0xee, 0x32, 0xfc, 0xde, 0x38, 0xfe, 0x52, 0x87, 0x2b, 0x63, 0x5c, 0xfb, 0x33, 0x03, 0xa5, 0x54,
-	0x1b, 0x1e, 0x52, 0xab, 0x2f, 0x21, 0xe9, 0x14, 0x71, 0x99, 0x4f, 0xef, 0x16, 0x57, 0xaa, 0x68,
-	0xe3, 0x8f, 0x54, 0x38, 0x3a, 0x86, 0xad, 0x90, 0x87, 0xa4, 0x47, 0x25, 0xf5, 0xd3, 0xc9, 0xe7,
-	0xde, 0x92, 0xfc, 0x46, 0xc8, 0xc3, 0xb6, 0x0a, 0x9f, 0xc8, 0xbd, 0xf5, 0x14, 0x9e, 0x18, 0x35,
-	0x73, 0x41, 0xd2, 0x12, 0xdc, 0x5d, 0x87, 0xb5, 0x0b, 0xcf, 0xb9, 0x6d, 0x7b, 0x22, 0x96, 0x46,
-	0x6c, 0xbb, 0x08, 0x2a, 0x98, 0x0f, 0x43, 0x17, 0xf3, 0xae, 0x17, 0x5a, 0xdb, 0x6f, 0x50, 0xfc,
-	0x56, 0x44, 0xce, 0x69, 0xd7, 0xae, 0xa1, 0xef, 0xa0, 0xe8, 0x0c, 0x3c, 0xdf, 0x4d, 0x94, 0x6c,
-	0xd6, 0x51, 0x7d, 0xae, 0x92, 0x4f, 0x39, 0x75, 0x5b, 0xd4, 0xa7, 0xa1, 0xe3, 0x85, 0x7d, 0x83,
-	0xc7, 0x05, 0x8d, 0xbe, 0x2f, 0x64, 0x02, 0xd1, 0xbb, 0xcd, 0x48, 0xb4, 0x60, 0x6d, 0x4a, 0xd9,
-	0xbb, 0xcf, 0x21, 0x7f, 0xe8, 0xc6, 0xf6, 0xf0, 0x2a, 0xac, 0x3a, 0xfe, 0x30, 0x96, 0x4c, 0x24,
-	0x32, 0xb7, 0x9f, 0xbb, 0xff, 0x64, 0x21, 0x7f, 0x33, 0x8e, 0xfb, 0x04, 0x4a, 0x5d, 0x7d, 0x2e,
-	0x13, 0xc4, 0x2e, 0xcd, 0x4c, 0x3d, 0xdf, 0xca, 0x56, 0x33, 0xb8, 0x98, 0x38, 0xf4, 0xdc, 0x4c,
-	0xdf, 0x26, 0xfb, 0x7f, 0x6e, 0xf3, 0x3d, 0xac, 0xf5, 0xa8, 0xef, 0x2b, 0x25, 0x26, 0x7c, 0xb9,
-	0x77, 0xe4, 0x2b, 0x27, 0x04, 0x96, 0xb2, 0x0e, 0x15, 0xe6, 0xc6, 0x24, 0x55, 0xa4, 0x65, 0x7d,
-	0xf3, 0x32, 0x73, 0xe3, 0xcb, 0xfb, 0x3a, 0xa1, 0x9f, 0xe1, 0x99, 0x2f, 0x62, 0xe2, 0x73, 0xea,
-	0x12, 0xc1, 0x22, 0x2e, 0xa4, 0xea, 0xb5, 0x02, 0x26, 0x35, 0x78, 0xdb, 0x5a, 0xbc, 0x94, 0xc2,
-	0x0b, 0xfb, 0x66, 0x2d, 0xd6, 0x7c, 0x11, 0xab, 0xbc, 0x70, 0xc2, 0x70, 0xa9, 0x09, 0x74, 0x27,
-	0x7e, 0xcf, 0xc1, 0xc6, 0x9c, 0x9c, 0x51, 0x1b, 0x20, 0xf2, 0x9c, 0x5b, 0xd2, 0x53, 0x4a, 0xb2,
-	0x6b, 0xfd, 0xa3, 0xb9, 0x37, 0x9e, 0xd2, 0x5b, 0x67, 0x09, 0x4f, 0x20, 0xd1, 0x09, 0x14, 0x84,
-	0x52, 0x1f, 0x11, 0x4a, 0x7e, 0xba, 0x65, 0x85, 0xe6, 0xf3, 0xb9, 0x44, 0xd3, 0x2a, 0xd5, 0x0b,
-	0xf7, 0x1e, 0x8b, 0xbe, 0x80, 0x15, 0x05, 0xf3, 0xbb, 0x76, 0xc1, 0xec, 0xcc, 0x65, 0x99, 0xd4,
-	0x75, 0x67, 0x09, 0x5b, 0x08, 0x6a, 0x42, 0xce, 0x71, 0x63, 0xbb, 0xf8, 0x9f, 0xcd, 0x45, 0x8e,
-	0x15, 0xd9, 0x59, 0xc2, 0x2a, 0x58, 0x61, 0x46, 0x6e, 0x6c, 0x87, 0x7c, 0x3e, 0xe6, 0x66, 0x12,
-	0x33, 0x72, 0x63, 0x74, 0x0a, 0x95, 0x91, 0x1b, 0x13, 0x36, 0x8a, 0x98, 0xf0, 0x02, 0x16, 0x4a,
-	0xea, 0xdb, 0x1e, 0x2d, 0x26, 0x98, 0x41, 0xb6, 0x1e, 0xc3, 0x8a, 0x1d, 0xec, 0xbf, 0x1e, 0x41,
-	0xc9, 0x2a, 0xc3, 0x76, 0x28, 0x84, 0x4d, 0xad, 0x8b, 0x6e, 0xd2, 0xb9, 0xfb, 0xe1, 0xcd, 0xd4,
-	0xcb, 0xcd, 0xcf, 0xe6, 0x1e, 0x97, 0xa2, 0x48, 0x8b, 0xd5, 0x88, 0x53, 0x8f, 0xd3, 0x86, 0x3f,
-	0xeb, 0x40, 0x3f, 0xcd, 0x9c, 0x67, 0x38, 0xab, 0xcb, 0xef, 0x38, 0x0e, 0x69, 0xf6, 0xb1, 0xde,
-	0x4a, 0xe6, 0x0d, 0x4b, 0x58, 0xcd, 0xd0, 0xee, 0x2c, 0x7c, 0x4c, 0x71, 0x31, 0x98, 0xfc, 0xa1,
-	0x72, 0xa0, 0x62, 0x76, 0xa3, 0x1c, 0x08, 0x2e, 0xa5, 0xef, 0x85, 0x7d, 0x2b, 0x97, 0x17, 0x0f,
-	0x28, 0x88, 0x7e, 0x3b, 0xaf, 0xc6, 0x48, 0x73, 0x73, 0xbc, 0x26, 0xd2, 0x66, 0xe4, 0xc2, 0xc6,
-	0x80, 0x51, 0x5f, 0x0e, 0x88, 0x33, 0x60, 0xce, 0x6d, 0x92, 0xb2, 0xe9, 0xf3, 0xa7, 0x0f, 0x38,
-	0xa7, 0xa3, 0xd1, 0x87, 0x0a, 0x6c, 0x6f, 0xb1, 0x3e, 0x98, 0x36, 0xd5, 0x5e, 0xc3, 0xe6, 0xdc,
-	0x7c, 0xd0, 0x07, 0xa0, 0x1e, 0x50, 0x22, 0xf9, 0x2d, 0x0b, 0x93, 0x77, 0x2a, 0x1f, 0xd0, 0xd1,
-	0x95, 0x36, 0xa0, 0x0f, 0xa1, 0xa0, 0x5d, 0x44, 0xbf, 0x41, 0x5a, 0xbe, 0x59, 0x0c, 0xda, 0x84,
-	0x95, 0xa5, 0x76, 0x05, 0xeb, 0x33, 0x09, 0xa0, 0xaf, 0xa6, 0xb6, 0x76, 0xe6, 0x01, 0x8b, 0x25,
-	0xb5, 0xd3, 0x3f, 0x9f, 0x5a, 0x24, 0x36, 0xd9, 0x35, 0x28, 0x5c, 0x9f, 0x5f, 0x5e, 0x1c, 0x1f,
-	0x9e, 0xb4, 0x4f, 0x8e, 0x8f, 0x2a, 0x4b, 0xca, 0x80, 0x5f, 0x5d, 0x9f, 0x1f, 0x11, 0xfc, 0xaa,
-	0x75, 0x72, 0x5e, 0xc9, 0xb4, 0xf6, 0x60, 0xd3, 0xe3, 0xa9, 0xa2, 0x99, 0x9a, 0xb5, 0x50, 0xaa,
-	0x68, 0x17, 0x2a, 0x83, 0x8b, 0x4c, 0x77, 0x45, 0xa7, 0x72, 0xf0, 0x6f, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0x2d, 0xca, 0x96, 0x84, 0xe2, 0x0b, 0x00, 0x00,
+	// 1589 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0x1a, 0x47,
+	0x16, 0x16, 0xa0, 0x1b, 0x07, 0x84, 0x50, 0x6b, 0x25, 0x61, 0xca, 0xab, 0x95, 0xa8, 0xf5, 0x9a,
+	0x17, 0xa3, 0x5a, 0x79, 0xcb, 0xeb, 0xd2, 0x6e, 0xed, 0x3a, 0xe8, 0x12, 0x54, 0x91, 0x65, 0xd2,
+	0x92, 0x6f, 0x55, 0xa9, 0x1a, 0x0d, 0x33, 0x0d, 0x4c, 0x69, 0x66, 0x9a, 0xf4, 0x34, 0x16, 0xe4,
+	0x21, 0x79, 0xcd, 0x2f, 0x49, 0x7e, 0x43, 0xfe, 0x49, 0x9e, 0x52, 0x79, 0xcf, 0x3f, 0xc8, 0x43,
+	0x2a, 0xd5, 0x97, 0x41, 0x0c, 0x8c, 0x10, 0xb2, 0xfc, 0xc6, 0x9c, 0xee, 0xef, 0x3b, 0xd7, 0x3e,
+	0x7d, 0x1a, 0x28, 0xb7, 0x58, 0xc7, 0xda, 0x09, 0x08, 0xfb, 0xe0, 0x58, 0xc4, 0xb0, 0xa8, 0xdf,
+	0x74, 0x5a, 0x23, 0x9f, 0x95, 0x0e, 0xa3, 0x9c, 0xa2, 0x55, 0xb1, 0xb3, 0x12, 0x5d, 0x2a, 0x6e,
+	0xb6, 0x28, 0x6d, 0xb9, 0x64, 0x47, 0x6e, 0x69, 0x74, 0x9b, 0x3b, 0x76, 0x97, 0x99, 0xdc, 0xa1,
+	0xbe, 0x02, 0x8d, 0xaf, 0x5f, 0x31, 0xb3, 0xd3, 0x21, 0x2c, 0xd0, 0xeb, 0x6b, 0x7a, 0x5d, 0x18,
+	0x61, 0x51, 0x9b, 0x28, 0x71, 0xe9, 0x97, 0x45, 0xc8, 0xbe, 0x24, 0xbc, 0x4d, 0xed, 0x7d, 0xa9,
+	0x07, 0xed, 0xc1, 0xac, 0x6f, 0x7a, 0xa4, 0x90, 0xd8, 0x4a, 0x95, 0x33, 0xbb, 0xff, 0xa8, 0xc4,
+	0xd8, 0x52, 0x19, 0x06, 0x54, 0x4e, 0x4d, 0x8f, 0x60, 0x89, 0x41, 0x2f, 0x20, 0x77, 0x65, 0x3a,
+	0xdc, 0x68, 0x52, 0x66, 0x30, 0x62, 0xda, 0xfd, 0x42, 0x72, 0x2b, 0x51, 0xce, 0xec, 0x16, 0x2b,
+	0x4a, 0x79, 0x25, 0x34, 0xae, 0x52, 0xa5, 0xd4, 0x7d, 0x63, 0xba, 0x5d, 0x82, 0xb3, 0x02, 0x71,
+	0x44, 0x19, 0x16, 0xfb, 0xd1, 0x53, 0x58, 0xe0, 0x8e, 0x47, 0x68, 0x97, 0x17, 0x52, 0x12, 0xfa,
+	0x60, 0x0c, 0x7a, 0xa0, 0xfd, 0xc6, 0xe1, 0x4e, 0xf4, 0x16, 0x1e, 0x78, 0x66, 0xcf, 0x60, 0xe4,
+	0xeb, 0x2e, 0x09, 0xb8, 0xe1, 0x91, 0x20, 0x30, 0x5b, 0xc4, 0x68, 0xf4, 0x39, 0x09, 0x0a, 0xb3,
+	0x92, 0xe6, 0xe1, 0x18, 0xcd, 0xeb, 0x63, 0x9f, 0x3f, 0xdd, 0x55, 0x36, 0xac, 0x7b, 0x66, 0x0f,
+	0x2b, 0xf4, 0x4b, 0x05, 0xae, 0x0a, 0x2c, 0x7a, 0x0f, 0x45, 0x45, 0x1c, 0x74, 0xa8, 0x1f, 0x90,
+	0x11, 0xe6, 0xb9, 0x29, 0x98, 0x37, 0x24, 0xb3, 0x82, 0x47, 0xa8, 0x31, 0x64, 0x19, 0xe1, 0xac,
+	0x6f, 0x74, 0xa8, 0xeb, 0x58, 0xfd, 0xc2, 0xbc, 0x24, 0x7b, 0x72, 0x7b, 0xb8, 0xb1, 0x40, 0xd5,
+	0x25, 0xa8, 0x36, 0x83, 0x33, 0xec, 0xfa, 0x13, 0xbd, 0x83, 0x5c, 0x9b, 0xd8, 0x2d, 0xc7, 0x6f,
+	0x85, 0xac, 0x0b, 0x92, 0x75, 0xe7, 0x76, 0xd6, 0x9a, 0xc2, 0x0d, 0x78, 0x97, 0xda, 0xc3, 0x82,
+	0xe2, 0x73, 0x98, 0x15, 0x69, 0x46, 0x05, 0x58, 0xd0, 0x2c, 0x85, 0xc4, 0x56, 0xa2, 0x9c, 0xc6,
+	0xe1, 0x27, 0x5a, 0x87, 0x79, 0x4f, 0x12, 0xca, 0x94, 0xa7, 0xb1, 0xfe, 0x2a, 0xfe, 0x98, 0x84,
+	0xcc, 0x90, 0xc9, 0x68, 0x1b, 0xb2, 0x22, 0xa4, 0x26, 0xe7, 0xc4, 0xeb, 0xf0, 0x40, 0xd2, 0x2c,
+	0xe1, 0x8c, 0x67, 0xf6, 0x3e, 0xd3, 0x22, 0x54, 0x85, 0x65, 0xc7, 0x77, 0xb8, 0x63, 0xba, 0x46,
+	0xc3, 0xb4, 0x2e, 0x69, 0xb3, 0xa9, 0xcb, 0x68, 0x42, 0x2d, 0xe4, 0x34, 0xa2, 0xaa, 0x00, 0x68,
+	0x0f, 0x04, 0xe5, 0x00, 0x7f, 0x6b, 0x2d, 0x81, 0x67, 0xf6, 0x42, 0xec, 0x13, 0x40, 0x1a, 0x67,
+	0x78, 0x5d, 0x97, 0x3b, 0x1d, 0xd7, 0x21, 0x4c, 0xd6, 0x51, 0x12, 0xaf, 0xe8, 0x95, 0x97, 0x83,
+	0x05, 0x74, 0x04, 0xeb, 0x32, 0x09, 0x66, 0xc3, 0x25, 0x46, 0xc0, 0x4d, 0xde, 0x0d, 0x0c, 0x71,
+	0xc0, 0x44, 0x81, 0xa4, 0xca, 0xb9, 0xdd, 0x7c, 0xa8, 0x55, 0xe4, 0x60, 0x9f, 0xda, 0x04, 0xff,
+	0x65, 0xb0, 0xff, 0x4c, 0x6e, 0x17, 0xc2, 0xa0, 0xf8, 0x53, 0x02, 0x96, 0x22, 0x69, 0x98, 0x26,
+	0x56, 0xff, 0x83, 0x30, 0x53, 0x86, 0x4d, 0x5c, 0xb3, 0x7f, 0x7b, 0xa4, 0xb2, 0x7a, 0xff, 0x81,
+	0xd8, 0x8e, 0x0e, 0x61, 0xdd, 0xa7, 0xbe, 0xd1, 0x34, 0xb9, 0xe9, 0x46, 0x8d, 0x4f, 0xdd, 0x60,
+	0xfc, 0xaa, 0x4f, 0xfd, 0x23, 0xb1, 0x7d, 0xc8, 0xf6, 0xea, 0x03, 0xd8, 0x50, 0xd5, 0x4c, 0x99,
+	0x11, 0x2d, 0xc1, 0xd2, 0x0a, 0x2c, 0xd7, 0x1d, 0xeb, 0xf2, 0xc8, 0x61, 0x01, 0x57, 0xc5, 0x56,
+	0x42, 0x90, 0xc7, 0xb4, 0xeb, 0xdb, 0x98, 0x36, 0x1c, 0x5f, 0xcb, 0x7e, 0x4b, 0xc2, 0x76, 0x9d,
+	0x39, 0x94, 0x39, 0xbc, 0x7f, 0x42, 0x4d, 0xbb, 0x6a, 0xba, 0xa6, 0x6f, 0x0d, 0x62, 0xa1, 0x9b,
+	0xd3, 0x05, 0x2c, 0x5a, 0x6d, 0xc7, 0xb5, 0x19, 0xf1, 0x75, 0x83, 0x3a, 0x88, 0xad, 0xed, 0x5b,
+	0x99, 0x2a, 0xfb, 0x9a, 0xe6, 0xd0, 0xe7, 0xac, 0x8f, 0x07, 0xac, 0x68, 0x13, 0xa0, 0xa3, 0xc0,
+	0x0e, 0x09, 0x0a, 0xc9, 0xad, 0x54, 0x39, 0x8d, 0x87, 0x24, 0xc5, 0x63, 0x98, 0x93, 0x50, 0xf4,
+	0x02, 0xe6, 0x95, 0x32, 0x6d, 0x48, 0x39, 0xd6, 0x90, 0x88, 0x01, 0x4a, 0x35, 0xd6, 0xb8, 0x62,
+	0x1f, 0x96, 0x22, 0x56, 0xa0, 0x3c, 0xa4, 0x2e, 0x49, 0x5f, 0x9f, 0x2c, 0xf1, 0x13, 0x61, 0x98,
+	0xfb, 0x20, 0xfa, 0x88, 0x4e, 0xeb, 0x7f, 0xef, 0xe3, 0x2c, 0x56, 0x54, 0x7b, 0xc9, 0xe7, 0x89,
+	0xd2, 0x1f, 0x49, 0x78, 0xfc, 0x96, 0x38, 0xad, 0x36, 0x27, 0xf6, 0xb9, 0xc9, 0x5a, 0x84, 0xdf,
+	0x1c, 0x73, 0x0b, 0x16, 0xb8, 0xdc, 0x12, 0x68, 0x4f, 0x8f, 0x63, 0xad, 0x98, 0x92, 0xae, 0xa2,
+	0xd6, 0x03, 0x15, 0xf7, 0x90, 0xb9, 0xe8, 0xc1, 0xbc, 0x5a, 0x10, 0x8d, 0xe4, 0x4a, 0x52, 0xe9,
+	0x72, 0xd7, 0x5f, 0xe8, 0x0b, 0xc8, 0xca, 0x24, 0x85, 0xad, 0x2d, 0x79, 0xc7, 0xa8, 0x67, 0x24,
+	0x5a, 0xf7, 0xb3, 0xef, 0x20, 0x3b, 0x6c, 0x47, 0x4c, 0xe4, 0xdf, 0x47, 0x23, 0xbf, 0xff, 0x09,
+	0x7c, 0x1e, 0x4e, 0xc0, 0xb7, 0x90, 0xfd, 0x9c, 0x75, 0xac, 0x93, 0x86, 0x0e, 0xf2, 0xa8, 0x77,
+	0x89, 0x7b, 0x78, 0x27, 0xfa, 0x46, 0x08, 0x91, 0x57, 0xb9, 0xea, 0xc8, 0x19, 0x2d, 0x13, 0x8d,
+	0xbc, 0xf4, 0x08, 0xd2, 0xfb, 0x76, 0xa0, 0x95, 0x17, 0x60, 0xc1, 0x72, 0xbb, 0x01, 0x27, 0x2c,
+	0xec, 0xea, 0xfa, 0xb3, 0xf4, 0x6b, 0x12, 0xd2, 0xef, 0x06, 0xfb, 0x1e, 0xc3, 0x52, 0x43, 0xea,
+	0x25, 0xcc, 0xd0, 0x33, 0x42, 0xa2, 0x9c, 0xae, 0x26, 0x0b, 0x09, 0x9c, 0x0d, 0x17, 0xe4, 0x35,
+	0xf1, 0x29, 0x73, 0x85, 0xbe, 0x84, 0xe5, 0xa6, 0xe9, 0xba, 0xa2, 0xf1, 0x86, 0x7c, 0xa9, 0x3b,
+	0xf2, 0xe5, 0x42, 0x02, 0x4d, 0x59, 0x86, 0x3c, 0xb1, 0x03, 0x23, 0x12, 0xa4, 0x59, 0xe9, 0x79,
+	0x8e, 0xd8, 0xc1, 0xd9, 0x75, 0x9c, 0xd0, 0x05, 0x6c, 0xba, 0x2c, 0x30, 0x5c, 0x6a, 0xda, 0x06,
+	0x23, 0x1d, 0xca, 0xb8, 0x68, 0x6d, 0x02, 0x18, 0xc6, 0xe0, 0xa6, 0x29, 0xe0, 0x8c, 0x33, 0xc7,
+	0x6f, 0xa9, 0x29, 0xa0, 0xe8, 0xb2, 0x40, 0xd8, 0x85, 0x43, 0x86, 0x33, 0x49, 0x20, 0x33, 0xf1,
+	0x7d, 0x0a, 0x1e, 0x1e, 0xda, 0xc1, 0xcd, 0xe7, 0xef, 0xc6, 0xec, 0xc4, 0xba, 0x91, 0xfc, 0x48,
+	0x37, 0x52, 0xf7, 0x73, 0x03, 0x5d, 0xc0, 0x86, 0x4b, 0x2d, 0xd3, 0x75, 0x78, 0xdf, 0xe8, 0x38,
+	0xd6, 0xe5, 0xd0, 0x10, 0x32, 0x7b, 0xc7, 0x6c, 0xad, 0x85, 0x44, 0x75, 0xc5, 0xa3, 0x93, 0x76,
+	0x01, 0x1b, 0xc4, 0xb7, 0x3b, 0xd4, 0xf1, 0xf9, 0xa8, 0x86, 0xb9, 0xbb, 0x6a, 0x08, 0x89, 0x22,
+	0x1a, 0x4a, 0x3f, 0xa4, 0xe0, 0xe1, 0x09, 0x9b, 0x90, 0x8a, 0x6d, 0xc8, 0xea, 0xd8, 0x0f, 0xd5,
+	0x3f, 0xce, 0x68, 0x99, 0x8c, 0xc3, 0xf4, 0x39, 0xa9, 0x4e, 0x95, 0x93, 0xf4, 0xc4, 0xa8, 0xbf,
+	0x81, 0xc5, 0x30, 0x58, 0x7a, 0xd0, 0xdd, 0x8b, 0x0f, 0xc2, 0x04, 0xaf, 0x2a, 0x27, 0x9a, 0x01,
+	0x0f, 0xb8, 0xc6, 0x0e, 0xf0, 0xdc, 0x7d, 0x9a, 0x6d, 0x1d, 0x16, 0x43, 0x15, 0xa2, 0xbb, 0x33,
+	0xd2, 0x72, 0xa8, 0xaf, 0x63, 0xa7, 0xbf, 0x10, 0x82, 0xd9, 0x6f, 0xa8, 0x1f, 0x86, 0x4a, 0xfe,
+	0x96, 0xc3, 0x66, 0xb7, 0x21, 0xc5, 0x29, 0x3d, 0x6c, 0xaa, 0xcf, 0xd2, 0xef, 0x73, 0xb0, 0x1a,
+	0xa3, 0x16, 0x1d, 0x01, 0x88, 0xca, 0x30, 0x9a, 0x62, 0xd8, 0xd0, 0x01, 0xf9, 0x7b, 0xfc, 0x9d,
+	0x19, 0x1d, 0x49, 0x6a, 0x33, 0x78, 0x08, 0x89, 0x8e, 0x21, 0xc3, 0xc4, 0x80, 0x62, 0x30, 0x31,
+	0xa1, 0x48, 0x53, 0x33, 0xbb, 0x8f, 0x62, 0x89, 0x46, 0x07, 0x19, 0x39, 0x93, 0x5f, 0x63, 0xd1,
+	0x7f, 0x60, 0x5e, 0xc0, 0xdc, 0x86, 0x3e, 0x61, 0xdb, 0xb1, 0x2c, 0xc3, 0x77, 0x41, 0x6d, 0x06,
+	0x6b, 0x08, 0x3a, 0x87, 0x45, 0x3d, 0x7a, 0xf4, 0x0b, 0x69, 0x09, 0x7f, 0xf6, 0x71, 0x13, 0x40,
+	0x6d, 0x06, 0x0f, 0x98, 0x50, 0x0b, 0x96, 0xaf, 0xf4, 0xc5, 0x65, 0xa8, 0xfb, 0xb7, 0x00, 0x13,
+	0xc6, 0x8b, 0x29, 0x2f, 0xb9, 0xda, 0x0c, 0xce, 0x5d, 0x45, 0xb6, 0xa2, 0x5d, 0x48, 0x59, 0x76,
+	0xa0, 0x9f, 0x36, 0x9b, 0xb1, 0xe4, 0x83, 0x4b, 0xa8, 0x36, 0x83, 0xc5, 0x66, 0x74, 0x08, 0x29,
+	0x62, 0x07, 0xfa, 0xe1, 0xf2, 0xcf, 0x58, 0xcc, 0xa4, 0x6e, 0x29, 0x68, 0x88, 0xa2, 0x71, 0x59,
+	0x50, 0x58, 0x9c, 0x40, 0x33, 0xe9, 0x4c, 0x08, 0x1a, 0x97, 0x05, 0xe8, 0x19, 0xa4, 0x7a, 0x76,
+	0xa0, 0x67, 0x80, 0x78, 0x0f, 0x06, 0xd7, 0xa3, 0xb8, 0x07, 0x05, 0xae, 0x67, 0x07, 0xa8, 0x0e,
+	0xf9, 0x9e, 0x1d, 0x18, 0xa4, 0xd7, 0x21, 0xcc, 0xf1, 0x88, 0xcf, 0x4d, 0x57, 0x5f, 0x14, 0xd3,
+	0x91, 0x8c, 0xa1, 0xab, 0x8b, 0x30, 0xaf, 0x07, 0xea, 0x9f, 0xe7, 0x60, 0x49, 0xf7, 0x11, 0x5d,
+	0xf6, 0x3e, 0xac, 0xc9, 0x2e, 0xd2, 0x08, 0x5d, 0xb9, 0x9e, 0x22, 0x12, 0xe5, 0xdc, 0x0d, 0x35,
+	0x13, 0xa1, 0xa8, 0xc4, 0x44, 0x42, 0xde, 0xeb, 0xab, 0xee, 0xf8, 0x02, 0xfa, 0x6a, 0x4c, 0x9f,
+	0x9e, 0x84, 0xef, 0xda, 0xe9, 0xa3, 0xec, 0x83, 0x43, 0xbc, 0xa4, 0xde, 0x8e, 0x21, 0xab, 0x9a,
+	0x1e, 0xb6, 0x6f, 0x7d, 0xc4, 0xe2, 0xac, 0x37, 0xfc, 0x47, 0x86, 0x05, 0x79, 0xf5, 0x26, 0xe1,
+	0x6d, 0x46, 0x39, 0x77, 0x1d, 0xbf, 0xa5, 0xcf, 0xe0, 0xf3, 0x29, 0x02, 0x22, 0xdf, 0xac, 0xe7,
+	0x03, 0xa4, 0xf2, 0x1c, 0x2f, 0xb3, 0xa8, 0x18, 0xd9, 0xb0, 0xda, 0x26, 0xa6, 0xcb, 0xdb, 0x86,
+	0xd5, 0x26, 0xd6, 0x65, 0x68, 0xb2, 0xca, 0xf5, 0xbf, 0xa6, 0xd0, 0x53, 0x93, 0xe8, 0x7d, 0x01,
+	0xd6, 0x5e, 0xac, 0xb4, 0x47, 0x45, 0xc5, 0xb7, 0xb0, 0x16, 0x6b, 0x0f, 0xfa, 0x2b, 0x88, 0x87,
+	0xab, 0xc1, 0xe9, 0x25, 0xf1, 0xc3, 0xf7, 0x61, 0xda, 0x33, 0x7b, 0xe7, 0x52, 0x80, 0xfe, 0x06,
+	0x19, 0xb9, 0x64, 0xc8, 0xb7, 0x9f, 0x2c, 0xe3, 0x24, 0x06, 0x29, 0xc2, 0x42, 0x52, 0x3c, 0x87,
+	0x95, 0x31, 0x03, 0xd0, 0xff, 0x47, 0xc6, 0xc7, 0xc4, 0x14, 0xa3, 0x41, 0x64, 0xb8, 0xfc, 0xf7,
+	0x48, 0x77, 0xd6, 0xc6, 0x2e, 0x43, 0xe6, 0xf5, 0xe9, 0x59, 0xfd, 0x70, 0xff, 0xf8, 0xe8, 0xf8,
+	0xf0, 0x20, 0x3f, 0x23, 0x04, 0xf8, 0xd5, 0xeb, 0xd3, 0x03, 0x03, 0xbf, 0xaa, 0x1e, 0x9f, 0xe6,
+	0x13, 0xd5, 0x27, 0xb0, 0xe6, 0xd0, 0x48, 0xd0, 0x54, 0xcc, 0xaa, 0x28, 0x12, 0xb4, 0xba, 0xb0,
+	0xa0, 0x9e, 0x68, 0xcc, 0x4b, 0x53, 0x9e, 0xfe, 0x19, 0x00, 0x00, 0xff, 0xff, 0xae, 0xd3, 0x19,
+	0x42, 0x5a, 0x13, 0x00, 0x00,
 }

From a906ca0441ceb1f7cd4f5c7de30b8e81ce2ff5e8 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 30 Apr 2020 14:24:06 -0700
Subject: [PATCH 057/481] balancergroup: fix connectivity state (#3585)

 - Stop sending empty update to sub-balancers at init time
 - At init, when only one sub-balancer reports transient failure, wait for other sub-balancers
 - When aggregating states, consider a sub-balancer turns Connecting from TransientFailure still in TransientFailure, so aggregated state doesn't stay connecting for a long time
---
 .../balancer/balancergroup/balancergroup.go   | 83 ++++++++++++++-----
 .../balancergroup/balancergroup_test.go       | 76 +++++++++++++++++
 2 files changed, 136 insertions(+), 23 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index 44468c104736..b08a5864e25b 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -62,7 +62,12 @@ type subBalancerWithConfig struct {
 	// The static part of sub-balancer. Keeps balancerBuilders and addresses.
 	// To be used when restarting sub-balancer.
 	builder balancer.Builder
-	ccState balancer.ClientConnState
+	// ccState is a cache of the addresses/balancer config, so when the balancer
+	// is restarted after close, it will get the previous update. It's a pointer
+	// and is set to nil at init, so when the balancer is built for the first
+	// time (not a restart), it won't receive an empty update. Note that this
+	// isn't reset to nil when the underlying balancer is closed.
+	ccState *balancer.ClientConnState
 	// The dynamic part of sub-balancer. Only used when balancer group is
 	// started. Gets cleared when sub-balancer is closed.
 	balancer balancer.Balancer
@@ -94,7 +99,9 @@ func (sbc *subBalancerWithConfig) startBalancer() {
 	b := sbc.builder.Build(sbc, balancer.BuildOptions{})
 	sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name())
 	sbc.balancer = b
-	b.UpdateClientConnState(sbc.ccState)
+	if sbc.ccState != nil {
+		b.UpdateClientConnState(*sbc.ccState)
+	}
 }
 
 func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
@@ -110,7 +117,7 @@ func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state
 }
 
 func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnState) error {
-	sbc.ccState = s
+	sbc.ccState = &s
 	b := sbc.balancer
 	if b == nil {
 		// This sub-balancer was closed. This should never happen because
@@ -149,12 +156,17 @@ func (sbc *subBalancerWithConfig) stopBalancer() {
 
 type pickerState struct {
 	weight uint32
-	picker balancer.Picker
-	state  connectivity.State
+	state  balancer.State
+	// stateToAggregate is the connectivity state used only for state
+	// aggregation. It could be different from state.ConnectivityState. For
+	// example when a sub-balancer transitions from TransientFailure to
+	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
+	// is still TransientFailure.
+	stateToAggregate connectivity.State
 }
 
 func (s *pickerState) String() string {
-	return fmt.Sprintf("weight:%v,picker:%p,state:%v", s.weight, s.picker, s.state)
+	return fmt.Sprintf("weight:%v,picker:%p,state:%v,stateToAggregate:%v", s.weight, s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
 }
 
 // BalancerGroup takes a list of balancers, and make them into one balancer.
@@ -288,10 +300,14 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, weight uint32, builder bala
 	bg.incomingMu.Lock()
 	bg.idToPickerState[id] = &pickerState{
 		weight: weight,
-		// Start everything in IDLE. It's doesn't affect the overall state
-		// because we don't count IDLE when aggregating (as opposite to e.g.
-		// READY, 1 READY results in overall READY).
-		state: connectivity.Idle,
+		// Start everything in CONNECTING, so if one of the sub-balancers
+		// reports TransientFailure, the RPCs will still wait for the other
+		// sub-balancers.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+		stateToAggregate: connectivity.Connecting,
 	}
 	bg.incomingMu.Unlock()
 
@@ -518,13 +534,18 @@ func (bg *BalancerGroup) updateBalancerState(id internal.LocalityID, state balan
 		bg.logger.Warningf("balancer group: pickerState for %v not found when update picker/state", id)
 		return
 	}
-	newPicker := state.Picker
 	if bg.loadStore != nil {
 		// Only wrap the picker to do load reporting if loadStore was set.
-		newPicker = newLoadReportPicker(state.Picker, id, bg.loadStore)
+		state.Picker = newLoadReportPicker(state.Picker, id, bg.loadStore)
+	}
+	if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) {
+		// If old state is TransientFailure, and new state is Connecting, don't
+		// update the state, to prevent the aggregated state from being always
+		// CONNECTING. Otherwise, stateToAggregate is the same as
+		// state.ConnectivityState.
+		pickerSt.stateToAggregate = state.ConnectivityState
 	}
-	pickerSt.picker = newPicker
-	pickerSt.state = state.ConnectivityState
+	pickerSt.state = state
 	if bg.incomingStarted {
 		bg.logger.Infof("Child pickers with weight: %+v", bg.idToPickerState)
 		bg.cc.UpdateState(buildPickerAndState(bg.idToPickerState))
@@ -539,10 +560,13 @@ func (bg *BalancerGroup) Close() {
 		bg.incomingStarted = false
 
 		for _, pState := range bg.idToPickerState {
-			// Reset everything to IDLE but keep the entry in map (to keep the
-			// weight).
-			pState.picker = nil
-			pState.state = connectivity.Idle
+			// Reset everything to init state (Connecting) but keep the entry in
+			// map (to keep the weight).
+			pState.state = balancer.State{
+				ConnectivityState: connectivity.Connecting,
+				Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+			}
+			pState.stateToAggregate = connectivity.Connecting
 		}
 
 		// Also remove all SubConns.
@@ -570,7 +594,7 @@ func buildPickerAndState(m map[internal.LocalityID]*pickerState) balancer.State
 	var readyN, connectingN int
 	readyPickerWithWeights := make([]pickerState, 0, len(m))
 	for _, ps := range m {
-		switch ps.state {
+		switch ps.stateToAggregate {
 		case connectivity.Ready:
 			readyN++
 			readyPickerWithWeights = append(readyPickerWithWeights, *ps)
@@ -587,10 +611,23 @@ func buildPickerAndState(m map[internal.LocalityID]*pickerState) balancer.State
 	default:
 		aggregatedState = connectivity.TransientFailure
 	}
-	if aggregatedState == connectivity.TransientFailure {
-		return balancer.State{ConnectivityState: aggregatedState, Picker: base.NewErrPicker(balancer.ErrTransientFailure)}
+
+	// Make sure picker's return error is consistent with the aggregatedState.
+	//
+	// TODO: This is true for balancers like weighted_target, but not for
+	// routing. For routing, we want to always build picker with all sub-pickers
+	// (not even ready sub-pickers), so even if the overall state is Ready, pick
+	// for certain RPCs can behave like Connecting or TransientFailure.
+	var picker balancer.Picker
+	switch aggregatedState {
+	case connectivity.TransientFailure:
+		picker = base.NewErrPicker(balancer.ErrTransientFailure)
+	case connectivity.Connecting:
+		picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable)
+	default:
+		picker = newPickerGroup(readyPickerWithWeights)
 	}
-	return balancer.State{ConnectivityState: aggregatedState, Picker: newPickerGroup(readyPickerWithWeights)}
+	return balancer.State{ConnectivityState: aggregatedState, Picker: picker}
 }
 
 // NewRandomWRR is the WRR constructor used to pick sub-pickers from
@@ -612,7 +649,7 @@ type pickerGroup struct {
 func newPickerGroup(readyPickerWithWeights []pickerState) *pickerGroup {
 	w := NewRandomWRR()
 	for _, ps := range readyPickerWithWeights {
-		w.Add(ps.picker, int64(ps.weight))
+		w.Add(ps.state.Picker, int64(ps.weight))
 	}
 
 	return &pickerGroup{
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 8c211c041612..8fda50bcb595 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -23,6 +23,7 @@ import (
 
 	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
@@ -33,6 +34,7 @@ import (
 
 var (
 	rrBuilder        = balancer.Get(roundrobin.Name)
+	pfBuilder        = balancer.Get(grpc.PickFirstBalancerName)
 	testBalancerIDs  = []internal.LocalityID{{Region: "b1"}, {Region: "b2"}, {Region: "b3"}}
 	testBackendAddrs []resolver.Address
 )
@@ -551,6 +553,80 @@ func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
 	bg.Start()
 }
 
+// Test that at init time, with two sub-balancers, if one sub-balancer reports
+// transient_failure, the picks won't fail with transient_failure, and should
+// instead wait for the other sub-balancer.
+func (s) TestBalancerGroup_InitOneSubBalancerTransientFailure(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
+
+	// Add two balancers to group and send one resolved address to both
+	// balancers.
+	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
+	sc1 := <-cc.NewSubConnCh
+
+	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
+	<-cc.NewSubConnCh
+
+	// Set one subconn to TransientFailure, this will trigger one sub-balancer
+	// to report transient failure.
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		r, err := p1.Pick(balancer.PickInfo{})
+		if err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrNoSubConnAvailable, r, err)
+		}
+	}
+}
+
+// Test that with two sub-balancers, both in transient_failure, if one turns
+// connecting, the overall state stays in transient_failure, and all picks
+// return transient failure error.
+func (s) TestBalancerGroup_SubBalancerTurnsConnectingFromTransientFailure(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, nil, nil)
+	bg.Start()
+
+	// Add two balancers to group and send one resolved address to both
+	// balancers.
+	bg.Add(testBalancerIDs[0], 1, pfBuilder)
+	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
+	sc1 := <-cc.NewSubConnCh
+
+	bg.Add(testBalancerIDs[1], 1, pfBuilder)
+	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
+	sc2 := <-cc.NewSubConnCh
+
+	// Set both subconn to TransientFailure, this will put both sub-balancers in
+	// transient failure.
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		r, err := p1.Pick(balancer.PickInfo{})
+		if err != balancer.ErrTransientFailure {
+			t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrTransientFailure, r, err)
+		}
+	}
+
+	// Set one subconn to Connecting, it shouldn't change the overall state.
+	bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		r, err := p2.Pick(balancer.PickInfo{})
+		if err != balancer.ErrTransientFailure {
+			t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrTransientFailure, r, err)
+		}
+	}
+}
+
 func replaceDefaultSubBalancerCloseTimeout(n time.Duration) func() {
 	old := DefaultSubBalancerCloseTimeout
 	DefaultSubBalancerCloseTimeout = n

From e7557c82823b655ba432ad88b63557d9b55744b7 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 4 May 2020 08:50:03 -0700
Subject: [PATCH 058/481] xds: reject RDS response containing match with
 case-sensitive false (#3592)

---
 xds/internal/client/v2client_rds.go      | 13 +++++++++++--
 xds/internal/client/v2client_rds_test.go | 20 ++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
index 82140c9c43d0..e03284fcb1e0 100644
--- a/xds/internal/client/v2client_rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -95,9 +95,18 @@ func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string)
 		return "", fmt.Errorf("matched virtual host has no routes")
 	}
 	dr := vh.Routes[len(vh.Routes)-1]
-	if match := dr.GetMatch(); match == nil || (match.GetPrefix() != "" && match.GetPrefix() != "/") {
+	match := dr.GetMatch()
+	if match == nil {
+		return "", fmt.Errorf("matched virtual host's default route doesn't have a match")
+	}
+	if prefix := match.GetPrefix(); prefix != "" && prefix != "/" {
 		// The matched virtual host is invalid. Match is not "" or "/".
-		return "", fmt.Errorf("matched virtual host is invalid")
+		return "", fmt.Errorf("matched virtual host's default route is %v, want Prefix empty string or /", match)
+	}
+	if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
+		// The case sensitive is set to false. Not set or set to true are both
+		// valid.
+		return "", fmt.Errorf("matches virtual host's default route set case-sensitive to false")
 	}
 	if route := dr.GetRoute(); route != nil {
 		return route.GetCluster(), nil
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
index f9936d793796..e85cdbc8782f 100644
--- a/xds/internal/client/v2client_rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -25,6 +25,7 @@ import (
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
 	"github.com/golang/protobuf/proto"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
@@ -141,6 +142,25 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 			wantCluster: "",
 			wantError:   true,
 		},
+		{
+			// default route's match sets case-sensitive to false.
+			name: "good-route-config-but-with-casesensitive-false",
+			rc: &xdspb.RouteConfiguration{
+				Name: goodRouteName1,
+				VirtualHosts: []*routepb.VirtualHost{{
+					Domains: []string{goodLDSTarget1},
+					Routes: []*routepb.Route{{
+						Match: &routepb.RouteMatch{
+							PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"},
+							CaseSensitive: &wrapperspb.BoolValue{Value: false},
+						},
+						Action: &routepb.Route_Route{
+							Route: &routepb.RouteAction{
+								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
+							}}}}}}},
+			wantCluster: "",
+			wantError:   true,
+		},
 		{
 			name:        "good-route-config-with-empty-string-route",
 			rc:          goodRouteConfig1,

From c8482678eb49c6f3a6ab9288a5d61e4b8af3cae6 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Mon, 4 May 2020 10:01:09 -0700
Subject: [PATCH 059/481] advancedtls: remove test.Fatal() from child goroutine
 (#3595)

---
 .../advancedtls_integration_test.go            | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 84ca953f00ba..075e1e9f6ca2 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -404,17 +404,13 @@ func TestEnd2End(t *testing.T) {
 			}
 			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
 			defer s.Stop()
-			go func(s *grpc.Server) {
-				lis, err := net.Listen("tcp", port)
-				// defer lis.Close()
-				if err != nil {
-					t.Fatalf("failed to listen: %v", err)
-				}
-				pb.RegisterGreeterServer(s, &serverImpl{})
-				if err := s.Serve(lis); err != nil {
-					t.Fatalf("failed to serve: %v", err)
-				}
-			}(s)
+			lis, err := net.Listen("tcp", port)
+			defer lis.Close()
+			if err != nil {
+				t.Fatalf("failed to listen: %v", err)
+			}
+			pb.RegisterGreeterServer(s, &serverImpl{})
+			go s.Serve(lis)
 			clientOptions := &ClientOptions{
 				Certificates:         test.clientCert,
 				GetClientCertificate: test.clientGetCert,

From 10ccd463592b737f466d18d0247472e059b24659 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 6 May 2020 11:07:39 -0700
Subject: [PATCH 060/481] xds_client: make sure LDS sets ConfigSource to ADS
 (#3588)

---
 xds/internal/client/v2client_lds.go      |  3 +++
 xds/internal/client/v2client_lds_test.go | 29 ++++++++++++++++++++++++
 xds/internal/client/v2client_test.go     |  3 +++
 3 files changed, 35 insertions(+)

diff --git a/xds/internal/client/v2client_lds.go b/xds/internal/client/v2client_lds.go
index 49f4a46a28cc..05eddb5e093d 100644
--- a/xds/internal/client/v2client_lds.go
+++ b/xds/internal/client/v2client_lds.go
@@ -68,6 +68,9 @@ func (v2c *v2Client) getRouteConfigNameFromListener(lis *xdspb.Listener) (string
 	v2c.logger.Infof("Resource with type %T, contains %v", apiLis, apiLis)
 	switch apiLis.RouteSpecifier.(type) {
 	case *httppb.HttpConnectionManager_Rds:
+		if apiLis.GetRds().GetConfigSource().GetAds() == nil {
+			return "", fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis)
+		}
 		name := apiLis.GetRds().GetRouteConfigName()
 		if name == "" {
 			return "", fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
diff --git a/xds/internal/client/v2client_lds_test.go b/xds/internal/client/v2client_lds_test.go
index a61b18bbbae0..893245d0ec54 100644
--- a/xds/internal/client/v2client_lds_test.go
+++ b/xds/internal/client/v2client_lds_test.go
@@ -22,7 +22,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang/protobuf/proto"
+
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
+	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
+	anypb "github.com/golang/protobuf/ptypes/any"
 )
 
 func (s) TestLDSGetRouteConfig(t *testing.T) {
@@ -62,6 +68,29 @@ func (s) TestLDSGetRouteConfig(t *testing.T) {
 			wantRoute: "",
 			wantErr:   true,
 		},
+		{
+			name: "rds.ConfigSource-in-apiListener-is-not-ADS",
+			lis: &xdspb.Listener{
+				Name: goodLDSTarget1,
+				ApiListener: &listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: httpConnManagerURL,
+						Value: func() []byte {
+							cm := &httppb.HttpConnectionManager{
+								RouteSpecifier: &httppb.HttpConnectionManager_Rds{
+									Rds: &httppb.Rds{
+										ConfigSource: &basepb.ConfigSource{
+											ConfigSourceSpecifier: &basepb.ConfigSource_Path{
+												Path: "/some/path",
+											},
+										},
+										RouteConfigName: goodRouteName1}}}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}()}}},
+			wantRoute: "",
+			wantErr:   true,
+		},
 		{
 			name:      "goodListener1",
 			lis:       goodListener1,
diff --git a/xds/internal/client/v2client_test.go b/xds/internal/client/v2client_test.go
index e52648cfeb1f..147558d5acc0 100644
--- a/xds/internal/client/v2client_test.go
+++ b/xds/internal/client/v2client_test.go
@@ -88,6 +88,9 @@ var (
 	goodHTTPConnManager1 = &httppb.HttpConnectionManager{
 		RouteSpecifier: &httppb.HttpConnectionManager_Rds{
 			Rds: &httppb.Rds{
+				ConfigSource: &basepb.ConfigSource{
+					ConfigSourceSpecifier: &basepb.ConfigSource_Ads{Ads: &basepb.AggregatedConfigSource{}},
+				},
 				RouteConfigName: goodRouteName1,
 			},
 		},

From 695df7e2f9e0c2a8de6793b77c1e752b33871ba0 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Fri, 8 May 2020 04:16:17 +0800
Subject: [PATCH 061/481] status: wrap status proto in a struct (#3556)

---
 internal/status/status.go | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/internal/status/status.go b/internal/status/status.go
index 981c265b4c1b..710223b8ded0 100644
--- a/internal/status/status.go
+++ b/internal/status/status.go
@@ -97,7 +97,7 @@ func (s *Status) Err() error {
 	if s.Code() == codes.OK {
 		return nil
 	}
-	return (*Error)(s.Proto())
+	return &Error{e: s.Proto()}
 }
 
 // WithDetails returns a new status with the provided details messages appended to the status.
@@ -136,18 +136,19 @@ func (s *Status) Details() []interface{} {
 	return details
 }
 
-// Error is an alias of a status proto. It implements error and Status,
+// Error wraps a pointer of a status proto. It implements error and Status,
 // and a nil *Error should never be returned by this package.
-type Error spb.Status
+type Error struct {
+	e *spb.Status
+}
 
 func (e *Error) Error() string {
-	p := (*spb.Status)(e)
-	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage())
+	return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(e.e.GetCode()), e.e.GetMessage())
 }
 
 // GRPCStatus returns the Status represented by se.
 func (e *Error) GRPCStatus() *Status {
-	return FromProto((*spb.Status)(e))
+	return FromProto(e.e)
 }
 
 // Is implements future error.Is functionality.
@@ -157,5 +158,5 @@ func (e *Error) Is(target error) bool {
 	if !ok {
 		return false
 	}
-	return proto.Equal((*spb.Status)(e), (*spb.Status)(tse))
+	return proto.Equal(e.e, tse.e)
 }

From 42e450fade2ebf999ee655b19a9042759a58ac1f Mon Sep 17 00:00:00 2001
From: Matjaz Debelak <KillerX@users.noreply.github.com>
Date: Thu, 7 May 2020 22:17:02 +0200
Subject: [PATCH 062/481] doc: update "A short introduction to Channelz" url
 (#3583)

---
 examples/features/debugging/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/features/debugging/README.md b/examples/features/debugging/README.md
index 41618e960a31..543280e9591b 100644
--- a/examples/features/debugging/README.md
+++ b/examples/features/debugging/README.md
@@ -13,7 +13,7 @@ To turn on the logs for debugging, run the code with the following environment v
 ## Channelz
 We also provide a runtime debugging tool, Channelz, to help users with live debugging.
 
-See the channelz blog post here ([link](https://grpc.io/blog/a_short_introduction_to_channelz)) for
+See the channelz blog post here ([link](https://grpc.io/blog/a-short-introduction-to-channelz/)) for
 details about how to use channelz service to debug live program.
 
 ## Try it

From a6ab4473c5a469332c1bdee691293affeaaece25 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 7 May 2020 14:15:30 -0700
Subject: [PATCH 063/481] cache: callback without cache's mutex (#3603)

---
 internal/cache/timeoutCache.go | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/internal/cache/timeoutCache.go b/internal/cache/timeoutCache.go
index 94cd33898be8..200b499ec81e 100644
--- a/internal/cache/timeoutCache.go
+++ b/internal/cache/timeoutCache.go
@@ -23,7 +23,9 @@ import (
 )
 
 type cacheEntry struct {
-	item     interface{}
+	item interface{}
+	// Note that to avoid deadlocks (potentially caused by lock ordering),
+	// callback can only be called without holding cache's mutex.
 	callback func()
 	timer    *time.Timer
 	// deleted is set to true in Remove() when the call to timer.Stop() fails.
@@ -89,7 +91,7 @@ func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{},
 func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	entry, ok := c.removeInternal(key, false)
+	entry, ok := c.removeInternal(key)
 	if !ok {
 		return nil, false
 	}
@@ -99,7 +101,7 @@ func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) {
 // removeInternal removes and returns the item with key.
 //
 // caller must hold c.mu.
-func (c *TimeoutCache) removeInternal(key interface{}, runCallback bool) (*cacheEntry, bool) {
+func (c *TimeoutCache) removeInternal(key interface{}) (*cacheEntry, bool) {
 	entry, ok := c.cache[key]
 	if !ok {
 		return nil, false
@@ -115,17 +117,28 @@ func (c *TimeoutCache) removeInternal(key interface{}, runCallback bool) (*cache
 		// of deleted and return.
 		entry.deleted = true
 	}
-	if runCallback {
-		entry.callback()
-	}
 	return entry, true
 }
 
 // Clear removes all entries, and runs the callbacks if runCallback is true.
 func (c *TimeoutCache) Clear(runCallback bool) {
+	var entries []*cacheEntry
 	c.mu.Lock()
-	defer c.mu.Unlock()
 	for key := range c.cache {
-		c.removeInternal(key, runCallback)
+		if e, ok := c.removeInternal(key); ok {
+			entries = append(entries, e)
+		}
+	}
+	c.mu.Unlock()
+
+	if !runCallback {
+		return
+	}
+
+	// removeInternal removes entries from cache, and also stops the timer, so
+	// the callback is guaranteed to be not called. If runCallback is true,
+	// manual execute all callbacks.
+	for _, entry := range entries {
+		entry.callback()
 	}
 }

From 36493bc37524eb8cbfd4d3fcb666bea6d106c274 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 13 May 2020 11:03:21 -0700
Subject: [PATCH 064/481] xds: stop expiry timer after receiving an update
 (#3615)

---
 xds/internal/client/client_callback.go        | 10 ++-
 xds/internal/client/client_watchers.go        | 78 +++++++++++++++++--
 .../client/client_watchers_cluster.go         |  4 +-
 .../client/client_watchers_cluster_test.go    | 45 +++++++++++
 .../client/client_watchers_endpoints.go       |  4 +-
 xds/internal/client/client_watchers_lds.go    |  4 +-
 xds/internal/client/client_watchers_rds.go    |  4 +-
 7 files changed, 132 insertions(+), 17 deletions(-)

diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go
index 3cd456ac5252..11dd6ccabc11 100644
--- a/xds/internal/client/client_callback.go
+++ b/xds/internal/client/client_callback.go
@@ -24,6 +24,8 @@ type watcherInfoWithUpdate struct {
 	err    error
 }
 
+// scheduleCallback should only be called by methods of watchInfo, which checks
+// for watcher states and maintain consistency.
 func (c *Client) scheduleCallback(wi *watchInfo, update interface{}, err error) {
 	c.updateCh.Put(&watcherInfoWithUpdate{
 		wi:     wi,
@@ -79,7 +81,7 @@ func (c *Client) newLDSUpdate(d map[string]ldsUpdate) {
 	for name, update := range d {
 		if s, ok := c.ldsWatchers[name]; ok {
 			for wi := range s {
-				c.scheduleCallback(wi, update, nil)
+				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, update)
@@ -105,7 +107,7 @@ func (c *Client) newRDSUpdate(d map[string]rdsUpdate) {
 	for name, update := range d {
 		if s, ok := c.rdsWatchers[name]; ok {
 			for wi := range s {
-				c.scheduleCallback(wi, update, nil)
+				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, update)
@@ -126,7 +128,7 @@ func (c *Client) newCDSUpdate(d map[string]ClusterUpdate) {
 	for name, update := range d {
 		if s, ok := c.cdsWatchers[name]; ok {
 			for wi := range s {
-				c.scheduleCallback(wi, update, nil)
+				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, update)
@@ -152,7 +154,7 @@ func (c *Client) newEDSUpdate(d map[string]EndpointsUpdate) {
 	for name, update := range d {
 		if s, ok := c.edsWatchers[name]; ok {
 			for wi := range s {
-				c.scheduleCallback(wi, update, nil)
+				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, update)
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index bb1e15db709a..7a5bd2614a03 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -19,6 +19,8 @@
 package client
 
 import (
+	"fmt"
+	"sync"
 	"time"
 )
 
@@ -33,8 +35,18 @@ const (
 	edsURL = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
 )
 
+type watchInfoState int
+
+const (
+	watchInfoStateStarted watchInfoState = iota
+	watchInfoStateRespReceived
+	watchInfoStateTimeout
+	watchInfoStateCanceled
+)
+
 // watchInfo holds all the information from a watch() call.
 type watchInfo struct {
+	c       *Client
 	typeURL string
 	target  string
 
@@ -42,7 +54,63 @@ type watchInfo struct {
 	rdsCallback rdsCallbackFunc
 	cdsCallback func(ClusterUpdate, error)
 	edsCallback func(EndpointsUpdate, error)
+
 	expiryTimer *time.Timer
+
+	// mu protects state, and c.scheduleCallback().
+	// - No callback should be scheduled after watchInfo is canceled.
+	// - No timeout error should be scheduled after watchInfo is resp received.
+	mu    sync.Mutex
+	state watchInfoState
+}
+
+func (wi *watchInfo) newUpdate(update interface{}) {
+	wi.mu.Lock()
+	defer wi.mu.Unlock()
+	if wi.state == watchInfoStateCanceled {
+		return
+	}
+	wi.state = watchInfoStateRespReceived
+	wi.expiryTimer.Stop()
+	wi.c.scheduleCallback(wi, update, nil)
+}
+
+func (wi *watchInfo) timeout() {
+	wi.mu.Lock()
+	defer wi.mu.Unlock()
+	if wi.state == watchInfoStateCanceled || wi.state == watchInfoStateRespReceived {
+		return
+	}
+	wi.state = watchInfoStateTimeout
+	var (
+		u interface{}
+		t string
+	)
+	switch wi.typeURL {
+	case ldsURL:
+		u = ldsUpdate{}
+		t = "LDS"
+	case rdsURL:
+		u = rdsUpdate{}
+		t = "RDS"
+	case cdsURL:
+		u = ClusterUpdate{}
+		t = "CDS"
+	case edsURL:
+		u = EndpointsUpdate{}
+		t = "EDS"
+	}
+	wi.c.scheduleCallback(wi, u, fmt.Errorf("xds: %s target %s not found, watcher timeout", t, wi.target))
+}
+
+func (wi *watchInfo) cancel() {
+	wi.mu.Lock()
+	defer wi.mu.Unlock()
+	if wi.state == watchInfoStateCanceled {
+		return
+	}
+	wi.expiryTimer.Stop()
+	wi.state = watchInfoStateCanceled
 }
 
 func (c *Client) watch(wi *watchInfo) (cancel func()) {
@@ -83,31 +151,31 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 	case ldsURL:
 		if v, ok := c.ldsCache[resourceName]; ok {
 			c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, v)
-			c.scheduleCallback(wi, v, nil)
+			wi.newUpdate(v)
 		}
 	case rdsURL:
 		if v, ok := c.rdsCache[resourceName]; ok {
 			c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, v)
-			c.scheduleCallback(wi, v, nil)
+			wi.newUpdate(v)
 		}
 	case cdsURL:
 		if v, ok := c.cdsCache[resourceName]; ok {
 			c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, v)
-			c.scheduleCallback(wi, v, nil)
+			wi.newUpdate(v)
 		}
 	case edsURL:
 		if v, ok := c.edsCache[resourceName]; ok {
 			c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, v)
-			c.scheduleCallback(wi, v, nil)
+			wi.newUpdate(v)
 		}
 	}
 
 	return func() {
 		c.logger.Debugf("watch for type %v, resource name %v canceled", wi.typeURL, wi.target)
+		wi.cancel()
 		c.mu.Lock()
 		defer c.mu.Unlock()
 		if s := watchers[resourceName]; s != nil {
-			wi.expiryTimer.Stop()
 			// Remove this watcher, so it's callback will not be called in the
 			// future.
 			delete(s, wi)
diff --git a/xds/internal/client/client_watchers_cluster.go b/xds/internal/client/client_watchers_cluster.go
index 90d905f7a246..2d90711308c0 100644
--- a/xds/internal/client/client_watchers_cluster.go
+++ b/xds/internal/client/client_watchers_cluster.go
@@ -19,7 +19,6 @@
 package client
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -44,13 +43,14 @@ type ClusterUpdate struct {
 // after the watcher is canceled. The caller needs to handle this case.
 func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
+		c:           c,
 		typeURL:     cdsURL,
 		target:      clusterName,
 		cdsCallback: cb,
 	}
 
 	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		c.scheduleCallback(wi, ClusterUpdate{}, fmt.Errorf("xds: CDS target %s not found, watcher timeout", clusterName))
+		wi.timeout()
 	})
 	return c.watch(wi)
 }
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index 3621d8dfcd19..ab7a62091b9f 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -44,6 +44,9 @@ func (s) TestClusterWatch(t *testing.T) {
 	}
 	defer c.Close()
 
+	// TODO: add a timeout to this recv.
+	// Note that this won't be necessary if we finish the TODO below to call
+	// Client directly instead of v2Client.r.
 	v2Client := <-v2ClientCh
 
 	clusterUpdateCh := testutils.NewChannel()
@@ -274,3 +277,45 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 		t.Errorf("unexpected clusterError: <nil>, want error watcher timeout")
 	}
 }
+
+// TestClusterWatchExpiryTimerStop tests the case where the client does receive
+// an CDS response for the request that it sends out. We want no error even
+// after expiry timeout.
+func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
+	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
+	defaultWatchExpiryTimeout = 500 * time.Millisecond
+	defer func() {
+		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
+	}()
+
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	clusterUpdateCh := testutils.NewChannel()
+	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
+		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
+	})
+
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName: wantUpdate,
+	})
+
+	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Wait for an error, the error should never happen.
+	u, err := clusterUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	if err != testutils.ErrRecvTimeout {
+		t.Fatalf("got unexpected: %v, %v, want recv timeout", u.(clusterUpdateErr).u, u.(clusterUpdateErr).err)
+	}
+}
diff --git a/xds/internal/client/client_watchers_endpoints.go b/xds/internal/client/client_watchers_endpoints.go
index db28865aebe3..439f18248a91 100644
--- a/xds/internal/client/client_watchers_endpoints.go
+++ b/xds/internal/client/client_watchers_endpoints.go
@@ -19,7 +19,6 @@
 package client
 
 import (
-	"fmt"
 	"time"
 
 	"google.golang.org/grpc/xds/internal"
@@ -81,13 +80,14 @@ type EndpointsUpdate struct {
 // after the watcher is canceled. The caller needs to handle this case.
 func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
+		c:           c,
 		typeURL:     edsURL,
 		target:      clusterName,
 		edsCallback: cb,
 	}
 
 	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		c.scheduleCallback(wi, EndpointsUpdate{}, fmt.Errorf("xds: EDS target %s not found, watcher timeout", clusterName))
+		wi.timeout()
 	})
 	return c.watch(wi)
 }
diff --git a/xds/internal/client/client_watchers_lds.go b/xds/internal/client/client_watchers_lds.go
index 9728cf8023d3..0f7860db991e 100644
--- a/xds/internal/client/client_watchers_lds.go
+++ b/xds/internal/client/client_watchers_lds.go
@@ -19,7 +19,6 @@
 package client
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -35,13 +34,14 @@ type ldsCallbackFunc func(ldsUpdate, error)
 // after the watcher is canceled. The caller needs to handle this case.
 func (c *Client) watchLDS(serviceName string, cb ldsCallbackFunc) (cancel func()) {
 	wi := &watchInfo{
+		c:           c,
 		typeURL:     ldsURL,
 		target:      serviceName,
 		ldsCallback: cb,
 	}
 
 	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		c.scheduleCallback(wi, ldsUpdate{}, fmt.Errorf("xds: LDS target %s not found, watcher timeout", serviceName))
+		wi.timeout()
 	})
 	return c.watch(wi)
 }
diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
index e7f1cc321f36..b241a350dfe2 100644
--- a/xds/internal/client/client_watchers_rds.go
+++ b/xds/internal/client/client_watchers_rds.go
@@ -19,7 +19,6 @@
 package client
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -35,13 +34,14 @@ type rdsCallbackFunc func(rdsUpdate, error)
 // after the watcher is canceled. The caller needs to handle this case.
 func (c *Client) watchRDS(routeName string, cb rdsCallbackFunc) (cancel func()) {
 	wi := &watchInfo{
+		c:           c,
 		typeURL:     rdsURL,
 		target:      routeName,
 		rdsCallback: cb,
 	}
 
 	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		c.scheduleCallback(wi, rdsUpdate{}, fmt.Errorf("xds: RDS target %s not found, watcher timeout", routeName))
+		wi.timeout()
 	})
 	return c.watch(wi)
 }

From f62d68d51371af260be14b6dae029757e354f773 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 18 May 2020 21:39:57 +0100
Subject: [PATCH 065/481] benchmark: exit from UnconstrainedStreamingCall()
 when stream is canceled (#3623)

---
 benchmark/benchmark.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index 8d9b20f28b1b..e844bb1ea52d 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -113,6 +113,7 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U
 			err := stream.RecvMsg(in)
 			switch status.Code(err) {
 			case codes.Canceled:
+				return
 			case codes.OK:
 			default:
 				log.Fatalf("server recv error: %v", err)
@@ -125,6 +126,7 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U
 			err := stream.Send(response)
 			switch status.Code(err) {
 			case codes.Unavailable:
+				return
 			case codes.OK:
 			default:
 				log.Fatalf("server send error: %v", err)

From f5cae43845b8b054a2173bb83c660ef944f7c71b Mon Sep 17 00:00:00 2001
From: Ben Wells <b.v.wells@gmail.com>
Date: Mon, 18 May 2020 22:18:06 +0100
Subject: [PATCH 066/481] Fix wording in keep alive documentation (#3625)

---
 Documentation/keepalive.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Documentation/keepalive.md b/Documentation/keepalive.md
index c10168014149..43cfda82daf7 100644
--- a/Documentation/keepalive.md
+++ b/Documentation/keepalive.md
@@ -2,7 +2,7 @@
 
 gRPC sends http2 pings on the transport to detect if the connection is down. If
 the ping is not acknowledged by the other side within a certain period, the
-connection will be close. Note that pings are only necessary when there's no
+connection will be closed. Note that pings are only necessary when there's no
 activity on the connection.
 
 For how to configure keepalive, see

From 96e5c84cc83240a5f4086a26ff37e715d7fa63ba Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 19 May 2020 10:04:23 -0700
Subject: [PATCH 067/481] xds: rename resolver scheme to xds (#3621)

https://github.com/grpc/grpc-go/pull/3621
---
 examples/features/xds/README.md            |  4 ++--
 examples/features/xds/client/main.go       |  2 +-
 test/kokoro/xds.sh                         |  2 +-
 xds/internal/resolver/xds_resolver.go      | 19 ++++++++++++++++---
 xds/internal/resolver/xds_resolver_test.go |  9 +++++++++
 5 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/examples/features/xds/README.md b/examples/features/xds/README.md
index 6aabc8bf6dea..eba96d38f37a 100644
--- a/examples/features/xds/README.md
+++ b/examples/features/xds/README.md
@@ -29,9 +29,9 @@ The client application needs to import the xDS package to install the resolver a
 _ "google.golang.org/grpc/xds/experimental" // To install the xds resolvers and balancers.
 ```
 
-Then, use `xds-experimental` target scheme for the ClientConn.
+Then, use `xds` target scheme for the ClientConn.
 
 ```
 $ export GRPC_XDS_BOOTSTRAP=/path/to/bootstrap.json
-$ go run client/main.go "xDS world" xds-experimental:///target_service
+$ go run client/main.go "xDS world" xds:///target_service
 ```
diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go
index 97e1edeb23b4..3f1a53871516 100644
--- a/examples/features/xds/client/main.go
+++ b/examples/features/xds/client/main.go
@@ -49,7 +49,7 @@ Usage: client [name [target]]
   name
         The name you wish to be greeted by. Defaults to %q
   target
-        The URI of the server, e.g. "xds-experimental:///helloworld-service". Defaults to %q
+        The URI of the server, e.g. "xds:///helloworld-service". Defaults to %q
 `, defaultName, defaultTarget)
 
 		flag.PrintDefaults()
diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index 06d826381084..c3fe3c29eefe 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -28,7 +28,7 @@ GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
     --gcp_suffix=$(date '+%s') \
     --verbose \
     --client_cmd="grpc-go/interop/xds/client/client \
-      --server=xds-experimental:///{server_uri} \
+      --server=xds:///{server_uri} \
       --stats_port={stats_port} \
       --qps={qps} \
       {fail_on_failed_rpc}"
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 04da4c37d6b8..66ec38c89d01 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -33,9 +33,7 @@ import (
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
 
-// xDS balancer name is xds_experimental while resolver scheme is
-// xds-experimental since "_" is not a valid character in the URL.
-const xdsScheme = "xds-experimental"
+const xdsScheme = "xds"
 
 // For overriding in unittests.
 var (
@@ -216,3 +214,18 @@ func (r *xdsResolver) Close() {
 	r.cancelCtx()
 	r.logger.Infof("Shutdown")
 }
+
+// Keep scheme with "-experimental" temporarily. Remove after one release.
+const schemeExperimental = "xds-experimental"
+
+type xdsResolverExperimentalBuilder struct {
+	xdsResolverBuilder
+}
+
+func (*xdsResolverExperimentalBuilder) Scheme() string {
+	return schemeExperimental
+}
+
+func init() {
+	resolver.Register(&xdsResolverExperimentalBuilder{})
+}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 7dbe7167ca78..57456730abab 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -54,6 +54,15 @@ var (
 	target = resolver.Target{Endpoint: targetStr}
 )
 
+func TestRegister(t *testing.T) {
+	for _, s := range []string{"xds", "xds-experimental"} {
+		b := resolver.Get(s)
+		if b == nil {
+			t.Errorf("scheme %v is not registered", s)
+		}
+	}
+}
+
 // testClientConn is a fake implemetation of resolver.ClientConn. All is does
 // is to store the state received from the resolver locally and signal that
 // event through a channel.

From 636b0d84dd99dd8f2532ed495d2071f0ebe1162a Mon Sep 17 00:00:00 2001
From: IceberGu <wei.cai-nat@daocloud.io>
Date: Wed, 20 May 2020 10:24:38 +0800
Subject: [PATCH 068/481] internal: fix typos (#3581)

---
 dialoptions.go | 2 +-
 server.go      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dialoptions.go b/dialoptions.go
index b5c810927b10..d5030c076178 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -459,7 +459,7 @@ func WithStreamInterceptor(f StreamClientInterceptor) DialOption {
 }
 
 // WithChainStreamInterceptor returns a DialOption that specifies the chained
-// interceptor for unary RPCs. The first interceptor will be the outer most,
+// interceptor for streaming RPCs. The first interceptor will be the outer most,
 // while the last interceptor will be the inner most wrapper around the real call.
 // All interceptors added by this method will be chained, and the interceptor
 // defined by WithStreamInterceptor will always be prepended to the chain.
diff --git a/server.go b/server.go
index bac3c04cbce4..c2c7cae6c5dc 100644
--- a/server.go
+++ b/server.go
@@ -345,7 +345,7 @@ func StreamInterceptor(i StreamServerInterceptor) ServerOption {
 }
 
 // ChainStreamInterceptor returns a ServerOption that specifies the chained interceptor
-// for stream RPCs. The first interceptor will be the outer most,
+// for streaming RPCs. The first interceptor will be the outer most,
 // while the last interceptor will be the inner most wrapper around the real call.
 // All stream interceptors added by this method will be chained.
 func ChainStreamInterceptor(interceptors ...StreamServerInterceptor) ServerOption {

From 9eb3e7ddf3bd7fd7069f04c704acf8cfc9fe78fb Mon Sep 17 00:00:00 2001
From: yihuaz <yihuaz@google.com>
Date: Wed, 20 May 2020 15:43:38 -0700
Subject: [PATCH 069/481] credentials: local creds implementation (#3517)

Local credentials should be used in either a UDS and local TCP connection. The former will be associated with the security level PrigvacyAndIntegrity while the latter is associated with NoSecurity. Local credentials should be used instead of WithInsecure for localhost connections.
---
 credentials/credentials.go         |   8 +-
 credentials/credentials_test.go    |   4 +-
 credentials/local/local.go         | 109 +++++++++++++++
 credentials/local/local_test.go    | 204 +++++++++++++++++++++++++++
 internal/transport/http2_client.go |   4 +-
 test/local_creds_test.go           | 217 +++++++++++++++++++++++++++++
 6 files changed, 540 insertions(+), 6 deletions(-)
 create mode 100644 credentials/local/local.go
 create mode 100644 credentials/local/local_test.go
 create mode 100644 test/local_creds_test.go

diff --git a/credentials/credentials.go b/credentials/credentials.go
index 53addd8c71e9..02766443ae74 100644
--- a/credentials/credentials.go
+++ b/credentials/credentials.go
@@ -58,9 +58,11 @@ type PerRPCCredentials interface {
 type SecurityLevel int
 
 const (
-	// NoSecurity indicates a connection is insecure.
+	// Invalid indicates an invalid security level.
 	// The zero SecurityLevel value is invalid for backward compatibility.
-	NoSecurity SecurityLevel = iota + 1
+	Invalid SecurityLevel = iota
+	// NoSecurity indicates a connection is insecure.
+	NoSecurity
 	// IntegrityOnly indicates a connection only provides integrity protection.
 	IntegrityOnly
 	// PrivacyAndIntegrity indicates a connection provides both privacy and integrity protection.
@@ -237,7 +239,7 @@ func CheckSecurityLevel(ctx context.Context, level SecurityLevel) error {
 	}
 	if ci, ok := ri.AuthInfo.(internalInfo); ok {
 		// CommonAuthInfo.SecurityLevel has an invalid value.
-		if ci.GetCommonAuthInfo().SecurityLevel == 0 {
+		if ci.GetCommonAuthInfo().SecurityLevel == Invalid {
 			return nil
 		}
 		if ci.GetCommonAuthInfo().SecurityLevel < level {
diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go
index c2a316281928..5ff4850453bd 100644
--- a/credentials/credentials_test.go
+++ b/credentials/credentials_test.go
@@ -86,12 +86,12 @@ func (s) TestCheckSecurityLevel(t *testing.T) {
 			want:      true,
 		},
 		{
-			authLevel: 0,
+			authLevel: Invalid,
 			testLevel: IntegrityOnly,
 			want:      true,
 		},
 		{
-			authLevel: 0,
+			authLevel: Invalid,
 			testLevel: PrivacyAndIntegrity,
 			want:      true,
 		},
diff --git a/credentials/local/local.go b/credentials/local/local.go
new file mode 100644
index 000000000000..23de34cf8a18
--- /dev/null
+++ b/credentials/local/local.go
@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package local implements local transport credentials.
+// Local credentials reports the security level based on the type
+// of connetion. If the connection is local TCP, NoSecurity will be
+// reported, and if the connection is UDS, PrivacyAndIntegrity will be
+// reported. If local credentials is not used in local connections
+// (local TCP or UDS), it will fail.
+//
+// This package is EXPERIMENTAL.
+package local
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+
+	"google.golang.org/grpc/credentials"
+)
+
+// Info contains the auth information for a local connection.
+// It implements the AuthInfo interface.
+type Info struct {
+	credentials.CommonAuthInfo
+}
+
+// AuthType returns the type of Info as a string.
+func (Info) AuthType() string {
+	return "local"
+}
+
+// localTC is the credentials required to establish a local connection.
+type localTC struct {
+	info credentials.ProtocolInfo
+}
+
+func (c *localTC) Info() credentials.ProtocolInfo {
+	return c.info
+}
+
+// getSecurityLevel returns the security level for a local connection.
+// It returns an error if a connection is not local.
+func getSecurityLevel(network, addr string) (credentials.SecurityLevel, error) {
+	switch {
+	// Local TCP connection
+	case strings.HasPrefix(addr, "127."), strings.HasPrefix(addr, "[::1]:"):
+		return credentials.NoSecurity, nil
+	// UDS connection
+	case network == "unix":
+		return credentials.PrivacyAndIntegrity, nil
+	// Not a local connection and should fail
+	default:
+		return credentials.Invalid, fmt.Errorf("local credentials rejected connection to non-local address %q", addr)
+	}
+}
+
+func (*localTC) ClientHandshake(ctx context.Context, authority string, conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	secLevel, err := getSecurityLevel(conn.RemoteAddr().Network(), conn.RemoteAddr().String())
+	if err != nil {
+		return nil, nil, err
+	}
+	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil
+}
+
+func (*localTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	secLevel, err := getSecurityLevel(conn.RemoteAddr().Network(), conn.RemoteAddr().String())
+	if err != nil {
+		return nil, nil, err
+	}
+	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil
+}
+
+// NewCredentials returns a local credential implementing credentials.TransportCredentials.
+func NewCredentials() credentials.TransportCredentials {
+	return &localTC{
+		info: credentials.ProtocolInfo{
+			SecurityProtocol: "local",
+		},
+	}
+}
+
+// Clone makes a copy of Local credentials.
+func (c *localTC) Clone() credentials.TransportCredentials {
+	return &localTC{info: c.info}
+}
+
+// OverrideServerName overrides the server name used to verify the hostname on the returned certificates from the server.
+// Since this feature is specific to TLS (SNI + hostname verification check), it does not take any effet for local credentials.
+func (c *localTC) OverrideServerName(serverNameOverride string) error {
+	c.info.ServerName = serverNameOverride
+	return nil
+}
diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go
new file mode 100644
index 000000000000..a508d89bcd06
--- /dev/null
+++ b/credentials/local/local_test.go
@@ -0,0 +1,204 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package local
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestGetSecurityLevel(t *testing.T) {
+	testCases := []struct {
+		testNetwork string
+		testAddr    string
+		want        credentials.SecurityLevel
+	}{
+		{
+			testNetwork: "tcp",
+			testAddr:    "127.0.0.1:10000",
+			want:        credentials.NoSecurity,
+		},
+		{
+			testNetwork: "tcp",
+			testAddr:    "[::1]:10000",
+			want:        credentials.NoSecurity,
+		},
+		{
+			testNetwork: "unix",
+			testAddr:    "/tmp/grpc_fullstack_test",
+			want:        credentials.PrivacyAndIntegrity,
+		},
+		{
+			testNetwork: "tcp",
+			testAddr:    "192.168.0.1:10000",
+			want:        credentials.Invalid,
+		},
+	}
+	for _, tc := range testCases {
+		got, _ := getSecurityLevel(tc.testNetwork, tc.testAddr)
+		if got != tc.want {
+			t.Fatalf("GetSeurityLevel(%s, %s) returned %s but want %s", tc.testNetwork, tc.testAddr, got.String(), tc.want.String())
+		}
+	}
+}
+
+type serverHandshake func(net.Conn) (credentials.AuthInfo, error)
+
+// Server local handshake implementation.
+func serverLocalHandshake(conn net.Conn) (credentials.AuthInfo, error) {
+	cred := NewCredentials()
+	_, authInfo, err := cred.ServerHandshake(conn)
+	if err != nil {
+		return nil, err
+	}
+	return authInfo, nil
+}
+
+// Client local handshake implementation.
+func clientLocalHandshake(conn net.Conn, lisAddr string) (credentials.AuthInfo, error) {
+	cred := NewCredentials()
+	_, authInfo, err := cred.ClientHandshake(context.Background(), lisAddr, conn)
+	if err != nil {
+		return nil, err
+	}
+	return authInfo, nil
+}
+
+// Client connects to a server with local credentials.
+func clientHandle(hs func(net.Conn, string) (credentials.AuthInfo, error), network, lisAddr string) (credentials.AuthInfo, error) {
+	conn, _ := net.Dial(network, lisAddr)
+	defer conn.Close()
+	clientAuthInfo, err := hs(conn, lisAddr)
+	if err != nil {
+		return nil, fmt.Errorf("Error on client while handshake")
+	}
+	return clientAuthInfo, nil
+}
+
+type testServerHandleResult struct {
+	authInfo credentials.AuthInfo
+	err      error
+}
+
+// Server accepts a client's connection with local credentials.
+func serverHandle(hs serverHandshake, done chan testServerHandleResult, lis net.Listener) {
+	serverRawConn, err := lis.Accept()
+	if err != nil {
+		done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed to accept connection. Error: %v", err)}
+	}
+	serverAuthInfo, err := hs(serverRawConn)
+	if err != nil {
+		serverRawConn.Close()
+		done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed while handshake. Error: %v", err)}
+	}
+	done <- testServerHandleResult{authInfo: serverAuthInfo, err: nil}
+}
+
+func serverAndClientHandshake(lis net.Listener) (credentials.SecurityLevel, error) {
+	done := make(chan testServerHandleResult, 1)
+	const timeout = 5 * time.Second
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+	go serverHandle(serverLocalHandshake, done, lis)
+	defer lis.Close()
+	clientAuthInfo, err := clientHandle(clientLocalHandshake, lis.Addr().Network(), lis.Addr().String())
+	if err != nil {
+		return credentials.Invalid, fmt.Errorf("Error at client-side: %v", err)
+	}
+	select {
+	case <-timer.C:
+		return credentials.Invalid, fmt.Errorf("Test didn't finish in time")
+	case serverHandleResult := <-done:
+		if serverHandleResult.err != nil {
+			return credentials.Invalid, fmt.Errorf("Error at server-side: %v", serverHandleResult.err)
+		}
+		clientLocal, _ := clientAuthInfo.(Info)
+		serverLocal, _ := serverHandleResult.authInfo.(Info)
+		clientSecLevel := clientLocal.CommonAuthInfo.SecurityLevel
+		serverSecLevel := serverLocal.CommonAuthInfo.SecurityLevel
+		if clientSecLevel != serverSecLevel {
+			return credentials.Invalid, fmt.Errorf("client's AuthInfo contains %s but server's AuthInfo contains %s", clientSecLevel.String(), serverSecLevel.String())
+		}
+		return clientSecLevel, nil
+	}
+}
+
+func (s) TestServerAndClientHandshake(t *testing.T) {
+	testCases := []struct {
+		testNetwork string
+		testAddr    string
+		want        credentials.SecurityLevel
+	}{
+		{
+			testNetwork: "tcp",
+			testAddr:    "127.0.0.1:10000",
+			want:        credentials.NoSecurity,
+		},
+		{
+			testNetwork: "tcp",
+			testAddr:    "[::1]:10000",
+			want:        credentials.NoSecurity,
+		},
+		{
+			testNetwork: "tcp",
+			testAddr:    "localhost:10000",
+			want:        credentials.NoSecurity,
+		},
+		{
+			testNetwork: "unix",
+			testAddr:    fmt.Sprintf("/tmp/grpc_fullstck_test%d", time.Now().UnixNano()),
+			want:        credentials.PrivacyAndIntegrity,
+		},
+	}
+	for _, tc := range testCases {
+		if runtime.GOOS == "windows" && tc.testNetwork == "unix" {
+			t.Skip("skipping tests for unix connections on Windows")
+		}
+		t.Run("serverAndClientHandshakeResult", func(t *testing.T) {
+			lis, err := net.Listen(tc.testNetwork, tc.testAddr)
+			if err != nil {
+				if strings.Contains(err.Error(), "bind: cannot assign requested address") ||
+					strings.Contains(err.Error(), "socket: address family not supported by protocol") {
+					t.Skipf("no support for address %v", tc.testAddr)
+				}
+				t.Fatalf("Failed to listen: %v", err)
+			}
+			got, err := serverAndClientHandshake(lis)
+			if got != tc.want {
+				t.Fatalf("serverAndClientHandshake(%s, %s) = %v, %v; want %v, nil", tc.testNetwork, tc.testAddr, got, err, tc.want)
+			}
+		})
+	}
+}
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index d1eb17e068fe..b4e55d0101ba 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -221,12 +221,14 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		// address specific arbitrary data to reach the credential handshaker.
 		contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
 		connectCtx = contextWithHandshakeInfo(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
-		scheme = "https"
 		conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.ServerName, conn)
 		if err != nil {
 			return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err)
 		}
 		isSecure = true
+		if transportCreds.Info().SecurityProtocol == "tls" {
+			scheme = "https"
+		}
 	}
 	dynamicWindow := true
 	icwz := int32(initialWindowSize)
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
new file mode 100644
index 000000000000..b55b73bdcbce
--- /dev/null
+++ b/test/local_creds_test.go
@@ -0,0 +1,217 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/local"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
+
+func testLocalCredsE2ESucceed(network, address string) error {
+	ss := &stubServer{
+		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			pr, ok := peer.FromContext(ctx)
+			if !ok {
+				return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
+			}
+			// Check security level
+			info := pr.AuthInfo.(local.Info)
+			secLevel := info.CommonAuthInfo.SecurityLevel
+			switch network {
+			case "unix":
+				if secLevel != credentials.PrivacyAndIntegrity {
+					return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.PrivacyAndIntegrity)
+				}
+			case "tcp":
+				if secLevel != credentials.NoSecurity {
+					return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity)
+				}
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+
+	sopts := []grpc.ServerOption{grpc.Creds(local.NewCredentials())}
+	s := grpc.NewServer(sopts...)
+	defer s.Stop()
+
+	testpb.RegisterTestServiceServer(s, ss)
+
+	lis, err := net.Listen(network, address)
+	if err != nil {
+		return fmt.Errorf("Failed to create listener: %v", err)
+	}
+
+	go s.Serve(lis)
+
+	var cc *grpc.ClientConn
+	lisAddr := lis.Addr().String()
+
+	switch network {
+	case "unix":
+		cc, err = grpc.Dial(lisAddr, grpc.WithTransportCredentials(local.NewCredentials()), grpc.WithContextDialer(
+			func(ctx context.Context, addr string) (net.Conn, error) {
+				return net.Dial("unix", addr)
+			}))
+	case "tcp":
+		cc, err = grpc.Dial(lisAddr, grpc.WithTransportCredentials(local.NewCredentials()))
+	default:
+		return fmt.Errorf("unsupported network %q", network)
+	}
+	if err != nil {
+		return fmt.Errorf("Failed to dial server: %v, %v", err, lisAddr)
+	}
+	defer cc.Close()
+
+	c := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		return fmt.Errorf("EmptyCall(_, _) = _, %v; want _, <nil>", err)
+	}
+	return nil
+}
+
+func (s) TestLocalCredsLocalhost(t *testing.T) {
+	if err := testLocalCredsE2ESucceed("tcp", "localhost:0"); err != nil {
+		t.Fatalf("Failed e2e test for localhost: %v", err)
+	}
+}
+
+func (s) TestLocalCredsUDS(t *testing.T) {
+	addr := fmt.Sprintf("/tmp/grpc_fullstck_test%d", time.Now().UnixNano())
+	if err := testLocalCredsE2ESucceed("unix", addr); err != nil {
+		t.Fatalf("Failed e2e test for UDS: %v", err)
+	}
+}
+
+type connWrapper struct {
+	net.Conn
+	remote net.Addr
+}
+
+func (c connWrapper) RemoteAddr() net.Addr {
+	return c.remote
+}
+
+type lisWrapper struct {
+	net.Listener
+	remote net.Addr
+}
+
+func spoofListener(l net.Listener, remote net.Addr) net.Listener {
+	return &lisWrapper{l, remote}
+}
+
+func (l *lisWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	return connWrapper{c, l.remote}, nil
+}
+
+func spoofDialer(addr net.Addr) func(target string, t time.Duration) (net.Conn, error) {
+	return func(t string, d time.Duration) (net.Conn, error) {
+		c, err := net.DialTimeout("tcp", t, d)
+		if err != nil {
+			return nil, err
+		}
+		return connWrapper{c, addr}, nil
+	}
+}
+
+func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
+	ss := &stubServer{
+		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+
+	sopts := []grpc.ServerOption{grpc.Creds(local.NewCredentials())}
+	s := grpc.NewServer(sopts...)
+	defer s.Stop()
+
+	testpb.RegisterTestServiceServer(s, ss)
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return fmt.Errorf("Failed to create listener: %v", err)
+	}
+
+	var fakeClientAddr, fakeServerAddr net.Addr
+	fakeClientAddr = &net.IPAddr{
+		IP:   net.ParseIP("10.8.9.10"),
+		Zone: "",
+	}
+	fakeServerAddr = &net.IPAddr{
+		IP:   net.ParseIP("10.8.9.11"),
+		Zone: "",
+	}
+
+	go s.Serve(spoofListener(lis, fakeClientAddr))
+
+	cc, err := grpc.Dial(lis.Addr().String(), append(dopts, grpc.WithDialer(spoofDialer(fakeServerAddr)))...)
+	if err != nil {
+		return fmt.Errorf("Failed to dial server: %v, %v", err, lis.Addr().String())
+	}
+	defer cc.Close()
+
+	c := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	_, err = c.EmptyCall(ctx, &testpb.Empty{})
+	return err
+}
+
+func isExpected(got, want error) bool {
+	return status.Code(got) == status.Code(want) && strings.Contains(status.Convert(got).Message(), status.Convert(want).Message())
+}
+
+func (s) TestLocalCredsClientFail(t *testing.T) {
+	// Use local creds at client-side which should lead to client-side failure.
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(local.NewCredentials())}
+	want := status.Error(codes.Unavailable, "transport: authentication handshake failed: local credentials rejected connection to non-local address")
+	if err := testLocalCredsE2EFail(opts); !isExpected(err, want) {
+		t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, want)
+	}
+}
+
+func (s) TestLocalCredsServerFail(t *testing.T) {
+	// Use insecure at client-side which should lead to server-side failure.
+	opts := []grpc.DialOption{grpc.WithInsecure()}
+	if err := testLocalCredsE2EFail(opts); status.Code(err) != codes.Unavailable {
+		t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, codes.Unavailable)
+	}
+}

From 8bd46590eb550787035bbfe02f8c637deac934e7 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 20 May 2020 17:41:03 -0700
Subject: [PATCH 070/481] xds: not cancel and restart RDS watch on the same LDS
 update (#3619)

---
 .../client/client_watchers_service.go         |  7 +++
 .../client/client_watchers_service_test.go    | 46 +++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
index 529de4d41677..7ba5e8435f36 100644
--- a/xds/internal/client/client_watchers_service.go
+++ b/xds/internal/client/client_watchers_service.go
@@ -62,6 +62,7 @@ type serviceUpdateWatcher struct {
 
 	mu        sync.Mutex
 	closed    bool
+	rdsName   string
 	rdsCancel func()
 }
 
@@ -82,6 +83,12 @@ func (w *serviceUpdateWatcher) handleLDSResp(update ldsUpdate, err error) {
 		return
 	}
 
+	if w.rdsName == update.routeName {
+		// If the new routeName is same as the previous, don't cancel and
+		// restart the RDS watch.
+		return
+	}
+	w.rdsName = update.routeName
 	if w.rdsCancel != nil {
 		w.rdsCancel()
 	}
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 004d97c9a164..d66458a4e2ae 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -337,3 +337,49 @@ func (s) TestServiceWatchWithClientClose(t *testing.T) {
 		t.Fatal(cbErr)
 	}
 }
+
+// TestServiceNotCancelRDSOnSameLDSUpdate covers the case that if the second LDS
+// update contains the same RDS name as the previous, the RDS watch isn't
+// canceled and restarted.
+func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+
+	<-v2Client.addWatches[ldsURL]
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	<-v2Client.addWatches[rdsURL]
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {clusterName: testCDSName},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Another LDS update with a the same RDS_name.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	select {
+	case <-v2Client.removeWatches[rdsURL]:
+		t.Fatalf("unexpected rds watch cancel")
+	case <-time.After(time.Second):
+	}
+}

From 5c35263ba8e53a27973205b9caed2116a9a9e028 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 21 May 2020 13:20:06 -0700
Subject: [PATCH 071/481] internal: update rls.pb.go (#3637)

---
 .../internal/proto/grpc_lookup_v1/rls.pb.go   | 60 +++++++------------
 1 file changed, 23 insertions(+), 37 deletions(-)

diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index 57cb1c93ba50..a3a3f8118ce9 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -95,15 +95,9 @@ func (m *RouteLookupRequest) GetKeyMap() map[string]string {
 }
 
 type RouteLookupResponse struct {
-	// Actual addressable entity to use for routing decision, using syntax
-	// requested by the request target_type.
-	// This field is deprecated in favor of the new "targets" field, below.
-	Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Deprecated: Do not use.
 	// Prioritized list (best one first) of addressable entities to use
 	// for routing, using syntax requested by the request target_type.
 	// The targets will be tried in order until a healthy one is found.
-	// If present, it should be used by proxy/gRPC client code instead of
-	// "target" (which is deprecated).
 	Targets []string `protobuf:"bytes,3,rep,name=targets,proto3" json:"targets,omitempty"`
 	// Optional header value to pass along to AFE in the X-Google-RLS-Data header.
 	// Cached with "target" and sent with all requests that match the request key.
@@ -139,14 +133,6 @@ func (m *RouteLookupResponse) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_RouteLookupResponse proto.InternalMessageInfo
 
-// Deprecated: Do not use.
-func (m *RouteLookupResponse) GetTarget() string {
-	if m != nil {
-		return m.Target
-	}
-	return ""
-}
-
 func (m *RouteLookupResponse) GetTargets() []string {
 	if m != nil {
 		return m.Targets
@@ -170,29 +156,29 @@ func init() {
 func init() { proto.RegisterFile("grpc/lookup/v1/rls.proto", fileDescriptor_3bab962d3362f3ca) }
 
 var fileDescriptor_3bab962d3362f3ca = []byte{
-	// 343 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x4b, 0xeb, 0x40,
-	0x10, 0xc6, 0x5f, 0x9a, 0xbe, 0xf4, 0x75, 0x02, 0x8f, 0xf7, 0x56, 0x91, 0xa5, 0x17, 0x4b, 0xbc,
-	0xf4, 0x20, 0x1b, 0x5a, 0x2f, 0xea, 0xb1, 0x28, 0x1e, 0xb4, 0x50, 0xa2, 0x07, 0xf1, 0x12, 0xd6,
-	0x76, 0x48, 0x4b, 0xd6, 0xec, 0xba, 0xbb, 0x09, 0xe4, 0x0f, 0xf6, 0xff, 0x90, 0x64, 0x23, 0xb4,
-	0x0a, 0x7a, 0xfb, 0xbe, 0xdf, 0x2c, 0x99, 0x6f, 0x26, 0x03, 0x34, 0xd3, 0x6a, 0x15, 0x0b, 0x29,
-	0xf3, 0x52, 0xc5, 0xd5, 0x34, 0xd6, 0xc2, 0x30, 0xa5, 0xa5, 0x95, 0xe4, 0x6f, 0x53, 0x61, 0xae,
-	0xc2, 0xaa, 0x69, 0xf4, 0xe6, 0x01, 0x49, 0x64, 0x69, 0xf1, 0xae, 0x45, 0x09, 0xbe, 0x96, 0x68,
-	0x2c, 0x39, 0x82, 0xc0, 0xa0, 0xae, 0x50, 0x53, 0x6f, 0xec, 0x4d, 0x86, 0x49, 0xe7, 0x08, 0x81,
-	0xbe, 0xe2, 0x76, 0x43, 0x7b, 0x2d, 0x6d, 0x35, 0x39, 0x86, 0xd0, 0x72, 0x9d, 0xa1, 0x4d, 0x6d,
-	0xad, 0x90, 0xfa, 0x6d, 0x09, 0x1c, 0x7a, 0xa8, 0x15, 0x92, 0x1b, 0x18, 0xe4, 0x58, 0xa7, 0x2f,
-	0x5c, 0xd1, 0xfe, 0xd8, 0x9f, 0x84, 0x33, 0xc6, 0xf6, 0x53, 0xb0, 0xaf, 0x09, 0xd8, 0x2d, 0xd6,
-	0x0b, 0xae, 0xae, 0x0b, 0xab, 0xeb, 0x24, 0xc8, 0x5b, 0x33, 0xba, 0x80, 0x70, 0x07, 0x93, 0x7f,
-	0xe0, 0xe7, 0x58, 0x77, 0x09, 0x1b, 0x49, 0x0e, 0xe1, 0x77, 0xc5, 0x45, 0x89, 0x5d, 0x3e, 0x67,
-	0x2e, 0x7b, 0xe7, 0x5e, 0x24, 0xe0, 0x60, 0xaf, 0x89, 0x51, 0xb2, 0x30, 0x48, 0x46, 0x10, 0xb8,
-	0xa0, 0xee, 0x2b, 0xf3, 0x1e, 0xf5, 0x92, 0x8e, 0x10, 0x0a, 0x03, 0xa7, 0x0c, 0xf5, 0xc7, 0xfe,
-	0x64, 0x98, 0x7c, 0xd8, 0x66, 0xe2, 0x0d, 0xf2, 0x35, 0xea, 0x74, 0xcd, 0x2d, 0xef, 0x9a, 0x81,
-	0x43, 0x57, 0xdc, 0xf2, 0x59, 0xb1, 0xb7, 0xd4, 0x7b, 0xd4, 0xd5, 0x76, 0x85, 0xe4, 0x11, 0xc2,
-	0x1d, 0x4a, 0xa2, 0x9f, 0xb7, 0x30, 0x3a, 0xf9, 0xf6, 0x8d, 0x1b, 0x22, 0xfa, 0x35, 0x5f, 0xc0,
-	0xff, 0xad, 0xfc, 0xf4, 0x74, 0xfe, 0x27, 0x11, 0x66, 0xd9, 0xfc, 0xf4, 0xa5, 0xf7, 0x74, 0x9a,
-	0x49, 0x99, 0x09, 0x64, 0x99, 0x14, 0xbc, 0xc8, 0x98, 0xd4, 0x59, 0xbc, 0x7b, 0x22, 0x8d, 0x4e,
-	0x9d, 0x4e, 0xab, 0xe9, 0x73, 0xd0, 0xde, 0xca, 0xd9, 0x7b, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf5,
-	0x11, 0x43, 0x0e, 0x47, 0x02, 0x00, 0x00,
+	// 341 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xd1, 0x4b, 0xeb, 0x30,
+	0x14, 0xc6, 0x6f, 0xd7, 0xdd, 0x6e, 0x3b, 0x85, 0xcb, 0x6e, 0xee, 0x45, 0xca, 0x5e, 0x1c, 0xf5,
+	0x65, 0x0f, 0x92, 0xb2, 0xf9, 0xa2, 0x3e, 0x0e, 0x45, 0x50, 0x07, 0xa3, 0xfa, 0x20, 0x3e, 0x58,
+	0xe2, 0x76, 0xc8, 0x46, 0x6b, 0x13, 0x93, 0xb4, 0xd0, 0x3f, 0xd8, 0xff, 0x43, 0xda, 0x54, 0xd8,
+	0x14, 0xf4, 0xed, 0xfb, 0xbe, 0x73, 0x48, 0x7e, 0x27, 0x39, 0x10, 0x70, 0x25, 0x57, 0x51, 0x26,
+	0x44, 0x5a, 0xc8, 0xa8, 0x9c, 0x46, 0x2a, 0xd3, 0x54, 0x2a, 0x61, 0x04, 0xf9, 0x53, 0x57, 0xa8,
+	0xad, 0xd0, 0x72, 0x1a, 0xbe, 0x39, 0x40, 0x62, 0x51, 0x18, 0xbc, 0x6d, 0xa2, 0x18, 0x5f, 0x0b,
+	0xd4, 0x86, 0x1c, 0x80, 0xa7, 0x51, 0x95, 0xa8, 0x02, 0x67, 0xec, 0x4c, 0x06, 0x71, 0xeb, 0x08,
+	0x81, 0xae, 0x64, 0x66, 0x13, 0x74, 0x9a, 0xb4, 0xd1, 0xe4, 0x10, 0x7c, 0xc3, 0x14, 0x47, 0x93,
+	0x98, 0x4a, 0x62, 0xe0, 0x36, 0x25, 0xb0, 0xd1, 0x7d, 0x25, 0x91, 0x5c, 0x41, 0x2f, 0xc5, 0x2a,
+	0x79, 0x61, 0x32, 0xe8, 0x8e, 0xdd, 0x89, 0x3f, 0xa3, 0x74, 0x9f, 0x82, 0x7e, 0x25, 0xa0, 0x37,
+	0x58, 0x2d, 0x98, 0xbc, 0xcc, 0x8d, 0xaa, 0x62, 0x2f, 0x6d, 0xcc, 0xe8, 0x0c, 0xfc, 0x9d, 0x98,
+	0x0c, 0xc1, 0x4d, 0xb1, 0x6a, 0x09, 0x6b, 0x49, 0xfe, 0xc3, 0xef, 0x92, 0x65, 0x05, 0xb6, 0x7c,
+	0xd6, 0x9c, 0x77, 0x4e, 0x9d, 0xf0, 0x09, 0xfe, 0xed, 0x5d, 0xa2, 0xa5, 0xc8, 0x35, 0x92, 0x00,
+	0x7a, 0x16, 0x54, 0x07, 0xee, 0xd8, 0x9d, 0x0c, 0xe2, 0x0f, 0x5b, 0x4f, 0xb5, 0x41, 0xb6, 0x46,
+	0x95, 0xac, 0x99, 0x61, 0xed, 0x81, 0x60, 0xa3, 0x0b, 0x66, 0xd8, 0x75, 0xb7, 0xef, 0x0c, 0x3b,
+	0xb1, 0x67, 0xfb, 0x67, 0xf9, 0xde, 0x33, 0xde, 0xa1, 0x2a, 0xb7, 0x2b, 0x24, 0x0f, 0xe0, 0xef,
+	0xa4, 0x24, 0xfc, 0x79, 0xee, 0xd1, 0xd1, 0xb7, 0x3d, 0x16, 0x3b, 0xfc, 0x35, 0x5f, 0xc0, 0xdf,
+	0xad, 0xf8, 0xd4, 0x3a, 0xef, 0xc7, 0x99, 0x5e, 0xd6, 0xdf, 0xbc, 0x74, 0x1e, 0x8f, 0xb9, 0x10,
+	0x3c, 0x43, 0xca, 0x45, 0xc6, 0x72, 0x4e, 0x85, 0xe2, 0xd1, 0xee, 0x52, 0xd4, 0x3a, 0xb1, 0x3a,
+	0x29, 0xa7, 0xcf, 0x5e, 0xb3, 0x1d, 0x27, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0xca, 0x8d, 0x5c,
+	0xc7, 0x39, 0x02, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.

From d6e95de1409e5da6c1d3744a86673282709984d1 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Fri, 22 May 2020 04:45:46 +0800
Subject: [PATCH 072/481] xds: rename subBalancerWithConfig to
 subBalancerWrapper (#3608)

---
 .../balancer/balancergroup/balancergroup.go   | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index b08a5864e25b..0aa30a2663fc 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -35,16 +35,16 @@ import (
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
 )
 
-// subBalancerWithConfig is used to keep the configurations that will be used to start
+// subBalancerWrapper is used to keep the configurations that will be used to start
 // the underlying balancer. It can be called to start/stop the underlying
 // balancer.
 //
 // When the config changes, it will pass the update to the underlying balancer
 // if it exists.
 //
-// TODO: rename to subBalanceWrapper (and move to a separate file?)
-type subBalancerWithConfig struct {
-	// subBalancerWithConfig is passed to the sub-balancer as a ClientConn
+// TODO: move to a separate file?
+type subBalancerWrapper struct {
+	// subBalancerWrapper is passed to the sub-balancer as a ClientConn
 	// wrapper, only to keep the state and picker.  When sub-balancer is
 	// restarted while in cache, the picker needs to be resent.
 	//
@@ -74,7 +74,7 @@ type subBalancerWithConfig struct {
 }
 
 // UpdateState overrides balancer.ClientConn, to keep state and picker.
-func (sbc *subBalancerWithConfig) UpdateState(state balancer.State) {
+func (sbc *subBalancerWrapper) UpdateState(state balancer.State) {
 	sbc.mu.Lock()
 	sbc.state = state
 	sbc.group.updateBalancerState(sbc.id, state)
@@ -83,11 +83,11 @@ func (sbc *subBalancerWithConfig) UpdateState(state balancer.State) {
 
 // NewSubConn overrides balancer.ClientConn, so balancer group can keep track of
 // the relation between subconns and sub-balancers.
-func (sbc *subBalancerWithConfig) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+func (sbc *subBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
 	return sbc.group.newSubConn(sbc, addrs, opts)
 }
 
-func (sbc *subBalancerWithConfig) updateBalancerStateWithCachedPicker() {
+func (sbc *subBalancerWrapper) updateBalancerStateWithCachedPicker() {
 	sbc.mu.Lock()
 	if sbc.state.Picker != nil {
 		sbc.group.updateBalancerState(sbc.id, sbc.state)
@@ -95,7 +95,7 @@ func (sbc *subBalancerWithConfig) updateBalancerStateWithCachedPicker() {
 	sbc.mu.Unlock()
 }
 
-func (sbc *subBalancerWithConfig) startBalancer() {
+func (sbc *subBalancerWrapper) startBalancer() {
 	b := sbc.builder.Build(sbc, balancer.BuildOptions{})
 	sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name())
 	sbc.balancer = b
@@ -104,7 +104,7 @@ func (sbc *subBalancerWithConfig) startBalancer() {
 	}
 }
 
-func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+func (sbc *subBalancerWrapper) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	b := sbc.balancer
 	if b == nil {
 		// This sub-balancer was closed. This can happen when EDS removes a
@@ -116,7 +116,7 @@ func (sbc *subBalancerWithConfig) updateSubConnState(sc balancer.SubConn, state
 	b.UpdateSubConnState(sc, state)
 }
 
-func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnState) error {
+func (sbc *subBalancerWrapper) updateClientConnState(s balancer.ClientConnState) error {
 	sbc.ccState = &s
 	b := sbc.balancer
 	if b == nil {
@@ -133,7 +133,7 @@ func (sbc *subBalancerWithConfig) updateClientConnState(s balancer.ClientConnSta
 	return b.UpdateClientConnState(s)
 }
 
-func (sbc *subBalancerWithConfig) resolverError(err error) {
+func (sbc *subBalancerWrapper) resolverError(err error) {
 	b := sbc.balancer
 	if b == nil {
 		// This sub-balancer was closed. This should never happen because
@@ -149,7 +149,7 @@ func (sbc *subBalancerWithConfig) resolverError(err error) {
 	b.ResolverError(err)
 }
 
-func (sbc *subBalancerWithConfig) stopBalancer() {
+func (sbc *subBalancerWrapper) stopBalancer() {
 	sbc.balancer.Close()
 	sbc.balancer = nil
 }
@@ -206,7 +206,7 @@ type BalancerGroup struct {
 	// to sub-balancers after they are closed.
 	outgoingMu         sync.Mutex
 	outgoingStarted    bool
-	idToBalancerConfig map[internal.LocalityID]*subBalancerWithConfig
+	idToBalancerConfig map[internal.LocalityID]*subBalancerWrapper
 	// Cache for sub-balancers when they are removed.
 	balancerCache *cache.TimeoutCache
 
@@ -233,7 +233,7 @@ type BalancerGroup struct {
 	// from sub-balancers after they are closed.
 	incomingMu      sync.Mutex
 	incomingStarted bool // This boolean only guards calls back to ClientConn.
-	scToSubBalancer map[balancer.SubConn]*subBalancerWithConfig
+	scToSubBalancer map[balancer.SubConn]*subBalancerWrapper
 	// All balancer IDs exist as keys in this map, even if balancer group is not
 	// started.
 	//
@@ -255,9 +255,9 @@ func New(cc balancer.ClientConn, loadStore lrs.Store, logger *grpclog.PrefixLogg
 		logger:    logger,
 		loadStore: loadStore,
 
-		idToBalancerConfig: make(map[internal.LocalityID]*subBalancerWithConfig),
+		idToBalancerConfig: make(map[internal.LocalityID]*subBalancerWrapper),
 		balancerCache:      cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout),
-		scToSubBalancer:    make(map[balancer.SubConn]*subBalancerWithConfig),
+		scToSubBalancer:    make(map[balancer.SubConn]*subBalancerWrapper),
 		idToPickerState:    make(map[internal.LocalityID]*pickerState),
 	}
 }
@@ -313,12 +313,12 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, weight uint32, builder bala
 
 	// Store data in static map, and then check to see if bg is started.
 	bg.outgoingMu.Lock()
-	var sbc *subBalancerWithConfig
+	var sbc *subBalancerWrapper
 	// If outgoingStarted is true, search in the cache. Otherwise, cache is
 	// guaranteed to be empty, searching is unnecessary.
 	if bg.outgoingStarted {
 		if old, ok := bg.balancerCache.Remove(id); ok {
-			sbc, _ = old.(*subBalancerWithConfig)
+			sbc, _ = old.(*subBalancerWrapper)
 			if sbc != nil && sbc.builder != builder {
 				// If the sub-balancer in cache was built with a different
 				// balancer builder, don't use it, cleanup this old-balancer,
@@ -336,7 +336,7 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, weight uint32, builder bala
 		}
 	}
 	if sbc == nil {
-		sbc = &subBalancerWithConfig{
+		sbc = &subBalancerWrapper{
 			ClientConn: bg.cc,
 			id:         id,
 			group:      bg,
@@ -401,7 +401,7 @@ func (bg *BalancerGroup) Remove(id internal.LocalityID) {
 
 // bg.remove(id) doesn't do cleanup for the sub-balancer. This function does
 // cleanup after the timeout.
-func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWithConfig) {
+func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWrapper) {
 	bg.incomingMu.Lock()
 	// Remove SubConns. This is only done after the balancer is
 	// actually closed.
@@ -501,7 +501,7 @@ func (bg *BalancerGroup) ResolverError(err error) {
 // from map. Delete sc from the map only when state changes to Shutdown. Since
 // it's just forwarding the action, there's no need for a removeSubConn()
 // wrapper function.
-func (bg *BalancerGroup) newSubConn(config *subBalancerWithConfig, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+func (bg *BalancerGroup) newSubConn(config *subBalancerWrapper, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
 	// NOTE: if balancer with id was already removed, this should also return
 	// error. But since we call balancer.stopBalancer when removing the balancer, this
 	// shouldn't happen.

From 5e50693410e514a28e2880d2690375056335d137 Mon Sep 17 00:00:00 2001
From: Ryan Kim <Ryanfsdf@users.noreply.github.com>
Date: Thu, 21 May 2020 20:56:55 +0000
Subject: [PATCH 073/481] credentials/alts: Fix ALTS record crypto interface
 comments (#3638)

---
 credentials/alts/internal/conn/record.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/credentials/alts/internal/conn/record.go b/credentials/alts/internal/conn/record.go
index 8a872c3cffd0..0d64fb37a125 100644
--- a/credentials/alts/internal/conn/record.go
+++ b/credentials/alts/internal/conn/record.go
@@ -32,7 +32,7 @@ import (
 // ALTSRecordCrypto is the interface for gRPC ALTS record protocol.
 type ALTSRecordCrypto interface {
 	// Encrypt encrypts the plaintext and computes the tag (if any) of dst
-	// and plaintext, dst and plaintext do not overlap.
+	// and plaintext. dst and plaintext may fully overlap or not at all.
 	Encrypt(dst, plaintext []byte) ([]byte, error)
 	// EncryptionOverhead returns the tag size (if any) in bytes.
 	EncryptionOverhead() int

From 4709b05f2c64bbeabdd296e47e3d6929399a501b Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Thu, 21 May 2020 22:21:54 +0100
Subject: [PATCH 074/481] transport: move append of header and data down to
 http2 write loop to save garbage (#3568)

---
 internal/transport/controlbuf.go   | 64 +++++++++++++++++-------------
 internal/transport/http2_client.go | 14 ++-----
 internal/transport/http2_server.go |  7 ----
 3 files changed, 40 insertions(+), 45 deletions(-)

diff --git a/internal/transport/controlbuf.go b/internal/transport/controlbuf.go
index ddee20b6bef2..d4bb19c3bb53 100644
--- a/internal/transport/controlbuf.go
+++ b/internal/transport/controlbuf.go
@@ -857,38 +857,45 @@ func (l *loopyWriter) processData() (bool, error) {
 		return false, nil
 	}
 	var (
-		idx int
 		buf []byte
 	)
-	if len(dataItem.h) != 0 { // data header has not been written out yet.
-		buf = dataItem.h
-	} else {
-		idx = 1
-		buf = dataItem.d
-	}
-	size := http2MaxFrameLen
-	if len(buf) < size {
-		size = len(buf)
-	}
+	// Figure out the maximum size we can send
+	maxSize := http2MaxFrameLen
 	if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota <= 0 { // stream-level flow control.
 		str.state = waitingOnStreamQuota
 		return false, nil
-	} else if strQuota < size {
-		size = strQuota
+	} else if maxSize > strQuota {
+		maxSize = strQuota
+	}
+	if maxSize > int(l.sendQuota) { // connection-level flow control.
+		maxSize = int(l.sendQuota)
+	}
+	// Compute how much of the header and data we can send within quota and max frame length
+	hSize := min(maxSize, len(dataItem.h))
+	dSize := min(maxSize-hSize, len(dataItem.d))
+	if hSize != 0 {
+		if dSize == 0 {
+			buf = dataItem.h
+		} else {
+			// We can add some data to grpc message header to distribute bytes more equally across frames.
+			// Copy on the stack to avoid generating garbage
+			var localBuf [http2MaxFrameLen]byte
+			copy(localBuf[:hSize], dataItem.h)
+			copy(localBuf[hSize:], dataItem.d[:dSize])
+			buf = localBuf[:hSize+dSize]
+		}
+	} else {
+		buf = dataItem.d
 	}
 
-	if l.sendQuota < uint32(size) { // connection-level flow control.
-		size = int(l.sendQuota)
-	}
+	size := hSize + dSize
+
 	// Now that outgoing flow controls are checked we can replenish str's write quota
 	str.wq.replenish(size)
 	var endStream bool
 	// If this is the last data message on this stream and all of it can be written in this iteration.
-	if dataItem.endStream && size == len(buf) {
-		// buf contains either data or it contains header but data is empty.
-		if idx == 1 || len(dataItem.d) == 0 {
-			endStream = true
-		}
+	if dataItem.endStream && len(dataItem.h)+len(dataItem.d) <= size {
+		endStream = true
 	}
 	if dataItem.onEachWrite != nil {
 		dataItem.onEachWrite()
@@ -896,14 +903,10 @@ func (l *loopyWriter) processData() (bool, error) {
 	if err := l.framer.fr.WriteData(dataItem.streamID, endStream, buf[:size]); err != nil {
 		return false, err
 	}
-	buf = buf[size:]
 	str.bytesOutStanding += size
 	l.sendQuota -= uint32(size)
-	if idx == 0 {
-		dataItem.h = buf
-	} else {
-		dataItem.d = buf
-	}
+	dataItem.h = dataItem.h[hSize:]
+	dataItem.d = dataItem.d[dSize:]
 
 	if len(dataItem.h) == 0 && len(dataItem.d) == 0 { // All the data from that message was written out.
 		str.itl.dequeue()
@@ -924,3 +927,10 @@ func (l *loopyWriter) processData() (bool, error) {
 	}
 	return false, nil
 }
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index b4e55d0101ba..28b0c1662415 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -863,18 +863,10 @@ func (t *http2Client) Write(s *Stream, hdr []byte, data []byte, opts *Options) e
 	df := &dataFrame{
 		streamID:  s.id,
 		endStream: opts.Last,
+		h:         hdr,
+		d:         data,
 	}
-	if hdr != nil || data != nil { // If it's not an empty data frame.
-		// Add some data to grpc message header so that we can equally
-		// distribute bytes across frames.
-		emptyLen := http2MaxFrameLen - len(hdr)
-		if emptyLen > len(data) {
-			emptyLen = len(data)
-		}
-		hdr = append(hdr, data[:emptyLen]...)
-		data = data[emptyLen:]
-		df.h, df.d = hdr, data
-		// TODO(mmukhi): The above logic in this if can be moved to loopyWriter's data handler.
+	if hdr != nil || data != nil { // If it's not an empty data frame, check quota.
 		if err := s.wq.get(int32(len(hdr) + len(data))); err != nil {
 			return err
 		}
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index fa33ffb18856..e8c757321287 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -909,13 +909,6 @@ func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) e
 			return ContextErr(s.ctx.Err())
 		}
 	}
-	// Add some data to header frame so that we can equally distribute bytes across frames.
-	emptyLen := http2MaxFrameLen - len(hdr)
-	if emptyLen > len(data) {
-		emptyLen = len(data)
-	}
-	hdr = append(hdr, data[:emptyLen]...)
-	data = data[emptyLen:]
 	df := &dataFrame{
 		streamID:    s.id,
 		h:           hdr,

From d071d568340c9f0a6b80a967a4d36ea3b394d526 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 26 May 2020 13:58:04 -0700
Subject: [PATCH 075/481] xds: handle weighted cluster as route action (#3613)

---
 internal/internal.go                          |   6 +
 service_config.go                             |  32 ++++
 xds/internal/balancer/balancer.go             |   5 +-
 xds/internal/client/client_watchers_rds.go    |   2 +-
 .../client/client_watchers_rds_test.go        |  25 +--
 .../client/client_watchers_service.go         |   8 +-
 .../client/client_watchers_service_test.go    |  41 ++--
 xds/internal/client/v2client_rds.go           |  71 +++++--
 xds/internal/client/v2client_rds_test.go      | 176 ++++++++++++++----
 xds/internal/resolver/serviceconfig.go        |  80 ++++++++
 xds/internal/resolver/serviceconfig_test.go   |  92 +++++++++
 xds/internal/resolver/xds_resolver.go         |  17 +-
 xds/internal/resolver/xds_resolver_test.go    |  62 ++++--
 xds/internal/testutils/fakeclient/client.go   |   4 +-
 14 files changed, 499 insertions(+), 122 deletions(-)
 create mode 100644 xds/internal/resolver/serviceconfig.go
 create mode 100644 xds/internal/resolver/serviceconfig_test.go

diff --git a/internal/internal.go b/internal/internal.go
index 883345eee939..818ca857998b 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/serviceconfig"
 )
 
 var (
@@ -46,6 +47,11 @@ var (
 	// ParseServiceConfigForTesting is for creating a fake
 	// ClientConn for resolver testing only
 	ParseServiceConfigForTesting interface{} // func(string) *serviceconfig.ParseResult
+	// EqualServiceConfigForTesting is for testing service config generation and
+	// parsing. Both a and b should be returned by ParseServiceConfigForTesting.
+	// This function compares the config without rawJSON stripped, in case the
+	// there's difference in white space.
+	EqualServiceConfigForTesting func(a, b serviceconfig.Config) bool
 )
 
 // HealthChecker defines the signature of the client-side LB channel health checking function.
diff --git a/service_config.go b/service_config.go
index 37d4a58f1222..3132a66cd68c 100644
--- a/service_config.go
+++ b/service_config.go
@@ -21,6 +21,7 @@ package grpc
 import (
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -400,3 +401,34 @@ func getMaxSize(mcMax, doptMax *int, defaultVal int) *int {
 func newInt(b int) *int {
 	return &b
 }
+
+func init() {
+	internal.EqualServiceConfigForTesting = equalServiceConfig
+}
+
+// equalServiceConfig compares two configs. The rawJSONString field is ignored,
+// because they may diff in white spaces.
+//
+// If any of them is NOT *ServiceConfig, return false.
+func equalServiceConfig(a, b serviceconfig.Config) bool {
+	aa, ok := a.(*ServiceConfig)
+	if !ok {
+		return false
+	}
+	bb, ok := b.(*ServiceConfig)
+	if !ok {
+		return false
+	}
+	aaRaw := aa.rawJSONString
+	aa.rawJSONString = ""
+	bbRaw := bb.rawJSONString
+	bb.rawJSONString = ""
+	defer func() {
+		aa.rawJSONString = aaRaw
+		bb.rawJSONString = bbRaw
+	}()
+	// Using reflect.DeepEqual instead of cmp.Equal because many balancer
+	// configs are unexported, and cmp.Equal cannot compare unexported fields
+	// from unexported structs.
+	return reflect.DeepEqual(aa, bb)
+}
diff --git a/xds/internal/balancer/balancer.go b/xds/internal/balancer/balancer.go
index 51cd45fc447e..489def3ce3bb 100644
--- a/xds/internal/balancer/balancer.go
+++ b/xds/internal/balancer/balancer.go
@@ -20,6 +20,7 @@
 package balancer
 
 import (
-	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // Register the CDS balancer
-	_ "google.golang.org/grpc/xds/internal/balancer/edsbalancer" // Register the EDS balancer
+	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"    // Register the CDS balancer
+	_ "google.golang.org/grpc/xds/internal/balancer/edsbalancer"    // Register the EDS balancer
+	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget" // Register the weighted_target balancer
 )
diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
index b241a350dfe2..06f527921981 100644
--- a/xds/internal/client/client_watchers_rds.go
+++ b/xds/internal/client/client_watchers_rds.go
@@ -23,7 +23,7 @@ import (
 )
 
 type rdsUpdate struct {
-	clusterName string
+	weightedCluster map[string]uint32
 }
 type rdsCallbackFunc func(rdsUpdate, error)
 
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 0b2f91455cbe..74bd3deac13b 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -21,6 +21,7 @@ package client
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -50,12 +51,12 @@ func (s) TestRDSWatch(t *testing.T) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := rdsUpdate{clusterName: testCDSName}
+	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -106,13 +107,13 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 		})
 	}
 
-	wantUpdate := rdsUpdate{clusterName: testCDSName}
+	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: wantUpdate,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -124,7 +125,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -166,20 +167,20 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate1 := rdsUpdate{clusterName: testCDSName + "1"}
-	wantUpdate2 := rdsUpdate{clusterName: testCDSName + "2"}
+	wantUpdate1 := rdsUpdate{weightedCluster: map[string]uint32{testCDSName + "1": 1}}
+	wantUpdate2 := rdsUpdate{weightedCluster: map[string]uint32{testCDSName + "2": 1}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName + "1": wantUpdate1,
 		testRDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || u != (rdsUpdateErr{wantUpdate1, nil}) {
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := rdsUpdateCh2.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate2, nil}) {
+	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -203,12 +204,12 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := rdsUpdate{clusterName: testCDSName}
+	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -219,7 +220,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	})
 
 	// New watch should receives the update.
-	if u, err := rdsUpdateCh2.Receive(); err != nil || u != (rdsUpdateErr{wantUpdate, nil}) {
+	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
 		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
index 7ba5e8435f36..b7fe57d6062b 100644
--- a/xds/internal/client/client_watchers_service.go
+++ b/xds/internal/client/client_watchers_service.go
@@ -25,7 +25,9 @@ import (
 
 // ServiceUpdate contains update about the service.
 type ServiceUpdate struct {
-	Cluster string
+	// WeightedCluster is a map from cluster names (CDS resource to watch) to
+	// their weights.
+	WeightedCluster map[string]uint32
 }
 
 // WatchService uses LDS and RDS to discover information about the provided
@@ -106,7 +108,9 @@ func (w *serviceUpdateWatcher) handleRDSResp(update rdsUpdate, err error) {
 		w.serviceCb(ServiceUpdate{}, err)
 		return
 	}
-	w.serviceCb(ServiceUpdate{Cluster: update.clusterName}, nil)
+	w.serviceCb(ServiceUpdate{
+		WeightedCluster: update.weightedCluster,
+	}, nil)
 }
 
 func (w *serviceUpdateWatcher) close() {
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index d66458a4e2ae..91bb90b60759 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -24,6 +24,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
@@ -54,7 +55,7 @@ func (s) TestServiceWatch(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
 	<-v2Client.addWatches[ldsURL]
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
@@ -62,10 +63,10 @@ func (s) TestServiceWatch(t *testing.T) {
 	})
 	<-v2Client.addWatches[rdsURL]
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {clusterName: testCDSName},
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -90,7 +91,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
 	<-v2Client.addWatches[ldsURL]
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
@@ -98,10 +99,10 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	})
 	<-v2Client.addWatches[rdsURL]
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {clusterName: testCDSName},
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -113,20 +114,20 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	// Another update for the old name.
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {clusterName: testCDSName},
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
-	wantUpdate2 := ServiceUpdate{Cluster: testCDSName + "2"}
+	wantUpdate2 := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName + "2": 1}}
 	// RDS update for the new name.
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName + "2": {clusterName: testCDSName + "2"},
+		testRDSName + "2": {weightedCluster: map[string]uint32{testCDSName + "2": 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate2, nil}) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -151,7 +152,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
 	<-v2Client.addWatches[ldsURL]
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
@@ -159,10 +160,10 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 	})
 	<-v2Client.addWatches[rdsURL]
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {clusterName: testCDSName},
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -177,7 +178,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
 	}
 	uu := u.(serviceUpdateErr)
-	if uu.u != (ServiceUpdate{}) {
+	if !cmp.Equal(uu.u, ServiceUpdate{}) {
 		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
 	}
 	if uu.err == nil {
@@ -190,10 +191,10 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		testLDSName: {routeName: testRDSName},
 	})
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {clusterName: testCDSName},
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -227,8 +228,8 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.Cluster != "" {
-			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want empty clusterName", su.Cluster))
+		if su.WeightedCluster != nil {
+			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.WeightedCluster))
 			return
 		}
 		if err == nil {
@@ -271,8 +272,8 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.Cluster != "" {
-			callbackCh.Send(fmt.Errorf("got clusterName: %+v, want empty clusterName", su.Cluster))
+		if su.WeightedCluster != nil {
+			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.WeightedCluster))
 			return
 		}
 		if err == nil {
diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
index e03284fcb1e0..3650c045c370 100644
--- a/xds/internal/client/v2client_rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -48,26 +48,25 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, v2c.hostname)
 
 		// Use the hostname (resourceName for LDS) to find the routes.
-		cluster, err := getClusterFromRouteConfiguration(rc, hostname)
-		if cluster == "" {
+		u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname)
+		if err != nil {
 			return fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
 		}
-
 		// If we get here, it means that this resource was a good one.
-		returnUpdate[rc.GetName()] = rdsUpdate{clusterName: cluster}
+		returnUpdate[rc.GetName()] = u
 	}
 
 	v2c.parent.newRDSUpdate(returnUpdate)
 	return nil
 }
 
-// getClusterFromRouteConfiguration checks if the provided RouteConfiguration
-// meets the expected criteria. If so, it returns a non-empty clusterName with
-// nil error.
+// generateRDSUpdateFromRouteConfiguration checks if the provided
+// RouteConfiguration meets the expected criteria. If so, it returns a rdsUpdate
+// with nil error.
 //
 // A RouteConfiguration resource is considered valid when only if it contains a
 // VirtualHost whose domain field matches the server name from the URI passed
-// to the gRPC channel, and it contains a clusterName.
+// to the gRPC channel, and it contains a clusterName or a weighted cluster.
 //
 // The RouteConfiguration includes a list of VirtualHosts, which may have zero
 // or more elements. We are interested in the element whose domains field
@@ -75,8 +74,9 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 // VirtualHost proto that the we are interested in is the list of routes. We
 // only look at the last route in the list (the default route), whose match
 // field must be empty and whose route field must be set.  Inside that route
-// message, the cluster field will contain the clusterName we are looking for.
-func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string) (string, error) {
+// message, the cluster field will contain the clusterName or weighted clusters
+// we are looking for.
+func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string) (rdsUpdate, error) {
 	//
 	// Currently this returns "" on error, and the caller will return an error.
 	// But the error doesn't contain details of why the response is invalid
@@ -87,31 +87,66 @@ func getClusterFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string)
 	vh := findBestMatchingVirtualHost(host, rc.GetVirtualHosts())
 	if vh == nil {
 		// No matching virtual host found.
-		return "", fmt.Errorf("no matching virtual host found")
+		return rdsUpdate{}, fmt.Errorf("no matching virtual host found")
 	}
 	if len(vh.Routes) == 0 {
 		// The matched virtual host has no routes, this is invalid because there
 		// should be at least one default route.
-		return "", fmt.Errorf("matched virtual host has no routes")
+		return rdsUpdate{}, fmt.Errorf("matched virtual host has no routes")
 	}
 	dr := vh.Routes[len(vh.Routes)-1]
 	match := dr.GetMatch()
 	if match == nil {
-		return "", fmt.Errorf("matched virtual host's default route doesn't have a match")
+		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route doesn't have a match")
 	}
 	if prefix := match.GetPrefix(); prefix != "" && prefix != "/" {
 		// The matched virtual host is invalid. Match is not "" or "/".
-		return "", fmt.Errorf("matched virtual host's default route is %v, want Prefix empty string or /", match)
+		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route is %v, want Prefix empty string or /", match)
 	}
 	if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
 		// The case sensitive is set to false. Not set or set to true are both
 		// valid.
-		return "", fmt.Errorf("matches virtual host's default route set case-sensitive to false")
+		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route set case-sensitive to false")
+	}
+	route := dr.GetRoute()
+	if route == nil {
+		return rdsUpdate{}, fmt.Errorf("matched route is nil")
+	}
+
+	if wc := route.GetWeightedClusters(); wc != nil {
+		m, err := weightedClustersProtoToMap(wc)
+		if err != nil {
+			return rdsUpdate{}, fmt.Errorf("matched weighted cluster is invalid: %v", err)
+		}
+		return rdsUpdate{weightedCluster: m}, nil
+	}
+
+	// When there's just one cluster, we set weightedCluster to map with one
+	// entry. This mean we will build a weighted_target balancer even if there's
+	// just one cluster.
+	//
+	// Otherwise, we will need to switch the top policy between weighted_target
+	// and CDS. In case when the action changes between one cluster and multiple
+	// clusters, changing top level policy means recreating TCP connection every
+	// time.
+	return rdsUpdate{weightedCluster: map[string]uint32{route.GetCluster(): 1}}, nil
+}
+
+func weightedClustersProtoToMap(wc *routepb.WeightedCluster) (map[string]uint32, error) {
+	ret := make(map[string]uint32)
+	var totalWeight uint32 = 100
+	if t := wc.GetTotalWeight().GetValue(); t != 0 {
+		totalWeight = t
+	}
+	for _, cw := range wc.Clusters {
+		w := cw.Weight.GetValue()
+		ret[cw.Name] = w
+		totalWeight -= w
 	}
-	if route := dr.GetRoute(); route != nil {
-		return route.GetCluster(), nil
+	if totalWeight != 0 {
+		return nil, fmt.Errorf("weights of clusters do not add up to total total weight, difference: %v", totalWeight)
 	}
-	return "", fmt.Errorf("matched route is nil")
+	return ret, nil
 }
 
 type domainMatchType int
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
index e85cdbc8782f..a099e32c5eed 100644
--- a/xds/internal/client/v2client_rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -30,24 +30,22 @@ import (
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
 
-func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
+func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 	tests := []struct {
-		name        string
-		rc          *xdspb.RouteConfiguration
-		wantCluster string
-		wantError   bool
+		name       string
+		rc         *xdspb.RouteConfiguration
+		wantUpdate rdsUpdate
+		wantError  bool
 	}{
 		{
-			name:        "no-virtual-hosts-in-rc",
-			rc:          emptyRouteConfig,
-			wantCluster: "",
-			wantError:   true,
+			name:      "no-virtual-hosts-in-rc",
+			rc:        emptyRouteConfig,
+			wantError: true,
 		},
 		{
-			name:        "no-domains-in-rc",
-			rc:          noDomainsInRouteConfig,
-			wantCluster: "",
-			wantError:   true,
+			name:      "no-domains-in-rc",
+			rc:        noDomainsInRouteConfig,
+			wantError: true,
 		},
 		{
 			name: "non-matching-domain-in-rc",
@@ -56,8 +54,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 					{Domains: []string{uninterestingDomain}},
 				},
 			},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
 			name: "no-routes-in-rc",
@@ -66,8 +63,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 					{Domains: []string{goodLDSTarget1}},
 				},
 			},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
 			name: "default-route-match-field-is-nil",
@@ -87,8 +83,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
 			name: "default-route-match-field-is-non-nil",
@@ -105,8 +100,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
 			name: "default-route-routeaction-field-is-nil",
@@ -118,8 +112,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
 			name: "default-route-cluster-field-is-empty",
@@ -139,8 +132,7 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
 			// default route's match sets case-sensitive to false.
@@ -158,14 +150,12 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 							Route: &routepb.RouteAction{
 								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
 							}}}}}}},
-			wantCluster: "",
-			wantError:   true,
+			wantError: true,
 		},
 		{
-			name:        "good-route-config-with-empty-string-route",
-			rc:          goodRouteConfig1,
-			wantCluster: goodClusterName1,
-			wantError:   false,
+			name:       "good-route-config-with-empty-string-route",
+			rc:         goodRouteConfig1,
+			wantUpdate: rdsUpdate{weightedCluster: map[string]uint32{goodClusterName1: 1}},
 		},
 		{
 			// default route's match is not empty string, but "/".
@@ -180,15 +170,59 @@ func (s) TestRDSGetClusterFromRouteConfiguration(t *testing.T) {
 							Route: &routepb.RouteAction{
 								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
 							}}}}}}},
-			wantCluster: goodClusterName1,
+			wantUpdate: rdsUpdate{weightedCluster: map[string]uint32{goodClusterName1: 1}},
+		},
+
+		{
+			// weights not add up to total-weight.
+			name: "route-config-with-weighted_clusters_weights_not_add_up",
+			rc: &xdspb.RouteConfiguration{
+				Name: goodRouteName1,
+				VirtualHosts: []*routepb.VirtualHost{{
+					Domains: []string{goodLDSTarget1},
+					Routes: []*routepb.Route{{
+						Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"}},
+						Action: &routepb.Route_Route{
+							Route: &routepb.RouteAction{
+								ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
+									WeightedClusters: &routepb.WeightedCluster{
+										Clusters: []*routepb.WeightedCluster_ClusterWeight{
+											{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
+											{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
+											{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
+										},
+										TotalWeight: &wrapperspb.UInt32Value{Value: 30},
+									}}}}}}}}},
+			wantError: true,
+		},
+		{
+			name: "good-route-config-with-weighted_clusters",
+			rc: &xdspb.RouteConfiguration{
+				Name: goodRouteName1,
+				VirtualHosts: []*routepb.VirtualHost{{
+					Domains: []string{goodLDSTarget1},
+					Routes: []*routepb.Route{{
+						Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"}},
+						Action: &routepb.Route_Route{
+							Route: &routepb.RouteAction{
+								ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
+									WeightedClusters: &routepb.WeightedCluster{
+										Clusters: []*routepb.WeightedCluster_ClusterWeight{
+											{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
+											{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
+											{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
+										},
+										TotalWeight: &wrapperspb.UInt32Value{Value: 10},
+									}}}}}}}}},
+			wantUpdate: rdsUpdate{weightedCluster: map[string]uint32{"a": 2, "b": 3, "c": 5}},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotCluster, gotError := getClusterFromRouteConfiguration(test.rc, goodLDSTarget1)
-			if gotCluster != test.wantCluster || (gotError != nil) != test.wantError {
-				t.Errorf("getClusterFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, goodLDSTarget1, gotCluster, test.wantCluster)
+			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, goodLDSTarget1)
+			if !cmp.Equal(gotUpdate, test.wantUpdate, cmp.AllowUnexported(rdsUpdate{})) || (gotError != nil) != test.wantError {
+				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, goodLDSTarget1, gotUpdate, test.wantUpdate)
 			}
 		})
 	}
@@ -256,7 +290,7 @@ func (s) TestRDSHandleResponse(t *testing.T) {
 			name:          "one-good-route-config",
 			rdsResponse:   goodRDSResponse1,
 			wantErr:       false,
-			wantUpdate:    &rdsUpdate{clusterName: goodClusterName1},
+			wantUpdate:    &rdsUpdate{weightedCluster: map[string]uint32{goodClusterName1: 1}},
 			wantUpdateErr: false,
 		},
 	}
@@ -412,3 +446,73 @@ func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestWeightedClustersProtoToMap(t *testing.T) {
+	tests := []struct {
+		name    string
+		wc      *routepb.WeightedCluster
+		want    map[string]uint32
+		wantErr bool
+	}{
+		{
+			name: "weight not add up to non default total",
+			wc: &routepb.WeightedCluster{
+				Clusters: []*routepb.WeightedCluster_ClusterWeight{
+					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 1}},
+					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 1}},
+					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 1}},
+				},
+				TotalWeight: &wrapperspb.UInt32Value{Value: 10},
+			},
+			wantErr: true,
+		},
+		{
+			name: "weight not add up to default total",
+			wc: &routepb.WeightedCluster{
+				Clusters: []*routepb.WeightedCluster_ClusterWeight{
+					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
+					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
+					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
+				},
+				TotalWeight: nil,
+			},
+			wantErr: true,
+		},
+		{
+			name: "ok non default total weight",
+			wc: &routepb.WeightedCluster{
+				Clusters: []*routepb.WeightedCluster_ClusterWeight{
+					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
+					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
+					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
+				},
+				TotalWeight: &wrapperspb.UInt32Value{Value: 10},
+			},
+			want: map[string]uint32{"a": 2, "b": 3, "c": 5},
+		},
+		{
+			name: "ok default total weight is 100",
+			wc: &routepb.WeightedCluster{
+				Clusters: []*routepb.WeightedCluster_ClusterWeight{
+					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 20}},
+					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 30}},
+					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 50}},
+				},
+				TotalWeight: nil,
+			},
+			want: map[string]uint32{"a": 20, "b": 30, "c": 50},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := weightedClustersProtoToMap(tt.wc)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("weightedClustersProtoToMap() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.want) {
+				t.Errorf("weightedClustersProtoToMap() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
new file mode 100644
index 000000000000..6a55217e7a87
--- /dev/null
+++ b/xds/internal/resolver/serviceconfig.go
@@ -0,0 +1,80 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"encoding/json"
+	"fmt"
+
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+const (
+	cdsName            = "cds_experimental"
+	weightedTargetName = "weighted_target_experimental"
+)
+
+type serviceConfig struct {
+	LoadBalancingConfig balancerConfig `json:"loadBalancingConfig"`
+}
+
+type balancerConfig []map[string]interface{}
+
+func newBalancerConfig(name string, config interface{}) balancerConfig {
+	return []map[string]interface{}{{name: config}}
+}
+
+type weightedCDSBalancerConfig struct {
+	Targets map[string]cdsWithWeight `json:"targets"`
+}
+
+type cdsWithWeight struct {
+	Weight      uint32         `json:"weight"`
+	ChildPolicy balancerConfig `json:"childPolicy"`
+}
+
+type cdsBalancerConfig struct {
+	Cluster string `json:"cluster"`
+}
+
+func serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) {
+	// Even if WeightedCluster has only one entry, we still use weighted_target
+	// as top level balancer, to avoid switching top policy between CDS and
+	// weighted_target, causing TCP connection to be recreated.
+	targets := make(map[string]cdsWithWeight)
+	for name, weight := range su.WeightedCluster {
+		targets[name] = cdsWithWeight{
+			Weight:      weight,
+			ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: name}),
+		}
+	}
+
+	sc := serviceConfig{
+		LoadBalancingConfig: newBalancerConfig(
+			weightedTargetName, weightedCDSBalancerConfig{
+				Targets: targets,
+			},
+		),
+	}
+	bs, err := json.Marshal(sc)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal json: %v", err)
+	}
+	return string(bs), nil
+}
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
new file mode 100644
index 000000000000..c4d97261e691
--- /dev/null
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -0,0 +1,92 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/serviceconfig"
+	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
+	"google.golang.org/grpc/xds/internal/client"
+)
+
+const (
+	testCluster1        = "test-cluster-1"
+	testClusterOnlyJSON = `{"loadBalancingConfig":[{
+    "weighted_target_experimental": {
+	  "targets": { "test-cluster-1" : { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}] } }
+	}
+}]}`
+	testWeightedCDSJSON = `{"loadBalancingConfig":[{
+    "weighted_target_experimental": {
+	  "targets": {
+		"cluster_1" : {
+		  "weight":75,
+		  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+		},
+		"cluster_2" : {
+		  "weight":25,
+		  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+		}
+	  }
+	}
+}]}`
+)
+
+func TestServiceUpdateToJSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		su       client.ServiceUpdate
+		wantJSON string // wantJSON is not to be compared verbatim.
+	}{
+		{
+			name:     "one cluster only",
+			su:       client.ServiceUpdate{WeightedCluster: map[string]uint32{testCluster1: 1}},
+			wantJSON: testClusterOnlyJSON,
+		},
+		{
+			name: "weighted clusters",
+			su: client.ServiceUpdate{WeightedCluster: map[string]uint32{
+				"cluster_1": 75,
+				"cluster_2": 25,
+			}},
+			wantJSON: testWeightedCDSJSON,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotJSON, err := serviceUpdateToJSON(tt.su)
+			if err != nil {
+				t.Errorf("serviceUpdateToJSON returned error: %v", err)
+				return
+			}
+
+			gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON)
+			wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
+
+			if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
+				t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON)
+				t.Error("gotParsed: ", cmp.Diff(nil, gotParsed))
+				t.Error("wantParsed: ", cmp.Diff(nil, wantParsed))
+			}
+		})
+	}
+}
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 66ec38c89d01..2b806cc62c8e 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -161,16 +161,6 @@ type xdsResolver struct {
 	cancelWatch func()
 }
 
-const jsonFormatSC = `{
-    "loadBalancingConfig":[
-      {
-        "cds_experimental":{
-          "Cluster": "%s"
-        }
-      }
-    ]
-  }`
-
 // run is a long running goroutine which blocks on receiving service updates
 // and passes it on the ClientConn.
 func (r *xdsResolver) run() {
@@ -183,7 +173,12 @@ func (r *xdsResolver) run() {
 				r.cc.ReportError(update.err)
 				continue
 			}
-			sc := fmt.Sprintf(jsonFormatSC, update.su.Cluster)
+			sc, err := serviceUpdateToJSON(update.su)
+			if err != nil {
+				r.logger.Warningf("failed to convert update to service config: %v", err)
+				r.cc.ReportError(err)
+				continue
+			}
 			r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc)
 			r.cc.UpdateState(resolver.State{
 				ServiceConfig: r.cc.ParseServiceConfig(sc),
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 57456730abab..a6d1f72229c5 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -25,12 +25,14 @@ import (
 	"net"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config
+	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/testutils"
@@ -273,7 +275,7 @@ func TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	// Call the watchAPI callback after closing the resolver, and make sure no
 	// update is triggerred on the ClientConn.
 	xdsR.Close()
-	xdsC.InvokeWatchServiceCallback(cluster, nil)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{WeightedCluster: map[string]uint32{cluster: 1}}, nil)
 	if gotVal, gotErr := tcc.stateCh.Receive(); gotErr != testutils.ErrRecvTimeout {
 		t.Fatalf("ClientConn.UpdateState called after xdsResolver is closed: %v", gotVal)
 	}
@@ -297,7 +299,7 @@ func TestXDSResolverBadServiceUpdate(t *testing.T) {
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr := errors.New("bad serviceupdate")
-	xdsC.InvokeWatchServiceCallback("", suErr)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
 	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr)
 	}
@@ -318,19 +320,43 @@ func TestXDSResolverGoodServiceUpdate(t *testing.T) {
 
 	waitForWatchService(t, xdsC, targetStr)
 
-	// Invoke the watchAPI callback with a good service update and wait for the
-	// UpdateState method to be called on the ClientConn.
-	xdsC.InvokeWatchServiceCallback(cluster, nil)
-	gotState, err := tcc.stateCh.Receive()
-	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
-	}
-	rState := gotState.(resolver.State)
-	if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != xdsC {
-		t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want %v", gotClient, xdsC)
-	}
-	if err := rState.ServiceConfig.Err; err != nil {
-		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	for _, tt := range []struct {
+		su       client.ServiceUpdate
+		wantJSON string
+	}{
+		{
+			su:       client.ServiceUpdate{WeightedCluster: map[string]uint32{testCluster1: 1}},
+			wantJSON: testClusterOnlyJSON,
+		},
+		{
+			su: client.ServiceUpdate{WeightedCluster: map[string]uint32{
+				"cluster_1": 75,
+				"cluster_2": 25,
+			}},
+			wantJSON: testWeightedCDSJSON,
+		},
+	} {
+		// Invoke the watchAPI callback with a good service update and wait for the
+		// UpdateState method to be called on the ClientConn.
+		xdsC.InvokeWatchServiceCallback(tt.su, nil)
+		gotState, err := tcc.stateCh.Receive()
+		if err != nil {
+			t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		}
+		rState := gotState.(resolver.State)
+		if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != xdsC {
+			t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want %v", gotClient, xdsC)
+		}
+		if err := rState.ServiceConfig.Err; err != nil {
+			t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+		}
+
+		wantSCParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
+		if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) {
+			t.Errorf("ClientConn.UpdateState received different service config")
+			t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+			t.Error("want: ", cmp.Diff(nil, wantSCParsed.Config))
+		}
 	}
 }
 
@@ -352,14 +378,14 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr := errors.New("bad serviceupdate")
-	xdsC.InvokeWatchServiceCallback("", suErr)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
 	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr)
 	}
 
 	// Invoke the watchAPI callback with a good service update and wait for the
 	// UpdateState method to be called on the ClientConn.
-	xdsC.InvokeWatchServiceCallback(cluster, nil)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{WeightedCluster: map[string]uint32{cluster: 1}}, nil)
 	gotState, err := tcc.stateCh.Receive()
 	if err != nil {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
@@ -375,7 +401,7 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr2 := errors.New("bad serviceupdate 2")
-	xdsC.InvokeWatchServiceCallback("", suErr2)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr2)
 	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr2 {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr2)
 	}
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 2b290101e9ce..922fab1e4371 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -69,11 +69,11 @@ func (xdsC *Client) WaitForWatchService() (string, error) {
 }
 
 // InvokeWatchServiceCallback invokes the registered service watch callback.
-func (xdsC *Client) InvokeWatchServiceCallback(cluster string, err error) {
+func (xdsC *Client) InvokeWatchServiceCallback(u xdsclient.ServiceUpdate, err error) {
 	xdsC.mu.Lock()
 	defer xdsC.mu.Unlock()
 
-	xdsC.serviceCb(xdsclient.ServiceUpdate{Cluster: cluster}, err)
+	xdsC.serviceCb(u, err)
 }
 
 // WatchCluster registers a CDS watch.

From eb827fbfd832d8d74550d92c139423f53778d3dd Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 26 May 2020 15:24:52 -0700
Subject: [PATCH 076/481] grpclb, dns: pass balancer addresses via
 resolver.State (#3614)

---
 balancer/grpclb/grpclb.go                  | 22 +++++---
 balancer/grpclb/grpclb_test.go             | 26 +++++----
 balancer/grpclb/state/state.go             | 51 ++++++++++++++++++
 internal/resolver/dns/dns_resolver.go      | 11 ++--
 internal/resolver/dns/dns_resolver_test.go | 61 +++++++++++++---------
 resolver/resolver.go                       |  5 +-
 6 files changed, 131 insertions(+), 45 deletions(-)
 create mode 100644 balancer/grpclb/state/state.go

diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go
index 193873b0fa13..59f38f9047d5 100644
--- a/balancer/grpclb/grpclb.go
+++ b/balancer/grpclb/grpclb.go
@@ -28,10 +28,9 @@ import (
 	"sync"
 	"time"
 
-	durationpb "github.com/golang/protobuf/ptypes/duration"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
-	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
+	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
@@ -39,6 +38,9 @@ import (
 	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/resolver/dns"
 	"google.golang.org/grpc/resolver"
+
+	durationpb "github.com/golang/protobuf/ptypes/duration"
+	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
 )
 
 const (
@@ -410,11 +412,6 @@ func (lb *lbBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error
 	lb.handleServiceConfig(gc)
 
 	addrs := ccs.ResolverState.Addresses
-	if len(addrs) == 0 {
-		// There should be at least one address, either grpclb server or
-		// fallback. Empty address is not valid.
-		return balancer.ErrBadResolverState
-	}
 
 	var remoteBalancerAddrs, backendAddrs []resolver.Address
 	for _, a := range addrs {
@@ -425,6 +422,17 @@ func (lb *lbBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error
 			backendAddrs = append(backendAddrs, a)
 		}
 	}
+	if sd := grpclbstate.Get(ccs.ResolverState); sd != nil {
+		// Override any balancer addresses provided via
+		// ccs.ResolverState.Addresses.
+		remoteBalancerAddrs = sd.BalancerAddresses
+	}
+
+	if len(backendAddrs)+len(remoteBalancerAddrs) == 0 {
+		// There should be at least one address, either grpclb server or
+		// fallback. Empty address is not valid.
+		return balancer.ErrBadResolverState
+	}
 
 	if len(remoteBalancerAddrs) == 0 {
 		if lb.ccRemoteLB != nil {
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index 19296ca3a9d5..d701b6d216a0 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -31,11 +31,9 @@ import (
 	"testing"
 	"time"
 
-	durationpb "github.com/golang/protobuf/ptypes/duration"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
-	lbgrpc "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
-	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
+	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/grpctest"
@@ -44,6 +42,10 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 	"google.golang.org/grpc/status"
+
+	durationpb "github.com/golang/protobuf/ptypes/duration"
+	lbgrpc "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
+	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
@@ -390,6 +392,8 @@ func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (ts
 	return
 }
 
+var grpclbConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}`
+
 func (s) TestGRPCLB(t *testing.T) {
 	r, cleanup := manual.GenerateAndRegisterManualResolver()
 	defer cleanup()
@@ -422,13 +426,17 @@ func (s) TestGRPCLB(t *testing.T) {
 	defer cc.Close()
 	testC := testpb.NewTestServiceClient(cc)
 
-	r.UpdateState(resolver.State{Addresses: []resolver.Address{{
-		Addr:       tss.lbAddr,
-		Type:       resolver.GRPCLB,
-		ServerName: lbServerName,
-	}}})
+	rs := grpclbstate.Set(resolver.State{ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig)},
+		&grpclbstate.State{BalancerAddresses: []resolver.Address{{
+			Addr:       tss.lbAddr,
+			Type:       resolver.Backend,
+			ServerName: lbServerName,
+		}}})
+	r.UpdateState(rs)
 
-	if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 	}
 }
diff --git a/balancer/grpclb/state/state.go b/balancer/grpclb/state/state.go
new file mode 100644
index 000000000000..a24264a34f5f
--- /dev/null
+++ b/balancer/grpclb/state/state.go
@@ -0,0 +1,51 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package state declares grpclb types to be set by resolvers wishing to pass
+// information to grpclb via resolver.State Attributes.
+package state
+
+import (
+	"google.golang.org/grpc/resolver"
+)
+
+// keyType is the key to use for storing State in Attributes.
+type keyType string
+
+const key = keyType("grpc.grpclb.state")
+
+// State contains gRPCLB-relevant data passed from the name resolver.
+type State struct {
+	// BalancerAddresses contains the remote load balancer address(es).  If
+	// set, overrides any resolver-provided addresses with Type of GRPCLB.
+	BalancerAddresses []resolver.Address
+}
+
+// Set returns a copy of the provided state with attributes containing s.  s's
+// data should not be mutated after calling Set.
+func Set(state resolver.State, s *State) resolver.State {
+	state.Attributes = state.Attributes.WithValues(key, s)
+	return state
+}
+
+// Get returns the grpclb State in the resolver.State, or nil if not present.
+// The returned data should not be mutated.
+func Get(state resolver.State) *State {
+	s, _ := state.Attributes.Value(key).(*State)
+	return s
+}
diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go
index c368db62ea11..9d08dd8ab092 100644
--- a/internal/resolver/dns/dns_resolver.go
+++ b/internal/resolver/dns/dns_resolver.go
@@ -32,6 +32,7 @@ import (
 	"sync"
 	"time"
 
+	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/envconfig"
 	"google.golang.org/grpc/internal/grpcrand"
@@ -251,7 +252,7 @@ func (d *dnsResolver) lookupSRV() ([]resolver.Address, error) {
 				return nil, fmt.Errorf("dns: error parsing A record IP address %v", a)
 			}
 			addr := ip + ":" + strconv.Itoa(int(s.Port))
-			newAddrs = append(newAddrs, resolver.Address{Addr: addr, Type: resolver.GRPCLB, ServerName: s.Target})
+			newAddrs = append(newAddrs, resolver.Address{Addr: addr, ServerName: s.Target})
 		}
 	}
 	return newAddrs, nil
@@ -326,13 +327,15 @@ func (d *dnsResolver) lookup() (*resolver.State, error) {
 	if hostErr != nil && (srvErr != nil || len(srv) == 0) {
 		return nil, hostErr
 	}
-	state := &resolver.State{
-		Addresses: append(addrs, srv...),
+
+	state := resolver.State{Addresses: addrs}
+	if len(srv) > 0 {
+		state = grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: srv})
 	}
 	if !d.disableServiceConfig {
 		state.ServiceConfig = d.lookupTXT()
 	}
-	return state, nil
+	return &state, nil
 }
 
 // formatIP returns ok = false if addr is not a valid textual representation of an IP address.
diff --git a/internal/resolver/dns/dns_resolver_test.go b/internal/resolver/dns/dns_resolver_test.go
index b7b39a7f645d..1c8469a275a7 100644
--- a/internal/resolver/dns/dns_resolver_test.go
+++ b/internal/resolver/dns/dns_resolver_test.go
@@ -30,6 +30,7 @@ import (
 	"testing"
 	"time"
 
+	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
 	"google.golang.org/grpc/internal/envconfig"
 	"google.golang.org/grpc/internal/leakcheck"
 	"google.golang.org/grpc/resolver"
@@ -725,11 +726,11 @@ func testDNSResolver(t *testing.T) {
 			t.Fatalf("UpdateState not called after 2s; aborting")
 		}
 		if !reflect.DeepEqual(a.addrWant, state.Addresses) {
-			t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", a.target, state.Addresses, a.addrWant)
+			t.Errorf("Resolved addresses of target: %q = %+v, want %+v", a.target, state.Addresses, a.addrWant)
 		}
 		sc := scFromState(state)
 		if a.scWant != sc {
-			t.Errorf("Resolved service config of target: %q = %+v, want %+v\n", a.target, sc, a.scWant)
+			t.Errorf("Resolved service config of target: %q = %+v, want %+v", a.target, sc, a.scWant)
 		}
 		r.Close()
 	}
@@ -742,45 +743,52 @@ func testDNSResolverWithSRV(t *testing.T) {
 	}()
 	defer leakcheck.Check(t)
 	tests := []struct {
-		target   string
-		addrWant []resolver.Address
-		scWant   string
+		target      string
+		addrWant    []resolver.Address
+		grpclbAddrs []resolver.Address
+		scWant      string
 	}{
 		{
 			"foo.bar.com",
 			[]resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}},
+			nil,
 			generateSC("foo.bar.com"),
 		},
 		{
 			"foo.bar.com:1234",
 			[]resolver.Address{{Addr: "1.2.3.4:1234"}, {Addr: "5.6.7.8:1234"}},
+			nil,
 			generateSC("foo.bar.com"),
 		},
 		{
 			"srv.ipv4.single.fake",
-			[]resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}, {Addr: "1.2.3.4:1234", Type: resolver.GRPCLB, ServerName: "ipv4.single.fake"}},
+			[]resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}},
+			[]resolver.Address{{Addr: "1.2.3.4:1234", ServerName: "ipv4.single.fake"}},
 			generateSC("srv.ipv4.single.fake"),
 		},
 		{
 			"srv.ipv4.multi.fake",
+			nil,
 			[]resolver.Address{
-				{Addr: "1.2.3.4:1234", Type: resolver.GRPCLB, ServerName: "ipv4.multi.fake"},
-				{Addr: "5.6.7.8:1234", Type: resolver.GRPCLB, ServerName: "ipv4.multi.fake"},
-				{Addr: "9.10.11.12:1234", Type: resolver.GRPCLB, ServerName: "ipv4.multi.fake"},
+				{Addr: "1.2.3.4:1234", ServerName: "ipv4.multi.fake"},
+				{Addr: "5.6.7.8:1234", ServerName: "ipv4.multi.fake"},
+				{Addr: "9.10.11.12:1234", ServerName: "ipv4.multi.fake"},
 			},
 			generateSC("srv.ipv4.multi.fake"),
 		},
 		{
 			"srv.ipv6.single.fake",
-			[]resolver.Address{{Addr: "[2607:f8b0:400a:801::1001]:1234", Type: resolver.GRPCLB, ServerName: "ipv6.single.fake"}},
+			nil,
+			[]resolver.Address{{Addr: "[2607:f8b0:400a:801::1001]:1234", ServerName: "ipv6.single.fake"}},
 			generateSC("srv.ipv6.single.fake"),
 		},
 		{
 			"srv.ipv6.multi.fake",
+			nil,
 			[]resolver.Address{
-				{Addr: "[2607:f8b0:400a:801::1001]:1234", Type: resolver.GRPCLB, ServerName: "ipv6.multi.fake"},
-				{Addr: "[2607:f8b0:400a:801::1002]:1234", Type: resolver.GRPCLB, ServerName: "ipv6.multi.fake"},
-				{Addr: "[2607:f8b0:400a:801::1003]:1234", Type: resolver.GRPCLB, ServerName: "ipv6.multi.fake"},
+				{Addr: "[2607:f8b0:400a:801::1001]:1234", ServerName: "ipv6.multi.fake"},
+				{Addr: "[2607:f8b0:400a:801::1002]:1234", ServerName: "ipv6.multi.fake"},
+				{Addr: "[2607:f8b0:400a:801::1003]:1234", ServerName: "ipv6.multi.fake"},
 			},
 			generateSC("srv.ipv6.multi.fake"),
 		},
@@ -807,11 +815,16 @@ func testDNSResolverWithSRV(t *testing.T) {
 			t.Fatalf("UpdateState not called after 2s; aborting")
 		}
 		if !reflect.DeepEqual(a.addrWant, state.Addresses) {
-			t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", a.target, state.Addresses, a.addrWant)
+			t.Errorf("Resolved addresses of target: %q = %+v, want %+v", a.target, state.Addresses, a.addrWant)
+		}
+		gs := grpclbstate.Get(state)
+		if (gs == nil && len(a.grpclbAddrs) > 0) ||
+			(gs != nil && !reflect.DeepEqual(a.grpclbAddrs, gs.BalancerAddresses)) {
+			t.Errorf("Resolved state of target: %q = %+v (State=%+v), want state.Attributes.State=%+v", a.target, state, gs, a.grpclbAddrs)
 		}
 		sc := scFromState(state)
 		if a.scWant != sc {
-			t.Errorf("Resolved service config of target: %q = %+v, want %+v\n", a.target, sc, a.scWant)
+			t.Errorf("Resolved service config of target: %q = %+v, want %+v", a.target, sc, a.scWant)
 		}
 	}
 }
@@ -879,11 +892,11 @@ func testDNSResolveNow(t *testing.T) {
 			t.Fatalf("UpdateState not called after 2s; aborting.  state=%v", state)
 		}
 		if !reflect.DeepEqual(a.addrWant, state.Addresses) {
-			t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", a.target, state.Addresses, a.addrWant)
+			t.Errorf("Resolved addresses of target: %q = %+v, want %+v", a.target, state.Addresses, a.addrWant)
 		}
 		sc := scFromState(state)
 		if a.scWant != sc {
-			t.Errorf("Resolved service config of target: %q = %+v, want %+v\n", a.target, sc, a.scWant)
+			t.Errorf("Resolved service config of target: %q = %+v, want %+v", a.target, sc, a.scWant)
 		}
 
 		revertTbl := mutateTbl(a.target)
@@ -900,10 +913,10 @@ func testDNSResolveNow(t *testing.T) {
 		}
 		sc = scFromState(state)
 		if !reflect.DeepEqual(a.addrNext, state.Addresses) {
-			t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", a.target, state.Addresses, a.addrNext)
+			t.Errorf("Resolved addresses of target: %q = %+v, want %+v", a.target, state.Addresses, a.addrNext)
 		}
 		if a.scNext != sc {
-			t.Errorf("Resolved service config of target: %q = %+v, want %+v\n", a.target, sc, a.scNext)
+			t.Errorf("Resolved service config of target: %q = %+v, want %+v", a.target, sc, a.scNext)
 		}
 		revertTbl()
 	}
@@ -946,7 +959,7 @@ func testIPResolver(t *testing.T) {
 			time.Sleep(time.Millisecond)
 		}
 		if !reflect.DeepEqual(v.want, state.Addresses) {
-			t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", v.target, state.Addresses, v.want)
+			t.Errorf("Resolved addresses of target: %q = %+v, want %+v", v.target, state.Addresses, v.want)
 		}
 		r.ResolveNow(resolver.ResolveNowOptions{})
 		for i := 0; i < 50; i++ {
@@ -1039,7 +1052,7 @@ func TestDisableServiceConfig(t *testing.T) {
 		}
 		sc := scFromState(state)
 		if a.scWant != sc {
-			t.Errorf("Resolved service config of target: %q = %+v, want %+v\n", a.target, sc, a.scWant)
+			t.Errorf("Resolved service config of target: %q = %+v, want %+v", a.target, sc, a.scWant)
 		}
 	}
 }
@@ -1098,7 +1111,7 @@ func TestDNSResolverRetry(t *testing.T) {
 	}
 	want := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}
 	if !reflect.DeepEqual(want, state.Addresses) {
-		t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", target, state.Addresses, want)
+		t.Errorf("Resolved addresses of target: %q = %+v, want %+v", target, state.Addresses, want)
 	}
 	// mutate the host lookup table so the target has 0 address returned.
 	revertTbl := mutateTbl(target)
@@ -1125,7 +1138,7 @@ func TestDNSResolverRetry(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 	if !reflect.DeepEqual(want, state.Addresses) {
-		t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", target, state.Addresses, want)
+		t.Errorf("Resolved addresses of target: %q = %+v, want %+v", target, state.Addresses, want)
 	}
 }
 
@@ -1330,7 +1343,7 @@ func TestRateLimitedResolve(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 	if !reflect.DeepEqual(state.Addresses, wantAddrs) {
-		t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", target, state.Addresses, wantAddrs)
+		t.Errorf("Resolved addresses of target: %q = %+v, want %+v", target, state.Addresses, wantAddrs)
 	}
 }
 
diff --git a/resolver/resolver.go b/resolver/resolver.go
index fe14b2fb9826..379275a2d9b4 100644
--- a/resolver/resolver.go
+++ b/resolver/resolver.go
@@ -85,7 +85,10 @@ const (
 	Backend AddressType = iota
 	// GRPCLB indicates the address is for a grpclb load balancer.
 	//
-	// Deprecated: use Attributes in Address instead.
+	// Deprecated: to select the GRPCLB load balancing policy, use a service
+	// config with a corresponding loadBalancingConfig.  To supply balancer
+	// addresses to the GRPCLB load balancing policy, set State.Attributes
+	// using balancer/grpclb/state.Set.
 	GRPCLB
 )
 

From 5cd45224f897e3d12e40aafbb4198c827fe47714 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 26 May 2020 17:59:18 -0700
Subject: [PATCH 077/481] internal: fix errors after merging (#3643)

---
 xds/internal/client/client_watchers_service_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 91bb90b60759..71de6c750146 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -359,7 +359,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{Cluster: testCDSName}
+	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
 	<-v2Client.addWatches[ldsURL]
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
@@ -367,10 +367,10 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	})
 	<-v2Client.addWatches[rdsURL]
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {clusterName: testCDSName},
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || u != (serviceUpdateErr{wantUpdate, nil}) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 

From e786c2dfb285929db56044f9e376dcdfcd2a9dac Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 27 May 2020 09:55:12 -0700
Subject: [PATCH 078/481] xds: eds balancer handleEDSUpdate cleanup (#3639)

---
 xds/internal/balancer/edsbalancer/eds.go      | 60 ++++++-------------
 .../balancer/edsbalancer/eds_impl_test.go     |  2 +-
 xds/internal/balancer/edsbalancer/eds_test.go |  4 +-
 .../edsbalancer/xds_client_wrapper.go         | 16 +----
 .../edsbalancer/xds_client_wrapper_test.go    | 23 ++++---
 5 files changed, 40 insertions(+), 65 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 02ac314cd71b..1cf9b16ed84a 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -23,7 +23,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
@@ -37,8 +36,7 @@ import (
 )
 
 const (
-	defaultTimeout = 10 * time.Second
-	edsName        = "eds_experimental"
+	edsName = "eds_experimental"
 )
 
 var (
@@ -62,13 +60,13 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 		cc:                cc,
 		buildOpts:         opts,
 		grpcUpdate:        make(chan interface{}),
-		xdsClientUpdate:   make(chan interface{}),
+		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
 	}
 	loadStore := lrs.NewStore()
 	x.logger = grpclog.NewPrefixLogger(loggingPrefix(x))
 	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, loadStore, x.logger)
-	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.loseContact, x.buildOpts, loadStore, x.logger)
+	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.buildOpts, loadStore, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
@@ -117,7 +115,7 @@ type edsBalancer struct {
 
 	// edsBalancer continuously monitor the channels below, and will handle events from them in sync.
 	grpcUpdate        chan interface{}
-	xdsClientUpdate   chan interface{}
+	xdsClientUpdate   chan *edsUpdate
 	childPolicyUpdate *buffer.Unbounded
 
 	client  *xdsclientWrapper // may change when passed a different service config
@@ -140,12 +138,8 @@ func (x *edsBalancer) run() {
 			u := update.(*balancerStateWithPriority)
 			x.edsImpl.updateState(u.priority, u.s)
 		case <-x.ctx.Done():
-			if x.client != nil {
-				x.client.close()
-			}
-			if x.edsImpl != nil {
-				x.edsImpl.close()
-			}
+			x.client.close()
+			x.edsImpl.close()
 			return
 		}
 	}
@@ -154,9 +148,7 @@ func (x *edsBalancer) run() {
 func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 	switch u := update.(type) {
 	case *subConnStateUpdate:
-		if x.edsImpl != nil {
-			x.edsImpl.handleSubConnStateChange(u.sc, u.state.ConnectivityState)
-		}
+		x.edsImpl.handleSubConnStateChange(u.sc, u.state.ConnectivityState)
 	case *balancer.ClientConnState:
 		x.logger.Infof("Receive update from resolver, balancer config: %+v", u.BalancerConfig)
 		cfg, _ := u.BalancerConfig.(*EDSConfig)
@@ -174,7 +166,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
-		if x.edsImpl != nil && !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy) {
+		if !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy) {
 			if cfg.ChildPolicy != nil {
 				x.edsImpl.handleChildPolicy(cfg.ChildPolicy.Name, cfg.ChildPolicy.Config)
 			} else {
@@ -189,17 +181,13 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 	}
 }
 
-func (x *edsBalancer) handleXDSClientUpdate(update interface{}) {
-	switch u := update.(type) {
-	// TODO: this func should accept (xdsclient.EndpointsUpdate, error), and process
-	// the error, instead of having a separate loseContact signal.
-	case xdsclient.EndpointsUpdate:
-		x.edsImpl.handleEDSResponse(u)
-	case *loseContact:
-		// loseContact can be useful for going into fallback.
-	default:
-		panic("unexpected xds client update type")
+func (x *edsBalancer) handleXDSClientUpdate(update *edsUpdate) {
+	if update.err != nil {
+		// TODO: handle errors from EDS callback. E.g. if CDS resource is
+		// removed, the EDS watch should be canceled.
+		return
 	}
+	x.edsImpl.handleEDSResponse(update.resp)
 }
 
 type subConnStateUpdate struct {
@@ -233,24 +221,14 @@ func (x *edsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	return nil
 }
 
-func (x *edsBalancer) handleEDSUpdate(resp xdsclient.EndpointsUpdate) error {
-	// TODO: this function should take (resp, error), and send them together on
-	// the channel. There doesn't need to be a separate `loseContact` function.
-	select {
-	case x.xdsClientUpdate <- resp:
-	case <-x.ctx.Done():
-	}
-
-	return nil
-}
-
-type loseContact struct {
+type edsUpdate struct {
+	resp xdsclient.EndpointsUpdate
+	err  error
 }
 
-// TODO: delete loseContact when handleEDSUpdate takes (resp, error).
-func (x *edsBalancer) loseContact() {
+func (x *edsBalancer) handleEDSUpdate(resp xdsclient.EndpointsUpdate, err error) {
 	select {
-	case x.xdsClientUpdate <- &loseContact{}:
+	case x.xdsClientUpdate <- &edsUpdate{resp: resp, err: err}:
 	case <-x.ctx.Done():
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 1e449430697f..d308f97c8f10 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -412,7 +412,7 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	for i := 0; i < 5; i++ {
 		_, err := p0.Pick(balancer.PickInfo{})
 		if err != testutils.ErrTestConstPicker {
-			t.Fatalf("picker.Pick, got err %q, want err %q", err, testutils.ErrTestConstPicker)
+			t.Fatalf("picker.Pick, got err %+v, want err %+v", err, testutils.ErrTestConstPicker)
 		}
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 5f8b04d6820f..ed1e44de902e 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -108,10 +108,12 @@ func (f *fakeEDSBalancer) handleChildPolicy(name string, config json.RawMessage)
 	f.childPolicy.Send(&loadBalancingConfig{Name: name, Config: config})
 }
 
-func (f *fakeEDSBalancer) close()                                              {}
 func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {}
+
 func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {}
 
+func (f *fakeEDSBalancer) close() {}
+
 func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) error {
 	val, err := f.childPolicy.Receive()
 	if err != nil {
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index ab06c1a28f35..15fdd7d4d36e 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -50,8 +50,7 @@ var (
 type xdsclientWrapper struct {
 	logger *grpclog.PrefixLogger
 
-	newEDSUpdate func(xdsclient.EndpointsUpdate) error
-	loseContact  func()
+	newEDSUpdate func(xdsclient.EndpointsUpdate, error)
 	bbo          balancer.BuildOptions
 	loadStore    lrs.Store
 
@@ -78,11 +77,10 @@ type xdsclientWrapper struct {
 //
 // The given callbacks won't be called until the underlying xds_client is
 // working and sends updates.
-func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate) error, loseContact func(), bbo balancer.BuildOptions, loadStore lrs.Store, logger *grpclog.PrefixLogger) *xdsclientWrapper {
+func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, loadStore lrs.Store, logger *grpclog.PrefixLogger) *xdsclientWrapper {
 	return &xdsclientWrapper{
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
-		loseContact:  loseContact,
 		bbo:          bbo,
 		loadStore:    loadStore,
 	}
@@ -188,16 +186,8 @@ func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
 		c.cancelEndpointsWatch()
 	}
 	cancelEDSWatch := c.xdsclient.WatchEndpoints(c.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
-		if err != nil {
-			// TODO: this should trigger a call to `c.loseContact`, when the
-			// error indicates "lose contact".
-			c.logger.Warningf("Watch error from xds-client %p: %v", c.xdsclient, err)
-			return
-		}
 		c.logger.Infof("Watch update from xds-client %p, content: %+v", c.xdsclient, update)
-		if err := c.newEDSUpdate(update); err != nil {
-			c.logger.Warningf("xds: processing new EDS update failed due to %v.", err)
-		}
+		c.newEDSUpdate(update, err)
 	})
 	c.logger.Infof("Watch started on resource name %v with xds-client %p", c.edsServiceName, c.xdsclient)
 	c.cancelEndpointsWatch = func() {
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index 15124a8c469a..f7cc4bda2d8c 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -64,7 +64,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 	}
 	defer cleanup()
 
-	cw := newXDSClientWrapper(nil, nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
+	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
 	defer cw.close()
 
 	for _, test := range []struct {
@@ -159,12 +159,11 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 //   edsBalancer with the received error.
 func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 	edsRespChan := testutils.NewChannel()
-	newEDS := func(update xdsclient.EndpointsUpdate) error {
-		edsRespChan.Send(update)
-		return nil
+	newEDS := func(update xdsclient.EndpointsUpdate, err error) {
+		edsRespChan.Send(&edsUpdate{resp: update, err: err})
 	}
 
-	cw := newXDSClientWrapper(newEDS, nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
+	cw := newXDSClientWrapper(newEDS, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
 	defer cw.close()
 
 	xdsC := fakeclient.NewClient()
@@ -176,14 +175,20 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 	if gotCluster != testEDSClusterName {
 		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
 	}
-	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, errors.New("EDS watch callback error"))
+	watchErr := errors.New("EDS watch callback error")
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, watchErr)
 
 	// The callback is called with an error, expect no update from edsRespChan.
 	//
 	// TODO: check for loseContact() when errors indicating "lose contact" are
 	// handled correctly.
-	if gotUpdate, gotErr := edsRespChan.Receive(); gotErr != testutils.ErrRecvTimeout {
-		t.Fatalf("edsBalancer got edsUpdate {%+v, %v}, when none was expected", gotUpdate, gotErr)
+	gotUpdate, err := edsRespChan.Receive()
+	if err != nil {
+		t.Fatalf("edsBalancer failed to get edsUpdate %v", err)
+	}
+	update := gotUpdate.(*edsUpdate)
+	if !cmp.Equal(update.resp, (xdsclient.EndpointsUpdate{})) || update.err != watchErr {
+		t.Fatalf("want update {nil, %v}, got %+v", watchErr, update)
 	}
 }
 
@@ -198,7 +203,7 @@ func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
 	}
 	defer func() { xdsclientNew = oldxdsclientNew }()
 
-	cw := newXDSClientWrapper(nil, nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
+	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
 	defer cw.close()
 
 	// Verify that the eds watch is registered for the expected resource name.

From 30c53e745c3e0fa52324bb50850a782e1ff9f136 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 27 May 2020 10:14:25 -0700
Subject: [PATCH 079/481] xds: move balancer/resolver registation to package
 xds (#3640)

---
 examples/features/xds/README.md      | 10 ++++------
 examples/features/xds/client/main.go |  1 +
 interop/xds/client/client.go         |  2 +-
 xds/experimental/xds_experimental.go | 13 +++++++------
 xds/xds.go                           | 29 ++++++++++++++++++++++++++++
 5 files changed, 42 insertions(+), 13 deletions(-)
 create mode 100644 xds/xds.go

diff --git a/examples/features/xds/README.md b/examples/features/xds/README.md
index eba96d38f37a..8cf8c4413957 100644
--- a/examples/features/xds/README.md
+++ b/examples/features/xds/README.md
@@ -9,13 +9,11 @@ hello world
 example](https://github.com/grpc/grpc-go/tree/master/examples/helloworld). The
 server replies with responses including its hostname.
 
-**Note** that xDS support is incomplete and experimental, with limited
-compatibility.
-
 ## xDS environment setup
 
-This example doesn't include instuctions to setup xDS environment. Please
-refer to documentation specific for your xDS management server.
+This example doesn't include instructions to setup xDS environment. Please refer
+to documentation specific for your xDS management server. Examples will be added
+later.
 
 The client also needs a bootstrap file. See [gRFC
 A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md#xdsclient-and-bootstrap-file)
@@ -26,7 +24,7 @@ for the bootstrap format.
 The client application needs to import the xDS package to install the resolver and balancers:
 
 ```go
-_ "google.golang.org/grpc/xds/experimental" // To install the xds resolvers and balancers.
+_ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers.
 ```
 
 Then, use `xds` target scheme for the ClientConn.
diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go
index 3f1a53871516..ea50f09450a8 100644
--- a/examples/features/xds/client/main.go
+++ b/examples/features/xds/client/main.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/grpc"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 
+	// TODO: change this to xds after the xds package is released.
 	_ "google.golang.org/grpc/xds/experimental" // To install the xds resolvers and balancers.
 )
 
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index de3e1ebf8093..009ad26f9d60 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -31,7 +31,7 @@ import (
 	"google.golang.org/grpc/grpclog"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/peer"
-	_ "google.golang.org/grpc/xds/experimental"
+	_ "google.golang.org/grpc/xds"
 )
 
 type statsWatcherKey struct {
diff --git a/xds/experimental/xds_experimental.go b/xds/experimental/xds_experimental.go
index 058beb65c70f..b7ac3019e56c 100644
--- a/xds/experimental/xds_experimental.go
+++ b/xds/experimental/xds_experimental.go
@@ -16,13 +16,14 @@
  *
  */
 
-// Package experimental contains xds implementation, still in experimental
-// state. Users only need to import this package to get all xds functionality.
-// Things are expected to change fast until we get to a stable state, at
-// which point, all this will be moved to the xds package.
+// Package experimental contains xds implementation. Users only need to import
+// this package to get all xds functionality.
+//
+// Deprecated: import package xds instead.
 package experimental
 
+// TODO: remove this package after one release.
+
 import (
-	_ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers.
-	_ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver
+	_ "google.golang.org/grpc/xds" // Register the balancers and resolvers.
 )
diff --git a/xds/xds.go b/xds/xds.go
new file mode 100644
index 000000000000..e077a69476eb
--- /dev/null
+++ b/xds/xds.go
@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds contains xds implementation. Users need to import this package to
+// get all xds functionality.
+//
+// See https://github.com/grpc/grpc-go/tree/master/examples/features/xds for
+// example.
+package xds
+
+import (
+	_ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers.
+	_ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver
+)

From 6c9e30c09db2765b8ac80ad109077d95e07ccef1 Mon Sep 17 00:00:00 2001
From: Damien Neil <neild@users.noreply.github.com>
Date: Wed, 27 May 2020 14:15:25 -0700
Subject: [PATCH 080/481] cmd/protoc-gen-go-grpc: add code generator (#3453)

---
 cmd/protoc-gen-go-grpc/go.mod  |   5 +
 cmd/protoc-gen-go-grpc/go.sum  |  16 ++
 cmd/protoc-gen-go-grpc/grpc.go | 405 +++++++++++++++++++++++++++++++++
 cmd/protoc-gen-go-grpc/main.go |  47 ++++
 4 files changed, 473 insertions(+)
 create mode 100644 cmd/protoc-gen-go-grpc/go.mod
 create mode 100644 cmd/protoc-gen-go-grpc/go.sum
 create mode 100644 cmd/protoc-gen-go-grpc/grpc.go
 create mode 100644 cmd/protoc-gen-go-grpc/main.go

diff --git a/cmd/protoc-gen-go-grpc/go.mod b/cmd/protoc-gen-go-grpc/go.mod
new file mode 100644
index 000000000000..d0cfd8ebf56f
--- /dev/null
+++ b/cmd/protoc-gen-go-grpc/go.mod
@@ -0,0 +1,5 @@
+module google.golang.org/grpc/cmd/protoc-gen-go-grpc
+
+go 1.9
+
+require google.golang.org/protobuf v1.23.0
diff --git a/cmd/protoc-gen-go-grpc/go.sum b/cmd/protoc-gen-go-grpc/go.sum
new file mode 100644
index 000000000000..3e741dc38700
--- /dev/null
+++ b/cmd/protoc-gen-go-grpc/go.sum
@@ -0,0 +1,16 @@
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
new file mode 100644
index 000000000000..38d444fc7a1d
--- /dev/null
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -0,0 +1,405 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"google.golang.org/protobuf/compiler/protogen"
+
+	"google.golang.org/protobuf/types/descriptorpb"
+)
+
+const (
+	contextPackage = protogen.GoImportPath("context")
+	grpcPackage    = protogen.GoImportPath("google.golang.org/grpc")
+	codesPackage   = protogen.GoImportPath("google.golang.org/grpc/codes")
+	statusPackage  = protogen.GoImportPath("google.golang.org/grpc/status")
+)
+
+// generateFile generates a _grpc.pb.go file containing gRPC service definitions.
+func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile {
+	if len(file.Services) == 0 {
+		return nil
+	}
+	filename := file.GeneratedFilenamePrefix + "_grpc.pb.go"
+	g := gen.NewGeneratedFile(filename, file.GoImportPath)
+	g.P("// Code generated by protoc-gen-go-grpc. DO NOT EDIT.")
+	g.P()
+	g.P("package ", file.GoPackageName)
+	g.P()
+	generateFileContent(gen, file, g)
+	return g
+}
+
+// generateFileContent generates the gRPC service definitions, excluding the package statement.
+func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile) {
+	if len(file.Services) == 0 {
+		return
+	}
+
+	g.P("// This is a compile-time assertion to ensure that this generated file")
+	g.P("// is compatible with the grpc package it is being compiled against.")
+	g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion6"))
+	g.P()
+	for _, service := range file.Services {
+		genService(gen, file, g, service)
+	}
+}
+
+func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	clientName := service.GoName + "Client"
+
+	g.P("// ", clientName, " is the client API for ", service.GoName, " service.")
+	g.P("//")
+	g.P("// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.")
+
+	// Client interface.
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.Annotate(clientName, service.Location)
+	g.P("type ", clientName, " interface {")
+	for _, method := range service.Methods {
+		g.Annotate(clientName+"."+method.GoName, method.Location)
+		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
+			g.P(deprecationComment)
+		}
+		g.P(method.Comments.Leading,
+			clientSignature(g, method))
+	}
+	g.P("}")
+	g.P()
+
+	// Client structure.
+	g.P("type ", unexport(clientName), " struct {")
+	g.P("cc ", grpcPackage.Ident("ClientConnInterface"))
+	g.P("}")
+	g.P()
+
+	// NewClient factory.
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P(deprecationComment)
+	}
+	g.P("func New", clientName, " (cc ", grpcPackage.Ident("ClientConnInterface"), ") ", clientName, " {")
+	g.P("return &", unexport(clientName), "{cc}")
+	g.P("}")
+	g.P()
+
+	var methodIndex, streamIndex int
+	// Client method implementations.
+	for _, method := range service.Methods {
+		if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() {
+			// Unary RPC method
+			genClientMethod(gen, file, g, method, methodIndex)
+			methodIndex++
+		} else {
+			// Streaming RPC method
+			genClientMethod(gen, file, g, method, streamIndex)
+			streamIndex++
+		}
+	}
+
+	// Server interface.
+	serverType := service.GoName + "Server"
+	g.P("// ", serverType, " is the server API for ", service.GoName, " service.")
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.Annotate(serverType, service.Location)
+	g.P("type ", serverType, " interface {")
+	for _, method := range service.Methods {
+		g.Annotate(serverType+"."+method.GoName, method.Location)
+		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
+			g.P(deprecationComment)
+		}
+		g.P(method.Comments.Leading,
+			serverSignature(g, method))
+	}
+	g.P("}")
+	g.P()
+
+	// Server Unimplemented struct for forward compatibility.
+	g.P("// Unimplemented", serverType, " can be embedded to have forward compatible implementations.")
+	g.P("type Unimplemented", serverType, " struct {")
+	g.P("}")
+	g.P()
+	for _, method := range service.Methods {
+		nilArg := ""
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			nilArg = "nil,"
+		}
+		g.P("func (*Unimplemented", serverType, ") ", serverSignature(g, method), "{")
+		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
+		g.P("}")
+	}
+	g.P()
+
+	// Server registration.
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P(deprecationComment)
+	}
+	serviceDescVar := "_" + service.GoName + "_serviceDesc"
+	g.P("func Register", service.GoName, "Server(s *", grpcPackage.Ident("Server"), ", srv ", serverType, ") {")
+	g.P("s.RegisterService(&", serviceDescVar, `, srv)`)
+	g.P("}")
+	g.P()
+
+	// Server handler implementations.
+	var handlerNames []string
+	for _, method := range service.Methods {
+		hname := genServerMethod(gen, file, g, method)
+		handlerNames = append(handlerNames, hname)
+	}
+
+	// Service descriptor.
+	g.P("var ", serviceDescVar, " = ", grpcPackage.Ident("ServiceDesc"), " {")
+	g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",")
+	g.P("HandlerType: (*", serverType, ")(nil),")
+	g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{")
+	for i, method := range service.Methods {
+		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+			continue
+		}
+		g.P("{")
+		g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",")
+		g.P("Handler: ", handlerNames[i], ",")
+		g.P("},")
+	}
+	g.P("},")
+	g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{")
+	for i, method := range service.Methods {
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			continue
+		}
+		g.P("{")
+		g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
+		g.P("Handler: ", handlerNames[i], ",")
+		if method.Desc.IsStreamingServer() {
+			g.P("ServerStreams: true,")
+		}
+		if method.Desc.IsStreamingClient() {
+			g.P("ClientStreams: true,")
+		}
+		g.P("},")
+	}
+	g.P("},")
+	g.P("Metadata: \"", file.Desc.Path(), "\",")
+	g.P("}")
+	g.P()
+}
+
+func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
+	s := method.GoName + "(ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context"))
+	if !method.Desc.IsStreamingClient() {
+		s += ", in *" + g.QualifiedGoIdent(method.Input.GoIdent)
+	}
+	s += ", opts ..." + g.QualifiedGoIdent(grpcPackage.Ident("CallOption")) + ") ("
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		s += "*" + g.QualifiedGoIdent(method.Output.GoIdent)
+	} else {
+		s += method.Parent.GoName + "_" + method.GoName + "Client"
+	}
+	s += ", error)"
+	return s
+}
+
+func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, index int) {
+	service := method.Parent
+	sname := fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name())
+
+	if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
+		g.P(deprecationComment)
+	}
+	g.P("func (c *", unexport(service.GoName), "Client) ", clientSignature(g, method), "{")
+	if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() {
+		g.P("out := new(", method.Output.GoIdent, ")")
+		g.P(`err := c.cc.Invoke(ctx, "`, sname, `", in, out, opts...)`)
+		g.P("if err != nil { return nil, err }")
+		g.P("return out, nil")
+		g.P("}")
+		g.P()
+		return
+	}
+	streamType := unexport(service.GoName) + method.GoName + "Client"
+	serviceDescVar := "_" + service.GoName + "_serviceDesc"
+	g.P("stream, err := c.cc.NewStream(ctx, &", serviceDescVar, ".Streams[", index, `], "`, sname, `", opts...)`)
+	g.P("if err != nil { return nil, err }")
+	g.P("x := &", streamType, "{stream}")
+	if !method.Desc.IsStreamingClient() {
+		g.P("if err := x.ClientStream.SendMsg(in); err != nil { return nil, err }")
+		g.P("if err := x.ClientStream.CloseSend(); err != nil { return nil, err }")
+	}
+	g.P("return x, nil")
+	g.P("}")
+	g.P()
+
+	genSend := method.Desc.IsStreamingClient()
+	genRecv := method.Desc.IsStreamingServer()
+	genCloseAndRecv := !method.Desc.IsStreamingServer()
+
+	// Stream auxiliary types and methods.
+	g.P("type ", service.GoName, "_", method.GoName, "Client interface {")
+	if genSend {
+		g.P("Send(*", method.Input.GoIdent, ") error")
+	}
+	if genRecv {
+		g.P("Recv() (*", method.Output.GoIdent, ", error)")
+	}
+	if genCloseAndRecv {
+		g.P("CloseAndRecv() (*", method.Output.GoIdent, ", error)")
+	}
+	g.P(grpcPackage.Ident("ClientStream"))
+	g.P("}")
+	g.P()
+
+	g.P("type ", streamType, " struct {")
+	g.P(grpcPackage.Ident("ClientStream"))
+	g.P("}")
+	g.P()
+
+	if genSend {
+		g.P("func (x *", streamType, ") Send(m *", method.Input.GoIdent, ") error {")
+		g.P("return x.ClientStream.SendMsg(m)")
+		g.P("}")
+		g.P()
+	}
+	if genRecv {
+		g.P("func (x *", streamType, ") Recv() (*", method.Output.GoIdent, ", error) {")
+		g.P("m := new(", method.Output.GoIdent, ")")
+		g.P("if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err }")
+		g.P("return m, nil")
+		g.P("}")
+		g.P()
+	}
+	if genCloseAndRecv {
+		g.P("func (x *", streamType, ") CloseAndRecv() (*", method.Output.GoIdent, ", error) {")
+		g.P("if err := x.ClientStream.CloseSend(); err != nil { return nil, err }")
+		g.P("m := new(", method.Output.GoIdent, ")")
+		g.P("if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err }")
+		g.P("return m, nil")
+		g.P("}")
+		g.P()
+	}
+}
+
+func serverSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
+	var reqArgs []string
+	ret := "error"
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context")))
+		ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)"
+	}
+	if !method.Desc.IsStreamingClient() {
+		reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent))
+	}
+	if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+		reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server")
+	}
+	return method.GoName + "(" + strings.Join(reqArgs, ", ") + ") " + ret
+}
+
+func genServerMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method) string {
+	service := method.Parent
+	hname := fmt.Sprintf("_%s_%s_Handler", service.GoName, method.GoName)
+
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		g.P("func ", hname, "(srv interface{}, ctx ", contextPackage.Ident("Context"), ", dec func(interface{}) error, interceptor ", grpcPackage.Ident("UnaryServerInterceptor"), ") (interface{}, error) {")
+		g.P("in := new(", method.Input.GoIdent, ")")
+		g.P("if err := dec(in); err != nil { return nil, err }")
+		g.P("if interceptor == nil { return srv.(", service.GoName, "Server).", method.GoName, "(ctx, in) }")
+		g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{")
+		g.P("Server: srv,")
+		g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.GoName)), ",")
+		g.P("}")
+		g.P("handler := func(ctx ", contextPackage.Ident("Context"), ", req interface{}) (interface{}, error) {")
+		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")
+		g.P("}")
+		g.P("return interceptor(ctx, in, info, handler)")
+		g.P("}")
+		g.P()
+		return hname
+	}
+	streamType := unexport(service.GoName) + method.GoName + "Server"
+	g.P("func ", hname, "(srv interface{}, stream ", grpcPackage.Ident("ServerStream"), ") error {")
+	if !method.Desc.IsStreamingClient() {
+		g.P("m := new(", method.Input.GoIdent, ")")
+		g.P("if err := stream.RecvMsg(m); err != nil { return err }")
+		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(m, &", streamType, "{stream})")
+	} else {
+		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(&", streamType, "{stream})")
+	}
+	g.P("}")
+	g.P()
+
+	genSend := method.Desc.IsStreamingServer()
+	genSendAndClose := !method.Desc.IsStreamingServer()
+	genRecv := method.Desc.IsStreamingClient()
+
+	// Stream auxiliary types and methods.
+	g.P("type ", service.GoName, "_", method.GoName, "Server interface {")
+	if genSend {
+		g.P("Send(*", method.Output.GoIdent, ") error")
+	}
+	if genSendAndClose {
+		g.P("SendAndClose(*", method.Output.GoIdent, ") error")
+	}
+	if genRecv {
+		g.P("Recv() (*", method.Input.GoIdent, ", error)")
+	}
+	g.P(grpcPackage.Ident("ServerStream"))
+	g.P("}")
+	g.P()
+
+	g.P("type ", streamType, " struct {")
+	g.P(grpcPackage.Ident("ServerStream"))
+	g.P("}")
+	g.P()
+
+	if genSend {
+		g.P("func (x *", streamType, ") Send(m *", method.Output.GoIdent, ") error {")
+		g.P("return x.ServerStream.SendMsg(m)")
+		g.P("}")
+		g.P()
+	}
+	if genSendAndClose {
+		g.P("func (x *", streamType, ") SendAndClose(m *", method.Output.GoIdent, ") error {")
+		g.P("return x.ServerStream.SendMsg(m)")
+		g.P("}")
+		g.P()
+	}
+	if genRecv {
+		g.P("func (x *", streamType, ") Recv() (*", method.Input.GoIdent, ", error) {")
+		g.P("m := new(", method.Input.GoIdent, ")")
+		g.P("if err := x.ServerStream.RecvMsg(m); err != nil { return nil, err }")
+		g.P("return m, nil")
+		g.P("}")
+		g.P()
+	}
+
+	return hname
+}
+
+const deprecationComment = "// Deprecated: Do not use."
+
+func unexport(s string) string { return strings.ToLower(s[:1]) + s[1:] }
diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
new file mode 100644
index 000000000000..dac0be207e23
--- /dev/null
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -0,0 +1,47 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// protoc-gen-go-grpc is a plugin for the Google protocol buffer compiler to
+// generate Go code. Install it by building this program and making it
+// accessible within your PATH with the name:
+//	protoc-gen-go-grpc
+//
+// The 'go-grpc' suffix becomes part of the argument for the protocol compiler,
+// such that it can be invoked as:
+//	protoc --go-grpc_out=. path/to/file.proto
+//
+// This generates Go service definitions for the protocol buffer defined by
+// file.proto.  With that input, the output will be written to:
+//	path/to/file_grpc.pb.go
+package main
+
+import (
+	"google.golang.org/protobuf/compiler/protogen"
+)
+
+func main() {
+	protogen.Options{}.Run(func(gen *protogen.Plugin) error {
+		for _, f := range gen.Files {
+			if !f.Generate {
+				continue
+			}
+			generateFile(gen, f)
+		}
+		return nil
+	})
+}

From acd5f849045e8584f4c58e511b9abb8433b34e3b Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 28 May 2020 09:54:18 -0700
Subject: [PATCH 081/481] xds: handle errors in eds balancer (#3645)

---
 xds/internal/balancer/edsbalancer/eds.go      |  43 ++++--
 xds/internal/balancer/edsbalancer/eds_impl.go |  14 ++
 .../balancer/edsbalancer/eds_impl_priority.go |   5 +-
 .../edsbalancer/eds_impl_priority_test.go     |   4 +-
 .../balancer/edsbalancer/eds_impl_test.go     |  65 +++++++++
 xds/internal/balancer/edsbalancer/eds_test.go | 131 +++++++++++++++++-
 .../edsbalancer/xds_client_wrapper.go         |  17 ++-
 xds/internal/client/errors.go                 |  60 ++++++++
 xds/internal/testutils/fakeclient/client.go   |   7 +
 9 files changed, 328 insertions(+), 18 deletions(-)
 create mode 100644 xds/internal/client/errors.go

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 1cf9b16ed84a..59bbe5fb4eec 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -145,6 +145,32 @@ func (x *edsBalancer) run() {
 	}
 }
 
+// handleErrorFromUpdate handles both the error from parent ClientConn (from CDS
+// balancer) and the error from xds client (from the watcher). fromParent is
+// true if error is from parent ClientConn.
+//
+// If the error is connection error, it should be handled for fallback purposes.
+//
+// If the error is resource-not-found:
+// - If it's from CDS balancer (shows as a resolver error), it means LDS or CDS
+// resources were removed. The EDS watch should be canceled.
+// - If it's from xds client, it means EDS resource were removed. The EDS
+// watcher should keep watching.
+// In both cases, the sub-balancers will be closed, and the future picks will
+// fail.
+func (x *edsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
+	if xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound {
+		if fromParent {
+			// This is an error from the parent ClientConn (can be the parent
+			// CDS balancer), and is a resource-not-found error. This means the
+			// resource (can be either LDS or CDS) was removed. Stop the EDS
+			// watch.
+			x.client.cancelWatch()
+		}
+		x.edsImpl.handleEDSResponse(xdsclient.EndpointsUpdate{})
+	}
+}
+
 func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 	switch u := update.(type) {
 	case *subConnStateUpdate:
@@ -175,6 +201,8 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 		}
 
 		x.config = cfg
+	case error:
+		x.handleErrorFromUpdate(u, true)
 	default:
 		// unreachable path
 		panic("wrong update type")
@@ -182,9 +210,8 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 }
 
 func (x *edsBalancer) handleXDSClientUpdate(update *edsUpdate) {
-	if update.err != nil {
-		// TODO: handle errors from EDS callback. E.g. if CDS resource is
-		// removed, the EDS watch should be canceled.
+	if err := update.err; err != nil {
+		x.handleErrorFromUpdate(err, false)
 		return
 	}
 	x.edsImpl.handleEDSResponse(update.resp)
@@ -206,11 +233,11 @@ func (x *edsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 	}
 }
 
-func (x *edsBalancer) ResolverError(error) {
-	// TODO: Need to distinguish between connection errors and resource removed
-	// errors. For the former, we will need to handle it later on for fallback.
-	// For the latter, handle it by stopping the watch, closing sub-balancers
-	// and pickers.
+func (x *edsBalancer) ResolverError(err error) {
+	select {
+	case x.grpcUpdate <- err:
+	case <-x.ctx.Done():
+	}
 }
 
 func (x *edsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 0aeb6cc328ad..979eee45bb76 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -18,11 +18,13 @@ package edsbalancer
 
 import (
 	"encoding/json"
+	"reflect"
 	"sync"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/balancer/weightedroundrobin"
 	"google.golang.org/grpc/codes"
@@ -66,6 +68,7 @@ type edsBalancerImpl struct {
 	subBalancerBuilder   balancer.Builder
 	loadStore            lrs.Store
 	priorityToLocalities map[priorityType]*balancerGroupWithConfig
+	respReceived         bool
 
 	// There's no need to hold any mutexes at the same time. The order to take
 	// mutex should be: priorityMu > subConnMu, but this is implicit via
@@ -183,6 +186,17 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
 	//     - socketAddress.GetNamedPort(), socketAddress.GetResolverName()
 	//     - resolve endpoint's name with another resolver
 
+	// If the first EDS update is an empty update, nothing is changing from the
+	// previous update (which is the default empty value). We need to explicitly
+	// handle first update being empty, and send a transient failure picker.
+	//
+	// TODO: define Equal() on type EndpointUpdate to avoid DeepEqual. And do
+	// the same for the other types.
+	if !edsImpl.respReceived && reflect.DeepEqual(edsResp, xdsclient.EndpointsUpdate{}) {
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(errAllPrioritiesRemoved)})
+	}
+	edsImpl.respReceived = true
+
 	edsImpl.updateDrops(edsResp.Drops)
 
 	// Filter out all localities with weight 0.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index a279ddc64374..c2ce31238542 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -18,6 +18,7 @@
 package edsbalancer
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -27,6 +28,8 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+var errAllPrioritiesRemoved = errors.New("eds: no locality is provided, all priorities are removed")
+
 // handlePriorityChange handles priority after EDS adds/removes a
 // priority.
 //
@@ -46,7 +49,7 @@ func (edsImpl *edsBalancerImpl) handlePriorityChange() {
 	// Everything was removed by EDS.
 	if !edsImpl.priorityLowest.isSet() {
 		edsImpl.priorityInUse = newPriorityTypeUnset()
-		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(balancer.ErrTransientFailure)})
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(errAllPrioritiesRemoved)})
 		return
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index c98f7205fc87..d9b9ff4a8145 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -553,8 +553,8 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	// Test pick return TransientFailure.
 	pFail := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
-		if _, err := pFail.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure {
-			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
+		if _, err := pFail.Pick(balancer.PickInfo{}); err != errAllPrioritiesRemoved {
+			t.Fatalf("want pick error %v, got %v", errAllPrioritiesRemoved, err)
 		}
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index d308f97c8f10..3f47b720ebbf 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -18,6 +18,7 @@ package edsbalancer
 
 import (
 	"fmt"
+	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -386,6 +387,70 @@ func (s) TestClose(t *testing.T) {
 	edsb.close()
 }
 
+// TestEDS_EmptyUpdate covers the cases when eds impl receives an empty update.
+//
+// It should send an error picker with transient failure to the parent.
+func (s) TestEDS_EmptyUpdate(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+
+	// The first update is an empty update.
+	edsb.handleEDSResponse(xdsclient.EndpointsUpdate{})
+	// Pick should fail with transient failure, and all priority removed error.
+	perr0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		_, err := perr0.Pick(balancer.PickInfo{})
+		if !reflect.DeepEqual(err, errAllPrioritiesRemoved) {
+			t.Fatalf("picker.Pick, got error %v, want error %v", err, errAllPrioritiesRemoved)
+		}
+	}
+
+	// One locality with one backend.
+	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+
+	sc1 := <-cc.NewSubConnCh
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
+
+	// Pick with only the first backend.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !reflect.DeepEqual(gotSCSt.SubConn, sc1) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+	}
+
+	edsb.handleEDSResponse(xdsclient.EndpointsUpdate{})
+	// Pick should fail with transient failure, and all priority removed error.
+	perr1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		_, err := perr1.Pick(balancer.PickInfo{})
+		if !reflect.DeepEqual(err, errAllPrioritiesRemoved) {
+			t.Fatalf("picker.Pick, got error %v, want error %v", err, errAllPrioritiesRemoved)
+		}
+	}
+
+	// Handle another update with priorities and localities.
+	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+
+	sc2 := <-cc.NewSubConnCh
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
+
+	// Pick with only the first backend.
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
+		if !reflect.DeepEqual(gotSCSt.SubConn, sc2) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
+		}
+	}
+}
+
 // Create XDS balancer, and update sub-balancer before handling eds responses.
 // Then switch between round-robin and test-const-balancer after handling first
 // eds response.
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index ed1e44de902e..02cb9263e4c9 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -22,6 +22,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"testing"
 
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
@@ -97,6 +98,7 @@ type fakeEDSBalancer struct {
 	cc                 balancer.ClientConn
 	childPolicy        *testutils.Channel
 	subconnStateChange *testutils.Channel
+	edsUpdate          *testutils.Channel
 	loadStore          lrs.Store
 }
 
@@ -108,7 +110,9 @@ func (f *fakeEDSBalancer) handleChildPolicy(name string, config json.RawMessage)
 	f.childPolicy.Send(&loadBalancingConfig{Name: name, Config: config})
 }
 
-func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {}
+func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {
+	f.edsUpdate.Send(edsResp)
+}
 
 func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {}
 
@@ -138,11 +142,24 @@ func (f *fakeEDSBalancer) waitForSubConnStateChange(wantState *scStateChange) er
 	return nil
 }
 
+func (f *fakeEDSBalancer) waitForEDSResponse(wantUpdate xdsclient.EndpointsUpdate) error {
+	val, err := f.edsUpdate.Receive()
+	if err != nil {
+		return fmt.Errorf("error waiting for edsUpdate: %v", err)
+	}
+	gotUpdate := val.(xdsclient.EndpointsUpdate)
+	if !reflect.DeepEqual(gotUpdate, wantUpdate) {
+		return fmt.Errorf("got edsUpdate %+v, want %+v", gotUpdate, wantUpdate)
+	}
+	return nil
+}
+
 func newFakeEDSBalancer(cc balancer.ClientConn, loadStore lrs.Store) edsBalancerImplInterface {
 	return &fakeEDSBalancer{
 		cc:                 cc,
 		childPolicy:        testutils.NewChannelWithSize(10),
 		subconnStateChange: testutils.NewChannelWithSize(10),
+		edsUpdate:          testutils.NewChannelWithSize(10),
 		loadStore:          loadStore,
 	}
 }
@@ -399,6 +416,118 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 	edsLB.waitForSubConnStateChange(&scStateChange{sc: fsc, state: state})
 }
 
+// TestErrorFromXDSClientUpdate verifies that errros from xdsclient update are
+// handled correctly.
+//
+// If it's resource-not-found, watch will NOT be canceled, the EDS impl will
+// receive an empty EDS update, and new RPCs will fail.
+//
+// If it's connection error, nothing will happen. This will need to change to
+// handle fallback.
+func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
+	edsLBCh := testutils.NewChannel()
+	xdsClientCh := testutils.NewChannel()
+	cancel := setup(edsLBCh, xdsClientCh)
+	defer cancel()
+
+	builder := balancer.Get(edsName)
+	cc := newNoopTestClientConn()
+	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
+	if !ok {
+		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	}
+	defer edsB.Close()
+
+	edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{
+			BalancerName:   testBalancerNameFooBar,
+			EDSServiceName: testEDSClusterName,
+		},
+	})
+
+	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
+	edsLB := waitForNewEDSLB(t, edsLBCh)
+	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
+	}
+
+	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, connectionErr)
+	if err := xdsC.WaitForCancelEDSWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil {
+		t.Fatal("eds impl got EDS resp, want timeout error")
+	}
+
+	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error")
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, resourceErr)
+	// Even if error is resource not found, watch shouldn't be canceled, because
+	// this is an EDS resource removed (and xds client actually never sends this
+	// error, but we still handles it).
+	if err := xdsC.WaitForCancelEDSWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+		t.Fatalf("eds impl expecting empty update, got %v", err)
+	}
+}
+
+// TestErrorFromResolver verifies that resolver errors are handled correctly.
+//
+// If it's resource-not-found, watch will be canceled, the EDS impl will receive
+// an empty EDS update, and new RPCs will fail.
+//
+// If it's connection error, nothing will happen. This will need to change to
+// handle fallback.
+func (s) TestErrorFromResolver(t *testing.T) {
+	edsLBCh := testutils.NewChannel()
+	xdsClientCh := testutils.NewChannel()
+	cancel := setup(edsLBCh, xdsClientCh)
+	defer cancel()
+
+	builder := balancer.Get(edsName)
+	cc := newNoopTestClientConn()
+	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
+	if !ok {
+		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	}
+	defer edsB.Close()
+
+	edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{
+			BalancerName:   testBalancerNameFooBar,
+			EDSServiceName: testEDSClusterName,
+		},
+	})
+
+	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
+	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
+	edsLB := waitForNewEDSLB(t, edsLBCh)
+	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
+	}
+
+	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
+	edsB.ResolverError(connectionErr)
+	if err := xdsC.WaitForCancelEDSWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil {
+		t.Fatal("eds impl got EDS resp, want timeout error")
+	}
+
+	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error")
+	edsB.ResolverError(resourceErr)
+	if err := xdsC.WaitForCancelEDSWatch(); err != nil {
+		t.Fatalf("want watch to be canceled, waitForCancel failed: %v", err)
+	}
+	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
+	}
+}
+
 func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 	const testEDSName = "eds.service"
 	var testLRSName = "lrs.server"
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index 15fdd7d4d36e..0f987a948fb9 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -258,20 +258,25 @@ func (c *xdsclientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attr
 	}
 }
 
-func (c *xdsclientWrapper) close() {
-	if c.xdsclient != nil && c.balancerName != "" {
-		// Only close xdsclient if it's not from attributes.
-		c.xdsclient.Close()
-	}
-
+func (c *xdsclientWrapper) cancelWatch() {
+	c.loadReportServer = nil
 	if c.cancelLoadReport != nil {
 		c.cancelLoadReport()
 	}
+	c.edsServiceName = ""
 	if c.cancelEndpointsWatch != nil {
 		c.cancelEndpointsWatch()
 	}
 }
 
+func (c *xdsclientWrapper) close() {
+	c.cancelWatch()
+	if c.xdsclient != nil && c.balancerName != "" {
+		// Only close xdsclient if it's not from attributes.
+		c.xdsclient.Close()
+	}
+}
+
 // defaultDialCreds builds a DialOption containing the credentials to be used
 // while talking to the xDS server (this is done only if the xds bootstrap
 // process does not return any credentials to use). If the parent channel
diff --git a/xds/internal/client/errors.go b/xds/internal/client/errors.go
new file mode 100644
index 000000000000..34ae2738db00
--- /dev/null
+++ b/xds/internal/client/errors.go
@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import "fmt"
+
+// ErrorType is the type of the error that the watcher will receive from the xds
+// client.
+type ErrorType int
+
+const (
+	// ErrorTypeUnknown indicates the error doesn't have a specific type. It is
+	// the default value, and is returned if the error is not an xds error.
+	ErrorTypeUnknown ErrorType = iota
+	// ErrorTypeConnection indicates a connection error from the gRPC client.
+	ErrorTypeConnection
+	// ErrorTypeResourceNotFound indicates a resource is not found from the xds
+	// response. It's typically returned if the resource is removed in the xds
+	// server.
+	ErrorTypeResourceNotFound
+)
+
+type xdsClientError struct {
+	t    ErrorType
+	desc string
+}
+
+func (e *xdsClientError) Error() string {
+	return e.desc
+}
+
+// NewErrorf creates an xds client error. The callbacks are called with this
+// error, to pass additional information about the error.
+func NewErrorf(t ErrorType, format string, args ...interface{}) error {
+	return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)}
+}
+
+// ErrType returns the error's type.
+func ErrType(e error) ErrorType {
+	if xe, ok := e.(*xdsClientError); ok {
+		return xe.t
+	}
+	return ErrorTypeUnknown
+}
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 922fab1e4371..aacade2fbd0a 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -143,6 +143,13 @@ func (xdsC *Client) InvokeWatchEDSCallback(update xdsclient.EndpointsUpdate, err
 	xdsC.edsCb(update, err)
 }
 
+// WaitForCancelEDSWatch waits for a EDS watch to be cancelled within a
+// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise.
+func (xdsC *Client) WaitForCancelEDSWatch() error {
+	_, err := xdsC.edsCancelCh.Receive()
+	return err
+}
+
 // ReportLoadArgs wraps the arguments passed to ReportLoad.
 type ReportLoadArgs struct {
 	// Server is the name of the server to which the load is reported.

From e0ec2b8320c9b9f0f32f850b4b6075496dca594c Mon Sep 17 00:00:00 2001
From: Bill Zong <billzong2012@gmail.com>
Date: Fri, 29 May 2020 02:08:21 +0800
Subject: [PATCH 082/481] doc: fix quickstart hyperlink in README (#3648)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 751aa6d80a92..249cd2063279 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
 The Go implementation of [gRPC](https://grpc.io/): A high performance, open
 source, general RPC framework that puts mobile and HTTP/2 first. For more
 information see the [gRPC Quick Start:
-Go](https://grpc.io/docs/quickstart/go.html) guide.
+Go](https://grpc.io/docs/languages/go/quickstart/) guide.
 
 Installation
 ------------

From f818fd2a025ea8805930f5cc1f6609b35bab7599 Mon Sep 17 00:00:00 2001
From: Damien Neil <neild@users.noreply.github.com>
Date: Thu, 28 May 2020 13:52:49 -0700
Subject: [PATCH 083/481] all: regenerate .pb.go files using
 cmd/protoc-gen-go-grpc (#3646)

Change regenerate.sh to use cmd/protoc-gen-go-grpc and rerun it.
This splits the gRPC service definitions into separate files, but
otherwise has no changes.
---
 .../grpclb/grpc_lb_v1/load_balancer.pb.go     | 118 ----
 .../grpc_lb_v1/load_balancer_grpc.pb.go       | 120 ++++
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  86 ---
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       |  88 +++
 benchmark/grpc_testing/services.pb.go         | 506 ----------------
 benchmark/grpc_testing/services_grpc.pb.go    | 508 ++++++++++++++++
 channelz/grpc_channelz_v1/channelz.pb.go      | 316 ----------
 channelz/grpc_channelz_v1/channelz_grpc.pb.go | 318 ++++++++++
 .../internal/proto/grpc_gcp/handshaker.pb.go  | 128 ----
 .../proto/grpc_gcp/handshaker_grpc.pb.go      | 130 ++++
 examples/features/proto/echo/echo.pb.go       | 294 ---------
 examples/features/proto/echo/echo_grpc.pb.go  | 296 +++++++++
 .../helloworld/helloworld/helloworld.pb.go    |  86 ---
 .../helloworld/helloworld_grpc.pb.go          |  88 +++
 .../route_guide/routeguide/route_guide.pb.go  | 326 ----------
 .../routeguide/route_guide_grpc.pb.go         | 328 ++++++++++
 health/grpc_health_v1/health.pb.go            | 182 ------
 health/grpc_health_v1/health_grpc.pb.go       | 184 ++++++
 interop/grpc_testing/test.pb.go               | 566 -----------------
 interop/grpc_testing/test_grpc.pb.go          | 568 ++++++++++++++++++
 profiling/proto/service.pb.go                 | 126 ----
 profiling/proto/service_grpc.pb.go            | 128 ++++
 .../grpc_reflection_v1alpha/reflection.pb.go  | 120 ----
 .../reflection_grpc.pb.go                     | 122 ++++
 reflection/grpc_testing/test.pb.go            | 153 -----
 reflection/grpc_testing/test_grpc.pb.go       | 155 +++++
 regenerate.sh                                 |   7 +-
 stats/grpc_testing/test.pb.go                 | 300 ---------
 stats/grpc_testing/test_grpc.pb.go            | 302 ++++++++++
 stress/grpc_testing/metrics.pb.go             | 154 -----
 stress/grpc_testing/metrics_grpc.pb.go        | 156 +++++
 test/grpc_testing/test.pb.go                  | 418 -------------
 test/grpc_testing/test_grpc.pb.go             | 420 +++++++++++++
 33 files changed, 3916 insertions(+), 3881 deletions(-)
 create mode 100644 balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
 create mode 100644 balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
 create mode 100644 benchmark/grpc_testing/services_grpc.pb.go
 create mode 100644 channelz/grpc_channelz_v1/channelz_grpc.pb.go
 create mode 100644 credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
 create mode 100644 examples/features/proto/echo/echo_grpc.pb.go
 create mode 100644 examples/helloworld/helloworld/helloworld_grpc.pb.go
 create mode 100644 examples/route_guide/routeguide/route_guide_grpc.pb.go
 create mode 100644 health/grpc_health_v1/health_grpc.pb.go
 create mode 100644 interop/grpc_testing/test_grpc.pb.go
 create mode 100644 profiling/proto/service_grpc.pb.go
 create mode 100644 reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
 create mode 100644 reflection/grpc_testing/test_grpc.pb.go
 create mode 100644 stats/grpc_testing/test_grpc.pb.go
 create mode 100644 stress/grpc_testing/metrics_grpc.pb.go
 create mode 100644 test/grpc_testing/test_grpc.pb.go

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
index b59191ac5825..7381dfc1ae4f 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
@@ -4,14 +4,10 @@
 package grpc_lb_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -642,117 +638,3 @@ var fileDescriptor_7cd3f6d792743fdf = []byte{
 	0x6d, 0xe1, 0xbe, 0xfb, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x47, 0x55, 0xac, 0xab, 0x06, 0x00,
 	0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// LoadBalancerClient is the client API for LoadBalancer service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type LoadBalancerClient interface {
-	// Bidirectional rpc to get a list of servers.
-	BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error)
-}
-
-type loadBalancerClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
-	return &loadBalancerClient{cc}
-}
-
-func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &loadBalancerBalanceLoadClient{stream}
-	return x, nil
-}
-
-type LoadBalancer_BalanceLoadClient interface {
-	Send(*LoadBalanceRequest) error
-	Recv() (*LoadBalanceResponse, error)
-	grpc.ClientStream
-}
-
-type loadBalancerBalanceLoadClient struct {
-	grpc.ClientStream
-}
-
-func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
-	m := new(LoadBalanceResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// LoadBalancerServer is the server API for LoadBalancer service.
-type LoadBalancerServer interface {
-	// Bidirectional rpc to get a list of servers.
-	BalanceLoad(LoadBalancer_BalanceLoadServer) error
-}
-
-// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations.
-type UnimplementedLoadBalancerServer struct {
-}
-
-func (*UnimplementedLoadBalancerServer) BalanceLoad(srv LoadBalancer_BalanceLoadServer) error {
-	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
-}
-
-func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
-	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
-}
-
-func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream})
-}
-
-type LoadBalancer_BalanceLoadServer interface {
-	Send(*LoadBalanceResponse) error
-	Recv() (*LoadBalanceRequest, error)
-	grpc.ServerStream
-}
-
-type loadBalancerBalanceLoadServer struct {
-	grpc.ServerStream
-}
-
-func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
-	m := new(LoadBalanceRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.lb.v1.LoadBalancer",
-	HandlerType: (*LoadBalancerServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "BalanceLoad",
-			Handler:       _LoadBalancer_BalanceLoad_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "grpc/lb/v1/load_balancer.proto",
-}
diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
new file mode 100644
index 000000000000..40cbf5724563
--- /dev/null
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -0,0 +1,120 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_lb_v1
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// LoadBalancerClient is the client API for LoadBalancer service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type LoadBalancerClient interface {
+	// Bidirectional rpc to get a list of servers.
+	BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error)
+}
+
+type loadBalancerClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
+	return &loadBalancerClient{cc}
+}
+
+func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &loadBalancerBalanceLoadClient{stream}
+	return x, nil
+}
+
+type LoadBalancer_BalanceLoadClient interface {
+	Send(*LoadBalanceRequest) error
+	Recv() (*LoadBalanceResponse, error)
+	grpc.ClientStream
+}
+
+type loadBalancerBalanceLoadClient struct {
+	grpc.ClientStream
+}
+
+func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
+	m := new(LoadBalanceResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// LoadBalancerServer is the server API for LoadBalancer service.
+type LoadBalancerServer interface {
+	// Bidirectional rpc to get a list of servers.
+	BalanceLoad(LoadBalancer_BalanceLoadServer) error
+}
+
+// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations.
+type UnimplementedLoadBalancerServer struct {
+}
+
+func (*UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
+	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
+}
+
+func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
+	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
+}
+
+func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream})
+}
+
+type LoadBalancer_BalanceLoadServer interface {
+	Send(*LoadBalanceResponse) error
+	Recv() (*LoadBalanceRequest, error)
+	grpc.ServerStream
+}
+
+type loadBalancerBalanceLoadServer struct {
+	grpc.ServerStream
+}
+
+func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
+	m := new(LoadBalanceRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.lb.v1.LoadBalancer",
+	HandlerType: (*LoadBalancerServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "BalanceLoad",
+			Handler:       _LoadBalancer_BalanceLoad_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "grpc/lb/v1/load_balancer.proto",
+}
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index a3a3f8118ce9..7ec14279109f 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -4,12 +4,8 @@
 package grpc_lookup_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -180,85 +176,3 @@ var fileDescriptor_3bab962d3362f3ca = []byte{
 	0x29, 0xa7, 0xcf, 0x5e, 0xb3, 0x1d, 0x27, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0xca, 0x8d, 0x5c,
 	0xc7, 0x39, 0x02, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// RouteLookupServiceClient is the client API for RouteLookupService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type RouteLookupServiceClient interface {
-	// Lookup returns a target for a single key.
-	RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error)
-}
-
-type routeLookupServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
-	return &routeLookupServiceClient{cc}
-}
-
-func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
-	out := new(RouteLookupResponse)
-	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// RouteLookupServiceServer is the server API for RouteLookupService service.
-type RouteLookupServiceServer interface {
-	// Lookup returns a target for a single key.
-	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
-}
-
-// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedRouteLookupServiceServer struct {
-}
-
-func (*UnimplementedRouteLookupServiceServer) RouteLookup(ctx context.Context, req *RouteLookupRequest) (*RouteLookupResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
-}
-
-func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
-	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
-}
-
-func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(RouteLookupRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(RouteLookupServiceServer).RouteLookup(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.lookup.v1.RouteLookupService",
-	HandlerType: (*RouteLookupServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "RouteLookup",
-			Handler:    _RouteLookupService_RouteLookup_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/lookup/v1/rls.proto",
-}
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
new file mode 100644
index 000000000000..6e4145dd5fc4
--- /dev/null
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -0,0 +1,88 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_lookup_v1
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// RouteLookupServiceClient is the client API for RouteLookupService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type RouteLookupServiceClient interface {
+	// Lookup returns a target for a single key.
+	RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error)
+}
+
+type routeLookupServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
+	return &routeLookupServiceClient{cc}
+}
+
+func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
+	out := new(RouteLookupResponse)
+	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// RouteLookupServiceServer is the server API for RouteLookupService service.
+type RouteLookupServiceServer interface {
+	// Lookup returns a target for a single key.
+	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
+}
+
+// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedRouteLookupServiceServer struct {
+}
+
+func (*UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
+}
+
+func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
+	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
+}
+
+func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RouteLookupRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RouteLookupServiceServer).RouteLookup(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.lookup.v1.RouteLookupService",
+	HandlerType: (*RouteLookupServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "RouteLookup",
+			Handler:    _RouteLookupService_RouteLookup_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/lookup/v1/rls.proto",
+}
diff --git a/benchmark/grpc_testing/services.pb.go b/benchmark/grpc_testing/services.pb.go
index 7c71cf193004..4d0acd35d4fb 100644
--- a/benchmark/grpc_testing/services.pb.go
+++ b/benchmark/grpc_testing/services.pb.go
@@ -4,12 +4,8 @@
 package grpc_testing
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -51,505 +47,3 @@ var fileDescriptor_e86b6b5d31c265e4 = []byte{
 	0x8e, 0xd8, 0xec, 0x5f, 0x93, 0xad, 0x8b, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x77, 0x96, 0x9e,
 	0xdb, 0xdf, 0x02, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// BenchmarkServiceClient is the client API for BenchmarkService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type BenchmarkServiceClient interface {
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error)
-	// Unconstrainted streaming.
-	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error)
-}
-
-type benchmarkServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient {
-	return &benchmarkServiceClient{cc}
-}
-
-func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
-	out := new(SimpleResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &benchmarkServiceStreamingCallClient{stream}
-	return x, nil
-}
-
-type BenchmarkService_StreamingCallClient interface {
-	Send(*SimpleRequest) error
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type benchmarkServiceStreamingCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *benchmarkServiceStreamingCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &benchmarkServiceUnconstrainedStreamingCallClient{stream}
-	return x, nil
-}
-
-type BenchmarkService_UnconstrainedStreamingCallClient interface {
-	Send(*SimpleRequest) error
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type benchmarkServiceUnconstrainedStreamingCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// BenchmarkServiceServer is the server API for BenchmarkService service.
-type BenchmarkServiceServer interface {
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	StreamingCall(BenchmarkService_StreamingCallServer) error
-	// Unconstrainted streaming.
-	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
-}
-
-// UnimplementedBenchmarkServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedBenchmarkServiceServer struct {
-}
-
-func (*UnimplementedBenchmarkServiceServer) UnaryCall(ctx context.Context, req *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedBenchmarkServiceServer) StreamingCall(srv BenchmarkService_StreamingCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
-}
-func (*UnimplementedBenchmarkServiceServer) UnconstrainedStreamingCall(srv BenchmarkService_UnconstrainedStreamingCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
-}
-
-func RegisterBenchmarkServiceServer(s *grpc.Server, srv BenchmarkServiceServer) {
-	s.RegisterService(&_BenchmarkService_serviceDesc, srv)
-}
-
-func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SimpleRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(BenchmarkServiceServer).UnaryCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.BenchmarkService/UnaryCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream})
-}
-
-type BenchmarkService_StreamingCallServer interface {
-	Send(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type benchmarkServiceStreamingCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *benchmarkServiceStreamingCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _BenchmarkService_UnconstrainedStreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(BenchmarkServiceServer).UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
-}
-
-type BenchmarkService_UnconstrainedStreamingCallServer interface {
-	Send(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type benchmarkServiceUnconstrainedStreamingCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _BenchmarkService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.BenchmarkService",
-	HandlerType: (*BenchmarkServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryCall",
-			Handler:    _BenchmarkService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingCall",
-			Handler:       _BenchmarkService_StreamingCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "UnconstrainedStreamingCall",
-			Handler:       _BenchmarkService_UnconstrainedStreamingCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "benchmark/grpc_testing/services.proto",
-}
-
-// WorkerServiceClient is the client API for WorkerService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type WorkerServiceClient interface {
-	// Start server with specified workload.
-	// First request sent specifies the ServerConfig followed by ServerStatus
-	// response. After that, a "Mark" can be sent anytime to request the latest
-	// stats. Closing the stream will initiate shutdown of the test server
-	// and once the shutdown has finished, the OK status is sent to terminate
-	// this RPC.
-	RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error)
-	// Start client with specified workload.
-	// First request sent specifies the ClientConfig followed by ClientStatus
-	// response. After that, a "Mark" can be sent anytime to request the latest
-	// stats. Closing the stream will initiate shutdown of the test client
-	// and once the shutdown has finished, the OK status is sent to terminate
-	// this RPC.
-	RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error)
-	// Just return the core count - unary call
-	CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error)
-	// Quit this worker
-	QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error)
-}
-
-type workerServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient {
-	return &workerServiceClient{cc}
-}
-
-func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[0], "/grpc.testing.WorkerService/RunServer", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &workerServiceRunServerClient{stream}
-	return x, nil
-}
-
-type WorkerService_RunServerClient interface {
-	Send(*ServerArgs) error
-	Recv() (*ServerStatus, error)
-	grpc.ClientStream
-}
-
-type workerServiceRunServerClient struct {
-	grpc.ClientStream
-}
-
-func (x *workerServiceRunServerClient) Send(m *ServerArgs) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) {
-	m := new(ServerStatus)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[1], "/grpc.testing.WorkerService/RunClient", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &workerServiceRunClientClient{stream}
-	return x, nil
-}
-
-type WorkerService_RunClientClient interface {
-	Send(*ClientArgs) error
-	Recv() (*ClientStatus, error)
-	grpc.ClientStream
-}
-
-type workerServiceRunClientClient struct {
-	grpc.ClientStream
-}
-
-func (x *workerServiceRunClientClient) Send(m *ClientArgs) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *workerServiceRunClientClient) Recv() (*ClientStatus, error) {
-	m := new(ClientStatus)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) {
-	out := new(CoreResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/CoreCount", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) {
-	out := new(Void)
-	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/QuitWorker", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// WorkerServiceServer is the server API for WorkerService service.
-type WorkerServiceServer interface {
-	// Start server with specified workload.
-	// First request sent specifies the ServerConfig followed by ServerStatus
-	// response. After that, a "Mark" can be sent anytime to request the latest
-	// stats. Closing the stream will initiate shutdown of the test server
-	// and once the shutdown has finished, the OK status is sent to terminate
-	// this RPC.
-	RunServer(WorkerService_RunServerServer) error
-	// Start client with specified workload.
-	// First request sent specifies the ClientConfig followed by ClientStatus
-	// response. After that, a "Mark" can be sent anytime to request the latest
-	// stats. Closing the stream will initiate shutdown of the test client
-	// and once the shutdown has finished, the OK status is sent to terminate
-	// this RPC.
-	RunClient(WorkerService_RunClientServer) error
-	// Just return the core count - unary call
-	CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
-	// Quit this worker
-	QuitWorker(context.Context, *Void) (*Void, error)
-}
-
-// UnimplementedWorkerServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedWorkerServiceServer struct {
-}
-
-func (*UnimplementedWorkerServiceServer) RunServer(srv WorkerService_RunServerServer) error {
-	return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
-}
-func (*UnimplementedWorkerServiceServer) RunClient(srv WorkerService_RunClientServer) error {
-	return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
-}
-func (*UnimplementedWorkerServiceServer) CoreCount(ctx context.Context, req *CoreRequest) (*CoreResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
-}
-func (*UnimplementedWorkerServiceServer) QuitWorker(ctx context.Context, req *Void) (*Void, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
-}
-
-func RegisterWorkerServiceServer(s *grpc.Server, srv WorkerServiceServer) {
-	s.RegisterService(&_WorkerService_serviceDesc, srv)
-}
-
-func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(WorkerServiceServer).RunServer(&workerServiceRunServerServer{stream})
-}
-
-type WorkerService_RunServerServer interface {
-	Send(*ServerStatus) error
-	Recv() (*ServerArgs, error)
-	grpc.ServerStream
-}
-
-type workerServiceRunServerServer struct {
-	grpc.ServerStream
-}
-
-func (x *workerServiceRunServerServer) Send(m *ServerStatus) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *workerServiceRunServerServer) Recv() (*ServerArgs, error) {
-	m := new(ServerArgs)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _WorkerService_RunClient_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(WorkerServiceServer).RunClient(&workerServiceRunClientServer{stream})
-}
-
-type WorkerService_RunClientServer interface {
-	Send(*ClientStatus) error
-	Recv() (*ClientArgs, error)
-	grpc.ServerStream
-}
-
-type workerServiceRunClientServer struct {
-	grpc.ServerStream
-}
-
-func (x *workerServiceRunClientServer) Send(m *ClientStatus) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) {
-	m := new(ClientArgs)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _WorkerService_CoreCount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(CoreRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(WorkerServiceServer).CoreCount(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.WorkerService/CoreCount",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(WorkerServiceServer).CoreCount(ctx, req.(*CoreRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Void)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(WorkerServiceServer).QuitWorker(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.WorkerService/QuitWorker",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(WorkerServiceServer).QuitWorker(ctx, req.(*Void))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _WorkerService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.WorkerService",
-	HandlerType: (*WorkerServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "CoreCount",
-			Handler:    _WorkerService_CoreCount_Handler,
-		},
-		{
-			MethodName: "QuitWorker",
-			Handler:    _WorkerService_QuitWorker_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "RunServer",
-			Handler:       _WorkerService_RunServer_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "RunClient",
-			Handler:       _WorkerService_RunClient_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "benchmark/grpc_testing/services.proto",
-}
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
new file mode 100644
index 000000000000..e923f72c5387
--- /dev/null
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -0,0 +1,508 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// BenchmarkServiceClient is the client API for BenchmarkService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type BenchmarkServiceClient interface {
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error)
+	// Unconstrainted streaming.
+	// Both server and client keep sending & receiving simultaneously.
+	UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error)
+}
+
+type benchmarkServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient {
+	return &benchmarkServiceClient{cc}
+}
+
+func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
+	out := new(SimpleResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &benchmarkServiceStreamingCallClient{stream}
+	return x, nil
+}
+
+type BenchmarkService_StreamingCallClient interface {
+	Send(*SimpleRequest) error
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type benchmarkServiceStreamingCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *benchmarkServiceStreamingCallClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &benchmarkServiceUnconstrainedStreamingCallClient{stream}
+	return x, nil
+}
+
+type BenchmarkService_UnconstrainedStreamingCallClient interface {
+	Send(*SimpleRequest) error
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type benchmarkServiceUnconstrainedStreamingCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *benchmarkServiceUnconstrainedStreamingCallClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// BenchmarkServiceServer is the server API for BenchmarkService service.
+type BenchmarkServiceServer interface {
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	StreamingCall(BenchmarkService_StreamingCallServer) error
+	// Unconstrainted streaming.
+	// Both server and client keep sending & receiving simultaneously.
+	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
+}
+
+// UnimplementedBenchmarkServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedBenchmarkServiceServer struct {
+}
+
+func (*UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (*UnimplementedBenchmarkServiceServer) StreamingCall(BenchmarkService_StreamingCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
+}
+func (*UnimplementedBenchmarkServiceServer) UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
+}
+
+func RegisterBenchmarkServiceServer(s *grpc.Server, srv BenchmarkServiceServer) {
+	s.RegisterService(&_BenchmarkService_serviceDesc, srv)
+}
+
+func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SimpleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BenchmarkServiceServer).UnaryCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.BenchmarkService/UnaryCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream})
+}
+
+type BenchmarkService_StreamingCallServer interface {
+	Send(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type benchmarkServiceStreamingCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *benchmarkServiceStreamingCallServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _BenchmarkService_UnconstrainedStreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
+}
+
+type BenchmarkService_UnconstrainedStreamingCallServer interface {
+	Send(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type benchmarkServiceUnconstrainedStreamingCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *benchmarkServiceUnconstrainedStreamingCallServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _BenchmarkService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.BenchmarkService",
+	HandlerType: (*BenchmarkServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryCall",
+			Handler:    _BenchmarkService_UnaryCall_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingCall",
+			Handler:       _BenchmarkService_StreamingCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "UnconstrainedStreamingCall",
+			Handler:       _BenchmarkService_UnconstrainedStreamingCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "benchmark/grpc_testing/services.proto",
+}
+
+// WorkerServiceClient is the client API for WorkerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type WorkerServiceClient interface {
+	// Start server with specified workload.
+	// First request sent specifies the ServerConfig followed by ServerStatus
+	// response. After that, a "Mark" can be sent anytime to request the latest
+	// stats. Closing the stream will initiate shutdown of the test server
+	// and once the shutdown has finished, the OK status is sent to terminate
+	// this RPC.
+	RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error)
+	// Start client with specified workload.
+	// First request sent specifies the ClientConfig followed by ClientStatus
+	// response. After that, a "Mark" can be sent anytime to request the latest
+	// stats. Closing the stream will initiate shutdown of the test client
+	// and once the shutdown has finished, the OK status is sent to terminate
+	// this RPC.
+	RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error)
+	// Just return the core count - unary call
+	CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error)
+	// Quit this worker
+	QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error)
+}
+
+type workerServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient {
+	return &workerServiceClient{cc}
+}
+
+func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[0], "/grpc.testing.WorkerService/RunServer", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &workerServiceRunServerClient{stream}
+	return x, nil
+}
+
+type WorkerService_RunServerClient interface {
+	Send(*ServerArgs) error
+	Recv() (*ServerStatus, error)
+	grpc.ClientStream
+}
+
+type workerServiceRunServerClient struct {
+	grpc.ClientStream
+}
+
+func (x *workerServiceRunServerClient) Send(m *ServerArgs) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) {
+	m := new(ServerStatus)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[1], "/grpc.testing.WorkerService/RunClient", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &workerServiceRunClientClient{stream}
+	return x, nil
+}
+
+type WorkerService_RunClientClient interface {
+	Send(*ClientArgs) error
+	Recv() (*ClientStatus, error)
+	grpc.ClientStream
+}
+
+type workerServiceRunClientClient struct {
+	grpc.ClientStream
+}
+
+func (x *workerServiceRunClientClient) Send(m *ClientArgs) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *workerServiceRunClientClient) Recv() (*ClientStatus, error) {
+	m := new(ClientStatus)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) {
+	out := new(CoreResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/CoreCount", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) {
+	out := new(Void)
+	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/QuitWorker", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// WorkerServiceServer is the server API for WorkerService service.
+type WorkerServiceServer interface {
+	// Start server with specified workload.
+	// First request sent specifies the ServerConfig followed by ServerStatus
+	// response. After that, a "Mark" can be sent anytime to request the latest
+	// stats. Closing the stream will initiate shutdown of the test server
+	// and once the shutdown has finished, the OK status is sent to terminate
+	// this RPC.
+	RunServer(WorkerService_RunServerServer) error
+	// Start client with specified workload.
+	// First request sent specifies the ClientConfig followed by ClientStatus
+	// response. After that, a "Mark" can be sent anytime to request the latest
+	// stats. Closing the stream will initiate shutdown of the test client
+	// and once the shutdown has finished, the OK status is sent to terminate
+	// this RPC.
+	RunClient(WorkerService_RunClientServer) error
+	// Just return the core count - unary call
+	CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
+	// Quit this worker
+	QuitWorker(context.Context, *Void) (*Void, error)
+}
+
+// UnimplementedWorkerServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedWorkerServiceServer struct {
+}
+
+func (*UnimplementedWorkerServiceServer) RunServer(WorkerService_RunServerServer) error {
+	return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
+}
+func (*UnimplementedWorkerServiceServer) RunClient(WorkerService_RunClientServer) error {
+	return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
+}
+func (*UnimplementedWorkerServiceServer) CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
+}
+func (*UnimplementedWorkerServiceServer) QuitWorker(context.Context, *Void) (*Void, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
+}
+
+func RegisterWorkerServiceServer(s *grpc.Server, srv WorkerServiceServer) {
+	s.RegisterService(&_WorkerService_serviceDesc, srv)
+}
+
+func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(WorkerServiceServer).RunServer(&workerServiceRunServerServer{stream})
+}
+
+type WorkerService_RunServerServer interface {
+	Send(*ServerStatus) error
+	Recv() (*ServerArgs, error)
+	grpc.ServerStream
+}
+
+type workerServiceRunServerServer struct {
+	grpc.ServerStream
+}
+
+func (x *workerServiceRunServerServer) Send(m *ServerStatus) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *workerServiceRunServerServer) Recv() (*ServerArgs, error) {
+	m := new(ServerArgs)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _WorkerService_RunClient_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(WorkerServiceServer).RunClient(&workerServiceRunClientServer{stream})
+}
+
+type WorkerService_RunClientServer interface {
+	Send(*ClientStatus) error
+	Recv() (*ClientArgs, error)
+	grpc.ServerStream
+}
+
+type workerServiceRunClientServer struct {
+	grpc.ServerStream
+}
+
+func (x *workerServiceRunClientServer) Send(m *ClientStatus) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) {
+	m := new(ClientArgs)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _WorkerService_CoreCount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CoreRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(WorkerServiceServer).CoreCount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.WorkerService/CoreCount",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(WorkerServiceServer).CoreCount(ctx, req.(*CoreRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Void)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(WorkerServiceServer).QuitWorker(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.WorkerService/QuitWorker",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(WorkerServiceServer).QuitWorker(ctx, req.(*Void))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _WorkerService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.WorkerService",
+	HandlerType: (*WorkerServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "CoreCount",
+			Handler:    _WorkerService_CoreCount_Handler,
+		},
+		{
+			MethodName: "QuitWorker",
+			Handler:    _WorkerService_QuitWorker_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "RunServer",
+			Handler:       _WorkerService_RunServer_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "RunClient",
+			Handler:       _WorkerService_RunClient_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "benchmark/grpc_testing/services.proto",
+}
diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go
index 34bfa5ab8f50..9c364e1ebd9c 100644
--- a/channelz/grpc_channelz_v1/channelz.pb.go
+++ b/channelz/grpc_channelz_v1/channelz.pb.go
@@ -4,16 +4,12 @@
 package grpc_channelz_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	any "github.com/golang/protobuf/ptypes/any"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	wrappers "github.com/golang/protobuf/ptypes/wrappers"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -2925,315 +2921,3 @@ var fileDescriptor_6ee37dfd35a8ab00 = []byte{
 	0xd3, 0x77, 0xc6, 0x68, 0xe7, 0x3c, 0xcf, 0x4f, 0xf3, 0x5f, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff,
 	0x54, 0xae, 0x0b, 0x93, 0xdf, 0x1f, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// ChannelzClient is the client API for Channelz service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ChannelzClient interface {
-	// Gets all root channels (i.e. channels the application has directly
-	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error)
-	// Gets all servers that exist in the process.
-	GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error)
-	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error)
-	// Gets all server sockets that exist in the process.
-	GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error)
-	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error)
-	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error)
-	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error)
-}
-
-type channelzClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
-	return &channelzClient{cc}
-}
-
-func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
-	out := new(GetTopChannelsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
-	out := new(GetServersResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
-	out := new(GetServerResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
-	out := new(GetServerSocketsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
-	out := new(GetChannelResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
-	out := new(GetSubchannelResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
-	out := new(GetSocketResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ChannelzServer is the server API for Channelz service.
-type ChannelzServer interface {
-	// Gets all root channels (i.e. channels the application has directly
-	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
-	// Gets all servers that exist in the process.
-	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
-	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
-	// Gets all server sockets that exist in the process.
-	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
-	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
-	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
-	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
-}
-
-// UnimplementedChannelzServer can be embedded to have forward compatible implementations.
-type UnimplementedChannelzServer struct {
-}
-
-func (*UnimplementedChannelzServer) GetTopChannels(ctx context.Context, req *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
-}
-func (*UnimplementedChannelzServer) GetServers(ctx context.Context, req *GetServersRequest) (*GetServersResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
-}
-func (*UnimplementedChannelzServer) GetServer(ctx context.Context, req *GetServerRequest) (*GetServerResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
-}
-func (*UnimplementedChannelzServer) GetServerSockets(ctx context.Context, req *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
-}
-func (*UnimplementedChannelzServer) GetChannel(ctx context.Context, req *GetChannelRequest) (*GetChannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
-}
-func (*UnimplementedChannelzServer) GetSubchannel(ctx context.Context, req *GetSubchannelRequest) (*GetSubchannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
-}
-func (*UnimplementedChannelzServer) GetSocket(ctx context.Context, req *GetSocketRequest) (*GetSocketResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
-}
-
-func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
-	s.RegisterService(&_Channelz_serviceDesc, srv)
-}
-
-func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetTopChannelsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetTopChannels(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetServersRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetServers(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetServers",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetServerRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetServer(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetServer",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetServerSocketsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetServerSockets(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetChannelRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetChannel(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetChannel",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetSubchannelRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetSubchannel(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetSocketRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetSocket(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetSocket",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Channelz_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.channelz.v1.Channelz",
-	HandlerType: (*ChannelzServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetTopChannels",
-			Handler:    _Channelz_GetTopChannels_Handler,
-		},
-		{
-			MethodName: "GetServers",
-			Handler:    _Channelz_GetServers_Handler,
-		},
-		{
-			MethodName: "GetServer",
-			Handler:    _Channelz_GetServer_Handler,
-		},
-		{
-			MethodName: "GetServerSockets",
-			Handler:    _Channelz_GetServerSockets_Handler,
-		},
-		{
-			MethodName: "GetChannel",
-			Handler:    _Channelz_GetChannel_Handler,
-		},
-		{
-			MethodName: "GetSubchannel",
-			Handler:    _Channelz_GetSubchannel_Handler,
-		},
-		{
-			MethodName: "GetSocket",
-			Handler:    _Channelz_GetSocket_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/channelz/v1/channelz.proto",
-}
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
new file mode 100644
index 000000000000..bacb448d1d85
--- /dev/null
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -0,0 +1,318 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_channelz_v1
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ChannelzClient is the client API for Channelz service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ChannelzClient interface {
+	// Gets all root channels (i.e. channels the application has directly
+	// created). This does not include subchannels nor non-top level channels.
+	GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error)
+	// Gets all servers that exist in the process.
+	GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error)
+	// Returns a single Server, or else a NOT_FOUND code.
+	GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error)
+	// Gets all server sockets that exist in the process.
+	GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error)
+	// Returns a single Channel, or else a NOT_FOUND code.
+	GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error)
+	// Returns a single Subchannel, or else a NOT_FOUND code.
+	GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error)
+	// Returns a single Socket or else a NOT_FOUND code.
+	GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error)
+}
+
+type channelzClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
+	return &channelzClient{cc}
+}
+
+func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
+	out := new(GetTopChannelsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
+	out := new(GetServersResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
+	out := new(GetServerResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
+	out := new(GetServerSocketsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
+	out := new(GetChannelResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
+	out := new(GetSubchannelResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
+	out := new(GetSocketResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ChannelzServer is the server API for Channelz service.
+type ChannelzServer interface {
+	// Gets all root channels (i.e. channels the application has directly
+	// created). This does not include subchannels nor non-top level channels.
+	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
+	// Gets all servers that exist in the process.
+	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
+	// Returns a single Server, or else a NOT_FOUND code.
+	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
+	// Gets all server sockets that exist in the process.
+	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
+	// Returns a single Channel, or else a NOT_FOUND code.
+	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
+	// Returns a single Subchannel, or else a NOT_FOUND code.
+	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
+	// Returns a single Socket or else a NOT_FOUND code.
+	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
+}
+
+// UnimplementedChannelzServer can be embedded to have forward compatible implementations.
+type UnimplementedChannelzServer struct {
+}
+
+func (*UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
+}
+func (*UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
+}
+func (*UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
+}
+func (*UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
+}
+func (*UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
+}
+func (*UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
+}
+func (*UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
+}
+
+func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
+	s.RegisterService(&_Channelz_serviceDesc, srv)
+}
+
+func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopChannelsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetTopChannels(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetServers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetServers",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServerRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetServer(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetServer",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServerSocketsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetServerSockets(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetChannelRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetChannel(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetChannel",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetSubchannelRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetSubchannel(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetSocketRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetSocket(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetSocket",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Channelz_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.channelz.v1.Channelz",
+	HandlerType: (*ChannelzServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetTopChannels",
+			Handler:    _Channelz_GetTopChannels_Handler,
+		},
+		{
+			MethodName: "GetServers",
+			Handler:    _Channelz_GetServers_Handler,
+		},
+		{
+			MethodName: "GetServer",
+			Handler:    _Channelz_GetServer_Handler,
+		},
+		{
+			MethodName: "GetServerSockets",
+			Handler:    _Channelz_GetServerSockets_Handler,
+		},
+		{
+			MethodName: "GetChannel",
+			Handler:    _Channelz_GetChannel_Handler,
+		},
+		{
+			MethodName: "GetSubchannel",
+			Handler:    _Channelz_GetSubchannel_Handler,
+		},
+		{
+			MethodName: "GetSocket",
+			Handler:    _Channelz_GetSocket_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/channelz/v1/channelz.proto",
+}
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
index a2060de402bc..6d9c304e7981 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
@@ -4,12 +4,8 @@
 package grpc_gcp
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -979,127 +975,3 @@ var fileDescriptor_54c074f40c7c7e99 = []byte{
 	0x5f, 0xef, 0xa8, 0xf5, 0x83, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc1, 0xf9, 0x9d, 0xf2, 0xd9,
 	0x0b, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// HandshakerServiceClient is the client API for HandshakerService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type HandshakerServiceClient interface {
-	// Handshaker service accepts a stream of handshaker request, returning a
-	// stream of handshaker response. Client is expected to send exactly one
-	// message with either client_start or server_start followed by one or more
-	// messages with next. Each time client sends a request, the handshaker
-	// service expects to respond. Client does not have to wait for service's
-	// response before sending next request.
-	DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error)
-}
-
-type handshakerServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient {
-	return &handshakerServiceClient{cc}
-}
-
-func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &handshakerServiceDoHandshakeClient{stream}
-	return x, nil
-}
-
-type HandshakerService_DoHandshakeClient interface {
-	Send(*HandshakerReq) error
-	Recv() (*HandshakerResp, error)
-	grpc.ClientStream
-}
-
-type handshakerServiceDoHandshakeClient struct {
-	grpc.ClientStream
-}
-
-func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
-	m := new(HandshakerResp)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// HandshakerServiceServer is the server API for HandshakerService service.
-type HandshakerServiceServer interface {
-	// Handshaker service accepts a stream of handshaker request, returning a
-	// stream of handshaker response. Client is expected to send exactly one
-	// message with either client_start or server_start followed by one or more
-	// messages with next. Each time client sends a request, the handshaker
-	// service expects to respond. Client does not have to wait for service's
-	// response before sending next request.
-	DoHandshake(HandshakerService_DoHandshakeServer) error
-}
-
-// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedHandshakerServiceServer struct {
-}
-
-func (*UnimplementedHandshakerServiceServer) DoHandshake(srv HandshakerService_DoHandshakeServer) error {
-	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
-}
-
-func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
-	s.RegisterService(&_HandshakerService_serviceDesc, srv)
-}
-
-func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream})
-}
-
-type HandshakerService_DoHandshakeServer interface {
-	Send(*HandshakerResp) error
-	Recv() (*HandshakerReq, error)
-	grpc.ServerStream
-}
-
-type handshakerServiceDoHandshakeServer struct {
-	grpc.ServerStream
-}
-
-func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
-	m := new(HandshakerReq)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _HandshakerService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.gcp.HandshakerService",
-	HandlerType: (*HandshakerServiceServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "DoHandshake",
-			Handler:       _HandshakerService_DoHandshake_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "grpc/gcp/handshaker.proto",
-}
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
new file mode 100644
index 000000000000..75c2e4732439
--- /dev/null
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -0,0 +1,130 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_gcp
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// HandshakerServiceClient is the client API for HandshakerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type HandshakerServiceClient interface {
+	// Handshaker service accepts a stream of handshaker request, returning a
+	// stream of handshaker response. Client is expected to send exactly one
+	// message with either client_start or server_start followed by one or more
+	// messages with next. Each time client sends a request, the handshaker
+	// service expects to respond. Client does not have to wait for service's
+	// response before sending next request.
+	DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error)
+}
+
+type handshakerServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient {
+	return &handshakerServiceClient{cc}
+}
+
+func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &handshakerServiceDoHandshakeClient{stream}
+	return x, nil
+}
+
+type HandshakerService_DoHandshakeClient interface {
+	Send(*HandshakerReq) error
+	Recv() (*HandshakerResp, error)
+	grpc.ClientStream
+}
+
+type handshakerServiceDoHandshakeClient struct {
+	grpc.ClientStream
+}
+
+func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
+	m := new(HandshakerResp)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// HandshakerServiceServer is the server API for HandshakerService service.
+type HandshakerServiceServer interface {
+	// Handshaker service accepts a stream of handshaker request, returning a
+	// stream of handshaker response. Client is expected to send exactly one
+	// message with either client_start or server_start followed by one or more
+	// messages with next. Each time client sends a request, the handshaker
+	// service expects to respond. Client does not have to wait for service's
+	// response before sending next request.
+	DoHandshake(HandshakerService_DoHandshakeServer) error
+}
+
+// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedHandshakerServiceServer struct {
+}
+
+func (*UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
+	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
+}
+
+func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
+	s.RegisterService(&_HandshakerService_serviceDesc, srv)
+}
+
+func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream})
+}
+
+type HandshakerService_DoHandshakeServer interface {
+	Send(*HandshakerResp) error
+	Recv() (*HandshakerReq, error)
+	grpc.ServerStream
+}
+
+type handshakerServiceDoHandshakeServer struct {
+	grpc.ServerStream
+}
+
+func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
+	m := new(HandshakerReq)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _HandshakerService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.gcp.HandshakerService",
+	HandlerType: (*HandshakerServiceServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "DoHandshake",
+			Handler:       _HandshakerService_DoHandshake_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "grpc/gcp/handshaker.proto",
+}
diff --git a/examples/features/proto/echo/echo.pb.go b/examples/features/proto/echo/echo.pb.go
index 520356d5affe..4048b43c0c2c 100644
--- a/examples/features/proto/echo/echo.pb.go
+++ b/examples/features/proto/echo/echo.pb.go
@@ -4,12 +4,8 @@
 package echo
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -131,293 +127,3 @@ var fileDescriptor_2fd1d686b7b805dc = []byte{
 	0xb0, 0xd1, 0x69, 0x62, 0x67, 0xca, 0xaa, 0xf9, 0xeb, 0x4c, 0x9e, 0x4f, 0xfb, 0xff, 0xe2, 0x3b,
 	0x00, 0x00, 0xff, 0xff, 0xf7, 0x79, 0x87, 0xf0, 0x4d, 0x02, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// EchoClient is the client API for Echo service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type EchoClient interface {
-	// UnaryEcho is unary echo.
-	UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error)
-	// ServerStreamingEcho is server side streaming.
-	ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error)
-	// ClientStreamingEcho is client side streaming.
-	ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error)
-	// BidirectionalStreamingEcho is bidi streaming.
-	BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error)
-}
-
-type echoClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewEchoClient(cc grpc.ClientConnInterface) EchoClient {
-	return &echoClient{cc}
-}
-
-func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) {
-	out := new(EchoResponse)
-	err := c.cc.Invoke(ctx, "/grpc.examples.echo.Echo/UnaryEcho", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[0], "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &echoServerStreamingEchoClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type Echo_ServerStreamingEchoClient interface {
-	Recv() (*EchoResponse, error)
-	grpc.ClientStream
-}
-
-type echoServerStreamingEchoClient struct {
-	grpc.ClientStream
-}
-
-func (x *echoServerStreamingEchoClient) Recv() (*EchoResponse, error) {
-	m := new(EchoResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[1], "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &echoClientStreamingEchoClient{stream}
-	return x, nil
-}
-
-type Echo_ClientStreamingEchoClient interface {
-	Send(*EchoRequest) error
-	CloseAndRecv() (*EchoResponse, error)
-	grpc.ClientStream
-}
-
-type echoClientStreamingEchoClient struct {
-	grpc.ClientStream
-}
-
-func (x *echoClientStreamingEchoClient) Send(m *EchoRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *echoClientStreamingEchoClient) CloseAndRecv() (*EchoResponse, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(EchoResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[2], "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &echoBidirectionalStreamingEchoClient{stream}
-	return x, nil
-}
-
-type Echo_BidirectionalStreamingEchoClient interface {
-	Send(*EchoRequest) error
-	Recv() (*EchoResponse, error)
-	grpc.ClientStream
-}
-
-type echoBidirectionalStreamingEchoClient struct {
-	grpc.ClientStream
-}
-
-func (x *echoBidirectionalStreamingEchoClient) Send(m *EchoRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *echoBidirectionalStreamingEchoClient) Recv() (*EchoResponse, error) {
-	m := new(EchoResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// EchoServer is the server API for Echo service.
-type EchoServer interface {
-	// UnaryEcho is unary echo.
-	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
-	// ServerStreamingEcho is server side streaming.
-	ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
-	// ClientStreamingEcho is client side streaming.
-	ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
-	// BidirectionalStreamingEcho is bidi streaming.
-	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
-}
-
-// UnimplementedEchoServer can be embedded to have forward compatible implementations.
-type UnimplementedEchoServer struct {
-}
-
-func (*UnimplementedEchoServer) UnaryEcho(ctx context.Context, req *EchoRequest) (*EchoResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
-}
-func (*UnimplementedEchoServer) ServerStreamingEcho(req *EchoRequest, srv Echo_ServerStreamingEchoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
-}
-func (*UnimplementedEchoServer) ClientStreamingEcho(srv Echo_ClientStreamingEchoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
-}
-func (*UnimplementedEchoServer) BidirectionalStreamingEcho(srv Echo_BidirectionalStreamingEchoServer) error {
-	return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
-}
-
-func RegisterEchoServer(s *grpc.Server, srv EchoServer) {
-	s.RegisterService(&_Echo_serviceDesc, srv)
-}
-
-func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(EchoRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(EchoServer).UnaryEcho(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.examples.echo.Echo/UnaryEcho",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(EchoServer).UnaryEcho(ctx, req.(*EchoRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Echo_ServerStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(EchoRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(EchoServer).ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
-}
-
-type Echo_ServerStreamingEchoServer interface {
-	Send(*EchoResponse) error
-	grpc.ServerStream
-}
-
-type echoServerStreamingEchoServer struct {
-	grpc.ServerStream
-}
-
-func (x *echoServerStreamingEchoServer) Send(m *EchoResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func _Echo_ClientStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(EchoServer).ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
-}
-
-type Echo_ClientStreamingEchoServer interface {
-	SendAndClose(*EchoResponse) error
-	Recv() (*EchoRequest, error)
-	grpc.ServerStream
-}
-
-type echoClientStreamingEchoServer struct {
-	grpc.ServerStream
-}
-
-func (x *echoClientStreamingEchoServer) SendAndClose(m *EchoResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *echoClientStreamingEchoServer) Recv() (*EchoRequest, error) {
-	m := new(EchoRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _Echo_BidirectionalStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(EchoServer).BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
-}
-
-type Echo_BidirectionalStreamingEchoServer interface {
-	Send(*EchoResponse) error
-	Recv() (*EchoRequest, error)
-	grpc.ServerStream
-}
-
-type echoBidirectionalStreamingEchoServer struct {
-	grpc.ServerStream
-}
-
-func (x *echoBidirectionalStreamingEchoServer) Send(m *EchoResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) {
-	m := new(EchoRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _Echo_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.examples.echo.Echo",
-	HandlerType: (*EchoServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryEcho",
-			Handler:    _Echo_UnaryEcho_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ServerStreamingEcho",
-			Handler:       _Echo_ServerStreamingEcho_Handler,
-			ServerStreams: true,
-		},
-		{
-			StreamName:    "ClientStreamingEcho",
-			Handler:       _Echo_ClientStreamingEcho_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "BidirectionalStreamingEcho",
-			Handler:       _Echo_BidirectionalStreamingEcho_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "examples/features/proto/echo/echo.proto",
-}
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
new file mode 100644
index 000000000000..38be6f65ce1a
--- /dev/null
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -0,0 +1,296 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package echo
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// EchoClient is the client API for Echo service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type EchoClient interface {
+	// UnaryEcho is unary echo.
+	UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error)
+	// ServerStreamingEcho is server side streaming.
+	ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error)
+	// ClientStreamingEcho is client side streaming.
+	ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error)
+	// BidirectionalStreamingEcho is bidi streaming.
+	BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error)
+}
+
+type echoClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewEchoClient(cc grpc.ClientConnInterface) EchoClient {
+	return &echoClient{cc}
+}
+
+func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) {
+	out := new(EchoResponse)
+	err := c.cc.Invoke(ctx, "/grpc.examples.echo.Echo/UnaryEcho", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[0], "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &echoServerStreamingEchoClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Echo_ServerStreamingEchoClient interface {
+	Recv() (*EchoResponse, error)
+	grpc.ClientStream
+}
+
+type echoServerStreamingEchoClient struct {
+	grpc.ClientStream
+}
+
+func (x *echoServerStreamingEchoClient) Recv() (*EchoResponse, error) {
+	m := new(EchoResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[1], "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &echoClientStreamingEchoClient{stream}
+	return x, nil
+}
+
+type Echo_ClientStreamingEchoClient interface {
+	Send(*EchoRequest) error
+	CloseAndRecv() (*EchoResponse, error)
+	grpc.ClientStream
+}
+
+type echoClientStreamingEchoClient struct {
+	grpc.ClientStream
+}
+
+func (x *echoClientStreamingEchoClient) Send(m *EchoRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *echoClientStreamingEchoClient) CloseAndRecv() (*EchoResponse, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(EchoResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[2], "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &echoBidirectionalStreamingEchoClient{stream}
+	return x, nil
+}
+
+type Echo_BidirectionalStreamingEchoClient interface {
+	Send(*EchoRequest) error
+	Recv() (*EchoResponse, error)
+	grpc.ClientStream
+}
+
+type echoBidirectionalStreamingEchoClient struct {
+	grpc.ClientStream
+}
+
+func (x *echoBidirectionalStreamingEchoClient) Send(m *EchoRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *echoBidirectionalStreamingEchoClient) Recv() (*EchoResponse, error) {
+	m := new(EchoResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// EchoServer is the server API for Echo service.
+type EchoServer interface {
+	// UnaryEcho is unary echo.
+	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
+	// ServerStreamingEcho is server side streaming.
+	ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
+	// ClientStreamingEcho is client side streaming.
+	ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
+	// BidirectionalStreamingEcho is bidi streaming.
+	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
+}
+
+// UnimplementedEchoServer can be embedded to have forward compatible implementations.
+type UnimplementedEchoServer struct {
+}
+
+func (*UnimplementedEchoServer) UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
+}
+func (*UnimplementedEchoServer) ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
+}
+func (*UnimplementedEchoServer) ClientStreamingEcho(Echo_ClientStreamingEchoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
+}
+func (*UnimplementedEchoServer) BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error {
+	return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
+}
+
+func RegisterEchoServer(s *grpc.Server, srv EchoServer) {
+	s.RegisterService(&_Echo_serviceDesc, srv)
+}
+
+func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EchoRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(EchoServer).UnaryEcho(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.examples.echo.Echo/UnaryEcho",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(EchoServer).UnaryEcho(ctx, req.(*EchoRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Echo_ServerStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(EchoRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(EchoServer).ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
+}
+
+type Echo_ServerStreamingEchoServer interface {
+	Send(*EchoResponse) error
+	grpc.ServerStream
+}
+
+type echoServerStreamingEchoServer struct {
+	grpc.ServerStream
+}
+
+func (x *echoServerStreamingEchoServer) Send(m *EchoResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _Echo_ClientStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(EchoServer).ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
+}
+
+type Echo_ClientStreamingEchoServer interface {
+	SendAndClose(*EchoResponse) error
+	Recv() (*EchoRequest, error)
+	grpc.ServerStream
+}
+
+type echoClientStreamingEchoServer struct {
+	grpc.ServerStream
+}
+
+func (x *echoClientStreamingEchoServer) SendAndClose(m *EchoResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *echoClientStreamingEchoServer) Recv() (*EchoRequest, error) {
+	m := new(EchoRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _Echo_BidirectionalStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(EchoServer).BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
+}
+
+type Echo_BidirectionalStreamingEchoServer interface {
+	Send(*EchoResponse) error
+	Recv() (*EchoRequest, error)
+	grpc.ServerStream
+}
+
+type echoBidirectionalStreamingEchoServer struct {
+	grpc.ServerStream
+}
+
+func (x *echoBidirectionalStreamingEchoServer) Send(m *EchoResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) {
+	m := new(EchoRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _Echo_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.examples.echo.Echo",
+	HandlerType: (*EchoServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryEcho",
+			Handler:    _Echo_UnaryEcho_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ServerStreamingEcho",
+			Handler:       _Echo_ServerStreamingEcho_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "ClientStreamingEcho",
+			Handler:       _Echo_ClientStreamingEcho_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "BidirectionalStreamingEcho",
+			Handler:       _Echo_BidirectionalStreamingEcho_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "examples/features/proto/echo/echo.proto",
+}
diff --git a/examples/helloworld/helloworld/helloworld.pb.go b/examples/helloworld/helloworld/helloworld.pb.go
index 9fde58df0a6e..531c792a1e7c 100644
--- a/examples/helloworld/helloworld/helloworld.pb.go
+++ b/examples/helloworld/helloworld/helloworld.pb.go
@@ -4,12 +4,8 @@
 package helloworld
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -129,85 +125,3 @@ var fileDescriptor_b83ea99a5323a2c7 = []byte{
 	0x5d, 0x2f, 0xbf, 0x28, 0x5d, 0x1f, 0xa4, 0x9d, 0x40, 0xe0, 0x24, 0xb1, 0x81, 0x83, 0xc4, 0x18,
 	0x10, 0x00, 0x00, 0xff, 0xff, 0x46, 0xfe, 0x45, 0x5c, 0x45, 0x01, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// GreeterClient is the client API for Greeter service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type GreeterClient interface {
-	// Sends a greeting
-	SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error)
-}
-
-type greeterClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient {
-	return &greeterClient{cc}
-}
-
-func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) {
-	out := new(HelloReply)
-	err := c.cc.Invoke(ctx, "/helloworld.Greeter/SayHello", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// GreeterServer is the server API for Greeter service.
-type GreeterServer interface {
-	// Sends a greeting
-	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
-}
-
-// UnimplementedGreeterServer can be embedded to have forward compatible implementations.
-type UnimplementedGreeterServer struct {
-}
-
-func (*UnimplementedGreeterServer) SayHello(ctx context.Context, req *HelloRequest) (*HelloReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
-}
-
-func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) {
-	s.RegisterService(&_Greeter_serviceDesc, srv)
-}
-
-func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(HelloRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(GreeterServer).SayHello(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/helloworld.Greeter/SayHello",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Greeter_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "helloworld.Greeter",
-	HandlerType: (*GreeterServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "SayHello",
-			Handler:    _Greeter_SayHello_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "examples/helloworld/helloworld/helloworld.proto",
-}
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
new file mode 100644
index 000000000000..ac81352958cb
--- /dev/null
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -0,0 +1,88 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package helloworld
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// GreeterClient is the client API for Greeter service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type GreeterClient interface {
+	// Sends a greeting
+	SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error)
+}
+
+type greeterClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient {
+	return &greeterClient{cc}
+}
+
+func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) {
+	out := new(HelloReply)
+	err := c.cc.Invoke(ctx, "/helloworld.Greeter/SayHello", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// GreeterServer is the server API for Greeter service.
+type GreeterServer interface {
+	// Sends a greeting
+	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
+}
+
+// UnimplementedGreeterServer can be embedded to have forward compatible implementations.
+type UnimplementedGreeterServer struct {
+}
+
+func (*UnimplementedGreeterServer) SayHello(context.Context, *HelloRequest) (*HelloReply, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
+}
+
+func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) {
+	s.RegisterService(&_Greeter_serviceDesc, srv)
+}
+
+func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HelloRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GreeterServer).SayHello(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/helloworld.Greeter/SayHello",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Greeter_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "helloworld.Greeter",
+	HandlerType: (*GreeterServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "SayHello",
+			Handler:    _Greeter_SayHello_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "examples/helloworld/helloworld/helloworld.proto",
+}
diff --git a/examples/route_guide/routeguide/route_guide.pb.go b/examples/route_guide/routeguide/route_guide.pb.go
index b0edc2c25558..78c856645a7b 100644
--- a/examples/route_guide/routeguide/route_guide.pb.go
+++ b/examples/route_guide/routeguide/route_guide.pb.go
@@ -4,12 +4,8 @@
 package routeguide
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -343,325 +339,3 @@ var fileDescriptor_af806a20656386f8 = []byte{
 	0x76, 0x4f, 0xf5, 0xe1, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x28, 0xef, 0x54, 0xdd, 0x89, 0x03,
 	0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// RouteGuideClient is the client API for RouteGuide service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type RouteGuideClient interface {
-	// A simple RPC.
-	//
-	// Obtains the feature at a given position.
-	//
-	// A feature with an empty name is returned if there's no feature at the given
-	// position.
-	GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error)
-	// A server-to-client streaming RPC.
-	//
-	// Obtains the Features available within the given Rectangle.  Results are
-	// streamed rather than returned at once (e.g. in a response message with a
-	// repeated field), as the rectangle may cover a large area and contain a
-	// huge number of features.
-	ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error)
-	// A client-to-server streaming RPC.
-	//
-	// Accepts a stream of Points on a route being traversed, returning a
-	// RouteSummary when traversal is completed.
-	RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error)
-	// A Bidirectional streaming RPC.
-	//
-	// Accepts a stream of RouteNotes sent while a route is being traversed,
-	// while receiving other RouteNotes (e.g. from other users).
-	RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error)
-}
-
-type routeGuideClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewRouteGuideClient(cc grpc.ClientConnInterface) RouteGuideClient {
-	return &routeGuideClient{cc}
-}
-
-func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) {
-	out := new(Feature)
-	err := c.cc.Invoke(ctx, "/routeguide.RouteGuide/GetFeature", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[0], "/routeguide.RouteGuide/ListFeatures", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &routeGuideListFeaturesClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type RouteGuide_ListFeaturesClient interface {
-	Recv() (*Feature, error)
-	grpc.ClientStream
-}
-
-type routeGuideListFeaturesClient struct {
-	grpc.ClientStream
-}
-
-func (x *routeGuideListFeaturesClient) Recv() (*Feature, error) {
-	m := new(Feature)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[1], "/routeguide.RouteGuide/RecordRoute", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &routeGuideRecordRouteClient{stream}
-	return x, nil
-}
-
-type RouteGuide_RecordRouteClient interface {
-	Send(*Point) error
-	CloseAndRecv() (*RouteSummary, error)
-	grpc.ClientStream
-}
-
-type routeGuideRecordRouteClient struct {
-	grpc.ClientStream
-}
-
-func (x *routeGuideRecordRouteClient) Send(m *Point) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *routeGuideRecordRouteClient) CloseAndRecv() (*RouteSummary, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(RouteSummary)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[2], "/routeguide.RouteGuide/RouteChat", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &routeGuideRouteChatClient{stream}
-	return x, nil
-}
-
-type RouteGuide_RouteChatClient interface {
-	Send(*RouteNote) error
-	Recv() (*RouteNote, error)
-	grpc.ClientStream
-}
-
-type routeGuideRouteChatClient struct {
-	grpc.ClientStream
-}
-
-func (x *routeGuideRouteChatClient) Send(m *RouteNote) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *routeGuideRouteChatClient) Recv() (*RouteNote, error) {
-	m := new(RouteNote)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// RouteGuideServer is the server API for RouteGuide service.
-type RouteGuideServer interface {
-	// A simple RPC.
-	//
-	// Obtains the feature at a given position.
-	//
-	// A feature with an empty name is returned if there's no feature at the given
-	// position.
-	GetFeature(context.Context, *Point) (*Feature, error)
-	// A server-to-client streaming RPC.
-	//
-	// Obtains the Features available within the given Rectangle.  Results are
-	// streamed rather than returned at once (e.g. in a response message with a
-	// repeated field), as the rectangle may cover a large area and contain a
-	// huge number of features.
-	ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
-	// A client-to-server streaming RPC.
-	//
-	// Accepts a stream of Points on a route being traversed, returning a
-	// RouteSummary when traversal is completed.
-	RecordRoute(RouteGuide_RecordRouteServer) error
-	// A Bidirectional streaming RPC.
-	//
-	// Accepts a stream of RouteNotes sent while a route is being traversed,
-	// while receiving other RouteNotes (e.g. from other users).
-	RouteChat(RouteGuide_RouteChatServer) error
-}
-
-// UnimplementedRouteGuideServer can be embedded to have forward compatible implementations.
-type UnimplementedRouteGuideServer struct {
-}
-
-func (*UnimplementedRouteGuideServer) GetFeature(ctx context.Context, req *Point) (*Feature, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
-}
-func (*UnimplementedRouteGuideServer) ListFeatures(req *Rectangle, srv RouteGuide_ListFeaturesServer) error {
-	return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
-}
-func (*UnimplementedRouteGuideServer) RecordRoute(srv RouteGuide_RecordRouteServer) error {
-	return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
-}
-func (*UnimplementedRouteGuideServer) RouteChat(srv RouteGuide_RouteChatServer) error {
-	return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
-}
-
-func RegisterRouteGuideServer(s *grpc.Server, srv RouteGuideServer) {
-	s.RegisterService(&_RouteGuide_serviceDesc, srv)
-}
-
-func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Point)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(RouteGuideServer).GetFeature(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/routeguide.RouteGuide/GetFeature",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(RouteGuideServer).GetFeature(ctx, req.(*Point))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _RouteGuide_ListFeatures_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(Rectangle)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(RouteGuideServer).ListFeatures(m, &routeGuideListFeaturesServer{stream})
-}
-
-type RouteGuide_ListFeaturesServer interface {
-	Send(*Feature) error
-	grpc.ServerStream
-}
-
-type routeGuideListFeaturesServer struct {
-	grpc.ServerStream
-}
-
-func (x *routeGuideListFeaturesServer) Send(m *Feature) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func _RouteGuide_RecordRoute_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(RouteGuideServer).RecordRoute(&routeGuideRecordRouteServer{stream})
-}
-
-type RouteGuide_RecordRouteServer interface {
-	SendAndClose(*RouteSummary) error
-	Recv() (*Point, error)
-	grpc.ServerStream
-}
-
-type routeGuideRecordRouteServer struct {
-	grpc.ServerStream
-}
-
-func (x *routeGuideRecordRouteServer) SendAndClose(m *RouteSummary) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *routeGuideRecordRouteServer) Recv() (*Point, error) {
-	m := new(Point)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _RouteGuide_RouteChat_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(RouteGuideServer).RouteChat(&routeGuideRouteChatServer{stream})
-}
-
-type RouteGuide_RouteChatServer interface {
-	Send(*RouteNote) error
-	Recv() (*RouteNote, error)
-	grpc.ServerStream
-}
-
-type routeGuideRouteChatServer struct {
-	grpc.ServerStream
-}
-
-func (x *routeGuideRouteChatServer) Send(m *RouteNote) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) {
-	m := new(RouteNote)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _RouteGuide_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "routeguide.RouteGuide",
-	HandlerType: (*RouteGuideServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetFeature",
-			Handler:    _RouteGuide_GetFeature_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ListFeatures",
-			Handler:       _RouteGuide_ListFeatures_Handler,
-			ServerStreams: true,
-		},
-		{
-			StreamName:    "RecordRoute",
-			Handler:       _RouteGuide_RecordRoute_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "RouteChat",
-			Handler:       _RouteGuide_RouteChat_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "examples/route_guide/routeguide/route_guide.proto",
-}
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
new file mode 100644
index 000000000000..326a7683ef0d
--- /dev/null
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -0,0 +1,328 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package routeguide
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// RouteGuideClient is the client API for RouteGuide service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type RouteGuideClient interface {
+	// A simple RPC.
+	//
+	// Obtains the feature at a given position.
+	//
+	// A feature with an empty name is returned if there's no feature at the given
+	// position.
+	GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error)
+	// A server-to-client streaming RPC.
+	//
+	// Obtains the Features available within the given Rectangle.  Results are
+	// streamed rather than returned at once (e.g. in a response message with a
+	// repeated field), as the rectangle may cover a large area and contain a
+	// huge number of features.
+	ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error)
+	// A client-to-server streaming RPC.
+	//
+	// Accepts a stream of Points on a route being traversed, returning a
+	// RouteSummary when traversal is completed.
+	RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error)
+	// A Bidirectional streaming RPC.
+	//
+	// Accepts a stream of RouteNotes sent while a route is being traversed,
+	// while receiving other RouteNotes (e.g. from other users).
+	RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error)
+}
+
+type routeGuideClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewRouteGuideClient(cc grpc.ClientConnInterface) RouteGuideClient {
+	return &routeGuideClient{cc}
+}
+
+func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) {
+	out := new(Feature)
+	err := c.cc.Invoke(ctx, "/routeguide.RouteGuide/GetFeature", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[0], "/routeguide.RouteGuide/ListFeatures", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &routeGuideListFeaturesClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type RouteGuide_ListFeaturesClient interface {
+	Recv() (*Feature, error)
+	grpc.ClientStream
+}
+
+type routeGuideListFeaturesClient struct {
+	grpc.ClientStream
+}
+
+func (x *routeGuideListFeaturesClient) Recv() (*Feature, error) {
+	m := new(Feature)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[1], "/routeguide.RouteGuide/RecordRoute", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &routeGuideRecordRouteClient{stream}
+	return x, nil
+}
+
+type RouteGuide_RecordRouteClient interface {
+	Send(*Point) error
+	CloseAndRecv() (*RouteSummary, error)
+	grpc.ClientStream
+}
+
+type routeGuideRecordRouteClient struct {
+	grpc.ClientStream
+}
+
+func (x *routeGuideRecordRouteClient) Send(m *Point) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *routeGuideRecordRouteClient) CloseAndRecv() (*RouteSummary, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(RouteSummary)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[2], "/routeguide.RouteGuide/RouteChat", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &routeGuideRouteChatClient{stream}
+	return x, nil
+}
+
+type RouteGuide_RouteChatClient interface {
+	Send(*RouteNote) error
+	Recv() (*RouteNote, error)
+	grpc.ClientStream
+}
+
+type routeGuideRouteChatClient struct {
+	grpc.ClientStream
+}
+
+func (x *routeGuideRouteChatClient) Send(m *RouteNote) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *routeGuideRouteChatClient) Recv() (*RouteNote, error) {
+	m := new(RouteNote)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// RouteGuideServer is the server API for RouteGuide service.
+type RouteGuideServer interface {
+	// A simple RPC.
+	//
+	// Obtains the feature at a given position.
+	//
+	// A feature with an empty name is returned if there's no feature at the given
+	// position.
+	GetFeature(context.Context, *Point) (*Feature, error)
+	// A server-to-client streaming RPC.
+	//
+	// Obtains the Features available within the given Rectangle.  Results are
+	// streamed rather than returned at once (e.g. in a response message with a
+	// repeated field), as the rectangle may cover a large area and contain a
+	// huge number of features.
+	ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
+	// A client-to-server streaming RPC.
+	//
+	// Accepts a stream of Points on a route being traversed, returning a
+	// RouteSummary when traversal is completed.
+	RecordRoute(RouteGuide_RecordRouteServer) error
+	// A Bidirectional streaming RPC.
+	//
+	// Accepts a stream of RouteNotes sent while a route is being traversed,
+	// while receiving other RouteNotes (e.g. from other users).
+	RouteChat(RouteGuide_RouteChatServer) error
+}
+
+// UnimplementedRouteGuideServer can be embedded to have forward compatible implementations.
+type UnimplementedRouteGuideServer struct {
+}
+
+func (*UnimplementedRouteGuideServer) GetFeature(context.Context, *Point) (*Feature, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
+}
+func (*UnimplementedRouteGuideServer) ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error {
+	return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
+}
+func (*UnimplementedRouteGuideServer) RecordRoute(RouteGuide_RecordRouteServer) error {
+	return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
+}
+func (*UnimplementedRouteGuideServer) RouteChat(RouteGuide_RouteChatServer) error {
+	return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
+}
+
+func RegisterRouteGuideServer(s *grpc.Server, srv RouteGuideServer) {
+	s.RegisterService(&_RouteGuide_serviceDesc, srv)
+}
+
+func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Point)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RouteGuideServer).GetFeature(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/routeguide.RouteGuide/GetFeature",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RouteGuideServer).GetFeature(ctx, req.(*Point))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _RouteGuide_ListFeatures_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(Rectangle)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(RouteGuideServer).ListFeatures(m, &routeGuideListFeaturesServer{stream})
+}
+
+type RouteGuide_ListFeaturesServer interface {
+	Send(*Feature) error
+	grpc.ServerStream
+}
+
+type routeGuideListFeaturesServer struct {
+	grpc.ServerStream
+}
+
+func (x *routeGuideListFeaturesServer) Send(m *Feature) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _RouteGuide_RecordRoute_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(RouteGuideServer).RecordRoute(&routeGuideRecordRouteServer{stream})
+}
+
+type RouteGuide_RecordRouteServer interface {
+	SendAndClose(*RouteSummary) error
+	Recv() (*Point, error)
+	grpc.ServerStream
+}
+
+type routeGuideRecordRouteServer struct {
+	grpc.ServerStream
+}
+
+func (x *routeGuideRecordRouteServer) SendAndClose(m *RouteSummary) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *routeGuideRecordRouteServer) Recv() (*Point, error) {
+	m := new(Point)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _RouteGuide_RouteChat_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(RouteGuideServer).RouteChat(&routeGuideRouteChatServer{stream})
+}
+
+type RouteGuide_RouteChatServer interface {
+	Send(*RouteNote) error
+	Recv() (*RouteNote, error)
+	grpc.ServerStream
+}
+
+type routeGuideRouteChatServer struct {
+	grpc.ServerStream
+}
+
+func (x *routeGuideRouteChatServer) Send(m *RouteNote) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) {
+	m := new(RouteNote)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _RouteGuide_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "routeguide.RouteGuide",
+	HandlerType: (*RouteGuideServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetFeature",
+			Handler:    _RouteGuide_GetFeature_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ListFeatures",
+			Handler:       _RouteGuide_ListFeatures_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "RecordRoute",
+			Handler:       _RouteGuide_RecordRoute_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "RouteChat",
+			Handler:       _RouteGuide_RouteChat_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "examples/route_guide/routeguide/route_guide.proto",
+}
diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go
index 4c2a527ec597..e9919c0073b6 100644
--- a/health/grpc_health_v1/health.pb.go
+++ b/health/grpc_health_v1/health.pb.go
@@ -4,12 +4,8 @@
 package grpc_health_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -163,181 +159,3 @@ var fileDescriptor_e265fd9d4e077217 = []byte{
 	0xd3, 0x20, 0x46, 0xe8, 0x85, 0x19, 0x26, 0xb1, 0x81, 0x93, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff,
 	0xff, 0x12, 0x7d, 0x96, 0xcb, 0x2d, 0x02, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// HealthClient is the client API for Health service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type HealthClient interface {
-	// If the requested service is unknown, the call will fail with status
-	// NOT_FOUND.
-	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
-	// Performs a watch for the serving status of the requested service.
-	// The server will immediately send back a message indicating the current
-	// serving status.  It will then subsequently send a new message whenever
-	// the service's serving status changes.
-	//
-	// If the requested service is unknown when the call is received, the
-	// server will send a message setting the serving status to
-	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
-	// future point, the serving status of the service becomes known, the
-	// server will send a new message with the service's serving status.
-	//
-	// If the call terminates with status UNIMPLEMENTED, then clients
-	// should assume this method is not supported and should not retry the
-	// call.  If the call terminates with any other status (including OK),
-	// clients should retry the call with appropriate exponential backoff.
-	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
-}
-
-type healthClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
-	return &healthClient{cc}
-}
-
-func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
-	out := new(HealthCheckResponse)
-	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &healthWatchClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type Health_WatchClient interface {
-	Recv() (*HealthCheckResponse, error)
-	grpc.ClientStream
-}
-
-type healthWatchClient struct {
-	grpc.ClientStream
-}
-
-func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
-	m := new(HealthCheckResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// HealthServer is the server API for Health service.
-type HealthServer interface {
-	// If the requested service is unknown, the call will fail with status
-	// NOT_FOUND.
-	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
-	// Performs a watch for the serving status of the requested service.
-	// The server will immediately send back a message indicating the current
-	// serving status.  It will then subsequently send a new message whenever
-	// the service's serving status changes.
-	//
-	// If the requested service is unknown when the call is received, the
-	// server will send a message setting the serving status to
-	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
-	// future point, the serving status of the service becomes known, the
-	// server will send a new message with the service's serving status.
-	//
-	// If the call terminates with status UNIMPLEMENTED, then clients
-	// should assume this method is not supported and should not retry the
-	// call.  If the call terminates with any other status (including OK),
-	// clients should retry the call with appropriate exponential backoff.
-	Watch(*HealthCheckRequest, Health_WatchServer) error
-}
-
-// UnimplementedHealthServer can be embedded to have forward compatible implementations.
-type UnimplementedHealthServer struct {
-}
-
-func (*UnimplementedHealthServer) Check(ctx context.Context, req *HealthCheckRequest) (*HealthCheckResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
-}
-func (*UnimplementedHealthServer) Watch(req *HealthCheckRequest, srv Health_WatchServer) error {
-	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
-}
-
-func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
-	s.RegisterService(&_Health_serviceDesc, srv)
-}
-
-func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(HealthCheckRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(HealthServer).Check(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.health.v1.Health/Check",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(HealthCheckRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
-}
-
-type Health_WatchServer interface {
-	Send(*HealthCheckResponse) error
-	grpc.ServerStream
-}
-
-type healthWatchServer struct {
-	grpc.ServerStream
-}
-
-func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-var _Health_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.health.v1.Health",
-	HandlerType: (*HealthServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Check",
-			Handler:    _Health_Check_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "Watch",
-			Handler:       _Health_Watch_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "grpc/health/v1/health.proto",
-}
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
new file mode 100644
index 000000000000..28cd72ea63c0
--- /dev/null
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -0,0 +1,184 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_health_v1
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// HealthClient is the client API for Health service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type HealthClient interface {
+	// If the requested service is unknown, the call will fail with status
+	// NOT_FOUND.
+	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
+	// Performs a watch for the serving status of the requested service.
+	// The server will immediately send back a message indicating the current
+	// serving status.  It will then subsequently send a new message whenever
+	// the service's serving status changes.
+	//
+	// If the requested service is unknown when the call is received, the
+	// server will send a message setting the serving status to
+	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
+	// future point, the serving status of the service becomes known, the
+	// server will send a new message with the service's serving status.
+	//
+	// If the call terminates with status UNIMPLEMENTED, then clients
+	// should assume this method is not supported and should not retry the
+	// call.  If the call terminates with any other status (including OK),
+	// clients should retry the call with appropriate exponential backoff.
+	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
+}
+
+type healthClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
+	return &healthClient{cc}
+}
+
+func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
+	out := new(HealthCheckResponse)
+	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &healthWatchClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Health_WatchClient interface {
+	Recv() (*HealthCheckResponse, error)
+	grpc.ClientStream
+}
+
+type healthWatchClient struct {
+	grpc.ClientStream
+}
+
+func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
+	m := new(HealthCheckResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// HealthServer is the server API for Health service.
+type HealthServer interface {
+	// If the requested service is unknown, the call will fail with status
+	// NOT_FOUND.
+	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
+	// Performs a watch for the serving status of the requested service.
+	// The server will immediately send back a message indicating the current
+	// serving status.  It will then subsequently send a new message whenever
+	// the service's serving status changes.
+	//
+	// If the requested service is unknown when the call is received, the
+	// server will send a message setting the serving status to
+	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
+	// future point, the serving status of the service becomes known, the
+	// server will send a new message with the service's serving status.
+	//
+	// If the call terminates with status UNIMPLEMENTED, then clients
+	// should assume this method is not supported and should not retry the
+	// call.  If the call terminates with any other status (including OK),
+	// clients should retry the call with appropriate exponential backoff.
+	Watch(*HealthCheckRequest, Health_WatchServer) error
+}
+
+// UnimplementedHealthServer can be embedded to have forward compatible implementations.
+type UnimplementedHealthServer struct {
+}
+
+func (*UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+}
+func (*UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
+	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+}
+
+func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
+	s.RegisterService(&_Health_serviceDesc, srv)
+}
+
+func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HealthCheckRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(HealthServer).Check(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.health.v1.Health/Check",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(HealthCheckRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
+}
+
+type Health_WatchServer interface {
+	Send(*HealthCheckResponse) error
+	grpc.ServerStream
+}
+
+type healthWatchServer struct {
+	grpc.ServerStream
+}
+
+func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+var _Health_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.health.v1.Health",
+	HandlerType: (*HealthServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Check",
+			Handler:    _Health_Check_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Watch",
+			Handler:       _Health_Watch_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "grpc/health/v1/health.proto",
+}
diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 8bbff7a69dff..1c8e33bc95a3 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -4,12 +4,8 @@
 package grpc_testing
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -848,565 +844,3 @@ var fileDescriptor_534063719f48d90d = []byte{
 	0x4b, 0x7e, 0xc7, 0xee, 0x67, 0x7d, 0xd4, 0x9e, 0x6f, 0xca, 0x0f, 0xda, 0xa7, 0xff, 0x06, 0x00,
 	0x00, 0xff, 0xff, 0xaa, 0x43, 0x4c, 0xeb, 0xf3, 0x0a, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// TestServiceClient is the client API for TestService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type TestServiceClient interface {
-	// One empty request followed by one empty response.
-	EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
-	// One request followed by a sequence of responses (streamed download).
-	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error)
-	// A sequence of requests followed by one response (streamed upload).
-	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
-	// A sequence of requests followed by a sequence of responses.
-	// The server buffers all the client requests and then serves them in order. A
-	// stream of responses are returned to the client when the server starts with
-	// first request.
-	HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error)
-}
-
-type testServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
-	return &testServiceClient{cc}
-}
-
-func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
-	out := new(Empty)
-	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
-	out := new(SimpleResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceStreamingOutputCallClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type TestService_StreamingOutputCallClient interface {
-	Recv() (*StreamingOutputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceStreamingOutputCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallResponse, error) {
-	m := new(StreamingOutputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceStreamingInputCallClient{stream}
-	return x, nil
-}
-
-type TestService_StreamingInputCallClient interface {
-	Send(*StreamingInputCallRequest) error
-	CloseAndRecv() (*StreamingInputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceStreamingInputCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceStreamingInputCallClient) Send(m *StreamingInputCallRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCallResponse, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(StreamingInputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceFullDuplexCallClient{stream}
-	return x, nil
-}
-
-type TestService_FullDuplexCallClient interface {
-	Send(*StreamingOutputCallRequest) error
-	Recv() (*StreamingOutputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceFullDuplexCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceFullDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
-	m := new(StreamingOutputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceHalfDuplexCallClient{stream}
-	return x, nil
-}
-
-type TestService_HalfDuplexCallClient interface {
-	Send(*StreamingOutputCallRequest) error
-	Recv() (*StreamingOutputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceHalfDuplexCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceHalfDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
-	m := new(StreamingOutputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// TestServiceServer is the server API for TestService service.
-type TestServiceServer interface {
-	// One empty request followed by one empty response.
-	EmptyCall(context.Context, *Empty) (*Empty, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by a sequence of responses (streamed download).
-	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
-	// A sequence of requests followed by one response (streamed upload).
-	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(TestService_StreamingInputCallServer) error
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// A sequence of requests followed by a sequence of responses.
-	// The server buffers all the client requests and then serves them in order. A
-	// stream of responses are returned to the client when the server starts with
-	// first request.
-	HalfDuplexCall(TestService_HalfDuplexCallServer) error
-}
-
-// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
-}
-
-func (*UnimplementedTestServiceServer) EmptyCall(ctx context.Context, req *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-}
-func (*UnimplementedTestServiceServer) UnaryCall(ctx context.Context, req *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingOutputCall(req *StreamingOutputCallRequest, srv TestService_StreamingOutputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingInputCall(srv TestService_StreamingInputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) FullDuplexCall(srv TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (*UnimplementedTestServiceServer) HalfDuplexCall(srv TestService_HalfDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-}
-
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
-}
-
-func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Empty)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(TestServiceServer).EmptyCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.TestService/EmptyCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SimpleRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.TestService/UnaryCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(StreamingOutputCallRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
-}
-
-type TestService_StreamingOutputCallServer interface {
-	Send(*StreamingOutputCallResponse) error
-	grpc.ServerStream
-}
-
-type testServiceStreamingOutputCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
-}
-
-type TestService_StreamingInputCallServer interface {
-	SendAndClose(*StreamingInputCallResponse) error
-	Recv() (*StreamingInputCallRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceStreamingInputCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceStreamingInputCallServer) SendAndClose(m *StreamingInputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest, error) {
-	m := new(StreamingInputCallRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-
-type TestService_FullDuplexCallServer interface {
-	Send(*StreamingOutputCallResponse) error
-	Recv() (*StreamingOutputCallRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceFullDuplexCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceFullDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
-	m := new(StreamingOutputCallRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
-}
-
-type TestService_HalfDuplexCallServer interface {
-	Send(*StreamingOutputCallResponse) error
-	Recv() (*StreamingOutputCallRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceHalfDuplexCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceHalfDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
-	m := new(StreamingOutputCallRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _TestService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "EmptyCall",
-			Handler:    _TestService_EmptyCall_Handler,
-		},
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingOutputCall",
-			Handler:       _TestService_StreamingOutputCall_Handler,
-			ServerStreams: true,
-		},
-		{
-			StreamName:    "StreamingInputCall",
-			Handler:       _TestService_StreamingInputCall_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "HalfDuplexCall",
-			Handler:       _TestService_HalfDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "interop/grpc_testing/test.proto",
-}
-
-// UnimplementedServiceClient is the client API for UnimplementedService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type UnimplementedServiceClient interface {
-	// A call that no server should implement
-	UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
-}
-
-type unimplementedServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewUnimplementedServiceClient(cc grpc.ClientConnInterface) UnimplementedServiceClient {
-	return &unimplementedServiceClient{cc}
-}
-
-func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
-	out := new(Empty)
-	err := c.cc.Invoke(ctx, "/grpc.testing.UnimplementedService/UnimplementedCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// UnimplementedServiceServer is the server API for UnimplementedService service.
-type UnimplementedServiceServer interface {
-	// A call that no server should implement
-	UnimplementedCall(context.Context, *Empty) (*Empty, error)
-}
-
-// UnimplementedUnimplementedServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedUnimplementedServiceServer struct {
-}
-
-func (*UnimplementedUnimplementedServiceServer) UnimplementedCall(ctx context.Context, req *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
-}
-
-func RegisterUnimplementedServiceServer(s *grpc.Server, srv UnimplementedServiceServer) {
-	s.RegisterService(&_UnimplementedService_serviceDesc, srv)
-}
-
-func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Empty)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.UnimplementedService/UnimplementedCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, req.(*Empty))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _UnimplementedService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.UnimplementedService",
-	HandlerType: (*UnimplementedServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnimplementedCall",
-			Handler:    _UnimplementedService_UnimplementedCall_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
-}
-
-// LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type LoadBalancerStatsServiceClient interface {
-	// Gets the backend distribution for RPCs sent by a test client.
-	GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error)
-}
-
-type loadBalancerStatsServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewLoadBalancerStatsServiceClient(cc grpc.ClientConnInterface) LoadBalancerStatsServiceClient {
-	return &loadBalancerStatsServiceClient{cc}
-}
-
-func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) {
-	out := new(LoadBalancerStatsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.LoadBalancerStatsService/GetClientStats", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service.
-type LoadBalancerStatsServiceServer interface {
-	// Gets the backend distribution for RPCs sent by a test client.
-	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
-}
-
-// UnimplementedLoadBalancerStatsServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedLoadBalancerStatsServiceServer struct {
-}
-
-func (*UnimplementedLoadBalancerStatsServiceServer) GetClientStats(ctx context.Context, req *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
-}
-
-func RegisterLoadBalancerStatsServiceServer(s *grpc.Server, srv LoadBalancerStatsServiceServer) {
-	s.RegisterService(&_LoadBalancerStatsService_serviceDesc, srv)
-}
-
-func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(LoadBalancerStatsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.LoadBalancerStatsService/GetClientStats",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, req.(*LoadBalancerStatsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _LoadBalancerStatsService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.LoadBalancerStatsService",
-	HandlerType: (*LoadBalancerStatsServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetClientStats",
-			Handler:    _LoadBalancerStatsService_GetClientStats_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
-}
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
new file mode 100644
index 000000000000..d731f4643ecd
--- /dev/null
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -0,0 +1,568 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// TestServiceClient is the client API for TestService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type TestServiceClient interface {
+	// One empty request followed by one empty response.
+	EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
+	// One request followed by a sequence of responses (streamed download).
+	// The server returns the payload with client desired type and sizes.
+	StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error)
+	// A sequence of requests followed by one response (streamed upload).
+	// The server returns the aggregated size of client payload as the result.
+	StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error)
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
+	// A sequence of requests followed by a sequence of responses.
+	// The server buffers all the client requests and then serves them in order. A
+	// stream of responses are returned to the client when the server starts with
+	// first request.
+	HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error)
+}
+
+type testServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
+	return &testServiceClient{cc}
+}
+
+func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
+	out := new(SimpleResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceStreamingOutputCallClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type TestService_StreamingOutputCallClient interface {
+	Recv() (*StreamingOutputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceStreamingOutputCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallResponse, error) {
+	m := new(StreamingOutputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceStreamingInputCallClient{stream}
+	return x, nil
+}
+
+type TestService_StreamingInputCallClient interface {
+	Send(*StreamingInputCallRequest) error
+	CloseAndRecv() (*StreamingInputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceStreamingInputCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceStreamingInputCallClient) Send(m *StreamingInputCallRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCallResponse, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(StreamingInputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceFullDuplexCallClient{stream}
+	return x, nil
+}
+
+type TestService_FullDuplexCallClient interface {
+	Send(*StreamingOutputCallRequest) error
+	Recv() (*StreamingOutputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceFullDuplexCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceFullDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
+	m := new(StreamingOutputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceHalfDuplexCallClient{stream}
+	return x, nil
+}
+
+type TestService_HalfDuplexCallClient interface {
+	Send(*StreamingOutputCallRequest) error
+	Recv() (*StreamingOutputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceHalfDuplexCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceHalfDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
+	m := new(StreamingOutputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// TestServiceServer is the server API for TestService service.
+type TestServiceServer interface {
+	// One empty request followed by one empty response.
+	EmptyCall(context.Context, *Empty) (*Empty, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by a sequence of responses (streamed download).
+	// The server returns the payload with client desired type and sizes.
+	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	// A sequence of requests followed by one response (streamed upload).
+	// The server returns the aggregated size of client payload as the result.
+	StreamingInputCall(TestService_StreamingInputCallServer) error
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(TestService_FullDuplexCallServer) error
+	// A sequence of requests followed by a sequence of responses.
+	// The server buffers all the client requests and then serves them in order. A
+	// stream of responses are returned to the client when the server starts with
+	// first request.
+	HalfDuplexCall(TestService_HalfDuplexCallServer) error
+}
+
+// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedTestServiceServer struct {
+}
+
+func (*UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+}
+func (*UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (*UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+}
+func (*UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+}
+func (*UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+}
+func (*UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+}
+
+func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+	s.RegisterService(&_TestService_serviceDesc, srv)
+}
+
+func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).EmptyCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/EmptyCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SimpleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).UnaryCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/UnaryCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(StreamingOutputCallRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
+}
+
+type TestService_StreamingOutputCallServer interface {
+	Send(*StreamingOutputCallResponse) error
+	grpc.ServerStream
+}
+
+type testServiceStreamingOutputCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
+}
+
+type TestService_StreamingInputCallServer interface {
+	SendAndClose(*StreamingInputCallResponse) error
+	Recv() (*StreamingInputCallRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceStreamingInputCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceStreamingInputCallServer) SendAndClose(m *StreamingInputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest, error) {
+	m := new(StreamingInputCallRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+
+type TestService_FullDuplexCallServer interface {
+	Send(*StreamingOutputCallResponse) error
+	Recv() (*StreamingOutputCallRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceFullDuplexCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceFullDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
+	m := new(StreamingOutputCallRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
+}
+
+type TestService_HalfDuplexCallServer interface {
+	Send(*StreamingOutputCallResponse) error
+	Recv() (*StreamingOutputCallRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceHalfDuplexCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceHalfDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
+	m := new(StreamingOutputCallRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _TestService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.TestService",
+	HandlerType: (*TestServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "EmptyCall",
+			Handler:    _TestService_EmptyCall_Handler,
+		},
+		{
+			MethodName: "UnaryCall",
+			Handler:    _TestService_UnaryCall_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingOutputCall",
+			Handler:       _TestService_StreamingOutputCall_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "StreamingInputCall",
+			Handler:       _TestService_StreamingInputCall_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "FullDuplexCall",
+			Handler:       _TestService_FullDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "HalfDuplexCall",
+			Handler:       _TestService_HalfDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "interop/grpc_testing/test.proto",
+}
+
+// UnimplementedServiceClient is the client API for UnimplementedService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type UnimplementedServiceClient interface {
+	// A call that no server should implement
+	UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
+}
+
+type unimplementedServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewUnimplementedServiceClient(cc grpc.ClientConnInterface) UnimplementedServiceClient {
+	return &unimplementedServiceClient{cc}
+}
+
+func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.UnimplementedService/UnimplementedCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// UnimplementedServiceServer is the server API for UnimplementedService service.
+type UnimplementedServiceServer interface {
+	// A call that no server should implement
+	UnimplementedCall(context.Context, *Empty) (*Empty, error)
+}
+
+// UnimplementedUnimplementedServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedUnimplementedServiceServer struct {
+}
+
+func (*UnimplementedUnimplementedServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
+}
+
+func RegisterUnimplementedServiceServer(s *grpc.Server, srv UnimplementedServiceServer) {
+	s.RegisterService(&_UnimplementedService_serviceDesc, srv)
+}
+
+func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.UnimplementedService/UnimplementedCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _UnimplementedService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.UnimplementedService",
+	HandlerType: (*UnimplementedServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnimplementedCall",
+			Handler:    _UnimplementedService_UnimplementedCall_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "interop/grpc_testing/test.proto",
+}
+
+// LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type LoadBalancerStatsServiceClient interface {
+	// Gets the backend distribution for RPCs sent by a test client.
+	GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error)
+}
+
+type loadBalancerStatsServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewLoadBalancerStatsServiceClient(cc grpc.ClientConnInterface) LoadBalancerStatsServiceClient {
+	return &loadBalancerStatsServiceClient{cc}
+}
+
+func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) {
+	out := new(LoadBalancerStatsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.LoadBalancerStatsService/GetClientStats", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service.
+type LoadBalancerStatsServiceServer interface {
+	// Gets the backend distribution for RPCs sent by a test client.
+	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
+}
+
+// UnimplementedLoadBalancerStatsServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedLoadBalancerStatsServiceServer struct {
+}
+
+func (*UnimplementedLoadBalancerStatsServiceServer) GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
+}
+
+func RegisterLoadBalancerStatsServiceServer(s *grpc.Server, srv LoadBalancerStatsServiceServer) {
+	s.RegisterService(&_LoadBalancerStatsService_serviceDesc, srv)
+}
+
+func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LoadBalancerStatsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.LoadBalancerStatsService/GetClientStats",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, req.(*LoadBalancerStatsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _LoadBalancerStatsService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.LoadBalancerStatsService",
+	HandlerType: (*LoadBalancerStatsServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetClientStats",
+			Handler:    _LoadBalancerStatsService_GetClientStats_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "interop/grpc_testing/test.proto",
+}
diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go
index 90f02824ef69..831a62721383 100644
--- a/profiling/proto/service.pb.go
+++ b/profiling/proto/service.pb.go
@@ -4,12 +4,8 @@
 package proto
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -366,125 +362,3 @@ var fileDescriptor_e1ab2aa17b47c6fb = []byte{
 	0x95, 0xbb, 0x2e, 0xf9, 0xc9, 0xfd, 0xaf, 0x42, 0xfa, 0xbc, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff,
 	0x5d, 0x47, 0x09, 0xa9, 0x19, 0x03, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// ProfilingClient is the client API for Profiling service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ProfilingClient interface {
-	// Enable allows users to toggle profiling on and off remotely.
-	Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error)
-	// GetStreamStats is used to retrieve an array of stream-level stats from a
-	// gRPC client/server.
-	GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error)
-}
-
-type profilingClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
-	return &profilingClient{cc}
-}
-
-func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
-	out := new(EnableResponse)
-	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
-	out := new(GetStreamStatsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ProfilingServer is the server API for Profiling service.
-type ProfilingServer interface {
-	// Enable allows users to toggle profiling on and off remotely.
-	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
-	// GetStreamStats is used to retrieve an array of stream-level stats from a
-	// gRPC client/server.
-	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
-}
-
-// UnimplementedProfilingServer can be embedded to have forward compatible implementations.
-type UnimplementedProfilingServer struct {
-}
-
-func (*UnimplementedProfilingServer) Enable(ctx context.Context, req *EnableRequest) (*EnableResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
-}
-func (*UnimplementedProfilingServer) GetStreamStats(ctx context.Context, req *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
-}
-
-func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
-	s.RegisterService(&_Profiling_serviceDesc, srv)
-}
-
-func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(EnableRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ProfilingServer).Enable(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetStreamStatsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ProfilingServer).GetStreamStats(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Profiling_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
-	HandlerType: (*ProfilingServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Enable",
-			Handler:    _Profiling_Enable_Handler,
-		},
-		{
-			MethodName: "GetStreamStats",
-			Handler:    _Profiling_GetStreamStats_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "profiling/proto/service.proto",
-}
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
new file mode 100644
index 000000000000..87f1b74b7d06
--- /dev/null
+++ b/profiling/proto/service_grpc.pb.go
@@ -0,0 +1,128 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ProfilingClient is the client API for Profiling service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ProfilingClient interface {
+	// Enable allows users to toggle profiling on and off remotely.
+	Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error)
+	// GetStreamStats is used to retrieve an array of stream-level stats from a
+	// gRPC client/server.
+	GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error)
+}
+
+type profilingClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
+	return &profilingClient{cc}
+}
+
+func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
+	out := new(EnableResponse)
+	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
+	out := new(GetStreamStatsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ProfilingServer is the server API for Profiling service.
+type ProfilingServer interface {
+	// Enable allows users to toggle profiling on and off remotely.
+	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
+	// GetStreamStats is used to retrieve an array of stream-level stats from a
+	// gRPC client/server.
+	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
+}
+
+// UnimplementedProfilingServer can be embedded to have forward compatible implementations.
+type UnimplementedProfilingServer struct {
+}
+
+func (*UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
+}
+func (*UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
+}
+
+func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
+	s.RegisterService(&_Profiling_serviceDesc, srv)
+}
+
+func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EnableRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ProfilingServer).Enable(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetStreamStatsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ProfilingServer).GetStreamStats(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Profiling_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
+	HandlerType: (*ProfilingServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Enable",
+			Handler:    _Profiling_Enable_Handler,
+		},
+		{
+			MethodName: "GetStreamStats",
+			Handler:    _Profiling_GetStreamStats_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "profiling/proto/service.proto",
+}
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 900bd6c05c78..382612d520dd 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -4,12 +4,8 @@
 package grpc_reflection_v1alpha
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -636,119 +632,3 @@ var fileDescriptor_e8cf9f2921ad6c95 = []byte{
 	0xcb, 0xb3, 0xdb, 0x8c, 0xdb, 0xea, 0x53, 0xd5, 0xb9, 0xfd, 0xd3, 0x35, 0xdc, 0x54, 0xbe, 0x39,
 	0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x74, 0x3a, 0x67, 0xe7, 0x06, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// ServerReflectionClient is the client API for ServerReflection service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ServerReflectionClient interface {
-	// The reflection service is structured as a bidirectional stream, ensuring
-	// all related requests go to a single server.
-	ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error)
-}
-
-type serverReflectionClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient {
-	return &serverReflectionClient{cc}
-}
-
-func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &serverReflectionServerReflectionInfoClient{stream}
-	return x, nil
-}
-
-type ServerReflection_ServerReflectionInfoClient interface {
-	Send(*ServerReflectionRequest) error
-	Recv() (*ServerReflectionResponse, error)
-	grpc.ClientStream
-}
-
-type serverReflectionServerReflectionInfoClient struct {
-	grpc.ClientStream
-}
-
-func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) {
-	m := new(ServerReflectionResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// ServerReflectionServer is the server API for ServerReflection service.
-type ServerReflectionServer interface {
-	// The reflection service is structured as a bidirectional stream, ensuring
-	// all related requests go to a single server.
-	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
-}
-
-// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations.
-type UnimplementedServerReflectionServer struct {
-}
-
-func (*UnimplementedServerReflectionServer) ServerReflectionInfo(srv ServerReflection_ServerReflectionInfoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
-}
-
-func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
-	s.RegisterService(&_ServerReflection_serviceDesc, srv)
-}
-
-func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
-}
-
-type ServerReflection_ServerReflectionInfoServer interface {
-	Send(*ServerReflectionResponse) error
-	Recv() (*ServerReflectionRequest, error)
-	grpc.ServerStream
-}
-
-type serverReflectionServerReflectionInfoServer struct {
-	grpc.ServerStream
-}
-
-func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) {
-	m := new(ServerReflectionRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _ServerReflection_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
-	HandlerType: (*ServerReflectionServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ServerReflectionInfo",
-			Handler:       _ServerReflection_ServerReflectionInfo_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
-}
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
new file mode 100644
index 000000000000..f04e2d4bb0f6
--- /dev/null
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -0,0 +1,122 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_reflection_v1alpha
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ServerReflectionClient is the client API for ServerReflection service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ServerReflectionClient interface {
+	// The reflection service is structured as a bidirectional stream, ensuring
+	// all related requests go to a single server.
+	ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error)
+}
+
+type serverReflectionClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient {
+	return &serverReflectionClient{cc}
+}
+
+func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &serverReflectionServerReflectionInfoClient{stream}
+	return x, nil
+}
+
+type ServerReflection_ServerReflectionInfoClient interface {
+	Send(*ServerReflectionRequest) error
+	Recv() (*ServerReflectionResponse, error)
+	grpc.ClientStream
+}
+
+type serverReflectionServerReflectionInfoClient struct {
+	grpc.ClientStream
+}
+
+func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) {
+	m := new(ServerReflectionResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// ServerReflectionServer is the server API for ServerReflection service.
+type ServerReflectionServer interface {
+	// The reflection service is structured as a bidirectional stream, ensuring
+	// all related requests go to a single server.
+	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
+}
+
+// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations.
+type UnimplementedServerReflectionServer struct {
+}
+
+func (*UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
+}
+
+func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
+	s.RegisterService(&_ServerReflection_serviceDesc, srv)
+}
+
+func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
+}
+
+type ServerReflection_ServerReflectionInfoServer interface {
+	Send(*ServerReflectionResponse) error
+	Recv() (*ServerReflectionRequest, error)
+	grpc.ServerStream
+}
+
+type serverReflectionServerReflectionInfoServer struct {
+	grpc.ServerStream
+}
+
+func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) {
+	m := new(ServerReflectionRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _ServerReflection_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
+	HandlerType: (*ServerReflectionServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ServerReflectionInfo",
+			Handler:       _ServerReflection_ServerReflectionInfo_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
+}
diff --git a/reflection/grpc_testing/test.pb.go b/reflection/grpc_testing/test.pb.go
index 72841c8ce216..464bf39dd5c4 100644
--- a/reflection/grpc_testing/test.pb.go
+++ b/reflection/grpc_testing/test.pb.go
@@ -4,12 +4,8 @@
 package grpc_testing
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -187,152 +183,3 @@ var fileDescriptor_b179ea967ba71047 = []byte{
 	0xa7, 0x95, 0x27, 0x3d, 0x56, 0x55, 0xfe, 0x50, 0xdd, 0x7d, 0x3c, 0xd6, 0x76, 0xfe, 0x1e, 0x00,
 	0x00, 0xff, 0xff, 0x38, 0x42, 0x3b, 0xd2, 0xdc, 0x01, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// SearchServiceClient is the client API for SearchService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type SearchServiceClient interface {
-	Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error)
-	StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error)
-}
-
-type searchServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient {
-	return &searchServiceClient{cc}
-}
-
-func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) {
-	out := new(SearchResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.SearchService/Search", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_SearchService_serviceDesc.Streams[0], "/grpc.testing.SearchService/StreamingSearch", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &searchServiceStreamingSearchClient{stream}
-	return x, nil
-}
-
-type SearchService_StreamingSearchClient interface {
-	Send(*SearchRequest) error
-	Recv() (*SearchResponse, error)
-	grpc.ClientStream
-}
-
-type searchServiceStreamingSearchClient struct {
-	grpc.ClientStream
-}
-
-func (x *searchServiceStreamingSearchClient) Send(m *SearchRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *searchServiceStreamingSearchClient) Recv() (*SearchResponse, error) {
-	m := new(SearchResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// SearchServiceServer is the server API for SearchService service.
-type SearchServiceServer interface {
-	Search(context.Context, *SearchRequest) (*SearchResponse, error)
-	StreamingSearch(SearchService_StreamingSearchServer) error
-}
-
-// UnimplementedSearchServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedSearchServiceServer struct {
-}
-
-func (*UnimplementedSearchServiceServer) Search(ctx context.Context, req *SearchRequest) (*SearchResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
-}
-func (*UnimplementedSearchServiceServer) StreamingSearch(srv SearchService_StreamingSearchServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
-}
-
-func RegisterSearchServiceServer(s *grpc.Server, srv SearchServiceServer) {
-	s.RegisterService(&_SearchService_serviceDesc, srv)
-}
-
-func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SearchRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(SearchServiceServer).Search(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.SearchService/Search",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _SearchService_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(SearchServiceServer).StreamingSearch(&searchServiceStreamingSearchServer{stream})
-}
-
-type SearchService_StreamingSearchServer interface {
-	Send(*SearchResponse) error
-	Recv() (*SearchRequest, error)
-	grpc.ServerStream
-}
-
-type searchServiceStreamingSearchServer struct {
-	grpc.ServerStream
-}
-
-func (x *searchServiceStreamingSearchServer) Send(m *SearchResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) {
-	m := new(SearchRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _SearchService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.SearchService",
-	HandlerType: (*SearchServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Search",
-			Handler:    _SearchService_Search_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingSearch",
-			Handler:       _SearchService_StreamingSearch_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "reflection/grpc_testing/test.proto",
-}
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
new file mode 100644
index 000000000000..52eac202bfeb
--- /dev/null
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -0,0 +1,155 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// SearchServiceClient is the client API for SearchService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type SearchServiceClient interface {
+	Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error)
+	StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error)
+}
+
+type searchServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient {
+	return &searchServiceClient{cc}
+}
+
+func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) {
+	out := new(SearchResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.SearchService/Search", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_SearchService_serviceDesc.Streams[0], "/grpc.testing.SearchService/StreamingSearch", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &searchServiceStreamingSearchClient{stream}
+	return x, nil
+}
+
+type SearchService_StreamingSearchClient interface {
+	Send(*SearchRequest) error
+	Recv() (*SearchResponse, error)
+	grpc.ClientStream
+}
+
+type searchServiceStreamingSearchClient struct {
+	grpc.ClientStream
+}
+
+func (x *searchServiceStreamingSearchClient) Send(m *SearchRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *searchServiceStreamingSearchClient) Recv() (*SearchResponse, error) {
+	m := new(SearchResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// SearchServiceServer is the server API for SearchService service.
+type SearchServiceServer interface {
+	Search(context.Context, *SearchRequest) (*SearchResponse, error)
+	StreamingSearch(SearchService_StreamingSearchServer) error
+}
+
+// UnimplementedSearchServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedSearchServiceServer struct {
+}
+
+func (*UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
+}
+func (*UnimplementedSearchServiceServer) StreamingSearch(SearchService_StreamingSearchServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
+}
+
+func RegisterSearchServiceServer(s *grpc.Server, srv SearchServiceServer) {
+	s.RegisterService(&_SearchService_serviceDesc, srv)
+}
+
+func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SearchRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SearchServiceServer).Search(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.SearchService/Search",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SearchService_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SearchServiceServer).StreamingSearch(&searchServiceStreamingSearchServer{stream})
+}
+
+type SearchService_StreamingSearchServer interface {
+	Send(*SearchResponse) error
+	Recv() (*SearchRequest, error)
+	grpc.ServerStream
+}
+
+type searchServiceStreamingSearchServer struct {
+	grpc.ServerStream
+}
+
+func (x *searchServiceStreamingSearchServer) Send(m *SearchResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) {
+	m := new(SearchRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _SearchService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.SearchService",
+	HandlerType: (*SearchServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Search",
+			Handler:    _SearchService_Search_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingSearch",
+			Handler:       _SearchService_StreamingSearch_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "reflection/grpc_testing/test.proto",
+}
diff --git a/regenerate.sh b/regenerate.sh
index 9dc72e16af33..6f47daf94bc3 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -29,6 +29,9 @@ mkdir -p ${GOBIN}
 echo "go install github.com/golang/protobuf/protoc-gen-go"
 (cd test/tools && go install github.com/golang/protobuf/protoc-gen-go)
 
+echo "go install cmd/protoc-gen-go-grpc"
+(cd cmd/protoc-gen-go-grpc && go install .)
+
 echo "git clone https://github.com/grpc/grpc-proto"
 git clone --quiet https://github.com/grpc/grpc-proto ${WORKDIR}/grpc-proto
 
@@ -54,7 +57,7 @@ SOURCES=(
 OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
-  protoc --go_opt=plugins=grpc --go_out=${OPTS}:${WORKDIR}/out \
+  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out \
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
@@ -68,7 +71,7 @@ mv ${WORKDIR}/out/google.golang.org/grpc/lookup/grpc_lookup_v1/* ${WORKDIR}/out/
 
 # grpc_testingv3/testv3.pb.go is not re-generated because it was
 # intentionally generated by an older version of protoc-gen-go.
-rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/testv3.pb.go
+rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/*.pb.go
 
 # grpc/service_config/service_config.proto does not have a go_package option.
 cp ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config
diff --git a/stats/grpc_testing/test.pb.go b/stats/grpc_testing/test.pb.go
index 9ce8e6819ef9..df3796ebbd30 100644
--- a/stats/grpc_testing/test.pb.go
+++ b/stats/grpc_testing/test.pb.go
@@ -4,12 +4,8 @@
 package grpc_testing
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -127,299 +123,3 @@ var fileDescriptor_d7a50f7a8e9e8e09 = []byte{
 	0x48, 0x76, 0x2b, 0xb2, 0xfe, 0xed, 0xf4, 0x65, 0x6f, 0xeb, 0xf3, 0xf6, 0x2b, 0x00, 0x00, 0xff,
 	0xff, 0x7c, 0x26, 0xce, 0x3c, 0xf0, 0x01, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// TestServiceClient is the client API for TestService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type TestServiceClient interface {
-	// One request followed by one response.
-	// The server returns the client id as-is.
-	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
-	// Client stream
-	ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error)
-	// Server stream
-	ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error)
-}
-
-type testServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
-	return &testServiceClient{cc}
-}
-
-func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
-	out := new(SimpleResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceFullDuplexCallClient{stream}
-	return x, nil
-}
-
-type TestService_FullDuplexCallClient interface {
-	Send(*SimpleRequest) error
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceFullDuplexCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceFullDuplexCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceClientStreamCallClient{stream}
-	return x, nil
-}
-
-type TestService_ClientStreamCallClient interface {
-	Send(*SimpleRequest) error
-	CloseAndRecv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceClientStreamCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceClientStreamCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceServerStreamCallClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type TestService_ServerStreamCallClient interface {
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceServerStreamCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// TestServiceServer is the server API for TestService service.
-type TestServiceServer interface {
-	// One request followed by one response.
-	// The server returns the client id as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// Client stream
-	ClientStreamCall(TestService_ClientStreamCallServer) error
-	// Server stream
-	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
-}
-
-// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
-}
-
-func (*UnimplementedTestServiceServer) UnaryCall(ctx context.Context, req *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedTestServiceServer) FullDuplexCall(srv TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (*UnimplementedTestServiceServer) ClientStreamCall(srv TestService_ClientStreamCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
-}
-func (*UnimplementedTestServiceServer) ServerStreamCall(req *SimpleRequest, srv TestService_ServerStreamCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
-}
-
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
-}
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SimpleRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.TestService/UnaryCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-
-type TestService_FullDuplexCallServer interface {
-	Send(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceFullDuplexCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceFullDuplexCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_ClientStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).ClientStreamCall(&testServiceClientStreamCallServer{stream})
-}
-
-type TestService_ClientStreamCallServer interface {
-	SendAndClose(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceClientStreamCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceClientStreamCallServer) SendAndClose(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceClientStreamCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_ServerStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(SimpleRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(TestServiceServer).ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
-}
-
-type TestService_ServerStreamCallServer interface {
-	Send(*SimpleResponse) error
-	grpc.ServerStream
-}
-
-type testServiceServerStreamCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-var _TestService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "ClientStreamCall",
-			Handler:       _TestService_ClientStreamCall_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "ServerStreamCall",
-			Handler:       _TestService_ServerStreamCall_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "stats/grpc_testing/test.proto",
-}
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
new file mode 100644
index 000000000000..a12bccebcfdf
--- /dev/null
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -0,0 +1,302 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// TestServiceClient is the client API for TestService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type TestServiceClient interface {
+	// One request followed by one response.
+	// The server returns the client id as-is.
+	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
+	// Client stream
+	ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error)
+	// Server stream
+	ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error)
+}
+
+type testServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
+	return &testServiceClient{cc}
+}
+
+func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
+	out := new(SimpleResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceFullDuplexCallClient{stream}
+	return x, nil
+}
+
+type TestService_FullDuplexCallClient interface {
+	Send(*SimpleRequest) error
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceFullDuplexCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceFullDuplexCallClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceClientStreamCallClient{stream}
+	return x, nil
+}
+
+type TestService_ClientStreamCallClient interface {
+	Send(*SimpleRequest) error
+	CloseAndRecv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceClientStreamCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceClientStreamCallClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceServerStreamCallClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type TestService_ServerStreamCallClient interface {
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceServerStreamCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// TestServiceServer is the server API for TestService service.
+type TestServiceServer interface {
+	// One request followed by one response.
+	// The server returns the client id as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(TestService_FullDuplexCallServer) error
+	// Client stream
+	ClientStreamCall(TestService_ClientStreamCallServer) error
+	// Server stream
+	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
+}
+
+// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedTestServiceServer struct {
+}
+
+func (*UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (*UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+}
+func (*UnimplementedTestServiceServer) ClientStreamCall(TestService_ClientStreamCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
+}
+func (*UnimplementedTestServiceServer) ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
+}
+
+func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+	s.RegisterService(&_TestService_serviceDesc, srv)
+}
+
+func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SimpleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).UnaryCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/UnaryCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+
+type TestService_FullDuplexCallServer interface {
+	Send(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceFullDuplexCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceFullDuplexCallServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceFullDuplexCallServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _TestService_ClientStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).ClientStreamCall(&testServiceClientStreamCallServer{stream})
+}
+
+type TestService_ClientStreamCallServer interface {
+	SendAndClose(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceClientStreamCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceClientStreamCallServer) SendAndClose(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceClientStreamCallServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _TestService_ServerStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(SimpleRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(TestServiceServer).ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
+}
+
+type TestService_ServerStreamCallServer interface {
+	Send(*SimpleResponse) error
+	grpc.ServerStream
+}
+
+type testServiceServerStreamCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+var _TestService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.TestService",
+	HandlerType: (*TestServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryCall",
+			Handler:    _TestService_UnaryCall_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "FullDuplexCall",
+			Handler:       _TestService_FullDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "ClientStreamCall",
+			Handler:       _TestService_ClientStreamCall_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "ServerStreamCall",
+			Handler:       _TestService_ServerStreamCall_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "stats/grpc_testing/test.proto",
+}
diff --git a/stress/grpc_testing/metrics.pb.go b/stress/grpc_testing/metrics.pb.go
index 16dfe4f1e85e..40a0123c44bd 100644
--- a/stress/grpc_testing/metrics.pb.go
+++ b/stress/grpc_testing/metrics.pb.go
@@ -4,12 +4,8 @@
 package grpc_testing
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -228,153 +224,3 @@ var fileDescriptor_028251bc41da09ab = []byte{
 	0x0a, 0x85, 0xd4, 0x2a, 0x2d, 0xa5, 0xd0, 0x46, 0xba, 0xba, 0xa2, 0x3d, 0xd5, 0xad, 0x4f, 0x5c,
 	0x67, 0x8f, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x8c, 0x62, 0x73, 0xd8, 0x01, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// MetricsServiceClient is the client API for MetricsService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type MetricsServiceClient interface {
-	// Returns the values of all the gauges that are currently being maintained by
-	// the service
-	GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error)
-	// Returns the value of one gauge
-	GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error)
-}
-
-type metricsServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient {
-	return &metricsServiceClient{cc}
-}
-
-func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_MetricsService_serviceDesc.Streams[0], "/grpc.testing.MetricsService/GetAllGauges", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &metricsServiceGetAllGaugesClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type MetricsService_GetAllGaugesClient interface {
-	Recv() (*GaugeResponse, error)
-	grpc.ClientStream
-}
-
-type metricsServiceGetAllGaugesClient struct {
-	grpc.ClientStream
-}
-
-func (x *metricsServiceGetAllGaugesClient) Recv() (*GaugeResponse, error) {
-	m := new(GaugeResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) {
-	out := new(GaugeResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.MetricsService/GetGauge", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// MetricsServiceServer is the server API for MetricsService service.
-type MetricsServiceServer interface {
-	// Returns the values of all the gauges that are currently being maintained by
-	// the service
-	GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
-	// Returns the value of one gauge
-	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
-}
-
-// UnimplementedMetricsServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedMetricsServiceServer struct {
-}
-
-func (*UnimplementedMetricsServiceServer) GetAllGauges(req *EmptyMessage, srv MetricsService_GetAllGaugesServer) error {
-	return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
-}
-func (*UnimplementedMetricsServiceServer) GetGauge(ctx context.Context, req *GaugeRequest) (*GaugeResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
-}
-
-func RegisterMetricsServiceServer(s *grpc.Server, srv MetricsServiceServer) {
-	s.RegisterService(&_MetricsService_serviceDesc, srv)
-}
-
-func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(EmptyMessage)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(MetricsServiceServer).GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
-}
-
-type MetricsService_GetAllGaugesServer interface {
-	Send(*GaugeResponse) error
-	grpc.ServerStream
-}
-
-type metricsServiceGetAllGaugesServer struct {
-	grpc.ServerStream
-}
-
-func (x *metricsServiceGetAllGaugesServer) Send(m *GaugeResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GaugeRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(MetricsServiceServer).GetGauge(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.MetricsService/GetGauge",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(MetricsServiceServer).GetGauge(ctx, req.(*GaugeRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _MetricsService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.MetricsService",
-	HandlerType: (*MetricsServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetGauge",
-			Handler:    _MetricsService_GetGauge_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "GetAllGauges",
-			Handler:       _MetricsService_GetAllGauges_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "stress/grpc_testing/metrics.proto",
-}
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
new file mode 100644
index 000000000000..06c8a6b5fa10
--- /dev/null
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -0,0 +1,156 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// MetricsServiceClient is the client API for MetricsService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type MetricsServiceClient interface {
+	// Returns the values of all the gauges that are currently being maintained by
+	// the service
+	GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error)
+	// Returns the value of one gauge
+	GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error)
+}
+
+type metricsServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient {
+	return &metricsServiceClient{cc}
+}
+
+func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_MetricsService_serviceDesc.Streams[0], "/grpc.testing.MetricsService/GetAllGauges", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &metricsServiceGetAllGaugesClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type MetricsService_GetAllGaugesClient interface {
+	Recv() (*GaugeResponse, error)
+	grpc.ClientStream
+}
+
+type metricsServiceGetAllGaugesClient struct {
+	grpc.ClientStream
+}
+
+func (x *metricsServiceGetAllGaugesClient) Recv() (*GaugeResponse, error) {
+	m := new(GaugeResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) {
+	out := new(GaugeResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.MetricsService/GetGauge", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// MetricsServiceServer is the server API for MetricsService service.
+type MetricsServiceServer interface {
+	// Returns the values of all the gauges that are currently being maintained by
+	// the service
+	GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
+	// Returns the value of one gauge
+	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
+}
+
+// UnimplementedMetricsServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedMetricsServiceServer struct {
+}
+
+func (*UnimplementedMetricsServiceServer) GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error {
+	return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
+}
+func (*UnimplementedMetricsServiceServer) GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
+}
+
+func RegisterMetricsServiceServer(s *grpc.Server, srv MetricsServiceServer) {
+	s.RegisterService(&_MetricsService_serviceDesc, srv)
+}
+
+func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(EmptyMessage)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(MetricsServiceServer).GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
+}
+
+type MetricsService_GetAllGaugesServer interface {
+	Send(*GaugeResponse) error
+	grpc.ServerStream
+}
+
+type metricsServiceGetAllGaugesServer struct {
+	grpc.ServerStream
+}
+
+func (x *metricsServiceGetAllGaugesServer) Send(m *GaugeResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GaugeRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MetricsServiceServer).GetGauge(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.MetricsService/GetGauge",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MetricsServiceServer).GetGauge(ctx, req.(*GaugeRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _MetricsService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.MetricsService",
+	HandlerType: (*MetricsServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetGauge",
+			Handler:    _MetricsService_GetGauge_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "GetAllGauges",
+			Handler:       _MetricsService_GetAllGauges_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "stress/grpc_testing/metrics.proto",
+}
diff --git a/test/grpc_testing/test.pb.go b/test/grpc_testing/test.pb.go
index 2b4e59cd20b6..70e3b89228bc 100644
--- a/test/grpc_testing/test.pb.go
+++ b/test/grpc_testing/test.pb.go
@@ -4,12 +4,8 @@
 package grpc_testing
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -570,417 +566,3 @@ var fileDescriptor_139516ae706ad4b7 = []byte{
 	0x9e, 0xba, 0xb3, 0x2e, 0x16, 0xed, 0x7d, 0xf5, 0xea, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e,
 	0x50, 0x51, 0x5b, 0xcf, 0x06, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// TestServiceClient is the client API for TestService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type TestServiceClient interface {
-	// One empty request followed by one empty response.
-	EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
-	// One request followed by a sequence of responses (streamed download).
-	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error)
-	// A sequence of requests followed by one response (streamed upload).
-	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
-	// A sequence of requests followed by a sequence of responses.
-	// The server buffers all the client requests and then serves them in order. A
-	// stream of responses are returned to the client when the server starts with
-	// first request.
-	HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error)
-}
-
-type testServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
-	return &testServiceClient{cc}
-}
-
-func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
-	out := new(Empty)
-	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
-	out := new(SimpleResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceStreamingOutputCallClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type TestService_StreamingOutputCallClient interface {
-	Recv() (*StreamingOutputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceStreamingOutputCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallResponse, error) {
-	m := new(StreamingOutputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceStreamingInputCallClient{stream}
-	return x, nil
-}
-
-type TestService_StreamingInputCallClient interface {
-	Send(*StreamingInputCallRequest) error
-	CloseAndRecv() (*StreamingInputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceStreamingInputCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceStreamingInputCallClient) Send(m *StreamingInputCallRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCallResponse, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(StreamingInputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceFullDuplexCallClient{stream}
-	return x, nil
-}
-
-type TestService_FullDuplexCallClient interface {
-	Send(*StreamingOutputCallRequest) error
-	Recv() (*StreamingOutputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceFullDuplexCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceFullDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
-	m := new(StreamingOutputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceHalfDuplexCallClient{stream}
-	return x, nil
-}
-
-type TestService_HalfDuplexCallClient interface {
-	Send(*StreamingOutputCallRequest) error
-	Recv() (*StreamingOutputCallResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceHalfDuplexCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceHalfDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
-	m := new(StreamingOutputCallResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// TestServiceServer is the server API for TestService service.
-type TestServiceServer interface {
-	// One empty request followed by one empty response.
-	EmptyCall(context.Context, *Empty) (*Empty, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by a sequence of responses (streamed download).
-	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
-	// A sequence of requests followed by one response (streamed upload).
-	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(TestService_StreamingInputCallServer) error
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// A sequence of requests followed by a sequence of responses.
-	// The server buffers all the client requests and then serves them in order. A
-	// stream of responses are returned to the client when the server starts with
-	// first request.
-	HalfDuplexCall(TestService_HalfDuplexCallServer) error
-}
-
-// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
-}
-
-func (*UnimplementedTestServiceServer) EmptyCall(ctx context.Context, req *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-}
-func (*UnimplementedTestServiceServer) UnaryCall(ctx context.Context, req *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingOutputCall(req *StreamingOutputCallRequest, srv TestService_StreamingOutputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingInputCall(srv TestService_StreamingInputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) FullDuplexCall(srv TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (*UnimplementedTestServiceServer) HalfDuplexCall(srv TestService_HalfDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-}
-
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
-}
-
-func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Empty)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(TestServiceServer).EmptyCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.TestService/EmptyCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SimpleRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.TestService/UnaryCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(StreamingOutputCallRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
-}
-
-type TestService_StreamingOutputCallServer interface {
-	Send(*StreamingOutputCallResponse) error
-	grpc.ServerStream
-}
-
-type testServiceStreamingOutputCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
-}
-
-type TestService_StreamingInputCallServer interface {
-	SendAndClose(*StreamingInputCallResponse) error
-	Recv() (*StreamingInputCallRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceStreamingInputCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceStreamingInputCallServer) SendAndClose(m *StreamingInputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest, error) {
-	m := new(StreamingInputCallRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-
-type TestService_FullDuplexCallServer interface {
-	Send(*StreamingOutputCallResponse) error
-	Recv() (*StreamingOutputCallRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceFullDuplexCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceFullDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
-	m := new(StreamingOutputCallRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
-}
-
-type TestService_HalfDuplexCallServer interface {
-	Send(*StreamingOutputCallResponse) error
-	Recv() (*StreamingOutputCallRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceHalfDuplexCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceHalfDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
-	m := new(StreamingOutputCallRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _TestService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "EmptyCall",
-			Handler:    _TestService_EmptyCall_Handler,
-		},
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingOutputCall",
-			Handler:       _TestService_StreamingOutputCall_Handler,
-			ServerStreams: true,
-		},
-		{
-			StreamName:    "StreamingInputCall",
-			Handler:       _TestService_StreamingInputCall_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "HalfDuplexCall",
-			Handler:       _TestService_HalfDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "test/grpc_testing/test.proto",
-}
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
new file mode 100644
index 000000000000..a211dcab9f35
--- /dev/null
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -0,0 +1,420 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// TestServiceClient is the client API for TestService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type TestServiceClient interface {
+	// One empty request followed by one empty response.
+	EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
+	// One request followed by a sequence of responses (streamed download).
+	// The server returns the payload with client desired type and sizes.
+	StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error)
+	// A sequence of requests followed by one response (streamed upload).
+	// The server returns the aggregated size of client payload as the result.
+	StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error)
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
+	// A sequence of requests followed by a sequence of responses.
+	// The server buffers all the client requests and then serves them in order. A
+	// stream of responses are returned to the client when the server starts with
+	// first request.
+	HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error)
+}
+
+type testServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
+	return &testServiceClient{cc}
+}
+
+func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
+	out := new(SimpleResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceStreamingOutputCallClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type TestService_StreamingOutputCallClient interface {
+	Recv() (*StreamingOutputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceStreamingOutputCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallResponse, error) {
+	m := new(StreamingOutputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceStreamingInputCallClient{stream}
+	return x, nil
+}
+
+type TestService_StreamingInputCallClient interface {
+	Send(*StreamingInputCallRequest) error
+	CloseAndRecv() (*StreamingInputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceStreamingInputCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceStreamingInputCallClient) Send(m *StreamingInputCallRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCallResponse, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(StreamingInputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceFullDuplexCallClient{stream}
+	return x, nil
+}
+
+type TestService_FullDuplexCallClient interface {
+	Send(*StreamingOutputCallRequest) error
+	Recv() (*StreamingOutputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceFullDuplexCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceFullDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
+	m := new(StreamingOutputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &testServiceHalfDuplexCallClient{stream}
+	return x, nil
+}
+
+type TestService_HalfDuplexCallClient interface {
+	Send(*StreamingOutputCallRequest) error
+	Recv() (*StreamingOutputCallResponse, error)
+	grpc.ClientStream
+}
+
+type testServiceHalfDuplexCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *testServiceHalfDuplexCallClient) Send(m *StreamingOutputCallRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) {
+	m := new(StreamingOutputCallResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// TestServiceServer is the server API for TestService service.
+type TestServiceServer interface {
+	// One empty request followed by one empty response.
+	EmptyCall(context.Context, *Empty) (*Empty, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by a sequence of responses (streamed download).
+	// The server returns the payload with client desired type and sizes.
+	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	// A sequence of requests followed by one response (streamed upload).
+	// The server returns the aggregated size of client payload as the result.
+	StreamingInputCall(TestService_StreamingInputCallServer) error
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(TestService_FullDuplexCallServer) error
+	// A sequence of requests followed by a sequence of responses.
+	// The server buffers all the client requests and then serves them in order. A
+	// stream of responses are returned to the client when the server starts with
+	// first request.
+	HalfDuplexCall(TestService_HalfDuplexCallServer) error
+}
+
+// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedTestServiceServer struct {
+}
+
+func (*UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+}
+func (*UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (*UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+}
+func (*UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+}
+func (*UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+}
+func (*UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+}
+
+func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+	s.RegisterService(&_TestService_serviceDesc, srv)
+}
+
+func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).EmptyCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/EmptyCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SimpleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).UnaryCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/UnaryCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(StreamingOutputCallRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
+}
+
+type TestService_StreamingOutputCallServer interface {
+	Send(*StreamingOutputCallResponse) error
+	grpc.ServerStream
+}
+
+type testServiceStreamingOutputCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
+}
+
+type TestService_StreamingInputCallServer interface {
+	SendAndClose(*StreamingInputCallResponse) error
+	Recv() (*StreamingInputCallRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceStreamingInputCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceStreamingInputCallServer) SendAndClose(m *StreamingInputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest, error) {
+	m := new(StreamingInputCallRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+
+type TestService_FullDuplexCallServer interface {
+	Send(*StreamingOutputCallResponse) error
+	Recv() (*StreamingOutputCallRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceFullDuplexCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceFullDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
+	m := new(StreamingOutputCallRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
+}
+
+type TestService_HalfDuplexCallServer interface {
+	Send(*StreamingOutputCallResponse) error
+	Recv() (*StreamingOutputCallRequest, error)
+	grpc.ServerStream
+}
+
+type testServiceHalfDuplexCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *testServiceHalfDuplexCallServer) Send(m *StreamingOutputCallResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) {
+	m := new(StreamingOutputCallRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _TestService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.TestService",
+	HandlerType: (*TestServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "EmptyCall",
+			Handler:    _TestService_EmptyCall_Handler,
+		},
+		{
+			MethodName: "UnaryCall",
+			Handler:    _TestService_UnaryCall_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingOutputCall",
+			Handler:       _TestService_StreamingOutputCall_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "StreamingInputCall",
+			Handler:       _TestService_StreamingInputCall_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "FullDuplexCall",
+			Handler:       _TestService_FullDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "HalfDuplexCall",
+			Handler:       _TestService_HalfDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "test/grpc_testing/test.proto",
+}

From 806b6fc5961483019a839ac031671b32a761ef3e Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 29 May 2020 09:43:40 -0700
Subject: [PATCH 084/481] xds: handle errors in cds balancer (#3649)

---
 .../balancer/cdsbalancer/cdsbalancer.go       |  58 +++++--
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 152 +++++++++++++++---
 xds/internal/testutils/channel.go             |  12 ++
 3 files changed, 183 insertions(+), 39 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 31ab7d697b34..8048a6fd0b80 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -25,6 +25,8 @@ import (
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/resolver"
@@ -183,13 +185,7 @@ func (b *cdsBalancer) run() {
 			// We first handle errors, if any, and then proceed with handling
 			// the update, only if the status quo has changed.
 			if err := update.err; err != nil {
-				// TODO: Should we cancel the watch only on specific errors?
-				if b.cancelWatch != nil {
-					b.cancelWatch()
-				}
-				if b.edsLB != nil {
-					b.edsLB.ResolverError(err)
-				}
+				b.handleErrorFromUpdate(err, true)
 			}
 			if b.client == update.client && b.clusterToWatch == update.clusterName {
 				break
@@ -221,9 +217,7 @@ func (b *cdsBalancer) run() {
 		case *watchUpdate:
 			if err := update.err; err != nil {
 				b.logger.Warningf("Watch error from xds-client %p: %v", b.client, err)
-				if b.edsLB != nil {
-					b.edsLB.ResolverError(err)
-				}
+				b.handleErrorFromUpdate(err, false)
 				break
 			}
 
@@ -271,6 +265,47 @@ func (b *cdsBalancer) run() {
 	}
 }
 
+// handleErrorFromUpdate handles both the error from parent ClientConn (from
+// resolver) and the error from xds client (from the watcher). fromParent is
+// true if error is from parent ClientConn.
+//
+// If the error is connection error, it's passed down to the child policy.
+// Nothing needs to be done in CDS (e.g. it doesn't go into fallback).
+//
+// If the error is resource-not-found:
+// - If it's from resolver, it means LDS resources were removed. The CDS watch
+// should be canceled.
+// - If it's from xds client, it means CDS resource were removed. The CDS
+// watcher should keep watching.
+//
+// In both cases, the error will be forwarded to EDS balancer. And if error is
+// resource-not-found, the child EDS balancer will stop watching EDS.
+func (b *cdsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
+	// TODO: connection errors will be sent to the eds balancers directly, and
+	// also forwarded by the parent balancers/resolvers. So the eds balancer may
+	// see the same error multiple times. We way want to only forward the error
+	// to eds if it's not a connection error.
+	//
+	// This is not necessary today, because xds client never sends connection
+	// errors.
+
+	if fromParent && xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound {
+		if b.cancelWatch != nil {
+			b.cancelWatch()
+		}
+	}
+	if b.edsLB != nil {
+		b.edsLB.ResolverError(err)
+	} else {
+		// If eds balancer was never created, fail the RPCs with
+		// errors.
+		b.cc.UpdateState(balancer.State{
+			ConnectivityState: connectivity.TransientFailure,
+			Picker:            base.NewErrPicker(err),
+		})
+	}
+}
+
 // handleClusterUpdate is the CDS watch API callback. It simply pushes the
 // received information on to the update channel for run() to pick it up.
 func (b *cdsBalancer) handleClusterUpdate(cu xdsclient.ClusterUpdate, err error) {
@@ -318,9 +353,6 @@ func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) erro
 }
 
 // ResolverError handles errors reported by the xdsResolver.
-//
-// TODO: Make it possible to differentiate between connection errors and
-// resource not found errors.
 func (b *cdsBalancer) ResolverError(err error) {
 	if b.isClosed() {
 		b.logger.Warningf("xds: received resolver error {%v} after cdsBalancer was closed", err)
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 683c40b89cb7..31e56c1449ee 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -55,6 +55,18 @@ func Test(t *testing.T) {
 
 type testClientConn struct {
 	balancer.ClientConn
+
+	newPickerCh *testutils.Channel // The last picker updated.
+}
+
+func newTestClientConn() *testClientConn {
+	return &testClientConn{
+		newPickerCh: testutils.NewChannelWithSize(1),
+	}
+}
+
+func (tcc *testClientConn) UpdateState(bs balancer.State) {
+	tcc.newPickerCh.Replace(bs)
 }
 
 // cdsWatchInfo wraps the update and the error sent in a CDS watch callback.
@@ -186,7 +198,7 @@ func cdsCCS(cluster string, xdsClient interface{}) balancer.ClientConnState {
       "loadBalancingConfig":[
         {
           "cds_experimental":{
-            "Cluster": %s
+            "Cluster": "%s"
           }
         }
       ]
@@ -216,9 +228,10 @@ func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.Clie
 
 // setup creates a cdsBalancer and an edsBalancer (and overrides the
 // newEDSBalancer function to return it), and also returns a cleanup function.
-func setup() (*cdsBalancer, *testEDSBalancer, func()) {
+func setup() (*cdsBalancer, *testEDSBalancer, *testClientConn, func()) {
 	builder := cdsBB{}
-	tcc := &testClientConn{}
+	tcc := newTestClientConn()
+	// cdsB := builder.Build(tcc, balancer.BuildOptions{}).(balancer.V2Balancer)
 	cdsB := builder.Build(tcc, balancer.BuildOptions{})
 
 	edsB := newTestEDSBalancer()
@@ -227,18 +240,18 @@ func setup() (*cdsBalancer, *testEDSBalancer, func()) {
 		return edsB, nil
 	}
 
-	return cdsB.(*cdsBalancer), edsB, func() {
+	return cdsB.(*cdsBalancer), edsB, tcc, func() {
 		newEDSBalancer = oldEDSBalancerBuilder
 	}
 }
 
 // setupWithWatch does everything that setup does, and also pushes a ClientConn
 // update to the cdsBalancer and waits for a CDS watch call to be registered.
-func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, func()) {
+func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *testClientConn, func()) {
 	t.Helper()
 
 	xdsC := fakeclient.NewClient()
-	cdsB, edsB, cancel := setup()
+	cdsB, edsB, tcc, cancel := setup()
 	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
@@ -249,7 +262,7 @@ func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBal
 	if gotCluster != clusterName {
 		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName)
 	}
-	return xdsC, cdsB, edsB, cancel
+	return xdsC, cdsB, edsB, tcc, cancel
 }
 
 // TestUpdateClientConnState invokes the UpdateClientConnState method on the
@@ -303,7 +316,7 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			cdsB, _, cancel := setup()
+			cdsB, _, _, cancel := setup()
 			defer func() {
 				cancel()
 				cdsB.Close()
@@ -330,7 +343,7 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 // TestUpdateClientConnStateAfterClose invokes the UpdateClientConnState method
 // on the cdsBalancer after close and verifies that it returns an error.
 func (s) TestUpdateClientConnStateAfterClose(t *testing.T) {
-	cdsB, _, cancel := setup()
+	cdsB, _, _, cancel := setup()
 	defer cancel()
 	cdsB.Close()
 
@@ -343,7 +356,7 @@ func (s) TestUpdateClientConnStateAfterClose(t *testing.T) {
 // update with the same cluster and xdsClient does not cause the cdsBalancer to
 // create a new watch.
 func (s) TestUpdateClientConnStateWithSameState(t *testing.T) {
-	xdsC, cdsB, _, cancel := setupWithWatch(t)
+	xdsC, cdsB, _, _, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -361,7 +374,7 @@ func (s) TestUpdateClientConnStateWithSameState(t *testing.T) {
 // different updates and verifies that the expect ClientConnState is propagated
 // to the edsBalancer.
 func (s) TestHandleClusterUpdate(t *testing.T) {
-	xdsC, cdsB, edsB, cancel := setupWithWatch(t)
+	xdsC, cdsB, edsB, _, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -383,10 +396,6 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
 			wantCCS:   edsCCS(serviceName, false, xdsC),
 		},
-		{
-			name:      "cdsWatch-returns-error",
-			updateErr: errors.New("cdsUpdate error"),
-		},
 	}
 
 	for _, test := range tests {
@@ -398,36 +407,127 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 	}
 }
 
-// TestResolverError verifies that an existing watch is cancelled when a
-// resolver error is received by the cdsBalancer, and also that the same error
-// is propagated to the edsBalancer.
+// TestHandleClusterUpdateError covers the cases that an error is returned from
+// the watcher.
+//
+// Includes error with and without a child eds balancer, and whether error is a
+// resource-not-found error.
+func (s) TestHandleClusterUpdateError(t *testing.T) {
+	xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// An error before eds balancer is built. Should result in an error picker.
+	// And this is not a resource not found error, watch shouldn't be canceled.
+	err1 := errors.New("cdsBalancer resolver error 1")
+	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err1)
+	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsB.waitForResolverError(err1); err == nil {
+		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
+	}
+	state, err := tcc.newPickerCh.Receive()
+	if err != nil {
+		t.Fatalf("failed to get picker, expect an error picker")
+	}
+	picker := state.(balancer.State).Picker
+	if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
+		t.Fatalf("want picker to always fail, got nil")
+	}
+
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// An error after eds balancer is build, eds should receive the error. This
+	// is not a resource not found error, watch shouldn't be canceled
+	err2 := errors.New("cdsBalancer resolver error 2")
+	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err2)
+	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsB.waitForResolverError(err2); err != nil {
+		t.Fatalf("eds balancer should get error, waitForError failed: %v", err)
+	}
+
+	// A resource not found error. Watch should not be canceled because this
+	// means CDS resource is removed, and eds should receive the error.
+	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
+	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, resourceErr)
+	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+		t.Fatalf("want watch to be not canceled, watchForCancel should timeout")
+	}
+	if err := edsB.waitForResolverError(resourceErr); err != nil {
+		t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err)
+	}
+}
+
+// TestResolverError verifies that resolvers errors (with type
+// resource-not-found or others) are handled correctly.
 func (s) TestResolverError(t *testing.T) {
-	xdsC, cdsB, edsB, cancel := setupWithWatch(t)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
+	// An error before eds balancer is built. Should result in an error picker.
+	// Not a resource not found error, watch shouldn't be canceled.
+	err1 := errors.New("cdsBalancer resolver error 1")
+	cdsB.ResolverError(err1)
+	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsB.waitForResolverError(err1); err == nil {
+		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
+	}
+	state, err := tcc.newPickerCh.Receive()
+	if err != nil {
+		t.Fatalf("failed to get picker, expect an error picker")
+	}
+	picker := state.(balancer.State).Picker
+	if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
+		t.Fatalf("want picker to always fail, got nil")
+	}
+
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
-	rErr := errors.New("cdsBalancer resolver error")
-	cdsB.ResolverError(rErr)
+	// Not a resource not found error, watch shouldn't be canceled, and eds
+	// should receive the error.
+	err2 := errors.New("cdsBalancer resolver error 2")
+	cdsB.ResolverError(err2)
+	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	}
+	if err := edsB.waitForResolverError(err2); err != nil {
+		t.Fatalf("eds balancer should get error, waitForError failed: %v", err)
+	}
+
+	// A resource not found error. Watch should be canceled, and eds should
+	// receive the error.
+	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
+	cdsB.ResolverError(resourceErr)
 	if err := xdsC.WaitForCancelClusterWatch(); err != nil {
-		t.Fatal(err)
+		t.Fatalf("want watch to be canceled, watchForCancel failed: %v", err)
 	}
-	if err := edsB.waitForResolverError(rErr); err != nil {
-		t.Fatal(err)
+	if err := edsB.waitForResolverError(resourceErr); err != nil {
+		t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err)
 	}
 }
 
 // TestUpdateSubConnState pushes a SubConn update to the cdsBalancer and
 // verifies that the update is propagated to the edsBalancer.
 func (s) TestUpdateSubConnState(t *testing.T) {
-	xdsC, cdsB, edsB, cancel := setupWithWatch(t)
+	xdsC, cdsB, edsB, _, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -450,7 +550,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 // TestClose calls Close() on the cdsBalancer, and verifies that the underlying
 // edsBalancer is also closed.
 func (s) TestClose(t *testing.T) {
-	xdsC, cdsB, edsB, cancel := setupWithWatch(t)
+	xdsC, cdsB, edsB, _, cancel := setupWithWatch(t)
 	defer cancel()
 
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
diff --git a/xds/internal/testutils/channel.go b/xds/internal/testutils/channel.go
index a2b34afaf36e..1715a01ff68f 100644
--- a/xds/internal/testutils/channel.go
+++ b/xds/internal/testutils/channel.go
@@ -45,6 +45,18 @@ func (cwt *Channel) Send(value interface{}) {
 	cwt.ch <- value
 }
 
+// Replace clears the value on the underlying channel, and sends the new value.
+//
+// It's expected to be used with a size-1 channel, to only keep the most
+// up-to-date item.
+func (cwt *Channel) Replace(value interface{}) {
+	select {
+	case <-cwt.ch:
+	default:
+	}
+	cwt.ch <- value
+}
+
 // TimedReceive returns the value received on the underlying channel, or
 // ErrRecvTimeout if timeout amount of time elapsed.
 func (cwt *Channel) TimedReceive(timeout time.Duration) (interface{}, error) {

From a085b3e7f600065bc998589ca128ae2dc1a93b50 Mon Sep 17 00:00:00 2001
From: yihuaz <yihuaz@google.com>
Date: Mon, 1 Jun 2020 14:55:19 -0700
Subject: [PATCH 085/481] credentials/local: fix incorrect binding of listener
 port (#3656)

---
 credentials/local/local_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go
index a508d89bcd06..3c65010e8b2a 100644
--- a/credentials/local/local_test.go
+++ b/credentials/local/local_test.go
@@ -163,17 +163,17 @@ func (s) TestServerAndClientHandshake(t *testing.T) {
 	}{
 		{
 			testNetwork: "tcp",
-			testAddr:    "127.0.0.1:10000",
+			testAddr:    "127.0.0.1:0",
 			want:        credentials.NoSecurity,
 		},
 		{
 			testNetwork: "tcp",
-			testAddr:    "[::1]:10000",
+			testAddr:    "[::1]:0",
 			want:        credentials.NoSecurity,
 		},
 		{
 			testNetwork: "tcp",
-			testAddr:    "localhost:10000",
+			testAddr:    "localhost:0",
 			want:        credentials.NoSecurity,
 		},
 		{

From cb7f5de3ff0b8395988343583f908d310f51a77a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 2 Jun 2020 13:17:04 -0700
Subject: [PATCH 086/481] xds: handle errors in xds resolver (#3651)

---
 xds/internal/resolver/serviceconfig_test.go | 10 +++++
 xds/internal/resolver/xds_resolver.go       | 12 ++++++
 xds/internal/resolver/xds_resolver_test.go  | 44 +++++++++++++++++++++
 3 files changed, 66 insertions(+)

diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index c4d97261e691..6e83a23cdd5d 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -48,6 +48,11 @@ const (
 		}
 	  }
 	}
+}]}`
+	testWeightedCDSNoChildJSON = `{"loadBalancingConfig":[{
+    "weighted_target_experimental": {
+	  "targets": {}
+	}
 }]}`
 )
 
@@ -62,6 +67,11 @@ func TestServiceUpdateToJSON(t *testing.T) {
 			su:       client.ServiceUpdate{WeightedCluster: map[string]uint32{testCluster1: 1}},
 			wantJSON: testClusterOnlyJSON,
 		},
+		{
+			name:     "empty weighted clusters",
+			su:       client.ServiceUpdate{WeightedCluster: nil},
+			wantJSON: testWeightedCDSNoChildJSON,
+		},
 		{
 			name: "weighted clusters",
 			su: client.ServiceUpdate{WeightedCluster: map[string]uint32{
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 2b806cc62c8e..cbba73bdbcda 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -170,6 +170,18 @@ func (r *xdsResolver) run() {
 		case update := <-r.updateCh:
 			if update.err != nil {
 				r.logger.Warningf("Watch error on resource %v from xds-client %p, %v", r.target.Endpoint, r.client, update.err)
+				if xdsclient.ErrType(update.err) == xdsclient.ErrorTypeResourceNotFound {
+					// If error is resource-not-found, it means the LDS resource
+					// was removed. Send an empty service config, which picks
+					// pick-first, with no address, and puts the ClientConn into
+					// transient failure..
+					r.cc.UpdateState(resolver.State{
+						ServiceConfig: r.cc.ParseServiceConfig("{}"),
+					})
+					continue
+				}
+				// Send error to ClientConn, and balancers, if error is not
+				// resource not found.
 				r.cc.ReportError(update.err)
 				continue
 			}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index a6d1f72229c5..7e1e49d2d92a 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -406,3 +406,47 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr2)
 	}
 }
+
+// TestXDSResolverResourceNotFoundError tests the cases where the resolver gets
+// a ResourceNotFoundError. It should generate a service config picking
+// weighted_target, but no child balancers.
+func TestXDSResolverResourceNotFoundError(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		config:        &validConfig,
+		xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer func() {
+		cancel()
+		xdsR.Close()
+	}()
+
+	waitForWatchService(t, xdsC, targetStr)
+
+	// Invoke the watchAPI callback with a bad service update and wait for the
+	// ReportError method to be called on the ClientConn.
+	suErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "resource removed error")
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
+	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != testutils.ErrRecvTimeout {
+		t.Fatalf("ClientConn.ReportError() received %v, %v, want channel recv timeout", gotErrVal, gotErr)
+	}
+	gotState, err := tcc.stateCh.Receive()
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState := gotState.(resolver.State)
+	// This update shouldn't have xds-client in it, because it doesn't pick an
+	// xds balancer.
+	if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != nil {
+		t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want <nil>", gotClient)
+	}
+	wantParsedConfig := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)("{}")
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantParsedConfig.Config) {
+		t.Error("ClientConn.UpdateState got wrong service config")
+		t.Errorf("gotParsed: %s", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Errorf("wantParsed: %s", cmp.Diff(nil, wantParsedConfig.Config))
+	}
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+}

From 42eed59dfa460aae8a92349be77f83aa7ee39672 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 3 Jun 2020 09:57:52 -0700
Subject: [PATCH 087/481] xds: handle errors in xds_client (#3658)

- xds_client
  - send resource-not-found error when a resource is removed for LDS or CDS
  - handle LDS resource-not-found to cancel RDS watch

- test update because it was expecting no update when resource is removed
- test cleanup to apply timeout to channels
---
 xds/internal/client/client_callback.go        |  54 +++++---
 xds/internal/client/client_test.go            |  36 +++--
 xds/internal/client/client_watchers.go        |  37 +++++-
 .../client/client_watchers_cluster_test.go    | 115 +++++++++++++++-
 .../client/client_watchers_endpoints_test.go  |  23 +++-
 .../client/client_watchers_lds_test.go        | 105 ++++++++++++++-
 .../client/client_watchers_rds_test.go        |  18 +++
 .../client/client_watchers_service.go         |  21 ++-
 .../client/client_watchers_service_test.go    | 124 ++++++++++++++++--
 9 files changed, 467 insertions(+), 66 deletions(-)

diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go
index 11dd6ccabc11..4bbdaabd2f7f 100644
--- a/xds/internal/client/client_callback.go
+++ b/xds/internal/client/client_callback.go
@@ -74,11 +74,11 @@ func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newLDSUpdate(d map[string]ldsUpdate) {
+func (c *Client) newLDSUpdate(updates map[string]ldsUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for name, update := range d {
+	for name, update := range updates {
 		if s, ok := c.ldsWatchers[name]; ok {
 			for wi := range s {
 				wi.newUpdate(update)
@@ -88,11 +88,20 @@ func (c *Client) newLDSUpdate(d map[string]ldsUpdate) {
 			c.ldsCache[name] = update
 		}
 	}
-	// TODO: handle removing resources, which means if a resource exists in the
-	// previous update, but not in the new update. This needs the balancers and
-	// resolvers to handle errors correctly.
-
-	// TODO: remove item from cache and remove corresponding RDS cached data.
+	for name := range c.ldsCache {
+		if _, ok := updates[name]; !ok {
+			// If resource exists in cache, but not in the new update, delete it
+			// from cache, and also send an resource not found error to indicate
+			// resource removed.
+			delete(c.ldsCache, name)
+			for wi := range c.ldsWatchers[name] {
+				wi.resourceNotFound()
+			}
+		}
+	}
+	// When LDS resource is removed, we don't delete corresponding RDS cached
+	// data. The RDS watch will be canceled, and cache entry is removed when the
+	// last watch is canceled.
 }
 
 // newRDSUpdate is called by the underlying xdsv2Client when it receives an xDS
@@ -100,11 +109,11 @@ func (c *Client) newLDSUpdate(d map[string]ldsUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newRDSUpdate(d map[string]rdsUpdate) {
+func (c *Client) newRDSUpdate(updates map[string]rdsUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for name, update := range d {
+	for name, update := range updates {
 		if s, ok := c.rdsWatchers[name]; ok {
 			for wi := range s {
 				wi.newUpdate(update)
@@ -121,11 +130,11 @@ func (c *Client) newRDSUpdate(d map[string]rdsUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newCDSUpdate(d map[string]ClusterUpdate) {
+func (c *Client) newCDSUpdate(updates map[string]ClusterUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for name, update := range d {
+	for name, update := range updates {
 		if s, ok := c.cdsWatchers[name]; ok {
 			for wi := range s {
 				wi.newUpdate(update)
@@ -135,11 +144,20 @@ func (c *Client) newCDSUpdate(d map[string]ClusterUpdate) {
 			c.cdsCache[name] = update
 		}
 	}
-	// TODO: handle removing resources, which means if a resource exists in the
-	// previous update, but not in the new update. This needs the balancers and
-	// resolvers to handle errors correctly.
-
-	// TODO: remove item from cache and remove corresponding EDS cached data.
+	for name := range c.cdsCache {
+		if _, ok := updates[name]; !ok {
+			// If resource exists in cache, but not in the new update, delete it
+			// from cache, and also send an resource not found error to indicate
+			// resource removed.
+			delete(c.cdsCache, name)
+			for wi := range c.cdsWatchers[name] {
+				wi.resourceNotFound()
+			}
+		}
+	}
+	// When CDS resource is removed, we don't delete corresponding EDS cached
+	// data. The EDS watch will be canceled, and cache entry is removed when the
+	// last watch is canceled.
 }
 
 // newEDSUpdate is called by the underlying xdsv2Client when it receives an xDS
@@ -147,11 +165,11 @@ func (c *Client) newCDSUpdate(d map[string]ClusterUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newEDSUpdate(d map[string]EndpointsUpdate) {
+func (c *Client) newEDSUpdate(updates map[string]EndpointsUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for name, update := range d {
+	for name, update := range updates {
 		if s, ok := c.edsWatchers[name]; ok {
 			for wi := range s {
 				wi.newUpdate(update)
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 0775032711c6..367e9e02642a 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -126,8 +126,8 @@ func (s) TestNew(t *testing.T) {
 type testXDSV2Client struct {
 	r updateHandler
 
-	addWatches    map[string]chan string
-	removeWatches map[string]chan string
+	addWatches    map[string]*testutils.Channel
+	removeWatches map[string]*testutils.Channel
 }
 
 func overrideNewXDSV2Client() (<-chan *testXDSV2Client, func()) {
@@ -142,16 +142,16 @@ func overrideNewXDSV2Client() (<-chan *testXDSV2Client, func()) {
 }
 
 func newTestXDSV2Client(r updateHandler) *testXDSV2Client {
-	addWatches := make(map[string]chan string)
-	addWatches[ldsURL] = make(chan string, 10)
-	addWatches[rdsURL] = make(chan string, 10)
-	addWatches[cdsURL] = make(chan string, 10)
-	addWatches[edsURL] = make(chan string, 10)
-	removeWatches := make(map[string]chan string)
-	removeWatches[ldsURL] = make(chan string, 10)
-	removeWatches[rdsURL] = make(chan string, 10)
-	removeWatches[cdsURL] = make(chan string, 10)
-	removeWatches[edsURL] = make(chan string, 10)
+	addWatches := make(map[string]*testutils.Channel)
+	addWatches[ldsURL] = testutils.NewChannel()
+	addWatches[rdsURL] = testutils.NewChannel()
+	addWatches[cdsURL] = testutils.NewChannel()
+	addWatches[edsURL] = testutils.NewChannel()
+	removeWatches := make(map[string]*testutils.Channel)
+	removeWatches[ldsURL] = testutils.NewChannel()
+	removeWatches[rdsURL] = testutils.NewChannel()
+	removeWatches[cdsURL] = testutils.NewChannel()
+	removeWatches[edsURL] = testutils.NewChannel()
 	return &testXDSV2Client{
 		r:             r,
 		addWatches:    addWatches,
@@ -160,11 +160,11 @@ func newTestXDSV2Client(r updateHandler) *testXDSV2Client {
 }
 
 func (c *testXDSV2Client) addWatch(resourceType, resourceName string) {
-	c.addWatches[resourceType] <- resourceName
+	c.addWatches[resourceType].Send(resourceName)
 }
 
 func (c *testXDSV2Client) removeWatch(resourceType, resourceName string) {
-	c.removeWatches[resourceType] <- resourceName
+	c.removeWatches[resourceType].Send(resourceName)
 }
 
 func (c *testXDSV2Client) close() {}
@@ -184,11 +184,19 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	clusterUpdateCh := testutils.NewChannel()
+	firstTime := true
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		// Calls another watch inline, to ensure there's deadlock.
 		c.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
+		if _, err := v2Client.addWatches[cdsURL].Receive(); firstTime && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
+		firstTime = false
 	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index 7a5bd2614a03..cfe31e861380 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -75,6 +75,17 @@ func (wi *watchInfo) newUpdate(update interface{}) {
 	wi.c.scheduleCallback(wi, update, nil)
 }
 
+func (wi *watchInfo) resourceNotFound() {
+	wi.mu.Lock()
+	defer wi.mu.Unlock()
+	if wi.state == watchInfoStateCanceled {
+		return
+	}
+	wi.state = watchInfoStateRespReceived
+	wi.expiryTimer.Stop()
+	wi.sendErrorLocked(NewErrorf(ErrorTypeResourceNotFound, "xds: %s target %s not found in received response", wi.typeURL, wi.target))
+}
+
 func (wi *watchInfo) timeout() {
 	wi.mu.Lock()
 	defer wi.mu.Unlock()
@@ -82,25 +93,25 @@ func (wi *watchInfo) timeout() {
 		return
 	}
 	wi.state = watchInfoStateTimeout
+	wi.sendErrorLocked(fmt.Errorf("xds: %s target %s not found, watcher timeout", wi.typeURL, wi.target))
+}
+
+// Caller must hold wi.mu.
+func (wi *watchInfo) sendErrorLocked(err error) {
 	var (
 		u interface{}
-		t string
 	)
 	switch wi.typeURL {
 	case ldsURL:
 		u = ldsUpdate{}
-		t = "LDS"
 	case rdsURL:
 		u = rdsUpdate{}
-		t = "RDS"
 	case cdsURL:
 		u = ClusterUpdate{}
-		t = "CDS"
 	case edsURL:
 		u = EndpointsUpdate{}
-		t = "EDS"
 	}
-	wi.c.scheduleCallback(wi, u, fmt.Errorf("xds: %s target %s not found, watcher timeout", t, wi.target))
+	wi.c.scheduleCallback(wi, u, err)
 }
 
 func (wi *watchInfo) cancel() {
@@ -185,7 +196,19 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 				// watching this resource.
 				delete(watchers, resourceName)
 				c.v2c.removeWatch(wi.typeURL, resourceName)
-				// TODO: remove item from cache.
+				// Remove the resource from cache. When a watch for this
+				// resource is added later, it will trigger a xDS request with
+				// resource names, and client will receive new xDS responses.
+				switch wi.typeURL {
+				case ldsURL:
+					delete(c.ldsCache, resourceName)
+				case rdsURL:
+					delete(c.rdsCache, resourceName)
+				case cdsURL:
+					delete(c.cdsCache, resourceName)
+				case edsURL:
+					delete(c.edsCache, resourceName)
+				}
 			}
 		}
 	}
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index ab7a62091b9f..c14ea18e3c0a 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -32,7 +32,7 @@ type clusterUpdateErr struct {
 
 // TestClusterWatch covers the cases:
 // - an update is received after a watch()
-// - an update for another resource name (which doesn't trigger callback)
+// - an update for another resource name
 // - an upate is received after cancel()
 func (s) TestClusterWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
@@ -53,6 +53,9 @@ func (s) TestClusterWatch(t *testing.T) {
 	cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	// This is calling v2Client.r to send the update, but r is set to Client, so
@@ -69,13 +72,14 @@ func (s) TestClusterWatch(t *testing.T) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
-	// Another update for a different resource name.
+	// Another update, with an extra resource for a different resource name.
 	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName:  wantUpdate,
 		"randomName": {},
 	})
 
-	if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
+	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		t.Errorf("unexpected clusterUpdate: %+v, %v, want channel recv timeout", u, err)
 	}
 
 	// Cancel watch, and send update again.
@@ -114,6 +118,9 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[cdsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
@@ -168,6 +175,9 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 		c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[cdsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	// Third watch for a different name.
@@ -175,6 +185,9 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
@@ -212,6 +225,9 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
@@ -227,6 +243,9 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
+	if n, err := v2Client.addWatches[cdsURL].Receive(); err == nil {
+		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
+	}
 
 	// New watch should receives the update.
 	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
@@ -258,12 +277,15 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 	}
 	defer c.Close()
 
-	<-v2ClientCh
+	v2Client := <-v2ClientCh
 
 	clusterUpdateCh := testutils.NewChannel()
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	u, err := clusterUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
 	if err != nil {
@@ -303,6 +325,9 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
@@ -319,3 +344,83 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 		t.Fatalf("got unexpected: %v, %v, want recv timeout", u.(clusterUpdateErr).u, u.(clusterUpdateErr).err)
 	}
 }
+
+// TestClusterResourceRemoved covers the cases:
+// - an update is received after a watch()
+// - another update is received, with one resource removed
+//   - this should trigger callback with resource removed error
+// - one more update without the removed resource
+//   - the callback (above) shouldn't receive any update
+func (s) TestClusterResourceRemoved(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	clusterUpdateCh1 := testutils.NewChannel()
+	c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
+		clusterUpdateCh1.Send(clusterUpdateErr{u: update, err: err})
+	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	// Another watch for a different name.
+	clusterUpdateCh2 := testutils.NewChannel()
+	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
+		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
+	})
+	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+
+	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName + "1": wantUpdate1,
+		testCDSName + "2": wantUpdate2,
+	})
+
+	if u, err := clusterUpdateCh1.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Send another update to remove resource 1.
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName + "2": wantUpdate2,
+	})
+
+	// watcher 1 should get an error.
+	if u, err := clusterUpdateCh1.Receive(); err != nil || ErrType(u.(clusterUpdateErr).err) != ErrorTypeResourceNotFound {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
+	}
+
+	// watcher 2 should get the same update again.
+	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Send one more update without resource 1.
+	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+		testCDSName + "2": wantUpdate2,
+	})
+
+	// watcher 1 should get an error.
+	if u, err := clusterUpdateCh1.Receive(); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected clusterUpdate: %v, want receiving from channel timeout", u)
+	}
+
+	// watcher 2 should get the same update again.
+	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index 46e62bf57ec7..53b1bc316abc 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -69,6 +69,9 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
 	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
@@ -124,6 +127,9 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[edsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
@@ -178,6 +184,9 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 		c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[edsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	// Third watch for a different name.
@@ -185,6 +194,9 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate1 := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
 	wantUpdate2 := EndpointsUpdate{Localities: []Locality{testLocalities[1]}}
@@ -222,6 +234,9 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
 	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
@@ -237,6 +252,9 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
+	if n, err := v2Client.addWatches[edsURL].Receive(); err == nil {
+		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
+	}
 
 	// New watch should receives the update.
 	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
@@ -268,12 +286,15 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 	}
 	defer c.Close()
 
-	<-v2ClientCh
+	v2Client := <-v2ClientCh
 
 	endpointsUpdateCh := testutils.NewChannel()
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	u, err := endpointsUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
 	if err != nil {
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index 114726c029a5..5db842d261ca 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -31,7 +31,7 @@ type ldsUpdateErr struct {
 
 // TestLDSWatch covers the cases:
 // - an update is received after a watch()
-// - an update for another resource name (which doesn't trigger callback)
+// - an update for another resource name
 // - an upate is received after cancel()
 func (s) TestLDSWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
@@ -49,6 +49,9 @@ func (s) TestLDSWatch(t *testing.T) {
 	cancelWatch := c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := ldsUpdate{routeName: testRDSName}
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
@@ -59,12 +62,13 @@ func (s) TestLDSWatch(t *testing.T) {
 		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
-	// Another update for a different resource name.
+	// Another update, with an extra resource for a different resource name.
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName:  wantUpdate,
 		"randomName": {},
 	})
 
-	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
@@ -104,6 +108,9 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[ldsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	wantUpdate := ldsUpdate{routeName: testRDSName}
@@ -158,6 +165,9 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 		c.watchLDS(testLDSName+"1", func(update ldsUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[ldsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	// Third watch for a different name.
@@ -165,6 +175,9 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	c.watchLDS(testLDSName+"2", func(update ldsUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate1 := ldsUpdate{routeName: testRDSName + "1"}
 	wantUpdate2 := ldsUpdate{routeName: testRDSName + "2"}
@@ -202,6 +215,9 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := ldsUpdate{routeName: testRDSName}
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
@@ -217,6 +233,9 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
+	if n, err := v2Client.addWatches[ldsURL].Receive(); err == nil {
+		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
+	}
 
 	// New watch should receives the update.
 	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
@@ -228,3 +247,83 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
+
+// TestLDSResourceRemoved covers the cases:
+// - an update is received after a watch()
+// - another update is received, with one resource removed
+//   - this should trigger callback with resource removed error
+// - one more update without the removed resource
+//   - the callback (above) shouldn't receive any update
+func (s) TestLDSResourceRemoved(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	ldsUpdateCh1 := testutils.NewChannel()
+	c.watchLDS(testLDSName+"1", func(update ldsUpdate, err error) {
+		ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err})
+	})
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	// Another watch for a different name.
+	ldsUpdateCh2 := testutils.NewChannel()
+	c.watchLDS(testLDSName+"2", func(update ldsUpdate, err error) {
+		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
+	})
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+
+	wantUpdate1 := ldsUpdate{routeName: testEDSName + "1"}
+	wantUpdate2 := ldsUpdate{routeName: testEDSName + "2"}
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName + "1": wantUpdate1,
+		testLDSName + "2": wantUpdate2,
+	})
+
+	if u, err := ldsUpdateCh1.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Send another update to remove resource 1.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName + "2": wantUpdate2,
+	})
+
+	// watcher 1 should get an error.
+	if u, err := ldsUpdateCh1.Receive(); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
+	}
+
+	// watcher 2 should get the same update again.
+	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Send one more update without resource 1.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName + "2": wantUpdate2,
+	})
+
+	// watcher 1 should get an error.
+	if u, err := ldsUpdateCh1.Receive(); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected ldsUpdate: %v, want receiving from channel timeout", u)
+	}
+
+	// watcher 2 should get the same update again.
+	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+	}
+}
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 74bd3deac13b..06ed7a377e2b 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -50,6 +50,9 @@ func (s) TestRDSWatch(t *testing.T) {
 	cancelWatch := c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
@@ -105,6 +108,9 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[rdsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
@@ -159,6 +165,9 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		c.watchRDS(testRDSName+"1", func(update rdsUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
+		if _, err := v2Client.addWatches[rdsURL].Receive(); i == 0 && err != nil {
+			t.Fatalf("want new watch to start, got error %v", err)
+		}
 	}
 
 	// Third watch for a different name.
@@ -166,6 +175,9 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	c.watchRDS(testRDSName+"2", func(update rdsUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate1 := rdsUpdate{weightedCluster: map[string]uint32{testCDSName + "1": 1}}
 	wantUpdate2 := rdsUpdate{weightedCluster: map[string]uint32{testCDSName + "2": 1}}
@@ -203,6 +215,9 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
@@ -218,6 +233,9 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
+	if n, err := v2Client.addWatches[rdsURL].Receive(); err == nil {
+		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
+	}
 
 	// New watch should receives the update.
 	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
index b7fe57d6062b..1eea1f316a3c 100644
--- a/xds/internal/client/client_watchers_service.go
+++ b/xds/internal/client/client_watchers_service.go
@@ -75,12 +75,18 @@ func (w *serviceUpdateWatcher) handleLDSResp(update ldsUpdate, err error) {
 	if w.closed {
 		return
 	}
-	// TODO: this error case returns early, without canceling the existing RDS
-	// watch. If we decided to stop the RDS watch when LDS errors, move this
-	// after rdsCancel(). We may also need to check the error type and do
-	// different things based on that (e.g. cancel RDS watch only on
-	// resourceRemovedError, but not on connectionError).
 	if err != nil {
+		// We check the error type and do different things. For now, the only
+		// type we check is ResourceNotFound, which indicates the LDS resource
+		// was removed, and besides sending the error to callback, we also
+		// cancel the RDS watch.
+		if ErrType(err) == ErrorTypeResourceNotFound && w.rdsCancel != nil {
+			w.rdsCancel()
+			w.rdsName = ""
+			w.rdsCancel = nil
+		}
+		// The other error cases still return early without canceling the
+		// existing RDS watch.
 		w.serviceCb(ServiceUpdate{}, err)
 		return
 	}
@@ -104,6 +110,11 @@ func (w *serviceUpdateWatcher) handleRDSResp(update rdsUpdate, err error) {
 	if w.closed {
 		return
 	}
+	if w.rdsCancel == nil {
+		// This mean only the RDS watch is canceled, can happen if the LDS
+		// resource is removed.
+		return
+	}
 	if err != nil {
 		w.serviceCb(ServiceUpdate{}, err)
 		return
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 71de6c750146..8b341683d094 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -57,11 +57,15 @@ func (s) TestServiceWatch(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
-	<-v2Client.addWatches[ldsURL]
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
 		testLDSName: {routeName: testRDSName},
 	})
-	<-v2Client.addWatches[rdsURL]
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
@@ -93,11 +97,15 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
-	<-v2Client.addWatches[ldsURL]
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
 		testLDSName: {routeName: testRDSName},
 	})
-	<-v2Client.addWatches[rdsURL]
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
@@ -110,7 +118,9 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
 		testLDSName: {routeName: testRDSName + "2"},
 	})
-	<-v2Client.addWatches[rdsURL]
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 
 	// Another update for the old name.
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
@@ -154,11 +164,15 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
-	<-v2Client.addWatches[ldsURL]
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
 		testLDSName: {routeName: testRDSName},
 	})
-	<-v2Client.addWatches[rdsURL]
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
@@ -361,11 +375,15 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
 
-	<-v2Client.addWatches[ldsURL]
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
 		testLDSName: {routeName: testRDSName},
 	})
-	<-v2Client.addWatches[rdsURL]
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
@@ -378,9 +396,89 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
 		testLDSName: {routeName: testRDSName},
 	})
-	select {
-	case <-v2Client.removeWatches[rdsURL]:
-		t.Fatalf("unexpected rds watch cancel")
-	case <-time.After(time.Second):
+	if v, err := v2Client.removeWatches[rdsURL].Receive(); err == nil {
+		t.Fatalf("unexpected rds watch cancel: %v", v)
+	}
+}
+
+// TestServiceResourceRemoved covers the cases:
+// - an update is received after a watch()
+// - another update is received, with one resource removed
+//   - this should trigger callback with resource removed error
+// - one more update without the removed resource
+//   - the callback (above) shouldn't receive any update
+func (s) TestServiceResourceRemoved(t *testing.T) {
+	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	defer cleanup()
+
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
+
+	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
+
+	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+	})
+
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
+
+	// Remove LDS resource, should cancel the RDS watch, and trigger resource
+	// removed error.
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{})
+	if _, err := v2Client.removeWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want watch to be canceled, got error %v", err)
+	}
+	if u, err := serviceUpdateCh.Receive(); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
+	}
+
+	// Send RDS update for the removed LDS resource, expect no updates to
+	// callback, because RDS should be canceled.
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName + "new": 1}},
+	})
+	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
+	}
+
+	// Add LDS resource, but not RDS resource, should
+	//  - start a new RDS watch
+	//  - timeout on service channel, because RDS cache was cleared
+	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+		testLDSName: {routeName: testRDSName},
+	})
+	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
+	}
+
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {weightedCluster: map[string]uint32{testCDSName + "new2": 1}},
+	})
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName + "new2": 1}}, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }

From ad51f572fd270f2323e3aa2c1d2775cab9087af2 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 4 Jun 2020 10:56:13 -0700
Subject: [PATCH 088/481] protoc-gen-go-grpc: add requirement of embedding
 UnimplementedServer in services (#3657)

---
 .../grpc_lb_v1/load_balancer_grpc.pb.go       |  4 +++-
 balancer/grpclb/grpclb_test.go                |  1 +
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       |  4 +++-
 .../testutils/fakeserver/fakeserver.go        |  1 +
 benchmark/benchmark.go                        |  2 ++
 benchmark/grpc_testing/services_grpc.pb.go    |  8 +++++--
 benchmark/worker/main.go                      |  1 +
 channelz/grpc_channelz_v1/channelz_grpc.pb.go |  4 +++-
 channelz/service/service.go                   |  4 +++-
 cmd/protoc-gen-go-grpc/README.md              | 21 +++++++++++++++++++
 cmd/protoc-gen-go-grpc/grpc.go                | 15 ++++++++++++-
 cmd/protoc-gen-go-grpc/main.go                | 11 +++++++++-
 .../proto/grpc_gcp/handshaker_grpc.pb.go      |  4 +++-
 examples/features/proto/echo/echo_grpc.pb.go  |  4 +++-
 .../helloworld/helloworld_grpc.pb.go          |  4 +++-
 .../routeguide/route_guide_grpc.pb.go         |  4 +++-
 health/grpc_health_v1/health_grpc.pb.go       |  4 +++-
 health/server.go                              |  1 +
 interop/fake_grpclb/fake_grpclb.go            |  1 +
 interop/grpc_testing/test_grpc.pb.go          | 12 ++++++++---
 interop/test_utils.go                         |  1 +
 interop/xds/client/client.go                  |  4 +++-
 profiling/proto/service_grpc.pb.go            |  4 +++-
 profiling/service/service.go                  |  1 +
 .../reflection_grpc.pb.go                     |  4 +++-
 reflection/grpc_testing/test_grpc.pb.go       |  4 +++-
 reflection/serverreflection.go                |  1 +
 regenerate.sh                                 |  2 +-
 stats/grpc_testing/test_grpc.pb.go            |  4 +++-
 stress/client/main.go                         |  1 +
 stress/grpc_testing/metrics_grpc.pb.go        |  4 +++-
 test/grpc_testing/test_grpc.pb.go             |  4 +++-
 32 files changed, 120 insertions(+), 24 deletions(-)
 create mode 100644 cmd/protoc-gen-go-grpc/README.md

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index 40cbf5724563..5a3a2ec57641 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -61,12 +61,14 @@ func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
 }
 
 // LoadBalancerServer is the server API for LoadBalancer service.
+// All implementations should embed UnimplementedLoadBalancerServer
+// for forward compatibility
 type LoadBalancerServer interface {
 	// Bidirectional rpc to get a list of servers.
 	BalanceLoad(LoadBalancer_BalanceLoadServer) error
 }
 
-// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations.
+// UnimplementedLoadBalancerServer should be embedded to have forward compatible implementations.
 type UnimplementedLoadBalancerServer struct {
 }
 
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index d701b6d216a0..45d2df64567f 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -187,6 +187,7 @@ func (s *rpcStats) String() string {
 }
 
 type remoteBalancer struct {
+	lbgrpc.UnimplementedLoadBalancerServer
 	sls       chan *lbpb.ServerList
 	statsDura time.Duration
 	done      chan struct{}
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index 6e4145dd5fc4..79fa758f7e97 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -39,12 +39,14 @@ func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLoo
 }
 
 // RouteLookupServiceServer is the server API for RouteLookupService service.
+// All implementations should embed UnimplementedRouteLookupServiceServer
+// for forward compatibility
 type RouteLookupServiceServer interface {
 	// Lookup returns a target for a single key.
 	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
 }
 
-// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedRouteLookupServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedRouteLookupServiceServer struct {
 }
 
diff --git a/balancer/rls/internal/testutils/fakeserver/fakeserver.go b/balancer/rls/internal/testutils/fakeserver/fakeserver.go
index 93947da4ccef..479e3036468f 100644
--- a/balancer/rls/internal/testutils/fakeserver/fakeserver.go
+++ b/balancer/rls/internal/testutils/fakeserver/fakeserver.go
@@ -49,6 +49,7 @@ type Response struct {
 // Server is a fake implementation of RLS. It exposes channels to send/receive
 // RLS requests and responses.
 type Server struct {
+	rlsgrpc.UnimplementedRouteLookupServiceServer
 	RequestChan  *testutils.Channel
 	ResponseChan chan Response
 	Address      string
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index e844bb1ea52d..d82b61c9c619 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -60,6 +60,7 @@ func NewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 }
 
 type testServer struct {
+	testpb.UnimplementedBenchmarkServiceServer
 }
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -141,6 +142,7 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U
 // byteBufServer is a gRPC server that sends and receives byte buffer.
 // The purpose is to benchmark the gRPC performance without protobuf serialization/deserialization overhead.
 type byteBufServer struct {
+	testpb.UnimplementedBenchmarkServiceServer
 	respSize int32
 }
 
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index e923f72c5387..385870ae5959 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -108,6 +108,8 @@ func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleRespon
 }
 
 // BenchmarkServiceServer is the server API for BenchmarkService service.
+// All implementations should embed UnimplementedBenchmarkServiceServer
+// for forward compatibility
 type BenchmarkServiceServer interface {
 	// One request followed by one response.
 	// The server returns the client payload as-is.
@@ -120,7 +122,7 @@ type BenchmarkServiceServer interface {
 	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
 }
 
-// UnimplementedBenchmarkServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedBenchmarkServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedBenchmarkServiceServer struct {
 }
 
@@ -347,6 +349,8 @@ func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...
 }
 
 // WorkerServiceServer is the server API for WorkerService service.
+// All implementations should embed UnimplementedWorkerServiceServer
+// for forward compatibility
 type WorkerServiceServer interface {
 	// Start server with specified workload.
 	// First request sent specifies the ServerConfig followed by ServerStatus
@@ -368,7 +372,7 @@ type WorkerServiceServer interface {
 	QuitWorker(context.Context, *Void) (*Void, error)
 }
 
-// UnimplementedWorkerServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedWorkerServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedWorkerServiceServer struct {
 }
 
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 2b760f54f9fa..fba451c4f27b 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -73,6 +73,7 @@ func (byteBufCodec) String() string {
 // workerServer implements WorkerService rpc handlers.
 // It can create benchmarkServer or benchmarkClient on demand.
 type workerServer struct {
+	testpb.UnimplementedWorkerServiceServer
 	stop       chan<- bool
 	serverPort int
 }
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index bacb448d1d85..20e1fe709796 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -106,6 +106,8 @@ func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, op
 }
 
 // ChannelzServer is the server API for Channelz service.
+// All implementations should embed UnimplementedChannelzServer
+// for forward compatibility
 type ChannelzServer interface {
 	// Gets all root channels (i.e. channels the application has directly
 	// created). This does not include subchannels nor non-top level channels.
@@ -124,7 +126,7 @@ type ChannelzServer interface {
 	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
 }
 
-// UnimplementedChannelzServer can be embedded to have forward compatible implementations.
+// UnimplementedChannelzServer should be embedded to have forward compatible implementations.
 type UnimplementedChannelzServer struct {
 }
 
diff --git a/channelz/service/service.go b/channelz/service/service.go
index b7650b3b29fa..702b74c03e60 100644
--- a/channelz/service/service.go
+++ b/channelz/service/service.go
@@ -48,7 +48,9 @@ func newCZServer() channelzgrpc.ChannelzServer {
 	return &serverImpl{}
 }
 
-type serverImpl struct{}
+type serverImpl struct {
+	channelzgrpc.UnimplementedChannelzServer
+}
 
 func connectivityStateToProto(s connectivity.State) *channelzpb.ChannelConnectivityState {
 	switch s {
diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md
new file mode 100644
index 000000000000..f76e419a26c7
--- /dev/null
+++ b/cmd/protoc-gen-go-grpc/README.md
@@ -0,0 +1,21 @@
+# protoc-gen-go-grpc
+
+This tool generates Go language bindings of `service`s in protobuf definition
+files for gRPC.  For usage information, please see our [quick start
+guide](https://grpc.io/docs/languages/go/quickstart/).
+
+## Future-proofing services
+
+By default, to register services using the methods generated by this tool, the
+service implementations must embed the corresponding
+`Unimplemented<ServiceName>Server` for future compatibility.  This is a behavior
+change from the grpc code generator previously included with `protoc-gen-go`.
+To restore this behavior, set the option `requireUnimplementedServers=false`.
+E.g.:
+
+```
+  protoc --go-grpc_out=requireUnimplementedServers=false[,other options...]:. \
+```
+
+Note that this is not recommended, and the option is only provided to restore
+backward compatibility with previously-generated code.
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 38d444fc7a1d..4f78bdd46018 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -119,9 +119,16 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 		}
 	}
 
+	mustOrShould := "must"
+	if !*requireUnimplemented {
+		mustOrShould = "should"
+	}
+
 	// Server interface.
 	serverType := service.GoName + "Server"
 	g.P("// ", serverType, " is the server API for ", service.GoName, " service.")
+	g.P("// All implementations ", mustOrShould, " embed Unimplemented", serverType)
+	g.P("// for forward compatibility")
 	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
 		g.P("//")
 		g.P(deprecationComment)
@@ -136,11 +143,14 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 		g.P(method.Comments.Leading,
 			serverSignature(g, method))
 	}
+	if *requireUnimplemented {
+		g.P("mustEmbedUnimplemented", serverType, "()")
+	}
 	g.P("}")
 	g.P()
 
 	// Server Unimplemented struct for forward compatibility.
-	g.P("// Unimplemented", serverType, " can be embedded to have forward compatible implementations.")
+	g.P("// Unimplemented", serverType, " ", mustOrShould, " be embedded to have forward compatible implementations.")
 	g.P("type Unimplemented", serverType, " struct {")
 	g.P("}")
 	g.P()
@@ -153,6 +163,9 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
 		g.P("}")
 	}
+	if *requireUnimplemented {
+		g.P("func (*Unimplemented", serverType, ") mustEmbedUnimplemented", serverType, "() {}")
+	}
 	g.P()
 
 	// Server registration.
diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index dac0be207e23..a8555fad23a1 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -31,11 +31,20 @@
 package main
 
 import (
+	"flag"
+
 	"google.golang.org/protobuf/compiler/protogen"
 )
 
+var requireUnimplemented *bool
+
 func main() {
-	protogen.Options{}.Run(func(gen *protogen.Plugin) error {
+	var flags flag.FlagSet
+	requireUnimplemented = flags.Bool("requireUnimplementedServers", true, "unset to match legacy behavior")
+
+	protogen.Options{
+		ParamFunc: flags.Set,
+	}.Run(func(gen *protogen.Plugin) error {
 		for _, f := range gen.Files {
 			if !f.Generate {
 				continue
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index 75c2e4732439..0e973b8250e3 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -66,6 +66,8 @@ func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
 }
 
 // HandshakerServiceServer is the server API for HandshakerService service.
+// All implementations should embed UnimplementedHandshakerServiceServer
+// for forward compatibility
 type HandshakerServiceServer interface {
 	// Handshaker service accepts a stream of handshaker request, returning a
 	// stream of handshaker response. Client is expected to send exactly one
@@ -76,7 +78,7 @@ type HandshakerServiceServer interface {
 	DoHandshake(HandshakerService_DoHandshakeServer) error
 }
 
-// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedHandshakerServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedHandshakerServiceServer struct {
 }
 
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index 38be6f65ce1a..ff36643530ef 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -142,6 +142,8 @@ func (x *echoBidirectionalStreamingEchoClient) Recv() (*EchoResponse, error) {
 }
 
 // EchoServer is the server API for Echo service.
+// All implementations should embed UnimplementedEchoServer
+// for forward compatibility
 type EchoServer interface {
 	// UnaryEcho is unary echo.
 	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
@@ -153,7 +155,7 @@ type EchoServer interface {
 	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
 }
 
-// UnimplementedEchoServer can be embedded to have forward compatible implementations.
+// UnimplementedEchoServer should be embedded to have forward compatible implementations.
 type UnimplementedEchoServer struct {
 }
 
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index ac81352958cb..435df1a0e5eb 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -39,12 +39,14 @@ func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...
 }
 
 // GreeterServer is the server API for Greeter service.
+// All implementations should embed UnimplementedGreeterServer
+// for forward compatibility
 type GreeterServer interface {
 	// Sends a greeting
 	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
 }
 
-// UnimplementedGreeterServer can be embedded to have forward compatible implementations.
+// UnimplementedGreeterServer should be embedded to have forward compatible implementations.
 type UnimplementedGreeterServer struct {
 }
 
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index 326a7683ef0d..9ae8b4483bfb 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -158,6 +158,8 @@ func (x *routeGuideRouteChatClient) Recv() (*RouteNote, error) {
 }
 
 // RouteGuideServer is the server API for RouteGuide service.
+// All implementations should embed UnimplementedRouteGuideServer
+// for forward compatibility
 type RouteGuideServer interface {
 	// A simple RPC.
 	//
@@ -185,7 +187,7 @@ type RouteGuideServer interface {
 	RouteChat(RouteGuide_RouteChatServer) error
 }
 
-// UnimplementedRouteGuideServer can be embedded to have forward compatible implementations.
+// UnimplementedRouteGuideServer should be embedded to have forward compatible implementations.
 type UnimplementedRouteGuideServer struct {
 }
 
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index 28cd72ea63c0..f87e3c92adbe 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -88,6 +88,8 @@ func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
 }
 
 // HealthServer is the server API for Health service.
+// All implementations should embed UnimplementedHealthServer
+// for forward compatibility
 type HealthServer interface {
 	// If the requested service is unknown, the call will fail with status
 	// NOT_FOUND.
@@ -110,7 +112,7 @@ type HealthServer interface {
 	Watch(*HealthCheckRequest, Health_WatchServer) error
 }
 
-// UnimplementedHealthServer can be embedded to have forward compatible implementations.
+// UnimplementedHealthServer should be embedded to have forward compatible implementations.
 type UnimplementedHealthServer struct {
 }
 
diff --git a/health/server.go b/health/server.go
index 6b65124242b8..ed2b3df70f4e 100644
--- a/health/server.go
+++ b/health/server.go
@@ -33,6 +33,7 @@ import (
 
 // Server implements `service Health`.
 type Server struct {
+	healthgrpc.UnimplementedHealthServer
 	mu sync.RWMutex
 	// If shutdown is true, it's expected all serving status is NOT_SERVING, and
 	// will stay in NOT_SERVING.
diff --git a/interop/fake_grpclb/fake_grpclb.go b/interop/fake_grpclb/fake_grpclb.go
index f6d55045ee22..3b2177ef0291 100644
--- a/interop/fake_grpclb/fake_grpclb.go
+++ b/interop/fake_grpclb/fake_grpclb.go
@@ -48,6 +48,7 @@ var (
 )
 
 type loadBalancerServer struct {
+	lbpb.UnimplementedLoadBalancerServer
 	serverListResponse *lbpb.LoadBalanceResponse
 }
 
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index d731f4643ecd..07b555a3f690 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -194,6 +194,8 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 }
 
 // TestServiceServer is the server API for TestService service.
+// All implementations should embed UnimplementedTestServiceServer
+// for forward compatibility
 type TestServiceServer interface {
 	// One empty request followed by one empty response.
 	EmptyCall(context.Context, *Empty) (*Empty, error)
@@ -217,7 +219,7 @@ type TestServiceServer interface {
 	HalfDuplexCall(TestService_HalfDuplexCallServer) error
 }
 
-// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedTestServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedTestServiceServer struct {
 }
 
@@ -445,12 +447,14 @@ func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *
 }
 
 // UnimplementedServiceServer is the server API for UnimplementedService service.
+// All implementations should embed UnimplementedUnimplementedServiceServer
+// for forward compatibility
 type UnimplementedServiceServer interface {
 	// A call that no server should implement
 	UnimplementedCall(context.Context, *Empty) (*Empty, error)
 }
 
-// UnimplementedUnimplementedServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedUnimplementedServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedUnimplementedServiceServer struct {
 }
 
@@ -519,12 +523,14 @@ func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in
 }
 
 // LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service.
+// All implementations should embed UnimplementedLoadBalancerStatsServiceServer
+// for forward compatibility
 type LoadBalancerStatsServiceServer interface {
 	// Gets the backend distribution for RPCs sent by a test client.
 	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
 }
 
-// UnimplementedLoadBalancerStatsServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedLoadBalancerStatsServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedLoadBalancerStatsServiceServer struct {
 }
 
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 63ba6a5ebd0a..74e839031881 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -672,6 +672,7 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) {
 }
 
 type testServer struct {
+	testpb.UnimplementedTestServiceServer
 }
 
 // NewTestServer creates a test server for test service.
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 009ad26f9d60..44fbf1e6dbb3 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -60,7 +60,9 @@ var (
 	watchers         = make(map[statsWatcherKey]*statsWatcher)
 )
 
-type statsService struct{}
+type statsService struct {
+	testpb.UnimplementedLoadBalancerStatsServiceServer
+}
 
 // Wait for the next LoadBalancerStatsRequest.GetNumRpcs to start and complete,
 // and return the distribution of remote peers. This is essentially a clientside
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index 87f1b74b7d06..aed8d58762d3 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -51,6 +51,8 @@ func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStats
 }
 
 // ProfilingServer is the server API for Profiling service.
+// All implementations should embed UnimplementedProfilingServer
+// for forward compatibility
 type ProfilingServer interface {
 	// Enable allows users to toggle profiling on and off remotely.
 	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
@@ -59,7 +61,7 @@ type ProfilingServer interface {
 	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
 }
 
-// UnimplementedProfilingServer can be embedded to have forward compatible implementations.
+// UnimplementedProfilingServer should be embedded to have forward compatible implementations.
 type UnimplementedProfilingServer struct {
 }
 
diff --git a/profiling/service/service.go b/profiling/service/service.go
index a36b8bdda883..52a7ce789717 100644
--- a/profiling/service/service.go
+++ b/profiling/service/service.go
@@ -76,6 +76,7 @@ func Init(pc *ProfilingConfig) error {
 }
 
 type profilingServer struct {
+	ppb.UnimplementedProfilingServer
 	drainMutex sync.Mutex
 }
 
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index f04e2d4bb0f6..2294b2c6c9e5 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -62,13 +62,15 @@ func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionRe
 }
 
 // ServerReflectionServer is the server API for ServerReflection service.
+// All implementations should embed UnimplementedServerReflectionServer
+// for forward compatibility
 type ServerReflectionServer interface {
 	// The reflection service is structured as a bidirectional stream, ensuring
 	// all related requests go to a single server.
 	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
 }
 
-// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations.
+// UnimplementedServerReflectionServer should be embedded to have forward compatible implementations.
 type UnimplementedServerReflectionServer struct {
 }
 
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 52eac202bfeb..95cd2bab1c22 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -70,12 +70,14 @@ func (x *searchServiceStreamingSearchClient) Recv() (*SearchResponse, error) {
 }
 
 // SearchServiceServer is the server API for SearchService service.
+// All implementations should embed UnimplementedSearchServiceServer
+// for forward compatibility
 type SearchServiceServer interface {
 	Search(context.Context, *SearchRequest) (*SearchResponse, error)
 	StreamingSearch(SearchService_StreamingSearchServer) error
 }
 
-// UnimplementedSearchServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedSearchServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedSearchServiceServer struct {
 }
 
diff --git a/reflection/serverreflection.go b/reflection/serverreflection.go
index de3e132f0451..7b6dd414a275 100644
--- a/reflection/serverreflection.go
+++ b/reflection/serverreflection.go
@@ -55,6 +55,7 @@ import (
 )
 
 type serverReflectionServer struct {
+	rpb.UnimplementedServerReflectionServer
 	s *grpc.Server
 
 	initSymbols  sync.Once
diff --git a/regenerate.sh b/regenerate.sh
index 6f47daf94bc3..987bc20251dd 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -57,7 +57,7 @@ SOURCES=(
 OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
-  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out \
+  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},requireUnimplementedServers=false:${WORKDIR}/out \
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index a12bccebcfdf..98da3340b76a 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -145,6 +145,8 @@ func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) {
 }
 
 // TestServiceServer is the server API for TestService service.
+// All implementations should embed UnimplementedTestServiceServer
+// for forward compatibility
 type TestServiceServer interface {
 	// One request followed by one response.
 	// The server returns the client id as-is.
@@ -159,7 +161,7 @@ type TestServiceServer interface {
 	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
 }
 
-// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedTestServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedTestServiceServer struct {
 }
 
diff --git a/stress/client/main.go b/stress/client/main.go
index 86d7539ae7f1..b8f2229a5b76 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -144,6 +144,7 @@ func (g *gauge) get() int64 {
 
 // server implements metrics server functions.
 type server struct {
+	metricspb.UnimplementedMetricsServiceServer
 	mutex sync.RWMutex
 	// gauges is a map from /stress_test/server_<n>/channel_<n>/stub_<n>/qps to its qps gauge.
 	gauges map[string]*gauge
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 06c8a6b5fa10..3a232ff8501b 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -74,6 +74,8 @@ func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, o
 }
 
 // MetricsServiceServer is the server API for MetricsService service.
+// All implementations should embed UnimplementedMetricsServiceServer
+// for forward compatibility
 type MetricsServiceServer interface {
 	// Returns the values of all the gauges that are currently being maintained by
 	// the service
@@ -82,7 +84,7 @@ type MetricsServiceServer interface {
 	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
 }
 
-// UnimplementedMetricsServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedMetricsServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedMetricsServiceServer struct {
 }
 
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index a211dcab9f35..2340ac05e890 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -194,6 +194,8 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 }
 
 // TestServiceServer is the server API for TestService service.
+// All implementations should embed UnimplementedTestServiceServer
+// for forward compatibility
 type TestServiceServer interface {
 	// One empty request followed by one empty response.
 	EmptyCall(context.Context, *Empty) (*Empty, error)
@@ -217,7 +219,7 @@ type TestServiceServer interface {
 	HalfDuplexCall(TestService_HalfDuplexCallServer) error
 }
 
-// UnimplementedTestServiceServer can be embedded to have forward compatible implementations.
+// UnimplementedTestServiceServer should be embedded to have forward compatible implementations.
 type UnimplementedTestServiceServer struct {
 }
 

From 479df5ea818c360b6c5bb3db292ec0afb354c226 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Fri, 5 Jun 2020 12:22:55 -0700
Subject: [PATCH 089/481] advancedtls: check error before deferring close
 (#3659)

---
 security/advancedtls/advancedtls_integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 075e1e9f6ca2..5f8c25f681b3 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -405,10 +405,10 @@ func TestEnd2End(t *testing.T) {
 			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
 			defer s.Stop()
 			lis, err := net.Listen("tcp", port)
-			defer lis.Close()
 			if err != nil {
 				t.Fatalf("failed to listen: %v", err)
 			}
+			defer lis.Close()
 			pb.RegisterGreeterServer(s, &serverImpl{})
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{

From d5bc6ecb590760edf37158ab03fb7c83962e3d5a Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 10 Jun 2020 08:53:19 -0700
Subject: [PATCH 090/481] xds: Add cmpopts.EquateEmpty option in tests. (#3671)

Recently I have started seeing a lot of xds tests fail in travis when
doing cmp.Equal. Adding the EquateEmpty option will treat all maps and
slices of length zero as equal whether they are empty or nil.
---
 .../client/client_watchers_endpoints_test.go  | 19 ++++++++++--------
 .../client/client_watchers_service_test.go    | 20 +++++++++++--------
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index 53b1bc316abc..e61bb7105230 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -23,6 +23,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
@@ -42,6 +44,7 @@ var (
 			Weight:    1,
 		},
 	}
+	endpointsCmpOpts = []cmp.Option{cmp.AllowUnexported(endpointsUpdateErr{}), cmpopts.EquateEmpty()}
 )
 
 type endpointsUpdateErr struct {
@@ -78,7 +81,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -138,7 +141,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -150,7 +153,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -206,12 +209,12 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, endpointsCmpOpts...) {
 			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -243,7 +246,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -257,7 +260,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	}
 
 	// New watch should receives the update.
-	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -301,7 +304,7 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 		t.Fatalf("failed to get endpointsUpdate: %v", err)
 	}
 	uu := u.(endpointsUpdateErr)
-	if !cmp.Equal(uu.u, EndpointsUpdate{}, cmp.AllowUnexported(endpointsUpdateErr{})) {
+	if !cmp.Equal(uu.u, EndpointsUpdate{}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, want %v", uu.u, EndpointsUpdate{})
 	}
 	if uu.err == nil {
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 8b341683d094..37d7fb45401d 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -25,6 +25,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
@@ -34,6 +36,8 @@ type serviceUpdateErr struct {
 	err error
 }
 
+var serviceCmpOpts = []cmp.Option{cmp.AllowUnexported(serviceUpdateErr{}), cmpopts.EquateEmpty()}
+
 // TestServiceWatch covers the cases:
 // - an update is received after a watch()
 // - an update for another resource name (which doesn't trigger callback)
@@ -70,7 +74,7 @@ func (s) TestServiceWatch(t *testing.T) {
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -110,7 +114,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -137,7 +141,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		testRDSName + "2": {weightedCluster: map[string]uint32{testCDSName + "2": 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -177,7 +181,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -208,7 +212,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -388,7 +392,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -439,7 +443,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -478,7 +482,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: {weightedCluster: map[string]uint32{testCDSName + "new2": 1}},
 	})
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName + "new2": 1}}, nil}, cmp.AllowUnexported(serviceUpdateErr{})) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName + "new2": 1}}, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }

From 9aa97f9cb41be5555bb5d255eca0eacd1fc00422 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 10 Jun 2020 18:00:24 -0700
Subject: [PATCH 091/481] stream: fix calloption.After() race in finish (#3672)

---
 rpc_util.go          | 45 ++++++++++++++-------------------
 stream.go            | 20 +++++++--------
 test/end2end_test.go | 59 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+), 37 deletions(-)

diff --git a/rpc_util.go b/rpc_util.go
index cf9dbe7fd97d..8644b8a7d0da 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -155,7 +155,6 @@ func (d *gzipDecompressor) Type() string {
 type callInfo struct {
 	compressorType        string
 	failFast              bool
-	stream                ClientStream
 	maxReceiveMessageSize *int
 	maxSendMessageSize    *int
 	creds                 credentials.PerRPCCredentials
@@ -180,7 +179,7 @@ type CallOption interface {
 
 	// after is called after the call has completed.  after cannot return an
 	// error, so any failures should be reported via output parameters.
-	after(*callInfo)
+	after(*callInfo, *csAttempt)
 }
 
 // EmptyCallOption does not alter the Call configuration.
@@ -188,8 +187,8 @@ type CallOption interface {
 // by interceptors.
 type EmptyCallOption struct{}
 
-func (EmptyCallOption) before(*callInfo) error { return nil }
-func (EmptyCallOption) after(*callInfo)        {}
+func (EmptyCallOption) before(*callInfo) error      { return nil }
+func (EmptyCallOption) after(*callInfo, *csAttempt) {}
 
 // Header returns a CallOptions that retrieves the header metadata
 // for a unary RPC.
@@ -205,10 +204,8 @@ type HeaderCallOption struct {
 }
 
 func (o HeaderCallOption) before(c *callInfo) error { return nil }
-func (o HeaderCallOption) after(c *callInfo) {
-	if c.stream != nil {
-		*o.HeaderAddr, _ = c.stream.Header()
-	}
+func (o HeaderCallOption) after(c *callInfo, attempt *csAttempt) {
+	*o.HeaderAddr, _ = attempt.s.Header()
 }
 
 // Trailer returns a CallOptions that retrieves the trailer metadata
@@ -225,10 +222,8 @@ type TrailerCallOption struct {
 }
 
 func (o TrailerCallOption) before(c *callInfo) error { return nil }
-func (o TrailerCallOption) after(c *callInfo) {
-	if c.stream != nil {
-		*o.TrailerAddr = c.stream.Trailer()
-	}
+func (o TrailerCallOption) after(c *callInfo, attempt *csAttempt) {
+	*o.TrailerAddr = attempt.s.Trailer()
 }
 
 // Peer returns a CallOption that retrieves peer information for a unary RPC.
@@ -245,11 +240,9 @@ type PeerCallOption struct {
 }
 
 func (o PeerCallOption) before(c *callInfo) error { return nil }
-func (o PeerCallOption) after(c *callInfo) {
-	if c.stream != nil {
-		if x, ok := peer.FromContext(c.stream.Context()); ok {
-			*o.PeerAddr = *x
-		}
+func (o PeerCallOption) after(c *callInfo, attempt *csAttempt) {
+	if x, ok := peer.FromContext(attempt.s.Context()); ok {
+		*o.PeerAddr = *x
 	}
 }
 
@@ -285,7 +278,7 @@ func (o FailFastCallOption) before(c *callInfo) error {
 	c.failFast = o.FailFast
 	return nil
 }
-func (o FailFastCallOption) after(c *callInfo) {}
+func (o FailFastCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // MaxCallRecvMsgSize returns a CallOption which sets the maximum message size
 // in bytes the client can receive.
@@ -304,7 +297,7 @@ func (o MaxRecvMsgSizeCallOption) before(c *callInfo) error {
 	c.maxReceiveMessageSize = &o.MaxRecvMsgSize
 	return nil
 }
-func (o MaxRecvMsgSizeCallOption) after(c *callInfo) {}
+func (o MaxRecvMsgSizeCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // MaxCallSendMsgSize returns a CallOption which sets the maximum message size
 // in bytes the client can send.
@@ -323,7 +316,7 @@ func (o MaxSendMsgSizeCallOption) before(c *callInfo) error {
 	c.maxSendMessageSize = &o.MaxSendMsgSize
 	return nil
 }
-func (o MaxSendMsgSizeCallOption) after(c *callInfo) {}
+func (o MaxSendMsgSizeCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // PerRPCCredentials returns a CallOption that sets credentials.PerRPCCredentials
 // for a call.
@@ -342,7 +335,7 @@ func (o PerRPCCredsCallOption) before(c *callInfo) error {
 	c.creds = o.Creds
 	return nil
 }
-func (o PerRPCCredsCallOption) after(c *callInfo) {}
+func (o PerRPCCredsCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // UseCompressor returns a CallOption which sets the compressor used when
 // sending the request.  If WithCompressor is also set, UseCompressor has
@@ -363,7 +356,7 @@ func (o CompressorCallOption) before(c *callInfo) error {
 	c.compressorType = o.CompressorType
 	return nil
 }
-func (o CompressorCallOption) after(c *callInfo) {}
+func (o CompressorCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // CallContentSubtype returns a CallOption that will set the content-subtype
 // for a call. For example, if content-subtype is "json", the Content-Type over
@@ -396,7 +389,7 @@ func (o ContentSubtypeCallOption) before(c *callInfo) error {
 	c.contentSubtype = o.ContentSubtype
 	return nil
 }
-func (o ContentSubtypeCallOption) after(c *callInfo) {}
+func (o ContentSubtypeCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // ForceCodec returns a CallOption that will set the given Codec to be
 // used for all request and response messages for a call. The result of calling
@@ -428,7 +421,7 @@ func (o ForceCodecCallOption) before(c *callInfo) error {
 	c.codec = o.Codec
 	return nil
 }
-func (o ForceCodecCallOption) after(c *callInfo) {}
+func (o ForceCodecCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // CallCustomCodec behaves like ForceCodec, but accepts a grpc.Codec instead of
 // an encoding.Codec.
@@ -450,7 +443,7 @@ func (o CustomCodecCallOption) before(c *callInfo) error {
 	c.codec = o.Codec
 	return nil
 }
-func (o CustomCodecCallOption) after(c *callInfo) {}
+func (o CustomCodecCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // MaxRetryRPCBufferSize returns a CallOption that limits the amount of memory
 // used for buffering this RPC's requests for retry purposes.
@@ -471,7 +464,7 @@ func (o MaxRetryRPCBufferSizeCallOption) before(c *callInfo) error {
 	c.maxRetryRPCBufferSize = o.MaxRetryRPCBufferSize
 	return nil
 }
-func (o MaxRetryRPCBufferSizeCallOption) after(c *callInfo) {}
+func (o MaxRetryRPCBufferSizeCallOption) after(c *callInfo, attempt *csAttempt) {}
 
 // The format of the payload: compressed or not?
 type payloadFormat uint8
diff --git a/stream.go b/stream.go
index 934ef68321cc..62d51334488d 100644
--- a/stream.go
+++ b/stream.go
@@ -277,7 +277,6 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 	}
 	cs.binlog = binarylog.GetMethodLogger(method)
 
-	cs.callInfo.stream = cs
 	// Only this initial attempt has stats/tracing.
 	// TODO(dfawley): move to newAttempt when per-attempt stats are implemented.
 	if err := cs.newAttemptLocked(sh, trInfo); err != nil {
@@ -799,6 +798,15 @@ func (cs *clientStream) finish(err error) {
 	}
 	cs.finished = true
 	cs.commitAttemptLocked()
+	if cs.attempt != nil {
+		cs.attempt.finish(err)
+		// after functions all rely upon having a stream.
+		if cs.attempt.s != nil {
+			for _, o := range cs.opts {
+				o.after(cs.callInfo, cs.attempt)
+			}
+		}
+	}
 	cs.mu.Unlock()
 	// For binary logging. only log cancel in finish (could be caused by RPC ctx
 	// canceled or ClientConn closed). Trailer will be logged in RecvMsg.
@@ -820,15 +828,6 @@ func (cs *clientStream) finish(err error) {
 			cs.cc.incrCallsSucceeded()
 		}
 	}
-	if cs.attempt != nil {
-		cs.attempt.finish(err)
-		// after functions all rely upon having a stream.
-		if cs.attempt.s != nil {
-			for _, o := range cs.opts {
-				o.after(cs.callInfo)
-			}
-		}
-	}
 	cs.cancel()
 }
 
@@ -1066,7 +1065,6 @@ func newNonRetryClientStream(ctx context.Context, desc *StreamDesc, method strin
 		t:        t,
 	}
 
-	as.callInfo.stream = as
 	s, err := as.t.NewStream(as.ctx, as.callHdr)
 	if err != nil {
 		err = toRPCErr(err)
diff --git a/test/end2end_test.go b/test/end2end_test.go
index f3a60de5a96f..82150fe11be9 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -7128,3 +7128,62 @@ func (s) TestGzipBadChecksum(t *testing.T) {
 		t.Errorf("ss.client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum)
 	}
 }
+
+// When an RPC is canceled, it's possible that the last Recv() returns before
+// all call options' after are executed.
+func (s) TestCanceledRPCCallOptionRace(t *testing.T) {
+	ss := &stubServer{
+		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+			err := stream.Send(&testpb.StreamingOutputCallResponse{})
+			if err != nil {
+				return err
+			}
+			<-stream.Context().Done()
+			return nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	const count = 1000
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	wg.Add(count)
+	for i := 0; i < count; i++ {
+		go func() {
+			defer wg.Done()
+			var p peer.Peer
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+			stream, err := ss.client.FullDuplexCall(ctx, grpc.Peer(&p))
+			if err != nil {
+				t.Errorf("_.FullDuplexCall(_) = _, %v", err)
+				return
+			}
+			if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+				t.Errorf("_ has error %v while sending", err)
+				return
+			}
+			if _, err := stream.Recv(); err != nil {
+				t.Errorf("%v.Recv() = %v", stream, err)
+				return
+			}
+			cancel()
+			if _, err := stream.Recv(); status.Code(err) != codes.Canceled {
+				t.Errorf("%v compleled with error %v, want %s", stream, err, codes.Canceled)
+				return
+			}
+			// If recv returns before call options are executed, peer.Addr is not set,
+			// fail the test.
+			if p.Addr == nil {
+				t.Errorf("peer.Addr is nil, want non-nil")
+				return
+			}
+		}()
+	}
+	wg.Wait()
+}

From eb11ffdf9bd6e525ab49264f15b2b8b3841aaad2 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 11 Jun 2020 09:18:17 -0700
Subject: [PATCH 092/481] retry: prevent per-RPC creds error from being
 transparently retried (#3677)

---
 balancer/grpclb/grpclb_test.go | 12 ++++++------
 stream.go                      | 17 +++++------------
 test/creds_test.go             |  6 +++++-
 3 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index 45d2df64567f..d55fe3eb8be5 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -1351,9 +1351,9 @@ func (s) TestGRPCLBStatsUnaryFailedToSend(t *testing.T) {
 			cc.Invoke(context.Background(), failtosendURI, &testpb.Empty{}, nil)
 		}
 	}, &rpcStats{
-		numCallsStarted:                        int64(countRPC)*2 - 1,
-		numCallsFinished:                       int64(countRPC)*2 - 1,
-		numCallsFinishedWithClientFailedToSend: int64(countRPC-1) * 2,
+		numCallsStarted:                        int64(countRPC),
+		numCallsFinished:                       int64(countRPC),
+		numCallsFinishedWithClientFailedToSend: int64(countRPC) - 1,
 		numCallsFinishedKnownReceived:          1,
 	}); err != nil {
 		t.Fatal(err)
@@ -1444,9 +1444,9 @@ func (s) TestGRPCLBStatsStreamingFailedToSend(t *testing.T) {
 			cc.NewStream(context.Background(), &grpc.StreamDesc{}, failtosendURI)
 		}
 	}, &rpcStats{
-		numCallsStarted:                        int64(countRPC)*2 - 1,
-		numCallsFinished:                       int64(countRPC)*2 - 1,
-		numCallsFinishedWithClientFailedToSend: int64(countRPC-1) * 2,
+		numCallsStarted:                        int64(countRPC),
+		numCallsFinished:                       int64(countRPC),
+		numCallsFinishedWithClientFailedToSend: int64(countRPC) - 1,
 		numCallsFinishedKnownReceived:          1,
 	}); err != nil {
 		t.Fatal(err)
diff --git a/stream.go b/stream.go
index 62d51334488d..68d5c14506d8 100644
--- a/stream.go
+++ b/stream.go
@@ -459,12 +459,6 @@ func (cs *clientStream) commitAttempt() {
 // shouldRetry returns nil if the RPC should be retried; otherwise it returns
 // the error that should be returned by the operation.
 func (cs *clientStream) shouldRetry(err error) error {
-	if cs.attempt.s == nil && !cs.callInfo.failFast {
-		// In the event of any error from NewStream (attempt.s == nil), we
-		// never attempted to write anything to the wire, so we can retry
-		// indefinitely for non-fail-fast RPCs.
-		return nil
-	}
 	if cs.finished || cs.committed {
 		// RPC is finished or committed; cannot retry.
 		return err
@@ -472,13 +466,11 @@ func (cs *clientStream) shouldRetry(err error) error {
 	// Wait for the trailers.
 	if cs.attempt.s != nil {
 		<-cs.attempt.s.Done()
+		if cs.firstAttempt && cs.attempt.s.Unprocessed() {
+			// First attempt, stream unprocessed: transparently retry.
+			return nil
+		}
 	}
-	if cs.firstAttempt && (cs.attempt.s == nil || cs.attempt.s.Unprocessed()) {
-		// First attempt, stream unprocessed: transparently retry.
-		cs.firstAttempt = false
-		return nil
-	}
-	cs.firstAttempt = false
 	if cs.cc.dopts.disableRetry {
 		return err
 	}
@@ -564,6 +556,7 @@ func (cs *clientStream) retryLocked(lastErr error) error {
 			cs.commitAttemptLocked()
 			return err
 		}
+		cs.firstAttempt = false
 		if err := cs.newAttemptLocked(nil, nil); err != nil {
 			return err
 		}
diff --git a/test/creds_test.go b/test/creds_test.go
index 7b607ef03716..8f87af125ec3 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -189,11 +189,15 @@ func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
 	cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{}))
 	tc := testpb.NewTestServiceClient(cc)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 	defer cancel()
 	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMethod {
 		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod)
 	}
+
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Convert(err).Message() != wantMethod {
+		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod)
+	}
 }
 
 const clientAlwaysFailCredErrorMsg = "clientAlwaysFailCred always fails"

From b6d6ede3e0bffb68045fe11848f6e3549b1bbefc Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 11 Jun 2020 09:48:55 -0700
Subject: [PATCH 093/481] xds: use google default creds (#3673)

- use google default creds, so the client works not only on GCE (e.g. it also reads env variable for creds).
- Change google default creds to use jwt directly if scope is not set.

- Leak check is disabled temporarily due to https://github.com/googleapis/google-cloud-go/issues/2417
---
 credentials/oauth/oauth.go                    | 40 ++++++++++++++++++-
 xds/internal/client/bootstrap/bootstrap.go    |  2 +-
 .../client/bootstrap/bootstrap_test.go        | 21 ++++------
 3 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/credentials/oauth/oauth.go b/credentials/oauth/oauth.go
index 899e3372ce3c..6657055d6609 100644
--- a/credentials/oauth/oauth.go
+++ b/credentials/oauth/oauth.go
@@ -74,6 +74,8 @@ func NewJWTAccessFromKey(jsonKey []byte) (credentials.PerRPCCredentials, error)
 }
 
 func (j jwtAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
+	// TODO: the returned TokenSource is reusable. Store it in a sync.Map, with
+	// uri as the key, to avoid recreating for every RPC.
 	ts, err := google.JWTAccessTokenSourceFromJSON(j.jsonKey, uri[0])
 	if err != nil {
 		return nil, err
@@ -177,9 +179,43 @@ func NewServiceAccountFromFile(keyFile string, scope ...string) (credentials.Per
 // NewApplicationDefault returns "Application Default Credentials". For more
 // detail, see https://developers.google.com/accounts/docs/application-default-credentials.
 func NewApplicationDefault(ctx context.Context, scope ...string) (credentials.PerRPCCredentials, error) {
-	t, err := google.DefaultTokenSource(ctx, scope...)
+	creds, err := google.FindDefaultCredentials(ctx, scope...)
 	if err != nil {
 		return nil, err
 	}
-	return TokenSource{t}, nil
+
+	// If JSON is nil, the authentication is provided by the environment and not
+	// with a credentials file, e.g. when code is running on Google Cloud
+	// Platform. Use the returned token source.
+	if creds.JSON == nil {
+		return TokenSource{creds.TokenSource}, nil
+	}
+
+	// If auth is provided by env variable or creds file, the behavior will be
+	// different based on whether scope is set. Because the returned
+	// creds.TokenSource does oauth with jwt by default, and it requires scope.
+	// We can only use it if scope is not empty, otherwise it will fail with
+	// missing scope error.
+	//
+	// If scope is set, use it, it should just work.
+	//
+	// If scope is not set, we try to use jwt directly without oauth (this only
+	// works if it's a service account).
+
+	if len(scope) != 0 {
+		return TokenSource{creds.TokenSource}, nil
+	}
+
+	// Try to convert JSON to a jwt config without setting the optional scope
+	// parameter to check if it's a service account (the function errors if it's
+	// not). This is necessary because the returned config doesn't show the type
+	// of the account.
+	if _, err := google.JWTConfigFromJSON(creds.JSON); err != nil {
+		// If this fails, it's not a service account, return the original
+		// TokenSource from above.
+		return TokenSource{creds.TokenSource}, nil
+	}
+
+	// If it's a service account, create a JWT only access with the key.
+	return NewJWTAccessFromKey(creds.JSON)
 }
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index a9db93b15461..1ee0f1d47c5e 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -139,7 +139,7 @@ func NewConfig() (*Config, error) {
 			config.BalancerName = xs.ServerURI
 			for _, cc := range xs.ChannelCreds {
 				if cc.Type == googleDefaultCreds {
-					config.Creds = grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())
+					config.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials())
 					// We stop at the first credential type that we support.
 					break
 				}
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index d3dedb0454d4..de0ef3954e72 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2019 gRPC authors.
@@ -22,13 +24,11 @@ import (
 	"os"
 	"testing"
 
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/golang/protobuf/proto"
+	structpb "github.com/golang/protobuf/ptypes/struct"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
-	"google.golang.org/grpc/internal/grpctest"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	structpb "github.com/golang/protobuf/ptypes/struct"
 )
 
 var (
@@ -58,19 +58,14 @@ var (
 	}
 )
 
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
+// TODO: enable leak check for this package when
+// https://github.com/googleapis/google-cloud-go/issues/2417 is fixed.
 
 // TestNewConfig exercises the functionality in NewConfig with different
 // bootstrap file contents. It overrides the fileReadFunc by returning
 // bootstrap file contents defined in this test, instead of reading from a
 // file.
-func (s) TestNewConfig(t *testing.T) {
+func TestNewConfig(t *testing.T) {
 	bootstrapFileMap := map[string]string{
 		"empty":          "",
 		"badJSON":        `["test": 123]`,
@@ -283,7 +278,7 @@ func (s) TestNewConfig(t *testing.T) {
 	}
 }
 
-func (s) TestNewConfigEnvNotSet(t *testing.T) {
+func TestNewConfigEnvNotSet(t *testing.T) {
 	os.Unsetenv(fileEnv)
 	config, err := NewConfig()
 	if err == nil {

From 6f5ecbe569c7c00bc64b1127a2701ec2125c6725 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 11 Jun 2020 11:25:28 -0700
Subject: [PATCH 094/481] Change version to 1.31.0-dev (#3681)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index c1926ea13edc..60b4b0d52b31 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.30.0-dev"
+const Version = "1.31.0-dev"

From 3b63c2b110b802a8c65a2a6bdf2bfa170475e7de Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 16 Jun 2020 10:03:59 -0700
Subject: [PATCH 095/481] retry: re-enable retrying on non-IO transport errors
 (#3691)

---
 internal/transport/http2_client.go | 15 ++++++++++++++-
 stream.go                          | 30 ++++++++++++++++++++++++++----
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 28b0c1662415..b43e21ffaf73 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -563,13 +563,26 @@ func (t *http2Client) getCallAuthData(ctx context.Context, audience string, call
 	return callAuthData, nil
 }
 
+// PerformedIOError wraps an error to indicate IO may have been performed
+// before the error occurred.
+type PerformedIOError struct {
+	Err error
+}
+
+// Error implements error.
+func (p PerformedIOError) Error() string {
+	return p.Err.Error()
+}
+
 // NewStream creates a stream and registers it into the transport as "active"
 // streams.
 func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) {
 	ctx = peer.NewContext(ctx, t.getPeer())
 	headerFields, err := t.createHeaderFields(ctx, callHdr)
 	if err != nil {
-		return nil, err
+		// We may have performed I/O in the per-RPC creds callback, so do not
+		// allow transparent retry.
+		return nil, PerformedIOError{err}
 	}
 	s := t.newStream(ctx, callHdr)
 	cleanup := func(err error) {
diff --git a/stream.go b/stream.go
index 68d5c14506d8..629af76bdfab 100644
--- a/stream.go
+++ b/stream.go
@@ -364,6 +364,11 @@ func (a *csAttempt) newStream() error {
 	cs.callHdr.PreviousAttempts = cs.numRetries
 	s, err := a.t.NewStream(cs.ctx, cs.callHdr)
 	if err != nil {
+		if _, ok := err.(transport.PerformedIOError); ok {
+			// Return without converting to an RPC error so retry code can
+			// inspect.
+			return err
+		}
 		return toRPCErr(err)
 	}
 	cs.attempt.s = s
@@ -459,6 +464,22 @@ func (cs *clientStream) commitAttempt() {
 // shouldRetry returns nil if the RPC should be retried; otherwise it returns
 // the error that should be returned by the operation.
 func (cs *clientStream) shouldRetry(err error) error {
+	unprocessed := false
+	if cs.attempt.s == nil {
+		pioErr, ok := err.(transport.PerformedIOError)
+		if ok {
+			// Unwrap error.
+			err = toRPCErr(pioErr.Err)
+		} else {
+			unprocessed = true
+		}
+		if !ok && !cs.callInfo.failFast {
+			// In the event of a non-IO operation error from NewStream, we
+			// never attempted to write anything to the wire, so we can retry
+			// indefinitely for non-fail-fast RPCs.
+			return nil
+		}
+	}
 	if cs.finished || cs.committed {
 		// RPC is finished or committed; cannot retry.
 		return err
@@ -466,10 +487,11 @@ func (cs *clientStream) shouldRetry(err error) error {
 	// Wait for the trailers.
 	if cs.attempt.s != nil {
 		<-cs.attempt.s.Done()
-		if cs.firstAttempt && cs.attempt.s.Unprocessed() {
-			// First attempt, stream unprocessed: transparently retry.
-			return nil
-		}
+		unprocessed = cs.attempt.s.Unprocessed()
+	}
+	if cs.firstAttempt && unprocessed {
+		// First attempt, stream unprocessed: transparently retry.
+		return nil
 	}
 	if cs.cc.dopts.disableRetry {
 		return err

From dfc058c6d9dd9b7395ec9b77b675fd54d3ff1512 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 16 Jun 2020 13:57:33 -0700
Subject: [PATCH 096/481] credentials: Add certificate provider plugin APIs.
 (#3654)

Also add an implementation for the `Distributor` type which makes it easier to implement new `Provider` types.
---
 credentials/tls/certprovider/distributor.go   | 110 ++++++
 .../tls/certprovider/distributor_test.go      | 159 ++++++++
 credentials/tls/certprovider/provider.go      | 104 ++++++
 credentials/tls/certprovider/store.go         | 120 ++++++
 credentials/tls/certprovider/store_test.go    | 351 ++++++++++++++++++
 5 files changed, 844 insertions(+)
 create mode 100644 credentials/tls/certprovider/distributor.go
 create mode 100644 credentials/tls/certprovider/distributor_test.go
 create mode 100644 credentials/tls/certprovider/provider.go
 create mode 100644 credentials/tls/certprovider/store.go
 create mode 100644 credentials/tls/certprovider/store_test.go

diff --git a/credentials/tls/certprovider/distributor.go b/credentials/tls/certprovider/distributor.go
new file mode 100644
index 000000000000..d6feb3afeba1
--- /dev/null
+++ b/credentials/tls/certprovider/distributor.go
@@ -0,0 +1,110 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"sync"
+
+	"google.golang.org/grpc/internal/grpcsync"
+)
+
+// Distributor makes it easy for provider implementations to furnish new key
+// materials by handling synchronization between the producer and consumers of
+// the key material.
+//
+// Provider implementations which choose to use a Distributor should do the
+// following:
+// - create a new Distributor using the NewDistributor() function.
+// - invoke the Set() method whenever they have new key material or errors to
+//   report.
+// - delegate to the distributor when handing calls to KeyMaterial().
+// - invoke the Stop() method when they are done using the distributor.
+type Distributor struct {
+	// mu protects the underlying key material.
+	mu   sync.Mutex
+	km   *KeyMaterial
+	pErr error
+
+	ready  *grpcsync.Event
+	closed *grpcsync.Event
+}
+
+// NewDistributor returns a new Distributor.
+func NewDistributor() *Distributor {
+	return &Distributor{
+		ready:  grpcsync.NewEvent(),
+		closed: grpcsync.NewEvent(),
+	}
+}
+
+// Set updates the key material in the distributor with km.
+//
+// Provider implementations which use the distributor must not modify the
+// contents of the KeyMaterial struct pointed to by km.
+//
+// A non-nil err value indicates the error that the provider implementation ran
+// into when trying to fetch key material, and makes it possible to surface the
+// error to the user. A non-nil error value passed here causes distributor's
+// KeyMaterial() method to return nil key material.
+func (d *Distributor) Set(km *KeyMaterial, err error) {
+	d.mu.Lock()
+	d.km = km
+	d.pErr = err
+	if err != nil {
+		// If a non-nil err is passed, we ignore the key material being passed.
+		d.km = nil
+	}
+	d.ready.Fire()
+	d.mu.Unlock()
+}
+
+// KeyMaterial returns the most recent key material provided to the distributor.
+// If no key material was provided at the time of this call, it will block until
+// the deadline on the context expires or fresh key material arrives.
+func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) {
+	if d.closed.HasFired() {
+		return nil, errProviderClosed
+	}
+
+	if d.ready.HasFired() {
+		return d.keyMaterial()
+	}
+
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-d.closed.Done():
+		return nil, errProviderClosed
+	case <-d.ready.Done():
+		return d.keyMaterial()
+	}
+}
+
+func (d *Distributor) keyMaterial() (*KeyMaterial, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.km, d.pErr
+}
+
+// Stop turns down the distributor, releases allocated resources and fails any
+// active KeyMaterial() call waiting for new key material.
+func (d *Distributor) Stop() {
+	d.closed.Fire()
+}
diff --git a/credentials/tls/certprovider/distributor_test.go b/credentials/tls/certprovider/distributor_test.go
new file mode 100644
index 000000000000..e6c41564c293
--- /dev/null
+++ b/credentials/tls/certprovider/distributor_test.go
@@ -0,0 +1,159 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+var errProviderTestInternal = errors.New("provider internal error")
+
+// TestDistributor invokes the different methods on the Distributor type and
+// verifies the results.
+func (s) TestDistributor(t *testing.T) {
+	dist := NewDistributor()
+
+	// Read cert/key files from testdata.
+	km, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// wantKM1 has both local and root certs.
+	wantKM1 := *km
+	// wantKM2 has only local certs. Roots are nil-ed out.
+	wantKM2 := *km
+	wantKM2.Roots = nil
+
+	// Create a goroutines which work in lockstep with the rest of the test.
+	// This goroutine reads the key material from the distributor while the rest
+	// of the test sets it.
+	var wg sync.WaitGroup
+	wg.Add(1)
+	errCh := make(chan error)
+	proceedCh := make(chan struct{})
+	go func() {
+		defer wg.Done()
+
+		// The first call to KeyMaterial() should timeout because no key
+		// material has been set on the distributor as yet.
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout/2)
+		defer cancel()
+		if _, err := dist.KeyMaterial(ctx); err != context.DeadlineExceeded {
+			errCh <- err
+			return
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should return the key material with both
+		// the local certs and the root certs.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		gotKM, err := dist.KeyMaterial(ctx)
+		if err != nil {
+			errCh <- err
+			return
+		}
+		if !reflect.DeepEqual(gotKM, &wantKM1) {
+			errCh <- fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM1)
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should eventually return key material with
+		// only the local certs.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		for {
+			gotKM, err := dist.KeyMaterial(ctx)
+			if err != nil {
+				errCh <- err
+				return
+			}
+			if reflect.DeepEqual(gotKM, &wantKM2) {
+				break
+			}
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should return nil key material and a
+		// non-nil error.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		for {
+			gotKM, err := dist.KeyMaterial(ctx)
+			if gotKM == nil && err == errProviderTestInternal {
+				break
+			}
+			if err != nil {
+				// If we have gotten any error other than
+				// errProviderTestInternal, we should bail out.
+				errCh <- err
+				return
+			}
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should eventually return errProviderClosed
+		// error.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		for {
+			if _, err := dist.KeyMaterial(ctx); err == errProviderClosed {
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Set(&wantKM1, nil)
+	})
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Set(&wantKM2, nil)
+	})
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Set(&wantKM2, errProviderTestInternal)
+	})
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Stop()
+	})
+
+}
+
+func waitAndDo(t *testing.T, proceedCh chan struct{}, errCh chan error, do func()) {
+	t.Helper()
+
+	timer := time.NewTimer(defaultTestTimeout)
+	select {
+	case <-timer.C:
+		t.Fatalf("test timed out when waiting for event from distributor")
+	case <-proceedCh:
+		do()
+	case err := <-errCh:
+		t.Fatal(err)
+	}
+}
diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go
new file mode 100644
index 000000000000..58c93c1e731f
--- /dev/null
+++ b/credentials/tls/certprovider/provider.go
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package certprovider defines APIs for certificate providers in gRPC.
+//
+// Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
+package certprovider
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+)
+
+var (
+	// errProviderClosed is returned by Distributor.KeyMaterial when it is
+	// closed.
+	errProviderClosed = errors.New("provider instance is closed")
+
+	// m is a map from name to provider builder.
+	m = make(map[string]Builder)
+)
+
+// Register registers the provider builder, whose name as returned by its Name()
+// method will be used as the name registered with this builder. Registered
+// Builders are used by the Store to create Providers.
+func Register(b Builder) {
+	m[b.Name()] = b
+}
+
+// getBuilder returns the provider builder registered with the given name.
+// If no builder is registered with the provided name, nil will be returned.
+func getBuilder(name string) Builder {
+	if b, ok := m[name]; ok {
+		return b
+	}
+	return nil
+}
+
+// Builder creates a Provider.
+type Builder interface {
+	// Build creates a new provider with the provided config.
+	Build(StableConfig) Provider
+
+	// ParseConfig converts config input in a format specific to individual
+	// implementations and returns an implementation of the StableConfig
+	// interface.
+	// Equivalent configurations must return StableConfig types whose
+	// Canonical() method returns the same output.
+	ParseConfig(interface{}) (StableConfig, error)
+
+	// Name returns the name of providers built by this builder.
+	Name() string
+}
+
+// StableConfig wraps the method to return a stable provider configuration.
+type StableConfig interface {
+	// Canonical returns provider config as an arbitrary byte slice.
+	// Equivalent configurations must return the same output.
+	Canonical() []byte
+}
+
+// Provider makes it possible to keep channel credential implementations up to
+// date with secrets that they rely on to secure communications on the
+// underlying channel.
+//
+// Provider implementations are free to rely on local or remote sources to fetch
+// the latest secrets, and free to share any state between different
+// instantiations as they deem fit.
+type Provider interface {
+	// KeyMaterial returns the key material sourced by the provider.
+	// Callers are expected to use the returned value as read-only.
+	KeyMaterial(ctx context.Context) (*KeyMaterial, error)
+
+	// Close cleans up resources allocated by the provider.
+	Close()
+}
+
+// KeyMaterial wraps the certificates and keys returned by a provider instance.
+type KeyMaterial struct {
+	// Certs contains a slice of cert/key pairs used to prove local identity.
+	Certs []tls.Certificate
+	// Roots contains the set of trusted roots to validate the peer's identity.
+	Roots *x509.CertPool
+}
diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go
new file mode 100644
index 000000000000..0d2b29b675b1
--- /dev/null
+++ b/credentials/tls/certprovider/store.go
@@ -0,0 +1,120 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"fmt"
+	"sync"
+)
+
+// provStore is the global singleton certificate provider store.
+var provStore = &store{
+	providers: make(map[storeKey]*wrappedProvider),
+}
+
+// storeKey acts as the key to the map of providers maintained by the store. A
+// combination of provider name and configuration is used to uniquely identify
+// every provider instance in the store. Go maps need to be indexed by
+// comparable types, so the provider configuration is converted from
+// `interface{}` to string using the ParseConfig method while creating this key.
+type storeKey struct {
+	// name of the certificate provider.
+	name string
+	// configuration of the certificate provider in string form.
+	config string
+}
+
+// wrappedProvider wraps a provider instance with a reference count.
+type wrappedProvider struct {
+	Provider
+	refCount int
+
+	// A reference to the key and store are also kept here to override the
+	// Close method on the provider.
+	storeKey storeKey
+	store    *store
+}
+
+// store is a collection of provider instances, safe for concurrent access.
+type store struct {
+	mu        sync.Mutex
+	providers map[storeKey]*wrappedProvider
+}
+
+// GetProvider returns a provider instance corresponding to name and config.
+// name is the registered name of the provider and config is the
+// provider-specific configuration. Implementations of the Builder interface
+// should clearly document the type of configuration accepted by them.
+//
+// If a provider exists for the (name+config) combination, its reference count
+// is incremented before returning. If no provider exists for the (name+config)
+// combination, a new one is created using the registered builder. If no
+// registered builder is found, or the provider configuration is rejected by it,
+// a non-nil error is returned.
+func GetProvider(name string, config interface{}) (Provider, error) {
+	provStore.mu.Lock()
+	defer provStore.mu.Unlock()
+
+	builder := getBuilder(name)
+	if builder == nil {
+		return nil, fmt.Errorf("no registered builder for provider name: %s", name)
+	}
+	stableConfig, err := builder.ParseConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	sk := storeKey{
+		name:   name,
+		config: string(stableConfig.Canonical()),
+	}
+	if wp, ok := provStore.providers[sk]; ok {
+		wp.refCount++
+		return wp, nil
+	}
+
+	provider := builder.Build(stableConfig)
+	if provider == nil {
+		return nil, fmt.Errorf("certprovider.Build(%v) failed", sk)
+	}
+	wp := &wrappedProvider{
+		Provider: provider,
+		refCount: 1,
+		storeKey: sk,
+		store:    provStore,
+	}
+	provStore.providers[sk] = wp
+	return wp, nil
+}
+
+// Close overrides the Close method of the embedded provider. It releases the
+// reference held by the caller on the underlying provider and if the
+// provider's reference count reaches zero, it is removed from the store, and
+// its Close method is also invoked.
+func (wp *wrappedProvider) Close() {
+	ps := wp.store
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+
+	wp.refCount--
+	if wp.refCount == 0 {
+		wp.Provider.Close()
+		delete(ps.providers, wp.storeKey)
+	}
+}
diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go
new file mode 100644
index 000000000000..6cd02e012dfb
--- /dev/null
+++ b/credentials/tls/certprovider/store_test.go
@@ -0,0 +1,351 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/testdata"
+)
+
+const (
+	fakeProvider1Name  = "fake-certificate-provider-1"
+	fakeProvider2Name  = "fake-certificate-provider-2"
+	fakeConfig         = "my fake config"
+	defaultTestTimeout = 1 * time.Second
+)
+
+func init() {
+	Register(&fakeProviderBuilder{name: fakeProvider1Name})
+	Register(&fakeProviderBuilder{name: fakeProvider2Name})
+}
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// fakeProviderBuilder builds new instances of fakeProvider and interprets the
+// config provided to it as a string.
+type fakeProviderBuilder struct {
+	name string
+}
+
+func (b *fakeProviderBuilder) Build(StableConfig) Provider {
+	ctx, cancel := context.WithCancel(context.Background())
+	p := &fakeProvider{
+		Distributor: NewDistributor(),
+		cancel:      cancel,
+		done:        make(chan struct{}),
+		kmCh:        make(chan *KeyMaterial, 2),
+	}
+	go p.run(ctx)
+	return p
+}
+
+func (b *fakeProviderBuilder) ParseConfig(config interface{}) (StableConfig, error) {
+	s, ok := config.(string)
+	if !ok {
+		return nil, fmt.Errorf("provider %s received bad config %v", b.name, config)
+	}
+	return &fakeStableConfig{config: s}, nil
+}
+
+func (b *fakeProviderBuilder) Name() string {
+	return b.name
+}
+
+type fakeStableConfig struct {
+	config string
+}
+
+func (c *fakeStableConfig) Canonical() []byte {
+	return []byte(c.config)
+}
+
+// fakeProvider is an implementation of the Provider interface which embeds a
+// Distributor and exposes two channels for the user:
+// 1. to be notified when the provider is closed
+// 2. to push new key material into the provider
+type fakeProvider struct {
+	*Distributor
+
+	// Used to cancel the run goroutine when the provider is closed.
+	cancel context.CancelFunc
+	// This channel is closed when the provider is closed. Tests should block on
+	// this to make sure the provider is closed.
+	done chan struct{}
+	// Tests can push new key material on this channel, and the provider will
+	// return this on subsequent calls to KeyMaterial().
+	kmCh chan *KeyMaterial
+}
+
+func (p *fakeProvider) run(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case km := <-p.kmCh:
+			p.Distributor.Set(km, nil)
+		}
+	}
+}
+
+func (p *fakeProvider) Close() {
+	p.cancel()
+	p.Distributor.Stop()
+}
+
+// loadKeyMaterials is a helper to read cert/key files from testdata and convert
+// them into a KeyMaterial struct.
+func loadKeyMaterials() (*KeyMaterial, error) {
+	certs, err := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	if err != nil {
+		return nil, err
+	}
+
+	pemData, err := ioutil.ReadFile(testdata.Path("ca.pem"))
+	if err != nil {
+		return nil, err
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots}, nil
+}
+
+func makeProvider(t *testing.T, name, config string) (Provider, *fakeProvider) {
+	t.Helper()
+
+	prov, err := GetProvider(name, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The store returns a wrappedProvider, which holds a reference to the
+	// actual provider, which in our case in the fakeProvider.
+	wp := prov.(*wrappedProvider)
+	fp := wp.Provider.(*fakeProvider)
+	return prov, fp
+}
+
+// TestStoreWithSingleProvider creates a single provider through the store and
+// calls methods on it.
+func (s) TestStoreWithSingleProvider(t *testing.T) {
+	prov, fp := makeProvider(t, fakeProvider1Name, fakeConfig)
+
+	// Push key materials into the provider.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp.kmCh <- wantKM
+
+	// Get key materials from the provider and compare it to the ones we pushed
+	// above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	// TODO(easwars): Remove all references to reflect.DeepEqual and use
+	// cmp.Equal instead. Currently, the later panics because x509.Certificate
+	// type defines an Equal method, but does not check for nil. This has been
+	// fixed in
+	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
+	// but this is only available starting go1.14. So, once we remove support
+	// for go1.13, we can make the switch.
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider and retry the KeyMaterial() call, and expect it to
+	// fail with a known error.
+	prov.Close()
+	if _, err := prov.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}
+
+// TestStoreWithSingleProviderWithSharing creates multiple instances of the same
+// type of provider through the store (and expects the store's sharing mechanism
+// to kick in) and calls methods on it.
+func (s) TestStoreWithSingleProviderWithSharing(t *testing.T) {
+	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig)
+	prov2, _ := makeProvider(t, fakeProvider1Name, fakeConfig)
+
+	// Push key materials into the fake provider1.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp1.kmCh <- wantKM
+
+	// Get key materials from the fake provider2 and compare it to the ones we
+	// pushed above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
+	// it to succeed.
+	prov1.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+
+	prov2.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}
+
+// TestStoreWithSingleProviderWithoutSharing creates multiple instances of the
+// same type of provider through the store with different configs. The store
+// would end up creating different provider instances for these and no sharing
+// would take place.
+func (s) TestStoreWithSingleProviderWithoutSharing(t *testing.T) {
+	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig+"1111")
+	prov2, fp2 := makeProvider(t, fakeProvider1Name, fakeConfig+"2222")
+
+	// Push the same key materials into the two providers.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp1.kmCh <- wantKM
+	fp2.kmCh <- wantKM
+
+	// Get key materials from the fake provider1 and compare it to the ones we
+	// pushed above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov1.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Get key materials from the fake provider2 and compare it to the ones we
+	// pushed above.
+	gotKM, err = prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Update the key materials used by provider1, and make sure provider2 is
+	// not affected.
+	newKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	newKM.Roots = nil
+	fp1.kmCh <- newKM
+
+	gotKM, err = prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
+	// it to succeed.
+	prov1.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+
+	prov2.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}
+
+// TestStoreWithMultipleProviders creates multiple providers of different types
+// and make sure closing of one does not affect the other.
+func (s) TestStoreWithMultipleProviders(t *testing.T) {
+	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig)
+	prov2, fp2 := makeProvider(t, fakeProvider2Name, fakeConfig)
+
+	// Push key materials into the fake providers.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp1.kmCh <- wantKM
+	fp2.kmCh <- wantKM
+
+	// Get key materials from the fake provider1 and compare it to the ones we
+	// pushed above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov1.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Get key materials from the fake provider2 and compare it to the ones we
+	// pushed above.
+	gotKM, err = prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
+	// it to succeed.
+	prov1.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+
+	prov2.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}

From 9a465503579e4f97b81d4e2ddafdd1daef80aa93 Mon Sep 17 00:00:00 2001
From: d-reidenbach <66145057+d-reidenbach@users.noreply.github.com>
Date: Wed, 17 Jun 2020 04:11:41 +0000
Subject: [PATCH 097/481] alts: Added peer attributes accessor for alts context
 and updated test method (#3675)

---
 .../alts/internal/authinfo/authinfo.go        |  6 +++++
 .../alts/internal/authinfo/authinfo_test.go   | 25 ++++++++++++++-----
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/credentials/alts/internal/authinfo/authinfo.go b/credentials/alts/internal/authinfo/authinfo.go
index 9c53d6b53fb8..ebea57da1ded 100644
--- a/credentials/alts/internal/authinfo/authinfo.go
+++ b/credentials/alts/internal/authinfo/authinfo.go
@@ -48,6 +48,7 @@ func newAuthInfo(result *altspb.HandshakerResult) *altsAuthInfo {
 			PeerServiceAccount:  result.GetPeerIdentity().GetServiceAccount(),
 			LocalServiceAccount: result.GetLocalIdentity().GetServiceAccount(),
 			PeerRpcVersions:     result.GetPeerRpcVersions(),
+			PeerAttributes:      result.GetPeerIdentity().GetAttributes(),
 		},
 		CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity},
 	}
@@ -87,3 +88,8 @@ func (s *altsAuthInfo) LocalServiceAccount() string {
 func (s *altsAuthInfo) PeerRPCVersions() *altspb.RpcProtocolVersions {
 	return s.p.GetPeerRpcVersions()
 }
+
+// PeerAttributes returns the context's peer attributes.
+func (s *altsAuthInfo) PeerAttributes() map[string]string {
+	return s.p.GetPeerAttributes()
+}
diff --git a/credentials/alts/internal/authinfo/authinfo_test.go b/credentials/alts/internal/authinfo/authinfo_test.go
index 10ac1b4bccac..722eeddc4a8d 100644
--- a/credentials/alts/internal/authinfo/authinfo_test.go
+++ b/credentials/alts/internal/authinfo/authinfo_test.go
@@ -35,15 +35,19 @@ func Test(t *testing.T) {
 }
 
 const (
-	testAppProtocol    = "my_app"
-	testRecordProtocol = "very_secure_protocol"
-	testPeerAccount    = "peer_service_account"
-	testLocalAccount   = "local_service_account"
-	testPeerHostname   = "peer_hostname"
-	testLocalHostname  = "local_hostname"
+	testAppProtocol             = "my_app"
+	testRecordProtocol          = "very_secure_protocol"
+	testPeerAccount             = "peer_service_account"
+	testLocalAccount            = "local_service_account"
+	testPeerHostname            = "peer_hostname"
+	testLocalHostname           = "local_hostname"
+	testLocalPeerAttributeKey   = "peer"
+	testLocalPeerAttributeValue = "attributes"
 )
 
 func (s) TestALTSAuthInfo(t *testing.T) {
+	testPeerAttributes := make(map[string]string)
+	testPeerAttributes[testLocalPeerAttributeKey] = testLocalPeerAttributeValue
 	for _, tc := range []struct {
 		result             *altspb.HandshakerResult
 		outAppProtocol     string
@@ -52,6 +56,7 @@ func (s) TestALTSAuthInfo(t *testing.T) {
 		outPeerAccount     string
 		outLocalAccount    string
 		outPeerRPCVersions *altspb.RpcProtocolVersions
+		outPeerAttributes  map[string]string
 	}{
 		{
 			&altspb.HandshakerResult{
@@ -61,6 +66,7 @@ func (s) TestALTSAuthInfo(t *testing.T) {
 					IdentityOneof: &altspb.Identity_ServiceAccount{
 						ServiceAccount: testPeerAccount,
 					},
+					Attributes: testPeerAttributes,
 				},
 				LocalIdentity: &altspb.Identity{
 					IdentityOneof: &altspb.Identity_ServiceAccount{
@@ -74,6 +80,7 @@ func (s) TestALTSAuthInfo(t *testing.T) {
 			testPeerAccount,
 			testLocalAccount,
 			nil,
+			testPeerAttributes,
 		},
 		{
 			&altspb.HandshakerResult{
@@ -83,6 +90,7 @@ func (s) TestALTSAuthInfo(t *testing.T) {
 					IdentityOneof: &altspb.Identity_Hostname{
 						Hostname: testPeerHostname,
 					},
+					Attributes: testPeerAttributes,
 				},
 				LocalIdentity: &altspb.Identity{
 					IdentityOneof: &altspb.Identity_Hostname{
@@ -115,6 +123,7 @@ func (s) TestALTSAuthInfo(t *testing.T) {
 					Minor: 11,
 				},
 			},
+			testPeerAttributes,
 		},
 	} {
 		authInfo := newAuthInfo(tc.result)
@@ -139,5 +148,9 @@ func (s) TestALTSAuthInfo(t *testing.T) {
 		if got, want := authInfo.PeerRPCVersions(), tc.outPeerRPCVersions; !reflect.DeepEqual(got, want) {
 			t.Errorf("authinfo.PeerRpcVersions()=%v, want %v", got, want)
 		}
+		if got, want := authInfo.PeerAttributes(), tc.outPeerAttributes; !reflect.DeepEqual(got, want) {
+			t.Errorf("authinfo.PeerAttributes()=%v, want %v", got, want)
+		}
+
 	}
 }

From 4e63bcab52b7ad5c264159e447231710de919b99 Mon Sep 17 00:00:00 2001
From: Zou Nengren <zounengren@cmss.chinamobile.com>
Date: Wed, 24 Jun 2020 00:49:44 +0800
Subject: [PATCH 098/481] test: replace
 manual.GenerateAndRegisterManualResolver with grpc.WithResolvers (#3700)

---
 balancer/grpclb/grpclb_test.go             | 45 +++++++---------
 balancer/roundrobin/roundrobin_test.go     | 35 +++++--------
 balancer_switching_test.go                 | 35 +++++--------
 clientconn_test.go                         | 39 ++++++--------
 examples/features/debugging/client/main.go |  5 +-
 examples/features/health/client/main.go    |  4 +-
 pickfirst_test.go                          | 38 ++++++--------
 resolver_conn_wrapper_test.go              | 14 +++--
 test/balancer_test.go                      | 30 +++++------
 test/channelz_test.go                      | 60 +++++++++-------------
 test/creds_test.go                         | 10 ++--
 test/end2end_test.go                       | 44 +++++++---------
 test/healthcheck_test.go                   |  8 +--
 13 files changed, 152 insertions(+), 215 deletions(-)

diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index d55fe3eb8be5..48082e2069fa 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -396,8 +396,7 @@ func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (ts
 var grpclbConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}`
 
 func (s) TestGRPCLB(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(1, nil)
 	if err != nil {
@@ -419,7 +418,7 @@ func (s) TestGRPCLB(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -444,8 +443,7 @@ func (s) TestGRPCLB(t *testing.T) {
 
 // The remote balancer sends response with duplicates to grpclb client.
 func (s) TestGRPCLBWeighted(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(2, nil)
 	if err != nil {
@@ -470,7 +468,7 @@ func (s) TestGRPCLBWeighted(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -510,8 +508,7 @@ func (s) TestGRPCLBWeighted(t *testing.T) {
 }
 
 func (s) TestDropRequest(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(2, nil)
 	if err != nil {
@@ -536,7 +533,7 @@ func (s) TestDropRequest(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -661,8 +658,7 @@ func (s) TestDropRequest(t *testing.T) {
 
 // When the balancer in use disconnects, grpclb should connect to the next address from resolved balancer address list.
 func (s) TestBalancerDisconnects(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	var (
 		tests []*testServers
@@ -694,7 +690,7 @@ func (s) TestBalancerDisconnects(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -739,8 +735,7 @@ func (s) TestFallback(t *testing.T) {
 	balancer.Register(newLBBuilderWithFallbackTimeout(100 * time.Millisecond))
 	defer balancer.Register(newLBBuilder())
 
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(1, nil)
 	if err != nil {
@@ -771,7 +766,7 @@ func (s) TestFallback(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -867,8 +862,7 @@ func (s) TestFallback(t *testing.T) {
 }
 
 func (s) TestExplicitFallback(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(1, nil)
 	if err != nil {
@@ -899,7 +893,7 @@ func (s) TestExplicitFallback(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -971,7 +965,7 @@ func (s) TestExplicitFallback(t *testing.T) {
 
 func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 	resolveNowCh := make(chan struct{}, 1)
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.ResolveNowCallback = func(resolver.ResolveNowOptions) {
 		select {
 		case <-resolveNowCh:
@@ -979,7 +973,6 @@ func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 		}
 		resolveNowCh <- struct{}{}
 	}
-	defer cleanup()
 
 	tss, cleanup, err := newLoadBalancer(1, nil)
 	if err != nil {
@@ -1009,7 +1002,7 @@ func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -1090,8 +1083,7 @@ func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 }
 
 func (s) TestGRPCLBPickFirst(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(3, nil)
 	if err != nil {
@@ -1120,7 +1112,7 @@ func (s) TestGRPCLBPickFirst(t *testing.T) {
 	creds := serverNameCheckCreds{}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
@@ -1245,8 +1237,7 @@ func checkStats(stats, expected *rpcStats) error {
 }
 
 func runAndCheckStats(t *testing.T, drop bool, statsChan chan *lbpb.ClientStats, runRPCs func(*grpc.ClientConn), statsWant *rpcStats) error {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	tss, cleanup, err := newLoadBalancer(1, statsChan)
 	if err != nil {
@@ -1270,7 +1261,7 @@ func runAndCheckStats(t *testing.T, drop bool, statsChan chan *lbpb.ClientStats,
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName,
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
 		grpc.WithTransportCredentials(&creds),
 		grpc.WithPerRPCCredentials(failPreRPCCred{}),
 		grpc.WithContextDialer(fakeNameDialer))
diff --git a/balancer/roundrobin/roundrobin_test.go b/balancer/roundrobin/roundrobin_test.go
index c5d066ed94c8..5a8ba481c9f0 100644
--- a/balancer/roundrobin/roundrobin_test.go
+++ b/balancer/roundrobin/roundrobin_test.go
@@ -98,8 +98,7 @@ func startTestServers(count int) (_ *test, err error) {
 }
 
 func (s) TestOneBackend(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	test, err := startTestServers(1)
 	if err != nil {
@@ -107,7 +106,7 @@ func (s) TestOneBackend(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -128,8 +127,7 @@ func (s) TestOneBackend(t *testing.T) {
 }
 
 func (s) TestBackendsRoundRobin(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	backendCount := 5
 	test, err := startTestServers(backendCount)
@@ -138,7 +136,7 @@ func (s) TestBackendsRoundRobin(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -187,8 +185,7 @@ func (s) TestBackendsRoundRobin(t *testing.T) {
 }
 
 func (s) TestAddressesRemoved(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	test, err := startTestServers(1)
 	if err != nil {
@@ -196,7 +193,7 @@ func (s) TestAddressesRemoved(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -233,8 +230,7 @@ func (s) TestAddressesRemoved(t *testing.T) {
 }
 
 func (s) TestCloseWithPendingRPC(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	test, err := startTestServers(1)
 	if err != nil {
@@ -242,7 +238,7 @@ func (s) TestCloseWithPendingRPC(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -266,8 +262,7 @@ func (s) TestCloseWithPendingRPC(t *testing.T) {
 }
 
 func (s) TestNewAddressWhileBlocking(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	test, err := startTestServers(1)
 	if err != nil {
@@ -275,7 +270,7 @@ func (s) TestNewAddressWhileBlocking(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -313,8 +308,7 @@ func (s) TestNewAddressWhileBlocking(t *testing.T) {
 }
 
 func (s) TestOneServerDown(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	backendCount := 3
 	test, err := startTestServers(backendCount)
@@ -323,7 +317,7 @@ func (s) TestOneServerDown(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -410,8 +404,7 @@ func (s) TestOneServerDown(t *testing.T) {
 }
 
 func (s) TestAllServersDown(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	backendCount := 3
 	test, err := startTestServers(backendCount)
@@ -420,7 +413,7 @@ func (s) TestAllServersDown(t *testing.T) {
 	}
 	defer test.cleanup()
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName(roundrobin.Name))
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
diff --git a/balancer_switching_test.go b/balancer_switching_test.go
index f47754bfdfeb..ed132121280c 100644
--- a/balancer_switching_test.go
+++ b/balancer_switching_test.go
@@ -147,14 +147,13 @@ func checkRoundRobin(cc *ClientConn, servers []*server) error {
 }
 
 func (s) TestSwitchBalancer(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	const numServers = 2
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -179,14 +178,13 @@ func (s) TestSwitchBalancer(t *testing.T) {
 
 // Test that balancer specified by dial option will not be overridden.
 func (s) TestBalancerDialOption(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	const numServers = 2
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}), WithBalancerName(roundrobin.Name))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}), WithBalancerName(roundrobin.Name))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -207,10 +205,9 @@ func (s) TestBalancerDialOption(t *testing.T) {
 
 // First addr update contains grpclb.
 func (s) TestSwitchBalancerGRPCLBFirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -268,10 +265,9 @@ func (s) TestSwitchBalancerGRPCLBFirst(t *testing.T) {
 
 // First addr update does not contain grpclb.
 func (s) TestSwitchBalancerGRPCLBSecond(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -345,10 +341,9 @@ func (s) TestSwitchBalancerGRPCLBSecond(t *testing.T) {
 // when the resolved address doesn't contain grpclb addresses, balancer will be
 // switched back to roundrobin.
 func (s) TestSwitchBalancerGRPCLBRoundRobin(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -408,10 +403,9 @@ func (s) TestSwitchBalancerGRPCLBRoundRobin(t *testing.T) {
 // service config won't take effect. But when there's no grpclb address in a new
 // resolved address list, balancer will be switched to the new one.
 func (s) TestSwitchBalancerGRPCLBServiceConfig(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -495,14 +489,13 @@ func (s) TestSwitchBalancerGRPCLBWithGRPCLBNotRegistered(t *testing.T) {
 	internal.BalancerUnregister("grpclb")
 	defer balancer.Register(&magicalLB{})
 
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	const numServers = 3
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
diff --git a/clientconn_test.go b/clientconn_test.go
index 524b9736c1a0..9b95f6cec087 100644
--- a/clientconn_test.go
+++ b/clientconn_test.go
@@ -66,10 +66,9 @@ func (s) TestDialWithTimeout(t *testing.T) {
 		<-dialDone // Close conn only after dial returns.
 	}()
 
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{lisAddr}})
-	client, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithTimeout(5*time.Second))
+	client, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithTimeout(5*time.Second))
 	close(dialDone)
 	if err != nil {
 		t.Fatalf("Dial failed. Err: %v", err)
@@ -119,10 +118,9 @@ func (s) TestDialWithMultipleBackendsNotSendingServerPreface(t *testing.T) {
 		}
 	}()
 
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{lis1Addr, lis2Addr}})
-	client, err := Dial(r.Scheme()+":///test.server", WithInsecure())
+	client, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r))
 	if err != nil {
 		t.Fatalf("Dial failed. Err: %v", err)
 	}
@@ -642,10 +640,9 @@ func (s) TestConnectParamsWithMinConnectTimeout(t *testing.T) {
 }
 
 func (s) TestResolverServiceConfigBeforeAddressNotPanic(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure())
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -660,10 +657,9 @@ func (s) TestResolverServiceConfigBeforeAddressNotPanic(t *testing.T) {
 
 func (s) TestResolverServiceConfigWhileClosingNotPanic(t *testing.T) {
 	for i := 0; i < 10; i++ { // Run this multiple times to make sure it doesn't panic.
-		r, rcleanup := manual.GenerateAndRegisterManualResolver()
-		defer rcleanup()
+		r := manual.NewBuilderWithScheme(fmt.Sprintf("whatever-%d", i))
 
-		cc, err := Dial(r.Scheme()+":///test.server", WithInsecure())
+		cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r))
 		if err != nil {
 			t.Fatalf("failed to dial: %v", err)
 		}
@@ -674,10 +670,9 @@ func (s) TestResolverServiceConfigWhileClosingNotPanic(t *testing.T) {
 }
 
 func (s) TestResolverEmptyUpdateNotPanic(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure())
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -754,10 +749,9 @@ func (s) TestClientUpdatesParamsAfterGoAway(t *testing.T) {
 }
 
 func (s) TestDisableServiceConfigOption(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	addr := r.Scheme() + ":///non.existent"
-	cc, err := Dial(addr, WithInsecure(), WithDisableServiceConfig())
+	cc, err := Dial(addr, WithInsecure(), WithResolvers(r), WithDisableServiceConfig())
 	if err != nil {
 		t.Fatalf("Dial(%s, _) = _, %v, want _, <nil>", addr, err)
 	}
@@ -1013,8 +1007,7 @@ func (s) TestUpdateAddresses_RetryFromFirstAddr(t *testing.T) {
 }
 
 func (s) TestDefaultServiceConfig(t *testing.T) {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	addr := r.Scheme() + ":///non.existent"
 	js := `{
     "methodConfig": [
@@ -1055,7 +1048,7 @@ func testInvalidDefaultServiceConfig(t *testing.T) {
 }
 
 func testDefaultServiceConfigWhenResolverServiceConfigDisabled(t *testing.T, r *manual.Resolver, addr string, js string) {
-	cc, err := Dial(addr, WithInsecure(), WithDisableServiceConfig(), WithDefaultServiceConfig(js))
+	cc, err := Dial(addr, WithInsecure(), WithDisableServiceConfig(), WithResolvers(r), WithDefaultServiceConfig(js))
 	if err != nil {
 		t.Fatalf("Dial(%s, _) = _, %v, want _, <nil>", addr, err)
 	}
@@ -1071,7 +1064,7 @@ func testDefaultServiceConfigWhenResolverServiceConfigDisabled(t *testing.T, r *
 }
 
 func testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig(t *testing.T, r *manual.Resolver, addr string, js string) {
-	cc, err := Dial(addr, WithInsecure(), WithDefaultServiceConfig(js))
+	cc, err := Dial(addr, WithInsecure(), WithResolvers(r), WithDefaultServiceConfig(js))
 	if err != nil {
 		t.Fatalf("Dial(%s, _) = _, %v, want _, <nil>", addr, err)
 	}
@@ -1085,7 +1078,7 @@ func testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig(t *testing.T
 }
 
 func testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig(t *testing.T, r *manual.Resolver, addr string, js string) {
-	cc, err := Dial(addr, WithInsecure(), WithDefaultServiceConfig(js))
+	cc, err := Dial(addr, WithInsecure(), WithResolvers(r), WithDefaultServiceConfig(js))
 	if err != nil {
 		t.Fatalf("Dial(%s, _) = _, %v, want _, <nil>", addr, err)
 	}
diff --git a/examples/features/debugging/client/main.go b/examples/features/debugging/client/main.go
index 33b7a0a1475f..29ac0fe89920 100644
--- a/examples/features/debugging/client/main.go
+++ b/examples/features/debugging/client/main.go
@@ -51,10 +51,9 @@ func main() {
 	defer s.Stop()
 
 	/***** Initialize manual resolver and Dial *****/
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	// Set up a connection to the server.
-	conn, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithBalancerName("round_robin"))
+	conn, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName("round_robin"))
 	if err != nil {
 		log.Fatalf("did not connect: %v", err)
 	}
diff --git a/examples/features/health/client/main.go b/examples/features/health/client/main.go
index 1a11782acf84..9cbc03f90a47 100644
--- a/examples/features/health/client/main.go
+++ b/examples/features/health/client/main.go
@@ -54,8 +54,7 @@ func callUnaryEcho(c pb.EchoClient) {
 func main() {
 	flag.Parse()
 
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{
 		Addresses: []resolver.Address{
 			{Addr: "localhost:50051"},
@@ -68,6 +67,7 @@ func main() {
 	options := []grpc.DialOption{
 		grpc.WithInsecure(),
 		grpc.WithBlock(),
+		grpc.WithResolvers(r),
 		grpc.WithDefaultServiceConfig(serviceConfig),
 	}
 
diff --git a/pickfirst_test.go b/pickfirst_test.go
index a69cec1c51de..9ece7844a355 100644
--- a/pickfirst_test.go
+++ b/pickfirst_test.go
@@ -39,14 +39,16 @@ func errorDesc(err error) string {
 }
 
 func (s) TestOneBackendPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 1
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server",
+		WithInsecure(),
+		WithResolvers(r),
+		WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -72,14 +74,13 @@ func (s) TestOneBackendPickfirst(t *testing.T) {
 }
 
 func (s) TestBackendsPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 2
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -105,14 +106,13 @@ func (s) TestBackendsPickfirst(t *testing.T) {
 }
 
 func (s) TestNewAddressWhileBlockingPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 1
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -141,14 +141,13 @@ func (s) TestNewAddressWhileBlockingPickfirst(t *testing.T) {
 }
 
 func (s) TestCloseWithPendingRPCPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 1
 	_, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -177,14 +176,13 @@ func (s) TestCloseWithPendingRPCPickfirst(t *testing.T) {
 }
 
 func (s) TestOneServerDownPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 2
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -218,14 +216,13 @@ func (s) TestOneServerDownPickfirst(t *testing.T) {
 }
 
 func (s) TestAllServersDownPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 2
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
@@ -261,14 +258,13 @@ func (s) TestAllServersDownPickfirst(t *testing.T) {
 }
 
 func (s) TestAddressesRemovedPickfirst(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	numServers := 3
 	servers, scleanup := startServers(t, numServers, math.MaxInt32)
 	defer scleanup()
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithCodec(testCodec{}))
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
 	}
diff --git a/resolver_conn_wrapper_test.go b/resolver_conn_wrapper_test.go
index 9f22c8b90f6c..e125976a5359 100644
--- a/resolver_conn_wrapper_test.go
+++ b/resolver_conn_wrapper_test.go
@@ -77,14 +77,14 @@ func testResolverErrorPolling(t *testing.T, badUpdate func(*manual.Resolver), go
 		return 0
 	}
 
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	rn := make(chan struct{})
 	defer func() { close(rn) }()
 	r.ResolveNowCallback = func(resolver.ResolveNowOptions) { rn <- struct{}{} }
 
 	defaultDialOptions := []DialOption{
 		WithInsecure(),
+		WithResolvers(r),
 		withResolveNowBackoff(resolverBackoff),
 	}
 	cc, err := Dial(r.Scheme()+":///test.server", append(defaultDialOptions, dopts...)...)
@@ -173,11 +173,10 @@ func (s) TestServiceConfigErrorPolling(t *testing.T) {
 // sure there is no data race in this code path, and also that there is no
 // deadlock.
 func (s) TestResolverErrorInBuild(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Err: errors.New("resolver build err")}})
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure())
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r))
 	if err != nil {
 		t.Fatalf("Dial(_, _) = _, %v; want _, nil", err)
 	}
@@ -194,10 +193,9 @@ func (s) TestResolverErrorInBuild(t *testing.T) {
 }
 
 func (s) TestServiceConfigErrorRPC(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure())
+	cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r))
 	if err != nil {
 		t.Fatalf("Dial(_, _) = _, %v; want _, nil", err)
 	}
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 3cd4e0e91fb7..7fa96d8680ee 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -321,13 +321,12 @@ func (testBalancerKeepAddresses) Close() {
 // Make sure that non-grpclb balancers don't get grpclb addresses even if name
 // resolver sends them
 func (s) TestNonGRPCLBBalancerGetsNoGRPCLBAddress(t *testing.T) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	b := newTestBalancerKeepAddresses()
 	balancer.Register(b)
 
-	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(),
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r),
 		grpc.WithBalancerName(b.Name()))
 	if err != nil {
 		t.Fatalf("failed to dial: %v", err)
@@ -433,8 +432,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	stub.Register(attrBalancerName, bf)
 	t.Logf("Registered balancer %s...", attrBalancerName)
 
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	t.Logf("Registered manual resolver with scheme %s...", r.Scheme())
 
 	lis, err := net.Listen("tcp", "localhost:0")
@@ -451,6 +449,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	creds := &attrTransportCreds{}
 	dopts := []grpc.DialOption{
 		grpc.WithTransportCredentials(creds),
+		grpc.WithResolvers(r),
 		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, attrBalancerName)),
 	}
 	cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...)
@@ -517,10 +516,9 @@ func (s) TestServersSwap(t *testing.T) {
 	defer cleanup()
 
 	// Initialize client
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: addr1}}})
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure())
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(r))
 	if err != nil {
 		t.Fatalf("Error creating client: %v", err)
 	}
@@ -570,15 +568,14 @@ func (s) TestEmptyAddrs(t *testing.T) {
 	go s.Serve(lis)
 
 	// Initialize pickfirst client
-	pfr, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	pfr := manual.NewBuilderWithScheme("whatever")
 	pfrnCalled := grpcsync.NewEvent()
 	pfr.ResolveNowCallback = func(resolver.ResolveNowOptions) {
 		pfrnCalled.Fire()
 	}
 	pfr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
 
-	pfcc, err := grpc.DialContext(ctx, pfr.Scheme()+":///", grpc.WithInsecure())
+	pfcc, err := grpc.DialContext(ctx, pfr.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(pfr))
 	if err != nil {
 		t.Fatalf("Error creating client: %v", err)
 	}
@@ -596,15 +593,15 @@ func (s) TestEmptyAddrs(t *testing.T) {
 	<-pfrnCalled.Done()
 
 	// Initialize roundrobin client
-	rrr, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	rrr := manual.NewBuilderWithScheme("whatever")
+
 	rrrnCalled := grpcsync.NewEvent()
 	rrr.ResolveNowCallback = func(resolver.ResolveNowOptions) {
 		rrrnCalled.Fire()
 	}
 	rrr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
 
-	rrcc, err := grpc.DialContext(ctx, rrr.Scheme()+":///", grpc.WithInsecure(),
+	rrcc, err := grpc.DialContext(ctx, rrr.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(rrr),
 		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, roundrobin.Name)))
 	if err != nil {
 		t.Fatalf("Error creating client: %v", err)
@@ -660,10 +657,9 @@ func (s) TestWaitForReady(t *testing.T) {
 	go s.Serve(lis)
 
 	// Initialize client
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure())
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(r))
 	if err != nil {
 		t.Fatalf("Error creating client: %v", err)
 	}
diff --git a/test/channelz_test.go b/test/channelz_test.go
index c69e0cec2e6e..37140bb2ce89 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -209,12 +209,11 @@ func (s) TestCZNestedChannelRegistrationAndDeletion(t *testing.T) {
 	// avoid calling API to set balancer type, which will void service config's change of balancer.
 	e.balancer = ""
 	te := newTest(t, e)
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", Type: resolver.GRPCLB, ServerName: "grpclb.server"}}
 	r.InitialState(resolver.State{Addresses: resolvedAddrs})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -255,14 +254,13 @@ func (s) TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) {
 	te := newTest(t, e)
 	var svrAddrs []resolver.Address
 	te.startServers(&testServer{security: e.security}, num)
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
 	}
 	r.InitialState(resolver.State{Addresses: svrAddrs})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	// Here, we just wait for all sockets to be up. In the future, if we implement
 	// IDLE, we may need to make several rpc calls to create the sockets.
@@ -507,14 +505,13 @@ func (s) TestCZChannelMetrics(t *testing.T) {
 	te.maxClientSendMsgSize = newInt(8)
 	var svrAddrs []resolver.Address
 	te.startServers(&testServer{security: e.security}, num)
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
 	}
 	r.InitialState(resolver.State{Addresses: svrAddrs})
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(cc)
 	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
@@ -1397,12 +1394,11 @@ func (s) TestCZChannelTraceCreationDeletion(t *testing.T) {
 	// avoid calling API to set balancer type, which will void service config's change of balancer.
 	e.balancer = ""
 	te := newTest(t, e)
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", Type: resolver.GRPCLB, ServerName: "grpclb.server"}}
 	r.InitialState(resolver.State{Addresses: resolvedAddrs})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	var nestedConn int64
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -1472,11 +1468,10 @@ func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
 	te.startServer(&testServer{security: e.security})
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	var subConn int64
 	// Here, we just wait for all sockets to be up. In the future, if we implement
@@ -1566,12 +1561,11 @@ func (s) TestCZChannelAddressResolutionChange(t *testing.T) {
 	e.balancer = ""
 	te := newTest(t, e)
 	te.startServer(&testServer{security: e.security})
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	addrs := []resolver.Address{{Addr: te.srvAddr}}
 	r.InitialState(resolver.State{Addresses: addrs})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	var cid int64
 	// Here, we just wait for all sockets to be up. In the future, if we implement
@@ -1670,15 +1664,14 @@ func (s) TestCZSubChannelPickedNewAddress(t *testing.T) {
 	e.balancer = ""
 	te := newTest(t, e)
 	te.startServers(&testServer{security: e.security}, 3)
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	var svrAddrs []resolver.Address
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
 	}
 	r.InitialState(resolver.State{Addresses: svrAddrs})
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(cc)
 	// make sure the connection is up
@@ -1730,11 +1723,10 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
 	te.startServer(&testServer{security: e.security})
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(cc)
 	// make sure the connection is up
@@ -1825,11 +1817,10 @@ func (s) TestCZChannelConnectivityState(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
 	te.startServer(&testServer{security: e.security})
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(cc)
 	// make sure the connection is up
@@ -1886,12 +1877,11 @@ func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) {
 	te := newTest(t, e)
 	channelz.SetMaxTraceEntry(1)
 	defer channelz.ResetMaxTraceEntryToDefault()
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", Type: resolver.GRPCLB, ServerName: "grpclb.server"}}
 	r.InitialState(resolver.State{Addresses: resolvedAddrs})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	var nestedConn int64
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -1950,11 +1940,10 @@ func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) {
 	channelz.SetMaxTraceEntry(1)
 	defer channelz.ResetMaxTraceEntryToDefault()
 	te.startServer(&testServer{security: e.security})
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	defer te.tearDown()
 	var subConn int64
 	// Here, we just wait for all sockets to be up. In the future, if we implement
@@ -2009,11 +1998,10 @@ func (s) TestCZTraceTopChannelDeletionTraceClear(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
 	te.startServer(&testServer{security: e.security})
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
-	defer cleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
-	te.clientConn()
+	te.clientConn(grpc.WithResolvers(r))
 	var subConn int64
 	// Here, we just wait for all sockets to be up. In the future, if we implement
 	// IDLE, we may need to make several rpc calls to create the sockets.
diff --git a/test/creds_test.go b/test/creds_test.go
index 8f87af125ec3..b25336908adb 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -411,10 +411,9 @@ func (s) TestCredsHandshakeAuthority(t *testing.T) {
 	go s.Serve(lis)
 	defer s.Stop()
 
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred))
+	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred), grpc.WithResolvers(r))
 	if err != nil {
 		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
 	}
@@ -452,10 +451,9 @@ func (s) TestCredsHandshakeServerNameAuthority(t *testing.T) {
 	go s.Serve(lis)
 	defer s.Stop()
 
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
-	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred))
+	cc, err := grpc.Dial(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred), grpc.WithResolvers(r))
 	if err != nil {
 		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
 	}
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 82150fe11be9..3e129c3fc159 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -1435,11 +1435,10 @@ func newDuration(b time.Duration) (a *time.Duration) {
 func (s) TestGetMethodConfig(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
 	defer te.tearDown()
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	addrs := []resolver.Address{{Addr: te.srvAddr}}
 	r.UpdateState(resolver.State{
 		Addresses: addrs,
@@ -1521,12 +1520,11 @@ func (s) TestGetMethodConfig(t *testing.T) {
 func (s) TestServiceConfigWaitForReady(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
 	defer te.tearDown()
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	// Case1: Client API set failfast to be false, and service config set wait_for_ready to be false, Client API should win, and the rpc will wait until deadline exceeds.
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	addrs := []resolver.Address{{Addr: te.srvAddr}}
 	r.UpdateState(resolver.State{
 		Addresses: addrs,
@@ -1610,12 +1608,11 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) {
 func (s) TestServiceConfigTimeout(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
 	defer te.tearDown()
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	// Case1: Client API sets timeout to be 1ns and ServiceConfig sets timeout to be 1hr. Timeout should be 1ns (min of 1ns and 1hr) and the rpc will wait until deadline exceeds.
 	te.resolverScheme = r.Scheme()
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	addrs := []resolver.Address{{Addr: te.srvAddr}}
 	r.UpdateState(resolver.State{
 		Addresses: addrs,
@@ -1708,8 +1705,7 @@ func (s) TestServiceConfigTimeout(t *testing.T) {
 
 func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	e := tcpClearRREnv
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	// Setting up values and objects shared across all test cases.
 	const smallSize = 1
@@ -1736,7 +1732,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	te1.resolverScheme = r.Scheme()
 	te1.nonBlockingDial = true
 	te1.startServer(&testServer{security: e.security})
-	cc1 := te1.clientConn()
+	cc1 := te1.clientConn(grpc.WithResolvers(r))
 
 	addrs := []resolver.Address{{Addr: te1.srvAddr}}
 	sc := parseCfg(r, `{
@@ -1827,7 +1823,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 
 	te2.startServer(&testServer{security: e.security})
 	defer te2.tearDown()
-	cc2 := te2.clientConn()
+	cc2 := te2.clientConn(grpc.WithResolvers(r))
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: te2.srvAddr}}, ServiceConfig: sc})
 	tc = testpb.NewTestServiceClient(cc2)
 
@@ -1888,7 +1884,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	te3.startServer(&testServer{security: e.security})
 	defer te3.tearDown()
 
-	cc3 := te3.clientConn()
+	cc3 := te3.clientConn(grpc.WithResolvers(r))
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: te3.srvAddr}}, ServiceConfig: sc})
 	tc = testpb.NewTestServiceClient(cc3)
 
@@ -1971,12 +1967,11 @@ func (s) TestStreamingRPCWithTimeoutInServiceConfigRecv(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
 	te.startServer(&testServer{security: tcpClearRREnv.security})
 	defer te.tearDown()
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	te.resolverScheme = r.Scheme()
 	te.nonBlockingDial = true
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	tc := testpb.NewTestServiceClient(cc)
 
 	r.UpdateState(resolver.State{
@@ -5017,9 +5012,8 @@ func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallSer
 
 // Start starts the server and creates a client connected to it.
 func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error {
-	r, cleanup := manual.GenerateAndRegisterManualResolver()
+	r := manual.NewBuilderWithScheme("whatever")
 	ss.r = r
-	ss.cleanups = append(ss.cleanups, cleanup)
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -5036,7 +5030,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption)
 
 	target := ss.r.Scheme() + ":///" + ss.addr
 
-	opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...)
+	opts := append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(r)}, dopts...)
 	cc, err := grpc.Dial(target, opts...)
 	if err != nil {
 		return fmt.Errorf("grpc.Dial(%q) = %v", target, err)
@@ -6693,12 +6687,11 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	testpb.RegisterTestServiceServer(s2, ts)
 	go s2.Serve(lis2)
 
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{
 		{Addr: lis1.Addr().String()},
 	}})
-	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure())
+	cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithInsecure())
 	if err != nil {
 		t.Fatalf("Error creating client: %v", err)
 	}
@@ -6763,12 +6756,11 @@ func (s) TestRPCWaitsForResolver(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
 	te.startServer(&testServer{security: tcpClearRREnv.security})
 	defer te.tearDown()
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
-	defer rcleanup()
+	r := manual.NewBuilderWithScheme("whatever")
 
 	te.resolverScheme = r.Scheme()
 	te.nonBlockingDial = true
-	cc := te.clientConn()
+	cc := te.clientConn(grpc.WithResolvers(r))
 	tc := testpb.NewTestServiceClient(cc)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go
index ec0c13e02743..0a60f8c927ac 100644
--- a/test/healthcheck_test.go
+++ b/test/healthcheck_test.go
@@ -152,19 +152,19 @@ type clientConfig struct {
 }
 
 func setupClient(c *clientConfig) (cc *grpc.ClientConn, r *manual.Resolver, deferFunc func(), err error) {
-	r, rcleanup := manual.GenerateAndRegisterManualResolver()
+	r = manual.NewBuilderWithScheme("whatever")
 	var opts []grpc.DialOption
-	opts = append(opts, grpc.WithInsecure(), grpc.WithBalancerName(c.balancerName))
+	opts = append(opts, grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(c.balancerName))
 	if c.testHealthCheckFuncWrapper != nil {
 		opts = append(opts, internal.WithHealthCheckFunc.(func(internal.HealthChecker) grpc.DialOption)(c.testHealthCheckFuncWrapper))
 	}
 	opts = append(opts, c.extraDialOption...)
 	cc, err = grpc.Dial(r.Scheme()+":///test.server", opts...)
 	if err != nil {
-		rcleanup()
+
 		return nil, nil, nil, fmt.Errorf("dial failed due to err: %v", err)
 	}
-	return cc, r, func() { cc.Close(); rcleanup() }, nil
+	return cc, r, func() { cc.Close() }, nil
 }
 
 func (s) TestHealthCheckWatchStateChange(t *testing.T) {

From 38aafd89f814f347db56a52efd055961651078ad Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 24 Jun 2020 12:44:51 -0700
Subject: [PATCH 099/481] vet.sh: require ALL modules are tidy; tidy some that
 are not (#3709)

---
 cmd/protoc-gen-go-grpc/go.sum |  2 ++
 security/advancedtls/go.mod   | 10 ++++++++-
 security/advancedtls/go.sum   | 42 +++++++++++++++++++++++++++++++++--
 vet.sh                        |  6 ++---
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/cmd/protoc-gen-go-grpc/go.sum b/cmd/protoc-gen-go-grpc/go.sum
index 3e741dc38700..92baf2631b73 100644
--- a/cmd/protoc-gen-go-grpc/go.sum
+++ b/cmd/protoc-gen-go-grpc/go.sum
@@ -5,7 +5,9 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index a9f7f46dbbbe..392985d74469 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -2,4 +2,12 @@ module google.golang.org/grpc/security/advancedtls
 
 go 1.13
 
-require google.golang.org/grpc v1.27.0
+require (
+	github.com/golang/protobuf v1.3.5 // indirect
+	github.com/google/go-cmp v0.4.0 // indirect
+	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
+	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
+	golang.org/x/text v0.3.3 // indirect
+	google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 // indirect
+	google.golang.org/grpc v1.29.1
+)
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 0a0042b33388..fd8b6c4732d7 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,53 +1,91 @@
 cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4 h1:c2HOrn5iMezYjSlGPncknSEr/8x5LELb/ilJbXi9DEA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 h1:fiNLklpBwWK1mth30Hlwk+fcdBmIALlgF5iy77O37Ig=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/vet.sh b/vet.sh
index f0a67298a5b0..bfd5d5411053 100755
--- a/vet.sh
+++ b/vet.sh
@@ -108,10 +108,10 @@ if [[ -z "${VET_SKIP_PROTO}" ]]; then
     (git status; git --no-pager diff; exit 1)
 fi
 
-# - Check that our module is tidy.
+# - Check that our modules are tidy.
 if go help mod >& /dev/null; then
-  go mod tidy && \
-    git status --porcelain 2>&1 | fail_on_output || \
+  find . -name 'go.mod' | xargs -IXXX bash -c 'cd $(dirname XXX); go mod tidy'
+  git status --porcelain 2>&1 | fail_on_output || \
     (git status; git --no-pager diff; exit 1)
 fi
 

From 7a808837ae926d5e44454a9a34af1d92cafeea10 Mon Sep 17 00:00:00 2001
From: Patrice Chalin <chalin@users.noreply.github.com>
Date: Thu, 25 Jun 2020 13:40:16 -0400
Subject: [PATCH 100/481] examples: make test script output easier to read
 (#3711)

---
 examples/examples_test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/examples_test.sh b/examples/examples_test.sh
index a382b18c8276..85e2958bff8e 100755
--- a/examples/examples_test.sh
+++ b/examples/examples_test.sh
@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-set +e -x
+set +e
 
 export TMPDIR=$(mktemp -d)
 trap "rm -rf ${TMPDIR}" EXIT
@@ -154,6 +154,6 @@ for example in ${EXAMPLES[@]}; do
             pass "client log contains expected output: ${EXPECTED_CLIENT_OUTPUT[$example]}"
         fi
     fi
-
     clean
+    echo ""
 done

From 31d22c78fb16fff5d12d823f797b3f9995285b68 Mon Sep 17 00:00:00 2001
From: Patrice Chalin <chalin@users.noreply.github.com>
Date: Thu, 25 Jun 2020 17:28:39 -0400
Subject: [PATCH 101/481] examples: use grpc v1.30.0 rather than v1.30.0-dev.1
 (#3710)

---
 examples/go.mod |  6 +++---
 examples/go.sum | 17 +++++++++++++++--
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/examples/go.mod b/examples/go.mod
index 8be62bc7b4a3..cbd35e6337c1 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -4,8 +4,8 @@ go 1.11
 
 require (
 	github.com/golang/mock v1.1.1
-	github.com/golang/protobuf v1.4.0
+	github.com/golang/protobuf v1.4.2
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
-	google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55
-	google.golang.org/grpc v1.30.0-dev.1
+	google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
+	google.golang.org/grpc v1.30.0
 )
diff --git a/examples/go.sum b/examples/go.sum
index 7aa49f559848..7e80c324a9af 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -6,6 +6,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
@@ -22,6 +23,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -60,16 +64,25 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad h1:uAwc13+y0Y8QZLTYhLCu6lHhnG99ecQU5FYTj8zxAng=
+google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.30.0-dev.1 h1:UPWdABFs9zu2kdq7GrCUcfnVgCT65hSpvHmy0RiKn0M=
-google.golang.org/grpc v1.30.0-dev.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From 42419544077668f30ecb673e1a16bd826c95fe88 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 25 Jun 2020 20:03:47 -0700
Subject: [PATCH 102/481] xds: Move test only utility method to testutils.
 (#3715)

---
 xds/internal/balancer/lrs/lrs_test.go |  5 +++--
 xds/internal/internal.go              | 11 ---------
 xds/internal/testutils/locality.go    | 32 +++++++++++++++++++++++++++
 3 files changed, 35 insertions(+), 13 deletions(-)
 create mode 100644 xds/internal/testutils/locality.go

diff --git a/xds/internal/balancer/lrs/lrs_test.go b/xds/internal/balancer/lrs/lrs_test.go
index 09b22ac9fbe6..b18c3d7e218d 100644
--- a/xds/internal/balancer/lrs/lrs_test.go
+++ b/xds/internal/balancer/lrs/lrs_test.go
@@ -39,6 +39,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 const (
@@ -286,7 +287,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 						)
 					}
 					upstreamLocalityStats = append(upstreamLocalityStats, &endpointpb.UpstreamLocalityStats{
-						Locality:                l.ToProto(),
+						Locality:                testutils.LocalityIDToProto(l),
 						TotalSuccessfulRequests: count.success,
 						TotalRequestsInProgress: tempInProgress,
 						TotalErrorRequests:      count.failure,
@@ -298,7 +299,7 @@ func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
 				for l, c := range inProgressCounts {
 					if _, ok := counts[l]; !ok {
 						upstreamLocalityStats = append(upstreamLocalityStats, &endpointpb.UpstreamLocalityStats{
-							Locality:                l.ToProto(),
+							Locality:                testutils.LocalityIDToProto(l),
 							TotalRequestsInProgress: c,
 						})
 					}
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index b2c980003dd8..8b17cf930242 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -21,8 +21,6 @@ package internal
 
 import (
 	"fmt"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 )
 
 type clientID string
@@ -49,12 +47,3 @@ type LocalityID struct {
 func (l LocalityID) String() string {
 	return fmt.Sprintf("%s-%s-%s", l.Region, l.Zone, l.SubZone)
 }
-
-// ToProto convert Locality to the proto representation.
-func (l LocalityID) ToProto() *corepb.Locality {
-	return &corepb.Locality{
-		Region:  l.Region,
-		Zone:    l.Zone,
-		SubZone: l.SubZone,
-	}
-}
diff --git a/xds/internal/testutils/locality.go b/xds/internal/testutils/locality.go
new file mode 100644
index 000000000000..a4e4cc59b865
--- /dev/null
+++ b/xds/internal/testutils/locality.go
@@ -0,0 +1,32 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package testutils
+
+import (
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	"google.golang.org/grpc/xds/internal"
+)
+
+// LocalityIDToProto converts a LocalityID to its proto representation.
+func LocalityIDToProto(l internal.LocalityID) *corepb.Locality {
+	return &corepb.Locality{
+		Region:  l.Region,
+		Zone:    l.Zone,
+		SubZone: l.SubZone,
+	}
+}

From 506b7730668b5a13465224b0d8133f974a3f843d Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Fri, 26 Jun 2020 12:04:47 -0700
Subject: [PATCH 103/481] Implemented component logging (#3617)

---
 balancer/base/balancer.go                     |  16 +--
 balancer/grpclb/grpclb.go                     |  21 ++--
 balancer/grpclb/grpclb_remote_balancer.go     |  23 ++--
 balancer/rls/internal/balancer.go             |  17 +--
 balancer/rls/internal/cache/cache.go          |   6 +-
 balancer/rls/internal/config.go               |   5 +-
 balancer/roundrobin/roundrobin.go             |   4 +-
 balancer_conn_wrappers.go                     |   2 +-
 channelz/service/service.go                   |   3 +
 channelz/service/service_test.go              |  20 +--
 clientconn.go                                 |  44 +++----
 credentials/alts/alts.go                      |   3 +-
 dialoptions.go                                |   3 +-
 grpclog/component.go                          | 117 ++++++++++++++++++
 grpclog/glogger/glogger.go                    |  34 ++---
 grpclog/loggerv2.go                           |   4 +
 internal/binarylog/binarylog.go               |   4 +-
 internal/binarylog/env_config.go              |   4 +-
 internal/binarylog/method_logger.go           |  13 +-
 internal/binarylog/sink.go                    |   3 +-
 internal/channelz/funcs.go                    |  18 +--
 internal/channelz/logging.go                  |  52 ++++----
 internal/channelz/types.go                    |  25 ++--
 internal/channelz/types_nonlinux.go           |   4 +-
 internal/grpclog/grpclog.go                   |  13 +-
 internal/resolver/dns/dns_resolver.go         |  10 +-
 internal/syscall/syscall_linux.go             |   4 +-
 internal/syscall/syscall_nonlinux.go          |   3 +-
 internal/transport/controlbuf.go              |  12 +-
 internal/transport/http2_client.go            |  16 ++-
 internal/transport/http2_server.go            |  43 +++++--
 internal/transport/http_util.go               |   6 +-
 internal/transport/log.go                     |  44 -------
 internal/transport/transport.go               |   2 +
 picker_wrapper.go                             |   5 +-
 pickfirst.go                                  |  17 ++-
 resolver_conn_wrapper.go                      |  14 +--
 server.go                                     |  27 ++--
 service_config.go                             |   9 +-
 stream.go                                     |   4 +-
 .../balancer/edsbalancer/eds_impl_priority.go |   5 +-
 xds/internal/balancer/lrs/lrs.go              |  24 ++--
 xds/internal/balancer/orca/orca.go            |   6 +-
 xds/internal/client/client_loadreport.go      |   4 +-
 44 files changed, 426 insertions(+), 287 deletions(-)
 create mode 100644 grpclog/component.go
 delete mode 100644 internal/transport/log.go

diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go
index d62b4b6069a8..32d782f1cf5c 100644
--- a/balancer/base/balancer.go
+++ b/balancer/base/balancer.go
@@ -28,6 +28,8 @@ import (
 	"google.golang.org/grpc/resolver"
 )
 
+var logger = grpclog.Component("balancer")
+
 type baseBuilder struct {
 	name          string
 	pickerBuilder PickerBuilder
@@ -91,8 +93,8 @@ func (b *baseBalancer) ResolverError(err error) {
 
 func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	// TODO: handle s.ResolverState.ServiceConfig?
-	if grpclog.V(2) {
-		grpclog.Infoln("base.baseBalancer: got new ClientConn state: ", s)
+	if logger.V(2) {
+		logger.Info("base.baseBalancer: got new ClientConn state: ", s)
 	}
 	// Successful resolution; clear resolver error and ensure we return nil.
 	b.resolverErr = nil
@@ -104,7 +106,7 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 			// a is a new address (not existing in b.subConns).
 			sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: b.config.HealthCheck})
 			if err != nil {
-				grpclog.Warningf("base.baseBalancer: failed to create new SubConn: %v", err)
+				logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err)
 				continue
 			}
 			b.subConns[a] = sc
@@ -168,13 +170,13 @@ func (b *baseBalancer) regeneratePicker() {
 
 func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	s := state.ConnectivityState
-	if grpclog.V(2) {
-		grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s)
+	if logger.V(2) {
+		logger.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s)
 	}
 	oldS, ok := b.scStates[sc]
 	if !ok {
-		if grpclog.V(2) {
-			grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s)
+		if logger.V(2) {
+			logger.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s)
 		}
 		return
 	}
diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go
index 59f38f9047d5..a7424cf8d2d7 100644
--- a/balancer/grpclb/grpclb.go
+++ b/balancer/grpclb/grpclb.go
@@ -50,6 +50,7 @@ const (
 )
 
 var errServerTerminatedConnection = errors.New("grpclb: failed to recv server list: server terminated connection")
+var logger = grpclog.Component("grpclb")
 
 func convertDuration(d *durationpb.Duration) time.Duration {
 	if d == nil {
@@ -150,11 +151,11 @@ func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) bal
 	if opt.CredsBundle != nil {
 		lb.grpclbClientConnCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBalancer)
 		if err != nil {
-			grpclog.Warningf("lbBalancer: client connection creds NewWithMode failed: %v", err)
+			logger.Warningf("lbBalancer: client connection creds NewWithMode failed: %v", err)
 		}
 		lb.grpclbBackendCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBackendFromBalancer)
 		if err != nil {
-			grpclog.Warningf("lbBalancer: backend creds NewWithMode failed: %v", err)
+			logger.Warningf("lbBalancer: backend creds NewWithMode failed: %v", err)
 		}
 	}
 
@@ -310,16 +311,16 @@ func (lb *lbBalancer) aggregateSubConnStates() connectivity.State {
 
 func (lb *lbBalancer) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) {
 	s := scs.ConnectivityState
-	if grpclog.V(2) {
-		grpclog.Infof("lbBalancer: handle SubConn state change: %p, %v", sc, s)
+	if logger.V(2) {
+		logger.Infof("lbBalancer: handle SubConn state change: %p, %v", sc, s)
 	}
 	lb.mu.Lock()
 	defer lb.mu.Unlock()
 
 	oldS, ok := lb.scStates[sc]
 	if !ok {
-		if grpclog.V(2) {
-			grpclog.Infof("lbBalancer: got state changes for an unknown SubConn: %p, %v", sc, s)
+		if logger.V(2) {
+			logger.Infof("lbBalancer: got state changes for an unknown SubConn: %p, %v", sc, s)
 		}
 		return
 	}
@@ -393,8 +394,8 @@ func (lb *lbBalancer) handleServiceConfig(gc *grpclbServiceConfig) {
 	if lb.usePickFirst == newUsePickFirst {
 		return
 	}
-	if grpclog.V(2) {
-		grpclog.Infof("lbBalancer: switching mode, new usePickFirst: %+v", newUsePickFirst)
+	if logger.V(2) {
+		logger.Infof("lbBalancer: switching mode, new usePickFirst: %+v", newUsePickFirst)
 	}
 	lb.refreshSubConns(lb.backendAddrs, lb.inFallback, newUsePickFirst)
 }
@@ -405,8 +406,8 @@ func (lb *lbBalancer) ResolverError(error) {
 }
 
 func (lb *lbBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error {
-	if grpclog.V(2) {
-		grpclog.Infof("lbBalancer: UpdateClientConnState: %+v", ccs)
+	if logger.V(2) {
+		logger.Infof("lbBalancer: UpdateClientConnState: %+v", ccs)
 	}
 	gc, _ := ccs.BalancerConfig.(*grpclbServiceConfig)
 	lb.handleServiceConfig(gc)
diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index 302d71316d59..8eb45be28e32 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -33,7 +33,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
 	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/keepalive"
@@ -44,8 +43,8 @@ import (
 // processServerList updates balancer's internal state, create/remove SubConns
 // and regenerates picker using the received serverList.
 func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
-	if grpclog.V(2) {
-		grpclog.Infof("lbBalancer: processing server list: %+v", l)
+	if logger.V(2) {
+		logger.Infof("lbBalancer: processing server list: %+v", l)
 	}
 	lb.mu.Lock()
 	defer lb.mu.Unlock()
@@ -56,8 +55,8 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
 
 	// If the new server list == old server list, do nothing.
 	if cmp.Equal(lb.fullServerList, l.Servers, cmp.Comparer(proto.Equal)) {
-		if grpclog.V(2) {
-			grpclog.Infof("lbBalancer: new serverlist same as the previous one, ignoring")
+		if logger.V(2) {
+			logger.Infof("lbBalancer: new serverlist same as the previous one, ignoring")
 		}
 		return
 	}
@@ -81,8 +80,8 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
 			Addr:     fmt.Sprintf("%s:%d", ipStr, s.Port),
 			Metadata: &md,
 		}
-		if grpclog.V(2) {
-			grpclog.Infof("lbBalancer: server list entry[%d]: ipStr:|%s|, port:|%d|, load balancer token:|%v|",
+		if logger.V(2) {
+			logger.Infof("lbBalancer: server list entry[%d]: ipStr:|%s|, port:|%d|, load balancer token:|%v|",
 				i, ipStr, s.Port, s.LoadBalanceToken)
 		}
 		backendAddrs = append(backendAddrs, addr)
@@ -150,7 +149,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback
 		// This bypasses the cc wrapper with SubConn cache.
 		sc, err := lb.cc.cc.NewSubConn(backendAddrs, opts)
 		if err != nil {
-			grpclog.Warningf("grpclb: failed to create new SubConn: %v", err)
+			logger.Warningf("grpclb: failed to create new SubConn: %v", err)
 			return
 		}
 		sc.Connect()
@@ -173,7 +172,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback
 			// Use addrWithMD to create the SubConn.
 			sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts)
 			if err != nil {
-				grpclog.Warningf("grpclb: failed to create new SubConn: %v", err)
+				logger.Warningf("grpclb: failed to create new SubConn: %v", err)
 				continue
 			}
 			lb.subConns[addrWithoutMD] = sc // Use the addr without MD as key for the map.
@@ -245,7 +244,7 @@ func (lb *lbBalancer) newRemoteBalancerCCWrapper() {
 	// receive ServerName as authority.
 	cc, err := grpc.DialContext(context.Background(), lb.manualResolver.Scheme()+":///grpclb.subClientConn", dopts...)
 	if err != nil {
-		grpclog.Fatalf("failed to dial: %v", err)
+		logger.Fatalf("failed to dial: %v", err)
 	}
 	ccw := &remoteBalancerCCWrapper{
 		cc:      cc,
@@ -373,9 +372,9 @@ func (ccw *remoteBalancerCCWrapper) watchRemoteBalancer() {
 		default:
 			if err != nil {
 				if err == errServerTerminatedConnection {
-					grpclog.Info(err)
+					logger.Info(err)
 				} else {
-					grpclog.Warning(err)
+					logger.Warning(err)
 				}
 			}
 		}
diff --git a/balancer/rls/internal/balancer.go b/balancer/rls/internal/balancer.go
index 2b8162485047..968e8e9310da 100644
--- a/balancer/rls/internal/balancer.go
+++ b/balancer/rls/internal/balancer.go
@@ -34,6 +34,7 @@ var (
 
 	// For overriding in tests.
 	newRLSClientFunc = newRLSClient
+	logger           = grpclog.Component("rls")
 )
 
 // rlsBalancer implements the RLS LB policy.
@@ -75,18 +76,18 @@ func (lb *rlsBalancer) run() {
 // channel accordingly.
 // TODO(easwars): Handle updates to other fields in the service config.
 func (lb *rlsBalancer) handleClientConnUpdate(ccs *balancer.ClientConnState) {
-	grpclog.Infof("rls: service config: %+v", ccs.BalancerConfig)
+	logger.Infof("rls: service config: %+v", ccs.BalancerConfig)
 	lb.mu.Lock()
 	defer lb.mu.Unlock()
 
 	if lb.done.HasFired() {
-		grpclog.Warning("rls: received service config after balancer close")
+		logger.Warning("rls: received service config after balancer close")
 		return
 	}
 
 	newCfg := ccs.BalancerConfig.(*lbConfig)
 	if lb.lbCfg.Equal(newCfg) {
-		grpclog.Info("rls: new service config matches existing config")
+		logger.Info("rls: new service config matches existing config")
 		return
 	}
 
@@ -109,12 +110,12 @@ func (lb *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error
 func (lb *rlsBalancer) ResolverError(error) {
 	// ResolverError is called by gRPC when the name resolver reports an error.
 	// TODO(easwars): How do we handle this?
-	grpclog.Fatal("rls: ResolverError is not yet unimplemented")
+	logger.Fatal("rls: ResolverError is not yet unimplemented")
 }
 
 // UpdateSubConnState implements balancer.V2Balancer interface.
 func (lb *rlsBalancer) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConnState) {
-	grpclog.Fatal("rls: UpdateSubConnState is not yet implemented")
+	logger.Fatal("rls: UpdateSubConnState is not yet implemented")
 }
 
 // Cleans up the resources allocated by the LB policy including the clientConn
@@ -162,7 +163,7 @@ func (lb *rlsBalancer) updateControlChannel(newCfg *lbConfig) {
 
 	cc, err := grpc.Dial(newCfg.lookupService, dopts...)
 	if err != nil {
-		grpclog.Errorf("rls: dialRLS(%s, %v): %v", newCfg.lookupService, lb.opts, err)
+		logger.Errorf("rls: dialRLS(%s, %v): %v", newCfg.lookupService, lb.opts, err)
 		// An error from a non-blocking dial indicates something serious. We
 		// should continue to use the old control channel if one exists, and
 		// return so that the rest of the config updates can be processes.
@@ -185,14 +186,14 @@ func dialCreds(opts balancer.BuildOptions) grpc.DialOption {
 	switch {
 	case opts.DialCreds != nil:
 		if err := opts.DialCreds.OverrideServerName(server); err != nil {
-			grpclog.Warningf("rls: OverrideServerName(%s) = (%v), using Insecure", server, err)
+			logger.Warningf("rls: OverrideServerName(%s) = (%v), using Insecure", server, err)
 			return grpc.WithInsecure()
 		}
 		return grpc.WithTransportCredentials(opts.DialCreds)
 	case opts.CredsBundle != nil:
 		return grpc.WithTransportCredentials(opts.CredsBundle.TransportCredentials())
 	default:
-		grpclog.Warning("rls: no credentials available, using Insecure")
+		logger.Warning("rls: no credentials available, using Insecure")
 		return grpc.WithInsecure()
 	}
 }
diff --git a/balancer/rls/internal/cache/cache.go b/balancer/rls/internal/cache/cache.go
index dd03695e0e9d..b975c3078fdb 100644
--- a/balancer/rls/internal/cache/cache.go
+++ b/balancer/rls/internal/cache/cache.go
@@ -30,6 +30,8 @@ import (
 	"google.golang.org/grpc/internal/backoff"
 )
 
+var logger = grpclog.Component("rls")
+
 // Key represents the cache key used to uniquely identify a cache entry.
 type Key struct {
 	// Path is the full path of the incoming RPC request.
@@ -175,7 +177,7 @@ func (lru *LRU) removeToFit(newSize int64) {
 		if elem == nil {
 			// This is a corner case where the cache is empty, but the new entry
 			// to be added is bigger than maxSize.
-			grpclog.Info("rls: newly added cache entry exceeds cache maxSize")
+			logger.Info("rls: newly added cache entry exceeds cache maxSize")
 			return
 		}
 
@@ -184,7 +186,7 @@ func (lru *LRU) removeToFit(newSize int64) {
 			// When the oldest entry is too new (it hasn't even spent a default
 			// minimum amount of time in the cache), we abort and allow the
 			// cache to grow bigger than the configured maxSize.
-			grpclog.Info("rls: LRU eviction finds oldest entry to be too new. Allowing cache to exceed maxSize momentarily")
+			logger.Info("rls: LRU eviction finds oldest entry to be too new. Allowing cache to exceed maxSize momentarily")
 			return
 		}
 		lru.removeElement(elem)
diff --git a/balancer/rls/internal/config.go b/balancer/rls/internal/config.go
index 816ab093a650..0a8d2d91fa82 100644
--- a/balancer/rls/internal/config.go
+++ b/balancer/rls/internal/config.go
@@ -32,7 +32,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/rls/internal/keys"
 	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
@@ -238,11 +237,11 @@ func (*rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 		return nil, fmt.Errorf("rls: stale_age is set, but max_age is not in service config {%+v}", string(c))
 	}
 	if staleAge >= maxAge {
-		grpclog.Info("rls: stale_age {%v} is greater than max_age {%v}, ignoring it", staleAge, maxAge)
+		logger.Info("rls: stale_age {%v} is greater than max_age {%v}, ignoring it", staleAge, maxAge)
 		staleAge = 0
 	}
 	if maxAge == 0 || maxAge > maxMaxAge {
-		grpclog.Infof("rls: max_age in service config is %v, using %v", maxAge, maxMaxAge)
+		logger.Infof("rls: max_age in service config is %v, using %v", maxAge, maxMaxAge)
 		maxAge = maxMaxAge
 	}
 
diff --git a/balancer/roundrobin/roundrobin.go b/balancer/roundrobin/roundrobin.go
index a02b372cf204..43c2a15373a1 100644
--- a/balancer/roundrobin/roundrobin.go
+++ b/balancer/roundrobin/roundrobin.go
@@ -33,6 +33,8 @@ import (
 // Name is the name of round_robin balancer.
 const Name = "round_robin"
 
+var logger = grpclog.Component("roundrobin")
+
 // newBuilder creates a new roundrobin balancer builder.
 func newBuilder() balancer.Builder {
 	return base.NewBalancerBuilder(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true})
@@ -45,7 +47,7 @@ func init() {
 type rrPickerBuilder struct{}
 
 func (*rrPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker {
-	grpclog.Infof("roundrobinPicker: newPicker called with info: %v", info)
+	logger.Infof("roundrobinPicker: newPicker called with info: %v", info)
 	if len(info.ReadySCs) == 0 {
 		return base.NewErrPicker(balancer.ErrNoSubConnAvailable)
 	}
diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go
index 807d1919777e..11e592aabb01 100644
--- a/balancer_conn_wrappers.go
+++ b/balancer_conn_wrappers.go
@@ -220,7 +220,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) {
 
 		ac, err := cc.newAddrConn(addrs, opts)
 		if err != nil {
-			channelz.Warningf(acbw.ac.channelzID, "acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err)
+			channelz.Warningf(logger, acbw.ac.channelzID, "acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err)
 			return
 		}
 		acbw.ac = ac
diff --git a/channelz/service/service.go b/channelz/service/service.go
index 702b74c03e60..4d175fef823d 100644
--- a/channelz/service/service.go
+++ b/channelz/service/service.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/status"
 )
@@ -39,6 +40,8 @@ func init() {
 	channelz.TurnOn()
 }
 
+var logger = grpclog.Component("channelz")
+
 // RegisterChannelzServiceToServer registers the channelz service to the given server.
 func RegisterChannelzServiceToServer(s *grpc.Server) {
 	channelzgrpc.RegisterChannelzServer(s, newCZServer())
diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go
index 37616a101ce8..e6a7d8eba3be 100644
--- a/channelz/service/service_test.go
+++ b/channelz/service/service_test.go
@@ -468,12 +468,12 @@ func (s) TestGetChannel(t *testing.T) {
 	refNames := []string{"top channel 1", "nested channel 1", "sub channel 2", "nested channel 3"}
 	ids := make([]int64, 4)
 	ids[0] = channelz.RegisterChannel(&dummyChannel{}, 0, refNames[0])
-	channelz.AddTraceEvent(ids[0], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
 		Severity: channelz.CtINFO,
 	})
 	ids[1] = channelz.RegisterChannel(&dummyChannel{}, ids[0], refNames[1])
-	channelz.AddTraceEvent(ids[1], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
 		Severity: channelz.CtINFO,
 		Parent: &channelz.TraceEventDesc{
@@ -483,7 +483,7 @@ func (s) TestGetChannel(t *testing.T) {
 	})
 
 	ids[2] = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[2])
-	channelz.AddTraceEvent(ids[2], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[2], 0, &channelz.TraceEventDesc{
 		Desc:     "SubChannel Created",
 		Severity: channelz.CtINFO,
 		Parent: &channelz.TraceEventDesc{
@@ -492,7 +492,7 @@ func (s) TestGetChannel(t *testing.T) {
 		},
 	})
 	ids[3] = channelz.RegisterChannel(&dummyChannel{}, ids[1], refNames[3])
-	channelz.AddTraceEvent(ids[3], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[3], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
 		Severity: channelz.CtINFO,
 		Parent: &channelz.TraceEventDesc{
@@ -500,11 +500,11 @@ func (s) TestGetChannel(t *testing.T) {
 			Severity: channelz.CtINFO,
 		},
 	})
-	channelz.AddTraceEvent(ids[0], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready),
 		Severity: channelz.CtINFO,
 	})
-	channelz.AddTraceEvent(ids[0], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     "Resolver returns an empty address list",
 		Severity: channelz.CtWarning,
 	})
@@ -571,12 +571,12 @@ func (s) TestGetSubChannel(t *testing.T) {
 	refNames := []string{"top channel 1", "sub channel 1", "socket 1", "socket 2"}
 	ids := make([]int64, 4)
 	ids[0] = channelz.RegisterChannel(&dummyChannel{}, 0, refNames[0])
-	channelz.AddTraceEvent(ids[0], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
 		Severity: channelz.CtINFO,
 	})
 	ids[1] = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[1])
-	channelz.AddTraceEvent(ids[1], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     subchanCreated,
 		Severity: channelz.CtINFO,
 		Parent: &channelz.TraceEventDesc{
@@ -586,11 +586,11 @@ func (s) TestGetSubChannel(t *testing.T) {
 	})
 	ids[2] = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[2])
 	ids[3] = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[3])
-	channelz.AddTraceEvent(ids[1], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     subchanConnectivityChange,
 		Severity: channelz.CtINFO,
 	})
-	channelz.AddTraceEvent(ids[1], 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     subChanPickNewAddress,
 		Severity: channelz.CtINFO,
 	})
diff --git a/clientconn.go b/clientconn.go
index ef327e8af4f7..3ed6034f1ad1 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -149,7 +149,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 	if channelz.IsOn() {
 		if cc.dopts.channelzParentID != 0 {
 			cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target)
-			channelz.AddTraceEvent(cc.channelzID, 0, &channelz.TraceEventDesc{
+			channelz.AddTraceEvent(logger, cc.channelzID, 0, &channelz.TraceEventDesc{
 				Desc:     "Channel Created",
 				Severity: channelz.CtINFO,
 				Parent: &channelz.TraceEventDesc{
@@ -159,7 +159,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 			})
 		} else {
 			cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, 0, target)
-			channelz.Info(cc.channelzID, "Channel Created")
+			channelz.Info(logger, cc.channelzID, "Channel Created")
 		}
 		cc.csMgr.channelzID = cc.channelzID
 	}
@@ -245,13 +245,13 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 
 	// Determine the resolver to use.
 	cc.parsedTarget = grpcutil.ParseTarget(cc.target)
-	channelz.Infof(cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme)
+	channelz.Infof(logger, cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme)
 	resolverBuilder := cc.getResolver(cc.parsedTarget.Scheme)
 	if resolverBuilder == nil {
 		// If resolver builder is still nil, the parsed target's scheme is
 		// not registered. Fallback to default resolver and set Endpoint to
 		// the original target.
-		channelz.Infof(cc.channelzID, "scheme %q not registered, fallback to default scheme", cc.parsedTarget.Scheme)
+		channelz.Infof(logger, cc.channelzID, "scheme %q not registered, fallback to default scheme", cc.parsedTarget.Scheme)
 		cc.parsedTarget = resolver.Target{
 			Scheme:   resolver.GetDefaultScheme(),
 			Endpoint: target,
@@ -422,7 +422,7 @@ func (csm *connectivityStateManager) updateState(state connectivity.State) {
 		return
 	}
 	csm.state = state
-	channelz.Infof(csm.channelzID, "Channel Connectivity change to %v", state)
+	channelz.Infof(logger, csm.channelzID, "Channel Connectivity change to %v", state)
 	if csm.notifyChan != nil {
 		// There are other goroutines waiting on this channel.
 		close(csm.notifyChan)
@@ -675,9 +675,9 @@ func (cc *ClientConn) switchBalancer(name string) {
 		return
 	}
 
-	channelz.Infof(cc.channelzID, "ClientConn switching balancer to %q", name)
+	channelz.Infof(logger, cc.channelzID, "ClientConn switching balancer to %q", name)
 	if cc.dopts.balancerBuilder != nil {
-		channelz.Info(cc.channelzID, "ignoring balancer switching: Balancer DialOption used instead")
+		channelz.Info(logger, cc.channelzID, "ignoring balancer switching: Balancer DialOption used instead")
 		return
 	}
 	if cc.balancerWrapper != nil {
@@ -686,11 +686,11 @@ func (cc *ClientConn) switchBalancer(name string) {
 
 	builder := balancer.Get(name)
 	if builder == nil {
-		channelz.Warningf(cc.channelzID, "Channel switches to new LB policy %q due to fallback from invalid balancer name", PickFirstBalancerName)
-		channelz.Infof(cc.channelzID, "failed to get balancer builder for: %v, using pick_first instead", name)
+		channelz.Warningf(logger, cc.channelzID, "Channel switches to new LB policy %q due to fallback from invalid balancer name", PickFirstBalancerName)
+		channelz.Infof(logger, cc.channelzID, "failed to get balancer builder for: %v, using pick_first instead", name)
 		builder = newPickfirstBuilder()
 	} else {
-		channelz.Infof(cc.channelzID, "Channel switches to new LB policy %q", name)
+		channelz.Infof(logger, cc.channelzID, "Channel switches to new LB policy %q", name)
 	}
 
 	cc.curBalancerName = builder.Name()
@@ -731,7 +731,7 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSub
 	}
 	if channelz.IsOn() {
 		ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "")
-		channelz.AddTraceEvent(ac.channelzID, 0, &channelz.TraceEventDesc{
+		channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{
 			Desc:     "Subchannel Created",
 			Severity: channelz.CtINFO,
 			Parent: &channelz.TraceEventDesc{
@@ -829,7 +829,7 @@ func (ac *addrConn) connect() error {
 func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
-	channelz.Infof(ac.channelzID, "addrConn: tryUpdateAddrs curAddr: %v, addrs: %v", ac.curAddr, addrs)
+	channelz.Infof(logger, ac.channelzID, "addrConn: tryUpdateAddrs curAddr: %v, addrs: %v", ac.curAddr, addrs)
 	if ac.state == connectivity.Shutdown ||
 		ac.state == connectivity.TransientFailure ||
 		ac.state == connectivity.Idle {
@@ -849,7 +849,7 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool {
 			break
 		}
 	}
-	channelz.Infof(ac.channelzID, "addrConn: tryUpdateAddrs curAddrFound: %v", curAddrFound)
+	channelz.Infof(logger, ac.channelzID, "addrConn: tryUpdateAddrs curAddrFound: %v", curAddrFound)
 	if curAddrFound {
 		ac.addrs = addrs
 	}
@@ -1020,7 +1020,7 @@ func (cc *ClientConn) Close() error {
 				Severity: channelz.CtINFO,
 			}
 		}
-		channelz.AddTraceEvent(cc.channelzID, 0, ted)
+		channelz.AddTraceEvent(logger, cc.channelzID, 0, ted)
 		// TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to
 		// the entity being deleted, and thus prevent it from being deleted right away.
 		channelz.RemoveEntry(cc.channelzID)
@@ -1064,7 +1064,7 @@ func (ac *addrConn) updateConnectivityState(s connectivity.State, lastErr error)
 		return
 	}
 	ac.state = s
-	channelz.Infof(ac.channelzID, "Subchannel Connectivity change to %v", s)
+	channelz.Infof(logger, ac.channelzID, "Subchannel Connectivity change to %v", s)
 	ac.cc.handleSubConnStateChange(ac.acbw, s, lastErr)
 }
 
@@ -1201,7 +1201,7 @@ func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.T
 		}
 		ac.mu.Unlock()
 
-		channelz.Infof(ac.channelzID, "Subchannel picks a new address %q to connect", addr.Addr)
+		channelz.Infof(logger, ac.channelzID, "Subchannel picks a new address %q to connect", addr.Addr)
 
 		newTr, reconnect, err := ac.createTransport(addr, copts, connectDeadline)
 		if err == nil {
@@ -1276,7 +1276,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 	newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, addr, copts, onPrefaceReceipt, onGoAway, onClose)
 	if err != nil {
 		// newTr is either nil, or closed.
-		channelz.Warningf(ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v. Err: %v. Reconnecting...", addr, err)
+		channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v. Err: %v. Reconnecting...", addr, err)
 		return nil, nil, err
 	}
 
@@ -1284,7 +1284,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 	case <-time.After(time.Until(connectDeadline)):
 		// We didn't get the preface in time.
 		newTr.Close()
-		channelz.Warningf(ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr)
+		channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr)
 		return nil, nil, errors.New("timed out waiting for server handshake")
 	case <-prefaceReceived:
 		// We got the preface - huzzah! things are good.
@@ -1331,7 +1331,7 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) {
 		// The health package is not imported to set health check function.
 		//
 		// TODO: add a link to the health check doc in the error message.
-		channelz.Error(ac.channelzID, "Health check is requested but health check function is not set.")
+		channelz.Error(logger, ac.channelzID, "Health check is requested but health check function is not set.")
 		return
 	}
 
@@ -1361,9 +1361,9 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) {
 		err := ac.cc.dopts.healthCheckFunc(ctx, newStream, setConnectivityState, healthCheckConfig.ServiceName)
 		if err != nil {
 			if status.Code(err) == codes.Unimplemented {
-				channelz.Error(ac.channelzID, "Subchannel health check is unimplemented at server side, thus health check is disabled")
+				channelz.Error(logger, ac.channelzID, "Subchannel health check is unimplemented at server side, thus health check is disabled")
 			} else {
-				channelz.Errorf(ac.channelzID, "HealthCheckFunc exits with unexpected error %v", err)
+				channelz.Errorf(logger, ac.channelzID, "HealthCheckFunc exits with unexpected error %v", err)
 			}
 		}
 	}()
@@ -1428,7 +1428,7 @@ func (ac *addrConn) tearDown(err error) {
 		ac.mu.Lock()
 	}
 	if channelz.IsOn() {
-		channelz.AddTraceEvent(ac.channelzID, 0, &channelz.TraceEventDesc{
+		channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{
 			Desc:     "Subchannel Deleted",
 			Severity: channelz.CtINFO,
 			Parent: &channelz.TraceEventDesc{
diff --git a/credentials/alts/alts.go b/credentials/alts/alts.go
index 5c9e8b1c471b..729c4b43b5fc 100644
--- a/credentials/alts/alts.go
+++ b/credentials/alts/alts.go
@@ -67,6 +67,7 @@ var (
 	// ServerHandshake is running on a platform where the trustworthiness of
 	// the handshaker service is not guaranteed.
 	ErrUntrustedPlatform = errors.New("ALTS: untrusted platform. ALTS is only supported on GCP")
+	logger               = grpclog.Component("alts")
 )
 
 // AuthInfo exposes security information from the ALTS handshake to the
@@ -307,7 +308,7 @@ func compareRPCVersions(v1, v2 *altspb.RpcProtocolVersions_Version) int {
 // agreed on.
 func checkRPCVersions(local, peer *altspb.RpcProtocolVersions) (bool, *altspb.RpcProtocolVersions_Version) {
 	if local == nil || peer == nil {
-		grpclog.Error("invalid checkRPCVersions argument, either local or peer is nil.")
+		logger.Error("invalid checkRPCVersions argument, either local or peer is nil.")
 		return false, nil
 	}
 
diff --git a/dialoptions.go b/dialoptions.go
index d5030c076178..decb4c5ee891 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -27,7 +27,6 @@ import (
 	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
 	internalbackoff "google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/envconfig"
@@ -423,7 +422,7 @@ func WithUserAgent(s string) DialOption {
 // for the client transport.
 func WithKeepaliveParams(kp keepalive.ClientParameters) DialOption {
 	if kp.Time < internal.KeepaliveMinPingTime {
-		grpclog.Warningf("Adjusting keepalive ping interval to minimum period of %v", internal.KeepaliveMinPingTime)
+		logger.Warningf("Adjusting keepalive ping interval to minimum period of %v", internal.KeepaliveMinPingTime)
 		kp.Time = internal.KeepaliveMinPingTime
 	}
 	return newFuncDialOption(func(o *dialOptions) {
diff --git a/grpclog/component.go b/grpclog/component.go
new file mode 100644
index 000000000000..b513281a34ce
--- /dev/null
+++ b/grpclog/component.go
@@ -0,0 +1,117 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpclog
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/internal/grpclog"
+)
+
+// componentData records the settings for a component.
+type componentData struct {
+	name string
+}
+
+var cache = map[string]*componentData{}
+
+func (c *componentData) InfoDepth(depth int, args ...interface{}) {
+	args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
+	grpclog.InfoDepth(depth+1, args...)
+}
+
+func (c *componentData) WarningDepth(depth int, args ...interface{}) {
+	args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
+	grpclog.WarningDepth(depth+1, args...)
+}
+
+func (c *componentData) ErrorDepth(depth int, args ...interface{}) {
+	args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
+	grpclog.ErrorDepth(depth+1, args...)
+}
+
+func (c *componentData) FatalDepth(depth int, args ...interface{}) {
+	args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
+	grpclog.FatalDepth(depth+1, args...)
+}
+
+func (c *componentData) Info(args ...interface{}) {
+	c.InfoDepth(1, args...)
+}
+
+func (c *componentData) Warning(args ...interface{}) {
+	c.WarningDepth(1, args...)
+}
+
+func (c *componentData) Error(args ...interface{}) {
+	c.ErrorDepth(1, args...)
+}
+
+func (c *componentData) Fatal(args ...interface{}) {
+	c.FatalDepth(1, args...)
+}
+
+func (c *componentData) Infof(format string, args ...interface{}) {
+	c.InfoDepth(1, fmt.Sprintf(format, args...))
+}
+
+func (c *componentData) Warningf(format string, args ...interface{}) {
+	c.WarningDepth(1, fmt.Sprintf(format, args...))
+}
+
+func (c *componentData) Errorf(format string, args ...interface{}) {
+	c.ErrorDepth(1, fmt.Sprintf(format, args...))
+}
+
+func (c *componentData) Fatalf(format string, args ...interface{}) {
+	c.FatalDepth(1, fmt.Sprintf(format, args...))
+}
+
+func (c *componentData) Infoln(args ...interface{}) {
+	c.InfoDepth(1, args...)
+}
+
+func (c *componentData) Warningln(args ...interface{}) {
+	c.WarningDepth(1, args...)
+}
+
+func (c *componentData) Errorln(args ...interface{}) {
+	c.ErrorDepth(1, args...)
+}
+
+func (c *componentData) Fatalln(args ...interface{}) {
+	c.FatalDepth(1, args...)
+}
+
+func (c *componentData) V(l int) bool {
+	return grpclog.Logger.V(l)
+}
+
+// Component creates a new component and returns it for logging. If a component
+// with the name already exists, nothing will be created and it will be
+// returned. SetLoggerV2 will panic if it is called with a logger created by
+// Component.
+func Component(componentName string) DepthLoggerV2 {
+	if cData, ok := cache[componentName]; ok {
+		return cData
+	}
+	c := &componentData{componentName}
+	cache[componentName] = c
+	return c
+}
diff --git a/grpclog/glogger/glogger.go b/grpclog/glogger/glogger.go
index 3d995c1c076f..4427dc078bb4 100644
--- a/grpclog/glogger/glogger.go
+++ b/grpclog/glogger/glogger.go
@@ -27,6 +27,8 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+const d = 2
+
 func init() {
 	grpclog.SetLoggerV2(&glogger{})
 }
@@ -34,67 +36,67 @@ func init() {
 type glogger struct{}
 
 func (g *glogger) Info(args ...interface{}) {
-	glog.InfoDepth(2, args...)
+	glog.InfoDepth(d, args...)
 }
 
 func (g *glogger) Infoln(args ...interface{}) {
-	glog.InfoDepth(2, fmt.Sprintln(args...))
+	glog.InfoDepth(d, fmt.Sprintln(args...))
 }
 
 func (g *glogger) Infof(format string, args ...interface{}) {
-	glog.InfoDepth(2, fmt.Sprintf(format, args...))
+	glog.InfoDepth(d, fmt.Sprintf(format, args...))
 }
 
 func (g *glogger) InfoDepth(depth int, args ...interface{}) {
-	glog.InfoDepth(depth+2, args...)
+	glog.InfoDepth(depth+d, args...)
 }
 
 func (g *glogger) Warning(args ...interface{}) {
-	glog.WarningDepth(2, args...)
+	glog.WarningDepth(d, args...)
 }
 
 func (g *glogger) Warningln(args ...interface{}) {
-	glog.WarningDepth(2, fmt.Sprintln(args...))
+	glog.WarningDepth(d, fmt.Sprintln(args...))
 }
 
 func (g *glogger) Warningf(format string, args ...interface{}) {
-	glog.WarningDepth(2, fmt.Sprintf(format, args...))
+	glog.WarningDepth(d, fmt.Sprintf(format, args...))
 }
 
 func (g *glogger) WarningDepth(depth int, args ...interface{}) {
-	glog.WarningDepth(depth+2, args...)
+	glog.WarningDepth(depth+d, args...)
 }
 
 func (g *glogger) Error(args ...interface{}) {
-	glog.ErrorDepth(2, args...)
+	glog.ErrorDepth(d, args...)
 }
 
 func (g *glogger) Errorln(args ...interface{}) {
-	glog.ErrorDepth(2, fmt.Sprintln(args...))
+	glog.ErrorDepth(d, fmt.Sprintln(args...))
 }
 
 func (g *glogger) Errorf(format string, args ...interface{}) {
-	glog.ErrorDepth(2, fmt.Sprintf(format, args...))
+	glog.ErrorDepth(d, fmt.Sprintf(format, args...))
 }
 
 func (g *glogger) ErrorDepth(depth int, args ...interface{}) {
-	glog.ErrorDepth(depth+2, args...)
+	glog.ErrorDepth(depth+d, args...)
 }
 
 func (g *glogger) Fatal(args ...interface{}) {
-	glog.FatalDepth(2, args...)
+	glog.FatalDepth(d, args...)
 }
 
 func (g *glogger) Fatalln(args ...interface{}) {
-	glog.FatalDepth(2, fmt.Sprintln(args...))
+	glog.FatalDepth(d, fmt.Sprintln(args...))
 }
 
 func (g *glogger) Fatalf(format string, args ...interface{}) {
-	glog.FatalDepth(2, fmt.Sprintf(format, args...))
+	glog.FatalDepth(d, fmt.Sprintf(format, args...))
 }
 
 func (g *glogger) FatalDepth(depth int, args ...interface{}) {
-	glog.FatalDepth(depth+2, args...)
+	glog.FatalDepth(depth+d, args...)
 }
 
 func (g *glogger) V(l int) bool {
diff --git a/grpclog/loggerv2.go b/grpclog/loggerv2.go
index 23612b7c41b5..8eba2d0e0eff 100644
--- a/grpclog/loggerv2.go
+++ b/grpclog/loggerv2.go
@@ -67,6 +67,9 @@ type LoggerV2 interface {
 // SetLoggerV2 sets logger that is used in grpc to a V2 logger.
 // Not mutex-protected, should be called before any gRPC functions.
 func SetLoggerV2(l LoggerV2) {
+	if _, ok := l.(*componentData); ok {
+		panic("cannot use component logger as grpclog logger")
+	}
 	grpclog.Logger = l
 	grpclog.DepthLogger, _ = l.(grpclog.DepthLoggerV2)
 }
@@ -203,6 +206,7 @@ func (g *loggerT) V(l int) bool {
 //
 // This API is EXPERIMENTAL.
 type DepthLoggerV2 interface {
+	LoggerV2
 	// InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Print.
 	InfoDepth(depth int, args ...interface{})
 	// WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Print.
diff --git a/internal/binarylog/binarylog.go b/internal/binarylog/binarylog.go
index b7a3dd8f918d..5cc3aeddb213 100644
--- a/internal/binarylog/binarylog.go
+++ b/internal/binarylog/binarylog.go
@@ -40,6 +40,8 @@ type Logger interface {
 // It is used to get a methodLogger for each individual method.
 var binLogger Logger
 
+var grpclogLogger = grpclog.Component("binarylog")
+
 // SetLogger sets the binarg logger.
 //
 // Only call this at init time.
@@ -149,7 +151,7 @@ func (l *logger) setBlacklist(method string) error {
 func (l *logger) getMethodLogger(methodName string) *MethodLogger {
 	s, m, err := grpcutil.ParseMethod(methodName)
 	if err != nil {
-		grpclog.Infof("binarylogging: failed to parse %q: %v", methodName, err)
+		grpclogLogger.Infof("binarylogging: failed to parse %q: %v", methodName, err)
 		return nil
 	}
 	if ml, ok := l.methods[s+"/"+m]; ok {
diff --git a/internal/binarylog/env_config.go b/internal/binarylog/env_config.go
index be30d0e65e70..d8f4e7602fde 100644
--- a/internal/binarylog/env_config.go
+++ b/internal/binarylog/env_config.go
@@ -24,8 +24,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-
-	"google.golang.org/grpc/grpclog"
 )
 
 // NewLoggerFromConfigString reads the string and build a logger. It can be used
@@ -52,7 +50,7 @@ func NewLoggerFromConfigString(s string) Logger {
 	methods := strings.Split(s, ",")
 	for _, method := range methods {
 		if err := l.fillMethodLoggerWithConfigString(method); err != nil {
-			grpclog.Warningf("failed to parse binary log config: %v", err)
+			grpclogLogger.Warningf("failed to parse binary log config: %v", err)
 			return nil
 		}
 	}
diff --git a/internal/binarylog/method_logger.go b/internal/binarylog/method_logger.go
index 160f6e8616f5..5e1083539b49 100644
--- a/internal/binarylog/method_logger.go
+++ b/internal/binarylog/method_logger.go
@@ -27,7 +27,6 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
 	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
@@ -219,12 +218,12 @@ func (c *ClientMessage) toProto() *pb.GrpcLogEntry {
 	if m, ok := c.Message.(proto.Message); ok {
 		data, err = proto.Marshal(m)
 		if err != nil {
-			grpclog.Infof("binarylogging: failed to marshal proto message: %v", err)
+			grpclogLogger.Infof("binarylogging: failed to marshal proto message: %v", err)
 		}
 	} else if b, ok := c.Message.([]byte); ok {
 		data = b
 	} else {
-		grpclog.Infof("binarylogging: message to log is neither proto.message nor []byte")
+		grpclogLogger.Infof("binarylogging: message to log is neither proto.message nor []byte")
 	}
 	ret := &pb.GrpcLogEntry{
 		Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE,
@@ -259,12 +258,12 @@ func (c *ServerMessage) toProto() *pb.GrpcLogEntry {
 	if m, ok := c.Message.(proto.Message); ok {
 		data, err = proto.Marshal(m)
 		if err != nil {
-			grpclog.Infof("binarylogging: failed to marshal proto message: %v", err)
+			grpclogLogger.Infof("binarylogging: failed to marshal proto message: %v", err)
 		}
 	} else if b, ok := c.Message.([]byte); ok {
 		data = b
 	} else {
-		grpclog.Infof("binarylogging: message to log is neither proto.message nor []byte")
+		grpclogLogger.Infof("binarylogging: message to log is neither proto.message nor []byte")
 	}
 	ret := &pb.GrpcLogEntry{
 		Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE,
@@ -315,7 +314,7 @@ type ServerTrailer struct {
 func (c *ServerTrailer) toProto() *pb.GrpcLogEntry {
 	st, ok := status.FromError(c.Err)
 	if !ok {
-		grpclog.Info("binarylogging: error in trailer is not a status error")
+		grpclogLogger.Info("binarylogging: error in trailer is not a status error")
 	}
 	var (
 		detailsBytes []byte
@@ -325,7 +324,7 @@ func (c *ServerTrailer) toProto() *pb.GrpcLogEntry {
 	if stProto != nil && len(stProto.Details) != 0 {
 		detailsBytes, err = proto.Marshal(stProto)
 		if err != nil {
-			grpclog.Infof("binarylogging: failed to marshal status proto: %v", err)
+			grpclogLogger.Infof("binarylogging: failed to marshal status proto: %v", err)
 		}
 	}
 	ret := &pb.GrpcLogEntry{
diff --git a/internal/binarylog/sink.go b/internal/binarylog/sink.go
index a2e7c346dd03..835f51040cb0 100644
--- a/internal/binarylog/sink.go
+++ b/internal/binarylog/sink.go
@@ -29,7 +29,6 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
-	"google.golang.org/grpc/grpclog"
 )
 
 var (
@@ -78,7 +77,7 @@ type writerSink struct {
 func (ws *writerSink) Write(e *pb.GrpcLogEntry) error {
 	b, err := proto.Marshal(e)
 	if err != nil {
-		grpclog.Infof("binary logging: failed to marshal proto message: %v", err)
+		grpclogLogger.Infof("binary logging: failed to marshal proto message: %v", err)
 	}
 	hdr := make([]byte, 4)
 	binary.BigEndian.PutUint32(hdr, uint32(len(b)))
diff --git a/internal/channelz/funcs.go b/internal/channelz/funcs.go
index e4252e5be9f2..81d3dd33e62c 100644
--- a/internal/channelz/funcs.go
+++ b/internal/channelz/funcs.go
@@ -30,7 +30,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/grpclog"
 )
 
 const (
@@ -216,7 +216,7 @@ func RegisterChannel(c Channel, pid int64, ref string) int64 {
 // by pid). It returns the unique channelz tracking id assigned to this subchannel.
 func RegisterSubChannel(c Channel, pid int64, ref string) int64 {
 	if pid == 0 {
-		grpclog.ErrorDepth(0, "a SubChannel's parent id cannot be 0")
+		logger.Error("a SubChannel's parent id cannot be 0")
 		return 0
 	}
 	id := idGen.genID()
@@ -253,7 +253,7 @@ func RegisterServer(s Server, ref string) int64 {
 // this listen socket.
 func RegisterListenSocket(s Socket, pid int64, ref string) int64 {
 	if pid == 0 {
-		grpclog.ErrorDepth(0, "a ListenSocket's parent id cannot be 0")
+		logger.Error("a ListenSocket's parent id cannot be 0")
 		return 0
 	}
 	id := idGen.genID()
@@ -268,7 +268,7 @@ func RegisterListenSocket(s Socket, pid int64, ref string) int64 {
 // this normal socket.
 func RegisterNormalSocket(s Socket, pid int64, ref string) int64 {
 	if pid == 0 {
-		grpclog.ErrorDepth(0, "a NormalSocket's parent id cannot be 0")
+		logger.Error("a NormalSocket's parent id cannot be 0")
 		return 0
 	}
 	id := idGen.genID()
@@ -294,17 +294,17 @@ type TraceEventDesc struct {
 }
 
 // AddTraceEvent adds trace related to the entity with specified id, using the provided TraceEventDesc.
-func AddTraceEvent(id int64, depth int, desc *TraceEventDesc) {
+func AddTraceEvent(l grpclog.DepthLoggerV2, id int64, depth int, desc *TraceEventDesc) {
 	for d := desc; d != nil; d = d.Parent {
 		switch d.Severity {
 		case CtUNKNOWN:
-			grpclog.InfoDepth(depth+1, d.Desc)
+			l.InfoDepth(depth+1, d.Desc)
 		case CtINFO:
-			grpclog.InfoDepth(depth+1, d.Desc)
+			l.InfoDepth(depth+1, d.Desc)
 		case CtWarning:
-			grpclog.WarningDepth(depth+1, d.Desc)
+			l.WarningDepth(depth+1, d.Desc)
 		case CtError:
-			grpclog.ErrorDepth(depth+1, d.Desc)
+			l.ErrorDepth(depth+1, d.Desc)
 		}
 	}
 	if getMaxTraceEntry() == 0 {
diff --git a/internal/channelz/logging.go b/internal/channelz/logging.go
index 59c7bedecd9f..e94039ee20b5 100644
--- a/internal/channelz/logging.go
+++ b/internal/channelz/logging.go
@@ -21,80 +21,82 @@ package channelz
 import (
 	"fmt"
 
-	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/grpclog"
 )
 
-// Info logs through grpclog.Info and adds a trace event if channelz is on.
-func Info(id int64, args ...interface{}) {
+var logger = grpclog.Component("channelz")
+
+// Info logs and adds a trace event if channelz is on.
+func Info(l grpclog.DepthLoggerV2, id int64, args ...interface{}) {
 	if IsOn() {
-		AddTraceEvent(id, 1, &TraceEventDesc{
+		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     fmt.Sprint(args...),
 			Severity: CtINFO,
 		})
 	} else {
-		grpclog.InfoDepth(1, args...)
+		l.InfoDepth(1, args...)
 	}
 }
 
-// Infof logs through grpclog.Infof and adds a trace event if channelz is on.
-func Infof(id int64, format string, args ...interface{}) {
+// Infof logs and adds a trace event if channelz is on.
+func Infof(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
 	if IsOn() {
-		AddTraceEvent(id, 1, &TraceEventDesc{
+		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     msg,
 			Severity: CtINFO,
 		})
 	} else {
-		grpclog.InfoDepth(1, msg)
+		l.InfoDepth(1, msg)
 	}
 }
 
-// Warning logs through grpclog.Warning and adds a trace event if channelz is on.
-func Warning(id int64, args ...interface{}) {
+// Warning logs and adds a trace event if channelz is on.
+func Warning(l grpclog.DepthLoggerV2, id int64, args ...interface{}) {
 	if IsOn() {
-		AddTraceEvent(id, 1, &TraceEventDesc{
+		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     fmt.Sprint(args...),
 			Severity: CtWarning,
 		})
 	} else {
-		grpclog.WarningDepth(1, args...)
+		l.WarningDepth(1, args...)
 	}
 }
 
-// Warningf logs through grpclog.Warningf and adds a trace event if channelz is on.
-func Warningf(id int64, format string, args ...interface{}) {
+// Warningf logs and adds a trace event if channelz is on.
+func Warningf(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
 	if IsOn() {
-		AddTraceEvent(id, 1, &TraceEventDesc{
+		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     msg,
 			Severity: CtWarning,
 		})
 	} else {
-		grpclog.WarningDepth(1, msg)
+		l.WarningDepth(1, msg)
 	}
 }
 
-// Error logs through grpclog.Error and adds a trace event if channelz is on.
-func Error(id int64, args ...interface{}) {
+// Error logs and adds a trace event if channelz is on.
+func Error(l grpclog.DepthLoggerV2, id int64, args ...interface{}) {
 	if IsOn() {
-		AddTraceEvent(id, 1, &TraceEventDesc{
+		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     fmt.Sprint(args...),
 			Severity: CtError,
 		})
 	} else {
-		grpclog.ErrorDepth(1, args...)
+		l.ErrorDepth(1, args...)
 	}
 }
 
-// Errorf logs through grpclog.Errorf and adds a trace event if channelz is on.
-func Errorf(id int64, format string, args ...interface{}) {
+// Errorf logs and adds a trace event if channelz is on.
+func Errorf(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
 	if IsOn() {
-		AddTraceEvent(id, 1, &TraceEventDesc{
+		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     msg,
 			Severity: CtError,
 		})
 	} else {
-		grpclog.ErrorDepth(1, msg)
+		l.ErrorDepth(1, msg)
 	}
 }
diff --git a/internal/channelz/types.go b/internal/channelz/types.go
index 17c2274cb3de..075dc7d16714 100644
--- a/internal/channelz/types.go
+++ b/internal/channelz/types.go
@@ -26,7 +26,6 @@ import (
 
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
 )
 
 // entry represents a node in the channelz database.
@@ -60,17 +59,17 @@ func (d *dummyEntry) addChild(id int64, e entry) {
 	// the addrConn will create a new transport. And when registering the new transport in
 	// channelz, its parent addrConn could have already been torn down and deleted
 	// from channelz tracking, and thus reach the code here.
-	grpclog.Infof("attempt to add child of type %T with id %d to a parent (id=%d) that doesn't currently exist", e, id, d.idNotFound)
+	logger.Infof("attempt to add child of type %T with id %d to a parent (id=%d) that doesn't currently exist", e, id, d.idNotFound)
 }
 
 func (d *dummyEntry) deleteChild(id int64) {
 	// It is possible for a normal program to reach here under race condition.
 	// Refer to the example described in addChild().
-	grpclog.Infof("attempt to delete child with id %d from a parent (id=%d) that doesn't currently exist", id, d.idNotFound)
+	logger.Infof("attempt to delete child with id %d from a parent (id=%d) that doesn't currently exist", id, d.idNotFound)
 }
 
 func (d *dummyEntry) triggerDelete() {
-	grpclog.Warningf("attempt to delete an entry (id=%d) that doesn't currently exist", d.idNotFound)
+	logger.Warningf("attempt to delete an entry (id=%d) that doesn't currently exist", d.idNotFound)
 }
 
 func (*dummyEntry) deleteSelfIfReady() {
@@ -215,7 +214,7 @@ func (c *channel) addChild(id int64, e entry) {
 	case *channel:
 		c.nestedChans[id] = v.refName
 	default:
-		grpclog.Errorf("cannot add a child (id = %d) of type %T to a channel", id, e)
+		logger.Errorf("cannot add a child (id = %d) of type %T to a channel", id, e)
 	}
 }
 
@@ -326,7 +325,7 @@ func (sc *subChannel) addChild(id int64, e entry) {
 	if v, ok := e.(*normalSocket); ok {
 		sc.sockets[id] = v.refName
 	} else {
-		grpclog.Errorf("cannot add a child (id = %d) of type %T to a subChannel", id, e)
+		logger.Errorf("cannot add a child (id = %d) of type %T to a subChannel", id, e)
 	}
 }
 
@@ -493,11 +492,11 @@ type listenSocket struct {
 }
 
 func (ls *listenSocket) addChild(id int64, e entry) {
-	grpclog.Errorf("cannot add a child (id = %d) of type %T to a listen socket", id, e)
+	logger.Errorf("cannot add a child (id = %d) of type %T to a listen socket", id, e)
 }
 
 func (ls *listenSocket) deleteChild(id int64) {
-	grpclog.Errorf("cannot delete a child (id = %d) from a listen socket", id)
+	logger.Errorf("cannot delete a child (id = %d) from a listen socket", id)
 }
 
 func (ls *listenSocket) triggerDelete() {
@@ -506,7 +505,7 @@ func (ls *listenSocket) triggerDelete() {
 }
 
 func (ls *listenSocket) deleteSelfIfReady() {
-	grpclog.Errorf("cannot call deleteSelfIfReady on a listen socket")
+	logger.Errorf("cannot call deleteSelfIfReady on a listen socket")
 }
 
 func (ls *listenSocket) getParentID() int64 {
@@ -522,11 +521,11 @@ type normalSocket struct {
 }
 
 func (ns *normalSocket) addChild(id int64, e entry) {
-	grpclog.Errorf("cannot add a child (id = %d) of type %T to a normal socket", id, e)
+	logger.Errorf("cannot add a child (id = %d) of type %T to a normal socket", id, e)
 }
 
 func (ns *normalSocket) deleteChild(id int64) {
-	grpclog.Errorf("cannot delete a child (id = %d) from a normal socket", id)
+	logger.Errorf("cannot delete a child (id = %d) from a normal socket", id)
 }
 
 func (ns *normalSocket) triggerDelete() {
@@ -535,7 +534,7 @@ func (ns *normalSocket) triggerDelete() {
 }
 
 func (ns *normalSocket) deleteSelfIfReady() {
-	grpclog.Errorf("cannot call deleteSelfIfReady on a normal socket")
+	logger.Errorf("cannot call deleteSelfIfReady on a normal socket")
 }
 
 func (ns *normalSocket) getParentID() int64 {
@@ -594,7 +593,7 @@ func (s *server) addChild(id int64, e entry) {
 	case *listenSocket:
 		s.listenSockets[id] = v.refName
 	default:
-		grpclog.Errorf("cannot add a child (id = %d) of type %T to a server", id, e)
+		logger.Errorf("cannot add a child (id = %d) of type %T to a server", id, e)
 	}
 }
 
diff --git a/internal/channelz/types_nonlinux.go b/internal/channelz/types_nonlinux.go
index 79edbefc4331..19c2fc521dcf 100644
--- a/internal/channelz/types_nonlinux.go
+++ b/internal/channelz/types_nonlinux.go
@@ -22,8 +22,6 @@ package channelz
 
 import (
 	"sync"
-
-	"google.golang.org/grpc/grpclog"
 )
 
 var once sync.Once
@@ -39,6 +37,6 @@ type SocketOptionData struct {
 // Windows OS doesn't support Socket Option
 func (s *SocketOptionData) Getsockopt(fd uintptr) {
 	once.Do(func() {
-		grpclog.Warningln("Channelz: socket options are not supported on non-linux os and appengine.")
+		logger.Warning("Channelz: socket options are not supported on non-linux os and appengine.")
 	})
 }
diff --git a/internal/grpclog/grpclog.go b/internal/grpclog/grpclog.go
index 8c8e19fce1d5..745a166f02cf 100644
--- a/internal/grpclog/grpclog.go
+++ b/internal/grpclog/grpclog.go
@@ -19,6 +19,10 @@
 // Package grpclog (internal) defines depth logging for grpc.
 package grpclog
 
+import (
+	"os"
+)
+
 // Logger is the logger used for the non-depth log functions.
 var Logger LoggerV2
 
@@ -30,7 +34,7 @@ func InfoDepth(depth int, args ...interface{}) {
 	if DepthLogger != nil {
 		DepthLogger.InfoDepth(depth, args...)
 	} else {
-		Logger.Info(args...)
+		Logger.Infoln(args...)
 	}
 }
 
@@ -39,7 +43,7 @@ func WarningDepth(depth int, args ...interface{}) {
 	if DepthLogger != nil {
 		DepthLogger.WarningDepth(depth, args...)
 	} else {
-		Logger.Warning(args...)
+		Logger.Warningln(args...)
 	}
 }
 
@@ -48,7 +52,7 @@ func ErrorDepth(depth int, args ...interface{}) {
 	if DepthLogger != nil {
 		DepthLogger.ErrorDepth(depth, args...)
 	} else {
-		Logger.Error(args...)
+		Logger.Errorln(args...)
 	}
 }
 
@@ -57,8 +61,9 @@ func FatalDepth(depth int, args ...interface{}) {
 	if DepthLogger != nil {
 		DepthLogger.FatalDepth(depth, args...)
 	} else {
-		Logger.Fatal(args...)
+		Logger.Fatalln(args...)
 	}
+	os.Exit(1)
 }
 
 // LoggerV2 does underlying logging work for grpclog.
diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go
index 9d08dd8ab092..304235566589 100644
--- a/internal/resolver/dns/dns_resolver.go
+++ b/internal/resolver/dns/dns_resolver.go
@@ -44,6 +44,8 @@ import (
 // addresses from SRV records.  Must not be changed after init time.
 var EnableSRVLookups = false
 
+var logger = grpclog.Component("dns")
+
 func init() {
 	resolver.Register(NewBuilder())
 }
@@ -272,7 +274,7 @@ func handleDNSError(err error, lookupType string) error {
 	err = filterError(err)
 	if err != nil {
 		err = fmt.Errorf("dns: %v record lookup error: %v", lookupType, err)
-		grpclog.Infoln(err)
+		logger.Info(err)
 	}
 	return err
 }
@@ -295,7 +297,7 @@ func (d *dnsResolver) lookupTXT() *serviceconfig.ParseResult {
 
 	// TXT record must have "grpc_config=" attribute in order to be used as service config.
 	if !strings.HasPrefix(res, txtAttribute) {
-		grpclog.Warningf("dns: TXT record %v missing %v attribute", res, txtAttribute)
+		logger.Warningf("dns: TXT record %v missing %v attribute", res, txtAttribute)
 		// This is not an error; it is the equivalent of not having a service config.
 		return nil
 	}
@@ -421,12 +423,12 @@ func canaryingSC(js string) string {
 	var rcs []rawChoice
 	err := json.Unmarshal([]byte(js), &rcs)
 	if err != nil {
-		grpclog.Warningf("dns: error parsing service config json: %v", err)
+		logger.Warningf("dns: error parsing service config json: %v", err)
 		return ""
 	}
 	cliHostname, err := os.Hostname()
 	if err != nil {
-		grpclog.Warningf("dns: error getting client hostname: %v", err)
+		logger.Warningf("dns: error getting client hostname: %v", err)
 		return ""
 	}
 	var sc string
diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go
index 43281a3e078d..c50468a0fc89 100644
--- a/internal/syscall/syscall_linux.go
+++ b/internal/syscall/syscall_linux.go
@@ -32,11 +32,13 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+var logger = grpclog.Component("core")
+
 // GetCPUTime returns the how much CPU time has passed since the start of this process.
 func GetCPUTime() int64 {
 	var ts unix.Timespec
 	if err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &ts); err != nil {
-		grpclog.Fatal(err)
+		logger.Fatal(err)
 	}
 	return ts.Nano()
 }
diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go
index ae0a9117e7b9..adae60d65188 100644
--- a/internal/syscall/syscall_nonlinux.go
+++ b/internal/syscall/syscall_nonlinux.go
@@ -31,10 +31,11 @@ import (
 )
 
 var once sync.Once
+var logger = grpclog.Component("core")
 
 func log() {
 	once.Do(func() {
-		grpclog.Info("CPU time info is unavailable on non-linux or appengine environment.")
+		logger.Info("CPU time info is unavailable on non-linux or appengine environment.")
 	})
 }
 
diff --git a/internal/transport/controlbuf.go b/internal/transport/controlbuf.go
index d4bb19c3bb53..40ef23923fda 100644
--- a/internal/transport/controlbuf.go
+++ b/internal/transport/controlbuf.go
@@ -505,7 +505,9 @@ func (l *loopyWriter) run() (err error) {
 			// 1. When the connection is closed by some other known issue.
 			// 2. User closed the connection.
 			// 3. A graceful close of connection.
-			infof("transport: loopyWriter.run returning. %v", err)
+			if logger.V(logLevel) {
+				logger.Infof("transport: loopyWriter.run returning. %v", err)
+			}
 			err = nil
 		}
 	}()
@@ -605,7 +607,9 @@ func (l *loopyWriter) headerHandler(h *headerFrame) error {
 	if l.side == serverSide {
 		str, ok := l.estdStreams[h.streamID]
 		if !ok {
-			warningf("transport: loopy doesn't recognize the stream: %d", h.streamID)
+			if logger.V(logLevel) {
+				logger.Warningf("transport: loopy doesn't recognize the stream: %d", h.streamID)
+			}
 			return nil
 		}
 		// Case 1.A: Server is responding back with headers.
@@ -658,7 +662,9 @@ func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.He
 	l.hBuf.Reset()
 	for _, f := range hf {
 		if err := l.hEnc.WriteField(f); err != nil {
-			warningf("transport: loopyWriter.writeHeader encountered error while encoding headers:", err)
+			if logger.V(logLevel) {
+				logger.Warningf("transport: loopyWriter.writeHeader encountered error while encoding headers: %v", err)
+			}
 		}
 	}
 	var (
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index b43e21ffaf73..47af2ebeade8 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -354,7 +354,9 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst)
 		err := t.loopy.run()
 		if err != nil {
-			errorf("transport: loopyWriter.run returning. Err: %v", err)
+			if logger.V(logLevel) {
+				logger.Errorf("transport: loopyWriter.run returning. Err: %v", err)
+			}
 		}
 		// If it's a connection error, let reader goroutine handle it
 		// since there might be data in the buffers.
@@ -1013,7 +1015,9 @@ func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) {
 	}
 	statusCode, ok := http2ErrConvTab[f.ErrCode]
 	if !ok {
-		warningf("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error %v", f.ErrCode)
+		if logger.V(logLevel) {
+			logger.Warningf("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error %v", f.ErrCode)
+		}
 		statusCode = codes.Unknown
 	}
 	if statusCode == codes.Canceled {
@@ -1095,7 +1099,9 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 		return
 	}
 	if f.ErrCode == http2.ErrCodeEnhanceYourCalm {
-		infof("Client received GoAway with http2.ErrCodeEnhanceYourCalm.")
+		if logger.V(logLevel) {
+			logger.Infof("Client received GoAway with http2.ErrCodeEnhanceYourCalm.")
+		}
 	}
 	id := f.LastStreamID
 	if id > 0 && id%2 != 1 {
@@ -1325,7 +1331,9 @@ func (t *http2Client) reader() {
 		case *http2.WindowUpdateFrame:
 			t.handleWindowUpdate(frame)
 		default:
-			errorf("transport: http2Client.reader got unhandled frame type %v.", frame)
+			if logger.V(logLevel) {
+				logger.Errorf("transport: http2Client.reader got unhandled frame type %v.", frame)
+			}
 		}
 	}
 }
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index e8c757321287..788cf1e4df01 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -37,7 +37,6 @@ import (
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/keepalive"
@@ -289,7 +288,9 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err
 		t.loopy = newLoopyWriter(serverSide, t.framer, t.controlBuf, t.bdpEst)
 		t.loopy.ssGoAwayHandler = t.outgoingGoAwayHandler
 		if err := t.loopy.run(); err != nil {
-			errorf("transport: loopyWriter.run returning. Err: %v", err)
+			if logger.V(logLevel) {
+				logger.Errorf("transport: loopyWriter.run returning. Err: %v", err)
+			}
 		}
 		t.conn.Close()
 		close(t.writerDone)
@@ -360,7 +361,9 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 		}
 		s.ctx, err = t.inTapHandle(s.ctx, info)
 		if err != nil {
-			warningf("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err)
+			if logger.V(logLevel) {
+				logger.Warningf("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err)
+			}
 			t.controlBuf.put(&cleanupStream{
 				streamID: s.id,
 				rst:      true,
@@ -391,7 +394,9 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 	if streamID%2 != 1 || streamID <= t.maxStreamID {
 		t.mu.Unlock()
 		// illegal gRPC stream id.
-		errorf("transport: http2Server.HandleStreams received an illegal stream id: %v", streamID)
+		if logger.V(logLevel) {
+			logger.Errorf("transport: http2Server.HandleStreams received an illegal stream id: %v", streamID)
+		}
 		s.cancel()
 		return true
 	}
@@ -454,7 +459,9 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.
 		atomic.StoreInt64(&t.lastRead, time.Now().UnixNano())
 		if err != nil {
 			if se, ok := err.(http2.StreamError); ok {
-				warningf("transport: http2Server.HandleStreams encountered http2.StreamError: %v", se)
+				if logger.V(logLevel) {
+					logger.Warningf("transport: http2Server.HandleStreams encountered http2.StreamError: %v", se)
+				}
 				t.mu.Lock()
 				s := t.activeStreams[se.StreamID]
 				t.mu.Unlock()
@@ -474,7 +481,9 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.
 				t.Close()
 				return
 			}
-			warningf("transport: http2Server.HandleStreams failed to read frame: %v", err)
+			if logger.V(logLevel) {
+				logger.Warningf("transport: http2Server.HandleStreams failed to read frame: %v", err)
+			}
 			t.Close()
 			return
 		}
@@ -497,7 +506,9 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.
 		case *http2.GoAwayFrame:
 			// TODO: Handle GoAway from the client appropriately.
 		default:
-			errorf("transport: http2Server.HandleStreams found unhandled frame type %v.", frame)
+			if logger.V(logLevel) {
+				logger.Errorf("transport: http2Server.HandleStreams found unhandled frame type %v.", frame)
+			}
 		}
 	}
 }
@@ -719,7 +730,9 @@ func (t *http2Server) handlePing(f *http2.PingFrame) {
 
 	if t.pingStrikes > maxPingStrikes {
 		// Send goaway and close the connection.
-		errorf("transport: Got too many pings from the client, closing the connection.")
+		if logger.V(logLevel) {
+			logger.Errorf("transport: Got too many pings from the client, closing the connection.")
+		}
 		t.controlBuf.put(&goAway{code: http2.ErrCodeEnhanceYourCalm, debugData: []byte("too_many_pings"), closeConn: true})
 	}
 }
@@ -752,7 +765,9 @@ func (t *http2Server) checkForHeaderListSize(it interface{}) bool {
 	var sz int64
 	for _, f := range hdrFrame.hf {
 		if sz += int64(f.Size()); sz > int64(*t.maxSendHeaderListSize) {
-			errorf("header list size to send violates the maximum size (%d bytes) set by client", *t.maxSendHeaderListSize)
+			if logger.V(logLevel) {
+				logger.Errorf("header list size to send violates the maximum size (%d bytes) set by client", *t.maxSendHeaderListSize)
+			}
 			return false
 		}
 	}
@@ -849,7 +864,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error {
 		stBytes, err := proto.Marshal(p)
 		if err != nil {
 			// TODO: return error instead, when callers are able to handle it.
-			grpclog.Errorf("transport: failed to marshal rpc status: %v, error: %v", p, err)
+			logger.Errorf("transport: failed to marshal rpc status: %v, error: %v", p, err)
 		} else {
 			headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status-details-bin", Value: encodeBinHeader(stBytes)})
 		}
@@ -980,7 +995,9 @@ func (t *http2Server) keepalive() {
 			select {
 			case <-ageTimer.C:
 				// Close the connection after grace period.
-				infof("transport: closing server transport due to maximum connection age.")
+				if logger.V(logLevel) {
+					logger.Infof("transport: closing server transport due to maximum connection age.")
+				}
 				t.Close()
 			case <-t.done:
 			}
@@ -997,7 +1014,9 @@ func (t *http2Server) keepalive() {
 				continue
 			}
 			if outstandingPing && kpTimeoutLeft <= 0 {
-				infof("transport: closing server transport due to idleness.")
+				if logger.V(logLevel) {
+					logger.Infof("transport: closing server transport due to idleness.")
+				}
 				t.Close()
 				return
 			}
diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go
index 8f5f3349d906..e68cdcb6cc2d 100644
--- a/internal/transport/http_util.go
+++ b/internal/transport/http_util.go
@@ -37,6 +37,7 @@ import (
 	"golang.org/x/net/http2/hpack"
 	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/status"
 )
 
@@ -97,6 +98,7 @@ var (
 		// 504 Gateway timeout - UNAVAILABLE.
 		http.StatusGatewayTimeout: codes.Unavailable,
 	}
+	logger = grpclog.Component("transport")
 )
 
 type parsedHeaderData struct {
@@ -412,7 +414,9 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) {
 		}
 		v, err := decodeMetadataHeader(f.Name, f.Value)
 		if err != nil {
-			errorf("Failed to decode metadata header (%q, %q): %v", f.Name, f.Value, err)
+			if logger.V(logLevel) {
+				logger.Errorf("Failed to decode metadata header (%q, %q): %v", f.Name, f.Value, err)
+			}
 			return
 		}
 		d.addMetadata(f.Name, v)
diff --git a/internal/transport/log.go b/internal/transport/log.go
deleted file mode 100644
index 879df80c4de7..000000000000
--- a/internal/transport/log.go
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// This file contains wrappers for grpclog functions.
-// The transport package only logs to verbose level 2 by default.
-
-package transport
-
-import "google.golang.org/grpc/grpclog"
-
-const logLevel = 2
-
-func infof(format string, args ...interface{}) {
-	if grpclog.V(logLevel) {
-		grpclog.Infof(format, args...)
-	}
-}
-
-func warningf(format string, args ...interface{}) {
-	if grpclog.V(logLevel) {
-		grpclog.Warningf(format, args...)
-	}
-}
-
-func errorf(format string, args ...interface{}) {
-	if grpclog.V(logLevel) {
-		grpclog.Errorf(format, args...)
-	}
-}
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index 1ffd96ff43d1..b74030a96878 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -41,6 +41,8 @@ import (
 	"google.golang.org/grpc/tap"
 )
 
+const logLevel = 2
+
 type bufferPool struct {
 	pool sync.Pool
 }
diff --git a/picker_wrapper.go b/picker_wrapper.go
index 7f3edaaedc60..a58174b6f436 100644
--- a/picker_wrapper.go
+++ b/picker_wrapper.go
@@ -25,7 +25,6 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/status"
@@ -145,7 +144,7 @@ func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.
 
 		acw, ok := pickResult.SubConn.(*acBalancerWrapper)
 		if !ok {
-			grpclog.Error("subconn returned from pick is not *acBalancerWrapper")
+			logger.Error("subconn returned from pick is not *acBalancerWrapper")
 			continue
 		}
 		if t, ok := acw.getAddrConn().getReadyTransport(); ok {
@@ -159,7 +158,7 @@ func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.
 			// DoneInfo with default value works.
 			pickResult.Done(balancer.DoneInfo{})
 		}
-		grpclog.Infof("blockingPicker: the picked transport is not ready, loop back to repick")
+		logger.Infof("blockingPicker: the picked transport is not ready, loop back to repick")
 		// If ok == false, ac.state is not READY.
 		// A valid picker always returns READY subConn. This means the state of ac
 		// just changed, and picker will be updated shortly.
diff --git a/pickfirst.go b/pickfirst.go
index 4b7340ad3ecc..56e33f6c76b7 100644
--- a/pickfirst.go
+++ b/pickfirst.go
@@ -24,7 +24,6 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/grpclog"
 )
 
 // PickFirstBalancerName is the name of the pick_first balancer.
@@ -58,8 +57,8 @@ func (b *pickfirstBalancer) ResolverError(err error) {
 			Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)},
 		})
 	}
-	if grpclog.V(2) {
-		grpclog.Infof("pickfirstBalancer: ResolverError called with error %v", err)
+	if logger.V(2) {
+		logger.Infof("pickfirstBalancer: ResolverError called with error %v", err)
 	}
 }
 
@@ -72,8 +71,8 @@ func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) e
 		var err error
 		b.sc, err = b.cc.NewSubConn(cs.ResolverState.Addresses, balancer.NewSubConnOptions{})
 		if err != nil {
-			if grpclog.V(2) {
-				grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err)
+			if logger.V(2) {
+				logger.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err)
 			}
 			b.state = connectivity.TransientFailure
 			b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure,
@@ -92,12 +91,12 @@ func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) e
 }
 
 func (b *pickfirstBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
-	if grpclog.V(2) {
-		grpclog.Infof("pickfirstBalancer: UpdateSubConnState: %p, %v", sc, s)
+	if logger.V(2) {
+		logger.Infof("pickfirstBalancer: UpdateSubConnState: %p, %v", sc, s)
 	}
 	if b.sc != sc {
-		if grpclog.V(2) {
-			grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized")
+		if logger.V(2) {
+			logger.Infof("pickfirstBalancer: ignored state change because sc is not recognized")
 		}
 		return
 	}
diff --git a/resolver_conn_wrapper.go b/resolver_conn_wrapper.go
index edfda866c001..265002a75e00 100644
--- a/resolver_conn_wrapper.go
+++ b/resolver_conn_wrapper.go
@@ -140,7 +140,7 @@ func (ccr *ccResolverWrapper) UpdateState(s resolver.State) {
 	if ccr.done.HasFired() {
 		return
 	}
-	channelz.Infof(ccr.cc.channelzID, "ccResolverWrapper: sending update to cc: %v", s)
+	channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending update to cc: %v", s)
 	if channelz.IsOn() {
 		ccr.addChannelzTraceEvent(s)
 	}
@@ -152,7 +152,7 @@ func (ccr *ccResolverWrapper) ReportError(err error) {
 	if ccr.done.HasFired() {
 		return
 	}
-	channelz.Warningf(ccr.cc.channelzID, "ccResolverWrapper: reporting error to cc: %v", err)
+	channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: reporting error to cc: %v", err)
 	ccr.poll(ccr.cc.updateResolverState(resolver.State{}, err))
 }
 
@@ -161,7 +161,7 @@ func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) {
 	if ccr.done.HasFired() {
 		return
 	}
-	channelz.Infof(ccr.cc.channelzID, "ccResolverWrapper: sending new addresses to cc: %v", addrs)
+	channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending new addresses to cc: %v", addrs)
 	if channelz.IsOn() {
 		ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig})
 	}
@@ -175,14 +175,14 @@ func (ccr *ccResolverWrapper) NewServiceConfig(sc string) {
 	if ccr.done.HasFired() {
 		return
 	}
-	channelz.Infof(ccr.cc.channelzID, "ccResolverWrapper: got new service config: %v", sc)
+	channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: got new service config: %v", sc)
 	if ccr.cc.dopts.disableServiceConfig {
-		channelz.Info(ccr.cc.channelzID, "Service config lookups disabled; ignoring config")
+		channelz.Info(logger, ccr.cc.channelzID, "Service config lookups disabled; ignoring config")
 		return
 	}
 	scpr := parseServiceConfig(sc)
 	if scpr.Err != nil {
-		channelz.Warningf(ccr.cc.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err)
+		channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err)
 		ccr.poll(balancer.ErrBadResolverState)
 		return
 	}
@@ -215,7 +215,7 @@ func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) {
 	} else if len(ccr.curState.Addresses) == 0 && len(s.Addresses) > 0 {
 		updates = append(updates, "resolver returned new addresses")
 	}
-	channelz.AddTraceEvent(ccr.cc.channelzID, 0, &channelz.TraceEventDesc{
+	channelz.AddTraceEvent(logger, ccr.cc.channelzID, 0, &channelz.TraceEventDesc{
 		Desc:     fmt.Sprintf("Resolver state updated: %+v (%v)", s, strings.Join(updates, "; ")),
 		Severity: channelz.CtINFO,
 	})
diff --git a/server.go b/server.go
index c2c7cae6c5dc..0cac3cf5f2f7 100644
--- a/server.go
+++ b/server.go
@@ -59,6 +59,7 @@ const (
 )
 
 var statusOK = status.New(codes.OK, "")
+var logger = grpclog.Component("core")
 
 type methodHandler func(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor UnaryServerInterceptor) (interface{}, error)
 
@@ -223,7 +224,7 @@ func InitialConnWindowSize(s int32) ServerOption {
 // KeepaliveParams returns a ServerOption that sets keepalive and max-age parameters for the server.
 func KeepaliveParams(kp keepalive.ServerParameters) ServerOption {
 	if kp.Time > 0 && kp.Time < time.Second {
-		grpclog.Warning("Adjusting keepalive ping interval to minimum period of 1s")
+		logger.Warning("Adjusting keepalive ping interval to minimum period of 1s")
 		kp.Time = time.Second
 	}
 
@@ -537,7 +538,7 @@ func (s *Server) RegisterService(sd *ServiceDesc, ss interface{}) {
 	ht := reflect.TypeOf(sd.HandlerType).Elem()
 	st := reflect.TypeOf(ss)
 	if !st.Implements(ht) {
-		grpclog.Fatalf("grpc: Server.RegisterService found the handler of type %v that does not satisfy %v", st, ht)
+		logger.Fatalf("grpc: Server.RegisterService found the handler of type %v that does not satisfy %v", st, ht)
 	}
 	s.register(sd, ss)
 }
@@ -547,10 +548,10 @@ func (s *Server) register(sd *ServiceDesc, ss interface{}) {
 	defer s.mu.Unlock()
 	s.printf("RegisterService(%q)", sd.ServiceName)
 	if s.serve {
-		grpclog.Fatalf("grpc: Server.RegisterService after Server.Serve for %q", sd.ServiceName)
+		logger.Fatalf("grpc: Server.RegisterService after Server.Serve for %q", sd.ServiceName)
 	}
 	if _, ok := s.m[sd.ServiceName]; ok {
-		grpclog.Fatalf("grpc: Server.RegisterService found duplicate service registration for %q", sd.ServiceName)
+		logger.Fatalf("grpc: Server.RegisterService found duplicate service registration for %q", sd.ServiceName)
 	}
 	srv := &service{
 		server: ss,
@@ -756,7 +757,7 @@ func (s *Server) handleRawConn(rawConn net.Conn) {
 			s.mu.Lock()
 			s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err)
 			s.mu.Unlock()
-			channelz.Warningf(s.channelzID, "grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err)
+			channelz.Warningf(logger, s.channelzID, "grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err)
 			rawConn.Close()
 		}
 		rawConn.SetDeadline(time.Time{})
@@ -803,7 +804,7 @@ func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) tr
 		s.errorf("NewServerTransport(%q) failed: %v", c.RemoteAddr(), err)
 		s.mu.Unlock()
 		c.Close()
-		channelz.Warning(s.channelzID, "grpc: Server.Serve failed to create ServerTransport: ", err)
+		channelz.Warning(logger, s.channelzID, "grpc: Server.Serve failed to create ServerTransport: ", err)
 		return nil
 	}
 
@@ -957,12 +958,12 @@ func (s *Server) incrCallsFailed() {
 func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options, comp encoding.Compressor) error {
 	data, err := encode(s.getCodec(stream.ContentSubtype()), msg)
 	if err != nil {
-		channelz.Error(s.channelzID, "grpc: server failed to encode response: ", err)
+		channelz.Error(logger, s.channelzID, "grpc: server failed to encode response: ", err)
 		return err
 	}
 	compData, err := compress(data, cp, comp)
 	if err != nil {
-		channelz.Error(s.channelzID, "grpc: server failed to compress response: ", err)
+		channelz.Error(logger, s.channelzID, "grpc: server failed to compress response: ", err)
 		return err
 	}
 	hdr, payload := msgHeader(data, compData)
@@ -1136,7 +1137,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 	if err != nil {
 		if st, ok := status.FromError(err); ok {
 			if e := t.WriteStatus(stream, st); e != nil {
-				channelz.Warningf(s.channelzID, "grpc: Server.processUnaryRPC failed to write status %v", e)
+				channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status %v", e)
 			}
 		}
 		return err
@@ -1181,7 +1182,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 			trInfo.tr.SetError()
 		}
 		if e := t.WriteStatus(stream, appStatus); e != nil {
-			channelz.Warningf(s.channelzID, "grpc: Server.processUnaryRPC failed to write status: %v", e)
+			channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status: %v", e)
 		}
 		if binlog != nil {
 			if h, _ := stream.Header(); h.Len() > 0 {
@@ -1210,7 +1211,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 		}
 		if sts, ok := status.FromError(err); ok {
 			if e := t.WriteStatus(stream, sts); e != nil {
-				channelz.Warningf(s.channelzID, "grpc: Server.processUnaryRPC failed to write status: %v", e)
+				channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status: %v", e)
 			}
 		} else {
 			switch st := err.(type) {
@@ -1478,7 +1479,7 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str
 				trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true)
 				trInfo.tr.SetError()
 			}
-			channelz.Warningf(s.channelzID, "grpc: Server.handleStream failed to write status: %v", err)
+			channelz.Warningf(logger, s.channelzID, "grpc: Server.handleStream failed to write status: %v", err)
 		}
 		if trInfo != nil {
 			trInfo.tr.Finish()
@@ -1519,7 +1520,7 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str
 			trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true)
 			trInfo.tr.SetError()
 		}
-		channelz.Warningf(s.channelzID, "grpc: Server.handleStream failed to write status: %v", err)
+		channelz.Warningf(logger, s.channelzID, "grpc: Server.handleStream failed to write status: %v", err)
 	}
 	if trInfo != nil {
 		trInfo.tr.Finish()
diff --git a/service_config.go b/service_config.go
index 3132a66cd68c..c9f164c9ae43 100644
--- a/service_config.go
+++ b/service_config.go
@@ -27,7 +27,6 @@ import (
 	"time"
 
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
 	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/serviceconfig"
@@ -269,7 +268,7 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 	var rsc jsonSC
 	err := json.Unmarshal([]byte(js), &rsc)
 	if err != nil {
-		grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
+		logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
 		return &serviceconfig.ParseResult{Err: err}
 	}
 	sc := ServiceConfig{
@@ -295,7 +294,7 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 		}
 		d, err := parseDuration(m.Timeout)
 		if err != nil {
-			grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
+			logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
 			return &serviceconfig.ParseResult{Err: err}
 		}
 
@@ -304,7 +303,7 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 			Timeout:      d,
 		}
 		if mc.retryPolicy, err = convertRetryPolicy(m.RetryPolicy); err != nil {
-			grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
+			logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
 			return &serviceconfig.ParseResult{Err: err}
 		}
 		if m.MaxRequestMessageBytes != nil {
@@ -357,7 +356,7 @@ func convertRetryPolicy(jrp *jsonRetryPolicy) (p *retryPolicy, err error) {
 		*mb <= 0 ||
 		jrp.BackoffMultiplier <= 0 ||
 		len(jrp.RetryableStatusCodes) == 0 {
-		grpclog.Warningf("grpc: ignoring retry policy %v due to illegal configuration", jrp)
+		logger.Warningf("grpc: ignoring retry policy %v due to illegal configuration", jrp)
 		return nil, nil
 	}
 
diff --git a/stream.go b/stream.go
index 629af76bdfab..34e4010d0dba 100644
--- a/stream.go
+++ b/stream.go
@@ -510,13 +510,13 @@ func (cs *clientStream) shouldRetry(err error) error {
 		if len(sps) == 1 {
 			var e error
 			if pushback, e = strconv.Atoi(sps[0]); e != nil || pushback < 0 {
-				channelz.Infof(cs.cc.channelzID, "Server retry pushback specified to abort (%q).", sps[0])
+				channelz.Infof(logger, cs.cc.channelzID, "Server retry pushback specified to abort (%q).", sps[0])
 				cs.retryThrottler.throttle() // This counts as a failure for throttling.
 				return err
 			}
 			hasPushback = true
 		} else if len(sps) > 1 {
-			channelz.Warningf(cs.cc.channelzID, "Server retry pushback specified multiple values (%q); not retrying.", sps)
+			channelz.Warningf(logger, cs.cc.channelzID, "Server retry pushback specified multiple values (%q); not retrying.", sps)
 			cs.retryThrottler.throttle() // This counts as a failure for throttling.
 			return err
 		}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index c2ce31238542..66854e17ea42 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -28,6 +28,7 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+var logger = grpclog.Component("xds")
 var errAllPrioritiesRemoved = errors.New("eds: no locality is provided, all priorities are removed")
 
 // handlePriorityChange handles priority after EDS adds/removes a
@@ -134,13 +135,13 @@ func (edsImpl *edsBalancerImpl) handlePriorityWithNewState(priority priorityType
 	defer edsImpl.priorityMu.Unlock()
 
 	if !edsImpl.priorityInUse.isSet() {
-		grpclog.Infof("eds: received picker update when no priority is in use (EDS returned an empty list)")
+		logger.Infof("eds: received picker update when no priority is in use (EDS returned an empty list)")
 		return false
 	}
 
 	if edsImpl.priorityInUse.higherThan(priority) {
 		// Lower priorities should all be closed, this is an unexpected update.
-		grpclog.Infof("eds: received picker update from priority lower then priorityInUse")
+		logger.Infof("eds: received picker update from priority lower then priorityInUse")
 		return false
 	}
 
diff --git a/xds/internal/balancer/lrs/lrs.go b/xds/internal/balancer/lrs/lrs.go
index eaf5997a9dc8..4bc20ef6e124 100644
--- a/xds/internal/balancer/lrs/lrs.go
+++ b/xds/internal/balancer/lrs/lrs.go
@@ -36,6 +36,8 @@ import (
 
 const negativeOneUInt64 = ^uint64(0)
 
+var logger = grpclog.Component("xds")
+
 // Store defines the interface for a load store. It keeps loads and can report
 // them to a server when requested.
 type Store interface {
@@ -310,25 +312,25 @@ func (ls *lrsStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterNa
 		doBackoff = true
 		stream, err := c.StreamLoadStats(ctx)
 		if err != nil {
-			grpclog.Warningf("lrs: failed to create stream: %v", err)
+			logger.Warningf("lrs: failed to create stream: %v", err)
 			continue
 		}
-		grpclog.Infof("lrs: created LRS stream")
+		logger.Infof("lrs: created LRS stream")
 		req := &lrspb.LoadStatsRequest{Node: node}
-		grpclog.Infof("lrs: sending init LoadStatsRequest: %v", req)
+		logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
 		if err := stream.Send(req); err != nil {
-			grpclog.Warningf("lrs: failed to send first request: %v", err)
+			logger.Warningf("lrs: failed to send first request: %v", err)
 			continue
 		}
 		first, err := stream.Recv()
 		if err != nil {
-			grpclog.Warningf("lrs: failed to receive first response: %v", err)
+			logger.Warningf("lrs: failed to receive first response: %v", err)
 			continue
 		}
-		grpclog.Infof("lrs: received first LoadStatsResponse: %+v", first)
+		logger.Infof("lrs: received first LoadStatsResponse: %+v", first)
 		interval, err := ptypes.Duration(first.LoadReportingInterval)
 		if err != nil {
-			grpclog.Warningf("lrs: failed to convert report interval: %v", err)
+			logger.Warningf("lrs: failed to convert report interval: %v", err)
 			continue
 		}
 		// The LRS client should join the clusters it knows with the cluster
@@ -343,12 +345,12 @@ func (ls *lrsStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterNa
 			}
 		}
 		if !clusterFoundInResponse {
-			grpclog.Warningf("lrs: received clusters %v does not contain expected {%v}", first.Clusters, clusterName)
+			logger.Warningf("lrs: received clusters %v does not contain expected {%v}", first.Clusters, clusterName)
 			continue
 		}
 		if first.ReportEndpointGranularity {
 			// TODO: fixme to support per endpoint loads.
-			grpclog.Warningf("lrs: endpoint loads requested, but not supported by current implementation")
+			logger.Warningf("lrs: endpoint loads requested, but not supported by current implementation")
 			continue
 		}
 
@@ -369,9 +371,9 @@ func (ls *lrsStore) sendLoads(ctx context.Context, stream lrsgrpc.LoadReportingS
 			return
 		}
 		req := &lrspb.LoadStatsRequest{ClusterStats: ls.buildStats(clusterName)}
-		grpclog.Infof("lrs: sending LRS loads: %+v", req)
+		logger.Infof("lrs: sending LRS loads: %+v", req)
 		if err := stream.Send(req); err != nil {
-			grpclog.Warningf("lrs: failed to send report: %v", err)
+			logger.Warningf("lrs: failed to send report: %v", err)
 			return
 		}
 	}
diff --git a/xds/internal/balancer/orca/orca.go b/xds/internal/balancer/orca/orca.go
index 2bcd5cfd0e13..28016806eec4 100644
--- a/xds/internal/balancer/orca/orca.go
+++ b/xds/internal/balancer/orca/orca.go
@@ -27,6 +27,8 @@ import (
 
 const mdKey = "X-Endpoint-Load-Metrics-Bin"
 
+var logger = grpclog.Component("xds")
+
 // toBytes converts a orca load report into bytes.
 func toBytes(r *orcapb.OrcaLoadReport) []byte {
 	if r == nil {
@@ -35,7 +37,7 @@ func toBytes(r *orcapb.OrcaLoadReport) []byte {
 
 	b, err := proto.Marshal(r)
 	if err != nil {
-		grpclog.Warningf("orca: failed to marshal load report: %v", err)
+		logger.Warningf("orca: failed to marshal load report: %v", err)
 		return nil
 	}
 	return b
@@ -54,7 +56,7 @@ func ToMetadata(r *orcapb.OrcaLoadReport) metadata.MD {
 func fromBytes(b []byte) *orcapb.OrcaLoadReport {
 	ret := new(orcapb.OrcaLoadReport)
 	if err := proto.Unmarshal(b, ret); err != nil {
-		grpclog.Warningf("orca: failed to unmarshal load report: %v", err)
+		logger.Warningf("orca: failed to unmarshal load report: %v", err)
 		return nil
 	}
 	return ret
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go
index 42766f997163..314a47a8defb 100644
--- a/xds/internal/client/client_loadreport.go
+++ b/xds/internal/client/client_loadreport.go
@@ -30,6 +30,8 @@ import (
 
 const nodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME"
 
+var logger = grpclog.Component("xds")
+
 // ReportLoad sends the load of the given clusterName from loadStore to the
 // given server. If the server is not an empty string, and is different from the
 // xds server, a new ClientConn will be created.
@@ -53,7 +55,7 @@ func (c *Client) ReportLoad(server string, clusterName string, loadStore lrs.Sto
 		ccNew, err := grpc.Dial(server, dopts...)
 		if err != nil {
 			// An error from a non-blocking dial indicates something serious.
-			grpclog.Infof("xds: failed to dial load report server {%s}: %v", server, err)
+			logger.Infof("xds: failed to dial load report server {%s}: %v", server, err)
 			return func() {}
 		}
 		cc = ccNew

From c95dc4da23cb6f3d4c8bedd53d4af72b46f55616 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 26 Jun 2020 12:56:03 -0700
Subject: [PATCH 104/481] doc: mark CustomCodec as deprecated (#3698)

---
 server.go | 14 +++++++++++---
 vet.sh    |  1 +
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/server.go b/server.go
index 0cac3cf5f2f7..1b56cc2d11fd 100644
--- a/server.go
+++ b/server.go
@@ -243,6 +243,12 @@ func KeepaliveEnforcementPolicy(kep keepalive.EnforcementPolicy) ServerOption {
 // CustomCodec returns a ServerOption that sets a codec for message marshaling and unmarshaling.
 //
 // This will override any lookups by content-subtype for Codecs registered with RegisterCodec.
+//
+// Deprecated: register codecs using encoding.RegisterCodec. The server will
+// automatically use registered codecs based on the incoming requests' headers.
+// See also
+// https://github.com/grpc/grpc-go/blob/master/Documentation/encoding.md#using-a-codec.
+// Will be supported throughout 1.x.
 func CustomCodec(codec Codec) ServerOption {
 	return newFuncServerOption(func(o *serverOptions) {
 		o.codec = codec
@@ -255,7 +261,8 @@ func CustomCodec(codec Codec) ServerOption {
 // default, server messages will be sent using the same compressor with which
 // request messages were sent.
 //
-// Deprecated: use encoding.RegisterCompressor instead.
+// Deprecated: use encoding.RegisterCompressor instead. Will be supported
+// throughout 1.x.
 func RPCCompressor(cp Compressor) ServerOption {
 	return newFuncServerOption(func(o *serverOptions) {
 		o.cp = cp
@@ -266,7 +273,8 @@ func RPCCompressor(cp Compressor) ServerOption {
 // messages.  It has higher priority than decompressors registered via
 // encoding.RegisterCompressor.
 //
-// Deprecated: use encoding.RegisterCompressor instead.
+// Deprecated: use encoding.RegisterCompressor instead. Will be supported
+// throughout 1.x.
 func RPCDecompressor(dc Decompressor) ServerOption {
 	return newFuncServerOption(func(o *serverOptions) {
 		o.dc = dc
@@ -276,7 +284,7 @@ func RPCDecompressor(dc Decompressor) ServerOption {
 // MaxMsgSize returns a ServerOption to set the max message size in bytes the server can receive.
 // If this is not set, gRPC uses the default limit.
 //
-// Deprecated: use MaxRecvMsgSize instead.
+// Deprecated: use MaxRecvMsgSize instead. Will be supported throughout 1.x.
 func MaxMsgSize(m int) ServerOption {
 	return MaxRecvMsgSize(m)
 }
diff --git a/vet.sh b/vet.sh
index bfd5d5411053..8b7dff19adb3 100755
--- a/vet.sh
+++ b/vet.sh
@@ -135,6 +135,7 @@ balancer.Picker
 grpc.CallCustomCodec
 grpc.Code
 grpc.Compressor
+grpc.CustomCodec
 grpc.Decompressor
 grpc.MaxMsgSize
 grpc.MethodConfig

From 68098483a7afa91b353453641408e3968ad92738 Mon Sep 17 00:00:00 2001
From: cindyxue <32377977+cindyxue@users.noreply.github.com>
Date: Sat, 27 Jun 2020 16:05:33 -0700
Subject: [PATCH 105/481] advancedtls: Add system default CAs to config
 function (#3663)

* Add system default CAs to config function
---
 security/advancedtls/advancedtls.go      | 28 ++++++++++--
 security/advancedtls/advancedtls_test.go | 56 ++++++++++++++++++++++++
 2 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index db78a566262e..745938ac5295 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -81,6 +81,8 @@ type GetRootCAsResults struct {
 // RootCertificateOptions contains a field and a function for obtaining root
 // trust certificates.
 // It is used by both ClientOptions and ServerOptions.
+// If users want to use default verification, but did not provide a valid
+// RootCertificateOptions, we use the system default trust certificates.
 type RootCertificateOptions struct {
 	// If field RootCACerts is set, field GetRootCAs will be ignored. RootCACerts
 	// will be used every time when verifying the peer certificates, without
@@ -184,15 +186,26 @@ func (o *ClientOptions) config() (*tls.Config, error) {
 		return nil, fmt.Errorf(
 			"client needs to provide custom verification mechanism if choose to skip default verification")
 	}
+	rootCAs := o.RootCACerts
+	if o.VType != SkipVerification && o.RootCACerts == nil && o.GetRootCAs == nil {
+		// Set rootCAs to system default.
+		systemRootCAs, err := x509.SystemCertPool()
+		if err != nil {
+			return nil, err
+		}
+		rootCAs = systemRootCAs
+	}
 	// We have to set InsecureSkipVerify to true to skip the default checks and
 	// use the verification function we built from buildVerifyFunc.
 	config := &tls.Config{
 		ServerName:           o.ServerNameOverride,
 		Certificates:         o.Certificates,
 		GetClientCertificate: o.GetClientCertificate,
-		RootCAs:              o.RootCACerts,
 		InsecureSkipVerify:   true,
 	}
+	if rootCAs != nil {
+		config.RootCAs = rootCAs
+	}
 	return config, nil
 }
 
@@ -204,6 +217,15 @@ func (o *ServerOptions) config() (*tls.Config, error) {
 		return nil, fmt.Errorf(
 			"server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)")
 	}
+	clientCAs := o.RootCACerts
+	if o.VType != SkipVerification && o.RootCACerts == nil && o.GetRootCAs == nil && o.RequireClientCert {
+		// Set clientCAs to system default.
+		systemRootCAs, err := x509.SystemCertPool()
+		if err != nil {
+			return nil, err
+		}
+		clientCAs = systemRootCAs
+	}
 	clientAuth := tls.NoClientCert
 	if o.RequireClientCert {
 		// We have to set clientAuth to RequireAnyClientCert to force underlying
@@ -216,8 +238,8 @@ func (o *ServerOptions) config() (*tls.Config, error) {
 		Certificates:   o.Certificates,
 		GetCertificate: o.GetCertificate,
 	}
-	if o.RootCACerts != nil {
-		config.ClientCAs = o.RootCACerts
+	if clientCAs != nil {
+		config.ClientCAs = clientCAs
 	}
 	return config, nil
 }
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index ab6ba59068ac..263bf7df418c 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -623,3 +623,59 @@ func TestWrapSyscallConn(t *testing.T) {
 			wrapConn)
 	}
 }
+
+func TestOptionsConfig(t *testing.T) {
+	serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"),
+		testdata.Path("server_key_1.pem"))
+	if err != nil {
+		t.Fatalf("Server is unable to parse peer certificates. Error: %v", err)
+	}
+	tests := []struct {
+		desc            string
+		clientVType     VerificationType
+		serverMutualTLS bool
+		serverCert      []tls.Certificate
+		serverVType     VerificationType
+	}{
+		{
+			desc:            "Client uses system-provided RootCAs; server uses system-provided ClientCAs",
+			clientVType:     CertVerification,
+			serverMutualTLS: true,
+			serverCert:      []tls.Certificate{serverPeerCert},
+			serverVType:     CertAndHostVerification,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			serverOptions := &ServerOptions{
+				Certificates:      test.serverCert,
+				RequireClientCert: test.serverMutualTLS,
+				VType:             test.serverVType,
+			}
+			serverConfig, err := serverOptions.config()
+			if err != nil {
+				t.Fatalf("Unable to generate serverConfig. Error: %v", err)
+			}
+			// Verify that the system-provided certificates would be used
+			// when no verification method was set in serverOptions.
+			if serverOptions.RootCACerts == nil && serverOptions.GetRootCAs == nil &&
+				serverOptions.RequireClientCert && serverConfig.ClientCAs == nil {
+				t.Fatalf("Failed to assign system-provided certificates on the server side.")
+			}
+			clientOptions := &ClientOptions{
+				VType: test.clientVType,
+			}
+			clientConfig, err := clientOptions.config()
+			if err != nil {
+				t.Fatalf("Unable to generate clientConfig. Error: %v", err)
+			}
+			// Verify that the system-provided certificates would be used
+			// when no verification method was set in clientOptions.
+			if clientOptions.RootCACerts == nil && clientOptions.GetRootCAs == nil &&
+				clientConfig.RootCAs == nil {
+				t.Fatalf("Failed to assign system-provided certificates on the client side.")
+			}
+		})
+	}
+}

From 3de8449f8555faf9e7196c89469a4d980ee81f6e Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 30 Jun 2020 12:04:42 -0700
Subject: [PATCH 106/481] xds: use logging components (#3718)

---
 internal/grpclog/prefixLogger.go              | 32 +++++++++++++++----
 .../balancer/cdsbalancer/cdsbalancer.go       |  2 +-
 xds/internal/balancer/cdsbalancer/logging.go  |  9 ++++--
 xds/internal/balancer/edsbalancer/eds.go      |  2 +-
 .../balancer/edsbalancer/eds_impl_priority.go |  6 ++--
 xds/internal/balancer/edsbalancer/logging.go  |  9 ++++--
 .../balancer/weightedtarget/logging.go        |  9 ++++--
 .../balancer/weightedtarget/weightedtarget.go |  2 +-
 xds/internal/client/bootstrap/logging.go      |  7 ++--
 xds/internal/client/client.go                 |  2 +-
 xds/internal/client/client_loadreport.go      |  5 +--
 xds/internal/client/client_logging.go         |  9 ++++--
 xds/internal/resolver/logging.go              |  9 ++++--
 xds/internal/resolver/xds_resolver.go         |  2 +-
 14 files changed, 73 insertions(+), 32 deletions(-)

diff --git a/internal/grpclog/prefixLogger.go b/internal/grpclog/prefixLogger.go
index f6e0dc1da8d2..82af70e96f15 100644
--- a/internal/grpclog/prefixLogger.go
+++ b/internal/grpclog/prefixLogger.go
@@ -18,10 +18,15 @@
 
 package grpclog
 
+import (
+	"fmt"
+)
+
 // PrefixLogger does logging with a prefix.
 //
 // Logging method on a nil logs without any prefix.
 type PrefixLogger struct {
+	logger DepthLoggerV2
 	prefix string
 }
 
@@ -30,34 +35,47 @@ func (pl *PrefixLogger) Infof(format string, args ...interface{}) {
 	if pl != nil {
 		// Handle nil, so the tests can pass in a nil logger.
 		format = pl.prefix + format
+		pl.logger.InfoDepth(1, fmt.Sprintf(format, args...))
+		return
 	}
-	Logger.Infof(format, args...)
+	InfoDepth(1, fmt.Sprintf(format, args...))
 }
 
 // Warningf does warning logging.
 func (pl *PrefixLogger) Warningf(format string, args ...interface{}) {
 	if pl != nil {
 		format = pl.prefix + format
+		pl.logger.WarningDepth(1, fmt.Sprintf(format, args...))
+		return
 	}
-	Logger.Warningf(format, args...)
+	WarningDepth(1, fmt.Sprintf(format, args...))
 }
 
 // Errorf does error logging.
 func (pl *PrefixLogger) Errorf(format string, args ...interface{}) {
 	if pl != nil {
 		format = pl.prefix + format
+		pl.logger.ErrorDepth(1, fmt.Sprintf(format, args...))
+		return
 	}
-	Logger.Errorf(format, args...)
+	ErrorDepth(1, fmt.Sprintf(format, args...))
 }
 
 // Debugf does info logging at verbose level 2.
 func (pl *PrefixLogger) Debugf(format string, args ...interface{}) {
-	if Logger.V(2) {
-		pl.Infof(format, args...)
+	if !Logger.V(2) {
+		return
+	}
+	if pl != nil {
+		// Handle nil, so the tests can pass in a nil logger.
+		format = pl.prefix + format
+		pl.logger.InfoDepth(1, fmt.Sprintf(format, args...))
+		return
 	}
+	InfoDepth(1, fmt.Sprintf(format, args...))
 }
 
 // NewPrefixLogger creates a prefix logger with the given prefix.
-func NewPrefixLogger(prefix string) *PrefixLogger {
-	return &PrefixLogger{prefix: prefix}
+func NewPrefixLogger(logger DepthLoggerV2, prefix string) *PrefixLogger {
+	return &PrefixLogger{logger: logger, prefix: prefix}
 }
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 8048a6fd0b80..c6ab764763aa 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -76,7 +76,7 @@ func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.
 		bOpts:    opts,
 		updateCh: buffer.NewUnbounded(),
 	}
-	b.logger = grpclog.NewPrefixLogger(loggingPrefix(b))
+	b.logger = prefixLogger((b))
 	b.logger.Infof("Created")
 	go b.run()
 	return b
diff --git a/xds/internal/balancer/cdsbalancer/logging.go b/xds/internal/balancer/cdsbalancer/logging.go
index 1b5369c1b460..e179bb1bf16d 100644
--- a/xds/internal/balancer/cdsbalancer/logging.go
+++ b/xds/internal/balancer/cdsbalancer/logging.go
@@ -20,10 +20,15 @@ package cdsbalancer
 
 import (
 	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 )
 
 const prefix = "[cds-lb %p] "
 
-func loggingPrefix(p *cdsBalancer) string {
-	return fmt.Sprintf(prefix, p)
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *cdsBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
 }
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 59bbe5fb4eec..5dc4a6a458ac 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -64,7 +64,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 		childPolicyUpdate: buffer.NewUnbounded(),
 	}
 	loadStore := lrs.NewStore()
-	x.logger = grpclog.NewPrefixLogger(loggingPrefix(x))
+	x.logger = prefixLogger((x))
 	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, loadStore, x.logger)
 	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.buildOpts, loadStore, x.logger)
 	x.logger.Infof("Created")
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index 66854e17ea42..3869bf594b47 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -25,10 +25,8 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/grpclog"
 )
 
-var logger = grpclog.Component("xds")
 var errAllPrioritiesRemoved = errors.New("eds: no locality is provided, all priorities are removed")
 
 // handlePriorityChange handles priority after EDS adds/removes a
@@ -135,13 +133,13 @@ func (edsImpl *edsBalancerImpl) handlePriorityWithNewState(priority priorityType
 	defer edsImpl.priorityMu.Unlock()
 
 	if !edsImpl.priorityInUse.isSet() {
-		logger.Infof("eds: received picker update when no priority is in use (EDS returned an empty list)")
+		edsImpl.logger.Infof("eds: received picker update when no priority is in use (EDS returned an empty list)")
 		return false
 	}
 
 	if edsImpl.priorityInUse.higherThan(priority) {
 		// Lower priorities should all be closed, this is an unexpected update.
-		logger.Infof("eds: received picker update from priority lower then priorityInUse")
+		edsImpl.logger.Infof("eds: received picker update from priority lower then priorityInUse")
 		return false
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/logging.go b/xds/internal/balancer/edsbalancer/logging.go
index 8a08e177f116..be4d0a512d16 100644
--- a/xds/internal/balancer/edsbalancer/logging.go
+++ b/xds/internal/balancer/edsbalancer/logging.go
@@ -20,10 +20,15 @@ package edsbalancer
 
 import (
 	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 )
 
 const prefix = "[eds-lb %p] "
 
-func loggingPrefix(p *edsBalancer) string {
-	return fmt.Sprintf(prefix, p)
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *edsBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
 }
diff --git a/xds/internal/balancer/weightedtarget/logging.go b/xds/internal/balancer/weightedtarget/logging.go
index 5f81da926fb8..e4061393071e 100644
--- a/xds/internal/balancer/weightedtarget/logging.go
+++ b/xds/internal/balancer/weightedtarget/logging.go
@@ -20,10 +20,15 @@ package weightedtarget
 
 import (
 	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 )
 
 const prefix = "[weighted-target-lb %p] "
 
-func loggingPrefix(p *weightedTargetBalancer) string {
-	return fmt.Sprintf(prefix, p)
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *weightedTargetBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
 }
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index f90f90f8b66d..a2ce14190a20 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -43,7 +43,7 @@ type weightedTargetBB struct{}
 
 func (wt *weightedTargetBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
 	b := &weightedTargetBalancer{}
-	b.logger = grpclog.NewPrefixLogger(loggingPrefix(b))
+	b.logger = prefixLogger((b))
 	b.bg = balancergroup.New(cc, nil, b.logger)
 	b.bg.Start()
 	b.logger.Infof("Created")
diff --git a/xds/internal/client/bootstrap/logging.go b/xds/internal/client/bootstrap/logging.go
index 0f8acda3958e..fdd811dd8a36 100644
--- a/xds/internal/client/bootstrap/logging.go
+++ b/xds/internal/client/bootstrap/logging.go
@@ -18,8 +18,11 @@
 
 package bootstrap
 
-import "google.golang.org/grpc/internal/grpclog"
+import (
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
 
 const prefix = "[xds-bootstrap] "
 
-var logger = grpclog.NewPrefixLogger(prefix)
+var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix)
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index bca195e8ce69..127734fd593a 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -128,7 +128,7 @@ func New(opts Options) (*Client, error) {
 		return nil, fmt.Errorf("xds: failed to dial balancer {%s}: %v", opts.Config.BalancerName, err)
 	}
 	c.cc = cc
-	c.logger = grpclog.NewPrefixLogger(loggingPrefix(c))
+	c.logger = prefixLogger((c))
 	c.logger.Infof("Created ClientConn to xDS server: %s", opts.Config.BalancerName)
 
 	c.v2c = newXDSV2Client(c, cc, opts.Config.NodeProto, backoff.DefaultExponential.Backoff, c.logger)
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go
index 314a47a8defb..8539e973511c 100644
--- a/xds/internal/client/client_loadreport.go
+++ b/xds/internal/client/client_loadreport.go
@@ -24,14 +24,11 @@ import (
 	"github.com/golang/protobuf/proto"
 	structpb "github.com/golang/protobuf/ptypes/struct"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
 )
 
 const nodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME"
 
-var logger = grpclog.Component("xds")
-
 // ReportLoad sends the load of the given clusterName from loadStore to the
 // given server. If the server is not an empty string, and is different from the
 // xds server, a new ClientConn will be created.
@@ -55,7 +52,7 @@ func (c *Client) ReportLoad(server string, clusterName string, loadStore lrs.Sto
 		ccNew, err := grpc.Dial(server, dopts...)
 		if err != nil {
 			// An error from a non-blocking dial indicates something serious.
-			logger.Infof("xds: failed to dial load report server {%s}: %v", server, err)
+			c.logger.Infof("xds: failed to dial load report server {%s}: %v", server, err)
 			return func() {}
 		}
 		cc = ccNew
diff --git a/xds/internal/client/client_logging.go b/xds/internal/client/client_logging.go
index db55b363c252..a47e5247fe22 100644
--- a/xds/internal/client/client_logging.go
+++ b/xds/internal/client/client_logging.go
@@ -20,10 +20,15 @@ package client
 
 import (
 	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 )
 
 const prefix = "[xds-client %p] "
 
-func loggingPrefix(p *Client) string {
-	return fmt.Sprintf(prefix, p)
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *Client) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
 }
diff --git a/xds/internal/resolver/logging.go b/xds/internal/resolver/logging.go
index 7c0de816fdc2..746b85af255f 100644
--- a/xds/internal/resolver/logging.go
+++ b/xds/internal/resolver/logging.go
@@ -20,10 +20,15 @@ package resolver
 
 import (
 	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 )
 
 const prefix = "[xds-resolver %p] "
 
-func loggingPrefix(p *xdsResolver) string {
-	return fmt.Sprintf(prefix, p)
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *xdsResolver) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
 }
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index cbba73bdbcda..690642094401 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -64,7 +64,7 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 		cc:       cc,
 		updateCh: make(chan suWithError, 1),
 	}
-	r.logger = grpclog.NewPrefixLogger(loggingPrefix(r))
+	r.logger = prefixLogger((r))
 	r.logger.Infof("Creating resolver for target: %+v", t)
 
 	if config.Creds == nil {

From 4258d12073b44ab3c6747c1b986681b3a648cc1d Mon Sep 17 00:00:00 2001
From: Aliaksandr Mianzhynski <amenzhinsky@gmail.com>
Date: Tue, 7 Jul 2020 03:56:02 +0300
Subject: [PATCH 107/481] service config: add default method config support
 (#3684)

---
 clientconn.go          | 19 ++++++-----
 clientconn_test.go     | 23 +++++++++++++
 service_config.go      | 45 ++++++++++++++++++-------
 service_config_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 143 insertions(+), 20 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index 3ed6034f1ad1..6dc5eda46f1a 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -860,9 +860,10 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool {
 // GetMethodConfig gets the method config of the input method.
 // If there's an exact match for input method (i.e. /service/method), we return
 // the corresponding MethodConfig.
-// If there isn't an exact match for the input method, we look for the default config
-// under the service (i.e /service/). If there is a default MethodConfig for
-// the service, we return it.
+// If there isn't an exact match for the input method, we look for the service's default
+// config under the service (i.e /service/) and then for the default for all services (empty string).
+//
+// If there is a default MethodConfig for the service, we return it.
 // Otherwise, we return an empty MethodConfig.
 func (cc *ClientConn) GetMethodConfig(method string) MethodConfig {
 	// TODO: Avoid the locking here.
@@ -871,12 +872,14 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig {
 	if cc.sc == nil {
 		return MethodConfig{}
 	}
-	m, ok := cc.sc.Methods[method]
-	if !ok {
-		i := strings.LastIndex(method, "/")
-		m = cc.sc.Methods[method[:i+1]]
+	if m, ok := cc.sc.Methods[method]; ok {
+		return m
+	}
+	i := strings.LastIndex(method, "/")
+	if m, ok := cc.sc.Methods[method[:i+1]]; ok {
+		return m
 	}
-	return m
+	return cc.sc.Methods[""]
 }
 
 func (cc *ClientConn) healthCheckConfig() *healthCheckConfig {
diff --git a/clientconn_test.go b/clientconn_test.go
index 9b95f6cec087..447a9bb14072 100644
--- a/clientconn_test.go
+++ b/clientconn_test.go
@@ -776,6 +776,29 @@ func (s) TestDisableServiceConfigOption(t *testing.T) {
 	}
 }
 
+func (s) TestMethodConfigDefaultService(t *testing.T) {
+	addr := "nonexist:///non.existent"
+	cc, err := Dial(addr, WithInsecure(), WithDefaultServiceConfig(`{
+  "methodConfig": [{
+    "name": [
+      {
+        "service": ""
+      }
+    ],
+    "waitForReady": true
+  }]
+}`))
+	if err != nil {
+		t.Fatalf("Dial(%s, _) = _, %v, want _, <nil>", addr, err)
+	}
+	defer cc.Close()
+
+	m := cc.GetMethodConfig("/foo/Bar")
+	if m.WaitForReady == nil {
+		t.Fatalf("want: method (%q) config to fallback to the default service", "/foo/Bar")
+	}
+}
+
 func (s) TestGetClientConnTarget(t *testing.T) {
 	addr := "nonexist:///non.existent"
 	cc, err := Dial(addr, WithInsecure())
diff --git a/service_config.go b/service_config.go
index c9f164c9ae43..5e434ca7f354 100644
--- a/service_config.go
+++ b/service_config.go
@@ -20,6 +20,7 @@ package grpc
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -224,19 +225,27 @@ func parseDuration(s *string) (*time.Duration, error) {
 }
 
 type jsonName struct {
-	Service *string
-	Method  *string
+	Service string
+	Method  string
 }
 
-func (j jsonName) generatePath() (string, bool) {
-	if j.Service == nil {
-		return "", false
+var (
+	errDuplicatedName             = errors.New("duplicated name")
+	errEmptyServiceNonEmptyMethod = errors.New("cannot combine empty 'service' and non-empty 'method'")
+)
+
+func (j jsonName) generatePath() (string, error) {
+	if j.Service == "" {
+		if j.Method != "" {
+			return "", errEmptyServiceNonEmptyMethod
+		}
+		return "", nil
 	}
-	res := "/" + *j.Service + "/"
-	if j.Method != nil {
-		res += *j.Method
+	res := "/" + j.Service + "/"
+	if j.Method != "" {
+		res += j.Method
 	}
-	return res, true
+	return res, nil
 }
 
 // TODO(lyuxuan): delete this struct after cleaning up old service config implementation.
@@ -288,6 +297,8 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 	if rsc.MethodConfig == nil {
 		return &serviceconfig.ParseResult{Config: &sc}
 	}
+
+	paths := map[string]struct{}{}
 	for _, m := range *rsc.MethodConfig {
 		if m.Name == nil {
 			continue
@@ -320,10 +331,20 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 				mc.MaxRespSize = newInt(int(*m.MaxResponseMessageBytes))
 			}
 		}
-		for _, n := range *m.Name {
-			if path, valid := n.generatePath(); valid {
-				sc.Methods[path] = mc
+		for i, n := range *m.Name {
+			path, err := n.generatePath()
+			if err != nil {
+				logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to methodConfig[%d]: %v", js, i, err)
+				return &serviceconfig.ParseResult{Err: err}
+			}
+
+			if _, ok := paths[path]; ok {
+				err = errDuplicatedName
+				logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to methodConfig[%d]: %v", js, i, err)
+				return &serviceconfig.ParseResult{Err: err}
 			}
+			paths[path] = struct{}{}
+			sc.Methods[path] = mc
 		}
 	}
 
diff --git a/service_config_test.go b/service_config_test.go
index ec9f56db2c71..b3c6988e8d97 100644
--- a/service_config_test.go
+++ b/service_config_test.go
@@ -372,6 +372,82 @@ func (s) TestParseMsgSize(t *testing.T) {
 
 	runParseTests(t, testcases)
 }
+func (s) TestParseDefaultMethodConfig(t *testing.T) {
+	dc := &ServiceConfig{
+		Methods: map[string]MethodConfig{
+			"": {WaitForReady: newBool(true)},
+		},
+	}
+
+	runParseTests(t, []parseTestCase{
+		{
+			`{
+  "methodConfig": [{
+    "name": [{}],
+    "waitForReady": true
+  }]
+}`,
+			dc,
+			false,
+		},
+		{
+			`{
+  "methodConfig": [{
+    "name": [{"service": null}],
+    "waitForReady": true
+  }]
+}`,
+			dc,
+			false,
+		},
+		{
+			`{
+  "methodConfig": [{
+    "name": [{"service": ""}],
+    "waitForReady": true
+  }]
+}`,
+			dc,
+			false,
+		},
+		{
+			`{
+  "methodConfig": [{
+    "name": [{"method": "Bar"}],
+    "waitForReady": true
+  }]
+}`,
+			nil,
+			true,
+		},
+		{
+			`{
+  "methodConfig": [{
+    "name": [{"service": "", "method": "Bar"}],
+    "waitForReady": true
+  }]
+}`,
+			nil,
+			true,
+		},
+	})
+}
+
+func (s) TestParseMethodConfigDuplicatedName(t *testing.T) {
+	runParseTests(t, []parseTestCase{
+		{
+			`{
+  "methodConfig": [{
+    "name": [
+      {"service": "foo"},
+      {"service": "foo"}
+    ],
+    "waitForReady": true
+  }]
+}`, nil, true,
+		},
+	})
+}
 
 func (s) TestParseDuration(t *testing.T) {
 	testCases := []struct {

From 9af290fac4b25c4f083f6b787e992096acbe516b Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 8 Jul 2020 17:41:40 -0700
Subject: [PATCH 108/481] balancergroup: move balancer state (connectivity
 state and picker) building out (#3685)

---
 .../balancer/balancergroup/balancergroup.go   | 247 +++-------------
 .../balancergroup/balancergroup_test.go       | 176 +++++++-----
 .../balancergroup/balancerstateaggregator.go  |  38 +++
 .../balancer/balancergroup/testutils_test.go  |   5 -
 xds/internal/balancer/edsbalancer/eds_impl.go |  31 +-
 .../balancer/edsbalancer/eds_impl_priority.go |   5 +-
 .../balancer/edsbalancer/eds_impl_test.go     |   1 -
 .../weightedaggregator/aggregator.go          | 264 ++++++++++++++++++
 .../balancer/weightedtarget/weightedtarget.go |  53 ++--
 9 files changed, 509 insertions(+), 311 deletions(-)
 create mode 100644 xds/internal/balancer/balancergroup/balancerstateaggregator.go
 create mode 100644 xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index 0aa30a2663fc..84de4b63ecfe 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -25,11 +25,9 @@ import (
 
 	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/cache"
 	"google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
@@ -154,21 +152,6 @@ func (sbc *subBalancerWrapper) stopBalancer() {
 	sbc.balancer = nil
 }
 
-type pickerState struct {
-	weight uint32
-	state  balancer.State
-	// stateToAggregate is the connectivity state used only for state
-	// aggregation. It could be different from state.ConnectivityState. For
-	// example when a sub-balancer transitions from TransientFailure to
-	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
-	// is still TransientFailure.
-	stateToAggregate connectivity.State
-}
-
-func (s *pickerState) String() string {
-	return fmt.Sprintf("weight:%v,picker:%p,state:%v,stateToAggregate:%v", s.weight, s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
-}
-
 // BalancerGroup takes a list of balancers, and make them into one balancer.
 //
 // Note that this struct doesn't implement balancer.Balancer, because it's not
@@ -185,8 +168,9 @@ func (s *pickerState) String() string {
 // Actions from sub-balances are forwarded to parent ClientConn
 //  - new/remove SubConn
 //  - picker update and health states change
-//     - sub-pickers are grouped into a group-picker
-//     - aggregated connectivity state is the overall state of all pickers.
+//     - sub-pickers are sent to an aggregator provided by the parent, which
+//     will group them into a group-picker. The aggregated connectivity state is
+//     also handled by the aggregator.
 //  - resolveNow
 //
 // Sub-balancers are only built when the balancer group is started. If the
@@ -198,6 +182,11 @@ type BalancerGroup struct {
 	logger    *grpclog.PrefixLogger
 	loadStore lrs.Store
 
+	// stateAggregator is where the state/picker updates will be sent to. It's
+	// provided by the parent balancer, to build a picker with all the
+	// sub-pickers.
+	stateAggregator BalancerStateAggregator
+
 	// outgoingMu guards all operations in the direction:
 	// ClientConn-->Sub-balancer. Including start, stop, resolver updates and
 	// SubConn state changes.
@@ -210,8 +199,8 @@ type BalancerGroup struct {
 	// Cache for sub-balancers when they are removed.
 	balancerCache *cache.TimeoutCache
 
-	// incomingMu and pickerMu are to make sure this balancer group doesn't send
-	// updates to cc after it's closed.
+	// incomingMu is to make sure this balancer group doesn't send updates to cc
+	// after it's closed.
 	//
 	// We don't share the mutex to avoid deadlocks (e.g. a call to sub-balancer
 	// may call back to balancer group inline. It causes deaclock if they
@@ -220,25 +209,22 @@ type BalancerGroup struct {
 	// We should never need to hold multiple locks at the same time in this
 	// struct. The case where two locks are held can only happen when the
 	// underlying balancer calls back into balancer group inline. So there's an
-	// implicit lock acquisition order that outgoingMu is locked before either
-	// incomingMu or pickerMu.
+	// implicit lock acquisition order that outgoingMu is locked before
+	// incomingMu.
 
 	// incomingMu guards all operations in the direction:
-	// Sub-balancer-->ClientConn. Including NewSubConn, RemoveSubConn, and
-	// updatePicker. It also guards the map from SubConn to balancer ID, so
-	// updateSubConnState needs to hold it shortly to find the
-	// sub-balancer to forward the update.
+	// Sub-balancer-->ClientConn. Including NewSubConn, RemoveSubConn. It also
+	// guards the map from SubConn to balancer ID, so updateSubConnState needs
+	// to hold it shortly to find the sub-balancer to forward the update.
+	//
+	// UpdateState is called by the balancer state aggretator, and it will
+	// decide when and whether to call.
 	//
 	// The corresponding boolean incomingStarted is used to stop further updates
 	// from sub-balancers after they are closed.
 	incomingMu      sync.Mutex
 	incomingStarted bool // This boolean only guards calls back to ClientConn.
 	scToSubBalancer map[balancer.SubConn]*subBalancerWrapper
-	// All balancer IDs exist as keys in this map, even if balancer group is not
-	// started.
-	//
-	// If an ID is not in map, it's either removed or never added.
-	idToPickerState map[internal.LocalityID]*pickerState
 }
 
 // DefaultSubBalancerCloseTimeout is defined as a variable instead of const for
@@ -249,16 +235,17 @@ var DefaultSubBalancerCloseTimeout = 15 * time.Minute
 
 // New creates a new BalancerGroup. Note that the BalancerGroup
 // needs to be started to work.
-func New(cc balancer.ClientConn, loadStore lrs.Store, logger *grpclog.PrefixLogger) *BalancerGroup {
+func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore lrs.Store, logger *grpclog.PrefixLogger) *BalancerGroup {
 	return &BalancerGroup{
 		cc:        cc,
 		logger:    logger,
 		loadStore: loadStore,
 
+		stateAggregator: stateAggregator,
+
 		idToBalancerConfig: make(map[internal.LocalityID]*subBalancerWrapper),
 		balancerCache:      cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout),
 		scToSubBalancer:    make(map[balancer.SubConn]*subBalancerWrapper),
-		idToPickerState:    make(map[internal.LocalityID]*pickerState),
 	}
 }
 
@@ -286,31 +273,8 @@ func (bg *BalancerGroup) Start() {
 	bg.outgoingMu.Unlock()
 }
 
-// Add adds a balancer built by builder to the group, with given id and weight.
-//
-// weight should never be zero.
-func (bg *BalancerGroup) Add(id internal.LocalityID, weight uint32, builder balancer.Builder) {
-	if weight == 0 {
-		bg.logger.Errorf("BalancerGroup.add called with weight 0, locality: %v. Locality is not added to balancer group", id)
-		return
-	}
-
-	// First, add things to the picker map. Do this even if incomingStarted is
-	// false, because the data is static.
-	bg.incomingMu.Lock()
-	bg.idToPickerState[id] = &pickerState{
-		weight: weight,
-		// Start everything in CONNECTING, so if one of the sub-balancers
-		// reports TransientFailure, the RPCs will still wait for the other
-		// sub-balancers.
-		state: balancer.State{
-			ConnectivityState: connectivity.Connecting,
-			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
-		},
-		stateToAggregate: connectivity.Connecting,
-	}
-	bg.incomingMu.Unlock()
-
+// Add adds a balancer built by builder to the group, with given id.
+func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) {
 	// Store data in static map, and then check to see if bg is started.
 	bg.outgoingMu.Lock()
 	var sbc *subBalancerWrapper
@@ -361,9 +325,6 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, weight uint32, builder bala
 // But doesn't close the balancer. The balancer is kept in a cache, and will be
 // closed after timeout. Cleanup work (closing sub-balancer and removing
 // subconns) will be done after timeout.
-//
-// It also removes the picker generated from this balancer from the picker
-// group. It always results in a picker update.
 func (bg *BalancerGroup) Remove(id internal.LocalityID) {
 	bg.outgoingMu.Lock()
 	if sbToRemove, ok := bg.idToBalancerConfig[id]; ok {
@@ -385,18 +346,6 @@ func (bg *BalancerGroup) Remove(id internal.LocalityID) {
 		bg.logger.Infof("balancer group: trying to remove a non-existing locality from balancer group: %v", id)
 	}
 	bg.outgoingMu.Unlock()
-
-	bg.incomingMu.Lock()
-	// Remove id and picker from picker map. This also results in future updates
-	// for this ID to be ignored.
-	delete(bg.idToPickerState, id)
-	if bg.incomingStarted {
-		// Normally picker update is triggered by SubConn state change. But we
-		// want to update state and picker to reflect the changes, too. Because
-		// we don't want `ClientConn` to pick this sub-balancer anymore.
-		bg.cc.UpdateState(buildPickerAndState(bg.idToPickerState))
-	}
-	bg.incomingMu.Unlock()
 }
 
 // bg.remove(id) doesn't do cleanup for the sub-balancer. This function does
@@ -420,36 +369,6 @@ func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWrapper) {
 	bg.incomingMu.Unlock()
 }
 
-// ChangeWeight changes the weight of the balancer.
-//
-// newWeight should never be zero.
-//
-// NOTE: It always results in a picker update now. This probably isn't
-// necessary. But it seems better to do the update because it's a change in the
-// picker (which is balancer's snapshot).
-func (bg *BalancerGroup) ChangeWeight(id internal.LocalityID, newWeight uint32) {
-	if newWeight == 0 {
-		bg.logger.Errorf("BalancerGroup.changeWeight called with newWeight 0. Weight is not changed")
-		return
-	}
-	bg.incomingMu.Lock()
-	defer bg.incomingMu.Unlock()
-	pState, ok := bg.idToPickerState[id]
-	if !ok {
-		return
-	}
-	if pState.weight == newWeight {
-		return
-	}
-	pState.weight = newWeight
-	if bg.incomingStarted {
-		// Normally picker update is triggered by SubConn state change. But we
-		// want to update state and picker to reflect the changes, too. Because
-		// `ClientConn` should do pick with the new weights now.
-		bg.cc.UpdateState(buildPickerAndState(bg.idToPickerState))
-	}
-}
-
 // Following are actions from the parent grpc.ClientConn, forward to sub-balancers.
 
 // UpdateSubConnState handles the state for the subconn. It finds the
@@ -520,35 +439,22 @@ func (bg *BalancerGroup) newSubConn(config *subBalancerWrapper, addrs []resolver
 	return sc, nil
 }
 
-// updateBalancerState: create an aggregated picker and an aggregated
-// connectivity state, then forward to ClientConn.
+// updateBalancerState: forward the new state to balancer state aggregator. The
+// aggregator will create an aggregated picker and an aggregated connectivity
+// state, then forward to ClientConn.
 func (bg *BalancerGroup) updateBalancerState(id internal.LocalityID, state balancer.State) {
 	bg.logger.Infof("Balancer state update from locality %v, new state: %+v", id, state)
-
-	bg.incomingMu.Lock()
-	defer bg.incomingMu.Unlock()
-	pickerSt, ok := bg.idToPickerState[id]
-	if !ok {
-		// All state starts in IDLE. If ID is not in map, it's either removed,
-		// or never existed.
-		bg.logger.Warningf("balancer group: pickerState for %v not found when update picker/state", id)
-		return
-	}
 	if bg.loadStore != nil {
 		// Only wrap the picker to do load reporting if loadStore was set.
 		state.Picker = newLoadReportPicker(state.Picker, id, bg.loadStore)
 	}
-	if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) {
-		// If old state is TransientFailure, and new state is Connecting, don't
-		// update the state, to prevent the aggregated state from being always
-		// CONNECTING. Otherwise, stateToAggregate is the same as
-		// state.ConnectivityState.
-		pickerSt.stateToAggregate = state.ConnectivityState
-	}
-	pickerSt.state = state
-	if bg.incomingStarted {
-		bg.logger.Infof("Child pickers with weight: %+v", bg.idToPickerState)
-		bg.cc.UpdateState(buildPickerAndState(bg.idToPickerState))
+
+	// Send new state to the aggregator, without holding the incomingMu.
+	// incomingMu is to protect all calls to the parent ClientConn, this update
+	// doesn't necessary trigger a call to ClientConn, and should already be
+	// protected by aggregator's mutex if necessary.
+	if bg.stateAggregator != nil {
+		bg.stateAggregator.UpdateState(id, state)
 	}
 }
 
@@ -558,17 +464,6 @@ func (bg *BalancerGroup) Close() {
 	bg.incomingMu.Lock()
 	if bg.incomingStarted {
 		bg.incomingStarted = false
-
-		for _, pState := range bg.idToPickerState {
-			// Reset everything to init state (Connecting) but keep the entry in
-			// map (to keep the weight).
-			pState.state = balancer.State{
-				ConnectivityState: connectivity.Connecting,
-				Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
-			}
-			pState.stateToAggregate = connectivity.Connecting
-		}
-
 		// Also remove all SubConns.
 		for sc := range bg.scToSubBalancer {
 			bg.cc.RemoveSubConn(sc)
@@ -590,82 +485,6 @@ func (bg *BalancerGroup) Close() {
 	bg.balancerCache.Clear(true)
 }
 
-func buildPickerAndState(m map[internal.LocalityID]*pickerState) balancer.State {
-	var readyN, connectingN int
-	readyPickerWithWeights := make([]pickerState, 0, len(m))
-	for _, ps := range m {
-		switch ps.stateToAggregate {
-		case connectivity.Ready:
-			readyN++
-			readyPickerWithWeights = append(readyPickerWithWeights, *ps)
-		case connectivity.Connecting:
-			connectingN++
-		}
-	}
-	var aggregatedState connectivity.State
-	switch {
-	case readyN > 0:
-		aggregatedState = connectivity.Ready
-	case connectingN > 0:
-		aggregatedState = connectivity.Connecting
-	default:
-		aggregatedState = connectivity.TransientFailure
-	}
-
-	// Make sure picker's return error is consistent with the aggregatedState.
-	//
-	// TODO: This is true for balancers like weighted_target, but not for
-	// routing. For routing, we want to always build picker with all sub-pickers
-	// (not even ready sub-pickers), so even if the overall state is Ready, pick
-	// for certain RPCs can behave like Connecting or TransientFailure.
-	var picker balancer.Picker
-	switch aggregatedState {
-	case connectivity.TransientFailure:
-		picker = base.NewErrPicker(balancer.ErrTransientFailure)
-	case connectivity.Connecting:
-		picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable)
-	default:
-		picker = newPickerGroup(readyPickerWithWeights)
-	}
-	return balancer.State{ConnectivityState: aggregatedState, Picker: picker}
-}
-
-// NewRandomWRR is the WRR constructor used to pick sub-pickers from
-// sub-balancers. It's to be modified in tests.
-var NewRandomWRR = wrr.NewRandom
-
-type pickerGroup struct {
-	length int
-	w      wrr.WRR
-}
-
-// newPickerGroup takes pickers with weights, and group them into one picker.
-//
-// Note it only takes ready pickers. The map shouldn't contain non-ready
-// pickers.
-//
-// TODO: (bg) confirm this is the expected behavior: non-ready balancers should
-// be ignored when picking. Only ready balancers are picked.
-func newPickerGroup(readyPickerWithWeights []pickerState) *pickerGroup {
-	w := NewRandomWRR()
-	for _, ps := range readyPickerWithWeights {
-		w.Add(ps.state.Picker, int64(ps.weight))
-	}
-
-	return &pickerGroup{
-		length: len(readyPickerWithWeights),
-		w:      w,
-	}
-}
-
-func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
-	if pg.length <= 0 {
-		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
-	}
-	p := pg.w.Next().(balancer.Picker)
-	return p.Pick(info)
-}
-
 const (
 	serverLoadCPUName    = "cpu_utilization"
 	serverLoadMemoryName = "mem_utilization"
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 8fda50bcb595..347404989d01 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -14,6 +14,15 @@
  * limitations under the License.
  */
 
+// All tests in this file are combination of balancer group and
+// weighted_balancerstate_aggregator, aka weighted_target tests. The difference
+// is weighted_target tests cannot add sub-balancers to balancer group directly,
+// they instead uses balancer config to control sub-balancers. Even though not
+// very suited, the tests still cover all the functionality.
+//
+// TODO: the tests should be moved to weighted_target, and balancer group's
+// tests should use a mock balancerstate_aggregator.
+
 package balancergroup
 
 import (
@@ -29,6 +38,8 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/balancer/lrs"
+	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -58,14 +69,22 @@ func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
 	}
 }
 
-// 1 balancer, 1 backend -> 2 backends -> 1 backend.
-func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
+func newTestBalancerGroup(t *testing.T, loadStore lrs.Store) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
 	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
+	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
+	gator.Start()
+	bg := New(cc, gator, loadStore, nil)
 	bg.Start()
+	return cc, gator, bg
+}
+
+// 1 balancer, 1 backend -> 2 backends -> 1 backend.
+func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add one balancer to group.
-	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	gator.Add(testBalancerIDs[0], 1)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	// Send one resolved address.
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 
@@ -117,17 +136,17 @@ func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
 
 // 2 balancers, each with 1 backend.
 func (s) TestBalancerGroup_TwoRR_OneBackend(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	gator.Add(testBalancerIDs[0], 1)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc1 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc2 := <-cc.NewSubConnCh
 
@@ -147,18 +166,18 @@ func (s) TestBalancerGroup_TwoRR_OneBackend(t *testing.T) {
 
 // 2 balancers, each with more than 1 backends.
 func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	gator.Add(testBalancerIDs[0], 1)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
@@ -232,18 +251,18 @@ func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
 
 // 2 balancers with different weights.
 func (s) TestBalancerGroup_TwoRR_DifferentWeight_MoreBackends(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	gator.Add(testBalancerIDs[0], 2)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
@@ -268,21 +287,22 @@ func (s) TestBalancerGroup_TwoRR_DifferentWeight_MoreBackends(t *testing.T) {
 
 // totally 3 balancers, add/remove balancer.
 func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add three balancers to group and send one resolved address to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	gator.Add(testBalancerIDs[0], 1)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc1 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}})
 	sc2 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[2], 1, rrBuilder)
+	gator.Add(testBalancerIDs[2], 1)
+	bg.Add(testBalancerIDs[2], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}})
 	sc3 := <-cc.NewSubConnCh
 
@@ -301,7 +321,9 @@ func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
 	}
 
 	// Remove the second balancer, while the others two are ready.
+	gator.Remove(testBalancerIDs[1])
 	bg.Remove(testBalancerIDs[1])
+	gator.BuildAndUpdate()
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
@@ -315,7 +337,9 @@ func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
 	// move balancer 3 into transient failure.
 	bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
 	// Remove the first balancer, while the third is transient failure.
+	gator.Remove(testBalancerIDs[0])
 	bg.Remove(testBalancerIDs[0])
+	gator.BuildAndUpdate()
 	scToRemove = <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
 		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
@@ -330,18 +354,18 @@ func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
 
 // 2 balancers, change balancer weight.
 func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	gator.Add(testBalancerIDs[0], 2)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
@@ -363,7 +387,8 @@ func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
-	bg.ChangeWeight(testBalancerIDs[0], 3)
+	gator.UpdateWeight(testBalancerIDs[0], 3)
+	gator.BuildAndUpdate()
 
 	// Test roundrobin with new weight.
 	p2 := <-cc.NewPickerCh
@@ -375,23 +400,22 @@ func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
 
 func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	testLoadStore := testutils.NewTestLoadStore()
-
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, testLoadStore, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, testLoadStore)
 
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	gator.Add(testBalancerIDs[0], 2)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	sc1 := <-cc.NewSubConnCh
 	sc2 := <-cc.NewSubConnCh
 	backendToBalancerID[sc1] = testBalancerIDs[0]
 	backendToBalancerID[sc2] = testBalancerIDs[0]
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 	sc3 := <-cc.NewSubConnCh
 	sc4 := <-cc.NewSubConnCh
@@ -459,13 +483,17 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 // Start the balancer group again and check for behavior.
 func (s) TestBalancerGroup_start_close(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
+	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
+	gator.Start()
+	bg := New(cc, gator, nil, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	gator.Add(testBalancerIDs[0], 2)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
@@ -490,22 +518,26 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
+	gator.Stop()
 	bg.Close()
 	for i := 0; i < 4; i++ {
 		bg.UpdateSubConnState(<-cc.RemoveSubConnCh, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
 	}
 
 	// Add b3, weight 1, backends [1,2].
-	bg.Add(testBalancerIDs[2], 1, rrBuilder)
+	gator.Add(testBalancerIDs[2], 1)
+	bg.Add(testBalancerIDs[2], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:3]}})
 
 	// Remove b1.
+	gator.Remove(testBalancerIDs[0])
 	bg.Remove(testBalancerIDs[0])
 
 	// Update b2 to weight 3, backends [0,3].
-	bg.ChangeWeight(testBalancerIDs[1], 3)
+	gator.UpdateWeight(testBalancerIDs[1], 3)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3])}})
 
+	gator.Start()
 	bg.Start()
 
 	m2 := make(map[resolver.Address]balancer.SubConn)
@@ -543,11 +575,15 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 // because of deadlock.
 func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
+	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
+	gator.Start()
+	bg := New(cc, gator, nil, nil)
 
-	bg.Add(testBalancerIDs[0], 2, &testutils.TestConstBalancerBuilder{})
+	gator.Add(testBalancerIDs[0], 2)
+	bg.Add(testBalancerIDs[0], &testutils.TestConstBalancerBuilder{})
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
-	bg.Add(testBalancerIDs[1], 1, &testutils.TestConstBalancerBuilder{})
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], &testutils.TestConstBalancerBuilder{})
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
@@ -557,17 +593,17 @@ func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
 // transient_failure, the picks won't fail with transient_failure, and should
 // instead wait for the other sub-balancer.
 func (s) TestBalancerGroup_InitOneSubBalancerTransientFailure(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 1, rrBuilder)
+	gator.Add(testBalancerIDs[0], 1)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc1 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	<-cc.NewSubConnCh
 
@@ -588,17 +624,17 @@ func (s) TestBalancerGroup_InitOneSubBalancerTransientFailure(t *testing.T) {
 // connecting, the overall state stays in transient_failure, and all picks
 // return transient failure error.
 func (s) TestBalancerGroup_SubBalancerTurnsConnectingFromTransientFailure(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
-	bg.Start()
+	cc, gator, bg := newTestBalancerGroup(t, nil)
 
 	// Add two balancers to group and send one resolved address to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 1, pfBuilder)
+	gator.Add(testBalancerIDs[0], 1)
+	bg.Add(testBalancerIDs[0], pfBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc1 := <-cc.NewSubConnCh
 
-	bg.Add(testBalancerIDs[1], 1, pfBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], pfBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}})
 	sc2 := <-cc.NewSubConnCh
 
@@ -639,15 +675,19 @@ func replaceDefaultSubBalancerCloseTimeout(n time.Duration) func() {
 // Two rr balancers are added to bg, each with 2 ready subConns. A sub-balancer
 // is removed later, so the balancer group returned has one sub-balancer in its
 // own map, and one sub-balancer in cache.
-func initBalancerGroupForCachingTest(t *testing.T) (*BalancerGroup, *testutils.TestClientConn, map[resolver.Address]balancer.SubConn) {
+func initBalancerGroupForCachingTest(t *testing.T) (*weightedaggregator.Aggregator, *BalancerGroup, *testutils.TestClientConn, map[resolver.Address]balancer.SubConn) {
 	cc := testutils.NewTestClientConn(t)
-	bg := New(cc, nil, nil)
+	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
+	gator.Start()
+	bg := New(cc, gator, nil, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
-	bg.Add(testBalancerIDs[0], 2, rrBuilder)
+	gator.Add(testBalancerIDs[0], 2)
+	bg.Add(testBalancerIDs[0], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
@@ -672,7 +712,9 @@ func initBalancerGroupForCachingTest(t *testing.T) (*BalancerGroup, *testutils.T
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
+	gator.Remove(testBalancerIDs[1])
 	bg.Remove(testBalancerIDs[1])
+	gator.BuildAndUpdate()
 	// Don't wait for SubConns to be removed after close, because they are only
 	// removed after close timeout.
 	for i := 0; i < 10; i++ {
@@ -692,14 +734,14 @@ func initBalancerGroupForCachingTest(t *testing.T) (*BalancerGroup, *testutils.T
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
-	return bg, cc, m1
+	return gator, bg, cc, m1
 }
 
 // Test that if a sub-balancer is removed, and re-added within close timeout,
 // the subConns won't be re-created.
 func (s) TestBalancerGroup_locality_caching(t *testing.T) {
 	defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)()
-	bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
+	gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
 
 	// Turn down subconn for addr2, shouldn't get picker update because
 	// sub-balancer1 was removed.
@@ -719,7 +761,8 @@ func (s) TestBalancerGroup_locality_caching(t *testing.T) {
 	// Re-add sub-balancer-1, because subconns were in cache, no new subconns
 	// should be created. But a new picker will still be generated, with subconn
 	// states update to date.
-	bg.Add(testBalancerIDs[1], 1, rrBuilder)
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], rrBuilder)
 
 	p3 := <-cc.NewPickerCh
 	want := []balancer.SubConn{
@@ -747,7 +790,7 @@ func (s) TestBalancerGroup_locality_caching(t *testing.T) {
 // immediately.
 func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) {
 	defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)()
-	bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
+	_, bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
 
 	bg.Close()
 	// The balancer group is closed. The subconns should be removed immediately.
@@ -776,7 +819,7 @@ func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) {
 // subConns will be removed.
 func (s) TestBalancerGroup_locality_caching_not_readd_within_timeout(t *testing.T) {
 	defer replaceDefaultSubBalancerCloseTimeout(time.Second)()
-	_, cc, addrToSC := initBalancerGroupForCachingTest(t)
+	_, _, cc, addrToSC := initBalancerGroupForCachingTest(t)
 
 	// The sub-balancer is not re-added withtin timeout. The subconns should be
 	// removed.
@@ -808,13 +851,14 @@ type noopBalancerBuilderWrapper struct {
 // builder. Old subconns should be removed, and new subconns should be created.
 func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *testing.T) {
 	defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)()
-	bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
+	gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t)
 
 	// Re-add sub-balancer-1, but with a different balancer builder. The
 	// sub-balancer was still in cache, but cann't be reused. This should cause
 	// old sub-balancer's subconns to be removed immediately, and new subconns
 	// to be created.
-	bg.Add(testBalancerIDs[1], 1, &noopBalancerBuilderWrapper{rrBuilder})
+	gator.Add(testBalancerIDs[1], 1)
+	bg.Add(testBalancerIDs[1], &noopBalancerBuilderWrapper{rrBuilder})
 
 	// The cached sub-balancer should be closed, and the subconns should be
 	// removed immediately.
diff --git a/xds/internal/balancer/balancergroup/balancerstateaggregator.go b/xds/internal/balancer/balancergroup/balancerstateaggregator.go
new file mode 100644
index 000000000000..0a555f3d9ba2
--- /dev/null
+++ b/xds/internal/balancer/balancergroup/balancerstateaggregator.go
@@ -0,0 +1,38 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package balancergroup
+
+import (
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/xds/internal"
+)
+
+// BalancerStateAggregator aggregates sub-picker and connectivity states into a
+// state.
+//
+// It takes care of merging sub-picker into one picker. The picking config is
+// passed directly from the the parent to the aggregator implementation (instead
+// via balancer group).
+type BalancerStateAggregator interface {
+	// UpdateState updates the state of the id.
+	//
+	// It's up to the implementation whether this will trigger an update to the
+	// parent ClientConn.
+	UpdateState(id internal.LocalityID, state balancer.State)
+}
diff --git a/xds/internal/balancer/balancergroup/testutils_test.go b/xds/internal/balancer/balancergroup/testutils_test.go
index bca79bf180af..1429fa87b3f2 100644
--- a/xds/internal/balancer/balancergroup/testutils_test.go
+++ b/xds/internal/balancer/balancergroup/testutils_test.go
@@ -22,7 +22,6 @@ import (
 	"testing"
 
 	"google.golang.org/grpc/internal/grpctest"
-	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 type s struct {
@@ -32,7 +31,3 @@ type s struct {
 func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
-
-func init() {
-	NewRandomWRR = testutils.NewTestWRR
-}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 979eee45bb76..95172cfb0ab9 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
+	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
@@ -49,8 +50,9 @@ type localityConfig struct {
 // balancerGroupWithConfig contains the localities with the same priority. It
 // manages all localities using a balancerGroup.
 type balancerGroupWithConfig struct {
-	bg      *balancergroup.BalancerGroup
-	configs map[internal.LocalityID]*localityConfig
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *weightedaggregator.Aggregator
+	configs         map[internal.LocalityID]*localityConfig
 }
 
 // edsBalancerImpl does load balancing based on the EDS responses. Note that it
@@ -141,10 +143,12 @@ func (edsImpl *edsBalancerImpl) handleChildPolicy(name string, config json.RawMe
 			//  switching sub-balancers (keep old balancer around until new
 			//  balancer becomes ready).
 			bgwc.bg.Remove(id)
-			bgwc.bg.Add(id, config.weight, edsImpl.subBalancerBuilder)
+			bgwc.bg.Add(id, edsImpl.subBalancerBuilder)
 			bgwc.bg.UpdateClientConnState(id, balancer.ClientConnState{
 				ResolverState: resolver.State{Addresses: config.addrs},
 			})
+			// This doesn't need to manually update picker, because the new
+			// sub-balancer will send it's picker later.
 		}
 	}
 }
@@ -233,9 +237,12 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
 			// time this priority is received). We don't start it here. It may
 			// be started when necessary (e.g. when higher is down, or if it's a
 			// new lowest priority).
+			ccPriorityWrapper := edsImpl.ccWrapperWithPriority(priority)
+			stateAggregator := weightedaggregator.New(ccPriorityWrapper, edsImpl.logger, newRandomWRR)
 			bgwc = &balancerGroupWithConfig{
-				bg:      balancergroup.New(edsImpl.ccWrapperWithPriority(priority), edsImpl.loadStore, edsImpl.logger),
-				configs: make(map[internal.LocalityID]*localityConfig),
+				bg:              balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.loadStore, edsImpl.logger),
+				stateAggregator: stateAggregator,
+				configs:         make(map[internal.LocalityID]*localityConfig),
 			}
 			edsImpl.priorityToLocalities[priority] = bgwc
 			priorityChanged = true
@@ -270,6 +277,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 	// for the same priority. It's used to delete localities that are removed in
 	// the new EDS response.
 	newLocalitiesSet := make(map[internal.LocalityID]struct{})
+	var rebuildStateAndPicker bool
 	for _, locality := range newLocalities {
 		// One balancer for each locality.
 
@@ -308,7 +316,8 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 		config, ok := bgwc.configs[lid]
 		if !ok {
 			// A new balancer, add it to balancer group and balancer map.
-			bgwc.bg.Add(lid, newWeight, edsImpl.subBalancerBuilder)
+			bgwc.stateAggregator.Add(lid, newWeight)
+			bgwc.bg.Add(lid, edsImpl.subBalancerBuilder)
 			config = &localityConfig{
 				weight: newWeight,
 			}
@@ -331,7 +340,8 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 
 		if weightChanged {
 			config.weight = newWeight
-			bgwc.bg.ChangeWeight(lid, newWeight)
+			bgwc.stateAggregator.UpdateWeight(lid, newWeight)
+			rebuildStateAndPicker = true
 		}
 
 		if addrsChanged {
@@ -345,11 +355,17 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 	// Delete localities that are removed in the latest response.
 	for lid := range bgwc.configs {
 		if _, ok := newLocalitiesSet[lid]; !ok {
+			bgwc.stateAggregator.Remove(lid)
 			bgwc.bg.Remove(lid)
 			delete(bgwc.configs, lid)
 			edsImpl.logger.Infof("Locality %v deleted", lid)
+			rebuildStateAndPicker = true
 		}
 	}
+
+	if rebuildStateAndPicker {
+		bgwc.stateAggregator.BuildAndUpdate()
+	}
 }
 
 // handleSubConnStateChange handles the state change and update pickers accordingly.
@@ -429,6 +445,7 @@ func (edsImpl *edsBalancerImpl) newSubConn(priority priorityType, addrs []resolv
 func (edsImpl *edsBalancerImpl) close() {
 	for _, bgwc := range edsImpl.priorityToLocalities {
 		if bg := bgwc.bg; bg != nil {
+			bgwc.stateAggregator.Stop()
 			bg.Close()
 		}
 	}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index 3869bf594b47..f21d64f60a71 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -104,6 +104,7 @@ func (edsImpl *edsBalancerImpl) startPriority(priority priorityType) {
 	// currently avoided by handling balancer update in a goroutine (the run
 	// goroutine in the parent eds balancer). When priority balancer is split
 	// into its own, this asynchronous state handling needs to be copied.
+	p.stateAggregator.Start()
 	p.bg.Start()
 	// startPriority can be called when
 	// 1. first EDS resp, start p0
@@ -191,7 +192,9 @@ func (edsImpl *edsBalancerImpl) handlePriorityWithNewStateReady(priority priorit
 		edsImpl.logger.Infof("Switching priority from %v to %v, because latter became Ready", edsImpl.priorityInUse, priority)
 		edsImpl.priorityInUse = priority
 		for i := priority.nextLower(); !i.lowerThan(edsImpl.priorityLowest); i = i.nextLower() {
-			edsImpl.priorityToLocalities[i].bg.Close()
+			bgwc := edsImpl.priorityToLocalities[i]
+			bgwc.stateAggregator.Stop()
+			bgwc.bg.Close()
 		}
 		return true
 	}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 3f47b720ebbf..db1fa328a228 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -46,7 +46,6 @@ func init() {
 	for i := 0; i < testBackendAddrsCount; i++ {
 		testEndpointAddrs = append(testEndpointAddrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
 	}
-	balancergroup.NewRandomWRR = testutils.NewTestWRR
 	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond
 }
 
diff --git a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go b/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
new file mode 100644
index 000000000000..6f2de711fa6c
--- /dev/null
+++ b/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
@@ -0,0 +1,264 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package weightedaggregator implements state aggregator for weighted_target
+// balancer.
+//
+// This is a separate package so it can be shared by weighted_target and eds.
+// The eds balancer will be refactored to use weighted_target directly. After
+// that, all functions and structs in this package can be moved to package
+// weightedtarget and unexported.
+package weightedaggregator
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/wrr"
+	"google.golang.org/grpc/xds/internal"
+)
+
+type weightedPickerState struct {
+	weight uint32
+	state  balancer.State
+	// stateToAggregate is the connectivity state used only for state
+	// aggregation. It could be different from state.ConnectivityState. For
+	// example when a sub-balancer transitions from TransientFailure to
+	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
+	// is still TransientFailure.
+	stateToAggregate connectivity.State
+}
+
+func (s *weightedPickerState) String() string {
+	return fmt.Sprintf("weight:%v,picker:%p,state:%v,stateToAggregate:%v", s.weight, s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
+}
+
+// Aggregator is the weighted balancer state aggregator.
+type Aggregator struct {
+	cc     balancer.ClientConn
+	logger *grpclog.PrefixLogger
+	newWRR func() wrr.WRR
+
+	mu sync.Mutex
+	// If started is false, no updates should be sent to the parent cc. A closed
+	// sub-balancer could still send pickers to this aggregator. This makes sure
+	// that no updates will be forwarded to parent when the whole balancer group
+	// and states aggregator is closed.
+	started bool
+	// All balancer IDs exist as keys in this map, even if balancer group is not
+	// started.
+	//
+	// If an ID is not in map, it's either removed or never added.
+	idToPickerState map[internal.LocalityID]*weightedPickerState
+}
+
+// New creates a new weighted balancer state aggregator.
+func New(cc balancer.ClientConn, logger *grpclog.PrefixLogger, newWRR func() wrr.WRR) *Aggregator {
+	return &Aggregator{
+		cc:              cc,
+		logger:          logger,
+		newWRR:          newWRR,
+		idToPickerState: make(map[internal.LocalityID]*weightedPickerState),
+	}
+}
+
+// Start starts the aggregator. It can be called after Close to restart the
+// aggretator.
+func (wbsa *Aggregator) Start() {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	wbsa.started = true
+}
+
+// Stop stops the aggregator. When the aggregator is closed, it won't call
+// parent ClientConn to upate balancer state.
+func (wbsa *Aggregator) Stop() {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	wbsa.started = false
+	wbsa.clearStates()
+}
+
+// Add adds a sub-balancer state with weight. It adds a place holder, and waits for
+// the real sub-balancer to update state.
+func (wbsa *Aggregator) Add(id internal.LocalityID, weight uint32) {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	wbsa.idToPickerState[id] = &weightedPickerState{
+		weight: weight,
+		// Start everything in CONNECTING, so if one of the sub-balancers
+		// reports TransientFailure, the RPCs will still wait for the other
+		// sub-balancers.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+		stateToAggregate: connectivity.Connecting,
+	}
+}
+
+// Remove removes the sub-balancer state. Future updates from this sub-balancer,
+// if any, will be ignored.
+func (wbsa *Aggregator) Remove(id internal.LocalityID) {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	if _, ok := wbsa.idToPickerState[id]; !ok {
+		return
+	}
+	// Remove id and picker from picker map. This also results in future updates
+	// for this ID to be ignored.
+	delete(wbsa.idToPickerState, id)
+}
+
+// UpdateWeight updates the weight for the given id. Note that this doesn't
+// trigger an update to the parent ClientConn. The caller should decide when
+// it's necessary, and call BuildAndUpdate.
+func (wbsa *Aggregator) UpdateWeight(id internal.LocalityID, newWeight uint32) {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	pState, ok := wbsa.idToPickerState[id]
+	if !ok {
+		return
+	}
+	pState.weight = newWeight
+}
+
+// UpdateState is called to report a balancer state change from sub-balancer.
+// It's usually called by the balancer group.
+//
+// It calls parent ClientConn's UpdateState with the new aggregated state.
+func (wbsa *Aggregator) UpdateState(id internal.LocalityID, newState balancer.State) {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	oldState, ok := wbsa.idToPickerState[id]
+	if !ok {
+		// All state starts with an entry in pickStateMap. If ID is not in map,
+		// it's either removed, or never existed.
+		return
+	}
+	if !(oldState.state.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting) {
+		// If old state is TransientFailure, and new state is Connecting, don't
+		// update the state, to prevent the aggregated state from being always
+		// CONNECTING. Otherwise, stateToAggregate is the same as
+		// state.ConnectivityState.
+		oldState.stateToAggregate = newState.ConnectivityState
+	}
+	oldState.state = newState
+
+	if !wbsa.started {
+		return
+	}
+	wbsa.cc.UpdateState(wbsa.build())
+}
+
+// clearState Reset everything to init state (Connecting) but keep the entry in
+// map (to keep the weight).
+//
+// Caller must hold wbsa.mu.
+func (wbsa *Aggregator) clearStates() {
+	for _, pState := range wbsa.idToPickerState {
+		pState.state = balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		}
+		pState.stateToAggregate = connectivity.Connecting
+	}
+}
+
+// BuildAndUpdate combines the sub-state from each sub-balancer into one state,
+// and update it to parent ClientConn.
+func (wbsa *Aggregator) BuildAndUpdate() {
+	wbsa.mu.Lock()
+	defer wbsa.mu.Unlock()
+	if !wbsa.started {
+		return
+	}
+	wbsa.cc.UpdateState(wbsa.build())
+}
+
+// build combines sub-states into one.
+//
+// Caller must hold wbsa.mu.
+func (wbsa *Aggregator) build() balancer.State {
+	wbsa.logger.Infof("Child pickers with config: %+v", wbsa.idToPickerState)
+	m := wbsa.idToPickerState
+	var readyN, connectingN int
+	readyPickerWithWeights := make([]weightedPickerState, 0, len(m))
+	for _, ps := range m {
+		switch ps.stateToAggregate {
+		case connectivity.Ready:
+			readyN++
+			readyPickerWithWeights = append(readyPickerWithWeights, *ps)
+		case connectivity.Connecting:
+			connectingN++
+		}
+	}
+	var aggregatedState connectivity.State
+	switch {
+	case readyN > 0:
+		aggregatedState = connectivity.Ready
+	case connectingN > 0:
+		aggregatedState = connectivity.Connecting
+	default:
+		aggregatedState = connectivity.TransientFailure
+	}
+
+	// Make sure picker's return error is consistent with the aggregatedState.
+	var picker balancer.Picker
+	switch aggregatedState {
+	case connectivity.TransientFailure:
+		picker = base.NewErrPicker(balancer.ErrTransientFailure)
+	case connectivity.Connecting:
+		picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable)
+	default:
+		picker = newWeightedPickerGroup(readyPickerWithWeights, wbsa.newWRR)
+	}
+	return balancer.State{ConnectivityState: aggregatedState, Picker: picker}
+}
+
+type weightedPickerGroup struct {
+	w wrr.WRR
+}
+
+// newWeightedPickerGroup takes pickers with weights, and groups them into one
+// picker.
+//
+// Note it only takes ready pickers. The map shouldn't contain non-ready
+// pickers.
+func newWeightedPickerGroup(readyWeightedPickers []weightedPickerState, newWRR func() wrr.WRR) *weightedPickerGroup {
+	w := newWRR()
+	for _, ps := range readyWeightedPickers {
+		w.Add(ps.state.Picker, int64(ps.weight))
+	}
+
+	return &weightedPickerGroup{
+		w: w,
+	}
+}
+
+func (pg *weightedPickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	p, ok := pg.w.Next().(balancer.Picker)
+	if !ok {
+		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+	}
+	return p.Pick(info)
+}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index a2ce14190a20..4172a4033641 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -24,17 +24,22 @@ import (
 	"fmt"
 
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 )
 
 const weightedTargetName = "weighted_target_experimental"
 
+// newRandomWRR is the WRR constructor used to pick sub-pickers from
+// sub-balancers. It's to be modified in tests.
+var newRandomWRR = wrr.NewRandom
+
 func init() {
 	balancer.Register(&weightedTargetBB{})
 }
@@ -43,8 +48,10 @@ type weightedTargetBB struct{}
 
 func (wt *weightedTargetBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
 	b := &weightedTargetBalancer{}
-	b.logger = prefixLogger((b))
-	b.bg = balancergroup.New(cc, nil, b.logger)
+	b.logger = prefixLogger(b)
+	b.stateAggregator = weightedaggregator.New(cc, b.logger, newRandomWRR)
+	b.stateAggregator.Start()
+	b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
 	b.bg.Start()
 	b.logger.Infof("Created")
 	return b
@@ -66,7 +73,8 @@ type weightedTargetBalancer struct {
 	// policies that it maintains and reports load using LRS. Once these two
 	// dependencies are removed from the balancerGroup, this package will not
 	// have any dependencies on xds code.
-	bg *balancergroup.BalancerGroup
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *weightedaggregator.Aggregator
 
 	targets map[string]target
 }
@@ -86,10 +94,17 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 	}
 	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
 
-	// Remove sub-balancers that are not in the new config.
+	var rebuildStateAndPicker bool
+
+	// Remove sub-pickers and sub-balancers that are not in the new config.
 	for name := range w.targets {
 		if _, ok := newConfig.Targets[name]; !ok {
-			w.bg.Remove(makeLocalityFromName(name))
+			l := makeLocalityFromName(name)
+			w.stateAggregator.Remove(l)
+			w.bg.Remove(l)
+			// Trigger a state/picker update, because we don't want `ClientConn`
+			// to pick this sub-balancer anymore.
+			rebuildStateAndPicker = true
 		}
 	}
 
@@ -103,11 +118,18 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 
 		oldT, ok := w.targets[name]
 		if !ok {
-			// If this is a new sub-balancer, add it.
-			w.bg.Add(l, newT.Weight, balancer.Get(newT.ChildPolicy.Name))
+			// If this is a new sub-balancer, add weights to the picker map.
+			w.stateAggregator.Add(l, newT.Weight)
+			// Then add to the balancer group.
+			w.bg.Add(l, balancer.Get(newT.ChildPolicy.Name))
+			// Not trigger a state/picker update. Wait for the new sub-balancer
+			// to send its updates.
 		} else if newT.Weight != oldT.Weight {
 			// If this is an existing sub-balancer, update weight if necessary.
-			w.bg.ChangeWeight(l, newT.Weight)
+			w.stateAggregator.UpdateWeight(l, newT.Weight)
+			// Trigger a state/picker update, because we don't want `ClientConn`
+			// should do picks with the new weights now.
+			rebuildStateAndPicker = true
 		}
 
 		// Forwards all the update:
@@ -127,6 +149,10 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 	}
 
 	w.targets = newConfig.Targets
+
+	if rebuildStateAndPicker {
+		w.stateAggregator.BuildAndUpdate()
+	}
 	return nil
 }
 
@@ -139,13 +165,6 @@ func (w *weightedTargetBalancer) UpdateSubConnState(sc balancer.SubConn, state b
 }
 
 func (w *weightedTargetBalancer) Close() {
+	w.stateAggregator.Stop()
 	w.bg.Close()
 }
-
-func (w *weightedTargetBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
-	w.logger.Errorf("UpdateSubConnState should be called instead of HandleSubConnStateChange")
-}
-
-func (w *weightedTargetBalancer) HandleResolvedAddrs([]resolver.Address, error) {
-	w.logger.Errorf("UpdateClientConnState should be called instead of HandleResolvedAddrs")
-}

From e8fb6c1752968c3ea7842d6013a36a55d1145f66 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 9 Jul 2020 12:15:45 -0700
Subject: [PATCH 109/481] credentials/sts: PerRPCCreds Implementation (#3696)

---
 credentials/sts/sts.go      | 395 +++++++++++++++++++
 credentials/sts/sts_test.go | 764 ++++++++++++++++++++++++++++++++++++
 2 files changed, 1159 insertions(+)
 create mode 100644 credentials/sts/sts.go
 create mode 100644 credentials/sts/sts_test.go

diff --git a/credentials/sts/sts.go b/credentials/sts/sts.go
new file mode 100644
index 000000000000..f07c4c402e76
--- /dev/null
+++ b/credentials/sts/sts.go
@@ -0,0 +1,395 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package sts implements call credentials using STS (Security Token Service) as
+// defined in https://tools.ietf.org/html/rfc8693.
+//
+// Experimental
+//
+// Notice: All APIs in this package are experimental and may be changed or
+// removed in a later release.
+package sts
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/grpclog"
+)
+
+const (
+	// HTTP request timeout set on the http.Client used to make STS requests.
+	stsRequestTimeout = 5 * time.Second
+	// If lifetime left in a cached token is lesser than this value, we fetch a
+	// new one instead of returning the current one.
+	minCachedTokenLifetime = 300 * time.Second
+
+	tokenExchangeGrantType    = "urn:ietf:params:oauth:grant-type:token-exchange"
+	defaultCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"
+)
+
+// For overriding in tests.
+var (
+	loadSystemCertPool   = x509.SystemCertPool
+	makeHTTPDoer         = makeHTTPClient
+	readSubjectTokenFrom = ioutil.ReadFile
+	readActorTokenFrom   = ioutil.ReadFile
+)
+
+// Options configures the parameters used for an STS based token exchange.
+type Options struct {
+	// TokenExchangeServiceURI is the address of the server which implements STS
+	// token exchange functionality.
+	TokenExchangeServiceURI string // Required.
+
+	// Resource is a URI that indicates the target service or resource where the
+	// client intends to use the requested security token.
+	Resource string // Optional.
+
+	// Audience is the logical name of the target service where the client
+	// intends to use the requested security token
+	Audience string // Optional.
+
+	// Scope is a list of space-delimited, case-sensitive strings, that allow
+	// the client to specify the desired scope of the requested security token
+	// in the context of the service or resource where the token will be used.
+	// If this field is left unspecified, a default value of
+	// https://www.googleapis.com/auth/cloud-platform will be used.
+	Scope string // Optional.
+
+	// RequestedTokenType is an identifier, as described in
+	// https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of
+	// the requested security token.
+	RequestedTokenType string // Optional.
+
+	// SubjectTokenPath is a filesystem path which contains the security token
+	// that represents the identity of the party on behalf of whom the request
+	// is being made.
+	SubjectTokenPath string // Required.
+
+	// SubjectTokenType is an identifier, as described in
+	// https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of
+	// the security token in the "subject_token_path" parameter.
+	SubjectTokenType string // Required.
+
+	// ActorTokenPath is a  security token that represents the identity of the
+	// acting party.
+	ActorTokenPath string // Optional.
+
+	// ActorTokenType is an identifier, as described in
+	// https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of
+	// the the security token in the "actor_token_path" parameter.
+	ActorTokenType string // Optional.
+}
+
+// NewCredentials returns a new PerRPCCredentials implementation, configured
+// using opts, which performs token exchange using STS.
+func NewCredentials(opts Options) (credentials.PerRPCCredentials, error) {
+	if err := validateOptions(opts); err != nil {
+		return nil, err
+	}
+
+	// Load the system roots to validate the certificate presented by the STS
+	// endpoint during the TLS handshake.
+	roots, err := loadSystemCertPool()
+	if err != nil {
+		return nil, err
+	}
+
+	return &callCreds{
+		opts:   opts,
+		client: makeHTTPDoer(roots),
+	}, nil
+}
+
+// callCreds provides the implementation of call credentials based on an STS
+// token exchange.
+type callCreds struct {
+	opts   Options
+	client httpDoer
+
+	// Cached accessToken to avoid an STS token exchange for every call to
+	// GetRequestMetadata.
+	mu            sync.Mutex
+	tokenMetadata map[string]string
+	tokenExpiry   time.Time
+}
+
+// GetRequestMetadata returns the cached accessToken, if available and valid, or
+// fetches a new one by performing an STS token exchange.
+func (c *callCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) {
+	if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil {
+		return nil, fmt.Errorf("unable to transfer STS PerRPCCredentials: %v", err)
+	}
+
+	// Holding the lock for the whole duration of the STS request and response
+	// processing ensures that concurrent RPCs don't end up in multiple
+	// requests being made.
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if md := c.cachedMetadata(); md != nil {
+		return md, nil
+	}
+	req, err := constructRequest(ctx, c.opts)
+	if err != nil {
+		return nil, err
+	}
+	respBody, err := sendRequest(c.client, req)
+	if err != nil {
+		return nil, err
+	}
+	ti, err := tokenInfoFromResponse(respBody)
+	if err != nil {
+		return nil, err
+	}
+	c.tokenMetadata = map[string]string{"Authorization": fmt.Sprintf("%s %s", ti.tokenType, ti.token)}
+	c.tokenExpiry = ti.expiryTime
+	return c.tokenMetadata, nil
+}
+
+// RequireTransportSecurity indicates whether the credentials requires
+// transport security.
+func (c *callCreds) RequireTransportSecurity() bool {
+	return true
+}
+
+// httpDoer wraps the single method on the http.Client type that we use. This
+// helps with overriding in unittests.
+type httpDoer interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+func makeHTTPClient(roots *x509.CertPool) httpDoer {
+	return &http.Client{
+		Timeout: stsRequestTimeout,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: roots,
+			},
+		},
+	}
+}
+
+// validateOptions performs the following validation checks on opts:
+// - tokenExchangeServiceURI is not empty
+// - tokenExchangeServiceURI is a valid URI with a http(s) scheme
+// - subjectTokenPath and subjectTokenType are not empty.
+func validateOptions(opts Options) error {
+	if opts.TokenExchangeServiceURI == "" {
+		return errors.New("empty token_exchange_service_uri in options")
+	}
+	u, err := url.Parse(opts.TokenExchangeServiceURI)
+	if err != nil {
+		return err
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return fmt.Errorf("scheme is not supported: %s. Only http(s) is supported", u.Scheme)
+	}
+
+	if opts.SubjectTokenPath == "" {
+		return errors.New("required field SubjectTokenPath is not specified")
+	}
+	if opts.SubjectTokenType == "" {
+		return errors.New("required field SubjectTokenType is not specified")
+	}
+	return nil
+}
+
+// cachedMetadata returns the cached metadata provided it is not going to
+// expire anytime soon.
+//
+// Caller must hold c.mu.
+func (c *callCreds) cachedMetadata() map[string]string {
+	now := time.Now()
+	// If the cached token has not expired and the lifetime remaining on that
+	// token is greater than the minimum value we are willing to accept, go
+	// ahead and use it.
+	if c.tokenExpiry.After(now) && c.tokenExpiry.Sub(now) > minCachedTokenLifetime {
+		return c.tokenMetadata
+	}
+	return nil
+}
+
+// constructRequest creates the STS request body in JSON based on the provided
+// options.
+// - Contents of the subjectToken are read from the file specified in
+//   options. If we encounter an error here, we bail out.
+// - Contents of the actorToken are read from the file specified in options.
+//   If we encounter an error here, we ignore this field because this is
+//   optional.
+// - Most of the other fields in the request come directly from options.
+//
+// A new HTTP request is created by calling http.NewRequestWithContext() and
+// passing the provided context, thereby enforcing any timeouts specified in
+// the latter.
+func constructRequest(ctx context.Context, opts Options) (*http.Request, error) {
+	subToken, err := readSubjectTokenFrom(opts.SubjectTokenPath)
+	if err != nil {
+		return nil, err
+	}
+	reqScope := opts.Scope
+	if reqScope == "" {
+		reqScope = defaultCloudPlatformScope
+	}
+	reqParams := &requestParameters{
+		GrantType:          tokenExchangeGrantType,
+		Resource:           opts.Resource,
+		Audience:           opts.Audience,
+		Scope:              reqScope,
+		RequestedTokenType: opts.RequestedTokenType,
+		SubjectToken:       string(subToken),
+		SubjectTokenType:   opts.SubjectTokenType,
+	}
+	if opts.ActorTokenPath != "" {
+		actorToken, err := readActorTokenFrom(opts.ActorTokenPath)
+		if err != nil {
+			return nil, err
+		}
+		reqParams.ActorToken = string(actorToken)
+		reqParams.ActorTokenType = opts.ActorTokenType
+	}
+	jsonBody, err := json.Marshal(reqParams)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, "POST", opts.TokenExchangeServiceURI, bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create http request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	return req, nil
+}
+
+func sendRequest(client httpDoer, req *http.Request) ([]byte, error) {
+	// http.Client returns a non-nil error only if it encounters an error
+	// caused by client policy (such as CheckRedirect), or failure to speak
+	// HTTP (such as a network connectivity problem). A non-2xx status code
+	// doesn't cause an error.
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// When the http.Client returns a non-nil error, it is the
+	// responsibility of the caller to read the response body till an EOF is
+	// encountered and to close it.
+	body, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return body, nil
+	}
+	grpclog.Warningf("http status %d, body: %s", resp.StatusCode, string(body))
+	return nil, fmt.Errorf("http status %d, body: %s", resp.StatusCode, string(body))
+}
+
+func tokenInfoFromResponse(respBody []byte) (*tokenInfo, error) {
+	respData := &responseParameters{}
+	if err := json.Unmarshal(respBody, respData); err != nil {
+		return nil, fmt.Errorf("json.Unmarshal(%v): %v", respBody, err)
+	}
+	if respData.AccessToken == "" {
+		return nil, fmt.Errorf("empty accessToken in response (%v)", string(respBody))
+	}
+	return &tokenInfo{
+		tokenType:  respData.TokenType,
+		token:      respData.AccessToken,
+		expiryTime: time.Now().Add(time.Duration(respData.ExpiresIn) * time.Second),
+	}, nil
+}
+
+// requestParameters stores all STS request attributes defined in
+// https://tools.ietf.org/html/rfc8693#section-2.1.
+type requestParameters struct {
+	// REQUIRED. The value "urn:ietf:params:oauth:grant-type:token-exchange"
+	// indicates that a token exchange is being performed.
+	GrantType string `json:"grant_type"`
+	// OPTIONAL. Indicates the location of the target service or resource where
+	// the client intends to use the requested security token.
+	Resource string `json:"resource,omitempty"`
+	// OPTIONAL. The logical name of the target service where the client intends
+	// to use the requested security token.
+	Audience string `json:"audience,omitempty"`
+	// OPTIONAL. A list of space-delimited, case-sensitive strings, that allow
+	// the client to specify the desired scope of the requested security token
+	// in the context of the service or Resource where the token will be used.
+	Scope string `json:"scope,omitempty"`
+	// OPTIONAL. An identifier, for the type of the requested security token.
+	RequestedTokenType string `json:"requested_token_type,omitempty"`
+	// REQUIRED. A security token that represents the identity of the party on
+	// behalf of whom the request is being made.
+	SubjectToken string `json:"subject_token"`
+	// REQUIRED. An identifier, that indicates the type of the security token in
+	// the "subject_token" parameter.
+	SubjectTokenType string `json:"subject_token_type"`
+	// OPTIONAL. A security token that represents the identity of the acting
+	// party.
+	ActorToken string `json:"actor_token,omitempty"`
+	// An identifier, that indicates the type of the security token in the
+	// "actor_token" parameter.
+	ActorTokenType string `json:"actor_token_type,omitempty"`
+}
+
+// nesponseParameters stores all attributes sent as JSON in a successful STS
+// response. These attributes are defined in
+// https://tools.ietf.org/html/rfc8693#section-2.2.1.
+type responseParameters struct {
+	// REQUIRED. The security token issued by the authorization server
+	// in response to the token exchange request.
+	AccessToken string `json:"access_token"`
+	// REQUIRED. An identifier, representation of the issued security token.
+	IssuedTokenType string `json:"issued_token_type"`
+	// REQUIRED. A case-insensitive value specifying the method of using the access
+	// token issued. It provides the client with information about how to utilize the
+	// access token to access protected resources.
+	TokenType string `json:"token_type"`
+	// RECOMMENDED. The validity lifetime, in seconds, of the token issued by the
+	// authorization server.
+	ExpiresIn int64 `json:"expires_in"`
+	// OPTIONAL, if the Scope of the issued security token is identical to the
+	// Scope requested by the client; otherwise, REQUIRED.
+	Scope string `json:"scope"`
+	// OPTIONAL. A refresh token will typically not be issued when the exchange is
+	// of one temporary credential (the subject_token) for a different temporary
+	// credential (the issued token) for use in some other context.
+	RefreshToken string `json:"refresh_token"`
+}
+
+// tokenInfo wraps the information received in a successful STS response.
+type tokenInfo struct {
+	tokenType  string
+	token      string
+	expiryTime time.Time
+}
diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go
new file mode 100644
index 000000000000..641bad5820bb
--- /dev/null
+++ b/credentials/sts/sts_test.go
@@ -0,0 +1,764 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package sts
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httputil"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+)
+
+const (
+	requestedTokenType   = "urn:ietf:params:oauth:token-type:access-token"
+	actorTokenPath       = "/var/run/secrets/token.jwt"
+	actorTokenType       = "urn:ietf:params:oauth:token-type:refresh_token"
+	actorTokenContents   = "actorToken.jwt.contents"
+	accessTokenContents  = "access_token"
+	subjectTokenPath     = "/var/run/secrets/token.jwt"
+	subjectTokenType     = "urn:ietf:params:oauth:token-type:id_token"
+	subjectTokenContents = "subjectToken.jwt.contents"
+	serviceURI           = "http://localhost"
+	exampleResource      = "https://backend.example.com/api"
+	exampleAudience      = "example-backend-service"
+	testScope            = "https://www.googleapis.com/auth/monitoring"
+)
+
+var (
+	goodOptions = Options{
+		TokenExchangeServiceURI: serviceURI,
+		Audience:                exampleAudience,
+		RequestedTokenType:      requestedTokenType,
+		SubjectTokenPath:        subjectTokenPath,
+		SubjectTokenType:        subjectTokenType,
+	}
+	goodRequestParams = &requestParameters{
+		GrantType:          tokenExchangeGrantType,
+		Audience:           exampleAudience,
+		Scope:              defaultCloudPlatformScope,
+		RequestedTokenType: requestedTokenType,
+		SubjectToken:       subjectTokenContents,
+		SubjectTokenType:   subjectTokenType,
+	}
+	goodMetadata = map[string]string{
+		"Authorization": fmt.Sprintf("Bearer %s", accessTokenContents),
+	}
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// A struct that implements AuthInfo interface and added to the context passed
+// to GetRequestMetadata from tests.
+type testAuthInfo struct {
+	credentials.CommonAuthInfo
+}
+
+func (ta testAuthInfo) AuthType() string {
+	return "testAuthInfo"
+}
+
+func createTestContext(ctx context.Context, s credentials.SecurityLevel) context.Context {
+	auth := &testAuthInfo{CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: s}}
+	ri := credentials.RequestInfo{
+		Method:   "testInfo",
+		AuthInfo: auth,
+	}
+	return internal.NewRequestInfoContext.(func(context.Context, credentials.RequestInfo) context.Context)(ctx, ri)
+}
+
+// errReader implements the io.Reader interface and returns an error from the
+// Read method.
+type errReader struct{}
+
+func (r errReader) Read(b []byte) (n int, err error) {
+	return 0, errors.New("read error")
+}
+
+// We need a function to construct the response instead of simply declaring it
+// as a variable since the the response body will be consumed by the
+// credentials, and therefore we will need a new one everytime.
+func makeGoodResponse() *http.Response {
+	respJSON, _ := json.Marshal(responseParameters{
+		AccessToken:     accessTokenContents,
+		IssuedTokenType: "urn:ietf:params:oauth:token-type:access_token",
+		TokenType:       "Bearer",
+		ExpiresIn:       3600,
+	})
+	respBody := ioutil.NopCloser(bytes.NewReader(respJSON))
+	return &http.Response{
+		Status:     "200 OK",
+		StatusCode: http.StatusOK,
+		Body:       respBody,
+	}
+}
+
+// fakeHTTPDoer helps mock out the http.Client.Do calls made by the credentials
+// code under test. It makes the http.Request made by the credentials available
+// through a channel, and makes it possible to inject various responses.
+type fakeHTTPDoer struct {
+	reqCh  *testutils.Channel
+	respCh *testutils.Channel
+	err    error
+}
+
+func (fc *fakeHTTPDoer) Do(req *http.Request) (*http.Response, error) {
+	fc.reqCh.Send(req)
+	val, err := fc.respCh.Receive()
+	if err != nil {
+		return nil, err
+	}
+	return val.(*http.Response), fc.err
+}
+
+// Overrides the http.Client with a fakeClient which sends a good response.
+func overrideHTTPClientGood() (*fakeHTTPDoer, func()) {
+	fc := &fakeHTTPDoer{
+		reqCh:  testutils.NewChannel(),
+		respCh: testutils.NewChannel(),
+	}
+	fc.respCh.Send(makeGoodResponse())
+
+	origMakeHTTPDoer := makeHTTPDoer
+	makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc }
+	return fc, func() { makeHTTPDoer = origMakeHTTPDoer }
+}
+
+// Overrides the http.Client with the provided fakeClient.
+func overrideHTTPClient(fc *fakeHTTPDoer) func() {
+	origMakeHTTPDoer := makeHTTPDoer
+	makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc }
+	return func() { makeHTTPDoer = origMakeHTTPDoer }
+}
+
+// Overrides the subject token read to return a const which we can compare in
+// our tests.
+func overrideSubjectTokenGood() func() {
+	origReadSubjectTokenFrom := readSubjectTokenFrom
+	readSubjectTokenFrom = func(path string) ([]byte, error) {
+		return []byte(subjectTokenContents), nil
+	}
+	return func() { readSubjectTokenFrom = origReadSubjectTokenFrom }
+}
+
+// Overrides the subject token read to always return an error.
+func overrideSubjectTokenError() func() {
+	origReadSubjectTokenFrom := readSubjectTokenFrom
+	readSubjectTokenFrom = func(path string) ([]byte, error) {
+		return nil, errors.New("error reading subject token")
+	}
+	return func() { readSubjectTokenFrom = origReadSubjectTokenFrom }
+}
+
+// Overrides the actor token read to return a const which we can compare in
+// our tests.
+func overrideActorTokenGood() func() {
+	origReadActorTokenFrom := readActorTokenFrom
+	readActorTokenFrom = func(path string) ([]byte, error) {
+		return []byte(actorTokenContents), nil
+	}
+	return func() { readActorTokenFrom = origReadActorTokenFrom }
+}
+
+// Overrides the actor token read to always return an error.
+func overrideActorTokenError() func() {
+	origReadActorTokenFrom := readActorTokenFrom
+	readActorTokenFrom = func(path string) ([]byte, error) {
+		return nil, errors.New("error reading actor token")
+	}
+	return func() { readActorTokenFrom = origReadActorTokenFrom }
+}
+
+// compareRequest compares the http.Request received in the test with the
+// expected requestParameters specified in wantReqParams.
+func compareRequest(gotRequest *http.Request, wantReqParams *requestParameters) error {
+	jsonBody, err := json.Marshal(wantReqParams)
+	if err != nil {
+		return err
+	}
+	wantReq, err := http.NewRequest("POST", serviceURI, bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return fmt.Errorf("failed to create http request: %v", err)
+	}
+	wantReq.Header.Set("Content-Type", "application/json")
+
+	wantR, err := httputil.DumpRequestOut(wantReq, true)
+	if err != nil {
+		return err
+	}
+	gotR, err := httputil.DumpRequestOut(gotRequest, true)
+	if err != nil {
+		return err
+	}
+	if diff := cmp.Diff(string(wantR), string(gotR)); diff != "" {
+		return fmt.Errorf("sts request diff (-want +got):\n%s", diff)
+	}
+	return nil
+}
+
+// receiveAndCompareRequest waits for a request to be sent out by the
+// credentials implementation using the fakeHTTPClient and compares it to an
+// expected goodRequest. This is expected to be called in a separate goroutine
+// by the tests. So, any errors encountered are pushed to an error channel
+// which is monitored by the test.
+func receiveAndCompareRequest(reqCh *testutils.Channel, errCh chan error) {
+	val, err := reqCh.Receive()
+	if err != nil {
+		errCh <- err
+		return
+	}
+	req := val.(*http.Request)
+	if err := compareRequest(req, goodRequestParams); err != nil {
+		errCh <- err
+		return
+	}
+	errCh <- nil
+}
+
+// TestGetRequestMetadataSuccess verifies the successful case of sending an
+// token exchange request and processing the response.
+func (s) TestGetRequestMetadataSuccess(t *testing.T) {
+	defer overrideSubjectTokenGood()()
+	fc, cancel := overrideHTTPClientGood()
+	defer cancel()
+
+	creds, err := NewCredentials(goodOptions)
+	if err != nil {
+		t.Fatalf("NewCredentials(%v) = %v", goodOptions, err)
+	}
+
+	errCh := make(chan error, 1)
+	go receiveAndCompareRequest(fc.reqCh, errCh)
+
+	gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
+	if err != nil {
+		t.Fatalf("creds.GetRequestMetadata() = %v", err)
+	}
+	if !cmp.Equal(gotMetadata, goodMetadata) {
+		t.Fatalf("creds.GetRequestMetadata() = %v, want %v", gotMetadata, goodMetadata)
+	}
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+
+	// Make another call to get request metadata and this should return contents
+	// from the cache. This will fail if the credentials tries to send a fresh
+	// request here since we have not configured our fakeClient to return any
+	// response on retries.
+	gotMetadata, err = creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
+	if err != nil {
+		t.Fatalf("creds.GetRequestMetadata() = %v", err)
+	}
+	if !cmp.Equal(gotMetadata, goodMetadata) {
+		t.Fatalf("creds.GetRequestMetadata() = %v, want %v", gotMetadata, goodMetadata)
+	}
+}
+
+// TestGetRequestMetadataBadSecurityLevel verifies the case where the
+// securityLevel specified in the context passed to GetRequestMetadata is not
+// sufficient.
+func (s) TestGetRequestMetadataBadSecurityLevel(t *testing.T) {
+	defer overrideSubjectTokenGood()()
+
+	creds, err := NewCredentials(goodOptions)
+	if err != nil {
+		t.Fatalf("NewCredentials(%v) = %v", goodOptions, err)
+	}
+
+	gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.IntegrityOnly), "")
+	if err == nil {
+		t.Fatalf("creds.GetRequestMetadata() succeeded with metadata %v, expected to fail", gotMetadata)
+	}
+}
+
+// TestGetRequestMetadataCacheExpiry verifies the case where the cached access
+// token has expired, and the credentials implementation will have to send a
+// fresh token exchange request.
+func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) {
+	const expiresInSecs = 1
+	defer overrideSubjectTokenGood()()
+	fc := &fakeHTTPDoer{
+		reqCh:  testutils.NewChannel(),
+		respCh: testutils.NewChannel(),
+	}
+	defer overrideHTTPClient(fc)()
+
+	creds, err := NewCredentials(goodOptions)
+	if err != nil {
+		t.Fatalf("NewCredentials(%v) = %v", goodOptions, err)
+	}
+
+	// The fakeClient is configured to return an access_token with a one second
+	// expiry. So, in the second iteration, the credentials will find the cache
+	// entry, but that would have expired, and therefore we expect it to send
+	// out a fresh request.
+	for i := 0; i < 2; i++ {
+		errCh := make(chan error, 1)
+		go receiveAndCompareRequest(fc.reqCh, errCh)
+
+		respJSON, _ := json.Marshal(responseParameters{
+			AccessToken:     accessTokenContents,
+			IssuedTokenType: "urn:ietf:params:oauth:token-type:access_token",
+			TokenType:       "Bearer",
+			ExpiresIn:       expiresInSecs,
+		})
+		respBody := ioutil.NopCloser(bytes.NewReader(respJSON))
+		resp := &http.Response{
+			Status:     "200 OK",
+			StatusCode: http.StatusOK,
+			Body:       respBody,
+		}
+		fc.respCh.Send(resp)
+
+		gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
+		if err != nil {
+			t.Fatalf("creds.GetRequestMetadata() = %v", err)
+		}
+		if !cmp.Equal(gotMetadata, goodMetadata) {
+			t.Fatalf("creds.GetRequestMetadata() = %v, want %v", gotMetadata, goodMetadata)
+		}
+		if err := <-errCh; err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(expiresInSecs * time.Second)
+	}
+}
+
+// TestGetRequestMetadataBadResponses verifies the scenario where the token
+// exchange server returns bad responses.
+func (s) TestGetRequestMetadataBadResponses(t *testing.T) {
+	tests := []struct {
+		name     string
+		response *http.Response
+	}{
+		{
+			name: "bad JSON",
+			response: &http.Response{
+				Status:     "200 OK",
+				StatusCode: http.StatusOK,
+				Body:       ioutil.NopCloser(strings.NewReader("not JSON")),
+			},
+		},
+		{
+			name: "no access token",
+			response: &http.Response{
+				Status:     "200 OK",
+				StatusCode: http.StatusOK,
+				Body:       ioutil.NopCloser(strings.NewReader("{}")),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			defer overrideSubjectTokenGood()()
+
+			fc := &fakeHTTPDoer{
+				reqCh:  testutils.NewChannel(),
+				respCh: testutils.NewChannel(),
+			}
+			defer overrideHTTPClient(fc)()
+
+			creds, err := NewCredentials(goodOptions)
+			if err != nil {
+				t.Fatalf("NewCredentials(%v) = %v", goodOptions, err)
+			}
+
+			errCh := make(chan error, 1)
+			go receiveAndCompareRequest(fc.reqCh, errCh)
+
+			fc.respCh.Send(test.response)
+			if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil {
+				t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail")
+			}
+			if err := <-errCh; err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// TestGetRequestMetadataBadSubjectTokenRead verifies the scenario where the
+// attempt to read the subjectToken fails.
+func (s) TestGetRequestMetadataBadSubjectTokenRead(t *testing.T) {
+	defer overrideSubjectTokenError()()
+	fc, cancel := overrideHTTPClientGood()
+	defer cancel()
+
+	creds, err := NewCredentials(goodOptions)
+	if err != nil {
+		t.Fatalf("NewCredentials(%v) = %v", goodOptions, err)
+	}
+
+	errCh := make(chan error, 1)
+	go func() {
+		if _, err := fc.reqCh.Receive(); err != testutils.ErrRecvTimeout {
+			errCh <- err
+			return
+		}
+		errCh <- nil
+	}()
+
+	if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil {
+		t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail")
+	}
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestNewCredentials(t *testing.T) {
+	tests := []struct {
+		name           string
+		opts           Options
+		errSystemRoots bool
+		wantErr        bool
+	}{
+		{
+			name: "invalid options - empty subjectTokenPath",
+			opts: Options{
+				TokenExchangeServiceURI: serviceURI,
+			},
+			wantErr: true,
+		},
+		{
+			name:           "invalid system root certs",
+			opts:           goodOptions,
+			errSystemRoots: true,
+			wantErr:        true,
+		},
+		{
+			name: "good case",
+			opts: goodOptions,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if test.errSystemRoots {
+				oldSystemRoots := loadSystemCertPool
+				loadSystemCertPool = func() (*x509.CertPool, error) {
+					return nil, errors.New("failed to load system cert pool")
+				}
+				defer func() {
+					loadSystemCertPool = oldSystemRoots
+				}()
+			}
+
+			creds, err := NewCredentials(test.opts)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("NewCredentials(%v) = %v, want %v", test.opts, err, test.wantErr)
+			}
+			if err == nil {
+				if !creds.RequireTransportSecurity() {
+					t.Errorf("creds.RequireTransportSecurity() returned false")
+				}
+			}
+		})
+	}
+}
+
+func (s) TestValidateOptions(t *testing.T) {
+	tests := []struct {
+		name          string
+		opts          Options
+		wantErrPrefix string
+	}{
+		{
+			name:          "empty token exchange service URI",
+			opts:          Options{},
+			wantErrPrefix: "empty token_exchange_service_uri in options",
+		},
+		{
+			name: "invalid URI",
+			opts: Options{
+				TokenExchangeServiceURI: "\tI'm a bad URI\n",
+			},
+			wantErrPrefix: "invalid control character in URL",
+		},
+		{
+			name: "unsupported scheme",
+			opts: Options{
+				TokenExchangeServiceURI: "unix:///path/to/socket",
+			},
+			wantErrPrefix: "scheme is not supported",
+		},
+		{
+			name: "empty subjectTokenPath",
+			opts: Options{
+				TokenExchangeServiceURI: serviceURI,
+			},
+			wantErrPrefix: "required field SubjectTokenPath is not specified",
+		},
+		{
+			name: "empty subjectTokenType",
+			opts: Options{
+				TokenExchangeServiceURI: serviceURI,
+				SubjectTokenPath:        subjectTokenPath,
+			},
+			wantErrPrefix: "required field SubjectTokenType is not specified",
+		},
+		{
+			name: "good options",
+			opts: goodOptions,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateOptions(test.opts)
+			if (err != nil) != (test.wantErrPrefix != "") {
+				t.Errorf("validateOptions(%v) = %v, want %v", test.opts, err, test.wantErrPrefix)
+			}
+			if err != nil && !strings.Contains(err.Error(), test.wantErrPrefix) {
+				t.Errorf("validateOptions(%v) = %v, want %v", test.opts, err, test.wantErrPrefix)
+			}
+		})
+	}
+}
+
+func (s) TestConstructRequest(t *testing.T) {
+	tests := []struct {
+		name                string
+		opts                Options
+		subjectTokenReadErr bool
+		actorTokenReadErr   bool
+		wantReqParams       *requestParameters
+		wantErr             bool
+	}{
+		{
+			name:                "subject token read failure",
+			subjectTokenReadErr: true,
+			opts:                goodOptions,
+			wantErr:             true,
+		},
+		{
+			name:              "actor token read failure",
+			actorTokenReadErr: true,
+			opts: Options{
+				TokenExchangeServiceURI: serviceURI,
+				Audience:                exampleAudience,
+				RequestedTokenType:      requestedTokenType,
+				SubjectTokenPath:        subjectTokenPath,
+				SubjectTokenType:        subjectTokenType,
+				ActorTokenPath:          actorTokenPath,
+				ActorTokenType:          actorTokenType,
+			},
+			wantErr: true,
+		},
+		{
+			name:          "default cloud platform scope",
+			opts:          goodOptions,
+			wantReqParams: goodRequestParams,
+		},
+		{
+			name: "all good",
+			opts: Options{
+				TokenExchangeServiceURI: serviceURI,
+				Resource:                exampleResource,
+				Audience:                exampleAudience,
+				Scope:                   testScope,
+				RequestedTokenType:      requestedTokenType,
+				SubjectTokenPath:        subjectTokenPath,
+				SubjectTokenType:        subjectTokenType,
+				ActorTokenPath:          actorTokenPath,
+				ActorTokenType:          actorTokenType,
+			},
+			wantReqParams: &requestParameters{
+				GrantType:          tokenExchangeGrantType,
+				Resource:           exampleResource,
+				Audience:           exampleAudience,
+				Scope:              testScope,
+				RequestedTokenType: requestedTokenType,
+				SubjectToken:       subjectTokenContents,
+				SubjectTokenType:   subjectTokenType,
+				ActorToken:         actorTokenContents,
+				ActorTokenType:     actorTokenType,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if test.subjectTokenReadErr {
+				defer overrideSubjectTokenError()()
+			} else {
+				defer overrideSubjectTokenGood()()
+			}
+
+			if test.actorTokenReadErr {
+				defer overrideActorTokenError()()
+			} else {
+				defer overrideActorTokenGood()()
+			}
+
+			gotRequest, err := constructRequest(context.Background(), test.opts)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("constructRequest(%v) = %v, wantErr: %v", test.opts, err, test.wantErr)
+			}
+			if test.wantErr {
+				return
+			}
+			if err := compareRequest(gotRequest, test.wantReqParams); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func (s) TestSendRequest(t *testing.T) {
+	defer overrideSubjectTokenGood()()
+	req, err := constructRequest(context.Background(), goodOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name    string
+		resp    *http.Response
+		respErr error
+		wantErr bool
+	}{
+		{
+			name:    "client error",
+			respErr: errors.New("http.Client.Do failed"),
+			wantErr: true,
+		},
+		{
+			name: "bad response body",
+			resp: &http.Response{
+				Status:     "200 OK",
+				StatusCode: http.StatusOK,
+				Body:       ioutil.NopCloser(errReader{}),
+			},
+			wantErr: true,
+		},
+		{
+			name: "nonOK status code",
+			resp: &http.Response{
+				Status:     "400 BadRequest",
+				StatusCode: http.StatusBadRequest,
+				Body:       ioutil.NopCloser(strings.NewReader("")),
+			},
+			wantErr: true,
+		},
+		{
+			name: "good case",
+			resp: makeGoodResponse(),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			client := &fakeHTTPDoer{
+				reqCh:  testutils.NewChannel(),
+				respCh: testutils.NewChannel(),
+				err:    test.respErr,
+			}
+			client.respCh.Send(test.resp)
+			_, err := sendRequest(client, req)
+			if (err != nil) != test.wantErr {
+				t.Errorf("sendRequest(%v) = %v, wantErr: %v", req, err, test.wantErr)
+			}
+		})
+	}
+}
+
+func (s) TestTokenInfoFromResponse(t *testing.T) {
+	noAccessToken, _ := json.Marshal(responseParameters{
+		IssuedTokenType: "urn:ietf:params:oauth:token-type:access_token",
+		TokenType:       "Bearer",
+		ExpiresIn:       3600,
+	})
+	goodResponse, _ := json.Marshal(responseParameters{
+		IssuedTokenType: requestedTokenType,
+		AccessToken:     accessTokenContents,
+		TokenType:       "Bearer",
+		ExpiresIn:       3600,
+	})
+
+	tests := []struct {
+		name          string
+		respBody      []byte
+		wantTokenInfo *tokenInfo
+		wantErr       bool
+	}{
+		{
+			name:     "bad JSON",
+			respBody: []byte("not JSON"),
+			wantErr:  true,
+		},
+		{
+			name:     "empty response",
+			respBody: []byte(""),
+			wantErr:  true,
+		},
+		{
+			name:     "non-empty response with no access token",
+			respBody: noAccessToken,
+			wantErr:  true,
+		},
+		{
+			name:     "good response",
+			respBody: goodResponse,
+			wantTokenInfo: &tokenInfo{
+				tokenType: "Bearer",
+				token:     accessTokenContents,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotTokenInfo, err := tokenInfoFromResponse(test.respBody)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("tokenInfoFromResponse(%+v) = %v, wantErr: %v", test.respBody, err, test.wantErr)
+			}
+			if test.wantErr {
+				return
+			}
+			// Can't do a cmp.Equal on the whole struct since the expiryField
+			// is populated based on time.Now().
+			if gotTokenInfo.tokenType != test.wantTokenInfo.tokenType || gotTokenInfo.token != test.wantTokenInfo.token {
+				t.Errorf("tokenInfoFromResponse(%+v) = %+v, want: %+v", test.respBody, gotTokenInfo, test.wantTokenInfo)
+			}
+		})
+	}
+}

From a86c873a73a6ae1c7dcc4cf2f69e3f4c0d09edd1 Mon Sep 17 00:00:00 2001
From: Sophos <hnlq.sysu@gmail.com>
Date: Fri, 10 Jul 2020 04:09:50 +0800
Subject: [PATCH 110/481] cleanup: fix mention of healthcheck to health (#3731)

---
 clientconn.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clientconn.go b/clientconn.go
index 6dc5eda46f1a..11d9ad4d72a2 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -1304,7 +1304,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 //
 // LB channel health checking is enabled when all requirements below are met:
 // 1. it is not disabled by the user with the WithDisableHealthCheck DialOption
-// 2. internal.HealthCheckFunc is set by importing the grpc/healthcheck package
+// 2. internal.HealthCheckFunc is set by importing the grpc/health package
 // 3. a service config with non-empty healthCheckConfig field is provided
 // 4. the load balancer requests it
 //

From 34053813f1817b6a3ccad6415afcec901379ea2f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 9 Jul 2020 13:32:33 -0700
Subject: [PATCH 111/481] codes: document which error codes can be returned by
 the framework (#3699)

This commit essentially copies the information from the second section of the gRPC core
documentation about status codes at
https://grpc.github.io/grpc/core/md_doc_statuscodes.html
into the Go documentation of the status codes, to increase visibility.
---
 codes/codes.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/codes/codes.go b/codes/codes.go
index 02738839dd98..11b106182db2 100644
--- a/codes/codes.go
+++ b/codes/codes.go
@@ -33,6 +33,9 @@ const (
 	OK Code = 0
 
 	// Canceled indicates the operation was canceled (typically by the caller).
+	//
+	// The gRPC framework will generate this error code when cancellation
+	// is requested.
 	Canceled Code = 1
 
 	// Unknown error. An example of where this error may be returned is
@@ -40,12 +43,17 @@ const (
 	// an error-space that is not known in this address space. Also
 	// errors raised by APIs that do not return enough error information
 	// may be converted to this error.
+	//
+	// The gRPC framework will generate this error code in the above two
+	// mentioned cases.
 	Unknown Code = 2
 
 	// InvalidArgument indicates client specified an invalid argument.
 	// Note that this differs from FailedPrecondition. It indicates arguments
 	// that are problematic regardless of the state of the system
 	// (e.g., a malformed file name).
+	//
+	// This error code will not be generated by the gRPC framework.
 	InvalidArgument Code = 3
 
 	// DeadlineExceeded means operation expired before completion.
@@ -53,14 +61,21 @@ const (
 	// returned even if the operation has completed successfully. For
 	// example, a successful response from a server could have been delayed
 	// long enough for the deadline to expire.
+	//
+	// The gRPC framework will generate this error code when the deadline is
+	// exceeded.
 	DeadlineExceeded Code = 4
 
 	// NotFound means some requested entity (e.g., file or directory) was
 	// not found.
+	//
+	// This error code will not be generated by the gRPC framework.
 	NotFound Code = 5
 
 	// AlreadyExists means an attempt to create an entity failed because one
 	// already exists.
+	//
+	// This error code will not be generated by the gRPC framework.
 	AlreadyExists Code = 6
 
 	// PermissionDenied indicates the caller does not have permission to
@@ -69,10 +84,17 @@ const (
 	// instead for those errors). It must not be
 	// used if the caller cannot be identified (use Unauthenticated
 	// instead for those errors).
+	//
+	// This error code will not be generated by the gRPC core framework,
+	// but expect authentication middleware to use it.
 	PermissionDenied Code = 7
 
 	// ResourceExhausted indicates some resource has been exhausted, perhaps
 	// a per-user quota, or perhaps the entire file system is out of space.
+	//
+	// This error code will be generated by the gRPC framework in
+	// out-of-memory and server overload situations, or when a message is
+	// larger than the configured maximum size.
 	ResourceExhausted Code = 8
 
 	// FailedPrecondition indicates operation was rejected because the
@@ -94,6 +116,8 @@ const (
 	//      REST Get/Update/Delete on a resource and the resource on the
 	//      server does not match the condition. E.g., conflicting
 	//      read-modify-write on the same resource.
+	//
+	// This error code will not be generated by the gRPC framework.
 	FailedPrecondition Code = 9
 
 	// Aborted indicates the operation was aborted, typically due to a
@@ -102,6 +126,8 @@ const (
 	//
 	// See litmus test above for deciding between FailedPrecondition,
 	// Aborted, and Unavailable.
+	//
+	// This error code will not be generated by the gRPC framework.
 	Aborted Code = 10
 
 	// OutOfRange means operation was attempted past the valid range.
@@ -119,15 +145,26 @@ const (
 	// error) when it applies so that callers who are iterating through
 	// a space can easily look for an OutOfRange error to detect when
 	// they are done.
+	//
+	// This error code will not be generated by the gRPC framework.
 	OutOfRange Code = 11
 
 	// Unimplemented indicates operation is not implemented or not
 	// supported/enabled in this service.
+	//
+	// This error code will be generated by the gRPC framework. Most
+	// commonly, you will see this error code when a method implementation
+	// is missing on the server. It can also be generated for unknown
+	// compression algorithms or a disagreement as to whether an RPC should
+	// be streaming.
 	Unimplemented Code = 12
 
 	// Internal errors. Means some invariants expected by underlying
 	// system has been broken. If you see one of these errors,
 	// something is very broken.
+	//
+	// This error code will be generated by the gRPC framework in several
+	// internal error conditions.
 	Internal Code = 13
 
 	// Unavailable indicates the service is currently unavailable.
@@ -137,13 +174,22 @@ const (
 	//
 	// See litmus test above for deciding between FailedPrecondition,
 	// Aborted, and Unavailable.
+	//
+	// This error code will be generated by the gRPC framework during
+	// abrupt shutdown of a server process or network connection.
 	Unavailable Code = 14
 
 	// DataLoss indicates unrecoverable data loss or corruption.
+	//
+	// This error code will not be generated by the gRPC framework.
 	DataLoss Code = 15
 
 	// Unauthenticated indicates the request does not have valid
 	// authentication credentials for the operation.
+	//
+	// The gRPC framework will generate this error code when the
+	// authentication metadata is invalid or a Credentials callback fails,
+	// but also expect authentication middleware to generate it.
 	Unauthenticated Code = 16
 
 	_maxCode = 17

From abfbf74f21d37c828ecae93451a06b513653d0ff Mon Sep 17 00:00:00 2001
From: Evan Limanto <evanlimanto@berkeley.edu>
Date: Thu, 9 Jul 2020 13:39:24 -0700
Subject: [PATCH 112/481] doc: fix references to status methods (#3702)

---
 Documentation/rpc-errors.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Documentation/rpc-errors.md b/Documentation/rpc-errors.md
index 8bd610d1780e..0854090df469 100644
--- a/Documentation/rpc-errors.md
+++ b/Documentation/rpc-errors.md
@@ -61,8 +61,8 @@ exit status 1
 [status]:       https://godoc.org/google.golang.org/grpc/status#Status
 [new-status]:   https://godoc.org/google.golang.org/grpc/status#New
 [code]:         https://godoc.org/google.golang.org/grpc/codes#Code
-[with-details]: https://godoc.org/google.golang.org/grpc/status#Status.WithDetails
-[details]:      https://godoc.org/google.golang.org/grpc/status#Status.Details
-[status-err]:   https://godoc.org/google.golang.org/grpc/status#Status.Err
+[with-details]: https://godoc.org/google.golang.org/grpc/internal/status#Status.WithDetails
+[details]:      https://godoc.org/google.golang.org/grpc/internal/status#Status.Details
+[status-err]:   https://godoc.org/google.golang.org/grpc/internal/status#Status.Err
 [status-error]: https://godoc.org/google.golang.org/grpc/status#Error
 [example]:      https://github.com/grpc/grpc-go/tree/master/examples/features/errors

From d8193ee9cc3e403ec488bf371c57fe8a63e34890 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 9 Jul 2020 16:23:28 -0700
Subject: [PATCH 113/481] xds: Add v3 support for client bootstrap. (#3723)

---
 xds/internal/balancer/edsbalancer/eds_test.go |   3 +-
 .../edsbalancer/xds_client_wrapper_test.go    |   5 +-
 xds/internal/client/bootstrap/bootstrap.go    | 131 +++++--
 .../client/bootstrap/bootstrap_test.go        | 356 +++++++++++++-----
 xds/internal/client/client.go                 |   9 +-
 xds/internal/client/client_test.go            |   6 +-
 xds/internal/resolver/xds_resolver_test.go    |   8 +-
 .../testutils/{locality.go => protos.go}      |   9 +-
 xds/internal/version/version.go               |  32 ++
 9 files changed, 422 insertions(+), 137 deletions(-)
 rename xds/internal/testutils/{locality.go => protos.go} (75%)
 create mode 100644 xds/internal/version/version.go

diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 02cb9263e4c9..f70a0f2673be 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -25,7 +25,6 @@ import (
 	"reflect"
 	"testing"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/golang/protobuf/jsonpb"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
@@ -51,7 +50,7 @@ func init() {
 		return &bootstrap.Config{
 			BalancerName: testBalancerNameFooBar,
 			Creds:        grpc.WithInsecure(),
-			NodeProto:    &corepb.Node{},
+			NodeProto:    testutils.EmptyNodeProtoV2,
 		}, nil
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index f7cc4bda2d8c..b87b7463edb9 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -24,7 +24,6 @@ import (
 	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
@@ -101,7 +100,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 				return &bootstrap.Config{
 					BalancerName: fakeServer.Address,
 					Creds:        grpc.WithInsecure(),
-					NodeProto:    &corepb.Node{},
+					NodeProto:    testutils.EmptyNodeProtoV2,
 				}, nil
 			}
 			defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
@@ -138,7 +137,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 			wantReq := &xdspb.DiscoveryRequest{
 				TypeUrl:       edsType,
 				ResourceNames: []string{test.wantResourceName},
-				Node:          &corepb.Node{},
+				Node:          testutils.EmptyNodeProtoV2,
 			}
 			if !proto.Equal(edsReq.Req, wantReq) {
 				t.Fatalf("got EDS request %v, expected: %v, diff: %s", edsReq.Req, wantReq, cmp.Diff(edsReq.Req, wantReq, cmp.Comparer(proto.Equal)))
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 1ee0f1d47c5e..1e2e05e8f9b2 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -27,15 +27,25 @@ import (
 	"io/ioutil"
 	"os"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 const (
 	// Environment variable which holds the name of the xDS bootstrap file.
-	fileEnv = "GRPC_XDS_BOOTSTRAP"
+	bootstrapFileEnv = "GRPC_XDS_BOOTSTRAP"
+	// Environment variable which controls the use of xDS v3 API.
+	v3SupportEnv = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
+	// The "server_features" field in the bootstrap file contains a list of
+	// features supported by the server. A value of "xds_v3" indicates that the
+	// server supports the v3 version of the xDS transport protocol.
+	serverFeaturesV3 = "xds_v3"
+
 	// Type name for Google default credentials.
 	googleDefaultCreds              = "google_default"
 	gRPCUserAgentName               = "gRPC Go"
@@ -45,7 +55,7 @@ const (
 var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version)
 
 // For overriding in unit tests.
-var fileReadFunc = ioutil.ReadFile
+var bootstrapFileReadFunc = ioutil.ReadFile
 
 // Config provides the xDS client with several key bits of information that it
 // requires in its interaction with an xDS server. The Config is initialized
@@ -59,8 +69,13 @@ type Config struct {
 	// Creds contains the credentials to be used while talking to the xDS
 	// server, as a grpc.DialOption.
 	Creds grpc.DialOption
-	// NodeProto contains the node proto to be used in xDS requests.
-	NodeProto *corepb.Node
+	// TransportAPI indicates the API version of xDS transport protocol to use.
+	// This describes the xDS gRPC endpoint and version of
+	// DiscoveryRequest/Response used on the wire.
+	TransportAPI version.TransportAPI
+	// NodeProto contains the Node proto to be used in xDS requests. The actual
+	// type depends on the transport protocol version used.
+	NodeProto proto.Message
 }
 
 type channelCreds struct {
@@ -85,9 +100,10 @@ type xdsServer struct {
 //          "type": <string containing channel cred type>,
 //          "config": <JSON object containing config for the type>
 //        }
-//      ]
+//      ],
+//      "server_features": [ ... ]
 //    },
-//    "node": <JSON form of corepb.Node proto>
+//    "node": <JSON form of Node proto>
 // }
 //
 // Currently, we support exactly one type of credential, which is
@@ -101,13 +117,13 @@ type xdsServer struct {
 func NewConfig() (*Config, error) {
 	config := &Config{}
 
-	fName, ok := os.LookupEnv(fileEnv)
+	fName, ok := os.LookupEnv(bootstrapFileEnv)
 	if !ok {
-		return nil, fmt.Errorf("xds: Environment variable %v not defined", fileEnv)
+		return nil, fmt.Errorf("xds: Environment variable %v not defined", bootstrapFileEnv)
 	}
-	logger.Infof("Got bootstrap file location from %v environment variable: %v", fileEnv, fName)
+	logger.Infof("Got bootstrap file location from %v environment variable: %v", bootstrapFileEnv, fName)
 
-	data, err := fileReadFunc(fName)
+	data, err := bootstrapFileReadFunc(fName)
 	if err != nil {
 		return nil, fmt.Errorf("xds: Failed to read bootstrap file %s with error %v", fName, err)
 	}
@@ -118,11 +134,18 @@ func NewConfig() (*Config, error) {
 		return nil, fmt.Errorf("xds: Failed to parse file %s (content %v) with error: %v", fName, string(data), err)
 	}
 
+	serverSupportsV3 := false
 	m := jsonpb.Unmarshaler{AllowUnknownFields: true}
 	for k, v := range jsonData {
 		switch k {
 		case "node":
-			n := &corepb.Node{}
+			// We unconditionally convert the JSON into a v3.Node proto. The v3
+			// proto does not contain the deprecated field "build_version" from
+			// the v2 proto. We do not expect the bootstrap file to contain the
+			// "build_version" field. In any case, the unmarshal will succeed
+			// because we have set the `AllowUnknownFields` option on the
+			// unmarshaler.
+			n := &v3corepb.Node{}
 			if err := m.Unmarshal(bytes.NewReader(v), n); err != nil {
 				return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
 			}
@@ -144,6 +167,17 @@ func NewConfig() (*Config, error) {
 					break
 				}
 			}
+		case "server_features":
+			var features []string
+			if err := json.Unmarshal(v, &features); err != nil {
+				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
+			}
+			for _, f := range features {
+				switch f {
+				case serverFeaturesV3:
+					serverSupportsV3 = true
+				}
+			}
 		}
 		// Do not fail the xDS bootstrap when an unknown field is seen. This can
 		// happen when an older version client reads a newer version bootstrap
@@ -154,20 +188,69 @@ func NewConfig() (*Config, error) {
 		return nil, fmt.Errorf("xds: Required field %q not found in bootstrap", "xds_servers.server_uri")
 	}
 
-	// If we don't find a nodeProto in the bootstrap file, we just create an
-	// empty one here. That way, callers of this function can always expect
-	// that the NodeProto field is non-nil.
-	if config.NodeProto == nil {
-		config.NodeProto = &corepb.Node{}
+	// We end up using v3 transport protocol version only if the following
+	// conditions are met:
+	// 1. Server supports v3, indicated by the presence of "xds_v3" in
+	//    server_features.
+	// 2. Environment variable "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" is set to
+	//    true.
+	// The default value of the enum type "version.TransportAPI" is v2.
+	if v3Env := os.Getenv(v3SupportEnv); v3Env == "true" {
+		if serverSupportsV3 {
+			config.TransportAPI = version.TransportV3
+		}
 	}
-	// BuildVersion is deprecated, and is replaced by user_agent_name and
-	// user_agent_version. But the management servers are still using the old
-	// field, so we will keep both set.
-	config.NodeProto.BuildVersion = gRPCVersion
-	config.NodeProto.UserAgentName = gRPCUserAgentName
-	config.NodeProto.UserAgentVersionType = &corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
-	config.NodeProto.ClientFeatures = append(config.NodeProto.ClientFeatures, clientFeatureNoOverprovisioning)
 
+	if err := config.updateNodeProto(); err != nil {
+		return nil, err
+	}
 	logger.Infof("Bootstrap config for creating xds-client: %+v", config)
 	return config, nil
 }
+
+// updateNodeProto updates the node proto read from the bootstrap file.
+//
+// Node proto in Config contains a v3.Node protobuf message corresponding to the
+// JSON contents found in the bootstrap file. This method performs some post
+// processing on it:
+// 1. If we don't find a nodeProto in the bootstrap file, we create an empty one
+// here. That way, callers of this function can always expect that the NodeProto
+// field is non-nil.
+// 2. If the transport protocol version to be used is not v3, we convert the
+// current v3.Node proto in a v2.Node proto.
+// 3. Some additional fields which are not expected to be set in the bootstrap
+// file are populated here.
+func (c *Config) updateNodeProto() error {
+	if c.TransportAPI == version.TransportV3 {
+		v3, _ := c.NodeProto.(*v3corepb.Node)
+		if v3 == nil {
+			v3 = &v3corepb.Node{}
+		}
+		v3.UserAgentName = gRPCUserAgentName
+		v3.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
+		v3.ClientFeatures = append(v3.ClientFeatures, clientFeatureNoOverprovisioning)
+		c.NodeProto = v3
+		return nil
+	}
+
+	v2 := &v2corepb.Node{}
+	if c.NodeProto != nil {
+		v3, err := proto.Marshal(c.NodeProto)
+		if err != nil {
+			return fmt.Errorf("xds: proto.Marshal(%v): %v", c.NodeProto, err)
+		}
+		if err := proto.Unmarshal(v3, v2); err != nil {
+			return fmt.Errorf("xds: proto.Unmarshal(%v): %v", v3, err)
+		}
+	}
+	c.NodeProto = v2
+
+	// BuildVersion is deprecated, and is replaced by user_agent_name and
+	// user_agent_version. But the management servers are still using the old
+	// field, so we will keep both set.
+	v2.BuildVersion = gRPCVersion
+	v2.UserAgentName = gRPCUserAgentName
+	v2.UserAgentVersionType = &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
+	v2.ClientFeatures = append(v2.ClientFeatures, clientFeatureNoOverprovisioning)
+	return nil
+}
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index de0ef3954e72..4a825aa35d84 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -21,70 +21,28 @@
 package bootstrap
 
 import (
+	"fmt"
 	"os"
 	"testing"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/proto"
 	structpb "github.com/golang/protobuf/ptypes/struct"
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 var (
-	nodeProto = &corepb.Node{
-		Id: "ENVOY_NODE_ID",
-		Metadata: &structpb.Struct{
-			Fields: map[string]*structpb.Value{
-				"TRAFFICDIRECTOR_GRPC_HOSTNAME": {
-					Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"},
-				},
-			},
-		},
-		BuildVersion:         gRPCVersion,
-		UserAgentName:        gRPCUserAgentName,
-		UserAgentVersionType: &corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
-		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
-	}
-	nilCredsConfig = &Config{
-		BalancerName: "trafficdirector.googleapis.com:443",
-		Creds:        nil,
-		NodeProto:    nodeProto,
-	}
-	nonNilCredsConfig = &Config{
-		BalancerName: "trafficdirector.googleapis.com:443",
-		Creds:        grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
-		NodeProto:    nodeProto,
-	}
-)
-
-// TODO: enable leak check for this package when
-// https://github.com/googleapis/google-cloud-go/issues/2417 is fixed.
-
-// TestNewConfig exercises the functionality in NewConfig with different
-// bootstrap file contents. It overrides the fileReadFunc by returning
-// bootstrap file contents defined in this test, instead of reading from a
-// file.
-func TestNewConfig(t *testing.T) {
-	bootstrapFileMap := map[string]string{
-		"empty":          "",
-		"badJSON":        `["test": 123]`,
-		"noBalancerName": `{"node": {"id": "ENVOY_NODE_ID"}}`,
+	v2BootstrapFileMap = map[string]string{
 		"emptyNodeProto": `
 		{
 			"xds_servers" : [{
 				"server_uri": "trafficdirector.googleapis.com:443"
 			}]
 		}`,
-		"emptyXdsServer": `
-		{
-			"node": {
-				"id": "ENVOY_NODE_ID",
-				"metadata": {
-				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
-			    }
-			}
-		}`,
 		"unknownTopLevelFieldInFile": `
 		{
 			"node": {
@@ -208,80 +166,286 @@ func TestNewConfig(t *testing.T) {
 			]
 		}`,
 	}
+	v3BootstrapFileMap = map[string]string{
+		"serverDoesNotSupportsV3": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar"]
+		}`,
+		"serverSupportsV3": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"]
+		}`,
+	}
+	metadata = &structpb.Struct{
+		Fields: map[string]*structpb.Value{
+			"TRAFFICDIRECTOR_GRPC_HOSTNAME": {
+				Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"},
+			},
+		},
+	}
+	v2NodeProto = &v2corepb.Node{
+		Id:                   "ENVOY_NODE_ID",
+		Metadata:             metadata,
+		BuildVersion:         gRPCVersion,
+		UserAgentName:        gRPCUserAgentName,
+		UserAgentVersionType: &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
+	}
+	v3NodeProto = &v3corepb.Node{
+		Id:                   "ENVOY_NODE_ID",
+		Metadata:             metadata,
+		UserAgentName:        gRPCUserAgentName,
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
+	}
+	nilCredsConfigV2 = &Config{
+		BalancerName: "trafficdirector.googleapis.com:443",
+		Creds:        nil,
+		NodeProto:    v2NodeProto,
+	}
+	nonNilCredsConfigV2 = &Config{
+		BalancerName: "trafficdirector.googleapis.com:443",
+		Creds:        grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
+		NodeProto:    v2NodeProto,
+	}
+	nonNilCredsConfigV3 = &Config{
+		BalancerName: "trafficdirector.googleapis.com:443",
+		Creds:        grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
+		TransportAPI: version.TransportV3,
+		NodeProto:    v3NodeProto,
+	}
+)
 
-	oldFileReadFunc := fileReadFunc
-	fileReadFunc = func(name string) ([]byte, error) {
+func (c *Config) compare(want *Config) error {
+	if c.BalancerName != want.BalancerName {
+		return fmt.Errorf("config.BalancerName is %s, want %s", c.BalancerName, want.BalancerName)
+	}
+	// Since Creds is of type grpc.DialOption interface, where the
+	// implementation is provided by a function, it is not possible to compare.
+	if (c.Creds != nil) != (want.Creds != nil) {
+		return fmt.Errorf("config.Creds is %#v, want %#v", c.Creds, want.Creds)
+	}
+	if c.TransportAPI != want.TransportAPI {
+		return fmt.Errorf("config.TransportAPI is %v, want %v", c.TransportAPI, want.TransportAPI)
+
+	}
+	if diff := cmp.Diff(want.NodeProto, c.NodeProto, cmp.Comparer(proto.Equal)); diff != "" {
+		return fmt.Errorf("config.NodeProto diff (-want, +got):\n%s", diff)
+	}
+	return nil
+}
+
+func setupBootstrapOverride(bootstrapFileMap map[string]string) func() {
+	oldFileReadFunc := bootstrapFileReadFunc
+	bootstrapFileReadFunc = func(name string) ([]byte, error) {
 		if b, ok := bootstrapFileMap[name]; ok {
 			return []byte(b), nil
 		}
 		return nil, os.ErrNotExist
 	}
-	defer func() {
-		fileReadFunc = oldFileReadFunc
-		os.Unsetenv(fileEnv)
-	}()
+	return func() {
+		bootstrapFileReadFunc = oldFileReadFunc
+		os.Unsetenv(bootstrapFileEnv)
+	}
+}
+
+// TODO: enable leak check for this package when
+// https://github.com/googleapis/google-cloud-go/issues/2417 is fixed.
+
+// TestNewConfigV2ProtoFailure exercises the functionality in NewConfig with
+// different bootstrap file contents which are expected to fail.
+func TestNewConfigV2ProtoFailure(t *testing.T) {
+	bootstrapFileMap := map[string]string{
+		"empty":          "",
+		"badJSON":        `["test": 123]`,
+		"noBalancerName": `{"node": {"id": "ENVOY_NODE_ID"}}`,
+		"emptyXdsServer": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			}
+		}`,
+	}
+	cancel := setupBootstrapOverride(bootstrapFileMap)
+	defer cancel()
+
+	tests := []struct {
+		name      string
+		wantError bool
+	}{
+		{"nonExistentBootstrapFile", true},
+		{"empty", true},
+		{"badJSON", true},
+		{"noBalancerName", true},
+		{"emptyXdsServer", true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
+				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
+			}
+			if _, err := NewConfig(); err == nil {
+				t.Fatalf("NewConfig() returned nil error, expected to fail")
+			}
+		})
+	}
+}
+
+// TestNewConfigV2ProtoSuccess exercises the functionality in NewConfig with
+// different bootstrap file contents. It overrides the fileReadFunc by returning
+// bootstrap file contents defined in this test, instead of reading from a file.
+func TestNewConfigV2ProtoSuccess(t *testing.T) {
+	cancel := setupBootstrapOverride(v2BootstrapFileMap)
+	defer cancel()
 
 	tests := []struct {
 		name       string
 		wantConfig *Config
-		wantError  bool
 	}{
-		{"nonExistentBootstrapFile", nil, true},
-		{"empty", nil, true},
-		{"badJSON", nil, true},
-		{"emptyNodeProto", &Config{
-			BalancerName: "trafficdirector.googleapis.com:443",
-			NodeProto: &corepb.Node{
-				BuildVersion:         gRPCVersion,
-				UserAgentName:        gRPCUserAgentName,
-				UserAgentVersionType: &corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
-				ClientFeatures:       []string{clientFeatureNoOverprovisioning},
+		{
+			"emptyNodeProto", &Config{
+				BalancerName: "trafficdirector.googleapis.com:443",
+				NodeProto: &v2corepb.Node{
+					BuildVersion:         gRPCVersion,
+					UserAgentName:        gRPCUserAgentName,
+					UserAgentVersionType: &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+					ClientFeatures:       []string{clientFeatureNoOverprovisioning},
+				},
 			},
-		}, false},
-		{"noBalancerName", nil, true},
-		{"emptyXdsServer", nil, true},
-		{"unknownTopLevelFieldInFile", nilCredsConfig, false},
-		{"unknownFieldInNodeProto", nilCredsConfig, false},
-		{"unknownFieldInXdsServer", nilCredsConfig, false},
-		{"emptyChannelCreds", nilCredsConfig, false},
-		{"nonGoogleDefaultCreds", nilCredsConfig, false},
-		{"multipleChannelCreds", nonNilCredsConfig, false},
-		{"goodBootstrap", nonNilCredsConfig, false},
-		{"multipleXDSServers", nonNilCredsConfig, false},
+		},
+		{"unknownTopLevelFieldInFile", nilCredsConfigV2},
+		{"unknownFieldInNodeProto", nilCredsConfigV2},
+		{"unknownFieldInXdsServer", nilCredsConfigV2},
+		{"emptyChannelCreds", nilCredsConfigV2},
+		{"nonGoogleDefaultCreds", nilCredsConfigV2},
+		{"multipleChannelCreds", nonNilCredsConfigV2},
+		{"goodBootstrap", nonNilCredsConfigV2},
+		{"multipleXDSServers", nonNilCredsConfigV2},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := os.Setenv(fileEnv, test.name); err != nil {
-				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", fileEnv, test.name, err)
+			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
+				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
 			}
-			config, err := NewConfig()
+			c, err := NewConfig()
 			if err != nil {
-				if !test.wantError {
-					t.Fatalf("unexpected error %v", err)
-				}
-				return
+				t.Fatalf("NewConfig() failed: %v", err)
 			}
-			if test.wantError {
-				t.Fatalf("wantError: %v, got error %v", test.wantError, err)
+			if err := c.compare(test.wantConfig); err != nil {
+				t.Fatal(err)
 			}
-			if config.BalancerName != test.wantConfig.BalancerName {
-				t.Errorf("config.BalancerName is %s, want %s", config.BalancerName, test.wantConfig.BalancerName)
+		})
+	}
+}
+
+// TestNewConfigV3SupportNotEnabledOnClient verifies bootstrap functionality
+// when the GRPC_XDS_EXPERIMENTAL_V3_SUPPORT environment variable is not enabled
+// on the client. In this case, whether the server supports v3 or not, the
+// client will end up using v2.
+func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) {
+	if err := os.Setenv(v3SupportEnv, "false"); err != nil {
+		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
+	}
+	defer os.Unsetenv(v3SupportEnv)
+
+	cancel := setupBootstrapOverride(v3BootstrapFileMap)
+	defer cancel()
+
+	tests := []struct {
+		name       string
+		wantConfig *Config
+	}{
+		{"serverDoesNotSupportsV3", nonNilCredsConfigV2},
+		{"serverSupportsV3", nonNilCredsConfigV2},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
+				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
 			}
-			if !proto.Equal(config.NodeProto, test.wantConfig.NodeProto) {
-				t.Errorf("config.NodeProto is %#v, want %#v", config.NodeProto, test.wantConfig.NodeProto)
+			c, err := NewConfig()
+			if err != nil {
+				t.Fatalf("NewConfig() failed: %v", err)
+			}
+			if err := c.compare(test.wantConfig); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// TestNewConfigV3SupportEnabledOnClient verifies bootstrap functionality when
+// the GRPC_XDS_EXPERIMENTAL_V3_SUPPORT environment variable is enabled on the
+// client. Here the client ends up using v2 or v3 based on what the server
+// supports.
+func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
+	if err := os.Setenv(v3SupportEnv, "true"); err != nil {
+		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
+	}
+	defer os.Unsetenv(v3SupportEnv)
+
+	cancel := setupBootstrapOverride(v3BootstrapFileMap)
+	defer cancel()
+
+	tests := []struct {
+		name       string
+		wantConfig *Config
+	}{
+		{"serverDoesNotSupportsV3", nonNilCredsConfigV2},
+		{"serverSupportsV3", nonNilCredsConfigV3},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
+				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
+			}
+			c, err := NewConfig()
+			if err != nil {
+				t.Fatalf("NewConfig() failed: %v", err)
 			}
-			if (config.Creds != nil) != (test.wantConfig.Creds != nil) {
-				t.Errorf("config.Creds is %#v, want %#v", config.Creds, test.wantConfig.Creds)
+			if err := c.compare(test.wantConfig); err != nil {
+				t.Fatal(err)
 			}
 		})
 	}
 }
 
-func TestNewConfigEnvNotSet(t *testing.T) {
-	os.Unsetenv(fileEnv)
-	config, err := NewConfig()
-	if err == nil {
-		t.Errorf("NewConfig() returned: %#v, <nil>, wanted non-nil error", config)
+// TestNewConfigBootstrapFileEnvNotSet tests the case where the bootstrap file
+// environment variable is not set.
+func TestNewConfigBootstrapFileEnvNotSet(t *testing.T) {
+	os.Unsetenv(bootstrapFileEnv)
+	if _, err := NewConfig(); err == nil {
+		t.Errorf("NewConfig() returned nil error, expected to fail")
 	}
 }
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 127734fd593a..07a0a2637c0a 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 // Options provides all parameters required for the creation of an xDS client.
@@ -131,7 +132,13 @@ func New(opts Options) (*Client, error) {
 	c.logger = prefixLogger((c))
 	c.logger.Infof("Created ClientConn to xDS server: %s", opts.Config.BalancerName)
 
-	c.v2c = newXDSV2Client(c, cc, opts.Config.NodeProto, backoff.DefaultExponential.Backoff, c.logger)
+	if opts.Config.TransportAPI == version.TransportV2 {
+		c.v2c = newXDSV2Client(c, cc, opts.Config.NodeProto.(*corepb.Node), backoff.DefaultExponential.Backoff, c.logger)
+	} else {
+		// TODO(easwars): Remove this once v3Client is ready.
+		return nil, errors.New("xds v3 client is not yet supported")
+	}
+
 	c.logger.Infof("Created")
 	go c.run()
 	return c, nil
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 367e9e02642a..043fa4f76de1 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -55,7 +55,7 @@ func clientOpts(balancerName string) Options {
 		Config: bootstrap.Config{
 			BalancerName: balancerName,
 			Creds:        grpc.WithInsecure(),
-			NodeProto:    &corepb.Node{},
+			NodeProto:    testutils.EmptyNodeProtoV2,
 		},
 	}
 }
@@ -78,7 +78,7 @@ func (s) TestNew(t *testing.T) {
 			opts: Options{
 				Config: bootstrap.Config{
 					Creds:     grpc.WithInsecure(),
-					NodeProto: &corepb.Node{},
+					NodeProto: testutils.EmptyNodeProtoV2,
 				},
 			},
 			wantErr: true,
@@ -88,7 +88,7 @@ func (s) TestNew(t *testing.T) {
 			opts: Options{
 				Config: bootstrap.Config{
 					BalancerName: "dummy",
-					NodeProto:    &corepb.Node{},
+					NodeProto:    testutils.EmptyNodeProtoV2,
 				},
 			},
 			wantErr: true,
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 7e1e49d2d92a..dcb5c9e8e944 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -37,8 +37,6 @@ import (
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 )
 
 const (
@@ -51,7 +49,7 @@ var (
 	validConfig = bootstrap.Config{
 		BalancerName: balancerName,
 		Creds:        grpc.WithInsecure(),
-		NodeProto:    &corepb.Node{},
+		NodeProto:    testutils.EmptyNodeProtoV2,
 	}
 	target = resolver.Target{Endpoint: targetStr}
 )
@@ -138,7 +136,7 @@ func TestResolverBuilder(t *testing.T) {
 			rbo:  resolver.BuildOptions{},
 			config: bootstrap.Config{
 				Creds:     grpc.WithInsecure(),
-				NodeProto: &corepb.Node{},
+				NodeProto: testutils.EmptyNodeProtoV2,
 			},
 			wantErr: true,
 		},
@@ -147,7 +145,7 @@ func TestResolverBuilder(t *testing.T) {
 			rbo:  resolver.BuildOptions{},
 			config: bootstrap.Config{
 				BalancerName: balancerName,
-				NodeProto:    &corepb.Node{},
+				NodeProto:    testutils.EmptyNodeProtoV2,
 			},
 			xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}),
 			wantErr:       false,
diff --git a/xds/internal/testutils/locality.go b/xds/internal/testutils/protos.go
similarity index 75%
rename from xds/internal/testutils/locality.go
rename to xds/internal/testutils/protos.go
index a4e4cc59b865..fb60e91f2e34 100644
--- a/xds/internal/testutils/locality.go
+++ b/xds/internal/testutils/protos.go
@@ -18,13 +18,16 @@
 package testutils
 
 import (
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"google.golang.org/grpc/xds/internal"
 )
 
+// EmptyNodeProtoV2 is a node proto with no fields set.
+var EmptyNodeProtoV2 = &v2corepb.Node{}
+
 // LocalityIDToProto converts a LocalityID to its proto representation.
-func LocalityIDToProto(l internal.LocalityID) *corepb.Locality {
-	return &corepb.Locality{
+func LocalityIDToProto(l internal.LocalityID) *v2corepb.Locality {
+	return &v2corepb.Locality{
 		Region:  l.Region,
 		Zone:    l.Zone,
 		SubZone: l.SubZone,
diff --git a/xds/internal/version/version.go b/xds/internal/version/version.go
new file mode 100644
index 000000000000..0425fa5e33fd
--- /dev/null
+++ b/xds/internal/version/version.go
@@ -0,0 +1,32 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package version defines supported xDS API versions.
+package version
+
+// TransportAPI refers to the API version for xDS transport protocol. This
+// describes the xDS gRPC endpoint and version of DiscoveryRequest/Response used
+// on the wire.
+type TransportAPI int
+
+const (
+	// TransportV2 refers to the v2 xDS transport protocol.
+	TransportV2 TransportAPI = iota
+	// TransportV3 refers to the v3 xDS transport protocol.
+	TransportV3
+)

From a6c3c6968e905ec78603b11cdbccc2374ba33179 Mon Sep 17 00:00:00 2001
From: Patrice Chalin <chalin@users.noreply.github.com>
Date: Tue, 14 Jul 2020 19:59:29 -0400
Subject: [PATCH 114/481] examples: cleanup README.md (#3738)

---
 examples/README.md | 75 +++++++++++++---------------------------------
 1 file changed, 20 insertions(+), 55 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index d6fd0fa04e41..bb2138f26ffb 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,64 +1,29 @@
-gRPC in 3 minutes (Go)
-======================
+# gRPC Hello World
 
-BACKGROUND
--------------
-For this sample, we've already generated the server and client stubs from [helloworld.proto](helloworld/helloworld/helloworld.proto).
+Follow these setup to run the [quick start][] example:
 
-PREREQUISITES
--------------
+ 1. Get the code:
 
-- This requires Go 1.9 or later
-- Requires that [GOPATH is set](https://golang.org/doc/code.html#GOPATH)
+    ```console
+    $ go get google.golang.org/grpc/examples/helloworld/greeter_client
+    $ go get google.golang.org/grpc/examples/helloworld/greeter_server
+    ```
 
-```
-$ go help gopath
-$ # ensure the PATH contains $GOPATH/bin
-$ export PATH=$PATH:$GOPATH/bin
-```
+ 2. Run the server:
 
-INSTALL
--------
+    ```console
+    $ $(go env GOPATH)/bin/greeter_server &
+    ```
 
-```
-$ go get -u google.golang.org/grpc/examples/helloworld/greeter_client
-$ go get -u google.golang.org/grpc/examples/helloworld/greeter_server
-```
+ 3. Run the client:
 
-TRY IT!
--------
+    ```console
+    $ $(go env GOPATH)/bin/greeter_client
+    Greeting: Hello world
+    ```
 
-- Run the server
+For more details (including instructions for making a small change to the
+example code) or if you're having trouble running this example, see [Quick
+Start][].
 
-  ```
-  $ greeter_server &
-  ```
-
-- Run the client
-
-  ```
-  $ greeter_client
-  ```
-
-OPTIONAL - Rebuilding the generated code
-----------------------------------------
-
-1. Install [protobuf compiler](https://github.com/google/protobuf/blob/master/README.md#protocol-compiler-installation)
-
-1. Install the protoc Go plugin
-
-   ```
-   $ go get -u github.com/golang/protobuf/protoc-gen-go
-   ```
-
-1. Rebuild the generated Go code
-
-   ```
-   $ go generate google.golang.org/grpc/examples/helloworld/...
-   ```
-
-   Or run `protoc` command (with the grpc plugin)
-
-   ```
-   $ protoc -I helloworld/ helloworld/helloworld.proto --go_out=plugins=grpc:helloworld
-   ```
+[quick start]: https://grpc.io/docs/languages/go/quickstart

From 46d42f139d84e0db342c37381dfb23be667c3438 Mon Sep 17 00:00:00 2001
From: Patrice Chalin <chalin@users.noreply.github.com>
Date: Wed, 15 Jul 2020 13:41:05 -0400
Subject: [PATCH 115/481] README: adjust links, and slight rework of page
 (#3736)

---
 README.md | 123 +++++++++++++++++++++++++++---------------------------
 1 file changed, 61 insertions(+), 62 deletions(-)

diff --git a/README.md b/README.md
index 249cd2063279..fef78e4a1dff 100644
--- a/README.md
+++ b/README.md
@@ -1,65 +1,53 @@
 # gRPC-Go
 
 [![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go)
-[![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc)
+[![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)][API]
 [![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go)
 
-The Go implementation of [gRPC](https://grpc.io/): A high performance, open
-source, general RPC framework that puts mobile and HTTP/2 first. For more
-information see the [gRPC Quick Start:
-Go](https://grpc.io/docs/languages/go/quickstart/) guide.
+The [Go][] implementation of [gRPC][]: A high performance, open source, general
+RPC framework that puts mobile and HTTP/2 first. For more information see the
+[Go gRPC docs][], or jump directly into the [quick start][].
 
-Installation
-------------
+## Prerequisites
 
-To install this package, you need to install Go and setup your Go workspace on
-your computer. The simplest way to install the library is to run:
+- **[Go][]**: any one of the **three latest major** [releases][go-releases].
 
-```
-$ go get -u google.golang.org/grpc
+## Installation
+
+With [Go module][] support (Go 1.11+), simply add the following import
+
+```go
+import "google.golang.org/grpc"
 ```
 
-With Go module support (Go 1.11+), simply `import "google.golang.org/grpc"` in
-your source code and `go [build|run|test]` will automatically download the
-necessary dependencies ([Go modules
-ref](https://github.com/golang/go/wiki/Modules)).
+to your code, and then `go [build|run|test]` will automatically fetch the
+necessary dependencies.
 
-If you are trying to access grpc-go from within China, please see the
-[FAQ](#FAQ) below.
+Otherwise, to install the `grpc-go` package, run the following command:
 
-Prerequisites
--------------
-gRPC-Go officially supports the
-[three latest major releases of Go](https://golang.org/doc/devel/release.html).
+```console
+$ go get -u google.golang.org/grpc
+```
 
-Documentation
--------------
-- See [godoc](https://godoc.org/google.golang.org/grpc) for package and API
-  descriptions.
-- Documentation on specific topics can be found in the [Documentation
-  directory](Documentation/).
-- Examples can be found in the [examples directory](examples/).
+> **Note:** If you are trying to access `grpc-go` from **China**, see the
+> [FAQ](#FAQ) below.
 
-Performance
------------
-Performance benchmark data for grpc-go and other languages is maintained in
-[this
-dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696).
+## Learn more
 
-Status
-------
-General Availability [Google Cloud Platform Launch
-Stages](https://cloud.google.com/terms/launch-stages).
+- [Go gRPC docs][], which include a [quick start][] and [API
+  reference][API] among other resources
+- [Low-level technical docs](Documentation) from this repository
+- [Performance benchmark][]
+- [Examples](examples)
 
-FAQ
----
+## FAQ
 
-#### I/O Timeout Errors
+### I/O Timeout Errors
 
-The `golang.org` domain may be blocked from some countries.  `go get` usually
+The `golang.org` domain may be blocked from some countries. `go get` usually
 produces an error like the following when this happens:
 
-```
+```console
 $ go get -u google.golang.org/grpc
 package google.golang.org/grpc: unrecognized import path "google.golang.org/grpc" (https fetch: Get https://google.golang.org/grpc?go-get=1: dial tcp 216.239.37.1:443: i/o timeout)
 ```
@@ -70,7 +58,7 @@ To build Go code, there are several options:
 
 - Without Go module support: `git clone` the repo manually:
 
-  ```
+  ```sh
   git clone https://github.com/grpc/grpc-go.git $GOPATH/src/google.golang.org/grpc
   ```
 
@@ -80,7 +68,7 @@ To build Go code, there are several options:
 - With Go module support: it is possible to use the `replace` feature of `go
   mod` to create aliases for golang.org packages.  In your project's directory:
 
-  ```
+  ```sh
   go mod edit -replace=google.golang.org/grpc=github.com/grpc/grpc-go@latest
   go mod tidy
   go mod vendor
@@ -88,19 +76,17 @@ To build Go code, there are several options:
   ```
 
   Again, this will need to be done for all transitive dependencies hosted on
-  golang.org as well.  Please refer to [this
-  issue](https://github.com/golang/go/issues/28652) in the golang repo regarding
-  this concern.
+  golang.org as well. For details, refer to [golang/go issue #28652](https://github.com/golang/go/issues/28652).
 
-#### Compiling error, undefined: grpc.SupportPackageIsVersion
+### Compiling error, undefined: grpc.SupportPackageIsVersion
 
-##### If you are using Go modules:
+#### If you are using Go modules:
 
-Please ensure your gRPC-Go version is `require`d at the appropriate version in
+Ensure your gRPC-Go version is `require`d at the appropriate version in
 the same module containing the generated `.pb.go` files.  For example,
 `SupportPackageIsVersion6` needs `v1.27.0`, so in your `go.mod` file:
 
-```
+```go
 module <your module name>
 
 require (
@@ -108,23 +94,27 @@ require (
 )
 ```
 
-##### If you are *not* using Go modules:
+#### If you are *not* using Go modules:
 
-Please update proto package, gRPC package and rebuild the proto files:
- - `go get -u github.com/golang/protobuf/{proto,protoc-gen-go}`
- - `go get -u google.golang.org/grpc`
- - `protoc --go_out=plugins=grpc:. *.proto`
+Update the `proto` package, gRPC package, and rebuild the `.proto` files:
 
-#### How to turn on logging
+```sh
+go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
+go get -u google.golang.org/grpc
+protoc --go_out=plugins=grpc:. *.proto
+```
 
-The default logger is controlled by the environment variables. Turn everything
-on by setting:
+### How to turn on logging
 
-```
-GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info
+The default logger is controlled by environment variables. Turn everything on
+like this:
+
+```console
+$ export GRPC_GO_LOG_VERBOSITY_LEVEL=99
+$ export GRPC_GO_LOG_SEVERITY_LEVEL=info
 ```
 
-#### The RPC failed with error `"code = Unavailable desc = transport is closing"`
+### The RPC failed with error `"code = Unavailable desc = transport is closing"`
 
 This error means the connection the RPC is using was closed, and there are many
 possible reasons, including:
@@ -140,3 +130,12 @@ It can be tricky to debug this because the error happens on the client side but
 the root cause of the connection being closed is on the server side. Turn on
 logging on __both client and server__, and see if there are any transport
 errors.
+
+[API]: https://grpc.io/docs/languages/go/api
+[Go]: https://golang.org
+[Go module]: https://github.com/golang/go/wiki/Modules
+[gRPC]: https://grpc.io
+[Go gRPC docs]: https://grpc.io/docs/languages/go
+[Performance benchmark]: https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696
+[quick start]: https://grpc.io/docs/languages/go/quickstart
+[go-releases]: https://golang.org/doc/devel/release.html

From 9fcde86ebe77f0508752f640ab4436f430b082ea Mon Sep 17 00:00:00 2001
From: Ryan Kim <Ryanfsdf@users.noreply.github.com>
Date: Wed, 15 Jul 2020 14:08:37 -0600
Subject: [PATCH 116/481] credentials/alts: Change ALTS hsConn to a map (#3741)

---
 .../internal/handshaker/service/service.go    | 13 ++++--
 .../handshaker/service/service_test.go        | 46 ++++++++++++-------
 2 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/credentials/alts/internal/handshaker/service/service.go b/credentials/alts/internal/handshaker/service/service.go
index 0c7b568354d2..77d759cd956f 100644
--- a/credentials/alts/internal/handshaker/service/service.go
+++ b/credentials/alts/internal/handshaker/service/service.go
@@ -27,9 +27,12 @@ import (
 )
 
 var (
-	// hsConn represents a connection to hypervisor handshaker service.
-	hsConn *grpc.ClientConn
-	mu     sync.Mutex
+	// mu guards hsConnMap and hsDialer.
+	mu sync.Mutex
+	// hsConn represents a mapping from a hypervisor handshaker service address
+	// to a corresponding connection to a hypervisor handshaker service
+	// instance.
+	hsConnMap = make(map[string]*grpc.ClientConn)
 	// hsDialer will be reassigned in tests.
 	hsDialer = grpc.Dial
 )
@@ -41,7 +44,8 @@ func Dial(hsAddress string) (*grpc.ClientConn, error) {
 	mu.Lock()
 	defer mu.Unlock()
 
-	if hsConn == nil {
+	hsConn, ok := hsConnMap[hsAddress]
+	if !ok {
 		// Create a new connection to the handshaker service. Note that
 		// this connection stays open until the application is closed.
 		var err error
@@ -49,6 +53,7 @@ func Dial(hsAddress string) (*grpc.ClientConn, error) {
 		if err != nil {
 			return nil, err
 		}
+		hsConnMap[hsAddress] = hsConn
 	}
 	return hsConn, nil
 }
diff --git a/credentials/alts/internal/handshaker/service/service_test.go b/credentials/alts/internal/handshaker/service/service_test.go
index 98160bf01c5f..28b4af757206 100644
--- a/credentials/alts/internal/handshaker/service/service_test.go
+++ b/credentials/alts/internal/handshaker/service/service_test.go
@@ -25,8 +25,8 @@ import (
 )
 
 const (
-	// The address is irrelevant in this test.
-	testAddress = "some_address"
+	testAddress1 = "some_address_1"
+	testAddress2 = "some_address_2"
 )
 
 func TestDial(t *testing.T) {
@@ -40,30 +40,44 @@ func TestDial(t *testing.T) {
 		}
 	}()
 
-	// Ensure that hsConn is nil at first.
-	hsConn = nil
-
-	// First call to Dial, it should create set hsConn.
-	conn1, err := Dial(testAddress)
+	// First call to Dial, it should create a connection to the server running
+	// at the given address.
+	conn1, err := Dial(testAddress1)
 	if err != nil {
-		t.Fatalf("first call to Dial failed: %v", err)
+		t.Fatalf("first call to Dial(%v) failed: %v", testAddress1, err)
 	}
 	if conn1 == nil {
-		t.Fatal("first call to Dial(_)=(nil, _), want not nil")
+		t.Fatalf("first call to Dial(%v)=(nil, _), want not nil", testAddress1)
 	}
-	if got, want := hsConn, conn1; got != want {
-		t.Fatalf("hsConn=%v, want %v", got, want)
+	if got, want := hsConnMap[testAddress1], conn1; got != want {
+		t.Fatalf("hsConnMap[%v]=%v, want %v", testAddress1, got, want)
 	}
 
 	// Second call to Dial should return conn1 above.
-	conn2, err := Dial(testAddress)
+	conn2, err := Dial(testAddress1)
 	if err != nil {
-		t.Fatalf("second call to Dial(_) failed: %v", err)
+		t.Fatalf("second call to Dial(%v) failed: %v", testAddress1, err)
 	}
 	if got, want := conn2, conn1; got != want {
-		t.Fatalf("second call to Dial(_)=(%v, _), want (%v,. _)", got, want)
+		t.Fatalf("second call to Dial(%v)=(%v, _), want (%v,. _)", testAddress1, got, want)
+	}
+	if got, want := hsConnMap[testAddress1], conn1; got != want {
+		t.Fatalf("hsConnMap[%v]=%v, want %v", testAddress1, got, want)
+	}
+
+	// Third call to Dial using a different address should create a new
+	// connection.
+	conn3, err := Dial(testAddress2)
+	if err != nil {
+		t.Fatalf("third call to Dial(%v) failed: %v", testAddress2, err)
+	}
+	if conn3 == nil {
+		t.Fatalf("third call to Dial(%v)=(nil, _), want not nil", testAddress2)
+	}
+	if got, want := hsConnMap[testAddress2], conn3; got != want {
+		t.Fatalf("hsConnMap[%v]=%v, want %v", testAddress2, got, want)
 	}
-	if got, want := hsConn, conn1; got != want {
-		t.Fatalf("hsConn=%v, want %v", got, want)
+	if got, want := conn2 == conn3, false; got != want {
+		t.Fatalf("(conn2==conn3)=%v, want %v", got, want)
 	}
 }

From dd8658f921014673870caacb6b3639b2b7e03017 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Wed, 15 Jul 2020 23:10:55 -0700
Subject: [PATCH 117/481] credentials: check and expose SPIFFE ID (#3626)

* credentials: check and expose SPIFFE ID
---
 credentials/tls.go                  |  28 ++++-
 internal/credentials/go110.go       |  65 ++++++++++
 internal/credentials/go110_test.go  | 182 ++++++++++++++++++++++++++++
 internal/credentials/gobefore110.go |  31 +++++
 4 files changed, 304 insertions(+), 2 deletions(-)
 create mode 100644 internal/credentials/go110.go
 create mode 100644 internal/credentials/go110_test.go
 create mode 100644 internal/credentials/gobefore110.go

diff --git a/credentials/tls.go b/credentials/tls.go
index 86e956bc8b77..1ba6f3a6b8f8 100644
--- a/credentials/tls.go
+++ b/credentials/tls.go
@@ -25,8 +25,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/url"
 
 	"google.golang.org/grpc/credentials/internal"
+	credinternal "google.golang.org/grpc/internal/credentials"
 )
 
 // TLSInfo contains the auth information for a TLS authenticated connection.
@@ -34,6 +36,8 @@ import (
 type TLSInfo struct {
 	State tls.ConnectionState
 	CommonAuthInfo
+	// This API is experimental.
+	SPIFFEID *url.URL
 }
 
 // AuthType returns the type of TLSInfo as a string.
@@ -94,7 +98,17 @@ func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawCon
 		conn.Close()
 		return nil, nil, ctx.Err()
 	}
-	return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState(), CommonAuthInfo{PrivacyAndIntegrity}}, nil
+	tlsInfo := TLSInfo{
+		State: conn.ConnectionState(),
+		CommonAuthInfo: CommonAuthInfo{
+			SecurityLevel: PrivacyAndIntegrity,
+		},
+	}
+	id := credinternal.SPIFFEIDFromState(conn.ConnectionState())
+	if id != nil {
+		tlsInfo.SPIFFEID = id
+	}
+	return internal.WrapSyscallConn(rawConn, conn), tlsInfo, nil
 }
 
 func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) {
@@ -103,7 +117,17 @@ func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error)
 		conn.Close()
 		return nil, nil, err
 	}
-	return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState(), CommonAuthInfo{PrivacyAndIntegrity}}, nil
+	tlsInfo := TLSInfo{
+		State: conn.ConnectionState(),
+		CommonAuthInfo: CommonAuthInfo{
+			SecurityLevel: PrivacyAndIntegrity,
+		},
+	}
+	id := credinternal.SPIFFEIDFromState(conn.ConnectionState())
+	if id != nil {
+		tlsInfo.SPIFFEID = id
+	}
+	return internal.WrapSyscallConn(rawConn, conn), tlsInfo, nil
 }
 
 func (c *tlsCreds) Clone() TransportCredentials {
diff --git a/internal/credentials/go110.go b/internal/credentials/go110.go
new file mode 100644
index 000000000000..d55b5203626e
--- /dev/null
+++ b/internal/credentials/go110.go
@@ -0,0 +1,65 @@
+// +build go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package credentials defines APIs for parsing SPIFFE ID.
+//
+// All APIs in this package are experimental.
+package credentials
+
+import (
+	"crypto/tls"
+	"net/url"
+
+	"google.golang.org/grpc/grpclog"
+)
+
+// SPIFFEIDFromState parses the SPIFFE ID from State. If the SPIFFE ID format
+// is invalid, return nil with warning.
+func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
+	if len(state.PeerCertificates) == 0 || len(state.PeerCertificates[0].URIs) == 0 {
+		return nil
+	}
+	var spiffeID *url.URL
+	for _, uri := range state.PeerCertificates[0].URIs {
+		if uri == nil || uri.Scheme != "spiffe" || uri.Opaque != "" || (uri.User != nil && uri.User.Username() != "") {
+			continue
+		}
+		// From this point, we assume the uri is intended for a SPIFFE ID.
+		if len(uri.String()) > 2048 {
+			grpclog.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes")
+			return nil
+		}
+		if len(uri.Host) == 0 || len(uri.RawPath) == 0 || len(uri.Path) == 0 {
+			grpclog.Warning("invalid SPIFFE ID: domain or workload ID is empty")
+			return nil
+		}
+		if len(uri.Host) > 255 {
+			grpclog.Warning("invalid SPIFFE ID: domain length larger than 255 characters")
+			return nil
+		}
+		// A valid SPIFFE certificate can only have exactly one URI SAN field.
+		if len(state.PeerCertificates[0].URIs) > 1 {
+			grpclog.Warning("invalid SPIFFE ID: multiple URI SANs")
+			return nil
+		}
+		spiffeID = uri
+	}
+	return spiffeID
+}
diff --git a/internal/credentials/go110_test.go b/internal/credentials/go110_test.go
new file mode 100644
index 000000000000..19266cced492
--- /dev/null
+++ b/internal/credentials/go110_test.go
@@ -0,0 +1,182 @@
+// +build go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net/url"
+	"testing"
+
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestSPIFFEIDFromState(t *testing.T) {
+	tests := []struct {
+		name string
+		urls []*url.URL
+		// If we expect a SPIFFE ID to be returned.
+		expectID bool
+	}{
+		{
+			name:     "empty URIs",
+			urls:     []*url.URL{},
+			expectID: false,
+		},
+		{
+			name: "good SPIFFE ID",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+			},
+			expectID: true,
+		},
+		{
+			name: "invalid host",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    "",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+			},
+			expectID: false,
+		},
+		{
+			name: "invalid path",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    "foo.bar.com",
+					Path:    "",
+					RawPath: "",
+				},
+			},
+			expectID: false,
+		},
+		{
+			name: "large path",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    "foo.bar.com",
+					Path:    string(make([]byte, 2050)),
+					RawPath: string(make([]byte, 2050)),
+				},
+			},
+			expectID: false,
+		},
+		{
+			name: "large host",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    string(make([]byte, 256)),
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+			},
+			expectID: false,
+		},
+		{
+			name: "multiple URI SANs",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+				{
+					Scheme:  "spiffe",
+					Host:    "bar.baz.com",
+					Path:    "workload/wl2",
+					RawPath: "workload/wl2",
+				},
+				{
+					Scheme:  "https",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+			},
+			expectID: false,
+		},
+		{
+			name: "multiple URI SANs without SPIFFE ID",
+			urls: []*url.URL{
+				{
+					Scheme:  "https",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+				{
+					Scheme:  "ssh",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+			},
+			expectID: false,
+		},
+		{
+			name: "multiple URI SANs with one SPIFFE ID",
+			urls: []*url.URL{
+				{
+					Scheme:  "spiffe",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+				{
+					Scheme:  "https",
+					Host:    "foo.bar.com",
+					Path:    "workload/wl1",
+					RawPath: "workload/wl1",
+				},
+			},
+			expectID: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: tt.urls}}}
+			id := SPIFFEIDFromState(state)
+			if got, want := id != nil, tt.expectID; got != want {
+				t.Errorf("want expectID = %v, but SPIFFE ID is %v", want, id)
+			}
+		})
+	}
+}
diff --git a/internal/credentials/gobefore110.go b/internal/credentials/gobefore110.go
new file mode 100644
index 000000000000..743713e19f8d
--- /dev/null
+++ b/internal/credentials/gobefore110.go
@@ -0,0 +1,31 @@
+// +build !go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import (
+	"crypto/tls"
+	"net/url"
+)
+
+//TODO(ZhenLian): delete this file when we remove Go 1.9 tests.
+func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
+	return nil
+}

From 1154df9a4e0958aea682a1b9eef3f9a14138763f Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 16 Jul 2020 13:20:46 -0700
Subject: [PATCH 118/481] balancer: remove deprecated type aliases as planned
 (#3742)

---
 balancer/balancer.go                                  | 10 ----------
 balancer/base/base.go                                 | 11 -----------
 balancer/rls/internal/balancer.go                     |  6 +++---
 xds/internal/balancer/cdsbalancer/cdsbalancer_test.go |  1 -
 4 files changed, 3 insertions(+), 25 deletions(-)

diff --git a/balancer/balancer.go b/balancer/balancer.go
index e75b28436047..8bf359dbfda3 100644
--- a/balancer/balancer.go
+++ b/balancer/balancer.go
@@ -312,16 +312,6 @@ type Balancer interface {
 	Close()
 }
 
-// V2Balancer is temporarily defined for backward compatibility reasons.
-//
-// Deprecated: use Balancer directly instead.
-type V2Balancer = Balancer
-
-// V2Picker is temporarily defined for backward compatibility reasons.
-//
-// Deprecated: use Picker directly instead.
-type V2Picker = Picker
-
 // SubConnState describes the state of a SubConn.
 type SubConnState struct {
 	// ConnectivityState is the connectivity state of the SubConn.
diff --git a/balancer/base/base.go b/balancer/base/base.go
index c4fc89111bf1..e31d76e338a5 100644
--- a/balancer/base/base.go
+++ b/balancer/base/base.go
@@ -69,14 +69,3 @@ func NewBalancerBuilder(name string, pb PickerBuilder, config Config) balancer.B
 		config:        config,
 	}
 }
-
-// NewBalancerBuilderV2 is temporarily defined for backward compatibility
-// reasons.
-//
-// Deprecated: use NewBalancerBuilder instead.
-var NewBalancerBuilderV2 = NewBalancerBuilder
-
-// V2PickerBuilder is temporarily defined for backward compatibility reasons.
-//
-// Deprecated: use PickerBuilder instead.
-type V2PickerBuilder = PickerBuilder
diff --git a/balancer/rls/internal/balancer.go b/balancer/rls/internal/balancer.go
index 968e8e9310da..aeb509aa894b 100644
--- a/balancer/rls/internal/balancer.go
+++ b/balancer/rls/internal/balancer.go
@@ -97,7 +97,7 @@ func (lb *rlsBalancer) handleClientConnUpdate(ccs *balancer.ClientConnState) {
 
 // UpdateClientConnState pushes the received ClientConnState update on the
 // update channel which will be processed asynchronously by the run goroutine.
-// Implements balancer.V2Balancer interface.
+// Implements balancer.Balancer interface.
 func (lb *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error {
 	select {
 	case lb.ccUpdateCh <- &ccs:
@@ -106,14 +106,14 @@ func (lb *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error
 	return nil
 }
 
-// ResolverErr implements balancer.V2Balancer interface.
+// ResolverErr implements balancer.Balancer interface.
 func (lb *rlsBalancer) ResolverError(error) {
 	// ResolverError is called by gRPC when the name resolver reports an error.
 	// TODO(easwars): How do we handle this?
 	logger.Fatal("rls: ResolverError is not yet unimplemented")
 }
 
-// UpdateSubConnState implements balancer.V2Balancer interface.
+// UpdateSubConnState implements balancer.Balancer interface.
 func (lb *rlsBalancer) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConnState) {
 	logger.Fatal("rls: UpdateSubConnState is not yet implemented")
 }
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 31e56c1449ee..358e6c3895ef 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -231,7 +231,6 @@ func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.Clie
 func setup() (*cdsBalancer, *testEDSBalancer, *testClientConn, func()) {
 	builder := cdsBB{}
 	tcc := newTestClientConn()
-	// cdsB := builder.Build(tcc, balancer.BuildOptions{}).(balancer.V2Balancer)
 	cdsB := builder.Build(tcc, balancer.BuildOptions{})
 
 	edsB := newTestEDSBalancer()

From 6e77a8b2f6a0c45a9fb9854734718482b827f6e5 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 16 Jul 2020 13:37:27 -0700
Subject: [PATCH 119/481] xds: delete experimental package and experimental
 scheme (#3729)

---
 xds/experimental/xds_experimental.go       | 29 ----------------------
 xds/internal/resolver/xds_resolver.go      | 15 -----------
 xds/internal/resolver/xds_resolver_test.go |  8 +++---
 3 files changed, 3 insertions(+), 49 deletions(-)
 delete mode 100644 xds/experimental/xds_experimental.go

diff --git a/xds/experimental/xds_experimental.go b/xds/experimental/xds_experimental.go
deleted file mode 100644
index b7ac3019e56c..000000000000
--- a/xds/experimental/xds_experimental.go
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Package experimental contains xds implementation. Users only need to import
-// this package to get all xds functionality.
-//
-// Deprecated: import package xds instead.
-package experimental
-
-// TODO: remove this package after one release.
-
-import (
-	_ "google.golang.org/grpc/xds" // Register the balancers and resolvers.
-)
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 690642094401..fb46a91603c2 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -221,18 +221,3 @@ func (r *xdsResolver) Close() {
 	r.cancelCtx()
 	r.logger.Infof("Shutdown")
 }
-
-// Keep scheme with "-experimental" temporarily. Remove after one release.
-const schemeExperimental = "xds-experimental"
-
-type xdsResolverExperimentalBuilder struct {
-	xdsResolverBuilder
-}
-
-func (*xdsResolverExperimentalBuilder) Scheme() string {
-	return schemeExperimental
-}
-
-func init() {
-	resolver.Register(&xdsResolverExperimentalBuilder{})
-}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index dcb5c9e8e944..025a8b2dbc1a 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -55,11 +55,9 @@ var (
 )
 
 func TestRegister(t *testing.T) {
-	for _, s := range []string{"xds", "xds-experimental"} {
-		b := resolver.Get(s)
-		if b == nil {
-			t.Errorf("scheme %v is not registered", s)
-		}
+	b := resolver.Get(xdsScheme)
+	if b == nil {
+		t.Errorf("scheme %v is not registered", xdsScheme)
 	}
 }
 

From 08e6bb1878677157ba4917754f3cb27c6ba93ce3 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 16 Jul 2020 15:42:53 -0700
Subject: [PATCH 120/481] examples: change xds example to import xds instead of
 experimental (#3745)

---
 examples/features/xds/client/main.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go
index ea50f09450a8..9a67aabc3764 100644
--- a/examples/features/xds/client/main.go
+++ b/examples/features/xds/client/main.go
@@ -31,8 +31,7 @@ import (
 	"google.golang.org/grpc"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 
-	// TODO: change this to xds after the xds package is released.
-	_ "google.golang.org/grpc/xds/experimental" // To install the xds resolvers and balancers.
+	_ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers.
 )
 
 const (

From e2f575e56f2eafd0400d1385e15b89ddc4564fbf Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 16 Jul 2020 16:05:47 -0700
Subject: [PATCH 121/481] xdsrouting: all matchers (#3733)

---
 xds/internal/balancer/xdsrouting/doc.go       |  20 ++
 xds/internal/balancer/xdsrouting/matcher.go   | 134 +++++++
 .../balancer/xdsrouting/matcher_header.go     | 245 +++++++++++++
 .../xdsrouting/matcher_header_test.go         | 333 ++++++++++++++++++
 .../balancer/xdsrouting/matcher_path.go       | 102 ++++++
 .../balancer/xdsrouting/matcher_path_test.go  |  84 +++++
 .../balancer/xdsrouting/matcher_test.go       | 148 ++++++++
 7 files changed, 1066 insertions(+)
 create mode 100644 xds/internal/balancer/xdsrouting/doc.go
 create mode 100644 xds/internal/balancer/xdsrouting/matcher.go
 create mode 100644 xds/internal/balancer/xdsrouting/matcher_header.go
 create mode 100644 xds/internal/balancer/xdsrouting/matcher_header_test.go
 create mode 100644 xds/internal/balancer/xdsrouting/matcher_path.go
 create mode 100644 xds/internal/balancer/xdsrouting/matcher_path_test.go
 create mode 100644 xds/internal/balancer/xdsrouting/matcher_test.go

diff --git a/xds/internal/balancer/xdsrouting/doc.go b/xds/internal/balancer/xdsrouting/doc.go
new file mode 100644
index 000000000000..2f00649a0eb0
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/doc.go
@@ -0,0 +1,20 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xdsrouting implements the routing balancer for xds.
+package xdsrouting
diff --git a/xds/internal/balancer/xdsrouting/matcher.go b/xds/internal/balancer/xdsrouting/matcher.go
new file mode 100644
index 000000000000..79fab828a62f
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/matcher.go
@@ -0,0 +1,134 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/metadata"
+)
+
+// compositeMatcher.match returns true if all matchers return true.
+type compositeMatcher struct {
+	pm  pathMatcherInterface
+	hms []headerMatcherInterface
+	fm  *fractionMatcher
+}
+
+func newCompositeMatcher(pm pathMatcherInterface, hms []headerMatcherInterface, fm *fractionMatcher) *compositeMatcher {
+	return &compositeMatcher{pm: pm, hms: hms, fm: fm}
+}
+
+func (a *compositeMatcher) match(info balancer.PickInfo) bool {
+	if a.pm != nil && !a.pm.match(info.FullMethodName) {
+		return false
+	}
+
+	// Call headerMatchers even if md is nil, because routes may match
+	// non-presence of some headers.
+	var md metadata.MD
+	if info.Ctx != nil {
+		md, _ = metadata.FromOutgoingContext(info.Ctx)
+	}
+	for _, m := range a.hms {
+		if !m.match(md) {
+			return false
+		}
+	}
+
+	if a.fm != nil && !a.fm.match() {
+		return false
+	}
+	return true
+}
+
+func (a *compositeMatcher) equal(mm *compositeMatcher) bool {
+	if a == mm {
+		return true
+	}
+
+	if a == nil || mm == nil {
+		return false
+	}
+
+	if (a.pm != nil || mm.pm != nil) && (a.pm == nil || !a.pm.equal(mm.pm)) {
+		return false
+	}
+
+	if len(a.hms) != len(mm.hms) {
+		return false
+	}
+	for i := range a.hms {
+		if !a.hms[i].equal(mm.hms[i]) {
+			return false
+		}
+	}
+
+	if (a.fm != nil || mm.fm != nil) && (a.fm == nil || !a.fm.equal(mm.fm)) {
+		return false
+	}
+
+	return true
+}
+
+func (a *compositeMatcher) String() string {
+	var ret string
+	if a.pm != nil {
+		ret += a.pm.String()
+	}
+	for _, m := range a.hms {
+		ret += m.String()
+	}
+	if a.fm != nil {
+		ret += a.fm.String()
+	}
+	return ret
+}
+
+type fractionMatcher struct {
+	fraction int64 // real fraction is fraction/1,000,000.
+}
+
+func newFractionMatcher(fraction uint32) *fractionMatcher {
+	return &fractionMatcher{fraction: int64(fraction)}
+}
+
+var grpcrandInt63n = grpcrand.Int63n
+
+func (fm *fractionMatcher) match() bool {
+	t := grpcrandInt63n(1000000)
+	return t <= fm.fraction
+}
+
+func (fm *fractionMatcher) equal(m *fractionMatcher) bool {
+	if fm == m {
+		return true
+	}
+	if fm == nil || m == nil {
+		return false
+	}
+
+	return fm.fraction == m.fraction
+}
+
+func (fm *fractionMatcher) String() string {
+	return fmt.Sprintf("fraction:%v", fm.fraction)
+}
diff --git a/xds/internal/balancer/xdsrouting/matcher_header.go b/xds/internal/balancer/xdsrouting/matcher_header.go
new file mode 100644
index 000000000000..a900e43f5492
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/matcher_header.go
@@ -0,0 +1,245 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"google.golang.org/grpc/metadata"
+)
+
+type headerMatcherInterface interface {
+	match(metadata.MD) bool
+	equal(headerMatcherInterface) bool
+	String() string
+}
+
+// mdValuesFromOutgoingCtx retrieves metadata from context. If there are
+// multiple values, the values are concatenated with "," (comma and no space).
+//
+// All header matchers only match against the comma-concatenated string.
+func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) {
+	vs, ok := md[key]
+	if !ok {
+		return "", false
+	}
+	return strings.Join(vs, ","), true
+}
+
+type headerExactMatcher struct {
+	key   string
+	exact string
+}
+
+func newHeaderExactMatcher(key, exact string) *headerExactMatcher {
+	return &headerExactMatcher{key: key, exact: exact}
+}
+
+func (hem *headerExactMatcher) match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hem.key)
+	if !ok {
+		return false
+	}
+	return v == hem.exact
+}
+
+func (hem *headerExactMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*headerExactMatcher)
+	if !ok {
+		return false
+	}
+	return hem.key == mm.key && hem.exact == mm.exact
+}
+
+func (hem *headerExactMatcher) String() string {
+	return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact)
+}
+
+type headerRegexMatcher struct {
+	key string
+	re  *regexp.Regexp
+}
+
+func newHeaderRegexMatcher(key string, re *regexp.Regexp) *headerRegexMatcher {
+	return &headerRegexMatcher{key: key, re: re}
+}
+
+func (hrm *headerRegexMatcher) match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
+	if !ok {
+		return false
+	}
+	return hrm.re.MatchString(v)
+}
+
+func (hrm *headerRegexMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*headerRegexMatcher)
+	if !ok {
+		return false
+	}
+	return hrm.key == mm.key && hrm.re.String() == mm.re.String()
+}
+
+func (hrm *headerRegexMatcher) String() string {
+	return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String())
+}
+
+type headerRangeMatcher struct {
+	key        string
+	start, end int64 // represents [start, end).
+}
+
+func newHeaderRangeMatcher(key string, start, end int64) *headerRangeMatcher {
+	return &headerRangeMatcher{key: key, start: start, end: end}
+}
+
+func (hrm *headerRangeMatcher) match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
+	if !ok {
+		return false
+	}
+	if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end {
+		return true
+	}
+	return false
+}
+
+func (hrm *headerRangeMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*headerRangeMatcher)
+	if !ok {
+		return false
+	}
+	return hrm.key == mm.key && hrm.start == mm.start && hrm.end == mm.end
+}
+
+func (hrm *headerRangeMatcher) String() string {
+	return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end)
+}
+
+type headerPresentMatcher struct {
+	key     string
+	present bool
+}
+
+func newHeaderPresentMatcher(key string, present bool) *headerPresentMatcher {
+	return &headerPresentMatcher{key: key, present: present}
+}
+
+func (hpm *headerPresentMatcher) match(md metadata.MD) bool {
+	vs, ok := mdValuesFromOutgoingCtx(md, hpm.key)
+	present := ok && len(vs) > 0
+	return present == hpm.present
+}
+
+func (hpm *headerPresentMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*headerPresentMatcher)
+	if !ok {
+		return false
+	}
+	return hpm.key == mm.key && hpm.present == mm.present
+}
+
+func (hpm *headerPresentMatcher) String() string {
+	return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present)
+}
+
+type headerPrefixMatcher struct {
+	key    string
+	prefix string
+}
+
+func newHeaderPrefixMatcher(key string, prefix string) *headerPrefixMatcher {
+	return &headerPrefixMatcher{key: key, prefix: prefix}
+}
+
+func (hpm *headerPrefixMatcher) match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hpm.key)
+	if !ok {
+		return false
+	}
+	return strings.HasPrefix(v, hpm.prefix)
+}
+
+func (hpm *headerPrefixMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*headerPrefixMatcher)
+	if !ok {
+		return false
+	}
+	return hpm.key == mm.key && hpm.prefix == mm.prefix
+}
+
+func (hpm *headerPrefixMatcher) String() string {
+	return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix)
+}
+
+type headerSuffixMatcher struct {
+	key    string
+	suffix string
+}
+
+func newHeaderSuffixMatcher(key string, suffix string) *headerSuffixMatcher {
+	return &headerSuffixMatcher{key: key, suffix: suffix}
+}
+
+func (hsm *headerSuffixMatcher) match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hsm.key)
+	if !ok {
+		return false
+	}
+	return strings.HasSuffix(v, hsm.suffix)
+}
+
+func (hsm *headerSuffixMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*headerSuffixMatcher)
+	if !ok {
+		return false
+	}
+	return hsm.key == mm.key && hsm.suffix == mm.suffix
+}
+
+func (hsm *headerSuffixMatcher) String() string {
+	return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix)
+}
+
+type invertMatcher struct {
+	m headerMatcherInterface
+}
+
+func newInvertMatcher(m headerMatcherInterface) *invertMatcher {
+	return &invertMatcher{m: m}
+}
+
+func (i *invertMatcher) match(md metadata.MD) bool {
+	return !i.m.match(md)
+}
+
+func (i *invertMatcher) equal(m headerMatcherInterface) bool {
+	mm, ok := m.(*invertMatcher)
+	if !ok {
+		return false
+	}
+	return i.m.equal(mm.m)
+}
+
+func (i *invertMatcher) String() string {
+	return fmt.Sprintf("invert{%s}", i.m)
+}
diff --git a/xds/internal/balancer/xdsrouting/matcher_header_test.go b/xds/internal/balancer/xdsrouting/matcher_header_test.go
new file mode 100644
index 000000000000..3ec73ee90f78
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/matcher_header_test.go
@@ -0,0 +1,333 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"regexp"
+	"testing"
+
+	"google.golang.org/grpc/metadata"
+)
+
+func TestHeaderExactMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name       string
+		key, exact string
+		md         metadata.MD
+		want       bool
+	}{
+		{
+			name:  "one value one match",
+			key:   "th",
+			exact: "tv",
+			md:    metadata.Pairs("th", "tv"),
+			want:  true,
+		},
+		{
+			name:  "two value one match",
+			key:   "th",
+			exact: "tv",
+			md:    metadata.Pairs("th", "abc", "th", "tv"),
+			// Doesn't match comma-concatenated string.
+			want: false,
+		},
+		{
+			name:  "two value match concatenated",
+			key:   "th",
+			exact: "abc,tv",
+			md:    metadata.Pairs("th", "abc", "th", "tv"),
+			want:  true,
+		},
+		{
+			name:  "not match",
+			key:   "th",
+			exact: "tv",
+			md:    metadata.Pairs("th", "abc"),
+			want:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hem := newHeaderExactMatcher(tt.key, tt.exact)
+			if got := hem.match(tt.md); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHeaderRegexMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name          string
+		key, regexStr string
+		md            metadata.MD
+		want          bool
+	}{
+		{
+			name:     "one value one match",
+			key:      "th",
+			regexStr: "^t+v*$",
+			md:       metadata.Pairs("th", "tttvv"),
+			want:     true,
+		},
+		{
+			name:     "two value one match",
+			key:      "th",
+			regexStr: "^t+v*$",
+			md:       metadata.Pairs("th", "abc", "th", "tttvv"),
+			want:     false,
+		},
+		{
+			name:     "two value match concatenated",
+			key:      "th",
+			regexStr: "^[abc]*,t+v*$",
+			md:       metadata.Pairs("th", "abc", "th", "tttvv"),
+			want:     true,
+		},
+		{
+			name:     "no match",
+			key:      "th",
+			regexStr: "^t+v*$",
+			md:       metadata.Pairs("th", "abc"),
+			want:     false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hrm := newHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr))
+			if got := hrm.match(tt.md); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHeaderRangeMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name       string
+		key        string
+		start, end int64
+		md         metadata.MD
+		want       bool
+	}{
+		{
+			name:  "match",
+			key:   "th",
+			start: 1, end: 10,
+			md:   metadata.Pairs("th", "5"),
+			want: true,
+		},
+		{
+			name:  "equal to start",
+			key:   "th",
+			start: 1, end: 10,
+			md:   metadata.Pairs("th", "1"),
+			want: true,
+		},
+		{
+			name:  "equal to end",
+			key:   "th",
+			start: 1, end: 10,
+			md:   metadata.Pairs("th", "10"),
+			want: false,
+		},
+		{
+			name:  "negative",
+			key:   "th",
+			start: -10, end: 10,
+			md:   metadata.Pairs("th", "-5"),
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hrm := newHeaderRangeMatcher(tt.key, tt.start, tt.end)
+			if got := hrm.match(tt.md); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHeaderPresentMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name    string
+		key     string
+		present bool
+		md      metadata.MD
+		want    bool
+	}{
+		{
+			name:    "want present is present",
+			key:     "th",
+			present: true,
+			md:      metadata.Pairs("th", "tv"),
+			want:    true,
+		},
+		{
+			name:    "want present not present",
+			key:     "th",
+			present: true,
+			md:      metadata.Pairs("abc", "tv"),
+			want:    false,
+		},
+		{
+			name:    "want not present is present",
+			key:     "th",
+			present: false,
+			md:      metadata.Pairs("th", "tv"),
+			want:    false,
+		},
+		{
+			name:    "want not present is not present",
+			key:     "th",
+			present: false,
+			md:      metadata.Pairs("abc", "tv"),
+			want:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hpm := newHeaderPresentMatcher(tt.key, tt.present)
+			if got := hpm.match(tt.md); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHeaderPrefixMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name        string
+		key, prefix string
+		md          metadata.MD
+		want        bool
+	}{
+		{
+			name:   "one value one match",
+			key:    "th",
+			prefix: "tv",
+			md:     metadata.Pairs("th", "tv123"),
+			want:   true,
+		},
+		{
+			name:   "two value one match",
+			key:    "th",
+			prefix: "tv",
+			md:     metadata.Pairs("th", "abc", "th", "tv123"),
+			want:   false,
+		},
+		{
+			name:   "two value match concatenated",
+			key:    "th",
+			prefix: "tv",
+			md:     metadata.Pairs("th", "tv123", "th", "abc"),
+			want:   true,
+		},
+		{
+			name:   "not match",
+			key:    "th",
+			prefix: "tv",
+			md:     metadata.Pairs("th", "abc"),
+			want:   false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hpm := newHeaderPrefixMatcher(tt.key, tt.prefix)
+			if got := hpm.match(tt.md); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHeaderSuffixMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name        string
+		key, suffix string
+		md          metadata.MD
+		want        bool
+	}{
+		{
+			name:   "one value one match",
+			key:    "th",
+			suffix: "tv",
+			md:     metadata.Pairs("th", "123tv"),
+			want:   true,
+		},
+		{
+			name:   "two value one match",
+			key:    "th",
+			suffix: "tv",
+			md:     metadata.Pairs("th", "123tv", "th", "abc"),
+			want:   false,
+		},
+		{
+			name:   "two value match concatenated",
+			key:    "th",
+			suffix: "tv",
+			md:     metadata.Pairs("th", "abc", "th", "123tv"),
+			want:   true,
+		},
+		{
+			name:   "not match",
+			key:    "th",
+			suffix: "tv",
+			md:     metadata.Pairs("th", "abc"),
+			want:   false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hsm := newHeaderSuffixMatcher(tt.key, tt.suffix)
+			if got := hsm.match(tt.md); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestInvertMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name string
+		m    headerMatcherInterface
+		md   metadata.MD
+	}{
+		{
+			name: "true->false",
+			m:    newHeaderExactMatcher("th", "tv"),
+			md:   metadata.Pairs("th", "tv"),
+		},
+		{
+			name: "false->true",
+			m:    newHeaderExactMatcher("th", "abc"),
+			md:   metadata.Pairs("th", "tv"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := newInvertMatcher(tt.m).match(tt.md)
+			want := !tt.m.match(tt.md)
+			if got != want {
+				t.Errorf("match() = %v, want %v", got, want)
+			}
+		})
+	}
+}
diff --git a/xds/internal/balancer/xdsrouting/matcher_path.go b/xds/internal/balancer/xdsrouting/matcher_path.go
new file mode 100644
index 000000000000..9c783acbb577
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/matcher_path.go
@@ -0,0 +1,102 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"regexp"
+	"strings"
+)
+
+type pathMatcherInterface interface {
+	match(path string) bool
+	equal(pathMatcherInterface) bool
+	String() string
+}
+
+type pathExactMatcher struct {
+	fullPath string
+}
+
+func newPathExactMatcher(p string) *pathExactMatcher {
+	return &pathExactMatcher{fullPath: p}
+}
+
+func (pem *pathExactMatcher) match(path string) bool {
+	return pem.fullPath == path
+}
+
+func (pem *pathExactMatcher) equal(m pathMatcherInterface) bool {
+	mm, ok := m.(*pathExactMatcher)
+	if !ok {
+		return false
+	}
+	return pem.fullPath == mm.fullPath
+}
+
+func (pem *pathExactMatcher) String() string {
+	return "pathExact:" + pem.fullPath
+}
+
+type pathPrefixMatcher struct {
+	prefix string
+}
+
+func newPathPrefixMatcher(p string) *pathPrefixMatcher {
+	return &pathPrefixMatcher{prefix: p}
+}
+
+func (ppm *pathPrefixMatcher) match(path string) bool {
+	return strings.HasPrefix(path, ppm.prefix)
+}
+
+func (ppm *pathPrefixMatcher) equal(m pathMatcherInterface) bool {
+	mm, ok := m.(*pathPrefixMatcher)
+	if !ok {
+		return false
+	}
+	return ppm.prefix == mm.prefix
+}
+
+func (ppm *pathPrefixMatcher) String() string {
+	return "pathPrefix:" + ppm.prefix
+}
+
+type pathRegexMatcher struct {
+	re *regexp.Regexp
+}
+
+func newPathRegexMatcher(re *regexp.Regexp) *pathRegexMatcher {
+	return &pathRegexMatcher{re: re}
+}
+
+func (prm *pathRegexMatcher) match(path string) bool {
+	return prm.re.MatchString(path)
+}
+
+func (prm *pathRegexMatcher) equal(m pathMatcherInterface) bool {
+	mm, ok := m.(*pathRegexMatcher)
+	if !ok {
+		return false
+	}
+	return prm.re.String() == mm.re.String()
+}
+
+func (prm *pathRegexMatcher) String() string {
+	return "pathRegex:" + prm.re.String()
+}
diff --git a/xds/internal/balancer/xdsrouting/matcher_path_test.go b/xds/internal/balancer/xdsrouting/matcher_path_test.go
new file mode 100644
index 000000000000..ad7db7481060
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/matcher_path_test.go
@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"regexp"
+	"testing"
+)
+
+func TestPathFullMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name     string
+		fullPath string
+		path     string
+		want     bool
+	}{
+		{name: "match", fullPath: "/s/m", path: "/s/m", want: true},
+		{name: "not match", fullPath: "/s/m", path: "/a/b", want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fpm := newPathExactMatcher(tt.fullPath)
+			if got := fpm.match(tt.path); got != tt.want {
+				t.Errorf("{%q}.match(%q) = %v, want %v", tt.fullPath, tt.path, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPathPrefixMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name   string
+		prefix string
+		path   string
+		want   bool
+	}{
+		{name: "match", prefix: "/s/", path: "/s/m", want: true},
+		{name: "not match", prefix: "/s/", path: "/a/b", want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fpm := newPathPrefixMatcher(tt.prefix)
+			if got := fpm.match(tt.path); got != tt.want {
+				t.Errorf("{%q}.match(%q) = %v, want %v", tt.prefix, tt.path, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPathRegexMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name      string
+		regexPath string
+		path      string
+		want      bool
+	}{
+		{name: "match", regexPath: "^/s+/m.*$", path: "/sss/me", want: true},
+		{name: "not match", regexPath: "^/s+/m*$", path: "/sss/b", want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fpm := newPathRegexMatcher(regexp.MustCompile(tt.regexPath))
+			if got := fpm.match(tt.path); got != tt.want {
+				t.Errorf("{%q}.match(%q) = %v, want %v", tt.regexPath, tt.path, got, tt.want)
+			}
+		})
+	}
+}
diff --git a/xds/internal/balancer/xdsrouting/matcher_test.go b/xds/internal/balancer/xdsrouting/matcher_test.go
new file mode 100644
index 000000000000..8c8a4b52e5cd
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/matcher_test.go
@@ -0,0 +1,148 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"context"
+	"testing"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestAndMatcherMatch(t *testing.T) {
+	tests := []struct {
+		name string
+		pm   pathMatcherInterface
+		hm   headerMatcherInterface
+		info balancer.PickInfo
+		want bool
+	}{
+		{
+			name: "both match",
+			pm:   newPathExactMatcher("/a/b"),
+			hm:   newHeaderExactMatcher("th", "tv"),
+			info: balancer.PickInfo{
+				FullMethodName: "/a/b",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			},
+			want: true,
+		},
+		{
+			name: "only one match",
+			pm:   newPathExactMatcher("/a/b"),
+			hm:   newHeaderExactMatcher("th", "tv"),
+			info: balancer.PickInfo{
+				FullMethodName: "/z/y",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			},
+			want: false,
+		},
+		{
+			name: "both not match",
+			pm:   newPathExactMatcher("/z/y"),
+			hm:   newHeaderExactMatcher("th", "abc"),
+			info: balancer.PickInfo{
+				FullMethodName: "/a/b",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := newCompositeMatcher(tt.pm, []headerMatcherInterface{tt.hm}, nil)
+			if got := a.match(tt.info); got != tt.want {
+				t.Errorf("match() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFractionMatcherMatch(t *testing.T) {
+	const fraction = 500000
+	fm := newFractionMatcher(fraction)
+	defer func() {
+		grpcrandInt63n = grpcrand.Int63n
+	}()
+
+	// rand > fraction, should return false.
+	grpcrandInt63n = func(n int64) int64 {
+		return fraction + 1
+	}
+	if matched := fm.match(); matched {
+		t.Errorf("match() = %v, want not match", matched)
+	}
+
+	// rand == fraction, should return true.
+	grpcrandInt63n = func(n int64) int64 {
+		return fraction
+	}
+	if matched := fm.match(); !matched {
+		t.Errorf("match() = %v, want match", matched)
+	}
+
+	// rand < fraction, should return true.
+	grpcrandInt63n = func(n int64) int64 {
+		return fraction - 1
+	}
+	if matched := fm.match(); !matched {
+		t.Errorf("match() = %v, want match", matched)
+	}
+}
+
+func TestCompositeMatcherEqual(t *testing.T) {
+	tests := []struct {
+		name string
+		pm   pathMatcherInterface
+		hms  []headerMatcherInterface
+		fm   *fractionMatcher
+		mm   *compositeMatcher
+		want bool
+	}{
+		{
+			name: "equal",
+			pm:   newPathExactMatcher("/a/b"),
+			mm:   newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil),
+			want: true,
+		},
+		{
+			name: "no path matcher",
+			pm:   nil,
+			mm:   newCompositeMatcher(nil, nil, nil),
+			want: true,
+		},
+		{
+			name: "not equal",
+			pm:   newPathExactMatcher("/a/b"),
+			fm:   newFractionMatcher(123),
+			mm:   newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil),
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := newCompositeMatcher(tt.pm, tt.hms, tt.fm)
+			if got := a.equal(tt.mm); got != tt.want {
+				t.Errorf("equal() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

From 6dc7938fe87560fdb44d6e8ba500758111ef5f37 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 16 Jul 2020 16:38:30 -0700
Subject: [PATCH 122/481] xdsrouting: balancer config parsing (#3734)

---
 .../balancer/xdsrouting/routing_config.go     | 205 ++++++++++
 .../xdsrouting/routing_config_test.go         | 369 ++++++++++++++++++
 xds/internal/client/client_watchers_rds.go    |  18 +
 3 files changed, 592 insertions(+)
 create mode 100644 xds/internal/balancer/xdsrouting/routing_config.go
 create mode 100644 xds/internal/balancer/xdsrouting/routing_config_test.go

diff --git a/xds/internal/balancer/xdsrouting/routing_config.go b/xds/internal/balancer/xdsrouting/routing_config.go
new file mode 100644
index 000000000000..6daea224a14d
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/routing_config.go
@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"encoding/json"
+	"fmt"
+
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+type action struct {
+	// ChildPolicy is the child policy and it's config.
+	ChildPolicy *internalserviceconfig.BalancerConfig
+}
+
+type int64Range struct {
+	start, end int64
+}
+
+type headerMatcher struct {
+	name string
+	// matchSpecifiers
+	invertMatch bool
+
+	// At most one of the following has non-default value.
+	exactMatch, regexMatch, prefixMatch, suffixMatch string
+	rangeMatch                                       *int64Range
+	presentMatch                                     bool
+}
+
+type route struct {
+	// Path, Prefix and Regex can have at most one set. This is guaranteed by
+	// config parsing.
+	path, prefix, regex string
+
+	headers  []headerMatcher
+	fraction *uint32
+
+	// Action is the name from the action list.
+	action string
+}
+
+// lbConfig is the balancer config for xds routing policy.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	routes  []route
+	actions map[string]action
+}
+
+// The following structs with `JSON` in name are temporary structs to unmarshal
+// json into. The fields will be read into lbConfig, to be used by the balancer.
+
+// routeJSON is temporary struct for json unmarshal.
+type routeJSON struct {
+	// Path, Prefix and Regex can have at most one non-nil.
+	Path, Prefix, Regex *string
+	// Zero or more header matchers.
+	Headers       []*xdsclient.HeaderMatcher
+	MatchFraction *wrapperspb.UInt32Value
+	// Action is the name from the action list.
+	Action string
+}
+
+// lbConfigJSON is temporary struct for json unmarshal.
+type lbConfigJSON struct {
+	Route  []routeJSON
+	Action map[string]action
+}
+
+func (jc lbConfigJSON) toLBConfig() *lbConfig {
+	var ret lbConfig
+	for _, r := range jc.Route {
+		var tempR route
+		switch {
+		case r.Path != nil:
+			tempR.path = *r.Path
+		case r.Prefix != nil:
+			tempR.prefix = *r.Prefix
+		case r.Regex != nil:
+			tempR.regex = *r.Regex
+		}
+		for _, h := range r.Headers {
+			if h.RangeMatch != nil {
+				fmt.Println("range not nil", *h.RangeMatch)
+			}
+			var tempHeader headerMatcher
+			switch {
+			case h.ExactMatch != nil:
+				tempHeader.exactMatch = *h.ExactMatch
+			case h.RegexMatch != nil:
+				tempHeader.regexMatch = *h.RegexMatch
+			case h.PrefixMatch != nil:
+				tempHeader.prefixMatch = *h.PrefixMatch
+			case h.SuffixMatch != nil:
+				tempHeader.suffixMatch = *h.SuffixMatch
+			case h.RangeMatch != nil:
+				tempHeader.rangeMatch = &int64Range{
+					start: h.RangeMatch.Start,
+					end:   h.RangeMatch.End,
+				}
+			case h.PresentMatch != nil:
+				tempHeader.presentMatch = *h.PresentMatch
+			}
+			tempHeader.name = h.Name
+			if h.InvertMatch != nil {
+				tempHeader.invertMatch = *h.InvertMatch
+			}
+			tempR.headers = append(tempR.headers, tempHeader)
+		}
+		if r.MatchFraction != nil {
+			tempR.fraction = &r.MatchFraction.Value
+		}
+		tempR.action = r.Action
+		ret.routes = append(ret.routes, tempR)
+	}
+	ret.actions = jc.Action
+	return &ret
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	var tempConfig lbConfigJSON
+	if err := json.Unmarshal(c, &tempConfig); err != nil {
+		return nil, err
+	}
+
+	// For each route:
+	// - at most one of path/prefix/regex.
+	// - action is in action list.
+
+	allRouteActions := make(map[string]bool)
+	for _, r := range tempConfig.Route {
+		var oneOfCount int
+		if r.Path != nil {
+			oneOfCount++
+		}
+		if r.Prefix != nil {
+			oneOfCount++
+		}
+		if r.Regex != nil {
+			oneOfCount++
+		}
+		if oneOfCount != 1 {
+			return nil, fmt.Errorf("%d (not exactly one) of path/prefix/regex is set in route %+v", oneOfCount, r)
+		}
+
+		for _, h := range r.Headers {
+			var oneOfCountH int
+			if h.ExactMatch != nil {
+				oneOfCountH++
+			}
+			if h.RegexMatch != nil {
+				oneOfCountH++
+			}
+			if h.PrefixMatch != nil {
+				oneOfCountH++
+			}
+			if h.SuffixMatch != nil {
+				oneOfCountH++
+			}
+			if h.RangeMatch != nil {
+				oneOfCountH++
+			}
+			if h.PresentMatch != nil {
+				oneOfCountH++
+			}
+			if oneOfCountH != 1 {
+				return nil, fmt.Errorf("%d (not exactly one) of header matcher specifier is set in route %+v", oneOfCountH, h)
+			}
+		}
+
+		if _, ok := tempConfig.Action[r.Action]; !ok {
+			return nil, fmt.Errorf("action %q from route %+v is not found in action list", r.Action, r)
+		}
+		allRouteActions[r.Action] = true
+	}
+
+	// Verify that actions are used by at least one route.
+	for n := range tempConfig.Action {
+		if _, ok := allRouteActions[n]; !ok {
+			return nil, fmt.Errorf("action %q is not used by any route", n)
+		}
+	}
+
+	return tempConfig.toLBConfig(), nil
+}
diff --git a/xds/internal/balancer/xdsrouting/routing_config_test.go b/xds/internal/balancer/xdsrouting/routing_config_test.go
new file mode 100644
index 000000000000..c1ff4611dac9
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/routing_config_test.go
@@ -0,0 +1,369 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"
+	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
+)
+
+const (
+	testJSONConfig = `{
+      "action":{
+        "cds:cluster_1":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_1"}
+          }]
+        },
+        "weighted:cluster_1_cluster_2_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1" : {
+                  "weight":75,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_2" : {
+                  "weight":25,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+                }
+              }
+            }
+          }]
+        },
+        "weighted:cluster_1_cluster_3_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1": {
+                  "weight":99,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_3": {
+                  "weight":1,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
+                }
+              }
+            }
+          }]
+        }
+      },
+
+      "route":[{
+        "path":"/service_1/method_1",
+        "action":"cds:cluster_1"
+      },
+      {
+        "path":"/service_1/method_2",
+        "action":"cds:cluster_1"
+      },
+      {
+        "prefix":"/service_2/method_1",
+        "action":"weighted:cluster_1_cluster_2_1"
+      },
+      {
+        "prefix":"/service_2",
+        "action":"weighted:cluster_1_cluster_2_1"
+      },
+      {
+        "regex":"^/service_2/method_3$",
+        "action":"weighted:cluster_1_cluster_3_1"
+      }]
+    }
+`
+
+	testJSONConfigWithAllMatchers = `{
+      "action":{
+        "cds:cluster_1":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_1"}
+          }]
+        },
+        "cds:cluster_2":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_2"}
+          }]
+        },
+        "cds:cluster_3":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_3"}
+          }]
+        }
+      },
+
+      "route":[{
+        "path":"/service_1/method_1",
+        "action":"cds:cluster_1"
+      },
+      {
+        "prefix":"/service_2/method_1",
+        "action":"cds:cluster_1"
+      },
+      {
+        "regex":"^/service_2/method_3$",
+        "action":"cds:cluster_1"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "exactMatch":"value-1", "invertMatch":true}],
+        "action":"cds:cluster_2"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "regexMatch":"^value-1$"}],
+        "action":"cds:cluster_2"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "rangeMatch":{"start":-1, "end":7}}],
+        "action":"cds:cluster_3"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "presentMatch":true}],
+        "action":"cds:cluster_3"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "prefixMatch":"value-1"}],
+        "action":"cds:cluster_2"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "suffixMatch":"value-1"}],
+        "action":"cds:cluster_2"
+      },
+      {
+        "prefix":"",
+        "matchFraction":{"value": 31415},
+        "action":"cds:cluster_3"
+      }]
+    }
+`
+
+	cdsName = "cds_experimental"
+	wtName  = "weighted_target_experimental"
+)
+
+var (
+	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
+	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
+	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
+	cdsConfigJSON2  = `{"cluster":"cluster_2"}`
+	cdsConfig2, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON2))
+	cdsConfigJSON3  = `{"cluster":"cluster_3"}`
+	cdsConfig3, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON3))
+
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON1  = `{
+	"targets": {
+	  "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+	  "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] }
+	} }`
+	wtConfig1, _  = wtConfigParser.ParseConfig([]byte(wtConfigJSON1))
+	wtConfigJSON2 = `{
+    "targets": {
+      "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+      "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] }
+    } }`
+	wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2))
+)
+
+func Test_parseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "more than one path matcher", // Path matcher is oneof, so this is an error.
+			js: `{
+              "Action":{
+                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
+              },
+              "Route": [{
+                "path":"/service_1/method_1",
+                "prefix":"/service_1/",
+                "action":"cds:cluster_1"
+              }]
+            }`,
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "no path matcher",
+			js: `{
+              "Action":{
+                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
+              },
+              "Route": [{
+                "action":"cds:cluster_1"
+              }]
+            }`,
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "route action not found in action list",
+			js: `{
+              "Action":{},
+              "Route": [{
+                "path":"/service_1/method_1",
+                "action":"cds:cluster_1"
+              }]
+            }`,
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "action list contains action not used",
+			js: `{
+              "Action":{
+                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]},
+                "cds:cluster_not_used":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
+              },
+              "Route": [{
+                "path":"/service_1/method_1",
+                "action":"cds:cluster_1"
+              }]
+            }`,
+			want:    nil,
+			wantErr: true,
+		},
+
+		{
+			name: "no header specifier in header matcher",
+			js: `{
+              "Action":{
+                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
+              },
+              "Route": [{
+                "path":"/service_1/method_1",
+                "headers":[{"name":"header-1"}],
+                "action":"cds:cluster_1"
+              }]
+            }`,
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "more than one header specifier in header matcher",
+			js: `{
+              "Action":{
+                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
+              },
+              "Route": [{
+                "path":"/service_1/method_1",
+                "headers":[{"name":"header-1", "prefixMatch":"a", "suffixMatch":"b"}],
+                "action":"cds:cluster_1"
+              }]
+            }`,
+			want:    nil,
+			wantErr: true,
+		},
+
+		{
+			name: "OK with path matchers only",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				routes: []route{
+					{path: "/service_1/method_1", action: "cds:cluster_1"},
+					{path: "/service_1/method_2", action: "cds:cluster_1"},
+					{prefix: "/service_2/method_1", action: "weighted:cluster_1_cluster_2_1"},
+					{prefix: "/service_2", action: "weighted:cluster_1_cluster_2_1"},
+					{regex: "^/service_2/method_3$", action: "weighted:cluster_1_cluster_3_1"},
+				},
+				actions: map[string]action{
+					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig1},
+					},
+					"weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig1},
+					},
+					"weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig2},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "OK with all matchers",
+			js:   testJSONConfigWithAllMatchers,
+			want: &lbConfig{
+				routes: []route{
+					{path: "/service_1/method_1", action: "cds:cluster_1"},
+					{prefix: "/service_2/method_1", action: "cds:cluster_1"},
+					{regex: "^/service_2/method_3$", action: "cds:cluster_1"},
+
+					{prefix: "", headers: []headerMatcher{{name: "header-1", exactMatch: "value-1", invertMatch: true}}, action: "cds:cluster_2"},
+					{prefix: "", headers: []headerMatcher{{name: "header-1", regexMatch: "^value-1$"}}, action: "cds:cluster_2"},
+					{prefix: "", headers: []headerMatcher{{name: "header-1", rangeMatch: &int64Range{start: -1, end: 7}}}, action: "cds:cluster_3"},
+					{prefix: "", headers: []headerMatcher{{name: "header-1", presentMatch: true}}, action: "cds:cluster_3"},
+					{prefix: "", headers: []headerMatcher{{name: "header-1", prefixMatch: "value-1"}}, action: "cds:cluster_2"},
+					{prefix: "", headers: []headerMatcher{{name: "header-1", suffixMatch: "value-1"}}, action: "cds:cluster_2"},
+					{prefix: "", fraction: newUInt32P(31415), action: "cds:cluster_3"},
+				},
+				actions: map[string]action{
+					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig1},
+					},
+					"cds:cluster_2": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig2},
+					},
+					"cds:cluster_3": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig3},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	cmpOptions := []cmp.Option{cmp.AllowUnexported(lbConfig{}, route{}, headerMatcher{}, int64Range{})}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.want, cmpOptions...) {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want, cmpOptions...))
+			}
+		})
+	}
+}
+
+func newUInt32P(i uint32) *uint32 {
+	return &i
+}
diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
index 06f527921981..4c8f5d42affd 100644
--- a/xds/internal/client/client_watchers_rds.go
+++ b/xds/internal/client/client_watchers_rds.go
@@ -22,6 +22,24 @@ import (
 	"time"
 )
 
+// Int64Range is a range for header range match.
+type Int64Range struct {
+	Start int64 `json:"start"`
+	End   int64 `json:"end"`
+}
+
+// HeaderMatcher represents header matchers.
+type HeaderMatcher struct {
+	Name         string      `json:"name"`
+	InvertMatch  *bool       `json:"invertMatch,omitempty"`
+	ExactMatch   *string     `json:"exactMatch,omitempty"`
+	RegexMatch   *string     `json:"regexMatch,omitempty"`
+	PrefixMatch  *string     `json:"prefixMatch,omitempty"`
+	SuffixMatch  *string     `json:"suffixMatch,omitempty"`
+	RangeMatch   *Int64Range `json:"rangeMatch,omitempty"`
+	PresentMatch *bool       `json:"presentMatch,omitempty"`
+}
+
 type rdsUpdate struct {
 	weightedCluster map[string]uint32
 }

From 266c7b6f8236ec13705a4d5639dddd02513b1cec Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 20 Jul 2020 13:40:03 -0700
Subject: [PATCH 123/481] xdsrouting: add fake headers (#3748)

---
 internal/grpcutil/encode_duration.go          | 63 +++++++++++++++
 internal/grpcutil/encode_duration_test.go     | 51 ++++++++++++
 internal/grpcutil/metadata.go                 | 40 ++++++++++
 internal/grpcutil/method.go                   | 44 ++++++++++
 internal/grpcutil/method_test.go              | 23 ++++++
 internal/transport/handler_server.go          |  3 +-
 internal/transport/http2_client.go            |  5 +-
 internal/transport/http2_server.go            |  5 +-
 internal/transport/http_util.go               | 80 +------------------
 internal/transport/http_util_test.go          | 50 ------------
 stream.go                                     | 12 ++-
 test/balancer_test.go                         | 58 ++++++++++++++
 xds/internal/balancer/xdsrouting/matcher.go   | 12 +++
 .../balancer/xdsrouting/matcher_test.go       | 27 +++++++
 .../balancer/xdsrouting/routing_config.go     |  3 -
 15 files changed, 340 insertions(+), 136 deletions(-)
 create mode 100644 internal/grpcutil/encode_duration.go
 create mode 100644 internal/grpcutil/encode_duration_test.go
 create mode 100644 internal/grpcutil/metadata.go

diff --git a/internal/grpcutil/encode_duration.go b/internal/grpcutil/encode_duration.go
new file mode 100644
index 000000000000..b25b0baec3cc
--- /dev/null
+++ b/internal/grpcutil/encode_duration.go
@@ -0,0 +1,63 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpcutil
+
+import (
+	"strconv"
+	"time"
+)
+
+const maxTimeoutValue int64 = 100000000 - 1
+
+// div does integer division and round-up the result. Note that this is
+// equivalent to (d+r-1)/r but has less chance to overflow.
+func div(d, r time.Duration) int64 {
+	if d%r > 0 {
+		return int64(d/r + 1)
+	}
+	return int64(d / r)
+}
+
+// EncodeDuration encodes the duration to the format grpc-timeout header
+// accepts.
+//
+// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests
+func EncodeDuration(t time.Duration) string {
+	// TODO: This is simplistic and not bandwidth efficient. Improve it.
+	if t <= 0 {
+		return "0n"
+	}
+	if d := div(t, time.Nanosecond); d <= maxTimeoutValue {
+		return strconv.FormatInt(d, 10) + "n"
+	}
+	if d := div(t, time.Microsecond); d <= maxTimeoutValue {
+		return strconv.FormatInt(d, 10) + "u"
+	}
+	if d := div(t, time.Millisecond); d <= maxTimeoutValue {
+		return strconv.FormatInt(d, 10) + "m"
+	}
+	if d := div(t, time.Second); d <= maxTimeoutValue {
+		return strconv.FormatInt(d, 10) + "S"
+	}
+	if d := div(t, time.Minute); d <= maxTimeoutValue {
+		return strconv.FormatInt(d, 10) + "M"
+	}
+	// Note that maxTimeoutValue * time.Hour > MaxInt64.
+	return strconv.FormatInt(div(t, time.Hour), 10) + "H"
+}
diff --git a/internal/grpcutil/encode_duration_test.go b/internal/grpcutil/encode_duration_test.go
new file mode 100644
index 000000000000..eea49e2e77f5
--- /dev/null
+++ b/internal/grpcutil/encode_duration_test.go
@@ -0,0 +1,51 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpcutil
+
+import (
+	"testing"
+	"time"
+)
+
+func TestEncodeDuration(t *testing.T) {
+	for _, test := range []struct {
+		in  string
+		out string
+	}{
+		{"12345678ns", "12345678n"},
+		{"123456789ns", "123457u"},
+		{"12345678us", "12345678u"},
+		{"123456789us", "123457m"},
+		{"12345678ms", "12345678m"},
+		{"123456789ms", "123457S"},
+		{"12345678s", "12345678S"},
+		{"123456789s", "2057614M"},
+		{"12345678m", "12345678M"},
+		{"123456789m", "2057614H"},
+	} {
+		d, err := time.ParseDuration(test.in)
+		if err != nil {
+			t.Fatalf("failed to parse duration string %s: %v", test.in, err)
+		}
+		out := EncodeDuration(d)
+		if out != test.out {
+			t.Fatalf("timeoutEncode(%s) = %s, want %s", test.in, out, test.out)
+		}
+	}
+}
diff --git a/internal/grpcutil/metadata.go b/internal/grpcutil/metadata.go
new file mode 100644
index 000000000000..6f22bd891153
--- /dev/null
+++ b/internal/grpcutil/metadata.go
@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpcutil
+
+import (
+	"context"
+
+	"google.golang.org/grpc/metadata"
+)
+
+type mdExtraKey struct{}
+
+// WithExtraMetadata creates a new context with incoming md attached.
+func WithExtraMetadata(ctx context.Context, md metadata.MD) context.Context {
+	return context.WithValue(ctx, mdExtraKey{}, md)
+}
+
+// ExtraMetadata returns the incoming metadata in ctx if it exists.  The
+// returned MD should not be modified. Writing to it may cause races.
+// Modification should be made to copies of the returned MD.
+func ExtraMetadata(ctx context.Context) (md metadata.MD, ok bool) {
+	md, ok = ctx.Value(mdExtraKey{}).(metadata.MD)
+	return
+}
diff --git a/internal/grpcutil/method.go b/internal/grpcutil/method.go
index 2c2ff7732a8f..4e7475060c1c 100644
--- a/internal/grpcutil/method.go
+++ b/internal/grpcutil/method.go
@@ -38,3 +38,47 @@ func ParseMethod(methodName string) (service, method string, _ error) {
 	}
 	return methodName[:pos], methodName[pos+1:], nil
 }
+
+const baseContentType = "application/grpc"
+
+// ContentSubtype returns the content-subtype for the given content-type.  The
+// given content-type must be a valid content-type that starts with
+// "application/grpc". A content-subtype will follow "application/grpc" after a
+// "+" or ";". See
+// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
+// more details.
+//
+// If contentType is not a valid content-type for gRPC, the boolean
+// will be false, otherwise true. If content-type == "application/grpc",
+// "application/grpc+", or "application/grpc;", the boolean will be true,
+// but no content-subtype will be returned.
+//
+// contentType is assumed to be lowercase already.
+func ContentSubtype(contentType string) (string, bool) {
+	if contentType == baseContentType {
+		return "", true
+	}
+	if !strings.HasPrefix(contentType, baseContentType) {
+		return "", false
+	}
+	// guaranteed since != baseContentType and has baseContentType prefix
+	switch contentType[len(baseContentType)] {
+	case '+', ';':
+		// this will return true for "application/grpc+" or "application/grpc;"
+		// which the previous validContentType function tested to be valid, so we
+		// just say that no content-subtype is specified in this case
+		return contentType[len(baseContentType)+1:], true
+	default:
+		return "", false
+	}
+}
+
+// ContentType builds full content type with the given sub-type.
+//
+// contentSubtype is assumed to be lowercase
+func ContentType(contentSubtype string) string {
+	if contentSubtype == "" {
+		return baseContentType
+	}
+	return baseContentType + "+" + contentSubtype
+}
diff --git a/internal/grpcutil/method_test.go b/internal/grpcutil/method_test.go
index 9880e1c43293..36c786cffbd2 100644
--- a/internal/grpcutil/method_test.go
+++ b/internal/grpcutil/method_test.go
@@ -44,3 +44,26 @@ func TestParseMethod(t *testing.T) {
 		}
 	}
 }
+
+func TestContentSubtype(t *testing.T) {
+	tests := []struct {
+		contentType string
+		want        string
+		wantValid   bool
+	}{
+		{"application/grpc", "", true},
+		{"application/grpc+", "", true},
+		{"application/grpc+blah", "blah", true},
+		{"application/grpc;", "", true},
+		{"application/grpc;blah", "blah", true},
+		{"application/grpcd", "", false},
+		{"application/grpd", "", false},
+		{"application/grp", "", false},
+	}
+	for _, tt := range tests {
+		got, gotValid := ContentSubtype(tt.contentType)
+		if got != tt.want || gotValid != tt.wantValid {
+			t.Errorf("contentSubtype(%q) = (%v, %v); want (%v, %v)", tt.contentType, got, gotValid, tt.want, tt.wantValid)
+		}
+	}
+}
diff --git a/internal/transport/handler_server.go b/internal/transport/handler_server.go
index fc44e976195d..05d3871e628d 100644
--- a/internal/transport/handler_server.go
+++ b/internal/transport/handler_server.go
@@ -39,6 +39,7 @@ import (
 	"golang.org/x/net/http2"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/stats"
@@ -57,7 +58,7 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats sta
 	}
 	contentType := r.Header.Get("Content-Type")
 	// TODO: do we assume contentType is lowercase? we did before
-	contentSubtype, validContentType := contentSubtype(contentType)
+	contentSubtype, validContentType := grpcutil.ContentSubtype(contentType)
 	if !validContentType {
 		return nil, errors.New("invalid gRPC request content-type")
 	}
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 47af2ebeade8..e7f2321131e4 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -32,6 +32,7 @@ import (
 
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
+	"google.golang.org/grpc/internal/grpcutil"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
@@ -436,7 +437,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 	headerFields = append(headerFields, hpack.HeaderField{Name: ":scheme", Value: t.scheme})
 	headerFields = append(headerFields, hpack.HeaderField{Name: ":path", Value: callHdr.Method})
 	headerFields = append(headerFields, hpack.HeaderField{Name: ":authority", Value: callHdr.Host})
-	headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(callHdr.ContentSubtype)})
+	headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: grpcutil.ContentType(callHdr.ContentSubtype)})
 	headerFields = append(headerFields, hpack.HeaderField{Name: "user-agent", Value: t.userAgent})
 	headerFields = append(headerFields, hpack.HeaderField{Name: "te", Value: "trailers"})
 	if callHdr.PreviousAttempts > 0 {
@@ -451,7 +452,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 		// Send out timeout regardless its value. The server can detect timeout context by itself.
 		// TODO(mmukhi): Perhaps this field should be updated when actually writing out to the wire.
 		timeout := time.Until(dl)
-		headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-timeout", Value: encodeTimeout(timeout)})
+		headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-timeout", Value: grpcutil.EncodeDuration(timeout)})
 	}
 	for k, v := range authData {
 		headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index 788cf1e4df01..04cbedf7945f 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -34,6 +34,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
+	"google.golang.org/grpc/internal/grpcutil"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
@@ -804,7 +805,7 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error {
 	// first and create a slice of that exact size.
 	headerFields := make([]hpack.HeaderField, 0, 2) // at least :status, content-type will be there if none else.
 	headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"})
-	headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(s.contentSubtype)})
+	headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: grpcutil.ContentType(s.contentSubtype)})
 	if s.sendCompress != "" {
 		headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: s.sendCompress})
 	}
@@ -854,7 +855,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error {
 			}
 		} else { // Send a trailer only response.
 			headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"})
-			headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(s.contentSubtype)})
+			headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: grpcutil.ContentType(s.contentSubtype)})
 		}
 	}
 	headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))})
diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go
index e68cdcb6cc2d..5e1e7a65da2b 100644
--- a/internal/transport/http_util.go
+++ b/internal/transport/http_util.go
@@ -38,6 +38,7 @@ import (
 	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/status"
 )
 
@@ -51,7 +52,7 @@ const (
 	// "proto" as a suffix after "+" or ";".  See
 	// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests
 	// for more details.
-	baseContentType = "application/grpc"
+
 )
 
 var (
@@ -184,46 +185,6 @@ func isWhitelistedHeader(hdr string) bool {
 	}
 }
 
-// contentSubtype returns the content-subtype for the given content-type.  The
-// given content-type must be a valid content-type that starts with
-// "application/grpc". A content-subtype will follow "application/grpc" after a
-// "+" or ";". See
-// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
-// more details.
-//
-// If contentType is not a valid content-type for gRPC, the boolean
-// will be false, otherwise true. If content-type == "application/grpc",
-// "application/grpc+", or "application/grpc;", the boolean will be true,
-// but no content-subtype will be returned.
-//
-// contentType is assumed to be lowercase already.
-func contentSubtype(contentType string) (string, bool) {
-	if contentType == baseContentType {
-		return "", true
-	}
-	if !strings.HasPrefix(contentType, baseContentType) {
-		return "", false
-	}
-	// guaranteed since != baseContentType and has baseContentType prefix
-	switch contentType[len(baseContentType)] {
-	case '+', ';':
-		// this will return true for "application/grpc+" or "application/grpc;"
-		// which the previous validContentType function tested to be valid, so we
-		// just say that no content-subtype is specified in this case
-		return contentType[len(baseContentType)+1:], true
-	default:
-		return "", false
-	}
-}
-
-// contentSubtype is assumed to be lowercase
-func contentType(contentSubtype string) string {
-	if contentSubtype == "" {
-		return baseContentType
-	}
-	return baseContentType + "+" + contentSubtype
-}
-
 func (d *decodeState) status() *status.Status {
 	if d.data.statusGen == nil {
 		// No status-details were provided; generate status using code/msg.
@@ -342,7 +303,7 @@ func (d *decodeState) addMetadata(k, v string) {
 func (d *decodeState) processHeaderField(f hpack.HeaderField) {
 	switch f.Name {
 	case "content-type":
-		contentSubtype, validContentType := contentSubtype(f.Value)
+		contentSubtype, validContentType := grpcutil.ContentSubtype(f.Value)
 		if !validContentType {
 			d.data.contentTypeErr = fmt.Sprintf("transport: received the unexpected content-type %q", f.Value)
 			return
@@ -453,41 +414,6 @@ func timeoutUnitToDuration(u timeoutUnit) (d time.Duration, ok bool) {
 	return
 }
 
-const maxTimeoutValue int64 = 100000000 - 1
-
-// div does integer division and round-up the result. Note that this is
-// equivalent to (d+r-1)/r but has less chance to overflow.
-func div(d, r time.Duration) int64 {
-	if m := d % r; m > 0 {
-		return int64(d/r + 1)
-	}
-	return int64(d / r)
-}
-
-// TODO(zhaoq): It is the simplistic and not bandwidth efficient. Improve it.
-func encodeTimeout(t time.Duration) string {
-	if t <= 0 {
-		return "0n"
-	}
-	if d := div(t, time.Nanosecond); d <= maxTimeoutValue {
-		return strconv.FormatInt(d, 10) + "n"
-	}
-	if d := div(t, time.Microsecond); d <= maxTimeoutValue {
-		return strconv.FormatInt(d, 10) + "u"
-	}
-	if d := div(t, time.Millisecond); d <= maxTimeoutValue {
-		return strconv.FormatInt(d, 10) + "m"
-	}
-	if d := div(t, time.Second); d <= maxTimeoutValue {
-		return strconv.FormatInt(d, 10) + "S"
-	}
-	if d := div(t, time.Minute); d <= maxTimeoutValue {
-		return strconv.FormatInt(d, 10) + "M"
-	}
-	// Note that maxTimeoutValue * time.Hour > MaxInt64.
-	return strconv.FormatInt(div(t, time.Hour), 10) + "H"
-}
-
 func decodeTimeout(s string) (time.Duration, error) {
 	size := len(s)
 	if size < 2 {
diff --git a/internal/transport/http_util_test.go b/internal/transport/http_util_test.go
index dd13bcc2ff1e..80b1c094a071 100644
--- a/internal/transport/http_util_test.go
+++ b/internal/transport/http_util_test.go
@@ -25,33 +25,6 @@ import (
 	"time"
 )
 
-func (s) TestTimeoutEncode(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"12345678ns", "12345678n"},
-		{"123456789ns", "123457u"},
-		{"12345678us", "12345678u"},
-		{"123456789us", "123457m"},
-		{"12345678ms", "12345678m"},
-		{"123456789ms", "123457S"},
-		{"12345678s", "12345678S"},
-		{"123456789s", "2057614M"},
-		{"12345678m", "12345678M"},
-		{"123456789m", "2057614H"},
-	} {
-		d, err := time.ParseDuration(test.in)
-		if err != nil {
-			t.Fatalf("failed to parse duration string %s: %v", test.in, err)
-		}
-		out := encodeTimeout(d)
-		if out != test.out {
-			t.Fatalf("timeoutEncode(%s) = %s, want %s", test.in, out, test.out)
-		}
-	}
-}
-
 func (s) TestTimeoutDecode(t *testing.T) {
 	for _, test := range []struct {
 		// input
@@ -72,29 +45,6 @@ func (s) TestTimeoutDecode(t *testing.T) {
 	}
 }
 
-func (s) TestContentSubtype(t *testing.T) {
-	tests := []struct {
-		contentType string
-		want        string
-		wantValid   bool
-	}{
-		{"application/grpc", "", true},
-		{"application/grpc+", "", true},
-		{"application/grpc+blah", "blah", true},
-		{"application/grpc;", "", true},
-		{"application/grpc;blah", "blah", true},
-		{"application/grpcd", "", false},
-		{"application/grpd", "", false},
-		{"application/grp", "", false},
-	}
-	for _, tt := range tests {
-		got, gotValid := contentSubtype(tt.contentType)
-		if got != tt.want || gotValid != tt.wantValid {
-			t.Errorf("contentSubtype(%q) = (%v, %v); want (%v, %v)", tt.contentType, got, gotValid, tt.want, tt.wantValid)
-		}
-	}
-}
-
 func (s) TestEncodeGrpcMessage(t *testing.T) {
 	for _, tt := range []struct {
 		input    string
diff --git a/stream.go b/stream.go
index 34e4010d0dba..fbc3fb11cb4d 100644
--- a/stream.go
+++ b/stream.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/internal/binarylog"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
@@ -346,7 +347,16 @@ func (cs *clientStream) newAttemptLocked(sh stats.Handler, trInfo *traceInfo) (r
 	if err := cs.ctx.Err(); err != nil {
 		return toRPCErr(err)
 	}
-	t, done, err := cs.cc.getTransport(cs.ctx, cs.callInfo.failFast, cs.callHdr.Method)
+
+	ctx := cs.ctx
+	if cs.cc.parsedTarget.Scheme == "xds" {
+		// Add extra metadata (metadata that will be added by transport) to context
+		// so the balancer can see them.
+		ctx = grpcutil.WithExtraMetadata(cs.ctx, metadata.Pairs(
+			"content-type", grpcutil.ContentType(cs.callHdr.ContentSubtype),
+		))
+	}
+	t, done, err := cs.cc.getTransport(ctx, cs.callInfo.failFast, cs.callHdr.Method)
 	if err != nil {
 		return err
 	}
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 7fa96d8680ee..9f8bc1a6e251 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -39,6 +39,7 @@ import (
 	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/balancerload"
 	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
@@ -60,6 +61,7 @@ type testBalancer struct {
 
 	newSubConnOptions balancer.NewSubConnOptions
 	pickInfos         []balancer.PickInfo
+	pickExtraMDs      []metadata.MD
 	doneInfo          []balancer.DoneInfo
 }
 
@@ -124,8 +126,10 @@ func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 	if p.err != nil {
 		return balancer.PickResult{}, p.err
 	}
+	extraMD, _ := grpcutil.ExtraMetadata(info.Ctx)
 	info.Ctx = nil // Do not validate context.
 	p.bal.pickInfos = append(p.bal.pickInfos, info)
+	p.bal.pickExtraMDs = append(p.bal.pickExtraMDs, extraMD)
 	return balancer.PickResult{SubConn: p.sc, Done: func(d balancer.DoneInfo) { p.bal.doneInfo = append(p.bal.doneInfo, d) }}, nil
 }
 
@@ -157,6 +161,60 @@ func (s) TestCredsBundleFromBalancer(t *testing.T) {
 	}
 }
 
+func (s) TestPickExtraMetadata(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPickExtraMetadata(t, e)
+	}
+}
+
+func testPickExtraMetadata(t *testing.T, e env) {
+	te := newTest(t, e)
+	b := &testBalancer{}
+	balancer.Register(b)
+	const (
+		testUserAgent      = "test-user-agent"
+		testSubContentType = "proto"
+	)
+
+	te.customDialOptions = []grpc.DialOption{
+		grpc.WithBalancerName(testBalancerName),
+		grpc.WithUserAgent(testUserAgent),
+	}
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	// Set resolver to xds to trigger the extra metadata code path.
+	r := manual.NewBuilderWithScheme("xds")
+	resolver.Register(r)
+	defer func() {
+		resolver.UnregisterForTesting("xds")
+	}()
+	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
+	te.resolverScheme = "xds"
+	cc := te.clientConn()
+	tc := testpb.NewTestServiceClient(cc)
+
+	// The RPCs will fail, but we don't care. We just need the pick to happen.
+	ctx1, cancel1 := context.WithTimeout(context.Background(), time.Second)
+	defer cancel1()
+	tc.EmptyCall(ctx1, &testpb.Empty{})
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second)
+	defer cancel2()
+	tc.EmptyCall(ctx2, &testpb.Empty{}, grpc.CallContentSubtype(testSubContentType))
+
+	want := []metadata.MD{
+		// First RPC doesn't have sub-content-type.
+		{"content-type": []string{"application/grpc"}},
+		// Second RPC has sub-content-type "proto".
+		{"content-type": []string{"application/grpc+proto"}},
+	}
+
+	if !cmp.Equal(b.pickExtraMDs, want) {
+		t.Fatalf("%s", cmp.Diff(b.pickExtraMDs, want))
+	}
+}
+
 func (s) TestDoneInfo(t *testing.T) {
 	for _, e := range listTestEnv() {
 		testDoneInfo(t, e)
diff --git a/xds/internal/balancer/xdsrouting/matcher.go b/xds/internal/balancer/xdsrouting/matcher.go
index 79fab828a62f..196aefae564b 100644
--- a/xds/internal/balancer/xdsrouting/matcher.go
+++ b/xds/internal/balancer/xdsrouting/matcher.go
@@ -20,9 +20,11 @@ package xdsrouting
 
 import (
 	"fmt"
+	"strings"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -47,6 +49,16 @@ func (a *compositeMatcher) match(info balancer.PickInfo) bool {
 	var md metadata.MD
 	if info.Ctx != nil {
 		md, _ = metadata.FromOutgoingContext(info.Ctx)
+		if extraMD, ok := grpcutil.ExtraMetadata(info.Ctx); ok {
+			md = metadata.Join(md, extraMD)
+			// Remove all binary headers. They are hard to match with. May need
+			// to add back if asked by users.
+			for k := range md {
+				if strings.HasSuffix(k, "-bin") {
+					delete(md, k)
+				}
+			}
+		}
 	}
 	for _, m := range a.hms {
 		if !m.match(md) {
diff --git a/xds/internal/balancer/xdsrouting/matcher_test.go b/xds/internal/balancer/xdsrouting/matcher_test.go
index 8c8a4b52e5cd..e7d76e27469f 100644
--- a/xds/internal/balancer/xdsrouting/matcher_test.go
+++ b/xds/internal/balancer/xdsrouting/matcher_test.go
@@ -24,6 +24,7 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/grpcutil"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -65,6 +66,32 @@ func TestAndMatcherMatch(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "fake header",
+			pm:   newPathPrefixMatcher("/"),
+			hm:   newHeaderExactMatcher("content-type", "fake"),
+			info: balancer.PickInfo{
+				FullMethodName: "/a/b",
+				Ctx: grpcutil.WithExtraMetadata(context.Background(), metadata.Pairs(
+					"content-type", "fake",
+				)),
+			},
+			want: true,
+		},
+		{
+			name: "binary header",
+			pm:   newPathPrefixMatcher("/"),
+			hm:   newHeaderPresentMatcher("t-bin", true),
+			info: balancer.PickInfo{
+				FullMethodName: "/a/b",
+				Ctx: grpcutil.WithExtraMetadata(
+					metadata.NewOutgoingContext(context.Background(), metadata.Pairs("t-bin", "123")), metadata.Pairs(
+						"content-type", "fake",
+					)),
+			},
+			// Shouldn't match binary header, even though it's in metadata.
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/xds/internal/balancer/xdsrouting/routing_config.go b/xds/internal/balancer/xdsrouting/routing_config.go
index 6daea224a14d..c8cc7a18c6db 100644
--- a/xds/internal/balancer/xdsrouting/routing_config.go
+++ b/xds/internal/balancer/xdsrouting/routing_config.go
@@ -100,9 +100,6 @@ func (jc lbConfigJSON) toLBConfig() *lbConfig {
 			tempR.regex = *r.Regex
 		}
 		for _, h := range r.Headers {
-			if h.RangeMatch != nil {
-				fmt.Println("range not nil", *h.RangeMatch)
-			}
 			var tempHeader headerMatcher
 			switch {
 			case h.ExactMatch != nil:

From ca3959a1b21a47ae53b2d5df691fb61d2f40a36c Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 20 Jul 2020 14:04:46 -0700
Subject: [PATCH 124/481] xdsrouting: handle route fields in xds_client (#3747)

---
 xds/internal/client/client_watchers_rds.go    |  12 +
 .../client/client_watchers_service.go         |   7 +
 .../client/client_watchers_service_test.go    |  21 +-
 xds/internal/client/envconfig.go              |  33 +++
 xds/internal/client/v2client_rds.go           | 131 +++++++++-
 xds/internal/client/v2client_rds_test.go      | 225 +++++++++++++++++-
 6 files changed, 416 insertions(+), 13 deletions(-)
 create mode 100644 xds/internal/client/envconfig.go

diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
index 4c8f5d42affd..91de78601b49 100644
--- a/xds/internal/client/client_watchers_rds.go
+++ b/xds/internal/client/client_watchers_rds.go
@@ -40,8 +40,20 @@ type HeaderMatcher struct {
 	PresentMatch *bool       `json:"presentMatch,omitempty"`
 }
 
+// Route represents route with matchers and action.
+type Route struct {
+	Path, Prefix, Regex *string
+	Headers             []*HeaderMatcher
+	Fraction            *uint32
+	Action              map[string]uint32 // action is weighted clusters.
+}
+
 type rdsUpdate struct {
+	// weightedCluster is only set when routing is disabled (env variable
+	// GRPC_XDS_EXPERIMENTAL_ROUTING is not true).
 	weightedCluster map[string]uint32
+
+	routes []*Route
 }
 type rdsCallbackFunc func(rdsUpdate, error)
 
diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
index 1eea1f316a3c..1cf4c0f988c1 100644
--- a/xds/internal/client/client_watchers_service.go
+++ b/xds/internal/client/client_watchers_service.go
@@ -27,7 +27,13 @@ import (
 type ServiceUpdate struct {
 	// WeightedCluster is a map from cluster names (CDS resource to watch) to
 	// their weights.
+	//
+	// This field is only set when routing is disabled (env variable
+	// GRPC_XDS_EXPERIMENTAL_ROUTING is not true).
 	WeightedCluster map[string]uint32
+
+	// Routes
+	Routes []*Route
 }
 
 // WatchService uses LDS and RDS to discover information about the provided
@@ -121,6 +127,7 @@ func (w *serviceUpdateWatcher) handleRDSResp(update rdsUpdate, err error) {
 	}
 	w.serviceCb(ServiceUpdate{
 		WeightedCluster: update.weightedCluster,
+		Routes:          update.routes,
 	}, nil)
 }
 
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 37d7fb45401d..4535285bd6b4 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -40,8 +40,7 @@ var serviceCmpOpts = []cmp.Option{cmp.AllowUnexported(serviceUpdateErr{}), cmpop
 
 // TestServiceWatch covers the cases:
 // - an update is received after a watch()
-// - an update for another resource name (which doesn't trigger callback)
-// - an upate is received after cancel()
+// - an update with routes received
 func (s) TestServiceWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
 	defer cleanup()
@@ -77,6 +76,24 @@ func (s) TestServiceWatch(t *testing.T) {
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
+
+	wantUpdate2 := ServiceUpdate{
+		Routes: []*Route{{
+			Prefix: newStringP(""),
+			Action: map[string]uint32{testCDSName: 1},
+		}},
+	}
+	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+		testRDSName: {
+			routes: []*Route{{
+				Prefix: newStringP(""),
+				Action: map[string]uint32{testCDSName: 1},
+			}},
+		},
+	})
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
+		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	}
 }
 
 // TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS
diff --git a/xds/internal/client/envconfig.go b/xds/internal/client/envconfig.go
new file mode 100644
index 000000000000..40f448e63711
--- /dev/null
+++ b/xds/internal/client/envconfig.go
@@ -0,0 +1,33 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"os"
+	"strings"
+)
+
+// TODO: there are multiple env variables, GRPC_XDS_BOOTSTRAP and
+// GRPC_XDS_EXPERIMENTAL_V3_SUPPORT, and this. Move all env variables into a
+// separate package.
+const routingEnabledConfigStr = "GRPC_XDS_EXPERIMENTAL_ROUTING"
+
+// routing is enabled only if env variable is set to true. The default is false.
+// We may flip the default later.
+var routingEnabled = strings.EqualFold(os.Getenv(routingEnabledConfigStr), "true")
diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
index 3650c045c370..1fc9ac9752bf 100644
--- a/xds/internal/client/v2client_rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -24,7 +24,9 @@ import (
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	typepb "github.com/envoyproxy/go-control-plane/envoy/type"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/internal/grpclog"
 )
 
 // handleRDSResponse processes an RDS response received from the xDS server. On
@@ -48,7 +50,7 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, v2c.hostname)
 
 		// Use the hostname (resourceName for LDS) to find the routes.
-		u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname)
+		u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname, v2c.logger)
 		if err != nil {
 			return fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
 		}
@@ -76,7 +78,7 @@ func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
 // field must be empty and whose route field must be set.  Inside that route
 // message, the cluster field will contain the clusterName or weighted clusters
 // we are looking for.
-func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string) (rdsUpdate, error) {
+func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string, logger *grpclog.PrefixLogger) (rdsUpdate, error) {
 	//
 	// Currently this returns "" on error, and the caller will return an error.
 	// But the error doesn't contain details of why the response is invalid
@@ -94,6 +96,16 @@ func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host
 		// should be at least one default route.
 		return rdsUpdate{}, fmt.Errorf("matched virtual host has no routes")
 	}
+
+	// Keep the old code path for routing disabled.
+	if routingEnabled {
+		routes, err := routesProtoToSlice(vh.Routes, logger)
+		if err != nil {
+			return rdsUpdate{}, fmt.Errorf("received route is invalid: %v", err)
+		}
+		return rdsUpdate{routes: routes}, nil
+	}
+
 	dr := vh.Routes[len(vh.Routes)-1]
 	match := dr.GetMatch()
 	if match == nil {
@@ -108,12 +120,12 @@ func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host
 		// valid.
 		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route set case-sensitive to false")
 	}
-	route := dr.GetRoute()
-	if route == nil {
+	routeAction := dr.GetRoute()
+	if routeAction == nil {
 		return rdsUpdate{}, fmt.Errorf("matched route is nil")
 	}
 
-	if wc := route.GetWeightedClusters(); wc != nil {
+	if wc := routeAction.GetWeightedClusters(); wc != nil {
 		m, err := weightedClustersProtoToMap(wc)
 		if err != nil {
 			return rdsUpdate{}, fmt.Errorf("matched weighted cluster is invalid: %v", err)
@@ -129,7 +141,114 @@ func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host
 	// and CDS. In case when the action changes between one cluster and multiple
 	// clusters, changing top level policy means recreating TCP connection every
 	// time.
-	return rdsUpdate{weightedCluster: map[string]uint32{route.GetCluster(): 1}}, nil
+	return rdsUpdate{weightedCluster: map[string]uint32{routeAction.GetCluster(): 1}}, nil
+}
+
+func routesProtoToSlice(routes []*routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) {
+	var routesRet []*Route
+
+	for _, r := range routes {
+		match := r.GetMatch()
+		if match == nil {
+			return nil, fmt.Errorf("route %+v doesn't have a match", r)
+		}
+
+		if len(match.GetQueryParameters()) != 0 {
+			// Ignore route with query parameters.
+			logger.Warningf("route %+v has query parameter matchers, the route will be ignored", r)
+			continue
+		}
+
+		if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
+			return nil, fmt.Errorf("route %+v has case-sensitive false", r)
+		}
+
+		pathSp := match.GetPathSpecifier()
+		if pathSp == nil {
+			return nil, fmt.Errorf("route %+v doesn't have a path specifier", r)
+		}
+
+		var route Route
+		switch pt := pathSp.(type) {
+		case *routepb.RouteMatch_Prefix:
+			route.Prefix = &pt.Prefix
+		case *routepb.RouteMatch_Path:
+			route.Path = &pt.Path
+		case *routepb.RouteMatch_SafeRegex:
+			route.Regex = &pt.SafeRegex.Regex
+		case *routepb.RouteMatch_Regex:
+			return nil, fmt.Errorf("route %+v has Regex, expected SafeRegex instead", r)
+		default:
+			logger.Warningf("route %+v has an unrecognized path specifier: %+v", r, pt)
+			continue
+		}
+
+		for _, h := range match.GetHeaders() {
+			var header HeaderMatcher
+			switch ht := h.GetHeaderMatchSpecifier().(type) {
+			case *routepb.HeaderMatcher_ExactMatch:
+				header.ExactMatch = &ht.ExactMatch
+			case *routepb.HeaderMatcher_SafeRegexMatch:
+				header.RegexMatch = &ht.SafeRegexMatch.Regex
+			case *routepb.HeaderMatcher_RangeMatch:
+				header.RangeMatch = &Int64Range{
+					Start: ht.RangeMatch.Start,
+					End:   ht.RangeMatch.End,
+				}
+			case *routepb.HeaderMatcher_PresentMatch:
+				header.PresentMatch = &ht.PresentMatch
+			case *routepb.HeaderMatcher_PrefixMatch:
+				header.PrefixMatch = &ht.PrefixMatch
+			case *routepb.HeaderMatcher_SuffixMatch:
+				header.SuffixMatch = &ht.SuffixMatch
+			case *routepb.HeaderMatcher_RegexMatch:
+				return nil, fmt.Errorf("route %+v has a header matcher with Regex, expected SafeRegex instead", r)
+			default:
+				logger.Warningf("route %+v has an unrecognized header matcher: %+v", r, ht)
+				continue
+			}
+			header.Name = h.GetName()
+			invert := h.GetInvertMatch()
+			header.InvertMatch = &invert
+			route.Headers = append(route.Headers, &header)
+		}
+
+		if fr := match.GetRuntimeFraction(); fr != nil {
+			d := fr.GetDefaultValue()
+			n := d.GetNumerator()
+			switch d.GetDenominator() {
+			case typepb.FractionalPercent_HUNDRED:
+				n *= 10000
+			case typepb.FractionalPercent_TEN_THOUSAND:
+				n *= 100
+			case typepb.FractionalPercent_MILLION:
+			}
+			route.Fraction = &n
+		}
+
+		clusters := make(map[string]uint32)
+		switch a := r.GetRoute().GetClusterSpecifier().(type) {
+		case *routepb.RouteAction_Cluster:
+			clusters[a.Cluster] = 1
+		case *routepb.RouteAction_WeightedClusters:
+			wcs := a.WeightedClusters
+			var totalWeight uint32
+			for _, c := range wcs.Clusters {
+				w := c.GetWeight().GetValue()
+				clusters[c.GetName()] = w
+				totalWeight += w
+			}
+			if totalWeight != wcs.GetTotalWeight().GetValue() {
+				return nil, fmt.Errorf("route %+v, action %+v, weights of clusters do not add up to total total weight, got: %v, want %v", r, a, wcs.GetTotalWeight().GetValue(), totalWeight)
+			}
+		case *routepb.RouteAction_ClusterHeader:
+			continue
+		}
+
+		route.Action = clusters
+		routesRet = append(routesRet, &route)
+	}
+	return routesRet, nil
 }
 
 func weightedClustersProtoToMap(wc *routepb.WeightedCluster) (map[string]uint32, error) {
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
index a099e32c5eed..e3ec4e01fd8f 100644
--- a/xds/internal/client/v2client_rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -23,10 +23,13 @@ import (
 	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	typepb "github.com/envoyproxy/go-control-plane/envoy/type"
 	"github.com/golang/protobuf/proto"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 )
 
@@ -220,7 +223,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, goodLDSTarget1)
+			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, goodLDSTarget1, nil)
 			if !cmp.Equal(gotUpdate, test.wantUpdate, cmp.AllowUnexported(rdsUpdate{})) || (gotError != nil) != test.wantError {
 				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, goodLDSTarget1, gotUpdate, test.wantUpdate)
 			}
@@ -240,10 +243,59 @@ func doLDS(t *testing.T, v2c *v2Client, fakeServer *fakeserver.Server) {
 	}
 }
 
-// TestRDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
-// and creates a v2Client using it. Then, it registers an LDS and RDS watcher
-// and tests different RDS responses.
-func (s) TestRDSHandleResponse(t *testing.T) {
+// TestRDSHandleResponseWithRoutingEnabled starts a fake xDS server, makes a
+// ClientConn to it, and creates a v2Client using it. Then, it registers an LDS
+// and RDS watcher and tests different RDS responses.
+//
+// Routing is protected by an env variable. This test sets it to true, so the
+// new fields will be parsed.
+func (s) TestRDSHandleResponseWithRoutingEnabled(t *testing.T) {
+	routingEnabled = true
+	defer func() {
+		routingEnabled = false
+	}()
+	tests := []struct {
+		name          string
+		rdsResponse   *xdspb.DiscoveryResponse
+		wantErr       bool
+		wantUpdate    *rdsUpdate
+		wantUpdateErr bool
+	}{
+		// Response contains one good interesting RouteConfiguration.
+		{
+			name:        "one-good-route-config",
+			rdsResponse: goodRDSResponse1,
+			wantErr:     false,
+			wantUpdate: &rdsUpdate{
+				// Instead of just weighted targets when routing is disabled,
+				// this result contains a route with perfix "", and action as
+				// weighted targets.
+				routes: []*Route{{
+					Prefix: newStringP(""),
+					Action: map[string]uint32{goodClusterName1: 1},
+				}},
+			},
+			wantUpdateErr: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testWatchHandle(t, &watchHandleTestcase{
+				typeURL:          rdsURL,
+				resourceName:     goodRouteName1,
+				responseToHandle: test.rdsResponse,
+				wantHandleErr:    test.wantErr,
+				wantUpdate:       test.wantUpdate,
+				wantUpdateErr:    test.wantUpdateErr,
+			})
+		})
+	}
+}
+
+// TestRDSHandleResponseWithRoutingDisabled starts a fake xDS server, makes a
+// ClientConn to it, and creates a v2Client using it. Then, it registers an LDS
+// and RDS watcher and tests different RDS responses.
+func (s) TestRDSHandleResponseWithRoutingDisabled(t *testing.T) {
 	tests := []struct {
 		name          string
 		rdsResponse   *xdspb.DiscoveryResponse
@@ -516,3 +568,166 @@ func (s) TestWeightedClustersProtoToMap(t *testing.T) {
 		})
 	}
 }
+
+func TestRoutesProtoToSlice(t *testing.T) {
+	tests := []struct {
+		name       string
+		routes     []*routepb.Route
+		wantRoutes []*Route
+		wantErr    bool
+	}{
+		{
+			name: "no path",
+			routes: []*routepb.Route{{
+				Match: &routepb.RouteMatch{},
+			}},
+			wantErr: true,
+		},
+		{
+			name: "path is regex instead of saferegex",
+			routes: []*routepb.Route{{
+				Match: &routepb.RouteMatch{
+					PathSpecifier: &routepb.RouteMatch_Regex{Regex: "*"},
+				},
+			}},
+			wantErr: true,
+		},
+		{
+			name: "header contains regex",
+			routes: []*routepb.Route{{
+				Match: &routepb.RouteMatch{
+					PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"},
+					Headers: []*routepb.HeaderMatcher{{
+						Name: "th",
+						HeaderMatchSpecifier: &routepb.HeaderMatcher_RegexMatch{
+							RegexMatch: "*",
+						},
+					}},
+				},
+			}},
+			wantErr: true,
+		},
+		{
+			name: "case_sensitive is false",
+			routes: []*routepb.Route{{
+				Match: &routepb.RouteMatch{
+					PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"},
+					CaseSensitive: &wrapperspb.BoolValue{Value: false},
+				},
+			}},
+			wantErr: true,
+		},
+		{
+			name: "good",
+			routes: []*routepb.Route{
+				{
+					Match: &routepb.RouteMatch{
+						PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/a/"},
+						Headers: []*routepb.HeaderMatcher{
+							{
+								Name: "th",
+								HeaderMatchSpecifier: &routepb.HeaderMatcher_PrefixMatch{
+									PrefixMatch: "tv",
+								},
+								InvertMatch: true,
+							},
+						},
+						RuntimeFraction: &corepb.RuntimeFractionalPercent{
+							DefaultValue: &typepb.FractionalPercent{
+								Numerator:   1,
+								Denominator: typepb.FractionalPercent_HUNDRED,
+							},
+						},
+					},
+					Action: &routepb.Route_Route{
+						Route: &routepb.RouteAction{
+							ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &routepb.WeightedCluster{
+									Clusters: []*routepb.WeightedCluster_ClusterWeight{
+										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
+										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+									},
+									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+								}}}},
+				},
+			},
+			wantRoutes: []*Route{{
+				Prefix: newStringP("/a/"),
+				Headers: []*HeaderMatcher{
+					{
+						Name:        "th",
+						InvertMatch: newBoolP(true),
+						PrefixMatch: newStringP("tv"),
+					},
+				},
+				Fraction: newUInt32P(10000),
+				Action:   map[string]uint32{"A": 40, "B": 60},
+			}},
+			wantErr: false,
+		},
+		{
+			name: "query is ignored",
+			routes: []*routepb.Route{
+				{
+					Match: &routepb.RouteMatch{
+						PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/a/"},
+					},
+					Action: &routepb.Route_Route{
+						Route: &routepb.RouteAction{
+							ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &routepb.WeightedCluster{
+									Clusters: []*routepb.WeightedCluster_ClusterWeight{
+										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
+										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+									},
+									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+								}}}},
+				},
+				{
+					Name: "with_query",
+					Match: &routepb.RouteMatch{
+						PathSpecifier:   &routepb.RouteMatch_Prefix{Prefix: "/b/"},
+						QueryParameters: []*routepb.QueryParameterMatcher{{Name: "route_will_be_ignored"}},
+					},
+				},
+			},
+			// Only one route in the result, because the second one with query
+			// parameters is ignored.
+			wantRoutes: []*Route{{
+				Prefix: newStringP("/a/"),
+				Action: map[string]uint32{"A": 40, "B": 60},
+			}},
+			wantErr: false,
+		},
+	}
+
+	cmpOpts := []cmp.Option{
+		cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}),
+		cmpopts.EquateEmpty(),
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := routesProtoToSlice(tt.routes, nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.wantRoutes, cmpOpts...) {
+				t.Errorf("routesProtoToSlice() got = %v, want %v, diff: %v", got, tt.wantRoutes, cmp.Diff(got, tt.wantRoutes, cmpOpts...))
+			}
+		})
+	}
+}
+
+func newStringP(s string) *string {
+	return &s
+}
+
+func newUInt32P(i uint32) *uint32 {
+	return &i
+}
+
+func newBoolP(b bool) *bool {
+	return &b
+}

From 5f0e72845ef27d2c2b7bf82dfdac2bcabbdd76ef Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 21 Jul 2020 11:55:49 -0700
Subject: [PATCH 125/481] xdsrouting: balancer implementation (#3746)

---
 xds/internal/balancer/balancer.go             |   1 +
 .../xdsrouting/balancerstateaggregator.go     | 227 ++++++
 xds/internal/balancer/xdsrouting/logging.go   |  34 +
 xds/internal/balancer/xdsrouting/routing.go   | 256 +++++++
 .../balancer/xdsrouting/routing_config.go     |  12 +-
 .../xdsrouting/routing_config_test.go         |  10 +-
 .../balancer/xdsrouting/routing_picker.go     |  61 ++
 .../xdsrouting/routing_picker_test.go         | 177 +++++
 .../balancer/xdsrouting/routing_test.go       | 710 ++++++++++++++++++
 9 files changed, 1477 insertions(+), 11 deletions(-)
 create mode 100644 xds/internal/balancer/xdsrouting/balancerstateaggregator.go
 create mode 100644 xds/internal/balancer/xdsrouting/logging.go
 create mode 100644 xds/internal/balancer/xdsrouting/routing.go
 create mode 100644 xds/internal/balancer/xdsrouting/routing_picker.go
 create mode 100644 xds/internal/balancer/xdsrouting/routing_picker_test.go
 create mode 100644 xds/internal/balancer/xdsrouting/routing_test.go

diff --git a/xds/internal/balancer/balancer.go b/xds/internal/balancer/balancer.go
index 489def3ce3bb..d0b1bb786c7d 100644
--- a/xds/internal/balancer/balancer.go
+++ b/xds/internal/balancer/balancer.go
@@ -23,4 +23,5 @@ import (
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"    // Register the CDS balancer
 	_ "google.golang.org/grpc/xds/internal/balancer/edsbalancer"    // Register the EDS balancer
 	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget" // Register the weighted_target balancer
+	_ "google.golang.org/grpc/xds/internal/balancer/xdsrouting"     // Register the xds_routing balancer
 )
diff --git a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go b/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
new file mode 100644
index 000000000000..83dbb7914b9d
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
@@ -0,0 +1,227 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/xds/internal"
+)
+
+type subBalancerState struct {
+	state balancer.State
+	// stateToAggregate is the connectivity state used only for state
+	// aggregation. It could be different from state.ConnectivityState. For
+	// example when a sub-balancer transitions from TransientFailure to
+	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
+	// is still TransientFailure.
+	stateToAggregate connectivity.State
+}
+
+func (s *subBalancerState) String() string {
+	return fmt.Sprintf("picker:%p,state:%v,stateToAggregate:%v", s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
+}
+
+type balancerStateAggregator struct {
+	cc     balancer.ClientConn
+	logger *grpclog.PrefixLogger
+
+	mu sync.Mutex
+	// routes, one for each matcher.
+	routes []route
+	// If started is false, no updates should be sent to the parent cc. A closed
+	// sub-balancer could still send pickers to this aggregator. This makes sure
+	// that no updates will be forwarded to parent when the whole balancer group
+	// and states aggregator is closed.
+	started bool
+	// All balancer IDs exist as keys in this map, even if balancer group is not
+	// started.
+	//
+	// If an ID is not in map, it's either removed or never added.
+	idToPickerState map[internal.LocalityID]*subBalancerState
+}
+
+func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator {
+	return &balancerStateAggregator{
+		cc:              cc,
+		logger:          logger,
+		idToPickerState: make(map[internal.LocalityID]*subBalancerState),
+	}
+}
+
+// Start starts the aggregator. It can be called after Close to restart the
+// aggretator.
+func (rbsa *balancerStateAggregator) start() {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	rbsa.started = true
+}
+
+// Close closes the aggregator. When the aggregator is closed, it won't call
+// parent ClientConn to upate balancer state.
+func (rbsa *balancerStateAggregator) close() {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	rbsa.started = false
+	rbsa.clearStates()
+}
+
+// add adds a sub-balancer state with weight. It adds a place holder, and waits
+// for the real sub-balancer to update state.
+//
+// This is called when there's a new action.
+func (rbsa *balancerStateAggregator) add(id internal.LocalityID) {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	rbsa.idToPickerState[id] = &subBalancerState{
+		// Start everything in CONNECTING, so if one of the sub-balancers
+		// reports TransientFailure, the RPCs will still wait for the other
+		// sub-balancers.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+		stateToAggregate: connectivity.Connecting,
+	}
+}
+
+// remove removes the sub-balancer state. Future updates from this sub-balancer,
+// if any, will be ignored.
+//
+// This is called when an action is removed.
+func (rbsa *balancerStateAggregator) remove(id internal.LocalityID) {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	if _, ok := rbsa.idToPickerState[id]; !ok {
+		return
+	}
+	// Remove id and picker from picker map. This also results in future updates
+	// for this ID to be ignored.
+	delete(rbsa.idToPickerState, id)
+}
+
+// updateRoutes updates the routes. Note that it doesn't trigger an update to
+// the parent ClientConn. The caller should decide when it's necessary, and call
+// buildAndUpdate.
+func (rbsa *balancerStateAggregator) updateRoutes(newRoutes []route) {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	rbsa.routes = newRoutes
+}
+
+// UpdateState is called to report a balancer state change from sub-balancer.
+// It's usually called by the balancer group.
+//
+// It calls parent ClientConn's UpdateState with the new aggregated state.
+func (rbsa *balancerStateAggregator) UpdateState(id internal.LocalityID, state balancer.State) {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	pickerSt, ok := rbsa.idToPickerState[id]
+	if !ok {
+		// All state starts with an entry in pickStateMap. If ID is not in map,
+		// it's either removed, or never existed.
+		return
+	}
+	if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) {
+		// If old state is TransientFailure, and new state is Connecting, don't
+		// update the state, to prevent the aggregated state from being always
+		// CONNECTING. Otherwise, stateToAggregate is the same as
+		// state.ConnectivityState.
+		pickerSt.stateToAggregate = state.ConnectivityState
+	}
+	pickerSt.state = state
+
+	if !rbsa.started {
+		return
+	}
+	rbsa.cc.UpdateState(rbsa.build())
+}
+
+// clearState Reset everything to init state (Connecting) but keep the entry in
+// map (to keep the weight).
+//
+// Caller must hold rbsa.mu.
+func (rbsa *balancerStateAggregator) clearStates() {
+	for _, pState := range rbsa.idToPickerState {
+		pState.state = balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		}
+		pState.stateToAggregate = connectivity.Connecting
+	}
+}
+
+// buildAndUpdate combines the sub-state from each sub-balancer into one state,
+// and update it to parent ClientConn.
+func (rbsa *balancerStateAggregator) buildAndUpdate() {
+	rbsa.mu.Lock()
+	defer rbsa.mu.Unlock()
+	if !rbsa.started {
+		return
+	}
+	rbsa.cc.UpdateState(rbsa.build())
+}
+
+// build combines sub-states into one. The picker will do routing pick.
+//
+// Caller must hold rbsa.mu.
+func (rbsa *balancerStateAggregator) build() balancer.State {
+	// TODO: the majority of this function (and UpdateState) is exactly the same
+	// as weighted_target's state aggregator. Try to make a general utility
+	// function/struct to handle the logic.
+	//
+	// One option: make a SubBalancerState that handles Update(State), including
+	// handling the special connecting after ready, as in UpdateState(). Then a
+	// function to calculate the aggregated connectivity state as in this
+	// function.
+	var readyN, connectingN int
+	for _, ps := range rbsa.idToPickerState {
+		switch ps.stateToAggregate {
+		case connectivity.Ready:
+			readyN++
+		case connectivity.Connecting:
+			connectingN++
+		}
+	}
+	var aggregatedState connectivity.State
+	switch {
+	case readyN > 0:
+		aggregatedState = connectivity.Ready
+	case connectingN > 0:
+		aggregatedState = connectivity.Connecting
+	default:
+		aggregatedState = connectivity.TransientFailure
+	}
+
+	// The picker's return error might not be consistent with the
+	// aggregatedState. Because for routing, we want to always build picker with
+	// all sub-pickers (not even ready sub-pickers), so even if the overall
+	// state is Ready, pick for certain RPCs can behave like Connecting or
+	// TransientFailure.
+	rbsa.logger.Infof("Child pickers with routes: %s, actions: %+v", rbsa.routes, rbsa.idToPickerState)
+	return balancer.State{
+		ConnectivityState: aggregatedState,
+		Picker:            newPickerGroup(rbsa.routes, rbsa.idToPickerState),
+	}
+}
diff --git a/xds/internal/balancer/xdsrouting/logging.go b/xds/internal/balancer/xdsrouting/logging.go
new file mode 100644
index 000000000000..5c4a6b3cb410
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/logging.go
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[xds-routing-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *routingBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}
diff --git a/xds/internal/balancer/xdsrouting/routing.go b/xds/internal/balancer/xdsrouting/routing.go
new file mode 100644
index 000000000000..ebbb2b8cf9d7
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/routing.go
@@ -0,0 +1,256 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+)
+
+const xdsRoutingName = "xds_routing_experimental"
+
+func init() {
+	balancer.Register(&routingBB{})
+}
+
+type routingBB struct{}
+
+func (rbb *routingBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+	b := &routingBalancer{}
+	b.logger = prefixLogger(b)
+	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
+	b.stateAggregator.start()
+	b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
+	b.bg.Start()
+	b.logger.Infof("Created")
+	return b
+}
+
+func (rbb *routingBB) Name() string {
+	return xdsRoutingName
+}
+
+func (rbb *routingBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type route struct {
+	action string
+	m      *compositeMatcher
+}
+
+func (r route) String() string {
+	return r.m.String() + "->" + r.action
+}
+
+type routingBalancer struct {
+	logger *grpclog.PrefixLogger
+
+	// TODO: make this package not dependent on xds specific code. Same as for
+	// weighted target balancer.
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *balancerStateAggregator
+
+	actions map[string]actionConfig
+	routes  []route
+}
+
+// TODO: remove this and use strings directly as keys for balancer group.
+func makeLocalityFromName(name string) internal.LocalityID {
+	return internal.LocalityID{Region: name}
+}
+
+// TODO: remove this and use strings directly as keys for balancer group.
+func getNameFromLocality(id internal.LocalityID) string {
+	return id.Region
+}
+
+func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *lbConfig) (needRebuild bool) {
+	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
+	var rebuildStateAndPicker bool
+
+	// Remove sub-pickers and sub-balancers that are not in the new action list.
+	for name := range rb.actions {
+		if _, ok := newConfig.actions[name]; !ok {
+			l := makeLocalityFromName(name)
+			rb.stateAggregator.remove(l)
+			rb.bg.Remove(l)
+			// Trigger a state/picker update, because we don't want `ClientConn`
+			// to pick this sub-balancer anymore.
+			rebuildStateAndPicker = true
+		}
+	}
+
+	// For sub-balancers in the new action list,
+	// - add to balancer group if it's new,
+	// - forward the address/balancer config update.
+	for name, newT := range newConfig.actions {
+		l := makeLocalityFromName(name)
+		if _, ok := rb.actions[name]; !ok {
+			// If this is a new sub-balancer, add weights to the picker map.
+			rb.stateAggregator.add(l)
+			// Then add to the balancer group.
+			rb.bg.Add(l, balancer.Get(newT.ChildPolicy.Name))
+			// Not trigger a state/picker update. Wait for the new sub-balancer
+			// to send its updates.
+		}
+		// Forwards all the update:
+		// - Addresses are from the map after splitting with hierarchy path,
+		// - Top level service config and attributes are the same,
+		// - Balancer config comes from the targets map.
+		//
+		// TODO: handle error? How to aggregate errors and return?
+		_ = rb.bg.UpdateClientConnState(l, balancer.ClientConnState{
+			ResolverState: resolver.State{
+				Addresses:     addressesSplit[name],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			},
+			BalancerConfig: newT.ChildPolicy.Config,
+		})
+	}
+
+	rb.actions = newConfig.actions
+	return rebuildStateAndPicker
+}
+
+func routeToMatcher(r routeConfig) (*compositeMatcher, error) {
+	var pathMatcher pathMatcherInterface
+	switch {
+	case r.regex != "":
+		re, err := regexp.Compile(r.regex)
+		if err != nil {
+			return nil, fmt.Errorf("failed to compile regex %q", r.regex)
+		}
+		pathMatcher = newPathRegexMatcher(re)
+	case r.path != "":
+		pathMatcher = newPathExactMatcher(r.path)
+	default:
+		pathMatcher = newPathPrefixMatcher(r.prefix)
+	}
+
+	var headerMatchers []headerMatcherInterface
+	for _, h := range r.headers {
+		var matcherT headerMatcherInterface
+		switch {
+		case h.exactMatch != "":
+			matcherT = newHeaderExactMatcher(h.name, h.exactMatch)
+		case h.regexMatch != "":
+			re, err := regexp.Compile(h.regexMatch)
+			if err != nil {
+				return nil, fmt.Errorf("failed to compile regex %q, skipping this matcher", h.regexMatch)
+			}
+			matcherT = newHeaderRegexMatcher(h.name, re)
+		case h.prefixMatch != "":
+			matcherT = newHeaderPrefixMatcher(h.name, h.prefixMatch)
+		case h.suffixMatch != "":
+			matcherT = newHeaderSuffixMatcher(h.name, h.suffixMatch)
+		case h.rangeMatch != nil:
+			matcherT = newHeaderRangeMatcher(h.name, h.rangeMatch.start, h.rangeMatch.end)
+		default:
+			matcherT = newHeaderPresentMatcher(h.name, h.presentMatch)
+		}
+		if h.invertMatch {
+			matcherT = newInvertMatcher(matcherT)
+		}
+		headerMatchers = append(headerMatchers, matcherT)
+	}
+
+	var fractionMatcher *fractionMatcher
+	if r.fraction != nil {
+		fractionMatcher = newFractionMatcher(*r.fraction)
+	}
+	return newCompositeMatcher(pathMatcher, headerMatchers, fractionMatcher), nil
+}
+
+func routesEqual(a, b []route) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		aa := a[i]
+		bb := b[i]
+		if aa.action != bb.action {
+			return false
+		}
+		if !aa.m.equal(bb.m) {
+			return false
+		}
+	}
+	return true
+}
+
+func (rb *routingBalancer) updateRoutes(newConfig *lbConfig) (needRebuild bool, _ error) {
+	var newRoutes []route
+	for _, rt := range newConfig.routes {
+		newMatcher, err := routeToMatcher(rt)
+		if err != nil {
+			return false, err
+		}
+		newRoutes = append(newRoutes, route{action: rt.action, m: newMatcher})
+	}
+	rebuildStateAndPicker := !routesEqual(newRoutes, rb.routes)
+	rb.routes = newRoutes
+
+	if rebuildStateAndPicker {
+		rb.stateAggregator.updateRoutes(rb.routes)
+	}
+	return rebuildStateAndPicker, nil
+}
+
+func (rb *routingBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+	rb.logger.Infof("update with config %+v, resolver state %+v", s.BalancerConfig, s.ResolverState)
+
+	rebuildForActions := rb.updateActions(s, newConfig)
+	rebuildForRoutes, err := rb.updateRoutes(newConfig)
+	if err != nil {
+		return fmt.Errorf("xds_routing balancer: failed to update routes: %v", err)
+	}
+
+	if rebuildForActions || rebuildForRoutes {
+		rb.stateAggregator.buildAndUpdate()
+	}
+	return nil
+}
+
+func (rb *routingBalancer) ResolverError(err error) {
+	rb.bg.ResolverError(err)
+}
+
+func (rb *routingBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	rb.bg.UpdateSubConnState(sc, state)
+}
+
+func (rb *routingBalancer) Close() {
+	rb.stateAggregator.close()
+	rb.bg.Close()
+}
diff --git a/xds/internal/balancer/xdsrouting/routing_config.go b/xds/internal/balancer/xdsrouting/routing_config.go
index c8cc7a18c6db..78716a098136 100644
--- a/xds/internal/balancer/xdsrouting/routing_config.go
+++ b/xds/internal/balancer/xdsrouting/routing_config.go
@@ -28,7 +28,7 @@ import (
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
-type action struct {
+type actionConfig struct {
 	// ChildPolicy is the child policy and it's config.
 	ChildPolicy *internalserviceconfig.BalancerConfig
 }
@@ -48,7 +48,7 @@ type headerMatcher struct {
 	presentMatch                                     bool
 }
 
-type route struct {
+type routeConfig struct {
 	// Path, Prefix and Regex can have at most one set. This is guaranteed by
 	// config parsing.
 	path, prefix, regex string
@@ -63,8 +63,8 @@ type route struct {
 // lbConfig is the balancer config for xds routing policy.
 type lbConfig struct {
 	serviceconfig.LoadBalancingConfig
-	routes  []route
-	actions map[string]action
+	routes  []routeConfig
+	actions map[string]actionConfig
 }
 
 // The following structs with `JSON` in name are temporary structs to unmarshal
@@ -84,13 +84,13 @@ type routeJSON struct {
 // lbConfigJSON is temporary struct for json unmarshal.
 type lbConfigJSON struct {
 	Route  []routeJSON
-	Action map[string]action
+	Action map[string]actionConfig
 }
 
 func (jc lbConfigJSON) toLBConfig() *lbConfig {
 	var ret lbConfig
 	for _, r := range jc.Route {
-		var tempR route
+		var tempR routeConfig
 		switch {
 		case r.Path != nil:
 			tempR.path = *r.Path
diff --git a/xds/internal/balancer/xdsrouting/routing_config_test.go b/xds/internal/balancer/xdsrouting/routing_config_test.go
index c1ff4611dac9..8cb4d22724e3 100644
--- a/xds/internal/balancer/xdsrouting/routing_config_test.go
+++ b/xds/internal/balancer/xdsrouting/routing_config_test.go
@@ -294,14 +294,14 @@ func Test_parseConfig(t *testing.T) {
 			name: "OK with path matchers only",
 			js:   testJSONConfig,
 			want: &lbConfig{
-				routes: []route{
+				routes: []routeConfig{
 					{path: "/service_1/method_1", action: "cds:cluster_1"},
 					{path: "/service_1/method_2", action: "cds:cluster_1"},
 					{prefix: "/service_2/method_1", action: "weighted:cluster_1_cluster_2_1"},
 					{prefix: "/service_2", action: "weighted:cluster_1_cluster_2_1"},
 					{regex: "^/service_2/method_3$", action: "weighted:cluster_1_cluster_3_1"},
 				},
-				actions: map[string]action{
+				actions: map[string]actionConfig{
 					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
 						Name: cdsName, Config: cdsConfig1},
 					},
@@ -319,7 +319,7 @@ func Test_parseConfig(t *testing.T) {
 			name: "OK with all matchers",
 			js:   testJSONConfigWithAllMatchers,
 			want: &lbConfig{
-				routes: []route{
+				routes: []routeConfig{
 					{path: "/service_1/method_1", action: "cds:cluster_1"},
 					{prefix: "/service_2/method_1", action: "cds:cluster_1"},
 					{regex: "^/service_2/method_3$", action: "cds:cluster_1"},
@@ -332,7 +332,7 @@ func Test_parseConfig(t *testing.T) {
 					{prefix: "", headers: []headerMatcher{{name: "header-1", suffixMatch: "value-1"}}, action: "cds:cluster_2"},
 					{prefix: "", fraction: newUInt32P(31415), action: "cds:cluster_3"},
 				},
-				actions: map[string]action{
+				actions: map[string]actionConfig{
 					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
 						Name: cdsName, Config: cdsConfig1},
 					},
@@ -348,7 +348,7 @@ func Test_parseConfig(t *testing.T) {
 		},
 	}
 
-	cmpOptions := []cmp.Option{cmp.AllowUnexported(lbConfig{}, route{}, headerMatcher{}, int64Range{})}
+	cmpOptions := []cmp.Option{cmp.AllowUnexported(lbConfig{}, routeConfig{}, headerMatcher{}, int64Range{})}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/xds/internal/balancer/xdsrouting/routing_picker.go b/xds/internal/balancer/xdsrouting/routing_picker.go
new file mode 100644
index 000000000000..36fdebd4ad2a
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/routing_picker.go
@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal"
+)
+
+// pickerGroup contains a list of route matchers and their corresponding
+// pickers. For each pick, the first matched picker is used. If the picker isn't
+// ready, the pick will be queued.
+type pickerGroup struct {
+	routes  []route
+	pickers map[string]balancer.Picker
+}
+
+func newPickerGroup(routes []route, idToPickerState map[internal.LocalityID]*subBalancerState) *pickerGroup {
+	pickers := make(map[string]balancer.Picker)
+	for id, st := range idToPickerState {
+		pickers[getNameFromLocality(id)] = st.state.Picker
+	}
+	return &pickerGroup{
+		routes:  routes,
+		pickers: pickers,
+	}
+}
+
+var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found")
+
+func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	for _, rt := range pg.routes {
+		if rt.m.match(info) {
+			// action from route is the ID for the sub-balancer to use.
+			p, ok := pg.pickers[rt.action]
+			if !ok {
+				return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+			}
+			return p.Pick(info)
+		}
+	}
+	return balancer.PickResult{}, errNoMatchedRouteFound
+}
diff --git a/xds/internal/balancer/xdsrouting/routing_picker_test.go b/xds/internal/balancer/xdsrouting/routing_picker_test.go
new file mode 100644
index 000000000000..2de40757a4f7
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/routing_picker_test.go
@@ -0,0 +1,177 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+var (
+	testPickers = []*testutils.TestConstPicker{
+		{SC: testutils.TestSubConns[0]},
+		{SC: testutils.TestSubConns[1]},
+	}
+)
+
+func (s) TestRoutingPickerGroupPick(t *testing.T) {
+	tests := []struct {
+		name string
+
+		routes  []route
+		pickers map[internal.LocalityID]*subBalancerState
+		info    balancer.PickInfo
+
+		want    balancer.PickResult
+		wantErr error
+	}{
+		{
+			name:    "empty",
+			wantErr: errNoMatchedRouteFound,
+		},
+		{
+			name: "one route no match",
+			routes: []route{
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+			},
+			pickers: map[internal.LocalityID]*subBalancerState{
+				makeLocalityFromName("action-0"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[0],
+				}},
+			},
+			info:    balancer.PickInfo{FullMethodName: "/z/y"},
+			wantErr: errNoMatchedRouteFound,
+		},
+		{
+			name: "one route one match",
+			routes: []route{
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+			},
+			pickers: map[internal.LocalityID]*subBalancerState{
+				makeLocalityFromName("action-0"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[0],
+				}},
+			},
+			info: balancer.PickInfo{FullMethodName: "/a/b"},
+			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
+		},
+		{
+			name: "two routes first match",
+			routes: []route{
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"},
+			},
+			pickers: map[internal.LocalityID]*subBalancerState{
+				makeLocalityFromName("action-0"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[0],
+				}},
+				makeLocalityFromName("action-1"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[1],
+				}},
+			},
+			info: balancer.PickInfo{FullMethodName: "/a/b"},
+			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
+		},
+		{
+			name: "two routes second match",
+			routes: []route{
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"},
+			},
+			pickers: map[internal.LocalityID]*subBalancerState{
+				makeLocalityFromName("action-0"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[0],
+				}},
+				makeLocalityFromName("action-1"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[1],
+				}},
+			},
+			info: balancer.PickInfo{FullMethodName: "/z/y"},
+			want: balancer.PickResult{SubConn: testutils.TestSubConns[1]},
+		},
+		{
+			name: "two routes both match former more specific",
+			routes: []route{
+				{m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-1"},
+			},
+			pickers: map[internal.LocalityID]*subBalancerState{
+				makeLocalityFromName("action-0"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[0],
+				}},
+				makeLocalityFromName("action-1"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[1],
+				}},
+			},
+			info: balancer.PickInfo{FullMethodName: "/a/b"},
+			// First route is a match, so first action is picked.
+			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
+		},
+		{
+			name: "tow routes both match latter more specific",
+			routes: []route{
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-1"},
+			},
+			pickers: map[internal.LocalityID]*subBalancerState{
+				makeLocalityFromName("action-0"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[0],
+				}},
+				makeLocalityFromName("action-1"): {state: balancer.State{
+					ConnectivityState: connectivity.Ready,
+					Picker:            testPickers[1],
+				}},
+			},
+			info: balancer.PickInfo{FullMethodName: "/a/b"},
+			// First route is a match, so first action is picked, even though
+			// second is an exact match.
+			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
+		},
+	}
+	cmpOpts := []cmp.Option{cmp.AllowUnexported(testutils.TestSubConn{})}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pg := newPickerGroup(tt.routes, tt.pickers)
+			got, err := pg.Pick(tt.info)
+			t.Logf("Pick(%+v) = {%+v, %+v}", tt.info, got, err)
+			if err != tt.wantErr {
+				t.Errorf("Pick() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.want, cmpOpts...) {
+				t.Errorf("Pick() got = %v, want %v, diff %s", got, tt.want, cmp.Diff(got, tt.want, cmpOpts...))
+			}
+		})
+	}
+
+}
diff --git a/xds/internal/balancer/xdsrouting/routing_test.go b/xds/internal/balancer/xdsrouting/routing_test.go
new file mode 100644
index 000000000000..90607bc18607
--- /dev/null
+++ b/xds/internal/balancer/xdsrouting/routing_test.go
@@ -0,0 +1,710 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsrouting
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+var (
+	rtBuilder           balancer.Builder
+	rtParser            balancer.ConfigParser
+	testBackendAddrStrs []string
+)
+
+const ignoreAttrsRRName = "ignore_attrs_round_robin"
+
+type ignoreAttrsRRBuilder struct {
+	balancer.Builder
+}
+
+func (trrb *ignoreAttrsRRBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	return &ignoreAttrsRRBalancer{trrb.Builder.Build(cc, opts)}
+}
+
+func (*ignoreAttrsRRBuilder) Name() string {
+	return ignoreAttrsRRName
+}
+
+// ignoreAttrsRRBalancer clears attributes from all addresses.
+//
+// It's necessary in this tests because hierarchy modifies address.Attributes.
+// Even if rr gets addresses with empty hierarchy, the attributes fields are
+// different. This is a temporary walkaround for the tests to ignore attributes.
+// Eventually, we need a way for roundrobin to know that two addresses with
+// empty attributes are equal.
+//
+// TODO: delete this when the issue is resolved:
+// https://github.com/grpc/grpc-go/issues/3611.
+type ignoreAttrsRRBalancer struct {
+	balancer.Balancer
+}
+
+func (trrb *ignoreAttrsRRBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	var newAddrs []resolver.Address
+	for _, a := range s.ResolverState.Addresses {
+		a.Attributes = nil
+		newAddrs = append(newAddrs, a)
+	}
+	s.ResolverState.Addresses = newAddrs
+	return trrb.Balancer.UpdateClientConnState(s)
+}
+
+const testBackendAddrsCount = 12
+
+func init() {
+	for i := 0; i < testBackendAddrsCount; i++ {
+		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
+	}
+	rtBuilder = balancer.Get(xdsRoutingName)
+	rtParser = rtBuilder.(balancer.ConfigParser)
+
+	balancer.Register(&ignoreAttrsRRBuilder{balancer.Get(roundrobin.Name)})
+
+	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond
+}
+
+func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) {
+	t.Helper()
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p.Pick(info)
+		if err != wantErr {
+			t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr)
+		}
+		if !cmp.Equal(gotSCSt.SubConn, wantSC, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC)
+		}
+	}
+}
+
+func TestRouting(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"Action": {
+	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
+},
+"Route": [
+	{"prefix":"/a/", "action":"cds:cluster_1"},
+	{"prefix":"", "headers":[{"name":"header-1", "exactMatch":"value-1"}], "action":"cds:cluster_2"}
+]
+}`
+
+	config1, err := rtParser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], Attributes: nil},
+		{Addr: testBackendAddrStrs[1], Attributes: nil},
+	}
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		if len(hierarchy.Get(addrs[0])) != 0 {
+			t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
+		}
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].Attributes = nil
+		m1[addrs[0]] = sc
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				FullMethodName: "/z/y",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")),
+			},
+			wantSC:  m1[wantAddrs[1]],
+			wantErr: nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				FullMethodName: "/z/y",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")),
+			},
+			wantSC:  nil,
+			wantErr: errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestRoutingConfigUpdateAddRoute covers the cases the routing balancer
+// receives config update with extra route, but the same actions.
+func TestRoutingConfigUpdateAddRoute(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"Action": {
+	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
+},
+"Route": [
+	{"prefix":"/a/", "action":"cds:cluster_1"},
+	{"path":"/z/y", "action":"cds:cluster_2"}
+]
+}`
+
+	config1, err := rtParser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], Attributes: nil},
+		{Addr: testBackendAddrStrs[1], Attributes: nil},
+	}
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		if len(hierarchy.Get(addrs[0])) != 0 {
+			t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
+		}
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].Attributes = nil
+		m1[addrs[0]] = sc
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
+			wantSC:   m1[wantAddrs[1]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
+			wantSC:   nil,
+			wantErr:  errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with different routes, but the same actions. Expect a
+	// picker update, but no subconn changes.
+	configJSON2 := `{
+"Action": {
+	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
+},
+"Route": [
+	{"prefix":"", "headers":[{"name":"header-1", "presentMatch":true}], "action":"cds:cluster_2"},
+	{"prefix":"/a/", "action":"cds:cluster_1"},
+	{"path":"/z/y", "action":"cds:cluster_2"}
+]
+}`
+	config2, err := rtParser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	// Send update with the same addresses.
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// New change to actions, expect no newSubConn.
+	select {
+	case <-time.After(time.Millisecond * 500):
+	case <-cc.NewSubConnCh:
+		addrs := <-cc.NewSubConnAddrsCh
+		t.Fatalf("unexpected NewSubConn with address %v", addrs)
+	}
+
+	p2 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
+			wantSC:   m1[wantAddrs[1]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				FullMethodName: "/a/z",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")),
+			},
+			wantSC:  m1[wantAddrs[1]],
+			wantErr: nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				FullMethodName: "/c/d",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")),
+			},
+			wantSC:  nil,
+			wantErr: errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestRoutingConfigUpdateAddRouteAndAction covers the cases the routing
+// balancer receives config update with extra route and actions.
+func TestRoutingConfigUpdateAddRouteAndAction(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"Action": {
+	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
+},
+"Route": [
+	{"prefix":"/a/", "action":"cds:cluster_1"},
+	{"path":"/z/y", "action":"cds:cluster_2"}
+]
+}`
+
+	config1, err := rtParser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], Attributes: nil},
+		{Addr: testBackendAddrStrs[1], Attributes: nil},
+	}
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		if len(hierarchy.Get(addrs[0])) != 0 {
+			t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
+		}
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].Attributes = nil
+		m1[addrs[0]] = sc
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
+			wantSC:   m1[wantAddrs[1]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
+			wantSC:   nil,
+			wantErr:  errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with different routes, and different actions. Expect a
+	// new subconn and a picker update.
+	configJSON2 := `{
+"Action": {
+	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_3":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
+},
+"Route": [
+	{"prefix":"", "headers":[{"name":"header-1", "presentMatch":false, "invertMatch":true}], "action":"cds:cluster_3"},
+	{"prefix":"/a/", "action":"cds:cluster_1"},
+	{"path":"/z/y", "action":"cds:cluster_2"}
+]
+}`
+	config2, err := rtParser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], Attributes: nil})
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+			hierarchy.Set(wantAddrs[2], []string{"cds:cluster_3"}),
+		}},
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect exactly one new subconn.
+	addrs := <-cc.NewSubConnAddrsCh
+	if len(hierarchy.Get(addrs[0])) != 0 {
+		t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
+	}
+	sc := <-cc.NewSubConnCh
+	// Clear the attributes before adding to map.
+	addrs[0].Attributes = nil
+	m1[addrs[0]] = sc
+	rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Should have no more newSubConn.
+	select {
+	case <-time.After(time.Millisecond * 500):
+	case <-cc.NewSubConnCh:
+		addrs := <-cc.NewSubConnAddrsCh
+		t.Fatalf("unexpected NewSubConn with address %v", addrs)
+	}
+
+	p2 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
+			wantSC:   m1[wantAddrs[1]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				FullMethodName: "/a/z",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")),
+			},
+			wantSC:  m1[wantAddrs[2]],
+			wantErr: nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				FullMethodName: "/c/d",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")),
+			},
+			wantSC:  nil,
+			wantErr: errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestRoutingConfigUpdateDeleteAll covers the cases the routing balancer receives config
+// update with no routes. Pick should fail with details in error.
+func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"Action": {
+	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
+},
+"Route": [
+	{"prefix":"/a/", "action":"cds:cluster_1"},
+	{"path":"/z/y", "action":"cds:cluster_2"}
+]
+}`
+
+	config1, err := rtParser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], Attributes: nil},
+		{Addr: testBackendAddrStrs[1], Attributes: nil},
+	}
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		if len(hierarchy.Get(addrs[0])) != 0 {
+			t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
+		}
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].Attributes = nil
+		m1[addrs[0]] = sc
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m1[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
+			wantSC:   m1[wantAddrs[1]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
+			wantSC:   nil,
+			wantErr:  errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with no routes.
+	configJSON2 := `{}`
+	config2, err := rtParser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect two remove subconn.
+	for range wantAddrs {
+		select {
+		case <-time.After(time.Millisecond * 500):
+			t.Fatalf("timeout waiting for remove subconn")
+		case <-cc.RemoveSubConnCh:
+		}
+	}
+
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p2.Pick(balancer.PickInfo{})
+		if err != errNoMatchedRouteFound {
+			t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, errNoMatchedRouteFound)
+		}
+	}
+
+	// Resend the previous config with routes and actions.
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
+			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m2 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		if len(hierarchy.Get(addrs[0])) != 0 {
+			t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
+		}
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].Attributes = nil
+		m2[addrs[0]] = sc
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p3 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
+			wantSC:   m2[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
+			wantSC:   m2[wantAddrs[0]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
+			wantSC:   m2[wantAddrs[1]],
+			wantErr:  nil,
+		},
+		{
+			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
+			wantSC:   nil,
+			wantErr:  errNoMatchedRouteFound,
+		},
+	} {
+		testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}

From a5a36bd3f0bbf60d6257ff57eb5749b4e58f5e94 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Tue, 21 Jul 2020 14:07:03 -0700
Subject: [PATCH 126/481] client: set auth header to localhost for unix target
 (#3730)

---
 clientconn.go          |   3 ++
 test/authority_test.go | 104 +++++++++++++++++++++++++++++++++++++++++
 test/end2end_test.go   |  42 ++++++++++++-----
 3 files changed, 137 insertions(+), 12 deletions(-)
 create mode 100644 test/authority_test.go

diff --git a/clientconn.go b/clientconn.go
index 11d9ad4d72a2..ae5ce4947e2e 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -245,6 +245,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 
 	// Determine the resolver to use.
 	cc.parsedTarget = grpcutil.ParseTarget(cc.target)
+	unixScheme := strings.HasPrefix(cc.target, "unix:")
 	channelz.Infof(logger, cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme)
 	resolverBuilder := cc.getResolver(cc.parsedTarget.Scheme)
 	if resolverBuilder == nil {
@@ -267,6 +268,8 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		cc.authority = creds.Info().ServerName
 	} else if cc.dopts.insecure && cc.dopts.authority != "" {
 		cc.authority = cc.dopts.authority
+	} else if unixScheme {
+		cc.authority = "localhost"
 	} else {
 		// Use endpoint from "scheme://authority/endpoint" as the default
 		// authority for ClientConn.
diff --git a/test/authority_test.go b/test/authority_test.go
new file mode 100644
index 000000000000..6cd5d82eec19
--- /dev/null
+++ b/test/authority_test.go
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
+
+func runUnixTest(t *testing.T, address, target, expectedAuthority string) {
+	if err := os.RemoveAll(address); err != nil {
+		t.Fatalf("Error removing socket file %v: %v\n", address, err)
+	}
+	us := &stubServer{
+		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			md, ok := metadata.FromIncomingContext(ctx)
+			if !ok {
+				return nil, status.Error(codes.InvalidArgument, "failed to parse metadata")
+			}
+			auths, ok := md[":authority"]
+			if !ok {
+				return nil, status.Error(codes.InvalidArgument, "no authority header")
+			}
+			if len(auths) < 1 {
+				return nil, status.Error(codes.InvalidArgument, "no authority header")
+			}
+			if auths[0] != expectedAuthority {
+				return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority))
+			}
+			return &testpb.Empty{}, nil
+		},
+		network: "unix",
+		address: address,
+		target:  target,
+	}
+	if err := us.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+		return
+	}
+	defer us.Stop()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	_, err := us.client.EmptyCall(ctx, &testpb.Empty{})
+	if err != nil {
+		t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+}
+
+func (s) TestUnix(t *testing.T) {
+	tests := []struct {
+		name      string
+		address   string
+		target    string
+		authority string
+	}{
+		{
+			name:      "Unix1",
+			address:   "sock.sock",
+			target:    "unix:sock.sock",
+			authority: "localhost",
+		},
+		{
+			name:      "Unix2",
+			address:   "/tmp/sock.sock",
+			target:    "unix:/tmp/sock.sock",
+			authority: "localhost",
+		},
+		{
+			name:      "Unix3",
+			address:   "/tmp/sock.sock",
+			target:    "unix:///tmp/sock.sock",
+			authority: "localhost",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			runUnixTest(t, test.address, test.target, test.authority)
+		})
+	}
+}
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 3e129c3fc159..0b735c159840 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -4991,7 +4991,11 @@ type stubServer struct {
 	cc     *grpc.ClientConn
 	s      *grpc.Server
 
-	addr string // address of listener
+	// Parameters for Listen and Dial. Defaults will be used if these are empty
+	// before Start.
+	network string
+	address string
+	target  string
 
 	cleanups []func() // Lambdas executed in Stop(); populated by Start().
 
@@ -5012,14 +5016,21 @@ func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallSer
 
 // Start starts the server and creates a client connected to it.
 func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error {
-	r := manual.NewBuilderWithScheme("whatever")
-	ss.r = r
+	if ss.network == "" {
+		ss.network = "tcp"
+	}
+	if ss.address == "" {
+		ss.address = "localhost:0"
+	}
+	if ss.target == "" {
+		ss.r = manual.NewBuilderWithScheme("whatever")
+	}
 
-	lis, err := net.Listen("tcp", "localhost:0")
+	lis, err := net.Listen(ss.network, ss.address)
 	if err != nil {
-		return fmt.Errorf(`net.Listen("tcp", "localhost:0") = %v`, err)
+		return fmt.Errorf("net.Listen(%q, %q) = %v", ss.network, ss.address, err)
 	}
-	ss.addr = lis.Addr().String()
+	ss.address = lis.Addr().String()
 	ss.cleanups = append(ss.cleanups, func() { lis.Close() })
 
 	s := grpc.NewServer(sopts...)
@@ -5028,15 +5039,20 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption)
 	ss.cleanups = append(ss.cleanups, s.Stop)
 	ss.s = s
 
-	target := ss.r.Scheme() + ":///" + ss.addr
+	opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...)
+	if ss.r != nil {
+		ss.target = ss.r.Scheme() + ":///" + ss.address
+		opts = append(opts, grpc.WithResolvers(ss.r))
+	}
 
-	opts := append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(r)}, dopts...)
-	cc, err := grpc.Dial(target, opts...)
+	cc, err := grpc.Dial(ss.target, opts...)
 	if err != nil {
-		return fmt.Errorf("grpc.Dial(%q) = %v", target, err)
+		return fmt.Errorf("grpc.Dial(%q) = %v", ss.target, err)
 	}
 	ss.cc = cc
-	ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}})
+	if ss.r != nil {
+		ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}})
+	}
 	if err := ss.waitForReady(cc); err != nil {
 		return err
 	}
@@ -5048,7 +5064,9 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption)
 }
 
 func (ss *stubServer) newServiceConfig(sc string) {
-	ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}, ServiceConfig: parseCfg(ss.r, sc)})
+	if ss.r != nil {
+		ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}, ServiceConfig: parseCfg(ss.r, sc)})
+	}
 }
 
 func (ss *stubServer) waitForReady(cc *grpc.ClientConn) error {

From a1ace9105a34ac2857f94bc8fb3436d13fc219cb Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 22 Jul 2020 14:36:22 -0700
Subject: [PATCH 127/481] xdsrouting: resolver to generate service config with
 routes, and pick routing balancer (#3751)

---
 xds/internal/resolver/serviceconfig.go        |  93 ++++-
 xds/internal/resolver/serviceconfig_action.go | 186 +++++++++
 .../resolver/serviceconfig_action_test.go     | 356 ++++++++++++++++++
 xds/internal/resolver/serviceconfig_test.go   | 321 +++++++++++++++-
 xds/internal/resolver/xds_resolver.go         |  12 +-
 xds/internal/resolver/xds_resolver_test.go    |  13 +
 6 files changed, 965 insertions(+), 16 deletions(-)
 create mode 100644 xds/internal/resolver/serviceconfig_action.go
 create mode 100644 xds/internal/resolver/serviceconfig_action_test.go

diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 6a55217e7a87..84c5753adf61 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -22,12 +22,14 @@ import (
 	"encoding/json"
 	"fmt"
 
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
 const (
 	cdsName            = "cds_experimental"
 	weightedTargetName = "weighted_target_experimental"
+	xdsRoutingName     = "xds_routing_experimental"
 )
 
 type serviceConfig struct {
@@ -53,24 +55,91 @@ type cdsBalancerConfig struct {
 	Cluster string `json:"cluster"`
 }
 
-func serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) {
+type route struct {
+	Path     *string                    `json:"path,omitempty"`
+	Prefix   *string                    `json:"prefix,omitempty"`
+	Regex    *string                    `json:"regex,omitempty"`
+	Headers  []*xdsclient.HeaderMatcher `json:"headers,omitempty"`
+	Fraction *wrapperspb.UInt32Value    `json:"matchFraction,omitempty"`
+	Action   string                     `json:"action"`
+}
+
+type xdsActionConfig struct {
+	ChildPolicy balancerConfig `json:"childPolicy"`
+}
+
+type xdsRoutingBalancerConfig struct {
+	Action map[string]xdsActionConfig `json:"action"`
+	Route  []*route                   `json:"route"`
+}
+
+func (r *xdsResolver) routesToJSON(routes []*xdsclient.Route) (string, error) {
+	r.updateActions(newActionsFromRoutes(routes))
+
+	// Generate routes.
+	var rts []*route
+	for _, rt := range routes {
+		t := &route{
+			Path:    rt.Path,
+			Prefix:  rt.Prefix,
+			Regex:   rt.Regex,
+			Headers: rt.Headers,
+		}
+
+		if f := rt.Fraction; f != nil {
+			t.Fraction = &wrapperspb.UInt32Value{Value: *f}
+		}
+
+		t.Action = r.getActionAssignedName(rt.Action)
+		rts = append(rts, t)
+	}
+
+	// Generate actions.
+	action := make(map[string]xdsActionConfig)
+	for _, act := range r.actions {
+		action[act.assignedName] = xdsActionConfig{
+			ChildPolicy: weightedClusterToBalancerConfig(act.clustersWithWeights),
+		}
+	}
+
+	sc := serviceConfig{
+		LoadBalancingConfig: newBalancerConfig(
+			xdsRoutingName, xdsRoutingBalancerConfig{
+				Route:  rts,
+				Action: action,
+			},
+		),
+	}
+
+	bs, err := json.Marshal(sc)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal json: %v", err)
+	}
+	return string(bs), nil
+}
+
+func weightedClusterToBalancerConfig(wc map[string]uint32) balancerConfig {
 	// Even if WeightedCluster has only one entry, we still use weighted_target
 	// as top level balancer, to avoid switching top policy between CDS and
 	// weighted_target, causing TCP connection to be recreated.
 	targets := make(map[string]cdsWithWeight)
-	for name, weight := range su.WeightedCluster {
+	for name, weight := range wc {
 		targets[name] = cdsWithWeight{
 			Weight:      weight,
 			ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: name}),
 		}
 	}
+	bc := newBalancerConfig(
+		weightedTargetName, weightedCDSBalancerConfig{
+			Targets: targets,
+		},
+	)
+	return bc
+}
 
+func weightedClusterToJSON(wc map[string]uint32) (string, error) {
 	sc := serviceConfig{
-		LoadBalancingConfig: newBalancerConfig(
-			weightedTargetName, weightedCDSBalancerConfig{
-				Targets: targets,
-			},
-		),
+		LoadBalancingConfig: weightedClusterToBalancerConfig(wc),
 	}
 	bs, err := json.Marshal(sc)
 	if err != nil {
@@ -78,3 +147,13 @@ func serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) {
 	}
 	return string(bs), nil
 }
+
+func (r *xdsResolver) serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) {
+	// If WeightedClusters is set, routing is disabled (by env variable). Use
+	// weighted target only.
+	if su.WeightedCluster != nil {
+		return weightedClusterToJSON(su.WeightedCluster)
+	}
+
+	return r.routesToJSON(su.Routes)
+}
diff --git a/xds/internal/resolver/serviceconfig_action.go b/xds/internal/resolver/serviceconfig_action.go
new file mode 100644
index 000000000000..d582048fda09
--- /dev/null
+++ b/xds/internal/resolver/serviceconfig_action.go
@@ -0,0 +1,186 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+
+	"google.golang.org/grpc/internal/grpcrand"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+type actionWithAssignedName struct {
+	// cluster:weight, "A":40, "B":60
+	clustersWithWeights map[string]uint32
+	// clusterNames, without weights, sorted and hashed, "A_B_"
+	clusterNames string
+	// The assigned name, clusters plus a random number, "A_B_1"
+	assignedName string
+	// randomNumber is the number appended to assignedName.
+	randomNumber int64
+}
+
+// newActionsFromRoutes gets actions from the routes, and turns them into a map
+// keyed by the hash of the clusters.
+//
+// In the returned map, all actions don't have assignedName. The assignedName
+// will be filled in after comparing the new actions with the existing actions,
+// so when a new and old action only diff in weights, the new action can reuse
+// the old action's name.
+//
+// from
+//   {B:60, A:40}, {A:30, B:70}, {B:90, C:10}
+//
+// to
+//   A40_B60_: {{A:40, B:60}, "A_B_", ""}
+//   A30_B70_: {{A:30, B:70}, "A_B_", ""}
+//   B90_C10_: {{B:90, C:10}, "B_C_", ""}
+func newActionsFromRoutes(routes []*xdsclient.Route) map[string]actionWithAssignedName {
+	newActions := make(map[string]actionWithAssignedName)
+	for _, route := range routes {
+		var clusterNames []string
+		for n := range route.Action {
+			clusterNames = append(clusterNames, n)
+		}
+
+		// Sort names to be consistent.
+		sort.Strings(clusterNames)
+		clustersOnly := ""
+		clustersWithWeight := ""
+		for _, c := range clusterNames {
+			// Generates A_B_
+			clustersOnly = clustersOnly + c + "_"
+			// Generates A40_B60_
+			clustersWithWeight = clustersWithWeight + c + strconv.FormatUint(uint64(route.Action[c]), 10) + "_"
+		}
+
+		if _, ok := newActions[clustersWithWeight]; !ok {
+			newActions[clustersWithWeight] = actionWithAssignedName{
+				clustersWithWeights: route.Action,
+				clusterNames:        clustersOnly,
+			}
+		}
+	}
+	return newActions
+}
+
+// updateActions takes a new map of actions, and updates the existing action map in the resolver.
+//
+// In the old map, all actions have assignedName set.
+// In the new map, all actions have no assignedName.
+//
+// After the update, the action map is updated to have all actions from the new
+// map, with assignedName:
+// - if the new action exists in old, get the old name
+// - if the new action doesn't exist in old
+//   - if there is an old action that will be removed, and has the same set of
+//   clusters, reuse the old action's name
+//   - otherwise, generate a new name
+func (r *xdsResolver) updateActions(newActions map[string]actionWithAssignedName) {
+	if r.actions == nil {
+		r.actions = make(map[string]actionWithAssignedName)
+	}
+
+	// Delete actions from existingActions if they are not in newActions. Keep
+	// the removed actions in a map, with key as clusterNames without weights,
+	// so their assigned names can be reused.
+	existingActions := r.actions
+	actionsRemoved := make(map[string][]string)
+	for actionHash, act := range existingActions {
+		if _, ok := newActions[actionHash]; !ok {
+			actionsRemoved[act.clusterNames] = append(actionsRemoved[act.clusterNames], act.assignedName)
+			delete(existingActions, actionHash)
+		}
+	}
+
+	// Find actions in newActions but not in oldActions. Add them, and try to
+	// reuse assigned names from actionsRemoved.
+	if r.usedActionNameRandomNumber == nil {
+		r.usedActionNameRandomNumber = make(map[int64]bool)
+	}
+	for actionHash, act := range newActions {
+		if _, ok := existingActions[actionHash]; !ok {
+			if assignedNamed, ok := actionsRemoved[act.clusterNames]; ok {
+				// Reuse the first assigned name from actionsRemoved.
+				act.assignedName = assignedNamed[0]
+				// If there are more names to reuse after this, update the slice
+				// in the map. Otherwise, remove the entry from the map.
+				if len(assignedNamed) > 1 {
+					actionsRemoved[act.clusterNames] = assignedNamed[1:]
+				} else {
+					delete(actionsRemoved, act.clusterNames)
+				}
+				existingActions[actionHash] = act
+				continue
+			}
+			// Generate a new name.
+			act.randomNumber = r.nextAssignedNameRandomNumber()
+			act.assignedName = fmt.Sprintf("%s%d", act.clusterNames, act.randomNumber)
+			existingActions[actionHash] = act
+		}
+	}
+
+	// Delete entry from nextIndex if all actions with the clusters are removed.
+	remainingRandomNumbers := make(map[int64]bool)
+	for _, act := range existingActions {
+		remainingRandomNumbers[act.randomNumber] = true
+	}
+	r.usedActionNameRandomNumber = remainingRandomNumbers
+}
+
+var grpcrandInt63n = grpcrand.Int63n
+
+func (r *xdsResolver) nextAssignedNameRandomNumber() int64 {
+	for {
+		t := grpcrandInt63n(math.MaxInt32)
+		if !r.usedActionNameRandomNumber[t] {
+			return t
+		}
+	}
+}
+
+// getActionAssignedName hashes the clusters from the action, and find the
+// assigned action name. The assigned action names are kept in r.actions, with
+// the clusters name hash as map key.
+//
+// The assigned action name is not simply the hash. For example, the hash can be
+// "A40_B60_", but the assigned name can be "A_B_0". It's this way so the action
+// can be reused if only weights are changing.
+func (r *xdsResolver) getActionAssignedName(action map[string]uint32) string {
+	var clusterNames []string
+	for n := range action {
+		clusterNames = append(clusterNames, n)
+	}
+	// Hash cluster names. Sort names to be consistent.
+	sort.Strings(clusterNames)
+	clustersWithWeight := ""
+	for _, c := range clusterNames {
+		// Generates hash "A40_B60_".
+		clustersWithWeight = clustersWithWeight + c + strconv.FormatUint(uint64(action[c]), 10) + "_"
+	}
+	// Look in r.actions for the assigned action name.
+	if act, ok := r.actions[clustersWithWeight]; ok {
+		return act.assignedName
+	}
+	r.logger.Warningf("no assigned name found for action %v", action)
+	return ""
+}
diff --git a/xds/internal/resolver/serviceconfig_action_test.go b/xds/internal/resolver/serviceconfig_action_test.go
new file mode 100644
index 000000000000..bfc0e6830155
--- /dev/null
+++ b/xds/internal/resolver/serviceconfig_action_test.go
@@ -0,0 +1,356 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+func TestNewActionsFromRoutes(t *testing.T) {
+	tests := []struct {
+		name   string
+		routes []*xdsclient.Route
+		want   map[string]actionWithAssignedName
+	}{
+		{
+			name: "temp",
+			routes: []*xdsclient.Route{
+				{Action: map[string]uint32{"B": 60, "A": 40}},
+				{Action: map[string]uint32{"A": 30, "B": 70}},
+				{Action: map[string]uint32{"B": 90, "C": 10}},
+			},
+			want: map[string]actionWithAssignedName{
+				"A40_B60_": {map[string]uint32{"A": 40, "B": 60}, "A_B_", "", 0},
+				"A30_B70_": {map[string]uint32{"A": 30, "B": 70}, "A_B_", "", 0},
+				"B90_C10_": {map[string]uint32{"B": 90, "C": 10}, "B_C_", "", 0},
+			},
+		},
+	}
+
+	cmpOpts := []cmp.Option{cmp.AllowUnexported(actionWithAssignedName{})}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := newActionsFromRoutes(tt.routes); !cmp.Equal(got, tt.want, cmpOpts...) {
+				t.Errorf("newActionsFromRoutes() got unexpected result, diff %v", cmp.Diff(got, tt.want, cmpOpts...))
+			}
+		})
+	}
+}
+
+func TestRemoveOrReuseName(t *testing.T) {
+	tests := []struct {
+		name         string
+		oldActions   map[string]actionWithAssignedName
+		oldRandNums  map[int64]bool
+		newActions   map[string]actionWithAssignedName
+		wantActions  map[string]actionWithAssignedName
+		wantRandNums map[int64]bool
+	}{
+		{
+			name: "add same cluster",
+			oldActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+			},
+			oldRandNums: map[int64]bool{
+				0: true,
+			},
+			newActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+				},
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+				},
+			},
+			wantActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_1000",
+					randomNumber:        1000,
+				},
+			},
+			wantRandNums: map[int64]bool{
+				0:    true,
+				1000: true,
+			},
+		},
+		{
+			name: "delete same cluster",
+			oldActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_1",
+					randomNumber:        1,
+				},
+			},
+			oldRandNums: map[int64]bool{
+				0: true,
+				1: true,
+			},
+			newActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+				},
+			},
+			wantActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+			},
+			wantRandNums: map[int64]bool{
+				0: true,
+			},
+		},
+		{
+			name: "add new clusters",
+			oldActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+			},
+			oldRandNums: map[int64]bool{
+				0: true,
+			},
+			newActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+				},
+				"a50_b50_": {
+					clustersWithWeights: map[string]uint32{"a": 50, "b": 50},
+					clusterNames:        "a_b_",
+				},
+			},
+			wantActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+				"a50_b50_": {
+					clustersWithWeights: map[string]uint32{"a": 50, "b": 50},
+					clusterNames:        "a_b_",
+					assignedName:        "a_b_1000",
+					randomNumber:        1000,
+				},
+			},
+			wantRandNums: map[int64]bool{
+				0:    true,
+				1000: true,
+			},
+		},
+		{
+			name: "reuse",
+			oldActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+			},
+			oldRandNums: map[int64]bool{
+				0: true,
+			},
+			newActions: map[string]actionWithAssignedName{
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+				},
+			},
+			wantActions: map[string]actionWithAssignedName{
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+			},
+			wantRandNums: map[int64]bool{
+				0: true,
+			},
+		},
+		{
+			name: "add and reuse",
+			oldActions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_1",
+					randomNumber:        1,
+				},
+				"a50_b50_": {
+					clustersWithWeights: map[string]uint32{"a": 50, "b": 50},
+					clusterNames:        "a_b_",
+					assignedName:        "a_b_2",
+					randomNumber:        2,
+				},
+			},
+			oldRandNums: map[int64]bool{
+				0: true,
+				1: true,
+				2: true,
+			},
+			newActions: map[string]actionWithAssignedName{
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+				},
+				"a30_b30_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 30, "b": 30, "c": 40},
+					clusterNames:        "a_b_c_",
+				},
+				"c50_d50_": {
+					clustersWithWeights: map[string]uint32{"c": 50, "d": 50},
+					clusterNames:        "c_d_",
+				},
+			},
+			wantActions: map[string]actionWithAssignedName{
+				"a10_b50_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_1",
+					randomNumber:        1,
+				},
+				"a30_b30_c40_": {
+					clustersWithWeights: map[string]uint32{"a": 30, "b": 30, "c": 40},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+					randomNumber:        0,
+				},
+				"c50_d50_": {
+					clustersWithWeights: map[string]uint32{"c": 50, "d": 50},
+					clusterNames:        "c_d_",
+					assignedName:        "c_d_1000",
+					randomNumber:        1000,
+				},
+			},
+			wantRandNums: map[int64]bool{
+				0:    true,
+				1:    true,
+				1000: true,
+			},
+		},
+	}
+	cmpOpts := []cmp.Option{cmp.AllowUnexported(actionWithAssignedName{})}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			defer replaceRandNumGenerator(1000)()
+			r := &xdsResolver{
+				actions:                    tt.oldActions,
+				usedActionNameRandomNumber: tt.oldRandNums,
+			}
+			r.updateActions(tt.newActions)
+			if !cmp.Equal(r.actions, tt.wantActions, cmpOpts...) {
+				t.Errorf("removeOrReuseName() got unexpected actions, diff %v", cmp.Diff(r.actions, tt.wantActions, cmpOpts...))
+			}
+			if !cmp.Equal(r.usedActionNameRandomNumber, tt.wantRandNums) {
+				t.Errorf("removeOrReuseName() got unexpected nextIndex, diff %v", cmp.Diff(r.usedActionNameRandomNumber, tt.wantRandNums))
+			}
+		})
+	}
+}
+
+func TestGetActionAssignedName(t *testing.T) {
+	tests := []struct {
+		name    string
+		actions map[string]actionWithAssignedName
+		action  map[string]uint32
+		want    string
+	}{
+		{
+			name: "good",
+			actions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+				},
+			},
+			action: map[string]uint32{"a": 20, "b": 30, "c": 50},
+			want:   "a_b_c_0",
+		},
+		{
+			name: "two",
+			actions: map[string]actionWithAssignedName{
+				"a20_b30_c50_": {
+					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
+					clusterNames:        "a_b_c_",
+					assignedName:        "a_b_c_0",
+				},
+				"c50_d50_": {
+					clustersWithWeights: map[string]uint32{"c": 50, "d": 50},
+					clusterNames:        "c_d_",
+					assignedName:        "c_d_0",
+				},
+			},
+			action: map[string]uint32{"c": 50, "d": 50},
+			want:   "c_d_0",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &xdsResolver{
+				actions: tt.actions,
+			}
+			if got := r.getActionAssignedName(tt.action); got != tt.want {
+				t.Errorf("getActionAssignedName() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index 6e83a23cdd5d..4e149893ee70 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -23,9 +23,12 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/serviceconfig"
 	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
+	_ "google.golang.org/grpc/xds/internal/balancer/xdsrouting"
 	"google.golang.org/grpc/xds/internal/client"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
 const (
@@ -34,7 +37,7 @@ const (
     "weighted_target_experimental": {
 	  "targets": { "test-cluster-1" : { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}] } }
 	}
-}]}`
+    }]}`
 	testWeightedCDSJSON = `{"loadBalancingConfig":[{
     "weighted_target_experimental": {
 	  "targets": {
@@ -48,30 +51,303 @@ const (
 		}
 	  }
 	}
-}]}`
+    }]}`
 	testWeightedCDSNoChildJSON = `{"loadBalancingConfig":[{
     "weighted_target_experimental": {
 	  "targets": {}
 	}
-}]}`
+    }]}`
+	testRoutingJSON = `{"loadBalancingConfig":[{
+    "xds_routing_experimental": {
+      "action":{
+        "cluster_1_cluster_2_0":{
+          "childPolicy":[{
+			"weighted_target_experimental": {
+			  "targets": {
+				"cluster_1" : {
+				  "weight":75,
+				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+				},
+				"cluster_2" : {
+				  "weight":25,
+				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+				}
+			  }
+			}
+          }]
+        }
+      },
+
+      "route":[{
+        "path":"/service_1/method_1",
+        "action":"cluster_1_cluster_2_0"
+      }]
+    }
+    }]}
+`
+	testRoutingAllMatchersJSON = `{"loadBalancingConfig":[{
+    "xds_routing_experimental": {
+      "action":{
+        "cluster_1_0":{
+          "childPolicy":[{
+			"weighted_target_experimental": {
+			  "targets": {
+				"cluster_1" : {
+				  "weight":1,
+				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+				}
+			  }
+			}
+          }]
+        },
+        "cluster_2_0":{
+          "childPolicy":[{
+			"weighted_target_experimental": {
+			  "targets": {
+				"cluster_2" : {
+				  "weight":1,
+				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+				}
+			  }
+			}
+          }]
+        },
+        "cluster_3_0":{
+          "childPolicy":[{
+			"weighted_target_experimental": {
+			  "targets": {
+				"cluster_3" : {
+				  "weight":1,
+				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
+				}
+			  }
+			}
+          }]
+        }
+      },
+
+      "route":[{
+        "path":"/service_1/method_1",
+        "action":"cluster_1_0"
+      },
+      {
+        "prefix":"/service_2/method_1",
+        "action":"cluster_1_0"
+      },
+      {
+        "regex":"^/service_2/method_3$",
+        "action":"cluster_1_0"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "exactMatch":"value-1", "invertMatch":true}],
+        "action":"cluster_2_0"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "regexMatch":"^value-1$"}],
+        "action":"cluster_2_0"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "rangeMatch":{"start":-1, "end":7}}],
+        "action":"cluster_3_0"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "presentMatch":true}],
+        "action":"cluster_3_0"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "prefixMatch":"value-1"}],
+        "action":"cluster_2_0"
+      },
+      {
+        "prefix":"",
+        "headers":[{"name":"header-1", "suffixMatch":"value-1"}],
+        "action":"cluster_2_0"
+      },
+      {
+        "prefix":"",
+        "matchFraction":{"value": 31415},
+        "action":"cluster_3_0"
+      }]
+    }
+    }]}
+`
 )
 
-func TestServiceUpdateToJSON(t *testing.T) {
+func TestWeightedClusterToJSON(t *testing.T) {
 	tests := []struct {
 		name     string
-		su       client.ServiceUpdate
+		wc       map[string]uint32
 		wantJSON string // wantJSON is not to be compared verbatim.
 	}{
 		{
 			name:     "one cluster only",
-			su:       client.ServiceUpdate{WeightedCluster: map[string]uint32{testCluster1: 1}},
+			wc:       map[string]uint32{testCluster1: 1},
 			wantJSON: testClusterOnlyJSON,
 		},
 		{
 			name:     "empty weighted clusters",
-			su:       client.ServiceUpdate{WeightedCluster: nil},
+			wc:       nil,
 			wantJSON: testWeightedCDSNoChildJSON,
 		},
+		{
+			name: "weighted clusters",
+			wc: map[string]uint32{
+				"cluster_1": 75,
+				"cluster_2": 25,
+			},
+			wantJSON: testWeightedCDSJSON,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotJSON, err := weightedClusterToJSON(tt.wc)
+			if err != nil {
+				t.Errorf("serviceUpdateToJSON returned error: %v", err)
+				return
+			}
+
+			gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON)
+			wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
+
+			if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
+				t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON)
+				t.Error("gotParsed: ", cmp.Diff(nil, gotParsed))
+				t.Error("wantParsed: ", cmp.Diff(nil, wantParsed))
+			}
+		})
+	}
+}
+
+func TestRoutesToJSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		routes   []*xdsclient.Route
+		wantJSON string
+		wantErr  bool
+	}{
+		{
+			name: "one route",
+			routes: []*xdsclient.Route{{
+				Path:   newStringP("/service_1/method_1"),
+				Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25},
+			}},
+			wantJSON: testRoutingJSON,
+			wantErr:  false,
+		},
+		{
+			name: "all matchers",
+			routes: []*xdsclient.Route{
+				{
+					Path:   newStringP("/service_1/method_1"),
+					Action: map[string]uint32{"cluster_1": 1},
+				},
+				{
+					Prefix: newStringP("/service_2/method_1"),
+					Action: map[string]uint32{"cluster_1": 1},
+				},
+				{
+					Regex:  newStringP("^/service_2/method_3$"),
+					Action: map[string]uint32{"cluster_1": 1},
+				},
+				{
+					Prefix: newStringP(""),
+					Headers: []*xdsclient.HeaderMatcher{{
+						Name:        "header-1",
+						InvertMatch: newBoolP(true),
+						ExactMatch:  newStringP("value-1"),
+					}},
+					Action: map[string]uint32{"cluster_2": 1},
+				},
+				{
+					Prefix: newStringP(""),
+					Headers: []*xdsclient.HeaderMatcher{{
+						Name:       "header-1",
+						RegexMatch: newStringP("^value-1$"),
+					}},
+					Action: map[string]uint32{"cluster_2": 1},
+				},
+				{
+					Prefix: newStringP(""),
+					Headers: []*xdsclient.HeaderMatcher{{
+						Name:       "header-1",
+						RangeMatch: &xdsclient.Int64Range{Start: -1, End: 7},
+					}},
+					Action: map[string]uint32{"cluster_3": 1},
+				},
+				{
+					Prefix: newStringP(""),
+					Headers: []*xdsclient.HeaderMatcher{{
+						Name:         "header-1",
+						PresentMatch: newBoolP(true),
+					}},
+					Action: map[string]uint32{"cluster_3": 1},
+				},
+				{
+					Prefix: newStringP(""),
+					Headers: []*xdsclient.HeaderMatcher{{
+						Name:        "header-1",
+						PrefixMatch: newStringP("value-1"),
+					}},
+					Action: map[string]uint32{"cluster_2": 1},
+				},
+				{
+					Prefix: newStringP(""),
+					Headers: []*xdsclient.HeaderMatcher{{
+						Name:        "header-1",
+						SuffixMatch: newStringP("value-1"),
+					}},
+					Action: map[string]uint32{"cluster_2": 1},
+				},
+				{
+					Prefix:   newStringP(""),
+					Fraction: newUint32P(31415),
+					Action:   map[string]uint32{"cluster_3": 1},
+				},
+			},
+			wantJSON: testRoutingAllMatchersJSON,
+			wantErr:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Note this random number function only generates 0. This is
+			// because the test doesn't handle action update, and there's only
+			// one action for each cluster bundle.
+			//
+			// This is necessary so the output is deterministic.
+			grpcrandInt63n = func(int64) int64 { return 0 }
+			defer func() { grpcrandInt63n = grpcrand.Int63n }()
+
+			gotJSON, err := (&xdsResolver{}).routesToJSON(tt.routes)
+			if err != nil {
+				t.Errorf("routesToJSON returned error: %v", err)
+				return
+			}
+
+			gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON)
+			wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
+
+			if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
+				t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON)
+				t.Error("gotParsed: ", cmp.Diff(nil, gotParsed))
+				t.Error("wantParsed: ", cmp.Diff(nil, wantParsed))
+			}
+		})
+	}
+}
+
+func TestServiceUpdateToJSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		su       client.ServiceUpdate
+		wantJSON string
+		wantErr  bool
+	}{
 		{
 			name: "weighted clusters",
 			su: client.ServiceUpdate{WeightedCluster: map[string]uint32{
@@ -79,11 +355,24 @@ func TestServiceUpdateToJSON(t *testing.T) {
 				"cluster_2": 25,
 			}},
 			wantJSON: testWeightedCDSJSON,
+			wantErr:  false,
+		},
+		{
+			name: "routing",
+			su: client.ServiceUpdate{
+				Routes: []*xdsclient.Route{{
+					Path:   newStringP("/service_1/method_1"),
+					Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25},
+				}},
+			},
+			wantJSON: testRoutingJSON,
+			wantErr:  false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotJSON, err := serviceUpdateToJSON(tt.su)
+			defer replaceRandNumGenerator(0)()
+			gotJSON, err := (&xdsResolver{}).serviceUpdateToJSON(tt.su)
 			if err != nil {
 				t.Errorf("serviceUpdateToJSON returned error: %v", err)
 				return
@@ -100,3 +389,19 @@ func TestServiceUpdateToJSON(t *testing.T) {
 		})
 	}
 }
+
+// Two updates to the same resolver, test that action names are reused.
+func TestServiceUpdateToJSON_TwoConfig_UpdateActions(t *testing.T) {
+}
+
+func newStringP(s string) *string {
+	return &s
+}
+
+func newBoolP(b bool) *bool {
+	return &b
+}
+
+func newUint32P(i uint32) *uint32 {
+	return &i
+}
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index fb46a91603c2..cdd103ef7dc3 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -159,6 +159,16 @@ type xdsResolver struct {
 	updateCh chan suWithError
 	// cancelWatch is the function to cancel the watcher.
 	cancelWatch func()
+
+	// actions is a map from hash of weighted cluster, to the weighted cluster
+	// map, and it's assigned name. E.g.
+	//   "A40_B60_": {{A:40, B:60}, "A_B_", "A_B_0"}
+	//   "A30_B70_": {{A:30, B:70}, "A_B_", "A_B_1"}
+	//   "B90_C10_": {{B:90, C:10}, "B_C_", "B_C_0"}
+	actions map[string]actionWithAssignedName
+	// usedActionNameRandomNumber contains random numbers that have been used in
+	// assigned names, to avoid collision.
+	usedActionNameRandomNumber map[int64]bool
 }
 
 // run is a long running goroutine which blocks on receiving service updates
@@ -185,7 +195,7 @@ func (r *xdsResolver) run() {
 				r.cc.ReportError(update.err)
 				continue
 			}
-			sc, err := serviceUpdateToJSON(update.su)
+			sc, err := r.serviceUpdateToJSON(update.su)
 			if err != nil {
 				r.logger.Warningf("failed to convert update to service config: %v", err)
 				r.cc.ReportError(err)
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 025a8b2dbc1a..d84ec44eb8c0 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	xdsinternal "google.golang.org/grpc/xds/internal"
@@ -446,3 +447,15 @@ func TestXDSResolverResourceNotFoundError(t *testing.T) {
 		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
 	}
 }
+
+func replaceRandNumGenerator(start int64) func() {
+	nextInt := start
+	grpcrandInt63n = func(int64) (ret int64) {
+		ret = nextInt
+		nextInt++
+		return
+	}
+	return func() {
+		grpcrandInt63n = grpcrand.Int63n
+	}
+}

From cee815dbe38c345e07496b0943582b987e0889a8 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Thu, 23 Jul 2020 09:01:20 -0700
Subject: [PATCH 128/481] Added support for proto3 field presence (#3752)

---
 cmd/protoc-gen-go-grpc/main.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index a8555fad23a1..9165761da6ff 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -34,6 +34,7 @@ import (
 	"flag"
 
 	"google.golang.org/protobuf/compiler/protogen"
+	"google.golang.org/protobuf/types/pluginpb"
 )
 
 var requireUnimplemented *bool
@@ -45,6 +46,7 @@ func main() {
 	protogen.Options{
 		ParamFunc: flags.Set,
 	}.Run(func(gen *protogen.Plugin) error {
+		gen.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL)
 		for _, f := range gen.Files {
 			if !f.Generate {
 				continue

From 9106c3fff5236fd664a8de183f1c27682c66b823 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 23 Jul 2020 11:26:53 -0700
Subject: [PATCH 129/481] Change version to 1.32.0-dev (#3757)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index 60b4b0d52b31..674c12f9a7e1 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.31.0-dev"
+const Version = "1.32.0-dev"

From 7408372e0e76a7cc1655d5e0397fd730ce16b9aa Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 27 Jul 2020 13:55:02 -0700
Subject: [PATCH 130/481] xdsrouting: remove env variable for routing (#3754)

---
 xds/internal/client/bootstrap/bootstrap.go    |   4 +
 xds/internal/client/client_watchers_rds.go    |   4 -
 .../client/client_watchers_rds_test.go        |  10 +-
 .../client/client_watchers_service.go         |  12 +-
 .../client/client_watchers_service_test.go    |  42 +++----
 xds/internal/client/envconfig.go              |  33 -----
 xds/internal/client/v2client_rds.go           |  48 +-------
 xds/internal/client/v2client_rds_test.go      |  80 ++++--------
 xds/internal/resolver/serviceconfig.go        |  17 ---
 xds/internal/resolver/serviceconfig_test.go   | 115 ++++++------------
 xds/internal/resolver/xds_resolver_test.go    |  13 +-
 11 files changed, 103 insertions(+), 275 deletions(-)
 delete mode 100644 xds/internal/client/envconfig.go

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 1e2e05e8f9b2..b2805bf73720 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -195,6 +195,10 @@ func NewConfig() (*Config, error) {
 	// 2. Environment variable "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" is set to
 	//    true.
 	// The default value of the enum type "version.TransportAPI" is v2.
+	//
+	// TODO: there are multiple env variables, GRPC_XDS_BOOTSTRAP and
+	// GRPC_XDS_EXPERIMENTAL_V3_SUPPORT. Move all env variables into a separate
+	// package.
 	if v3Env := os.Getenv(v3SupportEnv); v3Env == "true" {
 		if serverSupportsV3 {
 			config.TransportAPI = version.TransportV3
diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
index 91de78601b49..cc1b18c2d915 100644
--- a/xds/internal/client/client_watchers_rds.go
+++ b/xds/internal/client/client_watchers_rds.go
@@ -49,10 +49,6 @@ type Route struct {
 }
 
 type rdsUpdate struct {
-	// weightedCluster is only set when routing is disabled (env variable
-	// GRPC_XDS_EXPERIMENTAL_ROUTING is not true).
-	weightedCluster map[string]uint32
-
 	routes []*Route
 }
 type rdsCallbackFunc func(rdsUpdate, error)
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 06ed7a377e2b..16e042ecd961 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -54,7 +54,7 @@ func (s) TestRDSWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: wantUpdate,
 	})
@@ -113,7 +113,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 		}
 	}
 
-	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: wantUpdate,
 	})
@@ -179,8 +179,8 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := rdsUpdate{weightedCluster: map[string]uint32{testCDSName + "1": 1}}
-	wantUpdate2 := rdsUpdate{weightedCluster: map[string]uint32{testCDSName + "2": 1}}
+	wantUpdate1 := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}}}
+	wantUpdate2 := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName + "1": wantUpdate1,
 		testRDSName + "2": wantUpdate2,
@@ -219,7 +219,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := rdsUpdate{weightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
 		testRDSName: wantUpdate,
 	})
diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
index 1cf4c0f988c1..cc96622d71d3 100644
--- a/xds/internal/client/client_watchers_service.go
+++ b/xds/internal/client/client_watchers_service.go
@@ -25,14 +25,7 @@ import (
 
 // ServiceUpdate contains update about the service.
 type ServiceUpdate struct {
-	// WeightedCluster is a map from cluster names (CDS resource to watch) to
-	// their weights.
-	//
-	// This field is only set when routing is disabled (env variable
-	// GRPC_XDS_EXPERIMENTAL_ROUTING is not true).
-	WeightedCluster map[string]uint32
-
-	// Routes
+	// Routes contain matchers+actions to route RPCs.
 	Routes []*Route
 }
 
@@ -126,8 +119,7 @@ func (w *serviceUpdateWatcher) handleRDSResp(update rdsUpdate, err error) {
 		return
 	}
 	w.serviceCb(ServiceUpdate{
-		WeightedCluster: update.weightedCluster,
-		Routes:          update.routes,
+		Routes: update.routes,
 	}, nil)
 }
 
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 4535285bd6b4..4b72f63e8ee6 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -58,7 +58,7 @@ func (s) TestServiceWatch(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
 	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
@@ -70,7 +70,7 @@ func (s) TestServiceWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -116,7 +116,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
 	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
@@ -128,7 +128,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -145,17 +145,17 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	// Another update for the old name.
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
-	wantUpdate2 := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName + "2": 1}}
+	wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
 	// RDS update for the new name.
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName + "2": {weightedCluster: map[string]uint32{testCDSName + "2": 1}},
+		testRDSName + "2": {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
@@ -183,7 +183,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
 	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
@@ -195,7 +195,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -226,7 +226,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		testLDSName: {routeName: testRDSName},
 	})
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -263,8 +263,8 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.WeightedCluster != nil {
-			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.WeightedCluster))
+		if su.Routes != nil {
+			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.Routes))
 			return
 		}
 		if err == nil {
@@ -307,8 +307,8 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.WeightedCluster != nil {
-			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.WeightedCluster))
+		if su.Routes != nil {
+			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.Routes))
 			return
 		}
 		if err == nil {
@@ -394,7 +394,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
 	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
@@ -406,7 +406,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -445,7 +445,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	wantUpdate := ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName: 1}}
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
 	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
@@ -457,7 +457,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName: 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -477,7 +477,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	// Send RDS update for the removed LDS resource, expect no updates to
 	// callback, because RDS should be canceled.
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName + "new": 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}},
 	})
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
@@ -497,9 +497,9 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	}
 
 	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {weightedCluster: map[string]uint32{testCDSName + "new2": 1}},
+		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}},
 	})
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{WeightedCluster: map[string]uint32{testCDSName + "new2": 1}}, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
diff --git a/xds/internal/client/envconfig.go b/xds/internal/client/envconfig.go
deleted file mode 100644
index 40f448e63711..000000000000
--- a/xds/internal/client/envconfig.go
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"os"
-	"strings"
-)
-
-// TODO: there are multiple env variables, GRPC_XDS_BOOTSTRAP and
-// GRPC_XDS_EXPERIMENTAL_V3_SUPPORT, and this. Move all env variables into a
-// separate package.
-const routingEnabledConfigStr = "GRPC_XDS_EXPERIMENTAL_ROUTING"
-
-// routing is enabled only if env variable is set to true. The default is false.
-// We may flip the default later.
-var routingEnabled = strings.EqualFold(os.Getenv(routingEnabledConfigStr), "true")
diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
index 1fc9ac9752bf..cc71f6538f62 100644
--- a/xds/internal/client/v2client_rds.go
+++ b/xds/internal/client/v2client_rds.go
@@ -97,51 +97,11 @@ func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host
 		return rdsUpdate{}, fmt.Errorf("matched virtual host has no routes")
 	}
 
-	// Keep the old code path for routing disabled.
-	if routingEnabled {
-		routes, err := routesProtoToSlice(vh.Routes, logger)
-		if err != nil {
-			return rdsUpdate{}, fmt.Errorf("received route is invalid: %v", err)
-		}
-		return rdsUpdate{routes: routes}, nil
-	}
-
-	dr := vh.Routes[len(vh.Routes)-1]
-	match := dr.GetMatch()
-	if match == nil {
-		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route doesn't have a match")
-	}
-	if prefix := match.GetPrefix(); prefix != "" && prefix != "/" {
-		// The matched virtual host is invalid. Match is not "" or "/".
-		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route is %v, want Prefix empty string or /", match)
-	}
-	if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
-		// The case sensitive is set to false. Not set or set to true are both
-		// valid.
-		return rdsUpdate{}, fmt.Errorf("matched virtual host's default route set case-sensitive to false")
-	}
-	routeAction := dr.GetRoute()
-	if routeAction == nil {
-		return rdsUpdate{}, fmt.Errorf("matched route is nil")
+	routes, err := routesProtoToSlice(vh.Routes, logger)
+	if err != nil {
+		return rdsUpdate{}, fmt.Errorf("received route is invalid: %v", err)
 	}
-
-	if wc := routeAction.GetWeightedClusters(); wc != nil {
-		m, err := weightedClustersProtoToMap(wc)
-		if err != nil {
-			return rdsUpdate{}, fmt.Errorf("matched weighted cluster is invalid: %v", err)
-		}
-		return rdsUpdate{weightedCluster: m}, nil
-	}
-
-	// When there's just one cluster, we set weightedCluster to map with one
-	// entry. This mean we will build a weighted_target balancer even if there's
-	// just one cluster.
-	//
-	// Otherwise, we will need to switch the top policy between weighted_target
-	// and CDS. In case when the action changes between one cluster and multiple
-	// clusters, changing top level policy means recreating TCP connection every
-	// time.
-	return rdsUpdate{weightedCluster: map[string]uint32{routeAction.GetCluster(): 1}}, nil
+	return rdsUpdate{routes: routes}, nil
 }
 
 func routesProtoToSlice(routes []*routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) {
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
index e3ec4e01fd8f..3e5ef96fd6ba 100644
--- a/xds/internal/client/v2client_rds_test.go
+++ b/xds/internal/client/v2client_rds_test.go
@@ -158,7 +158,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 		{
 			name:       "good-route-config-with-empty-string-route",
 			rc:         goodRouteConfig1,
-			wantUpdate: rdsUpdate{weightedCluster: map[string]uint32{goodClusterName1: 1}},
+			wantUpdate: rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}},
 		},
 		{
 			// default route's match is not empty string, but "/".
@@ -173,7 +173,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 							Route: &routepb.RouteAction{
 								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
 							}}}}}}},
-			wantUpdate: rdsUpdate{weightedCluster: map[string]uint32{goodClusterName1: 1}},
+			wantUpdate: rdsUpdate{routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{goodClusterName1: 1}}}},
 		},
 
 		{
@@ -217,7 +217,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 										},
 										TotalWeight: &wrapperspb.UInt32Value{Value: 10},
 									}}}}}}}}},
-			wantUpdate: rdsUpdate{weightedCluster: map[string]uint32{"a": 2, "b": 3, "c": 5}},
+			wantUpdate: rdsUpdate{routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}}},
 		},
 	}
 
@@ -243,59 +243,10 @@ func doLDS(t *testing.T, v2c *v2Client, fakeServer *fakeserver.Server) {
 	}
 }
 
-// TestRDSHandleResponseWithRoutingEnabled starts a fake xDS server, makes a
-// ClientConn to it, and creates a v2Client using it. Then, it registers an LDS
-// and RDS watcher and tests different RDS responses.
-//
-// Routing is protected by an env variable. This test sets it to true, so the
-// new fields will be parsed.
-func (s) TestRDSHandleResponseWithRoutingEnabled(t *testing.T) {
-	routingEnabled = true
-	defer func() {
-		routingEnabled = false
-	}()
-	tests := []struct {
-		name          string
-		rdsResponse   *xdspb.DiscoveryResponse
-		wantErr       bool
-		wantUpdate    *rdsUpdate
-		wantUpdateErr bool
-	}{
-		// Response contains one good interesting RouteConfiguration.
-		{
-			name:        "one-good-route-config",
-			rdsResponse: goodRDSResponse1,
-			wantErr:     false,
-			wantUpdate: &rdsUpdate{
-				// Instead of just weighted targets when routing is disabled,
-				// this result contains a route with perfix "", and action as
-				// weighted targets.
-				routes: []*Route{{
-					Prefix: newStringP(""),
-					Action: map[string]uint32{goodClusterName1: 1},
-				}},
-			},
-			wantUpdateErr: false,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          rdsURL,
-				resourceName:     goodRouteName1,
-				responseToHandle: test.rdsResponse,
-				wantHandleErr:    test.wantErr,
-				wantUpdate:       test.wantUpdate,
-				wantUpdateErr:    test.wantUpdateErr,
-			})
-		})
-	}
-}
-
-// TestRDSHandleResponseWithRoutingDisabled starts a fake xDS server, makes a
-// ClientConn to it, and creates a v2Client using it. Then, it registers an LDS
-// and RDS watcher and tests different RDS responses.
-func (s) TestRDSHandleResponseWithRoutingDisabled(t *testing.T) {
+// TestRDSHandleResponseWithRouting starts a fake xDS server, makes a ClientConn
+// to it, and creates a v2Client using it. Then, it registers an LDS and RDS
+// watcher and tests different RDS responses.
+func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 	tests := []struct {
 		name          string
 		rdsResponse   *xdspb.DiscoveryResponse
@@ -342,7 +293,22 @@ func (s) TestRDSHandleResponseWithRoutingDisabled(t *testing.T) {
 			name:          "one-good-route-config",
 			rdsResponse:   goodRDSResponse1,
 			wantErr:       false,
-			wantUpdate:    &rdsUpdate{weightedCluster: map[string]uint32{goodClusterName1: 1}},
+			wantUpdate:    &rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}},
+			wantUpdateErr: false,
+		},
+		{
+			name:        "one-good-route-config with routes",
+			rdsResponse: goodRDSResponse1,
+			wantErr:     false,
+			wantUpdate: &rdsUpdate{
+				// Instead of just weighted targets when routing is disabled,
+				// this result contains a route with perfix "", and action as
+				// weighted targets.
+				routes: []*Route{{
+					Prefix: newStringP(""),
+					Action: map[string]uint32{goodClusterName1: 1},
+				}},
+			},
 			wantUpdateErr: false,
 		},
 	}
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 84c5753adf61..805d8d41104c 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -137,23 +137,6 @@ func weightedClusterToBalancerConfig(wc map[string]uint32) balancerConfig {
 	return bc
 }
 
-func weightedClusterToJSON(wc map[string]uint32) (string, error) {
-	sc := serviceConfig{
-		LoadBalancingConfig: weightedClusterToBalancerConfig(wc),
-	}
-	bs, err := json.Marshal(sc)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshal json: %v", err)
-	}
-	return string(bs), nil
-}
-
 func (r *xdsResolver) serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) {
-	// If WeightedClusters is set, routing is disabled (by env variable). Use
-	// weighted target only.
-	if su.WeightedCluster != nil {
-		return weightedClusterToJSON(su.WeightedCluster)
-	}
-
 	return r.routesToJSON(su.Routes)
 }
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index 4e149893ee70..ce4a7e8fab9c 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -32,31 +32,44 @@ import (
 )
 
 const (
-	testCluster1        = "test-cluster-1"
-	testClusterOnlyJSON = `{"loadBalancingConfig":[{
-    "weighted_target_experimental": {
-	  "targets": { "test-cluster-1" : { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}] } }
-	}
-    }]}`
+	testCluster1           = "test-cluster-1"
+	testOneClusterOnlyJSON = `{"loadBalancingConfig":[{
+    "xds_routing_experimental":{
+      "action":{
+        "test-cluster-1_0":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets":{
+                "test-cluster-1":{
+                  "weight":1,
+                  "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
+                }
+              }}}]
+        }
+      },
+      "route":[{"prefix":"","action":"test-cluster-1_0"}]
+    }}]}`
 	testWeightedCDSJSON = `{"loadBalancingConfig":[{
-    "weighted_target_experimental": {
-	  "targets": {
-		"cluster_1" : {
-		  "weight":75,
-		  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
-		},
-		"cluster_2" : {
-		  "weight":25,
-		  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
-		}
-	  }
-	}
-    }]}`
-	testWeightedCDSNoChildJSON = `{"loadBalancingConfig":[{
-    "weighted_target_experimental": {
-	  "targets": {}
-	}
-    }]}`
+    "xds_routing_experimental":{
+      "action":{
+        "cluster_1_cluster_2_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets":{
+                "cluster_1":{
+                  "weight":75,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_2":{
+                  "weight":25,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+                }
+              }}}]
+        }
+      },
+      "route":[{"prefix":"","action":"cluster_1_cluster_2_1"}]
+    }}]}`
+
 	testRoutingJSON = `{"loadBalancingConfig":[{
     "xds_routing_experimental": {
       "action":{
@@ -178,51 +191,6 @@ const (
 `
 )
 
-func TestWeightedClusterToJSON(t *testing.T) {
-	tests := []struct {
-		name     string
-		wc       map[string]uint32
-		wantJSON string // wantJSON is not to be compared verbatim.
-	}{
-		{
-			name:     "one cluster only",
-			wc:       map[string]uint32{testCluster1: 1},
-			wantJSON: testClusterOnlyJSON,
-		},
-		{
-			name:     "empty weighted clusters",
-			wc:       nil,
-			wantJSON: testWeightedCDSNoChildJSON,
-		},
-		{
-			name: "weighted clusters",
-			wc: map[string]uint32{
-				"cluster_1": 75,
-				"cluster_2": 25,
-			},
-			wantJSON: testWeightedCDSJSON,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			gotJSON, err := weightedClusterToJSON(tt.wc)
-			if err != nil {
-				t.Errorf("serviceUpdateToJSON returned error: %v", err)
-				return
-			}
-
-			gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON)
-			wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
-
-			if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
-				t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON)
-				t.Error("gotParsed: ", cmp.Diff(nil, gotParsed))
-				t.Error("wantParsed: ", cmp.Diff(nil, wantParsed))
-			}
-		})
-	}
-}
-
 func TestRoutesToJSON(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -348,15 +316,6 @@ func TestServiceUpdateToJSON(t *testing.T) {
 		wantJSON string
 		wantErr  bool
 	}{
-		{
-			name: "weighted clusters",
-			su: client.ServiceUpdate{WeightedCluster: map[string]uint32{
-				"cluster_1": 75,
-				"cluster_2": 25,
-			}},
-			wantJSON: testWeightedCDSJSON,
-			wantErr:  false,
-		},
 		{
 			name: "routing",
 			su: client.ServiceUpdate{
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index d84ec44eb8c0..5c3b0fce84e1 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -272,7 +272,7 @@ func TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	// Call the watchAPI callback after closing the resolver, and make sure no
 	// update is triggerred on the ClientConn.
 	xdsR.Close()
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{WeightedCluster: map[string]uint32{cluster: 1}}, nil)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil)
 	if gotVal, gotErr := tcc.stateCh.Receive(); gotErr != testutils.ErrRecvTimeout {
 		t.Fatalf("ClientConn.UpdateState called after xdsResolver is closed: %v", gotVal)
 	}
@@ -316,20 +316,21 @@ func TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	}()
 
 	waitForWatchService(t, xdsC, targetStr)
+	defer replaceRandNumGenerator(0)()
 
 	for _, tt := range []struct {
 		su       client.ServiceUpdate
 		wantJSON string
 	}{
 		{
-			su:       client.ServiceUpdate{WeightedCluster: map[string]uint32{testCluster1: 1}},
-			wantJSON: testClusterOnlyJSON,
+			su:       client.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}}},
+			wantJSON: testOneClusterOnlyJSON,
 		},
 		{
-			su: client.ServiceUpdate{WeightedCluster: map[string]uint32{
+			su: client.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
 				"cluster_1": 75,
 				"cluster_2": 25,
-			}},
+			}}}},
 			wantJSON: testWeightedCDSJSON,
 		},
 	} {
@@ -382,7 +383,7 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 
 	// Invoke the watchAPI callback with a good service update and wait for the
 	// UpdateState method to be called on the ClientConn.
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{WeightedCluster: map[string]uint32{cluster: 1}}, nil)
+	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil)
 	gotState, err := tcc.stateCh.Receive()
 	if err != nil {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)

From 8b7764bddbcab042f6e543ef87e5e1740a4469ae Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 27 Jul 2020 18:27:21 -0700
Subject: [PATCH 131/481] xds interop: update interop client to support new
 tests (#3737)

---
 interop/grpc_testing/test.pb.go | 194 ++++++++++++++++++++------------
 interop/grpc_testing/test.proto |   7 ++
 interop/xds/client/client.go    | 186 ++++++++++++++++++++++++++----
 interop/xds/server/server.go    |   7 ++
 4 files changed, 301 insertions(+), 93 deletions(-)

diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 1c8e33bc95a3..9b44623c25ae 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -712,10 +712,12 @@ type LoadBalancerStatsResponse struct {
 	// The number of completed RPCs for each peer.
 	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
 	// The number of RPCs that failed to record a remote peer.
-	NumFailures          int32    `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	NumFailures int32 `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"`
+	// The number of completed RPCs for each method (UnaryCall or EmptyCall).
+	RpcsByMethod         map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}                                         `json:"-"`
+	XXX_unrecognized     []byte                                           `json:"-"`
+	XXX_sizecache        int32                                            `json:"-"`
 }
 
 func (m *LoadBalancerStatsResponse) Reset()         { *m = LoadBalancerStatsResponse{} }
@@ -757,6 +759,53 @@ func (m *LoadBalancerStatsResponse) GetNumFailures() int32 {
 	return 0
 }
 
+func (m *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer {
+	if m != nil {
+		return m.RpcsByMethod
+	}
+	return nil
+}
+
+type LoadBalancerStatsResponse_RpcsByPeer struct {
+	// The number of completed RPCs for each peer.
+	RpcsByPeer           map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}         `json:"-"`
+	XXX_unrecognized     []byte           `json:"-"`
+	XXX_sizecache        int32            `json:"-"`
+}
+
+func (m *LoadBalancerStatsResponse_RpcsByPeer) Reset()         { *m = LoadBalancerStatsResponse_RpcsByPeer{} }
+func (m *LoadBalancerStatsResponse_RpcsByPeer) String() string { return proto.CompactTextString(m) }
+func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage()    {}
+func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) {
+	return fileDescriptor_534063719f48d90d, []int{11, 0}
+}
+
+func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Unmarshal(m, b)
+}
+func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Marshal(b, m, deterministic)
+}
+func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Merge(m, src)
+}
+func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Size() int {
+	return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Size(m)
+}
+func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_DiscardUnknown() {
+	xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer proto.InternalMessageInfo
+
+func (m *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 {
+	if m != nil {
+		return m.RpcsByPeer
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value)
 	proto.RegisterEnum("grpc.testing.GrpclbRouteType", GrpclbRouteType_name, GrpclbRouteType_value)
@@ -772,75 +821,82 @@ func init() {
 	proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse")
 	proto.RegisterType((*LoadBalancerStatsRequest)(nil), "grpc.testing.LoadBalancerStatsRequest")
 	proto.RegisterType((*LoadBalancerStatsResponse)(nil), "grpc.testing.LoadBalancerStatsResponse")
+	proto.RegisterMapType((map[string]*LoadBalancerStatsResponse_RpcsByPeer)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry")
 	proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry")
+	proto.RegisterType((*LoadBalancerStatsResponse_RpcsByPeer)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer")
+	proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry")
 }
 
 func init() { proto.RegisterFile("interop/grpc_testing/test.proto", fileDescriptor_534063719f48d90d) }
 
 var fileDescriptor_534063719f48d90d = []byte{
-	// 1019 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0xfd, 0x6e, 0x1b, 0x45,
-	0x10, 0xcf, 0x39, 0x71, 0x1c, 0x8f, 0x5d, 0xc7, 0xdd, 0xb4, 0x70, 0x71, 0x28, 0x35, 0x07, 0xa2,
-	0xa6, 0xa8, 0x0e, 0x72, 0xc5, 0x87, 0x2a, 0x15, 0x14, 0x27, 0x4e, 0x88, 0xea, 0xda, 0xe6, 0x1c,
-	0x83, 0xca, 0x3f, 0xa7, 0xcd, 0x79, 0x72, 0x39, 0x71, 0x5f, 0xec, 0xed, 0x45, 0xb8, 0xff, 0x20,
-	0xf1, 0x08, 0xbc, 0x02, 0x8f, 0xc1, 0x8b, 0xf0, 0x38, 0x68, 0xf7, 0xee, 0xfc, 0x79, 0x51, 0x13,
-	0x2a, 0xfa, 0x97, 0x77, 0xe7, 0xe3, 0x37, 0x33, 0xbf, 0x99, 0x1d, 0x1f, 0x3c, 0xb4, 0x3d, 0x8e,
-	0xcc, 0x0f, 0xf6, 0x2d, 0x16, 0x98, 0x06, 0xc7, 0x90, 0xdb, 0x9e, 0xb5, 0x2f, 0x7e, 0x9b, 0x01,
-	0xf3, 0xb9, 0x4f, 0xca, 0x42, 0xd1, 0x4c, 0x14, 0x5a, 0x01, 0xf2, 0x1d, 0x37, 0xe0, 0x13, 0xad,
-	0x0b, 0x85, 0x01, 0x9d, 0x38, 0x3e, 0x1d, 0x93, 0x27, 0xb0, 0xc1, 0x27, 0x01, 0xaa, 0x4a, 0x5d,
-	0x69, 0x54, 0x5a, 0xbb, 0xcd, 0x79, 0x87, 0x66, 0x62, 0x74, 0x36, 0x09, 0x50, 0x97, 0x66, 0x84,
-	0xc0, 0xc6, 0xb9, 0x3f, 0x9e, 0xa8, 0xb9, 0xba, 0xd2, 0x28, 0xeb, 0xf2, 0xac, 0x3d, 0x03, 0xe8,
-	0x98, 0x97, 0xfe, 0x90, 0x53, 0x1e, 0x85, 0xc2, 0xc2, 0xf4, 0xc7, 0x31, 0x60, 0x5e, 0x97, 0x67,
-	0xa2, 0x42, 0xc1, 0xc5, 0x30, 0xa4, 0x16, 0x4a, 0xc7, 0xa2, 0x9e, 0x5e, 0xb5, 0x3f, 0xd7, 0xe1,
-	0xce, 0xd0, 0x76, 0x03, 0x07, 0x75, 0xfc, 0x35, 0xc2, 0x90, 0x93, 0x6f, 0xe1, 0x0e, 0xc3, 0x30,
-	0xf0, 0xbd, 0x10, 0x8d, 0x9b, 0x65, 0x56, 0x4e, 0xed, 0xc5, 0x8d, 0x7c, 0x3c, 0xe7, 0x1f, 0xda,
-	0xaf, 0xe3, 0x88, 0xf9, 0x99, 0xd1, 0xd0, 0x7e, 0x8d, 0x64, 0x1f, 0x0a, 0x41, 0x8c, 0xa0, 0xae,
-	0xd7, 0x95, 0x46, 0xa9, 0x75, 0x3f, 0x13, 0x5e, 0x4f, 0xad, 0x04, 0xea, 0x85, 0xed, 0x38, 0x46,
-	0x14, 0x22, 0xf3, 0xa8, 0x8b, 0xea, 0x46, 0x5d, 0x69, 0x6c, 0xe9, 0x65, 0x21, 0x1c, 0x25, 0x32,
-	0xd2, 0x80, 0xaa, 0x34, 0xf2, 0x69, 0xc4, 0x2f, 0x8d, 0xd0, 0xf4, 0x03, 0x54, 0xf3, 0xd2, 0xae,
-	0x22, 0xe4, 0x7d, 0x21, 0x1e, 0x0a, 0x29, 0x39, 0x80, 0xed, 0x59, 0x92, 0x92, 0x37, 0xb5, 0x20,
-	0xf3, 0x50, 0x17, 0xf3, 0x98, 0xf1, 0xaa, 0x57, 0xa6, 0x05, 0xc4, 0x3c, 0x7f, 0x02, 0x12, 0xd4,
-	0x08, 0x91, 0x5d, 0x21, 0x33, 0xec, 0xb1, 0x5a, 0x9c, 0xa5, 0x34, 0x94, 0xc2, 0xd3, 0x31, 0x79,
-	0x0a, 0xef, 0x49, 0x2b, 0x81, 0xea, 0x9c, 0x1b, 0xcc, 0x8f, 0x78, 0x42, 0x2b, 0x48, 0xeb, 0x1d,
-	0xa1, 0x3d, 0x91, 0x4a, 0x5d, 0xe8, 0x04, 0x85, 0xda, 0x1f, 0x39, 0xa8, 0xa4, 0x4d, 0x89, 0x63,
-	0xce, 0x13, 0xa6, 0xdc, 0x88, 0xb0, 0x1a, 0x6c, 0x4d, 0xb9, 0x8a, 0x7b, 0x3e, 0xbd, 0x93, 0x87,
-	0x50, 0x9a, 0xa7, 0x68, 0x5d, 0xaa, 0xc1, 0x9f, 0xd1, 0xb3, 0x07, 0xc5, 0x59, 0x59, 0x1b, 0xb1,
-	0x77, 0x98, 0x96, 0x74, 0x0a, 0x77, 0x57, 0xab, 0xc9, 0xcb, 0x21, 0x79, 0xb0, 0x98, 0xd4, 0x52,
-	0x5d, 0xfa, 0xb6, 0xb5, 0x28, 0x10, 0x49, 0x5e, 0xfa, 0x21, 0x97, 0x49, 0x6e, 0xc6, 0x61, 0xd2,
-	0xbb, 0xd6, 0x85, 0xdd, 0x21, 0x67, 0x48, 0x5d, 0xdb, 0xb3, 0x4e, 0xbd, 0x20, 0xe2, 0x87, 0xd4,
-	0x71, 0xd2, 0x21, 0xbd, 0x2d, 0x1d, 0xda, 0x19, 0xd4, 0xb2, 0xd0, 0x12, 0x76, 0xbf, 0x82, 0xf7,
-	0xa9, 0x65, 0x31, 0xb4, 0x28, 0xc7, 0xb1, 0x91, 0xf8, 0xc4, 0xd3, 0x1b, 0x3f, 0xa3, 0xfb, 0x33,
-	0x75, 0x02, 0x2d, 0xc6, 0x58, 0x3b, 0x05, 0x92, 0x62, 0x0c, 0x28, 0xa3, 0x2e, 0x72, 0x64, 0xf2,
-	0x05, 0xce, 0xb9, 0xca, 0xb3, 0xa0, 0x5c, 0xee, 0x8a, 0x2b, 0x2a, 0x66, 0x38, 0x79, 0x13, 0x90,
-	0x8a, 0x46, 0xa1, 0xf6, 0x57, 0x6e, 0x2e, 0xc3, 0x7e, 0xc4, 0x97, 0x0a, 0x7e, 0xdb, 0x57, 0xf9,
-	0x03, 0xec, 0x4c, 0xfd, 0x83, 0x69, 0xaa, 0x6a, 0xae, 0xbe, 0xde, 0x28, 0xb5, 0xea, 0x8b, 0x28,
-	0xab, 0x25, 0xe9, 0x84, 0xad, 0x96, 0x79, 0xeb, 0x37, 0xfc, 0xf6, 0x8f, 0x4e, 0xeb, 0xc1, 0x5e,
-	0x26, 0x49, 0xff, 0xf1, 0x95, 0x68, 0x3f, 0x82, 0xda, 0xf5, 0xe9, 0xb8, 0x4d, 0x1d, 0xea, 0x99,
-	0xc8, 0x44, 0x94, 0x30, 0xa5, 0x7c, 0x17, 0xb6, 0xbc, 0xc8, 0x35, 0x58, 0x60, 0x86, 0x49, 0x2b,
-	0x0b, 0x5e, 0xe4, 0xea, 0x81, 0x19, 0x8a, 0x6e, 0x72, 0xdb, 0x45, 0x3f, 0xe2, 0x46, 0x88, 0x66,
-	0xda, 0xcd, 0x44, 0x34, 0x44, 0x53, 0xfb, 0x47, 0x81, 0xdd, 0x0c, 0xe0, 0x24, 0xcd, 0x57, 0x50,
-	0x16, 0xa8, 0xc6, 0xf9, 0xc4, 0x08, 0x10, 0x99, 0xaa, 0xc8, 0x2e, 0x7c, 0xbd, 0x98, 0xeb, 0xb5,
-	0xee, 0x4d, 0x91, 0x42, 0x7b, 0x32, 0x40, 0x64, 0x1d, 0x8f, 0xb3, 0x89, 0x0e, 0x6c, 0x2a, 0x20,
-	0x1f, 0x41, 0x59, 0x24, 0x7d, 0x41, 0x6d, 0x27, 0x62, 0x98, 0x0e, 0x5a, 0xc9, 0x8b, 0xdc, 0xe3,
-	0x44, 0x54, 0x7b, 0x0e, 0xdb, 0x4b, 0x08, 0xa4, 0x0a, 0xeb, 0xbf, 0xe0, 0x44, 0x56, 0x59, 0xd4,
-	0xc5, 0x91, 0xdc, 0x83, 0xfc, 0x15, 0x75, 0xa2, 0x74, 0x7b, 0xc7, 0x97, 0x67, 0xb9, 0x6f, 0x94,
-	0xc7, 0xdf, 0x41, 0x69, 0x6e, 0xcc, 0x48, 0x15, 0xca, 0x87, 0xfd, 0x97, 0x03, 0xbd, 0x33, 0x1c,
-	0x1e, 0xb4, 0xbb, 0x9d, 0xea, 0x1a, 0x21, 0x50, 0x19, 0xf5, 0x16, 0x64, 0x0a, 0x01, 0xd8, 0xd4,
-	0x0f, 0x7a, 0x47, 0xfd, 0x97, 0xd5, 0xdc, 0x63, 0x1f, 0xb6, 0x97, 0x16, 0x03, 0x79, 0x00, 0xbb,
-	0x27, 0xfa, 0xe0, 0xb0, 0xdb, 0x36, 0xf4, 0xfe, 0xe8, 0xac, 0x63, 0x9c, 0xbd, 0x1a, 0x74, 0x8c,
-	0x51, 0xef, 0x45, 0xaf, 0xff, 0x53, 0xaf, 0xba, 0x46, 0x3e, 0x84, 0xda, 0xaa, 0xfa, 0xf8, 0xa0,
-	0xdb, 0x6d, 0x1f, 0x1c, 0xbe, 0xa8, 0x2a, 0xd9, 0xee, 0x42, 0xd7, 0xe9, 0x1d, 0x55, 0x73, 0xad,
-	0xbf, 0x37, 0xa0, 0x74, 0x86, 0x21, 0x17, 0x4b, 0xd9, 0x36, 0x91, 0x7c, 0x09, 0x45, 0xf9, 0x37,
-	0x2c, 0x46, 0x87, 0xec, 0x2c, 0xcd, 0x9e, 0x50, 0xd4, 0xb2, 0x84, 0xe4, 0x18, 0x8a, 0x23, 0x8f,
-	0xb2, 0xd8, 0x6d, 0x6f, 0xd1, 0x62, 0xe1, 0x2f, 0xb4, 0xf6, 0x41, 0xb6, 0x32, 0xe9, 0xbe, 0x03,
-	0x3b, 0x19, 0x33, 0x4c, 0x1a, 0x4b, 0x4e, 0xd7, 0xee, 0x82, 0xda, 0x67, 0x37, 0xb0, 0x8c, 0x63,
-	0x7d, 0xa1, 0x10, 0x1b, 0xc8, 0xea, 0xe2, 0x23, 0x8f, 0xae, 0x81, 0x58, 0x5e, 0xb4, 0xb5, 0xc6,
-	0x9b, 0x0d, 0xe3, 0x50, 0x0d, 0x11, 0xaa, 0x72, 0x1c, 0x39, 0xce, 0x51, 0x14, 0x38, 0xf8, 0xdb,
-	0xff, 0x56, 0x53, 0x43, 0x91, 0x55, 0x55, 0xbe, 0xa7, 0xce, 0xc5, 0x3b, 0x08, 0xd5, 0x1a, 0xc1,
-	0xbd, 0x91, 0x27, 0x3b, 0xe8, 0xa2, 0xc7, 0x71, 0x9c, 0x4e, 0xd1, 0x73, 0xb8, 0xbb, 0x20, 0xbf,
-	0xdd, 0x34, 0xb5, 0x7e, 0xcf, 0xd8, 0x3c, 0x29, 0xb4, 0x09, 0x95, 0x13, 0xe4, 0x87, 0x8e, 0x8d,
-	0x1e, 0x97, 0x0a, 0xf2, 0xe9, 0x1b, 0x77, 0x43, 0x5c, 0xdb, 0xa3, 0x1b, 0xee, 0x10, 0x6d, 0xad,
-	0xfd, 0xe4, 0xe7, 0xcf, 0x2d, 0xdf, 0xb7, 0x1c, 0x6c, 0x5a, 0xbe, 0x43, 0x3d, 0xab, 0xe9, 0x33,
-	0x4b, 0x7e, 0xc7, 0xee, 0x67, 0x7d, 0xd4, 0x9e, 0x6f, 0xca, 0x0f, 0xda, 0xa7, 0xff, 0x06, 0x00,
-	0x00, 0xff, 0xff, 0xaa, 0x43, 0x4c, 0xeb, 0xf3, 0x0a, 0x00, 0x00,
+	// 1083 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xdd, 0x72, 0xdb, 0xc4,
+	0x17, 0x8f, 0x1c, 0x3b, 0x8e, 0x8f, 0x5d, 0xc7, 0xd9, 0xb4, 0xff, 0xbf, 0xe2, 0x50, 0x6a, 0x04,
+	0x43, 0x4d, 0x99, 0x3a, 0x8c, 0x3b, 0x7c, 0x75, 0xa6, 0x30, 0x71, 0xe2, 0xa4, 0x99, 0x3a, 0xb6,
+	0x91, 0x63, 0x98, 0x72, 0xa3, 0xd9, 0xc8, 0x1b, 0x45, 0x83, 0xa4, 0x15, 0xab, 0x55, 0x06, 0xf7,
+	0x86, 0x19, 0x1e, 0x81, 0x57, 0xe0, 0x09, 0xb8, 0xe6, 0x6d, 0x78, 0x12, 0x66, 0x57, 0x92, 0x3f,
+	0x15, 0x1a, 0x93, 0x81, 0x2b, 0xef, 0x9e, 0xcf, 0xdf, 0xf9, 0x9d, 0x3d, 0xbb, 0x16, 0x3c, 0xb2,
+	0x3d, 0x4e, 0x18, 0xf5, 0xf7, 0x2d, 0xe6, 0x9b, 0x06, 0x27, 0x01, 0xb7, 0x3d, 0x6b, 0x5f, 0xfc,
+	0x36, 0x7c, 0x46, 0x39, 0x45, 0x25, 0xa1, 0x68, 0xc4, 0x0a, 0x2d, 0x0f, 0xb9, 0xb6, 0xeb, 0xf3,
+	0xb1, 0xd6, 0x81, 0x7c, 0x1f, 0x8f, 0x1d, 0x8a, 0x47, 0xe8, 0x29, 0x64, 0xf9, 0xd8, 0x27, 0xaa,
+	0x52, 0x53, 0xea, 0xe5, 0xe6, 0x6e, 0x63, 0xd6, 0xa1, 0x11, 0x1b, 0x9d, 0x8f, 0x7d, 0xa2, 0x4b,
+	0x33, 0x84, 0x20, 0x7b, 0x41, 0x47, 0x63, 0x35, 0x53, 0x53, 0xea, 0x25, 0x5d, 0xae, 0xb5, 0xe7,
+	0x00, 0x6d, 0xf3, 0x8a, 0x0e, 0x38, 0xe6, 0x61, 0x20, 0x2c, 0x4c, 0x3a, 0x8a, 0x02, 0xe6, 0x74,
+	0xb9, 0x46, 0x2a, 0xe4, 0x5d, 0x12, 0x04, 0xd8, 0x22, 0xd2, 0xb1, 0xa0, 0x27, 0x5b, 0xed, 0xd7,
+	0x75, 0xb8, 0x37, 0xb0, 0x5d, 0xdf, 0x21, 0x3a, 0xf9, 0x31, 0x24, 0x01, 0x47, 0x5f, 0xc1, 0x3d,
+	0x46, 0x02, 0x9f, 0x7a, 0x01, 0x31, 0x6e, 0x87, 0xac, 0x94, 0xd8, 0x8b, 0x1d, 0x7a, 0x7f, 0xc6,
+	0x3f, 0xb0, 0xdf, 0x44, 0x19, 0x73, 0x53, 0xa3, 0x81, 0xfd, 0x86, 0xa0, 0x7d, 0xc8, 0xfb, 0x51,
+	0x04, 0x75, 0xbd, 0xa6, 0xd4, 0x8b, 0xcd, 0x07, 0xa9, 0xe1, 0xf5, 0xc4, 0x4a, 0x44, 0xbd, 0xb4,
+	0x1d, 0xc7, 0x08, 0x03, 0xc2, 0x3c, 0xec, 0x12, 0x35, 0x5b, 0x53, 0xea, 0x9b, 0x7a, 0x49, 0x08,
+	0x87, 0xb1, 0x0c, 0xd5, 0xa1, 0x22, 0x8d, 0x28, 0x0e, 0xf9, 0x95, 0x11, 0x98, 0xd4, 0x27, 0x6a,
+	0x4e, 0xda, 0x95, 0x85, 0xbc, 0x27, 0xc4, 0x03, 0x21, 0x45, 0x07, 0xb0, 0x35, 0x05, 0x29, 0x79,
+	0x53, 0xf3, 0x12, 0x87, 0x3a, 0x8f, 0x63, 0xca, 0xab, 0x5e, 0x9e, 0x14, 0x10, 0xf1, 0xfc, 0x01,
+	0xc8, 0xa0, 0x46, 0x40, 0xd8, 0x35, 0x61, 0x86, 0x3d, 0x52, 0x0b, 0x53, 0x48, 0x03, 0x29, 0x3c,
+	0x1d, 0xa1, 0x67, 0xf0, 0x3f, 0x69, 0x25, 0xa2, 0x3a, 0x17, 0x06, 0xa3, 0x21, 0x8f, 0x69, 0x05,
+	0x69, 0xbd, 0x23, 0xb4, 0x27, 0x52, 0xa9, 0x0b, 0x9d, 0xa0, 0x50, 0xfb, 0x25, 0x03, 0xe5, 0xa4,
+	0x29, 0x51, 0xce, 0x59, 0xc2, 0x94, 0x5b, 0x11, 0x56, 0x85, 0xcd, 0x09, 0x57, 0x51, 0xcf, 0x27,
+	0x7b, 0xf4, 0x08, 0x8a, 0xb3, 0x14, 0xad, 0x4b, 0x35, 0xd0, 0x29, 0x3d, 0x7b, 0x50, 0x98, 0x96,
+	0x95, 0x8d, 0xbc, 0x83, 0xa4, 0xa4, 0x53, 0xd8, 0x5e, 0xae, 0x26, 0x27, 0x0f, 0xc9, 0xc3, 0x79,
+	0x50, 0x0b, 0x75, 0xe9, 0x5b, 0xd6, 0xbc, 0x40, 0x80, 0xbc, 0xa2, 0x01, 0x97, 0x20, 0x37, 0xa2,
+	0x34, 0xc9, 0x5e, 0xeb, 0xc0, 0xee, 0x80, 0x33, 0x82, 0x5d, 0xdb, 0xb3, 0x4e, 0x3d, 0x3f, 0xe4,
+	0x87, 0xd8, 0x71, 0x92, 0x43, 0xba, 0x2a, 0x1d, 0xda, 0x39, 0x54, 0xd3, 0xa2, 0xc5, 0xec, 0x7e,
+	0x06, 0xff, 0xc7, 0x96, 0xc5, 0x88, 0x85, 0x39, 0x19, 0x19, 0xb1, 0x4f, 0x74, 0x7a, 0xa3, 0x31,
+	0x7a, 0x30, 0x55, 0xc7, 0xa1, 0xc5, 0x31, 0xd6, 0x4e, 0x01, 0x25, 0x31, 0xfa, 0x98, 0x61, 0x97,
+	0x70, 0xc2, 0xe4, 0x04, 0xce, 0xb8, 0xca, 0xb5, 0xa0, 0x5c, 0xde, 0x15, 0xd7, 0x58, 0x9c, 0xe1,
+	0x78, 0x26, 0x20, 0x11, 0x0d, 0x03, 0xed, 0xb7, 0xcc, 0x0c, 0xc2, 0x5e, 0xc8, 0x17, 0x0a, 0xbe,
+	0xeb, 0x54, 0x7e, 0x03, 0x3b, 0x13, 0x7f, 0x7f, 0x02, 0x55, 0xcd, 0xd4, 0xd6, 0xeb, 0xc5, 0x66,
+	0x6d, 0x3e, 0xca, 0x72, 0x49, 0x3a, 0x62, 0xcb, 0x65, 0xae, 0x3c, 0xc3, 0x77, 0x1f, 0x3a, 0xad,
+	0x0b, 0x7b, 0xa9, 0x24, 0xfd, 0xc3, 0x29, 0xd1, 0xbe, 0x05, 0xb5, 0x43, 0xf1, 0xa8, 0x85, 0x1d,
+	0xec, 0x99, 0x84, 0x89, 0x2c, 0x41, 0x42, 0xf9, 0x2e, 0x6c, 0x7a, 0xa1, 0x6b, 0x30, 0xdf, 0x0c,
+	0xe2, 0x56, 0xe6, 0xbd, 0xd0, 0xd5, 0x7d, 0x33, 0x10, 0xdd, 0xe4, 0xb6, 0x4b, 0x68, 0xc8, 0x8d,
+	0x80, 0x98, 0x49, 0x37, 0x63, 0xd1, 0x80, 0x98, 0xda, 0x9f, 0x59, 0xd8, 0x4d, 0x09, 0x1c, 0xc3,
+	0x7c, 0x0d, 0x25, 0x11, 0xd5, 0xb8, 0x18, 0x1b, 0x3e, 0x21, 0x4c, 0x55, 0x64, 0x17, 0x3e, 0x9f,
+	0xc7, 0x7a, 0xa3, 0x7b, 0x43, 0x40, 0x68, 0x8d, 0xfb, 0x84, 0xb0, 0xb6, 0xc7, 0xd9, 0x58, 0x07,
+	0x36, 0x11, 0xa0, 0xf7, 0xa0, 0x24, 0x40, 0x5f, 0x62, 0xdb, 0x09, 0x19, 0x49, 0x0e, 0x5a, 0xd1,
+	0x0b, 0xdd, 0xe3, 0x58, 0x84, 0x0c, 0x28, 0x27, 0xd9, 0x5d, 0xc2, 0xaf, 0xa8, 0x68, 0x9f, 0xc8,
+	0xff, 0xe5, 0x6a, 0xf9, 0xcf, 0xa4, 0x6f, 0x84, 0xa0, 0xc4, 0x66, 0x44, 0xd5, 0xdf, 0x15, 0x80,
+	0x29, 0x46, 0x34, 0x4a, 0xad, 0xb6, 0xb5, 0x7a, 0xb5, 0x7f, 0x57, 0x78, 0xf5, 0x05, 0x6c, 0x2d,
+	0xa8, 0x51, 0x05, 0xd6, 0x7f, 0x20, 0x63, 0xd9, 0xbb, 0x82, 0x2e, 0x96, 0xe8, 0x3e, 0xe4, 0xae,
+	0xb1, 0x13, 0x26, 0x6f, 0x52, 0xb4, 0x79, 0x9e, 0xf9, 0x42, 0xb9, 0xab, 0x7b, 0x00, 0xdb, 0x4b,
+	0xac, 0xa4, 0x04, 0x78, 0x39, 0x1b, 0xa0, 0xd8, 0x6c, 0xae, 0xce, 0xc1, 0x4c, 0xd2, 0x27, 0x5f,
+	0x43, 0x71, 0x66, 0xe0, 0x51, 0x05, 0x4a, 0x87, 0xbd, 0xb3, 0xbe, 0xde, 0x1e, 0x0c, 0x0e, 0x5a,
+	0x9d, 0x76, 0x65, 0x0d, 0x21, 0x28, 0x0f, 0xbb, 0x73, 0x32, 0x05, 0x01, 0x6c, 0xe8, 0x07, 0xdd,
+	0xa3, 0xde, 0x59, 0x25, 0xf3, 0x84, 0xc2, 0xd6, 0xc2, 0x15, 0x8d, 0x1e, 0xc2, 0xee, 0x89, 0xde,
+	0x3f, 0xec, 0xb4, 0x0c, 0xbd, 0x37, 0x3c, 0x6f, 0x1b, 0xe7, 0xaf, 0xfb, 0x6d, 0x63, 0xd8, 0x7d,
+	0xd5, 0xed, 0x7d, 0xd7, 0xad, 0xac, 0xa1, 0x77, 0xa1, 0xba, 0xac, 0x3e, 0x3e, 0xe8, 0x74, 0x5a,
+	0x07, 0x87, 0xaf, 0x2a, 0x4a, 0xba, 0xbb, 0xd0, 0xb5, 0xbb, 0x47, 0x95, 0x4c, 0xf3, 0x8f, 0x2c,
+	0x14, 0xcf, 0x49, 0xc0, 0xc5, 0xf3, 0x68, 0x9b, 0x04, 0x7d, 0x0a, 0x05, 0xf9, 0x87, 0x48, 0x0c,
+	0x31, 0xda, 0x59, 0xb8, 0x05, 0x84, 0xa2, 0x9a, 0x26, 0x44, 0xc7, 0x50, 0x18, 0x7a, 0x98, 0x45,
+	0x6e, 0x7b, 0xf3, 0x16, 0x73, 0x7f, 0x66, 0xaa, 0xef, 0xa4, 0x2b, 0xe3, 0x39, 0x74, 0x60, 0x27,
+	0xe5, 0x36, 0x41, 0xf5, 0x05, 0xa7, 0x1b, 0x6f, 0xe5, 0xea, 0x47, 0xb7, 0xb0, 0x8c, 0x72, 0x7d,
+	0xa2, 0x20, 0x1b, 0xd0, 0xf2, 0x13, 0x84, 0x1e, 0xdf, 0x10, 0x62, 0xf1, 0xc9, 0xab, 0xd6, 0xdf,
+	0x6e, 0x18, 0xa5, 0xaa, 0x8b, 0x54, 0xe5, 0xe3, 0xd0, 0x71, 0x8e, 0x42, 0xdf, 0x21, 0x3f, 0xfd,
+	0x6b, 0x35, 0xd5, 0x15, 0x59, 0x55, 0xf9, 0x25, 0x76, 0x2e, 0xff, 0x83, 0x54, 0xcd, 0x21, 0xdc,
+	0x1f, 0x7a, 0xb2, 0x83, 0x2e, 0xf1, 0x38, 0x19, 0x25, 0xa7, 0xe8, 0x05, 0x6c, 0xcf, 0xc9, 0x57,
+	0x3b, 0x4d, 0xcd, 0x9f, 0x53, 0xde, 0x80, 0x24, 0xb4, 0x09, 0xe5, 0x13, 0xc2, 0x0f, 0x1d, 0x9b,
+	0x78, 0x5c, 0x2a, 0xd0, 0x87, 0x6f, 0x9d, 0xd9, 0xa8, 0xb6, 0xc7, 0xb7, 0x9c, 0x6d, 0x6d, 0xad,
+	0xf5, 0xf4, 0xfb, 0x8f, 0x2d, 0x4a, 0x2d, 0x87, 0x34, 0x2c, 0xea, 0x60, 0xcf, 0x6a, 0x50, 0x66,
+	0xc9, 0x2f, 0x8a, 0xfd, 0xb4, 0xcf, 0x8b, 0x8b, 0x0d, 0xf9, 0x69, 0xf1, 0xec, 0xaf, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0x8c, 0x0b, 0x8c, 0x16, 0x7d, 0x0c, 0x00, 0x00,
 }
diff --git a/interop/grpc_testing/test.proto b/interop/grpc_testing/test.proto
index f827a3ef2d64..21451047e513 100644
--- a/interop/grpc_testing/test.proto
+++ b/interop/grpc_testing/test.proto
@@ -214,10 +214,17 @@ message LoadBalancerStatsRequest {
 }
 
 message LoadBalancerStatsResponse {
+  message RpcsByPeer {
+    // The number of completed RPCs for each peer.
+    map<string, int32> rpcs_by_peer = 1;
+  }
+
   // The number of completed RPCs for each peer.
   map<string, int32> rpcs_by_peer = 1;
   // The number of RPCs that failed to record a remote peer.
   int32 num_failures = 2;
+  // The number of completed RPCs for each method (UnaryCall or EmptyCall).
+  map<string, RpcsByPeer> rpcs_by_method = 3;
 }
 
 // A service used to obtain stats for verifying LB behavior.
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 44fbf1e6dbb3..27f954c30d28 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -23,13 +23,16 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	_ "google.golang.org/grpc/xds"
 )
@@ -39,18 +42,43 @@ type statsWatcherKey struct {
 	endID   int32
 }
 
+// rpcInfo contains the rpc type and the hostname where the response is received
+// from.
+type rpcInfo struct {
+	typ      string
+	hostname string
+}
+
 type statsWatcher struct {
 	rpcsByPeer    map[string]int32
+	rpcsByType    map[string]map[string]int32
 	numFailures   int32
 	remainingRpcs int32
-	c             chan *testpb.SimpleResponse
+	chanHosts     chan *rpcInfo
+}
+
+func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse {
+	rpcsByType := make(map[string]*testpb.LoadBalancerStatsResponse_RpcsByPeer, len(watcher.rpcsByType))
+	for t, rpcsByPeer := range watcher.rpcsByType {
+		rpcsByType[t] = &testpb.LoadBalancerStatsResponse_RpcsByPeer{
+			RpcsByPeer: rpcsByPeer,
+		}
+	}
+
+	return &testpb.LoadBalancerStatsResponse{
+		NumFailures:  watcher.numFailures + watcher.remainingRpcs,
+		RpcsByPeer:   watcher.rpcsByPeer,
+		RpcsByMethod: rpcsByType,
+	}
 }
 
 var (
 	failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail")
 	numChannels     = flag.Int("num_channels", 1, "Num of channels")
 	printResponse   = flag.Bool("print_response", false, "Write RPC response to stdout")
-	qps             = flag.Int("qps", 1, "QPS per channel")
+	qps             = flag.Int("qps", 1, "QPS per channel, for each type of RPC")
+	rpc             = flag.String("rpc", "UnaryCall", "Types of RPCs to make, ',' separated string. RPCs can be EmptyCall or UnaryCall")
+	rpcMetadata     = flag.String("metadata", "", "The metadata to send with RPC, in format EmptyCall:key1:value1,UnaryCall:key2:value2")
 	rpcTimeout      = flag.Duration("rpc_timeout", 20*time.Second, "Per RPC timeout")
 	server          = flag.String("server", "localhost:8080", "Address of server to connect to")
 	statsPort       = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service")
@@ -75,9 +103,10 @@ func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalanc
 	if !ok {
 		watcher = &statsWatcher{
 			rpcsByPeer:    make(map[string]int32),
+			rpcsByType:    make(map[string]map[string]int32),
 			numFailures:   0,
 			remainingRpcs: in.GetNumRpcs(),
-			c:             make(chan *testpb.SimpleResponse),
+			chanHosts:     make(chan *rpcInfo),
 		}
 		watchers[watcherKey] = watcher
 	}
@@ -95,25 +124,86 @@ func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalanc
 	// Wait until the requested RPCs have all been recorded or timeout occurs.
 	for {
 		select {
-		case r := <-watcher.c:
-			if r != nil {
-				watcher.rpcsByPeer[(*r).GetHostname()]++
+		case info := <-watcher.chanHosts:
+			if info != nil {
+				watcher.rpcsByPeer[info.hostname]++
+
+				rpcsByPeerForType := watcher.rpcsByType[info.typ]
+				if rpcsByPeerForType == nil {
+					rpcsByPeerForType = make(map[string]int32)
+					watcher.rpcsByType[info.typ] = rpcsByPeerForType
+				}
+				rpcsByPeerForType[info.hostname]++
 			} else {
 				watcher.numFailures++
 			}
 			watcher.remainingRpcs--
 			if watcher.remainingRpcs == 0 {
-				return &testpb.LoadBalancerStatsResponse{NumFailures: watcher.numFailures + watcher.remainingRpcs, RpcsByPeer: watcher.rpcsByPeer}, nil
+				return watcher.buildResp(), nil
 			}
 		case <-ctx.Done():
 			grpclog.Info("Timed out, returning partial stats")
-			return &testpb.LoadBalancerStatsResponse{NumFailures: watcher.numFailures + watcher.remainingRpcs, RpcsByPeer: watcher.rpcsByPeer}, nil
+			return watcher.buildResp(), nil
+		}
+	}
+}
+
+const (
+	unaryCall string = "UnaryCall"
+	emptyCall string = "EmptyCall"
+)
+
+func parseRPCTypes(rpcStr string) (ret []string) {
+	if len(rpcStr) == 0 {
+		return []string{unaryCall}
+	}
+
+	rpcs := strings.Split(rpcStr, ",")
+	for _, r := range rpcs {
+		switch r {
+		case unaryCall, emptyCall:
+			ret = append(ret, r)
+		default:
+			flag.PrintDefaults()
+			log.Fatalf("unsupported RPC type: %v", r)
+		}
+	}
+	return
+}
+
+type rpcConfig struct {
+	typ string
+	md  metadata.MD
+}
+
+// parseRPCMetadata turns EmptyCall:key1:value1 into
+//   {typ: emptyCall, md: {key1:value1}}.
+func parseRPCMetadata(rpcMetadataStr string, rpcs []string) []*rpcConfig {
+	rpcMetadataSplit := strings.Split(rpcMetadataStr, ",")
+	rpcsToMD := make(map[string][]string)
+	for _, rm := range rpcMetadataSplit {
+		rmSplit := strings.Split(rm, ":")
+		if len(rmSplit)%2 != 1 {
+			log.Fatalf("invalid metadata config %v, want EmptyCall:key1:value1", rm)
+		}
+		rpcsToMD[rmSplit[0]] = append(rpcsToMD[rmSplit[0]], rmSplit[1:]...)
+	}
+	ret := make([]*rpcConfig, 0, len(rpcs))
+	for _, rpcT := range rpcs {
+		rpcC := &rpcConfig{
+			typ: rpcT,
 		}
+		if md := rpcsToMD[string(rpcT)]; len(md) > 0 {
+			rpcC.md = metadata.Pairs(md...)
+		}
+		ret = append(ret, rpcC)
 	}
+	return ret
 }
 
 func main() {
 	flag.Parse()
+	rpcCfgs := parseRPCMetadata(*rpcMetadata, parseRPCTypes(*rpc))
 
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *statsPort))
 	if err != nil {
@@ -135,16 +225,52 @@ func main() {
 	}
 	ticker := time.NewTicker(time.Second / time.Duration(*qps**numChannels))
 	defer ticker.Stop()
-	sendRPCs(clients, ticker)
+	sendRPCs(clients, rpcCfgs, ticker)
 }
 
-func sendRPCs(clients []testpb.TestServiceClient, ticker *time.Ticker) {
+func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), *rpcTimeout)
+	defer cancel()
+
+	if len(cfg.md) != 0 {
+		ctx = metadata.NewOutgoingContext(ctx, cfg.md)
+	}
+	info := rpcInfo{typ: cfg.typ}
+
+	var (
+		p      peer.Peer
+		header metadata.MD
+		err    error
+	)
+	switch cfg.typ {
+	case unaryCall:
+		var resp *testpb.SimpleResponse
+		resp, err = c.UnaryCall(ctx, &testpb.SimpleRequest{FillServerId: true}, grpc.Peer(&p), grpc.Header(&header))
+		// For UnaryCall, also read hostname from response, in case the server
+		// isn't updated to send headers.
+		if resp != nil {
+			info.hostname = resp.Hostname
+		}
+	case emptyCall:
+		_, err = c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p), grpc.Header(&header))
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	hosts := header["hostname"]
+	if len(hosts) > 0 {
+		info.hostname = hosts[0]
+	}
+	return &p, &info, err
+}
+
+func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *time.Ticker) {
 	var i int
 	for range ticker.C {
 		go func(i int) {
-			c := clients[i]
-			ctx, cancel := context.WithTimeout(context.Background(), *rpcTimeout)
-			p := new(peer.Peer)
+			// Get and increment request ID, and save a list of watchers that
+			// are interested in this RPC.
 			mu.Lock()
 			savedRequestID := currentRequestID
 			currentRequestID++
@@ -155,20 +281,32 @@ func sendRPCs(clients []testpb.TestServiceClient, ticker *time.Ticker) {
 				}
 			}
 			mu.Unlock()
-			r, err := c.UnaryCall(ctx, &testpb.SimpleRequest{FillServerId: true}, grpc.Peer(p))
 
-			success := err == nil
-			cancel()
+			c := clients[i]
 
-			for _, watcher := range savedWatchers {
-				watcher.c <- r
-			}
+			for _, cfg := range cfgs {
+				p, info, err := makeOneRPC(c, cfg)
 
-			if err != nil && *failOnFailedRPC {
-				grpclog.Fatalf("RPC failed: %v", err)
-			}
-			if success && *printResponse {
-				fmt.Printf("Greeting: Hello world, this is %s, from %v\n", r.GetHostname(), p.Addr)
+				for _, watcher := range savedWatchers {
+					// This sends an empty string if the RPC failed.
+					watcher.chanHosts <- info
+				}
+				if err != nil && *failOnFailedRPC {
+					grpclog.Fatalf("RPC failed: %v", err)
+				}
+				if *printResponse {
+					if err == nil {
+						if cfg.typ == unaryCall {
+							// Need to keep this format, because some tests are
+							// relying on stdout.
+							fmt.Printf("Greeting: Hello world, this is %s, from %v\n", info.hostname, p.Addr)
+						} else {
+							fmt.Printf("RPC %q, from host %s, addr %v\n", cfg.typ, info.hostname, p.Addr)
+						}
+					} else {
+						fmt.Printf("RPC %q, failed with %v\n", cfg.typ, err)
+					}
+				}
 			}
 		}(i)
 		i = (i + 1) % len(clients)
diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go
index 34ada6b542e6..cfb2016f8670 100644
--- a/interop/xds/server/server.go
+++ b/interop/xds/server/server.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
 )
 
 var (
@@ -50,7 +51,13 @@ type server struct {
 	testpb.UnimplementedTestServiceServer
 }
 
+func (s *server) EmptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+	grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname))
+	return &testpb.Empty{}, nil
+}
+
 func (s *server) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname))
 	return &testpb.SimpleResponse{ServerId: *serverID, Hostname: hostname}, nil
 }
 

From dfc0c05b2da9889061710f8bb2f4a7df8e1ab1d7 Mon Sep 17 00:00:00 2001
From: cindyxue <32377977+cindyxue@users.noreply.github.com>
Date: Mon, 27 Jul 2020 23:50:43 -0700
Subject: [PATCH 132/481] advancedtls: Add SNI logic to
 ServerOptions.GetCertificate (#3697)

* Add SNI support in the user-provided GetCertificate callback
---
 security/advancedtls/advancedtls.go           |  20 ++--
 .../advancedtls_integration_test.go           |  18 +--
 security/advancedtls/advancedtls_test.go      |  26 ++---
 security/advancedtls/go.mod                   |   2 +-
 security/advancedtls/sni_110.go               |  53 +++++++++
 security/advancedtls/sni_before_110.go        |  41 +++++++
 security/advancedtls/sni_test_110.go          | 108 ++++++++++++++++++
 security/advancedtls/sni_test_before_110.go   | 108 ++++++++++++++++++
 .../advancedtls/testdata/server_cert_1.txt    |  90 +++++++++++++++
 .../advancedtls/testdata/server_cert_2.txt    |  91 +++++++++++++++
 .../advancedtls/testdata/server_cert_3.pem    |  23 ++++
 .../advancedtls/testdata/server_cert_3.txt    |  60 ++++++++++
 .../advancedtls/testdata/server_key_3.pem     |  28 +++++
 13 files changed, 638 insertions(+), 30 deletions(-)
 create mode 100644 security/advancedtls/sni_110.go
 create mode 100644 security/advancedtls/sni_before_110.go
 create mode 100644 security/advancedtls/sni_test_110.go
 create mode 100644 security/advancedtls/sni_test_before_110.go
 create mode 100644 security/advancedtls/testdata/server_cert_1.txt
 create mode 100644 security/advancedtls/testdata/server_cert_2.txt
 create mode 100644 security/advancedtls/testdata/server_cert_3.pem
 create mode 100644 security/advancedtls/testdata/server_cert_3.txt
 create mode 100644 security/advancedtls/testdata/server_key_3.pem

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index 745938ac5295..e028c623ba43 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -156,7 +156,7 @@ type ClientOptions struct {
 // Certificates or GetClientCertificate indicates the certificates sent from
 // the server to the client to prove server's identities. The rules for setting
 // these two fields are:
-// Either Certificates or GetCertificate must be set; the other will be ignored.
+// Either Certificates or GetCertificates must be set; the other will be ignored.
 type ServerOptions struct {
 	// If field Certificates is set, field GetClientCertificate will be ignored.
 	// The server will use Certificates every time when asked for a certificate,
@@ -166,7 +166,7 @@ type ServerOptions struct {
 	// invoke this function every time asked to present certificates to the
 	// client when a new connection is established. This is known as peer
 	// certificate reloading.
-	GetCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error)
+	GetCertificates func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
 	// VerifyPeer is a custom verification check after certificate signature
 	// check.
 	// If this is set, we will perform this customized check after doing the
@@ -210,8 +210,8 @@ func (o *ClientOptions) config() (*tls.Config, error) {
 }
 
 func (o *ServerOptions) config() (*tls.Config, error) {
-	if o.Certificates == nil && o.GetCertificate == nil {
-		return nil, fmt.Errorf("either Certificates or GetCertificate must be specified")
+	if o.Certificates == nil && o.GetCertificates == nil {
+		return nil, fmt.Errorf("either Certificates or GetCertificates must be specified")
 	}
 	if o.RequireClientCert && o.VType == SkipVerification && o.VerifyPeer == nil {
 		return nil, fmt.Errorf(
@@ -234,9 +234,15 @@ func (o *ServerOptions) config() (*tls.Config, error) {
 		clientAuth = tls.RequireAnyClientCert
 	}
 	config := &tls.Config{
-		ClientAuth:     clientAuth,
-		Certificates:   o.Certificates,
-		GetCertificate: o.GetCertificate,
+		ClientAuth:   clientAuth,
+		Certificates: o.Certificates,
+	}
+	if o.GetCertificates != nil {
+		// GetCertificate is only able to perform SNI logic for go1.10 and above.
+		// It will return the first certificate in o.GetCertificates for go1.9.
+		config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			return buildGetCertificates(clientHello, o)
+		}
 	}
 	if clientCAs != nil {
 		config.ClientCAs = clientCAs
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 5f8c25f681b3..3f9f6218035a 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -197,7 +197,7 @@ func TestEnd2End(t *testing.T) {
 		clientVerifyFunc CustomVerificationFunc
 		clientVType      VerificationType
 		serverCert       []tls.Certificate
-		serverGetCert    func(*tls.ClientHelloInfo) (*tls.Certificate, error)
+		serverGetCert    func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
 		serverRoot       *x509.CertPool
 		serverGetRoot    func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		serverVerifyFunc CustomVerificationFunc
@@ -271,12 +271,12 @@ func TestEnd2End(t *testing.T) {
 				return &VerificationResults{}, nil
 			},
 			clientVType: CertVerification,
-			serverGetCert: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
-					return &cs.serverPeer1, nil
+					return []*tls.Certificate{&cs.serverPeer1}, nil
 				default:
-					return &cs.serverPeer2, nil
+					return []*tls.Certificate{&cs.serverPeer2}, nil
 				}
 			},
 			serverRoot: cs.serverTrust1,
@@ -336,12 +336,12 @@ func TestEnd2End(t *testing.T) {
 				return nil, fmt.Errorf("custom authz check fails")
 			},
 			clientVType: CertVerification,
-			serverGetCert: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
-					return &cs.serverPeer1, nil
+					return []*tls.Certificate{&cs.serverPeer1}, nil
 				default:
-					return &cs.serverPeer2, nil
+					return []*tls.Certificate{&cs.serverPeer2}, nil
 				}
 			},
 			serverRoot: cs.serverTrust1,
@@ -388,8 +388,8 @@ func TestEnd2End(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			// Start a server using ServerOptions in another goroutine.
 			serverOptions := &ServerOptions{
-				Certificates:   test.serverCert,
-				GetCertificate: test.serverGetCert,
+				Certificates:    test.serverCert,
+				GetCertificates: test.serverGetCert,
 				RootCertificateOptions: RootCertificateOptions{
 					RootCACerts: test.serverRoot,
 					GetRootCAs:  test.serverGetRoot,
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 263bf7df418c..b2514547c174 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -102,7 +102,7 @@ func TestClientServerHandshake(t *testing.T) {
 		clientExpectHandshakeError bool
 		serverMutualTLS            bool
 		serverCert                 []tls.Certificate
-		serverGetCert              func(*tls.ClientHelloInfo) (*tls.Certificate, error)
+		serverGetCert              func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
 		serverRoot                 *x509.CertPool
 		serverGetRoot              func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		serverVerifyFunc           CustomVerificationFunc
@@ -279,8 +279,8 @@ func TestClientServerHandshake(t *testing.T) {
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-				return &serverPeerCert, nil
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverPeerCert}, nil
 			},
 			serverGetRoot:    getRootCAsForServer,
 			serverVerifyFunc: serverVerifyFunc,
@@ -300,8 +300,8 @@ func TestClientServerHandshake(t *testing.T) {
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-				return &serverPeerCert, nil
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverPeerCert}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -322,8 +322,8 @@ func TestClientServerHandshake(t *testing.T) {
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
-			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-				return &serverPeerCert, nil
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverPeerCert}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -344,8 +344,8 @@ func TestClientServerHandshake(t *testing.T) {
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-				return &clientPeerCert, nil
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&clientPeerCert}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -366,8 +366,8 @@ func TestClientServerHandshake(t *testing.T) {
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
-			serverGetCert: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-				return &serverPeerCert, nil
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverPeerCert}, nil
 			},
 			serverGetRoot:     getRootCAsForClient,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -402,8 +402,8 @@ func TestClientServerHandshake(t *testing.T) {
 			}
 			// Start a server using ServerOptions in another goroutine.
 			serverOptions := &ServerOptions{
-				Certificates:   test.serverCert,
-				GetCertificate: test.serverGetCert,
+				Certificates:    test.serverCert,
+				GetCertificates: test.serverGetCert,
 				RootCertificateOptions: RootCertificateOptions{
 					RootCACerts: test.serverRoot,
 					GetRootCAs:  test.serverGetRoot,
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index 392985d74469..f21b428ef4c0 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -4,7 +4,7 @@ go 1.13
 
 require (
 	github.com/golang/protobuf v1.3.5 // indirect
-	github.com/google/go-cmp v0.4.0 // indirect
+	github.com/google/go-cmp v0.4.0
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
 	golang.org/x/text v0.3.3 // indirect
diff --git a/security/advancedtls/sni_110.go b/security/advancedtls/sni_110.go
new file mode 100644
index 000000000000..5c9a6ae13a0f
--- /dev/null
+++ b/security/advancedtls/sni_110.go
@@ -0,0 +1,53 @@
+// +build go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+	"fmt"
+)
+
+// buildGetCertificates returns the certificate that matches the SNI field
+// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates.
+func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
+	if o.GetCertificates == nil {
+		return nil, fmt.Errorf("function GetCertificates must be specified")
+	}
+	certificates, err := o.GetCertificates(clientHello)
+	if err != nil {
+		return nil, err
+	}
+	if len(certificates) == 0 {
+		return nil, fmt.Errorf("no certificates configured")
+	}
+	// If users pass in only one certificate, return that certificate.
+	if len(certificates) == 1 {
+		return certificates[0], nil
+	}
+	// Choose the SNI certificate using SupportsCertificate.
+	for _, cert := range certificates {
+		if err := clientHello.SupportsCertificate(cert); err == nil {
+			return cert, nil
+		}
+	}
+	// If nothing matches, return the first certificate.
+	return certificates[0], nil
+}
diff --git a/security/advancedtls/sni_before_110.go b/security/advancedtls/sni_before_110.go
new file mode 100644
index 000000000000..180e3a05d494
--- /dev/null
+++ b/security/advancedtls/sni_before_110.go
@@ -0,0 +1,41 @@
+// +build !go1.10
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+	"fmt"
+)
+
+// buildGetCertificates returns the first element of o.GetCertificates.
+func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
+	if o.GetCertificates == nil {
+		return nil, fmt.Errorf("function GetCertificates must be specified")
+	}
+	certificates, err := o.GetCertificates(clientHello)
+	if err != nil {
+		return nil, err
+	}
+	if len(certificates) == 0 {
+		return nil, fmt.Errorf("no certificates configured")
+	}
+	return certificates[0], nil
+}
diff --git a/security/advancedtls/sni_test_110.go b/security/advancedtls/sni_test_110.go
new file mode 100644
index 000000000000..130ccde59050
--- /dev/null
+++ b/security/advancedtls/sni_test_110.go
@@ -0,0 +1,108 @@
+// +build go1.10
+
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/security/advancedtls/testdata"
+)
+
+// TestGetCertificatesSNI tests SNI logic for go1.10 and above.
+func TestGetCertificatesSNI(t *testing.T) {
+	// Load server certificates for setting the serverGetCert callback function.
+	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err)
+	}
+	serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err)
+	}
+	serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err)
+	}
+
+	tests := []struct {
+		desc          string
+		serverGetCert func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
+		serverName    string
+		wantCert      tls.Certificate
+	}{
+		{
+			desc: "Select serverCert1",
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+			},
+			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
+			serverName: "foo.bar.com",
+			wantCert:   serverCert1,
+		},
+		{
+			desc: "Select serverCert2",
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+			},
+			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
+			serverName: "foo.bar.server2.com",
+			wantCert:   serverCert2,
+		},
+		{
+			desc: "Select serverCert3",
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+			},
+			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
+			serverName: "google.com",
+			wantCert:   serverCert3,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			serverOptions := &ServerOptions{
+				GetCertificates: test.serverGetCert,
+			}
+			serverConfig, err := serverOptions.config()
+			if err != nil {
+				t.Fatalf("serverOptions.config() failed: %v", err)
+			}
+			pointFormatUncompressed := uint8(0)
+			clientHello := &tls.ClientHelloInfo{
+				CipherSuites:      []uint16{tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA},
+				ServerName:        test.serverName,
+				SupportedCurves:   []tls.CurveID{tls.CurveP256},
+				SupportedPoints:   []uint8{pointFormatUncompressed},
+				SupportedVersions: []uint16{tls.VersionTLS10},
+			}
+			gotCertificate, err := serverConfig.GetCertificate(clientHello)
+			if err != nil {
+				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
+			}
+			if !cmp.Equal(gotCertificate, test.wantCert, cmp.AllowUnexported(tls.Certificate{})) {
+				t.Errorf("GetCertificates() = %v, want %v", gotCertificate, test.wantCert)
+			}
+		})
+	}
+}
diff --git a/security/advancedtls/sni_test_before_110.go b/security/advancedtls/sni_test_before_110.go
new file mode 100644
index 000000000000..e31e2e6ee759
--- /dev/null
+++ b/security/advancedtls/sni_test_before_110.go
@@ -0,0 +1,108 @@
+// +build !go1.10
+
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/security/advancedtls/testdata"
+)
+
+// TestGetCertificatesSNI tests SNI logic for go1.9.
+func TestGetCertificatesSNI(t *testing.T) {
+	// Load server certificates for setting the serverGetCert callback function.
+	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err)
+	}
+	serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err)
+	}
+	serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err)
+	}
+
+	tests := []struct {
+		desc          string
+		serverGetCert func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
+		serverName    string
+		wantCert      tls.Certificate
+	}{
+		{
+			desc: "Select serverCert1",
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+			},
+			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
+			serverName: "foo.bar.com",
+			wantCert:   serverCert1,
+		},
+		{
+			desc: "Select serverCert2",
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+			},
+			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
+			serverName: "foo.bar.server2.com",
+			wantCert:   serverCert1,
+		},
+		{
+			desc: "Select serverCert3",
+			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+			},
+			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
+			serverName: "google.com",
+			wantCert:   serverCert1,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			serverOptions := &ServerOptions{
+				GetCertificates: test.serverGetCert,
+			}
+			serverConfig, err := serverOptions.config()
+			if err != nil {
+				t.Fatalf("serverOptions.config() failed: %v", err)
+			}
+			pointFormatUncompressed := uint8(0)
+			clientHello := &tls.ClientHelloInfo{
+				CipherSuites:      []uint16{tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA},
+				ServerName:        test.serverName,
+				SupportedCurves:   []tls.CurveID{tls.CurveP256},
+				SupportedPoints:   []uint8{pointFormatUncompressed},
+				SupportedVersions: []uint16{tls.VersionTLS10},
+			}
+			gotCertificate, err := serverConfig.GetCertificate(clientHello)
+			if err != nil {
+				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
+			}
+			if !cmp.Equal(gotCertificate, test.wantCert, cmp.AllowUnexported(tls.Certificate{})) {
+				t.Errorf("GetCertificates() = %v, want %v", gotCertificate, test.wantCert)
+			}
+		})
+	}
+}
diff --git a/security/advancedtls/testdata/server_cert_1.txt b/security/advancedtls/testdata/server_cert_1.txt
new file mode 100644
index 000000000000..5367569d0528
--- /dev/null
+++ b/security/advancedtls/testdata/server_cert_1.txt
@@ -0,0 +1,90 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 3 (0x3)
+        Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C = US, ST = CA, L = SVL, O = Internet Widgits Pty Ltd
+        Validity
+            Not Before: Nov  4 21:43:00 2019 GMT
+            Not After : Aug 18 21:43:00 2293 GMT
+        Subject: C = US, ST = CA, L = DUMMYCITY, O = Internet Widgits Pty Ltd, CN = foo.bar.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                RSA Public-Key: (4096 bit)
+                Modulus:
+                    00:ec:3f:24:2d:91:3a:bd:c3:fc:15:72:42:b3:fb:
+                    28:e6:04:a3:be:26:20:e6:ea:30:a8:aa:48:78:36:
+                    0e:0b:99:29:3b:4b:f9:f1:d5:bf:bd:0c:13:7c:ea:
+                    52:06:f4:bc:34:9e:2b:c0:b4:82:2c:87:fa:2f:e2:
+                    cd:7c:d7:b9:e1:8f:04:71:6d:85:77:ae:18:40:e4:
+                    b1:3a:4a:6b:e5:33:bf:3e:65:db:cf:94:64:87:1a:
+                    20:46:c0:37:3a:9f:93:3f:d4:4f:ac:c4:e4:e0:28:
+                    b6:0f:28:53:2a:cf:b9:fe:50:f2:ef:47:dc:7e:b6:
+                    60:c2:47:85:b8:cb:ca:48:5b:fa:9f:8a:97:30:01:
+                    f4:b3:51:0f:68:e1:60:ab:2f:a0:ad:fc:f0:10:4f:
+                    60:e1:92:db:be:83:04:5c:40:87:ce:51:3e:9a:9e:
+                    d6:1c:1b:19:cb:8c:c2:6c:57:74:6f:7b:af:94:3d:
+                    53:ad:17:a5:99:69:7c:41:f5:3e:7a:5b:48:c7:78:
+                    ff:d7:3b:a8:1f:f7:30:e7:83:26:78:e2:cb:a2:8f:
+                    58:92:61:cd:ca:e9:b8:d1:80:c0:40:58:e9:d8:d3:
+                    42:64:82:8f:e4:0c:b9:b1:36:db:9f:65:3f:3f:5b:
+                    24:59:31:b3:60:0c:fa:41:5a:1b:b8:9d:ec:99:37:
+                    90:fa:b5:e7:3f:cb:7c:e0:f9:ed:ea:27:ce:15:24:
+                    c7:77:3b:45:45:2d:19:8e:2e:7f:65:0e:85:df:66:
+                    50:69:24:2c:a4:6a:07:e5:3f:eb:28:84:53:94:4d:
+                    5f:9c:a8:65:a6:50:4c:c0:35:06:40:6a:a5:62:b1:
+                    93:60:e5:1c:85:28:34:9b:29:81:6f:e2:4f:cd:15:
+                    30:b9:19:d7:4b:bb:30:0c:4b:2d:64:fe:3b:dd:0e:
+                    a4:25:2c:4a:5c:de:d7:74:1f:5e:93:7b:1c:e8:c8:
+                    fa:72:1f:4a:eb:8d:3f:98:e4:55:98:b8:e0:8a:29:
+                    92:33:af:75:6b:05:84:05:d3:0c:2c:07:78:bc:0e:
+                    b2:6d:a7:00:35:c4:53:1f:7b:e6:ba:07:72:a8:24:
+                    c1:0a:a7:c4:46:e6:f2:6f:3a:79:23:00:0b:b8:e5:
+                    1f:e0:e2:ee:c6:13:a3:57:d9:86:1a:95:f7:a3:04:
+                    f1:46:d5:5f:21:d2:aa:d2:30:fb:f6:cb:e0:da:24:
+                    c6:c3:30:2f:d2:1f:21:fe:bc:0f:99:ac:ac:9b:65:
+                    9b:e4:83:9a:00:b8:2f:40:fc:3b:42:d3:7a:e8:b7:
+                    52:d7:f4:67:2a:a5:f7:eb:78:f1:0a:56:8b:56:12:
+                    d5:48:d8:48:70:ab:b8:69:5a:21:d3:71:b0:59:9d:
+                    17:b4:4b
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Key Identifier: 
+                C0:82:DA:FA:69:46:30:AE:FF:6F:CD:BB:93:49:94:A6:D0:E2:17:EB
+            X509v3 Authority Key Identifier: 
+                keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06
+
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            X509v3 Key Usage: 
+                Digital Signature, Key Encipherment
+    Signature Algorithm: sha256WithRSAEncryption
+         36:fd:cf:ec:f5:20:4b:52:dc:2e:38:f3:92:b1:e4:b6:a1:06:
+         86:aa:2d:c0:e6:f5:0a:58:97:a9:e3:be:13:09:61:79:ed:d4:
+         41:83:26:ad:ee:0b:43:83:d1:dd:19:1a:e8:7b:b2:1f:fe:d4:
+         c1:57:7d:6d:6b:d4:42:ea:7d:cd:34:8c:a4:1f:5b:3b:fa:de:
+         bb:2f:ae:56:b6:18:e5:53:a9:a3:99:58:ad:36:be:19:54:61:
+         0d:52:b6:a7:53:fc:60:e5:ff:f5:7f:82:3f:c1:49:06:cd:b2:
+         af:25:ee:de:bd:e0:e5:5e:ad:0b:dc:2e:b1:ec:7a:52:6f:9d:
+         e0:b9:84:18:db:49:53:ee:df:93:ee:8b:9d:9b:8e:3b:2a:82:
+         86:7f:45:c8:dd:d1:b0:40:17:ed:63:52:a1:5b:6e:d3:5c:a2:
+         72:05:fb:3a:39:71:0d:b4:2c:9d:15:23:1b:1f:8d:ac:89:dc:
+         c9:56:f2:19:c7:f3:2f:bb:d5:de:40:17:f1:52:ea:e8:93:ff:
+         56:43:f5:1d:cb:c0:51:52:25:d7:b0:81:a9:0e:4d:92:24:e7:
+         10:81:c7:31:26:ac:cb:66:c1:3f:f6:5f:69:7b:74:87:0d:b0:
+         8c:27:d4:24:29:59:e9:5b:a2:cb:0c:c0:f5:9b:1d:42:38:6b:
+         e3:c3:43:1e:ba:df:b1:51:0a:b7:33:55:26:39:01:2f:9f:c7:
+         88:ac:2f:4a:89:f3:69:de:72:43:48:49:08:59:36:86:84:09:
+         db:6a:82:84:3e:71:6a:9d:f9:bd:d8:b5:1e:7c:2c:29:e1:27:
+         45:4c:47:5b:88:b8:e6:fa:9d:9b:ff:d4:e9:8d:2d:5e:64:7f:
+         27:87:b2:8c:d8:7e:f5:52:3c:c4:d8:30:03:24:d7:ac:f8:53:
+         91:80:98:42:24:5a:6b:cb:34:48:57:e0:82:ac:96:d9:55:6c:
+         c2:c3:8c:19:7c:56:39:0a:a8:f1:b8:77:64:70:83:a8:04:c8:
+         3a:5d:0b:00:4c:e5:ba:f1:40:e5:57:cd:d9:67:48:21:e9:9c:
+         d3:f2:b8:01:b8:d1:c0:d1:3a:44:c0:97:db:e6:bc:8f:2e:33:
+         d5:e2:38:3d:d7:7b:50:13:01:36:28:61:cc:28:98:3c:f8:21:
+         5d:8c:fe:f5:d0:ab:e0:60:ec:36:22:8d:0b:71:30:1b:3d:56:
+         ae:96:e9:d2:89:c2:43:8b:ef:25:b7:d6:0d:82:e6:5a:c6:91:
+         8a:ad:8c:28:2a:2b:5c:4e:a1:de:cb:7d:cb:29:11:a2:66:c8:
+         a1:33:35:75:16:fe:28:0b:78:31:0a:1f:fa:d0:a8:f4:f1:69:
+         c7:97:1e:5d:fb:53:08:b5
diff --git a/security/advancedtls/testdata/server_cert_2.txt b/security/advancedtls/testdata/server_cert_2.txt
new file mode 100644
index 000000000000..8962204ff348
--- /dev/null
+++ b/security/advancedtls/testdata/server_cert_2.txt
@@ -0,0 +1,91 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 7 (0x7)
+        Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C = US, ST = CA, O = Internet Widgits Pty Ltd, CN = foo.bar.client2.trust.com
+        Validity
+            Not Before: Jan  9 22:51:54 2020 GMT
+            Not After : Oct 23 22:51:54 2293 GMT
+        Subject: C = US, ST = CA, O = Internet Widgits Pty Ltd, CN = foo.bar.server2.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                RSA Public-Key: (4096 bit)
+                Modulus:
+                    00:b1:0b:d3:7e:5b:61:30:db:b0:5f:3f:6d:d2:e0:
+                    3b:c6:4c:88:95:f5:7e:fd:cd:aa:20:5d:08:b9:6e:
+                    41:db:c4:ed:0d:f8:bc:cb:b4:ee:c5:87:11:05:a0:
+                    ac:12:3b:4e:0b:4c:e4:43:e4:17:89:c1:ae:b4:13:
+                    58:1c:31:58:6a:f2:01:ed:df:66:e9:f9:2e:9c:c5:
+                    85:e6:02:db:36:f4:f3:07:39:75:30:f1:b5:55:5b:
+                    46:2f:87:b0:d4:a0:ab:57:df:30:45:ae:bd:b0:49:
+                    9a:fc:ba:5e:bc:d0:5d:86:f4:24:45:4a:d5:4d:5b:
+                    b6:ba:e8:b7:a1:3b:c3:2f:46:2e:b3:ad:2c:63:03:
+                    df:cb:f4:56:62:91:bd:bc:23:00:af:a2:7a:3d:6f:
+                    f1:33:81:60:0e:bc:20:f5:8a:49:5f:ec:58:bc:64:
+                    d5:47:36:a0:2b:b8:1f:76:25:01:89:3e:ff:52:69:
+                    95:03:8f:bb:14:2f:1a:38:a3:9f:c1:45:20:22:77:
+                    70:97:5e:25:51:b8:3d:5d:89:7a:bb:15:12:cd:1d:
+                    96:d2:9c:72:67:12:85:72:6e:27:7a:ef:25:da:af:
+                    49:26:8d:eb:a0:34:a4:4d:64:c3:63:33:77:5d:ad:
+                    53:c7:ee:51:32:7b:cc:43:bb:86:8d:f9:52:ba:35:
+                    23:0e:30:5d:dc:3b:25:63:c1:e3:5f:4b:b2:02:fc:
+                    fe:5b:18:7f:84:aa:f3:71:e4:16:b5:98:bc:73:c5:
+                    58:13:41:38:eb:f3:a2:fa:8c:98:bd:f1:10:ee:b6:
+                    fe:7e:a5:81:c7:5e:f2:72:54:8e:db:09:f0:35:42:
+                    ca:b7:86:c2:48:b2:c6:18:08:ac:d1:f0:5d:de:b0:
+                    b8:25:8b:3b:bd:61:48:0f:71:3f:ed:97:72:02:c9:
+                    44:5d:0c:00:fc:30:ca:5d:1c:e5:13:1b:3a:d0:ce:
+                    d9:36:a0:db:f5:c2:ad:a6:95:26:4e:7b:29:2d:fc:
+                    c4:04:1d:47:6e:03:59:68:1e:7a:20:6d:e8:a8:e1:
+                    3c:57:59:f8:3d:2f:16:61:7e:24:e5:13:ca:48:0a:
+                    e6:f0:60:a3:2d:93:0b:8f:93:eb:b5:d1:06:26:52:
+                    c0:63:1f:fc:9b:73:fe:91:c3:04:40:32:8d:09:d5:
+                    9e:c4:f6:0b:61:3d:9f:a1:d7:94:a2:e1:3d:b6:bb:
+                    60:26:74:89:33:25:18:0f:c3:88:db:10:5e:a0:5b:
+                    f4:ee:d0:18:ab:36:50:c5:44:9b:6d:ba:ea:e2:6e:
+                    52:3a:55:49:a3:72:ae:04:af:1d:f6:f2:83:27:17:
+                    8b:9a:98:0a:f5:44:b1:c8:f2:a9:c8:ed:b0:75:ca:
+                    52:25:f3
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Key Identifier: 
+                74:BD:18:0B:32:AF:D0:51:8E:4C:4C:8D:B2:F6:4E:B8:6D:AB:BD:BA
+            X509v3 Authority Key Identifier: 
+                keyid:01:74:A9:44:61:3D:7A:BB:C2:32:CD:D0:ED:20:DA:3A:C4:C6:02:E8
+
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            X509v3 Key Usage: 
+                Digital Signature, Key Encipherment
+    Signature Algorithm: sha256WithRSAEncryption
+         b5:63:0c:d8:ed:af:74:2d:4c:94:36:41:05:2a:f2:ef:45:e5:
+         6a:0c:76:0c:f3:90:25:e0:54:56:f3:26:23:95:7e:24:74:6b:
+         fd:02:0a:bc:33:ba:e8:e8:8f:a3:b3:85:2e:59:4c:cf:e3:85:
+         1a:d6:70:5c:7c:86:e2:7a:11:99:a8:fa:43:9a:bf:50:54:00:
+         9e:6a:7b:72:7f:c5:20:89:6e:18:6c:46:64:ce:44:44:47:4d:
+         87:b5:fc:cf:f3:b9:9f:45:a3:cb:b0:91:00:96:2d:29:68:8b:
+         ff:c7:e0:f1:b7:8d:31:c2:01:be:5b:51:1d:af:42:b1:17:22:
+         bc:91:e4:d9:b9:96:6d:64:40:79:6c:71:ed:f6:e5:49:16:0a:
+         e3:bc:18:95:2e:89:ba:c4:a5:ce:ba:ab:3a:32:eb:bc:d8:91:
+         cd:f2:ee:d1:fc:67:3a:51:00:92:bd:b8:68:0b:54:04:d5:07:
+         0b:97:11:2c:42:64:7c:47:c1:68:b4:eb:21:c4:e4:ad:17:a7:
+         16:b9:e0:e6:cd:04:c6:89:36:40:d4:4b:c3:f7:7e:26:6b:3a:
+         d7:68:b3:b2:da:00:65:13:c8:fa:d0:1c:2e:10:ba:71:3e:0f:
+         aa:8b:d0:ff:b7:3e:83:9c:bc:b3:d1:52:0c:9f:3f:21:4a:10:
+         dc:8f:ab:38:45:d4:2c:2a:15:2d:71:45:fe:91:a2:d8:d9:dd:
+         0c:dc:a7:d9:cd:1b:f5:35:fe:14:ba:c5:1f:ed:ee:fb:87:cc:
+         87:a1:08:c2:2e:ff:5d:af:b3:3d:6e:11:94:79:0b:28:e6:83:
+         4e:fc:28:8f:7f:00:85:79:7f:3a:d1:07:ee:6e:fa:94:c4:0b:
+         4b:2c:05:b1:68:00:e8:37:bc:b8:b2:03:5c:5a:ca:13:f2:68:
+         57:df:ac:fc:da:be:27:24:7e:6d:c4:a9:53:2d:f2:43:0e:30:
+         9c:82:d5:fb:f1:a2:0a:83:e0:a5:d8:9f:09:3e:99:c8:39:d6:
+         69:6d:d6:c2:27:70:59:05:3c:3c:7d:d6:41:6a:b4:9c:1f:70:
+         7e:3e:ee:6f:67:de:95:1d:eb:31:8b:11:c8:0d:a1:25:4e:08:
+         ef:3a:11:2d:a7:98:0d:a1:d9:30:2d:da:d2:a0:05:6b:34:38:
+         a6:87:b2:bd:0f:9c:51:cc:e0:2e:a2:1b:a3:a0:a6:eb:1f:0a:
+         22:70:59:f0:0b:c9:bd:94:4e:1d:65:3b:99:5d:8e:6c:18:82:
+         1d:b5:cc:6f:14:21:c4:89:07:9b:81:1d:9a:79:ff:bf:fd:ce:
+         e4:77:11:0f:47:21:dc:d9:79:f3:40:26:56:5c:b4:86:32:8e:
+         28:b9:14:e7:b3:fe:86:47
+         
\ No newline at end of file
diff --git a/security/advancedtls/testdata/server_cert_3.pem b/security/advancedtls/testdata/server_cert_3.pem
new file mode 100644
index 000000000000..bfd2d095b227
--- /dev/null
+++ b/security/advancedtls/testdata/server_cert_3.pem
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIDyTCCArGgAwIBAgIUeoNdEiqhXVkpcYsmHaKiVS5W/tQwDQYJKoZIhvcNAQEL
+BQAwPjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMREwDwYDVQQHDAhTYW4gSm9z
+ZTEPMA0GA1UECgwGR29vZ2xlMB4XDTIwMDcxNjE2NTMxOVoXDTQwMDcxMTE2NTMx
+OVowgZMxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTERMA8GA1UEBwwIU2FuIEpv
+c2UxEjAQBgNVBAoMCUVuZCBQb2ludDEOMAwGA1UECwwFSW5mcmExIjAgBgkqhkiG
+9w0BCQEWE2NpbmR5eHVlQGdvb2dsZS5jb20xHDAaBgNVBAMME2Zvby5iYXIuc2Vy
+dmVyMy5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDcIyJEt3ZA
+xPn7H5f/IwXS8NcoAXWP8L6rWndcg+EWayx7W+wmUsFKGSFGzrFPPCFmKO8MrQqp
+8LSAxHAVtOC6Uw+INWJJw9BRlx2nvV7hfbqu3OnPkPVkN/siUQCqnEKJQHliNT9X
+Dl4/Mav75uQSWb3Vfi3KtG7mzPFNNbbe4yfHyGbC4e9RtKkGimDSJ413s3m4+scD
+vtpCcCXj9XXZNdCwD1CL3kNdmOdhgfkDBP+AMLBFKZKqpCo6m0s4JJTiej13dc27
+wTrnkFm1CP77SV+kQlWg5DAcVXYJkN9FqNqExqIPS/SxMk7H+3qSQACbttQK9UmC
+n3qR3pGbwqzNAgMBAAGjaTBnMB8GA1UdIwQYMBaAFG4bi8k0dOd7jSpPQQ6YUDAU
+ARaxMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgTwMCwGA1UdEQQlMCOCCmdvb2dsZS5j
+b22CCWFwcGxlLmNvbYIKYW1hem9uLmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAn5aW
+HEHNTDmcgC25oEtCj+IkoAslgFze4ZqkSz0HCzx76vj3AfMmIEvqB0W74wKqeZgm
+V0D7I0xHkM3ILH4RjoCotpol3nLooIPFflA6Z1ILTRZl8mE5kfBSHzKdPS0egOf6
+kgrNYgJjBEtGNsmq8RKxAHVVAPgH88di0JnQDN5LcV9ZBKTQM2R7EM6a8eWD/Jsi
+uujbNtdNERssSBV+Oil93MbsEcOT1RSKKxAiVvkHkR+45GRB889xBnqelcDVqcMK
+Vcdp6X7aD5/Bu/4fq9AZlcHSEQDixNtjp/pQR0B5FsCGrb5OAz0B2t9jykDiIyj4
+4lxhQz8ykXf7ue0/ag==
+-----END CERTIFICATE-----
diff --git a/security/advancedtls/testdata/server_cert_3.txt b/security/advancedtls/testdata/server_cert_3.txt
new file mode 100644
index 000000000000..e62c99cbf587
--- /dev/null
+++ b/security/advancedtls/testdata/server_cert_3.txt
@@ -0,0 +1,60 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number:
+            7a:83:5d:12:2a:a1:5d:59:29:71:8b:26:1d:a2:a2:55:2e:56:fe:d4
+        Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C = US, ST = CA, L = San Jose, O = Google
+        Validity
+            Not Before: Jul 16 16:53:19 2020 GMT
+            Not After : Jul 11 16:53:19 2040 GMT
+        Subject: C = US, ST = CA, L = San Jose, O = End Point, OU = Infra, emailAddress = cindyxue@google.com, CN = foo.bar.server3.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                RSA Public-Key: (2048 bit)
+                Modulus:
+                    00:dc:23:22:44:b7:76:40:c4:f9:fb:1f:97:ff:23:
+                    05:d2:f0:d7:28:01:75:8f:f0:be:ab:5a:77:5c:83:
+                    e1:16:6b:2c:7b:5b:ec:26:52:c1:4a:19:21:46:ce:
+                    b1:4f:3c:21:66:28:ef:0c:ad:0a:a9:f0:b4:80:c4:
+                    70:15:b4:e0:ba:53:0f:88:35:62:49:c3:d0:51:97:
+                    1d:a7:bd:5e:e1:7d:ba:ae:dc:e9:cf:90:f5:64:37:
+                    fb:22:51:00:aa:9c:42:89:40:79:62:35:3f:57:0e:
+                    5e:3f:31:ab:fb:e6:e4:12:59:bd:d5:7e:2d:ca:b4:
+                    6e:e6:cc:f1:4d:35:b6:de:e3:27:c7:c8:66:c2:e1:
+                    ef:51:b4:a9:06:8a:60:d2:27:8d:77:b3:79:b8:fa:
+                    c7:03:be:da:42:70:25:e3:f5:75:d9:35:d0:b0:0f:
+                    50:8b:de:43:5d:98:e7:61:81:f9:03:04:ff:80:30:
+                    b0:45:29:92:aa:a4:2a:3a:9b:4b:38:24:94:e2:7a:
+                    3d:77:75:cd:bb:c1:3a:e7:90:59:b5:08:fe:fb:49:
+                    5f:a4:42:55:a0:e4:30:1c:55:76:09:90:df:45:a8:
+                    da:84:c6:a2:0f:4b:f4:b1:32:4e:c7:fb:7a:92:40:
+                    00:9b:b6:d4:0a:f5:49:82:9f:7a:91:de:91:9b:c2:
+                    ac:cd
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Authority Key Identifier: 
+                keyid:6E:1B:8B:C9:34:74:E7:7B:8D:2A:4F:41:0E:98:50:30:14:01:16:B1
+
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            X509v3 Key Usage: 
+                Digital Signature, Non Repudiation, Key Encipherment, Data Encipherment
+            X509v3 Subject Alternative Name: 
+                DNS:google.com, DNS:apple.com, DNS:amazon.com
+    Signature Algorithm: sha256WithRSAEncryption
+         9f:96:96:1c:41:cd:4c:39:9c:80:2d:b9:a0:4b:42:8f:e2:24:
+         a0:0b:25:80:5c:de:e1:9a:a4:4b:3d:07:0b:3c:7b:ea:f8:f7:
+         01:f3:26:20:4b:ea:07:45:bb:e3:02:aa:79:98:26:57:40:fb:
+         23:4c:47:90:cd:c8:2c:7e:11:8e:80:a8:b6:9a:25:de:72:e8:
+         a0:83:c5:7e:50:3a:67:52:0b:4d:16:65:f2:61:39:91:f0:52:
+         1f:32:9d:3d:2d:1e:80:e7:fa:92:0a:cd:62:02:63:04:4b:46:
+         36:c9:aa:f1:12:b1:00:75:55:00:f8:07:f3:c7:62:d0:99:d0:
+         0c:de:4b:71:5f:59:04:a4:d0:33:64:7b:10:ce:9a:f1:e5:83:
+         fc:9b:22:ba:e8:db:36:d7:4d:11:1b:2c:48:15:7e:3a:29:7d:
+         dc:c6:ec:11:c3:93:d5:14:8a:2b:10:22:56:f9:07:91:1f:b8:
+         e4:64:41:f3:cf:71:06:7a:9e:95:c0:d5:a9:c3:0a:55:c7:69:
+         e9:7e:da:0f:9f:c1:bb:fe:1f:ab:d0:19:95:c1:d2:11:00:e2:
+         c4:db:63:a7:fa:50:47:40:79:16:c0:86:ad:be:4e:03:3d:01:
+         da:df:63:ca:40:e2:23:28:f8:e2:5c:61:43:3f:32:91:77:fb:
+         b9:ed:3f:6a
diff --git a/security/advancedtls/testdata/server_key_3.pem b/security/advancedtls/testdata/server_key_3.pem
new file mode 100644
index 000000000000..ae8ff1a75b4d
--- /dev/null
+++ b/security/advancedtls/testdata/server_key_3.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDcIyJEt3ZAxPn7
+H5f/IwXS8NcoAXWP8L6rWndcg+EWayx7W+wmUsFKGSFGzrFPPCFmKO8MrQqp8LSA
+xHAVtOC6Uw+INWJJw9BRlx2nvV7hfbqu3OnPkPVkN/siUQCqnEKJQHliNT9XDl4/
+Mav75uQSWb3Vfi3KtG7mzPFNNbbe4yfHyGbC4e9RtKkGimDSJ413s3m4+scDvtpC
+cCXj9XXZNdCwD1CL3kNdmOdhgfkDBP+AMLBFKZKqpCo6m0s4JJTiej13dc27wTrn
+kFm1CP77SV+kQlWg5DAcVXYJkN9FqNqExqIPS/SxMk7H+3qSQACbttQK9UmCn3qR
+3pGbwqzNAgMBAAECggEAU0Te84tKKdnYjUs4HYRL8ay0VienJpl0JjEEMXSZMfe8
+TbVJsH1hK/wxgC0zGLuwDoqxUeQqwnmQbZzgoPVYhGJi360BztFI/XPh/c8+EqGS
+eg6KSr+UcyJR1ns5e0+8Q1qmD6YAnZeLwu+xFIoT/3T+v8EI5UI3KQqgxAnrcIdc
+RWqIwWLkkPm1QVRYsvRaTvmgFd2LcIT9AdIruP/VsqF3GEzvEQSr0lgmwOe0izLb
+HKfZr+2JwOppwTLGhKo1wUyUUsglXCBOcFYAA03xdviQWBuCeKXamNrdB7M0T4zZ
+THheNRQq2g7bTYnncCYKcrNa5VY5wmHY767mf6ahAQKBgQD+AGk21CsRgiKCGzUI
+wTRynutAMkX55U1bud2+OMpzH9omvnccX5ezq5WTw/jZfz+jrUBF8YKSTtb3InUg
+yXcpJ/XjRDFMZp1Avy/44rOlYg1QYMD4JK96f8bbd1yej5NVw45V9synaJHuvoDV
+bkbZu00S0X8Pgvlh354MUH3CLQKBgQDd3oQdKMZgsX8gtmcQfQoEZ0VvlOYmTM5W
+Kw+24iGrBkfBt+NuKx8qm1CpoFGx4G2+TgttMywjF9RG3R2uGqbJZbCOAXzjRMQJ
+L9PuTiGAdYD3fA5cTmnrrNEhPydtRhF2M3p8FFeQtwsEBYreXux25rbmVOYTFMgJ
+hVUW9RdZIQKBgQDwEYdgMQw70hm3iuuHSMS/iQCkfl+xH08MYRH6FkcSpIpVkDOX
+96m0QXpwXQs41pJZqwhSkz9r9WQr1L+Lq58aoRBAK1XE9j+u0IUQ4YQVziTzUV9R
+qarJRze2eoxpuR3yM5C2IzuvBqDXW+r8zuvcIrFoFeXXzVzTar1AulsCSQKBgQCa
+UD+3QDrp2co/6F26vB0RfvpuZzPEA7undv/RBWrBVvblp46Je3iL28a4lAb+Hsh1
+ijasVuEl71b3iqcwBt1mSlIIEsTYFWX7tcZDgxgODqwKdcBPN0K4ZlR2OUSk3g0b
+Fybj0gotXwJMY8Z4b7Er6b/gZ8A2GUggRxotg34fwQKBgEh6a88SgTyu1KcR2yzN
+Zbs8MEfZ8hqfUj+GL0+6y9KQd4uSIngyHWGNETE9dCObNz5HEdJIpUNvB3vfN0Vk
+f1EEPDQLQKJjii7jZ9U9XfPVaUhqIVH3Aupmb30H3AQw7XOF23g63k9Yg1FWtDT6
+4CuBsUvjXywL4yHrWK+BuSXF
+-----END PRIVATE KEY-----

From 067255ffaeb0e7f93d875b83033a0c5e02dd6bcd Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 28 Jul 2020 10:33:47 -0700
Subject: [PATCH 133/481] xds testing: add path_matching and header_matching
 (#3764)

---
 test/kokoro/xds.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index c3fe3c29eefe..23c9d0119425 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -19,11 +19,17 @@ popd
 git clone -b "${branch}" --single-branch --depth=1 https://github.com/grpc/grpc.git
 
 grpc/tools/run_tests/helper_scripts/prep_xds.sh
+
+# Test cases "path_matching" and "header_matching" are not included in "all",
+# because not all interop clients in all languages support these new tests.
+#
+# TODO: remove "path_matching" and "header_matching" from --test_case after
+# they are added into "all".
 GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
-    --test_case=all \
+    --test_case="all,path_matching,header_matching" \
     --project_id=grpc-testing \
-    --source_image=projects/grpc-testing/global/images/xds-test-server \
+    --source_image=projects/grpc-testing/global/images/xds-test-server-2 \
     --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \
     --gcp_suffix=$(date '+%s') \
     --verbose \
@@ -31,4 +37,7 @@ GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
       --server=xds:///{server_uri} \
       --stats_port={stats_port} \
       --qps={qps} \
-      {fail_on_failed_rpc}"
+      {fail_on_failed_rpc} \
+      {rpcs_to_send} \
+      {metadata_to_send}"
+

From 0e6096e9a1f28320079fe39601d3fb8c3e51c950 Mon Sep 17 00:00:00 2001
From: Eric Gribkoff <ericgribkoff@google.com>
Date: Tue, 28 Jul 2020 11:43:27 -0700
Subject: [PATCH 134/481] interop: update --fail_on_failed_rpc to wait for
 initial success (#3763)

---
 interop/xds/client/client.go | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 27f954c30d28..7792094d54de 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -27,6 +27,7 @@ import (
 	"net"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"google.golang.org/grpc"
@@ -73,7 +74,7 @@ func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse {
 }
 
 var (
-	failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail")
+	failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail after first success")
 	numChannels     = flag.Int("num_channels", 1, "Num of channels")
 	printResponse   = flag.Bool("print_response", false, "Write RPC response to stdout")
 	qps             = flag.Int("qps", 1, "QPS per channel, for each type of RPC")
@@ -86,12 +87,24 @@ var (
 	mu               sync.Mutex
 	currentRequestID int32
 	watchers         = make(map[statsWatcherKey]*statsWatcher)
+
+	// 0 or 1 representing an RPC has succeeded. Use hasRPCSucceeded and
+	// setRPCSucceeded to access in a safe manner.
+	rpcSucceeded uint32
 )
 
 type statsService struct {
 	testpb.UnimplementedLoadBalancerStatsServiceServer
 }
 
+func hasRPCSucceeded() bool {
+	return atomic.LoadUint32(&rpcSucceeded) > 0
+}
+
+func setRPCSucceeded() {
+	atomic.StoreUint32(&rpcSucceeded, 1)
+}
+
 // Wait for the next LoadBalancerStatsRequest.GetNumRpcs to start and complete,
 // and return the distribution of remote peers. This is essentially a clientside
 // LB reporting mechanism that is designed to be queried by an external test
@@ -291,9 +304,12 @@ func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *tim
 					// This sends an empty string if the RPC failed.
 					watcher.chanHosts <- info
 				}
-				if err != nil && *failOnFailedRPC {
+				if err != nil && *failOnFailedRPC && hasRPCSucceeded() {
 					grpclog.Fatalf("RPC failed: %v", err)
 				}
+				if err == nil {
+					setRPCSucceeded()
+				}
 				if *printResponse {
 					if err == nil {
 						if cfg.typ == unaryCall {

From 1c32b02682dfa601b9515429c19a67539d83f565 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 28 Jul 2020 12:49:56 -0700
Subject: [PATCH 135/481] typo: upate->update (#3765)

---
 .../balancer/weightedtarget/weightedaggregator/aggregator.go    | 2 +-
 xds/internal/balancer/xdsrouting/balancerstateaggregator.go     | 2 +-
 xds/internal/client/client_watchers_cluster_test.go             | 2 +-
 xds/internal/client/client_watchers_endpoints_test.go           | 2 +-
 xds/internal/client/client_watchers_lds_test.go                 | 2 +-
 xds/internal/client/client_watchers_rds_test.go                 | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go b/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
index 6f2de711fa6c..a77a7c3f495f 100644
--- a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
+++ b/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
@@ -90,7 +90,7 @@ func (wbsa *Aggregator) Start() {
 }
 
 // Stop stops the aggregator. When the aggregator is closed, it won't call
-// parent ClientConn to upate balancer state.
+// parent ClientConn to update balancer state.
 func (wbsa *Aggregator) Stop() {
 	wbsa.mu.Lock()
 	defer wbsa.mu.Unlock()
diff --git a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go b/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
index 83dbb7914b9d..9f6250e5c804 100644
--- a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
+++ b/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
@@ -79,7 +79,7 @@ func (rbsa *balancerStateAggregator) start() {
 }
 
 // Close closes the aggregator. When the aggregator is closed, it won't call
-// parent ClientConn to upate balancer state.
+// parent ClientConn to update balancer state.
 func (rbsa *balancerStateAggregator) close() {
 	rbsa.mu.Lock()
 	defer rbsa.mu.Unlock()
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index c14ea18e3c0a..6d30f7dd5f7f 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -33,7 +33,7 @@ type clusterUpdateErr struct {
 // TestClusterWatch covers the cases:
 // - an update is received after a watch()
 // - an update for another resource name
-// - an upate is received after cancel()
+// - an update is received after cancel()
 func (s) TestClusterWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
 	defer cleanup()
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index e61bb7105230..7ac8d3e6cc64 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -55,7 +55,7 @@ type endpointsUpdateErr struct {
 // TestEndpointsWatch covers the cases:
 // - an update is received after a watch()
 // - an update for another resource name (which doesn't trigger callback)
-// - an upate is received after cancel()
+// - an update is received after cancel()
 func (s) TestEndpointsWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
 	defer cleanup()
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index 5db842d261ca..fa8bfdb6ca0f 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -32,7 +32,7 @@ type ldsUpdateErr struct {
 // TestLDSWatch covers the cases:
 // - an update is received after a watch()
 // - an update for another resource name
-// - an upate is received after cancel()
+// - an update is received after cancel()
 func (s) TestLDSWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
 	defer cleanup()
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 16e042ecd961..8476b275b496 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -33,7 +33,7 @@ type rdsUpdateErr struct {
 // TestRDSWatch covers the cases:
 // - an update is received after a watch()
 // - an update for another resource name (which doesn't trigger callback)
-// - an upate is received after cancel()
+// - an update is received after cancel()
 func (s) TestRDSWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewXDSV2Client()
 	defer cleanup()

From d6c4e49aab240de395fb0efba3890c3e965e3d4c Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 29 Jul 2020 13:03:27 -0700
Subject: [PATCH 136/481] internal: delete mock from examples (#3766)

---
 Documentation/gomock-example.md               | 182 ----------------
 examples/go.mod                               |   1 -
 .../helloworld/mock_helloworld/hw_mock.go     |  48 -----
 .../mock_helloworld/hw_mock_test.go           |  79 -------
 .../route_guide/mock_routeguide/rg_mock.go    | 200 ------------------
 .../mock_routeguide/rg_mock_test.go           |  92 --------
 vet.sh                                        |   4 +-
 7 files changed, 2 insertions(+), 604 deletions(-)
 delete mode 100644 Documentation/gomock-example.md
 delete mode 100644 examples/helloworld/mock_helloworld/hw_mock.go
 delete mode 100644 examples/helloworld/mock_helloworld/hw_mock_test.go
 delete mode 100644 examples/route_guide/mock_routeguide/rg_mock.go
 delete mode 100644 examples/route_guide/mock_routeguide/rg_mock_test.go

diff --git a/Documentation/gomock-example.md b/Documentation/gomock-example.md
deleted file mode 100644
index d93ce5ccef98..000000000000
--- a/Documentation/gomock-example.md
+++ /dev/null
@@ -1,182 +0,0 @@
-# Mocking Service for gRPC
-
-[Example code unary RPC](https://github.com/grpc/grpc-go/tree/master/examples/helloworld/mock_helloworld)
-
-[Example code streaming RPC](https://github.com/grpc/grpc-go/tree/master/examples/route_guide/mock_routeguide)
-
-## Why?
-
-To test client-side logic without the overhead of connecting to a real server. Mocking enables users to write light-weight unit tests to check functionalities on client-side without invoking RPC calls to a server.
-
-## Idea: Mock the client stub that connects to the server.
-
-We use Gomock to mock the client interface (in the generated code) and programmatically set its methods to expect and return pre-determined values. This enables users to write tests around the client logic and use this mocked stub while making RPC calls.
-
-## How to use Gomock?
-
-Documentation on Gomock can be found [here](https://github.com/golang/mock).
-A quick reading of the documentation should enable users to follow the code below.
-
-Consider a gRPC service based on following proto file:
-
-```proto
-//helloworld.proto
-
-package helloworld;
-
-message HelloRequest {
-    string name = 1;
-}
-
-message HelloReply {
-    string name = 1;
-}
-
-service Greeter {
-    rpc SayHello (HelloRequest) returns (HelloReply) {}
-}
-```
-
-The generated file helloworld.pb.go will have a client interface for each service defined in the proto file. This interface will have methods corresponding to each rpc inside that service.
-
-```Go
-type GreeterClient interface {
-    SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error)
-}
-```
-
-The generated code also contains a struct that implements this interface.
-
-```Go
-type greeterClient struct {
-   cc *grpc.ClientConn
-}
-func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error){
-    // ...
-    // gRPC specific code here
-    // ...
-}
-```
-
-Along with this the generated code has a method to create an instance of this struct.
-```Go
-func NewGreeterClient(cc *grpc.ClientConn) GreeterClient
-```
-
-The user code uses this function to create an instance of the struct greeterClient which then can be used to make rpc calls to the server.
-We will mock this interface GreeterClient and use an instance of that mock to make rpc calls. These calls instead of going to server will return pre-determined values.
-
-To create a mock we’ll use [mockgen](https://github.com/golang/mock#running-mockgen).
-From the directory ``` examples/helloworld/ ``` run ``` mockgen google.golang.org/grpc/examples/helloworld/helloworld GreeterClient > mock_helloworld/hw_mock.go ```
-
-Notice that in the above command we specify GreeterClient as the interface to be mocked.
-
-The user test code can import the package generated by mockgen along with library package gomock to write unit tests around client-side logic.
-```Go
-import "github.com/golang/mock/gomock"
-import hwmock "google.golang.org/grpc/examples/helloworld/mock_helloworld"
-```
-
-An instance of the mocked interface can be created as:
-```Go
-mockGreeterClient := hwmock.NewMockGreeterClient(ctrl)
-```
-This mocked object can be programmed to expect calls to its methods and return pre-determined values. For instance, we can program mockGreeterClient to expect a call to its method SayHello and return a HelloReply with message “Mocked RPC”.
-
-```Go
-mockGreeterClient.EXPECT().SayHello(
-    gomock.Any(), // expect any value for first parameter
-    gomock.Any(), // expect any value for second parameter
-).Return(&helloworld.HelloReply{Message: “Mocked RPC”}, nil)
-```
-
-gomock.Any() indicates that the parameter can have any value or type. We can indicate specific values for built-in types with gomock.Eq().
-However, if the test code needs to specify the parameter to have a proto message type, we can replace gomock.Any() with an instance of a struct that implements gomock.Matcher interface.
-
-```Go
-type rpcMsg struct {
-    msg proto.Message
-}
-
-func (r *rpcMsg) Matches(msg interface{}) bool {
-    m, ok := msg.(proto.Message)
-    if !ok {
-        return false
-    }
-    return proto.Equal(m, r.msg)
-}
-
-func (r *rpcMsg) String() string {
-    return fmt.Sprintf("is %s", r.msg)
-}
-
-...
-
-req := &helloworld.HelloRequest{Name: "unit_test"}
-mockGreeterClient.EXPECT().SayHello(
-    gomock.Any(),
-    &rpcMsg{msg: req},
-).Return(&helloworld.HelloReply{Message: "Mocked Interface"}, nil)
-```
-
-## Mock streaming RPCs:
-
-For our example we consider the case of bi-directional streaming RPCs. Concretely, we'll write a test for RouteChat function from the route guide example to demonstrate how to write mocks for streams.
-
-RouteChat is a bi-directional streaming RPC, which means calling RouteChat returns a stream that can __Send__ and __Recv__ messages to  and from the server, respectively. We'll start by creating a mock of this stream interface returned by RouteChat and then we'll mock the client interface and set expectation on the method RouteChat to return our mocked stream.
-
-### Generating mocking code:
-Like before we'll use [mockgen](https://github.com/golang/mock#running-mockgen). From the `examples/route_guide` directory run:  `mockgen google.golang.org/grpc/examples/route_guide/routeguide RouteGuideClient,RouteGuide_RouteChatClient > mock_route_guide/rg_mock.go`
-
-Notice that we are mocking both client(`RouteGuideClient`) and stream(`RouteGuide_RouteChatClient`) interfaces here.
-
-This will create a file `rg_mock.go` under directory `mock_route_guide`. This file contains all the mocking code we need to write our test.
-
-In our test code, like before, we import the this mocking code along with the generated code
-
-```go
-import (
-    rgmock "google.golang.org/grpc/examples/route_guide/mock_routeguide"
-    rgpb "google.golang.org/grpc/examples/route_guide/routeguide"
-)
-```
-
-Now considering a test that takes the RouteGuide client object as a parameter, makes a RouteChat rpc call and sends a message on the resulting stream. Furthermore, this test expects to see the same message to be received on the stream. 
-
-```go
-var msg = ...
-
-// Creates a RouteChat call and sends msg on it.
-// Checks if the received message was equal to msg.
-func testRouteChat(client rgb.RouteChatClient) error{
-    ...
-}
-```
-
-We can inject our mock in here by simply passing it as an argument to the method.
-
-Creating mock for stream interface:
-
-```go
-    stream := rgmock.NewMockRouteGuide_RouteChatClient(ctrl)
-}
-```
-
-Setting Expectations:
-
-```go
-    stream.EXPECT().Send(gomock.Any()).Return(nil)
-    stream.EXPECT().Recv().Return(msg, nil)
-```
-
-Creating mock for client interface:
-
-```go
-    rgclient := rgmock.NewMockRouteGuideClient(ctrl)
-```
-
-Setting Expectations:
-
-```go
-    rgclient.EXPECT().RouteChat(gomock.Any()).Return(stream, nil)
-```
diff --git a/examples/go.mod b/examples/go.mod
index cbd35e6337c1..c8e7dab30b70 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -3,7 +3,6 @@ module google.golang.org/grpc/examples
 go 1.11
 
 require (
-	github.com/golang/mock v1.1.1
 	github.com/golang/protobuf v1.4.2
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
diff --git a/examples/helloworld/mock_helloworld/hw_mock.go b/examples/helloworld/mock_helloworld/hw_mock.go
deleted file mode 100644
index 46e97435f6c1..000000000000
--- a/examples/helloworld/mock_helloworld/hw_mock.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: google.golang.org/grpc/examples/helloworld/helloworld (interfaces: GreeterClient)
-
-package mock_helloworld
-
-import (
-	context "context"
-	gomock "github.com/golang/mock/gomock"
-	grpc "google.golang.org/grpc"
-	helloworld "google.golang.org/grpc/examples/helloworld/helloworld"
-)
-
-// Mock of GreeterClient interface
-type MockGreeterClient struct {
-	ctrl     *gomock.Controller
-	recorder *_MockGreeterClientRecorder
-}
-
-// Recorder for MockGreeterClient (not exported)
-type _MockGreeterClientRecorder struct {
-	mock *MockGreeterClient
-}
-
-func NewMockGreeterClient(ctrl *gomock.Controller) *MockGreeterClient {
-	mock := &MockGreeterClient{ctrl: ctrl}
-	mock.recorder = &_MockGreeterClientRecorder{mock}
-	return mock
-}
-
-func (_m *MockGreeterClient) EXPECT() *_MockGreeterClientRecorder {
-	return _m.recorder
-}
-
-func (_m *MockGreeterClient) SayHello(_param0 context.Context, _param1 *helloworld.HelloRequest, _param2 ...grpc.CallOption) (*helloworld.HelloReply, error) {
-	_s := []interface{}{_param0, _param1}
-	for _, _x := range _param2 {
-		_s = append(_s, _x)
-	}
-	ret := _m.ctrl.Call(_m, "SayHello", _s...)
-	ret0, _ := ret[0].(*helloworld.HelloReply)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockGreeterClientRecorder) SayHello(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
-	_s := append([]interface{}{arg0, arg1}, arg2...)
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "SayHello", _s...)
-}
diff --git a/examples/helloworld/mock_helloworld/hw_mock_test.go b/examples/helloworld/mock_helloworld/hw_mock_test.go
deleted file mode 100644
index 4c6e0196d302..000000000000
--- a/examples/helloworld/mock_helloworld/hw_mock_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package mock_helloworld_test
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/golang/mock/gomock"
-	"github.com/golang/protobuf/proto"
-	helloworld "google.golang.org/grpc/examples/helloworld/helloworld"
-	hwmock "google.golang.org/grpc/examples/helloworld/mock_helloworld"
-	"google.golang.org/grpc/internal/grpctest"
-)
-
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
-// rpcMsg implements the gomock.Matcher interface
-type rpcMsg struct {
-	msg proto.Message
-}
-
-func (r *rpcMsg) Matches(msg interface{}) bool {
-	m, ok := msg.(proto.Message)
-	if !ok {
-		return false
-	}
-	return proto.Equal(m, r.msg)
-}
-
-func (r *rpcMsg) String() string {
-	return fmt.Sprintf("is %s", r.msg)
-}
-
-func (s) TestSayHello(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-	mockGreeterClient := hwmock.NewMockGreeterClient(ctrl)
-	req := &helloworld.HelloRequest{Name: "unit_test"}
-	mockGreeterClient.EXPECT().SayHello(
-		gomock.Any(),
-		&rpcMsg{msg: req},
-	).Return(&helloworld.HelloReply{Message: "Mocked Interface"}, nil)
-	testSayHello(t, mockGreeterClient)
-}
-
-func testSayHello(t *testing.T, client helloworld.GreeterClient) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-	r, err := client.SayHello(ctx, &helloworld.HelloRequest{Name: "unit_test"})
-	if err != nil || r.Message != "Mocked Interface" {
-		t.Errorf("mocking failed")
-	}
-	t.Log("Reply : ", r.Message)
-}
diff --git a/examples/route_guide/mock_routeguide/rg_mock.go b/examples/route_guide/mock_routeguide/rg_mock.go
deleted file mode 100644
index 564f98750d4b..000000000000
--- a/examples/route_guide/mock_routeguide/rg_mock.go
+++ /dev/null
@@ -1,200 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: google.golang.org/grpc/examples/route_guide/routeguide (interfaces: RouteGuideClient,RouteGuide_RouteChatClient)
-
-package mock_routeguide
-
-import (
-	context "context"
-	gomock "github.com/golang/mock/gomock"
-	grpc "google.golang.org/grpc"
-	routeguide "google.golang.org/grpc/examples/route_guide/routeguide"
-	metadata "google.golang.org/grpc/metadata"
-)
-
-// Mock of RouteGuideClient interface
-type MockRouteGuideClient struct {
-	ctrl     *gomock.Controller
-	recorder *_MockRouteGuideClientRecorder
-}
-
-// Recorder for MockRouteGuideClient (not exported)
-type _MockRouteGuideClientRecorder struct {
-	mock *MockRouteGuideClient
-}
-
-func NewMockRouteGuideClient(ctrl *gomock.Controller) *MockRouteGuideClient {
-	mock := &MockRouteGuideClient{ctrl: ctrl}
-	mock.recorder = &_MockRouteGuideClientRecorder{mock}
-	return mock
-}
-
-func (_m *MockRouteGuideClient) EXPECT() *_MockRouteGuideClientRecorder {
-	return _m.recorder
-}
-
-func (_m *MockRouteGuideClient) GetFeature(_param0 context.Context, _param1 *routeguide.Point, _param2 ...grpc.CallOption) (*routeguide.Feature, error) {
-	_s := []interface{}{_param0, _param1}
-	for _, _x := range _param2 {
-		_s = append(_s, _x)
-	}
-	ret := _m.ctrl.Call(_m, "GetFeature", _s...)
-	ret0, _ := ret[0].(*routeguide.Feature)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockRouteGuideClientRecorder) GetFeature(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
-	_s := append([]interface{}{arg0, arg1}, arg2...)
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetFeature", _s...)
-}
-
-func (_m *MockRouteGuideClient) ListFeatures(_param0 context.Context, _param1 *routeguide.Rectangle, _param2 ...grpc.CallOption) (routeguide.RouteGuide_ListFeaturesClient, error) {
-	_s := []interface{}{_param0, _param1}
-	for _, _x := range _param2 {
-		_s = append(_s, _x)
-	}
-	ret := _m.ctrl.Call(_m, "ListFeatures", _s...)
-	ret0, _ := ret[0].(routeguide.RouteGuide_ListFeaturesClient)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockRouteGuideClientRecorder) ListFeatures(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
-	_s := append([]interface{}{arg0, arg1}, arg2...)
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "ListFeatures", _s...)
-}
-
-func (_m *MockRouteGuideClient) RecordRoute(_param0 context.Context, _param1 ...grpc.CallOption) (routeguide.RouteGuide_RecordRouteClient, error) {
-	_s := []interface{}{_param0}
-	for _, _x := range _param1 {
-		_s = append(_s, _x)
-	}
-	ret := _m.ctrl.Call(_m, "RecordRoute", _s...)
-	ret0, _ := ret[0].(routeguide.RouteGuide_RecordRouteClient)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockRouteGuideClientRecorder) RecordRoute(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
-	_s := append([]interface{}{arg0}, arg1...)
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "RecordRoute", _s...)
-}
-
-func (_m *MockRouteGuideClient) RouteChat(_param0 context.Context, _param1 ...grpc.CallOption) (routeguide.RouteGuide_RouteChatClient, error) {
-	_s := []interface{}{_param0}
-	for _, _x := range _param1 {
-		_s = append(_s, _x)
-	}
-	ret := _m.ctrl.Call(_m, "RouteChat", _s...)
-	ret0, _ := ret[0].(routeguide.RouteGuide_RouteChatClient)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockRouteGuideClientRecorder) RouteChat(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
-	_s := append([]interface{}{arg0}, arg1...)
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "RouteChat", _s...)
-}
-
-// Mock of RouteGuide_RouteChatClient interface
-type MockRouteGuide_RouteChatClient struct {
-	ctrl     *gomock.Controller
-	recorder *_MockRouteGuide_RouteChatClientRecorder
-}
-
-// Recorder for MockRouteGuide_RouteChatClient (not exported)
-type _MockRouteGuide_RouteChatClientRecorder struct {
-	mock *MockRouteGuide_RouteChatClient
-}
-
-func NewMockRouteGuide_RouteChatClient(ctrl *gomock.Controller) *MockRouteGuide_RouteChatClient {
-	mock := &MockRouteGuide_RouteChatClient{ctrl: ctrl}
-	mock.recorder = &_MockRouteGuide_RouteChatClientRecorder{mock}
-	return mock
-}
-
-func (_m *MockRouteGuide_RouteChatClient) EXPECT() *_MockRouteGuide_RouteChatClientRecorder {
-	return _m.recorder
-}
-
-func (_m *MockRouteGuide_RouteChatClient) CloseSend() error {
-	ret := _m.ctrl.Call(_m, "CloseSend")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) CloseSend() *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "CloseSend")
-}
-
-func (_m *MockRouteGuide_RouteChatClient) Context() context.Context {
-	ret := _m.ctrl.Call(_m, "Context")
-	ret0, _ := ret[0].(context.Context)
-	return ret0
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) Context() *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "Context")
-}
-
-func (_m *MockRouteGuide_RouteChatClient) Header() (metadata.MD, error) {
-	ret := _m.ctrl.Call(_m, "Header")
-	ret0, _ := ret[0].(metadata.MD)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) Header() *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "Header")
-}
-
-func (_m *MockRouteGuide_RouteChatClient) Recv() (*routeguide.RouteNote, error) {
-	ret := _m.ctrl.Call(_m, "Recv")
-	ret0, _ := ret[0].(*routeguide.RouteNote)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) Recv() *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "Recv")
-}
-
-func (_m *MockRouteGuide_RouteChatClient) RecvMsg(_param0 interface{}) error {
-	ret := _m.ctrl.Call(_m, "RecvMsg", _param0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) RecvMsg(arg0 interface{}) *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "RecvMsg", arg0)
-}
-
-func (_m *MockRouteGuide_RouteChatClient) Send(_param0 *routeguide.RouteNote) error {
-	ret := _m.ctrl.Call(_m, "Send", _param0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) Send(arg0 interface{}) *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "Send", arg0)
-}
-
-func (_m *MockRouteGuide_RouteChatClient) SendMsg(_param0 interface{}) error {
-	ret := _m.ctrl.Call(_m, "SendMsg", _param0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) SendMsg(arg0 interface{}) *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "SendMsg", arg0)
-}
-
-func (_m *MockRouteGuide_RouteChatClient) Trailer() metadata.MD {
-	ret := _m.ctrl.Call(_m, "Trailer")
-	ret0, _ := ret[0].(metadata.MD)
-	return ret0
-}
-
-func (_mr *_MockRouteGuide_RouteChatClientRecorder) Trailer() *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "Trailer")
-}
diff --git a/examples/route_guide/mock_routeguide/rg_mock_test.go b/examples/route_guide/mock_routeguide/rg_mock_test.go
deleted file mode 100644
index 8713063a4822..000000000000
--- a/examples/route_guide/mock_routeguide/rg_mock_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package mock_routeguide_test
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/golang/mock/gomock"
-	"github.com/golang/protobuf/proto"
-	rgmock "google.golang.org/grpc/examples/route_guide/mock_routeguide"
-	rgpb "google.golang.org/grpc/examples/route_guide/routeguide"
-	"google.golang.org/grpc/internal/grpctest"
-)
-
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
-var msg = &rgpb.RouteNote{
-	Location: &rgpb.Point{Latitude: 17, Longitude: 29},
-	Message:  "Taxi-cab",
-}
-
-func (s) TestRouteChat(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-
-	// Create mock for the stream returned by RouteChat
-	stream := rgmock.NewMockRouteGuide_RouteChatClient(ctrl)
-	// set expectation on sending.
-	stream.EXPECT().Send(
-		gomock.Any(),
-	).Return(nil)
-	// Set expectation on receiving.
-	stream.EXPECT().Recv().Return(msg, nil)
-	stream.EXPECT().CloseSend().Return(nil)
-	// Create mock for the client interface.
-	rgclient := rgmock.NewMockRouteGuideClient(ctrl)
-	// Set expectation on RouteChat
-	rgclient.EXPECT().RouteChat(
-		gomock.Any(),
-	).Return(stream, nil)
-	if err := testRouteChat(rgclient); err != nil {
-		t.Fatalf("Test failed: %v", err)
-	}
-}
-
-func testRouteChat(client rgpb.RouteGuideClient) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	stream, err := client.RouteChat(ctx)
-	if err != nil {
-		return err
-	}
-	if err := stream.Send(msg); err != nil {
-		return err
-	}
-	if err := stream.CloseSend(); err != nil {
-		return err
-	}
-	got, err := stream.Recv()
-	if err != nil {
-		return err
-	}
-	if !proto.Equal(got, msg) {
-		return fmt.Errorf("stream.Recv() = %v, want %v", got, msg)
-	}
-	return nil
-}
diff --git a/vet.sh b/vet.sh
index 8b7dff19adb3..2116d1987eef 100755
--- a/vet.sh
+++ b/vet.sh
@@ -95,8 +95,8 @@ go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go
 
 # - gofmt, goimports, golint (with exceptions for generated code), go vet.
 gofmt -s -d -l . 2>&1 | fail_on_output
-goimports -l . 2>&1 | not grep -vE "(_mock|\.pb)\.go"
-golint ./... 2>&1 | not grep -vE "(_mock|\.pb)\.go:"
+goimports -l . 2>&1 | not grep -vE "\.pb\.go"
+golint ./... 2>&1 | not grep -vE "\.pb\.go:"
 go vet -all ./...
 
 misspell -error .

From 97c30a14193a97f3c63d041ef7497b6b4db87e88 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 30 Jul 2020 10:27:09 -0700
Subject: [PATCH 137/481] xds: Client refactor in preparation for xDS v3
 support (#3743)

---
 .../edsbalancer/eds_impl_priority_test.go     |  83 +-
 .../balancer/edsbalancer/eds_impl_test.go     |  78 +-
 xds/internal/balancer/edsbalancer/eds_test.go |   2 +
 .../edsbalancer/eds_testutil.go}              | 128 +--
 .../edsbalancer/xds_client_wrapper_test.go    |   7 +-
 xds/internal/client/client.go                 | 253 +++++-
 xds/internal/client/client_callback.go        |  36 +-
 xds/internal/client/client_cds_test.go        | 312 +++++++
 xds/internal/client/client_eds_test.go        | 312 +++++++
 xds/internal/client/client_lds_test.go        | 171 ++++
 xds/internal/client/client_rds_test.go        | 854 ++++++++++++++++++
 xds/internal/client/client_test.go            | 122 +--
 xds/internal/client/client_watchers.go        | 239 ++++-
 .../client/client_watchers_cluster.go         |  56 --
 .../client/client_watchers_cluster_test.go    |  57 +-
 .../client/client_watchers_endpoints.go       |  93 --
 .../client/client_watchers_endpoints_test.go  |  40 +-
 xds/internal/client/client_watchers_lds.go    |  47 -
 .../client/client_watchers_lds_test.go        | 113 +--
 xds/internal/client/client_watchers_rds.go    |  73 --
 .../client/client_watchers_rds_test.go        |  95 +-
 .../client/client_watchers_service.go         | 135 ---
 .../client/client_watchers_service_test.go    | 304 +++----
 xds/internal/client/client_xds.go             | 507 +++++++++++
 xds/internal/client/tests/client_test.go      | 122 +++
 xds/internal/client/testutil_test.go          | 209 -----
 .../client/{v2client.go => v2/client.go}      | 207 +++--
 .../client_ack_test.go}                       |  82 +-
 .../client_cds_test.go}                       | 224 ++---
 .../client_eds_test.go}                       | 137 +--
 .../client_lds_test.go}                       | 117 +--
 xds/internal/client/v2/client_rds_test.go     | 167 ++++
 .../{v2client_test.go => v2/client_test.go}   | 357 +++++---
 xds/internal/client/v2client_cds.go           |  75 --
 xds/internal/client/v2client_eds_testutil.go  | 128 ---
 xds/internal/client/v2client_lds.go           |  88 --
 xds/internal/client/v2client_rds.go           | 323 -------
 xds/internal/client/v2client_rds_test.go      | 699 --------------
 xds/internal/resolver/serviceconfig_test.go   |   5 +-
 xds/internal/resolver/xds_resolver_test.go    |   4 +-
 xds/internal/testutils/protos.go              | 112 ++-
 xds/internal/version/version.go               |  19 +-
 xds/xds.go                                    |   5 +-
 43 files changed, 4032 insertions(+), 3165 deletions(-)
 rename xds/internal/{client/v2client_eds.go => balancer/edsbalancer/eds_testutil.go} (56%)
 create mode 100644 xds/internal/client/client_cds_test.go
 create mode 100644 xds/internal/client/client_eds_test.go
 create mode 100644 xds/internal/client/client_lds_test.go
 create mode 100644 xds/internal/client/client_rds_test.go
 delete mode 100644 xds/internal/client/client_watchers_cluster.go
 delete mode 100644 xds/internal/client/client_watchers_endpoints.go
 delete mode 100644 xds/internal/client/client_watchers_lds.go
 delete mode 100644 xds/internal/client/client_watchers_rds.go
 delete mode 100644 xds/internal/client/client_watchers_service.go
 create mode 100644 xds/internal/client/client_xds.go
 create mode 100644 xds/internal/client/tests/client_test.go
 delete mode 100644 xds/internal/client/testutil_test.go
 rename xds/internal/client/{v2client.go => v2/client.go} (74%)
 rename xds/internal/client/{v2client_ack_test.go => v2/client_ack_test.go} (88%)
 rename xds/internal/client/{v2client_cds_test.go => v2/client_cds_test.go} (56%)
 rename xds/internal/client/{v2client_eds_test.go => v2/client_eds_test.go} (52%)
 rename xds/internal/client/{v2client_lds_test.go => v2/client_lds_test.go} (55%)
 create mode 100644 xds/internal/client/v2/client_rds_test.go
 rename xds/internal/client/{v2client_test.go => v2/client_test.go} (62%)
 delete mode 100644 xds/internal/client/v2client_cds.go
 delete mode 100644 xds/internal/client/v2client_eds_testutil.go
 delete mode 100644 xds/internal/client/v2client_lds.go
 delete mode 100644 xds/internal/client/v2client_rds.go
 delete mode 100644 xds/internal/client/v2client_rds_test.go

diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index d9b9ff4a8145..a82e64431420 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -25,7 +25,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -39,10 +38,10 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
@@ -62,11 +61,11 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	}
 
 	// Add p2, it shouldn't cause any udpates.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	select {
 	case <-cc.NewPickerCh:
@@ -79,10 +78,10 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	}
 
 	// Remove p2, no updates.
-	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build()))
 
 	select {
 	case <-cc.NewPickerCh:
@@ -105,10 +104,10 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -147,11 +146,11 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	}
 
 	// Add p2, it shouldn't cause any udpates.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	select {
 	case <-cc.NewPickerCh:
@@ -183,10 +182,10 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	}
 
 	// Remove 2, use 1.
-	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build()))
 
 	// p2 SubConns are removed.
 	scToRemove := <-cc.RemoveSubConnCh
@@ -212,10 +211,10 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -242,11 +241,11 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	}
 
 	// Add p2, it should create a new SubConn.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
@@ -277,11 +276,11 @@ func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0,1,2], each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab1.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -359,10 +358,10 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -409,10 +408,10 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
-	clab0 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab0 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab0.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -463,12 +462,12 @@ func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	}
 
 	// Add two localities, with two priorities, with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
 	clab1.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
 	clab1.AddLocality(testSubZones[3], 1, 1, testEndpointAddrs[3:4], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs2 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want {
@@ -520,10 +519,10 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
-	clab0 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab0 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab0.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab0.Build()))
 
 	addrs0 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want {
@@ -541,8 +540,8 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	}
 
 	// Remove all priorities.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	// p0 subconn should be removed.
 	scToRemove := <-cc.RemoveSubConnCh
@@ -559,10 +558,10 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	}
 
 	// Re-add two localities, with previous priorities, but different backends.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[3:4], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	addrs01 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs01[0].Addr, testEndpointAddrs[2]; got != want {
@@ -591,9 +590,9 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	}
 
 	// Remove p1 from EDS, to fallback to p0.
-	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build()))
 
 	// p1 subconn should be removed.
 	scToRemove1 := <-cc.RemoveSubConnCh
@@ -664,10 +663,10 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
@@ -687,10 +686,10 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	}
 
 	// Remove addresses from priority 0, should use p1.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, nil, nil)
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	// p0 will remove the subconn, and ClientConn will send a sc update to
 	// shutdown.
@@ -723,10 +722,10 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	addrs1 := <-cc.NewSubConnAddrsCh
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
@@ -746,12 +745,12 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	}
 
 	// Set priority 0 endpoints to all unhealthy, should use p1.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
-	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], &xdsclient.AddLocalityOptions{
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], &testutils.AddLocalityOptions{
 		Health: []corepb.HealthStatus{corepb.HealthStatus_UNHEALTHY},
 	})
 	clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	// p0 will remove the subconn, and ClientConn will send a sc update to
 	// transient failure.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index db1fa328a228..c0961e46c949 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -60,9 +60,9 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// One locality with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	sc1 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
@@ -78,9 +78,9 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	}
 
 	// The same locality, add one more backend.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	sc2 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
@@ -94,9 +94,9 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	}
 
 	// The same locality, delete first backend.
-	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build()))
 
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
@@ -114,9 +114,9 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	}
 
 	// The same locality, replace backend.
-	clab4 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab4 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab4.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab4.Build()))
 
 	sc3 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc3, connectivity.Connecting)
@@ -137,9 +137,9 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	}
 
 	// The same locality, different drop rate, dropping 50%.
-	clab5 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], []uint32{50})
+	clab5 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], []uint32{50})
 	clab5.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab5.Build()))
 
 	// Picks with drops.
 	p5 := <-cc.NewPickerCh
@@ -155,9 +155,9 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	}
 
 	// The same locality, remove drops.
-	clab6 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab6 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab6.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab6.Build()))
 
 	// Pick without drops.
 	p6 := <-cc.NewPickerCh
@@ -181,9 +181,9 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 	sc1 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
@@ -192,7 +192,7 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	// locality. Otherwise the test is flaky because of a map is used in EDS to
 	// keep localities.
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 	sc2 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
@@ -205,11 +205,11 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	}
 
 	// Add another locality, with one backend.
-	clab2 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab2.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	clab2.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab2.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
 	sc3 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc3, connectivity.Connecting)
@@ -223,10 +223,10 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	}
 
 	// Remove first locality.
-	clab3 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab3.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	clab3.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab3.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build()))
 
 	scToRemove := <-cc.RemoveSubConnCh
 	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
@@ -242,10 +242,10 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	}
 
 	// Add a backend to the last locality.
-	clab4 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab4 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab4.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
 	clab4.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab4.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab4.Build()))
 
 	sc4 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc4, connectivity.Connecting)
@@ -262,10 +262,10 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	}
 
 	// Change weight of the locality[1].
-	clab5 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab5 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab5.AddLocality(testSubZones[1], 2, 0, testEndpointAddrs[1:2], nil)
 	clab5.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab5.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab5.Build()))
 
 	// Test pick with two subconns different locality weight.
 	p5 := <-cc.NewPickerCh
@@ -278,10 +278,10 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 	}
 
 	// Change weight of the locality[1] to 0, it should never be picked.
-	clab6 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab6 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab6.AddLocality(testSubZones[1], 0, 0, testEndpointAddrs[1:2], nil)
 	clab6.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab6.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab6.Build()))
 
 	// Changing weight of locality[1] to 0 caused it to be removed. It's subconn
 	// should also be removed.
@@ -312,8 +312,8 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, each 3 backend, one Healthy, one Unhealthy, one Unknown.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
-	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:6], &xdsclient.AddLocalityOptions{
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:6], &testutils.AddLocalityOptions{
 		Health: []corepb.HealthStatus{
 			corepb.HealthStatus_HEALTHY,
 			corepb.HealthStatus_UNHEALTHY,
@@ -323,7 +323,7 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 			corepb.HealthStatus_DEGRADED,
 		},
 	})
-	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[6:12], &xdsclient.AddLocalityOptions{
+	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[6:12], &testutils.AddLocalityOptions{
 		Health: []corepb.HealthStatus{
 			corepb.HealthStatus_HEALTHY,
 			corepb.HealthStatus_UNHEALTHY,
@@ -333,7 +333,7 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 			corepb.HealthStatus_DEGRADED,
 		},
 	})
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	var (
 		readySCs           []balancer.SubConn
@@ -406,9 +406,9 @@ func (s) TestEDS_EmptyUpdate(t *testing.T) {
 	}
 
 	// One locality with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	sc1 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
@@ -434,7 +434,7 @@ func (s) TestEDS_EmptyUpdate(t *testing.T) {
 	}
 
 	// Handle another update with priorities and localities.
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	sc2 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
@@ -462,10 +462,10 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	edsb.handleChildPolicy("test-const-balancer", nil)
 
 	// Two localities, each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	for i := 0; i < 2; i++ {
 		sc := <-cc.NewSubConnCh
@@ -601,9 +601,9 @@ func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) {
 
 	edsb.handleChildPolicy("test-inline-update-balancer", nil)
 
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 
 	p0 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
@@ -689,9 +689,9 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
 	// Two localities, each with one backend.
-	clab1 := xdsclient.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 	sc1 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
@@ -703,7 +703,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// locality. Otherwise the test is flaky because of a map is used in EDS to
 	// keep localities.
 	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
-	edsb.handleEDSResponse(xdsclient.ParseEDSRespProtoForTesting(clab1.Build()))
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 	sc2 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index f70a0f2673be..312ad9efb030 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -41,6 +41,8 @@ import (
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+
+	_ "google.golang.org/grpc/xds/internal/client/v2" // V2 client registration.
 )
 
 func init() {
diff --git a/xds/internal/client/v2client_eds.go b/xds/internal/balancer/edsbalancer/eds_testutil.go
similarity index 56%
rename from xds/internal/client/v2client_eds.go
rename to xds/internal/balancer/edsbalancer/eds_testutil.go
index 0055de52b832..5e37cdcb47c7 100644
--- a/xds/internal/client/v2client_eds.go
+++ b/xds/internal/balancer/edsbalancer/eds_testutil.go
@@ -1,6 +1,5 @@
 /*
- *
- * Copyright 2019 gRPC authors.
+ * Copyright 2020 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +14,7 @@
  * limitations under the License.
  */
 
-package client
+package edsbalancer
 
 import (
 	"fmt"
@@ -26,53 +25,25 @@ import (
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
 	typepb "github.com/envoyproxy/go-control-plane/envoy/type"
-	"github.com/golang/protobuf/ptypes"
 	"google.golang.org/grpc/xds/internal"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
-func parseAddress(socketAddress *corepb.SocketAddress) string {
-	return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue())))
-}
-
-func parseDropPolicy(dropPolicy *xdspb.ClusterLoadAssignment_Policy_DropOverload) OverloadDropConfig {
-	percentage := dropPolicy.GetDropPercentage()
-	var (
-		numerator   = percentage.GetNumerator()
-		denominator uint32
-	)
-	switch percentage.GetDenominator() {
-	case typepb.FractionalPercent_HUNDRED:
-		denominator = 100
-	case typepb.FractionalPercent_TEN_THOUSAND:
-		denominator = 10000
-	case typepb.FractionalPercent_MILLION:
-		denominator = 1000000
-	}
-	return OverloadDropConfig{
-		Category:    dropPolicy.GetCategory(),
-		Numerator:   numerator,
-		Denominator: denominator,
-	}
-}
-
-func parseEndpoints(lbEndpoints []*endpointpb.LbEndpoint) []Endpoint {
-	endpoints := make([]Endpoint, 0, len(lbEndpoints))
-	for _, lbEndpoint := range lbEndpoints {
-		endpoints = append(endpoints, Endpoint{
-			HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()),
-			Address:      parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()),
-			Weight:       lbEndpoint.GetLoadBalancingWeight().GetValue(),
-		})
+// parseEDSRespProtoForTesting parses EDS response, and panic if parsing fails.
+//
+// TODO: delete this. The EDS balancer tests should build an EndpointsUpdate
+// directly, instead of building and parsing a proto message.
+func parseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) xdsclient.EndpointsUpdate {
+	u, err := parseEDSRespProto(m)
+	if err != nil {
+		panic(err.Error())
 	}
-	return endpoints
+	return u
 }
 
-// ParseEDSRespProto turns EDS response proto message to EndpointsUpdate.
-//
-// This is temporarily exported to be used in eds balancer, before it switches
-// to use xds client. TODO: unexport.
-func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (EndpointsUpdate, error) {
-	ret := EndpointsUpdate{}
+// parseEDSRespProto turns EDS response proto message to EndpointsUpdate.
+func parseEDSRespProto(m *xdspb.ClusterLoadAssignment) (xdsclient.EndpointsUpdate, error) {
+	ret := xdsclient.EndpointsUpdate{}
 	for _, dropPolicy := range m.GetPolicy().GetDropOverloads() {
 		ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy))
 	}
@@ -80,7 +51,7 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (EndpointsUpdate, error)
 	for _, locality := range m.Endpoints {
 		l := locality.GetLocality()
 		if l == nil {
-			return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
+			return xdsclient.EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
 		}
 		lid := internal.LocalityID{
 			Region:  l.Region,
@@ -89,7 +60,7 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (EndpointsUpdate, error)
 		}
 		priority := locality.GetPriority()
 		priorities[priority] = struct{}{}
-		ret.Localities = append(ret.Localities, Locality{
+		ret.Localities = append(ret.Localities, xdsclient.Locality{
 			ID:        lid,
 			Endpoints: parseEndpoints(locality.GetLbEndpoints()),
 			Weight:    locality.GetLoadBalancingWeight().GetValue(),
@@ -98,46 +69,45 @@ func ParseEDSRespProto(m *xdspb.ClusterLoadAssignment) (EndpointsUpdate, error)
 	}
 	for i := 0; i < len(priorities); i++ {
 		if _, ok := priorities[uint32(i)]; !ok {
-			return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities)
+			return xdsclient.EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities)
 		}
 	}
 	return ret, nil
 }
 
-// ParseEDSRespProtoForTesting parses EDS response, and panic if parsing fails.
-// This is used by EDS balancer tests.
-//
-// TODO: delete this. The EDS balancer tests should build an EndpointsUpdate directly,
-//  instead of building and parsing a proto message.
-func ParseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) EndpointsUpdate {
-	u, err := ParseEDSRespProto(m)
-	if err != nil {
-		panic(err.Error())
-	}
-	return u
+func parseAddress(socketAddress *corepb.SocketAddress) string {
+	return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue())))
 }
 
-func (v2c *v2Client) handleEDSResponse(resp *xdspb.DiscoveryResponse) error {
-	returnUpdate := make(map[string]EndpointsUpdate)
-	for _, r := range resp.GetResources() {
-		var resource ptypes.DynamicAny
-		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
-			return fmt.Errorf("xds: failed to unmarshal resource in EDS response: %v", err)
-		}
-		cla, ok := resource.Message.(*xdspb.ClusterLoadAssignment)
-		if !ok {
-			return fmt.Errorf("xds: unexpected resource type: %T in EDS response", resource.Message)
-		}
-		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
-
-		u, err := ParseEDSRespProto(cla)
-		if err != nil {
-			return err
-		}
-
-		returnUpdate[cla.GetClusterName()] = u
+func parseDropPolicy(dropPolicy *xdspb.ClusterLoadAssignment_Policy_DropOverload) xdsclient.OverloadDropConfig {
+	percentage := dropPolicy.GetDropPercentage()
+	var (
+		numerator   = percentage.GetNumerator()
+		denominator uint32
+	)
+	switch percentage.GetDenominator() {
+	case typepb.FractionalPercent_HUNDRED:
+		denominator = 100
+	case typepb.FractionalPercent_TEN_THOUSAND:
+		denominator = 10000
+	case typepb.FractionalPercent_MILLION:
+		denominator = 1000000
 	}
+	return xdsclient.OverloadDropConfig{
+		Category:    dropPolicy.GetCategory(),
+		Numerator:   numerator,
+		Denominator: denominator,
+	}
+}
 
-	v2c.parent.newEDSUpdate(returnUpdate)
-	return nil
+func parseEndpoints(lbEndpoints []*endpointpb.LbEndpoint) []xdsclient.Endpoint {
+	endpoints := make([]xdsclient.Endpoint, 0, len(lbEndpoints))
+	for _, lbEndpoint := range lbEndpoints {
+		endpoints = append(endpoints, xdsclient.Endpoint{
+			HealthStatus: xdsclient.EndpointHealthStatus(lbEndpoint.GetHealthStatus()),
+			Address:      parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()),
+			Weight:       lbEndpoint.GetLoadBalancingWeight().GetValue(),
+		})
+	}
+	return endpoints
 }
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index b87b7463edb9..90372b539ba7 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -36,10 +36,7 @@ import (
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-)
-
-const (
-	edsType = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 var (
@@ -135,7 +132,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 			}
 
 			wantReq := &xdspb.DiscoveryRequest{
-				TypeUrl:       edsType,
+				TypeUrl:       version.V2EndpointsURL,
 				ResourceNames: []string{test.wantResourceName},
 				Node:          testutils.EmptyNodeProtoV2,
 			}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 07a0a2637c0a..deade155b309 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -26,17 +26,204 @@ import (
 	"sync"
 	"time"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"github.com/golang/protobuf/proto"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/keepalive"
+	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
+var (
+	m = make(map[version.TransportAPI]APIClientBuilder)
+)
+
+// RegisterAPIClientBuilder registers a client builder for xDS transport protocol
+// version specified by b.Version().
+//
+// NOTE: this function must only be called during initialization time (i.e. in
+// an init() function), and is not thread-safe. If multiple builders are
+// registered for the same version, the one registered last will take effect.
+func RegisterAPIClientBuilder(b APIClientBuilder) {
+	m[b.Version()] = b
+}
+
+// getAPIClientBuilder returns the client builder registered for the provided
+// xDS transport API version.
+func getAPIClientBuilder(version version.TransportAPI) APIClientBuilder {
+	if b, ok := m[version]; ok {
+		return b
+	}
+	return nil
+}
+
+// BuildOptions contains options to be passed to client builders.
+type BuildOptions struct {
+	// Parent is a top-level xDS client or server which has the intelligence to
+	// take appropriate action based on xDS responses received from the
+	// management server.
+	Parent UpdateHandler
+	// NodeProto contains the Node proto to be used in xDS requests. The actual
+	// type depends on the transport protocol version used.
+	NodeProto proto.Message
+	// Backoff returns the amount of time to backoff before retrying broken
+	// streams.
+	Backoff func(int) time.Duration
+	// Logger provides enhanced logging capabilities.
+	Logger *grpclog.PrefixLogger
+}
+
+// APIClientBuilder creates an xDS client for a specific xDS transport protocol
+// version.
+type APIClientBuilder interface {
+	// Build builds a transport protocol specific implementation of the xDS
+	// client based on the provided clientConn to the management server and the
+	// provided options.
+	Build(*grpc.ClientConn, BuildOptions) (APIClient, error)
+	// Version returns the xDS transport protocol version used by clients build
+	// using this builder.
+	Version() version.TransportAPI
+}
+
+// APIClient represents the functionality provided by transport protocol
+// version specific implementations of the xDS client.
+type APIClient interface {
+	// AddWatch adds a watch for an xDS resource given its type and name.
+	AddWatch(resourceType, resourceName string)
+	// RemoveWatch cancels an already registered watch for an xDS resource
+	// given its type and name.
+	RemoveWatch(resourceType, resourceName string)
+	// Close cleans up resources allocated by the API client.
+	Close()
+}
+
+// UpdateHandler receives and processes (by taking appropriate actions) xDS
+// resource updates from an APIClient for a specific version.
+type UpdateHandler interface {
+	// NewListeners handles updates to xDS listener resources.
+	NewListeners(map[string]ListenerUpdate)
+	// NewRouteConfigs handles updates to xDS RouteConfiguration resources.
+	NewRouteConfigs(map[string]RouteConfigUpdate)
+	// NewClusters handles updates to xDS Cluster resources.
+	NewClusters(map[string]ClusterUpdate)
+	// NewEndpoints handles updates to xDS ClusterLoadAssignment (or tersely
+	// referred to as Endpoints) resources.
+	NewEndpoints(map[string]EndpointsUpdate)
+}
+
+// ListenerUpdate contains information received in an LDS response, which is of
+// interest to the registered LDS watcher.
+type ListenerUpdate struct {
+	// RouteConfigName is the route configuration name corresponding to the
+	// target which is being watched through LDS.
+	RouteConfigName string
+}
+
+// RouteConfigUpdate contains information received in an RDS response, which is
+// of interest to the registered RDS watcher.
+type RouteConfigUpdate struct {
+	// Routes contains a list of routes, each containing matchers and
+	// corresponding action.
+	Routes []*Route
+}
+
+// Route is both a specification of how to match a request as well as an
+// indication of the action to take upon match.
+type Route struct {
+	Path, Prefix, Regex *string
+	Headers             []*HeaderMatcher
+	Fraction            *uint32
+	Action              map[string]uint32 // action is weighted clusters.
+}
+
+// HeaderMatcher represents header matchers.
+type HeaderMatcher struct {
+	Name         string      `json:"name"`
+	InvertMatch  *bool       `json:"invertMatch,omitempty"`
+	ExactMatch   *string     `json:"exactMatch,omitempty"`
+	RegexMatch   *string     `json:"regexMatch,omitempty"`
+	PrefixMatch  *string     `json:"prefixMatch,omitempty"`
+	SuffixMatch  *string     `json:"suffixMatch,omitempty"`
+	RangeMatch   *Int64Range `json:"rangeMatch,omitempty"`
+	PresentMatch *bool       `json:"presentMatch,omitempty"`
+}
+
+// Int64Range is a range for header range match.
+type Int64Range struct {
+	Start int64 `json:"start"`
+	End   int64 `json:"end"`
+}
+
+// ServiceUpdate contains information received from LDS and RDS responses,
+// which is of interest to the registered service watcher.
+type ServiceUpdate struct {
+	// Routes contain matchers+actions to route RPCs.
+	Routes []*Route
+}
+
+// ClusterUpdate contains information from a received CDS response, which is of
+// interest to the registered CDS watcher.
+type ClusterUpdate struct {
+	// ServiceName is the service name corresponding to the clusterName which
+	// is being watched for through CDS.
+	ServiceName string
+	// EnableLRS indicates whether or not load should be reported through LRS.
+	EnableLRS bool
+}
+
+// OverloadDropConfig contains the config to drop overloads.
+type OverloadDropConfig struct {
+	Category    string
+	Numerator   uint32
+	Denominator uint32
+}
+
+// EndpointHealthStatus represents the health status of an endpoint.
+type EndpointHealthStatus int32
+
+const (
+	// EndpointHealthStatusUnknown represents HealthStatus UNKNOWN.
+	EndpointHealthStatusUnknown EndpointHealthStatus = iota
+	// EndpointHealthStatusHealthy represents HealthStatus HEALTHY.
+	EndpointHealthStatusHealthy
+	// EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY.
+	EndpointHealthStatusUnhealthy
+	// EndpointHealthStatusDraining represents HealthStatus DRAINING.
+	EndpointHealthStatusDraining
+	// EndpointHealthStatusTimeout represents HealthStatus TIMEOUT.
+	EndpointHealthStatusTimeout
+	// EndpointHealthStatusDegraded represents HealthStatus DEGRADED.
+	EndpointHealthStatusDegraded
+)
+
+// Endpoint contains information of an endpoint.
+type Endpoint struct {
+	Address      string
+	HealthStatus EndpointHealthStatus
+	Weight       uint32
+}
+
+// Locality contains information of a locality.
+type Locality struct {
+	Endpoints []Endpoint
+	ID        internal.LocalityID
+	Priority  uint32
+	Weight    uint32
+}
+
+// EndpointsUpdate contains an EDS update.
+type EndpointsUpdate struct {
+	Drops      []OverloadDropConfig
+	Localities []Locality
+}
+
 // Options provides all parameters required for the creation of an xDS client.
 type Options struct {
 	// Config contains a fully populated bootstrap config. It is the
@@ -49,16 +236,13 @@ type Options struct {
 	TargetName string
 }
 
-// Interface to be overridden in tests.
-type xdsv2Client interface {
-	addWatch(resourceType, resourceName string)
-	removeWatch(resourceType, resourceName string)
-	close()
-}
-
 // Function to be overridden in tests.
-var newXDSV2Client = func(parent *Client, cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) xdsv2Client {
-	return newV2Client(parent, cc, nodeProto, backoff, logger)
+var newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, opts BuildOptions) (APIClient, error) {
+	cb := getAPIClientBuilder(apiVersion)
+	if cb == nil {
+		return nil, fmt.Errorf("no client builder for xDS API version: %v", apiVersion)
+	}
+	return cb.Build(cc, opts)
 }
 
 // Client is a full fledged gRPC client which queries a set of discovery APIs
@@ -68,20 +252,25 @@ var newXDSV2Client = func(parent *Client, cc *grpc.ClientConn, nodeProto *corepb
 // A single client object will be shared by the xds resolver and balancer
 // implementations. But the same client can only be shared by the same parent
 // ClientConn.
+//
+// Implements UpdateHandler interface.
+// TODO(easwars): Make a wrapper struct which implements this interface in the
+// style of ccBalancerWrapper so that the Client type does not implement these
+// exported methods.
 type Client struct {
-	done *grpcsync.Event
-	opts Options
-	cc   *grpc.ClientConn // Connection to the xDS server
-	v2c  xdsv2Client      // Actual xDS client implementation using the v2 API
+	done      *grpcsync.Event
+	opts      Options
+	cc        *grpc.ClientConn // Connection to the xDS server
+	apiClient APIClient
 
 	logger *grpclog.PrefixLogger
 
 	updateCh    *buffer.Unbounded // chan *watcherInfoWithUpdate
 	mu          sync.Mutex
 	ldsWatchers map[string]map[*watchInfo]bool
-	ldsCache    map[string]ldsUpdate
+	ldsCache    map[string]ListenerUpdate
 	rdsWatchers map[string]map[*watchInfo]bool
-	rdsCache    map[string]rdsUpdate
+	rdsCache    map[string]RouteConfigUpdate
 	cdsWatchers map[string]map[*watchInfo]bool
 	cdsCache    map[string]ClusterUpdate
 	edsWatchers map[string]map[*watchInfo]bool
@@ -99,6 +288,17 @@ func New(opts Options) (*Client, error) {
 		return nil, errors.New("xds: no node_proto provided in options")
 	}
 
+	switch opts.Config.TransportAPI {
+	case version.TransportV2:
+		if _, ok := opts.Config.NodeProto.(*v2corepb.Node); !ok {
+			return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", opts.Config.NodeProto, opts.Config.TransportAPI)
+		}
+	case version.TransportV3:
+		if _, ok := opts.Config.NodeProto.(*v3corepb.Node); !ok {
+			return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", opts.Config.NodeProto, opts.Config.TransportAPI)
+		}
+	}
+
 	dopts := []grpc.DialOption{
 		opts.Config.Creds,
 		grpc.WithKeepaliveParams(keepalive.ClientParameters{
@@ -114,9 +314,9 @@ func New(opts Options) (*Client, error) {
 
 		updateCh:    buffer.NewUnbounded(),
 		ldsWatchers: make(map[string]map[*watchInfo]bool),
-		ldsCache:    make(map[string]ldsUpdate),
+		ldsCache:    make(map[string]ListenerUpdate),
 		rdsWatchers: make(map[string]map[*watchInfo]bool),
-		rdsCache:    make(map[string]rdsUpdate),
+		rdsCache:    make(map[string]RouteConfigUpdate),
 		cdsWatchers: make(map[string]map[*watchInfo]bool),
 		cdsCache:    make(map[string]ClusterUpdate),
 		edsWatchers: make(map[string]map[*watchInfo]bool),
@@ -132,13 +332,16 @@ func New(opts Options) (*Client, error) {
 	c.logger = prefixLogger((c))
 	c.logger.Infof("Created ClientConn to xDS server: %s", opts.Config.BalancerName)
 
-	if opts.Config.TransportAPI == version.TransportV2 {
-		c.v2c = newXDSV2Client(c, cc, opts.Config.NodeProto.(*corepb.Node), backoff.DefaultExponential.Backoff, c.logger)
-	} else {
-		// TODO(easwars): Remove this once v3Client is ready.
-		return nil, errors.New("xds v3 client is not yet supported")
+	apiClient, err := newAPIClient(opts.Config.TransportAPI, cc, BuildOptions{
+		Parent:    c,
+		NodeProto: opts.Config.NodeProto,
+		Backoff:   backoff.DefaultExponential.Backoff,
+		Logger:    c.logger,
+	})
+	if err != nil {
+		return nil, err
 	}
-
+	c.apiClient = apiClient
 	c.logger.Infof("Created")
 	go c.run()
 	return c, nil
@@ -173,7 +376,7 @@ func (c *Client) Close() {
 	c.done.Fire()
 	// TODO: Should we invoke the registered callbacks here with an error that
 	// the client is closed?
-	c.v2c.close()
+	c.apiClient.Close()
 	c.cc.Close()
 	c.logger.Infof("Shutdown")
 }
diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go
index 4bbdaabd2f7f..a00257e4e658 100644
--- a/xds/internal/client/client_callback.go
+++ b/xds/internal/client/client_callback.go
@@ -18,6 +18,8 @@
 
 package client
 
+import "google.golang.org/grpc/xds/internal/version"
+
 type watcherInfoWithUpdate struct {
 	wi     *watchInfo
 	update interface{}
@@ -45,19 +47,19 @@ func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
 	// canceled, and the user needs to take care of it.
 	var ccb func()
 	switch wiu.wi.typeURL {
-	case ldsURL:
+	case version.V2ListenerURL:
 		if s, ok := c.ldsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
-			ccb = func() { wiu.wi.ldsCallback(wiu.update.(ldsUpdate), wiu.err) }
+			ccb = func() { wiu.wi.ldsCallback(wiu.update.(ListenerUpdate), wiu.err) }
 		}
-	case rdsURL:
+	case version.V2RouteConfigURL:
 		if s, ok := c.rdsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
-			ccb = func() { wiu.wi.rdsCallback(wiu.update.(rdsUpdate), wiu.err) }
+			ccb = func() { wiu.wi.rdsCallback(wiu.update.(RouteConfigUpdate), wiu.err) }
 		}
-	case cdsURL:
+	case version.V2ClusterURL:
 		if s, ok := c.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
 			ccb = func() { wiu.wi.cdsCallback(wiu.update.(ClusterUpdate), wiu.err) }
 		}
-	case edsURL:
+	case version.V2EndpointsURL:
 		if s, ok := c.edsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
 			ccb = func() { wiu.wi.edsCallback(wiu.update.(EndpointsUpdate), wiu.err) }
 		}
@@ -69,12 +71,12 @@ func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
 	}
 }
 
-// newLDSUpdate is called by the underlying xdsv2Client when it receives an xDS
-// response.
+// NewListeners is called by the underlying xdsAPIClient when it receives an
+// xDS response.
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newLDSUpdate(updates map[string]ldsUpdate) {
+func (c *Client) NewListeners(updates map[string]ListenerUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -104,12 +106,12 @@ func (c *Client) newLDSUpdate(updates map[string]ldsUpdate) {
 	// last watch is canceled.
 }
 
-// newRDSUpdate is called by the underlying xdsv2Client when it receives an xDS
-// response.
+// NewRouteConfigs is called by the underlying xdsAPIClient when it receives an
+// xDS response.
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newRDSUpdate(updates map[string]rdsUpdate) {
+func (c *Client) NewRouteConfigs(updates map[string]RouteConfigUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -125,12 +127,12 @@ func (c *Client) newRDSUpdate(updates map[string]rdsUpdate) {
 	}
 }
 
-// newCDSUpdate is called by the underlying xdsv2Client when it receives an xDS
+// NewClusters is called by the underlying xdsAPIClient when it receives an xDS
 // response.
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newCDSUpdate(updates map[string]ClusterUpdate) {
+func (c *Client) NewClusters(updates map[string]ClusterUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -160,12 +162,12 @@ func (c *Client) newCDSUpdate(updates map[string]ClusterUpdate) {
 	// last watch is canceled.
 }
 
-// newEDSUpdate is called by the underlying xdsv2Client when it receives an xDS
-// response.
+// NewEndpoints is called by the underlying xdsAPIClient when it receives an
+// xDS response.
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) newEDSUpdate(updates map[string]EndpointsUpdate) {
+func (c *Client) NewEndpoints(updates map[string]EndpointsUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go
new file mode 100644
index 000000000000..d45baa0e823f
--- /dev/null
+++ b/xds/internal/client/client_cds_test.go
@@ -0,0 +1,312 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+func (s) TestValidateCluster(t *testing.T) {
+	const (
+		clusterName = "clusterName"
+		serviceName = "service"
+	)
+	var (
+		emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false}
+	)
+
+	tests := []struct {
+		name       string
+		cluster    *v3clusterpb.Cluster
+		wantUpdate ClusterUpdate
+		wantErr    bool
+	}{
+		{
+			name: "non-eds-cluster-type",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "no-eds-config",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				LbPolicy:             v3clusterpb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "no-ads-config-source",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig:     &v3clusterpb.Cluster_EdsClusterConfig{},
+				LbPolicy:             v3clusterpb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "non-round-robin-lb-policy",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
+		{
+			name: "happy-case-no-service-name-no-lrs",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: emptyUpdate,
+		},
+		{
+			name: "happy-case-no-lrs",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: false},
+		},
+		{
+			name: "happiest-case",
+			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				LrsServer: &v3corepb.ConfigSource{
+					ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{
+						Self: &v3corepb.SelfConfigSource{},
+					},
+				},
+			},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			update, err := validateCluster(test.cluster)
+			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("validateCluster(%+v) = (%v, %v), wantErr: (%v, %v)", test.cluster, update, err, test.wantUpdate, test.wantErr)
+			}
+		})
+	}
+}
+
+func (s) TestUnmarshalCluster(t *testing.T) {
+	const (
+		v2ClusterName = "v2clusterName"
+		v3ClusterName = "v3clusterName"
+		v2Service     = "v2Service"
+		v3Service     = "v2Service"
+	)
+	var (
+		v2Cluster = &v2xdspb.Cluster{
+			Name:                 v2ClusterName,
+			ClusterDiscoveryType: &v2xdspb.Cluster_Type{Type: v2xdspb.Cluster_EDS},
+			EdsClusterConfig: &v2xdspb.Cluster_EdsClusterConfig{
+				EdsConfig: &v2corepb.ConfigSource{
+					ConfigSourceSpecifier: &v2corepb.ConfigSource_Ads{
+						Ads: &v2corepb.AggregatedConfigSource{},
+					},
+				},
+				ServiceName: v2Service,
+			},
+			LbPolicy: v2xdspb.Cluster_ROUND_ROBIN,
+			LrsServer: &v2corepb.ConfigSource{
+				ConfigSourceSpecifier: &v2corepb.ConfigSource_Self{
+					Self: &v2corepb.SelfConfigSource{},
+				},
+			},
+		}
+
+		v3Cluster = &v3clusterpb.Cluster{
+			Name:                 v3ClusterName,
+			ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+			EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+				EdsConfig: &v3corepb.ConfigSource{
+					ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+						Ads: &v3corepb.AggregatedConfigSource{},
+					},
+				},
+				ServiceName: v3Service,
+			},
+			LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+			LrsServer: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{
+					Self: &v3corepb.SelfConfigSource{},
+				},
+			},
+		}
+	)
+
+	tests := []struct {
+		name       string
+		resources  []*anypb.Any
+		wantUpdate map[string]ClusterUpdate
+		wantErr    bool
+	}{
+		{
+			name:      "non-cluster resource type",
+			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
+			wantErr:   true,
+		},
+		{
+			name: "badly marshaled cluster resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ClusterURL,
+					Value:   []byte{1, 2, 3, 4},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "bad cluster resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ClusterURL,
+					Value: func() []byte {
+						cl := &v3clusterpb.Cluster{
+							ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
+						}
+						mcl, _ := proto.Marshal(cl)
+						return mcl
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "v2 cluster",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ClusterURL,
+					Value: func() []byte {
+						mcl, _ := proto.Marshal(v2Cluster)
+						return mcl
+					}(),
+				},
+			},
+			wantUpdate: map[string]ClusterUpdate{
+				v2ClusterName: {ServiceName: v2Service, EnableLRS: true},
+			},
+		},
+		{
+			name: "v3 cluster",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ClusterURL,
+					Value: func() []byte {
+						mcl, _ := proto.Marshal(v3Cluster)
+						return mcl
+					}(),
+				},
+			},
+			wantUpdate: map[string]ClusterUpdate{
+				v3ClusterName: {ServiceName: v3Service, EnableLRS: true},
+			},
+		},
+		{
+			name: "multiple clusters",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ClusterURL,
+					Value: func() []byte {
+						mcl, _ := proto.Marshal(v2Cluster)
+						return mcl
+					}(),
+				},
+				{
+					TypeUrl: version.V3ClusterURL,
+					Value: func() []byte {
+						mcl, _ := proto.Marshal(v3Cluster)
+						return mcl
+					}(),
+				},
+			},
+			wantUpdate: map[string]ClusterUpdate{
+				v2ClusterName: {ServiceName: v2Service, EnableLRS: true},
+				v3ClusterName: {ServiceName: v3Service, EnableLRS: true},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			update, err := UnmarshalCluster(test.resources, nil)
+			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("UnmarshalCluster(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
+			}
+		})
+	}
+}
diff --git a/xds/internal/client/client_eds_test.go b/xds/internal/client/client_eds_test.go
new file mode 100644
index 000000000000..01df225083b5
--- /dev/null
+++ b/xds/internal/client/client_eds_test.go
@@ -0,0 +1,312 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+func (s) TestEDSParseRespProto(t *testing.T) {
+	tests := []struct {
+		name    string
+		m       *v3endpointpb.ClusterLoadAssignment
+		want    EndpointsUpdate
+		wantErr bool
+	}{
+		{
+			name: "missing-priority",
+			m: func() *v3endpointpb.ClusterLoadAssignment {
+				clab0 := newClaBuilder("test", nil)
+				clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
+				clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
+				return clab0.Build()
+			}(),
+			want:    EndpointsUpdate{},
+			wantErr: true,
+		},
+		{
+			name: "missing-locality-ID",
+			m: func() *v3endpointpb.ClusterLoadAssignment {
+				clab0 := newClaBuilder("test", nil)
+				clab0.addLocality("", 1, 0, []string{"addr1:314"}, nil)
+				return clab0.Build()
+			}(),
+			want:    EndpointsUpdate{},
+			wantErr: true,
+		},
+		{
+			name: "good",
+			m: func() *v3endpointpb.ClusterLoadAssignment {
+				clab0 := newClaBuilder("test", nil)
+				clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{
+					Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY},
+					Weight: []uint32{271},
+				})
+				clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{
+					Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING},
+					Weight: []uint32{828},
+				})
+				return clab0.Build()
+			}(),
+			want: EndpointsUpdate{
+				Drops: nil,
+				Localities: []Locality{
+					{
+						Endpoints: []Endpoint{{
+							Address:      "addr1:314",
+							HealthStatus: EndpointHealthStatusUnhealthy,
+							Weight:       271,
+						}},
+						ID:       internal.LocalityID{SubZone: "locality-1"},
+						Priority: 1,
+						Weight:   1,
+					},
+					{
+						Endpoints: []Endpoint{{
+							Address:      "addr2:159",
+							HealthStatus: EndpointHealthStatusDraining,
+							Weight:       828,
+						}},
+						ID:       internal.LocalityID{SubZone: "locality-2"},
+						Priority: 0,
+						Weight:   1,
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseEDSRespProto(tt.m)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseEDSRespProto() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if d := cmp.Diff(got, tt.want); d != "" {
+				t.Errorf("parseEDSRespProto() got = %v, want %v, diff: %v", got, tt.want, d)
+			}
+		})
+	}
+}
+
+func (s) TestUnmarshalEndpoints(t *testing.T) {
+	tests := []struct {
+		name       string
+		resources  []*anypb.Any
+		wantUpdate map[string]EndpointsUpdate
+		wantErr    bool
+	}{
+		{
+			name:      "non-clusterLoadAssignment resource type",
+			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
+			wantErr:   true,
+		},
+		{
+			name: "badly marshaled clusterLoadAssignment resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3EndpointsURL,
+					Value:   []byte{1, 2, 3, 4},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "bad endpoints resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3EndpointsURL,
+					Value: func() []byte {
+						clab0 := newClaBuilder("test", nil)
+						clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
+						clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
+						e := clab0.Build()
+						me, _ := proto.Marshal(e)
+						return me
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "v3 endpoints",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3EndpointsURL,
+					Value: func() []byte {
+						clab0 := newClaBuilder("test", nil)
+						clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{
+							Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY},
+							Weight: []uint32{271},
+						})
+						clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{
+							Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING},
+							Weight: []uint32{828},
+						})
+						e := clab0.Build()
+						me, _ := proto.Marshal(e)
+						return me
+					}(),
+				},
+			},
+			wantUpdate: map[string]EndpointsUpdate{
+				"test": {
+					Drops: nil,
+					Localities: []Locality{
+						{
+							Endpoints: []Endpoint{{
+								Address:      "addr1:314",
+								HealthStatus: EndpointHealthStatusUnhealthy,
+								Weight:       271,
+							}},
+							ID:       internal.LocalityID{SubZone: "locality-1"},
+							Priority: 1,
+							Weight:   1,
+						},
+						{
+							Endpoints: []Endpoint{{
+								Address:      "addr2:159",
+								HealthStatus: EndpointHealthStatusDraining,
+								Weight:       828,
+							}},
+							ID:       internal.LocalityID{SubZone: "locality-2"},
+							Priority: 0,
+							Weight:   1,
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			update, err := UnmarshalEndpoints(test.resources, nil)
+			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("UnmarshalEndpoints(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
+			}
+		})
+	}
+}
+
+// claBuilder builds a ClusterLoadAssignment, aka EDS
+// response.
+type claBuilder struct {
+	v *v3endpointpb.ClusterLoadAssignment
+}
+
+// newClaBuilder creates a claBuilder.
+func newClaBuilder(clusterName string, dropPercents []uint32) *claBuilder {
+	var drops []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload
+	for i, d := range dropPercents {
+		drops = append(drops, &v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{
+			Category: fmt.Sprintf("test-drop-%d", i),
+			DropPercentage: &v3typepb.FractionalPercent{
+				Numerator:   d,
+				Denominator: v3typepb.FractionalPercent_HUNDRED,
+			},
+		})
+	}
+
+	return &claBuilder{
+		v: &v3endpointpb.ClusterLoadAssignment{
+			ClusterName: clusterName,
+			Policy: &v3endpointpb.ClusterLoadAssignment_Policy{
+				DropOverloads: drops,
+			},
+		},
+	}
+}
+
+// addLocalityOptions contains options when adding locality to the builder.
+type addLocalityOptions struct {
+	Health []v3corepb.HealthStatus
+	Weight []uint32
+}
+
+// addLocality adds a locality to the builder.
+func (clab *claBuilder) addLocality(subzone string, weight uint32, priority uint32, addrsWithPort []string, opts *addLocalityOptions) {
+	var lbEndPoints []*v3endpointpb.LbEndpoint
+	for i, a := range addrsWithPort {
+		host, portStr, err := net.SplitHostPort(a)
+		if err != nil {
+			panic("failed to split " + a)
+		}
+		port, err := strconv.Atoi(portStr)
+		if err != nil {
+			panic("failed to atoi " + portStr)
+		}
+
+		lbe := &v3endpointpb.LbEndpoint{
+			HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+				Endpoint: &v3endpointpb.Endpoint{
+					Address: &v3corepb.Address{
+						Address: &v3corepb.Address_SocketAddress{
+							SocketAddress: &v3corepb.SocketAddress{
+								Protocol: v3corepb.SocketAddress_TCP,
+								Address:  host,
+								PortSpecifier: &v3corepb.SocketAddress_PortValue{
+									PortValue: uint32(port)}}}}}},
+		}
+		if opts != nil {
+			if i < len(opts.Health) {
+				lbe.HealthStatus = opts.Health[i]
+			}
+			if i < len(opts.Weight) {
+				lbe.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: opts.Weight[i]}
+			}
+		}
+		lbEndPoints = append(lbEndPoints, lbe)
+	}
+
+	var localityID *v3corepb.Locality
+	if subzone != "" {
+		localityID = &v3corepb.Locality{
+			Region:  "",
+			Zone:    "",
+			SubZone: subzone,
+		}
+	}
+
+	clab.v.Endpoints = append(clab.v.Endpoints, &v3endpointpb.LocalityLbEndpoints{
+		Locality:            localityID,
+		LbEndpoints:         lbEndPoints,
+		LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight},
+		Priority:            priority,
+	})
+}
+
+// Build builds ClusterLoadAssignment.
+func (clab *claBuilder) Build() *v3endpointpb.ClusterLoadAssignment {
+	return clab.v
+}
diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
new file mode 100644
index 000000000000..eedfe72f94e2
--- /dev/null
+++ b/xds/internal/client/client_lds_test.go
@@ -0,0 +1,171 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v2httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
+	v2listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+func (s) TestUnmarshalListener(t *testing.T) {
+	const (
+		v2LDSTarget       = "lds.target.good:2222"
+		v3LDSTarget       = "lds.target.good:3333"
+		v2RouteConfigName = "v2RouteConfig"
+		v3RouteConfigName = "v3RouteConfig"
+	)
+
+	var (
+		v2Lis = &anypb.Any{
+			TypeUrl: version.V2ListenerURL,
+			Value: func() []byte {
+				cm := &v2httppb.HttpConnectionManager{
+					RouteSpecifier: &v2httppb.HttpConnectionManager_Rds{
+						Rds: &v2httppb.Rds{
+							ConfigSource: &v2corepb.ConfigSource{
+								ConfigSourceSpecifier: &v2corepb.ConfigSource_Ads{Ads: &v2corepb.AggregatedConfigSource{}},
+							},
+							RouteConfigName: v2RouteConfigName,
+						},
+					},
+				}
+				mcm, _ := proto.Marshal(cm)
+				lis := &v2xdspb.Listener{
+					Name: v2LDSTarget,
+					ApiListener: &v2listenerpb.ApiListener{
+						ApiListener: &anypb.Any{
+							TypeUrl: version.V2HTTPConnManagerURL,
+							Value:   mcm,
+						},
+					},
+				}
+				mLis, _ := proto.Marshal(lis)
+				return mLis
+			}(),
+		}
+		v3Lis = &anypb.Any{
+			TypeUrl: version.V3ListenerURL,
+			Value: func() []byte {
+				cm := &v3httppb.HttpConnectionManager{
+					RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+						Rds: &v3httppb.Rds{
+							ConfigSource: &v3corepb.ConfigSource{
+								ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+							},
+							RouteConfigName: v3RouteConfigName,
+						},
+					},
+				}
+				mcm, _ := proto.Marshal(cm)
+				lis := &v3listenerpb.Listener{
+					Name: v3LDSTarget,
+					ApiListener: &v3listenerpb.ApiListener{
+						ApiListener: &anypb.Any{
+							TypeUrl: version.V3HTTPConnManagerURL,
+							Value:   mcm,
+						},
+					},
+				}
+				mLis, _ := proto.Marshal(lis)
+				return mLis
+			}(),
+		}
+	)
+
+	tests := []struct {
+		name       string
+		resources  []*anypb.Any
+		wantUpdate map[string]ListenerUpdate
+		wantErr    bool
+	}{
+		{
+			name:      "non-listener resource",
+			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
+			wantErr:   true,
+		},
+		{
+			name: "badly marshaled listener resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V3HTTPConnManagerURL,
+									Value:   []byte{1, 2, 3, 4},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty resource list",
+		},
+		{
+			name:      "v2 listener resource",
+			resources: []*anypb.Any{v2Lis},
+			wantUpdate: map[string]ListenerUpdate{
+				v2LDSTarget: {RouteConfigName: v2RouteConfigName},
+			},
+		},
+		{
+			name:      "v3 listener resource",
+			resources: []*anypb.Any{v3Lis},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName},
+			},
+		},
+		{
+			name:      "multiple listener resources",
+			resources: []*anypb.Any{v2Lis, v3Lis},
+			wantUpdate: map[string]ListenerUpdate{
+				v2LDSTarget: {RouteConfigName: v2RouteConfigName},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			update, err := UnmarshalListener(test.resources, nil)
+			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("UnmarshalListener(%v) = (%v, %v) want (%v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
+			}
+		})
+	}
+}
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
new file mode 100644
index 000000000000..44e2cfc5dde8
--- /dev/null
+++ b/xds/internal/client/client_rds_test.go
@@ -0,0 +1,854 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"testing"
+
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v2routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+func (s) TestGetRouteConfigFromListener(t *testing.T) {
+	const (
+		goodLDSTarget       = "lds.target.good:1111"
+		goodRouteConfigName = "GoodRouteConfig"
+	)
+
+	tests := []struct {
+		name      string
+		lis       *v3listenerpb.Listener
+		wantRoute string
+		wantErr   bool
+	}{
+		{
+			name:      "no-apiListener-field",
+			lis:       &v3listenerpb.Listener{},
+			wantRoute: "",
+			wantErr:   true,
+		},
+		{
+			name: "badly-marshaled-apiListener",
+			lis: &v3listenerpb.Listener{
+				Name: goodLDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V3HTTPConnManagerURL,
+						Value:   []byte{1, 2, 3, 4},
+					},
+				},
+			},
+			wantRoute: "",
+			wantErr:   true,
+		},
+		{
+			name: "wrong-type-in-apiListener",
+			lis: &v3listenerpb.Listener{
+				Name: goodLDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V2ListenerURL,
+						Value: func() []byte {
+							cm := &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+									Rds: &v3httppb.Rds{
+										ConfigSource: &v3corepb.ConfigSource{
+											ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+										},
+										RouteConfigName: goodRouteConfigName}}}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}()}}},
+			wantRoute: "",
+			wantErr:   true,
+		},
+		{
+			name: "empty-httpConnMgr-in-apiListener",
+			lis: &v3listenerpb.Listener{
+				Name: goodLDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V3HTTPConnManagerURL,
+						Value: func() []byte {
+							cm := &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+									Rds: &v3httppb.Rds{},
+								},
+							}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}()}}},
+			wantRoute: "",
+			wantErr:   true,
+		},
+		{
+			name: "scopedRoutes-routeConfig-in-apiListener",
+			lis: &v3listenerpb.Listener{
+				Name: goodLDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V3HTTPConnManagerURL,
+						Value: func() []byte {
+							cm := &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+							}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}()}}},
+			wantRoute: "",
+			wantErr:   true,
+		},
+		{
+			name: "rds.ConfigSource-in-apiListener-is-not-ADS",
+			lis: &v3listenerpb.Listener{
+				Name: goodLDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V3HTTPConnManagerURL,
+						Value: func() []byte {
+							cm := &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+									Rds: &v3httppb.Rds{
+										ConfigSource: &v3corepb.ConfigSource{
+											ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{
+												Path: "/some/path",
+											},
+										},
+										RouteConfigName: goodRouteConfigName}}}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}()}}},
+			wantRoute: "",
+			wantErr:   true,
+		},
+		{
+			name: "goodListener",
+			lis: &v3listenerpb.Listener{
+				Name: goodLDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V3HTTPConnManagerURL,
+						Value: func() []byte {
+							cm := &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+									Rds: &v3httppb.Rds{
+										ConfigSource: &v3corepb.ConfigSource{
+											ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+										},
+										RouteConfigName: goodRouteConfigName}}}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}()}}},
+			wantRoute: goodRouteConfigName,
+			wantErr:   false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotRoute, err := getRouteConfigNameFromListener(test.lis, nil)
+			if (err != nil) != test.wantErr || gotRoute != test.wantRoute {
+				t.Errorf("getRouteConfigNameFromListener(%+v) = (%s, %v), want (%s, %v)", test.lis, gotRoute, err, test.wantRoute, test.wantErr)
+			}
+		})
+	}
+}
+
+func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
+	const (
+		uninterestingDomain      = "uninteresting.domain"
+		uninterestingClusterName = "uninterestingClusterName"
+		ldsTarget                = "lds.target.good:1111"
+		routeName                = "routeName"
+		clusterName              = "clusterName"
+	)
+
+	tests := []struct {
+		name       string
+		rc         *v3routepb.RouteConfiguration
+		wantUpdate RouteConfigUpdate
+		wantError  bool
+	}{
+		{
+			name:      "no-virtual-hosts-in-rc",
+			rc:        &v3routepb.RouteConfiguration{},
+			wantError: true,
+		},
+		{
+			name: "no-domains-in-rc",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{{}},
+			},
+			wantError: true,
+		},
+		{
+			name: "non-matching-domain-in-rc",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{Domains: []string{uninterestingDomain}},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "no-routes-in-rc",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{Domains: []string{ldsTarget}},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "default-route-match-field-is-nil",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "default-route-match-field-is-non-nil",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match:  &v3routepb.RouteMatch{},
+								Action: &v3routepb.Route_Route{},
+							},
+						},
+					},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "default-route-routeaction-field-is-nil",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*v3routepb.Route{{}},
+					},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "default-route-cluster-field-is-empty",
+			rc: &v3routepb.RouteConfiguration{
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_ClusterHeader{},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantError: true,
+		},
+		{
+			// default route's match sets case-sensitive to false.
+			name: "good-route-config-but-with-casesensitive-false",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{ldsTarget},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{
+							PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+							CaseSensitive: &wrapperspb.BoolValue{Value: false},
+						},
+						Action: &v3routepb.Route_Route{
+							Route: &v3routepb.RouteAction{
+								ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+							}}}}}}},
+			wantError: true,
+		},
+		{
+			name: "good-route-config-with-empty-string-route",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{uninterestingDomain},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName},
+									},
+								},
+							},
+						},
+					},
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{clusterName: 1}}}},
+		},
+		{
+			// default route's match is not empty string, but "/".
+			name: "good-route-config-with-slash-string-route",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}}}},
+		},
+		{
+			// weights not add up to total-weight.
+			name: "route-config-with-weighted_clusters_weights_not_add_up",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+											WeightedClusters: &v3routepb.WeightedCluster{
+												Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+													{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
+													{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
+													{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
+												},
+												TotalWeight: &wrapperspb.UInt32Value{Value: 30},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "good-route-config-with-weighted_clusters",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+											WeightedClusters: &v3routepb.WeightedCluster{
+												Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+													{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
+													{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
+													{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
+												},
+												TotalWeight: &wrapperspb.UInt32Value{Value: 10},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, ldsTarget, nil)
+			if (gotError != nil) != test.wantError || !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, ldsTarget, gotUpdate, test.wantUpdate)
+			}
+		})
+	}
+}
+
+func (s) TestUnmarshalRouteConfig(t *testing.T) {
+	const (
+		ldsTarget                = "lds.target.good:1111"
+		uninterestingDomain      = "uninteresting.domain"
+		uninterestingClusterName = "uninterestingClusterName"
+		v2RouteConfigName        = "v2RouteConfig"
+		v3RouteConfigName        = "v3RouteConfig"
+		v2ClusterName            = "v2Cluster"
+		v3ClusterName            = "v3Cluster"
+	)
+
+	var (
+		v2VirtualHost = []*v2routepb.VirtualHost{
+			{
+				Domains: []string{uninterestingDomain},
+				Routes: []*v2routepb.Route{
+					{
+						Match: &v2routepb.RouteMatch{PathSpecifier: &v2routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v2routepb.Route_Route{
+							Route: &v2routepb.RouteAction{
+								ClusterSpecifier: &v2routepb.RouteAction_Cluster{Cluster: uninterestingClusterName},
+							},
+						},
+					},
+				},
+			},
+			{
+				Domains: []string{ldsTarget},
+				Routes: []*v2routepb.Route{
+					{
+						Match: &v2routepb.RouteMatch{PathSpecifier: &v2routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v2routepb.Route_Route{
+							Route: &v2routepb.RouteAction{
+								ClusterSpecifier: &v2routepb.RouteAction_Cluster{Cluster: v2ClusterName},
+							},
+						},
+					},
+				},
+			},
+		}
+		v2RouteConfig = &anypb.Any{
+			TypeUrl: version.V2RouteConfigURL,
+			Value: func() []byte {
+				rc := &v2xdspb.RouteConfiguration{
+					Name:         v2RouteConfigName,
+					VirtualHosts: v2VirtualHost,
+				}
+				m, _ := proto.Marshal(rc)
+				return m
+			}(),
+		}
+		v3VirtualHost = []*v3routepb.VirtualHost{
+			{
+				Domains: []string{uninterestingDomain},
+				Routes: []*v3routepb.Route{
+					{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{
+							Route: &v3routepb.RouteAction{
+								ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName},
+							},
+						},
+					},
+				},
+			},
+			{
+				Domains: []string{ldsTarget},
+				Routes: []*v3routepb.Route{
+					{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{
+							Route: &v3routepb.RouteAction{
+								ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: v3ClusterName},
+							},
+						},
+					},
+				},
+			},
+		}
+		v3RouteConfig = &anypb.Any{
+			TypeUrl: version.V2RouteConfigURL,
+			Value: func() []byte {
+				rc := &v3routepb.RouteConfiguration{
+					Name:         v3RouteConfigName,
+					VirtualHosts: v3VirtualHost,
+				}
+				m, _ := proto.Marshal(rc)
+				return m
+			}(),
+		}
+	)
+
+	tests := []struct {
+		name       string
+		resources  []*anypb.Any
+		wantUpdate map[string]RouteConfigUpdate
+		wantErr    bool
+	}{
+		{
+			name:      "non-routeConfig resource type",
+			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
+			wantErr:   true,
+		},
+		{
+			name: "badly marshaled routeconfig resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3RouteConfigURL,
+					Value:   []byte{1, 2, 3, 4},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "bad routeConfig resource",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3RouteConfigURL,
+					Value: func() []byte {
+						rc := &v3routepb.RouteConfiguration{
+							VirtualHosts: []*v3routepb.VirtualHost{
+								{Domains: []string{uninterestingDomain}},
+							},
+						}
+						m, _ := proto.Marshal(rc)
+						return m
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty resource list",
+		},
+		{
+			name:      "v2 routeConfig resource",
+			resources: []*anypb.Any{v2RouteConfig},
+			wantUpdate: map[string]RouteConfigUpdate{
+				v2RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}}},
+			},
+		},
+		{
+			name:      "v3 routeConfig resource",
+			resources: []*anypb.Any{v3RouteConfig},
+			wantUpdate: map[string]RouteConfigUpdate{
+				v3RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}}},
+			},
+		},
+		{
+			name:      "multiple routeConfig resources",
+			resources: []*anypb.Any{v2RouteConfig, v3RouteConfig},
+			wantUpdate: map[string]RouteConfigUpdate{
+				v3RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}}},
+				v2RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}}},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			update, err := UnmarshalRouteConfig(test.resources, ldsTarget, nil)
+			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("UnmarshalRouteConfig(%v, %v) = (%v, %v) want (%v, %v)", test.resources, ldsTarget, update, err, test.wantUpdate, test.wantErr)
+			}
+		})
+	}
+}
+
+func (s) TestMatchTypeForDomain(t *testing.T) {
+	tests := []struct {
+		d    string
+		want domainMatchType
+	}{
+		{d: "", want: domainMatchTypeInvalid},
+		{d: "*", want: domainMatchTypeUniversal},
+		{d: "bar.*", want: domainMatchTypePrefix},
+		{d: "*.abc.com", want: domainMatchTypeSuffix},
+		{d: "foo.bar.com", want: domainMatchTypeExact},
+		{d: "foo.*.com", want: domainMatchTypeInvalid},
+	}
+	for _, tt := range tests {
+		if got := matchTypeForDomain(tt.d); got != tt.want {
+			t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want)
+		}
+	}
+}
+
+func (s) TestMatch(t *testing.T) {
+	tests := []struct {
+		name        string
+		domain      string
+		host        string
+		wantTyp     domainMatchType
+		wantMatched bool
+	}{
+		{name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
+		{name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
+		{name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true},
+		{name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true},
+		{name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false},
+		{name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true},
+		{name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false},
+		{name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true},
+		{name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched {
+				t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched)
+			}
+		})
+	}
+}
+
+func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
+	var (
+		oneExactMatch = &v3routepb.VirtualHost{
+			Name:    "one-exact-match",
+			Domains: []string{"foo.bar.com"},
+		}
+		oneSuffixMatch = &v3routepb.VirtualHost{
+			Name:    "one-suffix-match",
+			Domains: []string{"*.bar.com"},
+		}
+		onePrefixMatch = &v3routepb.VirtualHost{
+			Name:    "one-prefix-match",
+			Domains: []string{"foo.bar.*"},
+		}
+		oneUniversalMatch = &v3routepb.VirtualHost{
+			Name:    "one-universal-match",
+			Domains: []string{"*"},
+		}
+		longExactMatch = &v3routepb.VirtualHost{
+			Name:    "one-exact-match",
+			Domains: []string{"v2.foo.bar.com"},
+		}
+		multipleMatch = &v3routepb.VirtualHost{
+			Name:    "multiple-match",
+			Domains: []string{"pi.foo.bar.com", "314.*", "*.159"},
+		}
+		vhs = []*v3routepb.VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
+	)
+
+	tests := []struct {
+		name   string
+		host   string
+		vHosts []*v3routepb.VirtualHost
+		want   *v3routepb.VirtualHost
+	}{
+		{name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch},
+		{name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch},
+		{name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch},
+		{name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch},
+		{name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch},
+		// Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact.
+		{name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch},
+		// Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix.
+		{name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch},
+		// Matches suffix "*.bar.com" and prefix "314.*". Takes suffix.
+		{name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := findBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) {
+				t.Errorf("findBestMatchingVirtualHost() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func (s) TestRoutesProtoToSlice(t *testing.T) {
+	tests := []struct {
+		name       string
+		routes     []*v3routepb.Route
+		wantRoutes []*Route
+		wantErr    bool
+	}{
+		{
+			name: "no path",
+			routes: []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{},
+			}},
+			wantErr: true,
+		},
+		{
+			name: "case_sensitive is false",
+			routes: []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{
+					PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+					CaseSensitive: &wrapperspb.BoolValue{Value: false},
+				},
+			}},
+			wantErr: true,
+		},
+		{
+			name: "good",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"},
+						Headers: []*v3routepb.HeaderMatcher{
+							{
+								Name: "th",
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{
+									PrefixMatch: "tv",
+								},
+								InvertMatch: true,
+							},
+						},
+						RuntimeFraction: &v3corepb.RuntimeFractionalPercent{
+							DefaultValue: &v3typepb.FractionalPercent{
+								Numerator:   1,
+								Denominator: v3typepb.FractionalPercent_HUNDRED,
+							},
+						},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &v3routepb.WeightedCluster{
+									Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
+										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+									},
+									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+								}}}},
+				},
+			},
+			wantRoutes: []*Route{{
+				Prefix: newStringP("/a/"),
+				Headers: []*HeaderMatcher{
+					{
+						Name:        "th",
+						InvertMatch: newBoolP(true),
+						PrefixMatch: newStringP("tv"),
+					},
+				},
+				Fraction: newUInt32P(10000),
+				Action:   map[string]uint32{"A": 40, "B": 60},
+			}},
+			wantErr: false,
+		},
+		{
+			name: "query is ignored",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &v3routepb.WeightedCluster{
+									Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
+										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+									},
+									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+								}}}},
+				},
+				{
+					Name: "with_query",
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier:   &v3routepb.RouteMatch_Prefix{Prefix: "/b/"},
+						QueryParameters: []*v3routepb.QueryParameterMatcher{{Name: "route_will_be_ignored"}},
+					},
+				},
+			},
+			// Only one route in the result, because the second one with query
+			// parameters is ignored.
+			wantRoutes: []*Route{{
+				Prefix: newStringP("/a/"),
+				Action: map[string]uint32{"A": 40, "B": 60},
+			}},
+			wantErr: false,
+		},
+	}
+
+	cmpOpts := []cmp.Option{
+		cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}),
+		cmpopts.EquateEmpty(),
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := routesProtoToSlice(tt.routes, nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.wantRoutes, cmpOpts...) {
+				t.Errorf("routesProtoToSlice() got = %v, want %v, diff: %v", got, tt.wantRoutes, cmp.Diff(got, tt.wantRoutes, cmpOpts...))
+			}
+		})
+	}
+}
+
+func newStringP(s string) *string {
+	return &s
+}
+
+func newUInt32P(i uint32) *uint32 {
+	return &i
+}
+
+func newBoolP(b bool) *bool {
+	return &b
+}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 043fa4f76de1..849d91a4d852 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -23,13 +23,10 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 type s struct {
@@ -60,119 +57,56 @@ func clientOpts(balancerName string) Options {
 	}
 }
 
-func (s) TestNew(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
-	defer cleanup()
-
-	tests := []struct {
-		name    string
-		opts    Options
-		wantErr bool
-	}{
-		{name: "empty-opts", opts: Options{}, wantErr: true},
-		{
-			name: "empty-balancer-name",
-			opts: Options{
-				Config: bootstrap.Config{
-					Creds:     grpc.WithInsecure(),
-					NodeProto: testutils.EmptyNodeProtoV2,
-				},
-			},
-			wantErr: true,
-		},
-		{
-			name: "empty-dial-creds",
-			opts: Options{
-				Config: bootstrap.Config{
-					BalancerName: "dummy",
-					NodeProto:    testutils.EmptyNodeProtoV2,
-				},
-			},
-			wantErr: true,
-		},
-		{
-			name: "empty-node-proto",
-			opts: Options{
-				Config: bootstrap.Config{
-					BalancerName: "dummy",
-					Creds:        grpc.WithInsecure(),
-				},
-			},
-			wantErr: true,
-		},
-		{
-			name:    "happy-case",
-			opts:    clientOpts(fakeServer.Address),
-			wantErr: false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			c, err := New(test.opts)
-			if err == nil {
-				defer c.Close()
-			}
-			if (err != nil) != test.wantErr {
-				t.Fatalf("New(%+v) = %v, wantErr: %v", test.opts, err, test.wantErr)
-			}
-		})
-	}
-}
-
-type testXDSV2Client struct {
-	r updateHandler
+type testAPIClient struct {
+	r UpdateHandler
 
 	addWatches    map[string]*testutils.Channel
 	removeWatches map[string]*testutils.Channel
 }
 
-func overrideNewXDSV2Client() (<-chan *testXDSV2Client, func()) {
-	oldNewXDSV2Client := newXDSV2Client
-	ch := make(chan *testXDSV2Client, 1)
-	newXDSV2Client = func(parent *Client, cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) xdsv2Client {
-		ret := newTestXDSV2Client(parent)
+func overrideNewAPIClient() (<-chan *testAPIClient, func()) {
+	origNewAPIClient := newAPIClient
+	ch := make(chan *testAPIClient, 1)
+	newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, opts BuildOptions) (APIClient, error) {
+		ret := newTestAPIClient(opts.Parent)
 		ch <- ret
-		return ret
+		return ret, nil
 	}
-	return ch, func() { newXDSV2Client = oldNewXDSV2Client }
+	return ch, func() { newAPIClient = origNewAPIClient }
 }
 
-func newTestXDSV2Client(r updateHandler) *testXDSV2Client {
+func newTestAPIClient(r UpdateHandler) *testAPIClient {
 	addWatches := make(map[string]*testutils.Channel)
-	addWatches[ldsURL] = testutils.NewChannel()
-	addWatches[rdsURL] = testutils.NewChannel()
-	addWatches[cdsURL] = testutils.NewChannel()
-	addWatches[edsURL] = testutils.NewChannel()
+	addWatches[version.V2ListenerURL] = testutils.NewChannel()
+	addWatches[version.V2RouteConfigURL] = testutils.NewChannel()
+	addWatches[version.V2ClusterURL] = testutils.NewChannel()
+	addWatches[version.V2EndpointsURL] = testutils.NewChannel()
 	removeWatches := make(map[string]*testutils.Channel)
-	removeWatches[ldsURL] = testutils.NewChannel()
-	removeWatches[rdsURL] = testutils.NewChannel()
-	removeWatches[cdsURL] = testutils.NewChannel()
-	removeWatches[edsURL] = testutils.NewChannel()
-	return &testXDSV2Client{
+	removeWatches[version.V2ListenerURL] = testutils.NewChannel()
+	removeWatches[version.V2RouteConfigURL] = testutils.NewChannel()
+	removeWatches[version.V2ClusterURL] = testutils.NewChannel()
+	removeWatches[version.V2EndpointsURL] = testutils.NewChannel()
+	return &testAPIClient{
 		r:             r,
 		addWatches:    addWatches,
 		removeWatches: removeWatches,
 	}
 }
 
-func (c *testXDSV2Client) addWatch(resourceType, resourceName string) {
+func (c *testAPIClient) AddWatch(resourceType, resourceName string) {
 	c.addWatches[resourceType].Send(resourceName)
 }
 
-func (c *testXDSV2Client) removeWatch(resourceType, resourceName string) {
+func (c *testAPIClient) RemoveWatch(resourceType, resourceName string) {
 	c.removeWatches[resourceType].Send(resourceName)
 }
 
-func (c *testXDSV2Client) close() {}
+func (c *testAPIClient) Close() {}
 
 // TestWatchCallAnotherWatch covers the case where watch() is called inline by a
 // callback. It makes sure it doesn't cause a deadlock.
 func (s) TestWatchCallAnotherWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -189,17 +123,17 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		// Calls another watch inline, to ensure there's deadlock.
 		c.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
-		if _, err := v2Client.addWatches[cdsURL].Receive(); firstTime && err != nil {
+		if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); firstTime && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 		firstTime = false
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -208,7 +142,7 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 	}
 
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate2,
 	})
 
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index cfe31e861380..809a20fee768 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -22,19 +22,14 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 // The value chosen here is based on the default value of the
 // initial_fetch_timeout field in corepb.ConfigSource proto.
 var defaultWatchExpiryTimeout = 15 * time.Second
 
-const (
-	ldsURL = "type.googleapis.com/envoy.api.v2.Listener"
-	rdsURL = "type.googleapis.com/envoy.api.v2.RouteConfiguration"
-	cdsURL = "type.googleapis.com/envoy.api.v2.Cluster"
-	edsURL = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
-)
-
 type watchInfoState int
 
 const (
@@ -50,8 +45,8 @@ type watchInfo struct {
 	typeURL string
 	target  string
 
-	ldsCallback ldsCallbackFunc
-	rdsCallback rdsCallbackFunc
+	ldsCallback func(ListenerUpdate, error)
+	rdsCallback func(RouteConfigUpdate, error)
 	cdsCallback func(ClusterUpdate, error)
 	edsCallback func(EndpointsUpdate, error)
 
@@ -102,13 +97,13 @@ func (wi *watchInfo) sendErrorLocked(err error) {
 		u interface{}
 	)
 	switch wi.typeURL {
-	case ldsURL:
-		u = ldsUpdate{}
-	case rdsURL:
-		u = rdsUpdate{}
-	case cdsURL:
+	case version.V2ListenerURL:
+		u = ListenerUpdate{}
+	case version.V2RouteConfigURL:
+		u = RouteConfigUpdate{}
+	case version.V2ClusterURL:
 		u = ClusterUpdate{}
-	case edsURL:
+	case version.V2EndpointsURL:
 		u = EndpointsUpdate{}
 	}
 	wi.c.scheduleCallback(wi, u, err)
@@ -130,13 +125,13 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 	c.logger.Debugf("new watch for type %v, resource name %v", wi.typeURL, wi.target)
 	var watchers map[string]map[*watchInfo]bool
 	switch wi.typeURL {
-	case ldsURL:
+	case version.V2ListenerURL:
 		watchers = c.ldsWatchers
-	case rdsURL:
+	case version.V2RouteConfigURL:
 		watchers = c.rdsWatchers
-	case cdsURL:
+	case version.V2ClusterURL:
 		watchers = c.cdsWatchers
-	case edsURL:
+	case version.V2EndpointsURL:
 		watchers = c.edsWatchers
 	}
 
@@ -151,7 +146,7 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 		c.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.typeURL, wi.target)
 		s = make(map[*watchInfo]bool)
 		watchers[resourceName] = s
-		c.v2c.addWatch(wi.typeURL, resourceName)
+		c.apiClient.AddWatch(wi.typeURL, resourceName)
 	}
 	// No matter what, add the new watcher to the set, so it's callback will be
 	// call for new responses.
@@ -159,22 +154,22 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 
 	// If the resource is in cache, call the callback with the value.
 	switch wi.typeURL {
-	case ldsURL:
+	case version.V2ListenerURL:
 		if v, ok := c.ldsCache[resourceName]; ok {
 			c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
 		}
-	case rdsURL:
+	case version.V2RouteConfigURL:
 		if v, ok := c.rdsCache[resourceName]; ok {
 			c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
 		}
-	case cdsURL:
+	case version.V2ClusterURL:
 		if v, ok := c.cdsCache[resourceName]; ok {
 			c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
 		}
-	case edsURL:
+	case version.V2EndpointsURL:
 		if v, ok := c.edsCache[resourceName]; ok {
 			c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
@@ -195,21 +190,207 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 				// If this was the last watcher, also tell xdsv2Client to stop
 				// watching this resource.
 				delete(watchers, resourceName)
-				c.v2c.removeWatch(wi.typeURL, resourceName)
+				c.apiClient.RemoveWatch(wi.typeURL, resourceName)
 				// Remove the resource from cache. When a watch for this
 				// resource is added later, it will trigger a xDS request with
 				// resource names, and client will receive new xDS responses.
 				switch wi.typeURL {
-				case ldsURL:
+				case version.V2ListenerURL:
 					delete(c.ldsCache, resourceName)
-				case rdsURL:
+				case version.V2RouteConfigURL:
 					delete(c.rdsCache, resourceName)
-				case cdsURL:
+				case version.V2ClusterURL:
 					delete(c.cdsCache, resourceName)
-				case edsURL:
+				case version.V2EndpointsURL:
 					delete(c.edsCache, resourceName)
 				}
 			}
 		}
 	}
 }
+
+// watchLDS starts a listener watcher for the service..
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) {
+	wi := &watchInfo{
+		c:           c,
+		typeURL:     version.V2ListenerURL,
+		target:      serviceName,
+		ldsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		wi.timeout()
+	})
+	return c.watch(wi)
+}
+
+// watchRDS starts a listener watcher for the service..
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) watchRDS(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) {
+	wi := &watchInfo{
+		c:           c,
+		typeURL:     version.V2RouteConfigURL,
+		target:      routeName,
+		rdsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		wi.timeout()
+	})
+	return c.watch(wi)
+}
+
+// WatchService uses LDS and RDS to discover information about the provided
+// serviceName.
+//
+// WatchService can only be called once. The second call will not start a
+// watcher and the callback will get an error. It's this case because an xDS
+// client is expected to be used only by one ClientConn.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error)) (cancel func()) {
+	c.mu.Lock()
+	if len(c.ldsWatchers) != 0 {
+		go cb(ServiceUpdate{}, fmt.Errorf("unexpected WatchService when there's another service being watched"))
+		c.mu.Unlock()
+		return func() {}
+	}
+	c.mu.Unlock()
+
+	w := &serviceUpdateWatcher{c: c, serviceCb: cb}
+	w.ldsCancel = c.watchLDS(serviceName, w.handleLDSResp)
+
+	return w.close
+}
+
+// serviceUpdateWatcher handles LDS and RDS response, and calls the service
+// callback at the right time.
+type serviceUpdateWatcher struct {
+	c         *Client
+	ldsCancel func()
+	serviceCb func(ServiceUpdate, error)
+
+	mu        sync.Mutex
+	closed    bool
+	rdsName   string
+	rdsCancel func()
+}
+
+func (w *serviceUpdateWatcher) handleLDSResp(update ListenerUpdate, err error) {
+	w.c.logger.Infof("xds: client received LDS update: %+v, err: %v", update, err)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return
+	}
+	if err != nil {
+		// We check the error type and do different things. For now, the only
+		// type we check is ResourceNotFound, which indicates the LDS resource
+		// was removed, and besides sending the error to callback, we also
+		// cancel the RDS watch.
+		if ErrType(err) == ErrorTypeResourceNotFound && w.rdsCancel != nil {
+			w.rdsCancel()
+			w.rdsName = ""
+			w.rdsCancel = nil
+		}
+		// The other error cases still return early without canceling the
+		// existing RDS watch.
+		w.serviceCb(ServiceUpdate{}, err)
+		return
+	}
+
+	if w.rdsName == update.RouteConfigName {
+		// If the new RouteConfigName is same as the previous, don't cancel and
+		// restart the RDS watch.
+		return
+	}
+	w.rdsName = update.RouteConfigName
+	if w.rdsCancel != nil {
+		w.rdsCancel()
+	}
+	w.rdsCancel = w.c.watchRDS(update.RouteConfigName, w.handleRDSResp)
+}
+
+func (w *serviceUpdateWatcher) handleRDSResp(update RouteConfigUpdate, err error) {
+	w.c.logger.Infof("xds: client received RDS update: %+v, err: %v", update, err)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return
+	}
+	if w.rdsCancel == nil {
+		// This mean only the RDS watch is canceled, can happen if the LDS
+		// resource is removed.
+		return
+	}
+	if err != nil {
+		w.serviceCb(ServiceUpdate{}, err)
+		return
+	}
+	w.serviceCb(ServiceUpdate(update), nil)
+}
+
+func (w *serviceUpdateWatcher) close() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.closed = true
+	w.ldsCancel()
+	if w.rdsCancel != nil {
+		w.rdsCancel()
+		w.rdsCancel = nil
+	}
+}
+
+// WatchCluster uses CDS to discover information about the provided
+// clusterName.
+//
+// WatchCluster can be called multiple times, with same or different
+// clusterNames. Each call will start an independent watcher for the resource.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
+	wi := &watchInfo{
+		c:           c,
+		typeURL:     version.V2ClusterURL,
+		target:      clusterName,
+		cdsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		wi.timeout()
+	})
+	return c.watch(wi)
+}
+
+// WatchEndpoints uses EDS to discover endpoints in the provided clusterName.
+//
+// WatchEndpoints can be called multiple times, with same or different
+// clusterNames. Each call will start an independent watcher for the resource.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
+	wi := &watchInfo{
+		c:           c,
+		typeURL:     version.V2EndpointsURL,
+		target:      clusterName,
+		edsCallback: cb,
+	}
+
+	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+		wi.timeout()
+	})
+	return c.watch(wi)
+}
diff --git a/xds/internal/client/client_watchers_cluster.go b/xds/internal/client/client_watchers_cluster.go
deleted file mode 100644
index 2d90711308c0..000000000000
--- a/xds/internal/client/client_watchers_cluster.go
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"time"
-)
-
-// ClusterUpdate contains information from a received CDS response, which is of
-// interest to the registered CDS watcher.
-type ClusterUpdate struct {
-	// ServiceName is the service name corresponding to the clusterName which
-	// is being watched for through CDS.
-	ServiceName string
-	// EnableLRS indicates whether or not load should be reported through LRS.
-	EnableLRS bool
-}
-
-// WatchCluster uses CDS to discover information about the provided
-// clusterName.
-//
-// WatchCluster can be called multiple times, with same or different
-// clusterNames. Each call will start an independent watcher for the resource.
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
-	wi := &watchInfo{
-		c:           c,
-		typeURL:     cdsURL,
-		target:      clusterName,
-		cdsCallback: cb,
-	}
-
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		wi.timeout()
-	})
-	return c.watch(wi)
-}
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index 6d30f7dd5f7f..e6530fbf6823 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 type clusterUpdateErr struct {
@@ -35,7 +36,7 @@ type clusterUpdateErr struct {
 // - an update for another resource name
 // - an update is received after cancel()
 func (s) TestClusterWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -53,7 +54,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -64,7 +65,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	//
 	// TODO: in a future cleanup, this (and the same thing in other tests) can
 	// be changed call Client directly.
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -73,7 +74,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	}
 
 	// Another update, with an extra resource for a different resource name.
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName:  wantUpdate,
 		"randomName": {},
 	})
@@ -84,7 +85,7 @@ func (s) TestClusterWatch(t *testing.T) {
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -96,7 +97,7 @@ func (s) TestClusterWatch(t *testing.T) {
 // TestClusterTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -118,13 +119,13 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[cdsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -136,7 +137,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -154,7 +155,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 // TestClusterThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -175,7 +176,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 		c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[cdsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -185,13 +186,13 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
 	})
@@ -210,7 +211,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 // TestClusterWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestClusterWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -225,12 +226,12 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -243,7 +244,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[cdsURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
@@ -268,7 +269,7 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
 	}()
 
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -283,7 +284,7 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -310,7 +311,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
 	}()
 
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -325,12 +326,12 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -352,7 +353,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 // - one more update without the removed resource
 //   - the callback (above) shouldn't receive any update
 func (s) TestClusterResourceRemoved(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -367,7 +368,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 		clusterUpdateCh1.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
@@ -375,13 +376,13 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[cdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
 	})
@@ -395,7 +396,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	}
 
 	// Send another update to remove resource 1.
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "2": wantUpdate2,
 	})
 
@@ -410,7 +411,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	}
 
 	// Send one more update without resource 1.
-	v2Client.r.newCDSUpdate(map[string]ClusterUpdate{
+	v2Client.r.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "2": wantUpdate2,
 	})
 
diff --git a/xds/internal/client/client_watchers_endpoints.go b/xds/internal/client/client_watchers_endpoints.go
deleted file mode 100644
index 439f18248a91..000000000000
--- a/xds/internal/client/client_watchers_endpoints.go
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"time"
-
-	"google.golang.org/grpc/xds/internal"
-)
-
-// OverloadDropConfig contains the config to drop overloads.
-type OverloadDropConfig struct {
-	Category    string
-	Numerator   uint32
-	Denominator uint32
-}
-
-// EndpointHealthStatus represents the health status of an endpoint.
-type EndpointHealthStatus int32
-
-const (
-	// EndpointHealthStatusUnknown represents HealthStatus UNKNOWN.
-	EndpointHealthStatusUnknown EndpointHealthStatus = iota
-	// EndpointHealthStatusHealthy represents HealthStatus HEALTHY.
-	EndpointHealthStatusHealthy
-	// EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY.
-	EndpointHealthStatusUnhealthy
-	// EndpointHealthStatusDraining represents HealthStatus DRAINING.
-	EndpointHealthStatusDraining
-	// EndpointHealthStatusTimeout represents HealthStatus TIMEOUT.
-	EndpointHealthStatusTimeout
-	// EndpointHealthStatusDegraded represents HealthStatus DEGRADED.
-	EndpointHealthStatusDegraded
-)
-
-// Endpoint contains information of an endpoint.
-type Endpoint struct {
-	Address      string
-	HealthStatus EndpointHealthStatus
-	Weight       uint32
-}
-
-// Locality contains information of a locality.
-type Locality struct {
-	Endpoints []Endpoint
-	ID        internal.LocalityID
-	Priority  uint32
-	Weight    uint32
-}
-
-// EndpointsUpdate contains an EDS update.
-type EndpointsUpdate struct {
-	Drops      []OverloadDropConfig
-	Localities []Locality
-}
-
-// WatchEndpoints uses EDS to discover endpoints in the provided clusterName.
-//
-// WatchEndpoints can be called multiple times, with same or different
-// clusterNames. Each call will start an independent watcher for the resource.
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
-	wi := &watchInfo{
-		c:           c,
-		typeURL:     edsURL,
-		target:      clusterName,
-		edsCallback: cb,
-	}
-
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		wi.timeout()
-	})
-	return c.watch(wi)
-}
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index 7ac8d3e6cc64..27cf2e62d02e 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -24,9 +24,9 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 var (
@@ -57,7 +57,7 @@ type endpointsUpdateErr struct {
 // - an update for another resource name (which doesn't trigger callback)
 // - an update is received after cancel()
 func (s) TestEndpointsWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -72,12 +72,12 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -86,7 +86,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	}
 
 	// Another update for a different resource name.
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		"randomName": {},
 	})
 
@@ -96,7 +96,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -108,7 +108,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 // TestEndpointsTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -130,13 +130,13 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[edsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -148,7 +148,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -166,7 +166,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 // TestEndpointsThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -187,7 +187,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 		c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[edsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -197,13 +197,13 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
 	wantUpdate2 := EndpointsUpdate{Localities: []Locality{testLocalities[1]}}
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
 	})
@@ -222,7 +222,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 // TestEndpointsWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestEndpointsWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -237,12 +237,12 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	v2Client.r.newEDSUpdate(map[string]EndpointsUpdate{
+	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName: wantUpdate,
 	})
 
@@ -255,7 +255,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[edsURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
@@ -280,7 +280,7 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
 	}()
 
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -295,7 +295,7 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[edsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
diff --git a/xds/internal/client/client_watchers_lds.go b/xds/internal/client/client_watchers_lds.go
deleted file mode 100644
index 0f7860db991e..000000000000
--- a/xds/internal/client/client_watchers_lds.go
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"time"
-)
-
-type ldsUpdate struct {
-	routeName string
-}
-type ldsCallbackFunc func(ldsUpdate, error)
-
-// watchLDS starts a listener watcher for the service..
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) watchLDS(serviceName string, cb ldsCallbackFunc) (cancel func()) {
-	wi := &watchInfo{
-		c:           c,
-		typeURL:     ldsURL,
-		target:      serviceName,
-		ldsCallback: cb,
-	}
-
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		wi.timeout()
-	})
-	return c.watch(wi)
-}
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index fa8bfdb6ca0f..ce1e5ed3b086 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -22,10 +22,11 @@ import (
 	"testing"
 
 	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 type ldsUpdateErr struct {
-	u   ldsUpdate
+	u   ListenerUpdate
 	err error
 }
 
@@ -34,7 +35,7 @@ type ldsUpdateErr struct {
 // - an update for another resource name
 // - an update is received after cancel()
 func (s) TestLDSWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -46,47 +47,47 @@ func (s) TestLDSWatch(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	ldsUpdateCh := testutils.NewChannel()
-	cancelWatch := c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+	cancelWatch := c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ldsUpdate{routeName: testRDSName}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: wantUpdate,
 	})
 
 	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Another update, with an extra resource for a different resource name.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName:  wantUpdate,
 		"randomName": {},
 	})
 
 	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: wantUpdate,
 	})
 
 	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
 // TestLDSTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -105,46 +106,46 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
-		cancelLastWatch = c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+		cancelLastWatch = c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[ldsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
 
-	wantUpdate := ldsUpdate{routeName: testRDSName}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: wantUpdate,
 	})
 
 	for i := 0; i < count; i++ {
 		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-			t.Errorf("i=%v, unexpected ldsUpdate: %v, error receiving from channel: %v", i, u, err)
+			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: wantUpdate,
 	})
 
 	for i := 0; i < count-1; i++ {
 		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-			t.Errorf("i=%v, unexpected ldsUpdate: %v, error receiving from channel: %v", i, u, err)
+			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
 	if u, err := ldsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
 // TestLDSThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -162,45 +163,45 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
-		c.watchLDS(testLDSName+"1", func(update ldsUpdate, err error) {
+		c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[ldsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
 
 	// Third watch for a different name.
 	ldsUpdateCh2 := testutils.NewChannel()
-	c.watchLDS(testLDSName+"2", func(update ldsUpdate, err error) {
+	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ldsUpdate{routeName: testRDSName + "1"}
-	wantUpdate2 := ldsUpdate{routeName: testRDSName + "2"}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	wantUpdate1 := ListenerUpdate{RouteConfigName: testRDSName + "1"}
+	wantUpdate2 := ListenerUpdate{RouteConfigName: testRDSName + "2"}
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "1": wantUpdate1,
 		testLDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
 		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
-			t.Errorf("i=%v, unexpected ldsUpdate: %v, error receiving from channel: %v", i, u, err)
+			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
 	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
 
 // TestLDSWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestLDSWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -212,39 +213,39 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	ldsUpdateCh := testutils.NewChannel()
-	c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ldsUpdate{routeName: testRDSName}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: wantUpdate,
 	})
 
 	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Another watch for the resource in cache.
 	ldsUpdateCh2 := testutils.NewChannel()
-	c.watchLDS(testLDSName, func(update ldsUpdate, err error) {
+	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[ldsURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
 	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
 	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected ldsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
@@ -255,7 +256,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 // - one more update without the removed resource
 //   - the callback (above) shouldn't receive any update
 func (s) TestLDSResourceRemoved(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -267,63 +268,63 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	ldsUpdateCh1 := testutils.NewChannel()
-	c.watchLDS(testLDSName+"1", func(update ldsUpdate, err error) {
+	c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 		ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
 	ldsUpdateCh2 := testutils.NewChannel()
-	c.watchLDS(testLDSName+"2", func(update ldsUpdate, err error) {
+	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ldsUpdate{routeName: testEDSName + "1"}
-	wantUpdate2 := ldsUpdate{routeName: testEDSName + "2"}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	wantUpdate1 := ListenerUpdate{RouteConfigName: testEDSName + "1"}
+	wantUpdate2 := ListenerUpdate{RouteConfigName: testEDSName + "2"}
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "1": wantUpdate1,
 		testLDSName + "2": wantUpdate2,
 	})
 
 	if u, err := ldsUpdateCh1.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Send another update to remove resource 1.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "2": wantUpdate2,
 	})
 
 	// watcher 1 should get an error.
 	if u, err := ldsUpdateCh1.Receive(); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
 	}
 
 	// watcher 2 should get the same update again.
 	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Send one more update without resource 1.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "2": wantUpdate2,
 	})
 
 	// watcher 1 should get an error.
 	if u, err := ldsUpdateCh1.Receive(); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected ldsUpdate: %v, want receiving from channel timeout", u)
+		t.Errorf("unexpected ListenerUpdate: %v, want receiving from channel timeout", u)
 	}
 
 	// watcher 2 should get the same update again.
 	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ldsUpdate: %v, error receiving from channel: %v", u, err)
+		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_rds.go b/xds/internal/client/client_watchers_rds.go
deleted file mode 100644
index cc1b18c2d915..000000000000
--- a/xds/internal/client/client_watchers_rds.go
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"time"
-)
-
-// Int64Range is a range for header range match.
-type Int64Range struct {
-	Start int64 `json:"start"`
-	End   int64 `json:"end"`
-}
-
-// HeaderMatcher represents header matchers.
-type HeaderMatcher struct {
-	Name         string      `json:"name"`
-	InvertMatch  *bool       `json:"invertMatch,omitempty"`
-	ExactMatch   *string     `json:"exactMatch,omitempty"`
-	RegexMatch   *string     `json:"regexMatch,omitempty"`
-	PrefixMatch  *string     `json:"prefixMatch,omitempty"`
-	SuffixMatch  *string     `json:"suffixMatch,omitempty"`
-	RangeMatch   *Int64Range `json:"rangeMatch,omitempty"`
-	PresentMatch *bool       `json:"presentMatch,omitempty"`
-}
-
-// Route represents route with matchers and action.
-type Route struct {
-	Path, Prefix, Regex *string
-	Headers             []*HeaderMatcher
-	Fraction            *uint32
-	Action              map[string]uint32 // action is weighted clusters.
-}
-
-type rdsUpdate struct {
-	routes []*Route
-}
-type rdsCallbackFunc func(rdsUpdate, error)
-
-// watchRDS starts a listener watcher for the service..
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) watchRDS(routeName string, cb rdsCallbackFunc) (cancel func()) {
-	wi := &watchInfo{
-		c:           c,
-		typeURL:     rdsURL,
-		target:      routeName,
-		rdsCallback: cb,
-	}
-
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
-		wi.timeout()
-	})
-	return c.watch(wi)
-}
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 8476b275b496..009d70380610 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -23,10 +23,11 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 type rdsUpdateErr struct {
-	u   rdsUpdate
+	u   RouteConfigUpdate
 	err error
 }
 
@@ -35,7 +36,7 @@ type rdsUpdateErr struct {
 // - an update for another resource name (which doesn't trigger callback)
 // - an update is received after cancel()
 func (s) TestRDSWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -47,46 +48,46 @@ func (s) TestRDSWatch(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	rdsUpdateCh := testutils.NewChannel()
-	cancelWatch := c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+	cancelWatch := c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Another update for a different resource name.
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		"randomName": {},
 	})
 
 	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: wantUpdate,
 	})
 
 	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
 // TestRDSTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -105,46 +106,46 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
-		cancelLastWatch = c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+		cancelLastWatch = c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[rdsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
 
-	wantUpdate := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: wantUpdate,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: wantUpdate,
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
 	if u, err := rdsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
 // TestRDSThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -162,45 +163,45 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
-		c.watchRDS(testRDSName+"1", func(update rdsUpdate, err error) {
+		c.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[rdsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
 
 	// Third watch for a different name.
 	rdsUpdateCh2 := testutils.NewChannel()
-	c.watchRDS(testRDSName+"2", func(update rdsUpdate, err error) {
+	c.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}}}
-	wantUpdate2 := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	wantUpdate1 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}}}
+	wantUpdate2 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName + "1": wantUpdate1,
 		testRDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-			t.Errorf("i=%v, unexpected rdsUpdate: %v, error receiving from channel: %v", i, u, err)
+		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
 
 // TestRDSWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestRDSWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -212,38 +213,38 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	rdsUpdateCh := testutils.NewChannel()
-	c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Another watch for the resource in cache.
 	rdsUpdateCh2 := testutils.NewChannel()
-	c.watchRDS(testRDSName, func(update rdsUpdate, err error) {
+	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[rdsURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdate{}, rdsUpdateErr{})) {
-		t.Errorf("unexpected rdsUpdate: %v, error receiving from channel: %v", u, err)
+	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
 	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
-		t.Errorf("unexpected rdsUpdate: %v, %v, want channel recv timeout", u, err)
+		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_service.go b/xds/internal/client/client_watchers_service.go
deleted file mode 100644
index cc96622d71d3..000000000000
--- a/xds/internal/client/client_watchers_service.go
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"fmt"
-	"sync"
-)
-
-// ServiceUpdate contains update about the service.
-type ServiceUpdate struct {
-	// Routes contain matchers+actions to route RPCs.
-	Routes []*Route
-}
-
-// WatchService uses LDS and RDS to discover information about the provided
-// serviceName.
-//
-// WatchService can only be called once. The second call will not start a
-// watcher and the callback will get an error. It's this case because an xDS
-// client is expected to be used only by one ClientConn.
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error)) (cancel func()) {
-	c.mu.Lock()
-	if len(c.ldsWatchers) != 0 {
-		go cb(ServiceUpdate{}, fmt.Errorf("unexpected WatchService when there's another service being watched"))
-		c.mu.Unlock()
-		return func() {}
-	}
-	c.mu.Unlock()
-
-	w := &serviceUpdateWatcher{c: c, serviceCb: cb}
-	w.ldsCancel = c.watchLDS(serviceName, w.handleLDSResp)
-
-	return w.close
-}
-
-// serviceUpdateWatcher handles LDS and RDS response, and calls the service
-// callback at the right time.
-type serviceUpdateWatcher struct {
-	c         *Client
-	ldsCancel func()
-	serviceCb func(ServiceUpdate, error)
-
-	mu        sync.Mutex
-	closed    bool
-	rdsName   string
-	rdsCancel func()
-}
-
-func (w *serviceUpdateWatcher) handleLDSResp(update ldsUpdate, err error) {
-	w.c.logger.Infof("xds: client received LDS update: %+v, err: %v", update, err)
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	if w.closed {
-		return
-	}
-	if err != nil {
-		// We check the error type and do different things. For now, the only
-		// type we check is ResourceNotFound, which indicates the LDS resource
-		// was removed, and besides sending the error to callback, we also
-		// cancel the RDS watch.
-		if ErrType(err) == ErrorTypeResourceNotFound && w.rdsCancel != nil {
-			w.rdsCancel()
-			w.rdsName = ""
-			w.rdsCancel = nil
-		}
-		// The other error cases still return early without canceling the
-		// existing RDS watch.
-		w.serviceCb(ServiceUpdate{}, err)
-		return
-	}
-
-	if w.rdsName == update.routeName {
-		// If the new routeName is same as the previous, don't cancel and
-		// restart the RDS watch.
-		return
-	}
-	w.rdsName = update.routeName
-	if w.rdsCancel != nil {
-		w.rdsCancel()
-	}
-	w.rdsCancel = w.c.watchRDS(update.routeName, w.handleRDSResp)
-}
-
-func (w *serviceUpdateWatcher) handleRDSResp(update rdsUpdate, err error) {
-	w.c.logger.Infof("xds: client received RDS update: %+v, err: %v", update, err)
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	if w.closed {
-		return
-	}
-	if w.rdsCancel == nil {
-		// This mean only the RDS watch is canceled, can happen if the LDS
-		// resource is removed.
-		return
-	}
-	if err != nil {
-		w.serviceCb(ServiceUpdate{}, err)
-		return
-	}
-	w.serviceCb(ServiceUpdate{
-		Routes: update.routes,
-	}, nil)
-}
-
-func (w *serviceUpdateWatcher) close() {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	w.closed = true
-	w.ldsCancel()
-	if w.rdsCancel != nil {
-		w.rdsCancel()
-		w.rdsCancel = nil
-	}
-}
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 4b72f63e8ee6..08476e2f7f0a 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -19,16 +19,13 @@
 package client
 
 import (
-	"errors"
-	"fmt"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 type serviceUpdateErr struct {
@@ -42,7 +39,7 @@ var serviceCmpOpts = []cmp.Option{cmp.AllowUnexported(serviceUpdateErr{}), cmpop
 // - an update is received after a watch()
 // - an update with routes received
 func (s) TestServiceWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -60,17 +57,17 @@ func (s) TestServiceWatch(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -83,9 +80,9 @@ func (s) TestServiceWatch(t *testing.T) {
 			Action: map[string]uint32{testCDSName: 1},
 		}},
 	}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {
-			routes: []*Route{{
+			Routes: []*Route{{
 				Prefix: newStringP(""),
 				Action: map[string]uint32{testCDSName: 1},
 			}},
@@ -100,7 +97,7 @@ func (s) TestServiceWatch(t *testing.T) {
 // response, the second LDS response trigger an new RDS watch, and an update of
 // the old RDS watch doesn't trigger update to service callback.
 func (s) TestServiceWatchLDSUpdate(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -118,17 +115,17 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -136,16 +133,16 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	}
 
 	// Another LDS update with a different RDS_name.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName + "2"},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName + "2"},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	// Another update for the old name.
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
@@ -154,8 +151,8 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
 	// RDS update for the new name.
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName + "2": {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName + "2": {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
@@ -167,7 +164,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 // error (because only one is allowed). But the first watch still receives
 // updates.
 func (s) TestServiceWatchSecond(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -185,17 +182,17 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -222,11 +219,11 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 
 	// Send update again, first callback should be called, second should
 	// timeout.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -242,135 +239,130 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 // does not respond to the requests being sent out as part of registering a
 // service update watcher. The callback will get an error.
 func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
-	defer cleanup()
-
-	xdsClient, err := New(clientOpts(fakeServer.Address))
-	if err != nil {
-		t.Fatalf("New returned error: %v", err)
-	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
-
 	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
 	defaultWatchExpiryTimeout = 500 * time.Millisecond
 	defer func() {
 		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
 	}()
 
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.Routes != nil {
-			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.Routes))
-			return
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("xdsClient.WatchService returned error non-nil error"))
-			return
-		}
-		callbackCh.Send(nil)
-	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
-
-	// Wait for one request from the client, but send no reponses.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
-	}
-	waitForNilErr(t, callbackCh)
-}
+	v2ClientCh, cleanup := overrideNewAPIClient()
+	defer cleanup()
 
-// TestServiceWatchEmptyRDS tests the case where the underlying v2Client
-// receives an empty RDS response. The callback will get an error.
-func (s) TestServiceWatchEmptyRDS(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
+	c, err := New(clientOpts(testXDSServer))
 	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
+		t.Fatalf("failed to create client: %v", err)
 	}
-	defer cleanup()
+	defer c.Close()
 
-	xdsClient, err := New(clientOpts(fakeServer.Address))
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	u, err := serviceUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
 	if err != nil {
-		t.Fatalf("New returned error: %v", err)
+		t.Fatalf("failed to get serviceUpdate: %v", err)
+	}
+	uu := u.(serviceUpdateErr)
+	if !cmp.Equal(uu.u, ServiceUpdate{}) {
+		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
+	}
+	if uu.err == nil {
+		t.Errorf("unexpected serviceError: <nil>, want error watcher timeout")
 	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
+}
 
+// TestServiceWatchEmptyRDS tests the case where the underlying v2Client
+// receives an empty RDS response. The callback will get an error.
+func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
 	defaultWatchExpiryTimeout = 500 * time.Millisecond
 	defer func() {
 		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
 	}()
 
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		if su.Routes != nil {
-			callbackCh.Send(fmt.Errorf("got WeightedCluster: %+v, want nil", su.Routes))
-			return
-		}
-		if err == nil {
-			callbackCh.Send(errors.New("xdsClient.WatchService returned error non-nil error"))
-			return
-		}
-		callbackCh.Send(nil)
-	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
+	v2ClientCh, cleanup := overrideNewAPIClient()
+	defer cleanup()
 
-	// Make the fakeServer send LDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
+	c, err := New(clientOpts(testXDSServer))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
 	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
+
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	})
 
-	// Make the fakeServer send an empty RDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an RDS request")
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
+	})
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{})
+	u, err := serviceUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	if err != nil {
+		t.Fatalf("failed to get serviceUpdate: %v", err)
+	}
+	uu := u.(serviceUpdateErr)
+	if !cmp.Equal(uu.u, ServiceUpdate{}) {
+		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
+	}
+	if uu.err == nil {
+		t.Errorf("unexpected serviceError: <nil>, want error watcher timeout")
 	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: noVirtualHostsInRDSResponse}
-	waitForNilErr(t, callbackCh)
 }
 
 // TestServiceWatchWithClientClose tests the case where xDS responses are
 // received after the client is closed, and we make sure that the registered
 // watcher callback is not invoked.
 func (s) TestServiceWatchWithClientClose(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
+	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
+	defaultWatchExpiryTimeout = 500 * time.Millisecond
+	defer func() {
+		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
+	}()
+
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	xdsClient, err := New(clientOpts(fakeServer.Address))
+	c, err := New(clientOpts(testXDSServer))
 	if err != nil {
-		t.Fatalf("New returned error: %v", err)
+		t.Fatalf("failed to create client: %v", err)
 	}
-	defer xdsClient.Close()
-	t.Log("Created an xdsClient...")
+	defer c.Close()
+
+	v2Client := <-v2ClientCh
 
-	callbackCh := testutils.NewChannel()
-	cancelWatch := xdsClient.WatchService(goodLDSTarget1, func(su ServiceUpdate, err error) {
-		callbackCh.Send(errors.New("watcher callback invoked after client close"))
+	serviceUpdateCh := testutils.NewChannel()
+	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-	defer cancelWatch()
-	t.Log("Registered a watcher for service updates...")
 
-	// Make the fakeServer send LDS response.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout expired when expecting an LDS request")
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
-
-	xdsClient.Close()
-	t.Log("Closing the xdsClient...")
-
-	// Push an RDS response from the fakeserver
-	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodRDSResponse1}
-	if cbErr, err := callbackCh.Receive(); err != testutils.ErrRecvTimeout {
-		t.Fatal(cbErr)
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
+	})
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+		t.Fatalf("want new watch to start, got error %v", err)
+	}
+	// Client is closed before it receives the RDS response.
+	c.Close()
+	if u, err := serviceUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2); err != testutils.ErrRecvTimeout {
+		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
@@ -378,7 +370,7 @@ func (s) TestServiceWatchWithClientClose(t *testing.T) {
 // update contains the same RDS name as the previous, the RDS watch isn't
 // canceled and restarted.
 func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -396,17 +388,17 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -414,10 +406,10 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	}
 
 	// Another LDS update with a the same RDS_name.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if v, err := v2Client.removeWatches[rdsURL].Receive(); err == nil {
+	if v, err := v2Client.removeWatches[version.V2RouteConfigURL].Receive(); err == nil {
 		t.Fatalf("unexpected rds watch cancel: %v", v)
 	}
 }
@@ -429,7 +421,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 // - one more update without the removed resource
 //   - the callback (above) shouldn't receive any update
 func (s) TestServiceResourceRemoved(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewXDSV2Client()
+	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
 	c, err := New(clientOpts(testXDSServer))
@@ -447,17 +439,17 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ldsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
@@ -466,8 +458,8 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 
 	// Remove LDS resource, should cancel the RDS watch, and trigger resource
 	// removed error.
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{})
-	if _, err := v2Client.removeWatches[rdsURL].Receive(); err != nil {
+	v2Client.r.NewListeners(map[string]ListenerUpdate{})
+	if _, err := v2Client.removeWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want watch to be canceled, got error %v", err)
 	}
 	if u, err := serviceUpdateCh.Receive(); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
@@ -476,8 +468,8 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 
 	// Send RDS update for the removed LDS resource, expect no updates to
 	// callback, because RDS should be canceled.
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}},
 	})
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
@@ -486,18 +478,18 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	// Add LDS resource, but not RDS resource, should
 	//  - start a new RDS watch
 	//  - timeout on service channel, because RDS cache was cleared
-	v2Client.r.newLDSUpdate(map[string]ldsUpdate{
-		testLDSName: {routeName: testRDSName},
+	v2Client.r.NewListeners(map[string]ListenerUpdate{
+		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[rdsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
 	}
 
-	v2Client.r.newRDSUpdate(map[string]rdsUpdate{
-		testRDSName: {routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}},
+	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}},
 	})
 	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
new file mode 100644
index 000000000000..4a1a16df3add
--- /dev/null
+++ b/xds/internal/client/client_xds.go
@@ -0,0 +1,507 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+// UnmarshalListener processes resources received in an LDS response, validates
+// them, and transforms them into a native struct which contains only fields we
+// are interested in.
+func UnmarshalListener(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, error) {
+	update := make(map[string]ListenerUpdate)
+	for _, r := range resources {
+		if t := r.GetTypeUrl(); t != version.V2ListenerURL && t != version.V3ListenerURL {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in LDS response", t)
+		}
+		lis := &v3listenerpb.Listener{}
+		if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
+			return nil, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
+		}
+		logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
+		routeName, err := getRouteConfigNameFromListener(lis, logger)
+		if err != nil {
+			return nil, err
+		}
+		update[lis.GetName()] = ListenerUpdate{RouteConfigName: routeName}
+	}
+	return update, nil
+}
+
+// getRouteConfigNameFromListener checks if the provided Listener proto meets
+// the expected criteria. If so, it returns a non-empty routeConfigName.
+func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger) (string, error) {
+	if lis.GetApiListener() == nil {
+		return "", fmt.Errorf("xds: no api_listener field in LDS response %+v", lis)
+	}
+	apiLisAny := lis.GetApiListener().GetApiListener()
+	if t := apiLisAny.GetTypeUrl(); t != version.V3HTTPConnManagerURL && t != version.V2HTTPConnManagerURL {
+		return "", fmt.Errorf("xds: unexpected resource type: %s in LDS response", t)
+	}
+	apiLis := &v3httppb.HttpConnectionManager{}
+	if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
+		return "", fmt.Errorf("xds: failed to unmarshal api_listner in LDS response: %v", err)
+	}
+
+	logger.Infof("Resource with type %T, contains %v", apiLis, apiLis)
+	switch apiLis.RouteSpecifier.(type) {
+	case *v3httppb.HttpConnectionManager_Rds:
+		if apiLis.GetRds().GetConfigSource().GetAds() == nil {
+			return "", fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis)
+		}
+		name := apiLis.GetRds().GetRouteConfigName()
+		if name == "" {
+			return "", fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
+		}
+		return name, nil
+	case *v3httppb.HttpConnectionManager_RouteConfig:
+		// TODO: Add support for specifying the RouteConfiguration inline
+		// in the LDS response.
+		return "", fmt.Errorf("xds: LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
+	case nil:
+		return "", fmt.Errorf("xds: no RouteSpecifier in received LDS response: %+v", apiLis)
+	default:
+		return "", fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
+	}
+}
+
+// UnmarshalRouteConfig processes resources received in an RDS response,
+// validates them, and transforms them into a native struct which contains only
+// fields we are interested in. The provided hostname determines the route
+// configuration resources of interest.
+func UnmarshalRouteConfig(resources []*anypb.Any, hostname string, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, error) {
+	update := make(map[string]RouteConfigUpdate)
+	for _, r := range resources {
+		if t := r.GetTypeUrl(); t != version.V2RouteConfigURL && t != version.V3RouteConfigURL {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in RDS response", t)
+		}
+		rc := &v3routepb.RouteConfiguration{}
+		if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
+			return nil, fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err)
+		}
+		logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, hostname)
+
+		// Use the hostname (resourceName for LDS) to find the routes.
+		u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname, logger)
+		if err != nil {
+			return nil, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
+		}
+		update[rc.GetName()] = u
+	}
+	return update, nil
+}
+
+// generateRDSUpdateFromRouteConfiguration checks if the provided
+// RouteConfiguration meets the expected criteria. If so, it returns a
+// RouteConfigUpdate with nil error.
+//
+// A RouteConfiguration resource is considered valid when only if it contains a
+// VirtualHost whose domain field matches the server name from the URI passed
+// to the gRPC channel, and it contains a clusterName or a weighted cluster.
+//
+// The RouteConfiguration includes a list of VirtualHosts, which may have zero
+// or more elements. We are interested in the element whose domains field
+// matches the server name specified in the "xds:" URI. The only field in the
+// VirtualHost proto that the we are interested in is the list of routes. We
+// only look at the last route in the list (the default route), whose match
+// field must be empty and whose route field must be set.  Inside that route
+// message, the cluster field will contain the clusterName or weighted clusters
+// we are looking for.
+func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, host string, logger *grpclog.PrefixLogger) (RouteConfigUpdate, error) {
+	//
+	// Currently this returns "" on error, and the caller will return an error.
+	// But the error doesn't contain details of why the response is invalid
+	// (mismatch domain or empty route).
+	//
+	// For logging purposes, we can log in line. But if we want to populate
+	// error details for nack, a detailed error needs to be returned.
+	vh := findBestMatchingVirtualHost(host, rc.GetVirtualHosts())
+	if vh == nil {
+		// No matching virtual host found.
+		return RouteConfigUpdate{}, fmt.Errorf("no matching virtual host found")
+	}
+	if len(vh.Routes) == 0 {
+		// The matched virtual host has no routes, this is invalid because there
+		// should be at least one default route.
+		return RouteConfigUpdate{}, fmt.Errorf("matched virtual host has no routes")
+	}
+
+	routes, err := routesProtoToSlice(vh.Routes, logger)
+	if err != nil {
+		return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
+	}
+	return RouteConfigUpdate{Routes: routes}, nil
+}
+
+func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) {
+	var routesRet []*Route
+
+	for _, r := range routes {
+		match := r.GetMatch()
+		if match == nil {
+			return nil, fmt.Errorf("route %+v doesn't have a match", r)
+		}
+
+		if len(match.GetQueryParameters()) != 0 {
+			// Ignore route with query parameters.
+			logger.Warningf("route %+v has query parameter matchers, the route will be ignored", r)
+			continue
+		}
+
+		if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
+			return nil, fmt.Errorf("route %+v has case-sensitive false", r)
+		}
+
+		pathSp := match.GetPathSpecifier()
+		if pathSp == nil {
+			return nil, fmt.Errorf("route %+v doesn't have a path specifier", r)
+		}
+
+		var route Route
+		switch pt := pathSp.(type) {
+		case *v3routepb.RouteMatch_Prefix:
+			route.Prefix = &pt.Prefix
+		case *v3routepb.RouteMatch_Path:
+			route.Path = &pt.Path
+		case *v3routepb.RouteMatch_SafeRegex:
+			route.Regex = &pt.SafeRegex.Regex
+		default:
+			logger.Warningf("route %+v has an unrecognized path specifier: %+v", r, pt)
+			continue
+		}
+
+		for _, h := range match.GetHeaders() {
+			var header HeaderMatcher
+			switch ht := h.GetHeaderMatchSpecifier().(type) {
+			case *v3routepb.HeaderMatcher_ExactMatch:
+				header.ExactMatch = &ht.ExactMatch
+			case *v3routepb.HeaderMatcher_SafeRegexMatch:
+				header.RegexMatch = &ht.SafeRegexMatch.Regex
+			case *v3routepb.HeaderMatcher_RangeMatch:
+				header.RangeMatch = &Int64Range{
+					Start: ht.RangeMatch.Start,
+					End:   ht.RangeMatch.End,
+				}
+			case *v3routepb.HeaderMatcher_PresentMatch:
+				header.PresentMatch = &ht.PresentMatch
+			case *v3routepb.HeaderMatcher_PrefixMatch:
+				header.PrefixMatch = &ht.PrefixMatch
+			case *v3routepb.HeaderMatcher_SuffixMatch:
+				header.SuffixMatch = &ht.SuffixMatch
+			default:
+				logger.Warningf("route %+v has an unrecognized header matcher: %+v", r, ht)
+				continue
+			}
+			header.Name = h.GetName()
+			invert := h.GetInvertMatch()
+			header.InvertMatch = &invert
+			route.Headers = append(route.Headers, &header)
+		}
+
+		if fr := match.GetRuntimeFraction(); fr != nil {
+			d := fr.GetDefaultValue()
+			n := d.GetNumerator()
+			switch d.GetDenominator() {
+			case v3typepb.FractionalPercent_HUNDRED:
+				n *= 10000
+			case v3typepb.FractionalPercent_TEN_THOUSAND:
+				n *= 100
+			case v3typepb.FractionalPercent_MILLION:
+			}
+			route.Fraction = &n
+		}
+
+		clusters := make(map[string]uint32)
+		switch a := r.GetRoute().GetClusterSpecifier().(type) {
+		case *v3routepb.RouteAction_Cluster:
+			clusters[a.Cluster] = 1
+		case *v3routepb.RouteAction_WeightedClusters:
+			wcs := a.WeightedClusters
+			var totalWeight uint32
+			for _, c := range wcs.Clusters {
+				w := c.GetWeight().GetValue()
+				clusters[c.GetName()] = w
+				totalWeight += w
+			}
+			if totalWeight != wcs.GetTotalWeight().GetValue() {
+				return nil, fmt.Errorf("route %+v, action %+v, weights of clusters do not add up to total total weight, got: %v, want %v", r, a, wcs.GetTotalWeight().GetValue(), totalWeight)
+			}
+		case *v3routepb.RouteAction_ClusterHeader:
+			continue
+		}
+
+		route.Action = clusters
+		routesRet = append(routesRet, &route)
+	}
+	return routesRet, nil
+}
+
+type domainMatchType int
+
+const (
+	domainMatchTypeInvalid domainMatchType = iota
+	domainMatchTypeUniversal
+	domainMatchTypePrefix
+	domainMatchTypeSuffix
+	domainMatchTypeExact
+)
+
+// Exact > Suffix > Prefix > Universal > Invalid.
+func (t domainMatchType) betterThan(b domainMatchType) bool {
+	return t > b
+}
+
+func matchTypeForDomain(d string) domainMatchType {
+	if d == "" {
+		return domainMatchTypeInvalid
+	}
+	if d == "*" {
+		return domainMatchTypeUniversal
+	}
+	if strings.HasPrefix(d, "*") {
+		return domainMatchTypeSuffix
+	}
+	if strings.HasSuffix(d, "*") {
+		return domainMatchTypePrefix
+	}
+	if strings.Contains(d, "*") {
+		return domainMatchTypeInvalid
+	}
+	return domainMatchTypeExact
+}
+
+func match(domain, host string) (domainMatchType, bool) {
+	switch typ := matchTypeForDomain(domain); typ {
+	case domainMatchTypeInvalid:
+		return typ, false
+	case domainMatchTypeUniversal:
+		return typ, true
+	case domainMatchTypePrefix:
+		// abc.*
+		return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*"))
+	case domainMatchTypeSuffix:
+		// *.123
+		return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*"))
+	case domainMatchTypeExact:
+		return typ, domain == host
+	default:
+		return domainMatchTypeInvalid, false
+	}
+}
+
+// findBestMatchingVirtualHost returns the virtual host whose domains field best
+// matches host
+//
+// The domains field support 4 different matching pattern types:
+//  - Exact match
+//  - Suffix match (e.g. “*ABC”)
+//  - Prefix match (e.g. “ABC*)
+//  - Universal match (e.g. “*”)
+//
+// The best match is defined as:
+//  - A match is better if it’s matching pattern type is better
+//    - Exact match > suffix match > prefix match > universal match
+//  - If two matches are of the same pattern type, the longer match is better
+//    - This is to compare the length of the matching pattern, e.g. “*ABCDE” >
+//    “*ABC”
+func findBestMatchingVirtualHost(host string, vHosts []*v3routepb.VirtualHost) *v3routepb.VirtualHost {
+	var (
+		matchVh   *v3routepb.VirtualHost
+		matchType = domainMatchTypeInvalid
+		matchLen  int
+	)
+	for _, vh := range vHosts {
+		for _, domain := range vh.GetDomains() {
+			typ, matched := match(domain, host)
+			if typ == domainMatchTypeInvalid {
+				// The rds response is invalid.
+				return nil
+			}
+			if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched {
+				// The previous match has better type, or the previous match has
+				// better length, or this domain isn't a match.
+				continue
+			}
+			matchVh = vh
+			matchType = typ
+			matchLen = len(domain)
+		}
+	}
+	return matchVh
+}
+
+// UnmarshalCluster processes resources received in an CDS response, validates
+// them, and transforms them into a native struct which contains only fields we
+// are interested in.
+func UnmarshalCluster(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ClusterUpdate, error) {
+	update := make(map[string]ClusterUpdate)
+	for _, r := range resources {
+		if t := r.GetTypeUrl(); t != version.V2ClusterURL && t != version.V3ClusterURL {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in CDS response", t)
+		}
+
+		cluster := &v3clusterpb.Cluster{}
+		if err := proto.Unmarshal(r.GetValue(), cluster); err != nil {
+			return nil, fmt.Errorf("xds: failed to unmarshal resource in CDS response: %v", err)
+		}
+		logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
+		cu, err := validateCluster(cluster)
+		if err != nil {
+			return nil, err
+		}
+
+		// If the Cluster message in the CDS response did not contain a
+		// serviceName, we will just use the clusterName for EDS.
+		if cu.ServiceName == "" {
+			cu.ServiceName = cluster.GetName()
+		}
+		logger.Debugf("Resource with name %v, value %+v added to cache", cluster.GetName(), cu)
+		update[cluster.GetName()] = cu
+	}
+	return update, nil
+}
+
+func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
+	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
+	switch {
+	case cluster.GetType() != v3clusterpb.Cluster_EDS:
+		return emptyUpdate, fmt.Errorf("xds: unexpected cluster type %v in response: %+v", cluster.GetType(), cluster)
+	case cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil:
+		return emptyUpdate, fmt.Errorf("xds: unexpected edsConfig in response: %+v", cluster)
+	case cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN:
+		return emptyUpdate, fmt.Errorf("xds: unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
+	}
+
+	return ClusterUpdate{
+		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
+		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
+	}, nil
+}
+
+// UnmarshalEndpoints processes resources received in an EDS response,
+// validates them, and transforms them into a native struct which contains only
+// fields we are interested in.
+func UnmarshalEndpoints(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]EndpointsUpdate, error) {
+	update := make(map[string]EndpointsUpdate)
+	for _, r := range resources {
+		if t := r.GetTypeUrl(); t != version.V2EndpointsURL && t != version.V3EndpointsURL {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in EDS response", t)
+		}
+
+		cla := &v3endpointpb.ClusterLoadAssignment{}
+		if err := proto.Unmarshal(r.GetValue(), cla); err != nil {
+			return nil, fmt.Errorf("xds: failed to unmarshal resource in EDS response: %v", err)
+		}
+		logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
+
+		u, err := parseEDSRespProto(cla)
+		if err != nil {
+			return nil, err
+		}
+		update[cla.GetClusterName()] = u
+	}
+	return update, nil
+}
+
+func parseAddress(socketAddress *v3corepb.SocketAddress) string {
+	return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue())))
+}
+
+func parseDropPolicy(dropPolicy *v3endpointpb.ClusterLoadAssignment_Policy_DropOverload) OverloadDropConfig {
+	percentage := dropPolicy.GetDropPercentage()
+	var (
+		numerator   = percentage.GetNumerator()
+		denominator uint32
+	)
+	switch percentage.GetDenominator() {
+	case v3typepb.FractionalPercent_HUNDRED:
+		denominator = 100
+	case v3typepb.FractionalPercent_TEN_THOUSAND:
+		denominator = 10000
+	case v3typepb.FractionalPercent_MILLION:
+		denominator = 1000000
+	}
+	return OverloadDropConfig{
+		Category:    dropPolicy.GetCategory(),
+		Numerator:   numerator,
+		Denominator: denominator,
+	}
+}
+
+func parseEndpoints(lbEndpoints []*v3endpointpb.LbEndpoint) []Endpoint {
+	endpoints := make([]Endpoint, 0, len(lbEndpoints))
+	for _, lbEndpoint := range lbEndpoints {
+		endpoints = append(endpoints, Endpoint{
+			HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()),
+			Address:      parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()),
+			Weight:       lbEndpoint.GetLoadBalancingWeight().GetValue(),
+		})
+	}
+	return endpoints
+}
+
+func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate, error) {
+	ret := EndpointsUpdate{}
+	for _, dropPolicy := range m.GetPolicy().GetDropOverloads() {
+		ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy))
+	}
+	priorities := make(map[uint32]struct{})
+	for _, locality := range m.Endpoints {
+		l := locality.GetLocality()
+		if l == nil {
+			return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
+		}
+		lid := internal.LocalityID{
+			Region:  l.Region,
+			Zone:    l.Zone,
+			SubZone: l.SubZone,
+		}
+		priority := locality.GetPriority()
+		priorities[priority] = struct{}{}
+		ret.Localities = append(ret.Localities, Locality{
+			ID:        lid,
+			Endpoints: parseEndpoints(locality.GetLbEndpoints()),
+			Weight:    locality.GetLoadBalancingWeight().GetValue(),
+			Priority:  priority,
+		})
+	}
+	for i := 0; i < len(priorities); i++ {
+		if _, ok := priorities[uint32(i)]; !ok {
+			return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities)
+		}
+	}
+	return ret, nil
+}
diff --git a/xds/internal/client/tests/client_test.go b/xds/internal/client/tests/client_test.go
new file mode 100644
index 000000000000..3c96175b0d89
--- /dev/null
+++ b/xds/internal/client/tests/client_test.go
@@ -0,0 +1,122 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package tests
+
+import (
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/grpctest"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	_ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 API client.
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	testXDSServer = "xds-server"
+)
+
+func clientOpts(balancerName string) xdsclient.Options {
+	return xdsclient.Options{
+		Config: bootstrap.Config{
+			BalancerName: balancerName,
+			Creds:        grpc.WithInsecure(),
+			NodeProto:    testutils.EmptyNodeProtoV2,
+		},
+	}
+}
+
+func (s) TestNew(t *testing.T) {
+	tests := []struct {
+		name    string
+		opts    xdsclient.Options
+		wantErr bool
+	}{
+		{name: "empty-opts", opts: xdsclient.Options{}, wantErr: true},
+		{
+			name: "empty-balancer-name",
+			opts: xdsclient.Options{
+				Config: bootstrap.Config{
+					Creds:     grpc.WithInsecure(),
+					NodeProto: testutils.EmptyNodeProtoV2,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty-dial-creds",
+			opts: xdsclient.Options{
+				Config: bootstrap.Config{
+					BalancerName: testXDSServer,
+					NodeProto:    testutils.EmptyNodeProtoV2,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty-node-proto",
+			opts: xdsclient.Options{
+				Config: bootstrap.Config{
+					BalancerName: testXDSServer,
+					Creds:        grpc.WithInsecure(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "node-proto-version-mismatch",
+			opts: xdsclient.Options{
+				Config: bootstrap.Config{
+					BalancerName: testXDSServer,
+					Creds:        grpc.WithInsecure(),
+					NodeProto:    testutils.EmptyNodeProtoV3,
+					TransportAPI: version.TransportV2,
+				},
+			},
+			wantErr: true,
+		},
+		// TODO(easwars): Add cases for v3 API client.
+		{
+			name: "happy-case",
+			opts: clientOpts(testXDSServer),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c, err := xdsclient.New(test.opts)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("New(%+v) = %v, wantErr: %v", test.opts, err, test.wantErr)
+			}
+			if c != nil {
+				c.Close()
+			}
+		})
+	}
+}
diff --git a/xds/internal/client/testutil_test.go b/xds/internal/client/testutil_test.go
deleted file mode 100644
index a63fbdb38a91..000000000000
--- a/xds/internal/client/testutil_test.go
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package client
-
-import (
-	"reflect"
-	"testing"
-	"time"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-)
-
-type watchHandleTestcase struct {
-	typeURL      string
-	resourceName string
-
-	responseToHandle *xdspb.DiscoveryResponse
-	wantHandleErr    bool
-	wantUpdate       interface{}
-	wantUpdateErr    bool
-}
-
-type testUpdateReceiver struct {
-	f func(typeURL string, d map[string]interface{})
-}
-
-func (t *testUpdateReceiver) newLDSUpdate(d map[string]ldsUpdate) {
-	dd := make(map[string]interface{})
-	for k, v := range d {
-		dd[k] = v
-	}
-	t.newUpdate(ldsURL, dd)
-}
-
-func (t *testUpdateReceiver) newRDSUpdate(d map[string]rdsUpdate) {
-	dd := make(map[string]interface{})
-	for k, v := range d {
-		dd[k] = v
-	}
-	t.newUpdate(rdsURL, dd)
-}
-
-func (t *testUpdateReceiver) newCDSUpdate(d map[string]ClusterUpdate) {
-	dd := make(map[string]interface{})
-	for k, v := range d {
-		dd[k] = v
-	}
-	t.newUpdate(cdsURL, dd)
-}
-
-func (t *testUpdateReceiver) newEDSUpdate(d map[string]EndpointsUpdate) {
-	dd := make(map[string]interface{})
-	for k, v := range d {
-		dd[k] = v
-	}
-	t.newUpdate(edsURL, dd)
-}
-
-func (t *testUpdateReceiver) newUpdate(typeURL string, d map[string]interface{}) {
-	t.f(typeURL, d)
-}
-
-// testWatchHandle is called to test response handling for each xDS.
-//
-// It starts the xDS watch as configured in test, waits for the fake xds server
-// to receive the request (so watch callback is installed), and calls
-// handleXDSResp with responseToHandle (if it's set). It then compares the
-// update received by watch callback with the expected results.
-func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	type updateErr struct {
-		u   interface{}
-		err error
-	}
-	gotUpdateCh := testutils.NewChannel()
-
-	v2c := newV2Client(&testUpdateReceiver{
-		f: func(typeURL string, d map[string]interface{}) {
-			if typeURL == test.typeURL {
-				if u, ok := d[test.resourceName]; ok {
-					gotUpdateCh.Send(updateErr{u, nil})
-				}
-			}
-		},
-	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-
-	// RDS needs an existin LDS watch for the hostname.
-	if test.typeURL == rdsURL {
-		doLDS(t, v2c, fakeServer)
-	}
-
-	// Register the watcher, this will also trigger the v2Client to send the xDS
-	// request.
-	v2c.addWatch(test.typeURL, test.resourceName)
-
-	// Wait till the request makes it to the fakeServer. This ensures that
-	// the watch request has been processed by the v2Client.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout waiting for an xDS request: %v", err)
-	}
-
-	// Directly push the response through a call to handleXDSResp. This bypasses
-	// the fakeServer, so it's only testing the handle logic. Client response
-	// processing is covered elsewhere.
-	//
-	// Also note that this won't trigger ACK, so there's no need to clear the
-	// request channel afterwards.
-	var handleXDSResp func(response *xdspb.DiscoveryResponse) error
-	switch test.typeURL {
-	case ldsURL:
-		handleXDSResp = v2c.handleLDSResponse
-	case rdsURL:
-		handleXDSResp = v2c.handleRDSResponse
-	case cdsURL:
-		handleXDSResp = v2c.handleCDSResponse
-	case edsURL:
-		handleXDSResp = v2c.handleEDSResponse
-	}
-	if err := handleXDSResp(test.responseToHandle); (err != nil) != test.wantHandleErr {
-		t.Fatalf("v2c.handleRDSResponse() returned err: %v, wantErr: %v", err, test.wantHandleErr)
-	}
-
-	// If the test doesn't expect the callback to be invoked, verify that no
-	// update or error is pushed to the callback.
-	//
-	// Cannot directly compare test.wantUpdate with nil (typed vs non-typed nil:
-	// https://golang.org/doc/faq#nil_error).
-	if c := test.wantUpdate; c == nil || (reflect.ValueOf(c).Kind() == reflect.Ptr && reflect.ValueOf(c).IsNil()) {
-		update, err := gotUpdateCh.Receive()
-		if err == testutils.ErrRecvTimeout {
-			return
-		}
-		t.Fatalf("Unexpected update: +%v", update)
-	}
-
-	wantUpdate := reflect.ValueOf(test.wantUpdate).Elem().Interface()
-	uErr, err := gotUpdateCh.Receive()
-	if err == testutils.ErrRecvTimeout {
-		t.Fatal("Timeout expecting xDS update")
-	}
-	gotUpdate := uErr.(updateErr).u
-	opt := cmp.AllowUnexported(rdsUpdate{}, ldsUpdate{}, ClusterUpdate{}, EndpointsUpdate{})
-	if diff := cmp.Diff(gotUpdate, wantUpdate, opt); diff != "" {
-		t.Fatalf("got update : %+v, want %+v, diff: %s", gotUpdate, wantUpdate, diff)
-	}
-	gotUpdateErr := uErr.(updateErr).err
-	if (gotUpdateErr != nil) != test.wantUpdateErr {
-		t.Fatalf("got xDS update error {%v}, wantErr: %v", gotUpdateErr, test.wantUpdateErr)
-	}
-}
-
-// startServerAndGetCC starts a fake XDS server and also returns a ClientConn
-// connected to it.
-func startServerAndGetCC(t *testing.T) (*fakeserver.Server, *grpc.ClientConn, func()) {
-	t.Helper()
-
-	fs, sCleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
-
-	cc, ccCleanup, err := fs.XDSClientConn()
-	if err != nil {
-		sCleanup()
-		t.Fatalf("Failed to get a clientConn to the fake xDS server: %v", err)
-	}
-	return fs, cc, func() {
-		sCleanup()
-		ccCleanup()
-	}
-}
-
-// waitForNilErr waits for a nil error value to be received on the
-// provided channel.
-func waitForNilErr(t *testing.T, ch *testutils.Channel) {
-	t.Helper()
-
-	val, err := ch.Receive()
-	if err == testutils.ErrRecvTimeout {
-		t.Fatalf("Timeout expired when expecting update")
-	}
-	if val != nil {
-		if cbErr := val.(error); cbErr != nil {
-			t.Fatal(cbErr)
-		}
-	}
-}
diff --git a/xds/internal/client/v2client.go b/xds/internal/client/v2/client.go
similarity index 74%
rename from xds/internal/client/v2client.go
rename to xds/internal/client/v2/client.go
index 28536f26f286..627b9e8e5a95 100644
--- a/xds/internal/client/v2client.go
+++ b/xds/internal/client/v2/client.go
@@ -16,38 +16,68 @@
  *
  */
 
-package client
+// Package v2 provides xDS v2 transport protocol specific functionality.
+package v2
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/version"
 
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
 )
 
-type adsStream adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient
+func init() {
+	xdsclient.RegisterAPIClientBuilder(clientBuilder{})
+}
 
-var _ xdsv2Client = &v2Client{}
+type clientBuilder struct{}
 
-// updateHandler handles the update (parsed from xds responses). It's
-// implemented by the upper level Client.
-//
-// It's an interface to be overridden in test.
-type updateHandler interface {
-	newLDSUpdate(d map[string]ldsUpdate)
-	newRDSUpdate(d map[string]rdsUpdate)
-	newCDSUpdate(d map[string]ClusterUpdate)
-	newEDSUpdate(d map[string]EndpointsUpdate)
+func (clientBuilder) Build(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
+	return newClient(cc, opts)
 }
 
-// v2Client performs the actual xDS RPCs using the xDS v2 API. It creates a
+func (clientBuilder) Version() version.TransportAPI {
+	return version.TransportV2
+}
+
+func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
+	nodeProto, ok := opts.NodeProto.(*v2corepb.Node)
+	if !ok {
+		return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v2corepb.Node{})
+	}
+	v2c := &client{
+		cc:        cc,
+		parent:    opts.Parent,
+		nodeProto: nodeProto,
+		backoff:   opts.Backoff,
+		logger:    opts.Logger,
+
+		streamCh: make(chan adsStream, 1),
+		sendCh:   buffer.NewUnbounded(),
+
+		watchMap:   make(map[string]map[string]bool),
+		versionMap: make(map[string]string),
+		nonceMap:   make(map[string]string),
+	}
+	v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background())
+
+	go v2c.run()
+	return v2c, nil
+}
+
+type adsStream v2adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient
+
+// client performs the actual xDS RPCs using the xDS v2 API. It creates a
 // single ADS stream on which the different types of xDS requests and responses
 // are multiplexed.
 //
@@ -55,18 +85,14 @@ type updateHandler interface {
 // and do ACK/NACK. It's a naive implementation that sends whatever the upper
 // layer tells it to send. It will call the callback with everything in every
 // response. It doesn't keep a cache of responses, or check for duplicates.
-//
-// The reason for splitting this out from the top level xdsClient object is
-// because there is already an xDS v3Aplha API in development. If and when we
-// want to switch to that, this separation will ease that process.
-type v2Client struct {
+type client struct {
 	ctx       context.Context
 	cancelCtx context.CancelFunc
-	parent    updateHandler
+	parent    xdsclient.UpdateHandler
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
 	cc        *grpc.ClientConn
-	nodeProto *corepb.Node
+	nodeProto *v2corepb.Node
 	backoff   func(int) time.Duration
 
 	logger *grpclog.PrefixLogger
@@ -102,38 +128,31 @@ type v2Client struct {
 	hostname string
 }
 
-// newV2Client creates a new v2Client initialized with the passed arguments.
-func newV2Client(parent updateHandler, cc *grpc.ClientConn, nodeProto *corepb.Node, backoff func(int) time.Duration, logger *grpclog.PrefixLogger) *v2Client {
-	v2c := &v2Client{
-		cc:        cc,
-		parent:    parent,
-		nodeProto: nodeProto,
-		backoff:   backoff,
-
-		logger: logger,
-
-		streamCh: make(chan adsStream, 1),
-		sendCh:   buffer.NewUnbounded(),
-
-		watchMap:   make(map[string]map[string]bool),
-		versionMap: make(map[string]string),
-		nonceMap:   make(map[string]string),
-	}
-	v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background())
+func (v2c *client) AddWatch(resourceType, resourceName string) {
+	v2c.sendCh.Put(&watchAction{
+		typeURL:  resourceType,
+		remove:   false,
+		resource: resourceName,
+	})
+}
 
-	go v2c.run()
-	return v2c
+func (v2c *client) RemoveWatch(resourceType, resourceName string) {
+	v2c.sendCh.Put(&watchAction{
+		typeURL:  resourceType,
+		remove:   true,
+		resource: resourceName,
+	})
 }
 
 // close cleans up resources and goroutines allocated by this client.
-func (v2c *v2Client) close() {
+func (v2c *client) Close() {
 	v2c.cancelCtx()
 }
 
 // run starts an ADS stream (and backs off exponentially, if the previous
 // stream failed without receiving a single reply) and runs the sender and
 // receiver routines to send and receive data from the stream respectively.
-func (v2c *v2Client) run() {
+func (v2c *client) run() {
 	go v2c.send()
 	// TODO: start a goroutine monitoring ClientConn's connectivity state, and
 	// report error (and log) when stats is transient failure.
@@ -159,7 +178,7 @@ func (v2c *v2Client) run() {
 		}
 
 		retries++
-		cli := adsgrpc.NewAggregatedDiscoveryServiceClient(v2c.cc)
+		cli := v2adsgrpc.NewAggregatedDiscoveryServiceClient(v2c.cc)
 		stream, err := cli.StreamAggregatedResources(v2c.ctx, grpc.WaitForReady(true))
 		if err != nil {
 			v2c.logger.Warningf("xds: ADS stream creation failed: %v", err)
@@ -187,8 +206,8 @@ func (v2c *v2Client) run() {
 // - If this is an ack, version will be the version from the response
 // - If this is a nack, version will be the previous acked version (from
 // versionMap). If there was no ack before, it will be an empty string
-func (v2c *v2Client) sendRequest(stream adsStream, resourceNames []string, typeURL, version, nonce string) bool {
-	req := &xdspb.DiscoveryRequest{
+func (v2c *client) sendRequest(stream adsStream, resourceNames []string, typeURL, version, nonce string) bool {
+	req := &v2xdspb.DiscoveryRequest{
 		Node:          v2c.nodeProto,
 		TypeUrl:       typeURL,
 		ResourceNames: resourceNames,
@@ -210,7 +229,7 @@ func (v2c *v2Client) sendRequest(stream adsStream, resourceNames []string, typeU
 // that here because the stream has just started and Send() usually returns
 // quickly (once it pushes the message onto the transport layer) and is only
 // ever blocked if we don't have enough flow control quota.
-func (v2c *v2Client) sendExisting(stream adsStream) bool {
+func (v2c *client) sendExisting(stream adsStream) bool {
 	v2c.mu.Lock()
 	defer v2c.mu.Unlock()
 
@@ -236,7 +255,7 @@ type watchAction struct {
 // processWatchInfo pulls the fields needed by the request from a watchAction.
 //
 // It also updates the watch map in v2c.
-func (v2c *v2Client) processWatchInfo(t *watchAction) (target []string, typeURL, version, nonce string, send bool) {
+func (v2c *client) processWatchInfo(t *watchAction) (target []string, typeURL, ver, nonce string, send bool) {
 	v2c.mu.Lock()
 	defer v2c.mu.Unlock()
 
@@ -258,7 +277,7 @@ func (v2c *v2Client) processWatchInfo(t *watchAction) (target []string, typeURL,
 
 	// Special handling for LDS, because RDS needs the LDS resource_name for
 	// response host matching.
-	if t.typeURL == ldsURL {
+	if t.typeURL == version.V2ListenerURL {
 		// Set hostname to the first LDS resource_name, and reset it when the
 		// last LDS watch is removed. The upper level Client isn't expected to
 		// watchLDS more than once.
@@ -275,9 +294,9 @@ func (v2c *v2Client) processWatchInfo(t *watchAction) (target []string, typeURL,
 	// We don't reset version or nonce when a new watch is started. The version
 	// and nonce from previous response are carried by the request unless the
 	// stream is recreated.
-	version = v2c.versionMap[typeURL]
+	ver = v2c.versionMap[typeURL]
 	nonce = v2c.nonceMap[typeURL]
-	return target, typeURL, version, nonce, send
+	return target, typeURL, ver, nonce, send
 }
 
 type ackAction struct {
@@ -293,7 +312,7 @@ type ackAction struct {
 // processAckInfo pulls the fields needed by the ack request from a ackAction.
 //
 // If no active watch is found for this ack, it returns false for send.
-func (v2c *v2Client) processAckInfo(t *ackAction, stream adsStream) (target []string, typeURL, version, nonce string, send bool) {
+func (v2c *client) processAckInfo(t *ackAction, stream adsStream) (target []string, typeURL, version, nonce string, send bool) {
 	if t.stream != stream {
 		// If ACK's stream isn't the current sending stream, this means the ACK
 		// was pushed to queue before the old stream broke, and a new stream has
@@ -353,7 +372,7 @@ func (v2c *v2Client) processAckInfo(t *ackAction, stream adsStream) (target []st
 // Note that this goroutine doesn't do anything to the old stream when there's a
 // new one. In fact, there should be only one stream in progress, and new one
 // should only be created when the old one fails (recv returns an error).
-func (v2c *v2Client) send() {
+func (v2c *client) send() {
 	var stream adsStream
 	for {
 		select {
@@ -398,7 +417,7 @@ func (v2c *v2Client) send() {
 
 // recv receives xDS responses on the provided ADS stream and branches out to
 // message specific handlers.
-func (v2c *v2Client) recv(stream adsStream) bool {
+func (v2c *client) recv(stream adsStream) bool {
 	success := false
 	for {
 		resp, err := stream.Recv()
@@ -409,15 +428,20 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 		}
 		v2c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
 		v2c.logger.Debugf("ADS response received: %v", resp)
+
+		// Note that the xDS transport protocol is versioned independently of
+		// the resource types, and it is supported to transfer older versions
+		// of resource types using new versions of the transport protocol, or
+		// vice-versa. Hence we need to handle v3 type_urls as well here.
 		var respHandleErr error
 		switch resp.GetTypeUrl() {
-		case ldsURL:
+		case version.V2ListenerURL, version.V3ListenerURL:
 			respHandleErr = v2c.handleLDSResponse(resp)
-		case rdsURL:
+		case version.V2RouteConfigURL, version.V3RouteConfigURL:
 			respHandleErr = v2c.handleRDSResponse(resp)
-		case cdsURL:
+		case version.V2ClusterURL, version.V3ClusterURL:
 			respHandleErr = v2c.handleCDSResponse(resp)
-		case edsURL:
+		case version.V2EndpointsURL, version.V3EndpointsURL:
 			respHandleErr = v2c.handleEDSResponse(resp)
 		default:
 			v2c.logger.Warningf("Resource type %v unknown in response from server", resp.GetTypeUrl())
@@ -446,25 +470,56 @@ func (v2c *v2Client) recv(stream adsStream) bool {
 	}
 }
 
-func (v2c *v2Client) addWatch(resourceType, resourceName string) {
-	v2c.sendCh.Put(&watchAction{
-		typeURL:  resourceType,
-		remove:   false,
-		resource: resourceName,
-	})
-}
-
-func (v2c *v2Client) removeWatch(resourceType, resourceName string) {
-	v2c.sendCh.Put(&watchAction{
-		typeURL:  resourceType,
-		remove:   true,
-		resource: resourceName,
-	})
-}
-
 func mapToSlice(m map[string]bool) (ret []string) {
 	for i := range m {
 		ret = append(ret, i)
 	}
 	return
 }
+
+// handleLDSResponse processes an LDS response received from the xDS server. On
+// receipt of a good response, it also invokes the registered watcher callback.
+func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
+	update, err := xdsclient.UnmarshalListener(resp.GetResources(), v2c.logger)
+	if err != nil {
+		return err
+	}
+	v2c.parent.NewListeners(update)
+	return nil
+}
+
+// handleRDSResponse processes an RDS response received from the xDS server. On
+// receipt of a good response, it caches validated resources and also invokes
+// the registered watcher callback.
+func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
+	v2c.mu.Lock()
+	hostname := v2c.hostname
+	v2c.mu.Unlock()
+
+	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v2c.logger)
+	if err != nil {
+		return err
+	}
+	v2c.parent.NewRouteConfigs(update)
+	return nil
+}
+
+// handleCDSResponse processes an CDS response received from the xDS server. On
+// receipt of a good response, it also invokes the registered watcher callback.
+func (v2c *client) handleCDSResponse(resp *v2xdspb.DiscoveryResponse) error {
+	update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v2c.logger)
+	if err != nil {
+		return err
+	}
+	v2c.parent.NewClusters(update)
+	return nil
+}
+
+func (v2c *client) handleEDSResponse(resp *v2xdspb.DiscoveryResponse) error {
+	update, err := xdsclient.UnmarshalEndpoints(resp.GetResources(), v2c.logger)
+	if err != nil {
+		return err
+	}
+	v2c.parent.NewEndpoints(update)
+	return nil
+}
diff --git a/xds/internal/client/v2client_ack_test.go b/xds/internal/client/v2/client_ack_test.go
similarity index 88%
rename from xds/internal/client/v2client_ack_test.go
rename to xds/internal/client/v2/client_ack_test.go
index cbeb862eefff..d8d1ad834968 100644
--- a/xds/internal/client/v2client_ack_test.go
+++ b/xds/internal/client/v2/client_ack_test.go
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package client
+package v2
 
 import (
 	"fmt"
@@ -30,42 +30,46 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
-func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *v2Client, cbLDS, cbRDS, cbCDS, cbEDS *testutils.Channel, cleanup func()) {
+func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cbRDS, cbCDS, cbEDS *testutils.Channel, cleanup func()) {
 	cbLDS = testutils.NewChannel()
 	cbRDS = testutils.NewChannel()
 	cbCDS = testutils.NewChannel()
 	cbEDS = testutils.NewChannel()
-	v2c = newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(typeURL string, d map[string]interface{}) {
 			t.Logf("Received %s callback with {%+v}", typeURL, d)
 			switch typeURL {
-			case ldsURL:
+			case version.V2ListenerURL:
 				if _, ok := d[goodLDSTarget1]; ok {
 					cbLDS.Send(struct{}{})
 				}
-			case rdsURL:
+			case version.V2RouteConfigURL:
 				if _, ok := d[goodRouteName1]; ok {
 					cbRDS.Send(struct{}{})
 				}
-			case cdsURL:
+			case version.V2ClusterURL:
 				if _, ok := d[goodClusterName1]; ok {
 					cbCDS.Send(struct{}{})
 				}
-			case edsURL:
+			case version.V2EndpointsURL:
 				if _, ok := d[goodEDSName]; ok {
 					cbEDS.Send(struct{}{})
 				}
 			}
 		},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	t.Log("Started xds v2Client...")
-	return v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2c.close
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Started xds client...")
+	return v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2c.Close
 }
 
 // compareXDSRequest reads requests from channel, compare it with want.
-func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, version, nonce string) error {
+func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string) error {
 	val, err := ch.Receive()
 	if err != nil {
 		return err
@@ -75,7 +79,7 @@ func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, vers
 		return fmt.Errorf("unexpected error from request: %v", req.Err)
 	}
 	wantClone := proto.Clone(want).(*xdspb.DiscoveryRequest)
-	wantClone.VersionInfo = version
+	wantClone.VersionInfo = ver
 	wantClone.ResponseNonce = nonce
 	if !cmp.Equal(req.Req, wantClone, cmp.Comparer(proto.Equal)) {
 		return fmt.Errorf("received request different from want, diff: %s", cmp.Diff(req.Req, wantClone))
@@ -83,9 +87,9 @@ func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, vers
 	return nil
 }
 
-func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *xdspb.DiscoveryResponse, version int) (nonce string) {
+func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *xdspb.DiscoveryResponse, ver int) (nonce string) {
 	respToSend := proto.Clone(respWithoutVersion).(*xdspb.DiscoveryResponse)
-	respToSend.VersionInfo = strconv.Itoa(version)
+	respToSend.VersionInfo = strconv.Itoa(ver)
 	nonce = strconv.Itoa(int(time.Now().UnixNano()))
 	respToSend.Nonce = nonce
 	ch <- &fakeserver.Response{Resp: respToSend}
@@ -94,25 +98,25 @@ func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *
 
 // startXDS calls watch to send the first request. It then sends a good response
 // and checks for ack.
-func startXDS(t *testing.T, xdsname string, v2c *v2Client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
+func startXDS(t *testing.T, xdsname string, v2c *client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
 	var (
 		nameToWatch, typeURLToWatch string
 	)
 	switch xdsname {
 	case "LDS":
-		typeURLToWatch = ldsURL
+		typeURLToWatch = version.V2ListenerURL
 		nameToWatch = goodLDSTarget1
 	case "RDS":
-		typeURLToWatch = rdsURL
+		typeURLToWatch = version.V2RouteConfigURL
 		nameToWatch = goodRouteName1
 	case "CDS":
-		typeURLToWatch = cdsURL
+		typeURLToWatch = version.V2ClusterURL
 		nameToWatch = goodClusterName1
 	case "EDS":
-		typeURLToWatch = edsURL
+		typeURLToWatch = version.V2EndpointsURL
 		nameToWatch = goodEDSName
 	}
-	v2c.addWatch(typeURLToWatch, nameToWatch)
+	v2c.AddWatch(typeURLToWatch, nameToWatch)
 
 	if err := compareXDSRequest(reqChan, req, preVersion, preNonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
@@ -125,11 +129,11 @@ func startXDS(t *testing.T, xdsname string, v2c *v2Client, reqChan *testutils.Ch
 //
 // It also waits and checks that the ack request contains the given version, and
 // the generated nonce.
-func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
-	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, version)
+func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, ver int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
+	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, ver)
 	t.Logf("Good %s response pushed to fakeServer...", xdsname)
 
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(version), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce); err != nil {
 		return "", fmt.Errorf("failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("Good %s response acked", xdsname)
@@ -145,24 +149,24 @@ func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, v
 // be nacked, so we expect a request with the previous version (version-1).
 //
 // But the nonce in request should be the new nonce.
-func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, version int, wantReq *xdspb.DiscoveryRequest) error {
+func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, ver int, wantReq *xdspb.DiscoveryRequest) error {
 	var typeURL string
 	switch xdsname {
 	case "LDS":
-		typeURL = ldsURL
+		typeURL = version.V2ListenerURL
 	case "RDS":
-		typeURL = rdsURL
+		typeURL = version.V2RouteConfigURL
 	case "CDS":
-		typeURL = cdsURL
+		typeURL = version.V2ClusterURL
 	case "EDS":
-		typeURL = edsURL
+		typeURL = version.V2EndpointsURL
 	}
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
 		TypeUrl:   typeURL,
-	}, version)
+	}, ver)
 	t.Logf("Bad %s response pushed to fakeServer...", xdsname)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(version-1), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce); err != nil {
 		return fmt.Errorf("failed to receive %s request: %v", xdsname, err)
 	}
 	t.Logf("Bad %s response nacked", xdsname)
@@ -262,7 +266,7 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
-		TypeUrl:   ldsURL,
+		TypeUrl:   version.V2ListenerURL,
 	}, versionLDS)
 	t.Logf("Bad response pushed to fakeServer...")
 
@@ -303,7 +307,7 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	// This is an invalid response after the new watch.
 	nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
-		TypeUrl:   ldsURL,
+		TypeUrl:   version.V2ListenerURL,
 	}, versionLDS)
 	t.Logf("Bad response pushed to fakeServer...")
 
@@ -332,7 +336,7 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start a CDS watch.
-	v2c.addWatch(cdsURL, goodClusterName1)
+	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
 		t.Fatal(err)
 	}
@@ -346,14 +350,14 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	}
 	// Cancel the CDS watch, and start a new one. The new watch should have the
 	// version from the response above.
-	v2c.removeWatch(cdsURL, goodClusterName1)
+	v2c.RemoveWatch(version.V2ClusterURL, goodClusterName1)
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
-	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: cdsURL}
+	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
-	v2c.addWatch(cdsURL, goodClusterName1)
+	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
 	// Wait for a request with correct resource names and version.
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
@@ -387,7 +391,7 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start a CDS watch.
-	v2c.addWatch(cdsURL, goodClusterName1)
+	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
@@ -400,10 +404,10 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	}
 	// Cancel the watch before the next response is sent. This mimics the case
 	// watch is canceled while response is on wire.
-	v2c.removeWatch(cdsURL, goodClusterName1)
+	v2c.RemoveWatch(version.V2ClusterURL, goodClusterName1)
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
-	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: cdsURL}
+	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
@@ -428,7 +432,7 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 
 	// Start a new watch. The new watch should have the nonce from the response
 	// above, and version from the first good response.
-	v2c.addWatch(cdsURL, goodClusterName1)
+	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce); err != nil {
 		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
 	}
diff --git a/xds/internal/client/v2client_cds_test.go b/xds/internal/client/v2/client_cds_test.go
similarity index 56%
rename from xds/internal/client/v2client_cds_test.go
rename to xds/internal/client/v2/client_cds_test.go
index b93beede4485..e93f952e2a7e 100644
--- a/xds/internal/client/v2client_cds_test.go
+++ b/xds/internal/client/v2/client_cds_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package client
+package v2
 
 import (
 	"testing"
@@ -26,7 +26,8 @@ import (
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
-	"github.com/google/go-cmp/cmp"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 const (
@@ -34,115 +35,68 @@ const (
 	serviceName2 = "bar-service"
 )
 
-func (s) TestValidateCluster(t *testing.T) {
-	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
-	tests := []struct {
-		name       string
-		cluster    *xdspb.Cluster
-		wantUpdate ClusterUpdate
-		wantErr    bool
-	}{
-		{
-			name: "non-eds-cluster-type",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_STATIC},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-				},
-				LbPolicy: xdspb.Cluster_LEAST_REQUEST,
+var (
+	badlyMarshaledCDSResponse = &xdspb.DiscoveryResponse{
+		Resources: []*anypb.Any{
+			{
+				TypeUrl: version.V2ClusterURL,
+				Value:   []byte{1, 2, 3, 4},
 			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
 		},
-		{
-			name: "no-eds-config",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				LbPolicy:             xdspb.Cluster_ROUND_ROBIN,
+		TypeUrl: version.V2ClusterURL,
+	}
+	goodCluster1 = &xdspb.Cluster{
+		Name:                 goodClusterName1,
+		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+			EdsConfig: &corepb.ConfigSource{
+				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+					Ads: &corepb.AggregatedConfigSource{},
+				},
 			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
+			ServiceName: serviceName1,
 		},
-		{
-			name: "no-ads-config-source",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig:     &xdspb.Cluster_EdsClusterConfig{},
-				LbPolicy:             xdspb.Cluster_ROUND_ROBIN,
+		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+		LrsServer: &corepb.ConfigSource{
+			ConfigSourceSpecifier: &corepb.ConfigSource_Self{
+				Self: &corepb.SelfConfigSource{},
 			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
 		},
-		{
-			name: "non-round-robin-lb-policy",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
+	}
+	marshaledCluster1, _ = proto.Marshal(goodCluster1)
+	goodCluster2         = &xdspb.Cluster{
+		Name:                 goodClusterName2,
+		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
+		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
+			EdsConfig: &corepb.ConfigSource{
+				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
+					Ads: &corepb.AggregatedConfigSource{},
 				},
-				LbPolicy: xdspb.Cluster_LEAST_REQUEST,
 			},
-			wantUpdate: emptyUpdate,
-			wantErr:    true,
+			ServiceName: serviceName2,
 		},
-		{
-			name: "happy-case-no-service-name-no-lrs",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-				},
-				LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+	}
+	marshaledCluster2, _ = proto.Marshal(goodCluster2)
+	goodCDSResponse1     = &xdspb.DiscoveryResponse{
+		Resources: []*anypb.Any{
+			{
+				TypeUrl: version.V2ClusterURL,
+				Value:   marshaledCluster1,
 			},
-			wantUpdate: emptyUpdate,
 		},
-		{
-			name: "happy-case-no-lrs",
-			cluster: &xdspb.Cluster{
-				ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-				EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-					EdsConfig: &corepb.ConfigSource{
-						ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-							Ads: &corepb.AggregatedConfigSource{},
-						},
-					},
-					ServiceName: serviceName1,
-				},
-				LbPolicy: xdspb.Cluster_ROUND_ROBIN,
+		TypeUrl: version.V2ClusterURL,
+	}
+	goodCDSResponse2 = &xdspb.DiscoveryResponse{
+		Resources: []*anypb.Any{
+			{
+				TypeUrl: version.V2ClusterURL,
+				Value:   marshaledCluster2,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName1, EnableLRS: false},
-		},
-		{
-			name:       "happiest-case",
-			cluster:    goodCluster1,
-			wantUpdate: ClusterUpdate{ServiceName: serviceName1, EnableLRS: true},
 		},
+		TypeUrl: version.V2ClusterURL,
 	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, gotErr := validateCluster(test.cluster)
-			if (gotErr != nil) != test.wantErr {
-				t.Errorf("validateCluster(%+v) returned error: %v, wantErr: %v", test.cluster, gotErr, test.wantErr)
-			}
-			if !cmp.Equal(gotUpdate, test.wantUpdate) {
-				t.Errorf("validateCluster(%+v) = %v, want: %v", test.cluster, gotUpdate, test.wantUpdate)
-			}
-		})
-	}
-}
+)
 
 // TestCDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
 // and creates a v2Client using it. Then, it registers a CDS watcher and tests
@@ -152,7 +106,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 		name          string
 		cdsResponse   *xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *ClusterUpdate
+		wantUpdate    *xdsclient.ClusterUpdate
 		wantUpdateErr bool
 	}{
 		// Badly marshaled CDS response.
@@ -192,14 +146,14 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			name:          "one-good-cluster",
 			cdsResponse:   goodCDSResponse1,
 			wantErr:       false,
-			wantUpdate:    &ClusterUpdate{ServiceName: serviceName1, EnableLRS: true},
+			wantUpdate:    &xdsclient.ClusterUpdate{ServiceName: serviceName1, EnableLRS: true},
 			wantUpdateErr: false,
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:      cdsURL,
+				typeURL:      version.V2ClusterURL,
 				resourceName: goodClusterName1,
 
 				responseToHandle: test.cdsResponse,
@@ -217,10 +171,13 @@ func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) {
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(string, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
 
 	if v2c.handleCDSResponse(badResourceTypeInLDSResponse) == nil {
 		t.Fatal("v2c.handleCDSResponse() succeeded, should have failed")
@@ -230,66 +187,3 @@ func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) {
 		t.Fatal("v2c.handleCDSResponse() succeeded, should have failed")
 	}
 }
-
-var (
-	badlyMarshaledCDSResponse = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   []byte{1, 2, 3, 4},
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-	goodCluster1 = &xdspb.Cluster{
-		Name:                 goodClusterName1,
-		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-			EdsConfig: &corepb.ConfigSource{
-				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-					Ads: &corepb.AggregatedConfigSource{},
-				},
-			},
-			ServiceName: serviceName1,
-		},
-		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
-		LrsServer: &corepb.ConfigSource{
-			ConfigSourceSpecifier: &corepb.ConfigSource_Self{
-				Self: &corepb.SelfConfigSource{},
-			},
-		},
-	}
-	marshaledCluster1, _ = proto.Marshal(goodCluster1)
-	goodCluster2         = &xdspb.Cluster{
-		Name:                 goodClusterName2,
-		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
-		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
-			EdsConfig: &corepb.ConfigSource{
-				ConfigSourceSpecifier: &corepb.ConfigSource_Ads{
-					Ads: &corepb.AggregatedConfigSource{},
-				},
-			},
-			ServiceName: serviceName2,
-		},
-		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
-	}
-	marshaledCluster2, _ = proto.Marshal(goodCluster2)
-	goodCDSResponse1     = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   marshaledCluster1,
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-	goodCDSResponse2 = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: cdsURL,
-				Value:   marshaledCluster2,
-			},
-		},
-		TypeUrl: cdsURL,
-	}
-)
diff --git a/xds/internal/client/v2client_eds_test.go b/xds/internal/client/v2/client_eds_test.go
similarity index 52%
rename from xds/internal/client/v2client_eds_test.go
rename to xds/internal/client/v2/client_eds_test.go
index 009fbcb28190..170a96a42491 100644
--- a/xds/internal/client/v2client_eds_test.go
+++ b/xds/internal/client/v2/client_eds_test.go
@@ -13,157 +13,75 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package client
+package v2
 
 import (
 	"testing"
 	"time"
 
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	"github.com/golang/protobuf/ptypes"
 	anypb "github.com/golang/protobuf/ptypes/any"
-	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
-func (s) TestEDSParseRespProto(t *testing.T) {
-	tests := []struct {
-		name    string
-		m       *xdspb.ClusterLoadAssignment
-		want    EndpointsUpdate
-		wantErr bool
-	}{
-		{
-			name: "missing-priority",
-			m: func() *xdspb.ClusterLoadAssignment {
-				clab0 := NewClusterLoadAssignmentBuilder("test", nil)
-				clab0.AddLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
-				clab0.AddLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
-				return clab0.Build()
-			}(),
-			want:    EndpointsUpdate{},
-			wantErr: true,
-		},
-		{
-			name: "missing-locality-ID",
-			m: func() *xdspb.ClusterLoadAssignment {
-				clab0 := NewClusterLoadAssignmentBuilder("test", nil)
-				clab0.AddLocality("", 1, 0, []string{"addr1:314"}, nil)
-				return clab0.Build()
-			}(),
-			want:    EndpointsUpdate{},
-			wantErr: true,
-		},
-		{
-			name: "good",
-			m: func() *xdspb.ClusterLoadAssignment {
-				clab0 := NewClusterLoadAssignmentBuilder("test", nil)
-				clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, &AddLocalityOptions{
-					Health: []corepb.HealthStatus{corepb.HealthStatus_UNHEALTHY},
-					Weight: []uint32{271},
-				})
-				clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, &AddLocalityOptions{
-					Health: []corepb.HealthStatus{corepb.HealthStatus_DRAINING},
-					Weight: []uint32{828},
-				})
-				return clab0.Build()
-			}(),
-			want: EndpointsUpdate{
-				Drops: nil,
-				Localities: []Locality{
-					{
-						Endpoints: []Endpoint{{
-							Address:      "addr1:314",
-							HealthStatus: EndpointHealthStatusUnhealthy,
-							Weight:       271,
-						}},
-						ID:       internal.LocalityID{SubZone: "locality-1"},
-						Priority: 1,
-						Weight:   1,
-					},
-					{
-						Endpoints: []Endpoint{{
-							Address:      "addr2:159",
-							HealthStatus: EndpointHealthStatusDraining,
-							Weight:       828,
-						}},
-						ID:       internal.LocalityID{SubZone: "locality-2"},
-						Priority: 0,
-						Weight:   1,
-					},
-				},
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := ParseEDSRespProto(tt.m)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("ParseEDSRespProto() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if d := cmp.Diff(got, tt.want); d != "" {
-				t.Errorf("ParseEDSRespProto() got = %v, want %v, diff: %v", got, tt.want, d)
-			}
-		})
-	}
-}
-
 var (
-	badlyMarshaledEDSResponse = &xdspb.DiscoveryResponse{
+	badlyMarshaledEDSResponse = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: edsURL,
+				TypeUrl: version.V2EndpointsURL,
 				Value:   []byte{1, 2, 3, 4},
 			},
 		},
-		TypeUrl: edsURL,
+		TypeUrl: version.V2EndpointsURL,
 	}
-	badResourceTypeInEDSResponse = &xdspb.DiscoveryResponse{
+	badResourceTypeInEDSResponse = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
 				TypeUrl: httpConnManagerURL,
 				Value:   marshaledConnMgr1,
 			},
 		},
-		TypeUrl: edsURL,
+		TypeUrl: version.V2EndpointsURL,
 	}
-	goodEDSResponse1 = &xdspb.DiscoveryResponse{
+	goodEDSResponse1 = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			func() *anypb.Any {
-				clab0 := NewClusterLoadAssignmentBuilder(goodEDSName, nil)
+				clab0 := testutils.NewClusterLoadAssignmentBuilder(goodEDSName, nil)
 				clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil)
 				clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil)
 				a, _ := ptypes.MarshalAny(clab0.Build())
 				return a
 			}(),
 		},
-		TypeUrl: edsURL,
+		TypeUrl: version.V2EndpointsURL,
 	}
-	goodEDSResponse2 = &xdspb.DiscoveryResponse{
+	goodEDSResponse2 = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			func() *anypb.Any {
-				clab0 := NewClusterLoadAssignmentBuilder("not-goodEDSName", nil)
+				clab0 := testutils.NewClusterLoadAssignmentBuilder("not-goodEDSName", nil)
 				clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil)
 				clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil)
 				a, _ := ptypes.MarshalAny(clab0.Build())
 				return a
 			}(),
 		},
-		TypeUrl: edsURL,
+		TypeUrl: version.V2EndpointsURL,
 	}
 )
 
 func (s) TestEDSHandleResponse(t *testing.T) {
 	tests := []struct {
 		name          string
-		edsResponse   *xdspb.DiscoveryResponse
+		edsResponse   *v2xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *EndpointsUpdate
+		wantUpdate    *xdsclient.EndpointsUpdate
 		wantUpdateErr bool
 	}{
 		// Any in resource is badly marshaled.
@@ -195,16 +113,16 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 			name:        "one-good-assignment",
 			edsResponse: goodEDSResponse1,
 			wantErr:     false,
-			wantUpdate: &EndpointsUpdate{
-				Localities: []Locality{
+			wantUpdate: &xdsclient.EndpointsUpdate{
+				Localities: []xdsclient.Locality{
 					{
-						Endpoints: []Endpoint{{Address: "addr1:314"}},
+						Endpoints: []xdsclient.Endpoint{{Address: "addr1:314"}},
 						ID:        internal.LocalityID{SubZone: "locality-1"},
 						Priority:  1,
 						Weight:    1,
 					},
 					{
-						Endpoints: []Endpoint{{Address: "addr2:159"}},
+						Endpoints: []xdsclient.Endpoint{{Address: "addr2:159"}},
 						ID:        internal.LocalityID{SubZone: "locality-2"},
 						Priority:  0,
 						Weight:    1,
@@ -217,7 +135,7 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          edsURL,
+				typeURL:          version.V2EndpointsURL,
 				resourceName:     goodEDSName,
 				responseToHandle: test.edsResponse,
 				wantHandleErr:    test.wantErr,
@@ -234,10 +152,13 @@ func (s) TestEDSHandleResponseWithoutWatch(t *testing.T) {
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(string, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
 
 	if v2c.handleEDSResponse(badResourceTypeInEDSResponse) == nil {
 		t.Fatal("v2c.handleEDSResponse() succeeded, should have failed")
diff --git a/xds/internal/client/v2client_lds_test.go b/xds/internal/client/v2/client_lds_test.go
similarity index 55%
rename from xds/internal/client/v2client_lds_test.go
rename to xds/internal/client/v2/client_lds_test.go
index 893245d0ec54..ca8161f504f4 100644
--- a/xds/internal/client/v2client_lds_test.go
+++ b/xds/internal/client/v2/client_lds_test.go
@@ -16,114 +16,26 @@
  *
  */
 
-package client
+package v2
 
 import (
 	"testing"
 	"time"
 
-	"github.com/golang/protobuf/proto"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
-	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
-	anypb "github.com/golang/protobuf/ptypes/any"
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
-func (s) TestLDSGetRouteConfig(t *testing.T) {
-	tests := []struct {
-		name      string
-		lis       *xdspb.Listener
-		wantRoute string
-		wantErr   bool
-	}{
-		{
-			name:      "no-apiListener-field",
-			lis:       &xdspb.Listener{},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name:      "badly-marshaled-apiListener",
-			lis:       badAPIListener1,
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name:      "wrong-type-in-apiListener",
-			lis:       badResourceListener,
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name:      "empty-httpConnMgr-in-apiListener",
-			lis:       listenerWithEmptyHTTPConnMgr,
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name:      "scopedRoutes-routeConfig-in-apiListener",
-			lis:       listenerWithScopedRoutesRouteConfig,
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "rds.ConfigSource-in-apiListener-is-not-ADS",
-			lis: &xdspb.Listener{
-				Name: goodLDSTarget1,
-				ApiListener: &listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: httpConnManagerURL,
-						Value: func() []byte {
-							cm := &httppb.HttpConnectionManager{
-								RouteSpecifier: &httppb.HttpConnectionManager_Rds{
-									Rds: &httppb.Rds{
-										ConfigSource: &basepb.ConfigSource{
-											ConfigSourceSpecifier: &basepb.ConfigSource_Path{
-												Path: "/some/path",
-											},
-										},
-										RouteConfigName: goodRouteName1}}}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}()}}},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name:      "goodListener1",
-			lis:       goodListener1,
-			wantRoute: goodRouteName1,
-			wantErr:   false,
-		},
-	}
-	_, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-	v2c := newV2Client(nil, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			gotRoute, err := v2c.getRouteConfigNameFromListener(test.lis)
-			if gotRoute != test.wantRoute {
-				t.Errorf("getRouteConfigNameFromListener(%+v) = %v, want %v", test.lis, gotRoute, test.wantRoute)
-			}
-			if (err != nil) != test.wantErr {
-				t.Errorf("getRouteConfigNameFromListener(%+v) = %v, want %v", test.lis, err, test.wantErr)
-			}
-		})
-	}
-}
-
 // TestLDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
-// and creates a v2Client using it. Then, it registers a watchLDS and tests
+// and creates a client using it. Then, it registers a watchLDS and tests
 // different LDS responses.
 func (s) TestLDSHandleResponse(t *testing.T) {
 	tests := []struct {
 		name          string
-		ldsResponse   *xdspb.DiscoveryResponse
+		ldsResponse   *v2xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *ldsUpdate
+		wantUpdate    *xdsclient.ListenerUpdate
 		wantUpdateErr bool
 	}{
 		// Badly marshaled LDS response.
@@ -157,7 +69,7 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 			name:          "one-good-listener",
 			ldsResponse:   goodLDSResponse1,
 			wantErr:       false,
-			wantUpdate:    &ldsUpdate{routeName: goodRouteName1},
+			wantUpdate:    &xdsclient.ListenerUpdate{RouteConfigName: goodRouteName1},
 			wantUpdateErr: false,
 		},
 		// Response contains multiple good listeners, including the one we are
@@ -166,7 +78,7 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 			name:          "multiple-good-listener",
 			ldsResponse:   ldsResponseWithMultipleResources,
 			wantErr:       false,
-			wantUpdate:    &ldsUpdate{routeName: goodRouteName1},
+			wantUpdate:    &xdsclient.ListenerUpdate{RouteConfigName: goodRouteName1},
 			wantUpdateErr: false,
 		},
 		// Response contains two good listeners (one interesting and one
@@ -201,7 +113,7 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          ldsURL,
+				typeURL:          version.V2ListenerURL,
 				resourceName:     goodLDSTarget1,
 				responseToHandle: test.ldsResponse,
 				wantHandleErr:    test.wantErr,
@@ -212,16 +124,19 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 	}
 }
 
-// TestLDSHandleResponseWithoutWatch tests the case where the v2Client receives
+// TestLDSHandleResponseWithoutWatch tests the case where the client receives
 // an LDS response without a registered watcher.
 func (s) TestLDSHandleResponseWithoutWatch(t *testing.T) {
 	_, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
-	v2c := newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(string, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
 
 	if v2c.handleLDSResponse(badResourceTypeInLDSResponse) == nil {
 		t.Fatal("v2c.handleLDSResponse() succeeded, should have failed")
diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/client_rds_test.go
new file mode 100644
index 000000000000..aa44371ed350
--- /dev/null
+++ b/xds/internal/client/v2/client_rds_test.go
@@ -0,0 +1,167 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package v2
+
+import (
+	"testing"
+	"time"
+
+	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+// doLDS makes a LDS watch, and waits for the response and ack to finish.
+//
+// This is called by RDS tests to start LDS first, because LDS is a
+// pre-requirement for RDS, and RDS handle would fail without an existing LDS
+// watch.
+func doLDS(t *testing.T, v2c xdsclient.APIClient, fakeServer *fakeserver.Server) {
+	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout waiting for LDS request: %v", err)
+	}
+}
+
+// TestRDSHandleResponseWithRouting starts a fake xDS server, makes a ClientConn
+// to it, and creates a v2Client using it. Then, it registers an LDS and RDS
+// watcher and tests different RDS responses.
+func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
+	tests := []struct {
+		name          string
+		rdsResponse   *xdspb.DiscoveryResponse
+		wantErr       bool
+		wantUpdate    *xdsclient.RouteConfigUpdate
+		wantUpdateErr bool
+	}{
+		// Badly marshaled RDS response.
+		{
+			name:          "badly-marshaled-response",
+			rdsResponse:   badlyMarshaledRDSResponse,
+			wantErr:       true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response does not contain RouteConfiguration proto.
+		{
+			name:          "no-route-config-in-response",
+			rdsResponse:   badResourceTypeInRDSResponse,
+			wantErr:       true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// No VirtualHosts in the response. Just one test case here for a bad
+		// RouteConfiguration, since the others are covered in
+		// TestGetClusterFromRouteConfiguration.
+		{
+			name:          "no-virtual-hosts-in-response",
+			rdsResponse:   noVirtualHostsInRDSResponse,
+			wantErr:       true,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response contains one good RouteConfiguration, uninteresting though.
+		{
+			name:          "one-uninteresting-route-config",
+			rdsResponse:   goodRDSResponse2,
+			wantErr:       false,
+			wantUpdate:    nil,
+			wantUpdateErr: false,
+		},
+		// Response contains one good interesting RouteConfiguration.
+		{
+			name:          "one-good-route-config",
+			rdsResponse:   goodRDSResponse1,
+			wantErr:       false,
+			wantUpdate:    &xdsclient.RouteConfigUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}},
+			wantUpdateErr: false,
+		},
+		{
+			name:        "one-good-route-config with routes",
+			rdsResponse: goodRDSResponse1,
+			wantErr:     false,
+			wantUpdate: &xdsclient.RouteConfigUpdate{
+				// Instead of just weighted targets when routing is disabled,
+				// this result contains a route with perfix "", and action as
+				// weighted targets.
+				Routes: []*xdsclient.Route{{
+					Prefix: newStringP(""),
+					Action: map[string]uint32{goodClusterName1: 1},
+				}},
+			},
+			wantUpdateErr: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testWatchHandle(t, &watchHandleTestcase{
+				typeURL:          version.V2RouteConfigURL,
+				resourceName:     goodRouteName1,
+				responseToHandle: test.rdsResponse,
+				wantHandleErr:    test.wantErr,
+				wantUpdate:       test.wantUpdate,
+				wantUpdateErr:    test.wantUpdateErr,
+			})
+		})
+	}
+}
+
+// TestRDSHandleResponseWithoutLDSWatch tests the case where the v2Client
+// receives an RDS response without a registered LDS watcher.
+func (s) TestRDSHandleResponseWithoutLDSWatch(t *testing.T) {
+	_, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+
+	v2c, err := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
+
+	if v2c.handleRDSResponse(goodRDSResponse1) == nil {
+		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
+	}
+}
+
+// TestRDSHandleResponseWithoutRDSWatch tests the case where the v2Client
+// receives an RDS response without a registered RDS watcher.
+func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
+	fakeServer, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+
+	v2c, err := newV2Client(&testUpdateReceiver{
+		f: func(string, map[string]interface{}) {},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
+	doLDS(t, v2c, fakeServer)
+
+	if v2c.handleRDSResponse(badResourceTypeInRDSResponse) == nil {
+		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
+	}
+
+	if v2c.handleRDSResponse(goodRDSResponse1) != nil {
+		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
+	}
+}
diff --git a/xds/internal/client/v2client_test.go b/xds/internal/client/v2/client_test.go
similarity index 62%
rename from xds/internal/client/v2client_test.go
rename to xds/internal/client/v2/client_test.go
index 147558d5acc0..fa4a3c9c0ce5 100644
--- a/xds/internal/client/v2client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -16,19 +16,25 @@
  *
  */
 
-package client
+package v2
 
 import (
 	"errors"
+	"reflect"
 	"testing"
 	"time"
 
 	"github.com/golang/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/version"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
@@ -39,8 +45,15 @@ import (
 	structpb "github.com/golang/protobuf/ptypes/struct"
 )
 
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
 const (
-	defaultTestTimeout       = 1 * time.Second
 	goodLDSTarget1           = "lds.target.good:1111"
 	goodLDSTarget2           = "lds.target.good:2222"
 	goodRouteName1           = "GoodRouteConfig1"
@@ -67,22 +80,22 @@ var (
 	}
 	goodLDSRequest = &xdspb.DiscoveryRequest{
 		Node:          goodNodeProto,
-		TypeUrl:       ldsURL,
+		TypeUrl:       version.V2ListenerURL,
 		ResourceNames: []string{goodLDSTarget1},
 	}
 	goodRDSRequest = &xdspb.DiscoveryRequest{
 		Node:          goodNodeProto,
-		TypeUrl:       rdsURL,
+		TypeUrl:       version.V2RouteConfigURL,
 		ResourceNames: []string{goodRouteName1},
 	}
 	goodCDSRequest = &xdspb.DiscoveryRequest{
 		Node:          goodNodeProto,
-		TypeUrl:       cdsURL,
+		TypeUrl:       version.V2ClusterURL,
 		ResourceNames: []string{goodClusterName1},
 	}
 	goodEDSRequest = &xdspb.DiscoveryRequest{
 		Node:          goodNodeProto,
-		TypeUrl:       edsURL,
+		TypeUrl:       version.V2EndpointsURL,
 		ResourceNames: []string{goodEDSName},
 	}
 	goodHTTPConnManager1 = &httppb.HttpConnectionManager{
@@ -96,17 +109,7 @@ var (
 		},
 	}
 	marshaledConnMgr1, _ = proto.Marshal(goodHTTPConnManager1)
-	emptyHTTPConnManager = &httppb.HttpConnectionManager{
-		RouteSpecifier: &httppb.HttpConnectionManager_Rds{
-			Rds: &httppb.Rds{},
-		},
-	}
-	emptyMarshaledConnMgr, _ = proto.Marshal(emptyHTTPConnManager)
-	connMgrWithScopedRoutes  = &httppb.HttpConnectionManager{
-		RouteSpecifier: &httppb.HttpConnectionManager_ScopedRoutes{},
-	}
-	marshaledConnMgrWithScopedRoutes, _ = proto.Marshal(connMgrWithScopedRoutes)
-	goodListener1                       = &xdspb.Listener{
+	goodListener1        = &xdspb.Listener{
 		Name: goodLDSTarget1,
 		ApiListener: &listenerpb.ApiListener{
 			ApiListener: &anypb.Any{
@@ -128,16 +131,7 @@ var (
 	marshaledListener2, _     = proto.Marshal(goodListener2)
 	noAPIListener             = &xdspb.Listener{Name: goodLDSTarget1}
 	marshaledNoAPIListener, _ = proto.Marshal(noAPIListener)
-	badAPIListener1           = &xdspb.Listener{
-		Name: goodLDSTarget1,
-		ApiListener: &listenerpb.ApiListener{
-			ApiListener: &anypb.Any{
-				TypeUrl: httpConnManagerURL,
-				Value:   []byte{1, 2, 3, 4},
-			},
-		},
-	}
-	badAPIListener2 = &xdspb.Listener{
+	badAPIListener2           = &xdspb.Listener{
 		Name: goodLDSTarget2,
 		ApiListener: &listenerpb.ApiListener{
 			ApiListener: &anypb.Any{
@@ -147,60 +141,33 @@ var (
 		},
 	}
 	badlyMarshaledAPIListener2, _ = proto.Marshal(badAPIListener2)
-	badResourceListener           = &xdspb.Listener{
-		Name: goodLDSTarget1,
-		ApiListener: &listenerpb.ApiListener{
-			ApiListener: &anypb.Any{
-				TypeUrl: ldsURL,
-				Value:   marshaledListener1,
-			},
-		},
-	}
-	listenerWithEmptyHTTPConnMgr = &xdspb.Listener{
-		Name: goodLDSTarget1,
-		ApiListener: &listenerpb.ApiListener{
-			ApiListener: &anypb.Any{
-				TypeUrl: httpConnManagerURL,
-				Value:   emptyMarshaledConnMgr,
-			},
-		},
-	}
-	listenerWithScopedRoutesRouteConfig = &xdspb.Listener{
-		Name: goodLDSTarget1,
-		ApiListener: &listenerpb.ApiListener{
-			ApiListener: &anypb.Any{
-				TypeUrl: httpConnManagerURL,
-				Value:   marshaledConnMgrWithScopedRoutes,
-			},
-		},
-	}
-	goodLDSResponse1 = &xdspb.DiscoveryResponse{
+	goodLDSResponse1              = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledListener1,
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
 	goodLDSResponse2 = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledListener2,
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
-	emptyLDSResponse          = &xdspb.DiscoveryResponse{TypeUrl: ldsURL}
+	emptyLDSResponse          = &xdspb.DiscoveryResponse{TypeUrl: version.V2ListenerURL}
 	badlyMarshaledLDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   []byte{1, 2, 3, 4},
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
 	badResourceTypeInLDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
@@ -209,55 +176,55 @@ var (
 				Value:   marshaledConnMgr1,
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
 	ldsResponseWithMultipleResources = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledListener2,
 			},
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledListener1,
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
 	noAPIListenerLDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledNoAPIListener,
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
 	goodBadUglyLDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledListener2,
 			},
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   marshaledListener1,
 			},
 			{
-				TypeUrl: ldsURL,
+				TypeUrl: version.V2ListenerURL,
 				Value:   badlyMarshaledAPIListener2,
 			},
 		},
-		TypeUrl: ldsURL,
+		TypeUrl: version.V2ListenerURL,
 	}
 	badlyMarshaledRDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: rdsURL,
+				TypeUrl: version.V2RouteConfigURL,
 				Value:   []byte{1, 2, 3, 4},
 			},
 		},
-		TypeUrl: rdsURL,
+		TypeUrl: version.V2RouteConfigURL,
 	}
 	badResourceTypeInRDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
@@ -266,21 +233,18 @@ var (
 				Value:   marshaledConnMgr1,
 			},
 		},
-		TypeUrl: rdsURL,
+		TypeUrl: version.V2RouteConfigURL,
 	}
 	emptyRouteConfig             = &xdspb.RouteConfiguration{}
 	marshaledEmptyRouteConfig, _ = proto.Marshal(emptyRouteConfig)
-	noDomainsInRouteConfig       = &xdspb.RouteConfiguration{
-		VirtualHosts: []*routepb.VirtualHost{{}},
-	}
-	noVirtualHostsInRDSResponse = &xdspb.DiscoveryResponse{
+	noVirtualHostsInRDSResponse  = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: rdsURL,
+				TypeUrl: version.V2RouteConfigURL,
 				Value:   marshaledEmptyRouteConfig,
 			},
 		},
-		TypeUrl: rdsURL,
+		TypeUrl: version.V2RouteConfigURL,
 	}
 	goodRouteConfig1 = &xdspb.RouteConfiguration{
 		Name: goodRouteName1,
@@ -349,23 +313,201 @@ var (
 	goodRDSResponse1             = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: rdsURL,
+				TypeUrl: version.V2RouteConfigURL,
 				Value:   marshaledGoodRouteConfig1,
 			},
 		},
-		TypeUrl: rdsURL,
+		TypeUrl: version.V2RouteConfigURL,
 	}
 	goodRDSResponse2 = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
-				TypeUrl: rdsURL,
+				TypeUrl: version.V2RouteConfigURL,
 				Value:   marshaledGoodRouteConfig2,
 			},
 		},
-		TypeUrl: rdsURL,
+		TypeUrl: version.V2RouteConfigURL,
 	}
 )
 
+type watchHandleTestcase struct {
+	typeURL      string
+	resourceName string
+
+	responseToHandle *xdspb.DiscoveryResponse
+	wantHandleErr    bool
+	wantUpdate       interface{}
+	wantUpdateErr    bool
+}
+
+type testUpdateReceiver struct {
+	f func(typeURL string, d map[string]interface{})
+}
+
+func (t *testUpdateReceiver) NewListeners(d map[string]xdsclient.ListenerUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(version.V2ListenerURL, dd)
+}
+
+func (t *testUpdateReceiver) NewRouteConfigs(d map[string]xdsclient.RouteConfigUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(version.V2RouteConfigURL, dd)
+}
+
+func (t *testUpdateReceiver) NewClusters(d map[string]xdsclient.ClusterUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(version.V2ClusterURL, dd)
+}
+
+func (t *testUpdateReceiver) NewEndpoints(d map[string]xdsclient.EndpointsUpdate) {
+	dd := make(map[string]interface{})
+	for k, v := range d {
+		dd[k] = v
+	}
+	t.newUpdate(version.V2EndpointsURL, dd)
+}
+
+func (t *testUpdateReceiver) newUpdate(typeURL string, d map[string]interface{}) {
+	t.f(typeURL, d)
+}
+
+// testWatchHandle is called to test response handling for each xDS.
+//
+// It starts the xDS watch as configured in test, waits for the fake xds server
+// to receive the request (so watch callback is installed), and calls
+// handleXDSResp with responseToHandle (if it's set). It then compares the
+// update received by watch callback with the expected results.
+func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
+	fakeServer, cc, cleanup := startServerAndGetCC(t)
+	defer cleanup()
+
+	type updateErr struct {
+		u   interface{}
+		err error
+	}
+	gotUpdateCh := testutils.NewChannel()
+
+	v2c, err := newV2Client(&testUpdateReceiver{
+		f: func(typeURL string, d map[string]interface{}) {
+			if typeURL == test.typeURL {
+				if u, ok := d[test.resourceName]; ok {
+					gotUpdateCh.Send(updateErr{u, nil})
+				}
+			}
+		},
+	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
+
+	// RDS needs an existin LDS watch for the hostname.
+	if test.typeURL == version.V2RouteConfigURL {
+		doLDS(t, v2c, fakeServer)
+	}
+
+	// Register the watcher, this will also trigger the v2Client to send the xDS
+	// request.
+	v2c.AddWatch(test.typeURL, test.resourceName)
+
+	// Wait till the request makes it to the fakeServer. This ensures that
+	// the watch request has been processed by the v2Client.
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout waiting for an xDS request: %v", err)
+	}
+
+	// Directly push the response through a call to handleXDSResp. This bypasses
+	// the fakeServer, so it's only testing the handle logic. Client response
+	// processing is covered elsewhere.
+	//
+	// Also note that this won't trigger ACK, so there's no need to clear the
+	// request channel afterwards.
+	var handleXDSResp func(response *xdspb.DiscoveryResponse) error
+	switch test.typeURL {
+	case version.V2ListenerURL:
+		handleXDSResp = v2c.handleLDSResponse
+	case version.V2RouteConfigURL:
+		handleXDSResp = v2c.handleRDSResponse
+	case version.V2ClusterURL:
+		handleXDSResp = v2c.handleCDSResponse
+	case version.V2EndpointsURL:
+		handleXDSResp = v2c.handleEDSResponse
+	}
+	if err := handleXDSResp(test.responseToHandle); (err != nil) != test.wantHandleErr {
+		t.Fatalf("v2c.handleRDSResponse() returned err: %v, wantErr: %v", err, test.wantHandleErr)
+	}
+
+	// If the test doesn't expect the callback to be invoked, verify that no
+	// update or error is pushed to the callback.
+	//
+	// Cannot directly compare test.wantUpdate with nil (typed vs non-typed nil:
+	// https://golang.org/doc/faq#nil_error).
+	if c := test.wantUpdate; c == nil || (reflect.ValueOf(c).Kind() == reflect.Ptr && reflect.ValueOf(c).IsNil()) {
+		update, err := gotUpdateCh.Receive()
+		if err == testutils.ErrRecvTimeout {
+			return
+		}
+		t.Fatalf("Unexpected update: +%v", update)
+	}
+
+	wantUpdate := reflect.ValueOf(test.wantUpdate).Elem().Interface()
+	uErr, err := gotUpdateCh.Receive()
+	if err == testutils.ErrRecvTimeout {
+		t.Fatal("Timeout expecting xDS update")
+	}
+	gotUpdate := uErr.(updateErr).u
+	if diff := cmp.Diff(gotUpdate, wantUpdate); diff != "" {
+		t.Fatalf("got update : %+v, want %+v, diff: %s", gotUpdate, wantUpdate, diff)
+	}
+	gotUpdateErr := uErr.(updateErr).err
+	if (gotUpdateErr != nil) != test.wantUpdateErr {
+		t.Fatalf("got xDS update error {%v}, wantErr: %v", gotUpdateErr, test.wantUpdateErr)
+	}
+}
+
+// startServerAndGetCC starts a fake XDS server and also returns a ClientConn
+// connected to it.
+func startServerAndGetCC(t *testing.T) (*fakeserver.Server, *grpc.ClientConn, func()) {
+	t.Helper()
+
+	fs, sCleanup, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("Failed to start fake xDS server: %v", err)
+	}
+
+	cc, ccCleanup, err := fs.XDSClientConn()
+	if err != nil {
+		sCleanup()
+		t.Fatalf("Failed to get a clientConn to the fake xDS server: %v", err)
+	}
+	return fs, cc, func() {
+		sCleanup()
+		ccCleanup()
+	}
+}
+
+func newV2Client(p xdsclient.UpdateHandler, cc *grpc.ClientConn, n *basepb.Node, b func(int) time.Duration, l *grpclog.PrefixLogger) (*client, error) {
+	c, err := newClient(cc, xdsclient.BuildOptions{
+		Parent:    p,
+		NodeProto: n,
+		Backoff:   b,
+		Logger:    l,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return c.(*client), nil
+}
+
 // TestV2ClientBackoffAfterRecvError verifies if the v2Client backoffs when it
 // encounters a Recv error while receiving an LDS response.
 func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
@@ -381,14 +523,17 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	}
 
 	callbackCh := make(chan struct{})
-	v2c := newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(string, map[string]interface{}) { close(callbackCh) },
 	}, cc, goodNodeProto, clientBackoff, nil)
-	defer v2c.close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
 	t.Log("Started xds v2Client...")
 
 	// v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {})
-	v2c.addWatch(ldsURL, goodLDSTarget1)
+	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
@@ -397,7 +542,7 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")}
 	t.Log("Bad LDS response pushed to fakeServer...")
 
-	timer := time.NewTimer(defaultTestTimeout)
+	timer := time.NewTimer(1 * time.Second)
 	select {
 	case <-timer.C:
 		t.Fatal("Timeout when expecting LDS update")
@@ -417,9 +562,9 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	defer cleanup()
 
 	callbackCh := testutils.NewChannel()
-	v2c := newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(typeURL string, d map[string]interface{}) {
-			if typeURL == ldsURL {
+			if typeURL == version.V2ListenerURL {
 				if u, ok := d[goodLDSTarget1]; ok {
 					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
 					callbackCh.Send(struct{}{})
@@ -427,10 +572,13 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 			}
 		},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
 	t.Log("Started xds v2Client...")
 
-	v2c.addWatch(ldsURL, goodLDSTarget1)
+	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
@@ -467,12 +615,6 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 // level). And when the stream is re-created, the watcher should get future
 // updates.
 func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	fakeServer, sCleanup, err := fakeserver.StartServer()
 	if err != nil {
 		t.Fatalf("Failed to start fake xDS server: %v", err)
@@ -490,9 +632,9 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	defer cc.Close()
 
 	callbackCh := testutils.NewChannel()
-	v2c := newV2Client(&testUpdateReceiver{
+	v2c, err := newV2Client(&testUpdateReceiver{
 		f: func(typeURL string, d map[string]interface{}) {
-			if typeURL == ldsURL {
+			if typeURL == version.V2ListenerURL {
 				if u, ok := d[goodLDSTarget1]; ok {
 					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
 					callbackCh.Send(u)
@@ -500,12 +642,15 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 			}
 		},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer v2c.Close()
 	t.Log("Started xds v2Client...")
 
 	// This watch is started when the xds-ClientConn is in Transient Failure,
 	// and no xds stream is created.
-	v2c.addWatch(ldsURL, goodLDSTarget1)
+	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
 
 	// The watcher should receive an update, with a timeout error in it.
 	if v, err := callbackCh.TimedReceive(100 * time.Millisecond); err == nil {
@@ -528,7 +673,11 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 
 	if v, err := callbackCh.Receive(); err != nil {
 		t.Fatal("Timeout when expecting LDS update")
-	} else if _, ok := v.(ldsUpdate); !ok {
+	} else if _, ok := v.(xdsclient.ListenerUpdate); !ok {
 		t.Fatalf("Expect an LDS update from watcher, got %v", v)
 	}
 }
+
+func newStringP(s string) *string {
+	return &s
+}
diff --git a/xds/internal/client/v2client_cds.go b/xds/internal/client/v2client_cds.go
deleted file mode 100644
index 6fc9ae86d098..000000000000
--- a/xds/internal/client/v2client_cds.go
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"fmt"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	"github.com/golang/protobuf/ptypes"
-)
-
-// handleCDSResponse processes an CDS response received from the xDS server. On
-// receipt of a good response, it also invokes the registered watcher callback.
-func (v2c *v2Client) handleCDSResponse(resp *xdspb.DiscoveryResponse) error {
-	returnUpdate := make(map[string]ClusterUpdate)
-	for _, r := range resp.GetResources() {
-		var resource ptypes.DynamicAny
-		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
-			return fmt.Errorf("xds: failed to unmarshal resource in CDS response: %v", err)
-		}
-		cluster, ok := resource.Message.(*xdspb.Cluster)
-		if !ok {
-			return fmt.Errorf("xds: unexpected resource type: %T in CDS response", resource.Message)
-		}
-		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
-		update, err := validateCluster(cluster)
-		if err != nil {
-			return err
-		}
-
-		// If the Cluster message in the CDS response did not contain a
-		// serviceName, we will just use the clusterName for EDS.
-		if update.ServiceName == "" {
-			update.ServiceName = cluster.GetName()
-		}
-		v2c.logger.Debugf("Resource with name %v, type %T, value %+v added to cache", cluster.GetName(), update, update)
-		returnUpdate[cluster.GetName()] = update
-	}
-
-	v2c.parent.newCDSUpdate(returnUpdate)
-	return nil
-}
-
-func validateCluster(cluster *xdspb.Cluster) (ClusterUpdate, error) {
-	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
-	switch {
-	case cluster.GetType() != xdspb.Cluster_EDS:
-		return emptyUpdate, fmt.Errorf("xds: unexpected cluster type %v in response: %+v", cluster.GetType(), cluster)
-	case cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil:
-		return emptyUpdate, fmt.Errorf("xds: unexpected edsConfig in response: %+v", cluster)
-	case cluster.GetLbPolicy() != xdspb.Cluster_ROUND_ROBIN:
-		return emptyUpdate, fmt.Errorf("xds: unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
-	}
-
-	return ClusterUpdate{
-		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
-		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
-	}, nil
-}
diff --git a/xds/internal/client/v2client_eds_testutil.go b/xds/internal/client/v2client_eds_testutil.go
deleted file mode 100644
index 2d187a510365..000000000000
--- a/xds/internal/client/v2client_eds_testutil.go
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// All structs/functions in this file should be unexported. They are used in EDS
-// balancer tests now, to generate test inputs. Eventually, EDS balancer tests
-// should generate EndpointsUpdate directly, instead of generating and parsing the
-// proto message.
-// TODO: unexported everything in this file.
-
-package client
-
-import (
-	"fmt"
-	"net"
-	"strconv"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
-	typepb "github.com/envoyproxy/go-control-plane/envoy/type"
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
-)
-
-// ClusterLoadAssignmentBuilder builds a ClusterLoadAssignment, aka EDS
-// response.
-type ClusterLoadAssignmentBuilder struct {
-	v *xdspb.ClusterLoadAssignment
-}
-
-// NewClusterLoadAssignmentBuilder creates a ClusterLoadAssignmentBuilder.
-func NewClusterLoadAssignmentBuilder(clusterName string, dropPercents []uint32) *ClusterLoadAssignmentBuilder {
-	var drops []*xdspb.ClusterLoadAssignment_Policy_DropOverload
-	for i, d := range dropPercents {
-		drops = append(drops, &xdspb.ClusterLoadAssignment_Policy_DropOverload{
-			Category: fmt.Sprintf("test-drop-%d", i),
-			DropPercentage: &typepb.FractionalPercent{
-				Numerator:   d,
-				Denominator: typepb.FractionalPercent_HUNDRED,
-			},
-		})
-	}
-
-	return &ClusterLoadAssignmentBuilder{
-		v: &xdspb.ClusterLoadAssignment{
-			ClusterName: clusterName,
-			Policy: &xdspb.ClusterLoadAssignment_Policy{
-				DropOverloads: drops,
-			},
-		},
-	}
-}
-
-// AddLocalityOptions contains options when adding locality to the builder.
-type AddLocalityOptions struct {
-	Health []corepb.HealthStatus
-	Weight []uint32
-}
-
-// AddLocality adds a locality to the builder.
-func (clab *ClusterLoadAssignmentBuilder) AddLocality(subzone string, weight uint32, priority uint32, addrsWithPort []string, opts *AddLocalityOptions) {
-	var lbEndPoints []*endpointpb.LbEndpoint
-	for i, a := range addrsWithPort {
-		host, portStr, err := net.SplitHostPort(a)
-		if err != nil {
-			panic("failed to split " + a)
-		}
-		port, err := strconv.Atoi(portStr)
-		if err != nil {
-			panic("failed to atoi " + portStr)
-		}
-
-		lbe := &endpointpb.LbEndpoint{
-			HostIdentifier: &endpointpb.LbEndpoint_Endpoint{
-				Endpoint: &endpointpb.Endpoint{
-					Address: &corepb.Address{
-						Address: &corepb.Address_SocketAddress{
-							SocketAddress: &corepb.SocketAddress{
-								Protocol: corepb.SocketAddress_TCP,
-								Address:  host,
-								PortSpecifier: &corepb.SocketAddress_PortValue{
-									PortValue: uint32(port)}}}}}},
-		}
-		if opts != nil {
-			if i < len(opts.Health) {
-				lbe.HealthStatus = opts.Health[i]
-			}
-			if i < len(opts.Weight) {
-				lbe.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: opts.Weight[i]}
-			}
-		}
-		lbEndPoints = append(lbEndPoints, lbe)
-	}
-
-	var localityID *corepb.Locality
-	if subzone != "" {
-		localityID = &corepb.Locality{
-			Region:  "",
-			Zone:    "",
-			SubZone: subzone,
-		}
-	}
-
-	clab.v.Endpoints = append(clab.v.Endpoints, &endpointpb.LocalityLbEndpoints{
-		Locality:            localityID,
-		LbEndpoints:         lbEndPoints,
-		LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight},
-		Priority:            priority,
-	})
-}
-
-// Build builds ClusterLoadAssignment.
-func (clab *ClusterLoadAssignmentBuilder) Build() *xdspb.ClusterLoadAssignment {
-	return clab.v
-}
diff --git a/xds/internal/client/v2client_lds.go b/xds/internal/client/v2client_lds.go
deleted file mode 100644
index 05eddb5e093d..000000000000
--- a/xds/internal/client/v2client_lds.go
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"fmt"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
-	"github.com/golang/protobuf/ptypes"
-)
-
-// handleLDSResponse processes an LDS response received from the xDS server. On
-// receipt of a good response, it also invokes the registered watcher callback.
-func (v2c *v2Client) handleLDSResponse(resp *xdspb.DiscoveryResponse) error {
-	returnUpdate := make(map[string]ldsUpdate)
-	for _, r := range resp.GetResources() {
-		var resource ptypes.DynamicAny
-		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
-			return fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
-		}
-		lis, ok := resource.Message.(*xdspb.Listener)
-		if !ok {
-			return fmt.Errorf("xds: unexpected resource type: %T in LDS response", resource.Message)
-		}
-		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
-		routeName, err := v2c.getRouteConfigNameFromListener(lis)
-		if err != nil {
-			return err
-		}
-		returnUpdate[lis.GetName()] = ldsUpdate{routeName: routeName}
-	}
-
-	v2c.parent.newLDSUpdate(returnUpdate)
-	return nil
-}
-
-// getRouteConfigNameFromListener checks if the provided Listener proto meets
-// the expected criteria. If so, it returns a non-empty routeConfigName.
-func (v2c *v2Client) getRouteConfigNameFromListener(lis *xdspb.Listener) (string, error) {
-	if lis.GetApiListener() == nil {
-		return "", fmt.Errorf("xds: no api_listener field in LDS response %+v", lis)
-	}
-	var apiAny ptypes.DynamicAny
-	if err := ptypes.UnmarshalAny(lis.GetApiListener().GetApiListener(), &apiAny); err != nil {
-		return "", fmt.Errorf("xds: failed to unmarshal api_listner in LDS response: %v", err)
-	}
-	apiLis, ok := apiAny.Message.(*httppb.HttpConnectionManager)
-	if !ok {
-		return "", fmt.Errorf("xds: unexpected api_listener type: %T in LDS response", apiAny.Message)
-	}
-	v2c.logger.Infof("Resource with type %T, contains %v", apiLis, apiLis)
-	switch apiLis.RouteSpecifier.(type) {
-	case *httppb.HttpConnectionManager_Rds:
-		if apiLis.GetRds().GetConfigSource().GetAds() == nil {
-			return "", fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis)
-		}
-		name := apiLis.GetRds().GetRouteConfigName()
-		if name == "" {
-			return "", fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
-		}
-		return name, nil
-	case *httppb.HttpConnectionManager_RouteConfig:
-		// TODO: Add support for specifying the RouteConfiguration inline
-		// in the LDS response.
-		return "", fmt.Errorf("xds: LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
-	case nil:
-		return "", fmt.Errorf("xds: no RouteSpecifier in received LDS response: %+v", apiLis)
-	default:
-		return "", fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
-	}
-}
diff --git a/xds/internal/client/v2client_rds.go b/xds/internal/client/v2client_rds.go
deleted file mode 100644
index cc71f6538f62..000000000000
--- a/xds/internal/client/v2client_rds.go
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"fmt"
-	"strings"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
-	typepb "github.com/envoyproxy/go-control-plane/envoy/type"
-	"github.com/golang/protobuf/ptypes"
-	"google.golang.org/grpc/internal/grpclog"
-)
-
-// handleRDSResponse processes an RDS response received from the xDS server. On
-// receipt of a good response, it caches validated resources and also invokes
-// the registered watcher callback.
-func (v2c *v2Client) handleRDSResponse(resp *xdspb.DiscoveryResponse) error {
-	v2c.mu.Lock()
-	hostname := v2c.hostname
-	v2c.mu.Unlock()
-
-	returnUpdate := make(map[string]rdsUpdate)
-	for _, r := range resp.GetResources() {
-		var resource ptypes.DynamicAny
-		if err := ptypes.UnmarshalAny(r, &resource); err != nil {
-			return fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err)
-		}
-		rc, ok := resource.Message.(*xdspb.RouteConfiguration)
-		if !ok {
-			return fmt.Errorf("xds: unexpected resource type: %T in RDS response", resource.Message)
-		}
-		v2c.logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, v2c.hostname)
-
-		// Use the hostname (resourceName for LDS) to find the routes.
-		u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname, v2c.logger)
-		if err != nil {
-			return fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
-		}
-		// If we get here, it means that this resource was a good one.
-		returnUpdate[rc.GetName()] = u
-	}
-
-	v2c.parent.newRDSUpdate(returnUpdate)
-	return nil
-}
-
-// generateRDSUpdateFromRouteConfiguration checks if the provided
-// RouteConfiguration meets the expected criteria. If so, it returns a rdsUpdate
-// with nil error.
-//
-// A RouteConfiguration resource is considered valid when only if it contains a
-// VirtualHost whose domain field matches the server name from the URI passed
-// to the gRPC channel, and it contains a clusterName or a weighted cluster.
-//
-// The RouteConfiguration includes a list of VirtualHosts, which may have zero
-// or more elements. We are interested in the element whose domains field
-// matches the server name specified in the "xds:" URI. The only field in the
-// VirtualHost proto that the we are interested in is the list of routes. We
-// only look at the last route in the list (the default route), whose match
-// field must be empty and whose route field must be set.  Inside that route
-// message, the cluster field will contain the clusterName or weighted clusters
-// we are looking for.
-func generateRDSUpdateFromRouteConfiguration(rc *xdspb.RouteConfiguration, host string, logger *grpclog.PrefixLogger) (rdsUpdate, error) {
-	//
-	// Currently this returns "" on error, and the caller will return an error.
-	// But the error doesn't contain details of why the response is invalid
-	// (mismatch domain or empty route).
-	//
-	// For logging purposes, we can log in line. But if we want to populate
-	// error details for nack, a detailed error needs to be returned.
-	vh := findBestMatchingVirtualHost(host, rc.GetVirtualHosts())
-	if vh == nil {
-		// No matching virtual host found.
-		return rdsUpdate{}, fmt.Errorf("no matching virtual host found")
-	}
-	if len(vh.Routes) == 0 {
-		// The matched virtual host has no routes, this is invalid because there
-		// should be at least one default route.
-		return rdsUpdate{}, fmt.Errorf("matched virtual host has no routes")
-	}
-
-	routes, err := routesProtoToSlice(vh.Routes, logger)
-	if err != nil {
-		return rdsUpdate{}, fmt.Errorf("received route is invalid: %v", err)
-	}
-	return rdsUpdate{routes: routes}, nil
-}
-
-func routesProtoToSlice(routes []*routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) {
-	var routesRet []*Route
-
-	for _, r := range routes {
-		match := r.GetMatch()
-		if match == nil {
-			return nil, fmt.Errorf("route %+v doesn't have a match", r)
-		}
-
-		if len(match.GetQueryParameters()) != 0 {
-			// Ignore route with query parameters.
-			logger.Warningf("route %+v has query parameter matchers, the route will be ignored", r)
-			continue
-		}
-
-		if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
-			return nil, fmt.Errorf("route %+v has case-sensitive false", r)
-		}
-
-		pathSp := match.GetPathSpecifier()
-		if pathSp == nil {
-			return nil, fmt.Errorf("route %+v doesn't have a path specifier", r)
-		}
-
-		var route Route
-		switch pt := pathSp.(type) {
-		case *routepb.RouteMatch_Prefix:
-			route.Prefix = &pt.Prefix
-		case *routepb.RouteMatch_Path:
-			route.Path = &pt.Path
-		case *routepb.RouteMatch_SafeRegex:
-			route.Regex = &pt.SafeRegex.Regex
-		case *routepb.RouteMatch_Regex:
-			return nil, fmt.Errorf("route %+v has Regex, expected SafeRegex instead", r)
-		default:
-			logger.Warningf("route %+v has an unrecognized path specifier: %+v", r, pt)
-			continue
-		}
-
-		for _, h := range match.GetHeaders() {
-			var header HeaderMatcher
-			switch ht := h.GetHeaderMatchSpecifier().(type) {
-			case *routepb.HeaderMatcher_ExactMatch:
-				header.ExactMatch = &ht.ExactMatch
-			case *routepb.HeaderMatcher_SafeRegexMatch:
-				header.RegexMatch = &ht.SafeRegexMatch.Regex
-			case *routepb.HeaderMatcher_RangeMatch:
-				header.RangeMatch = &Int64Range{
-					Start: ht.RangeMatch.Start,
-					End:   ht.RangeMatch.End,
-				}
-			case *routepb.HeaderMatcher_PresentMatch:
-				header.PresentMatch = &ht.PresentMatch
-			case *routepb.HeaderMatcher_PrefixMatch:
-				header.PrefixMatch = &ht.PrefixMatch
-			case *routepb.HeaderMatcher_SuffixMatch:
-				header.SuffixMatch = &ht.SuffixMatch
-			case *routepb.HeaderMatcher_RegexMatch:
-				return nil, fmt.Errorf("route %+v has a header matcher with Regex, expected SafeRegex instead", r)
-			default:
-				logger.Warningf("route %+v has an unrecognized header matcher: %+v", r, ht)
-				continue
-			}
-			header.Name = h.GetName()
-			invert := h.GetInvertMatch()
-			header.InvertMatch = &invert
-			route.Headers = append(route.Headers, &header)
-		}
-
-		if fr := match.GetRuntimeFraction(); fr != nil {
-			d := fr.GetDefaultValue()
-			n := d.GetNumerator()
-			switch d.GetDenominator() {
-			case typepb.FractionalPercent_HUNDRED:
-				n *= 10000
-			case typepb.FractionalPercent_TEN_THOUSAND:
-				n *= 100
-			case typepb.FractionalPercent_MILLION:
-			}
-			route.Fraction = &n
-		}
-
-		clusters := make(map[string]uint32)
-		switch a := r.GetRoute().GetClusterSpecifier().(type) {
-		case *routepb.RouteAction_Cluster:
-			clusters[a.Cluster] = 1
-		case *routepb.RouteAction_WeightedClusters:
-			wcs := a.WeightedClusters
-			var totalWeight uint32
-			for _, c := range wcs.Clusters {
-				w := c.GetWeight().GetValue()
-				clusters[c.GetName()] = w
-				totalWeight += w
-			}
-			if totalWeight != wcs.GetTotalWeight().GetValue() {
-				return nil, fmt.Errorf("route %+v, action %+v, weights of clusters do not add up to total total weight, got: %v, want %v", r, a, wcs.GetTotalWeight().GetValue(), totalWeight)
-			}
-		case *routepb.RouteAction_ClusterHeader:
-			continue
-		}
-
-		route.Action = clusters
-		routesRet = append(routesRet, &route)
-	}
-	return routesRet, nil
-}
-
-func weightedClustersProtoToMap(wc *routepb.WeightedCluster) (map[string]uint32, error) {
-	ret := make(map[string]uint32)
-	var totalWeight uint32 = 100
-	if t := wc.GetTotalWeight().GetValue(); t != 0 {
-		totalWeight = t
-	}
-	for _, cw := range wc.Clusters {
-		w := cw.Weight.GetValue()
-		ret[cw.Name] = w
-		totalWeight -= w
-	}
-	if totalWeight != 0 {
-		return nil, fmt.Errorf("weights of clusters do not add up to total total weight, difference: %v", totalWeight)
-	}
-	return ret, nil
-}
-
-type domainMatchType int
-
-const (
-	domainMatchTypeInvalid domainMatchType = iota
-	domainMatchTypeUniversal
-	domainMatchTypePrefix
-	domainMatchTypeSuffix
-	domainMatchTypeExact
-)
-
-// Exact > Suffix > Prefix > Universal > Invalid.
-func (t domainMatchType) betterThan(b domainMatchType) bool {
-	return t > b
-}
-
-func matchTypeForDomain(d string) domainMatchType {
-	if d == "" {
-		return domainMatchTypeInvalid
-	}
-	if d == "*" {
-		return domainMatchTypeUniversal
-	}
-	if strings.HasPrefix(d, "*") {
-		return domainMatchTypeSuffix
-	}
-	if strings.HasSuffix(d, "*") {
-		return domainMatchTypePrefix
-	}
-	if strings.Contains(d, "*") {
-		return domainMatchTypeInvalid
-	}
-	return domainMatchTypeExact
-}
-
-func match(domain, host string) (domainMatchType, bool) {
-	switch typ := matchTypeForDomain(domain); typ {
-	case domainMatchTypeInvalid:
-		return typ, false
-	case domainMatchTypeUniversal:
-		return typ, true
-	case domainMatchTypePrefix:
-		// abc.*
-		return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*"))
-	case domainMatchTypeSuffix:
-		// *.123
-		return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*"))
-	case domainMatchTypeExact:
-		return typ, domain == host
-	default:
-		return domainMatchTypeInvalid, false
-	}
-}
-
-// findBestMatchingVirtualHost returns the virtual host whose domains field best
-// matches host
-//
-// The domains field support 4 different matching pattern types:
-//  - Exact match
-//  - Suffix match (e.g. “*ABC”)
-//  - Prefix match (e.g. “ABC*)
-//  - Universal match (e.g. “*”)
-//
-// The best match is defined as:
-//  - A match is better if it’s matching pattern type is better
-//    - Exact match > suffix match > prefix match > universal match
-//  - If two matches are of the same pattern type, the longer match is better
-//    - This is to compare the length of the matching pattern, e.g. “*ABCDE” >
-//    “*ABC”
-func findBestMatchingVirtualHost(host string, vHosts []*routepb.VirtualHost) *routepb.VirtualHost {
-	var (
-		matchVh   *routepb.VirtualHost
-		matchType = domainMatchTypeInvalid
-		matchLen  int
-	)
-	for _, vh := range vHosts {
-		for _, domain := range vh.GetDomains() {
-			typ, matched := match(domain, host)
-			if typ == domainMatchTypeInvalid {
-				// The rds response is invalid.
-				return nil
-			}
-			if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched {
-				// The previous match has better type, or the previous match has
-				// better length, or this domain isn't a match.
-				continue
-			}
-			matchVh = vh
-			matchType = typ
-			matchLen = len(domain)
-		}
-	}
-	return matchVh
-}
diff --git a/xds/internal/client/v2client_rds_test.go b/xds/internal/client/v2client_rds_test.go
deleted file mode 100644
index 3e5ef96fd6ba..000000000000
--- a/xds/internal/client/v2client_rds_test.go
+++ /dev/null
@@ -1,699 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"testing"
-	"time"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
-	typepb "github.com/envoyproxy/go-control-plane/envoy/type"
-	"github.com/golang/protobuf/proto"
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-)
-
-func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
-	tests := []struct {
-		name       string
-		rc         *xdspb.RouteConfiguration
-		wantUpdate rdsUpdate
-		wantError  bool
-	}{
-		{
-			name:      "no-virtual-hosts-in-rc",
-			rc:        emptyRouteConfig,
-			wantError: true,
-		},
-		{
-			name:      "no-domains-in-rc",
-			rc:        noDomainsInRouteConfig,
-			wantError: true,
-		},
-		{
-			name: "non-matching-domain-in-rc",
-			rc: &xdspb.RouteConfiguration{
-				VirtualHosts: []*routepb.VirtualHost{
-					{Domains: []string{uninterestingDomain}},
-				},
-			},
-			wantError: true,
-		},
-		{
-			name: "no-routes-in-rc",
-			rc: &xdspb.RouteConfiguration{
-				VirtualHosts: []*routepb.VirtualHost{
-					{Domains: []string{goodLDSTarget1}},
-				},
-			},
-			wantError: true,
-		},
-		{
-			name: "default-route-match-field-is-nil",
-			rc: &xdspb.RouteConfiguration{
-				VirtualHosts: []*routepb.VirtualHost{
-					{
-						Domains: []string{goodLDSTarget1},
-						Routes: []*routepb.Route{
-							{
-								Action: &routepb.Route_Route{
-									Route: &routepb.RouteAction{
-										ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			wantError: true,
-		},
-		{
-			name: "default-route-match-field-is-non-nil",
-			rc: &xdspb.RouteConfiguration{
-				VirtualHosts: []*routepb.VirtualHost{
-					{
-						Domains: []string{goodLDSTarget1},
-						Routes: []*routepb.Route{
-							{
-								Match:  &routepb.RouteMatch{},
-								Action: &routepb.Route_Route{},
-							},
-						},
-					},
-				},
-			},
-			wantError: true,
-		},
-		{
-			name: "default-route-routeaction-field-is-nil",
-			rc: &xdspb.RouteConfiguration{
-				VirtualHosts: []*routepb.VirtualHost{
-					{
-						Domains: []string{goodLDSTarget1},
-						Routes:  []*routepb.Route{{}},
-					},
-				},
-			},
-			wantError: true,
-		},
-		{
-			name: "default-route-cluster-field-is-empty",
-			rc: &xdspb.RouteConfiguration{
-				VirtualHosts: []*routepb.VirtualHost{
-					{
-						Domains: []string{goodLDSTarget1},
-						Routes: []*routepb.Route{
-							{
-								Action: &routepb.Route_Route{
-									Route: &routepb.RouteAction{
-										ClusterSpecifier: &routepb.RouteAction_ClusterHeader{},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			wantError: true,
-		},
-		{
-			// default route's match sets case-sensitive to false.
-			name: "good-route-config-but-with-casesensitive-false",
-			rc: &xdspb.RouteConfiguration{
-				Name: goodRouteName1,
-				VirtualHosts: []*routepb.VirtualHost{{
-					Domains: []string{goodLDSTarget1},
-					Routes: []*routepb.Route{{
-						Match: &routepb.RouteMatch{
-							PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"},
-							CaseSensitive: &wrapperspb.BoolValue{Value: false},
-						},
-						Action: &routepb.Route_Route{
-							Route: &routepb.RouteAction{
-								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
-							}}}}}}},
-			wantError: true,
-		},
-		{
-			name:       "good-route-config-with-empty-string-route",
-			rc:         goodRouteConfig1,
-			wantUpdate: rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}},
-		},
-		{
-			// default route's match is not empty string, but "/".
-			name: "good-route-config-with-slash-string-route",
-			rc: &xdspb.RouteConfiguration{
-				Name: goodRouteName1,
-				VirtualHosts: []*routepb.VirtualHost{{
-					Domains: []string{goodLDSTarget1},
-					Routes: []*routepb.Route{{
-						Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"}},
-						Action: &routepb.Route_Route{
-							Route: &routepb.RouteAction{
-								ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1},
-							}}}}}}},
-			wantUpdate: rdsUpdate{routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{goodClusterName1: 1}}}},
-		},
-
-		{
-			// weights not add up to total-weight.
-			name: "route-config-with-weighted_clusters_weights_not_add_up",
-			rc: &xdspb.RouteConfiguration{
-				Name: goodRouteName1,
-				VirtualHosts: []*routepb.VirtualHost{{
-					Domains: []string{goodLDSTarget1},
-					Routes: []*routepb.Route{{
-						Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"}},
-						Action: &routepb.Route_Route{
-							Route: &routepb.RouteAction{
-								ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
-									WeightedClusters: &routepb.WeightedCluster{
-										Clusters: []*routepb.WeightedCluster_ClusterWeight{
-											{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
-											{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
-											{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
-										},
-										TotalWeight: &wrapperspb.UInt32Value{Value: 30},
-									}}}}}}}}},
-			wantError: true,
-		},
-		{
-			name: "good-route-config-with-weighted_clusters",
-			rc: &xdspb.RouteConfiguration{
-				Name: goodRouteName1,
-				VirtualHosts: []*routepb.VirtualHost{{
-					Domains: []string{goodLDSTarget1},
-					Routes: []*routepb.Route{{
-						Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"}},
-						Action: &routepb.Route_Route{
-							Route: &routepb.RouteAction{
-								ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
-									WeightedClusters: &routepb.WeightedCluster{
-										Clusters: []*routepb.WeightedCluster_ClusterWeight{
-											{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
-											{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
-											{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
-										},
-										TotalWeight: &wrapperspb.UInt32Value{Value: 10},
-									}}}}}}}}},
-			wantUpdate: rdsUpdate{routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}}},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, goodLDSTarget1, nil)
-			if !cmp.Equal(gotUpdate, test.wantUpdate, cmp.AllowUnexported(rdsUpdate{})) || (gotError != nil) != test.wantError {
-				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, goodLDSTarget1, gotUpdate, test.wantUpdate)
-			}
-		})
-	}
-}
-
-// doLDS makes a LDS watch, and waits for the response and ack to finish.
-//
-// This is called by RDS tests to start LDS first, because LDS is a
-// pre-requirement for RDS, and RDS handle would fail without an existing LDS
-// watch.
-func doLDS(t *testing.T, v2c *v2Client, fakeServer *fakeserver.Server) {
-	v2c.addWatch(ldsURL, goodLDSTarget1)
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
-		t.Fatalf("Timeout waiting for LDS request: %v", err)
-	}
-}
-
-// TestRDSHandleResponseWithRouting starts a fake xDS server, makes a ClientConn
-// to it, and creates a v2Client using it. Then, it registers an LDS and RDS
-// watcher and tests different RDS responses.
-func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
-	tests := []struct {
-		name          string
-		rdsResponse   *xdspb.DiscoveryResponse
-		wantErr       bool
-		wantUpdate    *rdsUpdate
-		wantUpdateErr bool
-	}{
-		// Badly marshaled RDS response.
-		{
-			name:          "badly-marshaled-response",
-			rdsResponse:   badlyMarshaledRDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
-			wantUpdateErr: false,
-		},
-		// Response does not contain RouteConfiguration proto.
-		{
-			name:          "no-route-config-in-response",
-			rdsResponse:   badResourceTypeInRDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
-			wantUpdateErr: false,
-		},
-		// No VirtualHosts in the response. Just one test case here for a bad
-		// RouteConfiguration, since the others are covered in
-		// TestGetClusterFromRouteConfiguration.
-		{
-			name:          "no-virtual-hosts-in-response",
-			rdsResponse:   noVirtualHostsInRDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
-			wantUpdateErr: false,
-		},
-		// Response contains one good RouteConfiguration, uninteresting though.
-		{
-			name:          "one-uninteresting-route-config",
-			rdsResponse:   goodRDSResponse2,
-			wantErr:       false,
-			wantUpdate:    nil,
-			wantUpdateErr: false,
-		},
-		// Response contains one good interesting RouteConfiguration.
-		{
-			name:          "one-good-route-config",
-			rdsResponse:   goodRDSResponse1,
-			wantErr:       false,
-			wantUpdate:    &rdsUpdate{routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}},
-			wantUpdateErr: false,
-		},
-		{
-			name:        "one-good-route-config with routes",
-			rdsResponse: goodRDSResponse1,
-			wantErr:     false,
-			wantUpdate: &rdsUpdate{
-				// Instead of just weighted targets when routing is disabled,
-				// this result contains a route with perfix "", and action as
-				// weighted targets.
-				routes: []*Route{{
-					Prefix: newStringP(""),
-					Action: map[string]uint32{goodClusterName1: 1},
-				}},
-			},
-			wantUpdateErr: false,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          rdsURL,
-				resourceName:     goodRouteName1,
-				responseToHandle: test.rdsResponse,
-				wantHandleErr:    test.wantErr,
-				wantUpdate:       test.wantUpdate,
-				wantUpdateErr:    test.wantUpdateErr,
-			})
-		})
-	}
-}
-
-// TestRDSHandleResponseWithoutLDSWatch tests the case where the v2Client
-// receives an RDS response without a registered LDS watcher.
-func (s) TestRDSHandleResponseWithoutLDSWatch(t *testing.T) {
-	_, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
-	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-
-	if v2c.handleRDSResponse(goodRDSResponse1) == nil {
-		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
-	}
-}
-
-// TestRDSHandleResponseWithoutRDSWatch tests the case where the v2Client
-// receives an RDS response without a registered RDS watcher.
-func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
-	fakeServer, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
-	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	defer v2c.close()
-	doLDS(t, v2c, fakeServer)
-
-	if v2c.handleRDSResponse(badResourceTypeInRDSResponse) == nil {
-		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
-	}
-
-	if v2c.handleRDSResponse(goodRDSResponse1) != nil {
-		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
-	}
-}
-
-func (s) TestMatchTypeForDomain(t *testing.T) {
-	tests := []struct {
-		d    string
-		want domainMatchType
-	}{
-		{d: "", want: domainMatchTypeInvalid},
-		{d: "*", want: domainMatchTypeUniversal},
-		{d: "bar.*", want: domainMatchTypePrefix},
-		{d: "*.abc.com", want: domainMatchTypeSuffix},
-		{d: "foo.bar.com", want: domainMatchTypeExact},
-		{d: "foo.*.com", want: domainMatchTypeInvalid},
-	}
-	for _, tt := range tests {
-		if got := matchTypeForDomain(tt.d); got != tt.want {
-			t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want)
-		}
-	}
-}
-
-func (s) TestMatch(t *testing.T) {
-	tests := []struct {
-		name        string
-		domain      string
-		host        string
-		wantTyp     domainMatchType
-		wantMatched bool
-	}{
-		{name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
-		{name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
-		{name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true},
-		{name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true},
-		{name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false},
-		{name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true},
-		{name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false},
-		{name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true},
-		{name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched {
-				t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched)
-			}
-		})
-	}
-}
-
-func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
-	var (
-		oneExactMatch = &routepb.VirtualHost{
-			Name:    "one-exact-match",
-			Domains: []string{"foo.bar.com"},
-		}
-		oneSuffixMatch = &routepb.VirtualHost{
-			Name:    "one-suffix-match",
-			Domains: []string{"*.bar.com"},
-		}
-		onePrefixMatch = &routepb.VirtualHost{
-			Name:    "one-prefix-match",
-			Domains: []string{"foo.bar.*"},
-		}
-		oneUniversalMatch = &routepb.VirtualHost{
-			Name:    "one-universal-match",
-			Domains: []string{"*"},
-		}
-		longExactMatch = &routepb.VirtualHost{
-			Name:    "one-exact-match",
-			Domains: []string{"v2.foo.bar.com"},
-		}
-		multipleMatch = &routepb.VirtualHost{
-			Name:    "multiple-match",
-			Domains: []string{"pi.foo.bar.com", "314.*", "*.159"},
-		}
-		vhs = []*routepb.VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
-	)
-
-	tests := []struct {
-		name   string
-		host   string
-		vHosts []*routepb.VirtualHost
-		want   *routepb.VirtualHost
-	}{
-		{name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch},
-		{name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch},
-		{name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch},
-		{name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch},
-		{name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch},
-		// Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact.
-		{name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch},
-		// Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix.
-		{name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch},
-		// Matches suffix "*.bar.com" and prefix "314.*". Takes suffix.
-		{name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := findBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) {
-				t.Errorf("findBestMatchingVirtualHost() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func (s) TestWeightedClustersProtoToMap(t *testing.T) {
-	tests := []struct {
-		name    string
-		wc      *routepb.WeightedCluster
-		want    map[string]uint32
-		wantErr bool
-	}{
-		{
-			name: "weight not add up to non default total",
-			wc: &routepb.WeightedCluster{
-				Clusters: []*routepb.WeightedCluster_ClusterWeight{
-					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 1}},
-					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 1}},
-					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 1}},
-				},
-				TotalWeight: &wrapperspb.UInt32Value{Value: 10},
-			},
-			wantErr: true,
-		},
-		{
-			name: "weight not add up to default total",
-			wc: &routepb.WeightedCluster{
-				Clusters: []*routepb.WeightedCluster_ClusterWeight{
-					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
-					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
-					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
-				},
-				TotalWeight: nil,
-			},
-			wantErr: true,
-		},
-		{
-			name: "ok non default total weight",
-			wc: &routepb.WeightedCluster{
-				Clusters: []*routepb.WeightedCluster_ClusterWeight{
-					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}},
-					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}},
-					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}},
-				},
-				TotalWeight: &wrapperspb.UInt32Value{Value: 10},
-			},
-			want: map[string]uint32{"a": 2, "b": 3, "c": 5},
-		},
-		{
-			name: "ok default total weight is 100",
-			wc: &routepb.WeightedCluster{
-				Clusters: []*routepb.WeightedCluster_ClusterWeight{
-					{Name: "a", Weight: &wrapperspb.UInt32Value{Value: 20}},
-					{Name: "b", Weight: &wrapperspb.UInt32Value{Value: 30}},
-					{Name: "c", Weight: &wrapperspb.UInt32Value{Value: 50}},
-				},
-				TotalWeight: nil,
-			},
-			want: map[string]uint32{"a": 20, "b": 30, "c": 50},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := weightedClustersProtoToMap(tt.wc)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("weightedClustersProtoToMap() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !cmp.Equal(got, tt.want) {
-				t.Errorf("weightedClustersProtoToMap() got = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestRoutesProtoToSlice(t *testing.T) {
-	tests := []struct {
-		name       string
-		routes     []*routepb.Route
-		wantRoutes []*Route
-		wantErr    bool
-	}{
-		{
-			name: "no path",
-			routes: []*routepb.Route{{
-				Match: &routepb.RouteMatch{},
-			}},
-			wantErr: true,
-		},
-		{
-			name: "path is regex instead of saferegex",
-			routes: []*routepb.Route{{
-				Match: &routepb.RouteMatch{
-					PathSpecifier: &routepb.RouteMatch_Regex{Regex: "*"},
-				},
-			}},
-			wantErr: true,
-		},
-		{
-			name: "header contains regex",
-			routes: []*routepb.Route{{
-				Match: &routepb.RouteMatch{
-					PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"},
-					Headers: []*routepb.HeaderMatcher{{
-						Name: "th",
-						HeaderMatchSpecifier: &routepb.HeaderMatcher_RegexMatch{
-							RegexMatch: "*",
-						},
-					}},
-				},
-			}},
-			wantErr: true,
-		},
-		{
-			name: "case_sensitive is false",
-			routes: []*routepb.Route{{
-				Match: &routepb.RouteMatch{
-					PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/"},
-					CaseSensitive: &wrapperspb.BoolValue{Value: false},
-				},
-			}},
-			wantErr: true,
-		},
-		{
-			name: "good",
-			routes: []*routepb.Route{
-				{
-					Match: &routepb.RouteMatch{
-						PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/a/"},
-						Headers: []*routepb.HeaderMatcher{
-							{
-								Name: "th",
-								HeaderMatchSpecifier: &routepb.HeaderMatcher_PrefixMatch{
-									PrefixMatch: "tv",
-								},
-								InvertMatch: true,
-							},
-						},
-						RuntimeFraction: &corepb.RuntimeFractionalPercent{
-							DefaultValue: &typepb.FractionalPercent{
-								Numerator:   1,
-								Denominator: typepb.FractionalPercent_HUNDRED,
-							},
-						},
-					},
-					Action: &routepb.Route_Route{
-						Route: &routepb.RouteAction{
-							ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
-								WeightedClusters: &routepb.WeightedCluster{
-									Clusters: []*routepb.WeightedCluster_ClusterWeight{
-										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
-										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
-									},
-									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
-								}}}},
-				},
-			},
-			wantRoutes: []*Route{{
-				Prefix: newStringP("/a/"),
-				Headers: []*HeaderMatcher{
-					{
-						Name:        "th",
-						InvertMatch: newBoolP(true),
-						PrefixMatch: newStringP("tv"),
-					},
-				},
-				Fraction: newUInt32P(10000),
-				Action:   map[string]uint32{"A": 40, "B": 60},
-			}},
-			wantErr: false,
-		},
-		{
-			name: "query is ignored",
-			routes: []*routepb.Route{
-				{
-					Match: &routepb.RouteMatch{
-						PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/a/"},
-					},
-					Action: &routepb.Route_Route{
-						Route: &routepb.RouteAction{
-							ClusterSpecifier: &routepb.RouteAction_WeightedClusters{
-								WeightedClusters: &routepb.WeightedCluster{
-									Clusters: []*routepb.WeightedCluster_ClusterWeight{
-										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
-										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
-									},
-									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
-								}}}},
-				},
-				{
-					Name: "with_query",
-					Match: &routepb.RouteMatch{
-						PathSpecifier:   &routepb.RouteMatch_Prefix{Prefix: "/b/"},
-						QueryParameters: []*routepb.QueryParameterMatcher{{Name: "route_will_be_ignored"}},
-					},
-				},
-			},
-			// Only one route in the result, because the second one with query
-			// parameters is ignored.
-			wantRoutes: []*Route{{
-				Prefix: newStringP("/a/"),
-				Action: map[string]uint32{"A": 40, "B": 60},
-			}},
-			wantErr: false,
-		},
-	}
-
-	cmpOpts := []cmp.Option{
-		cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}),
-		cmpopts.EquateEmpty(),
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := routesProtoToSlice(tt.routes, nil)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !cmp.Equal(got, tt.wantRoutes, cmpOpts...) {
-				t.Errorf("routesProtoToSlice() got = %v, want %v, diff: %v", got, tt.wantRoutes, cmp.Diff(got, tt.wantRoutes, cmpOpts...))
-			}
-		})
-	}
-}
-
-func newStringP(s string) *string {
-	return &s
-}
-
-func newUInt32P(i uint32) *uint32 {
-	return &i
-}
-
-func newBoolP(b bool) *bool {
-	return &b
-}
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index ce4a7e8fab9c..1fc03d1fac16 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -27,7 +27,6 @@ import (
 	"google.golang.org/grpc/serviceconfig"
 	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
 	_ "google.golang.org/grpc/xds/internal/balancer/xdsrouting"
-	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
@@ -312,13 +311,13 @@ func TestRoutesToJSON(t *testing.T) {
 func TestServiceUpdateToJSON(t *testing.T) {
 	tests := []struct {
 		name     string
-		su       client.ServiceUpdate
+		su       xdsclient.ServiceUpdate
 		wantJSON string
 		wantErr  bool
 	}{
 		{
 			name: "routing",
-			su: client.ServiceUpdate{
+			su: xdsclient.ServiceUpdate{
 				Routes: []*xdsclient.Route{{
 					Path:   newStringP("/service_1/method_1"),
 					Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25},
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 5c3b0fce84e1..33f2e2695a0c 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -319,11 +319,11 @@ func TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	defer replaceRandNumGenerator(0)()
 
 	for _, tt := range []struct {
-		su       client.ServiceUpdate
+		su       xdsclient.ServiceUpdate
 		wantJSON string
 	}{
 		{
-			su:       client.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}}},
+			su:       xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}}},
 			wantJSON: testOneClusterOnlyJSON,
 		},
 		{
diff --git a/xds/internal/testutils/protos.go b/xds/internal/testutils/protos.go
index fb60e91f2e34..25a0944d96dc 100644
--- a/xds/internal/testutils/protos.go
+++ b/xds/internal/testutils/protos.go
@@ -18,13 +18,25 @@
 package testutils
 
 import (
+	"fmt"
+	"net"
+	"strconv"
+
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v2typepb "github.com/envoyproxy/go-control-plane/envoy/type"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"google.golang.org/grpc/xds/internal"
 )
 
-// EmptyNodeProtoV2 is a node proto with no fields set.
+// EmptyNodeProtoV2 is a v2 Node proto with no fields set.
 var EmptyNodeProtoV2 = &v2corepb.Node{}
 
+// EmptyNodeProtoV3 is a v3 Node proto with no fields set.
+var EmptyNodeProtoV3 = &v3corepb.Node{}
+
 // LocalityIDToProto converts a LocalityID to its proto representation.
 func LocalityIDToProto(l internal.LocalityID) *v2corepb.Locality {
 	return &v2corepb.Locality{
@@ -33,3 +45,101 @@ func LocalityIDToProto(l internal.LocalityID) *v2corepb.Locality {
 		SubZone: l.SubZone,
 	}
 }
+
+// The helper structs/functions related to EDS protos are used in EDS balancer
+// tests now, to generate test inputs. Eventually, EDS balancer tests should
+// generate EndpointsUpdate directly, instead of generating and parsing the
+// proto message.
+// TODO: Once EDS balancer tests don't use these, these can be moved to v2 client code.
+
+// ClusterLoadAssignmentBuilder builds a ClusterLoadAssignment, aka EDS
+// response.
+type ClusterLoadAssignmentBuilder struct {
+	v *v2xdspb.ClusterLoadAssignment
+}
+
+// NewClusterLoadAssignmentBuilder creates a ClusterLoadAssignmentBuilder.
+func NewClusterLoadAssignmentBuilder(clusterName string, dropPercents []uint32) *ClusterLoadAssignmentBuilder {
+	var drops []*v2xdspb.ClusterLoadAssignment_Policy_DropOverload
+	for i, d := range dropPercents {
+		drops = append(drops, &v2xdspb.ClusterLoadAssignment_Policy_DropOverload{
+			Category: fmt.Sprintf("test-drop-%d", i),
+			DropPercentage: &v2typepb.FractionalPercent{
+				Numerator:   d,
+				Denominator: v2typepb.FractionalPercent_HUNDRED,
+			},
+		})
+	}
+
+	return &ClusterLoadAssignmentBuilder{
+		v: &v2xdspb.ClusterLoadAssignment{
+			ClusterName: clusterName,
+			Policy: &v2xdspb.ClusterLoadAssignment_Policy{
+				DropOverloads: drops,
+			},
+		},
+	}
+}
+
+// AddLocalityOptions contains options when adding locality to the builder.
+type AddLocalityOptions struct {
+	Health []v2corepb.HealthStatus
+	Weight []uint32
+}
+
+// AddLocality adds a locality to the builder.
+func (clab *ClusterLoadAssignmentBuilder) AddLocality(subzone string, weight uint32, priority uint32, addrsWithPort []string, opts *AddLocalityOptions) {
+	var lbEndPoints []*v2endpointpb.LbEndpoint
+	for i, a := range addrsWithPort {
+		host, portStr, err := net.SplitHostPort(a)
+		if err != nil {
+			panic("failed to split " + a)
+		}
+		port, err := strconv.Atoi(portStr)
+		if err != nil {
+			panic("failed to atoi " + portStr)
+		}
+
+		lbe := &v2endpointpb.LbEndpoint{
+			HostIdentifier: &v2endpointpb.LbEndpoint_Endpoint{
+				Endpoint: &v2endpointpb.Endpoint{
+					Address: &v2corepb.Address{
+						Address: &v2corepb.Address_SocketAddress{
+							SocketAddress: &v2corepb.SocketAddress{
+								Protocol: v2corepb.SocketAddress_TCP,
+								Address:  host,
+								PortSpecifier: &v2corepb.SocketAddress_PortValue{
+									PortValue: uint32(port)}}}}}},
+		}
+		if opts != nil {
+			if i < len(opts.Health) {
+				lbe.HealthStatus = opts.Health[i]
+			}
+			if i < len(opts.Weight) {
+				lbe.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: opts.Weight[i]}
+			}
+		}
+		lbEndPoints = append(lbEndPoints, lbe)
+	}
+
+	var localityID *v2corepb.Locality
+	if subzone != "" {
+		localityID = &v2corepb.Locality{
+			Region:  "",
+			Zone:    "",
+			SubZone: subzone,
+		}
+	}
+
+	clab.v.Endpoints = append(clab.v.Endpoints, &v2endpointpb.LocalityLbEndpoints{
+		Locality:            localityID,
+		LbEndpoints:         lbEndPoints,
+		LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight},
+		Priority:            priority,
+	})
+}
+
+// Build builds ClusterLoadAssignment.
+func (clab *ClusterLoadAssignmentBuilder) Build() *v2xdspb.ClusterLoadAssignment {
+	return clab.v
+}
diff --git a/xds/internal/version/version.go b/xds/internal/version/version.go
index 0425fa5e33fd..d1ca46f658a1 100644
--- a/xds/internal/version/version.go
+++ b/xds/internal/version/version.go
@@ -16,7 +16,8 @@
  *
  */
 
-// Package version defines supported xDS API versions.
+// Package version defines constants to distinguish between supported xDS API
+// versions.
 package version
 
 // TransportAPI refers to the API version for xDS transport protocol. This
@@ -30,3 +31,19 @@ const (
 	// TransportV3 refers to the v3 xDS transport protocol.
 	TransportV3
 )
+
+// Resource URLs. We need to be able to accept either version of the resource
+// regardless of the version of the transport protocol in use.
+const (
+	V2ListenerURL        = "type.googleapis.com/envoy.api.v2.Listener"
+	V2RouteConfigURL     = "type.googleapis.com/envoy.api.v2.RouteConfiguration"
+	V2ClusterURL         = "type.googleapis.com/envoy.api.v2.Cluster"
+	V2EndpointsURL       = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
+	V2HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"
+
+	V3ListenerURL        = "type.googleapis.com/envoy.config.listener.v3.Listener"
+	V3RouteConfigURL     = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"
+	V3ClusterURL         = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
+	V3EndpointsURL       = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"
+	V3HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"
+)
diff --git a/xds/xds.go b/xds/xds.go
index e077a69476eb..40d62605c263 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -24,6 +24,7 @@
 package xds
 
 import (
-	_ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers.
-	_ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver
+	_ "google.golang.org/grpc/xds/internal/balancer"  // Register the balancers.
+	_ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 xDS API client.
+	_ "google.golang.org/grpc/xds/internal/resolver"  // Register the xds_resolver.
 )

From f92f534396d5991013707fbcd7a3c38caea7e9bd Mon Sep 17 00:00:00 2001
From: Hiroyuki Tanaka <h.tanaka.0325@gmail.com>
Date: Fri, 31 Jul 2020 05:39:09 +0900
Subject: [PATCH 138/481] README: change badge to pkg.go.dev (#3768)

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fef78e4a1dff..3949a683fb58 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # gRPC-Go
 
 [![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go)
-[![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)][API]
+[![GoDoc](https://pkg.go.dev/badge/google.golang.org/grpc)][API]
 [![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go)
 
 The [Go][] implementation of [gRPC][]: A high performance, open source, general
@@ -131,7 +131,7 @@ the root cause of the connection being closed is on the server side. Turn on
 logging on __both client and server__, and see if there are any transport
 errors.
 
-[API]: https://grpc.io/docs/languages/go/api
+[API]: https://pkg.go.dev/google.golang.org/grpc
 [Go]: https://golang.org
 [Go module]: https://github.com/golang/go/wiki/Modules
 [gRPC]: https://grpc.io

From b2e49701d9f410d503f183779ec11eecf1295e64 Mon Sep 17 00:00:00 2001
From: Achyuta Das <61797154+aka-achu@users.noreply.github.com>
Date: Fri, 31 Jul 2020 02:13:56 +0530
Subject: [PATCH 139/481] examples: break from 'PerRPCCredsCallOption' status
 checking loop (#3772)

If *grpc.PerRPCCredsCallOption is not found in the option array then change credentialConfigStatus to true and break from the loop. No need to iterate further.
---
 examples/features/interceptor/client/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/features/interceptor/client/main.go b/examples/features/interceptor/client/main.go
index 0642477c4e3f..b8f2733ca497 100644
--- a/examples/features/interceptor/client/main.go
+++ b/examples/features/interceptor/client/main.go
@@ -93,6 +93,7 @@ func streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.Clie
 		_, ok := o.(*grpc.PerRPCCredsCallOption)
 		if ok {
 			credsConfigured = true
+			break
 		}
 	}
 	if !credsConfigured {

From b72d07f81697106b9b4514c0bfd1c1d017065d1a Mon Sep 17 00:00:00 2001
From: cindyxue <32377977+cindyxue@users.noreply.github.com>
Date: Thu, 30 Jul 2020 13:48:30 -0700
Subject: [PATCH 140/481] advancedtls: migrate testing to grpctest (#3773)

advancedtls: migrate tests to use grpctest
---
 .../advancedtls_integration_test.go           |  2 +-
 security/advancedtls/advancedtls_test.go      | 21 +++++++++++++------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 3f9f6218035a..195da18385cb 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -181,7 +181,7 @@ func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds cred
 // the factor we modified in stage 1.
 // (could be change the client's trust certificate, or change custom
 // verification function, etc)
-func TestEnd2End(t *testing.T) {
+func (s) TestEnd2End(t *testing.T) {
 	cs := &certStore{}
 	err := cs.loadCerts()
 	if err != nil {
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index b2514547c174..4e79d0d89742 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -32,10 +32,19 @@ import (
 	"testing"
 
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/security/advancedtls/testdata"
 )
 
-func TestClientServerHandshake(t *testing.T) {
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestClientServerHandshake(t *testing.T) {
 	// ------------------Load Client Trust Cert and Peer Cert-------------------
 	clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem"))
 	if err != nil {
@@ -523,7 +532,7 @@ func compare(a1, a2 credentials.AuthInfo) bool {
 	}
 }
 
-func TestAdvancedTLSOverrideServerName(t *testing.T) {
+func (s) TestAdvancedTLSOverrideServerName(t *testing.T) {
 	expectedServerName := "server.name"
 	clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem"))
 	if err != nil {
@@ -545,7 +554,7 @@ func TestAdvancedTLSOverrideServerName(t *testing.T) {
 	}
 }
 
-func TestTLSClone(t *testing.T) {
+func (s) TestTLSClone(t *testing.T) {
 	expectedServerName := "server.name"
 	clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem"))
 	if err != nil {
@@ -573,7 +582,7 @@ func TestTLSClone(t *testing.T) {
 
 }
 
-func TestAppendH2ToNextProtos(t *testing.T) {
+func (s) TestAppendH2ToNextProtos(t *testing.T) {
 	tests := []struct {
 		name string
 		ps   []string
@@ -613,7 +622,7 @@ type nonSyscallConn struct {
 	net.Conn
 }
 
-func TestWrapSyscallConn(t *testing.T) {
+func (s) TestWrapSyscallConn(t *testing.T) {
 	sc := &syscallConn{}
 	nsc := &nonSyscallConn{}
 
@@ -624,7 +633,7 @@ func TestWrapSyscallConn(t *testing.T) {
 	}
 }
 
-func TestOptionsConfig(t *testing.T) {
+func (s) TestOptionsConfig(t *testing.T) {
 	serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"),
 		testdata.Path("server_key_1.pem"))
 	if err != nil {

From bc714cd8aeaa4f1a59e055b5a6411a04b17fd319 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 30 Jul 2020 16:15:35 -0700
Subject: [PATCH 141/481] Remove Go 1.9 / AppEngine support; assume go1.12
 build tag (#3767)

---
 .travis.yml                                   |   4 -
 CONTRIBUTING.md                               |   1 -
 Makefile                                      |   8 --
 balancer/rls/internal/balancer.go             |   2 -
 balancer/rls/internal/balancer_test.go        |   2 -
 balancer/rls/internal/builder.go              |   2 -
 balancer/rls/internal/client_test.go          |   2 -
 balancer/rls/internal/config.go               |   2 -
 balancer/rls/internal/config_test.go          |   2 -
 balancer/rls/internal/keys/builder.go         |   2 -
 balancer/rls/internal/keys/builder_test.go    |   2 -
 balancer/rls/internal/picker.go               |   2 -
 balancer/rls/internal/picker_test.go          |   2 -
 channelz/service/func_linux.go                |   2 -
 channelz/service/func_nonlinux.go             |   2 +-
 channelz/service/service_sktopt_test.go       |   2 +-
 channelz/service/service_test.go              |   2 +-
 channelz/service/util_sktopt_386_test.go      |   2 +-
 channelz/service/util_sktopt_amd64_test.go    |   2 +-
 credentials/go12.go                           |  30 -----
 credentials/internal/syscallconn.go           |   2 -
 credentials/internal/syscallconn_appengine.go |  30 -----
 credentials/internal/syscallconn_test.go      |   2 -
 credentials/tls.go                            |   4 +
 examples/features/xds/client/main.go          |   2 -
 examples/features/xds/server/main.go          |   2 -
 install_gae.sh                                |   6 -
 internal/channelz/types_linux.go              |   2 -
 internal/channelz/types_nonlinux.go           |   4 +-
 internal/channelz/util_linux.go               |   2 +-
 internal/channelz/util_nonlinux.go            |   2 +-
 internal/channelz/util_test.go                |   2 +-
 internal/credentials/gobefore110.go           |  31 -----
 internal/credentials/{go110.go => spiffe.go}  |   2 -
 .../{go110_test.go => spiffe_test.go}         |   2 -
 internal/profiling/buffer/buffer.go           |   2 -
 internal/profiling/buffer/buffer_appengine.go |  43 -------
 internal/profiling/buffer/buffer_test.go      |   2 -
 internal/profiling/profiling_test.go          |   2 -
 internal/syscall/syscall_linux.go             |   4 +-
 internal/syscall/syscall_nonlinux.go          |  16 +--
 interop/grpclb_fallback/client.go             |   2 -
 security/advancedtls/{sni_110.go => sni.go}   |   2 -
 security/advancedtls/sni_before_110.go        |  41 -------
 security/advancedtls/sni_test_before_110.go   | 108 ------------------
 .../{sni_test_110.go => sni_test_disabled.go} |   2 -
 test/channelz_linux_go110_test.go             |   2 +-
 test/go_vet/vet.go                            |  53 ---------
 vet.sh                                        |   4 -
 .../client/bootstrap/bootstrap_test.go        |   2 -
 50 files changed, 24 insertions(+), 431 deletions(-)
 delete mode 100644 credentials/go12.go
 delete mode 100644 credentials/internal/syscallconn_appengine.go
 delete mode 100755 install_gae.sh
 delete mode 100644 internal/credentials/gobefore110.go
 rename internal/credentials/{go110.go => spiffe.go} (99%)
 rename internal/credentials/{go110_test.go => spiffe_test.go} (99%)
 delete mode 100644 internal/profiling/buffer/buffer_appengine.go
 rename security/advancedtls/{sni_110.go => sni.go} (98%)
 delete mode 100644 security/advancedtls/sni_before_110.go
 delete mode 100644 security/advancedtls/sni_test_before_110.go
 rename security/advancedtls/{sni_test_110.go => sni_test_disabled.go} (99%)
 delete mode 100644 test/go_vet/vet.go

diff --git a/.travis.yml b/.travis.yml
index 0e24e59f0567..00287d666a5a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,8 +16,6 @@ matrix:
     env: GO111MODULE=on
   - go: 1.12.x
     env: GO111MODULE=on
-  - go: 1.9.x
-    env: GAE=1
 
 go_import_path: google.golang.org/grpc
 
@@ -30,13 +28,11 @@ before_install:
 install:
   - try3() { eval "$*" || eval "$*" || eval "$*"; }
   - try3 'if [[ "${GO111MODULE}" = "on" ]]; then go mod download; else make testdeps; fi'
-  - if [[ -n "${GAE}" ]]; then source ./install_gae.sh; make testappenginedeps; fi
   - if [[ -n "${VET}" ]]; then ./vet.sh -install; fi
 
 script:
   - set -e
   - if [[ -n "${TESTEXTRAS}" ]]; then examples/examples_test.sh; interop/interop_test.sh; make testsubmodule; exit 0; fi
   - if [[ -n "${VET}" ]]; then ./vet.sh; fi
-  - if [[ -n "${GAE}" ]]; then make testappengine; exit 0; fi
   - if [[ -n "${RACE}" ]]; then make testrace; exit 0; fi
   - make test
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4f1567e2f95e..cd03f8c76888 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -57,6 +57,5 @@ How to get your contributions merged smoothly and quickly.
   - `make vet` to catch vet errors
   - `make test` to run the tests
   - `make testrace` to run tests in race mode
-  - optional `make testappengine` to run tests with appengine
 
 - Exceptions to the rules can be made if there's a compelling reason for doing so.
diff --git a/Makefile b/Makefile
index 410f7d56d4c2..bd892e592490 100644
--- a/Makefile
+++ b/Makefile
@@ -22,12 +22,6 @@ test: testdeps
 testsubmodule: testdeps
 	cd security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
 
-testappengine: testappenginedeps
-	goapp test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
-
-testappenginedeps:
-	goapp get -d -v -t -tags 'appengine appenginevm' google.golang.org/grpc/...
-
 testdeps:
 	go get -d -v -t google.golang.org/grpc/...
 
@@ -53,8 +47,6 @@ vetdeps:
 	deps \
 	proto \
 	test \
-	testappengine \
-	testappenginedeps \
 	testdeps \
 	testrace \
 	updatedeps \
diff --git a/balancer/rls/internal/balancer.go b/balancer/rls/internal/balancer.go
index aeb509aa894b..7af97b76faf1 100644
--- a/balancer/rls/internal/balancer.go
+++ b/balancer/rls/internal/balancer.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/balancer_test.go b/balancer/rls/internal/balancer_test.go
index 9df769c2a8c3..973f5e0cd608 100644
--- a/balancer/rls/internal/balancer_test.go
+++ b/balancer/rls/internal/balancer_test.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/builder.go b/balancer/rls/internal/builder.go
index c38babff4d3d..7c29caef4047 100644
--- a/balancer/rls/internal/builder.go
+++ b/balancer/rls/internal/builder.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/client_test.go b/balancer/rls/internal/client_test.go
index 1a1a75d1be98..a45850dce11e 100644
--- a/balancer/rls/internal/client_test.go
+++ b/balancer/rls/internal/client_test.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/config.go b/balancer/rls/internal/config.go
index 0a8d2d91fa82..94ed3c5ee70b 100644
--- a/balancer/rls/internal/config.go
+++ b/balancer/rls/internal/config.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/config_test.go b/balancer/rls/internal/config_test.go
index 9200a29d8a3c..fa22e15a6501 100644
--- a/balancer/rls/internal/config_test.go
+++ b/balancer/rls/internal/config_test.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/keys/builder.go b/balancer/rls/internal/keys/builder.go
index abc076ab4da3..5ce5a9da508a 100644
--- a/balancer/rls/internal/keys/builder.go
+++ b/balancer/rls/internal/keys/builder.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/keys/builder_test.go b/balancer/rls/internal/keys/builder_test.go
index f5e03ac51dd7..a5cad29e0c93 100644
--- a/balancer/rls/internal/keys/builder_test.go
+++ b/balancer/rls/internal/keys/builder_test.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/picker.go b/balancer/rls/internal/picker.go
index 698185b1595b..ce7100536d95 100644
--- a/balancer/rls/internal/picker.go
+++ b/balancer/rls/internal/picker.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/balancer/rls/internal/picker_test.go b/balancer/rls/internal/picker_test.go
index b14a9fad340a..fc5f935d6929 100644
--- a/balancer/rls/internal/picker_test.go
+++ b/balancer/rls/internal/picker_test.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/channelz/service/func_linux.go b/channelz/service/func_linux.go
index cfa457a6a8b1..ce38a921b974 100644
--- a/channelz/service/func_linux.go
+++ b/channelz/service/func_linux.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/channelz/service/func_nonlinux.go b/channelz/service/func_nonlinux.go
index eb53334ed0d1..64ecea947de0 100644
--- a/channelz/service/func_nonlinux.go
+++ b/channelz/service/func_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux appengine
+// +build !linux
 
 /*
  *
diff --git a/channelz/service/service_sktopt_test.go b/channelz/service/service_sktopt_test.go
index 1a12ed0ed3cc..e2d024f83652 100644
--- a/channelz/service/service_sktopt_test.go
+++ b/channelz/service/service_sktopt_test.go
@@ -1,4 +1,4 @@
-// +build linux,!appengine
+// +build linux
 // +build 386 amd64
 
 /*
diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go
index e6a7d8eba3be..41f4f31a5b94 100644
--- a/channelz/service/service_test.go
+++ b/channelz/service/service_test.go
@@ -58,7 +58,7 @@ type protoToSocketOptFunc func([]*channelzpb.SocketOption) *channelz.SocketOptio
 
 // protoToSocketOpt is used in function socketProtoToStruct to extract socket option
 // data from unmarshaled proto message.
-// It is only defined under linux, non-appengine environment on x86 architecture.
+// It is only defined under linux environment on x86 architecture.
 var protoToSocketOpt protoToSocketOptFunc
 
 // emptyTime is used for detecting unset value of time.Time type.
diff --git a/channelz/service/util_sktopt_386_test.go b/channelz/service/util_sktopt_386_test.go
index cdc2fda4028d..d9c981271361 100644
--- a/channelz/service/util_sktopt_386_test.go
+++ b/channelz/service/util_sktopt_386_test.go
@@ -1,4 +1,4 @@
-// +build 386,linux,!appengine
+// +build 386,linux
 
 /*
  *
diff --git a/channelz/service/util_sktopt_amd64_test.go b/channelz/service/util_sktopt_amd64_test.go
index 7ebe9c70eb68..0ff06d128330 100644
--- a/channelz/service/util_sktopt_amd64_test.go
+++ b/channelz/service/util_sktopt_amd64_test.go
@@ -1,4 +1,4 @@
-// +build amd64,linux,!appengine
+// +build amd64,linux
 
 /*
  *
diff --git a/credentials/go12.go b/credentials/go12.go
deleted file mode 100644
index ccbf35b33125..000000000000
--- a/credentials/go12.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// +build go1.12
-
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package credentials
-
-import "crypto/tls"
-
-// This init function adds cipher suite constants only defined in Go 1.12.
-func init() {
-	cipherSuiteLookup[tls.TLS_AES_128_GCM_SHA256] = "TLS_AES_128_GCM_SHA256"
-	cipherSuiteLookup[tls.TLS_AES_256_GCM_SHA384] = "TLS_AES_256_GCM_SHA384"
-	cipherSuiteLookup[tls.TLS_CHACHA20_POLY1305_SHA256] = "TLS_CHACHA20_POLY1305_SHA256"
-}
diff --git a/credentials/internal/syscallconn.go b/credentials/internal/syscallconn.go
index 2f4472becc7c..264c7298085a 100644
--- a/credentials/internal/syscallconn.go
+++ b/credentials/internal/syscallconn.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/credentials/internal/syscallconn_appengine.go b/credentials/internal/syscallconn_appengine.go
deleted file mode 100644
index d4346e9eabe6..000000000000
--- a/credentials/internal/syscallconn_appengine.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// +build appengine
-
-/*
- *
- * Copyright 2018 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package internal
-
-import (
-	"net"
-)
-
-// WrapSyscallConn returns newConn on appengine.
-func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn {
-	return newConn
-}
diff --git a/credentials/internal/syscallconn_test.go b/credentials/internal/syscallconn_test.go
index a5a7d72689e6..6700e7546853 100644
--- a/credentials/internal/syscallconn_test.go
+++ b/credentials/internal/syscallconn_test.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/credentials/tls.go b/credentials/tls.go
index 1ba6f3a6b8f8..dfc4bfe06047 100644
--- a/credentials/tls.go
+++ b/credentials/tls.go
@@ -241,6 +241,10 @@ var cipherSuiteLookup = map[uint16]string{
 	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256:   "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
 	tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305:    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
 	tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305:  "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+	// Go 1.12
+	tls.TLS_AES_128_GCM_SHA256:       "TLS_AES_128_GCM_SHA256",
+	tls.TLS_AES_256_GCM_SHA384:       "TLS_AES_256_GCM_SHA384",
+	tls.TLS_CHACHA20_POLY1305_SHA256: "TLS_CHACHA20_POLY1305_SHA256",
 }
 
 // cloneTLSConfig returns a shallow clone of the exported
diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go
index 9a67aabc3764..b1daa1cae9c8 100644
--- a/examples/features/xds/client/main.go
+++ b/examples/features/xds/client/main.go
@@ -1,5 +1,3 @@
-// +build go1.11
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/examples/features/xds/server/main.go b/examples/features/xds/server/main.go
index 512c4839711a..7e0815645e5a 100644
--- a/examples/features/xds/server/main.go
+++ b/examples/features/xds/server/main.go
@@ -1,5 +1,3 @@
-// +build go1.11
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/install_gae.sh b/install_gae.sh
deleted file mode 100755
index 7c7bcada5044..000000000000
--- a/install_gae.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-TMP=$(mktemp -d /tmp/sdk.XXX) \
-&& curl -o $TMP.zip "https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_linux_amd64-1.9.68.zip" \
-&& unzip -q $TMP.zip -d $TMP \
-&& export PATH="$PATH:$TMP/go_appengine"
diff --git a/internal/channelz/types_linux.go b/internal/channelz/types_linux.go
index 692dd6181778..1b1c4cce34a9 100644
--- a/internal/channelz/types_linux.go
+++ b/internal/channelz/types_linux.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/internal/channelz/types_nonlinux.go b/internal/channelz/types_nonlinux.go
index 19c2fc521dcf..501770791524 100644
--- a/internal/channelz/types_nonlinux.go
+++ b/internal/channelz/types_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux appengine
+// +build !linux
 
 /*
  *
@@ -37,6 +37,6 @@ type SocketOptionData struct {
 // Windows OS doesn't support Socket Option
 func (s *SocketOptionData) Getsockopt(fd uintptr) {
 	once.Do(func() {
-		logger.Warning("Channelz: socket options are not supported on non-linux os and appengine.")
+		logger.Warning("Channelz: socket options are not supported on non-linux os.")
 	})
 }
diff --git a/internal/channelz/util_linux.go b/internal/channelz/util_linux.go
index fdf409d55de3..49432bebf5f4 100644
--- a/internal/channelz/util_linux.go
+++ b/internal/channelz/util_linux.go
@@ -1,4 +1,4 @@
-// +build linux,!appengine
+// +build linux
 
 /*
  *
diff --git a/internal/channelz/util_nonlinux.go b/internal/channelz/util_nonlinux.go
index 8864a0811164..d600417fb8c2 100644
--- a/internal/channelz/util_nonlinux.go
+++ b/internal/channelz/util_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux appengine
+// +build !linux
 
 /*
  *
diff --git a/internal/channelz/util_test.go b/internal/channelz/util_test.go
index 9efe3873924c..2621e7410d61 100644
--- a/internal/channelz/util_test.go
+++ b/internal/channelz/util_test.go
@@ -1,4 +1,4 @@
-// +build linux,go1.10,!appengine
+// +build linux
 
 /*
  *
diff --git a/internal/credentials/gobefore110.go b/internal/credentials/gobefore110.go
deleted file mode 100644
index 743713e19f8d..000000000000
--- a/internal/credentials/gobefore110.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// +build !go1.10
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package credentials
-
-import (
-	"crypto/tls"
-	"net/url"
-)
-
-//TODO(ZhenLian): delete this file when we remove Go 1.9 tests.
-func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
-	return nil
-}
diff --git a/internal/credentials/go110.go b/internal/credentials/spiffe.go
similarity index 99%
rename from internal/credentials/go110.go
rename to internal/credentials/spiffe.go
index d55b5203626e..c86813061ef5 100644
--- a/internal/credentials/go110.go
+++ b/internal/credentials/spiffe.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/internal/credentials/go110_test.go b/internal/credentials/spiffe_test.go
similarity index 99%
rename from internal/credentials/go110_test.go
rename to internal/credentials/spiffe_test.go
index 19266cced492..324874e6e396 100644
--- a/internal/credentials/go110_test.go
+++ b/internal/credentials/spiffe_test.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/internal/profiling/buffer/buffer.go b/internal/profiling/buffer/buffer.go
index 45745cd09197..f4cd4201de11 100644
--- a/internal/profiling/buffer/buffer.go
+++ b/internal/profiling/buffer/buffer.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/internal/profiling/buffer/buffer_appengine.go b/internal/profiling/buffer/buffer_appengine.go
deleted file mode 100644
index c92599e5b9c0..000000000000
--- a/internal/profiling/buffer/buffer_appengine.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// +build appengine
-
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package buffer
-
-// CircularBuffer is a no-op implementation for appengine builds.
-//
-// Appengine does not support stats because of lack of the support for unsafe
-// pointers, which are necessary to efficiently store and retrieve things into
-// and from a circular buffer. As a result, Push does not do anything and Drain
-// returns an empty slice.
-type CircularBuffer struct{}
-
-// NewCircularBuffer returns a no-op for appengine builds.
-func NewCircularBuffer(size uint32) (*CircularBuffer, error) {
-	return nil, nil
-}
-
-// Push returns a no-op for appengine builds.
-func (cb *CircularBuffer) Push(x interface{}) {
-}
-
-// Drain returns a no-op for appengine builds.
-func (cb *CircularBuffer) Drain() []interface{} {
-	return nil
-}
diff --git a/internal/profiling/buffer/buffer_test.go b/internal/profiling/buffer/buffer_test.go
index 86bd77d4a2e6..a7f3b61e4afa 100644
--- a/internal/profiling/buffer/buffer_test.go
+++ b/internal/profiling/buffer/buffer_test.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/internal/profiling/profiling_test.go b/internal/profiling/profiling_test.go
index f20654492693..a6184a16bb4f 100644
--- a/internal/profiling/profiling_test.go
+++ b/internal/profiling/profiling_test.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go
index c50468a0fc89..3743bd45afbc 100644
--- a/internal/syscall/syscall_linux.go
+++ b/internal/syscall/syscall_linux.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2018 gRPC authors.
@@ -43,7 +41,7 @@ func GetCPUTime() int64 {
 	return ts.Nano()
 }
 
-// Rusage is an alias for syscall.Rusage under linux non-appengine environment.
+// Rusage is an alias for syscall.Rusage under linux environment.
 type Rusage syscall.Rusage
 
 // GetRusage returns the resource usage of current process.
diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go
index adae60d65188..95b713437489 100644
--- a/internal/syscall/syscall_nonlinux.go
+++ b/internal/syscall/syscall_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux appengine
+// +build !linux
 
 /*
  *
@@ -35,40 +35,40 @@ var logger = grpclog.Component("core")
 
 func log() {
 	once.Do(func() {
-		logger.Info("CPU time info is unavailable on non-linux or appengine environment.")
+		logger.Info("CPU time info is unavailable on non-linux environment.")
 	})
 }
 
 // GetCPUTime returns the how much CPU time has passed since the start of this process.
-// It always returns 0 under non-linux or appengine environment.
+// It always returns 0 under non-linux environment.
 func GetCPUTime() int64 {
 	log()
 	return 0
 }
 
-// Rusage is an empty struct under non-linux or appengine environment.
+// Rusage is an empty struct under non-linux environment.
 type Rusage struct{}
 
-// GetRusage is a no-op function under non-linux or appengine environment.
+// GetRusage is a no-op function under non-linux environment.
 func GetRusage() (rusage *Rusage) {
 	log()
 	return nil
 }
 
 // CPUTimeDiff returns the differences of user CPU time and system CPU time used
-// between two Rusage structs. It a no-op function for non-linux or appengine environment.
+// between two Rusage structs. It a no-op function for non-linux environment.
 func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) {
 	log()
 	return 0, 0
 }
 
-// SetTCPUserTimeout is a no-op function under non-linux or appengine environments
+// SetTCPUserTimeout is a no-op function under non-linux environments
 func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error {
 	log()
 	return nil
 }
 
-// GetTCPUserTimeout is a no-op function under non-linux or appengine environments
+// GetTCPUserTimeout is a no-op function under non-linux environments
 // a negative return value indicates the operation is not supported
 func GetTCPUserTimeout(conn net.Conn) (int, error) {
 	log()
diff --git a/interop/grpclb_fallback/client.go b/interop/grpclb_fallback/client.go
index ce7cc8e66cde..853c9a4353f1 100644
--- a/interop/grpclb_fallback/client.go
+++ b/interop/grpclb_fallback/client.go
@@ -1,6 +1,4 @@
 // +build linux
-// +build !appengine
-// +build go1.11
 
 /*
  *
diff --git a/security/advancedtls/sni_110.go b/security/advancedtls/sni.go
similarity index 98%
rename from security/advancedtls/sni_110.go
rename to security/advancedtls/sni.go
index 5c9a6ae13a0f..00e551fac296 100644
--- a/security/advancedtls/sni_110.go
+++ b/security/advancedtls/sni.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/security/advancedtls/sni_before_110.go b/security/advancedtls/sni_before_110.go
deleted file mode 100644
index 180e3a05d494..000000000000
--- a/security/advancedtls/sni_before_110.go
+++ /dev/null
@@ -1,41 +0,0 @@
-// +build !go1.10
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package advancedtls
-
-import (
-	"crypto/tls"
-	"fmt"
-)
-
-// buildGetCertificates returns the first element of o.GetCertificates.
-func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
-	if o.GetCertificates == nil {
-		return nil, fmt.Errorf("function GetCertificates must be specified")
-	}
-	certificates, err := o.GetCertificates(clientHello)
-	if err != nil {
-		return nil, err
-	}
-	if len(certificates) == 0 {
-		return nil, fmt.Errorf("no certificates configured")
-	}
-	return certificates[0], nil
-}
diff --git a/security/advancedtls/sni_test_before_110.go b/security/advancedtls/sni_test_before_110.go
deleted file mode 100644
index e31e2e6ee759..000000000000
--- a/security/advancedtls/sni_test_before_110.go
+++ /dev/null
@@ -1,108 +0,0 @@
-// +build !go1.10
-
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package advancedtls
-
-import (
-	"crypto/tls"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/security/advancedtls/testdata"
-)
-
-// TestGetCertificatesSNI tests SNI logic for go1.9.
-func TestGetCertificatesSNI(t *testing.T) {
-	// Load server certificates for setting the serverGetCert callback function.
-	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err)
-	}
-	serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err)
-	}
-	serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err)
-	}
-
-	tests := []struct {
-		desc          string
-		serverGetCert func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
-		serverName    string
-		wantCert      tls.Certificate
-	}{
-		{
-			desc: "Select serverCert1",
-			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
-			},
-			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
-			serverName: "foo.bar.com",
-			wantCert:   serverCert1,
-		},
-		{
-			desc: "Select serverCert2",
-			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
-			},
-			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
-			serverName: "foo.bar.server2.com",
-			wantCert:   serverCert1,
-		},
-		{
-			desc: "Select serverCert3",
-			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
-			},
-			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
-			serverName: "google.com",
-			wantCert:   serverCert1,
-		},
-	}
-	for _, test := range tests {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			serverOptions := &ServerOptions{
-				GetCertificates: test.serverGetCert,
-			}
-			serverConfig, err := serverOptions.config()
-			if err != nil {
-				t.Fatalf("serverOptions.config() failed: %v", err)
-			}
-			pointFormatUncompressed := uint8(0)
-			clientHello := &tls.ClientHelloInfo{
-				CipherSuites:      []uint16{tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA},
-				ServerName:        test.serverName,
-				SupportedCurves:   []tls.CurveID{tls.CurveP256},
-				SupportedPoints:   []uint8{pointFormatUncompressed},
-				SupportedVersions: []uint16{tls.VersionTLS10},
-			}
-			gotCertificate, err := serverConfig.GetCertificate(clientHello)
-			if err != nil {
-				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
-			}
-			if !cmp.Equal(gotCertificate, test.wantCert, cmp.AllowUnexported(tls.Certificate{})) {
-				t.Errorf("GetCertificates() = %v, want %v", gotCertificate, test.wantCert)
-			}
-		})
-	}
-}
diff --git a/security/advancedtls/sni_test_110.go b/security/advancedtls/sni_test_disabled.go
similarity index 99%
rename from security/advancedtls/sni_test_110.go
rename to security/advancedtls/sni_test_disabled.go
index 130ccde59050..3e9e19c11be9 100644
--- a/security/advancedtls/sni_test_110.go
+++ b/security/advancedtls/sni_test_disabled.go
@@ -1,5 +1,3 @@
-// +build go1.10
-
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/test/channelz_linux_go110_test.go b/test/channelz_linux_go110_test.go
index ede329ad6297..dea374bfc08b 100644
--- a/test/channelz_linux_go110_test.go
+++ b/test/channelz_linux_go110_test.go
@@ -1,4 +1,4 @@
-// +build go1.10,linux,!appengine
+// +build linux
 
 /*
  *
diff --git a/test/go_vet/vet.go b/test/go_vet/vet.go
deleted file mode 100644
index 475e8d683fc3..000000000000
--- a/test/go_vet/vet.go
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- *
- * Copyright 2018 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// vet checks whether files that are supposed to be built on appengine running
-// Go 1.10 or earlier import an unsupported package (e.g. "unsafe", "syscall").
-package main
-
-import (
-	"fmt"
-	"go/build"
-	"os"
-)
-
-func main() {
-	fail := false
-	b := build.Default
-	b.BuildTags = []string{"appengine", "appenginevm"}
-	argsWithoutProg := os.Args[1:]
-	for _, dir := range argsWithoutProg {
-		p, err := b.Import(".", dir, 0)
-		if _, ok := err.(*build.NoGoError); ok {
-			continue
-		} else if err != nil {
-			fmt.Printf("build.Import failed due to %v\n", err)
-			fail = true
-			continue
-		}
-		for _, pkg := range p.Imports {
-			if pkg == "syscall" || pkg == "unsafe" {
-				fmt.Printf("Package %s/%s importing %s package without appengine build tag is NOT ALLOWED!\n", p.Dir, p.Name, pkg)
-				fail = true
-			}
-		}
-	}
-	if fail {
-		os.Exit(1)
-	}
-}
diff --git a/vet.sh b/vet.sh
index 2116d1987eef..8c14c7d3618e 100755
--- a/vet.sh
+++ b/vet.sh
@@ -89,10 +89,6 @@ not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go"
 # - Ensure all xds proto imports are renamed to *pb or *grpc.
 git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' | not grep -v 'pb "\|grpc "'
 
-# - Check imports that are illegal in appengine (until Go 1.11).
-# TODO: Remove when we drop Go 1.10 support
-go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go
-
 # - gofmt, goimports, golint (with exceptions for generated code), go vet.
 gofmt -s -d -l . 2>&1 | fail_on_output
 goimports -l . 2>&1 | not grep -vE "\.pb\.go"
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 4a825aa35d84..18c28db346be 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -1,5 +1,3 @@
-// +build !appengine
-
 /*
  *
  * Copyright 2019 gRPC authors.

From 8bec2f5d898fdbcbc4dbfc486df8a7d4a7219a39 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 31 Jul 2020 11:00:10 -0700
Subject: [PATCH 142/481] Re-add Go1.11 support (#3776)

---
 .travis.yml         |  2 ++
 credentials/go12.go | 30 ++++++++++++++++++++++++++++++
 credentials/tls.go  |  4 ----
 3 files changed, 32 insertions(+), 4 deletions(-)
 create mode 100644 credentials/go12.go

diff --git a/.travis.yml b/.travis.yml
index 00287d666a5a..9097478f856d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,8 @@ matrix:
     env: GO111MODULE=on
   - go: 1.12.x
     env: GO111MODULE=on
+  - go: 1.11.x  # Keep until interop tests no longer require Go1.11
+    env: GO111MODULE=on
 
 go_import_path: google.golang.org/grpc
 
diff --git a/credentials/go12.go b/credentials/go12.go
new file mode 100644
index 000000000000..ccbf35b33125
--- /dev/null
+++ b/credentials/go12.go
@@ -0,0 +1,30 @@
+// +build go1.12
+
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import "crypto/tls"
+
+// This init function adds cipher suite constants only defined in Go 1.12.
+func init() {
+	cipherSuiteLookup[tls.TLS_AES_128_GCM_SHA256] = "TLS_AES_128_GCM_SHA256"
+	cipherSuiteLookup[tls.TLS_AES_256_GCM_SHA384] = "TLS_AES_256_GCM_SHA384"
+	cipherSuiteLookup[tls.TLS_CHACHA20_POLY1305_SHA256] = "TLS_CHACHA20_POLY1305_SHA256"
+}
diff --git a/credentials/tls.go b/credentials/tls.go
index dfc4bfe06047..1ba6f3a6b8f8 100644
--- a/credentials/tls.go
+++ b/credentials/tls.go
@@ -241,10 +241,6 @@ var cipherSuiteLookup = map[uint16]string{
 	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256:   "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
 	tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305:    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
 	tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305:  "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
-	// Go 1.12
-	tls.TLS_AES_128_GCM_SHA256:       "TLS_AES_128_GCM_SHA256",
-	tls.TLS_AES_256_GCM_SHA384:       "TLS_AES_256_GCM_SHA384",
-	tls.TLS_CHACHA20_POLY1305_SHA256: "TLS_CHACHA20_POLY1305_SHA256",
 }
 
 // cloneTLSConfig returns a shallow clone of the exported

From 5f7b337d951f2b7e13cdea854c5893ca940a0c77 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 4 Aug 2020 17:46:48 -0700
Subject: [PATCH 143/481] Add some appengine build constraints. (#3787)

Support for Go1.9 and appengine was removed in https://github.com/grpc/grpc-go/pull/3767. But it looks like we still need some way to do things differently for certain appengine builds.
---
 channelz/service/func_nonlinux.go        |  2 +-
 internal/channelz/types_nonlinux.go      |  2 +-
 internal/channelz/util_nonlinux.go       |  2 +-
 internal/credentials/spiffe_appengine.go | 31 ++++++++++++++++++++++++
 internal/syscall/syscall_nonlinux.go     |  2 +-
 security/advancedtls/sni_appengine.go    | 30 +++++++++++++++++++++++
 6 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 internal/credentials/spiffe_appengine.go
 create mode 100644 security/advancedtls/sni_appengine.go

diff --git a/channelz/service/func_nonlinux.go b/channelz/service/func_nonlinux.go
index 64ecea947de0..eb53334ed0d1 100644
--- a/channelz/service/func_nonlinux.go
+++ b/channelz/service/func_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux appengine
 
 /*
  *
diff --git a/internal/channelz/types_nonlinux.go b/internal/channelz/types_nonlinux.go
index 501770791524..da7351e9709a 100644
--- a/internal/channelz/types_nonlinux.go
+++ b/internal/channelz/types_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux appengine
 
 /*
  *
diff --git a/internal/channelz/util_nonlinux.go b/internal/channelz/util_nonlinux.go
index d600417fb8c2..8864a0811164 100644
--- a/internal/channelz/util_nonlinux.go
+++ b/internal/channelz/util_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux appengine
 
 /*
  *
diff --git a/internal/credentials/spiffe_appengine.go b/internal/credentials/spiffe_appengine.go
new file mode 100644
index 000000000000..af6f57719768
--- /dev/null
+++ b/internal/credentials/spiffe_appengine.go
@@ -0,0 +1,31 @@
+// +build appengine
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import (
+	"crypto/tls"
+	"net/url"
+)
+
+// SPIFFEIDFromState is a no-op for appengine builds.
+func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
+	return nil
+}
diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go
index 95b713437489..51159e638d77 100644
--- a/internal/syscall/syscall_nonlinux.go
+++ b/internal/syscall/syscall_nonlinux.go
@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux appengine
 
 /*
  *
diff --git a/security/advancedtls/sni_appengine.go b/security/advancedtls/sni_appengine.go
new file mode 100644
index 000000000000..fffbb0107ddd
--- /dev/null
+++ b/security/advancedtls/sni_appengine.go
@@ -0,0 +1,30 @@
+// +build appengine
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+)
+
+// buildGetCertificates is a no-op for appengine builds.
+func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
+	return nil, nil
+}

From 94d8df7536d3c4d178cb498e0bcc8623ae4651e2 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 5 Aug 2020 08:54:38 -0700
Subject: [PATCH 144/481] xds: Avoid mutex copy in log statement. (#3788)

---
 xds/internal/client/v2/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index 627b9e8e5a95..c5a79f54fc33 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -53,7 +53,7 @@ func (clientBuilder) Version() version.TransportAPI {
 func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
 	nodeProto, ok := opts.NodeProto.(*v2corepb.Node)
 	if !ok {
-		return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v2corepb.Node{})
+		return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, (*v2corepb.Node)(nil))
 	}
 	v2c := &client{
 		cc:        cc,

From fd393c8989c2d053f34469cc34c9df27f423526b Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 5 Aug 2020 09:55:07 -0700
Subject: [PATCH 145/481] testdata: Update testdata certs. (#3786)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* This will be used in certprovider tests where we would want more than one server and client certs.
* Also, updated existing usages of these certs to point to the new  files.
* Also copy over the required certs/key files. This avoids the example gomodule from depending on gRPC testdata package which should be able to change independently.
* Fix interop test's SAN.
---
 balancer/rls/internal/balancer_test.go        |   4 +-
 benchmark/worker/benchmark_client.go          |   2 +-
 benchmark/worker/benchmark_server.go          |   4 +-
 clientconn_test.go                            |   8 +-
 credentials/credentials_test.go               |   4 +-
 credentials/tls/certprovider/store_test.go    |   4 +-
 examples/data/data.go                         |  44 ++++++++
 examples/data/x509/ca_cert.pem                |  34 ++++++
 examples/data/x509/server_cert.pem            |  32 ++++++
 examples/data/x509/server_key.pem             |  51 +++++++++
 .../features/authentication/client/main.go    |   4 +-
 .../features/authentication/server/main.go    |   4 +-
 examples/features/encryption/README.md        |  10 +-
 .../features/encryption/TLS/client/main.go    |   4 +-
 .../features/encryption/TLS/server/main.go    |   4 +-
 examples/features/interceptor/client/main.go  |   4 +-
 examples/features/interceptor/server/main.go  |   4 +-
 examples/route_guide/client/client.go         |   4 +-
 examples/route_guide/server/server.go         |   6 +-
 interop/client/client.go                      |   2 +-
 interop/fake_grpclb/fake_grpclb.go            |   4 +-
 interop/interop_test.sh                       |   2 +-
 interop/server/server.go                      |   4 +-
 stress/client/main.go                         |   2 +-
 test/balancer_test.go                         |   2 +-
 test/channelz_test.go                         |   2 +-
 test/creds_test.go                            |   6 +-
 test/end2end_test.go                          |   8 +-
 testdata/ca.pem                               |  20 ----
 testdata/server1.key                          |  28 -----
 testdata/server1.pem                          |  22 ----
 testdata/x509/README.md                       | 106 ++++++++++++++++++
 testdata/x509/client1_cert.pem                |  32 ++++++
 testdata/x509/client1_key.pem                 |  51 +++++++++
 testdata/x509/client2_cert.pem                |  32 ++++++
 testdata/x509/client2_key.pem                 |  51 +++++++++
 testdata/x509/client_ca_cert.pem              |  34 ++++++
 testdata/x509/client_ca_key.pem               |  52 +++++++++
 testdata/x509/create.sh                       | 104 +++++++++++++++++
 testdata/x509/openssl.cnf                     |  28 +++++
 testdata/x509/server1_cert.pem                |  32 ++++++
 testdata/x509/server1_key.pem                 |  51 +++++++++
 testdata/x509/server2_cert.pem                |  32 ++++++
 testdata/x509/server2_key.pem                 |  51 +++++++++
 testdata/x509/server_ca_cert.pem              |  34 ++++++
 testdata/x509/server_ca_key.pem               |  52 +++++++++
 46 files changed, 954 insertions(+), 121 deletions(-)
 create mode 100644 examples/data/data.go
 create mode 100644 examples/data/x509/ca_cert.pem
 create mode 100644 examples/data/x509/server_cert.pem
 create mode 100644 examples/data/x509/server_key.pem
 delete mode 100644 testdata/ca.pem
 delete mode 100644 testdata/server1.key
 delete mode 100644 testdata/server1.pem
 create mode 100644 testdata/x509/README.md
 create mode 100644 testdata/x509/client1_cert.pem
 create mode 100644 testdata/x509/client1_key.pem
 create mode 100644 testdata/x509/client2_cert.pem
 create mode 100644 testdata/x509/client2_key.pem
 create mode 100644 testdata/x509/client_ca_cert.pem
 create mode 100644 testdata/x509/client_ca_key.pem
 create mode 100755 testdata/x509/create.sh
 create mode 100644 testdata/x509/openssl.cnf
 create mode 100644 testdata/x509/server1_cert.pem
 create mode 100644 testdata/x509/server1_key.pem
 create mode 100644 testdata/x509/server2_cert.pem
 create mode 100644 testdata/x509/server2_key.pem
 create mode 100644 testdata/x509/server_ca_cert.pem
 create mode 100644 testdata/x509/server_ca_key.pem

diff --git a/balancer/rls/internal/balancer_test.go b/balancer/rls/internal/balancer_test.go
index 973f5e0cd608..d6a98aa9ab00 100644
--- a/balancer/rls/internal/balancer_test.go
+++ b/balancer/rls/internal/balancer_test.go
@@ -189,11 +189,11 @@ func (s) TestUpdateControlChannelTimeout(t *testing.T) {
 // TestUpdateControlChannelWithCreds tests the scenario where the control
 // channel is to established with credentials from the parent channel.
 func (s) TestUpdateControlChannelWithCreds(t *testing.T) {
-	sCreds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	sCreds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		t.Fatalf("credentials.NewServerTLSFromFile(server1.pem, server1.key) = %v", err)
 	}
-	cCreds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "")
+	cCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "")
 	if err != nil {
 		t.Fatalf("credentials.NewClientTLSFromFile(ca.pem) = %v", err)
 	}
diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go
index abb5bc9928aa..bb3097f8c835 100644
--- a/benchmark/worker/benchmark_client.go
+++ b/benchmark/worker/benchmark_client.go
@@ -124,7 +124,7 @@ func createConns(config *testpb.ClientConfig) ([]*grpc.ClientConn, func(), error
 	// Check and set security options.
 	if config.SecurityParams != nil {
 		if *caFile == "" {
-			*caFile = testdata.Path("ca.pem")
+			*caFile = testdata.Path("x509/server_ca.pem")
 		}
 		creds, err := credentials.NewClientTLSFromFile(*caFile, config.SecurityParams.ServerHostOverride)
 		if err != nil {
diff --git a/benchmark/worker/benchmark_server.go b/benchmark/worker/benchmark_server.go
index 56f7e6ee265f..0c7c64804abb 100644
--- a/benchmark/worker/benchmark_server.go
+++ b/benchmark/worker/benchmark_server.go
@@ -95,10 +95,10 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma
 	// Set security options.
 	if config.SecurityParams != nil {
 		if *certFile == "" {
-			*certFile = testdata.Path("server1.pem")
+			*certFile = testdata.Path("x509/server1_cert.pem")
 		}
 		if *keyFile == "" {
-			*keyFile = testdata.Path("server1.key")
+			*keyFile = testdata.Path("x509/server1_key.pem")
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {
diff --git a/clientconn_test.go b/clientconn_test.go
index 447a9bb14072..6c61666b7efb 100644
--- a/clientconn_test.go
+++ b/clientconn_test.go
@@ -362,7 +362,7 @@ func (s) TestWithTimeout(t *testing.T) {
 func (s) TestWithTransportCredentialsTLS(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
 	defer cancel()
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		t.Fatalf("Failed to create credentials %v", err)
 	}
@@ -389,7 +389,7 @@ func (s) TestDefaultAuthority(t *testing.T) {
 
 func (s) TestTLSServerNameOverwrite(t *testing.T) {
 	overwriteServerName := "over.write.server.name"
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), overwriteServerName)
+	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), overwriteServerName)
 	if err != nil {
 		t.Fatalf("Failed to create credentials %v", err)
 	}
@@ -417,7 +417,7 @@ func (s) TestWithAuthority(t *testing.T) {
 
 func (s) TestWithAuthorityAndTLS(t *testing.T) {
 	overwriteServerName := "over.write.server.name"
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), overwriteServerName)
+	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), overwriteServerName)
 	if err != nil {
 		t.Fatalf("Failed to create credentials %v", err)
 	}
@@ -556,7 +556,7 @@ func (c securePerRPCCredentials) RequireTransportSecurity() bool {
 }
 
 func (s) TestCredentialsMisuse(t *testing.T) {
-	tlsCreds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+	tlsCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		t.Fatalf("Failed to create authenticator %v", err)
 	}
diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go
index 5ff4850453bd..e36145250695 100644
--- a/credentials/credentials_test.go
+++ b/credentials/credentials_test.go
@@ -283,7 +283,7 @@ func clientHandle(t *testing.T, hs func(net.Conn, string) (AuthInfo, error), lis
 
 // Server handshake implementation in gRPC.
 func gRPCServerHandshake(conn net.Conn) (AuthInfo, error) {
-	serverTLS, err := NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	serverTLS, err := NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		return nil, err
 	}
@@ -305,7 +305,7 @@ func gRPCClientHandshake(conn net.Conn, lisAddr string) (AuthInfo, error) {
 }
 
 func tlsServerHandshake(conn net.Conn) (AuthInfo, error) {
-	cert, err := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		return nil, err
 	}
diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go
index 6cd02e012dfb..33135d41c854 100644
--- a/credentials/tls/certprovider/store_test.go
+++ b/credentials/tls/certprovider/store_test.go
@@ -126,12 +126,12 @@ func (p *fakeProvider) Close() {
 // loadKeyMaterials is a helper to read cert/key files from testdata and convert
 // them into a KeyMaterial struct.
 func loadKeyMaterials() (*KeyMaterial, error) {
-	certs, err := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	certs, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		return nil, err
 	}
 
-	pemData, err := ioutil.ReadFile(testdata.Path("ca.pem"))
+	pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
 	if err != nil {
 		return nil, err
 	}
diff --git a/examples/data/data.go b/examples/data/data.go
new file mode 100644
index 000000000000..c583755fdf34
--- /dev/null
+++ b/examples/data/data.go
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package data provides convenience routines to access files in the data
+// directory.
+package data
+
+import (
+	"path/filepath"
+	"runtime"
+)
+
+// basepath is the root directory of this package.
+var basepath string
+
+func init() {
+	_, currentFile, _, _ := runtime.Caller(0)
+	basepath = filepath.Dir(currentFile)
+}
+
+// Path returns the absolute path the given relative file or directory path,
+// relative to the google.golang.org/grpc/examples/data directory in the
+// user's GOPATH.  If rel is already absolute, it is returned unmodified.
+func Path(rel string) string {
+	if filepath.IsAbs(rel) {
+		return rel
+	}
+
+	return filepath.Join(basepath, rel)
+}
diff --git a/examples/data/x509/ca_cert.pem b/examples/data/x509/ca_cert.pem
new file mode 100644
index 000000000000..eee033e8cb05
--- /dev/null
+++ b/examples/data/x509/ca_cert.pem
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF6jCCA9KgAwIBAgIJAKnJpgBC9CHNMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD
+MRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTAeFw0yMDA4MDQwMTU5NTdaFw0zMDA4
+MDIwMTU5NTdaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD
+U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMZFKSUi+PlQ6z/aTz1Jp9lqrFAY
+38cEIzpxS9ktQiWvLoYICImXRFhCH/h+WjmiyV8zYHcbft63BTUwgXJFuE0cxsJY
+mqOUYL2wTD5PzgoN0B9KVgKyyi0SQ6WH9+D2ZvYAolHb1l6pYuxxk1bQL2OA80Cc
+K659UioynIQtJ52NRqGRDI2EYsC9XRuhfddnDu/RwBaiv3ix84R3VAqcgRyOeGwH
+cX2e+aX0m6ULnsiyPXG9y9wQi956CGGZimInV63S+sU3Mc6PuUt8rwFlmSXCZ/07
+D8No5ljNUo6Vt2BpAMQzSz+SU4PUFE7Vxbq4ypI+2ZbkI80YjDwF52/pMauqZFIP
+Kjw0b2yyWD/F4hLmR7Rx9d8EFWRLZm2VYSVMiQTwANpb+uL7+kH8UE3QF7tryH8K
+G65mMh18XiERgSAWgs5Z8j/B1W5bl17PVx2Ii1dYp0IquyAVjCIKRrFituvoXXZj
+FHHpb/aUDpW0SYrT5dmDhAAGFkYfMTFd4EOj6bWepZtRRjPeIHR9B2yx8U0tFSMf
+tuHCj95l2izJDUfKhVIkigpbRrElI2QqXAPIyIOqcdzlgtI6DIanCd/CwsfdyaEs
+7AnW2mFWarbkxpw92RdGxYy6WXbdM+2EdY+cWKys06upINcnG2zvkCflAE39fg9F
+BVCJC71oO3laXnf7AgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
+FgQUBuToaw2a+AV/vfbooJn3yzwA3lMwgYAGA1UdIwR5MHeAFAbk6GsNmvgFf732
+6KCZ98s8AN5ToVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV
+BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1zZXJ2ZXJfY2GC
+CQCpyaYAQvQhzTAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBALUz
+P2SiZAXZDwCH8kzHbLqsqacSM81bUSuG153t3fhwZU8hzXgQqifFububLkrLaRCj
+VvtIS3XsbHmKYD1TBOOCZy5zE2KdpWYW47LmogBqUllKCSD099UHFB2YUepK9Zci
+oxYJMhNWIhkoJ/NJMp70A8PZtxUvZafeUQl6xueo1yPbfQubg0lG9Pp2xkmTypSv
+WJkpRyX8GSJYFoFFYdNcvICVw7E/Zg+PGXe8gjpAGWW8KxxaohPsdLid6f3KauJM
+UCi/WQECzIpNzxQDSqnGeoqbZp+2y6mhgECQ3mG/K75n0fX0aV88DNwTd1o0xOpv
+lHJo8VD9mvwnapbm/Bc7NWIzCjL8fo0IviRkmAuoz525eBy6NsUCf1f432auvNbg
+OUaGGrY6Kse9sF8Tsc8XMoT9AfGQaR8Ay7oJHjaCZccvuxpB2n//L1UAjMRPYd2y
+XAiSN2xz7WauUh4+v48lKbWa+dwn1G0pa6ZGB7IGBUbgva8Fi3iqVh3UZoz+0PFM
+qVLG2SzhfMTMHg0kF+rI4eOcEKc1j3A83DmTTPZDz3APn53weJLJhKzrgQiI1JRW
+boAJ4VFQF6zjxeecCIIiekH6saYKnol2yL6ksm0jyHoFejkrHWrzoRAwIhTf9avj
+G7QS5fiSQk4PXCX42J5aS/zISy85RT120bkBjV/P
+-----END CERTIFICATE-----
diff --git a/examples/data/x509/server_cert.pem b/examples/data/x509/server_cert.pem
new file mode 100644
index 000000000000..3e48a52fd108
--- /dev/null
+++ b/examples/data/x509/server_cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx
+CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV
+BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIwMDgwNDAxNTk1OFoXDTMwMDgwMjAxNTk1
+OFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMTCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAKonkszKvSg1IUvpfW3PAeDPLgLrXboOWJCXv3RD
+5q6vf29+IBCaljSJmU6T7SplokUML5ZkY6adjX6awG+LH3tOMg9zvXpHuSPRpFUk
+2oLFtaWuzJ+NC5HIM0wWDvdZ6KQsiPFbNxk2Rhkk+QKsiiptZy2yf/AbDY0sVieZ
+BJZJ+os+BdFIk7+XUgDutPdSAutTANhrGycYa4iYAfDGQApz3sndSSsM2KVc0w5F
+gW6w2UBC4ggc1ZaWdbVtkYo+0dCsrl1J7WUNsz8v8mjGsvm9eFuJjKFBiDhCF+xg
+4Xzu1Wz7zV97994la/xMImQR4QDdky9IgKcJMVUGua6U0GE5lmt2wnd3aAI228Vm
+6SnK7kKvnD8vRUyM9ByeRoMlrAuYb0AjnVBr/MTFbOaii6w2v3RjU0j6YFzp8+67
+ihOW9nkb1ayqSXD3T4QUD0p75Ne7/zz1r2amIh9pmSJlugLexVDpb86vXg9RnXjb
+Zn2HTEkXsL5eHUIlQzuhK+gdmj+MLGf/Yzp3fdaJsA0cJfMjj5Ubb2gR4VwzrHy9
+AD2Kjjzs06pTtpULChwpr9IBTLEsZfw/4uW4II4pfe6Rwn4bGHFifjx0+3svlsSo
+jdHcXEMHvdRPhWGUZ0rne+IK6Qxgb3OMZu7a04vV0RqvgovxM6hre3e0UzBJG45Y
+qlQjAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFFL5HUzehgKNfgdz
+4nuw5fru5OTPMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh
+bXBsZS5jb20wDQYJKoZIhvcNAQEFBQADggIBAHMPYTF4StfSx9869EoitlEi7Oz2
+YTOForDbsY9i0VnIamhIi9CpjekAGLo8SVojeAk7UV3ayiu0hEMAHJWbicgWTwWM
+JvZWWfrIk/2WYyBWWTa711DuW26cvtbSebFzXsovNeTqMICiTeYbvOAK826UdH/o
+OqNiHL+UO5xR1Xmqa2hKmLSl5J1n+zgm94l6SROzc9c5YDzn03U+8dlhoyXCwlTv
+JRprOD+lupccxcKj5Tfh9/G6PjKsgxW+DZ+rvQV5f/l7c4m/bBrgS8tru4t2Xip0
+NhQW4qHnL0wXdTjaOG/1liLppjcp7SsP+vKF4shUvp+P8NQuAswBp/QtqUse5EYl
+EUARWrjEpV4OHSKThkMackMg5E32keiOvQE6iICxtU+m2V+C3xXM3G2cGlDDx5Ob
+tan0c9fZXoygrN2mc94GPogfwFGxwivajvvJIs/bsB3RkcIuLbi2UB76Wwoq+ZvH
+15xxNZI1rpaDhjEuqwbSGPMPVpFtF5VERgYQ9LaDgj7yorwSQ1YLY8R1y0vSiAR2
+2YeOaBH1ZLPF9v9os1iK4TIC8XQfPv7ll2WdDwfbe2ux5GVbDBD4bPhP9s3F4a+f
+oPhikWsUY4eN5CfS76x6xL0L60TL1AlWLlwuubTxpvNhv3GSyxjfunjcGiXDml20
+6S80qO4hepxzzjol
+-----END CERTIFICATE-----
diff --git a/examples/data/x509/server_key.pem b/examples/data/x509/server_key.pem
new file mode 100644
index 000000000000..e71ad0ac9753
--- /dev/null
+++ b/examples/data/x509/server_key.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEAqieSzMq9KDUhS+l9bc8B4M8uAutdug5YkJe/dEPmrq9/b34g
+EJqWNImZTpPtKmWiRQwvlmRjpp2NfprAb4sfe04yD3O9eke5I9GkVSTagsW1pa7M
+n40LkcgzTBYO91nopCyI8Vs3GTZGGST5AqyKKm1nLbJ/8BsNjSxWJ5kElkn6iz4F
+0UiTv5dSAO6091IC61MA2GsbJxhriJgB8MZACnPeyd1JKwzYpVzTDkWBbrDZQELi
+CBzVlpZ1tW2Rij7R0KyuXUntZQ2zPy/yaMay+b14W4mMoUGIOEIX7GDhfO7VbPvN
+X3v33iVr/EwiZBHhAN2TL0iApwkxVQa5rpTQYTmWa3bCd3doAjbbxWbpKcruQq+c
+Py9FTIz0HJ5GgyWsC5hvQCOdUGv8xMVs5qKLrDa/dGNTSPpgXOnz7ruKE5b2eRvV
+rKpJcPdPhBQPSnvk17v/PPWvZqYiH2mZImW6At7FUOlvzq9eD1GdeNtmfYdMSRew
+vl4dQiVDO6Er6B2aP4wsZ/9jOnd91omwDRwl8yOPlRtvaBHhXDOsfL0APYqOPOzT
+qlO2lQsKHCmv0gFMsSxl/D/i5bggjil97pHCfhsYcWJ+PHT7ey+WxKiN0dxcQwe9
+1E+FYZRnSud74grpDGBvc4xm7trTi9XRGq+Ci/EzqGt7d7RTMEkbjliqVCMCAwEA
+AQKCAgEAjU6UEVMFSBDnd/2OVtUlQCeOlIoWql8jmeEL9Gg3eTbx5AugYWmf+D2V
+fbZHrX/+BM2b74+rWkFZspyd14R4PpSv6jk6UASkcmS1zqfud8/tjIzgDli6FPVn
+9HYVM8IM+9qoV5hi56M1D8iuq1PS4m081Kx6p1IwLN93JSdksdL6KQz3E9jsKp5m
+UbPrwcDv/7JM723zfMJA+40Rf32EzalwicAl9YSTnrC57g428VAY+88Pm6EmmAqX
+8nXt+hs1b9EYdQziA5wfEgiljfIFzHVXMN3IVlrv35iz+XBzkqddw0ZSRkvTiz8U
+sNAhd22JqIhapVfWz+FIgM43Ag9ABUMNWoQlaT0+2KlhkL+cZ6J1nfpMTBEIatz0
+A/l4TGcvdDhREODrS5jrxwJNx/LMRENtFFnRzAPzX4RdkFvi8SOioAWRBvs1TZFo
+ZLq2bzDOzDjs+EPQVx0SmjZEiBRhI6nC8Way00IdQi3T546r6qTKfPmXgjl5/fVO
+J4adGVbEUnI/7+fqL2N82WVr+Le585EFP/6IL5FO++sAIGDqAOzEQhyRaLhmnz+D
+GboeS/Tac9XdymFbrEvEMB4EFS3nsZHTeahfiqVd/SuXFDTHZ6kiqXweuhfsP1uW
+7tGlnqtn+3zmLO6XRENPVvmjn7DhU255yjiKFdUqkajcoOYyWPECggEBANuYk+sr
+UTScvJoh/VRHuqd9NkVVIoqfoTN61x6V1OuNNcmjMWsOIsH+n4SifLlUW6xCKaSK
+8x8RJYfE9bnObv/NqM4DMhuaNd52bPKFi8IBbHSZpuRE/UEyJhMDpoto04H1GXx4
+1S49tndiNxQOv1/VojB4BH7kapY0yp30drK1CrocGN+YOUddxI9lOQpgt2AyoXVk
+ehdyamK4uzQmkMyyGQljrV5EQbmyPCqZ1l/d0MJ9DixOBxnPDR9Ov9qrG4Dy6S/k
+cH8PythqHTGTdlXgsBJaWEl2PyQupo3OhfiCV+79B9uxPfKvk5CIMVbnYxKgu+ly
+RKSTSX+GHVgNwicCggEBAMZcwQIAA+I39sTRg/Vn/MxmUBAu3h2+oJcuZ3FQh4v5
+SL80BWEsooK9Oe4MzxyWkU+8FieFu5G6iXaSx8f3Wv6j90IzA3g6Xr9M5xBm5qUN
+IqzF+hUZuKAEMY1NcPlFTa2NlrkT8JdfQvJ+D5QrcBIMFmg9cKG5x9yD7MfHTJkf
+ztMDFOwP3n7ahKRBowfe7/unAEFf6hYFtYjV+bqMDmBFVmk2CIVtjFgO9BNBQ/LB
+zGcnwo2VigWBIjRDF5BgV0v+2g0PZGaxJ362RigZjzJojx3gYj6kaZYX8yb6ttGo
+RPGt1A9woz6m0G0fLLMlce1dpbBAna14UVY7AEVt56UCggEAVvii/Oz3CINbHyB/
+GLYf8t3gdK03NPfr/FuWf4KQBYqz1txPYjsDARo7S2ifRTdn51186LIvgApmdtNH
+DwP3alClnpIdclktJKJ6m8LQi1HNBpEkTBwWwY9/DODRQT2PJ1VPdsDUja/baIT5
+k3QTz3zo85FVFnyYyky2QsDjkfup9/PQ1h2P8fftNW29naKYff0PfVMCF+80u0y2
+t/zeNHQE/nb/3unhrg4tTiIHiYhsedrVli6BGXOrms6xpYVHK1cJi/JJq8kxaWz9
+ivkAURrgISSu+sleUJI5XMiCvt3AveJxDk2wX0Gyi/eksuqJjoMiaV7cWOIMpfkT
+/h/U2QKCAQAFirvduXBiVpvvXccpCRG4CDe+bADKpfPIpYRAVzaiQ4GzzdlEoMGd
+k3nV28fBjbdbme6ohgT6ilKi3HD2dkO1j5Et6Uz0g/T3tUdTXvycqeRJHXLiOgi9
+d8CGqR456KTF74nBe/whzoiJS9pVkm0cI/hQSz8lVZJu58SqxDewo4HcxV5FRiA6
+PRKtoCPU6Xac+kp4iRx6JwiuXQQQIS+ZovZKFDdiuu/L2gcZrp4eXym9zA+UcxQb
+GUOCYEl9QCPQPLuM19w/Pj3TPXZyUlx81Q0Cka1NALzuc5bYhPKsot3iPrAJCmWV
+L4XtNozCKI6pSg+CABwnp4/mL9nPFsX9AoIBAQDHiDhG9jtBdgtAEog6oL2Z98qR
+u5+nONtLQ61I5R22eZYOgWfxnz08fTtpaHaVWNLNzF0ApyxjxD+zkFHcMJDUuHkR
+O0yxUbCaof7u8EFtq8P9ux4xjtCnZW+9da0Y07zBrcXTsHYnAOiqNbtvVYd6RPiW
+AaE61hgvj1c9/BQh2lUcroQx+yJI8uAAQrfYtXzm90rb6qk6rWy4li2ybMjB+LmP
+cIQIXIUzdwE5uhBnwIre74cIZRXFJBqFY01+mT8ShPUWJkpOe0Fojrkl633TUuNf
+9thZ++Fjvs4s7alFH5Hc7Ulk4v/O1+owdjqERd8zlu7+568C9s50CGwFnH0d
+-----END RSA PRIVATE KEY-----
diff --git a/examples/features/authentication/client/main.go b/examples/features/authentication/client/main.go
index 7790e58eeac2..0c5c9d948e37 100644
--- a/examples/features/authentication/client/main.go
+++ b/examples/features/authentication/client/main.go
@@ -30,8 +30,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/oauth"
+	"google.golang.org/grpc/examples/data"
 	ecpb "google.golang.org/grpc/examples/features/proto/echo"
-	"google.golang.org/grpc/testdata"
 )
 
 var addr = flag.String("addr", "localhost:50051", "the address to connect to")
@@ -51,7 +51,7 @@ func main() {
 
 	// Set up the credentials for the connection.
 	perRPC := oauth.NewOauthAccess(fetchToken())
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+	creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		log.Fatalf("failed to load credentials: %v", err)
 	}
diff --git a/examples/features/authentication/server/main.go b/examples/features/authentication/server/main.go
index 63983e1a60d0..5163fc31671b 100644
--- a/examples/features/authentication/server/main.go
+++ b/examples/features/authentication/server/main.go
@@ -32,9 +32,9 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/examples/data"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/testdata"
 
 	pb "google.golang.org/grpc/examples/features/proto/echo"
 )
@@ -50,7 +50,7 @@ func main() {
 	flag.Parse()
 	fmt.Printf("server starting on port %d...\n", *port)
 
-	cert, err := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	cert, err := tls.LoadX509KeyPair(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem"))
 	if err != nil {
 		log.Fatalf("failed to load key pair: %s", err)
 	}
diff --git a/examples/features/encryption/README.md b/examples/features/encryption/README.md
index ddae8e2626d0..a00188d66a2d 100644
--- a/examples/features/encryption/README.md
+++ b/examples/features/encryption/README.md
@@ -30,16 +30,16 @@ base on TLS. Refer to the
 [godoc](https://godoc.org/google.golang.org/grpc/credentials) for details.
 
 In our example, we use the public/private keys created ahead: 
-* "server1.pem" contains the server certificate (public key). 
-* "server1.key" contains the server private key. 
-* "ca.pem" contains the certificate (certificate authority)
+* "server_cert.pem" contains the server certificate (public key). 
+* "server_key.pem" contains the server private key. 
+* "ca_cert.pem" contains the certificate (certificate authority)
 that can verify the server's certificate.
 
-On server side, we provide the paths to "server1.pem" and "server1.key" to
+On server side, we provide the paths to "server.pem" and "server.key" to
 configure TLS and create the server credential using
 [`credentials.NewServerTLSFromFile`](https://godoc.org/google.golang.org/grpc/credentials#NewServerTLSFromFile).
 
-On client side, we provide the path to the "ca.pem" to configure TLS and create
+On client side, we provide the path to the "ca_cert.pem" to configure TLS and create
 the client credential using
 [`credentials.NewClientTLSFromFile`](https://godoc.org/google.golang.org/grpc/credentials#NewClientTLSFromFile).
 Note that we override the server name with "x.test.youtube.com", as the server
diff --git a/examples/features/encryption/TLS/client/main.go b/examples/features/encryption/TLS/client/main.go
index 71bd23bd84b1..718196b1bb41 100644
--- a/examples/features/encryption/TLS/client/main.go
+++ b/examples/features/encryption/TLS/client/main.go
@@ -28,8 +28,8 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/examples/data"
 	ecpb "google.golang.org/grpc/examples/features/proto/echo"
-	"google.golang.org/grpc/testdata"
 )
 
 var addr = flag.String("addr", "localhost:50051", "the address to connect to")
@@ -48,7 +48,7 @@ func main() {
 	flag.Parse()
 
 	// Create tls based credential.
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+	creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		log.Fatalf("failed to load credentials: %v", err)
 	}
diff --git a/examples/features/encryption/TLS/server/main.go b/examples/features/encryption/TLS/server/main.go
index 10795178de76..81bf1f3acc3e 100644
--- a/examples/features/encryption/TLS/server/main.go
+++ b/examples/features/encryption/TLS/server/main.go
@@ -28,7 +28,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/testdata"
+	"google.golang.org/grpc/examples/data"
 
 	pb "google.golang.org/grpc/examples/features/proto/echo"
 )
@@ -52,7 +52,7 @@ func main() {
 	}
 
 	// Create tls based credential.
-	creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	creds, err := credentials.NewServerTLSFromFile(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem"))
 	if err != nil {
 		log.Fatalf("failed to create credentials: %v", err)
 	}
diff --git a/examples/features/interceptor/client/main.go b/examples/features/interceptor/client/main.go
index b8f2733ca497..0c2015169d17 100644
--- a/examples/features/interceptor/client/main.go
+++ b/examples/features/interceptor/client/main.go
@@ -31,8 +31,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/oauth"
+	"google.golang.org/grpc/examples/data"
 	ecpb "google.golang.org/grpc/examples/features/proto/echo"
-	"google.golang.org/grpc/testdata"
 )
 
 var addr = flag.String("addr", "localhost:50051", "the address to connect to")
@@ -147,7 +147,7 @@ func main() {
 	flag.Parse()
 
 	// Create tls based credential.
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+	creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		log.Fatalf("failed to load credentials: %v", err)
 	}
diff --git a/examples/features/interceptor/server/main.go b/examples/features/interceptor/server/main.go
index 2ad04c1584c5..1b07cdecd6ca 100644
--- a/examples/features/interceptor/server/main.go
+++ b/examples/features/interceptor/server/main.go
@@ -32,9 +32,9 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/examples/data"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/testdata"
 
 	pb "google.golang.org/grpc/examples/features/proto/echo"
 )
@@ -149,7 +149,7 @@ func main() {
 	}
 
 	// Create tls based credential.
-	creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	creds, err := credentials.NewServerTLSFromFile(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem"))
 	if err != nil {
 		log.Fatalf("failed to create credentials: %v", err)
 	}
diff --git a/examples/route_guide/client/client.go b/examples/route_guide/client/client.go
index f7e5c7564b40..3e9d4e1183cc 100644
--- a/examples/route_guide/client/client.go
+++ b/examples/route_guide/client/client.go
@@ -32,8 +32,8 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/examples/data"
 	pb "google.golang.org/grpc/examples/route_guide/routeguide"
-	"google.golang.org/grpc/testdata"
 )
 
 var (
@@ -155,7 +155,7 @@ func main() {
 	var opts []grpc.DialOption
 	if *tls {
 		if *caFile == "" {
-			*caFile = testdata.Path("ca.pem")
+			*caFile = data.Path("x509/ca_cert.pem")
 		}
 		creds, err := credentials.NewClientTLSFromFile(*caFile, *serverHostOverride)
 		if err != nil {
diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go
index aa157f0be099..dd804406afcd 100644
--- a/examples/route_guide/server/server.go
+++ b/examples/route_guide/server/server.go
@@ -38,7 +38,7 @@ import (
 	"google.golang.org/grpc"
 
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/testdata"
+	"google.golang.org/grpc/examples/data"
 
 	"github.com/golang/protobuf/proto"
 
@@ -226,10 +226,10 @@ func main() {
 	var opts []grpc.ServerOption
 	if *tls {
 		if *certFile == "" {
-			*certFile = testdata.Path("server1.pem")
+			*certFile = data.Path("x509/server_cert.pem")
 		}
 		if *keyFile == "" {
-			*keyFile = testdata.Path("server1.key")
+			*keyFile = data.Path("x509/server_key.pem")
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {
diff --git a/interop/client/client.go b/interop/client/client.go
index 58b5465ae4d2..eb4477f66d13 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -135,7 +135,7 @@ func main() {
 		if *testCA {
 			var err error
 			if *caFile == "" {
-				*caFile = testdata.Path("ca.pem")
+				*caFile = testdata.Path("x509/server_ca_cert.pem")
 			}
 			creds, err = credentials.NewClientTLSFromFile(*caFile, sn)
 			if err != nil {
diff --git a/interop/fake_grpclb/fake_grpclb.go b/interop/fake_grpclb/fake_grpclb.go
index 3b2177ef0291..f7ccd7e92d4d 100644
--- a/interop/fake_grpclb/fake_grpclb.go
+++ b/interop/fake_grpclb/fake_grpclb.go
@@ -112,8 +112,8 @@ func main() {
 	flag.Parse()
 	var opts []grpc.ServerOption
 	if *useTLS {
-		certFile := testdata.Path("server1.pem")
-		keyFile := testdata.Path("server1.key")
+		certFile := testdata.Path("x509/server1_cert.pem")
+		keyFile := testdata.Path("x509/server1_key.pem")
 		creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
 		if err != nil {
 			grpclog.Fatalf("Failed to generate credentials %v", err)
diff --git a/interop/interop_test.sh b/interop/interop_test.sh
index 15bf9cc0f951..f4103c44666b 100755
--- a/interop/interop_test.sh
+++ b/interop/interop_test.sh
@@ -87,7 +87,7 @@ for case in ${CASES[@]}; do
     echo "$(tput setaf 4) testing: ${case} $(tput sgr 0)"
 
     CLIENT_LOG="$(mktemp)"
-    if ! timeout 20 go run ./interop/client --use_tls --server_host_override=foo.test.google.fr --use_test_ca --test_case="${case}" &> $CLIENT_LOG; then
+    if ! timeout 20 go run ./interop/client --use_tls --server_host_override=x.test.example.com --use_test_ca --test_case="${case}" &> $CLIENT_LOG; then
         fail "FAIL: test case ${case}
         got server log:
         $(cat $SERVER_LOG)
diff --git a/interop/server/server.go b/interop/server/server.go
index 93c90dab5018..c029fd9858de 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -55,10 +55,10 @@ func main() {
 	var opts []grpc.ServerOption
 	if *useTLS {
 		if *certFile == "" {
-			*certFile = testdata.Path("server1.pem")
+			*certFile = testdata.Path("x509/server1_cert.pem")
 		}
 		if *keyFile == "" {
-			*keyFile = testdata.Path("server1.key")
+			*keyFile = testdata.Path("x509/server1_key.pem")
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {
diff --git a/stress/client/main.go b/stress/client/main.go
index b8f2229a5b76..f44bfa53d83f 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -279,7 +279,7 @@ func newConn(address string, useTLS, testCA bool, tlsServerName string) (*grpc.C
 		if testCA {
 			var err error
 			if *caFile == "" {
-				*caFile = testdata.Path("ca.pem")
+				*caFile = testdata.Path("x509/server_ca_cert.pem")
 			}
 			creds, err = credentials.NewClientTLSFromFile(*caFile, sn)
 			if err != nil {
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 9f8bc1a6e251..7be226708581 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -144,7 +144,7 @@ func (s) TestCredsBundleFromBalancer(t *testing.T) {
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithBalancerName(testBalancerName),
 	}
-	creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		t.Fatalf("Failed to generate credentials %v", err)
 	}
diff --git a/test/channelz_test.go b/test/channelz_test.go
index 37140bb2ce89..db510d4c6df5 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -1368,7 +1368,7 @@ func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) {
 			break
 		}
 		skt := channelz.GetSocket(id)
-		cert, _ := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+		cert, _ := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 		securityVal, ok := skt.SocketData.Security.(*credentials.TLSChannelzSecurityValue)
 		if !ok {
 			return false, fmt.Errorf("the SocketData.Security is of type: %T, want: *credentials.TLSChannelzSecurityValue", skt.SocketData.Security)
diff --git a/test/creds_test.go b/test/creds_test.go
index b25336908adb..46bdd30dc85e 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -55,7 +55,7 @@ func (c *testCredsBundle) TransportCredentials() credentials.TransportCredential
 		return nil
 	}
 
-	creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		c.t.Logf("Failed to load credentials: %v", err)
 		return nil
@@ -80,7 +80,7 @@ func (s) TestCredsBundleBoth(t *testing.T) {
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t}),
 	}
-	creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		t.Fatalf("Failed to generate credentials %v", err)
 	}
@@ -102,7 +102,7 @@ func (s) TestCredsBundleTransportCredentials(t *testing.T) {
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundleTLSOnly}),
 	}
-	creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		t.Fatalf("Failed to generate credentials %v", err)
 	}
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 0b735c159840..bdc70affa2b9 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -224,7 +224,7 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 		if authType != s.security {
 			return nil, status.Errorf(codes.Unauthenticated, "Wrong auth type: got %q, want %q", authType, s.security)
 		}
-		if serverName != "x.test.youtube.com" {
+		if serverName != "x.test.example.com" {
 			return nil, status.Errorf(codes.Unauthenticated, "Unknown server name %q", serverName)
 		}
 	}
@@ -617,7 +617,7 @@ func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network,
 		te.t.Fatalf("Failed to listen: %v", err)
 	}
 	if te.e.security == "tls" {
-		creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+		creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 		if err != nil {
 			te.t.Fatalf("Failed to generate credentials %v", err)
 		}
@@ -658,7 +658,7 @@ func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network,
 		if te.e.security != "tls" {
 			te.t.Fatalf("unsupported environment settings")
 		}
-		cert, err := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+		cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 		if err != nil {
 			te.t.Fatal("tls.LoadX509KeyPair(server1.pem, server1.key) failed: ", err)
 		}
@@ -793,7 +793,7 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string)
 	}
 	switch te.e.security {
 	case "tls":
-		creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com")
+		creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
 		if err != nil {
 			te.t.Fatalf("Failed to load credentials: %v", err)
 		}
diff --git a/testdata/ca.pem b/testdata/ca.pem
deleted file mode 100644
index 49d39cd8ed5f..000000000000
--- a/testdata/ca.pem
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDWjCCAkKgAwIBAgIUWrP0VvHcy+LP6UuYNtiL9gBhD5owDQYJKoZIhvcNAQEL
-BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
-MDMxNzE4NTk1MVoXDTMwMDMxNTE4NTk1MVowVjELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAsGL0oXflF0LzoM+Bh+qUU9yhqzw2w8OOX5mu/iNCyUOBrqaHi7mGHx73GD01
-diNzCzvlcQqdNIH6NQSL7DTpBjca66jYT9u73vZe2MDrr1nVbuLvfu9850cdxiUO
-Inv5xf8+sTHG0C+a+VAvMhsLiRjsq+lXKRJyk5zkbbsETybqpxoJ+K7CoSy3yc/k
-QIY3TipwEtwkKP4hzyo6KiGd/DPexie4nBUInN3bS1BUeNZ5zeaIC2eg3bkeeW7c
-qT55b+Yen6CxY0TEkzBK6AKt/WUialKMgT0wbTxRZO7kUCH3Sq6e/wXeFdJ+HvdV
-LPlAg5TnMaNpRdQih/8nRFpsdwIDAQABoyAwHjAMBgNVHRMEBTADAQH/MA4GA1Ud
-DwEB/wQEAwICBDANBgkqhkiG9w0BAQsFAAOCAQEAkTrKZjBrJXHps/HrjNCFPb5a
-THuGPCSsepe1wkKdSp1h4HGRpLoCgcLysCJ5hZhRpHkRihhef+rFHEe60UePQO3S
-CVTtdJB4CYWpcNyXOdqefrbJW5QNljxgi6Fhvs7JJkBqdXIkWXtFk2eRgOIP2Eo9
-/OHQHlYnwZFrk6sp4wPyR+A95S0toZBcyDVz7u+hOW0pGK3wviOe9lvRgj/H3Pwt
-bewb0l+MhRig0/DVHamyVxrDRbqInU1/GTNCwcZkXKYFWSf92U+kIcTth24Q1gcw
-eZiLl5FfrWokUNytFElXob0V0a5/kbhiLc3yWmvWqHTpqCALbVyF+rKJo2f5Kw==
------END CERTIFICATE-----
diff --git a/testdata/server1.key b/testdata/server1.key
deleted file mode 100644
index 086462992cfb..000000000000
--- a/testdata/server1.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDnE443EknxvxBq
-6+hvn/t09hl8hx366EBYvZmVM/NC+7igXRAjiJiA/mIaCvL3MS0Iz5hBLxSGICU+
-WproA3GCIFITIwcf/ETyWj/5xpgZ4AKrLrjQmmX8mhwUajfF3UvwMJrCOVqPp67t
-PtP+2kBXaqrXdvnvXR41FsIB8V7zIAuIZB6bHQhiGVlc1sgZYsE2EGG9WMmHtS86
-qkAOTjG2XyjmPTGAwhGDpYkYrpzp99IiDh4/Veai81hn0ssQkbry0XRD/Ig3jcHh
-23WiriPNJ0JsbgXUSLKRPZObA9VgOLy2aXoN84IMaeK3yy+cwSYG/99w93fUZJte
-MXwz4oYZAgMBAAECggEBAIVn2Ncai+4xbH0OLWckabwgyJ4IM9rDc0LIU368O1kU
-koais8qP9dujAWgfoh3sGh/YGgKn96VnsZjKHlyMgF+r4TaDJn3k2rlAOWcurGlj
-1qaVlsV4HiEzp7pxiDmHhWvp4672Bb6iBG+bsjCUOEk/n9o9KhZzIBluRhtxCmw5
-nw4Do7z00PTvN81260uPWSc04IrytvZUiAIx/5qxD72bij2xJ8t/I9GI8g4FtoVB
-8pB6S/hJX1PZhh9VlU6Yk+TOfOVnbebG4W5138LkB835eqk3Zz0qsbc2euoi8Hxi
-y1VGwQEmMQ63jXz4c6g+X55ifvUK9Jpn5E8pq+pMd7ECgYEA93lYq+Cr54K4ey5t
-sWMa+ye5RqxjzgXj2Kqr55jb54VWG7wp2iGbg8FMlkQwzTJwebzDyCSatguEZLuB
-gRGroRnsUOy9vBvhKPOch9bfKIl6qOgzMJB267fBVWx5ybnRbWN/I7RvMQf3k+9y
-biCIVnxDLEEYyx7z85/5qxsXg/MCgYEA7wmWKtCTn032Hy9P8OL49T0X6Z8FlkDC
-Rk42ygrc/MUbugq9RGUxcCxoImOG9JXUpEtUe31YDm2j+/nbvrjl6/bP2qWs0V7l
-dTJl6dABP51pCw8+l4cWgBBX08Lkeen812AAFNrjmDCjX6rHjWHLJcpS18fnRRkP
-V1d/AHWX7MMCgYEA6Gsw2guhp0Zf2GCcaNK5DlQab8OL4Hwrpttzo4kuTlwtqNKp
-Q9H4al9qfF4Cr1TFya98+EVYf8yFRM3NLNjZpe3gwYf2EerlJj7VLcahw0KKzoN1
-QBENfwgPLRk5sDkx9VhSmcfl/diLroZdpAwtv3vo4nEoxeuGFbKTGx3Qkf0CgYEA
-xyR+dcb05Ygm3w4klHQTowQ10s1H80iaUcZBgQuR1ghEtDbUPZHsoR5t1xCB02ys
-DgAwLv1bChIvxvH/L6KM8ovZ2LekBX4AviWxoBxJnfz/EVau98B0b1auRN6eSC83
-FRuGldlSOW1z/nSh8ViizSYE5H5HX1qkXEippvFRE88CgYB3Bfu3YQY60ITWIShv
-nNkdcbTT9eoP9suaRJjw92Ln+7ZpALYlQMKUZmJ/5uBmLs4RFwUTQruLOPL4yLTH
-awADWUzs3IRr1fwn9E+zM8JVyKCnUEM3w4N5UZskGO2klashAd30hWO+knRv/y0r
-uGIYs9Ek7YXlXIRVrzMwcsrt1w==
------END PRIVATE KEY-----
diff --git a/testdata/server1.pem b/testdata/server1.pem
deleted file mode 100644
index 88244f856c62..000000000000
--- a/testdata/server1.pem
+++ /dev/null
@@ -1,22 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDtDCCApygAwIBAgIUbJfTREJ6k6/+oInWhV1O1j3ZT0IwDQYJKoZIhvcNAQEL
-BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
-MDMxODAzMTA0MloXDTMwMDMxNjAzMTA0MlowZTELMAkGA1UEBhMCVVMxETAPBgNV
-BAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxl
-LCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0B
-AQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPz
-Qvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caY
-GeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe
-8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c
-6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPV
-YDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo2swaTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIF4DBPBgNVHREESDBGghAqLnRlc3QuZ29vZ2xlLmZy
-ghh3YXRlcnpvb2kudGVzdC5nb29nbGUuYmWCEioudGVzdC55b3V0dWJlLmNvbYcE
-wKgBAzANBgkqhkiG9w0BAQsFAAOCAQEAS8hDQA8PSgipgAml7Q3/djwQ644ghWQv
-C2Kb+r30RCY1EyKNhnQnIIh/OUbBZvh0M0iYsy6xqXgfDhCB93AA6j0i5cS8fkhH
-Jl4RK0tSkGQ3YNY4NzXwQP/vmUgfkw8VBAZ4Y4GKxppdATjffIW+srbAmdDruIRM
-wPeikgOoRrXf0LA1fi4TqxARzeRwenQpayNfGHTvVF9aJkl8HoaMunTAdG5pIVcr
-9GKi/gEMpXUJbbVv3U5frX1Wo4CFo+rZWJ/LyCMeb0jciNLxSdMwj/E/ZuExlyeZ
-gc9ctPjSMvgSyXEKv6Vwobleeg88V2ZgzenziORoWj4KszG/lbQZvg==
------END CERTIFICATE-----
diff --git a/testdata/x509/README.md b/testdata/x509/README.md
new file mode 100644
index 000000000000..e64a385e5f97
--- /dev/null
+++ b/testdata/x509/README.md
@@ -0,0 +1,106 @@
+This directory contains x509 certificates and associated private keys used in
+gRPC-Go tests.
+
+How were these test certs/keys generated ?
+------------------------------------------
+0. Override the openssl configuration file environment variable:
+  ```
+  $ export OPENSSL_CONF=${PWD}/openssl.cnf
+  ```
+
+1. Generate a self-signed CA certificate along with its private key:
+  ```
+  $ openssl req -x509                             \
+      -newkey rsa:4096                            \
+      -nodes                                      \
+      -days 3650                                  \
+      -keyout ca_key.pem                          \
+      -out ca_cert.pem                            \
+      -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-ca/  \
+      -config ./openssl.cnf                       \
+      -extensions test_ca
+  ```
+
+  To view the CA cert:
+  ```
+  $ openssl x509 -text -noout -in ca_cert.pem
+  ```
+
+2.a Generate a private key for the server:
+  ```
+  $ openssl genrsa -out server_key.pem 4096
+  ```
+
+2.b Generate a private key for the client:
+  ```
+  $ openssl genrsa -out client_key.pem 4096
+  ```
+
+3.a Generate a CSR for the server:
+  ```
+  $ openssl req -new                                \
+    -key server_key.pem                             \
+    -days 3650                                      \
+    -out server_csr.pem                             \
+    -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server/  \
+    -config ./openssl.cnf                           \
+    -reqexts test_server
+  ```
+
+  To view the CSR:
+  ```
+  $ openssl req -text -noout -in server_csr.pem
+  ```
+
+3.b Generate a CSR for the client:
+  ```
+  $ openssl req -new                                \
+    -key client_key.pem                             \
+    -days 3650                                      \
+    -out client_csr.pem                             \
+    -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client/  \
+    -config ./openssl.cnf                           \
+    -reqexts test_client
+  ```
+
+  To view the CSR:
+  ```
+  $ openssl req -text -noout -in client_csr.pem
+  ```
+
+4.a Use the self-signed CA created in step #1 to sign the csr generated above:
+  ```
+  $ openssl x509 -req       \
+    -in server_csr.pem      \
+    -CAkey ca_key.pem       \
+    -CA ca_cert.pem         \
+    -days 3650              \
+    -set_serial 1000        \
+    -out server_cert.pem    \
+    -extfile ./openssl.cnf  \
+    -extensions test_server
+  ```
+
+4.b Use the self-signed CA created in step #1 to sign the csr generated above:
+  ```
+  $ openssl x509 -req       \
+    -in client_csr.pem      \
+    -CAkey ca_key.pem       \
+    -CA ca_cert.pem         \
+    -days 3650              \
+    -set_serial 1000        \
+    -out client_cert.pem    \
+    -extfile ./openssl.cnf  \
+    -extensions test_client
+  ```
+
+5.a Verify the `server_cert.pem` is trusted by `ca_cert.pem`:
+  ```
+  $ openssl verify -verbose -CAfile ca_cert.pem  server_cert.pem
+  ```
+
+5.b Verify the `client_cert.pem` is trusted by `ca_cert.pem`:
+  ```
+  $ openssl verify -verbose -CAfile ca_cert.pem  client_cert.pem
+  ```
+
diff --git a/testdata/x509/client1_cert.pem b/testdata/x509/client1_cert.pem
new file mode 100644
index 000000000000..714136918f30
--- /dev/null
+++ b/testdata/x509/client1_cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx
+CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV
+BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIwMDgwNDAyMDAwMFoXDTMwMDgwMjAyMDAw
+MFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAK3fSafgFyHediP0fonPcc/pH010l2jqryUNsEfr
+PhxR//ccr7sBvbcInwvj3NJ9XqF4V4ws9h/QbPMLXg1FBcC/LpYjo6VZoNjuJLt2
+DTG2gGcTEL+4G2w/4ztrrmunLxa53P3URIgMgMYhCTIXK2enVbpy637X8WhPYOrq
+w+NXnDaTwT8uLGfMVEAKNvXzf8Ras8OHjgTZJEpkgXVjREhUhOPszrBsyYKnI/f2
+QSDnvgSJbrkBLFRqluT/ciqccryBWy0qJOStVhha1I2tId+dvJsTgQa/NLBASbsU
+LkIIUV375K0raINYeg/kA6MK6YDwcCtrVbQa8fu7drxxBiY3tSoDLVn1FYz7iTJ3
+PvtpwsGAqTEsSW3k7l2MTz3iuqcAgL8tI1CpyacwNPfy7j67mH3akY95sh2nmTVj
+rsW2uuFSC/cc00bH+IMVZnztE7+fpgZvU63BVnf9d9TMgDe8kwMwbq7dFi9irr12
+8Szpbdnt028dgsrjpbOgPpMYJehRK2Q0I7+99cLeJa1V5ySeFhf+uhNpW9RDi/qp
+TJGAG+rE3qAbVVoD9GrOispNZW7Hby4/q8pkNoafXmilqIf6mOri/88AYOMXbH4X
+i8mJgIeN2AjJmEGVPBPM25ZjN+ZurWqfdSasXuiIJmJzw9ExcIEjzAjoMl9CNVVy
+c77lAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFEdlWQy5/06l3GTu
+rqJTuMgy4JuKMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD
+AjANBgkqhkiG9w0BAQUFAAOCAgEAAmzrEprlWBxCQDzFZy5pZVIa1FniD+23qXlV
+n0Fhhr0eF2udYR2tfzf0VM9WcBHHoRzX5fwNkGmIWiXAISgamMl4sHHZn6Ig0i9h
+k9/fI4bYtrCiOqjYRG6VA8OZSD98bD+NtQEPQneO5F5buL0by4FUugu6Ls0Ovpk0
+yhb2pgKFhbFbMC6ev1AK9IpJZgz2q9/rjkJedGjnu35ze+94tw5Fe1FIIkg24ZQk
+C4e4DzSpRz5s51LS+dS5hDGuvglWn7SrwGuGujz8iMQdAJa3WSP5WmjbuUFaD8pH
+6afrjAhMZoWgxNubLkypkFUW/3W5JwTLnj5wPhPpBtHX6NQ30/FgN89j7+0Zp064
+i4Ur1ykhHgbdUb/EB28sXs+/CkmfmFx44M68yhoJ5euUzRF5gGmxSgRn3+RVsw2E
+ju0YQBVvH8JjAt0XCi9SY+vCpe+EG0uV4HxEO6DDdSslsMkuiAeM7pvZTM3FbZyt
+BXpWs/L71OF37ouUbt1TD+C1fsCUovjGi4AE0KXeO1rv4u2mTGfxtOOUFKt2dFDa
+E1sjyJm1+WjDgIqNjbubM6zpvNtix0xaOXqg7MAt4OJnKAeRQELgJhe6Rt2ROKGq
+Hoy8uIjcA26/lwclj2h7fwiKznlxqfDxVsiwmCTdJJb76w69UQvyIRY3tlJr3c3+
+O4VSONQ=
+-----END CERTIFICATE-----
diff --git a/testdata/x509/client1_key.pem b/testdata/x509/client1_key.pem
new file mode 100644
index 000000000000..b7a3930254fd
--- /dev/null
+++ b/testdata/x509/client1_key.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEArd9Jp+AXId52I/R+ic9xz+kfTXSXaOqvJQ2wR+s+HFH/9xyv
+uwG9twifC+Pc0n1eoXhXjCz2H9Bs8wteDUUFwL8uliOjpVmg2O4ku3YNMbaAZxMQ
+v7gbbD/jO2uua6cvFrnc/dREiAyAxiEJMhcrZ6dVunLrftfxaE9g6urD41ecNpPB
+Py4sZ8xUQAo29fN/xFqzw4eOBNkkSmSBdWNESFSE4+zOsGzJgqcj9/ZBIOe+BIlu
+uQEsVGqW5P9yKpxyvIFbLSok5K1WGFrUja0h3528mxOBBr80sEBJuxQuQghRXfvk
+rStog1h6D+QDowrpgPBwK2tVtBrx+7t2vHEGJje1KgMtWfUVjPuJMnc++2nCwYCp
+MSxJbeTuXYxPPeK6pwCAvy0jUKnJpzA09/LuPruYfdqRj3myHaeZNWOuxba64VIL
+9xzTRsf4gxVmfO0Tv5+mBm9TrcFWd/131MyAN7yTAzBurt0WL2KuvXbxLOlt2e3T
+bx2CyuOls6A+kxgl6FErZDQjv731wt4lrVXnJJ4WF/66E2lb1EOL+qlMkYAb6sTe
+oBtVWgP0as6Kyk1lbsdvLj+rymQ2hp9eaKWoh/qY6uL/zwBg4xdsfheLyYmAh43Y
+CMmYQZU8E8zblmM35m6tap91Jqxe6IgmYnPD0TFwgSPMCOgyX0I1VXJzvuUCAwEA
+AQKCAgEAidNL4aUC8TgU0h+HBtrHzxVuWMmpE+OkfmzBZeEV1QEzM8Erk8OnjSVq
+XdR8QOZcUwa/7z/cwg9Hrck+/qnOC6IA3cbWe8X2eL8dovPLNbMDSbGVP0RDiKWE
+DKApHPDjpNIkWZkf0fCHS4b4cRpor7u3exqJjnzCwfraSp1aNiZGkATD1L9XN9iC
+mFkAhCpHB3EWulIDw9gUqlvNOy46/FLzHHGkzbkOa2DuZCpyKhFJUPNYL5K8fxYX
+EuNirmBhmwe3LLARmqvEaX3mq3+oMEgrL4pgZua+b1AmogM3P+S0CxoXhSW5rRQ/
+fcUzFNUbj7gIUoK85w3M780ELBAz3F0j9cy1/DcidV0T8SAzKVrpiJvvK59XYzzn
+3J4JFmAsZ0PYgkPhZyPY6hNysRFapPwJyNC+I1NVRpSNHifMsYNEX5dV4M6Qtmv4
+7QmtvUubpJ+vo75W0DNzQ8Ar4BaBVZ6YzKTW58/Ob9Y1o6knUJv/lElE9RLyJBrn
+PgtFMPDjf2FzYaA45+zVtQBDk3rljLatS6WZxWg+qh+4RPQjS6sKzNB7U728oiZj
+1PRMbeUGKAZDb6FWTZ5nlvai3Z1VDwmLdBBSACnUWLOhXqmnkWY0q9d3kSGnMih4
+Au1A2sCFhhoowoyEkbbmlvORDSo6jfqdYKxP2rUQV1DJBPepo6kCggEBANQzH//s
+CTcB9fMSIpiEUmcFBBineA+uMXDbmNRzeJMoFwSxt7TXQpVqfMOXTaa4aDuCSCNX
+VLIf10aP4Myx9A0i+t4A9IAn9Ps+wCu5Yb43XmiEc2PC/AZYuviYfP/rIptTS0o8
+z8zAc1cLdDYBww76DcKdagAQABZQaqPARlGEHAvqmr5fjR0oWfcGeEvzqdv7WbGf
+9nyuAWl1ldMmILysW0GRDudFhp5rit6A3uCq7LB5Qb14dGrek5k+y8lnzjp30r0O
+9QxUuxZVuvh4ujiDnQI5tVWbhD/jgIUF91Nm/Vw0bZMdcp0iA9r8EGmFaHNi0by7
+rMw/6Pqcxd75qP8CggEBANHC5PZLyZEmt6C/c1ohTIZtf2g5ghXSHARfwMvRTVJ5
+4HksZp/FQSe3080l4yACXDpfxJ6pm5WNFNhy4Xi09wkEIWk+bSOqBgk+DvItgqkY
+em3q1EUUdhzIB3OXqWRcpgmc78hLiD33GkCTM9BR6W2Q/5TY7o5ULOjkiDKiVL+r
++juFlXQtUTOak0Mwu1RRDQE6z96N5Ffg2rHxjNu1HxQK7OsSfc/lrwOyqnXaB7kR
+7CThI4xpSmtyMq4prxehM1YhKk2rJmT4hW+M636uyxZCBg1Aoqqnoxv0sQTHH6k/
+RU1+ZU38RYLzid2qNBom86RS1fWX60H3CH4EX3AVFBsCggEAFMOv9O4W9MAHXjK/
+GeeQ3K3b+cGheP9VrTJ/4QIvoU7B+d6eGF8cD9zsuoL6wT64TGJyRqsMCaYd/bSk
+jcM4G3T50XGMe2HtkgxQ57ZrPx7R6S5U0EVLPh++pAbf7HcI2uQqsOgEeYe3gaQI
+SiSf/r4vTIT00269Y3GZDc8J0n439F6Pp+NXvqutKgQDD4OXcoRFAaGikA7C6pvr
+/k5z06KWB3N3XuApzSS+4QkBRkDTim1DJpQ76B1BmjRP4rR6tLP29jMZfYxpBkV7
+V0cRCeivG4GkIe1m4o2TjPDJg+rHDhe/RS8TgRbMA8i4nmrEjs3zsiE3RoFWffeL
+UUdi5wKCAQB86+rb26rBbSNy8lHKXYZrkI6ODaGxSR4yZKw3NgEsmzTaNV0wzZLO
+CqZyyJuJFp7CjQJV04C7AfhmJ5SsBGoSzojvWqQ41ysdGf5gsEXeWpufFnkwYs0s
+utvlNW9GO/8OPo525LTQ4naZ+pCjAgVYoT/073SzAuJ0GJYcQZzjQZKXHCkztUFk
+0CvfmggWYOaz0si1LB/PTjQwQUC4IBfQIemS3cJbq9gdBayK3zw2NbxDAmnfV11g
+u/P+0QhbtD8Ujk/ZTZJiE7e0BWLCYWrFaLCd995ob8mt/n3l8IikjO/DBQFj/leP
+c2apwpGg+Y2kUUjnKICNGofONOB5qbP9AoIBAQCSKpGUVqnsb0PSqjrhr8B1VFvS
+4MZfe0ds6/GrB02D7owHPhPaSJsXhBXVri/ECSx2WripMujbZ3tZH4IPub848PYv
+9668O1RxKRkyoknyUn5TO58dhYbp3VO7P7EqfVfqEezyQ8bDfVGxrIbMA+kXJosi
+T052e3yNin6Q1r+R3cWCg0dHBGDCCkpKdD861LkYjfyipw+u8c4O+CefTHvd8XV8
+EXGn+NBBAPG42bBsJMa+P/1k9qJbflbUfQy/lPGxMspVD8xwWWeEOJEFTgGmoLWE
+cNtabvDCEiQ6+DjBBE2Cl656MjX9uv0Dn830so/PLr6FWK0JSy9sGIRTrPC/
+-----END RSA PRIVATE KEY-----
diff --git a/testdata/x509/client2_cert.pem b/testdata/x509/client2_cert.pem
new file mode 100644
index 000000000000..6777622f891b
--- /dev/null
+++ b/testdata/x509/client2_cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx
+CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV
+BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIwMDgwNDAyMDAwMloXDTMwMDgwMjAyMDAw
+MlowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MjCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAOBsqW26eD9t1JvEsQH2PjcstaknRoeNi+yqxQ4w
+DIWrngdeL9/achgzCx4lCbcdTv0QatGg3manPEXem5qsmR9dKj+EKPXnV2qI6PSI
+fv1AwcLma0Ph2F/zMASNP5wkwgv6MaIyYx3n+F4iBGQToUaj1l9XS5E30w7k2VxN
+KR7zDOGSKifavuGP2nVT8NKgXUjsh3X9F72ZIPZwvaPYkbmikOshDr8TchSgof6+
+9ng+sSmYt/Vm0yWspjJfk2qJldeIXgGRVCwOl0qBsziEk0HJSpAjjy/u9GcTGz3A
+qRQ7wPmoKU5MwnNQKZGE73JRra6zk64YiqWdkg7x2WuE1Dp661bvP9iwC4EgUqXL
+ZEQkISsDpT8RkWqp2G5crvyrk/cf8I8TbsPi9Q6Eg3dRkqCN8H1mkZT1assOp+G3
+2F7jOvagZfLik3xoSbvpD+u2vMRe30uPKZBNhEZv2PU2YaSEXu+a5qT328uFK254
+rLFi1DZU0eXlj9Y///nMo5kUoq3z4WcL1rnDRSJk2JZJ6Ln5SXN4lbNuvn7dFjKA
+VoQa4texrCSf8jtRKzexhBi28n6LAorJT57E/mo0ZvfL6aJb7cUjbhQZZmC5Kqoa
+lMaiEnoPxhMqG4m+n6bfYGLqfZlsDiTVzcgEd+RxGTlhaUIg65ZMGK5982PvV3vr
+AeyRAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFHoqiLSZ1N8EEtPo
+vmr1u4X0tKyKMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD
+AjANBgkqhkiG9w0BAQUFAAOCAgEAZ+NQzQk5L/55+58WYQk81SyFWXLjj3RVO3fB
+jUgIaxd87IrVeLKrnfoa9mMaS2Qf3SfEMhovRy6Jb2jfxbG0wLQnhx1bqNtaNLAr
+2pGG/Yu+4ZzN4iIloP9dn98tYFkHOLOLfIwNEh4Yg6IB9eg+qcbDg5JlqGkGRSsu
+IOS9XD2CY9zQ3KLGlVCWZ1EfW8u+du1GIUMx0DEwEYZ/zYnyTa2bCdBD7aetmbKZ
+yjSQ0Ole1W3z1Q3uF8CQjZ2dr/wQ3nmxj5Km9PN7/Q9iHn7RyeypWxt5utzSG5Bf
+egL0ER8kmYeKHZeagdRbKWPRyUjEligndLzh8Vi75hGFBDAx/pB0aVf81HEStKKw
+WCuL0PKpKIoIqNE8aJ6jTo0OEL+Z+6uam0vSnuVqHkeigbNsmefyR82TmiJYDahM
+3CBp6Q5gfw4WKIY/0JuJnN4Ym+zIgv2kKRVHGK3SHhiaCUGt2BydN9MxSjl1/B+v
+U7kYVj73MJZHSl96w1mnXXFOevxb7SOP23QmTKfqmU0NakfRMcHcjnG6M5mlnIDg
+DjpSJd1TLoCS1SfIyc+Fibd4grsRucnuo0iHuFqV8TZ4hi0qKKE8UG8En5KNiQDl
+exFgZo6FGQa1mJxQiSfhL3VoyeZ/b3QRG+mNVDmgHsZSTjfMppfmyBJRT4HIlsHS
+dWeIeN4=
+-----END CERTIFICATE-----
diff --git a/testdata/x509/client2_key.pem b/testdata/x509/client2_key.pem
new file mode 100644
index 000000000000..dc4cca65a312
--- /dev/null
+++ b/testdata/x509/client2_key.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEA4Gypbbp4P23Um8SxAfY+Nyy1qSdGh42L7KrFDjAMhaueB14v
+39pyGDMLHiUJtx1O/RBq0aDeZqc8Rd6bmqyZH10qP4Qo9edXaojo9Ih+/UDBwuZr
+Q+HYX/MwBI0/nCTCC/oxojJjHef4XiIEZBOhRqPWX1dLkTfTDuTZXE0pHvMM4ZIq
+J9q+4Y/adVPw0qBdSOyHdf0XvZkg9nC9o9iRuaKQ6yEOvxNyFKCh/r72eD6xKZi3
+9WbTJaymMl+TaomV14heAZFULA6XSoGzOISTQclKkCOPL+70ZxMbPcCpFDvA+agp
+TkzCc1ApkYTvclGtrrOTrhiKpZ2SDvHZa4TUOnrrVu8/2LALgSBSpctkRCQhKwOl
+PxGRaqnYblyu/KuT9x/wjxNuw+L1DoSDd1GSoI3wfWaRlPVqyw6n4bfYXuM69qBl
+8uKTfGhJu+kP67a8xF7fS48pkE2ERm/Y9TZhpIRe75rmpPfby4UrbnissWLUNlTR
+5eWP1j//+cyjmRSirfPhZwvWucNFImTYlknouflJc3iVs26+ft0WMoBWhBri17Gs
+JJ/yO1ErN7GEGLbyfosCislPnsT+ajRm98vpolvtxSNuFBlmYLkqqhqUxqISeg/G
+Eyobib6fpt9gYup9mWwOJNXNyAR35HEZOWFpQiDrlkwYrn3zY+9Xe+sB7JECAwEA
+AQKCAgA4kiuDRWXaV00olsQnwnKcZeDE6umUcdG7rrBNiz8c0s3a/ZsDyoTIJNXA
+m4V/axvmHqVOgkaNicpfsmV279sJVOq5aA8LLW2TpT9TpLSeEhzFjF+tlNh+F0cb
+Xp+SNJHVgxPP1vO1LiwlTl3c/DXDILmA/vhFetTxBC7mXWzoKEwu8DFAKpvDMAfZ
+W3dxIItjPnxG+a1qVZdBh9nF22mgaaIuIv8cm0I+gN9U374xQVxXJ+/3JBxFeufJ
++t2mFVh4JB/ONVwKXwMz/M24iXK1OpBZFR2a75kcAmzzfAUi3I0gYYtH+YFqn+Ja
+lC/nmT82sn2ffQA2DyoqKjysJad5PWHByyepPGA6mkrAwaxn8YFsd0Yu14LaWCfO
+5jKQzMvDhuAavAkaeT8EJnQdOeztXHYGV7S8rDQOgXM58W8e9+SchceJzkl1MYKf
+99xXveelRaTaGOWBK1E6xPQP7iKJTeh1/Xjk0ylEnWPG5VvjcbNFwleDAnhyDTwB
+OqcW2L3IV208MmDEmLuSBAFjHg8u5+/hLnsv+qozAX4yWhITZL67uBufVjKbhTi9
+viFUJ8/yGP9kIrJosQ4iDZgZv1juQLEhAw/W1eIV0gCxy/ZFfxAJXgKThZJWgSAI
+FTNf3mKZOiUpuG5+Pe5fFtDa1/vmvQaE5y2lzh8ztLtFboaboQKCAQEA+a4nh2bD
+WR6UC/3xQ22/Uwvntw91P18L+HyzNtgKCKKKVpwjWdaK/o9jdnRajK4s/hYKcIND
+szaSjnD1vXWezw61aXZgOBai+xGdMWJFbTIRFfFcJqvFwN4cOmURX2NzLn7JPCp2
+y8HUdP0u55n0Ax9/qSkh4Eysxcy9+RMAcJ7LIsqSSlsSY9tQ78QS8ymJeePdf8xl
+Ha3rlaGLpoLt/8gfYLjMfpyfUnuWrwRK79aBBKbkG7sdi5Cahnw1ZN1vxdOjpKcu
+5/NhJ5OZxU9OSm91uzSkQFfsLe2t1JLnjuvcPASlhMIskhpGof8qCrjct1e7sYeo
+UpyVknF7InNGMwKCAQEA5hrbpi7Nny2H4Uu7Z6aIUf3dIuPLl6f9VDkIqRon1HXx
+4+1gQWhEwclB18FzFVDv4h4YAGv4upGHYo8DNl8GYrcIpZQ2dxz3QfTH5jl+tKmF
+FfHIRKuBJVgXw+nVrE8HzF1M1UTCwCb8SnDg1dV8U5OfJy01LOEnp1sNW64T/pDy
+unCnY2+k/ncqGmeWUKL4mbKN8GmfzIGMhwi8yiM3Cdbmk0kETDK/NIwgl+YLX6dt
+lHe2g5OVoDgVatC8ViVmoQVmuuPASP1K4TPUAtRi0A0BYqPB2O/vFZ1f+yD1sJM7
+kILtz91DPB5v+7txwjD5S558TC1l8L9JCH12R7BWKwKCAQEAz4Z5RImdhM1tsCn6
+BlmJ1LToe7dVdL7DbF35d3RJorO22BYfK+Su0rbLrQE44gVDUE1xj+MKukJ5vfsV
+xculm+RV1LqXbwchoB0b0pgjrIcYvGxIc7wCOjRisgafUfGPIu4uxNtmsiUBOdvW
+yJmlv5LGwQt3JL+WOzHaFNQ+YV0a6mgE/9iCiI0Z0K/gMEwuACntSPPSd8C/Nzd2
+o4ff2eG0cugm0HXN1vjyXbXrsz1PL1an8oSsIfym83D50ERdSsiGE60Bx7j637JG
+9UDdifDqohc3DmQF4obTHQSdgqV4AEq8aIQcF7PPUYaMoyzUB2/cicp/lWqgx3+b
+IR8/EQKCAQBtZA9P7agrKEYUwSASooTkFb/vOkQrkN1KEOMhISIWSwv3w32jGqK1
+TaxTmc/QLm4cHRpj+PCCIXUvUbXBP2OVwlYGAXPzJH4XiPsPY/3sfTqbuBnxK2d2
+DW8e4CeIhvm6GhDQwqOjHeWKrib1AUzdnqxmv4MsFs33Lb4n+5Xdy6LZJ30sNINH
+xfbqHpzDMPbmepAn3s7tNhlMiMbXge5Eazmqg2fbobRsksFb9S0rCDl7/31xB9R2
+GrNz2E/w1E759ctkxalACcpzTWRZBAcFyWkDL76UF1yd9fcPOBgVHamPhe7whsvT
+5NRv5CisnQOnA20r+dkgno9lzd9RLW+JAoIBADJ0vUL2nJZkM6reh4+bDAoRDP3s
+U6JNPAmkMvWsiMckm+WKUtUo84VDBSIKX897z5sZ1AfkWS8P9MqyiDbPiJCuuIkq
+h9OJIHVEQ8NfmD/sl/3TE+ig0OzIbZUL3sssL1Iadkkn9hNnYIY1nt5QsKsWJ1m7
+u2+6DHTkj0TAM6SGt41TvRQyLS/fGomqmAkqYNuN3jdEGF5cFJoeyhOh/EoMP3RC
+LabPAhwUZzIH+JO93Ws5nuKOTPnryDQOM4Ug09aPLaJW5GRmfKVie1iDV6sp7KBI
+7OqHcuieCyxXHrFRESmxkMj87DaQ5mTo/q8qoZ1nOZ58vohAjbPvIaQ+vL8=
+-----END RSA PRIVATE KEY-----
diff --git a/testdata/x509/client_ca_cert.pem b/testdata/x509/client_ca_cert.pem
new file mode 100644
index 000000000000..ad6fea8ecfeb
--- /dev/null
+++ b/testdata/x509/client_ca_cert.pem
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF6jCCA9KgAwIBAgIJAKa2/29Hc+P6MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD
+MRcwFQYDVQQDDA50ZXN0LWNsaWVudF9jYTAeFw0yMDA4MDQwMTU5NTdaFw0zMDA4
+MDIwMTU5NTdaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD
+U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LWNsaWVudF9jYTCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL9LRGKPAszSvRzSKwgP6niaPy0w
+wbSILjrp60WVHB9jjOSIvgCaTev9Tz/+zCaxCqM/hIrBNXI+ITZuzNUBx3+rz4Ns
+VdYVhEsilc5gjl/dqsvD/FJdRKHKDrSzvKznwEs7KpGX1AdYoWBYZ8jNaQcDdopU
+VhZdE/196akrTRejZQhnjNaaCXKCjrubfeFGpZ4hTsDHLjzuTYkiZ7m5q0Kdiri0
+9gKNdp6b5edyLuuMimEviEsZbYritZbwP1kwGiOMSQi2tzBGUcIANugqxMhSUrgy
+JQ45Eew8mLnNqEOgk3nuWf4m0LPzTlJ/R70TmLIVyJrZ51GcLYmTZ/czsfkhXaPT
+sTuBRgqFhJNb2ukjq8XPJH7O0wOhbUKT7MCRXSlFttUCIZ8aOmufv5mYLuaGx0sd
+8uJEEMZHKDeMZOZNsyTZNaged77Onf+AoUkSH25aTdjU+bpUn/0CO2aJDqwp04Rq
+7qOrtGQ76miNnw4Fe/eHJuUoqp8VH4dUmFO3vZ24N+kSzF5LDwEbgyybQN/cot0i
+rjm8iqcimwS+BISEm7UvIeK0AEzXmxNC1mXEwvY0lkIci6TpH2Fy7OGaCu5MTru0
+XrOORWqxMLo65bTQ0ciUSxw8DartL4xobOW2UY+EUO6Da8yhVRbO59cC8dBbA9J4
+fH60efPhziFt4aKvAgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
+FgQUaGCg8RtquvpSIbS9Va1yqdqyXuYwgYAGA1UdIwR5MHeAFGhgoPEbarr6UiG0
+vVWtcqnasl7moVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV
+BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1jbGllbnRfY2GC
+CQCmtv9vR3Pj+jAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAEmD
+XMO4s+WP86pM35pFMbrz3qsjiB6Cw3tqSoUaw0eIi88uWk3ismREV/24wY4j+10+
+5NACcPLykZLh0XpQjHMIZu4FEZSsQP0ExnHluaS/XaFf8hIy/qLFcm5x6wZ08AeU
+M+daf9BmCSrjuW7u2bMxIrRLcnLMQG1kX3t3aEQLl/GA62g6Ll3MlHBGDILdvdNA
+jIscctNhnrCPLBc+ykifa5NIBhz1PWU1RTr9JyNJwLaO2To9LJcpZKda2LJJ6xYQ
+/lzPBg0aJgw9rOOgdenhb4ijQ5nMWZqCDZZFiKej3e6pj+M9E4a6OlelHiRPZT7j
+q0bSoDDNTCviGlap/LDCBTvzyU/c8hgJ2XSUMfOL5RTXQTmqF7eQEMepmNl+J9HT
+FYv80eOtk3O6rnIVHJ25zjLcLTD8iDzH3eX61bhMphI65jr4ltC6fGetXn9xINX4
+lpuxpMg5sRIYLl6lUdBcp1pMdsjEWUdiPcAxhjYqthb9MeSgmAG0cEJ+EbgGbiJA
+m2DpQ8HkQjd5gc2mCs1X5HKiFWr3ERTeQwzBwUZmNaupfgbDWpKi8xrz91r3tLVN
+eFjyd2z+0VtM82KP8D34ZVqssjp3jS8N9H1h3NoPqZPtFN3DjXfFV7BsfrcGR9CN
+mwNfZlxB487I+gXYIwAG2Tp1UYNQ1JDDfkF39Uu5
+-----END CERTIFICATE-----
diff --git a/testdata/x509/client_ca_key.pem b/testdata/x509/client_ca_key.pem
new file mode 100644
index 000000000000..3fcfdb7074e1
--- /dev/null
+++ b/testdata/x509/client_ca_key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC/S0RijwLM0r0c
+0isID+p4mj8tMMG0iC466etFlRwfY4zkiL4Amk3r/U8//swmsQqjP4SKwTVyPiE2
+bszVAcd/q8+DbFXWFYRLIpXOYI5f3arLw/xSXUShyg60s7ys58BLOyqRl9QHWKFg
+WGfIzWkHA3aKVFYWXRP9fempK00Xo2UIZ4zWmglygo67m33hRqWeIU7Axy487k2J
+Ime5uatCnYq4tPYCjXaem+Xnci7rjIphL4hLGW2K4rWW8D9ZMBojjEkItrcwRlHC
+ADboKsTIUlK4MiUOORHsPJi5zahDoJN57ln+JtCz805Sf0e9E5iyFcia2edRnC2J
+k2f3M7H5IV2j07E7gUYKhYSTW9rpI6vFzyR+ztMDoW1Ck+zAkV0pRbbVAiGfGjpr
+n7+ZmC7mhsdLHfLiRBDGRyg3jGTmTbMk2TWoHne+zp3/gKFJEh9uWk3Y1Pm6VJ/9
+AjtmiQ6sKdOEau6jq7RkO+pojZ8OBXv3hyblKKqfFR+HVJhTt72duDfpEsxeSw8B
+G4Msm0Df3KLdIq45vIqnIpsEvgSEhJu1LyHitABM15sTQtZlxML2NJZCHIuk6R9h
+cuzhmgruTE67tF6zjkVqsTC6OuW00NHIlEscPA2q7S+MaGzltlGPhFDug2vMoVUW
+zufXAvHQWwPSeHx+tHnz4c4hbeGirwIDAQABAoICAQC9otcLQazL8kpprOvd1TFj
+F75zhTcySiJSYxzKYTR85YqB8BEztcRzoy2SSnyGCtJ53Xj+uOTL+U2hkZvbuiTU
+qzVPmvFJBxGcDpAmBFCANtafpA2adT2Zih6kAt6TJjfaHLBpnvMhyTpJsbpJNWDe
+BA/auBqTlvg/PziJbRTCz0dUWpsjD5c3/reSwmW7EvcSWQCiWZK78p3IyeO8GZTu
+uBESZMrQ4v5p5DC5Ddf3yN5R0/YwROf0XCUamdajCu2Ouf6Y9dGKuNtKED5eUC++
+SuYYFhXoEKl04OmioH8jc6dfo+tw6XfSPOwzGly60xd3y+KPqF8J52K5VPkm9geC
+NEttAEKEpwLX4cAsxzQ09WaL0fq+XSpwWZYuAJI4F8zPadckbzittkAFGnwH6t5N
+ydaYoAcGxz9x97qbu2iS9SiN1cWQ+OSMF+o3o02WcLNcIBOVIKivV1FuLgQEPfXw
+bi9egAOUI5TUvoVO8mG3Drk5+Ii6PPxEaCKfp6x0xXA+t8JrmOCsEoYRiPhCc65B
+gHZC1+mgngYUs6PYmkPgTgBfYAe2wYpn7uaCEo06tNfe0kPqLzr2uMEKZNY1IfoM
+5RMxic9qKac3Qp2Lf/XG/90L/wO+kVpv/HSWh8JAZXezYD9f+EhrDuYae8KlsKXE
+Z+XGmMdgIarHLGnXoAqpeQKCAQEA3002LSywsvGM1KZz5tQoIyaor2kD8tW8vyS8
+7TlozM1TI58ALtDyV/LCrvS5jJEIbsdlrrOeBhQOS3RPjSQQdEQKfSly1TF9mhE2
+vDLznxFOQNpdkkzGwfLxI/5mMbeHN6960XAcfVD5QTDdpKPY+74uQU3HzQCx6Net
++UK3aT6CeIvgWn0xNnR5Fk2EnQHKUduqm0sRj5c2S8qUO6HxD/VPNRCT7G+faex9
+tP2VIHxwF4iH1WOmwQWxTLpy9wR7UYYxpFBvQN6gglHuMeY6tublTnvfhpMdZ5NU
+Zd1Trzrh4w4sXRWStHkphJK9aQzHEclZq5ktvdJtFd/GGZfsrQKCAQEA2040LYoT
+JcassmJs9UzgJeVkJIB61wmBc/qvqwKqy28niLubSYNaETO6cQzPrnlZjk6LKa+M
+HumrlA3DjobbkmA1YI/OAhIHjGMEtaxsOTUz0rMR5RDOvRc3hXo2qKsXfDQGUtr+
+1DrCmnI/iVm8+F8HtV9tEHzrGEaCmeMLHQWCxveNoGDnZRCZds52ApoFxiLnVq3N
++ocQEsWwdOg+8ZdyfF7RqzW2e22WoCkTJYApGutDfu1eXHXlOeBrBNPiHMzK3pbA
+n6+oqcxB23NRttNeUkge3UezjfQfuGqR7CLi0yF2L236MGBOGuXo4bGaUMgEz277
+ZBT7YfWhZpn8SwKCAQADqM5Ee0ECDbdTHM81bzChMtb82Om5pwsKzt1Rvekbwhmk
+scxc+AugqVfLajNIPHA48IeYD1V9oAKD9gn/tCGY5iyN1IoPOFpolfOhrewUJUJ1
+CZ8S8LMpJoQRJPAjzHAo13VZzU6KNzN+gACB3DWIGpvDcjTeBS7lM/Oj7BX5YY7d
+zt0EXpzZ2ZrKZMbRk9/u63ymQtqs0buQDmfTelnq+wgrRHRIIaQpJjkBKE6zU5a6
+rAAd3R40d5VqPnv31Fj5Awv5N2A7XeqfeBxBMRaxPKNxX9JP8EVBF0cAzFm8u2hM
+QkUz2VCoKHwnsgfsmssAXZ5ck4wOWk5zV1F1xemZAoIBAAR+esVAIhpREvLo33C7
+bZB5Pe8djubfM/7rcTQg7t0SXw4HQixke7EEjVqJt6vMotAuvd1R0p5DjZeQHKTM
+EK3UOOPMrp0OP4dZ9BvA98rIU1KLBt/Z01K+qg2bLomQT//klQiXokc5GQnPM4we
+AahZUjAeT37aAHtT3pNGutCSb1aidg2GTtecWni7zGFLRLkFuBXno+PxZpvr3yzW
+IYwT3W29B7Dpfd7TpRWNIe5PzQfXMF/mf1uHsvXXqnnD2ctbSwD6t+HN2Lf6DpNv
+ron/lNw8zB0evgg3q3q8/FaJdHp9Ig3gxBK/tnoIohgV6qKjJq4ViSNI5sngHbmb
+iDcCggEBAJ+Wg1Y3UnPShQjAAUyOeqfdLnb0h6ocz5Flog4I49023ro102xav/Rr
+O6NzaH8nBHt4OKYWPgwa1ANZ1ujXfnqU531OlB7p8vllDcECSR9qnSE0vMO8hvbU
+flREfjy2inQ9kVwCqLbYHh2XEYZ7sEwQ7p0dz1v9G1ytBslwyeC3h2aMIg4utT/k
+73y0T5Nq0e6Mas5w0ZBemzKNHoKw7N05g2rrELL4hRfkGMrEIsSaANPDRM+4cI1k
+a3CAv0mex+5XeBskUCtvU+xrCH6isDovDhCT/CSAjuEatezby6tLk8PeaH0uEaxr
+MhPlrQvyfY9eITe9uSQtiTQRg+Z4U5E=
+-----END PRIVATE KEY-----
diff --git a/testdata/x509/create.sh b/testdata/x509/create.sh
new file mode 100755
index 000000000000..ecfcfd19f544
--- /dev/null
+++ b/testdata/x509/create.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Create the server CA certs.
+openssl req -x509                                     \
+  -newkey rsa:4096                                    \
+  -nodes                                              \
+  -days 3650                                          \
+  -keyout server_ca_key.pem                           \
+  -out server_ca_cert.pem                             \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server_ca/   \
+  -config ./openssl.cnf                               \
+  -extensions test_ca
+
+# Create the client CA certs.
+openssl req -x509                                     \
+  -newkey rsa:4096                                    \
+  -nodes                                              \
+  -days 3650                                          \
+  -keyout client_ca_key.pem                           \
+  -out client_ca_cert.pem                             \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client_ca/   \
+  -config ./openssl.cnf                               \
+  -extensions test_ca
+
+# Generate two server certs.
+openssl genrsa -out server1_key.pem 4096
+openssl req -new                                    \
+  -key server1_key.pem                              \
+  -days 3650                                        \
+  -out server1_csr.pem                              \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server1/   \
+  -config ./openssl.cnf                             \
+  -reqexts test_server
+openssl x509 -req           \
+  -in server1_csr.pem       \
+  -CAkey server_ca_key.pem  \
+  -CA server_ca_cert.pem    \
+  -days 3650                \
+  -set_serial 1000          \
+  -out server1_cert.pem     \
+  -extfile ./openssl.cnf    \
+  -extensions test_server
+openssl verify -verbose -CAfile server_ca_cert.pem  server1_cert.pem
+
+openssl genrsa -out server2_key.pem 4096
+openssl req -new                                    \
+  -key server2_key.pem                              \
+  -days 3650                                        \
+  -out server2_csr.pem                              \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server2/   \
+  -config ./openssl.cnf                             \
+  -reqexts test_server
+openssl x509 -req           \
+  -in server2_csr.pem       \
+  -CAkey server_ca_key.pem  \
+  -CA server_ca_cert.pem    \
+  -days 3650                \
+  -set_serial 1000          \
+  -out server2_cert.pem     \
+  -extfile ./openssl.cnf    \
+  -extensions test_server
+openssl verify -verbose -CAfile server_ca_cert.pem  server2_cert.pem
+
+# Generate two client certs.
+openssl genrsa -out client1_key.pem 4096
+openssl req -new                                    \
+  -key client1_key.pem                              \
+  -days 3650                                        \
+  -out client1_csr.pem                              \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/   \
+  -config ./openssl.cnf                             \
+  -reqexts test_client
+openssl x509 -req           \
+  -in client1_csr.pem       \
+  -CAkey client_ca_key.pem  \
+  -CA client_ca_cert.pem    \
+  -days 3650                \
+  -set_serial 1000          \
+  -out client1_cert.pem     \
+  -extfile ./openssl.cnf    \
+  -extensions test_client
+openssl verify -verbose -CAfile client_ca_cert.pem  client1_cert.pem
+
+openssl genrsa -out client2_key.pem 4096
+openssl req -new                                    \
+  -key client2_key.pem                              \
+  -days 3650                                        \
+  -out client2_csr.pem                              \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client2/   \
+  -config ./openssl.cnf                             \
+  -reqexts test_client
+openssl x509 -req           \
+  -in client2_csr.pem       \
+  -CAkey client_ca_key.pem  \
+  -CA client_ca_cert.pem    \
+  -days 3650                \
+  -set_serial 1000          \
+  -out client2_cert.pem     \
+  -extfile ./openssl.cnf    \
+  -extensions test_client
+openssl verify -verbose -CAfile client_ca_cert.pem  client2_cert.pem
+
+# Cleanup the CSRs.
+rm *_csr.pem
diff --git a/testdata/x509/openssl.cnf b/testdata/x509/openssl.cnf
new file mode 100644
index 000000000000..d1034214e1d3
--- /dev/null
+++ b/testdata/x509/openssl.cnf
@@ -0,0 +1,28 @@
+[req]
+distinguished_name = req_distinguished_name
+attributes = req_attributes
+
+[req_distinguished_name]
+
+[req_attributes]
+
+[test_ca]
+basicConstraints        = critical,CA:TRUE
+subjectKeyIdentifier    = hash
+authorityKeyIdentifier  = keyid:always,issuer:always
+keyUsage                = critical,keyCertSign
+
+[test_server]
+basicConstraints        = critical,CA:FALSE
+subjectKeyIdentifier    = hash
+keyUsage                = critical,digitalSignature,keyEncipherment,keyAgreement
+subjectAltName          = @server_alt_names
+
+[server_alt_names]
+DNS.1 = *.test.example.com
+
+[test_client]
+basicConstraints        = critical,CA:FALSE
+subjectKeyIdentifier    = hash
+keyUsage                = critical,nonRepudiation,digitalSignature,keyEncipherment
+extendedKeyUsage        = critical,clientAuth
diff --git a/testdata/x509/server1_cert.pem b/testdata/x509/server1_cert.pem
new file mode 100644
index 000000000000..3e48a52fd108
--- /dev/null
+++ b/testdata/x509/server1_cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx
+CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV
+BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIwMDgwNDAxNTk1OFoXDTMwMDgwMjAxNTk1
+OFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMTCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAKonkszKvSg1IUvpfW3PAeDPLgLrXboOWJCXv3RD
+5q6vf29+IBCaljSJmU6T7SplokUML5ZkY6adjX6awG+LH3tOMg9zvXpHuSPRpFUk
+2oLFtaWuzJ+NC5HIM0wWDvdZ6KQsiPFbNxk2Rhkk+QKsiiptZy2yf/AbDY0sVieZ
+BJZJ+os+BdFIk7+XUgDutPdSAutTANhrGycYa4iYAfDGQApz3sndSSsM2KVc0w5F
+gW6w2UBC4ggc1ZaWdbVtkYo+0dCsrl1J7WUNsz8v8mjGsvm9eFuJjKFBiDhCF+xg
+4Xzu1Wz7zV97994la/xMImQR4QDdky9IgKcJMVUGua6U0GE5lmt2wnd3aAI228Vm
+6SnK7kKvnD8vRUyM9ByeRoMlrAuYb0AjnVBr/MTFbOaii6w2v3RjU0j6YFzp8+67
+ihOW9nkb1ayqSXD3T4QUD0p75Ne7/zz1r2amIh9pmSJlugLexVDpb86vXg9RnXjb
+Zn2HTEkXsL5eHUIlQzuhK+gdmj+MLGf/Yzp3fdaJsA0cJfMjj5Ubb2gR4VwzrHy9
+AD2Kjjzs06pTtpULChwpr9IBTLEsZfw/4uW4II4pfe6Rwn4bGHFifjx0+3svlsSo
+jdHcXEMHvdRPhWGUZ0rne+IK6Qxgb3OMZu7a04vV0RqvgovxM6hre3e0UzBJG45Y
+qlQjAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFFL5HUzehgKNfgdz
+4nuw5fru5OTPMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh
+bXBsZS5jb20wDQYJKoZIhvcNAQEFBQADggIBAHMPYTF4StfSx9869EoitlEi7Oz2
+YTOForDbsY9i0VnIamhIi9CpjekAGLo8SVojeAk7UV3ayiu0hEMAHJWbicgWTwWM
+JvZWWfrIk/2WYyBWWTa711DuW26cvtbSebFzXsovNeTqMICiTeYbvOAK826UdH/o
+OqNiHL+UO5xR1Xmqa2hKmLSl5J1n+zgm94l6SROzc9c5YDzn03U+8dlhoyXCwlTv
+JRprOD+lupccxcKj5Tfh9/G6PjKsgxW+DZ+rvQV5f/l7c4m/bBrgS8tru4t2Xip0
+NhQW4qHnL0wXdTjaOG/1liLppjcp7SsP+vKF4shUvp+P8NQuAswBp/QtqUse5EYl
+EUARWrjEpV4OHSKThkMackMg5E32keiOvQE6iICxtU+m2V+C3xXM3G2cGlDDx5Ob
+tan0c9fZXoygrN2mc94GPogfwFGxwivajvvJIs/bsB3RkcIuLbi2UB76Wwoq+ZvH
+15xxNZI1rpaDhjEuqwbSGPMPVpFtF5VERgYQ9LaDgj7yorwSQ1YLY8R1y0vSiAR2
+2YeOaBH1ZLPF9v9os1iK4TIC8XQfPv7ll2WdDwfbe2ux5GVbDBD4bPhP9s3F4a+f
+oPhikWsUY4eN5CfS76x6xL0L60TL1AlWLlwuubTxpvNhv3GSyxjfunjcGiXDml20
+6S80qO4hepxzzjol
+-----END CERTIFICATE-----
diff --git a/testdata/x509/server1_key.pem b/testdata/x509/server1_key.pem
new file mode 100644
index 000000000000..e71ad0ac9753
--- /dev/null
+++ b/testdata/x509/server1_key.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEAqieSzMq9KDUhS+l9bc8B4M8uAutdug5YkJe/dEPmrq9/b34g
+EJqWNImZTpPtKmWiRQwvlmRjpp2NfprAb4sfe04yD3O9eke5I9GkVSTagsW1pa7M
+n40LkcgzTBYO91nopCyI8Vs3GTZGGST5AqyKKm1nLbJ/8BsNjSxWJ5kElkn6iz4F
+0UiTv5dSAO6091IC61MA2GsbJxhriJgB8MZACnPeyd1JKwzYpVzTDkWBbrDZQELi
+CBzVlpZ1tW2Rij7R0KyuXUntZQ2zPy/yaMay+b14W4mMoUGIOEIX7GDhfO7VbPvN
+X3v33iVr/EwiZBHhAN2TL0iApwkxVQa5rpTQYTmWa3bCd3doAjbbxWbpKcruQq+c
+Py9FTIz0HJ5GgyWsC5hvQCOdUGv8xMVs5qKLrDa/dGNTSPpgXOnz7ruKE5b2eRvV
+rKpJcPdPhBQPSnvk17v/PPWvZqYiH2mZImW6At7FUOlvzq9eD1GdeNtmfYdMSRew
+vl4dQiVDO6Er6B2aP4wsZ/9jOnd91omwDRwl8yOPlRtvaBHhXDOsfL0APYqOPOzT
+qlO2lQsKHCmv0gFMsSxl/D/i5bggjil97pHCfhsYcWJ+PHT7ey+WxKiN0dxcQwe9
+1E+FYZRnSud74grpDGBvc4xm7trTi9XRGq+Ci/EzqGt7d7RTMEkbjliqVCMCAwEA
+AQKCAgEAjU6UEVMFSBDnd/2OVtUlQCeOlIoWql8jmeEL9Gg3eTbx5AugYWmf+D2V
+fbZHrX/+BM2b74+rWkFZspyd14R4PpSv6jk6UASkcmS1zqfud8/tjIzgDli6FPVn
+9HYVM8IM+9qoV5hi56M1D8iuq1PS4m081Kx6p1IwLN93JSdksdL6KQz3E9jsKp5m
+UbPrwcDv/7JM723zfMJA+40Rf32EzalwicAl9YSTnrC57g428VAY+88Pm6EmmAqX
+8nXt+hs1b9EYdQziA5wfEgiljfIFzHVXMN3IVlrv35iz+XBzkqddw0ZSRkvTiz8U
+sNAhd22JqIhapVfWz+FIgM43Ag9ABUMNWoQlaT0+2KlhkL+cZ6J1nfpMTBEIatz0
+A/l4TGcvdDhREODrS5jrxwJNx/LMRENtFFnRzAPzX4RdkFvi8SOioAWRBvs1TZFo
+ZLq2bzDOzDjs+EPQVx0SmjZEiBRhI6nC8Way00IdQi3T546r6qTKfPmXgjl5/fVO
+J4adGVbEUnI/7+fqL2N82WVr+Le585EFP/6IL5FO++sAIGDqAOzEQhyRaLhmnz+D
+GboeS/Tac9XdymFbrEvEMB4EFS3nsZHTeahfiqVd/SuXFDTHZ6kiqXweuhfsP1uW
+7tGlnqtn+3zmLO6XRENPVvmjn7DhU255yjiKFdUqkajcoOYyWPECggEBANuYk+sr
+UTScvJoh/VRHuqd9NkVVIoqfoTN61x6V1OuNNcmjMWsOIsH+n4SifLlUW6xCKaSK
+8x8RJYfE9bnObv/NqM4DMhuaNd52bPKFi8IBbHSZpuRE/UEyJhMDpoto04H1GXx4
+1S49tndiNxQOv1/VojB4BH7kapY0yp30drK1CrocGN+YOUddxI9lOQpgt2AyoXVk
+ehdyamK4uzQmkMyyGQljrV5EQbmyPCqZ1l/d0MJ9DixOBxnPDR9Ov9qrG4Dy6S/k
+cH8PythqHTGTdlXgsBJaWEl2PyQupo3OhfiCV+79B9uxPfKvk5CIMVbnYxKgu+ly
+RKSTSX+GHVgNwicCggEBAMZcwQIAA+I39sTRg/Vn/MxmUBAu3h2+oJcuZ3FQh4v5
+SL80BWEsooK9Oe4MzxyWkU+8FieFu5G6iXaSx8f3Wv6j90IzA3g6Xr9M5xBm5qUN
+IqzF+hUZuKAEMY1NcPlFTa2NlrkT8JdfQvJ+D5QrcBIMFmg9cKG5x9yD7MfHTJkf
+ztMDFOwP3n7ahKRBowfe7/unAEFf6hYFtYjV+bqMDmBFVmk2CIVtjFgO9BNBQ/LB
+zGcnwo2VigWBIjRDF5BgV0v+2g0PZGaxJ362RigZjzJojx3gYj6kaZYX8yb6ttGo
+RPGt1A9woz6m0G0fLLMlce1dpbBAna14UVY7AEVt56UCggEAVvii/Oz3CINbHyB/
+GLYf8t3gdK03NPfr/FuWf4KQBYqz1txPYjsDARo7S2ifRTdn51186LIvgApmdtNH
+DwP3alClnpIdclktJKJ6m8LQi1HNBpEkTBwWwY9/DODRQT2PJ1VPdsDUja/baIT5
+k3QTz3zo85FVFnyYyky2QsDjkfup9/PQ1h2P8fftNW29naKYff0PfVMCF+80u0y2
+t/zeNHQE/nb/3unhrg4tTiIHiYhsedrVli6BGXOrms6xpYVHK1cJi/JJq8kxaWz9
+ivkAURrgISSu+sleUJI5XMiCvt3AveJxDk2wX0Gyi/eksuqJjoMiaV7cWOIMpfkT
+/h/U2QKCAQAFirvduXBiVpvvXccpCRG4CDe+bADKpfPIpYRAVzaiQ4GzzdlEoMGd
+k3nV28fBjbdbme6ohgT6ilKi3HD2dkO1j5Et6Uz0g/T3tUdTXvycqeRJHXLiOgi9
+d8CGqR456KTF74nBe/whzoiJS9pVkm0cI/hQSz8lVZJu58SqxDewo4HcxV5FRiA6
+PRKtoCPU6Xac+kp4iRx6JwiuXQQQIS+ZovZKFDdiuu/L2gcZrp4eXym9zA+UcxQb
+GUOCYEl9QCPQPLuM19w/Pj3TPXZyUlx81Q0Cka1NALzuc5bYhPKsot3iPrAJCmWV
+L4XtNozCKI6pSg+CABwnp4/mL9nPFsX9AoIBAQDHiDhG9jtBdgtAEog6oL2Z98qR
+u5+nONtLQ61I5R22eZYOgWfxnz08fTtpaHaVWNLNzF0ApyxjxD+zkFHcMJDUuHkR
+O0yxUbCaof7u8EFtq8P9ux4xjtCnZW+9da0Y07zBrcXTsHYnAOiqNbtvVYd6RPiW
+AaE61hgvj1c9/BQh2lUcroQx+yJI8uAAQrfYtXzm90rb6qk6rWy4li2ybMjB+LmP
+cIQIXIUzdwE5uhBnwIre74cIZRXFJBqFY01+mT8ShPUWJkpOe0Fojrkl633TUuNf
+9thZ++Fjvs4s7alFH5Hc7Ulk4v/O1+owdjqERd8zlu7+568C9s50CGwFnH0d
+-----END RSA PRIVATE KEY-----
diff --git a/testdata/x509/server2_cert.pem b/testdata/x509/server2_cert.pem
new file mode 100644
index 000000000000..dc20b468e5d9
--- /dev/null
+++ b/testdata/x509/server2_cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx
+CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV
+BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIwMDgwNDAxNTk1OVoXDTMwMDgwMjAxNTk1
+OVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMjCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBANluCTNFJz8gsMgn2ixQuk4YphdLfbsgOlk2lRFx
+mYBpfD2hfZpnr6c67WNIWBuvMy57z+FWcmmA2iVabEs4OGPaQj5R6cngai01QNPO
+d0gPpcAW/4KuVAYOYiYWSrVOTj8aTZm4buG/VMZMUKUMS0JNXSuYLZrgD23Rsr5K
+j6q2fqRFtcC89QW9opafa4oTmkp6Kz/WrphF4EsK1fbelZ8xQ4+TOkIJegZMS+vA
+r3itgA3ha1xqzUU9+A4xTg8HybRzJMAbtzO0DJMzmfDXXwIzAsdsYerDgaoYlBtP
+5Fnod19g8k8NIJduF8dPRfnyn8fFVisT4fWet59/1jcXUbdsgdPLuuY59sxT/C8o
+HLfn26w4Wda0Sc2XN5qhXwezkPX51mOw2siP81jFeHRQE+J0IOfxjfpdbI1+xdIF
+vsu42NdmYa7a7ejhilZxDYRZSaJLLYE/ZDiGfTBZVoVKRNbM0EZ7VRCN9pN6i5jd
+WsHCjdq1u9rzplA0D3KrycUvlpZc7xFaJxTiVFGiJugJmTJoUpQHnF6chZsGukhA
+pypSB/f+r4tPa81N5X9f9vG0WBXiKGaoWVJXmNOQHaqAYz7maO/JCetjtUn6IH7V
+Ti0qK4yeVh/5GZzC7xFfTmO4oWbz6Cb9FKPSsVjvo/n2Zo0e7CSVKc9oFFBwgjg+
+p6bvAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFNpFUJbVy5fb+Uvm
+jKzuUDbuWctCMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh
+bXBsZS5jb20wDQYJKoZIhvcNAQEFBQADggIBAIaUCF04BpWeQkeUsslTSN44Q95U
+oNlRD19fNXWF8eae7Wl53dFkRhn2nyqx0uoHvFZ5oRhF4v8kzM1cyW4RyLk9WTnh
+Lmg/jfr84bSdWvN8nW5T2jNvq0ltSY414MFu4fHf8/GMbpIKtafFkisFXmhKm8Uc
+zVilTn9Wn087Lkg3FHYVU2v0oWfupM5Qvq6tvZxT2v+7nmES6Cip8Z9U7km04yxV
+hDy6YFdz2UDUYlZaQCsLPmaiIxR/EclSsL6KnMW3UjMyxX8Eft1WPwvTzlQKQFDs
+uEfbq+Cl+cogMaGq1VvAA9cvCUSa1hTathWayKH2q1mPH8sqtbFyged7XXh8mkkf
+8qeYTqfeL74I405Gl3u3/EjVnhSLpOqQOgn2E5HnV0bZaJmGHdU0DIvOyKauinyg
+U4hnL8WBv5en9owQvE+DrivbcG9brqEY3wot0XNzB7pxXjrWdw/PMc/HNPbBsT8s
+Zg0gwxwvpffGemc1L8tiM8aHOp8eR1oVr4szuNDAbAfdEgpwBctXs5JJg81zsmGe
+2jJfHFAeqwhUZgCoF/FjJ+IHxOFZx9IVwrlawPadIFgVh2I0rFUcME0B1/Vk46Gg
+BOiuP9keVX+qhKtqjnfabN9l5iX+zpniHIarke2o6W7nYIgdOtdbmH4YNZxjyidj
+9w/3d/4ItCavbKAn
+-----END CERTIFICATE-----
diff --git a/testdata/x509/server2_key.pem b/testdata/x509/server2_key.pem
new file mode 100644
index 000000000000..b0f6ddf70dce
--- /dev/null
+++ b/testdata/x509/server2_key.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEA2W4JM0UnPyCwyCfaLFC6ThimF0t9uyA6WTaVEXGZgGl8PaF9
+mmevpzrtY0hYG68zLnvP4VZyaYDaJVpsSzg4Y9pCPlHpyeBqLTVA0853SA+lwBb/
+gq5UBg5iJhZKtU5OPxpNmbhu4b9UxkxQpQxLQk1dK5gtmuAPbdGyvkqPqrZ+pEW1
+wLz1Bb2ilp9rihOaSnorP9aumEXgSwrV9t6VnzFDj5M6Qgl6BkxL68CveK2ADeFr
+XGrNRT34DjFODwfJtHMkwBu3M7QMkzOZ8NdfAjMCx2xh6sOBqhiUG0/kWeh3X2Dy
+Tw0gl24Xx09F+fKfx8VWKxPh9Z63n3/WNxdRt2yB08u65jn2zFP8Lygct+fbrDhZ
+1rRJzZc3mqFfB7OQ9fnWY7DayI/zWMV4dFAT4nQg5/GN+l1sjX7F0gW+y7jY12Zh
+rtrt6OGKVnENhFlJokstgT9kOIZ9MFlWhUpE1szQRntVEI32k3qLmN1awcKN2rW7
+2vOmUDQPcqvJxS+WllzvEVonFOJUUaIm6AmZMmhSlAecXpyFmwa6SECnKlIH9/6v
+i09rzU3lf1/28bRYFeIoZqhZUleY05AdqoBjPuZo78kJ62O1SfogftVOLSorjJ5W
+H/kZnMLvEV9OY7ihZvPoJv0Uo9KxWO+j+fZmjR7sJJUpz2gUUHCCOD6npu8CAwEA
+AQKCAgB1i31B0HLlN+EadCEIsCPoMH8qPM+eKFAjBtUT9xwLRfu6veFPZhqaB8tq
+TyQC43aB/MFnivqTeut0IixFhgFGSiph0prXXpFIG3AOkaH+vSbYcBZ2KZSXKZN6
+D7cXyVuX1bp6DjEzreJAyeUXNUxCbdyewsh04Ai3UBSXt2tv2PUiDeWyavTzw49w
+aoMSxII3HVDgVElTXQNizlrZ+X9d7p4dsnReWw0y9nBc5XB3hyShXGpULhEHC/dc
+hN80VPuAqHcHvHQQaZgaxFzGzUg5wiYQddGBv2wL7vmywkArMvfGAn08q1YhR41n
+XL3x4G7s6wwogbk4tjOC8PN4GQ09YxbxJVLSyVIHX/v8tYe8H8acsw4LonkawZVm
+HOgwMpz/hcm7P+ClYjAVUWZjCJt02svDV9U1BPEdBtOXrMDwlBfVuFxtM4GDkKmZ
+GjCLnthpvBXfw6stDKuwE9g+TYVcRMsPhksjE9ZasTTVtFU/qZXhc2bDuJkWaUAd
+yAtxBOQYF9mBN4g35NSE7k8FE3HxNDJx+zstodweq6qhinXchuKAeViap94tneeG
+hoSt9PgMnOnx7V0wIK7DaGCH3ssxbjRQ2wRLdTNYAzhV+tkeDex2zf2xtOvqtWIC
+l5gUSTUnaEYX5wVbCPAJIOAI1TtMe501PfXyZa8wb6p9eSHMAQKCAQEA9DJPKkjI
+p+FLBn5iFgGE6DgiFD+K6OyGtr0Mle8D+XFDtlZClF1sjZHacb3OQgDnlSYXZaZR
+iN5jKuVJsrhCgm9g0QYDwtp+m7JMMX9A19qZbbK71w9Qi80wuRZze0nktr6RKiyS
++x8VXkeSHPUSw7VbzbE/CCm551Z/ORoU5fXnDstPKk/M8K2NSYywwzwaEkEuu0NQ
+/syGxaAW8mThruDAZ4gtJns6IyTmM+8KgkSnbwK5mlOMPhJ+6bHDyeV3OJe2lSVW
+ZRA9kzDFAKlotpwRaSwBdu6chCdDhQGn/WlofJHCt2t5Fh9mK89AXQsXfjAh0O1N
+7zrU/yeNIXJd7wKCAQEA4/CD665RVUwNffb7fa0vnt6Rkj47FdM/BmWpLnb1IC7L
+87Fe9uryaNtghLD6T87vF3MtH2rEfQ2qwR9VRC4MyB5kNvozBVtJbKLy2oRD0/Lp
+GSLhjAiKrzu8Dmwv/5iQhrSRr3mqn/eoIx5ydgot/+OzxgH5Q4CGYvzZUcIMVpi+
+eq4/39vLPQoa5tvT+n0G81sCCVR+sBtBbgVq8WaiqW6UunqP+B4+bPG6jbYMjdcD
+w+ylakjJdAofl5SqcUcUy0UzI1pEjKnlLYyCyuVMlkhVZoaQiX9TTOTZ0jAXnbps
+sDS0fwW1/8J5cSXxIA3q1WVtshst2LwwaCgYlVhHAQKCAQEA0x+v7BnzSZnx+JJK
+EUaM9wyZAjKR0aG1Msat2+9C22W+qiVX+Nfw41EHsLDuY4hOsFe3gM3TzmafDFYi
+ap79+bF73hu6IrwvHEOBtoWTtUusvPf7iQsXk1b62fr8KsqPMCQAc5sIFI8iNVnh
+jKGh8Iya63Jj0ZXpwYW6Bs9y5AK/Gr5SGn3V7PvPnJhDtvf+fmvWkFa57yE7IB+x
+1y27JSvxjVFh39RIRlw/nwT7a/cZX1PWzgOPy5bIHRnw8VwvwEECvV4DnOr2oYxX
+tqPBAahbMTe3qHDR5zvfF16ANArvKEwJMfV8QdExz4ym1Aqj7BiHFBAnAj82Kcez
+MAimBwKCAQADw2LKL1SUbe8DF2LLjmJs4wvQOErNb3Fo76C9baVaZKtlWJZSyUo7
+RPPw/OMFEkuMPZCPJjocPm+FRLkpqQD5BNduuO7CteEedApCZVChXS9QBO1oXHO9
+tOTD8DFSrPgl4TFOjlmszm/uNIB7Rmu//8hmCn5NCQAu/jGwUd3WSCtM5zeSwJQ4
+a8RJ73MufYXx2pzL/qMg0TJhWKGNXr5swbCe64sY85bgQZVs5YaLiPM89tk8SftZ
+eRlQbVnrCNtlB71yZfkfwWZRPDKkmuiKyqLuUGZufrWnXVfjSnv5VKyatCQOvM9m
+a5WJsrCqcNBhuYz4Fc7J90FtVswhGxYBAoIBAQCzNj4K/OrC5fX2sidbcaEU/9S+
+r8JZeCaxAAFepoFKE0LyspNrsW0CZ3Ana3B7SqfH3nAFLoLxExCgMm1WkvwLp22X
+23Gav6cRG4XJjZjyLKW+rcowuhI2Hb6FE2UvshcDzlHpkISpjeY62Qx5gcoLeLlj
+eQpqg59wL5ZweCOcgV/K2nrOILlmQR/GQ68XxvBLoj3J46fc+/iI8G1roGI2H6n6
+tRqmOxRFdmchkPfLPYq5Z71LTWD7m1E27k8apttT8P2mfQhZZ3YYERyiRTTYdO0i
+0ZIi5+OqzTuZuefgurtHDnJe4rFT3/jZzKmI3IfbuRITxmxSgPd7cpuM22uo
+-----END RSA PRIVATE KEY-----
diff --git a/testdata/x509/server_ca_cert.pem b/testdata/x509/server_ca_cert.pem
new file mode 100644
index 000000000000..eee033e8cb05
--- /dev/null
+++ b/testdata/x509/server_ca_cert.pem
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF6jCCA9KgAwIBAgIJAKnJpgBC9CHNMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD
+MRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTAeFw0yMDA4MDQwMTU5NTdaFw0zMDA4
+MDIwMTU5NTdaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD
+U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMZFKSUi+PlQ6z/aTz1Jp9lqrFAY
+38cEIzpxS9ktQiWvLoYICImXRFhCH/h+WjmiyV8zYHcbft63BTUwgXJFuE0cxsJY
+mqOUYL2wTD5PzgoN0B9KVgKyyi0SQ6WH9+D2ZvYAolHb1l6pYuxxk1bQL2OA80Cc
+K659UioynIQtJ52NRqGRDI2EYsC9XRuhfddnDu/RwBaiv3ix84R3VAqcgRyOeGwH
+cX2e+aX0m6ULnsiyPXG9y9wQi956CGGZimInV63S+sU3Mc6PuUt8rwFlmSXCZ/07
+D8No5ljNUo6Vt2BpAMQzSz+SU4PUFE7Vxbq4ypI+2ZbkI80YjDwF52/pMauqZFIP
+Kjw0b2yyWD/F4hLmR7Rx9d8EFWRLZm2VYSVMiQTwANpb+uL7+kH8UE3QF7tryH8K
+G65mMh18XiERgSAWgs5Z8j/B1W5bl17PVx2Ii1dYp0IquyAVjCIKRrFituvoXXZj
+FHHpb/aUDpW0SYrT5dmDhAAGFkYfMTFd4EOj6bWepZtRRjPeIHR9B2yx8U0tFSMf
+tuHCj95l2izJDUfKhVIkigpbRrElI2QqXAPIyIOqcdzlgtI6DIanCd/CwsfdyaEs
+7AnW2mFWarbkxpw92RdGxYy6WXbdM+2EdY+cWKys06upINcnG2zvkCflAE39fg9F
+BVCJC71oO3laXnf7AgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
+FgQUBuToaw2a+AV/vfbooJn3yzwA3lMwgYAGA1UdIwR5MHeAFAbk6GsNmvgFf732
+6KCZ98s8AN5ToVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV
+BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1zZXJ2ZXJfY2GC
+CQCpyaYAQvQhzTAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBALUz
+P2SiZAXZDwCH8kzHbLqsqacSM81bUSuG153t3fhwZU8hzXgQqifFububLkrLaRCj
+VvtIS3XsbHmKYD1TBOOCZy5zE2KdpWYW47LmogBqUllKCSD099UHFB2YUepK9Zci
+oxYJMhNWIhkoJ/NJMp70A8PZtxUvZafeUQl6xueo1yPbfQubg0lG9Pp2xkmTypSv
+WJkpRyX8GSJYFoFFYdNcvICVw7E/Zg+PGXe8gjpAGWW8KxxaohPsdLid6f3KauJM
+UCi/WQECzIpNzxQDSqnGeoqbZp+2y6mhgECQ3mG/K75n0fX0aV88DNwTd1o0xOpv
+lHJo8VD9mvwnapbm/Bc7NWIzCjL8fo0IviRkmAuoz525eBy6NsUCf1f432auvNbg
+OUaGGrY6Kse9sF8Tsc8XMoT9AfGQaR8Ay7oJHjaCZccvuxpB2n//L1UAjMRPYd2y
+XAiSN2xz7WauUh4+v48lKbWa+dwn1G0pa6ZGB7IGBUbgva8Fi3iqVh3UZoz+0PFM
+qVLG2SzhfMTMHg0kF+rI4eOcEKc1j3A83DmTTPZDz3APn53weJLJhKzrgQiI1JRW
+boAJ4VFQF6zjxeecCIIiekH6saYKnol2yL6ksm0jyHoFejkrHWrzoRAwIhTf9avj
+G7QS5fiSQk4PXCX42J5aS/zISy85RT120bkBjV/P
+-----END CERTIFICATE-----
diff --git a/testdata/x509/server_ca_key.pem b/testdata/x509/server_ca_key.pem
new file mode 100644
index 000000000000..114e2a37a11e
--- /dev/null
+++ b/testdata/x509/server_ca_key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDGRSklIvj5UOs/
+2k89SafZaqxQGN/HBCM6cUvZLUIlry6GCAiJl0RYQh/4flo5oslfM2B3G37etwU1
+MIFyRbhNHMbCWJqjlGC9sEw+T84KDdAfSlYCssotEkOlh/fg9mb2AKJR29ZeqWLs
+cZNW0C9jgPNAnCuufVIqMpyELSedjUahkQyNhGLAvV0boX3XZw7v0cAWor94sfOE
+d1QKnIEcjnhsB3F9nvml9JulC57Isj1xvcvcEIveeghhmYpiJ1et0vrFNzHOj7lL
+fK8BZZklwmf9Ow/DaOZYzVKOlbdgaQDEM0s/klOD1BRO1cW6uMqSPtmW5CPNGIw8
+Bedv6TGrqmRSDyo8NG9sslg/xeIS5ke0cfXfBBVkS2ZtlWElTIkE8ADaW/ri+/pB
+/FBN0Be7a8h/ChuuZjIdfF4hEYEgFoLOWfI/wdVuW5dez1cdiItXWKdCKrsgFYwi
+CkaxYrbr6F12YxRx6W/2lA6VtEmK0+XZg4QABhZGHzExXeBDo+m1nqWbUUYz3iB0
+fQdssfFNLRUjH7bhwo/eZdosyQ1HyoVSJIoKW0axJSNkKlwDyMiDqnHc5YLSOgyG
+pwnfwsLH3cmhLOwJ1tphVmq25MacPdkXRsWMull23TPthHWPnFisrNOrqSDXJxts
+75An5QBN/X4PRQVQiQu9aDt5Wl53+wIDAQABAoICADoDco6TNRZ+PtdoIVdlfd93
+/wNQw+mPpF8tV2wsefZc09gT8auQv0az0nb7QZsrrpBUkB1Jxk2Ub8mob7fn/o1R
+pjanhlfmyoe2VhjFcRwv/n2pWpFfjxixB2of5r/EWUwR02zwTkFUfsWAVgRI1hTf
+Xk3BZGah9LC0LmfeboEDHW+Y6XtfCSYsQlobXp7wYMZ7MSFubWf7aa2Q3N5d/MlG
+RqYVZ3fCVHnioMgiJkvDG4d0aXnyvXpTarBkJMGjkVwjJ40dIU23cBhOW0alW7JY
+t+S4q1waDYxeR5HA7O8gykCeYZ4wSo+ANpD6q+h+uYchLLmh93fDfwTxFU8BhK6a
+Dp8ikyZe7hjEba5a7ZvfOXedOZoLqGuUF4P5wI0Hfdslqwq34QSqMiHJuQGa+dM+
+tqnxTw8TjylYysMJxkqipA91uhO9AWxUc37jkWOY255kXcQdKwx5TdQN25XDDjK3
+BNiGtWIEuRMoflO2tL8AmaATOYbVuC3rSm9vtK0jre09MwLxihuzd8fgGBrtEx5S
+UMaBAGDG1F0lcdxQY/h1byL5g1y//N472Ir0PLGczMPBigy+ZEy2GNtwUniwWOWH
+z8CE8BbCr4PMxaqR/qU4hmEw6E3mB8w0WMMGQRn9+jKwxSZaIsE518Wa7oVEx02d
+LZOu9b4xNslw8HjwaSKBAoIBAQDvas21s1EhtgKZaLXNyVsWaX6Mg1h0puylvqlg
+G7t7F7XRV4gPb21e65y29V42vG/r2KB/AJh8eHTFYrOPSPPT1ZZxfxD7yuJGliYc
+LwMU9QWkks5bFEP8nHogBv5nA47Ve+ctgrkwhZneWS896EI0Ulzw90oeOYgzJAmP
+u0IVx6k0SlYKw5b31xWdwRehAIiz0UFufn88QtM3Fhj530It/+mqvrT/MR93XIIm
+0tFLOIGz0Tp4yLleB1h//9xFdLUgDAGXgyC2ivlq5H31rGwkZr0Ixiwm8VOq1yvF
+/ZofDN37RIrIbC2O0shFbU/L4KC99Uu5gDk/bu7INwLrmK3JAoIBAQDUAMNz0Ewg
+cR1hlJ1mDD0IuKHjgjwoOslJ+r/P9tMfXkNudx7IRrnsNlJzB0RYYFaZiyXi4dXn
+nN1C1ePIXo/kfw18Bvl+GIUrHV9EZrMJ+OfdWyXOzv6kfkT4B+axUJpqCTA3aalr
++mI+EpSjw5IHzgEL9cZBlms2YSu6cDxKYXm8sjQ7w0OKSQFsdv4rIfT+xwVyVHMW
+1vn2tYdxnnidzuGUFt1Fhx8SnNHSu6K3rvjoc80jjg3TuOeKtil5AwVhtxqpX5HV
+XAdQwFSZigSkjypnvIlJ9YLVl+64U24UQXBZc3qZdImZqKn38Dfalaiz52CWZPtt
+N6HFzJTAjcmjAoIBAEgWe4wLSxGAcTXp3lvxFfkgmJcMcVTmcfjR+MPUddXxZLB8
+z53+KgtbnBtGtDB8+qIj3ud+sWtBBb/tIS2yhKAy/pJ79QwroYgpa54u1Zm40RMl
+lPa9ml70ap08Hdu8qYREQ25jnwkqIRNe/SeByHVim1N+0hVZs1XasvpRIuvV62+w
+NkoVbF6Bp6ORYWD7/S1Pg4kWk478fAZpI+oQvCeHl77unyb7joLtGs8/yP8CK6OO
+CzIVFiNmyNH5o0RSiLr2goAxXmc4XzM9S2Pun70yJhb/PIoZPd0B3s9FteNFh41B
+rRv93pXTh7PH3y//Gcc4la1sG1CrQUCNt9ZiaWkCggEABHHXpy/wyKVWdltFSYRs
+KyijzD9Iv5cr7S8ioluMZZX2V/SLYquI7ljdNagrWKb8ac+vBaiycV6qjOIrGmJR
+Jfs77yO+S1R8RkEhZC+7BTSAt/VXP5S7Zft3urN/tKv58MsshZzjfm4LbT26fAx3
+nU5GW1fVxj4/FS7IWepMeUq94KTjz3Tyj42kR//eqEzX9Bd8F7+JgisTpoZ7xngK
+E1TpCc/I59JDZoJ/K6nfaXZzpXv4CwzJYWz4/cF/8ReNH1VVa8OjLRP220yM+YMZ
+QdH2k6IyRqitC4lZ6edl4WrVzipLobf9woj0t0wD/8MvfEYXkk+frdSCwcDeRYMz
+fQKCAQB/kbirzZfwH60I6FIlCHohRDJ3csBU3n94UqSOqi/zl6ts+Vgrdq/36UuK
+lww51o1FmtnpI1jaVw8Sug7N4xYUkgCmyFvLU3SUOw68xzPxi7k9NwI+M1jH4ZMK
+JVJXHaxx2bY35rf+y1NKOge24uw//C1aEmKq4Dolql6ZiJlVGUna9lp+VmcDa+XW
+OzGfJWMZeSh2kI8cJrTCrar21zRfF2c6IsoKdDBAmZV1qSgzymzUYtYQ2P1s+qRS
+Cs891gpYRQMchfec7FefWdFYXgEfLRp+nz4WoLaIwK+oftPHl96V1z9rS1Zs2HXD
+okA9YtMucwgrhGFv9T0QtBuq4aEC
+-----END PRIVATE KEY-----

From b2f0b7930d0607d7a64d5c55f4f17430ee74b1ae Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 5 Aug 2020 14:37:15 -0700
Subject: [PATCH 146/481] xds: Pass WatchExpiryTimeout to Client via Options.
 (#3784)

Tests can pass a different value when necessary. This avoids overriding a global variable from tests, leading to unnecessary races.
---
 xds/internal/client/client.go                 | 16 ++++++++
 xds/internal/client/client_test.go            | 11 ++++-
 xds/internal/client/client_watchers.go        | 12 ++----
 .../client/client_watchers_cluster_test.go    | 34 +++++----------
 .../client/client_watchers_endpoints_test.go  | 19 +++------
 .../client/client_watchers_lds_test.go        | 10 ++---
 .../client/client_watchers_rds_test.go        |  8 ++--
 .../client/client_watchers_service_test.go    | 41 +++++--------------
 8 files changed, 65 insertions(+), 86 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index deade155b309..7088c63befcb 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -234,6 +234,16 @@ type Options struct {
 	DialOpts []grpc.DialOption
 	// TargetName is the target of the parent ClientConn.
 	TargetName string
+	// WatchExpiryTimeout is the amount of time the client is willing to wait
+	// for the first response from the server for any resource being watched.
+	// Expiry will not cause cancellation of the watch. It will only trigger the
+	// invocation of the registered callback and it is left up to the caller to
+	// decide whether or not they want to cancel the watch.
+	//
+	// If this field is left unspecified, a default value of 15 seconds will be
+	// used. This is based on the default value of the initial_fetch_timeout
+	// field in corepb.ConfigSource proto.
+	WatchExpiryTimeout time.Duration
 }
 
 // Function to be overridden in tests.
@@ -308,6 +318,12 @@ func New(opts Options) (*Client, error) {
 	}
 	dopts = append(dopts, opts.DialOpts...)
 
+	if opts.WatchExpiryTimeout == 0 {
+		// This is based on the default value of the initial_fetch_timeout field
+		// in corepb.ConfigSource proto.
+		opts.WatchExpiryTimeout = 15 * time.Second
+	}
+
 	c := &Client{
 		done: grpcsync.NewEvent(),
 		opts: opts,
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 849d91a4d852..9584e28b9d17 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -45,15 +45,22 @@ const (
 	testRDSName = "test-rds"
 	testCDSName = "test-cds"
 	testEDSName = "test-eds"
+
+	defaultTestWatchExpiryTimeout = 500 * time.Millisecond
 )
 
-func clientOpts(balancerName string) Options {
+func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
+	watchExpiryTimeout := time.Duration(0)
+	if overrideWatchExpiryTImeout {
+		watchExpiryTimeout = defaultTestWatchExpiryTimeout
+	}
 	return Options{
 		Config: bootstrap.Config{
 			BalancerName: balancerName,
 			Creds:        grpc.WithInsecure(),
 			NodeProto:    testutils.EmptyNodeProtoV2,
 		},
+		WatchExpiryTimeout: watchExpiryTimeout,
 	}
 }
 
@@ -109,7 +116,7 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index 809a20fee768..2cfec37dcc5f 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -26,10 +26,6 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-// The value chosen here is based on the default value of the
-// initial_fetch_timeout field in corepb.ConfigSource proto.
-var defaultWatchExpiryTimeout = 15 * time.Second
-
 type watchInfoState int
 
 const (
@@ -222,7 +218,7 @@ func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (c
 		ldsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
@@ -241,7 +237,7 @@ func (c *Client) watchRDS(routeName string, cb func(RouteConfigUpdate, error)) (
 		rdsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
@@ -367,7 +363,7 @@ func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error))
 		cdsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
@@ -389,7 +385,7 @@ func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, err
 		edsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(defaultWatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index e6530fbf6823..93d32113a310 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -20,7 +20,6 @@ package client
 
 import (
 	"testing"
-	"time"
 
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/version"
@@ -39,7 +38,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -100,7 +99,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -109,10 +108,9 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	v2Client := <-v2ClientCh
 
 	var clusterUpdateChs []*testutils.Channel
-	const count = 2
-
 	var cancelLastWatch func()
 
+	const count = 2
 	for i := 0; i < count; i++ {
 		clusterUpdateCh := testutils.NewChannel()
 		clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh)
@@ -158,7 +156,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -214,7 +212,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -263,16 +261,10 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 // an CDS response for the request that it sends out. We want the watch callback
 // to be invoked with an error once the watchExpiryTimer fires.
 func (s) TestClusterWatchExpiryTimer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -288,7 +280,7 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	u, err := clusterUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	u, err := clusterUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
 	if err != nil {
 		t.Fatalf("failed to get clusterUpdate: %v", err)
 	}
@@ -305,16 +297,10 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 // an CDS response for the request that it sends out. We want no error even
 // after expiry timeout.
 func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -340,7 +326,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	}
 
 	// Wait for an error, the error should never happen.
-	u, err := clusterUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	u, err := clusterUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
 	if err != testutils.ErrRecvTimeout {
 		t.Fatalf("got unexpected: %v, %v, want recv timeout", u.(clusterUpdateErr).u, u.(clusterUpdateErr).err)
 	}
@@ -356,7 +342,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index 27cf2e62d02e..c7e8bd8e52e0 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -20,7 +20,6 @@ package client
 
 import (
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -60,7 +59,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -111,7 +110,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -169,7 +168,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -225,7 +224,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -274,16 +273,10 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 // an CDS response for the request that it sends out. We want the watch callback
 // to be invoked with an error once the watchExpiryTimer fires.
 func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -299,7 +292,7 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	u, err := endpointsUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	u, err := endpointsUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
 	if err != nil {
 		t.Fatalf("failed to get endpointsUpdate: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index ce1e5ed3b086..c052e8bf4ce5 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -38,7 +38,7 @@ func (s) TestLDSWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -90,7 +90,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -148,7 +148,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -204,7 +204,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -259,7 +259,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 009d70380610..6156d195c740 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -39,7 +39,7 @@ func (s) TestRDSWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -90,7 +90,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -148,7 +148,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -204,7 +204,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 08476e2f7f0a..7009373b4e92 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -20,7 +20,6 @@ package client
 
 import (
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -42,7 +41,7 @@ func (s) TestServiceWatch(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -100,7 +99,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -167,7 +166,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -239,16 +238,10 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 // does not respond to the requests being sent out as part of registering a
 // service update watcher. The callback will get an error.
 func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -263,7 +256,7 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	u, err := serviceUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
 	}
@@ -279,16 +272,10 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 // TestServiceWatchEmptyRDS tests the case where the underlying v2Client
 // receives an empty RDS response. The callback will get an error.
 func (s) TestServiceWatchEmptyRDS(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -311,7 +298,7 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{})
-	u, err := serviceUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2)
+	u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
 	}
@@ -328,16 +315,10 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 // received after the client is closed, and we make sure that the registered
 // watcher callback is not invoked.
 func (s) TestServiceWatchWithClientClose(t *testing.T) {
-	oldWatchExpiryTimeout := defaultWatchExpiryTimeout
-	defaultWatchExpiryTimeout = 500 * time.Millisecond
-	defer func() {
-		defaultWatchExpiryTimeout = oldWatchExpiryTimeout
-	}()
-
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -361,7 +342,7 @@ func (s) TestServiceWatchWithClientClose(t *testing.T) {
 	}
 	// Client is closed before it receives the RDS response.
 	c.Close()
-	if u, err := serviceUpdateCh.TimedReceive(defaultWatchExpiryTimeout * 2); err != testutils.ErrRecvTimeout {
+	if u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2); err != testutils.ErrRecvTimeout {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -373,7 +354,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -424,7 +405,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	v2ClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer))
+	c, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}

From 446b4dba547f36b5c5f469bd1b50c136add3186e Mon Sep 17 00:00:00 2001
From: cindyxue <32377977+cindyxue@users.noreply.github.com>
Date: Thu, 6 Aug 2020 10:40:31 -0700
Subject: [PATCH 147/481] advancedtls: add spiffe support in aTLS (#3777)

* Add SPIFFE ID in TLSInfo
---
 security/advancedtls/advancedtls.go |  3 ++
 security/advancedtls/go.mod         |  5 +--
 security/advancedtls/go.sum         | 61 +++++++++++++----------------
 3 files changed, 33 insertions(+), 36 deletions(-)

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index e028c623ba43..40a9fd5f735e 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc/credentials"
+	credinternal "google.golang.org/grpc/internal/credentials"
 )
 
 // VerificationFuncParams contains parameters available to users when
@@ -299,6 +300,7 @@ func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string
 			SecurityLevel: credentials.PrivacyAndIntegrity,
 		},
 	}
+	info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState())
 	return WrapSyscallConn(rawConn, conn), info, nil
 }
 
@@ -316,6 +318,7 @@ func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credenti
 			SecurityLevel: credentials.PrivacyAndIntegrity,
 		},
 	}
+	info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState())
 	return WrapSyscallConn(rawConn, conn), info, nil
 }
 
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index f21b428ef4c0..58823b03e877 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -3,11 +3,10 @@ module google.golang.org/grpc/security/advancedtls
 go 1.13
 
 require (
-	github.com/golang/protobuf v1.3.5 // indirect
 	github.com/google/go-cmp v0.4.0
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
 	golang.org/x/text v0.3.3 // indirect
-	google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 // indirect
-	google.golang.org/grpc v1.29.1
+	google.golang.org/grpc v1.31.0
+	google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f
 )
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index fd8b6c4732d7..8283cf07e736 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,64 +1,51 @@
-cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4 h1:c2HOrn5iMezYjSlGPncknSEr/8x5LELb/ilJbXi9DEA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -66,26 +53,34 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 h1:fiNLklpBwWK1mth30Hlwk+fcdBmIALlgF5iy77O37Ig=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad h1:uAwc13+y0Y8QZLTYhLCu6lHhnG99ecQU5FYTj8zxAng=
+google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f h1:HNNmM2dnxUknBEJDvuCRZcUzhGbhkz00ckV2ha2gFJY=
+google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f/go.mod h1:TGiSRL2BBv2WqzfsFNWYp/pkWdtf5kbZS/DQ9Ee3mWk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From b830b5f361072689b94d99055e8ffc51821eabf3 Mon Sep 17 00:00:00 2001
From: cindyxue <32377977+cindyxue@users.noreply.github.com>
Date: Thu, 6 Aug 2020 11:10:47 -0700
Subject: [PATCH 148/481] advancedtls: fixed SNI testing and put SNI functions
 back in advancedtls.go (#3774)

* Fixed sni unit test
---
 security/advancedtls/advancedtls.go       |  27 ++++++
 security/advancedtls/advancedtls_test.go  |  72 +++++++++++++++
 security/advancedtls/sni.go               |  51 -----------
 security/advancedtls/sni_test_disabled.go | 106 ----------------------
 4 files changed, 99 insertions(+), 157 deletions(-)
 delete mode 100644 security/advancedtls/sni.go
 delete mode 100644 security/advancedtls/sni_test_disabled.go

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index 40a9fd5f735e..c21ff753aab6 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -506,3 +506,30 @@ func cloneTLSConfig(cfg *tls.Config) *tls.Config {
 	}
 	return cfg.Clone()
 }
+
+// buildGetCertificates returns the certificate that matches the SNI field
+// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates.
+func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
+	if o.GetCertificates == nil {
+		return nil, fmt.Errorf("function GetCertificates must be specified")
+	}
+	certificates, err := o.GetCertificates(clientHello)
+	if err != nil {
+		return nil, err
+	}
+	if len(certificates) == 0 {
+		return nil, fmt.Errorf("no certificates configured")
+	}
+	// If users pass in only one certificate, return that certificate.
+	if len(certificates) == 1 {
+		return certificates[0], nil
+	}
+	// Choose the SNI certificate using SupportsCertificate.
+	for _, cert := range certificates {
+		if err := clientHello.SupportsCertificate(cert); err == nil {
+			return cert, nil
+		}
+	}
+	// If nothing matches, return the first certificate.
+	return certificates[0], nil
+}
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 4e79d0d89742..8800a51a94d6 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -26,11 +26,13 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"math/big"
 	"net"
 	"reflect"
 	"syscall"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/security/advancedtls/testdata"
@@ -688,3 +690,73 @@ func (s) TestOptionsConfig(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestGetCertificatesSNI(t *testing.T) {
+	// Load server certificates for setting the serverGetCert callback function.
+	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err)
+	}
+	serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err)
+	}
+	serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err)
+	}
+
+	tests := []struct {
+		desc       string
+		serverName string
+		wantCert   tls.Certificate
+	}{
+		{
+			desc: "Select serverCert1",
+			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
+			serverName: "foo.bar.com",
+			wantCert:   serverCert1,
+		},
+		{
+			desc: "Select serverCert2",
+			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
+			serverName: "foo.bar.server2.com",
+			wantCert:   serverCert2,
+		},
+		{
+			desc: "Select serverCert3",
+			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
+			serverName: "google.com",
+			wantCert:   serverCert3,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			serverOptions := &ServerOptions{
+				GetCertificates: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+					return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+				},
+			}
+			serverConfig, err := serverOptions.config()
+			if err != nil {
+				t.Fatalf("serverOptions.config() failed: %v", err)
+			}
+			pointFormatUncompressed := uint8(0)
+			clientHello := &tls.ClientHelloInfo{
+				CipherSuites:      []uint16{tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA},
+				ServerName:        test.serverName,
+				SupportedCurves:   []tls.CurveID{tls.CurveP256},
+				SupportedPoints:   []uint8{pointFormatUncompressed},
+				SupportedVersions: []uint16{tls.VersionTLS10},
+			}
+			gotCertificate, err := serverConfig.GetCertificate(clientHello)
+			if err != nil {
+				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
+			}
+			if !cmp.Equal(*gotCertificate, test.wantCert, cmp.AllowUnexported(big.Int{})) {
+				t.Errorf("GetCertificates() = %v, want %v", *gotCertificate, test.wantCert)
+			}
+		})
+	}
+}
diff --git a/security/advancedtls/sni.go b/security/advancedtls/sni.go
deleted file mode 100644
index 00e551fac296..000000000000
--- a/security/advancedtls/sni.go
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package advancedtls
-
-import (
-	"crypto/tls"
-	"fmt"
-)
-
-// buildGetCertificates returns the certificate that matches the SNI field
-// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates.
-func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
-	if o.GetCertificates == nil {
-		return nil, fmt.Errorf("function GetCertificates must be specified")
-	}
-	certificates, err := o.GetCertificates(clientHello)
-	if err != nil {
-		return nil, err
-	}
-	if len(certificates) == 0 {
-		return nil, fmt.Errorf("no certificates configured")
-	}
-	// If users pass in only one certificate, return that certificate.
-	if len(certificates) == 1 {
-		return certificates[0], nil
-	}
-	// Choose the SNI certificate using SupportsCertificate.
-	for _, cert := range certificates {
-		if err := clientHello.SupportsCertificate(cert); err == nil {
-			return cert, nil
-		}
-	}
-	// If nothing matches, return the first certificate.
-	return certificates[0], nil
-}
diff --git a/security/advancedtls/sni_test_disabled.go b/security/advancedtls/sni_test_disabled.go
deleted file mode 100644
index 3e9e19c11be9..000000000000
--- a/security/advancedtls/sni_test_disabled.go
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package advancedtls
-
-import (
-	"crypto/tls"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/security/advancedtls/testdata"
-)
-
-// TestGetCertificatesSNI tests SNI logic for go1.10 and above.
-func TestGetCertificatesSNI(t *testing.T) {
-	// Load server certificates for setting the serverGetCert callback function.
-	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err)
-	}
-	serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err)
-	}
-	serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err)
-	}
-
-	tests := []struct {
-		desc          string
-		serverGetCert func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
-		serverName    string
-		wantCert      tls.Certificate
-	}{
-		{
-			desc: "Select serverCert1",
-			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
-			},
-			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
-			serverName: "foo.bar.com",
-			wantCert:   serverCert1,
-		},
-		{
-			desc: "Select serverCert2",
-			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
-			},
-			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
-			serverName: "foo.bar.server2.com",
-			wantCert:   serverCert2,
-		},
-		{
-			desc: "Select serverCert3",
-			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
-			},
-			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
-			serverName: "google.com",
-			wantCert:   serverCert3,
-		},
-	}
-	for _, test := range tests {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			serverOptions := &ServerOptions{
-				GetCertificates: test.serverGetCert,
-			}
-			serverConfig, err := serverOptions.config()
-			if err != nil {
-				t.Fatalf("serverOptions.config() failed: %v", err)
-			}
-			pointFormatUncompressed := uint8(0)
-			clientHello := &tls.ClientHelloInfo{
-				CipherSuites:      []uint16{tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA},
-				ServerName:        test.serverName,
-				SupportedCurves:   []tls.CurveID{tls.CurveP256},
-				SupportedPoints:   []uint8{pointFormatUncompressed},
-				SupportedVersions: []uint16{tls.VersionTLS10},
-			}
-			gotCertificate, err := serverConfig.GetCertificate(clientHello)
-			if err != nil {
-				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
-			}
-			if !cmp.Equal(gotCertificate, test.wantCert, cmp.AllowUnexported(tls.Certificate{})) {
-				t.Errorf("GetCertificates() = %v, want %v", gotCertificate, test.wantCert)
-			}
-		})
-	}
-}

From a5514c9e508fbd88377313da1a94fc0d69aa0d25 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 6 Aug 2020 13:10:09 -0700
Subject: [PATCH 149/481] grpc: Minor refactor in server code. (#3779)

---
 server.go | 87 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 44 insertions(+), 43 deletions(-)

diff --git a/server.go b/server.go
index 1b56cc2d11fd..867b012deff5 100644
--- a/server.go
+++ b/server.go
@@ -80,13 +80,14 @@ type ServiceDesc struct {
 	Metadata    interface{}
 }
 
-// service consists of the information of the server serving this service and
-// the methods in this service.
-type service struct {
-	server interface{} // the server for service methods
-	md     map[string]*MethodDesc
-	sd     map[string]*StreamDesc
-	mdata  interface{}
+// serviceInfo wraps information about a service. It is very similar to
+// ServiceDesc and is constructed from it for internal purposes.
+type serviceInfo struct {
+	// Contains the implementation for the methods in this service.
+	serviceImpl interface{}
+	methods     map[string]*MethodDesc
+	streams     map[string]*StreamDesc
+	mdata       interface{}
 }
 
 type serverWorkerData struct {
@@ -99,14 +100,14 @@ type serverWorkerData struct {
 type Server struct {
 	opts serverOptions
 
-	mu     sync.Mutex // guards following
-	lis    map[net.Listener]bool
-	conns  map[transport.ServerTransport]bool
-	serve  bool
-	drain  bool
-	cv     *sync.Cond          // signaled when connections close for GracefulStop
-	m      map[string]*service // service name -> service info
-	events trace.EventLog
+	mu       sync.Mutex // guards following
+	lis      map[net.Listener]bool
+	conns    map[transport.ServerTransport]bool
+	serve    bool
+	drain    bool
+	cv       *sync.Cond              // signaled when connections close for GracefulStop
+	services map[string]*serviceInfo // service name -> service info
+	events   trace.EventLog
 
 	quit               *grpcsync.Event
 	done               *grpcsync.Event
@@ -497,13 +498,13 @@ func NewServer(opt ...ServerOption) *Server {
 		o.apply(&opts)
 	}
 	s := &Server{
-		lis:    make(map[net.Listener]bool),
-		opts:   opts,
-		conns:  make(map[transport.ServerTransport]bool),
-		m:      make(map[string]*service),
-		quit:   grpcsync.NewEvent(),
-		done:   grpcsync.NewEvent(),
-		czData: new(channelzData),
+		lis:      make(map[net.Listener]bool),
+		opts:     opts,
+		conns:    make(map[transport.ServerTransport]bool),
+		services: make(map[string]*serviceInfo),
+		quit:     grpcsync.NewEvent(),
+		done:     grpcsync.NewEvent(),
+		czData:   new(channelzData),
 	}
 	chainUnaryServerInterceptors(s)
 	chainStreamServerInterceptors(s)
@@ -558,24 +559,24 @@ func (s *Server) register(sd *ServiceDesc, ss interface{}) {
 	if s.serve {
 		logger.Fatalf("grpc: Server.RegisterService after Server.Serve for %q", sd.ServiceName)
 	}
-	if _, ok := s.m[sd.ServiceName]; ok {
+	if _, ok := s.services[sd.ServiceName]; ok {
 		logger.Fatalf("grpc: Server.RegisterService found duplicate service registration for %q", sd.ServiceName)
 	}
-	srv := &service{
-		server: ss,
-		md:     make(map[string]*MethodDesc),
-		sd:     make(map[string]*StreamDesc),
-		mdata:  sd.Metadata,
+	info := &serviceInfo{
+		serviceImpl: ss,
+		methods:     make(map[string]*MethodDesc),
+		streams:     make(map[string]*StreamDesc),
+		mdata:       sd.Metadata,
 	}
 	for i := range sd.Methods {
 		d := &sd.Methods[i]
-		srv.md[d.MethodName] = d
+		info.methods[d.MethodName] = d
 	}
 	for i := range sd.Streams {
 		d := &sd.Streams[i]
-		srv.sd[d.StreamName] = d
+		info.streams[d.StreamName] = d
 	}
-	s.m[sd.ServiceName] = srv
+	s.services[sd.ServiceName] = info
 }
 
 // MethodInfo contains the information of an RPC including its method name and type.
@@ -599,16 +600,16 @@ type ServiceInfo struct {
 // Service names include the package names, in the form of <package>.<service>.
 func (s *Server) GetServiceInfo() map[string]ServiceInfo {
 	ret := make(map[string]ServiceInfo)
-	for n, srv := range s.m {
-		methods := make([]MethodInfo, 0, len(srv.md)+len(srv.sd))
-		for m := range srv.md {
+	for n, srv := range s.services {
+		methods := make([]MethodInfo, 0, len(srv.methods)+len(srv.streams))
+		for m := range srv.methods {
 			methods = append(methods, MethodInfo{
 				Name:           m,
 				IsClientStream: false,
 				IsServerStream: false,
 			})
 		}
-		for m, d := range srv.sd {
+		for m, d := range srv.streams {
 			methods = append(methods, MethodInfo{
 				Name:           m,
 				IsClientStream: d.ClientStreams,
@@ -1020,7 +1021,7 @@ func getChainUnaryHandler(interceptors []UnaryServerInterceptor, curr int, info
 	}
 }
 
-func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, md *MethodDesc, trInfo *traceInfo) (err error) {
+func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, info *serviceInfo, md *MethodDesc, trInfo *traceInfo) (err error) {
 	sh := s.opts.statsHandler
 	if sh != nil || trInfo != nil || channelz.IsOn() {
 		if channelz.IsOn() {
@@ -1177,7 +1178,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 		return nil
 	}
 	ctx := NewContextWithServerTransportStream(stream.Context(), stream)
-	reply, appErr := md.Handler(srv.server, ctx, df, s.opts.unaryInt)
+	reply, appErr := md.Handler(info.serviceImpl, ctx, df, s.opts.unaryInt)
 	if appErr != nil {
 		appStatus, ok := status.FromError(appErr)
 		if !ok {
@@ -1303,7 +1304,7 @@ func getChainStreamHandler(interceptors []StreamServerInterceptor, curr int, inf
 	}
 }
 
-func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) {
+func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, info *serviceInfo, sd *StreamDesc, trInfo *traceInfo) (err error) {
 	if channelz.IsOn() {
 		s.incrCallsStarted()
 	}
@@ -1420,8 +1421,8 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp
 	}
 	var appErr error
 	var server interface{}
-	if srv != nil {
-		server = srv.server
+	if info != nil {
+		server = info.serviceImpl
 	}
 	if s.opts.streamInt == nil {
 		appErr = sd.Handler(server, ss)
@@ -1497,13 +1498,13 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str
 	service := sm[:pos]
 	method := sm[pos+1:]
 
-	srv, knownService := s.m[service]
+	srv, knownService := s.services[service]
 	if knownService {
-		if md, ok := srv.md[method]; ok {
+		if md, ok := srv.methods[method]; ok {
 			s.processUnaryRPC(t, stream, srv, md, trInfo)
 			return
 		}
-		if sd, ok := srv.sd[method]; ok {
+		if sd, ok := srv.streams[method]; ok {
 			s.processStreamingRPC(t, stream, srv, sd, trInfo)
 			return
 		}

From 8beeedb8b1be952f00e9d0a4227089b6961dbf49 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 6 Aug 2020 13:19:17 -0700
Subject: [PATCH 150/481] internal: clean up all usages of grpclog, and replace
 with component (#3719)

---
 benchmark/benchmain/main.go                  |  14 +-
 benchmark/benchmark.go                       |  14 +-
 benchmark/client/main.go                     |  18 ++-
 benchmark/server/main.go                     |  12 +-
 benchmark/worker/benchmark_client.go         |  29 ++--
 benchmark/worker/benchmark_server.go         |  23 ++-
 benchmark/worker/main.go                     |  34 ++--
 connectivity/connectivity.go                 |   4 +-
 credentials/google/google.go                 |   8 +-
 credentials/sts/sts.go                       |   3 +-
 grpclog/component.go                         |   2 +-
 health/logging.go                            |  23 +++
 health/server.go                             |   3 +-
 internal/binarylog/binarylog_end2end_test.go |  10 +-
 internal/credentials/spiffe.go               |  10 +-
 internal/serviceconfig/serviceconfig.go      |   4 +-
 interop/alts/client/client.go                |  10 +-
 interop/alts/server/server.go                |  20 +--
 interop/client/client.go                     |  76 ++++-----
 interop/fake_grpclb/fake_grpclb.go           |  38 ++---
 interop/http2/negative_http2_client.go       |  30 ++--
 interop/server/server.go                     |   8 +-
 interop/test_utils.go                        | 160 ++++++++++---------
 interop/xds/client/client.go                 |  10 +-
 interop/xds/server/server.go                 |   4 +-
 profiling/cmd/catapult.go                    |  33 ++--
 profiling/cmd/local.go                       |  10 +-
 profiling/cmd/main.go                        |   8 +-
 profiling/cmd/remote.go                      |  23 ++-
 profiling/service/service.go                 |  10 +-
 stress/client/main.go                        |  38 ++---
 stress/metrics_client/main.go                |  14 +-
 test/balancer_test.go                        |   7 +-
 test/end2end_test.go                         |  11 +-
 test/logging.go                              |  23 +++
 vet.sh                                       |   3 +
 36 files changed, 417 insertions(+), 330 deletions(-)
 create mode 100644 health/logging.go
 create mode 100644 test/logging.go

diff --git a/benchmark/benchmain/main.go b/benchmark/benchmain/main.go
index a8bcb9deeec2..956024c7f047 100644
--- a/benchmark/benchmain/main.go
+++ b/benchmark/benchmain/main.go
@@ -100,6 +100,8 @@ var (
 	useBufconn          = flag.Bool("bufconn", false, "Use in-memory connection instead of system network I/O")
 	enableKeepalive     = flag.Bool("enable_keepalive", false, "Enable client keepalive. \n"+
 		"Keepalive.Time is set to 10s, Keepalive.Timeout is set to 1s, Keepalive.PermitWithoutStream is set to true.")
+
+	logger = grpclog.Component("benchmark")
 )
 
 const (
@@ -314,7 +316,7 @@ func makeClient(bf stats.Features) (testpb.BenchmarkServiceClient, func()) {
 		var err error
 		lis, err = net.Listen("tcp", "localhost:0")
 		if err != nil {
-			grpclog.Fatalf("Failed to listen: %v", err)
+			logger.Fatalf("Failed to listen: %v", err)
 		}
 		opts = append(opts, grpc.WithContextDialer(func(ctx context.Context, address string) (net.Conn, error) {
 			return nw.ContextDialer((&net.Dialer{}).DialContext)(ctx, "tcp", lis.Addr().String())
@@ -351,7 +353,7 @@ func makeFuncStream(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) {
 	for i := 0; i < bf.MaxConcurrentCalls; i++ {
 		stream, err := tc.StreamingCall(context.Background())
 		if err != nil {
-			grpclog.Fatalf("%v.StreamingCall(_) = _, %v", tc, err)
+			logger.Fatalf("%v.StreamingCall(_) = _, %v", tc, err)
 		}
 		streams[i] = stream
 	}
@@ -377,7 +379,7 @@ func makeFuncUnconstrainedStreamPreloaded(bf stats.Features) (rpcSendFunc, rpcRe
 		preparedMsg[i] = &grpc.PreparedMsg{}
 		err := preparedMsg[i].Encode(stream, req)
 		if err != nil {
-			grpclog.Fatalf("%v.Encode(%v, %v) = %v", preparedMsg[i], req, stream, err)
+			logger.Fatalf("%v.Encode(%v, %v) = %v", preparedMsg[i], req, stream, err)
 		}
 	}
 
@@ -405,7 +407,7 @@ func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_Stre
 	for i := 0; i < bf.MaxConcurrentCalls; i++ {
 		stream, err := tc.UnconstrainedStreamingCall(context.Background())
 		if err != nil {
-			grpclog.Fatalf("%v.UnconstrainedStreamingCall(_) = _, %v", tc, err)
+			logger.Fatalf("%v.UnconstrainedStreamingCall(_) = _, %v", tc, err)
 		}
 		streams[i] = stream
 	}
@@ -424,13 +426,13 @@ func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_Stre
 // request and response sizes.
 func unaryCaller(client testpb.BenchmarkServiceClient, reqSize, respSize int) {
 	if err := bm.DoUnaryCall(client, reqSize, respSize); err != nil {
-		grpclog.Fatalf("DoUnaryCall failed: %v", err)
+		logger.Fatalf("DoUnaryCall failed: %v", err)
 	}
 }
 
 func streamCaller(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) {
 	if err := bm.DoStreamingRoundTrip(stream, reqSize, respSize); err != nil {
-		grpclog.Fatalf("DoStreamingRoundTrip failed: %v", err)
+		logger.Fatalf("DoStreamingRoundTrip failed: %v", err)
 	}
 }
 
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index d82b61c9c619..56841e172366 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -35,18 +35,20 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+var logger = grpclog.Component("benchmark")
+
 // Allows reuse of the same testpb.Payload object.
 func setPayload(p *testpb.Payload, t testpb.PayloadType, size int) {
 	if size < 0 {
-		grpclog.Fatalf("Requested a response with invalid length %d", size)
+		logger.Fatalf("Requested a response with invalid length %d", size)
 	}
 	body := make([]byte, size)
 	switch t {
 	case testpb.PayloadType_COMPRESSABLE:
 	case testpb.PayloadType_UNCOMPRESSABLE:
-		grpclog.Fatalf("PayloadType UNCOMPRESSABLE is not supported")
+		logger.Fatalf("PayloadType UNCOMPRESSABLE is not supported")
 	default:
-		grpclog.Fatalf("Unsupported payload type: %d", t)
+		logger.Fatalf("Unsupported payload type: %d", t)
 	}
 	p.Type = t
 	p.Body = body
@@ -213,11 +215,11 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() {
 	case "bytebuf":
 		respSize, ok := info.Metadata.(int32)
 		if !ok {
-			grpclog.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type)
+			logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type)
 		}
 		testpb.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize})
 	default:
-		grpclog.Fatalf("failed to StartServer, unknown Type: %v", info.Type)
+		logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type)
 	}
 	go s.Serve(info.Listener)
 	return func() {
@@ -288,7 +290,7 @@ func NewClientConnWithContext(ctx context.Context, addr string, opts ...grpc.Dia
 	opts = append(opts, grpc.WithReadBufferSize(128*1024))
 	conn, err := grpc.DialContext(ctx, addr, opts...)
 	if err != nil {
-		grpclog.Fatalf("NewClientConn(%q) failed to create a ClientConn %v", addr, err)
+		logger.Fatalf("NewClientConn(%q) failed to create a ClientConn %v", addr, err)
 	}
 	return conn
 }
diff --git a/benchmark/client/main.go b/benchmark/client/main.go
index 762024397810..f750c5c30721 100644
--- a/benchmark/client/main.go
+++ b/benchmark/client/main.go
@@ -76,12 +76,14 @@ var (
 	}
 	mu    sync.Mutex
 	hists []*stats.Histogram
+
+	logger = grpclog.Component("benchmark")
 )
 
 func main() {
 	flag.Parse()
 	if *testName == "" {
-		grpclog.Fatalf("test_name not set")
+		logger.Fatalf("test_name not set")
 	}
 	req := &testpb.SimpleRequest{
 		ResponseType: testpb.PayloadType_COMPRESSABLE,
@@ -98,7 +100,7 @@ func main() {
 	endDeadline := warmDeadline.Add(time.Duration(*duration) * time.Second)
 	cf, err := os.Create("/tmp/" + *testName + ".cpu")
 	if err != nil {
-		grpclog.Fatalf("Error creating file: %v", err)
+		logger.Fatalf("Error creating file: %v", err)
 	}
 	defer cf.Close()
 	pprof.StartCPUProfile(cf)
@@ -111,12 +113,12 @@ func main() {
 	pprof.StopCPUProfile()
 	mf, err := os.Create("/tmp/" + *testName + ".mem")
 	if err != nil {
-		grpclog.Fatalf("Error creating file: %v", err)
+		logger.Fatalf("Error creating file: %v", err)
 	}
 	defer mf.Close()
 	runtime.GC() // materialize all statistics
 	if err := pprof.WriteHeapProfile(mf); err != nil {
-		grpclog.Fatalf("Error writing memory profile: %v", err)
+		logger.Fatalf("Error writing memory profile: %v", err)
 	}
 	hist := stats.NewHistogram(hopts)
 	for _, h := range hists {
@@ -166,20 +168,20 @@ func makeCaller(cc *grpc.ClientConn, req *testpb.SimpleRequest) func() {
 	if *rpcType == "unary" {
 		return func() {
 			if _, err := client.UnaryCall(context.Background(), req); err != nil {
-				grpclog.Fatalf("RPC failed: %v", err)
+				logger.Fatalf("RPC failed: %v", err)
 			}
 		}
 	}
 	stream, err := client.StreamingCall(context.Background())
 	if err != nil {
-		grpclog.Fatalf("RPC failed: %v", err)
+		logger.Fatalf("RPC failed: %v", err)
 	}
 	return func() {
 		if err := stream.Send(req); err != nil {
-			grpclog.Fatalf("Streaming RPC failed to send: %v", err)
+			logger.Fatalf("Streaming RPC failed to send: %v", err)
 		}
 		if _, err := stream.Recv(); err != nil {
-			grpclog.Fatalf("Streaming RPC failed to read: %v", err)
+			logger.Fatalf("Streaming RPC failed to read: %v", err)
 		}
 	}
 }
diff --git a/benchmark/server/main.go b/benchmark/server/main.go
index 59987366b459..5a82b1c78012 100644
--- a/benchmark/server/main.go
+++ b/benchmark/server/main.go
@@ -46,22 +46,24 @@ import (
 var (
 	port     = flag.String("port", "50051", "Localhost port to listen on.")
 	testName = flag.String("test_name", "", "Name of the test used for creating profiles.")
+
+	logger = grpclog.Component("benchmark")
 )
 
 func main() {
 	flag.Parse()
 	if *testName == "" {
-		grpclog.Fatalf("test name not set")
+		logger.Fatalf("test name not set")
 	}
 	lis, err := net.Listen("tcp", ":"+*port)
 	if err != nil {
-		grpclog.Fatalf("Failed to listen: %v", err)
+		logger.Fatalf("Failed to listen: %v", err)
 	}
 	defer lis.Close()
 
 	cf, err := os.Create("/tmp/" + *testName + ".cpu")
 	if err != nil {
-		grpclog.Fatalf("Failed to create file: %v", err)
+		logger.Fatalf("Failed to create file: %v", err)
 	}
 	defer cf.Close()
 	pprof.StartCPUProfile(cf)
@@ -77,12 +79,12 @@ func main() {
 	pprof.StopCPUProfile()
 	mf, err := os.Create("/tmp/" + *testName + ".mem")
 	if err != nil {
-		grpclog.Fatalf("Failed to create file: %v", err)
+		logger.Fatalf("Failed to create file: %v", err)
 	}
 	defer mf.Close()
 	runtime.GC() // materialize all statistics
 	if err := pprof.WriteHeapProfile(mf); err != nil {
-		grpclog.Fatalf("Failed to write memory profile: %v", err)
+		logger.Fatalf("Failed to write memory profile: %v", err)
 	}
 	fmt.Println("Server CPU utilization:", cpu)
 	fmt.Println("Server CPU profile:", cf.Name())
diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go
index bb3097f8c835..3fa682d5a77e 100644
--- a/benchmark/worker/benchmark_client.go
+++ b/benchmark/worker/benchmark_client.go
@@ -32,7 +32,6 @@ import (
 	"google.golang.org/grpc/benchmark/stats"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/syscall"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/testdata"
@@ -81,20 +80,20 @@ func printClientConfig(config *testpb.ClientConfig) {
 	//     will always create sync client
 	// - async client threads.
 	// - core list
-	grpclog.Infof(" * client type: %v (ignored, always creates sync client)", config.ClientType)
-	grpclog.Infof(" * async client threads: %v (ignored)", config.AsyncClientThreads)
+	logger.Infof(" * client type: %v (ignored, always creates sync client)", config.ClientType)
+	logger.Infof(" * async client threads: %v (ignored)", config.AsyncClientThreads)
 	// TODO: use cores specified by CoreList when setting list of cores is supported in go.
-	grpclog.Infof(" * core list: %v (ignored)", config.CoreList)
-
-	grpclog.Infof(" - security params: %v", config.SecurityParams)
-	grpclog.Infof(" - core limit: %v", config.CoreLimit)
-	grpclog.Infof(" - payload config: %v", config.PayloadConfig)
-	grpclog.Infof(" - rpcs per chann: %v", config.OutstandingRpcsPerChannel)
-	grpclog.Infof(" - channel number: %v", config.ClientChannels)
-	grpclog.Infof(" - load params: %v", config.LoadParams)
-	grpclog.Infof(" - rpc type: %v", config.RpcType)
-	grpclog.Infof(" - histogram params: %v", config.HistogramParams)
-	grpclog.Infof(" - server targets: %v", config.ServerTargets)
+	logger.Infof(" * core list: %v (ignored)", config.CoreList)
+
+	logger.Infof(" - security params: %v", config.SecurityParams)
+	logger.Infof(" - core limit: %v", config.CoreLimit)
+	logger.Infof(" - payload config: %v", config.PayloadConfig)
+	logger.Infof(" - rpcs per chann: %v", config.OutstandingRpcsPerChannel)
+	logger.Infof(" - channel number: %v", config.ClientChannels)
+	logger.Infof(" - load params: %v", config.LoadParams)
+	logger.Infof(" - rpc type: %v", config.RpcType)
+	logger.Infof(" - histogram params: %v", config.HistogramParams)
+	logger.Infof(" - server targets: %v", config.ServerTargets)
 }
 
 func setupClientEnv(config *testpb.ClientConfig) {
@@ -298,7 +297,7 @@ func (bc *benchmarkClient) doCloseLoopStreaming(conns []*grpc.ClientConn, rpcCou
 			c := testpb.NewBenchmarkServiceClient(conn)
 			stream, err := c.StreamingCall(context.Background())
 			if err != nil {
-				grpclog.Fatalf("%v.StreamingCall(_) = _, %v", c, err)
+				logger.Fatalf("%v.StreamingCall(_) = _, %v", c, err)
 			}
 			// Create histogram for each goroutine.
 			idx := ic*rpcCountPerConn + j
diff --git a/benchmark/worker/benchmark_server.go b/benchmark/worker/benchmark_server.go
index 0c7c64804abb..c2e5e2609786 100644
--- a/benchmark/worker/benchmark_server.go
+++ b/benchmark/worker/benchmark_server.go
@@ -33,7 +33,6 @@ import (
 	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/syscall"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/testdata"
@@ -59,15 +58,15 @@ func printServerConfig(config *testpb.ServerConfig) {
 	//     will always start sync server
 	// - async server threads
 	// - core list
-	grpclog.Infof(" * server type: %v (ignored, always starts sync server)", config.ServerType)
-	grpclog.Infof(" * async server threads: %v (ignored)", config.AsyncServerThreads)
+	logger.Infof(" * server type: %v (ignored, always starts sync server)", config.ServerType)
+	logger.Infof(" * async server threads: %v (ignored)", config.AsyncServerThreads)
 	// TODO: use cores specified by CoreList when setting list of cores is supported in go.
-	grpclog.Infof(" * core list: %v (ignored)", config.CoreList)
+	logger.Infof(" * core list: %v (ignored)", config.CoreList)
 
-	grpclog.Infof(" - security params: %v", config.SecurityParams)
-	grpclog.Infof(" - core limit: %v", config.CoreLimit)
-	grpclog.Infof(" - port: %v", config.Port)
-	grpclog.Infof(" - payload config: %v", config.PayloadConfig)
+	logger.Infof(" - security params: %v", config.SecurityParams)
+	logger.Infof(" - core limit: %v", config.CoreLimit)
+	logger.Infof(" - port: %v", config.Port)
+	logger.Infof(" - payload config: %v", config.PayloadConfig)
 }
 
 func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchmarkServer, error) {
@@ -102,7 +101,7 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {
-			grpclog.Fatalf("failed to generate credentials %v", err)
+			logger.Fatalf("failed to generate credentials %v", err)
 		}
 		opts = append(opts, grpc.Creds(creds))
 	}
@@ -114,7 +113,7 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma
 	}
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
 	if err != nil {
-		grpclog.Fatalf("Failed to listen: %v", err)
+		logger.Fatalf("Failed to listen: %v", err)
 	}
 	addr := lis.Addr().String()
 
@@ -147,11 +146,11 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma
 		}, opts...)
 	}
 
-	grpclog.Infof("benchmark server listening at %v", addr)
+	logger.Infof("benchmark server listening at %v", addr)
 	addrSplitted := strings.Split(addr, ":")
 	p, err := strconv.Atoi(addrSplitted[len(addrSplitted)-1])
 	if err != nil {
-		grpclog.Fatalf("failed to get port number from server address: %v", err)
+		logger.Fatalf("failed to get port number from server address: %v", err)
 	}
 
 	return &benchmarkServer{
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index fba451c4f27b..634f09e658c5 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -44,6 +44,8 @@ var (
 	serverPort    = flag.Int("server_port", 0, "port for benchmark server if not specified by server config message")
 	pprofPort     = flag.Int("pprof_port", -1, "Port for pprof debug server to listen on. Pprof server doesn't start if unset")
 	blockProfRate = flag.Int("block_prof_rate", 0, "fraction of goroutine blocking events to report in blocking profile")
+
+	logger = grpclog.Component("benchmark")
 )
 
 type byteBufCodec struct {
@@ -82,7 +84,7 @@ func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) er
 	var bs *benchmarkServer
 	defer func() {
 		// Close benchmark server when stream ends.
-		grpclog.Infof("closing benchmark server")
+		logger.Infof("closing benchmark server")
 		if bs != nil {
 			bs.closeFunc()
 		}
@@ -99,9 +101,9 @@ func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) er
 		var out *testpb.ServerStatus
 		switch argtype := in.Argtype.(type) {
 		case *testpb.ServerArgs_Setup:
-			grpclog.Infof("server setup received:")
+			logger.Infof("server setup received:")
 			if bs != nil {
-				grpclog.Infof("server setup received when server already exists, closing the existing server")
+				logger.Infof("server setup received when server already exists, closing the existing server")
 				bs.closeFunc()
 			}
 			bs, err = startBenchmarkServer(argtype.Setup, s.serverPort)
@@ -115,8 +117,8 @@ func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) er
 			}
 
 		case *testpb.ServerArgs_Mark:
-			grpclog.Infof("server mark received:")
-			grpclog.Infof(" - %v", argtype)
+			logger.Infof("server mark received:")
+			logger.Infof(" - %v", argtype)
 			if bs == nil {
 				return status.Error(codes.InvalidArgument, "server does not exist when mark received")
 			}
@@ -137,7 +139,7 @@ func (s *workerServer) RunClient(stream testpb.WorkerService_RunClientServer) er
 	var bc *benchmarkClient
 	defer func() {
 		// Shut down benchmark client when stream ends.
-		grpclog.Infof("shuting down benchmark client")
+		logger.Infof("shuting down benchmark client")
 		if bc != nil {
 			bc.shutdown()
 		}
@@ -154,9 +156,9 @@ func (s *workerServer) RunClient(stream testpb.WorkerService_RunClientServer) er
 		var out *testpb.ClientStatus
 		switch t := in.Argtype.(type) {
 		case *testpb.ClientArgs_Setup:
-			grpclog.Infof("client setup received:")
+			logger.Infof("client setup received:")
 			if bc != nil {
-				grpclog.Infof("client setup received when client already exists, shuting down the existing client")
+				logger.Infof("client setup received when client already exists, shuting down the existing client")
 				bc.shutdown()
 			}
 			bc, err = startBenchmarkClient(t.Setup)
@@ -168,8 +170,8 @@ func (s *workerServer) RunClient(stream testpb.WorkerService_RunClientServer) er
 			}
 
 		case *testpb.ClientArgs_Mark:
-			grpclog.Infof("client mark received:")
-			grpclog.Infof(" - %v", t)
+			logger.Infof("client mark received:")
+			logger.Infof(" - %v", t)
 			if bc == nil {
 				return status.Error(codes.InvalidArgument, "client does not exist when mark received")
 			}
@@ -185,12 +187,12 @@ func (s *workerServer) RunClient(stream testpb.WorkerService_RunClientServer) er
 }
 
 func (s *workerServer) CoreCount(ctx context.Context, in *testpb.CoreRequest) (*testpb.CoreResponse, error) {
-	grpclog.Infof("core count: %v", runtime.NumCPU())
+	logger.Infof("core count: %v", runtime.NumCPU())
 	return &testpb.CoreResponse{Cores: int32(runtime.NumCPU())}, nil
 }
 
 func (s *workerServer) QuitWorker(ctx context.Context, in *testpb.Void) (*testpb.Void, error) {
-	grpclog.Infof("quitting worker")
+	logger.Infof("quitting worker")
 	s.stop <- true
 	return &testpb.Void{}, nil
 }
@@ -201,9 +203,9 @@ func main() {
 	flag.Parse()
 	lis, err := net.Listen("tcp", ":"+strconv.Itoa(*driverPort))
 	if err != nil {
-		grpclog.Fatalf("failed to listen: %v", err)
+		logger.Fatalf("failed to listen: %v", err)
 	}
-	grpclog.Infof("worker listening at port %v", *driverPort)
+	logger.Infof("worker listening at port %v", *driverPort)
 
 	s := grpc.NewServer()
 	stop := make(chan bool)
@@ -224,8 +226,8 @@ func main() {
 
 	if *pprofPort >= 0 {
 		go func() {
-			grpclog.Infoln("Starting pprof server on port " + strconv.Itoa(*pprofPort))
-			grpclog.Infoln(http.ListenAndServe("localhost:"+strconv.Itoa(*pprofPort), nil))
+			logger.Infoln("Starting pprof server on port " + strconv.Itoa(*pprofPort))
+			logger.Infoln(http.ListenAndServe("localhost:"+strconv.Itoa(*pprofPort), nil))
 		}()
 	}
 
diff --git a/connectivity/connectivity.go b/connectivity/connectivity.go
index 34ec36fbf6d9..0a8d682ac6af 100644
--- a/connectivity/connectivity.go
+++ b/connectivity/connectivity.go
@@ -27,6 +27,8 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+var logger = grpclog.Component("core")
+
 // State indicates the state of connectivity.
 // It can be the state of a ClientConn or SubConn.
 type State int
@@ -44,7 +46,7 @@ func (s State) String() string {
 	case Shutdown:
 		return "SHUTDOWN"
 	default:
-		grpclog.Errorf("unknown connectivity state: %d", s)
+		logger.Errorf("unknown connectivity state: %d", s)
 		return "Invalid-State"
 	}
 }
diff --git a/credentials/google/google.go b/credentials/google/google.go
index 04b349abcfb6..7f3e240e475b 100644
--- a/credentials/google/google.go
+++ b/credentials/google/google.go
@@ -33,6 +33,8 @@ import (
 
 const tokenRequestTimeout = 30 * time.Second
 
+var logger = grpclog.Component("credentials")
+
 // NewDefaultCredentials returns a credentials bundle that is configured to work
 // with google services.
 //
@@ -44,14 +46,14 @@ func NewDefaultCredentials() credentials.Bundle {
 			defer cancel()
 			perRPCCreds, err := oauth.NewApplicationDefault(ctx)
 			if err != nil {
-				grpclog.Warningf("google default creds: failed to create application oauth: %v", err)
+				logger.Warningf("google default creds: failed to create application oauth: %v", err)
 			}
 			return perRPCCreds
 		},
 	}
 	bundle, err := c.NewWithMode(internal.CredsBundleModeFallback)
 	if err != nil {
-		grpclog.Warningf("google default creds: failed to create new creds: %v", err)
+		logger.Warningf("google default creds: failed to create new creds: %v", err)
 	}
 	return bundle
 }
@@ -69,7 +71,7 @@ func NewComputeEngineCredentials() credentials.Bundle {
 	}
 	bundle, err := c.NewWithMode(internal.CredsBundleModeFallback)
 	if err != nil {
-		grpclog.Warningf("compute engine creds: failed to create new creds: %v", err)
+		logger.Warningf("compute engine creds: failed to create new creds: %v", err)
 	}
 	return bundle
 }
diff --git a/credentials/sts/sts.go b/credentials/sts/sts.go
index f07c4c402e76..a97b20a9eb72 100644
--- a/credentials/sts/sts.go
+++ b/credentials/sts/sts.go
@@ -62,6 +62,7 @@ var (
 	makeHTTPDoer         = makeHTTPClient
 	readSubjectTokenFrom = ioutil.ReadFile
 	readActorTokenFrom   = ioutil.ReadFile
+	logger               = grpclog.Component("credentials")
 )
 
 // Options configures the parameters used for an STS based token exchange.
@@ -311,7 +312,7 @@ func sendRequest(client httpDoer, req *http.Request) ([]byte, error) {
 	if resp.StatusCode == http.StatusOK {
 		return body, nil
 	}
-	grpclog.Warningf("http status %d, body: %s", resp.StatusCode, string(body))
+	logger.Warningf("http status %d, body: %s", resp.StatusCode, string(body))
 	return nil, fmt.Errorf("http status %d, body: %s", resp.StatusCode, string(body))
 }
 
diff --git a/grpclog/component.go b/grpclog/component.go
index b513281a34ce..8358dd6e2abb 100644
--- a/grpclog/component.go
+++ b/grpclog/component.go
@@ -100,7 +100,7 @@ func (c *componentData) Fatalln(args ...interface{}) {
 }
 
 func (c *componentData) V(l int) bool {
-	return grpclog.Logger.V(l)
+	return V(l)
 }
 
 // Component creates a new component and returns it for logging. If a component
diff --git a/health/logging.go b/health/logging.go
new file mode 100644
index 000000000000..83c6acf55ef6
--- /dev/null
+++ b/health/logging.go
@@ -0,0 +1,23 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package health
+
+import "google.golang.org/grpc/grpclog"
+
+var logger = grpclog.Component("health_service")
diff --git a/health/server.go b/health/server.go
index ed2b3df70f4e..cce6312d77f9 100644
--- a/health/server.go
+++ b/health/server.go
@@ -25,7 +25,6 @@ import (
 	"sync"
 
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/grpclog"
 	healthgrpc "google.golang.org/grpc/health/grpc_health_v1"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/status"
@@ -114,7 +113,7 @@ func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthC
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.shutdown {
-		grpclog.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus)
+		logger.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus)
 		return
 	}
 
diff --git a/internal/binarylog/binarylog_end2end_test.go b/internal/binarylog/binarylog_end2end_test.go
index 9a7964f0569d..999bd90fb052 100644
--- a/internal/binarylog/binarylog_end2end_test.go
+++ b/internal/binarylog/binarylog_end2end_test.go
@@ -40,6 +40,8 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+var grpclogLogger = grpclog.Component("binarylog")
+
 type s struct {
 	grpctest.Tester
 }
@@ -539,7 +541,7 @@ func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64
 	}
 	data, err := proto.Marshal(msg)
 	if err != nil {
-		grpclog.Infof("binarylogging_testing: failed to marshal proto message: %v", err)
+		grpclogLogger.Infof("binarylogging_testing: failed to marshal proto message: %v", err)
 	}
 	return &pb.GrpcLogEntry{
 		Timestamp:            nil,
@@ -563,7 +565,7 @@ func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64
 	}
 	data, err := proto.Marshal(msg)
 	if err != nil {
-		grpclog.Infof("binarylogging_testing: failed to marshal proto message: %v", err)
+		grpclogLogger.Infof("binarylogging_testing: failed to marshal proto message: %v", err)
 	}
 	return &pb.GrpcLogEntry{
 		Timestamp:            nil,
@@ -612,7 +614,7 @@ func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64
 	}
 	st, ok := status.FromError(stErr)
 	if !ok {
-		grpclog.Info("binarylogging: error in trailer is not a status error")
+		grpclogLogger.Info("binarylogging: error in trailer is not a status error")
 	}
 	stProto := st.Proto()
 	var (
@@ -622,7 +624,7 @@ func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64
 	if stProto != nil && len(stProto.Details) != 0 {
 		detailsBytes, err = proto.Marshal(stProto)
 		if err != nil {
-			grpclog.Infof("binarylogging: failed to marshal status proto: %v", err)
+			grpclogLogger.Infof("binarylogging: failed to marshal status proto: %v", err)
 		}
 	}
 	return &pb.GrpcLogEntry{
diff --git a/internal/credentials/spiffe.go b/internal/credentials/spiffe.go
index c86813061ef5..406023035288 100644
--- a/internal/credentials/spiffe.go
+++ b/internal/credentials/spiffe.go
@@ -28,6 +28,8 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+var logger = grpclog.Component("credentials")
+
 // SPIFFEIDFromState parses the SPIFFE ID from State. If the SPIFFE ID format
 // is invalid, return nil with warning.
 func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
@@ -41,20 +43,20 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
 		}
 		// From this point, we assume the uri is intended for a SPIFFE ID.
 		if len(uri.String()) > 2048 {
-			grpclog.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes")
+			logger.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes")
 			return nil
 		}
 		if len(uri.Host) == 0 || len(uri.RawPath) == 0 || len(uri.Path) == 0 {
-			grpclog.Warning("invalid SPIFFE ID: domain or workload ID is empty")
+			logger.Warning("invalid SPIFFE ID: domain or workload ID is empty")
 			return nil
 		}
 		if len(uri.Host) > 255 {
-			grpclog.Warning("invalid SPIFFE ID: domain length larger than 255 characters")
+			logger.Warning("invalid SPIFFE ID: domain length larger than 255 characters")
 			return nil
 		}
 		// A valid SPIFFE certificate can only have exactly one URI SAN field.
 		if len(state.PeerCertificates[0].URIs) > 1 {
-			grpclog.Warning("invalid SPIFFE ID: multiple URI SANs")
+			logger.Warning("invalid SPIFFE ID: multiple URI SANs")
 			return nil
 		}
 		spiffeID = uri
diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go
index 9b26414d40e2..2b9b403a7a72 100644
--- a/internal/serviceconfig/serviceconfig.go
+++ b/internal/serviceconfig/serviceconfig.go
@@ -28,6 +28,8 @@ import (
 	externalserviceconfig "google.golang.org/grpc/serviceconfig"
 )
 
+var logger = grpclog.Component("core")
+
 // BalancerConfig is the balancer config part that service config's
 // loadBalancingConfig fields can be unmarshalled to. It's a json unmarshaller.
 //
@@ -69,7 +71,7 @@ func (bc *BalancerConfig) UnmarshalJSON(b []byte) error {
 		parser, ok := builder.(balancer.ConfigParser)
 		if !ok {
 			if string(jsonCfg) != "{}" {
-				grpclog.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg))
+				logger.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg))
 			}
 			// Stop at this, though the builder doesn't support parsing config.
 			return nil
diff --git a/interop/alts/client/client.go b/interop/alts/client/client.go
index f3b1028bfbf6..9fc0e3c15650 100644
--- a/interop/alts/client/client.go
+++ b/interop/alts/client/client.go
@@ -24,7 +24,7 @@ import (
 	"flag"
 	"time"
 
-	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/alts"
 	"google.golang.org/grpc/grpclog"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
@@ -33,6 +33,8 @@ import (
 var (
 	hsAddr     = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address")
 	serverAddr = flag.String("server_address", ":8080", "The port on which the server is listening")
+
+	logger = grpclog.Component("interop")
 )
 
 func main() {
@@ -46,7 +48,7 @@ func main() {
 	// Block until the server is ready.
 	conn, err := grpc.Dial(*serverAddr, grpc.WithTransportCredentials(altsTC), grpc.WithBlock())
 	if err != nil {
-		grpclog.Fatalf("gRPC Client: failed to dial the server at %v: %v", *serverAddr, err)
+		logger.Fatalf("gRPC Client: failed to dial the server at %v: %v", *serverAddr, err)
 	}
 	defer conn.Close()
 	grpcClient := testpb.NewTestServiceClient(conn)
@@ -55,9 +57,9 @@ func main() {
 	ctx := context.Background()
 	request := &testpb.Empty{}
 	if _, err := grpcClient.EmptyCall(ctx, request); err != nil {
-		grpclog.Fatalf("grpc Client: EmptyCall(_, %v) failed: %v", request, err)
+		logger.Fatalf("grpc Client: EmptyCall(_, %v) failed: %v", request, err)
 	}
-	grpclog.Info("grpc Client: empty call succeeded")
+	logger.Info("grpc Client: empty call succeeded")
 
 	// This sleep prevents the connection from being abruptly disconnected
 	// when running this binary (along with grpc_server) on GCP dev cluster.
diff --git a/interop/alts/server/server.go b/interop/alts/server/server.go
index 488608b22ec0..0d0f375a0d1a 100644
--- a/interop/alts/server/server.go
+++ b/interop/alts/server/server.go
@@ -25,7 +25,7 @@ import (
 	"net"
 	"strings"
 
-	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/alts"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/interop"
@@ -40,6 +40,8 @@ const (
 var (
 	hsAddr     = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address")
 	serverAddr = flag.String("server_address", ":8080", "The address on which the server is listening. Only two types of addresses are supported, 'host:port' and 'unix:/path'.")
+
+	logger = grpclog.Component("interop")
 )
 
 func main() {
@@ -54,7 +56,7 @@ func main() {
 	}
 	lis, err := net.Listen(network, address)
 	if err != nil {
-		grpclog.Fatalf("gRPC Server: failed to start the server at %v: %v", address, err)
+		logger.Fatalf("gRPC Server: failed to start the server at %v: %v", address, err)
 	}
 	opts := alts.DefaultServerOptions()
 	if *hsAddr != "" {
@@ -74,12 +76,12 @@ func authz(ctx context.Context, info *tap.Info) (context.Context, error) {
 		return nil, err
 	}
 	// Access all alts.AuthInfo data:
-	grpclog.Infof("authInfo.ApplicationProtocol() = %v", authInfo.ApplicationProtocol())
-	grpclog.Infof("authInfo.RecordProtocol() = %v", authInfo.RecordProtocol())
-	grpclog.Infof("authInfo.SecurityLevel() = %v", authInfo.SecurityLevel())
-	grpclog.Infof("authInfo.PeerServiceAccount() = %v", authInfo.PeerServiceAccount())
-	grpclog.Infof("authInfo.LocalServiceAccount() = %v", authInfo.LocalServiceAccount())
-	grpclog.Infof("authInfo.PeerRPCVersions() = %v", authInfo.PeerRPCVersions())
-	grpclog.Infof("info.FullMethodName = %v", info.FullMethodName)
+	logger.Infof("authInfo.ApplicationProtocol() = %v", authInfo.ApplicationProtocol())
+	logger.Infof("authInfo.RecordProtocol() = %v", authInfo.RecordProtocol())
+	logger.Infof("authInfo.SecurityLevel() = %v", authInfo.SecurityLevel())
+	logger.Infof("authInfo.PeerServiceAccount() = %v", authInfo.PeerServiceAccount())
+	logger.Infof("authInfo.LocalServiceAccount() = %v", authInfo.LocalServiceAccount())
+	logger.Infof("authInfo.PeerRPCVersions() = %v", authInfo.PeerRPCVersions())
+	logger.Infof("info.FullMethodName = %v", info.FullMethodName)
 	return ctx, nil
 }
diff --git a/interop/client/client.go b/interop/client/client.go
index eb4477f66d13..0c05c19a5e9f 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -79,6 +79,8 @@ var (
         unimplemented_method: client attempts to call unimplemented method;
         unimplemented_service: client attempts to call unimplemented service;
         pick_first_unary: all requests are sent to one server despite multiple servers are resolved.`)
+
+	logger = grpclog.Component("interop")
 )
 
 type credsMode uint8
@@ -102,12 +104,12 @@ func main() {
 		case computeEngineCredsName:
 			useCEC = true
 		default:
-			grpclog.Fatalf("If set, custom_credentials_type can only be set to one of %v or %v",
+			logger.Fatalf("If set, custom_credentials_type can only be set to one of %v or %v",
 				googleDefaultCredsName, computeEngineCredsName)
 		}
 	}
 	if (*useTLS && *useALTS) || (*useTLS && useGDC) || (*useALTS && useGDC) || (*useTLS && useCEC) || (*useALTS && useCEC) {
-		grpclog.Fatalf("only one of TLS, ALTS, google default creds, or compute engine creds can be used")
+		logger.Fatalf("only one of TLS, ALTS, google default creds, or compute engine creds can be used")
 	}
 
 	var credsChosen credsMode
@@ -139,7 +141,7 @@ func main() {
 			}
 			creds, err = credentials.NewClientTLSFromFile(*caFile, sn)
 			if err != nil {
-				grpclog.Fatalf("Failed to create TLS credentials %v", err)
+				logger.Fatalf("Failed to create TLS credentials %v", err)
 			}
 		} else {
 			creds = credentials.NewClientTLSFromCert(nil, sn)
@@ -159,7 +161,7 @@ func main() {
 	case credsNone:
 		opts = append(opts, grpc.WithInsecure())
 	default:
-		grpclog.Fatal("Invalid creds")
+		logger.Fatal("Invalid creds")
 	}
 	if credsChosen == credsTLS {
 		if *testCase == "compute_engine_creds" {
@@ -167,13 +169,13 @@ func main() {
 		} else if *testCase == "service_account_creds" {
 			jwtCreds, err := oauth.NewServiceAccountFromFile(*serviceAccountKeyFile, *oauthScope)
 			if err != nil {
-				grpclog.Fatalf("Failed to create JWT credentials: %v", err)
+				logger.Fatalf("Failed to create JWT credentials: %v", err)
 			}
 			opts = append(opts, grpc.WithPerRPCCredentials(jwtCreds))
 		} else if *testCase == "jwt_token_creds" {
 			jwtCreds, err := oauth.NewJWTAccessFromFile(*serviceAccountKeyFile)
 			if err != nil {
-				grpclog.Fatalf("Failed to create JWT credentials: %v", err)
+				logger.Fatalf("Failed to create JWT credentials: %v", err)
 			}
 			opts = append(opts, grpc.WithPerRPCCredentials(jwtCreds))
 		} else if *testCase == "oauth2_auth_token" {
@@ -183,99 +185,99 @@ func main() {
 	opts = append(opts, grpc.WithBlock())
 	conn, err := grpc.Dial(serverAddr, opts...)
 	if err != nil {
-		grpclog.Fatalf("Fail to dial: %v", err)
+		logger.Fatalf("Fail to dial: %v", err)
 	}
 	defer conn.Close()
 	tc := testpb.NewTestServiceClient(conn)
 	switch *testCase {
 	case "empty_unary":
 		interop.DoEmptyUnaryCall(tc)
-		grpclog.Infoln("EmptyUnaryCall done")
+		logger.Infoln("EmptyUnaryCall done")
 	case "large_unary":
 		interop.DoLargeUnaryCall(tc)
-		grpclog.Infoln("LargeUnaryCall done")
+		logger.Infoln("LargeUnaryCall done")
 	case "client_streaming":
 		interop.DoClientStreaming(tc)
-		grpclog.Infoln("ClientStreaming done")
+		logger.Infoln("ClientStreaming done")
 	case "server_streaming":
 		interop.DoServerStreaming(tc)
-		grpclog.Infoln("ServerStreaming done")
+		logger.Infoln("ServerStreaming done")
 	case "ping_pong":
 		interop.DoPingPong(tc)
-		grpclog.Infoln("Pingpong done")
+		logger.Infoln("Pingpong done")
 	case "empty_stream":
 		interop.DoEmptyStream(tc)
-		grpclog.Infoln("Emptystream done")
+		logger.Infoln("Emptystream done")
 	case "timeout_on_sleeping_server":
 		interop.DoTimeoutOnSleepingServer(tc)
-		grpclog.Infoln("TimeoutOnSleepingServer done")
+		logger.Infoln("TimeoutOnSleepingServer done")
 	case "compute_engine_creds":
 		if credsChosen != credsTLS {
-			grpclog.Fatalf("TLS credentials need to be set for compute_engine_creds test case.")
+			logger.Fatalf("TLS credentials need to be set for compute_engine_creds test case.")
 		}
 		interop.DoComputeEngineCreds(tc, *defaultServiceAccount, *oauthScope)
-		grpclog.Infoln("ComputeEngineCreds done")
+		logger.Infoln("ComputeEngineCreds done")
 	case "service_account_creds":
 		if credsChosen != credsTLS {
-			grpclog.Fatalf("TLS credentials need to be set for service_account_creds test case.")
+			logger.Fatalf("TLS credentials need to be set for service_account_creds test case.")
 		}
 		interop.DoServiceAccountCreds(tc, *serviceAccountKeyFile, *oauthScope)
-		grpclog.Infoln("ServiceAccountCreds done")
+		logger.Infoln("ServiceAccountCreds done")
 	case "jwt_token_creds":
 		if credsChosen != credsTLS {
-			grpclog.Fatalf("TLS credentials need to be set for jwt_token_creds test case.")
+			logger.Fatalf("TLS credentials need to be set for jwt_token_creds test case.")
 		}
 		interop.DoJWTTokenCreds(tc, *serviceAccountKeyFile)
-		grpclog.Infoln("JWTtokenCreds done")
+		logger.Infoln("JWTtokenCreds done")
 	case "per_rpc_creds":
 		if credsChosen != credsTLS {
-			grpclog.Fatalf("TLS credentials need to be set for per_rpc_creds test case.")
+			logger.Fatalf("TLS credentials need to be set for per_rpc_creds test case.")
 		}
 		interop.DoPerRPCCreds(tc, *serviceAccountKeyFile, *oauthScope)
-		grpclog.Infoln("PerRPCCreds done")
+		logger.Infoln("PerRPCCreds done")
 	case "oauth2_auth_token":
 		if credsChosen != credsTLS {
-			grpclog.Fatalf("TLS credentials need to be set for oauth2_auth_token test case.")
+			logger.Fatalf("TLS credentials need to be set for oauth2_auth_token test case.")
 		}
 		interop.DoOauth2TokenCreds(tc, *serviceAccountKeyFile, *oauthScope)
-		grpclog.Infoln("Oauth2TokenCreds done")
+		logger.Infoln("Oauth2TokenCreds done")
 	case "google_default_credentials":
 		if credsChosen != credsGoogleDefaultCreds {
-			grpclog.Fatalf("GoogleDefaultCredentials need to be set for google_default_credentials test case.")
+			logger.Fatalf("GoogleDefaultCredentials need to be set for google_default_credentials test case.")
 		}
 		interop.DoGoogleDefaultCredentials(tc, *defaultServiceAccount)
-		grpclog.Infoln("GoogleDefaultCredentials done")
+		logger.Infoln("GoogleDefaultCredentials done")
 	case "compute_engine_channel_credentials":
 		if credsChosen != credsComputeEngineCreds {
-			grpclog.Fatalf("ComputeEngineCreds need to be set for compute_engine_channel_credentials test case.")
+			logger.Fatalf("ComputeEngineCreds need to be set for compute_engine_channel_credentials test case.")
 		}
 		interop.DoComputeEngineChannelCredentials(tc, *defaultServiceAccount)
-		grpclog.Infoln("ComputeEngineChannelCredentials done")
+		logger.Infoln("ComputeEngineChannelCredentials done")
 	case "cancel_after_begin":
 		interop.DoCancelAfterBegin(tc)
-		grpclog.Infoln("CancelAfterBegin done")
+		logger.Infoln("CancelAfterBegin done")
 	case "cancel_after_first_response":
 		interop.DoCancelAfterFirstResponse(tc)
-		grpclog.Infoln("CancelAfterFirstResponse done")
+		logger.Infoln("CancelAfterFirstResponse done")
 	case "status_code_and_message":
 		interop.DoStatusCodeAndMessage(tc)
-		grpclog.Infoln("StatusCodeAndMessage done")
+		logger.Infoln("StatusCodeAndMessage done")
 	case "special_status_message":
 		interop.DoSpecialStatusMessage(tc)
-		grpclog.Infoln("SpecialStatusMessage done")
+		logger.Infoln("SpecialStatusMessage done")
 	case "custom_metadata":
 		interop.DoCustomMetadata(tc)
-		grpclog.Infoln("CustomMetadata done")
+		logger.Infoln("CustomMetadata done")
 	case "unimplemented_method":
 		interop.DoUnimplementedMethod(conn)
-		grpclog.Infoln("UnimplementedMethod done")
+		logger.Infoln("UnimplementedMethod done")
 	case "unimplemented_service":
 		interop.DoUnimplementedService(testpb.NewUnimplementedServiceClient(conn))
-		grpclog.Infoln("UnimplementedService done")
+		logger.Infoln("UnimplementedService done")
 	case "pick_first_unary":
 		interop.DoPickFirstUnary(tc)
-		grpclog.Infoln("PickFirstUnary done")
+		logger.Infoln("PickFirstUnary done")
 	default:
-		grpclog.Fatal("Unsupported test case: ", *testCase)
+		logger.Fatal("Unsupported test case: ", *testCase)
 	}
 }
diff --git a/interop/fake_grpclb/fake_grpclb.go b/interop/fake_grpclb/fake_grpclb.go
index f7ccd7e92d4d..6746462b107a 100644
--- a/interop/fake_grpclb/fake_grpclb.go
+++ b/interop/fake_grpclb/fake_grpclb.go
@@ -45,6 +45,8 @@ var (
 	useTLS       = flag.Bool("use_tls", false, "Listen on TLS credentials, using a test certificate.")
 	shortStream  = flag.Bool("short_stream", false, "End the balancer stream immediately after sending the first server list.")
 	serviceName  = flag.String("service_name", "UNSET", "Name of the service being load balanced for.")
+
+	logger = grpclog.Component("interop")
 )
 
 type loadBalancerServer struct {
@@ -53,17 +55,17 @@ type loadBalancerServer struct {
 }
 
 func (l *loadBalancerServer) BalanceLoad(stream lbpb.LoadBalancer_BalanceLoadServer) error {
-	grpclog.Info("Begin handling new BalancerLoad request.")
+	logger.Info("Begin handling new BalancerLoad request.")
 	var lbReq *lbpb.LoadBalanceRequest
 	var err error
 	if lbReq, err = stream.Recv(); err != nil {
-		grpclog.Errorf("Error receiving LoadBalanceRequest: %v", err)
+		logger.Errorf("Error receiving LoadBalanceRequest: %v", err)
 		return err
 	}
-	grpclog.Info("LoadBalancerRequest received.")
+	logger.Info("LoadBalancerRequest received.")
 	initialReq := lbReq.GetInitialRequest()
 	if initialReq == nil {
-		grpclog.Info("Expected first request to be an InitialRequest. Got: %v", lbReq)
+		logger.Info("Expected first request to be an InitialRequest. Got: %v", lbReq)
 		return status.Error(codes.Unknown, "First request not an InitialRequest")
 	}
 	// gRPC clients targeting foo.bar.com:443 can sometimes include the ":443" suffix in
@@ -74,12 +76,12 @@ func (l *loadBalancerServer) BalanceLoad(stream lbpb.LoadBalancer_BalanceLoadSer
 		cleanedName = initialReq.Name
 	} else {
 		if requestedNamePortNumber != "443" {
-			grpclog.Info("Bad requested service name port number: %v.", requestedNamePortNumber)
+			logger.Info("Bad requested service name port number: %v.", requestedNamePortNumber)
 			return status.Error(codes.Unknown, "Bad requested service name port number")
 		}
 	}
 	if cleanedName != *serviceName {
-		grpclog.Info("Expected requested service name: %v. Got: %v", *serviceName, initialReq.Name)
+		logger.Info("Expected requested service name: %v. Got: %v", *serviceName, initialReq.Name)
 		return status.Error(codes.NotFound, "Bad requested service name")
 	}
 	if err := stream.Send(&lbpb.LoadBalanceResponse{
@@ -87,21 +89,21 @@ func (l *loadBalancerServer) BalanceLoad(stream lbpb.LoadBalancer_BalanceLoadSer
 			InitialResponse: &lbpb.InitialLoadBalanceResponse{},
 		},
 	}); err != nil {
-		grpclog.Errorf("Error sending initial LB response: %v", err)
+		logger.Errorf("Error sending initial LB response: %v", err)
 		return status.Error(codes.Unknown, "Error sending initial response")
 	}
-	grpclog.Info("Send LoadBalanceResponse: %v", l.serverListResponse)
+	logger.Info("Send LoadBalanceResponse: %v", l.serverListResponse)
 	if err := stream.Send(l.serverListResponse); err != nil {
-		grpclog.Errorf("Error sending LB response: %v", err)
+		logger.Errorf("Error sending LB response: %v", err)
 		return status.Error(codes.Unknown, "Error sending response")
 	}
 	if *shortStream {
 		return nil
 	}
 	for {
-		grpclog.Info("Send LoadBalanceResponse: %v", l.serverListResponse)
+		logger.Info("Send LoadBalanceResponse: %v", l.serverListResponse)
 		if err := stream.Send(l.serverListResponse); err != nil {
-			grpclog.Errorf("Error sending LB response: %v", err)
+			logger.Errorf("Error sending LB response: %v", err)
 			return status.Error(codes.Unknown, "Error sending response")
 		}
 		time.Sleep(10 * time.Second)
@@ -116,7 +118,7 @@ func main() {
 		keyFile := testdata.Path("x509/server1_key.pem")
 		creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
 		if err != nil {
-			grpclog.Fatalf("Failed to generate credentials %v", err)
+			logger.Fatalf("Failed to generate credentials %v", err)
 		}
 		opts = append(opts, grpc.Creds(creds))
 	} else if *useALTS {
@@ -133,17 +135,17 @@ func main() {
 		for i := range rawBackendAddrs {
 			rawIP, rawPort, err := net.SplitHostPort(rawBackendAddrs[i])
 			if err != nil {
-				grpclog.Fatalf("Failed to parse --backend_addrs[%d]=%v, error: %v", i, rawBackendAddrs[i], err)
+				logger.Fatalf("Failed to parse --backend_addrs[%d]=%v, error: %v", i, rawBackendAddrs[i], err)
 			}
 			ip := net.ParseIP(rawIP)
 			if ip == nil {
-				grpclog.Fatalf("Failed to parse ip: %v", rawIP)
+				logger.Fatalf("Failed to parse ip: %v", rawIP)
 			}
 			numericPort, err := strconv.Atoi(rawPort)
 			if err != nil {
-				grpclog.Fatalf("Failed to convert port %v to int", rawPort)
+				logger.Fatalf("Failed to convert port %v to int", rawPort)
 			}
-			grpclog.Infof("Adding backend ip: %v, port: %d", ip.String(), numericPort)
+			logger.Infof("Adding backend ip: %v, port: %d", ip.String(), numericPort)
 			serverList[i] = &lbpb.Server{
 				IpAddress: ip,
 				Port:      int32(numericPort),
@@ -158,10 +160,10 @@ func main() {
 		},
 	}
 	server := grpc.NewServer(opts...)
-	grpclog.Infof("Begin listening on %d.", *port)
+	logger.Infof("Begin listening on %d.", *port)
 	lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port))
 	if err != nil {
-		grpclog.Fatalf("Failed to listen on port %v: %v", *port, err)
+		logger.Fatalf("Failed to listen on port %v: %v", *port, err)
 	}
 	lbpb.RegisterLoadBalancerServer(server, &loadBalancerServer{
 		serverListResponse: serverListResponse,
diff --git a/interop/http2/negative_http2_client.go b/interop/http2/negative_http2_client.go
index e5d18c3ef0ae..8ead7f49c6b4 100644
--- a/interop/http2/negative_http2_client.go
+++ b/interop/http2/negative_http2_client.go
@@ -52,6 +52,8 @@ var (
         max_streams : server will ensure that the max_concurrent_streams limit is upheld;`)
 	largeReqSize  = 271828
 	largeRespSize = 314159
+
+	logger = grpclog.Component("interop")
 )
 
 func largeSimpleRequest() *testpb.SimpleRequest {
@@ -76,10 +78,10 @@ func rstAfterHeader(tc testpb.TestServiceClient) {
 	req := largeSimpleRequest()
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if reply != nil {
-		grpclog.Fatalf("Client received reply despite server sending rst stream after header")
+		logger.Fatalf("Client received reply despite server sending rst stream after header")
 	}
 	if status.Code(err) != codes.Internal {
-		grpclog.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal)
+		logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal)
 	}
 }
 
@@ -87,10 +89,10 @@ func rstDuringData(tc testpb.TestServiceClient) {
 	req := largeSimpleRequest()
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if reply != nil {
-		grpclog.Fatalf("Client received reply despite server sending rst stream during data")
+		logger.Fatalf("Client received reply despite server sending rst stream during data")
 	}
 	if status.Code(err) != codes.Unknown {
-		grpclog.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unknown)
+		logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unknown)
 	}
 }
 
@@ -98,10 +100,10 @@ func rstAfterData(tc testpb.TestServiceClient) {
 	req := largeSimpleRequest()
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if reply != nil {
-		grpclog.Fatalf("Client received reply despite server sending rst stream after data")
+		logger.Fatalf("Client received reply despite server sending rst stream after data")
 	}
 	if status.Code(err) != codes.Internal {
-		grpclog.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal)
+		logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal)
 	}
 }
 
@@ -130,30 +132,30 @@ func main() {
 	opts = append(opts, grpc.WithInsecure())
 	conn, err := grpc.Dial(serverAddr, opts...)
 	if err != nil {
-		grpclog.Fatalf("Fail to dial: %v", err)
+		logger.Fatalf("Fail to dial: %v", err)
 	}
 	defer conn.Close()
 	tc := testpb.NewTestServiceClient(conn)
 	switch *testCase {
 	case "goaway":
 		goaway(tc)
-		grpclog.Infoln("goaway done")
+		logger.Infoln("goaway done")
 	case "rst_after_header":
 		rstAfterHeader(tc)
-		grpclog.Infoln("rst_after_header done")
+		logger.Infoln("rst_after_header done")
 	case "rst_during_data":
 		rstDuringData(tc)
-		grpclog.Infoln("rst_during_data done")
+		logger.Infoln("rst_during_data done")
 	case "rst_after_data":
 		rstAfterData(tc)
-		grpclog.Infoln("rst_after_data done")
+		logger.Infoln("rst_after_data done")
 	case "ping":
 		ping(tc)
-		grpclog.Infoln("ping done")
+		logger.Infoln("ping done")
 	case "max_streams":
 		maxStreams(tc)
-		grpclog.Infoln("max_streams done")
+		logger.Infoln("max_streams done")
 	default:
-		grpclog.Fatal("Unsupported test case: ", *testCase)
+		logger.Fatal("Unsupported test case: ", *testCase)
 	}
 }
diff --git a/interop/server/server.go b/interop/server/server.go
index c029fd9858de..52cfb2f4c458 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -40,17 +40,19 @@ var (
 	certFile   = flag.String("tls_cert_file", "", "The TLS cert file")
 	keyFile    = flag.String("tls_key_file", "", "The TLS key file")
 	port       = flag.Int("port", 10000, "The server port")
+
+	logger = grpclog.Component("interop")
 )
 
 func main() {
 	flag.Parse()
 	if *useTLS && *useALTS {
-		grpclog.Fatalf("use_tls and use_alts cannot be both set to true")
+		logger.Fatalf("use_tls and use_alts cannot be both set to true")
 	}
 	p := strconv.Itoa(*port)
 	lis, err := net.Listen("tcp", ":"+p)
 	if err != nil {
-		grpclog.Fatalf("failed to listen: %v", err)
+		logger.Fatalf("failed to listen: %v", err)
 	}
 	var opts []grpc.ServerOption
 	if *useTLS {
@@ -62,7 +64,7 @@ func main() {
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {
-			grpclog.Fatalf("Failed to generate credentials %v", err)
+			logger.Fatalf("Failed to generate credentials %v", err)
 		}
 		opts = append(opts, grpc.Creds(creds))
 	} else if *useALTS {
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 74e839031881..7e3aaa95f0ca 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -45,20 +45,22 @@ var (
 	largeRespSize       = 314159
 	initialMetadataKey  = "x-grpc-test-echo-initial"
 	trailingMetadataKey = "x-grpc-test-echo-trailing-bin"
+
+	logger = grpclog.Component("interop")
 )
 
 // ClientNewPayload returns a payload of the given type and size.
 func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 	if size < 0 {
-		grpclog.Fatalf("Requested a response with invalid length %d", size)
+		logger.Fatalf("Requested a response with invalid length %d", size)
 	}
 	body := make([]byte, size)
 	switch t {
 	case testpb.PayloadType_COMPRESSABLE:
 	case testpb.PayloadType_UNCOMPRESSABLE:
-		grpclog.Fatalf("PayloadType UNCOMPRESSABLE is not supported")
+		logger.Fatalf("PayloadType UNCOMPRESSABLE is not supported")
 	default:
-		grpclog.Fatalf("Unsupported payload type: %d", t)
+		logger.Fatalf("Unsupported payload type: %d", t)
 	}
 	return &testpb.Payload{
 		Type: t,
@@ -70,10 +72,10 @@ func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 func DoEmptyUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	reply, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, args...)
 	if err != nil {
-		grpclog.Fatal("/TestService/EmptyCall RPC failed: ", err)
+		logger.Fatal("/TestService/EmptyCall RPC failed: ", err)
 	}
 	if !proto.Equal(&testpb.Empty{}, reply) {
-		grpclog.Fatalf("/TestService/EmptyCall receives %v, want %v", reply, testpb.Empty{})
+		logger.Fatalf("/TestService/EmptyCall receives %v, want %v", reply, testpb.Empty{})
 	}
 }
 
@@ -87,12 +89,12 @@ func DoLargeUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	}
 	reply, err := tc.UnaryCall(context.Background(), req, args...)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	t := reply.GetPayload().GetType()
 	s := len(reply.GetPayload().GetBody())
 	if t != testpb.PayloadType_COMPRESSABLE || s != largeRespSize {
-		grpclog.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, largeRespSize)
+		logger.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, largeRespSize)
 	}
 }
 
@@ -100,7 +102,7 @@ func DoLargeUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 func DoClientStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	stream, err := tc.StreamingInputCall(context.Background(), args...)
 	if err != nil {
-		grpclog.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
 	}
 	var sum int
 	for _, s := range reqSizes {
@@ -109,16 +111,16 @@ func DoClientStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 			Payload: pl,
 		}
 		if err := stream.Send(req); err != nil {
-			grpclog.Fatalf("%v has error %v while sending %v", stream, err, req)
+			logger.Fatalf("%v has error %v while sending %v", stream, err, req)
 		}
 		sum += s
 	}
 	reply, err := stream.CloseAndRecv()
 	if err != nil {
-		grpclog.Fatalf("%v.CloseAndRecv() got error %v, want %v", stream, err, nil)
+		logger.Fatalf("%v.CloseAndRecv() got error %v, want %v", stream, err, nil)
 	}
 	if reply.GetAggregatedPayloadSize() != int32(sum) {
-		grpclog.Fatalf("%v.CloseAndRecv().GetAggregatePayloadSize() = %v; want %v", stream, reply.GetAggregatedPayloadSize(), sum)
+		logger.Fatalf("%v.CloseAndRecv().GetAggregatePayloadSize() = %v; want %v", stream, reply.GetAggregatedPayloadSize(), sum)
 	}
 }
 
@@ -136,7 +138,7 @@ func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	}
 	stream, err := tc.StreamingOutputCall(context.Background(), req, args...)
 	if err != nil {
-		grpclog.Fatalf("%v.StreamingOutputCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.StreamingOutputCall(_) = _, %v", tc, err)
 	}
 	var rpcStatus error
 	var respCnt int
@@ -149,20 +151,20 @@ func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 		}
 		t := reply.GetPayload().GetType()
 		if t != testpb.PayloadType_COMPRESSABLE {
-			grpclog.Fatalf("Got the reply of type %d, want %d", t, testpb.PayloadType_COMPRESSABLE)
+			logger.Fatalf("Got the reply of type %d, want %d", t, testpb.PayloadType_COMPRESSABLE)
 		}
 		size := len(reply.GetPayload().GetBody())
 		if size != respSizes[index] {
-			grpclog.Fatalf("Got reply body of length %d, want %d", size, respSizes[index])
+			logger.Fatalf("Got reply body of length %d, want %d", size, respSizes[index])
 		}
 		index++
 		respCnt++
 	}
 	if rpcStatus != io.EOF {
-		grpclog.Fatalf("Failed to finish the server streaming rpc: %v", rpcStatus)
+		logger.Fatalf("Failed to finish the server streaming rpc: %v", rpcStatus)
 	}
 	if respCnt != len(respSizes) {
-		grpclog.Fatalf("Got %d reply, want %d", len(respSizes), respCnt)
+		logger.Fatalf("Got %d reply, want %d", len(respSizes), respCnt)
 	}
 }
 
@@ -170,7 +172,7 @@ func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	stream, err := tc.FullDuplexCall(context.Background(), args...)
 	if err != nil {
-		grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
 	}
 	var index int
 	for index < len(reqSizes) {
@@ -186,27 +188,27 @@ func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 			Payload:            pl,
 		}
 		if err := stream.Send(req); err != nil {
-			grpclog.Fatalf("%v has error %v while sending %v", stream, err, req)
+			logger.Fatalf("%v has error %v while sending %v", stream, err, req)
 		}
 		reply, err := stream.Recv()
 		if err != nil {
-			grpclog.Fatalf("%v.Recv() = %v", stream, err)
+			logger.Fatalf("%v.Recv() = %v", stream, err)
 		}
 		t := reply.GetPayload().GetType()
 		if t != testpb.PayloadType_COMPRESSABLE {
-			grpclog.Fatalf("Got the reply of type %d, want %d", t, testpb.PayloadType_COMPRESSABLE)
+			logger.Fatalf("Got the reply of type %d, want %d", t, testpb.PayloadType_COMPRESSABLE)
 		}
 		size := len(reply.GetPayload().GetBody())
 		if size != respSizes[index] {
-			grpclog.Fatalf("Got reply body of length %d, want %d", size, respSizes[index])
+			logger.Fatalf("Got reply body of length %d, want %d", size, respSizes[index])
 		}
 		index++
 	}
 	if err := stream.CloseSend(); err != nil {
-		grpclog.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil)
+		logger.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil)
 	}
 	if _, err := stream.Recv(); err != io.EOF {
-		grpclog.Fatalf("%v failed to complele the ping pong test: %v", stream, err)
+		logger.Fatalf("%v failed to complele the ping pong test: %v", stream, err)
 	}
 }
 
@@ -214,13 +216,13 @@ func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 func DoEmptyStream(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	stream, err := tc.FullDuplexCall(context.Background(), args...)
 	if err != nil {
-		grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
 	}
 	if err := stream.CloseSend(); err != nil {
-		grpclog.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil)
+		logger.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil)
 	}
 	if _, err := stream.Recv(); err != io.EOF {
-		grpclog.Fatalf("%v failed to complete the empty stream test: %v", stream, err)
+		logger.Fatalf("%v failed to complete the empty stream test: %v", stream, err)
 	}
 }
 
@@ -233,7 +235,7 @@ func DoTimeoutOnSleepingServer(tc testpb.TestServiceClient, args ...grpc.CallOpt
 		if status.Code(err) == codes.DeadlineExceeded {
 			return
 		}
-		grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
 	}
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 27182)
 	req := &testpb.StreamingOutputCallRequest{
@@ -241,10 +243,10 @@ func DoTimeoutOnSleepingServer(tc testpb.TestServiceClient, args ...grpc.CallOpt
 		Payload:      pl,
 	}
 	if err := stream.Send(req); err != nil && err != io.EOF {
-		grpclog.Fatalf("%v.Send(_) = %v", stream, err)
+		logger.Fatalf("%v.Send(_) = %v", stream, err)
 	}
 	if _, err := stream.Recv(); status.Code(err) != codes.DeadlineExceeded {
-		grpclog.Fatalf("%v.Recv() = _, %v, want error code %d", stream, err, codes.DeadlineExceeded)
+		logger.Fatalf("%v.Recv() = _, %v, want error code %d", stream, err, codes.DeadlineExceeded)
 	}
 }
 
@@ -260,22 +262,22 @@ func DoComputeEngineCreds(tc testpb.TestServiceClient, serviceAccount, oauthScop
 	}
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	user := reply.GetUsername()
 	scope := reply.GetOauthScope()
 	if user != serviceAccount {
-		grpclog.Fatalf("Got user name %q, want %q.", user, serviceAccount)
+		logger.Fatalf("Got user name %q, want %q.", user, serviceAccount)
 	}
 	if !strings.Contains(oauthScope, scope) {
-		grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
+		logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
 	}
 }
 
 func getServiceAccountJSONKey(keyFile string) []byte {
 	jsonKey, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		grpclog.Fatalf("Failed to read the service account key file: %v", err)
+		logger.Fatalf("Failed to read the service account key file: %v", err)
 	}
 	return jsonKey
 }
@@ -292,16 +294,16 @@ func DoServiceAccountCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, o
 	}
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
 	user := reply.GetUsername()
 	scope := reply.GetOauthScope()
 	if !strings.Contains(string(jsonKey), user) {
-		grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
+		logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
 	}
 	if !strings.Contains(oauthScope, scope) {
-		grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
+		logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
 	}
 }
 
@@ -316,12 +318,12 @@ func DoJWTTokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile string)
 	}
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
 	user := reply.GetUsername()
 	if !strings.Contains(string(jsonKey), user) {
-		grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
+		logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
 	}
 }
 
@@ -330,11 +332,11 @@ func GetToken(serviceAccountKeyFile string, oauthScope string) *oauth2.Token {
 	jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
 	config, err := google.JWTConfigFromJSON(jsonKey, oauthScope)
 	if err != nil {
-		grpclog.Fatalf("Failed to get the config: %v", err)
+		logger.Fatalf("Failed to get the config: %v", err)
 	}
 	token, err := config.TokenSource(context.Background()).Token()
 	if err != nil {
-		grpclog.Fatalf("Failed to get the token: %v", err)
+		logger.Fatalf("Failed to get the token: %v", err)
 	}
 	return token
 }
@@ -351,16 +353,16 @@ func DoOauth2TokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oaut
 	}
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
 	user := reply.GetUsername()
 	scope := reply.GetOauthScope()
 	if !strings.Contains(string(jsonKey), user) {
-		grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
+		logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
 	}
 	if !strings.Contains(oauthScope, scope) {
-		grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
+		logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
 	}
 }
 
@@ -380,15 +382,15 @@ func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScop
 	ctx := metadata.NewOutgoingContext(context.Background(), metadata.MD{"authorization": []string{kv["authorization"]}})
 	reply, err := tc.UnaryCall(ctx, req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	user := reply.GetUsername()
 	scope := reply.GetOauthScope()
 	if !strings.Contains(string(jsonKey), user) {
-		grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
+		logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
 	}
 	if !strings.Contains(oauthScope, scope) {
-		grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
+		logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
 	}
 }
 
@@ -404,10 +406,10 @@ func DoGoogleDefaultCredentials(tc testpb.TestServiceClient, defaultServiceAccou
 	}
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	if reply.GetUsername() != defaultServiceAccount {
-		grpclog.Fatalf("Got user name %q; wanted %q. ", reply.GetUsername(), defaultServiceAccount)
+		logger.Fatalf("Got user name %q; wanted %q. ", reply.GetUsername(), defaultServiceAccount)
 	}
 }
 
@@ -423,10 +425,10 @@ func DoComputeEngineChannelCredentials(tc testpb.TestServiceClient, defaultServi
 	}
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	if reply.GetUsername() != defaultServiceAccount {
-		grpclog.Fatalf("Got user name %q; wanted %q. ", reply.GetUsername(), defaultServiceAccount)
+		logger.Fatalf("Got user name %q; wanted %q. ", reply.GetUsername(), defaultServiceAccount)
 	}
 }
 
@@ -440,12 +442,12 @@ func DoCancelAfterBegin(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 	ctx, cancel := context.WithCancel(metadata.NewOutgoingContext(context.Background(), testMetadata))
 	stream, err := tc.StreamingInputCall(ctx, args...)
 	if err != nil {
-		grpclog.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
 	}
 	cancel()
 	_, err = stream.CloseAndRecv()
 	if status.Code(err) != codes.Canceled {
-		grpclog.Fatalf("%v.CloseAndRecv() got error code %d, want %d", stream, status.Code(err), codes.Canceled)
+		logger.Fatalf("%v.CloseAndRecv() got error code %d, want %d", stream, status.Code(err), codes.Canceled)
 	}
 }
 
@@ -454,7 +456,7 @@ func DoCancelAfterFirstResponse(tc testpb.TestServiceClient, args ...grpc.CallOp
 	ctx, cancel := context.WithCancel(context.Background())
 	stream, err := tc.FullDuplexCall(ctx, args...)
 	if err != nil {
-		grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
+		logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
 	}
 	respParam := []*testpb.ResponseParameters{
 		{
@@ -468,14 +470,14 @@ func DoCancelAfterFirstResponse(tc testpb.TestServiceClient, args ...grpc.CallOp
 		Payload:            pl,
 	}
 	if err := stream.Send(req); err != nil {
-		grpclog.Fatalf("%v has error %v while sending %v", stream, err, req)
+		logger.Fatalf("%v has error %v while sending %v", stream, err, req)
 	}
 	if _, err := stream.Recv(); err != nil {
-		grpclog.Fatalf("%v.Recv() = %v", stream, err)
+		logger.Fatalf("%v.Recv() = %v", stream, err)
 	}
 	cancel()
 	if _, err := stream.Recv(); status.Code(err) != codes.Canceled {
-		grpclog.Fatalf("%v compleled with error code %d, want %d", stream, status.Code(err), codes.Canceled)
+		logger.Fatalf("%v compleled with error code %d, want %d", stream, status.Code(err), codes.Canceled)
 	}
 }
 
@@ -490,16 +492,16 @@ var (
 
 func validateMetadata(header, trailer metadata.MD) {
 	if len(header[initialMetadataKey]) != 1 {
-		grpclog.Fatalf("Expected exactly one header from server. Received %d", len(header[initialMetadataKey]))
+		logger.Fatalf("Expected exactly one header from server. Received %d", len(header[initialMetadataKey]))
 	}
 	if header[initialMetadataKey][0] != initialMetadataValue {
-		grpclog.Fatalf("Got header %s; want %s", header[initialMetadataKey][0], initialMetadataValue)
+		logger.Fatalf("Got header %s; want %s", header[initialMetadataKey][0], initialMetadataValue)
 	}
 	if len(trailer[trailingMetadataKey]) != 1 {
-		grpclog.Fatalf("Expected exactly one trailer from server. Received %d", len(trailer[trailingMetadataKey]))
+		logger.Fatalf("Expected exactly one trailer from server. Received %d", len(trailer[trailingMetadataKey]))
 	}
 	if trailer[trailingMetadataKey][0] != trailingMetadataValue {
-		grpclog.Fatalf("Got trailer %s; want %s", trailer[trailingMetadataKey][0], trailingMetadataValue)
+		logger.Fatalf("Got trailer %s; want %s", trailer[trailingMetadataKey][0], trailingMetadataValue)
 	}
 }
 
@@ -521,19 +523,19 @@ func DoCustomMetadata(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 		args...,
 	)
 	if err != nil {
-		grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
+		logger.Fatal("/TestService/UnaryCall RPC failed: ", err)
 	}
 	t := reply.GetPayload().GetType()
 	s := len(reply.GetPayload().GetBody())
 	if t != testpb.PayloadType_COMPRESSABLE || s != 1 {
-		grpclog.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, 1)
+		logger.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, 1)
 	}
 	validateMetadata(header, trailer)
 
 	// Testing with FullDuplex.
 	stream, err := tc.FullDuplexCall(ctx, args...)
 	if err != nil {
-		grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+		logger.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
 	respParam := []*testpb.ResponseParameters{
 		{
@@ -546,20 +548,20 @@ func DoCustomMetadata(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 		Payload:            pl,
 	}
 	if err := stream.Send(streamReq); err != nil {
-		grpclog.Fatalf("%v has error %v while sending %v", stream, err, streamReq)
+		logger.Fatalf("%v has error %v while sending %v", stream, err, streamReq)
 	}
 	streamHeader, err := stream.Header()
 	if err != nil {
-		grpclog.Fatalf("%v.Header() = %v", stream, err)
+		logger.Fatalf("%v.Header() = %v", stream, err)
 	}
 	if _, err := stream.Recv(); err != nil {
-		grpclog.Fatalf("%v.Recv() = %v", stream, err)
+		logger.Fatalf("%v.Recv() = %v", stream, err)
 	}
 	if err := stream.CloseSend(); err != nil {
-		grpclog.Fatalf("%v.CloseSend() = %v, want <nil>", stream, err)
+		logger.Fatalf("%v.CloseSend() = %v, want <nil>", stream, err)
 	}
 	if _, err := stream.Recv(); err != io.EOF {
-		grpclog.Fatalf("%v failed to complete the custom metadata test: %v", stream, err)
+		logger.Fatalf("%v failed to complete the custom metadata test: %v", stream, err)
 	}
 	streamTrailer := stream.Trailer()
 	validateMetadata(streamHeader, streamTrailer)
@@ -579,24 +581,24 @@ func DoStatusCodeAndMessage(tc testpb.TestServiceClient, args ...grpc.CallOption
 		ResponseStatus: respStatus,
 	}
 	if _, err := tc.UnaryCall(context.Background(), req, args...); err == nil || err.Error() != expectedErr.Error() {
-		grpclog.Fatalf("%v.UnaryCall(_, %v) = _, %v, want _, %v", tc, req, err, expectedErr)
+		logger.Fatalf("%v.UnaryCall(_, %v) = _, %v, want _, %v", tc, req, err, expectedErr)
 	}
 	// Test FullDuplexCall.
 	stream, err := tc.FullDuplexCall(context.Background(), args...)
 	if err != nil {
-		grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+		logger.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
 	streamReq := &testpb.StreamingOutputCallRequest{
 		ResponseStatus: respStatus,
 	}
 	if err := stream.Send(streamReq); err != nil {
-		grpclog.Fatalf("%v has error %v while sending %v, want <nil>", stream, err, streamReq)
+		logger.Fatalf("%v has error %v while sending %v, want <nil>", stream, err, streamReq)
 	}
 	if err := stream.CloseSend(); err != nil {
-		grpclog.Fatalf("%v.CloseSend() = %v, want <nil>", stream, err)
+		logger.Fatalf("%v.CloseSend() = %v, want <nil>", stream, err)
 	}
 	if _, err = stream.Recv(); err.Error() != expectedErr.Error() {
-		grpclog.Fatalf("%v.Recv() returned error %v, want %v", stream, err, expectedErr)
+		logger.Fatalf("%v.Recv() returned error %v, want %v", stream, err, expectedErr)
 	}
 }
 
@@ -617,7 +619,7 @@ func DoSpecialStatusMessage(tc testpb.TestServiceClient, args ...grpc.CallOption
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	if _, err := tc.UnaryCall(ctx, req, args...); err == nil || err.Error() != expectedErr.Error() {
-		grpclog.Fatalf("%v.UnaryCall(_, %v) = _, %v, want _, %v", tc, req, err, expectedErr)
+		logger.Fatalf("%v.UnaryCall(_, %v) = _, %v, want _, %v", tc, req, err, expectedErr)
 	}
 }
 
@@ -625,7 +627,7 @@ func DoSpecialStatusMessage(tc testpb.TestServiceClient, args ...grpc.CallOption
 func DoUnimplementedService(tc testpb.UnimplementedServiceClient) {
 	_, err := tc.UnimplementedCall(context.Background(), &testpb.Empty{})
 	if status.Code(err) != codes.Unimplemented {
-		grpclog.Fatalf("%v.UnimplementedCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unimplemented)
+		logger.Fatalf("%v.UnimplementedCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unimplemented)
 	}
 }
 
@@ -633,7 +635,7 @@ func DoUnimplementedService(tc testpb.UnimplementedServiceClient) {
 func DoUnimplementedMethod(cc *grpc.ClientConn) {
 	var req, reply proto.Message
 	if err := cc.Invoke(context.Background(), "/grpc.testing.TestService/UnimplementedCall", req, reply); err == nil || status.Code(err) != codes.Unimplemented {
-		grpclog.Fatalf("ClientConn.Invoke(_, _, _, _, _) = %v, want error code %s", err, codes.Unimplemented)
+		logger.Fatalf("ClientConn.Invoke(_, _, _, _, _) = %v, want error code %s", err, codes.Unimplemented)
 	}
 }
 
@@ -655,18 +657,18 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) {
 	for i := 0; i < rpcCount; i++ {
 		resp, err := tc.UnaryCall(ctx, req)
 		if err != nil {
-			grpclog.Fatalf("iteration %d, failed to do UnaryCall: %v", i, err)
+			logger.Fatalf("iteration %d, failed to do UnaryCall: %v", i, err)
 		}
 		id := resp.ServerId
 		if id == "" {
-			grpclog.Fatalf("iteration %d, got empty server ID", i)
+			logger.Fatalf("iteration %d, got empty server ID", i)
 		}
 		if i == 0 {
 			serverID = id
 			continue
 		}
 		if serverID != id {
-			grpclog.Fatalf("iteration %d, got different server ids: %q vs %q", i, serverID, id)
+			logger.Fatalf("iteration %d, got different server ids: %q vs %q", i, serverID, id)
 		}
 	}
 }
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 7792094d54de..b119bcd2b865 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -91,6 +91,8 @@ var (
 	// 0 or 1 representing an RPC has succeeded. Use hasRPCSucceeded and
 	// setRPCSucceeded to access in a safe manner.
 	rpcSucceeded uint32
+
+	logger = grpclog.Component("interop")
 )
 
 type statsService struct {
@@ -155,7 +157,7 @@ func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalanc
 				return watcher.buildResp(), nil
 			}
 		case <-ctx.Done():
-			grpclog.Info("Timed out, returning partial stats")
+			logger.Info("Timed out, returning partial stats")
 			return watcher.buildResp(), nil
 		}
 	}
@@ -220,7 +222,7 @@ func main() {
 
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *statsPort))
 	if err != nil {
-		grpclog.Fatalf("failed to listen: %v", err)
+		logger.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
 	defer s.Stop()
@@ -231,7 +233,7 @@ func main() {
 	for i := 0; i < *numChannels; i++ {
 		conn, err := grpc.DialContext(context.Background(), *server, grpc.WithInsecure())
 		if err != nil {
-			grpclog.Fatalf("Fail to dial: %v", err)
+			logger.Fatalf("Fail to dial: %v", err)
 		}
 		defer conn.Close()
 		clients[i] = testpb.NewTestServiceClient(conn)
@@ -305,7 +307,7 @@ func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *tim
 					watcher.chanHosts <- info
 				}
 				if err != nil && *failOnFailedRPC && hasRPCSucceeded() {
-					grpclog.Fatalf("RPC failed: %v", err)
+					logger.Fatalf("RPC failed: %v", err)
 				}
 				if err == nil {
 					setRPCSucceeded()
diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go
index cfb2016f8670..45b8448822b2 100644
--- a/interop/xds/server/server.go
+++ b/interop/xds/server/server.go
@@ -37,6 +37,8 @@ var (
 	port     = flag.Int("port", 8080, "The server port")
 	serverID = flag.String("server_id", "go_server", "Server ID included in response")
 	hostname = getHostname()
+
+	logger = grpclog.Component("interop")
 )
 
 func getHostname() string {
@@ -66,7 +68,7 @@ func main() {
 	p := strconv.Itoa(*port)
 	lis, err := net.Listen("tcp", ":"+p)
 	if err != nil {
-		grpclog.Fatalf("failed to listen: %v", err)
+		logger.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
 	testpb.RegisterTestServiceServer(s, &server{})
diff --git a/profiling/cmd/catapult.go b/profiling/cmd/catapult.go
index 953304b3ed59..4b3848d0d198 100644
--- a/profiling/cmd/catapult.go
+++ b/profiling/cmd/catapult.go
@@ -26,7 +26,6 @@ import (
 	"sort"
 	"strings"
 
-	"google.golang.org/grpc/grpclog"
 	ppb "google.golang.org/grpc/profiling/proto"
 )
 
@@ -177,7 +176,7 @@ func streamStatsCatapultJSONSingle(stat *ppb.Stat, baseSec int64, baseNsec int32
 					flowEndPID = opid
 					flowEndTID = fmt.Sprintf("%d", stat.Timers[flowEndID].GoId)
 				} else {
-					grpclog.Infof("cannot find %s/grpc/stream/recv/header for %s/http2/recv/header", opid, opid)
+					logger.Infof("cannot find %s/grpc/stream/recv/header for %s/http2/recv/header", opid, opid)
 				}
 			case "/http2/recv/dataFrame/loopyReader":
 				flowEndID = filterCounter(stat, "/recvAndDecompress", lrc.GetAndInc("/http2/recv/dataFrame/loopyReader"))
@@ -185,7 +184,7 @@ func streamStatsCatapultJSONSingle(stat *ppb.Stat, baseSec int64, baseNsec int32
 					flowEndPID = opid
 					flowEndTID = fmt.Sprintf("%d", stat.Timers[flowEndID].GoId)
 				} else {
-					grpclog.Infof("cannot find %s/recvAndDecompress for %s/http2/recv/dataFrame/loopyReader", opid, opid)
+					logger.Infof("cannot find %s/recvAndDecompress for %s/http2/recv/dataFrame/loopyReader", opid, opid)
 				}
 			default:
 				flowEndID = -1
@@ -232,7 +231,7 @@ func streamStatsCatapultJSONSingle(stat *ppb.Stat, baseSec int64, baseNsec int32
 					flowBeginPID = opid
 					flowBeginTID = fmt.Sprintf("%d", stat.Timers[flowBeginID].GoId)
 				} else {
-					grpclog.Infof("cannot find /%d/transport/enqueue for /%d/http2/send/dataFrame/loopyWriter/preprocess", connectionCounter, connectionCounter)
+					logger.Infof("cannot find /%d/transport/enqueue for /%d/http2/send/dataFrame/loopyWriter/preprocess", connectionCounter, connectionCounter)
 				}
 			default:
 				flowBeginID = -1
@@ -305,14 +304,14 @@ func timerBeginIsBefore(ti *ppb.Timer, tj *ppb.Timer) bool {
 // write to. The grpc-go profiling snapshot is processed and converted to a
 // JSON format that can be understood by trace-viewer.
 func streamStatsCatapultJSON(s *snapshot, streamStatsCatapultJSONFileName string) (err error) {
-	grpclog.Infof("calculating stream stats filters")
+	logger.Infof("calculating stream stats filters")
 	filterArray := strings.Split(*flagStreamStatsFilter, ",")
 	filter := make(map[string]bool)
 	for _, f := range filterArray {
 		filter[f] = true
 	}
 
-	grpclog.Infof("filter stream stats for %s", *flagStreamStatsFilter)
+	logger.Infof("filter stream stats for %s", *flagStreamStatsFilter)
 	var streamStats []*ppb.Stat
 	for _, stat := range s.StreamStats {
 		if _, ok := filter[stat.Tags]; ok {
@@ -320,14 +319,14 @@ func streamStatsCatapultJSON(s *snapshot, streamStatsCatapultJSONFileName string
 		}
 	}
 
-	grpclog.Infof("sorting timers within all stats")
+	logger.Infof("sorting timers within all stats")
 	for id := range streamStats {
 		sort.Slice(streamStats[id].Timers, func(i, j int) bool {
 			return timerBeginIsBefore(streamStats[id].Timers[i], streamStats[id].Timers[j])
 		})
 	}
 
-	grpclog.Infof("sorting stream stats")
+	logger.Infof("sorting stream stats")
 	sort.Slice(streamStats, func(i, j int) bool {
 		if len(streamStats[j].Timers) == 0 {
 			return true
@@ -352,7 +351,7 @@ func streamStatsCatapultJSON(s *snapshot, streamStatsCatapultJSONFileName string
 	}
 
 	// All timestamps use the earliest timestamp available as the reference.
-	grpclog.Infof("calculating the earliest timestamp across all timers")
+	logger.Infof("calculating the earliest timestamp across all timers")
 	var base *ppb.Timer
 	for _, stat := range streamStats {
 		for _, timer := range stat.Timers {
@@ -362,34 +361,34 @@ func streamStatsCatapultJSON(s *snapshot, streamStatsCatapultJSONFileName string
 		}
 	}
 
-	grpclog.Infof("converting %d stats to catapult JSON format", len(streamStats))
+	logger.Infof("converting %d stats to catapult JSON format", len(streamStats))
 	var jsonNodes []jsonNode
 	for _, stat := range streamStats {
 		jsonNodes = append(jsonNodes, streamStatsCatapultJSONSingle(stat, base.BeginSec, base.BeginNsec)...)
 	}
 
-	grpclog.Infof("marshalling catapult JSON")
+	logger.Infof("marshalling catapult JSON")
 	b, err := json.Marshal(jsonNodes)
 	if err != nil {
-		grpclog.Errorf("cannot marshal JSON: %v", err)
+		logger.Errorf("cannot marshal JSON: %v", err)
 		return err
 	}
 
-	grpclog.Infof("creating catapult JSON file")
+	logger.Infof("creating catapult JSON file")
 	streamStatsCatapultJSONFile, err := os.Create(streamStatsCatapultJSONFileName)
 	if err != nil {
-		grpclog.Errorf("cannot create file %s: %v", streamStatsCatapultJSONFileName, err)
+		logger.Errorf("cannot create file %s: %v", streamStatsCatapultJSONFileName, err)
 		return err
 	}
 	defer streamStatsCatapultJSONFile.Close()
 
-	grpclog.Infof("writing catapult JSON to disk")
+	logger.Infof("writing catapult JSON to disk")
 	_, err = streamStatsCatapultJSONFile.Write(b)
 	if err != nil {
-		grpclog.Errorf("cannot write marshalled JSON: %v", err)
+		logger.Errorf("cannot write marshalled JSON: %v", err)
 		return err
 	}
 
-	grpclog.Infof("successfully wrote catapult JSON file %s", streamStatsCatapultJSONFileName)
+	logger.Infof("successfully wrote catapult JSON file %s", streamStatsCatapultJSONFileName)
 	return nil
 }
diff --git a/profiling/cmd/local.go b/profiling/cmd/local.go
index 7ac1bb90cf79..7c312ac361ee 100644
--- a/profiling/cmd/local.go
+++ b/profiling/cmd/local.go
@@ -22,24 +22,22 @@ import (
 	"encoding/gob"
 	"fmt"
 	"os"
-
-	"google.golang.org/grpc/grpclog"
 )
 
 func loadSnapshot(snapshotFileName string) (*snapshot, error) {
-	grpclog.Infof("opening snapshot file %s", snapshotFileName)
+	logger.Infof("opening snapshot file %s", snapshotFileName)
 	snapshotFile, err := os.Open(snapshotFileName)
 	if err != nil {
-		grpclog.Errorf("cannot open %s: %v", snapshotFileName, err)
+		logger.Errorf("cannot open %s: %v", snapshotFileName, err)
 		return nil, err
 	}
 	defer snapshotFile.Close()
 
-	grpclog.Infof("decoding snapshot file %s", snapshotFileName)
+	logger.Infof("decoding snapshot file %s", snapshotFileName)
 	s := &snapshot{}
 	decoder := gob.NewDecoder(snapshotFile)
 	if err = decoder.Decode(s); err != nil {
-		grpclog.Errorf("cannot decode %s: %v", snapshotFileName, err)
+		logger.Errorf("cannot decode %s: %v", snapshotFileName, err)
 		return nil, err
 	}
 
diff --git a/profiling/cmd/main.go b/profiling/cmd/main.go
index 1471dc9b7c37..2328fbd9f800 100644
--- a/profiling/cmd/main.go
+++ b/profiling/cmd/main.go
@@ -27,24 +27,26 @@ import (
 	ppb "google.golang.org/grpc/profiling/proto"
 )
 
+var logger = grpclog.Component("profiling")
+
 type snapshot struct {
 	StreamStats []*ppb.Stat
 }
 
 func main() {
 	if err := parseArgs(); err != nil {
-		grpclog.Errorf("error parsing flags: %v", err)
+		logger.Errorf("error parsing flags: %v", err)
 		os.Exit(1)
 	}
 
 	if *flagAddress != "" {
 		if err := remoteCommand(); err != nil {
-			grpclog.Errorf("error: %v", err)
+			logger.Errorf("error: %v", err)
 			os.Exit(1)
 		}
 	} else {
 		if err := localCommand(); err != nil {
-			grpclog.Errorf("error: %v", err)
+			logger.Errorf("error: %v", err)
 			os.Exit(1)
 		}
 	}
diff --git a/profiling/cmd/remote.go b/profiling/cmd/remote.go
index 930aec4f7ac4..b6adfd6a6bef 100644
--- a/profiling/cmd/remote.go
+++ b/profiling/cmd/remote.go
@@ -26,47 +26,46 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/grpclog"
 	ppb "google.golang.org/grpc/profiling/proto"
 )
 
 func setEnabled(ctx context.Context, c ppb.ProfilingClient, enabled bool) error {
 	_, err := c.Enable(ctx, &ppb.EnableRequest{Enabled: enabled})
 	if err != nil {
-		grpclog.Infof("error calling Enable: %v\n", err)
+		logger.Infof("error calling Enable: %v\n", err)
 		return err
 	}
 
-	grpclog.Infof("successfully set enabled = %v", enabled)
+	logger.Infof("successfully set enabled = %v", enabled)
 	return nil
 }
 
 func retrieveSnapshot(ctx context.Context, c ppb.ProfilingClient, f string) error {
-	grpclog.Infof("getting stream stats")
+	logger.Infof("getting stream stats")
 	resp, err := c.GetStreamStats(ctx, &ppb.GetStreamStatsRequest{})
 	if err != nil {
-		grpclog.Errorf("error calling GetStreamStats: %v\n", err)
+		logger.Errorf("error calling GetStreamStats: %v\n", err)
 		return err
 	}
 	s := &snapshot{StreamStats: resp.StreamStats}
 
-	grpclog.Infof("creating snapshot file %s", f)
+	logger.Infof("creating snapshot file %s", f)
 	file, err := os.Create(f)
 	if err != nil {
-		grpclog.Errorf("cannot create %s: %v", f, err)
+		logger.Errorf("cannot create %s: %v", f, err)
 		return err
 	}
 	defer file.Close()
 
-	grpclog.Infof("encoding data and writing to snapshot file %s", f)
+	logger.Infof("encoding data and writing to snapshot file %s", f)
 	encoder := gob.NewEncoder(file)
 	err = encoder.Encode(s)
 	if err != nil {
-		grpclog.Infof("error encoding: %v", err)
+		logger.Infof("error encoding: %v", err)
 		return err
 	}
 
-	grpclog.Infof("successfully wrote profiling snapshot to %s", f)
+	logger.Infof("successfully wrote profiling snapshot to %s", f)
 	return nil
 }
 
@@ -78,10 +77,10 @@ func remoteCommand() error {
 		defer cancel()
 	}
 
-	grpclog.Infof("dialing %s", *flagAddress)
+	logger.Infof("dialing %s", *flagAddress)
 	cc, err := grpc.Dial(*flagAddress, grpc.WithInsecure())
 	if err != nil {
-		grpclog.Errorf("cannot dial %s: %v", *flagAddress, err)
+		logger.Errorf("cannot dial %s: %v", *flagAddress, err)
 		return err
 	}
 	defer cc.Close()
diff --git a/profiling/service/service.go b/profiling/service/service.go
index 52a7ce789717..e2ce8926d6c5 100644
--- a/profiling/service/service.go
+++ b/profiling/service/service.go
@@ -35,6 +35,8 @@ import (
 	ppb "google.golang.org/grpc/profiling/proto"
 )
 
+var logger = grpclog.Component("profiling")
+
 // ProfilingConfig defines configuration options for the Init method.
 type ProfilingConfig struct {
 	// Setting this to true will enable profiling.
@@ -96,9 +98,9 @@ func getProfilingServerInstance() *profilingServer {
 
 func (s *profilingServer) Enable(ctx context.Context, req *ppb.EnableRequest) (*ppb.EnableResponse, error) {
 	if req.Enabled {
-		grpclog.Infof("profilingServer: Enable: enabling profiling")
+		logger.Infof("profilingServer: Enable: enabling profiling")
 	} else {
-		grpclog.Infof("profilingServer: Enable: disabling profiling")
+		logger.Infof("profilingServer: Enable: disabling profiling")
 	}
 	profiling.Enable(req.Enabled)
 
@@ -131,12 +133,12 @@ func statToProtoStat(stat *profiling.Stat) *ppb.Stat {
 func (s *profilingServer) GetStreamStats(ctx context.Context, req *ppb.GetStreamStatsRequest) (*ppb.GetStreamStatsResponse, error) {
 	// Since the drain operation is destructive, only one client request should
 	// be served at a time.
-	grpclog.Infof("profilingServer: GetStreamStats: processing request")
+	logger.Infof("profilingServer: GetStreamStats: processing request")
 	s.drainMutex.Lock()
 	results := profiling.StreamStats.Drain()
 	s.drainMutex.Unlock()
 
-	grpclog.Infof("profilingServer: GetStreamStats: returning %v records", len(results))
+	logger.Infof("profilingServer: GetStreamStats: returning %v records", len(results))
 	streamStats := make([]*ppb.Stat, 0)
 	for _, stat := range results {
 		streamStats = append(streamStats, statToProtoStat(stat.(*profiling.Stat)))
diff --git a/stress/client/main.go b/stress/client/main.go
index f44bfa53d83f..c5bfffa4e519 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -52,6 +52,8 @@ var (
 	testCA               = flag.Bool("use_test_ca", false, "Whether to replace platform root CAs with test CA as the CA root")
 	tlsServerName        = flag.String("server_host_override", "foo.test.google.fr", "The server name use to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.")
 	caFile               = flag.String("ca_file", "", "The file containing the CA root cert file")
+
+	logger = grpclog.Component("stress")
 )
 
 // testCaseWithWeight contains the test case type and its weight.
@@ -196,7 +198,7 @@ func (s *server) createGauge(name string) *gauge {
 func startServer(server *server, port int) {
 	lis, err := net.Listen("tcp", ":"+strconv.Itoa(port))
 	if err != nil {
-		grpclog.Fatalf("failed to listen: %v", err)
+		logger.Fatalf("failed to listen: %v", err)
 	}
 
 	s := grpc.NewServer()
@@ -248,23 +250,23 @@ func performRPCs(gauge *gauge, conn *grpc.ClientConn, selector *weightedRandomTe
 }
 
 func logParameterInfo(addresses []string, tests []testCaseWithWeight) {
-	grpclog.Infof("server_addresses: %s", *serverAddresses)
-	grpclog.Infof("test_cases: %s", *testCases)
-	grpclog.Infof("test_duration_secs: %d", *testDurationSecs)
-	grpclog.Infof("num_channels_per_server: %d", *numChannelsPerServer)
-	grpclog.Infof("num_stubs_per_channel: %d", *numStubsPerChannel)
-	grpclog.Infof("metrics_port: %d", *metricsPort)
-	grpclog.Infof("use_tls: %t", *useTLS)
-	grpclog.Infof("use_test_ca: %t", *testCA)
-	grpclog.Infof("server_host_override: %s", *tlsServerName)
-
-	grpclog.Infoln("addresses:")
+	logger.Infof("server_addresses: %s", *serverAddresses)
+	logger.Infof("test_cases: %s", *testCases)
+	logger.Infof("test_duration_secs: %d", *testDurationSecs)
+	logger.Infof("num_channels_per_server: %d", *numChannelsPerServer)
+	logger.Infof("num_stubs_per_channel: %d", *numStubsPerChannel)
+	logger.Infof("metrics_port: %d", *metricsPort)
+	logger.Infof("use_tls: %t", *useTLS)
+	logger.Infof("use_test_ca: %t", *testCA)
+	logger.Infof("server_host_override: %s", *tlsServerName)
+
+	logger.Infoln("addresses:")
 	for i, addr := range addresses {
-		grpclog.Infof("%d. %s\n", i+1, addr)
+		logger.Infof("%d. %s\n", i+1, addr)
 	}
-	grpclog.Infoln("tests:")
+	logger.Infoln("tests:")
 	for i, test := range tests {
-		grpclog.Infof("%d. %v\n", i+1, test)
+		logger.Infof("%d. %v\n", i+1, test)
 	}
 }
 
@@ -283,7 +285,7 @@ func newConn(address string, useTLS, testCA bool, tlsServerName string) (*grpc.C
 			}
 			creds, err = credentials.NewClientTLSFromFile(*caFile, sn)
 			if err != nil {
-				grpclog.Fatalf("Failed to create TLS credentials %v", err)
+				logger.Fatalf("Failed to create TLS credentials %v", err)
 			}
 		} else {
 			creds = credentials.NewClientTLSFromCert(nil, sn)
@@ -311,7 +313,7 @@ func main() {
 		for connIndex := 0; connIndex < *numChannelsPerServer; connIndex++ {
 			conn, err := newConn(address, *useTLS, *testCA, *tlsServerName)
 			if err != nil {
-				grpclog.Fatalf("Fail to dial: %v", err)
+				logger.Fatalf("Fail to dial: %v", err)
 			}
 			defer conn.Close()
 			for clientIndex := 0; clientIndex < *numStubsPerChannel; clientIndex++ {
@@ -331,6 +333,6 @@ func main() {
 		close(stop)
 	}
 	wg.Wait()
-	grpclog.Infof(" ===== ALL DONE ===== ")
+	logger.Infof(" ===== ALL DONE ===== ")
 
 }
diff --git a/stress/metrics_client/main.go b/stress/metrics_client/main.go
index 655b97a7ae2d..ad6db6dd7a19 100644
--- a/stress/metrics_client/main.go
+++ b/stress/metrics_client/main.go
@@ -33,12 +33,14 @@ import (
 var (
 	metricsServerAddress = flag.String("metrics_server_address", "", "The metrics server addresses in the format <hostname>:<port>")
 	totalOnly            = flag.Bool("total_only", false, "If true, this prints only the total value of all gauges")
+
+	logger = grpclog.Component("stress")
 )
 
 func printMetrics(client metricspb.MetricsServiceClient, totalOnly bool) {
 	stream, err := client.GetAllGauges(context.Background(), &metricspb.EmptyMessage{})
 	if err != nil {
-		grpclog.Fatalf("failed to call GetAllGauges: %v", err)
+		logger.Fatalf("failed to call GetAllGauges: %v", err)
 	}
 
 	var (
@@ -56,25 +58,25 @@ func printMetrics(client metricspb.MetricsServiceClient, totalOnly bool) {
 		}
 		v := gaugeResponse.GetLongValue()
 		if !totalOnly {
-			grpclog.Infof("%s: %d", gaugeResponse.Name, v)
+			logger.Infof("%s: %d", gaugeResponse.Name, v)
 		}
 		overallQPS += v
 	}
 	if rpcStatus != io.EOF {
-		grpclog.Fatalf("failed to finish server streaming: %v", rpcStatus)
+		logger.Fatalf("failed to finish server streaming: %v", rpcStatus)
 	}
-	grpclog.Infof("overall qps: %d", overallQPS)
+	logger.Infof("overall qps: %d", overallQPS)
 }
 
 func main() {
 	flag.Parse()
 	if *metricsServerAddress == "" {
-		grpclog.Fatalf("Metrics server address is empty.")
+		logger.Fatalf("Metrics server address is empty.")
 	}
 
 	conn, err := grpc.Dial(*metricsServerAddress, grpc.WithInsecure())
 	if err != nil {
-		grpclog.Fatalf("cannot connect to metrics server: %v", err)
+		logger.Fatalf("cannot connect to metrics server: %v", err)
 	}
 	defer conn.Close()
 
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 7be226708581..f0189cf26444 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -35,7 +35,6 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/balancerload"
 	"google.golang.org/grpc/internal/grpcsync"
@@ -84,7 +83,7 @@ func (b *testBalancer) UpdateClientConnState(state balancer.ClientConnState) err
 		var err error
 		b.sc, err = b.cc.NewSubConn(state.ResolverState.Addresses, b.newSubConnOptions)
 		if err != nil {
-			grpclog.Errorf("testBalancer: failed to NewSubConn: %v", err)
+			logger.Errorf("testBalancer: failed to NewSubConn: %v", err)
 			return nil
 		}
 		b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &picker{sc: b.sc, bal: b}})
@@ -94,9 +93,9 @@ func (b *testBalancer) UpdateClientConnState(state balancer.ClientConnState) err
 }
 
 func (b *testBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
-	grpclog.Infof("testBalancer: UpdateSubConnState: %p, %v", sc, s)
+	logger.Infof("testBalancer: UpdateSubConnState: %p, %v", sc, s)
 	if b.sc != sc {
-		grpclog.Infof("testBalancer: ignored state change because sc is not recognized")
+		logger.Infof("testBalancer: ignored state change because sc is not recognized")
 		return
 	}
 	if s.ConnectivityState == connectivity.Shutdown {
diff --git a/test/end2end_test.go b/test/end2end_test.go
index bdc70affa2b9..d1a2cdf68612 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -4919,15 +4919,14 @@ func logOutputHasContents(v []byte, wakeup chan<- bool) bool {
 	return false
 }
 
-var verboseLogs = flag.Bool("verbose_logs", false, "show all grpclog output, without filtering")
+var verboseLogs = flag.Bool("verbose_logs", false, "show all log output, without filtering")
 
 func noop() {}
 
-// declareLogNoise declares that t is expected to emit the following noisy phrases,
-// even on success. Those phrases will be filtered from grpclog output
-// and only be shown if *verbose_logs or t ends up failing.
-// The returned restore function should be called with defer to be run
-// before the test ends.
+// declareLogNoise declares that t is expected to emit the following noisy
+// phrases, even on success. Those phrases will be filtered from log output and
+// only be shown if *verbose_logs or t ends up failing. The returned restore
+// function should be called with defer to be run before the test ends.
 func declareLogNoise(t *testing.T, phrases ...string) (restore func()) {
 	if *verboseLogs {
 		return noop
diff --git a/test/logging.go b/test/logging.go
new file mode 100644
index 000000000000..15a923ab557e
--- /dev/null
+++ b/test/logging.go
@@ -0,0 +1,23 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import "google.golang.org/grpc/grpclog"
+
+var logger = grpclog.Component("testing")
diff --git a/vet.sh b/vet.sh
index 8c14c7d3618e..29e2e371e544 100755
--- a/vet.sh
+++ b/vet.sh
@@ -83,6 +83,9 @@ not git grep -l 'x/net/context' -- "*.go"
 #   thread safety.
 git grep -l '"math/rand"' -- "*.go" 2>&1 | not grep -v '^examples\|^stress\|grpcrand\|^benchmark\|wrr_test'
 
+# - Do not call grpclog directly. Use grpclog.Component instead.
+git grep -l 'grpclog.I\|grpclog.W\|grpclog.E\|grpclog.F\|grpclog.V' -- "*.go" | not grep -v '^grpclog/component.go\|^internal/grpctest/tlogger_test.go'
+
 # - Ensure all ptypes proto packages are renamed when importing.
 not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go"
 

From 7419b444ee1718a247695a367bb0ef292e512187 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 6 Aug 2020 15:36:15 -0700
Subject: [PATCH 151/481] xds: Improve TestClientWrapperWatchEDS. (#3783)

* One subtest should not depend on a previous one. So, refactored the test to be a single one, instead of having multiple subtests.
* Added a channel on the fake server to notify when it receives a connection. This test was sometimes failing because it went ahead and sent EDS requests and expected responses, before the connection to the fake server was up.
---
 .../edsbalancer/xds_client_wrapper_test.go    | 151 +++++++++---------
 xds/internal/testutils/fakeserver/server.go   |  25 ++-
 2 files changed, 98 insertions(+), 78 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index 90372b539ba7..6ae65dc7c94b 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -20,12 +20,14 @@ package edsbalancer
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
@@ -44,6 +46,32 @@ var (
 	testEDSClusterName = "test/service/eds"
 )
 
+// Given a list of resource names, verifies that EDS requests for the same are
+// received at the fake server.
+func verifyExpectedRequests(fs *fakeserver.Server, resourceNames ...string) error {
+	wantReq := &xdspb.DiscoveryRequest{
+		TypeUrl: version.V2EndpointsURL,
+		Node:    testutils.EmptyNodeProtoV2,
+	}
+	for _, name := range resourceNames {
+		if name != "" {
+			wantReq.ResourceNames = []string{name}
+		}
+		req, err := fs.XDSRequestChan.TimedReceive(time.Millisecond * 100)
+		if err != nil {
+			return fmt.Errorf("timed out when expecting request {%+v} at fake server", wantReq)
+		}
+		edsReq := req.(*fakeserver.Request)
+		if edsReq.Err != nil {
+			return fmt.Errorf("eds RPC failed with err: %v", edsReq.Err)
+		}
+		if !proto.Equal(edsReq.Req, wantReq) {
+			return fmt.Errorf("got EDS request %v, expected: %v, diff: %s", edsReq.Req, wantReq, cmp.Diff(edsReq.Req, wantReq, cmp.Comparer(proto.Equal)))
+		}
+	}
+	return nil
+}
+
 // TestClientWrapperWatchEDS verifies that the clientWrapper registers an
 // EDS watch for expected resource upon receiving an update from the top-level
 // edsBalancer.
@@ -59,87 +87,56 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 		t.Fatalf("Failed to start fake xDS server: %v", err)
 	}
 	defer cleanup()
+	t.Logf("Started fake xDS server at %s...", fakeServer.Address)
 
 	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
 	defer cw.close()
+	t.Logf("Started xDS client wrapper for endpoint %s...", testServiceName)
+
+	oldBootstrapConfigNew := bootstrapConfigNew
+	bootstrapConfigNew = func() (*bootstrap.Config, error) {
+		return &bootstrap.Config{
+			BalancerName: fakeServer.Address,
+			Creds:        grpc.WithInsecure(),
+			NodeProto:    testutils.EmptyNodeProtoV2,
+		}, nil
+	}
+	defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
+
+	// Update with an empty edsServiceName should trigger an EDS watch
+	// for the user's dial target.
+	cw.handleUpdate(&EDSConfig{
+		BalancerName:   fakeServer.Address,
+		EDSServiceName: "",
+	}, nil)
+	if _, err := fakeServer.NewConnChan.TimedReceive(1 * time.Second); err != nil {
+		t.Fatal("Failed to connect to fake server")
+	}
+	t.Log("Client connection established to fake server...")
+	if err := verifyExpectedRequests(fakeServer, testServiceName); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update with an non-empty edsServiceName should trigger an EDS watch for
+	// the same. The previously registered watch will be cancelled, which will
+	// result in an EDS request with no resource names being sent to the server.
+	cw.handleUpdate(&EDSConfig{
+		BalancerName:   fakeServer.Address,
+		EDSServiceName: "foobar-1",
+	}, nil)
+	if err := verifyExpectedRequests(fakeServer, "", "foobar-1"); err != nil {
+		t.Fatal(err)
+	}
 
-	for _, test := range []struct {
-		name             string
-		edsServiceName   string
-		wantResourceName string
-	}{
-		{
-			// Update with an empty edsServiceName should trigger an EDS watch
-			// for the user's dial target.
-			name:             "empty-edsServiceName",
-			edsServiceName:   "",
-			wantResourceName: testServiceName,
-		},
-		{
-			// Update with an non-empty edsServiceName should trigger an EDS
-			// watch for the same.
-			name:             "first-non-empty-edsServiceName",
-			edsServiceName:   "foobar-1",
-			wantResourceName: "foobar-1",
-		},
-		{
-			// Also test the case where the edsServerName changes from one
-			// non-empty name to another, and make sure a new watch is
-			// registered.
-			name:             "second-non-empty-edsServiceName",
-			edsServiceName:   "foobar-2",
-			wantResourceName: "foobar-2",
-		},
-	} {
-		t.Run(test.name, func(t *testing.T) {
-			oldBootstrapConfigNew := bootstrapConfigNew
-			bootstrapConfigNew = func() (*bootstrap.Config, error) {
-				return &bootstrap.Config{
-					BalancerName: fakeServer.Address,
-					Creds:        grpc.WithInsecure(),
-					NodeProto:    testutils.EmptyNodeProtoV2,
-				}, nil
-			}
-			defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
-			cw.handleUpdate(&EDSConfig{
-				BalancerName:   fakeServer.Address,
-				EDSServiceName: test.edsServiceName,
-			}, nil)
-
-			var req interface{}
-			for i := 0; i < 2; i++ {
-				// Each new watch will first cancel the previous watch, and then
-				// start a new watch. The cancel will trigger a request as well.
-				// This loop receives the two requests and keeps the last.
-				r, err := fakeServer.XDSRequestChan.TimedReceive(time.Millisecond * 100)
-				if err != nil {
-					t.Fatalf("i: %v, expected xDS request, but got error: %v", i, err)
-				}
-				req = r
-				// If edsServiceName is empty string, the client doesn't cancel
-				// and resend request. The request from channel was from client
-				// init, and we don't expect a second request.
-				if test.edsServiceName == "" {
-					break
-				}
-			}
-			if r, err := fakeServer.XDSRequestChan.TimedReceive(time.Millisecond * 100); err == nil {
-				t.Fatalf("Expected req channel recv timeout, but got request: %v", r)
-			}
-			edsReq := req.(*fakeserver.Request)
-			if edsReq.Err != nil {
-				t.Fatalf("EDS RPC failed with err: %v", edsReq.Err)
-			}
-
-			wantReq := &xdspb.DiscoveryRequest{
-				TypeUrl:       version.V2EndpointsURL,
-				ResourceNames: []string{test.wantResourceName},
-				Node:          testutils.EmptyNodeProtoV2,
-			}
-			if !proto.Equal(edsReq.Req, wantReq) {
-				t.Fatalf("got EDS request %v, expected: %v, diff: %s", edsReq.Req, wantReq, cmp.Diff(edsReq.Req, wantReq, cmp.Comparer(proto.Equal)))
-			}
-		})
+	// Also test the case where the edsServerName changes from one
+	// non-empty name to another, and make sure a new watch is
+	// registered.
+	cw.handleUpdate(&EDSConfig{
+		BalancerName:   fakeServer.Address,
+		EDSServiceName: "foobar-2",
+	}, nil)
+	if err := verifyExpectedRequests(fakeServer, "", "foobar-2"); err != nil {
+		t.Fatal(err)
 	}
 }
 
diff --git a/xds/internal/testutils/fakeserver/server.go b/xds/internal/testutils/fakeserver/server.go
index 3185386cc794..e1663c60444c 100644
--- a/xds/internal/testutils/fakeserver/server.go
+++ b/xds/internal/testutils/fakeserver/server.go
@@ -75,6 +75,10 @@ type Server struct {
 	// LRSResponseChan is a channel on which the Server accepts the LRS
 	// response to be sent to the client.
 	LRSResponseChan chan *Response
+	// NewConnChan is a channel on which the fake server notifies receipt of new
+	// connection attempts. Tests can gate on this event before proceeding to
+	// other actions which depend on a connection to the fake server being up.
+	NewConnChan *testutils.Channel
 	// Address is the host:port on which the Server is listening for requests.
 	Address string
 
@@ -83,6 +87,20 @@ type Server struct {
 	lrsS *lrsServer
 }
 
+type wrappedListener struct {
+	net.Listener
+	server *Server
+}
+
+func (wl *wrappedListener) Accept() (net.Conn, error) {
+	c, err := wl.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	wl.server.NewConnChan.Send(struct{}{})
+	return c, err
+}
+
 // StartServer makes a new Server and gets it to start listening on a local
 // port for gRPC requests. The returned cancel function should be invoked by
 // the caller upon completion of the test.
@@ -95,17 +113,22 @@ func StartServer() (*Server, func(), error) {
 	s := &Server{
 		XDSRequestChan:  testutils.NewChannelWithSize(defaultChannelBufferSize),
 		LRSRequestChan:  testutils.NewChannelWithSize(defaultChannelBufferSize),
+		NewConnChan:     testutils.NewChannelWithSize(defaultChannelBufferSize),
 		XDSResponseChan: make(chan *Response, defaultChannelBufferSize),
 		LRSResponseChan: make(chan *Response, 1), // The server only ever sends one response.
 		Address:         lis.Addr().String(),
 	}
 	s.xdsS = &xdsServer{reqChan: s.XDSRequestChan, respChan: s.XDSResponseChan}
 	s.lrsS = &lrsServer{reqChan: s.LRSRequestChan, respChan: s.LRSResponseChan}
+	wp := &wrappedListener{
+		Listener: lis,
+		server:   s,
+	}
 
 	server := grpc.NewServer()
 	lrsgrpc.RegisterLoadReportingServiceServer(server, s.lrsS)
 	adsgrpc.RegisterAggregatedDiscoveryServiceServer(server, s.xdsS)
-	go server.Serve(lis)
+	go server.Serve(wp)
 
 	return s, func() { server.Stop() }, nil
 }

From d3e3e7a46f57b9fa7369ec5878e5f4954dea338c Mon Sep 17 00:00:00 2001
From: Elizabeth Zou <elizabeth.zou.0@gmail.com>
Date: Fri, 7 Aug 2020 11:49:45 -0500
Subject: [PATCH 152/481] security/authorization: Added CEL-based authorization
 engine (#3707)

* Add CEL-based authorization engine in a new module under security/authorization
---
 Makefile                                     |   1 +
 security/authorization/engine/engine.go      | 374 +++++++++++++++++++
 security/authorization/engine/engine_test.go | 272 ++++++++++++++
 security/authorization/go.mod                |  11 +
 security/authorization/go.sum                | 104 ++++++
 5 files changed, 762 insertions(+)
 create mode 100644 security/authorization/engine/engine.go
 create mode 100644 security/authorization/engine/engine_test.go
 create mode 100644 security/authorization/go.mod
 create mode 100644 security/authorization/go.sum

diff --git a/Makefile b/Makefile
index bd892e592490..cf474ae2fb97 100644
--- a/Makefile
+++ b/Makefile
@@ -21,6 +21,7 @@ test: testdeps
 
 testsubmodule: testdeps
 	cd security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
+	cd security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/...
 
 testdeps:
 	go get -d -v -t google.golang.org/grpc/...
diff --git a/security/authorization/engine/engine.go b/security/authorization/engine/engine.go
new file mode 100644
index 000000000000..e964c89b6821
--- /dev/null
+++ b/security/authorization/engine/engine.go
@@ -0,0 +1,374 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package engine
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+
+	pb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2"
+	cel "github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
+	types "github.com/google/cel-go/common/types"
+	interpreter "github.com/google/cel-go/interpreter"
+	expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/protobuf/proto"
+)
+
+var logger = grpclog.Component("channelz")
+
+var stringAttributeMap = map[string]func(*AuthorizationArgs) (string, error){
+	"request.url_path":                    (*AuthorizationArgs).getRequestURLPath,
+	"request.host":                        (*AuthorizationArgs).getRequestHost,
+	"request.method":                      (*AuthorizationArgs).getRequestMethod,
+	"source.address":                      (*AuthorizationArgs).getSourceAddress,
+	"destination.address":                 (*AuthorizationArgs).getDestinationAddress,
+	"connection.uri_san_peer_certificate": (*AuthorizationArgs).getURISanPeerCertificate,
+	"source.principal":                    (*AuthorizationArgs).getSourcePrincipal,
+}
+
+var intAttributeMap = map[string]func(*AuthorizationArgs) (int, error){
+	"source.port":      (*AuthorizationArgs).getSourcePort,
+	"destination.port": (*AuthorizationArgs).getDestinationPort,
+}
+
+// activationImpl is an implementation of interpreter.Activation.
+// An Activation is the primary mechanism by which a caller supplies input into a CEL program.
+type activationImpl struct {
+	dict map[string]interface{}
+}
+
+// ResolveName returns a value from the activation by qualified name, or false if the name
+// could not be found.
+func (activation activationImpl) ResolveName(name string) (interface{}, bool) {
+	result, ok := activation.dict[name]
+	return result, ok
+}
+
+// Parent returns the parent of the current activation, may be nil.
+// If non-nil, the parent will be searched during resolve calls.
+func (activation activationImpl) Parent() interpreter.Activation {
+	return activationImpl{}
+}
+
+// AuthorizationArgs is the input of the CEL-based authorization engine.
+type AuthorizationArgs struct {
+	md         metadata.MD
+	peerInfo   *peer.Peer
+	fullMethod string
+}
+
+// newActivation converts AuthorizationArgs into the activation for CEL.
+func newActivation(args *AuthorizationArgs) interpreter.Activation {
+	// Fill out evaluation map, only adding the attributes that can be extracted.
+	evalMap := make(map[string]interface{})
+	for key, function := range stringAttributeMap {
+		val, err := function(args)
+		if err == nil {
+			evalMap[key] = val
+		}
+	}
+	for key, function := range intAttributeMap {
+		val, err := function(args)
+		if err == nil {
+			evalMap[key] = val
+		}
+	}
+	val, err := args.getRequestHeaders()
+	if err == nil {
+		evalMap["request.headers"] = val
+	}
+	// Convert evaluation map to activation.
+	return activationImpl{dict: evalMap}
+}
+
+func (args *AuthorizationArgs) getRequestURLPath() (string, error) {
+	if args.fullMethod == "" {
+		return "", fmt.Errorf("authorization args doesn't have a valid request url path")
+	}
+	return args.fullMethod, nil
+}
+
+func (args *AuthorizationArgs) getRequestHost() (string, error) {
+	// TODO(@zhenlian): fill out attribute extraction for request.host
+	return "", fmt.Errorf("authorization args doesn't have a valid request host")
+}
+
+func (args *AuthorizationArgs) getRequestMethod() (string, error) {
+	// TODO(@zhenlian): fill out attribute extraction for request.method
+	return "", fmt.Errorf("authorization args doesn't have a valid request method")
+}
+
+func (args *AuthorizationArgs) getRequestHeaders() (map[string]string, error) {
+	// TODO(@zhenlian): fill out attribute extraction for request.headers
+	return nil, fmt.Errorf("authorization args doesn't have valid request headers")
+}
+
+func (args *AuthorizationArgs) getSourceAddress() (string, error) {
+	if args.peerInfo == nil {
+		return "", fmt.Errorf("authorization args doesn't have a valid source address")
+	}
+	addr := args.peerInfo.Addr.String()
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return "", err
+	}
+	return host, nil
+}
+
+func (args *AuthorizationArgs) getSourcePort() (int, error) {
+	if args.peerInfo == nil {
+		return 0, fmt.Errorf("authorization args doesn't have a valid source port")
+	}
+	addr := args.peerInfo.Addr.String()
+	_, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(port)
+}
+
+func (args *AuthorizationArgs) getDestinationAddress() (string, error) {
+	// TODO(@zhenlian): fill out attribute extraction for destination.address
+	return "", fmt.Errorf("authorization args doesn't have a valid destination address")
+}
+
+func (args *AuthorizationArgs) getDestinationPort() (int, error) {
+	// TODO(@zhenlian): fill out attribute extraction for destination.port
+	return 0, fmt.Errorf("authorization args doesn't have a valid destination port")
+}
+
+func (args *AuthorizationArgs) getURISanPeerCertificate() (string, error) {
+	// TODO(@zhenlian): fill out attribute extraction for connection.uri_san_peer_certificate
+	return "", fmt.Errorf("authorization args doesn't have a valid URI in SAN field of the peer certificate")
+}
+
+func (args *AuthorizationArgs) getSourcePrincipal() (string, error) {
+	// TODO(@zhenlian): fill out attribute extraction for source.principal
+	return "", fmt.Errorf("authorization args doesn't have a valid source principal")
+}
+
+// Decision represents different authorization decisions a CEL-based
+// authorization engine can return.
+type Decision int32
+
+const (
+	// DecisionAllow indicates allowing the RPC to go through.
+	DecisionAllow Decision = iota
+	// DecisionDeny indicates denying the RPC from going through.
+	DecisionDeny
+	// DecisionUnknown indicates that there is insufficient information to
+	// determine whether or not an RPC call is authorized.
+	DecisionUnknown
+)
+
+// String returns the string representation of a Decision object.
+func (d Decision) String() string {
+	return [...]string{"DecisionAllow", "DecisionDeny", "DecisionUnknown"}[d]
+}
+
+// AuthorizationDecision is the output of CEL-based authorization engines.
+// If decision is allow or deny, policyNames will either contain the names of
+// all the policies matched in the engine that permitted the action, or be
+// empty as the decision was made after all conditions evaluated to false.
+// If decision is unknown, policyNames will contain the list of policies that
+// evaluated to unknown.
+type AuthorizationDecision struct {
+	decision    Decision
+	policyNames []string
+}
+
+// Converts an expression to a parsed expression, with SourceInfo nil.
+func exprToParsedExpr(condition *expr.Expr) *expr.ParsedExpr {
+	return &expr.ParsedExpr{Expr: condition}
+}
+
+// Converts an expression to a CEL program.
+func exprToProgram(condition *expr.Expr, env *cel.Env) (cel.Program, error) {
+	// Converts condition to ParsedExpr by setting SourceInfo empty.
+	pexpr := exprToParsedExpr(condition)
+	// pretend cel.ExprToAst exists
+	ast, iss := env.Check(cel.ParsedExprToAst(pexpr))
+	if iss.Err() != nil {
+		return nil, iss.Err()
+	}
+	// Check that the expression will evaluate to a boolean.
+	if !proto.Equal(ast.ResultType(), decls.Bool) {
+		return nil, fmt.Errorf("expected boolean condition")
+	}
+	// Build the program plan.
+	return env.Program(ast,
+		cel.EvalOptions(cel.OptOptimize),
+	)
+}
+
+// policyEngine is the struct for an engine created from one RBAC proto.
+type policyEngine struct {
+	action   pb.RBAC_Action
+	programs map[string]cel.Program
+}
+
+// Creates a new policyEngine from an RBAC policy proto.
+func newPolicyEngine(rbac *pb.RBAC, env *cel.Env) (*policyEngine, error) {
+	if rbac == nil {
+		return nil, nil
+	}
+	action := rbac.Action
+	programs := make(map[string]cel.Program)
+	for policyName, policy := range rbac.Policies {
+		prg, err := exprToProgram(policy.Condition, env)
+		if err != nil {
+			return &policyEngine{}, fmt.Errorf("failed to create CEL program from condition: %v", err)
+		}
+		programs[policyName] = prg
+	}
+	return &policyEngine{action, programs}, nil
+}
+
+// Returns the decision of an engine based on whether or not AuthorizationArgs is a match,
+// i.e. if engine's action is ALLOW and match is true, we will return DecisionAllow;
+// if engine's action is ALLOW and match is false, we will return DecisionDeny.
+func getDecision(engine *policyEngine, match bool) Decision {
+	if engine.action == pb.RBAC_ALLOW && match || engine.action == pb.RBAC_DENY && !match {
+		return DecisionAllow
+	}
+	return DecisionDeny
+}
+
+// Returns the authorization decision of a single policy engine based on activation.
+// If any policy matches, the decision matches the engine's action, and the first
+//  matching policy name will be returned.
+// Else if any policy is missing attributes, the decision is unknown, and the list of
+//  policy names that can't be evaluated due to missing attributes will be returned.
+// Else, the decision is the opposite of the engine's action, i.e. an ALLOW engine
+//  will return DecisionDeny, and vice versa.
+func (engine *policyEngine) evaluate(activation interpreter.Activation) (Decision, []string) {
+	unknownPolicyNames := []string{}
+	for policyName, program := range engine.programs {
+		// Evaluate program against activation.
+		var match bool
+		out, _, err := program.Eval(activation)
+		if err != nil {
+			if out == nil {
+				// Unsuccessful evaluation, typically the result of a series of incompatible
+				// `EnvOption` or `ProgramOption` values used in the creation of the evaluation
+				// environment or executable program.
+				logger.Warning("Unsuccessful evaluation encountered during AuthorizationEngine.Evaluate: %s", err.Error())
+			}
+			// Unsuccessful evaluation or successful evaluation to an error result, i.e. missing attributes.
+			match = false
+		} else {
+			// Successful evaluation to a non-error result.
+			if !types.IsBool(out) {
+				logger.Warning("'Successful evaluation', but output isn't a boolean: %v", out)
+				match = false
+			} else {
+				match = out.Value().(bool)
+			}
+		}
+
+		// Process evaluation results.
+		if err != nil {
+			unknownPolicyNames = append(unknownPolicyNames, policyName)
+		} else if match {
+			return getDecision(engine, true), []string{policyName}
+		}
+	}
+	if len(unknownPolicyNames) > 0 {
+		return DecisionUnknown, unknownPolicyNames
+	}
+	return getDecision(engine, false), []string{}
+}
+
+// AuthorizationEngine is the struct for the CEL-based authorization engine.
+type AuthorizationEngine struct {
+	allow *policyEngine
+	deny  *policyEngine
+}
+
+// NewAuthorizationEngine builds a CEL evaluation engine from at most one allow and one deny Envoy RBAC.
+func NewAuthorizationEngine(allow, deny *pb.RBAC) (*AuthorizationEngine, error) {
+	if allow == nil && deny == nil {
+		return &AuthorizationEngine{}, fmt.Errorf("at least one of allow, deny must be non-nil")
+	}
+	if allow != nil && allow.Action != pb.RBAC_ALLOW || deny != nil && deny.Action != pb.RBAC_DENY {
+		return nil, fmt.Errorf("allow must have action ALLOW, deny must have action DENY")
+	}
+	// Note: env can be shared across multiple Checks / Program constructions.
+	env, err := cel.NewEnv(
+		cel.Declarations(
+			decls.NewVar("request.url_path", decls.String),
+			decls.NewVar("request.host", decls.String),
+			decls.NewVar("request.method", decls.String),
+			decls.NewVar("request.headers", decls.NewMapType(decls.String, decls.String)),
+			decls.NewVar("source.address", decls.String),
+			decls.NewVar("source.port", decls.Int),
+			decls.NewVar("destination.address", decls.String),
+			decls.NewVar("destination.port", decls.Int),
+			decls.NewVar("connection.uri_san_peer_certificate", decls.String),
+		),
+	)
+	if err != nil {
+		return &AuthorizationEngine{}, fmt.Errorf("failed to create CEL Env: %v", err)
+	}
+	// create policy engines
+	allowEngine, err := newPolicyEngine(allow, env)
+	if err != nil {
+		return &AuthorizationEngine{}, err
+	}
+	denyEngine, err := newPolicyEngine(deny, env)
+	if err != nil {
+		return &AuthorizationEngine{}, err
+	}
+	return &AuthorizationEngine{allow: allowEngine, deny: denyEngine}, nil
+}
+
+// Evaluate is the core function that evaluates whether an RPC is authorized.
+//
+// ALLOW policy. If one of the RBAC conditions is evaluated as true, then the
+// CEL-based authorization engine evaluation returns allow. If all of the RBAC
+// conditions are evaluated as false, then it returns deny. Otherwise, some
+// conditions are false and some are unknown, it returns undecided.
+//
+// DENY policy. If one of the RBAC conditions is evaluated as true, then the
+// CEL-based authorization engine evaluation returns deny. If all of the RBAC
+// conditions are evaluated as false, then it returns allow. Otherwise, some
+// conditions are false and some are unknown, it returns undecided.
+//
+// DENY policy + ALLOW policy. Evaluation is in the following order: If one
+// of the expressions in the DENY policy is true, the authorization engine
+// returns deny. If one of the expressions in the DENY policy is unknown, it
+// returns undecided. Now all the expressions in the DENY policy are false,
+// it returns the evaluation of the ALLOW policy.
+func (authorizationEngine *AuthorizationEngine) Evaluate(args *AuthorizationArgs) (AuthorizationDecision, error) {
+	activation := newActivation(args)
+	decision := DecisionAllow
+	var policyNames []string
+	// Evaluate the deny engine, if it exists.
+	if authorizationEngine.deny != nil {
+		decision, policyNames = authorizationEngine.deny.evaluate(activation)
+	}
+	// Evaluate the allow engine, if it exists and if the deny engine doesn't exist or is unmatched.
+	if authorizationEngine.allow != nil && decision == DecisionAllow {
+		decision, policyNames = authorizationEngine.allow.evaluate(activation)
+	}
+	return AuthorizationDecision{decision, policyNames}, nil
+}
diff --git a/security/authorization/engine/engine_test.go b/security/authorization/engine/engine_test.go
new file mode 100644
index 000000000000..a87a7f53d858
--- /dev/null
+++ b/security/authorization/engine/engine_test.go
@@ -0,0 +1,272 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package engine
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+
+	pb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2"
+	cel "github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	interpreter "github.com/google/cel-go/interpreter"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type programMock struct {
+	out ref.Val
+	err error
+}
+
+func (mock programMock) Eval(vars interface{}) (ref.Val, *cel.EvalDetails, error) {
+	return mock.out, nil, mock.err
+}
+
+type valMock struct {
+	val interface{}
+}
+
+func (mock valMock) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
+	return nil, nil
+}
+
+func (mock valMock) ConvertToType(typeValue ref.Type) ref.Val {
+	return nil
+}
+
+func (mock valMock) Equal(other ref.Val) ref.Val {
+	return nil
+}
+
+func (mock valMock) Type() ref.Type {
+	if mock.val == true || mock.val == false {
+		return types.BoolType
+	}
+	return nil
+}
+
+func (mock valMock) Value() interface{} {
+	return mock.val
+}
+
+var (
+	emptyActivation     = interpreter.EmptyActivation()
+	unsuccessfulProgram = programMock{out: nil, err: status.Errorf(codes.InvalidArgument, "Unsuccessful program evaluation")}
+	errProgram          = programMock{out: valMock{"missing attributes"}, err: status.Errorf(codes.InvalidArgument, "Successful program evaluation to an error result -- missing attributes")}
+	trueProgram         = programMock{out: valMock{true}, err: nil}
+	falseProgram        = programMock{out: valMock{false}, err: nil}
+
+	allowMatchEngine = &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{
+		"allow match policy1": unsuccessfulProgram,
+		"allow match policy2": trueProgram,
+		"allow match policy3": falseProgram,
+		"allow match policy4": errProgram,
+	}}
+	denyFailEngine = &policyEngine{action: pb.RBAC_DENY, programs: map[string]cel.Program{
+		"deny fail policy1": falseProgram,
+		"deny fail policy2": falseProgram,
+		"deny fail policy3": falseProgram,
+	}}
+	denyUnknownEngine = &policyEngine{action: pb.RBAC_DENY, programs: map[string]cel.Program{
+		"deny unknown policy1": falseProgram,
+		"deny unknown policy2": unsuccessfulProgram,
+		"deny unknown policy3": errProgram,
+		"deny unknown policy4": falseProgram,
+	}}
+)
+
+func TestNewAuthorizationEngine(t *testing.T) {
+	tests := map[string]struct {
+		allow   *pb.RBAC
+		deny    *pb.RBAC
+		wantErr string
+		errStr  string
+	}{
+		"too few rbacs": {
+			allow:   nil,
+			deny:    nil,
+			wantErr: "at least one of allow, deny must be non-nil",
+			errStr:  "Expected error: at least one of allow, deny must be non-nil",
+		},
+		"one rbac allow": {
+			allow:   &pb.RBAC{Action: pb.RBAC_ALLOW},
+			deny:    nil,
+			wantErr: "",
+			errStr:  "Expected 1 ALLOW RBAC to be successful",
+		},
+		"one rbac deny": {
+			allow:   nil,
+			deny:    &pb.RBAC{Action: pb.RBAC_DENY},
+			wantErr: "",
+			errStr:  "Expected 1 DENY RBAC to be successful",
+		},
+		"two rbacs": {
+			allow:   &pb.RBAC{Action: pb.RBAC_ALLOW},
+			deny:    &pb.RBAC{Action: pb.RBAC_DENY},
+			wantErr: "",
+			errStr:  "Expected 2 RBACs (DENY + ALLOW) to be successful",
+		},
+		"wrong rbac actions": {
+			allow:   &pb.RBAC{Action: pb.RBAC_DENY},
+			deny:    nil,
+			wantErr: "allow must have action ALLOW, deny must have action DENY",
+			errStr:  "Expected error: allow must have action ALLOW, deny must have action DENY",
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			_, gotErr := NewAuthorizationEngine(tc.allow, tc.deny)
+			if tc.wantErr == "" && gotErr == nil {
+				return
+			}
+			if gotErr == nil || gotErr.Error() != tc.wantErr {
+				t.Errorf(tc.errStr)
+			}
+		})
+	}
+}
+
+func TestGetDecision(t *testing.T) {
+	tests := map[string]struct {
+		engine *policyEngine
+		match  bool
+		want   Decision
+	}{
+		"ALLOW engine match": {
+			engine: &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{}},
+			match:  true,
+			want:   DecisionAllow,
+		},
+		"ALLOW engine fail": {
+			engine: &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{}},
+			match:  false,
+			want:   DecisionDeny,
+		},
+		"DENY engine match": {
+			engine: &policyEngine{action: pb.RBAC_DENY, programs: map[string]cel.Program{}},
+			match:  true,
+			want:   DecisionDeny,
+		},
+		"DENY engine fail": {
+			engine: &policyEngine{action: pb.RBAC_DENY, programs: map[string]cel.Program{}},
+			match:  false,
+			want:   DecisionAllow,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := getDecision(tc.engine, tc.match); got != tc.want {
+				t.Errorf("Expected %v, instead got %v", tc.want, got)
+			}
+		})
+	}
+}
+
+func TestPolicyEngineEvaluate(t *testing.T) {
+	tests := map[string]struct {
+		engine          *policyEngine
+		activation      interpreter.Activation
+		wantDecision    Decision
+		wantPolicyNames []string
+	}{
+		"no policies": {
+			engine:          &policyEngine{},
+			activation:      emptyActivation,
+			wantDecision:    DecisionDeny,
+			wantPolicyNames: []string{},
+		},
+		"match succeed": {
+			engine:          allowMatchEngine,
+			activation:      emptyActivation,
+			wantDecision:    DecisionAllow,
+			wantPolicyNames: []string{"allow match policy2"},
+		},
+		"match fail": {
+			engine:          denyFailEngine,
+			activation:      emptyActivation,
+			wantDecision:    DecisionAllow,
+			wantPolicyNames: []string{},
+		},
+		"unknown": {
+			engine:          denyUnknownEngine,
+			activation:      emptyActivation,
+			wantDecision:    DecisionUnknown,
+			wantPolicyNames: []string{"deny unknown policy2", "deny unknown policy3"},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			gotDecision, gotPolicyNames := tc.engine.evaluate(tc.activation)
+			sort.Strings(gotPolicyNames)
+			if gotDecision != tc.wantDecision || !reflect.DeepEqual(gotPolicyNames, tc.wantPolicyNames) {
+				t.Errorf("Expected (%v, %v), instead got (%v, %v)", tc.wantDecision, tc.wantPolicyNames, gotDecision, gotPolicyNames)
+			}
+		})
+	}
+}
+
+func TestAuthorizationEngineEvaluate(t *testing.T) {
+	tests := map[string]struct {
+		engine           *AuthorizationEngine
+		authArgs         *AuthorizationArgs
+		wantAuthDecision *AuthorizationDecision
+		wantErr          error
+	}{
+		"allow match": {
+			engine:           &AuthorizationEngine{allow: allowMatchEngine},
+			authArgs:         &AuthorizationArgs{},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"allow match policy2"}},
+			wantErr:          nil,
+		},
+		"deny fail": {
+			engine:           &AuthorizationEngine{deny: denyFailEngine},
+			authArgs:         &AuthorizationArgs{},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{}},
+			wantErr:          nil,
+		},
+		"first engine unknown": {
+			engine:           &AuthorizationEngine{allow: allowMatchEngine, deny: denyUnknownEngine},
+			authArgs:         &AuthorizationArgs{},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"deny unknown policy2", "deny unknown policy3"}},
+			wantErr:          nil,
+		},
+		"second engine match": {
+			engine:           &AuthorizationEngine{allow: allowMatchEngine, deny: denyFailEngine},
+			authArgs:         &AuthorizationArgs{},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"allow match policy2"}},
+			wantErr:          nil,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			gotAuthDecision, gotErr := tc.engine.Evaluate(tc.authArgs)
+			sort.Strings(gotAuthDecision.policyNames)
+			if tc.wantErr != nil && (gotErr == nil || gotErr.Error() != tc.wantErr.Error()) {
+				t.Errorf("Expected error to be %v, instead got %v", tc.wantErr, gotErr)
+			} else if tc.wantErr == nil && (gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !reflect.DeepEqual(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames)) {
+				t.Errorf("Expected authorization decision to be (%v, %v), instead got (%v, %v)", tc.wantAuthDecision.decision, tc.wantAuthDecision.policyNames, gotAuthDecision.decision, gotAuthDecision.policyNames)
+			}
+		})
+	}
+}
diff --git a/security/authorization/go.mod b/security/authorization/go.mod
new file mode 100644
index 000000000000..075cbe9158c0
--- /dev/null
+++ b/security/authorization/go.mod
@@ -0,0 +1,11 @@
+module google.golang.org/grpc/security/authorization
+
+go 1.12
+
+require (
+	github.com/envoyproxy/go-control-plane v0.9.5
+	github.com/google/cel-go v0.5.1
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
+	google.golang.org/grpc v1.31.0
+	google.golang.org/protobuf v1.25.0
+)
diff --git a/security/authorization/go.sum b/security/authorization/go.sum
new file mode 100644
index 000000000000..a953711e01e6
--- /dev/null
+++ b/security/authorization/go.sum
@@ -0,0 +1,104 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f h1:0cEys61Sr2hUBEXfNV8eyQP01oZuBgoMeHunebPirK8=
+github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533 h1:8wZizuKuZVu5COB7EsBYxBQz8nRcXXn5d4Gt91eJLvU=
+github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.5 h1:lRJIqDD8yjV1YyPRqecMdytjDLs2fTXq363aCib5xPU=
+github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s=
+github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/google/cel-go v0.5.1 h1:oDsbtAwlwFPEcC8dMoRWNuVzWJUDeDZeHjoet9rXjTs=
+github.com/google/cel-go v0.5.1/go.mod h1:9SvtVVTtZV4DTB1/RuAD1D2HhuqEIdmZEE/r/lrFyKE=
+github.com/google/cel-spec v0.4.0/go.mod h1:2pBM5cU4UKjbPDXBgwWkiwBsVgnxknuEJ7C5TDWwORQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171 h1:xes2Q2k+d/+YNXVw0FpZkIDJiaux4OVrRKXRAzH6A0U=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From 244f9ece7c3fbd0cde99b50552cd8aa9c0ad1c86 Mon Sep 17 00:00:00 2001
From: cindyxue <32377977+cindyxue@users.noreply.github.com>
Date: Mon, 10 Aug 2020 11:52:32 -0700
Subject: [PATCH 153/481] credentials: Migrate helper functions to
 /internal/credentials for future cleanup (#3780)

* credentials: migrate functions in credentials/internal to internal/credentials to be used for other modules
---
 credentials/credentials_test.go               | 37 ------------
 credentials/tls.go                            | 39 ++----------
 .../credentials}/syscallconn.go               |  3 +-
 .../credentials}/syscallconn_test.go          | 21 +------
 internal/credentials/util.go                  | 50 ++++++++++++++++
 internal/credentials/util_test.go             | 60 +++++++++++++++++++
 6 files changed, 119 insertions(+), 91 deletions(-)
 rename {credentials/internal => internal/credentials}/syscallconn.go (96%)
 rename {credentials/internal => internal/credentials}/syscallconn_test.go (79%)
 create mode 100644 internal/credentials/util.go
 create mode 100644 internal/credentials/util_test.go

diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go
index e36145250695..ea0cf5819ff8 100644
--- a/credentials/credentials_test.go
+++ b/credentials/credentials_test.go
@@ -22,7 +22,6 @@ import (
 	"context"
 	"crypto/tls"
 	"net"
-	"reflect"
 	"strings"
 	"testing"
 
@@ -326,39 +325,3 @@ func tlsClientHandshake(conn net.Conn, _ string) (AuthInfo, error) {
 	}
 	return TLSInfo{State: clientConn.ConnectionState(), CommonAuthInfo: CommonAuthInfo{SecurityLevel: PrivacyAndIntegrity}}, nil
 }
-
-func (s) TestAppendH2ToNextProtos(t *testing.T) {
-	tests := []struct {
-		name string
-		ps   []string
-		want []string
-	}{
-		{
-			name: "empty",
-			ps:   nil,
-			want: []string{"h2"},
-		},
-		{
-			name: "only h2",
-			ps:   []string{"h2"},
-			want: []string{"h2"},
-		},
-		{
-			name: "with h2",
-			ps:   []string{"alpn", "h2"},
-			want: []string{"alpn", "h2"},
-		},
-		{
-			name: "no h2",
-			ps:   []string{"alpn"},
-			want: []string{"alpn", "h2"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := appendH2ToNextProtos(tt.ps); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("appendH2ToNextProtos() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
diff --git a/credentials/tls.go b/credentials/tls.go
index 1ba6f3a6b8f8..48384f5050e7 100644
--- a/credentials/tls.go
+++ b/credentials/tls.go
@@ -27,7 +27,6 @@ import (
 	"net"
 	"net/url"
 
-	"google.golang.org/grpc/credentials/internal"
 	credinternal "google.golang.org/grpc/internal/credentials"
 )
 
@@ -73,7 +72,7 @@ func (c tlsCreds) Info() ProtocolInfo {
 
 func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) {
 	// use local cfg to avoid clobbering ServerName if using multiple endpoints
-	cfg := cloneTLSConfig(c.config)
+	cfg := credinternal.CloneTLSConfig(c.config)
 	if cfg.ServerName == "" {
 		serverName, _, err := net.SplitHostPort(authority)
 		if err != nil {
@@ -108,7 +107,7 @@ func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawCon
 	if id != nil {
 		tlsInfo.SPIFFEID = id
 	}
-	return internal.WrapSyscallConn(rawConn, conn), tlsInfo, nil
+	return credinternal.WrapSyscallConn(rawConn, conn), tlsInfo, nil
 }
 
 func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) {
@@ -127,7 +126,7 @@ func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error)
 	if id != nil {
 		tlsInfo.SPIFFEID = id
 	}
-	return internal.WrapSyscallConn(rawConn, conn), tlsInfo, nil
+	return credinternal.WrapSyscallConn(rawConn, conn), tlsInfo, nil
 }
 
 func (c *tlsCreds) Clone() TransportCredentials {
@@ -139,23 +138,10 @@ func (c *tlsCreds) OverrideServerName(serverNameOverride string) error {
 	return nil
 }
 
-const alpnProtoStrH2 = "h2"
-
-func appendH2ToNextProtos(ps []string) []string {
-	for _, p := range ps {
-		if p == alpnProtoStrH2 {
-			return ps
-		}
-	}
-	ret := make([]string, 0, len(ps)+1)
-	ret = append(ret, ps...)
-	return append(ret, alpnProtoStrH2)
-}
-
 // NewTLS uses c to construct a TransportCredentials based on TLS.
 func NewTLS(c *tls.Config) TransportCredentials {
-	tc := &tlsCreds{cloneTLSConfig(c)}
-	tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos)
+	tc := &tlsCreds{credinternal.CloneTLSConfig(c)}
+	tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos)
 	return tc
 }
 
@@ -242,18 +228,3 @@ var cipherSuiteLookup = map[uint16]string{
 	tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305:    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
 	tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305:  "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
 }
-
-// cloneTLSConfig returns a shallow clone of the exported
-// fields of cfg, ignoring the unexported sync.Once, which
-// contains a mutex and must not be copied.
-//
-// If cfg is nil, a new zero tls.Config is returned.
-//
-// TODO: inline this function if possible.
-func cloneTLSConfig(cfg *tls.Config) *tls.Config {
-	if cfg == nil {
-		return &tls.Config{}
-	}
-
-	return cfg.Clone()
-}
diff --git a/credentials/internal/syscallconn.go b/internal/credentials/syscallconn.go
similarity index 96%
rename from credentials/internal/syscallconn.go
rename to internal/credentials/syscallconn.go
index 264c7298085a..2919632d657e 100644
--- a/credentials/internal/syscallconn.go
+++ b/internal/credentials/syscallconn.go
@@ -16,8 +16,7 @@
  *
  */
 
-// Package internal contains credentials-internal code.
-package internal
+package credentials
 
 import (
 	"net"
diff --git a/credentials/internal/syscallconn_test.go b/internal/credentials/syscallconn_test.go
similarity index 79%
rename from credentials/internal/syscallconn_test.go
rename to internal/credentials/syscallconn_test.go
index 6700e7546853..b229a47d116e 100644
--- a/credentials/internal/syscallconn_test.go
+++ b/internal/credentials/syscallconn_test.go
@@ -16,29 +16,14 @@
  *
  */
 
-package internal_test
+package credentials
 
 import (
 	"net"
 	"syscall"
 	"testing"
-
-	"google.golang.org/grpc/credentials/internal"
-	"google.golang.org/grpc/internal/grpctest"
 )
 
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
-type syscallConn struct {
-	net.Conn
-}
-
 func (*syscallConn) SyscallConn() (syscall.RawConn, error) {
 	return nil, nil
 }
@@ -51,7 +36,7 @@ func (s) TestWrapSyscallConn(t *testing.T) {
 	sc := &syscallConn{}
 	nsc := &nonSyscallConn{}
 
-	wrapConn := internal.WrapSyscallConn(sc, nsc)
+	wrapConn := WrapSyscallConn(sc, nsc)
 	if _, ok := wrapConn.(syscall.Conn); !ok {
 		t.Errorf("returned conn (type %T) doesn't implement syscall.Conn, want implement", wrapConn)
 	}
@@ -61,7 +46,7 @@ func (s) TestWrapSyscallConnNoWrap(t *testing.T) {
 	nscRaw := &nonSyscallConn{}
 	nsc := &nonSyscallConn{}
 
-	wrapConn := internal.WrapSyscallConn(nscRaw, nsc)
+	wrapConn := WrapSyscallConn(nscRaw, nsc)
 	if _, ok := wrapConn.(syscall.Conn); ok {
 		t.Errorf("returned conn (type %T) implements syscall.Conn, want not implement", wrapConn)
 	}
diff --git a/internal/credentials/util.go b/internal/credentials/util.go
new file mode 100644
index 000000000000..55664fa46b81
--- /dev/null
+++ b/internal/credentials/util.go
@@ -0,0 +1,50 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import "crypto/tls"
+
+const alpnProtoStrH2 = "h2"
+
+// AppendH2ToNextProtos appends h2 to next protos.
+func AppendH2ToNextProtos(ps []string) []string {
+	for _, p := range ps {
+		if p == alpnProtoStrH2 {
+			return ps
+		}
+	}
+	ret := make([]string, 0, len(ps)+1)
+	ret = append(ret, ps...)
+	return append(ret, alpnProtoStrH2)
+}
+
+// CloneTLSConfig returns a shallow clone of the exported
+// fields of cfg, ignoring the unexported sync.Once, which
+// contains a mutex and must not be copied.
+//
+// If cfg is nil, a new zero tls.Config is returned.
+//
+// TODO: inline this function if possible.
+func CloneTLSConfig(cfg *tls.Config) *tls.Config {
+	if cfg == nil {
+		return &tls.Config{}
+	}
+
+	return cfg.Clone()
+}
diff --git a/internal/credentials/util_test.go b/internal/credentials/util_test.go
new file mode 100644
index 000000000000..69e01eeefe93
--- /dev/null
+++ b/internal/credentials/util_test.go
@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import (
+	"reflect"
+	"testing"
+)
+
+func (s) TestAppendH2ToNextProtos(t *testing.T) {
+	tests := []struct {
+		name string
+		ps   []string
+		want []string
+	}{
+		{
+			name: "empty",
+			ps:   nil,
+			want: []string{"h2"},
+		},
+		{
+			name: "only h2",
+			ps:   []string{"h2"},
+			want: []string{"h2"},
+		},
+		{
+			name: "with h2",
+			ps:   []string{"alpn", "h2"},
+			want: []string{"alpn", "h2"},
+		},
+		{
+			name: "no h2",
+			ps:   []string{"alpn"},
+			want: []string{"alpn", "h2"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := AppendH2ToNextProtos(tt.ps); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("AppendH2ToNextProtos() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

From 2983360ff4e746cf3bee8603ff71a8dad861b3b9 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 10 Aug 2020 15:53:34 -0700
Subject: [PATCH 154/481] Fix interop tests which rely on the old certs/keys.
 (#3804)

---
 interop/client/client.go           |  2 +-
 interop/fake_grpclb/fake_grpclb.go |  4 ++--
 interop/interop_test.sh            |  2 +-
 interop/server/server.go           |  4 ++--
 testdata/README.md                 |  3 +++
 testdata/ca.pem                    | 20 ++++++++++++++++++++
 testdata/server1.key               | 28 ++++++++++++++++++++++++++++
 testdata/server1.pem               | 22 ++++++++++++++++++++++
 8 files changed, 79 insertions(+), 6 deletions(-)
 create mode 100644 testdata/README.md
 create mode 100644 testdata/ca.pem
 create mode 100644 testdata/server1.key
 create mode 100644 testdata/server1.pem

diff --git a/interop/client/client.go b/interop/client/client.go
index 0c05c19a5e9f..2d7823062309 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -137,7 +137,7 @@ func main() {
 		if *testCA {
 			var err error
 			if *caFile == "" {
-				*caFile = testdata.Path("x509/server_ca_cert.pem")
+				*caFile = testdata.Path("ca.pem")
 			}
 			creds, err = credentials.NewClientTLSFromFile(*caFile, sn)
 			if err != nil {
diff --git a/interop/fake_grpclb/fake_grpclb.go b/interop/fake_grpclb/fake_grpclb.go
index 6746462b107a..6804235486ba 100644
--- a/interop/fake_grpclb/fake_grpclb.go
+++ b/interop/fake_grpclb/fake_grpclb.go
@@ -114,8 +114,8 @@ func main() {
 	flag.Parse()
 	var opts []grpc.ServerOption
 	if *useTLS {
-		certFile := testdata.Path("x509/server1_cert.pem")
-		keyFile := testdata.Path("x509/server1_key.pem")
+		certFile := testdata.Path("server1.pem")
+		keyFile := testdata.Path("server1.key")
 		creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
 		if err != nil {
 			logger.Fatalf("Failed to generate credentials %v", err)
diff --git a/interop/interop_test.sh b/interop/interop_test.sh
index f4103c44666b..5aeaa2aa10a8 100755
--- a/interop/interop_test.sh
+++ b/interop/interop_test.sh
@@ -87,7 +87,7 @@ for case in ${CASES[@]}; do
     echo "$(tput setaf 4) testing: ${case} $(tput sgr 0)"
 
     CLIENT_LOG="$(mktemp)"
-    if ! timeout 20 go run ./interop/client --use_tls --server_host_override=x.test.example.com --use_test_ca --test_case="${case}" &> $CLIENT_LOG; then
+    if ! timeout 20 go run ./interop/client --use_tls --server_host_override=foo.test.google.fr --use_test_ca --test_case="${case}" &> $CLIENT_LOG; then  
         fail "FAIL: test case ${case}
         got server log:
         $(cat $SERVER_LOG)
diff --git a/interop/server/server.go b/interop/server/server.go
index 52cfb2f4c458..c70e450bb108 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -57,10 +57,10 @@ func main() {
 	var opts []grpc.ServerOption
 	if *useTLS {
 		if *certFile == "" {
-			*certFile = testdata.Path("x509/server1_cert.pem")
+			*certFile = testdata.Path("server1.pem")
 		}
 		if *keyFile == "" {
-			*keyFile = testdata.Path("x509/server1_key.pem")
+			*keyFile = testdata.Path("server1.key")
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {
diff --git a/testdata/README.md b/testdata/README.md
new file mode 100644
index 000000000000..c0cb7187098a
--- /dev/null
+++ b/testdata/README.md
@@ -0,0 +1,3 @@
+This directory contains x509 certificates used in cloud-to-prod interop tests.
+For tests within gRPC-Go repo, please use the files in testsdata/x509
+directory.
diff --git a/testdata/ca.pem b/testdata/ca.pem
new file mode 100644
index 000000000000..49d39cd8ed5f
--- /dev/null
+++ b/testdata/ca.pem
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDWjCCAkKgAwIBAgIUWrP0VvHcy+LP6UuYNtiL9gBhD5owDQYJKoZIhvcNAQEL
+BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
+MDMxNzE4NTk1MVoXDTMwMDMxNTE4NTk1MVowVjELMAkGA1UEBhMCQVUxEzARBgNV
+BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
+ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAsGL0oXflF0LzoM+Bh+qUU9yhqzw2w8OOX5mu/iNCyUOBrqaHi7mGHx73GD01
+diNzCzvlcQqdNIH6NQSL7DTpBjca66jYT9u73vZe2MDrr1nVbuLvfu9850cdxiUO
+Inv5xf8+sTHG0C+a+VAvMhsLiRjsq+lXKRJyk5zkbbsETybqpxoJ+K7CoSy3yc/k
+QIY3TipwEtwkKP4hzyo6KiGd/DPexie4nBUInN3bS1BUeNZ5zeaIC2eg3bkeeW7c
+qT55b+Yen6CxY0TEkzBK6AKt/WUialKMgT0wbTxRZO7kUCH3Sq6e/wXeFdJ+HvdV
+LPlAg5TnMaNpRdQih/8nRFpsdwIDAQABoyAwHjAMBgNVHRMEBTADAQH/MA4GA1Ud
+DwEB/wQEAwICBDANBgkqhkiG9w0BAQsFAAOCAQEAkTrKZjBrJXHps/HrjNCFPb5a
+THuGPCSsepe1wkKdSp1h4HGRpLoCgcLysCJ5hZhRpHkRihhef+rFHEe60UePQO3S
+CVTtdJB4CYWpcNyXOdqefrbJW5QNljxgi6Fhvs7JJkBqdXIkWXtFk2eRgOIP2Eo9
+/OHQHlYnwZFrk6sp4wPyR+A95S0toZBcyDVz7u+hOW0pGK3wviOe9lvRgj/H3Pwt
+bewb0l+MhRig0/DVHamyVxrDRbqInU1/GTNCwcZkXKYFWSf92U+kIcTth24Q1gcw
+eZiLl5FfrWokUNytFElXob0V0a5/kbhiLc3yWmvWqHTpqCALbVyF+rKJo2f5Kw==
+-----END CERTIFICATE-----
diff --git a/testdata/server1.key b/testdata/server1.key
new file mode 100644
index 000000000000..086462992cfb
--- /dev/null
+++ b/testdata/server1.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDnE443EknxvxBq
+6+hvn/t09hl8hx366EBYvZmVM/NC+7igXRAjiJiA/mIaCvL3MS0Iz5hBLxSGICU+
+WproA3GCIFITIwcf/ETyWj/5xpgZ4AKrLrjQmmX8mhwUajfF3UvwMJrCOVqPp67t
+PtP+2kBXaqrXdvnvXR41FsIB8V7zIAuIZB6bHQhiGVlc1sgZYsE2EGG9WMmHtS86
+qkAOTjG2XyjmPTGAwhGDpYkYrpzp99IiDh4/Veai81hn0ssQkbry0XRD/Ig3jcHh
+23WiriPNJ0JsbgXUSLKRPZObA9VgOLy2aXoN84IMaeK3yy+cwSYG/99w93fUZJte
+MXwz4oYZAgMBAAECggEBAIVn2Ncai+4xbH0OLWckabwgyJ4IM9rDc0LIU368O1kU
+koais8qP9dujAWgfoh3sGh/YGgKn96VnsZjKHlyMgF+r4TaDJn3k2rlAOWcurGlj
+1qaVlsV4HiEzp7pxiDmHhWvp4672Bb6iBG+bsjCUOEk/n9o9KhZzIBluRhtxCmw5
+nw4Do7z00PTvN81260uPWSc04IrytvZUiAIx/5qxD72bij2xJ8t/I9GI8g4FtoVB
+8pB6S/hJX1PZhh9VlU6Yk+TOfOVnbebG4W5138LkB835eqk3Zz0qsbc2euoi8Hxi
+y1VGwQEmMQ63jXz4c6g+X55ifvUK9Jpn5E8pq+pMd7ECgYEA93lYq+Cr54K4ey5t
+sWMa+ye5RqxjzgXj2Kqr55jb54VWG7wp2iGbg8FMlkQwzTJwebzDyCSatguEZLuB
+gRGroRnsUOy9vBvhKPOch9bfKIl6qOgzMJB267fBVWx5ybnRbWN/I7RvMQf3k+9y
+biCIVnxDLEEYyx7z85/5qxsXg/MCgYEA7wmWKtCTn032Hy9P8OL49T0X6Z8FlkDC
+Rk42ygrc/MUbugq9RGUxcCxoImOG9JXUpEtUe31YDm2j+/nbvrjl6/bP2qWs0V7l
+dTJl6dABP51pCw8+l4cWgBBX08Lkeen812AAFNrjmDCjX6rHjWHLJcpS18fnRRkP
+V1d/AHWX7MMCgYEA6Gsw2guhp0Zf2GCcaNK5DlQab8OL4Hwrpttzo4kuTlwtqNKp
+Q9H4al9qfF4Cr1TFya98+EVYf8yFRM3NLNjZpe3gwYf2EerlJj7VLcahw0KKzoN1
+QBENfwgPLRk5sDkx9VhSmcfl/diLroZdpAwtv3vo4nEoxeuGFbKTGx3Qkf0CgYEA
+xyR+dcb05Ygm3w4klHQTowQ10s1H80iaUcZBgQuR1ghEtDbUPZHsoR5t1xCB02ys
+DgAwLv1bChIvxvH/L6KM8ovZ2LekBX4AviWxoBxJnfz/EVau98B0b1auRN6eSC83
+FRuGldlSOW1z/nSh8ViizSYE5H5HX1qkXEippvFRE88CgYB3Bfu3YQY60ITWIShv
+nNkdcbTT9eoP9suaRJjw92Ln+7ZpALYlQMKUZmJ/5uBmLs4RFwUTQruLOPL4yLTH
+awADWUzs3IRr1fwn9E+zM8JVyKCnUEM3w4N5UZskGO2klashAd30hWO+knRv/y0r
+uGIYs9Ek7YXlXIRVrzMwcsrt1w==
+-----END PRIVATE KEY-----
diff --git a/testdata/server1.pem b/testdata/server1.pem
new file mode 100644
index 000000000000..88244f856c62
--- /dev/null
+++ b/testdata/server1.pem
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDtDCCApygAwIBAgIUbJfTREJ6k6/+oInWhV1O1j3ZT0IwDQYJKoZIhvcNAQEL
+BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
+MDMxODAzMTA0MloXDTMwMDMxNjAzMTA0MlowZTELMAkGA1UEBhMCVVMxETAPBgNV
+BAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxl
+LCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPz
+Qvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caY
+GeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe
+8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c
+6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPV
+YDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo2swaTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIF4DBPBgNVHREESDBGghAqLnRlc3QuZ29vZ2xlLmZy
+ghh3YXRlcnpvb2kudGVzdC5nb29nbGUuYmWCEioudGVzdC55b3V0dWJlLmNvbYcE
+wKgBAzANBgkqhkiG9w0BAQsFAAOCAQEAS8hDQA8PSgipgAml7Q3/djwQ644ghWQv
+C2Kb+r30RCY1EyKNhnQnIIh/OUbBZvh0M0iYsy6xqXgfDhCB93AA6j0i5cS8fkhH
+Jl4RK0tSkGQ3YNY4NzXwQP/vmUgfkw8VBAZ4Y4GKxppdATjffIW+srbAmdDruIRM
+wPeikgOoRrXf0LA1fi4TqxARzeRwenQpayNfGHTvVF9aJkl8HoaMunTAdG5pIVcr
+9GKi/gEMpXUJbbVv3U5frX1Wo4CFo+rZWJ/LyCMeb0jciNLxSdMwj/E/ZuExlyeZ
+gc9ctPjSMvgSyXEKv6Vwobleeg88V2ZgzenziORoWj4KszG/lbQZvg==
+-----END CERTIFICATE-----

From 6aaac03d175ae0dc2eeae3b147d620d57cfd239a Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 11 Aug 2020 06:57:51 -0700
Subject: [PATCH 155/481] internal/serviceconfig: Fix typo and improve docs.
 (#3802)

---
 internal/serviceconfig/serviceconfig.go | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go
index 2b9b403a7a72..af3e2b5f7e8b 100644
--- a/internal/serviceconfig/serviceconfig.go
+++ b/internal/serviceconfig/serviceconfig.go
@@ -30,8 +30,11 @@ import (
 
 var logger = grpclog.Component("core")
 
-// BalancerConfig is the balancer config part that service config's
-// loadBalancingConfig fields can be unmarshalled to. It's a json unmarshaller.
+// BalancerConfig wraps the name and config associated with one load balancing
+// policy. It corresponds to a single entry of the loadBalancingConfig field
+// from ServiceConfig.
+//
+// It implements the json.Unmarshaler interface.
 //
 // https://github.com/grpc/grpc-proto/blob/54713b1e8bc6ed2d4f25fb4dff527842150b91b2/grpc/service_config/service_config.proto#L247
 type BalancerConfig struct {
@@ -41,7 +44,15 @@ type BalancerConfig struct {
 
 type intermediateBalancerConfig []map[string]json.RawMessage
 
-// UnmarshalJSON implements json unmarshaller.
+// UnmarshalJSON implements the json.Unmarshaler interface.
+//
+// ServiceConfig contains a list of loadBalancingConfigs, each with a name and
+// config. This method iterates through that list in order, and stops at the
+// first policy that is supported.
+// - If the config for the first supported policy is invalid, the whole service
+//   config is invalid.
+// - If the list doesn't contain any supported policy, the whole service config
+//   is invalid.
 func (bc *BalancerConfig) UnmarshalJSON(b []byte) error {
 	var ir intermediateBalancerConfig
 	err := json.Unmarshal(b, &ir)
@@ -53,13 +64,16 @@ func (bc *BalancerConfig) UnmarshalJSON(b []byte) error {
 		if len(lbcfg) != 1 {
 			return fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg)
 		}
+
 		var (
 			name    string
 			jsonCfg json.RawMessage
 		)
-		// Get the key:value pair from the map.
+		// Get the key:value pair from the map. We have already made sure that
+		// the map contains a single entry.
 		for name, jsonCfg = range lbcfg {
 		}
+
 		builder := balancer.Get(name)
 		if builder == nil {
 			// If the balancer is not registered, move on to the next config.

From b54ea173dcf1990a3fc2b73e434aa4389fc8e395 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 11 Aug 2020 17:09:24 -0700
Subject: [PATCH 156/481] xds: Support xDS v3 transport protocol. (#3778)

---
 xds/internal/client/transport_helper.go | 422 +++++++++++++++++++++++
 xds/internal/client/v2/client.go        | 439 +++++-------------------
 xds/internal/client/v3/client.go        | 256 ++++++++++++++
 3 files changed, 764 insertions(+), 353 deletions(-)
 create mode 100644 xds/internal/client/transport_helper.go
 create mode 100644 xds/internal/client/v3/client.go

diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
new file mode 100644
index 000000000000..6627fbffb9d5
--- /dev/null
+++ b/xds/internal/client/transport_helper.go
@@ -0,0 +1,422 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/grpclog"
+)
+
+// ErrResourceTypeUnsupported is an error used to indicate an unsupported xDS
+// resource type. The wrapped ErrStr contains the details.
+type ErrResourceTypeUnsupported struct {
+	ErrStr string
+}
+
+// Error helps implements the error interface.
+func (e ErrResourceTypeUnsupported) Error() string {
+	return e.ErrStr
+}
+
+// VersionedClient is the interface to be provided by the transport protocol
+// specific client implementations. This mainly deals with the actual sending
+// and receiving of messages.
+type VersionedClient interface {
+	// NewStream returns a new grpc.ClientStream specific to the underlying
+	// transport protocol version.
+	NewStream(ctx context.Context) (grpc.ClientStream, error)
+
+	// params: resources, typeURL, version, nonce
+
+	// SendRequest constructs and sends out a DiscoveryRequest message specific
+	// to the underlying transport protocol version.
+	SendRequest(s grpc.ClientStream, resourceNames []string, typeURL string, version string, nonce string) error
+
+	// RecvResponse uses the provided stream to receive a response specific to
+	// the underlying transport protocol version.
+	RecvResponse(stream grpc.ClientStream) (proto.Message, error)
+
+	// HandleResponse parses and validates the received response and notifies
+	// the top-level client which in turn notifies the registered watchers.
+	//
+	// Return values are: typeURL, version, nonce, error.
+	// If the provided protobuf message contains a resource type which is not
+	// supported, implementations must return an error of type
+	// ErrResourceTypeUnsupported.
+	HandleResponse(proto.Message) (string, string, string, error)
+}
+
+// TransportHelper contains all xDS transport protocol related functionality
+// which is common across different versioned client implementations.
+//
+// TransportHelper takes care of sending and receiving xDS requests and
+// responses on an ADS stream. It also takes care of ACK/NACK handling. It
+// delegates to the actual versioned client implementations wherever
+// appropriate.
+//
+// Implements the APIClient interface which makes it possible for versioned
+// client implementations to embed this type, and thereby satisfy the interface
+// requirements.
+type TransportHelper struct {
+	cancelCtx context.CancelFunc
+
+	vClient  VersionedClient
+	logger   *grpclog.PrefixLogger
+	backoff  func(int) time.Duration
+	streamCh chan grpc.ClientStream
+	sendCh   *buffer.Unbounded
+
+	mu sync.Mutex
+	// Message specific watch infos, protected by the above mutex. These are
+	// written to, after successfully reading from the update channel, and are
+	// read from when recovering from a broken stream to resend the xDS
+	// messages. When the user of this client object cancels a watch call,
+	// these are set to nil. All accesses to the map protected and any value
+	// inside the map should be protected with the above mutex.
+	watchMap map[string]map[string]bool
+	// versionMap contains the version that was acked (the version in the ack
+	// request that was sent on wire). The key is typeURL, the value is the
+	// version string, becaues the versions for different resource types should
+	// be independent.
+	versionMap map[string]string
+	// nonceMap contains the nonce from the most recent received response.
+	nonceMap map[string]string
+}
+
+// NewTransportHelper creates a new transport helper to be used by versioned
+// client implementations.
+func NewTransportHelper(vc VersionedClient, logger *grpclog.PrefixLogger, backoff func(int) time.Duration) *TransportHelper {
+	ctx, cancelCtx := context.WithCancel(context.Background())
+	t := &TransportHelper{
+		cancelCtx: cancelCtx,
+		vClient:   vc,
+		logger:    logger,
+		backoff:   backoff,
+
+		streamCh:   make(chan grpc.ClientStream, 1),
+		sendCh:     buffer.NewUnbounded(),
+		watchMap:   make(map[string]map[string]bool),
+		versionMap: make(map[string]string),
+		nonceMap:   make(map[string]string),
+	}
+
+	go t.run(ctx)
+	return t
+}
+
+// AddWatch adds a watch for an xDS resource given its type and name.
+func (t *TransportHelper) AddWatch(resourceType, resourceName string) {
+	t.sendCh.Put(&watchAction{
+		typeURL:  resourceType,
+		remove:   false,
+		resource: resourceName,
+	})
+}
+
+// RemoveWatch cancels an already registered watch for an xDS resource
+// given its type and name.
+func (t *TransportHelper) RemoveWatch(resourceType, resourceName string) {
+	t.sendCh.Put(&watchAction{
+		typeURL:  resourceType,
+		remove:   true,
+		resource: resourceName,
+	})
+}
+
+// Close closes the transport helper.
+func (t *TransportHelper) Close() {
+	t.cancelCtx()
+}
+
+// run starts an ADS stream (and backs off exponentially, if the previous
+// stream failed without receiving a single reply) and runs the sender and
+// receiver routines to send and receive data from the stream respectively.
+func (t *TransportHelper) run(ctx context.Context) {
+	go t.send(ctx)
+	// TODO: start a goroutine monitoring ClientConn's connectivity state, and
+	// report error (and log) when stats is transient failure.
+
+	retries := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if retries != 0 {
+			timer := time.NewTimer(t.backoff(retries))
+			select {
+			case <-timer.C:
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			}
+		}
+
+		retries++
+		stream, err := t.vClient.NewStream(ctx)
+		if err != nil {
+			t.logger.Warningf("xds: ADS stream creation failed: %v", err)
+			continue
+		}
+		t.logger.Infof("ADS stream created")
+
+		select {
+		case <-t.streamCh:
+		default:
+		}
+		t.streamCh <- stream
+		if t.recv(stream) {
+			retries = 0
+		}
+	}
+}
+
+// send is a separate goroutine for sending watch requests on the xds stream.
+//
+// It watches the stream channel for new streams, and the request channel for
+// new requests to send on the stream.
+//
+// For each new request (watchAction), it's
+//  - processed and added to the watch map
+//    - so resend will pick them up when there are new streams
+//  - sent on the current stream if there's one
+//    - the current stream is cleared when any send on it fails
+//
+// For each new stream, all the existing requests will be resent.
+//
+// Note that this goroutine doesn't do anything to the old stream when there's a
+// new one. In fact, there should be only one stream in progress, and new one
+// should only be created when the old one fails (recv returns an error).
+func (t *TransportHelper) send(ctx context.Context) {
+	var stream grpc.ClientStream
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case stream = <-t.streamCh:
+			if !t.sendExisting(stream) {
+				// send failed, clear the current stream.
+				stream = nil
+			}
+		case u := <-t.sendCh.Get():
+			t.sendCh.Load()
+
+			var (
+				target                  []string
+				typeURL, version, nonce string
+				send                    bool
+			)
+			switch update := u.(type) {
+			case *watchAction:
+				target, typeURL, version, nonce = t.processWatchInfo(update)
+			case *ackAction:
+				target, typeURL, version, nonce, send = t.processAckInfo(update, stream)
+				if !send {
+					continue
+				}
+			}
+			if stream == nil {
+				// There's no stream yet. Skip the request. This request
+				// will be resent to the new streams. If no stream is
+				// created, the watcher will timeout (same as server not
+				// sending response back).
+				continue
+			}
+			if err := t.vClient.SendRequest(stream, target, typeURL, version, nonce); err != nil {
+				t.logger.Warningf("ADS request for {target: %q, type: %q, version: %q, nonce: %q} failed: %v", target, typeURL, version, nonce, err)
+				// send failed, clear the current stream.
+				stream = nil
+			}
+		}
+	}
+}
+
+// sendExisting sends out xDS requests for registered watchers when recovering
+// from a broken stream.
+//
+// We call stream.Send() here with the lock being held. It should be OK to do
+// that here because the stream has just started and Send() usually returns
+// quickly (once it pushes the message onto the transport layer) and is only
+// ever blocked if we don't have enough flow control quota.
+func (t *TransportHelper) sendExisting(stream grpc.ClientStream) bool {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	// Reset the ack versions when the stream restarts.
+	t.versionMap = make(map[string]string)
+	t.nonceMap = make(map[string]string)
+
+	for typeURL, s := range t.watchMap {
+		if err := t.vClient.SendRequest(stream, mapToSlice(s), typeURL, "", ""); err != nil {
+			t.logger.Errorf("ADS request failed: %v", err)
+			return false
+		}
+	}
+
+	return true
+}
+
+// recv receives xDS responses on the provided ADS stream and branches out to
+// message specific handlers.
+func (t *TransportHelper) recv(stream grpc.ClientStream) bool {
+	success := false
+	for {
+		resp, err := t.vClient.RecvResponse(stream)
+		if err != nil {
+			t.logger.Warningf("ADS stream is closed with error: %v", err)
+			return success
+		}
+		typeURL, version, nonce, err := t.vClient.HandleResponse(resp)
+		if e, ok := err.(ErrResourceTypeUnsupported); ok {
+			t.logger.Warningf("%s", e.ErrStr)
+			continue
+		}
+		if err != nil {
+			t.sendCh.Put(&ackAction{
+				typeURL: typeURL,
+				version: "",
+				nonce:   nonce,
+				stream:  stream,
+			})
+			t.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", typeURL, version, nonce, err)
+			continue
+		}
+		t.sendCh.Put(&ackAction{
+			typeURL: typeURL,
+			version: version,
+			nonce:   nonce,
+			stream:  stream,
+		})
+		t.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", typeURL, version, nonce)
+		success = true
+	}
+}
+
+func mapToSlice(m map[string]bool) (ret []string) {
+	for i := range m {
+		ret = append(ret, i)
+	}
+	return
+}
+
+type watchAction struct {
+	typeURL  string
+	remove   bool // Whether this is to remove watch for the resource.
+	resource string
+}
+
+// processWatchInfo pulls the fields needed by the request from a watchAction.
+//
+// It also updates the watch map in v2c.
+func (t *TransportHelper) processWatchInfo(w *watchAction) (target []string, typeURL, ver, nonce string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	var current map[string]bool
+	current, ok := t.watchMap[w.typeURL]
+	if !ok {
+		current = make(map[string]bool)
+		t.watchMap[w.typeURL] = current
+	}
+
+	if w.remove {
+		delete(current, w.resource)
+		if len(current) == 0 {
+			delete(t.watchMap, w.typeURL)
+		}
+	} else {
+		current[w.resource] = true
+	}
+
+	typeURL = w.typeURL
+	target = mapToSlice(current)
+	// We don't reset version or nonce when a new watch is started. The version
+	// and nonce from previous response are carried by the request unless the
+	// stream is recreated.
+	ver = t.versionMap[typeURL]
+	nonce = t.nonceMap[typeURL]
+	return target, typeURL, ver, nonce
+}
+
+type ackAction struct {
+	typeURL string
+	version string // NACK if version is an empty string.
+	nonce   string
+	// ACK/NACK are tagged with the stream it's for. When the stream is down,
+	// all the ACK/NACK for this stream will be dropped, and the version/nonce
+	// won't be updated.
+	stream grpc.ClientStream
+}
+
+// processAckInfo pulls the fields needed by the ack request from a ackAction.
+//
+// If no active watch is found for this ack, it returns false for send.
+func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStream) (target []string, typeURL, version, nonce string, send bool) {
+	if ack.stream != stream {
+		// If ACK's stream isn't the current sending stream, this means the ACK
+		// was pushed to queue before the old stream broke, and a new stream has
+		// been started since. Return immediately here so we don't update the
+		// nonce for the new stream.
+		return nil, "", "", "", false
+	}
+	typeURL = ack.typeURL
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	// Update the nonce no matter if we are going to send the ACK request on
+	// wire. We may not send the request if the watch is canceled. But the nonce
+	// needs to be updated so the next request will have the right nonce.
+	nonce = ack.nonce
+	t.nonceMap[typeURL] = nonce
+
+	s, ok := t.watchMap[typeURL]
+	if !ok || len(s) == 0 {
+		// We don't send the request ack if there's no active watch (this can be
+		// either the server sends responses before any request, or the watch is
+		// canceled while the ackAction is in queue), because there's no resource
+		// name. And if we send a request with empty resource name list, the
+		// server may treat it as a wild card and send us everything.
+		return nil, "", "", "", false
+	}
+	send = true
+	target = mapToSlice(s)
+
+	version = ack.version
+	if version == "" {
+		// This is a nack, get the previous acked version.
+		version = t.versionMap[typeURL]
+		// version will still be an empty string if typeURL isn't
+		// found in versionMap, this can happen if there wasn't any ack
+		// before.
+	} else {
+		t.versionMap[typeURL] = version
+	}
+	return target, typeURL, version, nonce, send
+}
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index c5a79f54fc33..278dc4381d2c 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -23,10 +23,9 @@ import (
 	"context"
 	"fmt"
 	"sync"
-	"time"
 
+	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/version"
@@ -59,19 +58,10 @@ func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIC
 		cc:        cc,
 		parent:    opts.Parent,
 		nodeProto: nodeProto,
-		backoff:   opts.Backoff,
 		logger:    opts.Logger,
-
-		streamCh: make(chan adsStream, 1),
-		sendCh:   buffer.NewUnbounded(),
-
-		watchMap:   make(map[string]map[string]bool),
-		versionMap: make(map[string]string),
-		nonceMap:   make(map[string]string),
 	}
 	v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background())
-
-	go v2c.run()
+	v2c.TransportHelper = xdsclient.NewTransportHelper(v2c, opts.Logger, opts.Backoff)
 	return v2c, nil
 }
 
@@ -80,121 +70,69 @@ type adsStream v2adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesCli
 // client performs the actual xDS RPCs using the xDS v2 API. It creates a
 // single ADS stream on which the different types of xDS requests and responses
 // are multiplexed.
-//
-// This client's main purpose is to make the RPC, build/parse proto messages,
-// and do ACK/NACK. It's a naive implementation that sends whatever the upper
-// layer tells it to send. It will call the callback with everything in every
-// response. It doesn't keep a cache of responses, or check for duplicates.
 type client struct {
+	*xdsclient.TransportHelper
+
 	ctx       context.Context
 	cancelCtx context.CancelFunc
 	parent    xdsclient.UpdateHandler
+	logger    *grpclog.PrefixLogger
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
 	cc        *grpc.ClientConn
 	nodeProto *v2corepb.Node
-	backoff   func(int) time.Duration
-
-	logger *grpclog.PrefixLogger
-
-	// streamCh is the channel where new ADS streams are pushed to (when the old
-	// stream is broken). It's monitored by the sending goroutine, so requests
-	// are sent to the most up-to-date stream.
-	streamCh chan adsStream
-	// sendCh is the channel onto which watchAction objects are pushed by the
-	// watch API, and it is read and acted upon by the send() goroutine.
-	sendCh *buffer.Unbounded
 
 	mu sync.Mutex
-	// Message specific watch infos, protected by the above mutex. These are
-	// written to, after successfully reading from the update channel, and are
-	// read from when recovering from a broken stream to resend the xDS
-	// messages. When the user of this client object cancels a watch call,
-	// these are set to nil. All accesses to the map protected and any value
-	// inside the map should be protected with the above mutex.
-	watchMap map[string]map[string]bool
-	// versionMap contains the version that was acked (the version in the ack
-	// request that was sent on wire). The key is typeURL, the value is the
-	// version string, becaues the versions for different resource types should
-	// be independent.
-	versionMap map[string]string
-	// nonceMap contains the nonce from the most recent received response.
-	nonceMap map[string]string
-	// hostname is the LDS resource_name to watch. It is set to the first LDS
-	// resource_name to watch, and removed when the LDS watch is canceled.
+	// ldsResourceName is the LDS resource_name to watch. It is set to the first
+	// LDS resource_name to watch, and removed when the LDS watch is canceled.
 	//
 	// It's from the dial target of the parent ClientConn. RDS resource
 	// processing needs this to do the host matching.
-	hostname string
+	ldsResourceName string
+	ldsWatchCount   int
 }
 
+// AddWatch overrides the transport helper's AddWatch to save the LDS
+// resource_name. This is required when handling an RDS response to perform host
+// matching.
 func (v2c *client) AddWatch(resourceType, resourceName string) {
-	v2c.sendCh.Put(&watchAction{
-		typeURL:  resourceType,
-		remove:   false,
-		resource: resourceName,
-	})
+	v2c.mu.Lock()
+	// Special handling for LDS, because RDS needs the LDS resource_name for
+	// response host matching.
+	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+		// Set hostname to the first LDS resource_name, and reset it when the
+		// last LDS watch is removed. The upper level Client isn't expected to
+		// watchLDS more than once.
+		v2c.ldsWatchCount++
+		if v2c.ldsWatchCount == 1 {
+			v2c.ldsResourceName = resourceName
+		}
+	}
+	v2c.mu.Unlock()
+	v2c.TransportHelper.AddWatch(resourceType, resourceName)
 }
 
+// RemoveWatch overrides the transport helper's RemoveWatch to clear the LDS
+// resource_name when the last watch is removed.
 func (v2c *client) RemoveWatch(resourceType, resourceName string) {
-	v2c.sendCh.Put(&watchAction{
-		typeURL:  resourceType,
-		remove:   true,
-		resource: resourceName,
-	})
-}
-
-// close cleans up resources and goroutines allocated by this client.
-func (v2c *client) Close() {
-	v2c.cancelCtx()
-}
-
-// run starts an ADS stream (and backs off exponentially, if the previous
-// stream failed without receiving a single reply) and runs the sender and
-// receiver routines to send and receive data from the stream respectively.
-func (v2c *client) run() {
-	go v2c.send()
-	// TODO: start a goroutine monitoring ClientConn's connectivity state, and
-	// report error (and log) when stats is transient failure.
-
-	retries := 0
-	for {
-		select {
-		case <-v2c.ctx.Done():
-			return
-		default:
-		}
-
-		if retries != 0 {
-			t := time.NewTimer(v2c.backoff(retries))
-			select {
-			case <-t.C:
-			case <-v2c.ctx.Done():
-				if !t.Stop() {
-					<-t.C
-				}
-				return
-			}
-		}
-
-		retries++
-		cli := v2adsgrpc.NewAggregatedDiscoveryServiceClient(v2c.cc)
-		stream, err := cli.StreamAggregatedResources(v2c.ctx, grpc.WaitForReady(true))
-		if err != nil {
-			v2c.logger.Warningf("xds: ADS stream creation failed: %v", err)
-			continue
-		}
-		v2c.logger.Infof("ADS stream created")
-
-		select {
-		case <-v2c.streamCh:
-		default:
-		}
-		v2c.streamCh <- stream
-		if v2c.recv(stream) {
-			retries = 0
+	v2c.mu.Lock()
+	// Special handling for LDS, because RDS needs the LDS resource_name for
+	// response host matching.
+	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+		// Set hostname to the first LDS resource_name, and reset it when the
+		// last LDS watch is removed. The upper level Client isn't expected to
+		// watchLDS more than once.
+		v2c.ldsWatchCount--
+		if v2c.ldsWatchCount == 0 {
+			v2c.ldsResourceName = ""
 		}
 	}
+	v2c.mu.Unlock()
+	v2c.TransportHelper.RemoveWatch(resourceType, resourceName)
+}
+
+func (v2c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
+	return v2adsgrpc.NewAggregatedDiscoveryServiceClient(v2c.cc).StreamAggregatedResources(v2c.ctx, grpc.WaitForReady(true))
 }
 
 // sendRequest sends a request for provided typeURL and resource on the provided
@@ -206,7 +144,11 @@ func (v2c *client) run() {
 // - If this is an ack, version will be the version from the response
 // - If this is a nack, version will be the previous acked version (from
 // versionMap). If there was no ack before, it will be an empty string
-func (v2c *client) sendRequest(stream adsStream, resourceNames []string, typeURL, version, nonce string) bool {
+func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, typeURL, version, nonce string) error {
+	stream, ok := s.(adsStream)
+	if !ok {
+		return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s)
+	}
 	req := &v2xdspb.DiscoveryRequest{
 		Node:          v2c.nodeProto,
 		TypeUrl:       typeURL,
@@ -216,265 +158,56 @@ func (v2c *client) sendRequest(stream adsStream, resourceNames []string, typeURL
 		// TODO: populate ErrorDetails for nack.
 	}
 	if err := stream.Send(req); err != nil {
-		return false
+		return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err)
 	}
 	v2c.logger.Debugf("ADS request sent: %v", req)
-	return true
-}
-
-// sendExisting sends out xDS requests for registered watchers when recovering
-// from a broken stream.
-//
-// We call stream.Send() here with the lock being held. It should be OK to do
-// that here because the stream has just started and Send() usually returns
-// quickly (once it pushes the message onto the transport layer) and is only
-// ever blocked if we don't have enough flow control quota.
-func (v2c *client) sendExisting(stream adsStream) bool {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	// Reset the ack versions when the stream restarts.
-	v2c.versionMap = make(map[string]string)
-	v2c.nonceMap = make(map[string]string)
-
-	for typeURL, s := range v2c.watchMap {
-		if !v2c.sendRequest(stream, mapToSlice(s), typeURL, "", "") {
-			return false
-		}
-	}
-
-	return true
-}
-
-type watchAction struct {
-	typeURL  string
-	remove   bool // Whether this is to remove watch for the resource.
-	resource string
+	return nil
 }
 
-// processWatchInfo pulls the fields needed by the request from a watchAction.
-//
-// It also updates the watch map in v2c.
-func (v2c *client) processWatchInfo(t *watchAction) (target []string, typeURL, ver, nonce string, send bool) {
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	var current map[string]bool
-	current, ok := v2c.watchMap[t.typeURL]
+// RecvResponse blocks on the receipt of one response message on the provided
+// stream.
+func (v2c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
+	stream, ok := s.(adsStream)
 	if !ok {
-		current = make(map[string]bool)
-		v2c.watchMap[t.typeURL] = current
-	}
-
-	if t.remove {
-		delete(current, t.resource)
-		if len(current) == 0 {
-			delete(v2c.watchMap, t.typeURL)
-		}
-	} else {
-		current[t.resource] = true
-	}
-
-	// Special handling for LDS, because RDS needs the LDS resource_name for
-	// response host matching.
-	if t.typeURL == version.V2ListenerURL {
-		// Set hostname to the first LDS resource_name, and reset it when the
-		// last LDS watch is removed. The upper level Client isn't expected to
-		// watchLDS more than once.
-		if l := len(current); l == 1 {
-			v2c.hostname = t.resource
-		} else if l == 0 {
-			v2c.hostname = ""
-		}
-	}
-
-	send = true
-	typeURL = t.typeURL
-	target = mapToSlice(current)
-	// We don't reset version or nonce when a new watch is started. The version
-	// and nonce from previous response are carried by the request unless the
-	// stream is recreated.
-	ver = v2c.versionMap[typeURL]
-	nonce = v2c.nonceMap[typeURL]
-	return target, typeURL, ver, nonce, send
-}
-
-type ackAction struct {
-	typeURL string
-	version string // NACK if version is an empty string.
-	nonce   string
-	// ACK/NACK are tagged with the stream it's for. When the stream is down,
-	// all the ACK/NACK for this stream will be dropped, and the version/nonce
-	// won't be updated.
-	stream adsStream
-}
-
-// processAckInfo pulls the fields needed by the ack request from a ackAction.
-//
-// If no active watch is found for this ack, it returns false for send.
-func (v2c *client) processAckInfo(t *ackAction, stream adsStream) (target []string, typeURL, version, nonce string, send bool) {
-	if t.stream != stream {
-		// If ACK's stream isn't the current sending stream, this means the ACK
-		// was pushed to queue before the old stream broke, and a new stream has
-		// been started since. Return immediately here so we don't update the
-		// nonce for the new stream.
-		return nil, "", "", "", false
-	}
-	typeURL = t.typeURL
-
-	v2c.mu.Lock()
-	defer v2c.mu.Unlock()
-
-	// Update the nonce no matter if we are going to send the ACK request on
-	// wire. We may not send the request if the watch is canceled. But the nonce
-	// needs to be updated so the next request will have the right nonce.
-	nonce = t.nonce
-	v2c.nonceMap[typeURL] = nonce
-
-	s, ok := v2c.watchMap[typeURL]
-	if !ok || len(s) == 0 {
-		// We don't send the request ack if there's no active watch (this can be
-		// either the server sends responses before any request, or the watch is
-		// canceled while the ackAction is in queue), because there's no resource
-		// name. And if we send a request with empty resource name list, the
-		// server may treat it as a wild card and send us everything.
-		return nil, "", "", "", false
+		return nil, fmt.Errorf("xds: Attempt to receive response on unsupported stream type: %T", s)
 	}
-	send = true
-	target = mapToSlice(s)
 
-	version = t.version
-	if version == "" {
-		// This is a nack, get the previous acked version.
-		version = v2c.versionMap[typeURL]
-		// version will still be an empty string if typeURL isn't
-		// found in versionMap, this can happen if there wasn't any ack
-		// before.
-	} else {
-		v2c.versionMap[typeURL] = version
+	resp, err := stream.Recv()
+	if err != nil {
+		// TODO: call watch callbacks with error when stream is broken.
+		return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err)
 	}
-	return target, typeURL, version, nonce, send
+	v2c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
+	v2c.logger.Debugf("ADS response received: %v", resp)
+	return resp, nil
 }
 
-// send is a separate goroutine for sending watch requests on the xds stream.
-//
-// It watches the stream channel for new streams, and the request channel for
-// new requests to send on the stream.
-//
-// For each new request (watchAction), it's
-//  - processed and added to the watch map
-//    - so resend will pick them up when there are new streams)
-//  - sent on the current stream if there's one
-//    - the current stream is cleared when any send on it fails
-//
-// For each new stream, all the existing requests will be resent.
-//
-// Note that this goroutine doesn't do anything to the old stream when there's a
-// new one. In fact, there should be only one stream in progress, and new one
-// should only be created when the old one fails (recv returns an error).
-func (v2c *client) send() {
-	var stream adsStream
-	for {
-		select {
-		case <-v2c.ctx.Done():
-			return
-		case stream = <-v2c.streamCh:
-			if !v2c.sendExisting(stream) {
-				// send failed, clear the current stream.
-				stream = nil
-			}
-		case u := <-v2c.sendCh.Get():
-			v2c.sendCh.Load()
-
-			var (
-				target                  []string
-				typeURL, version, nonce string
-				send                    bool
-			)
-			switch t := u.(type) {
-			case *watchAction:
-				target, typeURL, version, nonce, send = v2c.processWatchInfo(t)
-			case *ackAction:
-				target, typeURL, version, nonce, send = v2c.processAckInfo(t, stream)
-			}
-			if !send {
-				continue
-			}
-			if stream == nil {
-				// There's no stream yet. Skip the request. This request
-				// will be resent to the new streams. If no stream is
-				// created, the watcher will timeout (same as server not
-				// sending response back).
-				continue
-			}
-			if !v2c.sendRequest(stream, target, typeURL, version, nonce) {
-				// send failed, clear the current stream.
-				stream = nil
-			}
-		}
+func (v2c *client) HandleResponse(r proto.Message) (string, string, string, error) {
+	resp, ok := r.(*v2xdspb.DiscoveryResponse)
+	if !ok {
+		return "", "", "", fmt.Errorf("xds: unsupported message type: %T", resp)
 	}
-}
-
-// recv receives xDS responses on the provided ADS stream and branches out to
-// message specific handlers.
-func (v2c *client) recv(stream adsStream) bool {
-	success := false
-	for {
-		resp, err := stream.Recv()
-		// TODO: call watch callbacks with error when stream is broken.
-		if err != nil {
-			v2c.logger.Warningf("ADS stream is closed with error: %v", err)
-			return success
-		}
-		v2c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
-		v2c.logger.Debugf("ADS response received: %v", resp)
-
-		// Note that the xDS transport protocol is versioned independently of
-		// the resource types, and it is supported to transfer older versions
-		// of resource types using new versions of the transport protocol, or
-		// vice-versa. Hence we need to handle v3 type_urls as well here.
-		var respHandleErr error
-		switch resp.GetTypeUrl() {
-		case version.V2ListenerURL, version.V3ListenerURL:
-			respHandleErr = v2c.handleLDSResponse(resp)
-		case version.V2RouteConfigURL, version.V3RouteConfigURL:
-			respHandleErr = v2c.handleRDSResponse(resp)
-		case version.V2ClusterURL, version.V3ClusterURL:
-			respHandleErr = v2c.handleCDSResponse(resp)
-		case version.V2EndpointsURL, version.V3EndpointsURL:
-			respHandleErr = v2c.handleEDSResponse(resp)
-		default:
-			v2c.logger.Warningf("Resource type %v unknown in response from server", resp.GetTypeUrl())
-			continue
-		}
 
-		typeURL := resp.GetTypeUrl()
-		if respHandleErr != nil {
-			v2c.sendCh.Put(&ackAction{
-				typeURL: typeURL,
-				version: "",
-				nonce:   resp.GetNonce(),
-				stream:  stream,
-			})
-			v2c.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", typeURL, resp.GetVersionInfo(), resp.GetNonce(), respHandleErr)
-			continue
+	// Note that the xDS transport protocol is versioned independently of
+	// the resource types, and it is supported to transfer older versions
+	// of resource types using new versions of the transport protocol, or
+	// vice-versa. Hence we need to handle v3 type_urls as well here.
+	var err error
+	switch resp.GetTypeUrl() {
+	case version.V2ListenerURL, version.V3ListenerURL:
+		err = v2c.handleLDSResponse(resp)
+	case version.V2RouteConfigURL, version.V3RouteConfigURL:
+		err = v2c.handleRDSResponse(resp)
+	case version.V2ClusterURL, version.V3ClusterURL:
+		err = v2c.handleCDSResponse(resp)
+	case version.V2EndpointsURL, version.V3EndpointsURL:
+		err = v2c.handleEDSResponse(resp)
+	default:
+		return "", "", "", xdsclient.ErrResourceTypeUnsupported{
+			ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()),
 		}
-		v2c.sendCh.Put(&ackAction{
-			typeURL: typeURL,
-			version: resp.GetVersionInfo(),
-			nonce:   resp.GetNonce(),
-			stream:  stream,
-		})
-		v2c.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", typeURL, resp.GetVersionInfo(), resp.GetNonce())
-		success = true
-	}
-}
-
-func mapToSlice(m map[string]bool) (ret []string) {
-	for i := range m {
-		ret = append(ret, i)
 	}
-	return
+	return resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), err
 }
 
 // handleLDSResponse processes an LDS response received from the xDS server. On
@@ -493,7 +226,7 @@ func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 // the registered watcher callback.
 func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	v2c.mu.Lock()
-	hostname := v2c.hostname
+	hostname := v2c.ldsResourceName
 	v2c.mu.Unlock()
 
 	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v2c.logger)
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
new file mode 100644
index 000000000000..56bb245e32a6
--- /dev/null
+++ b/xds/internal/client/v3/client.go
@@ -0,0 +1,256 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package v3 provides xDS v3 transport protocol specific functionality.
+package v3
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/grpclog"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/version"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+func init() {
+	xdsclient.RegisterAPIClientBuilder(clientBuilder{})
+}
+
+type clientBuilder struct{}
+
+func (clientBuilder) Build(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
+	return newClient(cc, opts)
+}
+
+func (clientBuilder) Version() version.TransportAPI {
+	return version.TransportV3
+}
+
+func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
+	nodeProto, ok := opts.NodeProto.(*v3corepb.Node)
+	if !ok {
+		return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v3corepb.Node{})
+	}
+	v3c := &client{
+		cc:        cc,
+		parent:    opts.Parent,
+		nodeProto: nodeProto,
+		logger:    opts.Logger,
+	}
+	v3c.ctx, v3c.cancelCtx = context.WithCancel(context.Background())
+	v3c.TransportHelper = xdsclient.NewTransportHelper(v3c, opts.Logger, opts.Backoff)
+	return v3c, nil
+}
+
+type adsStream v3adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient
+
+// client performs the actual xDS RPCs using the xDS v3 API. It creates a
+// single ADS stream on which the different types of xDS requests and responses
+// are multiplexed.
+type client struct {
+	*xdsclient.TransportHelper
+
+	ctx       context.Context
+	cancelCtx context.CancelFunc
+	parent    xdsclient.UpdateHandler
+	logger    *grpclog.PrefixLogger
+
+	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
+	cc        *grpc.ClientConn
+	nodeProto *v3corepb.Node
+
+	mu sync.Mutex
+	// ldsResourceName is the LDS resource_name to watch. It is set to the first
+	// LDS resource_name to watch, and removed when the LDS watch is canceled.
+	//
+	// It's from the dial target of the parent ClientConn. RDS resource
+	// processing needs this to do the host matching.
+	ldsResourceName string
+	ldsWatchCount   int
+}
+
+// AddWatch overrides the transport helper's AddWatch to save the LDS
+// resource_name. This is required when handling an RDS response to perform hot
+// matching.
+func (v3c *client) AddWatch(resourceType, resourceName string) {
+	v3c.mu.Lock()
+	// Special handling for LDS, because RDS needs the LDS resource_name for
+	// response host matching.
+	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+		// Set hostname to the first LDS resource_name, and reset it when the
+		// last LDS watch is removed. The upper level Client isn't expected to
+		// watchLDS more than once.
+		v3c.ldsWatchCount++
+		if v3c.ldsWatchCount == 1 {
+			v3c.ldsResourceName = resourceName
+		}
+	}
+	v3c.mu.Unlock()
+	v3c.TransportHelper.AddWatch(resourceType, resourceName)
+}
+
+func (v3c *client) RemoveWatch(resourceType, resourceName string) {
+	v3c.mu.Lock()
+	// Special handling for LDS, because RDS needs the LDS resource_name for
+	// response host matching.
+	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+		// Set hostname to the first LDS resource_name, and reset it when the
+		// last LDS watch is removed. The upper level Client isn't expected to
+		// watchLDS more than once.
+		v3c.ldsWatchCount--
+		if v3c.ldsWatchCount == 0 {
+			v3c.ldsResourceName = ""
+		}
+	}
+	v3c.mu.Unlock()
+	v3c.TransportHelper.RemoveWatch(resourceType, resourceName)
+}
+
+func (v3c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
+	return v3adsgrpc.NewAggregatedDiscoveryServiceClient(v3c.cc).StreamAggregatedResources(v3c.ctx, grpc.WaitForReady(true))
+}
+
+// sendRequest sends a request for provided typeURL and resource on the provided
+// stream.
+//
+// version is the ack version to be sent with the request
+// - If this is the new request (not an ack/nack), version will be an empty
+// string
+// - If this is an ack, version will be the version from the response
+// - If this is a nack, version will be the previous acked version (from
+// versionMap). If there was no ack before, it will be an empty string
+func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, typeURL, version, nonce string) error {
+	stream, ok := s.(adsStream)
+	if !ok {
+		return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s)
+	}
+	req := &v3discoverypb.DiscoveryRequest{
+		Node:          v3c.nodeProto,
+		TypeUrl:       typeURL,
+		ResourceNames: resourceNames,
+		VersionInfo:   version,
+		ResponseNonce: nonce,
+		// TODO: populate ErrorDetails for nack.
+	}
+	if err := stream.Send(req); err != nil {
+		return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err)
+	}
+	v3c.logger.Debugf("ADS request sent: %v", req)
+	return nil
+}
+
+// RecvResponse blocks on the receipt of one response message on the provided
+// stream.
+func (v3c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
+	stream, ok := s.(adsStream)
+	if !ok {
+		return nil, fmt.Errorf("xds: Attempt to receive response on unsupported stream type: %T", s)
+	}
+
+	resp, err := stream.Recv()
+	if err != nil {
+		// TODO: call watch callbacks with error when stream is broken.
+		return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err)
+	}
+	v3c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
+	v3c.logger.Debugf("ADS response received: %v", resp)
+	return resp, nil
+}
+
+func (v3c *client) HandleResponse(r proto.Message) (string, string, string, error) {
+	resp, ok := r.(*v3discoverypb.DiscoveryResponse)
+	if !ok {
+		return "", "", "", fmt.Errorf("xds: unsupported message type: %T", resp)
+	}
+
+	// Note that the xDS transport protocol is versioned independently of
+	// the resource types, and it is supported to transfer older versions
+	// of resource types using new versions of the transport protocol, or
+	// vice-versa. Hence we need to handle v3 type_urls as well here.
+	var err error
+	switch resp.GetTypeUrl() {
+	case version.V2ListenerURL, version.V3ListenerURL:
+		err = v3c.handleLDSResponse(resp)
+	case version.V2RouteConfigURL, version.V3RouteConfigURL:
+		err = v3c.handleRDSResponse(resp)
+	case version.V2ClusterURL, version.V3ClusterURL:
+		err = v3c.handleCDSResponse(resp)
+	case version.V2EndpointsURL, version.V3EndpointsURL:
+		err = v3c.handleEDSResponse(resp)
+	default:
+		return "", "", "", xdsclient.ErrResourceTypeUnsupported{
+			ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()),
+		}
+	}
+	return resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), err
+}
+
+// handleLDSResponse processes an LDS response received from the xDS server. On
+// receipt of a good response, it also invokes the registered watcher callback.
+func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
+	update, err := xdsclient.UnmarshalListener(resp.GetResources(), v3c.logger)
+	if err != nil {
+		return err
+	}
+	v3c.parent.NewListeners(update)
+	return nil
+}
+
+// handleRDSResponse processes an RDS response received from the xDS server. On
+// receipt of a good response, it caches validated resources and also invokes
+// the registered watcher callback.
+func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
+	v3c.mu.Lock()
+	hostname := v3c.ldsResourceName
+	v3c.mu.Unlock()
+
+	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v3c.logger)
+	if err != nil {
+		return err
+	}
+	v3c.parent.NewRouteConfigs(update)
+	return nil
+}
+
+// handleCDSResponse processes an CDS response received from the xDS server. On
+// receipt of a good response, it also invokes the registered watcher callback.
+func (v3c *client) handleCDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
+	update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v3c.logger)
+	if err != nil {
+		return err
+	}
+	v3c.parent.NewClusters(update)
+	return nil
+}
+
+func (v3c *client) handleEDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
+	update, err := xdsclient.UnmarshalEndpoints(resp.GetResources(), v3c.logger)
+	if err != nil {
+		return err
+	}
+	v3c.parent.NewEndpoints(update)
+	return nil
+}

From 7d8921505e1b4d41da1c1bbb4ae20a4625aa5476 Mon Sep 17 00:00:00 2001
From: Yi-Shu Tai <echo80313@gmail.com>
Date: Wed, 12 Aug 2020 11:47:16 -0700
Subject: [PATCH 157/481] internal/wrr: introduce order offset to edf scheduler
 as a tiebreaker for entries with same deadline (#3800)

---
 internal/wrr/edf.go      | 40 ++++++++++++++++++++--------------------
 internal/wrr/edf_test.go | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 20 deletions(-)
 create mode 100644 internal/wrr/edf_test.go

diff --git a/internal/wrr/edf.go b/internal/wrr/edf.go
index 307ad2a747d1..b4fb3f9d3bea 100644
--- a/internal/wrr/edf.go
+++ b/internal/wrr/edf.go
@@ -24,8 +24,10 @@ import (
 
 // edfWrr is a struct for EDF weighted round robin implementation.
 type edfWrr struct {
-	lock  sync.Mutex
-	items edfPriorityQueue
+	lock               sync.Mutex
+	items              edfPriorityQueue
+	currentOrderOffset uint64
+	currentTime        float64
 }
 
 // NewEDF creates Earliest Deadline First (EDF)
@@ -38,17 +40,20 @@ func NewEDF() WRR {
 
 // edfEntry is an internal wrapper for item that also stores weight and relative position in the queue.
 type edfEntry struct {
-	deadline float64
-	weight   int64
-	item     interface{}
+	deadline    float64
+	weight      int64
+	orderOffset uint64
+	item        interface{}
 }
 
 // edfPriorityQueue is a heap.Interface implementation for edfEntry elements.
 type edfPriorityQueue []*edfEntry
 
-func (pq edfPriorityQueue) Len() int           { return len(pq) }
-func (pq edfPriorityQueue) Less(i, j int) bool { return pq[i].deadline < pq[j].deadline }
-func (pq edfPriorityQueue) Swap(i, j int)      { pq[i], pq[j] = pq[j], pq[i] }
+func (pq edfPriorityQueue) Len() int { return len(pq) }
+func (pq edfPriorityQueue) Less(i, j int) bool {
+	return pq[i].deadline < pq[j].deadline || pq[i].deadline == pq[j].deadline && pq[i].orderOffset < pq[j].orderOffset
+}
+func (pq edfPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] }
 
 func (pq *edfPriorityQueue) Push(x interface{}) {
 	*pq = append(*pq, x.(*edfEntry))
@@ -60,22 +65,16 @@ func (pq *edfPriorityQueue) Pop() interface{} {
 	return old[len(old)-1]
 }
 
-// Current time in EDF scheduler.
-func (edf *edfWrr) currentTime() float64 {
-	if len(edf.items) == 0 {
-		return 0.0
-	}
-	return edf.items[0].deadline
-}
-
 func (edf *edfWrr) Add(item interface{}, weight int64) {
 	edf.lock.Lock()
 	defer edf.lock.Unlock()
 	entry := edfEntry{
-		deadline: edf.currentTime() + 1.0/float64(weight),
-		weight:   weight,
-		item:     item,
+		deadline:    edf.currentTime + 1.0/float64(weight),
+		weight:      weight,
+		item:        item,
+		orderOffset: edf.currentOrderOffset,
 	}
+	edf.currentOrderOffset++
 	heap.Push(&edf.items, &entry)
 }
 
@@ -86,7 +85,8 @@ func (edf *edfWrr) Next() interface{} {
 		return nil
 	}
 	item := edf.items[0]
-	item.deadline = edf.currentTime() + 1.0/float64(item.weight)
+	edf.currentTime = item.deadline
+	item.deadline = edf.currentTime + 1.0/float64(item.weight)
 	heap.Fix(&edf.items, 0)
 	return item.item
 }
diff --git a/internal/wrr/edf_test.go b/internal/wrr/edf_test.go
new file mode 100644
index 000000000000..c3d0c540f466
--- /dev/null
+++ b/internal/wrr/edf_test.go
@@ -0,0 +1,35 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package wrr
+
+import (
+	"testing"
+)
+
+func (s) TestEDFOnEndpointsWithSameWeight(t *testing.T) {
+	wrr := NewEDF()
+	wrr.Add("1", 1)
+	wrr.Add("2", 1)
+	wrr.Add("3", 1)
+	expected := []string{"1", "2", "3", "1", "2", "3", "1", "2", "3", "1", "2", "3"}
+	for i := 0; i < len(expected); i++ {
+		item := wrr.Next().(string)
+		if item != expected[i] {
+			t.Errorf("wrr Next=%s, want=%s", item, expected[i])
+		}
+	}
+}

From 0baf4c2d4c69ba8eaecdfe4a061d35f11da33544 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 13 Aug 2020 08:58:51 -0700
Subject: [PATCH 158/481] meshca: Pull in the MeshCA config proto from
 grpc-proto repo. (#3807)

---
 .../internal/meshca_experimental/config.pb.go | 198 ++++++++++++++++++
 regenerate.sh                                 |  23 +-
 vet.sh                                        |   2 +-
 3 files changed, 221 insertions(+), 2 deletions(-)
 create mode 100644 credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go

diff --git a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
new file mode 100644
index 000000000000..936dfb6ed4bf
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
@@ -0,0 +1,198 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: grpc/tls/provider/meshca/experimental/config.proto
+
+// NOTE: This proto will very likely move to a different namespace and a
+// different git repo in the future.
+
+package meshca_experimental
+
+import (
+	fmt "fmt"
+	v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	proto "github.com/golang/protobuf/proto"
+	duration "github.com/golang/protobuf/ptypes/duration"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+// Type of key to be embedded in CSRs sent to the MeshCA.
+type GoogleMeshCaConfig_KeyType int32
+
+const (
+	GoogleMeshCaConfig_KEY_TYPE_UNKNOWN GoogleMeshCaConfig_KeyType = 0
+	GoogleMeshCaConfig_KEY_TYPE_RSA     GoogleMeshCaConfig_KeyType = 1
+)
+
+var GoogleMeshCaConfig_KeyType_name = map[int32]string{
+	0: "KEY_TYPE_UNKNOWN",
+	1: "KEY_TYPE_RSA",
+}
+
+var GoogleMeshCaConfig_KeyType_value = map[string]int32{
+	"KEY_TYPE_UNKNOWN": 0,
+	"KEY_TYPE_RSA":     1,
+}
+
+func (x GoogleMeshCaConfig_KeyType) String() string {
+	return proto.EnumName(GoogleMeshCaConfig_KeyType_name, int32(x))
+}
+
+func (GoogleMeshCaConfig_KeyType) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_2f29f6dba7c86653, []int{0, 0}
+}
+
+// GoogleMeshCaConfig contains all configuration parameters required by the
+// MeshCA CertificateProvider plugin implementation.
+type GoogleMeshCaConfig struct {
+	// GoogleMeshCA server endpoint to get CSRs signed via the *CreateCertificate*
+	// unary call. This must have :ref:`api_type
+	// <envoy_api_field_config.core.v3.ApiConfigSource.api_type>` :ref:`GRPC
+	// <envoy_api_enum_value_config.core.v3.ApiConfigSource.ApiType.GRPC>`.
+	// STS based call credentials need to be supplied in :ref:`call_credentials
+	// <envoy_api_field_config.core.v3.GrpcService.GoogleGrpc.call_credentials>`.
+	//
+	// If :ref:`timeout envoy_api_field_config.core.v3.GrpcService.timeout` is
+	// left unspecified, a default value of 10s will be used.
+	Server *v3.ApiConfigSource `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"`
+	// Certificate lifetime to request in CSRs sent to the MeshCA.
+	//
+	// A default value of 24h will be used if left unspecified.
+	CertificateLifetime *duration.Duration `protobuf:"bytes,2,opt,name=certificate_lifetime,json=certificateLifetime,proto3" json:"certificate_lifetime,omitempty"`
+	// How long before certificate expiration should the certificate be renewed.
+	//
+	// A default value of 12h will be used if left unspecified.
+	RenewalGracePeriod *duration.Duration `protobuf:"bytes,3,opt,name=renewal_grace_period,json=renewalGracePeriod,proto3" json:"renewal_grace_period,omitempty"`
+	// Type of key.
+	//
+	// RSA keys will be used if left unspecified.
+	KeyType GoogleMeshCaConfig_KeyType `protobuf:"varint,4,opt,name=key_type,json=keyType,proto3,enum=grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig_KeyType" json:"key_type,omitempty"`
+	// Size of the key in bits.
+	//
+	// 2048 bit keys will be used if left unspecified.
+	KeySize uint32 `protobuf:"varint,5,opt,name=key_size,json=keySize,proto3" json:"key_size,omitempty"`
+	// GCE location (region/zone) where the workload is located.
+	//
+	// GCE/GKE Metadata Server will be contacted if left unspecified.
+	Location             string   `protobuf:"bytes,6,opt,name=location,proto3" json:"location,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *GoogleMeshCaConfig) Reset()         { *m = GoogleMeshCaConfig{} }
+func (m *GoogleMeshCaConfig) String() string { return proto.CompactTextString(m) }
+func (*GoogleMeshCaConfig) ProtoMessage()    {}
+func (*GoogleMeshCaConfig) Descriptor() ([]byte, []int) {
+	return fileDescriptor_2f29f6dba7c86653, []int{0}
+}
+
+func (m *GoogleMeshCaConfig) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GoogleMeshCaConfig.Unmarshal(m, b)
+}
+func (m *GoogleMeshCaConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GoogleMeshCaConfig.Marshal(b, m, deterministic)
+}
+func (m *GoogleMeshCaConfig) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GoogleMeshCaConfig.Merge(m, src)
+}
+func (m *GoogleMeshCaConfig) XXX_Size() int {
+	return xxx_messageInfo_GoogleMeshCaConfig.Size(m)
+}
+func (m *GoogleMeshCaConfig) XXX_DiscardUnknown() {
+	xxx_messageInfo_GoogleMeshCaConfig.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GoogleMeshCaConfig proto.InternalMessageInfo
+
+func (m *GoogleMeshCaConfig) GetServer() *v3.ApiConfigSource {
+	if m != nil {
+		return m.Server
+	}
+	return nil
+}
+
+func (m *GoogleMeshCaConfig) GetCertificateLifetime() *duration.Duration {
+	if m != nil {
+		return m.CertificateLifetime
+	}
+	return nil
+}
+
+func (m *GoogleMeshCaConfig) GetRenewalGracePeriod() *duration.Duration {
+	if m != nil {
+		return m.RenewalGracePeriod
+	}
+	return nil
+}
+
+func (m *GoogleMeshCaConfig) GetKeyType() GoogleMeshCaConfig_KeyType {
+	if m != nil {
+		return m.KeyType
+	}
+	return GoogleMeshCaConfig_KEY_TYPE_UNKNOWN
+}
+
+func (m *GoogleMeshCaConfig) GetKeySize() uint32 {
+	if m != nil {
+		return m.KeySize
+	}
+	return 0
+}
+
+func (m *GoogleMeshCaConfig) GetLocation() string {
+	if m != nil {
+		return m.Location
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterEnum("grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig_KeyType", GoogleMeshCaConfig_KeyType_name, GoogleMeshCaConfig_KeyType_value)
+	proto.RegisterType((*GoogleMeshCaConfig)(nil), "grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig")
+}
+
+func init() {
+	proto.RegisterFile("grpc/tls/provider/meshca/experimental/config.proto", fileDescriptor_2f29f6dba7c86653)
+}
+
+var fileDescriptor_2f29f6dba7c86653 = []byte{
+	// 439 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x4f, 0x6f, 0xd3, 0x30,
+	0x18, 0xc6, 0x09, 0x83, 0x6e, 0x98, 0x3f, 0x2a, 0xa6, 0x87, 0xac, 0x07, 0x54, 0x4d, 0x9a, 0x94,
+	0x93, 0x2d, 0xda, 0x33, 0x87, 0x6e, 0x4c, 0x3b, 0x74, 0x94, 0x2a, 0x1d, 0x9a, 0x86, 0x90, 0x22,
+	0xcf, 0x7d, 0x9b, 0x59, 0x73, 0xed, 0xe8, 0x8d, 0x1b, 0xc8, 0x3e, 0x09, 0x9f, 0x84, 0xcf, 0x87,
+	0xe2, 0x78, 0x53, 0x04, 0x07, 0x7a, 0xcb, 0x1b, 0xfb, 0xf7, 0x3c, 0x8f, 0x1e, 0xbf, 0x64, 0x9c,
+	0x63, 0x21, 0xb9, 0xd3, 0x25, 0x2f, 0xd0, 0x56, 0x6a, 0x05, 0xc8, 0x37, 0x50, 0xde, 0x4a, 0xc1,
+	0xe1, 0x67, 0x01, 0xa8, 0x36, 0x60, 0x9c, 0xd0, 0x5c, 0x5a, 0xb3, 0x56, 0x39, 0x2b, 0xd0, 0x3a,
+	0x4b, 0x8f, 0x1b, 0x86, 0x39, 0x5d, 0xb2, 0x07, 0x86, 0xb5, 0x0c, 0xeb, 0x32, 0xc3, 0x04, 0x4c,
+	0x65, 0xeb, 0x80, 0x72, 0x69, 0x11, 0x78, 0x35, 0x09, 0x63, 0x56, 0xda, 0x2d, 0x4a, 0x68, 0x05,
+	0x87, 0xef, 0x73, 0x6b, 0x73, 0x0d, 0xdc, 0x4f, 0x37, 0xdb, 0x35, 0x5f, 0x6d, 0x51, 0x38, 0x65,
+	0x4d, 0x7b, 0x7e, 0xf4, 0x7b, 0x8f, 0xd0, 0x73, 0x7f, 0xe5, 0x33, 0x94, 0xb7, 0xa7, 0xe2, 0xd4,
+	0x6b, 0xd0, 0x8f, 0xa4, 0x57, 0x02, 0x56, 0x80, 0x71, 0x34, 0x8a, 0x92, 0x97, 0xe3, 0x63, 0xe6,
+	0x1d, 0x59, 0x08, 0xdb, 0x38, 0xb2, 0x6a, 0xc2, 0xa6, 0x85, 0x6a, 0x81, 0xa5, 0xf7, 0x4c, 0x03,
+	0x44, 0x2f, 0xc8, 0x40, 0x02, 0x3a, 0xb5, 0x56, 0x52, 0x38, 0xc8, 0xb4, 0x5a, 0x83, 0x53, 0x1b,
+	0x88, 0x9f, 0x7a, 0xb1, 0x43, 0xd6, 0x86, 0x62, 0x0f, 0xa1, 0xd8, 0xa7, 0x10, 0x2a, 0x7d, 0xd7,
+	0xc1, 0x2e, 0x02, 0x45, 0x67, 0x64, 0x80, 0x60, 0xe0, 0x87, 0xd0, 0x59, 0x8e, 0x42, 0x42, 0xd6,
+	0x34, 0x61, 0x57, 0xf1, 0xde, 0xff, 0xd4, 0x68, 0xc0, 0xce, 0x1b, 0x6a, 0xe1, 0x21, 0xfa, 0x9d,
+	0x1c, 0xdc, 0x41, 0x9d, 0xb9, 0xba, 0x80, 0xf8, 0xd9, 0x28, 0x4a, 0xde, 0x8c, 0xa7, 0x6c, 0xa7,
+	0xd2, 0xd9, 0xbf, 0x35, 0xb1, 0x19, 0xd4, 0x97, 0x75, 0x01, 0xe9, 0xfe, 0x5d, 0xfb, 0x41, 0x0f,
+	0x5b, 0xf5, 0x52, 0xdd, 0x43, 0xfc, 0x7c, 0x14, 0x25, 0xaf, 0xfd, 0xd1, 0x52, 0xdd, 0x03, 0x1d,
+	0x92, 0x03, 0x6d, 0xa5, 0x0f, 0x16, 0xf7, 0x46, 0x51, 0xf2, 0x22, 0x7d, 0x9c, 0x8f, 0x3e, 0x90,
+	0xfd, 0x20, 0x45, 0x07, 0xa4, 0x3f, 0x3b, 0xbb, 0xce, 0x2e, 0xaf, 0x17, 0x67, 0xd9, 0xd7, 0xf9,
+	0x6c, 0xfe, 0xe5, 0x6a, 0xde, 0x7f, 0x42, 0xfb, 0xe4, 0xd5, 0xe3, 0xdf, 0x74, 0x39, 0xed, 0x47,
+	0x27, 0xbf, 0x22, 0x92, 0x28, 0xbb, 0x5b, 0xf4, 0x93, 0xb7, 0xdd, 0xd4, 0x8b, 0xa6, 0xa7, 0x45,
+	0xf4, 0xed, 0x2a, 0xf4, 0x96, 0x5b, 0x2d, 0x4c, 0xce, 0x2c, 0xe6, 0xdc, 0x6f, 0xac, 0x44, 0x58,
+	0x81, 0x71, 0x4a, 0xe8, 0xd2, 0x6f, 0x6f, 0xf3, 0x24, 0x7f, 0x6f, 0xb0, 0x32, 0x0e, 0xd0, 0x08,
+	0x1d, 0xe6, 0xac, 0xeb, 0x76, 0xd3, 0xf3, 0x2f, 0x31, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff, 0x72,
+	0x44, 0x68, 0xc5, 0x01, 0x03, 0x00, 0x00,
+}
diff --git a/regenerate.sh b/regenerate.sh
index 987bc20251dd..e743241d53c8 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -39,6 +39,19 @@ mkdir -p ${WORKDIR}/googleapis/google/rpc
 echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
 curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > ${WORKDIR}/googleapis/google/rpc/code.proto
 
+# Pull in the following repos to build the MeshCA config proto.
+ENVOY_API_REPOS=(
+  "https://github.com/envoyproxy/data-plane-api"
+  "https://github.com/cncf/udpa"
+  "https://github.com/envoyproxy/protoc-gen-validate"
+)
+for repo in ${ENVOY_API_REPOS[@]}; do
+  dirname=$(basename ${repo})
+  mkdir -p ${WORKDIR}/${dirname}
+  echo "git clone ${repo}"
+  git clone --quiet ${repo} ${WORKDIR}/${dirname}
+done
+
 mkdir -p ${WORKDIR}/out
 
 SOURCES=(
@@ -52,15 +65,23 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/lb/v1/load_balancer.proto
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
+  ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
   $(git ls-files --exclude-standard --cached --others "*.proto")
 )
-OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config
+# These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
+# import path of 'bar' in the generated code when 'foo.proto' is imported in
+# one of the sources.
+OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config,\
+Menvoy/config/core/v3/config_source.proto=github.com/envoyproxy/go-control-plane/envoy/config/core/v3
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
   protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},requireUnimplementedServers=false:${WORKDIR}/out \
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
+    -I${WORKDIR}/data-plane-api \
+    -I${WORKDIR}/udpa \
+    -I${WORKDIR}/protoc-gen-validate \
     ${src}
 done
 
diff --git a/vet.sh b/vet.sh
index 29e2e371e544..c52c1bdbf4c9 100755
--- a/vet.sh
+++ b/vet.sh
@@ -90,7 +90,7 @@ git grep -l 'grpclog.I\|grpclog.W\|grpclog.E\|grpclog.F\|grpclog.V' -- "*.go" |
 not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go"
 
 # - Ensure all xds proto imports are renamed to *pb or *grpc.
-git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' | not grep -v 'pb "\|grpc "'
+git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' ':(exclude)*.pb.go' | not grep -v 'pb "\|grpc "'
 
 # - gofmt, goimports, golint (with exceptions for generated code), go vet.
 gofmt -s -d -l . 2>&1 | fail_on_output

From 1605756d150e52ee9b5e7e90af6c8afa6884a54f Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 13 Aug 2020 11:47:23 -0700
Subject: [PATCH 159/481] xds: add LRS balancing policy (#3799)

---
 xds/internal/balancer/lrs/balancer.go         | 216 ++++++++++++++++++
 xds/internal/balancer/lrs/balancer_test.go    | 175 ++++++++++++++
 xds/internal/balancer/lrs/config.go           |  54 +++++
 xds/internal/balancer/lrs/config_test.go      | 127 ++++++++++
 xds/internal/balancer/lrs/logging.go          |  34 +++
 xds/internal/balancer/lrs/picker.go           |  75 ++++++
 .../balancer/lrs/{lrs.go => store.go}         |   4 -
 .../lrs/{lrs_test.go => store_test.go}        |   0
 xds/internal/internal.go                      |  10 +-
 9 files changed, 684 insertions(+), 11 deletions(-)
 create mode 100644 xds/internal/balancer/lrs/balancer.go
 create mode 100644 xds/internal/balancer/lrs/balancer_test.go
 create mode 100644 xds/internal/balancer/lrs/config.go
 create mode 100644 xds/internal/balancer/lrs/config_test.go
 create mode 100644 xds/internal/balancer/lrs/logging.go
 create mode 100644 xds/internal/balancer/lrs/picker.go
 rename xds/internal/balancer/lrs/{lrs.go => store.go} (98%)
 rename xds/internal/balancer/lrs/{lrs_test.go => store_test.go} (100%)

diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
new file mode 100644
index 000000000000..4af91c76498a
--- /dev/null
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -0,0 +1,216 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package lrs implements load reporting balancer for xds.
+package lrs
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal"
+	xdsinternal "google.golang.org/grpc/xds/internal"
+)
+
+func init() {
+	balancer.Register(&lrsBB{})
+}
+
+const lrsBalancerName = "lrs_experimental"
+
+type lrsBB struct{}
+
+func (l *lrsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	b := &lrsBalancer{
+		cc:        cc,
+		buildOpts: opts,
+	}
+	b.loadStore = NewStore()
+	b.client = newXDSClientWrapper(b.loadStore)
+	b.logger = prefixLogger(b)
+	b.logger.Infof("Created")
+	return b
+}
+
+func (l *lrsBB) Name() string {
+	return lrsBalancerName
+}
+
+func (l *lrsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type lrsBalancer struct {
+	cc        balancer.ClientConn
+	buildOpts balancer.BuildOptions
+
+	logger    *grpclog.PrefixLogger
+	loadStore Store
+	client    *xdsClientWrapper
+
+	config *lbConfig
+	lb     balancer.Balancer // The sub balancer.
+}
+
+func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	// If child policy is a different type, recreate the sub-balancer.
+	if b.config == nil || b.config.ChildPolicy.Name != newConfig.ChildPolicy.Name {
+		bb := balancer.Get(newConfig.ChildPolicy.Name)
+		if bb == nil {
+			return fmt.Errorf("balancer %q not registered", newConfig.ChildPolicy.Name)
+		}
+		if b.lb != nil {
+			b.lb.Close()
+		}
+		b.lb = bb.Build(newCCWrapper(b.cc, b.loadStore, newConfig.Locality), b.buildOpts)
+	}
+	// Update load reporting config or xds client.
+	b.client.update(newConfig, s.ResolverState.Attributes)
+	b.config = newConfig
+
+	// Addresses and sub-balancer config are sent to sub-balancer.
+	return b.lb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: b.config.ChildPolicy.Config,
+	})
+}
+
+func (b *lrsBalancer) ResolverError(err error) {
+	if b.lb != nil {
+		b.lb.ResolverError(err)
+	}
+}
+
+func (b *lrsBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	if b.lb != nil {
+		b.lb.UpdateSubConnState(sc, s)
+	}
+}
+
+func (b *lrsBalancer) Close() {
+	if b.lb != nil {
+		b.lb.Close()
+		b.lb = nil
+	}
+	b.client.close()
+}
+
+type ccWrapper struct {
+	balancer.ClientConn
+	loadStore  Store
+	localityID *internal.LocalityID
+}
+
+func newCCWrapper(cc balancer.ClientConn, loadStore Store, localityID *internal.LocalityID) *ccWrapper {
+	return &ccWrapper{
+		ClientConn: cc,
+		loadStore:  loadStore,
+		localityID: localityID,
+	}
+}
+
+func (ccw *ccWrapper) UpdateState(s balancer.State) {
+	s.Picker = newLoadReportPicker(s.Picker, *ccw.localityID, ccw.loadStore)
+	ccw.ClientConn.UpdateState(s)
+}
+
+// xdsClientInterface contains only the xds_client methods needed by LRS
+// balancer. It's defined so we can override xdsclient in tests.
+type xdsClientInterface interface {
+	ReportLoad(server string, clusterName string, loadStore Store) (cancel func())
+	Close()
+}
+
+type xdsClientWrapper struct {
+	loadStore Store
+
+	c                xdsClientInterface
+	cancelLoadReport func()
+	clusterName      string
+	lrsServerName    string
+}
+
+func newXDSClientWrapper(loadStore Store) *xdsClientWrapper {
+	return &xdsClientWrapper{
+		loadStore: loadStore,
+	}
+}
+
+// update checks the config and xdsclient, and decides whether it needs to
+// restart the load reporting stream.
+//
+// TODO: refactor lrs to share one stream instead of one per EDS.
+func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attributes) {
+	var restartLoadReport bool
+	if attr != nil {
+		if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil {
+			if w.c != clientFromAttr {
+				// xds client is different, restart.
+				restartLoadReport = true
+				w.c = clientFromAttr
+			}
+		}
+	}
+
+	// ClusterName is different, restart. ClusterName is from ClusterName and
+	// EdsServiceName.
+	//
+	// TODO: LRS request actually has separate fields from these two values.
+	// Update lrs.Store to set both.
+	newClusterName := newConfig.EdsServiceName
+	if newClusterName == "" {
+		newClusterName = newConfig.ClusterName
+	}
+	if w.clusterName != newClusterName {
+		restartLoadReport = true
+		w.clusterName = newClusterName
+	}
+
+	if w.lrsServerName != newConfig.LrsLoadReportingServerName {
+		// LrsLoadReportingServerName is different, load should be report to a
+		// different server, restart.
+		restartLoadReport = true
+		w.lrsServerName = newConfig.LrsLoadReportingServerName
+	}
+
+	if restartLoadReport {
+		if w.cancelLoadReport != nil {
+			w.cancelLoadReport()
+			w.cancelLoadReport = nil
+		}
+		if w.c != nil {
+			w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName, w.clusterName, w.loadStore)
+		}
+	}
+}
+
+func (w *xdsClientWrapper) close() {
+	if w.cancelLoadReport != nil {
+		w.cancelLoadReport()
+		w.cancelLoadReport = nil
+	}
+}
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
new file mode 100644
index 000000000000..c57b5e1a127f
--- /dev/null
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrs
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+	xdsinternal "google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+var (
+	testBackendAddrs = []resolver.Address{
+		{Addr: "1.1.1.1:1"},
+	}
+	testLocality = &xdsinternal.LocalityID{
+		Region:  "test-region",
+		Zone:    "test-zone",
+		SubZone: "test-sub-zone",
+	}
+)
+
+// This is a subset of testutils.fakeclient. Cannot use testutils.fakeclient
+// because testutils imports package lrs.
+//
+// TODO: after refactoring xdsclient to support load reporting, the testutils
+// package won't need to depend on lrs package for the store. And we can use the
+// testutils for this.
+type fakeXDSClient struct {
+	loadReportCh chan *reportLoadArgs
+}
+
+func newFakeXDSClient() *fakeXDSClient {
+	return &fakeXDSClient{
+		loadReportCh: make(chan *reportLoadArgs, 10),
+	}
+}
+
+// reportLoadArgs wraps the arguments passed to ReportLoad.
+type reportLoadArgs struct {
+	// server is the name of the server to which the load is reported.
+	server string
+	// cluster is the name of the cluster for which load is reported.
+	cluster string
+	// loadStore is the store where loads are stored.
+	loadStore interface{}
+}
+
+// ReportLoad starts reporting load about clusterName to server.
+func (xdsC *fakeXDSClient) ReportLoad(server string, clusterName string, loadStore Store) (cancel func()) {
+	xdsC.loadReportCh <- &reportLoadArgs{server: server, cluster: clusterName, loadStore: loadStore}
+	return func() {}
+}
+
+// waitForReportLoad waits for ReportLoad to be invoked on this client within a
+// reasonable timeout, and returns the arguments passed to it.
+func (xdsC *fakeXDSClient) waitForReportLoad() (*reportLoadArgs, error) {
+	select {
+	case <-time.After(time.Second):
+		return nil, fmt.Errorf("timeout")
+	case a := <-xdsC.loadReportCh:
+		return a, nil
+	}
+}
+
+// Close closes the xds client.
+func (xdsC *fakeXDSClient) Close() {
+}
+
+// TestLoadReporting verifies that the lrs balancer starts the loadReport
+// stream when the lbConfig passed to it contains a valid value for the LRS
+// server (empty string).
+func TestLoadReporting(t *testing.T) {
+	builder := balancer.Get(lrsBalancerName)
+	cc := testutils.NewTestClientConn(t)
+	lrsB := builder.Build(cc, balancer.BuildOptions{})
+	defer lrsB.Close()
+
+	xdsC := newFakeXDSClient()
+	if err := lrsB.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses:  testBackendAddrs,
+			Attributes: attributes.New(xdsinternal.XDSClientID, xdsC),
+		},
+		BalancerConfig: &lbConfig{
+			EdsServiceName:             testClusterName,
+			LrsLoadReportingServerName: testLRSServerName,
+			Locality:                   testLocality,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	got, err := xdsC.waitForReportLoad()
+	if err != nil {
+		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
+	}
+	if got.server != testLRSServerName || got.cluster != testClusterName {
+		t.Fatalf("xdsClient.ReportLoad called with {%q, %q}: want {%q, %q}", got.server, got.cluster, testLRSServerName, testClusterName)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	lrsB.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	lrsB.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with one backend.
+	p1 := <-cc.NewPickerCh
+	const successCount = 5
+	for i := 0; i < successCount; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+		gotSCSt.Done(balancer.DoneInfo{})
+	}
+	const errorCount = 5
+	for i := 0; i < errorCount; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+		gotSCSt.Done(balancer.DoneInfo{Err: fmt.Errorf("error")})
+	}
+
+	loads := make(map[xdsinternal.LocalityID]*rpcCountData)
+
+	got.loadStore.(*lrsStore).localityRPCCount.Range(
+		func(key, value interface{}) bool {
+			loads[key.(xdsinternal.LocalityID)] = value.(*rpcCountData)
+			return true
+		},
+	)
+
+	countData, ok := loads[*testLocality]
+	if !ok {
+		t.Fatalf("loads for %v not found in store", testLocality)
+	}
+	if *countData.succeeded != successCount {
+		t.Errorf("got succeeded %v, want %v", *countData.succeeded, successCount)
+	}
+	if *countData.errored != errorCount {
+		t.Errorf("got errord %v, want %v", *countData.errored, errorCount)
+	}
+	if *countData.inProgress != 0 {
+		t.Errorf("got inProgress %v, want %v", *countData.inProgress, 0)
+	}
+}
diff --git a/xds/internal/balancer/lrs/config.go b/xds/internal/balancer/lrs/config.go
new file mode 100644
index 000000000000..3d39961401b5
--- /dev/null
+++ b/xds/internal/balancer/lrs/config.go
@@ -0,0 +1,54 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrs
+
+import (
+	"encoding/json"
+	"fmt"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal"
+)
+
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	ClusterName                string
+	EdsServiceName             string
+	LrsLoadReportingServerName string
+	Locality                   *internal.LocalityID
+	ChildPolicy                *internalserviceconfig.BalancerConfig
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	var cfg lbConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+	if cfg.ClusterName == "" {
+		return nil, fmt.Errorf("required ClusterName is not set in %+v", cfg)
+	}
+	if cfg.LrsLoadReportingServerName == "" {
+		return nil, fmt.Errorf("required LrsLoadReportingServerName is not set in %+v", cfg)
+	}
+	if cfg.Locality == nil {
+		return nil, fmt.Errorf("required Locality is not set in %+v", cfg)
+	}
+	return &cfg, nil
+}
diff --git a/xds/internal/balancer/lrs/config_test.go b/xds/internal/balancer/lrs/config_test.go
new file mode 100644
index 000000000000..f49430569fed
--- /dev/null
+++ b/xds/internal/balancer/lrs/config_test.go
@@ -0,0 +1,127 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrs
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer/roundrobin"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	xdsinternal "google.golang.org/grpc/xds/internal"
+)
+
+const (
+	testClusterName   = "test-cluster"
+	testServiceName   = "test-eds-service"
+	testLRSServerName = "test-lrs-name"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name: "no cluster name",
+			js: `{
+  "edsServiceName": "test-eds-service",
+  "lrsLoadReportingServerName": "test-lrs-name",
+  "locality": {
+    "region": "test-region",
+    "zone": "test-zone",
+    "subZone": "test-sub-zone"
+  },
+  "childPolicy":[{"round_robin":{}}]
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "no LRS server name",
+			js: `{
+  "clusterName": "test-cluster",
+  "edsServiceName": "test-eds-service",
+  "locality": {
+    "region": "test-region",
+    "zone": "test-zone",
+    "subZone": "test-sub-zone"
+  },
+  "childPolicy":[{"round_robin":{}}]
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "no locality",
+			js: `{
+  "clusterName": "test-cluster",
+  "edsServiceName": "test-eds-service",
+  "lrsLoadReportingServerName": "test-lrs-name",
+  "childPolicy":[{"round_robin":{}}]
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "good",
+			js: `{
+  "clusterName": "test-cluster",
+  "edsServiceName": "test-eds-service",
+  "lrsLoadReportingServerName": "test-lrs-name",
+  "locality": {
+    "region": "test-region",
+    "zone": "test-zone",
+    "subZone": "test-sub-zone"
+  },
+  "childPolicy":[{"round_robin":{}}]
+}
+			`,
+			want: &lbConfig{
+				ClusterName:                testClusterName,
+				EdsServiceName:             testServiceName,
+				LrsLoadReportingServerName: testLRSServerName,
+				Locality: &xdsinternal.LocalityID{
+					Region:  "test-region",
+					Zone:    "test-zone",
+					SubZone: "test-sub-zone",
+				},
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name:   roundrobin.Name,
+					Config: nil,
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(got, tt.want); diff != "" {
+				t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff)
+			}
+		})
+	}
+}
diff --git a/xds/internal/balancer/lrs/logging.go b/xds/internal/balancer/lrs/logging.go
new file mode 100644
index 000000000000..602dac099597
--- /dev/null
+++ b/xds/internal/balancer/lrs/logging.go
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrs
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[lrs-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *lrsBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}
diff --git a/xds/internal/balancer/lrs/picker.go b/xds/internal/balancer/lrs/picker.go
new file mode 100644
index 000000000000..1fcc6e9e5b31
--- /dev/null
+++ b/xds/internal/balancer/lrs/picker.go
@@ -0,0 +1,75 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrs
+
+import (
+	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/xds/internal"
+)
+
+const (
+	serverLoadCPUName    = "cpu_utilization"
+	serverLoadMemoryName = "mem_utilization"
+)
+
+type loadReportPicker struct {
+	p balancer.Picker
+
+	id        internal.LocalityID
+	loadStore Store
+}
+
+func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore Store) *loadReportPicker {
+	return &loadReportPicker{
+		p:         p,
+		id:        id,
+		loadStore: loadStore,
+	}
+}
+
+func (lrp *loadReportPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	res, err := lrp.p.Pick(info)
+	if err != nil {
+		return res, err
+	}
+
+	lrp.loadStore.CallStarted(lrp.id)
+	oldDone := res.Done
+	res.Done = func(info balancer.DoneInfo) {
+		if oldDone != nil {
+			oldDone(info)
+		}
+		lrp.loadStore.CallFinished(lrp.id, info.Err)
+
+		load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport)
+		if !ok {
+			return
+		}
+		lrp.loadStore.CallServerLoad(lrp.id, serverLoadCPUName, load.CpuUtilization)
+		lrp.loadStore.CallServerLoad(lrp.id, serverLoadMemoryName, load.MemUtilization)
+		for n, d := range load.RequestCost {
+			lrp.loadStore.CallServerLoad(lrp.id, n, d)
+		}
+		for n, d := range load.Utilization {
+			lrp.loadStore.CallServerLoad(lrp.id, n, d)
+		}
+	}
+	return res, err
+}
diff --git a/xds/internal/balancer/lrs/lrs.go b/xds/internal/balancer/lrs/store.go
similarity index 98%
rename from xds/internal/balancer/lrs/lrs.go
rename to xds/internal/balancer/lrs/store.go
index 4bc20ef6e124..96c85f9cc9cd 100644
--- a/xds/internal/balancer/lrs/lrs.go
+++ b/xds/internal/balancer/lrs/store.go
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-// Package lrs implements load reporting service for xds balancer.
 package lrs
 
 import (
@@ -29,15 +28,12 @@ import (
 	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
 	"github.com/golang/protobuf/ptypes"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/xds/internal"
 )
 
 const negativeOneUInt64 = ^uint64(0)
 
-var logger = grpclog.Component("xds")
-
 // Store defines the interface for a load store. It keeps loads and can report
 // them to a server when requested.
 type Store interface {
diff --git a/xds/internal/balancer/lrs/lrs_test.go b/xds/internal/balancer/lrs/store_test.go
similarity index 100%
rename from xds/internal/balancer/lrs/lrs_test.go
rename to xds/internal/balancer/lrs/store_test.go
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index 8b17cf930242..462a8bac59b2 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -34,14 +34,10 @@ const XDSClientID = clientID("xdsClientID")
 // keys.
 //
 // xds.Locality cannot be map keys because one of the XXX fields is a slice.
-//
-// This struct should only be used as map keys. Use the proto message directly
-// in all other places.
-//
 type LocalityID struct {
-	Region  string
-	Zone    string
-	SubZone string
+	Region  string `json:"region,omitempty"`
+	Zone    string `json:"zone,omitempty"`
+	SubZone string `json:"subZone,omitempty"`
 }
 
 func (l LocalityID) String() string {

From 9310253e119a6398cb09973a9822d90b93d9a53a Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 13 Aug 2020 12:27:18 -0700
Subject: [PATCH 160/481] meshca: Pull in the MeshCA service proto from istio
 repo. (#3814)

---
 .../meshca/internal/v1/meshca.pb.go           | 155 ++++++++++++++++++
 .../meshca/internal/v1/meshca_grpc.pb.go      |  92 +++++++++++
 regenerate.sh                                 |  13 +-
 3 files changed, 259 insertions(+), 1 deletion(-)
 create mode 100644 credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
 create mode 100644 credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go

diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
new file mode 100644
index 000000000000..16122f768488
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
@@ -0,0 +1,155 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: istio/google/security/meshca/v1/meshca.proto
+
+package google_security_meshca_v1
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	duration "github.com/golang/protobuf/ptypes/duration"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+// Certificate request message.
+type MeshCertificateRequest struct {
+	// The request ID must be a valid UUID with the exception that zero UUID is
+	// not supported (00000000-0000-0000-0000-000000000000).
+	RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	// PEM-encoded certificate request.
+	Csr string `protobuf:"bytes,2,opt,name=csr,proto3" json:"csr,omitempty"`
+	// Optional: requested certificate validity period.
+	Validity             *duration.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
+	XXX_unrecognized     []byte             `json:"-"`
+	XXX_sizecache        int32              `json:"-"`
+}
+
+func (m *MeshCertificateRequest) Reset()         { *m = MeshCertificateRequest{} }
+func (m *MeshCertificateRequest) String() string { return proto.CompactTextString(m) }
+func (*MeshCertificateRequest) ProtoMessage()    {}
+func (*MeshCertificateRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_f72841047b94fe5e, []int{0}
+}
+
+func (m *MeshCertificateRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_MeshCertificateRequest.Unmarshal(m, b)
+}
+func (m *MeshCertificateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_MeshCertificateRequest.Marshal(b, m, deterministic)
+}
+func (m *MeshCertificateRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MeshCertificateRequest.Merge(m, src)
+}
+func (m *MeshCertificateRequest) XXX_Size() int {
+	return xxx_messageInfo_MeshCertificateRequest.Size(m)
+}
+func (m *MeshCertificateRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_MeshCertificateRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MeshCertificateRequest proto.InternalMessageInfo
+
+func (m *MeshCertificateRequest) GetRequestId() string {
+	if m != nil {
+		return m.RequestId
+	}
+	return ""
+}
+
+func (m *MeshCertificateRequest) GetCsr() string {
+	if m != nil {
+		return m.Csr
+	}
+	return ""
+}
+
+func (m *MeshCertificateRequest) GetValidity() *duration.Duration {
+	if m != nil {
+		return m.Validity
+	}
+	return nil
+}
+
+// Certificate response message.
+type MeshCertificateResponse struct {
+	// PEM-encoded certificate chain.
+	// Leaf cert is element '0'. Root cert is element 'n'.
+	CertChain            []string `protobuf:"bytes,1,rep,name=cert_chain,json=certChain,proto3" json:"cert_chain,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *MeshCertificateResponse) Reset()         { *m = MeshCertificateResponse{} }
+func (m *MeshCertificateResponse) String() string { return proto.CompactTextString(m) }
+func (*MeshCertificateResponse) ProtoMessage()    {}
+func (*MeshCertificateResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_f72841047b94fe5e, []int{1}
+}
+
+func (m *MeshCertificateResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_MeshCertificateResponse.Unmarshal(m, b)
+}
+func (m *MeshCertificateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_MeshCertificateResponse.Marshal(b, m, deterministic)
+}
+func (m *MeshCertificateResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MeshCertificateResponse.Merge(m, src)
+}
+func (m *MeshCertificateResponse) XXX_Size() int {
+	return xxx_messageInfo_MeshCertificateResponse.Size(m)
+}
+func (m *MeshCertificateResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_MeshCertificateResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MeshCertificateResponse proto.InternalMessageInfo
+
+func (m *MeshCertificateResponse) GetCertChain() []string {
+	if m != nil {
+		return m.CertChain
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*MeshCertificateRequest)(nil), "google.security.meshca.v1.MeshCertificateRequest")
+	proto.RegisterType((*MeshCertificateResponse)(nil), "google.security.meshca.v1.MeshCertificateResponse")
+}
+
+func init() {
+	proto.RegisterFile("istio/google/security/meshca/v1/meshca.proto", fileDescriptor_f72841047b94fe5e)
+}
+
+var fileDescriptor_f72841047b94fe5e = []byte{
+	// 284 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4b, 0xc3, 0x30,
+	0x1c, 0xc5, 0x8d, 0x05, 0x71, 0xd9, 0x45, 0x73, 0xd0, 0x6e, 0x30, 0x29, 0x3d, 0xed, 0x20, 0x29,
+	0xad, 0x08, 0x9e, 0x57, 0x2f, 0x1e, 0x84, 0x51, 0x3f, 0xc0, 0xc8, 0xd2, 0xff, 0xd6, 0x3f, 0x6c,
+	0xcd, 0x4c, 0xd2, 0xc2, 0xc0, 0x83, 0x9f, 0xc2, 0xcf, 0x2b, 0x69, 0x33, 0x11, 0xe6, 0x0e, 0xde,
+	0x1e, 0x2f, 0xef, 0xe5, 0xff, 0xe3, 0xd1, 0x7b, 0x34, 0x16, 0x55, 0xb2, 0x56, 0x6a, 0xbd, 0x81,
+	0xc4, 0x80, 0x6c, 0x34, 0xda, 0x7d, 0xb2, 0x05, 0x53, 0x49, 0x91, 0xb4, 0xa9, 0x57, 0x7c, 0xa7,
+	0x95, 0x55, 0x6c, 0xd4, 0xe7, 0xf8, 0x21, 0xc7, 0xfd, 0x6b, 0x9b, 0x8e, 0xef, 0xfc, 0x17, 0x5d,
+	0x70, 0xd9, 0xac, 0x92, 0xb2, 0xd1, 0xc2, 0xa2, 0xaa, 0xfb, 0x6a, 0xfc, 0x49, 0xe8, 0xcd, 0x2b,
+	0x98, 0x2a, 0x07, 0x6d, 0x71, 0x85, 0x52, 0x58, 0x28, 0xe0, 0xbd, 0x01, 0x63, 0xd9, 0x84, 0x52,
+	0xdd, 0xcb, 0x05, 0x96, 0x21, 0x89, 0xc8, 0x74, 0x50, 0x0c, 0xbc, 0xf3, 0x52, 0xb2, 0x2b, 0x1a,
+	0x48, 0xa3, 0xc3, 0xf3, 0xce, 0x77, 0x92, 0x3d, 0xd2, 0xcb, 0x56, 0x6c, 0xb0, 0x44, 0xbb, 0x0f,
+	0x83, 0x88, 0x4c, 0x87, 0xd9, 0x88, 0x7b, 0xb2, 0xc3, 0x79, 0xfe, 0xec, 0xcf, 0x17, 0x3f, 0xd1,
+	0xf8, 0x89, 0xde, 0x1e, 0x11, 0x98, 0x9d, 0xaa, 0x0d, 0x38, 0x04, 0x09, 0xda, 0x2e, 0x64, 0x25,
+	0xb0, 0x0e, 0x49, 0x14, 0x38, 0x04, 0xe7, 0xe4, 0xce, 0xc8, 0xbe, 0x8e, 0xe1, 0xdf, 0x40, 0xb7,
+	0x28, 0x81, 0x7d, 0xd0, 0xeb, 0x5c, 0x83, 0xb0, 0xf0, 0xeb, 0x8d, 0xa5, 0xfc, 0xe4, 0x50, 0xfc,
+	0xef, 0x11, 0xc6, 0xd9, 0x7f, 0x2a, 0x3d, 0x75, 0x7c, 0x36, 0xe3, 0x74, 0x22, 0xd5, 0xf6, 0x74,
+	0x75, 0x36, 0xec, 0xba, 0x62, 0xee, 0x66, 0x99, 0x93, 0xe5, 0x45, 0xb7, 0xcf, 0xc3, 0x77, 0x00,
+	0x00, 0x00, 0xff, 0xff, 0xb7, 0x0d, 0xfd, 0xff, 0xf7, 0x01, 0x00, 0x00,
+}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
new file mode 100644
index 000000000000..2361ac632705
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -0,0 +1,92 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package google_security_meshca_v1
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// MeshCertificateServiceClient is the client API for MeshCertificateService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type MeshCertificateServiceClient interface {
+	// Using provided CSR, returns a signed certificate that represents a GCP
+	// service account identity.
+	CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error)
+}
+
+type meshCertificateServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient {
+	return &meshCertificateServiceClient{cc}
+}
+
+func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
+	out := new(MeshCertificateResponse)
+	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// MeshCertificateServiceServer is the server API for MeshCertificateService service.
+// All implementations should embed UnimplementedMeshCertificateServiceServer
+// for forward compatibility
+type MeshCertificateServiceServer interface {
+	// Using provided CSR, returns a signed certificate that represents a GCP
+	// service account identity.
+	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
+}
+
+// UnimplementedMeshCertificateServiceServer should be embedded to have forward compatible implementations.
+type UnimplementedMeshCertificateServiceServer struct {
+}
+
+func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
+}
+
+func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) {
+	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
+}
+
+func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(MeshCertificateRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "google.security.meshca.v1.MeshCertificateService",
+	HandlerType: (*MeshCertificateServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "CreateCertificate",
+			Handler:    _MeshCertificateService_CreateCertificate_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "istio/google/security/meshca/v1/meshca.proto",
+}
diff --git a/regenerate.sh b/regenerate.sh
index e743241d53c8..cb7e5b61beb0 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -52,6 +52,11 @@ for repo in ${ENVOY_API_REPOS[@]}; do
   git clone --quiet ${repo} ${WORKDIR}/${dirname}
 done
 
+# Pull in the MeshCA service proto.
+mkdir -p ${WORKDIR}/istio/istio/google/security/meshca/v1
+echo "curl https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto"
+curl --silent https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto > ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
+
 mkdir -p ${WORKDIR}/out
 
 SOURCES=(
@@ -66,6 +71,7 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
   ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
+  ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
   $(git ls-files --exclude-standard --cached --others "*.proto")
 )
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
@@ -82,6 +88,7 @@ for src in ${SOURCES[@]}; do
     -I${WORKDIR}/data-plane-api \
     -I${WORKDIR}/udpa \
     -I${WORKDIR}/protoc-gen-validate \
+    -I${WORKDIR}/istio \
     ${src}
 done
 
@@ -95,6 +102,10 @@ mv ${WORKDIR}/out/google.golang.org/grpc/lookup/grpc_lookup_v1/* ${WORKDIR}/out/
 rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/*.pb.go
 
 # grpc/service_config/service_config.proto does not have a go_package option.
-cp ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config
+mv ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config
+
+# istio/google/security/meshca/v1/meshca.proto does not have a go_package option.
+mkdir -p ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/
+mv ${WORKDIR}/out/istio/google/security/meshca/v1/* ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/
 
 cp -R ${WORKDIR}/out/google.golang.org/grpc/* .

From 6768e33c1f90bcaf0d06cc67a207cf9b46e6d3e2 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 13 Aug 2020 12:40:42 -0700
Subject: [PATCH 161/481] xds: Fix TestV2ClientBackoffAfterRecvError (#3815)

The failure seems to have been because of the following:
* After the stream failure, and after the backoff, the client tries to
  register all its existing watches. And there seems to have been a race
  between the re-register and the client close. Hence a log message was
  getting printing saying "send failed because of EOF".

I've fixed this by making sure that the test waits for the xDS request
corresponding to the re-registered watch. I've also tried running it a
1000 times without a failure.
---
 xds/internal/client/v2/client_test.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index fa4a3c9c0ce5..024357d8476e 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -508,14 +508,14 @@ func newV2Client(p xdsclient.UpdateHandler, cc *grpc.ClientConn, n *basepb.Node,
 	return c.(*client), nil
 }
 
-// TestV2ClientBackoffAfterRecvError verifies if the v2Client backoffs when it
+// TestV2ClientBackoffAfterRecvError verifies if the v2Client backs off when it
 // encounters a Recv error while receiving an LDS response.
 func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
 	// Override the v2Client backoff function with this, so that we can verify
-	// that a backoff actually was triggerred.
+	// that a backoff actually was triggered.
 	boCh := make(chan int, 1)
 	clientBackoff := func(v int) time.Duration {
 		boCh <- v
@@ -532,7 +532,6 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	defer v2c.Close()
 	t.Log("Started xds v2Client...")
 
-	// v2c.watchLDS(goodLDSTarget1, func(u ldsUpdate, err error) {})
 	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
@@ -552,6 +551,11 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	case <-callbackCh:
 		t.Fatal("Received unexpected LDS callback")
 	}
+
+	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+		t.Fatalf("Timeout expired when expecting an LDS request")
+	}
+	t.Log("FakeServer received request after backoff...")
 }
 
 // TestV2ClientRetriesAfterBrokenStream verifies the case where a stream

From 1dbe29f6699217a28876877811024e9778b3bc34 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 13 Aug 2020 14:26:49 -0700
Subject: [PATCH 162/481] examples: update grpc version (#3813)

---
 examples/features/debugging/client/main.go      | 2 +-
 examples/features/load_balancing/README.md      | 7 ++++---
 examples/features/load_balancing/client/main.go | 4 ++--
 examples/go.mod                                 | 2 +-
 examples/go.sum                                 | 4 ++--
 5 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/examples/features/debugging/client/main.go b/examples/features/debugging/client/main.go
index 29ac0fe89920..faf6b5d5fa2f 100644
--- a/examples/features/debugging/client/main.go
+++ b/examples/features/debugging/client/main.go
@@ -53,7 +53,7 @@ func main() {
 	/***** Initialize manual resolver and Dial *****/
 	r := manual.NewBuilderWithScheme("whatever")
 	// Set up a connection to the server.
-	conn, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName("round_robin"))
+	conn, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`))
 	if err != nil {
 		log.Fatalf("did not connect: %v", err)
 	}
diff --git a/examples/features/load_balancing/README.md b/examples/features/load_balancing/README.md
index dccdf706fbb3..f874f3b8f6fe 100644
--- a/examples/features/load_balancing/README.md
+++ b/examples/features/load_balancing/README.md
@@ -25,9 +25,10 @@ with `this is examples/load_balancing (from :50051)`.
 Two clients are created, to connect to both of these servers (they get both
 server addresses from the name resolver).
 
-Each client picks a different load balancer (using `grpc.WithBalancerName`):
-`pick_first` or `round_robin`. (These two policies are supported in gRPC by
-default. To add a custom balancing policy, implement the interfaces defined in
+Each client picks a different load balancer (using
+`grpc.WithDefaultServiceConfig`): `pick_first` or `round_robin`. (These two
+policies are supported in gRPC by default. To add a custom balancing policy,
+implement the interfaces defined in
 https://godoc.org/google.golang.org/grpc/balancer).
 
 Note that balancers can also be switched using service config, which allows
diff --git a/examples/features/load_balancing/client/main.go b/examples/features/load_balancing/client/main.go
index 8cf0ba03b392..1578df16671b 100644
--- a/examples/features/load_balancing/client/main.go
+++ b/examples/features/load_balancing/client/main.go
@@ -55,9 +55,9 @@ func makeRPCs(cc *grpc.ClientConn, n int) {
 }
 
 func main() {
+	// "pick_first" is the default, so there's no need to set the load balancer.
 	pickfirstConn, err := grpc.Dial(
 		fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName),
-		// grpc.WithBalancerName("pick_first"), // "pick_first" is the default, so this DialOption is not necessary.
 		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
@@ -74,7 +74,7 @@ func main() {
 	// Make another ClientConn with round_robin policy.
 	roundrobinConn, err := grpc.Dial(
 		fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName),
-		grpc.WithBalancerName("round_robin"), // This sets the initial balancing policy.
+		grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`), // This sets the initial balancing policy.
 		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
diff --git a/examples/go.mod b/examples/go.mod
index c8e7dab30b70..5235f543f576 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -6,5 +6,5 @@ require (
 	github.com/golang/protobuf v1.4.2
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
-	google.golang.org/grpc v1.30.0
+	google.golang.org/grpc v1.31.0
 )
diff --git a/examples/go.sum b/examples/go.sum
index 7e80c324a9af..f55a46171240 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -71,8 +71,8 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

From f30caa90add585c4778cc480700acd755ea2c1f7 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 14 Aug 2020 10:26:20 -0700
Subject: [PATCH 163/481] server: Add ServiceRegistrar interface. (#3816)

---
 server.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/server.go b/server.go
index 867b012deff5..85beec413974 100644
--- a/server.go
+++ b/server.go
@@ -540,6 +540,18 @@ func (s *Server) errorf(format string, a ...interface{}) {
 	}
 }
 
+// ServiceRegistrar wraps a single method that supports service registration. It
+// enables users to pass concrete types other than grpc.Server to the service
+// registration methods exported by the IDL generated code.
+type ServiceRegistrar interface {
+	// RegisterService registers a service and its implementation to the
+	// concrete type implementing this interface.  It may not be called
+	// once the server has started serving.
+	// desc describes the service and its methods and handlers. impl is the
+	// service implementation which is passed to the method handlers.
+	RegisterService(desc *ServiceDesc, impl interface{})
+}
+
 // RegisterService registers a service and its implementation to the gRPC
 // server. It is called from the IDL generated code. This must be called before
 // invoking Serve.

From a3740e5ed326410515aa921bbfe5688595732b77 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 14 Aug 2020 13:07:10 -0700
Subject: [PATCH 164/481] grpc: Bump up support package version. (#3818)

This was missed out in https://github.com/grpc/grpc-go/pull/3816.
---
 rpc_util.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rpc_util.go b/rpc_util.go
index 8644b8a7d0da..d4870ba4a996 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -866,7 +866,7 @@ type channelzData struct {
 
 // The SupportPackageIsVersion variables are referenced from generated protocol
 // buffer files to ensure compatibility with the gRPC version used.  The latest
-// support package version is 6.
+// support package version is 7.
 //
 // Older versions are kept for compatibility. They may be removed if
 // compatibility cannot be maintained.
@@ -877,6 +877,7 @@ const (
 	SupportPackageIsVersion4 = true
 	SupportPackageIsVersion5 = true
 	SupportPackageIsVersion6 = true
+	SupportPackageIsVersion7 = true
 )
 
 const grpcUA = "grpc-go/" + Version

From 0f73133e3aa3bf4505007ebd0cf3508dfc952a22 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 18 Aug 2020 15:40:27 -0700
Subject: [PATCH 165/481] xds: Export a WatchListener() method on the
 xdsClient. (#3817)

---
 xds/internal/client/client.go                 |  66 +++++++-
 xds/internal/client/client_callback.go        |  12 +-
 xds/internal/client/client_test.go            |  34 ++--
 xds/internal/client/client_watchers.go        |  92 +++++-----
 .../client/client_watchers_cluster_test.go    |  21 ++-
 .../client/client_watchers_endpoints_test.go  |  16 +-
 .../client/client_watchers_lds_test.go        |  17 +-
 .../client/client_watchers_rds_test.go        |  13 +-
 .../client/client_watchers_service_test.go    |  40 ++---
 xds/internal/client/client_xds.go             |  22 +--
 xds/internal/client/transport_helper.go       | 105 ++++++------
 xds/internal/client/v2/client.go              |  62 ++++---
 xds/internal/client/v2/client_ack_test.go     | 157 +++++++++---------
 xds/internal/client/v2/client_cds_test.go     |   4 +-
 xds/internal/client/v2/client_eds_test.go     |   4 +-
 xds/internal/client/v2/client_lds_test.go     |   6 +-
 xds/internal/client/v2/client_rds_test.go     |  10 +-
 xds/internal/client/v2/client_test.go         |  52 +++---
 xds/internal/client/v3/client.go              |  64 ++++---
 xds/xds.go                                    |   1 +
 20 files changed, 445 insertions(+), 353 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 7088c63befcb..2f260643777b 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -96,10 +96,10 @@ type APIClientBuilder interface {
 // version specific implementations of the xDS client.
 type APIClient interface {
 	// AddWatch adds a watch for an xDS resource given its type and name.
-	AddWatch(resourceType, resourceName string)
+	AddWatch(ResourceType, string)
 	// RemoveWatch cancels an already registered watch for an xDS resource
 	// given its type and name.
-	RemoveWatch(resourceType, resourceName string)
+	RemoveWatch(ResourceType, string)
 	// Close cleans up resources allocated by the API client.
 	Close()
 }
@@ -396,3 +396,65 @@ func (c *Client) Close() {
 	c.cc.Close()
 	c.logger.Infof("Shutdown")
 }
+
+// ResourceType identifies resources in a transport protocol agnostic way. These
+// will be used in transport version agnostic code, while the versioned API
+// clients will map these to appropriate version URLs.
+type ResourceType int
+
+// Version agnostic resource type constants.
+const (
+	UnknownResource ResourceType = iota
+	ListenerResource
+	HTTPConnManagerResource
+	RouteConfigResource
+	ClusterResource
+	EndpointsResource
+)
+
+func (r ResourceType) String() string {
+	switch r {
+	case ListenerResource:
+		return "ListenerResource"
+	case HTTPConnManagerResource:
+		return "HTTPConnManagerResource"
+	case RouteConfigResource:
+		return "RouteConfigResource"
+	case ClusterResource:
+		return "ClusterResource"
+	case EndpointsResource:
+		return "EndpointsResource"
+	default:
+		return "UnknownResource"
+	}
+}
+
+// IsListenerResource returns true if the provider URL corresponds to an xDS
+// Listener resource.
+func IsListenerResource(url string) bool {
+	return url == version.V2ListenerURL || url == version.V3ListenerURL
+}
+
+// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS
+// HTTPConnManager resource.
+func IsHTTPConnManagerResource(url string) bool {
+	return url == version.V2HTTPConnManagerURL || url == version.V3HTTPConnManagerURL
+}
+
+// IsRouteConfigResource returns true if the provider URL corresponds to an xDS
+// RouteConfig resource.
+func IsRouteConfigResource(url string) bool {
+	return url == version.V2RouteConfigURL || url == version.V3RouteConfigURL
+}
+
+// IsClusterResource returns true if the provider URL corresponds to an xDS
+// Cluster resource.
+func IsClusterResource(url string) bool {
+	return url == version.V2ClusterURL || url == version.V3ClusterURL
+}
+
+// IsEndpointsResource returns true if the provider URL corresponds to an xDS
+// Endpoints resource.
+func IsEndpointsResource(url string) bool {
+	return url == version.V2EndpointsURL || url == version.V3EndpointsURL
+}
diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go
index a00257e4e658..a135dae745b9 100644
--- a/xds/internal/client/client_callback.go
+++ b/xds/internal/client/client_callback.go
@@ -18,8 +18,6 @@
 
 package client
 
-import "google.golang.org/grpc/xds/internal/version"
-
 type watcherInfoWithUpdate struct {
 	wi     *watchInfo
 	update interface{}
@@ -46,20 +44,20 @@ func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
 	// window that a watcher's callback could be called after the watcher is
 	// canceled, and the user needs to take care of it.
 	var ccb func()
-	switch wiu.wi.typeURL {
-	case version.V2ListenerURL:
+	switch wiu.wi.rType {
+	case ListenerResource:
 		if s, ok := c.ldsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
 			ccb = func() { wiu.wi.ldsCallback(wiu.update.(ListenerUpdate), wiu.err) }
 		}
-	case version.V2RouteConfigURL:
+	case RouteConfigResource:
 		if s, ok := c.rdsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
 			ccb = func() { wiu.wi.rdsCallback(wiu.update.(RouteConfigUpdate), wiu.err) }
 		}
-	case version.V2ClusterURL:
+	case ClusterResource:
 		if s, ok := c.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
 			ccb = func() { wiu.wi.cdsCallback(wiu.update.(ClusterUpdate), wiu.err) }
 		}
-	case version.V2EndpointsURL:
+	case EndpointsResource:
 		if s, ok := c.edsWatchers[wiu.wi.target]; ok && s[wiu.wi] {
 			ccb = func() { wiu.wi.edsCallback(wiu.update.(EndpointsUpdate), wiu.err) }
 		}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 9584e28b9d17..729be4cc0726 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -67,8 +67,8 @@ func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
 type testAPIClient struct {
 	r UpdateHandler
 
-	addWatches    map[string]*testutils.Channel
-	removeWatches map[string]*testutils.Channel
+	addWatches    map[ResourceType]*testutils.Channel
+	removeWatches map[ResourceType]*testutils.Channel
 }
 
 func overrideNewAPIClient() (<-chan *testAPIClient, func()) {
@@ -83,16 +83,18 @@ func overrideNewAPIClient() (<-chan *testAPIClient, func()) {
 }
 
 func newTestAPIClient(r UpdateHandler) *testAPIClient {
-	addWatches := make(map[string]*testutils.Channel)
-	addWatches[version.V2ListenerURL] = testutils.NewChannel()
-	addWatches[version.V2RouteConfigURL] = testutils.NewChannel()
-	addWatches[version.V2ClusterURL] = testutils.NewChannel()
-	addWatches[version.V2EndpointsURL] = testutils.NewChannel()
-	removeWatches := make(map[string]*testutils.Channel)
-	removeWatches[version.V2ListenerURL] = testutils.NewChannel()
-	removeWatches[version.V2RouteConfigURL] = testutils.NewChannel()
-	removeWatches[version.V2ClusterURL] = testutils.NewChannel()
-	removeWatches[version.V2EndpointsURL] = testutils.NewChannel()
+	addWatches := map[ResourceType]*testutils.Channel{
+		ListenerResource:    testutils.NewChannel(),
+		RouteConfigResource: testutils.NewChannel(),
+		ClusterResource:     testutils.NewChannel(),
+		EndpointsResource:   testutils.NewChannel(),
+	}
+	removeWatches := map[ResourceType]*testutils.Channel{
+		ListenerResource:    testutils.NewChannel(),
+		RouteConfigResource: testutils.NewChannel(),
+		ClusterResource:     testutils.NewChannel(),
+		EndpointsResource:   testutils.NewChannel(),
+	}
 	return &testAPIClient{
 		r:             r,
 		addWatches:    addWatches,
@@ -100,11 +102,11 @@ func newTestAPIClient(r UpdateHandler) *testAPIClient {
 	}
 }
 
-func (c *testAPIClient) AddWatch(resourceType, resourceName string) {
+func (c *testAPIClient) AddWatch(resourceType ResourceType, resourceName string) {
 	c.addWatches[resourceType].Send(resourceName)
 }
 
-func (c *testAPIClient) RemoveWatch(resourceType, resourceName string) {
+func (c *testAPIClient) RemoveWatch(resourceType ResourceType, resourceName string) {
 	c.removeWatches[resourceType].Send(resourceName)
 }
 
@@ -130,12 +132,12 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		// Calls another watch inline, to ensure there's deadlock.
 		c.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
-		if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); firstTime && err != nil {
+		if _, err := v2Client.addWatches[ClusterResource].Receive(); firstTime && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 		firstTime = false
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index 2cfec37dcc5f..7f67be894226 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -22,8 +22,6 @@ import (
 	"fmt"
 	"sync"
 	"time"
-
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 type watchInfoState int
@@ -37,9 +35,9 @@ const (
 
 // watchInfo holds all the information from a watch() call.
 type watchInfo struct {
-	c       *Client
-	typeURL string
-	target  string
+	c      *Client
+	rType  ResourceType
+	target string
 
 	ldsCallback func(ListenerUpdate, error)
 	rdsCallback func(RouteConfigUpdate, error)
@@ -74,7 +72,7 @@ func (wi *watchInfo) resourceNotFound() {
 	}
 	wi.state = watchInfoStateRespReceived
 	wi.expiryTimer.Stop()
-	wi.sendErrorLocked(NewErrorf(ErrorTypeResourceNotFound, "xds: %s target %s not found in received response", wi.typeURL, wi.target))
+	wi.sendErrorLocked(NewErrorf(ErrorTypeResourceNotFound, "xds: %v target %s not found in received response", wi.rType, wi.target))
 }
 
 func (wi *watchInfo) timeout() {
@@ -84,7 +82,7 @@ func (wi *watchInfo) timeout() {
 		return
 	}
 	wi.state = watchInfoStateTimeout
-	wi.sendErrorLocked(fmt.Errorf("xds: %s target %s not found, watcher timeout", wi.typeURL, wi.target))
+	wi.sendErrorLocked(fmt.Errorf("xds: %v target %s not found, watcher timeout", wi.rType, wi.target))
 }
 
 // Caller must hold wi.mu.
@@ -92,14 +90,14 @@ func (wi *watchInfo) sendErrorLocked(err error) {
 	var (
 		u interface{}
 	)
-	switch wi.typeURL {
-	case version.V2ListenerURL:
+	switch wi.rType {
+	case ListenerResource:
 		u = ListenerUpdate{}
-	case version.V2RouteConfigURL:
+	case RouteConfigResource:
 		u = RouteConfigUpdate{}
-	case version.V2ClusterURL:
+	case ClusterResource:
 		u = ClusterUpdate{}
-	case version.V2EndpointsURL:
+	case EndpointsResource:
 		u = EndpointsUpdate{}
 	}
 	wi.c.scheduleCallback(wi, u, err)
@@ -118,54 +116,54 @@ func (wi *watchInfo) cancel() {
 func (c *Client) watch(wi *watchInfo) (cancel func()) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.logger.Debugf("new watch for type %v, resource name %v", wi.typeURL, wi.target)
+	c.logger.Debugf("new watch for type %v, resource name %v", wi.rType, wi.target)
 	var watchers map[string]map[*watchInfo]bool
-	switch wi.typeURL {
-	case version.V2ListenerURL:
+	switch wi.rType {
+	case ListenerResource:
 		watchers = c.ldsWatchers
-	case version.V2RouteConfigURL:
+	case RouteConfigResource:
 		watchers = c.rdsWatchers
-	case version.V2ClusterURL:
+	case ClusterResource:
 		watchers = c.cdsWatchers
-	case version.V2EndpointsURL:
+	case EndpointsResource:
 		watchers = c.edsWatchers
 	}
 
 	resourceName := wi.target
 	s, ok := watchers[wi.target]
 	if !ok {
-		// If this is a new watcher, will ask lower level to send a new request with
-		// the resource name.
+		// If this is a new watcher, will ask lower level to send a new request
+		// with the resource name.
 		//
-		// If this type+name is already being watched, will not notify the
-		// underlying xdsv2Client.
-		c.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.typeURL, wi.target)
+		// If this (type+name) is already being watched, will not notify the
+		// underlying versioned apiClient.
+		c.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.rType, wi.target)
 		s = make(map[*watchInfo]bool)
 		watchers[resourceName] = s
-		c.apiClient.AddWatch(wi.typeURL, resourceName)
+		c.apiClient.AddWatch(wi.rType, resourceName)
 	}
 	// No matter what, add the new watcher to the set, so it's callback will be
 	// call for new responses.
 	s[wi] = true
 
 	// If the resource is in cache, call the callback with the value.
-	switch wi.typeURL {
-	case version.V2ListenerURL:
+	switch wi.rType {
+	case ListenerResource:
 		if v, ok := c.ldsCache[resourceName]; ok {
 			c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
 		}
-	case version.V2RouteConfigURL:
+	case RouteConfigResource:
 		if v, ok := c.rdsCache[resourceName]; ok {
 			c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
 		}
-	case version.V2ClusterURL:
+	case ClusterResource:
 		if v, ok := c.cdsCache[resourceName]; ok {
 			c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
 		}
-	case version.V2EndpointsURL:
+	case EndpointsResource:
 		if v, ok := c.edsCache[resourceName]; ok {
 			c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, v)
 			wi.newUpdate(v)
@@ -173,7 +171,7 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 	}
 
 	return func() {
-		c.logger.Debugf("watch for type %v, resource name %v canceled", wi.typeURL, wi.target)
+		c.logger.Debugf("watch for type %v, resource name %v canceled", wi.rType, wi.target)
 		wi.cancel()
 		c.mu.Lock()
 		defer c.mu.Unlock()
@@ -182,22 +180,22 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 			// future.
 			delete(s, wi)
 			if len(s) == 0 {
-				c.logger.Debugf("last watch for type %v, resource name %v canceled, will send a new xDS request", wi.typeURL, wi.target)
+				c.logger.Debugf("last watch for type %v, resource name %v canceled, will send a new xDS request", wi.rType, wi.target)
 				// If this was the last watcher, also tell xdsv2Client to stop
 				// watching this resource.
 				delete(watchers, resourceName)
-				c.apiClient.RemoveWatch(wi.typeURL, resourceName)
+				c.apiClient.RemoveWatch(wi.rType, resourceName)
 				// Remove the resource from cache. When a watch for this
 				// resource is added later, it will trigger a xDS request with
 				// resource names, and client will receive new xDS responses.
-				switch wi.typeURL {
-				case version.V2ListenerURL:
+				switch wi.rType {
+				case ListenerResource:
 					delete(c.ldsCache, resourceName)
-				case version.V2RouteConfigURL:
+				case RouteConfigResource:
 					delete(c.rdsCache, resourceName)
-				case version.V2ClusterURL:
+				case ClusterResource:
 					delete(c.cdsCache, resourceName)
-				case version.V2EndpointsURL:
+				case EndpointsResource:
 					delete(c.edsCache, resourceName)
 				}
 			}
@@ -205,6 +203,18 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 	}
 }
 
+// WatchListener uses LDS to discover information about the provided listener.
+//
+// WatchListener is expected to called only from the server side implementation
+// of xDS. Clients will use WatchService instead.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func (c *Client) WatchListener(listener string, cb func(ListenerUpdate, error)) (cancel func()) {
+	return c.watchLDS(listener, cb)
+}
+
 // watchLDS starts a listener watcher for the service..
 //
 // Note that during race (e.g. an xDS response is received while the user is
@@ -213,7 +223,7 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
-		typeURL:     version.V2ListenerURL,
+		rType:       ListenerResource,
 		target:      serviceName,
 		ldsCallback: cb,
 	}
@@ -232,7 +242,7 @@ func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (c
 func (c *Client) watchRDS(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
-		typeURL:     version.V2RouteConfigURL,
+		rType:       RouteConfigResource,
 		target:      routeName,
 		rdsCallback: cb,
 	}
@@ -358,7 +368,7 @@ func (w *serviceUpdateWatcher) close() {
 func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
-		typeURL:     version.V2ClusterURL,
+		rType:       ClusterResource,
 		target:      clusterName,
 		cdsCallback: cb,
 	}
@@ -380,7 +390,7 @@ func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error))
 func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
-		typeURL:     version.V2EndpointsURL,
+		rType:       EndpointsResource,
 		target:      clusterName,
 		edsCallback: cb,
 	}
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index 93d32113a310..eedda3e0a2d5 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -22,7 +22,6 @@ import (
 	"testing"
 
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 type clusterUpdateErr struct {
@@ -53,7 +52,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -117,7 +116,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[ClusterResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -174,7 +173,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 		c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[ClusterResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -184,7 +183,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -224,7 +223,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -242,7 +241,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[ClusterResource].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
@@ -276,7 +275,7 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -312,7 +311,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -354,7 +353,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 		clusterUpdateCh1.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
@@ -362,7 +361,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ClusterURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index c7e8bd8e52e0..09dc80817284 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -23,9 +23,9 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 var (
@@ -71,7 +71,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -129,7 +129,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[EndpointsResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -186,7 +186,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 		c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[EndpointsResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -196,7 +196,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -236,7 +236,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -254,7 +254,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[EndpointsResource].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
@@ -288,7 +288,7 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2EndpointsURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index c052e8bf4ce5..b2bcef8c7237 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -22,7 +22,6 @@ import (
 	"testing"
 
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 type ldsUpdateErr struct {
@@ -50,7 +49,7 @@ func (s) TestLDSWatch(t *testing.T) {
 	cancelWatch := c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -109,7 +108,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[ListenerResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -166,7 +165,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 		c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[ListenerResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -176,7 +175,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -216,7 +215,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -234,7 +233,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[ListenerResource].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
@@ -271,7 +270,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 		ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
@@ -279,7 +278,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index 6156d195c740..f9bb30d559f4 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -23,7 +23,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 type rdsUpdateErr struct {
@@ -51,7 +50,7 @@ func (s) TestRDSWatch(t *testing.T) {
 	cancelWatch := c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -109,7 +108,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[RouteConfigResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -166,7 +165,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		c.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); i == 0 && err != nil {
+		if _, err := v2Client.addWatches[RouteConfigResource].Receive(); i == 0 && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 	}
@@ -176,7 +175,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	c.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -216,7 +215,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -234,7 +233,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err == nil {
+	if n, err := v2Client.addWatches[RouteConfigResource].Receive(); err == nil {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 7009373b4e92..6f994686b8e4 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -23,8 +23,8 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 type serviceUpdateErr struct {
@@ -56,13 +56,13 @@ func (s) TestServiceWatch(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
@@ -114,13 +114,13 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
@@ -135,7 +135,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName + "2"},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -181,13 +181,13 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
@@ -253,7 +253,7 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
@@ -288,13 +288,13 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{})
@@ -331,13 +331,13 @@ func (s) TestServiceWatchWithClientClose(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Client is closed before it receives the RDS response.
@@ -369,13 +369,13 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
@@ -390,7 +390,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if v, err := v2Client.removeWatches[version.V2RouteConfigURL].Receive(); err == nil {
+	if v, err := v2Client.removeWatches[RouteConfigResource].Receive(); err == nil {
 		t.Fatalf("unexpected rds watch cancel: %v", v)
 	}
 }
@@ -420,13 +420,13 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[version.V2ListenerURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
@@ -440,7 +440,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	// Remove LDS resource, should cancel the RDS watch, and trigger resource
 	// removed error.
 	v2Client.r.NewListeners(map[string]ListenerUpdate{})
-	if _, err := v2Client.removeWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.removeWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want watch to be canceled, got error %v", err)
 	}
 	if u, err := serviceUpdateCh.Receive(); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
@@ -462,7 +462,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[version.V2RouteConfigURL].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 4a1a16df3add..f5f60a4f9460 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -33,9 +33,9 @@ import (
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
+
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 // UnmarshalListener processes resources received in an LDS response, validates
@@ -44,8 +44,8 @@ import (
 func UnmarshalListener(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, error) {
 	update := make(map[string]ListenerUpdate)
 	for _, r := range resources {
-		if t := r.GetTypeUrl(); t != version.V2ListenerURL && t != version.V3ListenerURL {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in LDS response", t)
+		if !IsListenerResource(r.GetTypeUrl()) {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in LDS response", r.GetTypeUrl())
 		}
 		lis := &v3listenerpb.Listener{}
 		if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
@@ -68,8 +68,8 @@ func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.
 		return "", fmt.Errorf("xds: no api_listener field in LDS response %+v", lis)
 	}
 	apiLisAny := lis.GetApiListener().GetApiListener()
-	if t := apiLisAny.GetTypeUrl(); t != version.V3HTTPConnManagerURL && t != version.V2HTTPConnManagerURL {
-		return "", fmt.Errorf("xds: unexpected resource type: %s in LDS response", t)
+	if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
+		return "", fmt.Errorf("xds: unexpected resource type: %s in LDS response", apiLisAny.GetTypeUrl())
 	}
 	apiLis := &v3httppb.HttpConnectionManager{}
 	if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
@@ -105,8 +105,8 @@ func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.
 func UnmarshalRouteConfig(resources []*anypb.Any, hostname string, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, error) {
 	update := make(map[string]RouteConfigUpdate)
 	for _, r := range resources {
-		if t := r.GetTypeUrl(); t != version.V2RouteConfigURL && t != version.V3RouteConfigURL {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in RDS response", t)
+		if !IsRouteConfigResource(r.GetTypeUrl()) {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in RDS response", r.GetTypeUrl())
 		}
 		rc := &v3routepb.RouteConfiguration{}
 		if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
@@ -369,8 +369,8 @@ func findBestMatchingVirtualHost(host string, vHosts []*v3routepb.VirtualHost) *
 func UnmarshalCluster(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ClusterUpdate, error) {
 	update := make(map[string]ClusterUpdate)
 	for _, r := range resources {
-		if t := r.GetTypeUrl(); t != version.V2ClusterURL && t != version.V3ClusterURL {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in CDS response", t)
+		if !IsClusterResource(r.GetTypeUrl()) {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in CDS response", r.GetTypeUrl())
 		}
 
 		cluster := &v3clusterpb.Cluster{}
@@ -417,8 +417,8 @@ func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 func UnmarshalEndpoints(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]EndpointsUpdate, error) {
 	update := make(map[string]EndpointsUpdate)
 	for _, r := range resources {
-		if t := r.GetTypeUrl(); t != version.V2EndpointsURL && t != version.V3EndpointsURL {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in EDS response", t)
+		if !IsEndpointsResource(r.GetTypeUrl()) {
+			return nil, fmt.Errorf("xds: unexpected resource type: %s in EDS response", r.GetTypeUrl())
 		}
 
 		cla := &v3endpointpb.ClusterLoadAssignment{}
diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
index 6627fbffb9d5..976003605041 100644
--- a/xds/internal/client/transport_helper.go
+++ b/xds/internal/client/transport_helper.go
@@ -48,11 +48,9 @@ type VersionedClient interface {
 	// transport protocol version.
 	NewStream(ctx context.Context) (grpc.ClientStream, error)
 
-	// params: resources, typeURL, version, nonce
-
 	// SendRequest constructs and sends out a DiscoveryRequest message specific
 	// to the underlying transport protocol version.
-	SendRequest(s grpc.ClientStream, resourceNames []string, typeURL string, version string, nonce string) error
+	SendRequest(s grpc.ClientStream, resourceNames []string, rType ResourceType, version string, nonce string) error
 
 	// RecvResponse uses the provided stream to receive a response specific to
 	// the underlying transport protocol version.
@@ -61,11 +59,11 @@ type VersionedClient interface {
 	// HandleResponse parses and validates the received response and notifies
 	// the top-level client which in turn notifies the registered watchers.
 	//
-	// Return values are: typeURL, version, nonce, error.
+	// Return values are: resourceType, version, nonce, error.
 	// If the provided protobuf message contains a resource type which is not
 	// supported, implementations must return an error of type
 	// ErrResourceTypeUnsupported.
-	HandleResponse(proto.Message) (string, string, string, error)
+	HandleResponse(proto.Message) (ResourceType, string, string, error)
 }
 
 // TransportHelper contains all xDS transport protocol related functionality
@@ -95,14 +93,14 @@ type TransportHelper struct {
 	// messages. When the user of this client object cancels a watch call,
 	// these are set to nil. All accesses to the map protected and any value
 	// inside the map should be protected with the above mutex.
-	watchMap map[string]map[string]bool
+	watchMap map[ResourceType]map[string]bool
 	// versionMap contains the version that was acked (the version in the ack
-	// request that was sent on wire). The key is typeURL, the value is the
+	// request that was sent on wire). The key is rType, the value is the
 	// version string, becaues the versions for different resource types should
 	// be independent.
-	versionMap map[string]string
+	versionMap map[ResourceType]string
 	// nonceMap contains the nonce from the most recent received response.
-	nonceMap map[string]string
+	nonceMap map[ResourceType]string
 }
 
 // NewTransportHelper creates a new transport helper to be used by versioned
@@ -117,9 +115,9 @@ func NewTransportHelper(vc VersionedClient, logger *grpclog.PrefixLogger, backof
 
 		streamCh:   make(chan grpc.ClientStream, 1),
 		sendCh:     buffer.NewUnbounded(),
-		watchMap:   make(map[string]map[string]bool),
-		versionMap: make(map[string]string),
-		nonceMap:   make(map[string]string),
+		watchMap:   make(map[ResourceType]map[string]bool),
+		versionMap: make(map[ResourceType]string),
+		nonceMap:   make(map[ResourceType]string),
 	}
 
 	go t.run(ctx)
@@ -127,9 +125,9 @@ func NewTransportHelper(vc VersionedClient, logger *grpclog.PrefixLogger, backof
 }
 
 // AddWatch adds a watch for an xDS resource given its type and name.
-func (t *TransportHelper) AddWatch(resourceType, resourceName string) {
+func (t *TransportHelper) AddWatch(rType ResourceType, resourceName string) {
 	t.sendCh.Put(&watchAction{
-		typeURL:  resourceType,
+		rType:    rType,
 		remove:   false,
 		resource: resourceName,
 	})
@@ -137,9 +135,9 @@ func (t *TransportHelper) AddWatch(resourceType, resourceName string) {
 
 // RemoveWatch cancels an already registered watch for an xDS resource
 // given its type and name.
-func (t *TransportHelper) RemoveWatch(resourceType, resourceName string) {
+func (t *TransportHelper) RemoveWatch(rType ResourceType, resourceName string) {
 	t.sendCh.Put(&watchAction{
-		typeURL:  resourceType,
+		rType:    rType,
 		remove:   true,
 		resource: resourceName,
 	})
@@ -228,15 +226,16 @@ func (t *TransportHelper) send(ctx context.Context) {
 			t.sendCh.Load()
 
 			var (
-				target                  []string
-				typeURL, version, nonce string
-				send                    bool
+				target         []string
+				rType          ResourceType
+				version, nonce string
+				send           bool
 			)
 			switch update := u.(type) {
 			case *watchAction:
-				target, typeURL, version, nonce = t.processWatchInfo(update)
+				target, rType, version, nonce = t.processWatchInfo(update)
 			case *ackAction:
-				target, typeURL, version, nonce, send = t.processAckInfo(update, stream)
+				target, rType, version, nonce, send = t.processAckInfo(update, stream)
 				if !send {
 					continue
 				}
@@ -248,8 +247,8 @@ func (t *TransportHelper) send(ctx context.Context) {
 				// sending response back).
 				continue
 			}
-			if err := t.vClient.SendRequest(stream, target, typeURL, version, nonce); err != nil {
-				t.logger.Warningf("ADS request for {target: %q, type: %q, version: %q, nonce: %q} failed: %v", target, typeURL, version, nonce, err)
+			if err := t.vClient.SendRequest(stream, target, rType, version, nonce); err != nil {
+				t.logger.Warningf("ADS request for {target: %q, type: %v, version: %q, nonce: %q} failed: %v", target, rType, version, nonce, err)
 				// send failed, clear the current stream.
 				stream = nil
 			}
@@ -269,11 +268,11 @@ func (t *TransportHelper) sendExisting(stream grpc.ClientStream) bool {
 	defer t.mu.Unlock()
 
 	// Reset the ack versions when the stream restarts.
-	t.versionMap = make(map[string]string)
-	t.nonceMap = make(map[string]string)
+	t.versionMap = make(map[ResourceType]string)
+	t.nonceMap = make(map[ResourceType]string)
 
-	for typeURL, s := range t.watchMap {
-		if err := t.vClient.SendRequest(stream, mapToSlice(s), typeURL, "", ""); err != nil {
+	for rType, s := range t.watchMap {
+		if err := t.vClient.SendRequest(stream, mapToSlice(s), rType, "", ""); err != nil {
 			t.logger.Errorf("ADS request failed: %v", err)
 			return false
 		}
@@ -292,28 +291,28 @@ func (t *TransportHelper) recv(stream grpc.ClientStream) bool {
 			t.logger.Warningf("ADS stream is closed with error: %v", err)
 			return success
 		}
-		typeURL, version, nonce, err := t.vClient.HandleResponse(resp)
+		rType, version, nonce, err := t.vClient.HandleResponse(resp)
 		if e, ok := err.(ErrResourceTypeUnsupported); ok {
 			t.logger.Warningf("%s", e.ErrStr)
 			continue
 		}
 		if err != nil {
 			t.sendCh.Put(&ackAction{
-				typeURL: typeURL,
+				rType:   rType,
 				version: "",
 				nonce:   nonce,
 				stream:  stream,
 			})
-			t.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", typeURL, version, nonce, err)
+			t.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", rType, version, nonce, err)
 			continue
 		}
 		t.sendCh.Put(&ackAction{
-			typeURL: typeURL,
+			rType:   rType,
 			version: version,
 			nonce:   nonce,
 			stream:  stream,
 		})
-		t.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", typeURL, version, nonce)
+		t.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", rType, version, nonce)
 		success = true
 	}
 }
@@ -326,46 +325,46 @@ func mapToSlice(m map[string]bool) (ret []string) {
 }
 
 type watchAction struct {
-	typeURL  string
+	rType    ResourceType
 	remove   bool // Whether this is to remove watch for the resource.
 	resource string
 }
 
 // processWatchInfo pulls the fields needed by the request from a watchAction.
 //
-// It also updates the watch map in v2c.
-func (t *TransportHelper) processWatchInfo(w *watchAction) (target []string, typeURL, ver, nonce string) {
+// It also updates the watch map.
+func (t *TransportHelper) processWatchInfo(w *watchAction) (target []string, rType ResourceType, ver, nonce string) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	var current map[string]bool
-	current, ok := t.watchMap[w.typeURL]
+	current, ok := t.watchMap[w.rType]
 	if !ok {
 		current = make(map[string]bool)
-		t.watchMap[w.typeURL] = current
+		t.watchMap[w.rType] = current
 	}
 
 	if w.remove {
 		delete(current, w.resource)
 		if len(current) == 0 {
-			delete(t.watchMap, w.typeURL)
+			delete(t.watchMap, w.rType)
 		}
 	} else {
 		current[w.resource] = true
 	}
 
-	typeURL = w.typeURL
+	rType = w.rType
 	target = mapToSlice(current)
 	// We don't reset version or nonce when a new watch is started. The version
 	// and nonce from previous response are carried by the request unless the
 	// stream is recreated.
-	ver = t.versionMap[typeURL]
-	nonce = t.nonceMap[typeURL]
-	return target, typeURL, ver, nonce
+	ver = t.versionMap[rType]
+	nonce = t.nonceMap[rType]
+	return target, rType, ver, nonce
 }
 
 type ackAction struct {
-	typeURL string
+	rType   ResourceType
 	version string // NACK if version is an empty string.
 	nonce   string
 	// ACK/NACK are tagged with the stream it's for. When the stream is down,
@@ -377,15 +376,15 @@ type ackAction struct {
 // processAckInfo pulls the fields needed by the ack request from a ackAction.
 //
 // If no active watch is found for this ack, it returns false for send.
-func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStream) (target []string, typeURL, version, nonce string, send bool) {
+func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStream) (target []string, rType ResourceType, version, nonce string, send bool) {
 	if ack.stream != stream {
 		// If ACK's stream isn't the current sending stream, this means the ACK
 		// was pushed to queue before the old stream broke, and a new stream has
 		// been started since. Return immediately here so we don't update the
 		// nonce for the new stream.
-		return nil, "", "", "", false
+		return nil, UnknownResource, "", "", false
 	}
-	typeURL = ack.typeURL
+	rType = ack.rType
 
 	t.mu.Lock()
 	defer t.mu.Unlock()
@@ -394,16 +393,16 @@ func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStrea
 	// wire. We may not send the request if the watch is canceled. But the nonce
 	// needs to be updated so the next request will have the right nonce.
 	nonce = ack.nonce
-	t.nonceMap[typeURL] = nonce
+	t.nonceMap[rType] = nonce
 
-	s, ok := t.watchMap[typeURL]
+	s, ok := t.watchMap[rType]
 	if !ok || len(s) == 0 {
 		// We don't send the request ack if there's no active watch (this can be
 		// either the server sends responses before any request, or the watch is
 		// canceled while the ackAction is in queue), because there's no resource
 		// name. And if we send a request with empty resource name list, the
 		// server may treat it as a wild card and send us everything.
-		return nil, "", "", "", false
+		return nil, UnknownResource, "", "", false
 	}
 	send = true
 	target = mapToSlice(s)
@@ -411,12 +410,12 @@ func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStrea
 	version = ack.version
 	if version == "" {
 		// This is a nack, get the previous acked version.
-		version = t.versionMap[typeURL]
-		// version will still be an empty string if typeURL isn't
+		version = t.versionMap[rType]
+		// version will still be an empty string if rType isn't
 		// found in versionMap, this can happen if there wasn't any ack
 		// before.
 	} else {
-		t.versionMap[typeURL] = version
+		t.versionMap[rType] = version
 	}
-	return target, typeURL, version, nonce, send
+	return target, rType, version, nonce, send
 }
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index 278dc4381d2c..96bd5e9b5686 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -39,6 +39,15 @@ func init() {
 	xdsclient.RegisterAPIClientBuilder(clientBuilder{})
 }
 
+var (
+	resourceTypeToURL = map[xdsclient.ResourceType]string{
+		xdsclient.ListenerResource:    version.V2ListenerURL,
+		xdsclient.RouteConfigResource: version.V2RouteConfigURL,
+		xdsclient.ClusterResource:     version.V2ClusterURL,
+		xdsclient.EndpointsResource:   version.V2EndpointsURL,
+	}
+)
+
 type clientBuilder struct{}
 
 func (clientBuilder) Build(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
@@ -95,30 +104,30 @@ type client struct {
 // AddWatch overrides the transport helper's AddWatch to save the LDS
 // resource_name. This is required when handling an RDS response to perform host
 // matching.
-func (v2c *client) AddWatch(resourceType, resourceName string) {
+func (v2c *client) AddWatch(rType xdsclient.ResourceType, rName string) {
 	v2c.mu.Lock()
 	// Special handling for LDS, because RDS needs the LDS resource_name for
 	// response host matching.
-	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+	if rType == xdsclient.ListenerResource {
 		// Set hostname to the first LDS resource_name, and reset it when the
 		// last LDS watch is removed. The upper level Client isn't expected to
 		// watchLDS more than once.
 		v2c.ldsWatchCount++
 		if v2c.ldsWatchCount == 1 {
-			v2c.ldsResourceName = resourceName
+			v2c.ldsResourceName = rName
 		}
 	}
 	v2c.mu.Unlock()
-	v2c.TransportHelper.AddWatch(resourceType, resourceName)
+	v2c.TransportHelper.AddWatch(rType, rName)
 }
 
 // RemoveWatch overrides the transport helper's RemoveWatch to clear the LDS
 // resource_name when the last watch is removed.
-func (v2c *client) RemoveWatch(resourceType, resourceName string) {
+func (v2c *client) RemoveWatch(rType xdsclient.ResourceType, rName string) {
 	v2c.mu.Lock()
 	// Special handling for LDS, because RDS needs the LDS resource_name for
 	// response host matching.
-	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+	if rType == xdsclient.ListenerResource {
 		// Set hostname to the first LDS resource_name, and reset it when the
 		// last LDS watch is removed. The upper level Client isn't expected to
 		// watchLDS more than once.
@@ -128,30 +137,29 @@ func (v2c *client) RemoveWatch(resourceType, resourceName string) {
 		}
 	}
 	v2c.mu.Unlock()
-	v2c.TransportHelper.RemoveWatch(resourceType, resourceName)
+	v2c.TransportHelper.RemoveWatch(rType, rName)
 }
 
 func (v2c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
 	return v2adsgrpc.NewAggregatedDiscoveryServiceClient(v2c.cc).StreamAggregatedResources(v2c.ctx, grpc.WaitForReady(true))
 }
 
-// sendRequest sends a request for provided typeURL and resource on the provided
-// stream.
+// sendRequest sends out a DiscoveryRequest for the given resourceNames, of type
+// rType, on the provided stream.
 //
 // version is the ack version to be sent with the request
-// - If this is the new request (not an ack/nack), version will be an empty
-// string
-// - If this is an ack, version will be the version from the response
+// - If this is the new request (not an ack/nack), version will be empty.
+// - If this is an ack, version will be the version from the response.
 // - If this is a nack, version will be the previous acked version (from
-// versionMap). If there was no ack before, it will be an empty string
-func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, typeURL, version, nonce string) error {
+//   versionMap). If there was no ack before, it will be empty.
+func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce string) error {
 	stream, ok := s.(adsStream)
 	if !ok {
 		return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s)
 	}
 	req := &v2xdspb.DiscoveryRequest{
 		Node:          v2c.nodeProto,
-		TypeUrl:       typeURL,
+		TypeUrl:       resourceTypeToURL[rType],
 		ResourceNames: resourceNames,
 		VersionInfo:   version,
 		ResponseNonce: nonce,
@@ -182,10 +190,11 @@ func (v2c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
 	return resp, nil
 }
 
-func (v2c *client) HandleResponse(r proto.Message) (string, string, string, error) {
+func (v2c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, string, string, error) {
+	rType := xdsclient.UnknownResource
 	resp, ok := r.(*v2xdspb.DiscoveryResponse)
 	if !ok {
-		return "", "", "", fmt.Errorf("xds: unsupported message type: %T", resp)
+		return rType, "", "", fmt.Errorf("xds: unsupported message type: %T", resp)
 	}
 
 	// Note that the xDS transport protocol is versioned independently of
@@ -193,21 +202,26 @@ func (v2c *client) HandleResponse(r proto.Message) (string, string, string, erro
 	// of resource types using new versions of the transport protocol, or
 	// vice-versa. Hence we need to handle v3 type_urls as well here.
 	var err error
-	switch resp.GetTypeUrl() {
-	case version.V2ListenerURL, version.V3ListenerURL:
+	url := resp.GetTypeUrl()
+	switch {
+	case xdsclient.IsListenerResource(url):
 		err = v2c.handleLDSResponse(resp)
-	case version.V2RouteConfigURL, version.V3RouteConfigURL:
+		rType = xdsclient.ListenerResource
+	case xdsclient.IsRouteConfigResource(url):
 		err = v2c.handleRDSResponse(resp)
-	case version.V2ClusterURL, version.V3ClusterURL:
+		rType = xdsclient.RouteConfigResource
+	case xdsclient.IsClusterResource(url):
 		err = v2c.handleCDSResponse(resp)
-	case version.V2EndpointsURL, version.V3EndpointsURL:
+		rType = xdsclient.ClusterResource
+	case xdsclient.IsEndpointsResource(url):
 		err = v2c.handleEDSResponse(resp)
+		rType = xdsclient.EndpointsResource
 	default:
-		return "", "", "", xdsclient.ErrResourceTypeUnsupported{
+		return rType, "", "", xdsclient.ErrResourceTypeUnsupported{
 			ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()),
 		}
 	}
-	return resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), err
+	return rType, resp.GetVersionInfo(), resp.GetNonce(), err
 }
 
 // handleLDSResponse processes an LDS response received from the xDS server. On
diff --git a/xds/internal/client/v2/client_ack_test.go b/xds/internal/client/v2/client_ack_test.go
index d8d1ad834968..40f5668297e2 100644
--- a/xds/internal/client/v2/client_ack_test.go
+++ b/xds/internal/client/v2/client_ack_test.go
@@ -28,6 +28,7 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
@@ -39,22 +40,22 @@ func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cb
 	cbCDS = testutils.NewChannel()
 	cbEDS = testutils.NewChannel()
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(typeURL string, d map[string]interface{}) {
-			t.Logf("Received %s callback with {%+v}", typeURL, d)
-			switch typeURL {
-			case version.V2ListenerURL:
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+			t.Logf("Received %v callback with {%+v}", rType, d)
+			switch rType {
+			case xdsclient.ListenerResource:
 				if _, ok := d[goodLDSTarget1]; ok {
 					cbLDS.Send(struct{}{})
 				}
-			case version.V2RouteConfigURL:
+			case xdsclient.RouteConfigResource:
 				if _, ok := d[goodRouteName1]; ok {
 					cbRDS.Send(struct{}{})
 				}
-			case version.V2ClusterURL:
+			case xdsclient.ClusterResource:
 				if _, ok := d[goodClusterName1]; ok {
 					cbCDS.Send(struct{}{})
 				}
-			case version.V2EndpointsURL:
+			case xdsclient.EndpointsResource:
 				if _, ok := d[goodEDSName]; ok {
 					cbEDS.Send(struct{}{})
 				}
@@ -98,30 +99,24 @@ func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *
 
 // startXDS calls watch to send the first request. It then sends a good response
 // and checks for ack.
-func startXDS(t *testing.T, xdsname string, v2c *client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
-	var (
-		nameToWatch, typeURLToWatch string
-	)
-	switch xdsname {
-	case "LDS":
-		typeURLToWatch = version.V2ListenerURL
+func startXDS(t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
+	nameToWatch := ""
+	switch rType {
+	case xdsclient.ListenerResource:
 		nameToWatch = goodLDSTarget1
-	case "RDS":
-		typeURLToWatch = version.V2RouteConfigURL
+	case xdsclient.RouteConfigResource:
 		nameToWatch = goodRouteName1
-	case "CDS":
-		typeURLToWatch = version.V2ClusterURL
+	case xdsclient.ClusterResource:
 		nameToWatch = goodClusterName1
-	case "EDS":
-		typeURLToWatch = version.V2EndpointsURL
+	case xdsclient.EndpointsResource:
 		nameToWatch = goodEDSName
 	}
-	v2c.AddWatch(typeURLToWatch, nameToWatch)
+	v2c.AddWatch(rType, nameToWatch)
 
 	if err := compareXDSRequest(reqChan, req, preVersion, preNonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", xdsname, err)
+		t.Fatalf("Failed to receive %v request: %v", rType, err)
 	}
-	t.Logf("FakeServer received %s request...", xdsname)
+	t.Logf("FakeServer received %v request...", rType)
 }
 
 // sendGoodResp sends the good response, with the given version, and a random
@@ -129,19 +124,19 @@ func startXDS(t *testing.T, xdsname string, v2c *client, reqChan *testutils.Chan
 //
 // It also waits and checks that the ack request contains the given version, and
 // the generated nonce.
-func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, ver int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
+func sendGoodResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, ver)
-	t.Logf("Good %s response pushed to fakeServer...", xdsname)
+	t.Logf("Good %v response pushed to fakeServer...", rType)
 
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce); err != nil {
-		return "", fmt.Errorf("failed to receive %s request: %v", xdsname, err)
+		return "", fmt.Errorf("failed to receive %v request: %v", rType, err)
 	}
-	t.Logf("Good %s response acked", xdsname)
+	t.Logf("Good %v response acked", rType)
 
 	if _, err := callbackCh.Receive(); err != nil {
-		return "", fmt.Errorf("timeout when expecting %s update", xdsname)
+		return "", fmt.Errorf("timeout when expecting %v update", rType)
 	}
-	t.Logf("Good %s response callback executed", xdsname)
+	t.Logf("Good %v response callback executed", rType)
 	return nonce, nil
 }
 
@@ -149,27 +144,27 @@ func sendGoodResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, v
 // be nacked, so we expect a request with the previous version (version-1).
 //
 // But the nonce in request should be the new nonce.
-func sendBadResp(t *testing.T, xdsname string, fakeServer *fakeserver.Server, ver int, wantReq *xdspb.DiscoveryRequest) error {
+func sendBadResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, wantReq *xdspb.DiscoveryRequest) error {
 	var typeURL string
-	switch xdsname {
-	case "LDS":
+	switch rType {
+	case xdsclient.ListenerResource:
 		typeURL = version.V2ListenerURL
-	case "RDS":
+	case xdsclient.RouteConfigResource:
 		typeURL = version.V2RouteConfigURL
-	case "CDS":
+	case xdsclient.ClusterResource:
 		typeURL = version.V2ClusterURL
-	case "EDS":
+	case xdsclient.EndpointsResource:
 		typeURL = version.V2EndpointsURL
 	}
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
 		TypeUrl:   typeURL,
 	}, ver)
-	t.Logf("Bad %s response pushed to fakeServer...", xdsname)
+	t.Logf("Bad %v response pushed to fakeServer...", rType)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce); err != nil {
-		return fmt.Errorf("failed to receive %s request: %v", xdsname, err)
+		return fmt.Errorf("failed to receive %v request: %v", rType, err)
 	}
-	t.Logf("Bad %s response nacked", xdsname)
+	t.Logf("Bad %v response nacked", rType)
 	return nil
 }
 
@@ -192,59 +187,59 @@ func (s) TestV2ClientAck(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start the watch, send a good response, and check for ack.
-	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
-	if _, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
-	startXDS(t, "RDS", v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
-	if _, err := sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
+	startXDS(t, xdsclient.RouteConfigResource, v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
+	if _, err := sendGoodResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
 		t.Fatal(err)
 	}
 	versionRDS++
-	startXDS(t, "CDS", v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
-	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	startXDS(t, xdsclient.ClusterResource, v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
+	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
-	startXDS(t, "EDS", v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
-	if _, err := sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
+	startXDS(t, xdsclient.EndpointsResource, v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
+	if _, err := sendGoodResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
 		t.Fatal(err)
 	}
 	versionEDS++
 
 	// Send a bad response, and check for nack.
-	if err := sendBadResp(t, "LDS", fakeServer, versionLDS, goodLDSRequest); err != nil {
+	if err := sendBadResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
-	if err := sendBadResp(t, "RDS", fakeServer, versionRDS, goodRDSRequest); err != nil {
+	if err := sendBadResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionRDS++
-	if err := sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest); err != nil {
+	if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
-	if err := sendBadResp(t, "EDS", fakeServer, versionEDS, goodEDSRequest); err != nil {
+	if err := sendBadResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionEDS++
 
 	// send another good response, and check for ack, with the new version.
-	if _, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
-	if _, err := sendGoodResp(t, "RDS", fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
 		t.Fatal(err)
 	}
 	versionRDS++
-	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
-	if _, err := sendGoodResp(t, "EDS", fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
 		t.Fatal(err)
 	}
 	versionEDS++
@@ -262,7 +257,7 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start the watch, send a good response, and check for ack.
-	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
 
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
@@ -278,7 +273,7 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 	t.Logf("Bad response nacked")
 	versionLDS++
 
-	sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 	versionLDS++
 }
 
@@ -294,14 +289,14 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start the watch, send a good response, and check for ack.
-	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
-	nonce, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	nonce, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Start a new watch. The version in the new request should be the version
 	// from the previous response, thus versionLDS before ++.
-	startXDS(t, "LDS", v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
+	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
 	versionLDS++
 
 	// This is an invalid response after the new watch.
@@ -318,7 +313,7 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	t.Logf("Bad response nacked")
 	versionLDS++
 
-	if _, err := sendGoodResp(t, "LDS", fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
@@ -336,42 +331,42 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start a CDS watch.
-	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
+	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
 		t.Fatal(err)
 	}
-	t.Logf("FakeServer received %s request...", "CDS")
+	t.Logf("FakeServer received %v request...", xdsclient.ClusterResource)
 
 	// Send a good CDS response, this function waits for the ACK with the right
 	// version.
-	nonce, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	nonce, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Cancel the CDS watch, and start a new one. The new watch should have the
 	// version from the response above.
-	v2c.RemoveWatch(version.V2ClusterURL, goodClusterName1)
+	v2c.RemoveWatch(xdsclient.ClusterResource, goodClusterName1)
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
 	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
-	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
+	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
 	// Wait for a request with correct resource names and version.
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	versionCDS++
 
 	// Send a bad response with the next version.
-	if err := sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest); err != nil {
+	if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
@@ -391,25 +386,25 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	defer v2cCleanup()
 
 	// Start a CDS watch.
-	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
+	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
-	t.Logf("FakeServer received %s request...", "CDS")
+	t.Logf("FakeServer received %v request...", xdsclient.ClusterResource)
 
 	// send a good response, and check for ack, with the new version.
-	nonce, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	nonce, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Cancel the watch before the next response is sent. This mimics the case
 	// watch is canceled while response is on wire.
-	v2c.RemoveWatch(version.V2ClusterURL, goodClusterName1)
+	v2c.RemoveWatch(xdsclient.ClusterResource, goodClusterName1)
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
 	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	versionCDS++
 
@@ -419,7 +414,7 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 
 	// Send a good response.
 	nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodCDSResponse1, versionCDS)
-	t.Logf("Good %s response pushed to fakeServer...", "CDS")
+	t.Logf("Good %v response pushed to fakeServer...", xdsclient.ClusterResource)
 
 	// Expect no ACK because watch was canceled.
 	if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout {
@@ -427,24 +422,24 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	}
 	// Still expected an callback update, because response was good.
 	if _, err := cbCDS.Receive(); err != nil {
-		t.Fatalf("Timeout when expecting %s update", "CDS")
+		t.Fatalf("Timeout when expecting %v update", xdsclient.ClusterResource)
 	}
 
 	// Start a new watch. The new watch should have the nonce from the response
 	// above, and version from the first good response.
-	v2c.AddWatch(version.V2ClusterURL, goodClusterName1)
+	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
 	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce); err != nil {
-		t.Fatalf("Failed to receive %s request: %v", "CDS", err)
+		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 
 	// Send a bad response with the next version.
-	if err := sendBadResp(t, "CDS", fakeServer, versionCDS, goodCDSRequest); err != nil {
+	if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	if _, err := sendGoodResp(t, "CDS", fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
diff --git a/xds/internal/client/v2/client_cds_test.go b/xds/internal/client/v2/client_cds_test.go
index e93f952e2a7e..8538f4e46afc 100644
--- a/xds/internal/client/v2/client_cds_test.go
+++ b/xds/internal/client/v2/client_cds_test.go
@@ -153,7 +153,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:      version.V2ClusterURL,
+				rType:        xdsclient.ClusterResource,
 				resourceName: goodClusterName1,
 
 				responseToHandle: test.cdsResponse,
@@ -172,7 +172,7 @@ func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/client_eds_test.go b/xds/internal/client/v2/client_eds_test.go
index 170a96a42491..7af74f0b6393 100644
--- a/xds/internal/client/v2/client_eds_test.go
+++ b/xds/internal/client/v2/client_eds_test.go
@@ -135,7 +135,7 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          version.V2EndpointsURL,
+				rType:            xdsclient.EndpointsResource,
 				resourceName:     goodEDSName,
 				responseToHandle: test.edsResponse,
 				wantHandleErr:    test.wantErr,
@@ -153,7 +153,7 @@ func (s) TestEDSHandleResponseWithoutWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/client_lds_test.go b/xds/internal/client/v2/client_lds_test.go
index ca8161f504f4..854cf3ccee73 100644
--- a/xds/internal/client/v2/client_lds_test.go
+++ b/xds/internal/client/v2/client_lds_test.go
@@ -23,8 +23,8 @@ import (
 	"time"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 // TestLDSHandleResponse starts a fake xDS server, makes a ClientConn to it,
@@ -113,7 +113,7 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          version.V2ListenerURL,
+				rType:            xdsclient.ListenerResource,
 				resourceName:     goodLDSTarget1,
 				responseToHandle: test.ldsResponse,
 				wantHandleErr:    test.wantErr,
@@ -131,7 +131,7 @@ func (s) TestLDSHandleResponseWithoutWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/client_rds_test.go
index aa44371ed350..59e63d83e287 100644
--- a/xds/internal/client/v2/client_rds_test.go
+++ b/xds/internal/client/v2/client_rds_test.go
@@ -23,9 +23,9 @@ import (
 	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 // doLDS makes a LDS watch, and waits for the response and ack to finish.
@@ -34,7 +34,7 @@ import (
 // pre-requirement for RDS, and RDS handle would fail without an existing LDS
 // watch.
 func doLDS(t *testing.T, v2c xdsclient.APIClient, fakeServer *fakeserver.Server) {
-	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
+	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout waiting for LDS request: %v", err)
 	}
@@ -112,7 +112,7 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testWatchHandle(t, &watchHandleTestcase{
-				typeURL:          version.V2RouteConfigURL,
+				rType:            xdsclient.RouteConfigResource,
 				resourceName:     goodRouteName1,
 				responseToHandle: test.rdsResponse,
 				wantHandleErr:    test.wantErr,
@@ -130,7 +130,7 @@ func (s) TestRDSHandleResponseWithoutLDSWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -149,7 +149,7 @@ func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index 024357d8476e..fecda6ad06d9 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -331,7 +331,7 @@ var (
 )
 
 type watchHandleTestcase struct {
-	typeURL      string
+	rType        xdsclient.ResourceType
 	resourceName string
 
 	responseToHandle *xdspb.DiscoveryResponse
@@ -341,7 +341,7 @@ type watchHandleTestcase struct {
 }
 
 type testUpdateReceiver struct {
-	f func(typeURL string, d map[string]interface{})
+	f func(rType xdsclient.ResourceType, d map[string]interface{})
 }
 
 func (t *testUpdateReceiver) NewListeners(d map[string]xdsclient.ListenerUpdate) {
@@ -349,7 +349,7 @@ func (t *testUpdateReceiver) NewListeners(d map[string]xdsclient.ListenerUpdate)
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(version.V2ListenerURL, dd)
+	t.newUpdate(xdsclient.ListenerResource, dd)
 }
 
 func (t *testUpdateReceiver) NewRouteConfigs(d map[string]xdsclient.RouteConfigUpdate) {
@@ -357,7 +357,7 @@ func (t *testUpdateReceiver) NewRouteConfigs(d map[string]xdsclient.RouteConfigU
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(version.V2RouteConfigURL, dd)
+	t.newUpdate(xdsclient.RouteConfigResource, dd)
 }
 
 func (t *testUpdateReceiver) NewClusters(d map[string]xdsclient.ClusterUpdate) {
@@ -365,7 +365,7 @@ func (t *testUpdateReceiver) NewClusters(d map[string]xdsclient.ClusterUpdate) {
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(version.V2ClusterURL, dd)
+	t.newUpdate(xdsclient.ClusterResource, dd)
 }
 
 func (t *testUpdateReceiver) NewEndpoints(d map[string]xdsclient.EndpointsUpdate) {
@@ -373,11 +373,11 @@ func (t *testUpdateReceiver) NewEndpoints(d map[string]xdsclient.EndpointsUpdate
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(version.V2EndpointsURL, dd)
+	t.newUpdate(xdsclient.EndpointsResource, dd)
 }
 
-func (t *testUpdateReceiver) newUpdate(typeURL string, d map[string]interface{}) {
-	t.f(typeURL, d)
+func (t *testUpdateReceiver) newUpdate(rType xdsclient.ResourceType, d map[string]interface{}) {
+	t.f(rType, d)
 }
 
 // testWatchHandle is called to test response handling for each xDS.
@@ -397,8 +397,8 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	gotUpdateCh := testutils.NewChannel()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(typeURL string, d map[string]interface{}) {
-			if typeURL == test.typeURL {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+			if rType == test.rType {
 				if u, ok := d[test.resourceName]; ok {
 					gotUpdateCh.Send(updateErr{u, nil})
 				}
@@ -410,14 +410,14 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	}
 	defer v2c.Close()
 
-	// RDS needs an existin LDS watch for the hostname.
-	if test.typeURL == version.V2RouteConfigURL {
+	// RDS needs an existing LDS watch for the hostname.
+	if test.rType == xdsclient.RouteConfigResource {
 		doLDS(t, v2c, fakeServer)
 	}
 
 	// Register the watcher, this will also trigger the v2Client to send the xDS
 	// request.
-	v2c.AddWatch(test.typeURL, test.resourceName)
+	v2c.AddWatch(test.rType, test.resourceName)
 
 	// Wait till the request makes it to the fakeServer. This ensures that
 	// the watch request has been processed by the v2Client.
@@ -432,14 +432,14 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	// Also note that this won't trigger ACK, so there's no need to clear the
 	// request channel afterwards.
 	var handleXDSResp func(response *xdspb.DiscoveryResponse) error
-	switch test.typeURL {
-	case version.V2ListenerURL:
+	switch test.rType {
+	case xdsclient.ListenerResource:
 		handleXDSResp = v2c.handleLDSResponse
-	case version.V2RouteConfigURL:
+	case xdsclient.RouteConfigResource:
 		handleXDSResp = v2c.handleRDSResponse
-	case version.V2ClusterURL:
+	case xdsclient.ClusterResource:
 		handleXDSResp = v2c.handleCDSResponse
-	case version.V2EndpointsURL:
+	case xdsclient.EndpointsResource:
 		handleXDSResp = v2c.handleEDSResponse
 	}
 	if err := handleXDSResp(test.responseToHandle); (err != nil) != test.wantHandleErr {
@@ -524,7 +524,7 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 
 	callbackCh := make(chan struct{})
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(string, map[string]interface{}) { close(callbackCh) },
+		f: func(xdsclient.ResourceType, map[string]interface{}) { close(callbackCh) },
 	}, cc, goodNodeProto, clientBackoff, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -532,7 +532,7 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	defer v2c.Close()
 	t.Log("Started xds v2Client...")
 
-	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
+	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
@@ -567,8 +567,8 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(typeURL string, d map[string]interface{}) {
-			if typeURL == version.V2ListenerURL {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+			if rType == xdsclient.ListenerResource {
 				if u, ok := d[goodLDSTarget1]; ok {
 					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
 					callbackCh.Send(struct{}{})
@@ -582,7 +582,7 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	defer v2c.Close()
 	t.Log("Started xds v2Client...")
 
-	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
+	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
 	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
@@ -637,8 +637,8 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(typeURL string, d map[string]interface{}) {
-			if typeURL == version.V2ListenerURL {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+			if rType == xdsclient.ListenerResource {
 				if u, ok := d[goodLDSTarget1]; ok {
 					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
 					callbackCh.Send(u)
@@ -654,7 +654,7 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 
 	// This watch is started when the xds-ClientConn is in Transient Failure,
 	// and no xds stream is created.
-	v2c.AddWatch(version.V2ListenerURL, goodLDSTarget1)
+	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
 
 	// The watcher should receive an update, with a timeout error in it.
 	if v, err := callbackCh.TimedReceive(100 * time.Millisecond); err == nil {
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 56bb245e32a6..a1103f8f8b85 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -39,6 +39,15 @@ func init() {
 	xdsclient.RegisterAPIClientBuilder(clientBuilder{})
 }
 
+var (
+	resourceTypeToURL = map[xdsclient.ResourceType]string{
+		xdsclient.ListenerResource:    version.V2ListenerURL,
+		xdsclient.RouteConfigResource: version.V2RouteConfigURL,
+		xdsclient.ClusterResource:     version.V2ClusterURL,
+		xdsclient.EndpointsResource:   version.V2EndpointsURL,
+	}
+)
+
 type clientBuilder struct{}
 
 func (clientBuilder) Build(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) {
@@ -93,30 +102,30 @@ type client struct {
 }
 
 // AddWatch overrides the transport helper's AddWatch to save the LDS
-// resource_name. This is required when handling an RDS response to perform hot
+// resource_name. This is required when handling an RDS response to perform host
 // matching.
-func (v3c *client) AddWatch(resourceType, resourceName string) {
+func (v3c *client) AddWatch(rType xdsclient.ResourceType, rName string) {
 	v3c.mu.Lock()
 	// Special handling for LDS, because RDS needs the LDS resource_name for
 	// response host matching.
-	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+	if rType == xdsclient.ListenerResource {
 		// Set hostname to the first LDS resource_name, and reset it when the
 		// last LDS watch is removed. The upper level Client isn't expected to
 		// watchLDS more than once.
 		v3c.ldsWatchCount++
 		if v3c.ldsWatchCount == 1 {
-			v3c.ldsResourceName = resourceName
+			v3c.ldsResourceName = rName
 		}
 	}
 	v3c.mu.Unlock()
-	v3c.TransportHelper.AddWatch(resourceType, resourceName)
+	v3c.TransportHelper.AddWatch(rType, rName)
 }
 
-func (v3c *client) RemoveWatch(resourceType, resourceName string) {
+func (v3c *client) RemoveWatch(rType xdsclient.ResourceType, rName string) {
 	v3c.mu.Lock()
 	// Special handling for LDS, because RDS needs the LDS resource_name for
 	// response host matching.
-	if resourceType == version.V2ListenerURL || resourceType == version.V3ListenerURL {
+	if rType == xdsclient.ListenerResource {
 		// Set hostname to the first LDS resource_name, and reset it when the
 		// last LDS watch is removed. The upper level Client isn't expected to
 		// watchLDS more than once.
@@ -126,30 +135,29 @@ func (v3c *client) RemoveWatch(resourceType, resourceName string) {
 		}
 	}
 	v3c.mu.Unlock()
-	v3c.TransportHelper.RemoveWatch(resourceType, resourceName)
+	v3c.TransportHelper.RemoveWatch(rType, rName)
 }
 
 func (v3c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
 	return v3adsgrpc.NewAggregatedDiscoveryServiceClient(v3c.cc).StreamAggregatedResources(v3c.ctx, grpc.WaitForReady(true))
 }
 
-// sendRequest sends a request for provided typeURL and resource on the provided
-// stream.
+// sendRequest sends out a DiscoveryRequest for the given resourceNames, of type
+// rType, on the provided stream.
 //
 // version is the ack version to be sent with the request
-// - If this is the new request (not an ack/nack), version will be an empty
-// string
-// - If this is an ack, version will be the version from the response
+// - If this is the new request (not an ack/nack), version will be empty.
+// - If this is an ack, version will be the version from the response.
 // - If this is a nack, version will be the previous acked version (from
-// versionMap). If there was no ack before, it will be an empty string
-func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, typeURL, version, nonce string) error {
+//   versionMap). If there was no ack before, it will be empty.
+func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce string) error {
 	stream, ok := s.(adsStream)
 	if !ok {
 		return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s)
 	}
 	req := &v3discoverypb.DiscoveryRequest{
 		Node:          v3c.nodeProto,
-		TypeUrl:       typeURL,
+		TypeUrl:       resourceTypeToURL[rType],
 		ResourceNames: resourceNames,
 		VersionInfo:   version,
 		ResponseNonce: nonce,
@@ -180,10 +188,11 @@ func (v3c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
 	return resp, nil
 }
 
-func (v3c *client) HandleResponse(r proto.Message) (string, string, string, error) {
+func (v3c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, string, string, error) {
+	rType := xdsclient.UnknownResource
 	resp, ok := r.(*v3discoverypb.DiscoveryResponse)
 	if !ok {
-		return "", "", "", fmt.Errorf("xds: unsupported message type: %T", resp)
+		return rType, "", "", fmt.Errorf("xds: unsupported message type: %T", resp)
 	}
 
 	// Note that the xDS transport protocol is versioned independently of
@@ -191,21 +200,26 @@ func (v3c *client) HandleResponse(r proto.Message) (string, string, string, erro
 	// of resource types using new versions of the transport protocol, or
 	// vice-versa. Hence we need to handle v3 type_urls as well here.
 	var err error
-	switch resp.GetTypeUrl() {
-	case version.V2ListenerURL, version.V3ListenerURL:
+	url := resp.GetTypeUrl()
+	switch {
+	case xdsclient.IsListenerResource(url):
 		err = v3c.handleLDSResponse(resp)
-	case version.V2RouteConfigURL, version.V3RouteConfigURL:
+		rType = xdsclient.ListenerResource
+	case xdsclient.IsRouteConfigResource(url):
 		err = v3c.handleRDSResponse(resp)
-	case version.V2ClusterURL, version.V3ClusterURL:
+		rType = xdsclient.RouteConfigResource
+	case xdsclient.IsClusterResource(url):
 		err = v3c.handleCDSResponse(resp)
-	case version.V2EndpointsURL, version.V3EndpointsURL:
+		rType = xdsclient.ClusterResource
+	case xdsclient.IsEndpointsResource(url):
 		err = v3c.handleEDSResponse(resp)
+		rType = xdsclient.EndpointsResource
 	default:
-		return "", "", "", xdsclient.ErrResourceTypeUnsupported{
+		return rType, "", "", xdsclient.ErrResourceTypeUnsupported{
 			ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()),
 		}
 	}
-	return resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), err
+	return rType, resp.GetVersionInfo(), resp.GetNonce(), err
 }
 
 // handleLDSResponse processes an LDS response received from the xDS server. On
diff --git a/xds/xds.go b/xds/xds.go
index 40d62605c263..fa0d699e8734 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -26,5 +26,6 @@ package xds
 import (
 	_ "google.golang.org/grpc/xds/internal/balancer"  // Register the balancers.
 	_ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 xDS API client.
+	_ "google.golang.org/grpc/xds/internal/client/v3" // Register the v3 xDS API client.
 	_ "google.golang.org/grpc/xds/internal/resolver"  // Register the xds_resolver.
 )

From f640ae6a4f4368a09e7d60e0303d40a3c9b50517 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Wed, 19 Aug 2020 12:01:00 -0700
Subject: [PATCH 166/481] security/authorization: util function for converting
 CEL expression string (#3822)

* security/authorization: add the util function to compile string cel expr
---
 security/authorization/engine/engine.go  |   2 +-
 security/authorization/util/util.go      |  63 ++++++++++++
 security/authorization/util/util_test.go | 121 +++++++++++++++++++++++
 3 files changed, 185 insertions(+), 1 deletion(-)
 create mode 100644 security/authorization/util/util.go
 create mode 100644 security/authorization/util/util_test.go

diff --git a/security/authorization/engine/engine.go b/security/authorization/engine/engine.go
index e964c89b6821..d598ff409fa6 100644
--- a/security/authorization/engine/engine.go
+++ b/security/authorization/engine/engine.go
@@ -33,7 +33,7 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-var logger = grpclog.Component("channelz")
+var logger = grpclog.Component("authorization")
 
 var stringAttributeMap = map[string]func(*AuthorizationArgs) (string, error){
 	"request.url_path":                    (*AuthorizationArgs).getRequestURLPath,
diff --git a/security/authorization/util/util.go b/security/authorization/util/util.go
new file mode 100644
index 000000000000..8aadad3862ba
--- /dev/null
+++ b/security/authorization/util/util.go
@@ -0,0 +1,63 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package engine
+
+import (
+	"errors"
+
+	expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
+)
+
+func compileCel(env *cel.Env, expr string) (*cel.Ast, error) {
+	ast, iss := env.Parse(expr)
+	// Report syntactic errors, if present.
+	if iss.Err() != nil {
+		return nil, iss.Err()
+	}
+	// Type-check the expression for correctness.
+	checked, iss := env.Check(ast)
+	if iss.Err() != nil {
+		return nil, iss.Err()
+	}
+	// Check the result type is a Boolean.
+	if !proto.Equal(checked.ResultType(), decls.Bool) {
+		return nil, errors.New("failed to compile CEL string: get non-bool value")
+	}
+	return checked, nil
+}
+
+func compileStringToCheckedExpr(expr string, declarations []*expr.Decl) (*expr.CheckedExpr, error) {
+	env, err := cel.NewEnv(cel.Declarations(declarations...))
+	if err != nil {
+		return nil, err
+	}
+	checked, err := compileCel(env, expr)
+	if err != nil {
+		return nil, err
+	}
+	checkedExpr, err := cel.AstToCheckedExpr(checked)
+	if err != nil {
+		return nil, err
+	}
+	return checkedExpr, nil
+}
diff --git a/security/authorization/util/util_test.go b/security/authorization/util/util_test.go
new file mode 100644
index 000000000000..981a00864d58
--- /dev/null
+++ b/security/authorization/util/util_test.go
@@ -0,0 +1,121 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package engine
+
+import (
+	"testing"
+
+	expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
+	"google.golang.org/grpc/internal/grpctest"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestStringConvert(t *testing.T) {
+	declarations := []*expr.Decl{
+		decls.NewIdent("request.url_path", decls.String, nil),
+		decls.NewIdent("request.host", decls.String, nil),
+		decls.NewIdent("connection.uri_san_peer_certificate", decls.String, nil),
+	}
+	env, err := cel.NewEnv()
+	if err != nil {
+		t.Fatalf("Failed to create the CEL environment")
+	}
+	for _, test := range []struct {
+		desc             string
+		wantEvalOutcome  bool
+		wantParsingError bool
+		wantEvalError    bool
+		expr             string
+		authzArgs        map[string]interface{}
+	}{
+		{
+			desc:            "single primitive match",
+			wantEvalOutcome: true,
+			expr:            "request.url_path.startsWith('/pkg.service/test')",
+			authzArgs:       map[string]interface{}{"request.url_path": "/pkg.service/test"},
+		},
+		{
+			desc:            "single compare match",
+			wantEvalOutcome: true,
+			expr:            "connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'",
+			authzArgs:       map[string]interface{}{"connection.uri_san_peer_certificate": "cluster/ns/default/sa/admin"},
+		},
+		{
+			desc:            "single primitive no match",
+			wantEvalOutcome: false,
+			expr:            "request.url_path.startsWith('/pkg.service/test')",
+			authzArgs:       map[string]interface{}{"request.url_path": "/source/pkg.service/test"},
+		},
+		{
+			desc:            "primitive and compare match",
+			wantEvalOutcome: true,
+			expr:            "request.url_path == '/pkg.service/test' && connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'",
+			authzArgs: map[string]interface{}{"request.url_path": "/pkg.service/test",
+				"connection.uri_san_peer_certificate": "cluster/ns/default/sa/admin"},
+		},
+		{
+			desc:             "parse error field not present in environment",
+			wantParsingError: true,
+			expr:             "request.source_path.startsWith('/pkg.service/test')",
+			authzArgs:        map[string]interface{}{"request.url_path": "/pkg.service/test"},
+		},
+		{
+			desc:          "eval error argument not included in environment",
+			wantEvalError: true,
+			expr:          "request.url_path.startsWith('/pkg.service/test')",
+			authzArgs:     map[string]interface{}{"request.source_path": "/pkg.service/test"},
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			checked, err := compileStringToCheckedExpr(test.expr, declarations)
+			if (err != nil) != test.wantParsingError {
+				t.Fatalf("Error mismatch in conversion, wantParsingError =%v, got %v", test.wantParsingError, err != nil)
+			}
+			if test.wantParsingError {
+				return
+			}
+			ast := cel.CheckedExprToAst(checked)
+			program, err := env.Program(ast)
+			if err != nil {
+				t.Fatalf("Failed to create CEL Program: %v", err)
+			}
+			eval, _, err := program.Eval(test.authzArgs)
+			if (err != nil) != test.wantEvalError {
+				t.Fatalf("Error mismatch in evaluation, wantEvalError =%v, got %v", test.wantEvalError, err != nil)
+			}
+			if test.wantEvalError {
+				return
+			}
+			if eval.Value() != test.wantEvalOutcome {
+				t.Fatalf("Error in evaluating converted CheckedExpr: want %v, got %v", test.wantEvalOutcome, eval.Value())
+			}
+		})
+	}
+}

From d16bb4c338a5470a3b3bc30750fafc26cafd215c Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 20 Aug 2020 12:50:29 -0700
Subject: [PATCH 167/481] transport: use escaped string instead of int->string
 conversion (#3831)

---
 internal/transport/http_util_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/transport/http_util_test.go b/internal/transport/http_util_test.go
index 80b1c094a071..a3616f7389f5 100644
--- a/internal/transport/http_util_test.go
+++ b/internal/transport/http_util_test.go
@@ -142,7 +142,7 @@ func (s) TestDecodeEncodeGrpcMessage(t *testing.T) {
 	}
 }
 
-const binaryValue = string(128)
+const binaryValue = "\u0080"
 
 func (s) TestEncodeMetadataHeader(t *testing.T) {
 	for _, test := range []struct {

From 6c0171fad0b597250588aa2a25330a373d4a9074 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 21 Aug 2020 10:41:53 -0700
Subject: [PATCH 168/481] rls: Update rls protos and code/test to reflect those
 changes. (#3832)

---
 balancer/rls/internal/config.go               |  22 +-
 balancer/rls/internal/config_test.go          |  47 +-
 balancer/rls/internal/picker.go               |  43 +-
 balancer/rls/internal/picker_test.go          | 683 +++++++++---------
 .../proto/grpc_lookup_v1/rls_config.pb.go     | 183 ++---
 regenerate.sh                                 |   1 +
 6 files changed, 430 insertions(+), 549 deletions(-)

diff --git a/balancer/rls/internal/config.go b/balancer/rls/internal/config.go
index 94ed3c5ee70b..305c09106ba3 100644
--- a/balancer/rls/internal/config.go
+++ b/balancer/rls/internal/config.go
@@ -61,7 +61,6 @@ type lbConfig struct {
 	maxAge               time.Duration
 	staleAge             time.Duration
 	cacheSizeBytes       int64
-	rpStrategy           rlspb.RouteLookupConfig_RequestProcessingStrategy
 	defaultTarget        string
 	cpName               string
 	cpTargetField        string
@@ -75,7 +74,6 @@ func (lbCfg *lbConfig) Equal(other *lbConfig) bool {
 		lbCfg.maxAge == other.maxAge &&
 		lbCfg.staleAge == other.staleAge &&
 		lbCfg.cacheSizeBytes == other.cacheSizeBytes &&
-		lbCfg.rpStrategy == other.rpStrategy &&
 		lbCfg.defaultTarget == other.defaultTarget &&
 		lbCfg.cpName == other.cpName &&
 		lbCfg.cpTargetField == other.cpTargetField &&
@@ -167,11 +165,6 @@ func (l *loadBalancingConfig) UnmarshalJSON(data []byte) error {
 //      - must be greater than zero
 //      - TODO(easwars): Define a minimum value for this field, to be used when
 //        left unspecified
-//   ** request_processing_strategy field:
-//      - must have a value other than STRATEGY_UNSPECIFIED
-//      - if set to SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR or
-//        ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS, the default_target field must be
-//        set to a non-empty value
 // * childPolicy field:
 //  - must find a valid child policy with a valid config (the child policy must
 //    be able to parse the provided config successfully when we pass it a dummy
@@ -242,22 +235,10 @@ func (*rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 		logger.Infof("rls: max_age in service config is %v, using %v", maxAge, maxMaxAge)
 		maxAge = maxMaxAge
 	}
-
 	cacheSizeBytes := rlsProto.GetCacheSizeBytes()
 	if cacheSizeBytes <= 0 {
 		return nil, fmt.Errorf("rls: cache_size_bytes must be greater than 0 in service config {%+v}", string(c))
 	}
-
-	rpStrategy := rlsProto.GetRequestProcessingStrategy()
-	if rpStrategy == rlspb.RouteLookupConfig_STRATEGY_UNSPECIFIED {
-		return nil, fmt.Errorf("rls: request_processing_strategy cannot be left unspecified in service config {%+v}", string(c))
-	}
-	defaultTarget := rlsProto.GetDefaultTarget()
-	if (rpStrategy == rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR ||
-		rpStrategy == rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS) && defaultTarget == "" {
-		return nil, fmt.Errorf("rls: request_processing_strategy is %s, but default_target is not set", rpStrategy.String())
-	}
-
 	if childPolicy == nil {
 		return nil, fmt.Errorf("rls: childPolicy is invalid in service config {%+v}", string(c))
 	}
@@ -283,8 +264,7 @@ func (*rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 		maxAge:               maxAge,
 		staleAge:             staleAge,
 		cacheSizeBytes:       cacheSizeBytes,
-		rpStrategy:           rpStrategy,
-		defaultTarget:        defaultTarget,
+		defaultTarget:        rlsProto.GetDefaultTarget(),
 		// TODO(easwars): Once we refactor validateChildPolicyConfig and make
 		// it a method on the lbConfig object, we could directly store the
 		// balancer.Builder and/or balancer.ConfigParser here instead of the
diff --git a/balancer/rls/internal/config_test.go b/balancer/rls/internal/config_test.go
index fa22e15a6501..1efd054512b2 100644
--- a/balancer/rls/internal/config_test.go
+++ b/balancer/rls/internal/config_test.go
@@ -28,8 +28,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 
 	"google.golang.org/grpc/balancer"
-	_ "google.golang.org/grpc/balancer/grpclb" // grpclb for config parsing.
-	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
+	_ "google.golang.org/grpc/balancer/grpclb"               // grpclb for config parsing.
 	_ "google.golang.org/grpc/internal/resolver/passthrough" // passthrough resolver.
 )
 
@@ -58,7 +57,6 @@ func testEqual(a, b *lbConfig) bool {
 		a.maxAge == b.maxAge &&
 		a.staleAge == b.staleAge &&
 		a.cacheSizeBytes == b.cacheSizeBytes &&
-		a.rpStrategy == b.rpStrategy &&
 		a.defaultTarget == b.defaultTarget &&
 		a.cpName == b.cpName &&
 		a.cpTargetField == b.cpTargetField &&
@@ -91,7 +89,6 @@ func TestParseConfig(t *testing.T) {
 					"maxAge" : "500s",
 					"staleAge": "600s",
 					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
 					"defaultTarget": "passthrough:///default"
 				},
 				"childPolicy": [
@@ -107,7 +104,6 @@ func TestParseConfig(t *testing.T) {
 				maxAge:               5 * time.Minute,  // This is max maxAge.
 				staleAge:             time.Duration(0), // StaleAge is ignore because it was higher than maxAge.
 				cacheSizeBytes:       1000,
-				rpStrategy:           rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
 				defaultTarget:        "passthrough:///default",
 				cpName:               "grpclb",
 				cpTargetField:        "service_name",
@@ -127,7 +123,6 @@ func TestParseConfig(t *testing.T) {
 					"maxAge": "60s",
 					"staleAge" : "50s",
 					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
 					"defaultTarget": "passthrough:///default"
 				},
 				"childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}],
@@ -139,7 +134,6 @@ func TestParseConfig(t *testing.T) {
 				maxAge:               60 * time.Second,
 				staleAge:             50 * time.Second,
 				cacheSizeBytes:       1000,
-				rpStrategy:           rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
 				defaultTarget:        "passthrough:///default",
 				cpName:               "grpclb",
 				cpTargetField:        "service_name",
@@ -288,41 +282,6 @@ func TestParseConfigErrors(t *testing.T) {
 			}`),
 			wantErr: "rls: cache_size_bytes must be greater than 0 in service config",
 		},
-		{
-			desc: "invalid request processing strategy",
-			input: []byte(`{
-				"routeLookupConfig": {
-					"grpcKeybuilders": [{
-						"names": [{"service": "service", "method": "method"}],
-						"headers": [{"key": "k1", "names": ["v1"]}]
-					}],
-					"lookupService": "passthrough:///target",
-					"lookupServiceTimeout" : "10s",
-					"maxAge": "30s",
-					"staleAge" : "25s",
-					"cacheSizeBytes": 1000
-				}
-			}`),
-			wantErr: "rls: request_processing_strategy cannot be left unspecified in service config",
-		},
-		{
-			desc: "request processing strategy without default target",
-			input: []byte(`{
-				"routeLookupConfig": {
-					"grpcKeybuilders": [{
-						"names": [{"service": "service", "method": "method"}],
-						"headers": [{"key": "k1", "names": ["v1"]}]
-					}],
-					"lookupService": "passthrough:///target",
-					"lookupServiceTimeout" : "10s",
-					"maxAge": "30s",
-					"staleAge" : "25s",
-					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS"
-				}
-			}`),
-			wantErr: "default_target is not set",
-		},
 		{
 			desc: "no child policy",
 			input: []byte(`{
@@ -336,7 +295,6 @@ func TestParseConfigErrors(t *testing.T) {
 					"maxAge": "30s",
 					"staleAge" : "25s",
 					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
 					"defaultTarget": "passthrough:///default"
 				}
 			}`),
@@ -355,7 +313,6 @@ func TestParseConfigErrors(t *testing.T) {
 					"maxAge": "30s",
 					"staleAge" : "25s",
 					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
 					"defaultTarget": "passthrough:///default"
 				},
 				"childPolicy": [
@@ -378,7 +335,6 @@ func TestParseConfigErrors(t *testing.T) {
 					"maxAge": "30s",
 					"staleAge" : "25s",
 					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
 					"defaultTarget": "passthrough:///default"
 				},
 				"childPolicy": [
@@ -402,7 +358,6 @@ func TestParseConfigErrors(t *testing.T) {
 					"maxAge": "30s",
 					"staleAge" : "25s",
 					"cacheSizeBytes": 1000,
-					"request_processing_strategy": "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
 					"defaultTarget": "passthrough:///default"
 				},
 				"childPolicy": [
diff --git a/balancer/rls/internal/picker.go b/balancer/rls/internal/picker.go
index ce7100536d95..738449446558 100644
--- a/balancer/rls/internal/picker.go
+++ b/balancer/rls/internal/picker.go
@@ -25,7 +25,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/rls/internal/cache"
 	"google.golang.org/grpc/balancer/rls/internal/keys"
-	rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -42,10 +41,6 @@ type rlsPicker struct {
 	// The keyBuilder map used to generate RLS keys for the RPC. This is built
 	// by the LB policy based on the received ServiceConfig.
 	kbm keys.BuilderMap
-	// This is the request processing strategy as indicated by the LB policy's
-	// ServiceConfig. This controls how to process a RPC when the data required
-	// to make the pick decision is not in the cache.
-	strategy rlspb.RouteLookupConfig_RequestProcessingStrategy
 
 	// The following hooks are setup by the LB policy to enable the rlsPicker to
 	// access state stored in the policy. This approach has the following
@@ -79,15 +74,6 @@ type rlsPicker struct {
 	defaultPick func(balancer.PickInfo) (balancer.PickResult, error)
 }
 
-// This helper function decides if the pick should delegate to the default
-// rlsPicker based on the request processing strategy. This is used when the
-// data cache does not have a valid entry for the current RPC and the RLS
-// request is throttled, or if the current data cache entry is in backoff.
-func (p *rlsPicker) shouldDelegateToDefault() bool {
-	return p.strategy == rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR ||
-		p.strategy == rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS
-}
-
 // Pick makes the routing decision for every outbound RPC.
 func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 	// For every incoming request, we first build the RLS keys using the
@@ -123,9 +109,9 @@ func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 		if p.shouldThrottle() {
 			// The entry doesn't exist or has expired and the new RLS request
 			// has been throttled. Treat it as an error and delegate to default
-			// pick or fail the pick, based on the request processing strategy.
+			// pick, if one exists, or fail the pick.
 			if entry == nil || entry.ExpiryTime.Before(now) {
-				if p.shouldDelegateToDefault() {
+				if p.defaultPick != nil {
 					return p.defaultPick(info)
 				}
 				return balancer.PickResult{}, errRLSThrottled
@@ -144,25 +130,20 @@ func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 			// this cache entry.
 			return entry.ChildPicker.Pick(info)
 		} else if entry.BackoffTime.After(now) {
-			// The entry has expired, but is in backoff. We either delegate to
-			// the default rlsPicker or return the error from the last failed
-			// RLS request for this entry.
-			if p.shouldDelegateToDefault() {
+			// The entry has expired, but is in backoff. We delegate to the
+			// default pick, if one exists, or return the error from the last
+			// failed RLS request for this entry.
+			if p.defaultPick != nil {
 				return p.defaultPick(info)
 			}
 			return balancer.PickResult{}, entry.CallStatus
 		}
 	}
 
-	// Either we didn't find an entry or found an entry which had expired and
-	// was not in backoff (which is also essentially equivalent to not finding
-	// an entry), and we started an RLS request in the background. We either
-	// queue the pick or delegate to the default pick. In the former case, upon
-	// receipt of an RLS response, the LB policy will send a new rlsPicker to
-	// the channel, and the pick will be retried.
-	if p.strategy == rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR ||
-		p.strategy == rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR {
-		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
-	}
-	return p.defaultPick(info)
+	// We get here only in the following cases:
+	// * No data cache entry or expired entry, RLS request sent out
+	// * No valid data cache entry and Pending cache entry exists
+	// We need to queue to pick which will be handled once the RLS response is
+	// received.
+	return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
 }
diff --git a/balancer/rls/internal/picker_test.go b/balancer/rls/internal/picker_test.go
index fc5f935d6929..1397bf8085d8 100644
--- a/balancer/rls/internal/picker_test.go
+++ b/balancer/rls/internal/picker_test.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/rls/internal/cache"
 	"google.golang.org/grpc/balancer/rls/internal/keys"
@@ -38,6 +39,12 @@ import (
 
 const defaultTestMaxAge = 5 * time.Second
 
+// initKeyBuilderMap initializes a keyBuilderMap of the form:
+// {
+// 		"gFoo": "k1=n1",
+//		"gBar/method1": "k2=n21,n22"
+// 		"gFoobar": "k3=n3",
+// }
 func initKeyBuilderMap() (keys.BuilderMap, error) {
 	kb1 := &rlspb.GrpcKeyBuilder{
 		Names:   []*rlspb.GrpcKeyBuilder_Name{{Service: "gFoo"}},
@@ -63,15 +70,33 @@ type fakeSubConn struct {
 	id int
 }
 
-// fakeChildPicker sends a PickResult with a fakeSubConn with the configured id.
-type fakeChildPicker struct {
+// fakePicker sends a PickResult with a fakeSubConn with the configured id.
+type fakePicker struct {
 	id int
 }
 
-func (p *fakeChildPicker) Pick(_ balancer.PickInfo) (balancer.PickResult, error) {
+func (p *fakePicker) Pick(_ balancer.PickInfo) (balancer.PickResult, error) {
 	return balancer.PickResult{SubConn: &fakeSubConn{id: p.id}}, nil
 }
 
+// newFakePicker returns a fakePicker configured with a random ID. The subConns
+// returned by this picker are of type fakefakeSubConn, and contain the same
+// random ID, which tests can use to verify.
+func newFakePicker() *fakePicker {
+	return &fakePicker{id: grpcrand.Intn(math.MaxInt32)}
+}
+
+func verifySubConn(sc balancer.SubConn, wantID int) error {
+	fsc, ok := sc.(*fakeSubConn)
+	if !ok {
+		return fmt.Errorf("Pick() returned a SubConn of type %T, want %T", sc, &fakeSubConn{})
+	}
+	if fsc.id != wantID {
+		return fmt.Errorf("Pick() returned SubConn %d, want %d", fsc.id, wantID)
+	}
+	return nil
+}
+
 // TestPickKeyBuilder verifies the different possible scenarios for forming an
 // RLS key for an incoming RPC.
 func TestPickKeyBuilder(t *testing.T) {
@@ -110,8 +135,7 @@ func TestPickKeyBuilder(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			randID := grpcrand.Intn(math.MaxInt32)
 			p := rlsPicker{
-				kbm:      kbm,
-				strategy: rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
+				kbm: kbm,
 				readCache: func(key cache.Key) (*cache.Entry, bool) {
 					if !cmp.Equal(key, test.wantKey) {
 						t.Fatalf("rlsPicker using cacheKey %v, want %v", key, test.wantKey)
@@ -124,7 +148,7 @@ func TestPickKeyBuilder(t *testing.T) {
 						// Cache entry is configured with a child policy whose
 						// rlsPicker always returns an empty PickResult and nil
 						// error.
-						ChildPicker: &fakeChildPicker{id: randID},
+						ChildPicker: &fakePicker{id: randID},
 					}, false
 				},
 				// The other hooks are not set here because they are not expected to be
@@ -149,7 +173,9 @@ func TestPickKeyBuilder(t *testing.T) {
 	}
 }
 
-func TestPick(t *testing.T) {
+// TestPick_DataCacheMiss_PendingCacheMiss verifies different Pick scenarios
+// where the entry is neither found in the data cache nor in the pending cache.
+func TestPick_DataCacheMiss_PendingCacheMiss(t *testing.T) {
 	const (
 		rpcPath       = "/gFoo/method"
 		wantKeyMapStr = "k1=v1"
@@ -160,369 +186,275 @@ func TestPick(t *testing.T) {
 	}
 	md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"})
 	wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr}
-	rlsLastErr := errors.New("last RLS request failed")
 
 	tests := []struct {
 		desc string
-		// The cache entry, as returned by the overridden readCache hook.
-		cacheEntry *cache.Entry
-		// Whether or not a pending entry exists, as returned by the overridden
-		// readCache hook.
-		pending bool
+		// Whether or not a default target is configured.
+		defaultPickExists bool
 		// Whether or not the RLS request should be throttled.
 		throttle bool
 		// Whether or not the test is expected to make a new RLS request.
-		newRLSRequest bool
-		// Whether or not the test ends up delegating to the default pick.
-		useDefaultPick bool
-		// Whether or not the test ends up delegating to the child policy in
-		// the cache entry.
-		useChildPick bool
-		// Request processing strategy as used by the rlsPicker.
-		strategy rlspb.RouteLookupConfig_RequestProcessingStrategy
+		wantRLSRequest bool
 		// Expected error returned by the rlsPicker under test.
 		wantErr error
 	}{
 		{
-			desc:     "cacheMiss_pending_defaultTargetOnError",
-			pending:  true,
-			strategy: rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:  balancer.ErrNoSubConnAvailable,
+			desc:              "rls request throttled with default pick",
+			defaultPickExists: true,
+			throttle:          true,
 		},
 		{
-			desc:     "cacheMiss_pending_clientSeesError",
-			pending:  true,
-			strategy: rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:  balancer.ErrNoSubConnAvailable,
-		},
-		{
-			desc:           "cacheMiss_pending_defaultTargetOnMiss",
-			pending:        true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
-		},
-		{
-			desc:          "cacheMiss_noPending_notThrottled_defaultTargetOnError",
-			newRLSRequest: true,
-			strategy:      rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:       balancer.ErrNoSubConnAvailable,
-		},
-		{
-			desc:          "cacheMiss_noPending_notThrottled_clientSeesError",
-			newRLSRequest: true,
-			strategy:      rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:       balancer.ErrNoSubConnAvailable,
-		},
-		{
-			desc:           "cacheMiss_noPending_notThrottled_defaultTargetOnMiss",
-			newRLSRequest:  true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
-		},
-		{
-			desc:           "cacheMiss_noPending_throttled_defaultTargetOnError",
-			throttle:       true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:        nil,
-		},
-		{
-			desc:     "cacheMiss_noPending_throttled_clientSeesError",
+			desc:     "rls request throttled without default pick",
 			throttle: true,
-			strategy: rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
 			wantErr:  errRLSThrottled,
 		},
 		{
-			desc:           "cacheMiss_noPending_throttled_defaultTargetOnMiss",
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			throttle:       true,
-			useDefaultPick: true,
-			wantErr:        nil,
-		},
-		{
-			desc:           "cacheHit_noPending_boExpired_dataExpired_throttled_defaultTargetOnError",
-			cacheEntry:     &cache.Entry{}, // Everything is expired in this entry
-			throttle:       true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:        nil,
-		},
-		{
-			desc:       "cacheHit_noPending_boExpired_dataExpired_throttled_clientSeesError",
-			cacheEntry: &cache.Entry{}, // Everything is expired in this entry
-			throttle:   true,
-			strategy:   rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:    errRLSThrottled,
-		},
-		{
-			desc:           "cacheHit_noPending_boExpired_dataExpired_throttled_defaultTargetOnMiss",
-			cacheEntry:     &cache.Entry{}, // Everything is expired in this entry
-			throttle:       true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
-		},
-		{
-			desc:         "cacheHit_noPending_stale_boExpired_dataNotExpired_throttled_defaultTargetOnMiss",
-			cacheEntry:   &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			throttle:     true, // Proactive refresh is throttled.
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:      nil,
-		},
-		{
-			desc:         "cacheHit_noPending_stale_boExpired_dataNotExpired_throttled_clientSeesError",
-			cacheEntry:   &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			throttle:     true, // Proactive refresh is throttled.
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:      nil,
-		},
-		{
-			desc:         "cacheHit_noPending_stale_boExpired_dataNotExpired_throttled_defaultTargetOnError",
-			cacheEntry:   &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			throttle:     true, // Proactive refresh is throttled.
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:      nil,
-		},
-		{
-			desc:          "cacheHit_noPending_boExpired_dataExpired_notThrottled_defaultTargetOnError",
-			cacheEntry:    &cache.Entry{}, // Everything is expired in this entry
-			newRLSRequest: true,
-			strategy:      rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:       balancer.ErrNoSubConnAvailable,
-		},
-		{
-			desc:          "cacheHit_noPending_boExpired_dataExpired_notThrottled_clientSeesError",
-			cacheEntry:    &cache.Entry{}, // Everything is expired in this entry
-			newRLSRequest: true,
-			strategy:      rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:       balancer.ErrNoSubConnAvailable,
-		},
-		{
-			desc:           "cacheHit_noPending_boExpired_dataExpired_notThrottled_defaultTargetOnMiss",
-			cacheEntry:     &cache.Entry{}, // Everything is expired in this entry
-			newRLSRequest:  true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
-		},
-		{
-			desc:          "cacheHit_noPending_stale_boExpired_dataNotExpired_notThrottled_defaultTargetOnMiss",
-			cacheEntry:    &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			newRLSRequest: true, // Proactive refresh.
-			useChildPick:  true,
-			strategy:      rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:       nil,
-		},
-		{
-			desc:          "cacheHit_noPending_stale_boExpired_dataNotExpired_notThrottled_clientSeesError",
-			cacheEntry:    &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			newRLSRequest: true, // Proactive refresh.
-			useChildPick:  true,
-			strategy:      rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:       nil,
-		},
-		{
-			desc:          "cacheHit_noPending_stale_boExpired_dataNotExpired_notThrottled_defaultTargetOnError",
-			cacheEntry:    &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			newRLSRequest: true, // Proactive refresh.
-			useChildPick:  true,
-			strategy:      rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:       nil,
-		},
-		{
-			desc:           "cacheHit_noPending_stale_boNotExpired_dataExpired_defaultTargetOnError",
-			cacheEntry:     &cache.Entry{BackoffTime: time.Now().Add(defaultTestMaxAge)},
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:        nil,
-		},
-		{
-			desc:           "cacheHit_noPending_stale_boNotExpired_dataExpired_defaultTargetOnMiss",
-			cacheEntry:     &cache.Entry{BackoffTime: time.Now().Add(defaultTestMaxAge)},
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
-		},
-		{
-			desc: "cacheHit_noPending_stale_boNotExpired_dataExpired_clientSeesError",
-			cacheEntry: &cache.Entry{
-				BackoffTime: time.Now().Add(defaultTestMaxAge),
-				CallStatus:  rlsLastErr,
-			},
-			strategy: rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:  rlsLastErr,
-		},
-		{
-			desc: "cacheHit_noPending_stale_boNotExpired_dataNotExpired_defaultTargetOnError",
-			cacheEntry: &cache.Entry{
-				ExpiryTime:  time.Now().Add(defaultTestMaxAge),
-				BackoffTime: time.Now().Add(defaultTestMaxAge),
-				CallStatus:  rlsLastErr,
-			},
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:      nil,
-		},
-		{
-			desc: "cacheHit_noPending_stale_boNotExpired_dataNotExpired_defaultTargetOnMiss",
-			cacheEntry: &cache.Entry{
-				ExpiryTime:  time.Now().Add(defaultTestMaxAge),
-				BackoffTime: time.Now().Add(defaultTestMaxAge),
-				CallStatus:  rlsLastErr,
-			},
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:      nil,
+			desc:           "rls request not throttled",
+			wantRLSRequest: true,
+			wantErr:        balancer.ErrNoSubConnAvailable,
 		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			rlsCh := testutils.NewChannel()
+			defaultPicker := newFakePicker()
+
+			p := rlsPicker{
+				kbm: kbm,
+				// Cache lookup fails, no pending entry.
+				readCache: func(key cache.Key) (*cache.Entry, bool) {
+					if !cmp.Equal(key, wantKey) {
+						t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey)
+					}
+					return nil, false
+				},
+				shouldThrottle: func() bool { return test.throttle },
+				startRLS: func(path string, km keys.KeyMap) {
+					if !test.wantRLSRequest {
+						rlsCh.Send(errors.New("RLS request attempted when none was expected"))
+						return
+					}
+					if path != rpcPath {
+						rlsCh.Send(fmt.Errorf("RLS request initiated for rpcPath %s, want %s", path, rpcPath))
+						return
+					}
+					if km.Str != wantKeyMapStr {
+						rlsCh.Send(fmt.Errorf("RLS request initiated with keys %v, want %v", km.Str, wantKeyMapStr))
+						return
+					}
+					rlsCh.Send(nil)
+				},
+			}
+			if test.defaultPickExists {
+				p.defaultPick = defaultPicker.Pick
+			}
+
+			gotResult, err := p.Pick(balancer.PickInfo{
+				FullMethodName: rpcPath,
+				Ctx:            metadata.NewOutgoingContext(context.Background(), md),
+			})
+			if err != test.wantErr {
+				t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr)
+			}
+			// If the test specified that a new RLS request should be made,
+			// verify it.
+			if test.wantRLSRequest {
+				if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil {
+					t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err)
+				}
+			}
+			if test.wantErr != nil {
+				return
+			}
+
+			// We get here only for cases where we expect the pick to be
+			// delegated to the default picker.
+			if err := verifySubConn(gotResult.SubConn, defaultPicker.id); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// TestPick_DataCacheMiss_PendingCacheMiss verifies different Pick scenarios
+// where the entry is not found in the data cache, but there is a entry in the
+// pending cache. For all of these scenarios, no new RLS request will be sent.
+func TestPick_DataCacheMiss_PendingCacheHit(t *testing.T) {
+	const (
+		rpcPath       = "/gFoo/method"
+		wantKeyMapStr = "k1=v1"
+	)
+	kbm, err := initKeyBuilderMap()
+	if err != nil {
+		t.Fatalf("Failed to create keyBuilderMap: %v", err)
+	}
+	md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"})
+	wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr}
+
+	tests := []struct {
+		desc              string
+		defaultPickExists bool
+	}{
 		{
-			desc: "cacheHit_noPending_stale_boNotExpired_dataNotExpired_clientSeesError",
-			cacheEntry: &cache.Entry{
-				ExpiryTime:  time.Now().Add(defaultTestMaxAge),
-				BackoffTime: time.Now().Add(defaultTestMaxAge),
-				CallStatus:  rlsLastErr,
-			},
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:      nil,
+			desc:              "default pick exists",
+			defaultPickExists: true,
 		},
 		{
-			desc: "cacheHit_noPending_notStale_dataNotExpired_defaultTargetOnError",
-			cacheEntry: &cache.Entry{
-				ExpiryTime: time.Now().Add(defaultTestMaxAge),
-				StaleTime:  time.Now().Add(defaultTestMaxAge),
-			},
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:      nil,
+			desc: "default pick does not exists",
 		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			rlsCh := testutils.NewChannel()
+			p := rlsPicker{
+				kbm: kbm,
+				// Cache lookup fails, pending entry exists.
+				readCache: func(key cache.Key) (*cache.Entry, bool) {
+					if !cmp.Equal(key, wantKey) {
+						t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey)
+					}
+					return nil, true
+				},
+				// Never throttle. We do not expect an RLS request to be sent out anyways.
+				shouldThrottle: func() bool { return false },
+				startRLS: func(_ string, _ keys.KeyMap) {
+					rlsCh.Send(nil)
+				},
+			}
+			if test.defaultPickExists {
+				p.defaultPick = func(info balancer.PickInfo) (balancer.PickResult, error) {
+					// We do not expect the default picker to be invoked at all.
+					// So, if we get here, the test will fail, because it
+					// expects the pick to be queued.
+					return balancer.PickResult{}, nil
+				}
+			}
+
+			if _, err := p.Pick(balancer.PickInfo{
+				FullMethodName: rpcPath,
+				Ctx:            metadata.NewOutgoingContext(context.Background(), md),
+			}); err != balancer.ErrNoSubConnAvailable {
+				t.Fatalf("Pick() returned error {%v}, want {%v}", err, balancer.ErrNoSubConnAvailable)
+			}
+
+			// Make sure that no RLS request was sent out.
+			if _, err := rlsCh.Receive(); err != testutils.ErrRecvTimeout {
+				t.Fatalf("RLS request sent out when pending entry exists")
+			}
+		})
+	}
+}
+
+// TestPick_DataCacheHit_PendingCacheMiss verifies different Pick scenarios
+// where the entry is found in the data cache, and there is no entry in the
+// pending cache. This includes cases where the entry in the data cache is
+// stale, expired or in backoff.
+func TestPick_DataCacheHit_PendingCacheMiss(t *testing.T) {
+	const (
+		rpcPath       = "/gFoo/method"
+		wantKeyMapStr = "k1=v1"
+	)
+	kbm, err := initKeyBuilderMap()
+	if err != nil {
+		t.Fatalf("Failed to create keyBuilderMap: %v", err)
+	}
+	md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"})
+	wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr}
+	rlsLastErr := errors.New("last RLS request failed")
+
+	tests := []struct {
+		desc string
+		// The cache entry, as returned by the overridden readCache hook.
+		cacheEntry *cache.Entry
+		// Whether or not a default target is configured.
+		defaultPickExists bool
+		// Whether or not the RLS request should be throttled.
+		throttle bool
+		// Whether or not the test is expected to make a new RLS request.
+		wantRLSRequest bool
+		// Whether or not the rlsPicker should delegate to the child picker.
+		wantChildPick bool
+		// Whether or not the rlsPicker should delegate to the default picker.
+		wantDefaultPick bool
+		// Expected error returned by the rlsPicker under test.
+		wantErr error
+	}{
 		{
-			desc: "cacheHit_noPending_notStale_dataNotExpired_defaultTargetOnMiss",
+			desc: "valid entry",
 			cacheEntry: &cache.Entry{
 				ExpiryTime: time.Now().Add(defaultTestMaxAge),
 				StaleTime:  time.Now().Add(defaultTestMaxAge),
 			},
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:      nil,
+			wantChildPick: true,
 		},
 		{
-			desc: "cacheHit_noPending_notStale_dataNotExpired_clientSeesError",
-			cacheEntry: &cache.Entry{
-				ExpiryTime: time.Now().Add(defaultTestMaxAge),
-				StaleTime:  time.Now().Add(defaultTestMaxAge),
-			},
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:      nil,
+			desc:          "entryStale_requestThrottled",
+			cacheEntry:    &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
+			throttle:      true,
+			wantChildPick: true,
 		},
 		{
-			desc:       "cacheHit_pending_dataExpired_boExpired_defaultTargetOnError",
-			cacheEntry: &cache.Entry{},
-			strategy:   rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:    balancer.ErrNoSubConnAvailable,
+			desc:           "entryStale_requestNotThrottled",
+			cacheEntry:     &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
+			wantRLSRequest: true,
+			wantChildPick:  true,
 		},
 		{
-			desc:           "cacheHit_pending_dataExpired_boExpired_defaultTargetOnMiss",
-			cacheEntry:     &cache.Entry{},
-			pending:        true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
+			desc:              "entryExpired_requestThrottled_defaultPickExists",
+			cacheEntry:        &cache.Entry{},
+			throttle:          true,
+			defaultPickExists: true,
+			wantDefaultPick:   true,
 		},
 		{
-			desc:       "cacheHit_pending_dataExpired_boExpired_clientSeesError",
+			desc:       "entryExpired_requestThrottled_defaultPickNotExists",
 			cacheEntry: &cache.Entry{},
-			pending:    true,
-			strategy:   rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:    balancer.ErrNoSubConnAvailable,
+			throttle:   true,
+			wantErr:    errRLSThrottled,
 		},
 		{
-			desc: "cacheHit_pending_dataExpired_boNotExpired_defaultTargetOnError",
-			cacheEntry: &cache.Entry{
-				BackoffTime: time.Now().Add(defaultTestMaxAge),
-				CallStatus:  rlsLastErr,
-			},
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:        nil,
+			desc:           "entryExpired_requestNotThrottled",
+			cacheEntry:     &cache.Entry{},
+			wantRLSRequest: true,
+			wantErr:        balancer.ErrNoSubConnAvailable,
 		},
 		{
-			desc: "cacheHit_pending_dataExpired_boNotExpired_defaultTargetOnMiss",
+			desc: "entryExpired_backoffNotExpired_defaultPickExists",
 			cacheEntry: &cache.Entry{
 				BackoffTime: time.Now().Add(defaultTestMaxAge),
 				CallStatus:  rlsLastErr,
 			},
-			pending:        true,
-			useDefaultPick: true,
-			strategy:       rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:        nil,
+			defaultPickExists: true,
 		},
 		{
-			desc: "cacheHit_pending_dataExpired_boNotExpired_clientSeesError",
+			desc: "entryExpired_backoffNotExpired_defaultPickNotExists",
 			cacheEntry: &cache.Entry{
 				BackoffTime: time.Now().Add(defaultTestMaxAge),
 				CallStatus:  rlsLastErr,
 			},
-			pending:  true,
-			strategy: rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:  rlsLastErr,
-		},
-		{
-			desc:         "cacheHit_pending_dataNotExpired_defaultTargetOnError",
-			cacheEntry:   &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			pending:      true,
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR,
-			wantErr:      nil,
-		},
-		{
-			desc:         "cacheHit_pending_dataNotExpired_defaultTargetOnMiss",
-			cacheEntry:   &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			pending:      true,
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS,
-			wantErr:      nil,
-		},
-		{
-			desc:         "cacheHit_pending_dataNotExpired_clientSeesError",
-			cacheEntry:   &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
-			pending:      true,
-			useChildPick: true,
-			strategy:     rlspb.RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR,
-			wantErr:      nil,
+			wantErr: rlsLastErr,
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			rlsCh := testutils.NewChannel()
-			randID := grpcrand.Intn(math.MaxInt32)
-			// We instantiate a fakeChildPicker which will return a fakeSubConn
-			// with configured id. Either the childPicker or the defaultPicker
-			// is configured to use this fakePicker based on whether
-			// useChidlPick or useDefaultPick is set in the test.
-			childPicker := &fakeChildPicker{id: randID}
+			childPicker := newFakePicker()
+			defaultPicker := newFakePicker()
 
 			p := rlsPicker{
-				kbm:      kbm,
-				strategy: test.strategy,
+				kbm: kbm,
 				readCache: func(key cache.Key) (*cache.Entry, bool) {
 					if !cmp.Equal(key, wantKey) {
 						t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey)
 					}
-					if test.useChildPick {
-						test.cacheEntry.ChildPicker = childPicker
-					}
-					return test.cacheEntry, test.pending
+					test.cacheEntry.ChildPicker = childPicker
+					return test.cacheEntry, false
 				},
 				shouldThrottle: func() bool { return test.throttle },
 				startRLS: func(path string, km keys.KeyMap) {
-					if !test.newRLSRequest {
+					if !test.wantRLSRequest {
 						rlsCh.Send(errors.New("RLS request attempted when none was expected"))
 						return
 					}
@@ -536,12 +468,9 @@ func TestPick(t *testing.T) {
 					}
 					rlsCh.Send(nil)
 				},
-				defaultPick: func(info balancer.PickInfo) (balancer.PickResult, error) {
-					if !test.useDefaultPick {
-						return balancer.PickResult{}, errors.New("Using default pick when the test doesn't want to use default pick")
-					}
-					return childPicker.Pick(info)
-				},
+			}
+			if test.defaultPickExists {
+				p.defaultPick = defaultPicker.Pick
 			}
 
 			gotResult, err := p.Pick(balancer.PickInfo{
@@ -551,26 +480,128 @@ func TestPick(t *testing.T) {
 			if err != test.wantErr {
 				t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr)
 			}
-			if test.useChildPick || test.useDefaultPick {
-				// For cases where the pick is not queued, but is delegated to
-				// either the child rlsPicker or the default rlsPicker, we
-				// verify that the expected fakeSubConn is returned.
-				sc, ok := gotResult.SubConn.(*fakeSubConn)
-				if !ok {
-					t.Fatalf("Pick() returned a SubConn of type %T, want %T", gotResult.SubConn, &fakeSubConn{})
-				}
-				if sc.id != randID {
-					t.Fatalf("Pick() returned SubConn %d, want %d", sc.id, randID)
-				}
-			}
-
 			// If the test specified that a new RLS request should be made,
 			// verify it.
-			if test.newRLSRequest {
+			if test.wantRLSRequest {
 				if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil {
 					t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err)
 				}
 			}
+			if test.wantErr != nil {
+				return
+			}
+
+			// We get here only for cases where we expect the pick to be
+			// delegated to the child picker or the default picker.
+			if test.wantChildPick {
+				if err := verifySubConn(gotResult.SubConn, childPicker.id); err != nil {
+					t.Fatal(err)
+				}
+
+			}
+			if test.wantDefaultPick {
+				if err := verifySubConn(gotResult.SubConn, defaultPicker.id); err != nil {
+					t.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// TestPick_DataCacheHit_PendingCacheHit verifies different Pick scenarios where
+// the entry is found both in the data cache and in the pending cache. This
+// mostly verifies cases where the entry is stale, but there is already a
+// pending RLS request, so no new request should be sent out.
+func TestPick_DataCacheHit_PendingCacheHit(t *testing.T) {
+	const (
+		rpcPath       = "/gFoo/method"
+		wantKeyMapStr = "k1=v1"
+	)
+	kbm, err := initKeyBuilderMap()
+	if err != nil {
+		t.Fatalf("Failed to create keyBuilderMap: %v", err)
+	}
+	md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"})
+	wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr}
+
+	tests := []struct {
+		desc string
+		// The cache entry, as returned by the overridden readCache hook.
+		cacheEntry *cache.Entry
+		// Whether or not a default target is configured.
+		defaultPickExists bool
+		// Expected error returned by the rlsPicker under test.
+		wantErr error
+	}{
+		{
+			desc:       "stale entry",
+			cacheEntry: &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
+		},
+		{
+			desc:              "stale entry with default picker",
+			cacheEntry:        &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)},
+			defaultPickExists: true,
+		},
+		{
+			desc:              "entryExpired_defaultPickExists",
+			cacheEntry:        &cache.Entry{},
+			defaultPickExists: true,
+			wantErr:           balancer.ErrNoSubConnAvailable,
+		},
+		{
+			desc:       "entryExpired_defaultPickNotExists",
+			cacheEntry: &cache.Entry{},
+			wantErr:    balancer.ErrNoSubConnAvailable,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			rlsCh := testutils.NewChannel()
+			childPicker := newFakePicker()
+
+			p := rlsPicker{
+				kbm: kbm,
+				readCache: func(key cache.Key) (*cache.Entry, bool) {
+					if !cmp.Equal(key, wantKey) {
+						t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey)
+					}
+					test.cacheEntry.ChildPicker = childPicker
+					return test.cacheEntry, true
+				},
+				// Never throttle. We do not expect an RLS request to be sent out anyways.
+				shouldThrottle: func() bool { return false },
+				startRLS: func(path string, km keys.KeyMap) {
+					rlsCh.Send(nil)
+				},
+			}
+			if test.defaultPickExists {
+				p.defaultPick = func(info balancer.PickInfo) (balancer.PickResult, error) {
+					// We do not expect the default picker to be invoked at all.
+					// So, if we get here, we return an error.
+					return balancer.PickResult{}, errors.New("default picker invoked when expecting a child pick")
+				}
+			}
+
+			gotResult, err := p.Pick(balancer.PickInfo{
+				FullMethodName: rpcPath,
+				Ctx:            metadata.NewOutgoingContext(context.Background(), md),
+			})
+			if err != test.wantErr {
+				t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr)
+			}
+			// Make sure that no RLS request was sent out.
+			if _, err := rlsCh.Receive(); err != testutils.ErrRecvTimeout {
+				t.Fatalf("RLS request sent out when pending entry exists")
+			}
+			if test.wantErr != nil {
+				return
+			}
+
+			// We get here only for cases where we expect the pick to be
+			// delegated to the child picker.
+			if err := verifySubConn(gotResult.SubConn, childPicker.id); err != nil {
+				t.Fatal(err)
+			}
 		})
 	}
 }
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
index 88479f41e397..01d5b656ebe2 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: grpc/rls/grpc_lookup_v1/rls_config.proto
+// source: grpc/lookup/v1/rls_config.proto
 
 package grpc_lookup_v1
 
@@ -21,51 +21,6 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
-// Specify how to process a request when not already in the cache.
-type RouteLookupConfig_RequestProcessingStrategy int32
-
-const (
-	RouteLookupConfig_STRATEGY_UNSPECIFIED RouteLookupConfig_RequestProcessingStrategy = 0
-	// Query the RLS and process the request using target returned by the
-	// lookup. The target will then be cached and used for processing
-	// subsequent requests for the same key. Any errors during lookup service
-	// processing will fall back to default target for request processing.
-	RouteLookupConfig_SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR RouteLookupConfig_RequestProcessingStrategy = 1
-	// Query the RLS and process the request using target returned by the
-	// lookup. The target will then be cached and used for processing
-	// subsequent requests for the same key. Any errors during lookup service
-	// processing will return an error back to the client.  Services with
-	// strict regional routing requirements should use this strategy.
-	RouteLookupConfig_SYNC_LOOKUP_CLIENT_SEES_ERROR RouteLookupConfig_RequestProcessingStrategy = 2
-	// Query the RLS asynchronously but respond with the default target.  The
-	// target in the lookup response will then be cached and used for
-	// subsequent requests.  Services with strict latency requirements (but not
-	// strict regional routing requirements) should use this strategy.
-	RouteLookupConfig_ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS RouteLookupConfig_RequestProcessingStrategy = 3
-)
-
-var RouteLookupConfig_RequestProcessingStrategy_name = map[int32]string{
-	0: "STRATEGY_UNSPECIFIED",
-	1: "SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR",
-	2: "SYNC_LOOKUP_CLIENT_SEES_ERROR",
-	3: "ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS",
-}
-
-var RouteLookupConfig_RequestProcessingStrategy_value = map[string]int32{
-	"STRATEGY_UNSPECIFIED":                0,
-	"SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR": 1,
-	"SYNC_LOOKUP_CLIENT_SEES_ERROR":       2,
-	"ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS": 3,
-}
-
-func (x RouteLookupConfig_RequestProcessingStrategy) String() string {
-	return proto.EnumName(RouteLookupConfig_RequestProcessingStrategy_name, int32(x))
-}
-
-func (RouteLookupConfig_RequestProcessingStrategy) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_f013e3228551a7a8, []int{3, 0}
-}
-
 // Extract a key based on a given name (e.g. header name or query parameter
 // name).  The name must match one of the names listed in the "name" field.  If
 // the "required_match" field is true, one of the specified names must be
@@ -88,7 +43,7 @@ func (m *NameMatcher) Reset()         { *m = NameMatcher{} }
 func (m *NameMatcher) String() string { return proto.CompactTextString(m) }
 func (*NameMatcher) ProtoMessage()    {}
 func (*NameMatcher) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f013e3228551a7a8, []int{0}
+	return fileDescriptor_5fe74d4f6e8f01c1, []int{0}
 }
 
 func (m *NameMatcher) XXX_Unmarshal(b []byte) error {
@@ -146,7 +101,7 @@ func (m *GrpcKeyBuilder) Reset()         { *m = GrpcKeyBuilder{} }
 func (m *GrpcKeyBuilder) String() string { return proto.CompactTextString(m) }
 func (*GrpcKeyBuilder) ProtoMessage()    {}
 func (*GrpcKeyBuilder) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f013e3228551a7a8, []int{1}
+	return fileDescriptor_5fe74d4f6e8f01c1, []int{1}
 }
 
 func (m *GrpcKeyBuilder) XXX_Unmarshal(b []byte) error {
@@ -197,7 +152,7 @@ func (m *GrpcKeyBuilder_Name) Reset()         { *m = GrpcKeyBuilder_Name{} }
 func (m *GrpcKeyBuilder_Name) String() string { return proto.CompactTextString(m) }
 func (*GrpcKeyBuilder_Name) ProtoMessage()    {}
 func (*GrpcKeyBuilder_Name) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f013e3228551a7a8, []int{1, 0}
+	return fileDescriptor_5fe74d4f6e8f01c1, []int{1, 0}
 }
 
 func (m *GrpcKeyBuilder_Name) XXX_Unmarshal(b []byte) error {
@@ -311,7 +266,7 @@ func (m *HttpKeyBuilder) Reset()         { *m = HttpKeyBuilder{} }
 func (m *HttpKeyBuilder) String() string { return proto.CompactTextString(m) }
 func (*HttpKeyBuilder) ProtoMessage()    {}
 func (*HttpKeyBuilder) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f013e3228551a7a8, []int{2}
+	return fileDescriptor_5fe74d4f6e8f01c1, []int{2}
 }
 
 func (m *HttpKeyBuilder) XXX_Unmarshal(b []byte) error {
@@ -396,27 +351,23 @@ type RouteLookupConfig struct {
 	CacheSizeBytes int64 `protobuf:"varint,7,opt,name=cache_size_bytes,json=cacheSizeBytes,proto3" json:"cache_size_bytes,omitempty"`
 	// This is a list of all the possible targets that can be returned by the
 	// lookup service.  If a target not on this list is returned, it will be
-	// treated the same as an RPC error from the RLS.
+	// treated the same as an unhealthy target.
 	ValidTargets []string `protobuf:"bytes,8,rep,name=valid_targets,json=validTargets,proto3" json:"valid_targets,omitempty"`
-	// This value provides a default target to use if needed.  It will be used for
-	// request processing strategy SYNC_LOOKUP_DEFAULT_TARGET_ON_ERROR if RLS
-	// returns an error, or strategy ASYNC_LOOKUP_DEFAULT_TARGET_ON_MISS if RLS
-	// returns an error or there is a cache miss in the client.  It will also be
-	// used if there are no healthy backends for an RLS target.  Note that
-	// requests can be routed only to a subdomain of the original target,
-	// e.g. "us_east_1.cloudbigtable.googleapis.com".
-	DefaultTarget             string                                      `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"`
-	RequestProcessingStrategy RouteLookupConfig_RequestProcessingStrategy `protobuf:"varint,10,opt,name=request_processing_strategy,json=requestProcessingStrategy,proto3,enum=grpc.lookup.v1.RouteLookupConfig_RequestProcessingStrategy" json:"request_processing_strategy,omitempty"`
-	XXX_NoUnkeyedLiteral      struct{}                                    `json:"-"`
-	XXX_unrecognized          []byte                                      `json:"-"`
-	XXX_sizecache             int32                                       `json:"-"`
+	// This value provides a default target to use if needed.  If set, it will be
+	// used if RLS returns an error, times out, or returns an invalid response.
+	// Note that requests can be routed only to a subdomain of the original
+	// target, e.g. "us_east_1.cloudbigtable.googleapis.com".
+	DefaultTarget        string   `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
 }
 
 func (m *RouteLookupConfig) Reset()         { *m = RouteLookupConfig{} }
 func (m *RouteLookupConfig) String() string { return proto.CompactTextString(m) }
 func (*RouteLookupConfig) ProtoMessage()    {}
 func (*RouteLookupConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f013e3228551a7a8, []int{3}
+	return fileDescriptor_5fe74d4f6e8f01c1, []int{3}
 }
 
 func (m *RouteLookupConfig) XXX_Unmarshal(b []byte) error {
@@ -500,15 +451,7 @@ func (m *RouteLookupConfig) GetDefaultTarget() string {
 	return ""
 }
 
-func (m *RouteLookupConfig) GetRequestProcessingStrategy() RouteLookupConfig_RequestProcessingStrategy {
-	if m != nil {
-		return m.RequestProcessingStrategy
-	}
-	return RouteLookupConfig_STRATEGY_UNSPECIFIED
-}
-
 func init() {
-	proto.RegisterEnum("grpc.lookup.v1.RouteLookupConfig_RequestProcessingStrategy", RouteLookupConfig_RequestProcessingStrategy_name, RouteLookupConfig_RequestProcessingStrategy_value)
 	proto.RegisterType((*NameMatcher)(nil), "grpc.lookup.v1.NameMatcher")
 	proto.RegisterType((*GrpcKeyBuilder)(nil), "grpc.lookup.v1.GrpcKeyBuilder")
 	proto.RegisterType((*GrpcKeyBuilder_Name)(nil), "grpc.lookup.v1.GrpcKeyBuilder.Name")
@@ -516,57 +459,47 @@ func init() {
 	proto.RegisterType((*RouteLookupConfig)(nil), "grpc.lookup.v1.RouteLookupConfig")
 }
 
-func init() {
-	proto.RegisterFile("grpc/rls/grpc_lookup_v1/rls_config.proto", fileDescriptor_f013e3228551a7a8)
-}
-
-var fileDescriptor_f013e3228551a7a8 = []byte{
-	// 742 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xdd, 0x6e, 0xdb, 0x36,
-	0x18, 0x9d, 0xa2, 0xd4, 0x89, 0x99, 0x46, 0x71, 0x85, 0xa0, 0x50, 0x5a, 0xac, 0xf0, 0x1c, 0x14,
-	0xd3, 0xc5, 0x20, 0xa3, 0x1e, 0x36, 0x6c, 0xd8, 0x95, 0xed, 0x28, 0x99, 0x51, 0xd7, 0x36, 0x28,
-	0xe5, 0xa2, 0xc3, 0x00, 0x82, 0x96, 0xbf, 0x48, 0x42, 0x24, 0x53, 0xa5, 0x28, 0xa3, 0xee, 0xde,
-	0x68, 0xc0, 0xde, 0x60, 0x2f, 0xb2, 0xb7, 0x19, 0x28, 0x4a, 0x9e, 0xed, 0x2d, 0x4b, 0xef, 0xf4,
-	0x1d, 0x9e, 0x73, 0xc4, 0xef, 0x8f, 0xc8, 0x0e, 0x79, 0x16, 0x74, 0x79, 0x92, 0x77, 0xe5, 0x07,
-	0x49, 0x18, 0xbb, 0x2f, 0x32, 0xb2, 0x7a, 0x23, 0x21, 0x12, 0xb0, 0xe5, 0x5d, 0x1c, 0x3a, 0x19,
-	0x67, 0x82, 0x99, 0x86, 0x24, 0x38, 0x8a, 0xe0, 0xac, 0xde, 0xbc, 0x78, 0x15, 0x32, 0x16, 0x26,
-	0xd0, 0x2d, 0x4f, 0xe7, 0xc5, 0x5d, 0x77, 0x51, 0x70, 0x2a, 0x62, 0xb6, 0x54, 0xfc, 0xce, 0xaf,
-	0xe8, 0x64, 0x42, 0x53, 0x78, 0x47, 0x45, 0x10, 0x01, 0x37, 0x5b, 0x48, 0xbf, 0x87, 0xb5, 0xa5,
-	0xb5, 0x35, 0xbb, 0x89, 0xe5, 0xa7, 0x79, 0x8e, 0x9e, 0x2c, 0x69, 0x0a, 0xb9, 0x75, 0xd0, 0xd6,
-	0xed, 0x26, 0x56, 0x81, 0xf9, 0x1a, 0x19, 0x1c, 0x3e, 0x14, 0x31, 0x87, 0x05, 0x49, 0xa5, 0xd6,
-	0xd2, 0xdb, 0x9a, 0x7d, 0x8c, 0x4f, 0x6b, 0xb4, 0x34, 0xec, 0xfc, 0xa9, 0x21, 0xe3, 0x86, 0x67,
-	0xc1, 0x5b, 0x58, 0x0f, 0x8a, 0x38, 0x59, 0x00, 0x37, 0x7f, 0xac, 0xfd, 0xb4, 0xb6, 0x6e, 0x9f,
-	0xf4, 0x2e, 0x9d, 0xdd, 0x0b, 0x3b, 0xbb, 0x74, 0x47, 0x5e, 0xae, 0xfe, 0xe9, 0x77, 0xe8, 0x28,
-	0x02, 0xba, 0x00, 0xae, 0x2e, 0x73, 0xd2, 0x7b, 0xb9, 0x2f, 0xde, 0x4a, 0x05, 0xd7, 0xdc, 0x17,
-	0x3f, 0xa0, 0x43, 0x89, 0x9b, 0x16, 0x3a, 0xca, 0x81, 0xaf, 0xe2, 0x00, 0xaa, 0xfc, 0xea, 0xd0,
-	0x7c, 0x8e, 0x1a, 0x29, 0x88, 0x88, 0x2d, 0xac, 0x83, 0xf2, 0xa0, 0x8a, 0x3a, 0x7f, 0x69, 0xc8,
-	0xf8, 0x59, 0x88, 0x6c, 0xeb, 0xfa, 0x97, 0xe8, 0x34, 0x62, 0xb9, 0x20, 0x19, 0x15, 0x02, 0xf8,
-	0x52, 0xa5, 0xd1, 0xc4, 0x4f, 0x25, 0x38, 0xab, 0x30, 0x49, 0xca, 0xa8, 0x88, 0xfe, 0x21, 0xa9,
-	0xda, 0x3d, 0x95, 0xe0, 0x86, 0x74, 0x8d, 0x5a, 0x1f, 0x0a, 0xe0, 0x6b, 0x92, 0x51, 0x4e, 0x53,
-	0x10, 0x32, 0x2d, 0xfd, 0xf1, 0xb4, 0xce, 0x4a, 0xd1, 0x6c, 0xa3, 0xd9, 0xae, 0xca, 0xe1, 0xe7,
-	0x57, 0xa5, 0xf3, 0x47, 0x03, 0x3d, 0xc3, 0xac, 0x10, 0x30, 0x2e, 0x79, 0xc3, 0x72, 0x88, 0xcc,
-	0x11, 0x6a, 0x45, 0x42, 0x64, 0xe4, 0x1e, 0xd6, 0x73, 0x95, 0x71, 0xdd, 0xa8, 0x57, 0xfb, 0xae,
-	0xbb, 0x85, 0xc1, 0x67, 0x91, 0x8a, 0x6b, 0x99, 0xb4, 0x2a, 0x87, 0x75, 0xdb, 0xea, 0xe0, 0xbf,
-	0xad, 0x76, 0x7b, 0x8e, 0xcf, 0x42, 0x15, 0x6f, 0xac, 0x5e, 0x23, 0xa3, 0x1a, 0xf9, 0xba, 0x81,
-	0x7a, 0xd9, 0xa7, 0x53, 0x85, 0x7a, 0x55, 0x1b, 0xa7, 0xe8, 0xf9, 0x2e, 0x8d, 0x88, 0x38, 0x05,
-	0x56, 0x08, 0xeb, 0xb0, 0xad, 0xd9, 0x27, 0xbd, 0x0b, 0x47, 0x2d, 0x83, 0x53, 0x2f, 0x83, 0x73,
-	0x55, 0x2d, 0x03, 0x3e, 0xdf, 0x71, 0xf2, 0x95, 0xcc, 0xec, 0xa1, 0xa3, 0x94, 0x7e, 0x24, 0x34,
-	0x04, 0xeb, 0xc9, 0x63, 0x0e, 0x8d, 0x94, 0x7e, 0xec, 0x87, 0x60, 0x7e, 0x8f, 0x9a, 0xb9, 0xa0,
-	0x09, 0x94, 0xaa, 0xc6, 0x63, 0xaa, 0xe3, 0x92, 0x2b, 0x75, 0x36, 0x6a, 0x05, 0x34, 0x88, 0x80,
-	0xe4, 0xf1, 0x27, 0x20, 0xf3, 0xb5, 0x80, 0xdc, 0x3a, 0x6a, 0x6b, 0xb6, 0x8e, 0x8d, 0x12, 0xf7,
-	0xe2, 0x4f, 0x30, 0x90, 0xa8, 0x9c, 0xae, 0x15, 0x4d, 0xe2, 0x05, 0x11, 0x94, 0x87, 0x20, 0x72,
-	0xeb, 0x58, 0x4d, 0x57, 0x09, 0xfa, 0x0a, 0x93, 0x25, 0x5b, 0xc0, 0x1d, 0x2d, 0x12, 0x51, 0xd1,
-	0xac, 0xa6, 0x2a, 0x59, 0x85, 0x2a, 0x9e, 0xf9, 0x1b, 0x7a, 0x29, 0x37, 0x16, 0xe4, 0x44, 0x73,
-	0x16, 0x40, 0x9e, 0xc7, 0xcb, 0x90, 0xe4, 0x82, 0x53, 0x01, 0xe1, 0xda, 0x42, 0x6d, 0xcd, 0x36,
-	0x7a, 0x3f, 0xed, 0xf7, 0xeb, 0x5f, 0x73, 0xe3, 0x60, 0x65, 0x32, 0xdb, 0x78, 0x78, 0x95, 0x05,
-	0xbe, 0xe0, 0x0f, 0x1d, 0x75, 0x7e, 0xd7, 0xd0, 0xc5, 0x83, 0x42, 0xd3, 0x42, 0xe7, 0x9e, 0x8f,
-	0xfb, 0xbe, 0x7b, 0xf3, 0x9e, 0xdc, 0x4e, 0xbc, 0x99, 0x3b, 0x1c, 0x5d, 0x8f, 0xdc, 0xab, 0xd6,
-	0x17, 0xe6, 0xd7, 0xe8, 0xd2, 0x7b, 0x3f, 0x19, 0x92, 0xf1, 0x74, 0xfa, 0xf6, 0x76, 0x46, 0xae,
-	0xdc, 0xeb, 0xfe, 0xed, 0xd8, 0x27, 0x7e, 0x1f, 0xdf, 0xb8, 0x3e, 0x99, 0x4e, 0x88, 0x8b, 0xf1,
-	0x14, 0xb7, 0x34, 0xf3, 0x2b, 0xf4, 0xe5, 0x36, 0x71, 0x38, 0x1e, 0xb9, 0x13, 0x9f, 0x78, 0xae,
-	0xeb, 0x55, 0x94, 0x03, 0xe9, 0xd5, 0xff, 0x7f, 0xb3, 0x77, 0x23, 0xcf, 0x6b, 0xe9, 0x03, 0x0f,
-	0x3d, 0x8b, 0xd9, 0x5e, 0x21, 0x06, 0x06, 0x4e, 0x72, 0x55, 0x81, 0x99, 0x6c, 0xed, 0x4c, 0xfb,
-	0xe5, 0x9b, 0xaa, 0xd5, 0x21, 0x4b, 0xe8, 0x32, 0x74, 0x18, 0x0f, 0xcb, 0x27, 0xbb, 0xab, 0x34,
-	0x7b, 0xcf, 0xf7, 0xbc, 0x51, 0x4e, 0xc4, 0xb7, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x47, 0xa7,
-	0x94, 0xbe, 0xe0, 0x05, 0x00, 0x00,
+func init() { proto.RegisterFile("grpc/lookup/v1/rls_config.proto", fileDescriptor_5fe74d4f6e8f01c1) }
+
+var fileDescriptor_5fe74d4f6e8f01c1 = []byte{
+	// 615 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xed, 0x6a, 0xdb, 0x30,
+	0x14, 0xc5, 0x75, 0x9a, 0x0f, 0xa5, 0x75, 0x53, 0x53, 0x8a, 0xd7, 0x42, 0x17, 0x52, 0x0a, 0xfe,
+	0x31, 0x1c, 0x9a, 0xb1, 0xb1, 0xfd, 0x5c, 0x36, 0xf6, 0xfd, 0x11, 0xdc, 0xfe, 0x1a, 0x03, 0xa1,
+	0xd8, 0xb7, 0xb2, 0xa9, 0x1d, 0xb9, 0x92, 0x1c, 0x9a, 0x3e, 0xd0, 0x9e, 0x62, 0x2f, 0xb2, 0xb7,
+	0x19, 0x92, 0xec, 0x2e, 0x2e, 0x83, 0xec, 0x5f, 0xee, 0xd1, 0x39, 0x27, 0x3a, 0xf7, 0x5e, 0x19,
+	0x3d, 0xa6, 0xbc, 0x88, 0xc6, 0x19, 0x63, 0xd7, 0x65, 0x31, 0x5e, 0x9e, 0x8f, 0x79, 0x26, 0x70,
+	0xc4, 0x16, 0x57, 0x29, 0x0d, 0x0a, 0xce, 0x24, 0x73, 0x1d, 0x45, 0x08, 0x0c, 0x21, 0x58, 0x9e,
+	0x1f, 0x9d, 0x50, 0xc6, 0x68, 0x06, 0x63, 0x7d, 0x3a, 0x2f, 0xaf, 0xc6, 0x71, 0xc9, 0x89, 0x4c,
+	0xd9, 0xc2, 0xf0, 0x47, 0x3f, 0x50, 0xff, 0x2b, 0xc9, 0xe1, 0x0b, 0x91, 0x51, 0x02, 0xdc, 0x1d,
+	0x20, 0xfb, 0x1a, 0x56, 0x9e, 0x35, 0xb4, 0xfc, 0x5e, 0xa8, 0x7e, 0xba, 0x07, 0x68, 0x7b, 0x41,
+	0x72, 0x10, 0xde, 0xd6, 0xd0, 0xf6, 0x7b, 0xa1, 0x29, 0xdc, 0x33, 0xe4, 0x70, 0xb8, 0x29, 0x53,
+	0x0e, 0x31, 0xce, 0x95, 0xd6, 0xb3, 0x87, 0x96, 0xdf, 0x0d, 0x77, 0x6b, 0x54, 0x1b, 0x8e, 0x7e,
+	0x59, 0xc8, 0x79, 0xc7, 0x8b, 0xe8, 0x13, 0xac, 0xa6, 0x65, 0x9a, 0xc5, 0xc0, 0xdd, 0x97, 0xb5,
+	0x9f, 0x35, 0xb4, 0xfd, 0xfe, 0xe4, 0x34, 0x68, 0x5e, 0x38, 0x68, 0xd2, 0x03, 0x75, 0xb9, 0xfa,
+	0x4f, 0x9f, 0xa1, 0x4e, 0x02, 0x24, 0x06, 0x6e, 0x2e, 0xd3, 0x9f, 0x1c, 0x3f, 0x14, 0xaf, 0x45,
+	0x09, 0x6b, 0xee, 0xd1, 0x0b, 0xd4, 0x52, 0xb8, 0xeb, 0xa1, 0x8e, 0x00, 0xbe, 0x4c, 0x23, 0xa8,
+	0xf2, 0xd5, 0xa5, 0x7b, 0x88, 0xda, 0x39, 0xc8, 0x84, 0xc5, 0xde, 0x96, 0x3e, 0xa8, 0xaa, 0xd1,
+	0x6f, 0x0b, 0x39, 0xef, 0xa5, 0x2c, 0xd6, 0xae, 0x7f, 0x8a, 0x76, 0x13, 0x26, 0x24, 0x2e, 0x88,
+	0x94, 0xc0, 0x17, 0x26, 0x46, 0x2f, 0xdc, 0x51, 0xe0, 0xac, 0xc2, 0x14, 0xa9, 0x20, 0x32, 0xf9,
+	0x4b, 0x32, 0xbd, 0xdb, 0x51, 0xe0, 0x3d, 0xe9, 0x2d, 0x1a, 0xdc, 0x94, 0xc0, 0x57, 0xb8, 0x20,
+	0x9c, 0xe4, 0x20, 0x55, 0x2c, 0x7b, 0x73, 0xac, 0x3d, 0x2d, 0x9a, 0xdd, 0x6b, 0xd6, 0xbb, 0xd2,
+	0xfa, 0xff, 0xae, 0x8c, 0x7e, 0xb6, 0xd0, 0x7e, 0xc8, 0x4a, 0x09, 0x9f, 0x35, 0xef, 0xb5, 0x5e,
+	0x22, 0xf7, 0x03, 0x1a, 0x24, 0x52, 0x16, 0xf8, 0x1a, 0x56, 0x73, 0x93, 0xb8, 0x1e, 0xd4, 0xc9,
+	0x43, 0xd7, 0x66, 0x63, 0xc2, 0xbd, 0xc4, 0xd4, 0xb5, 0x4c, 0x59, 0x29, 0x45, 0xc3, 0x6a, 0xeb,
+	0xdf, 0x56, 0xcd, 0x99, 0x87, 0x7b, 0xd4, 0xd4, 0xf7, 0x56, 0x67, 0xc8, 0x31, 0x64, 0x5c, 0x0f,
+	0xd0, 0xd6, 0x73, 0xda, 0x35, 0xe8, 0x45, 0x35, 0xc6, 0x6f, 0xe8, 0xb0, 0x49, 0xc3, 0x32, 0xcd,
+	0x81, 0x95, 0xd2, 0x6b, 0x0d, 0x2d, 0xbf, 0x3f, 0x79, 0x14, 0x98, 0xc7, 0x10, 0xd4, 0x8f, 0x21,
+	0x78, 0x53, 0x3d, 0x86, 0xf0, 0xa0, 0xe1, 0x74, 0x69, 0x64, 0xee, 0x04, 0x75, 0x72, 0x72, 0x8b,
+	0x09, 0x05, 0x6f, 0x7b, 0x93, 0x43, 0x3b, 0x27, 0xb7, 0xaf, 0x28, 0xb8, 0xcf, 0x51, 0x4f, 0x48,
+	0x92, 0x81, 0x56, 0xb5, 0x37, 0xa9, 0xba, 0x9a, 0xab, 0x74, 0x3e, 0x1a, 0x44, 0x24, 0x4a, 0x00,
+	0x8b, 0xf4, 0x0e, 0xf0, 0x7c, 0x25, 0x41, 0x78, 0x9d, 0xa1, 0xe5, 0xdb, 0xa1, 0xa3, 0xf1, 0x8b,
+	0xf4, 0x0e, 0xa6, 0x0a, 0x55, 0xdb, 0xb5, 0x24, 0x59, 0x1a, 0x63, 0x49, 0x38, 0x05, 0x29, 0xbc,
+	0xae, 0xd9, 0x2e, 0x0d, 0x5e, 0x1a, 0x4c, 0xb5, 0x2c, 0x86, 0x2b, 0x52, 0x66, 0xb2, 0xa2, 0x79,
+	0x3d, 0xd3, 0xb2, 0x0a, 0x35, 0xbc, 0x8f, 0xad, 0x2e, 0x1a, 0xf4, 0xc3, 0x63, 0xf5, 0x6a, 0x41,
+	0x6d, 0x35, 0x67, 0x11, 0x08, 0x91, 0x2e, 0x28, 0x16, 0x92, 0x13, 0x09, 0x74, 0x35, 0xbd, 0x40,
+	0xfb, 0x29, 0x7b, 0x30, 0xb1, 0xa9, 0x13, 0x66, 0xc2, 0xac, 0xcc, 0x4c, 0x65, 0x9a, 0x59, 0xdf,
+	0x9f, 0x54, 0x19, 0x29, 0xcb, 0xc8, 0x82, 0x06, 0x8c, 0xd3, 0xf1, 0xfa, 0xb7, 0x4a, 0xaf, 0x42,
+	0x35, 0x9d, 0xe5, 0xf9, 0xbc, 0xad, 0x5b, 0xf1, 0xf4, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff,
+	0xaa, 0xbd, 0xb2, 0xd0, 0x04, 0x00, 0x00,
 }
diff --git a/regenerate.sh b/regenerate.sh
index cb7e5b61beb0..7e33b45e0862 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -69,6 +69,7 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/health/v1/health.proto
   ${WORKDIR}/grpc-proto/grpc/lb/v1/load_balancer.proto
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
+  ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
   ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
   ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto

From e14f1c23f642a1e3564db11cd5e934963773a193 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 21 Aug 2020 13:59:11 -0700
Subject: [PATCH 169/481] certprovider: API update to include certificate name.
 (#3797)

---
 credentials/tls/certprovider/distributor.go   |   8 +-
 .../tls/certprovider/distributor_test.go      | 177 +++-----
 credentials/tls/certprovider/provider.go      |  35 +-
 credentials/tls/certprovider/store.go         |  28 +-
 credentials/tls/certprovider/store_test.go    | 426 ++++++++++--------
 5 files changed, 353 insertions(+), 321 deletions(-)

diff --git a/credentials/tls/certprovider/distributor.go b/credentials/tls/certprovider/distributor.go
index d6feb3afeba1..fdb38a663fe8 100644
--- a/credentials/tls/certprovider/distributor.go
+++ b/credentials/tls/certprovider/distributor.go
@@ -42,7 +42,11 @@ type Distributor struct {
 	km   *KeyMaterial
 	pErr error
 
-	ready  *grpcsync.Event
+	// ready channel to unblock KeyMaterial() invocations blocked on
+	// availability of key material.
+	ready *grpcsync.Event
+	// done channel to notify provider implementations and unblock any
+	// KeyMaterial() calls, once the Distributor is closed.
 	closed *grpcsync.Event
 }
 
@@ -75,7 +79,7 @@ func (d *Distributor) Set(km *KeyMaterial, err error) {
 	d.mu.Unlock()
 }
 
-// KeyMaterial returns the most recent key material provided to the distributor.
+// KeyMaterial returns the most recent key material provided to the Distributor.
 // If no key material was provided at the time of this call, it will block until
 // the deadline on the context expires or fresh key material arrives.
 func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) {
diff --git a/credentials/tls/certprovider/distributor_test.go b/credentials/tls/certprovider/distributor_test.go
index e6c41564c293..bec00e919bcf 100644
--- a/credentials/tls/certprovider/distributor_test.go
+++ b/credentials/tls/certprovider/distributor_test.go
@@ -1,3 +1,5 @@
+// +build go1.13
+
 /*
  *
  * Copyright 2020 gRPC authors.
@@ -21,139 +23,90 @@ package certprovider
 import (
 	"context"
 	"errors"
-	"fmt"
-	"reflect"
-	"sync"
 	"testing"
 	"time"
 )
 
 var errProviderTestInternal = errors.New("provider internal error")
 
+// TestDistributorEmpty tries to read key material from an empty distributor and
+// expects the call to timeout.
+func (s) TestDistributorEmpty(t *testing.T) {
+	dist := NewDistributor()
+
+	// This call to KeyMaterial() should timeout because no key material has
+	// been set on the distributor as yet.
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, dist, nil); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal(err)
+	}
+}
+
 // TestDistributor invokes the different methods on the Distributor type and
 // verifies the results.
 func (s) TestDistributor(t *testing.T) {
 	dist := NewDistributor()
 
 	// Read cert/key files from testdata.
-	km, err := loadKeyMaterials()
-	if err != nil {
+	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+
+	// Push key material into the distributor and make sure that a call to
+	// KeyMaterial() returns the expected key material, with both the local
+	// certs and root certs.
+	dist.Set(km1, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, dist, km1); err != nil {
 		t.Fatal(err)
 	}
-	// wantKM1 has both local and root certs.
-	wantKM1 := *km
-	// wantKM2 has only local certs. Roots are nil-ed out.
-	wantKM2 := *km
-	wantKM2.Roots = nil
-
-	// Create a goroutines which work in lockstep with the rest of the test.
-	// This goroutine reads the key material from the distributor while the rest
-	// of the test sets it.
-	var wg sync.WaitGroup
-	wg.Add(1)
-	errCh := make(chan error)
-	proceedCh := make(chan struct{})
-	go func() {
-		defer wg.Done()
-
-		// The first call to KeyMaterial() should timeout because no key
-		// material has been set on the distributor as yet.
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout/2)
-		defer cancel()
-		if _, err := dist.KeyMaterial(ctx); err != context.DeadlineExceeded {
-			errCh <- err
-			return
-		}
-		proceedCh <- struct{}{}
-
-		// This call to KeyMaterial() should return the key material with both
-		// the local certs and the root certs.
-		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		gotKM, err := dist.KeyMaterial(ctx)
-		if err != nil {
-			errCh <- err
-			return
-		}
-		if !reflect.DeepEqual(gotKM, &wantKM1) {
-			errCh <- fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM1)
-		}
-		proceedCh <- struct{}{}
-
-		// This call to KeyMaterial() should eventually return key material with
-		// only the local certs.
-		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		for {
-			gotKM, err := dist.KeyMaterial(ctx)
-			if err != nil {
-				errCh <- err
-				return
-			}
-			if reflect.DeepEqual(gotKM, &wantKM2) {
-				break
-			}
-		}
-		proceedCh <- struct{}{}
-
-		// This call to KeyMaterial() should return nil key material and a
-		// non-nil error.
-		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		for {
-			gotKM, err := dist.KeyMaterial(ctx)
-			if gotKM == nil && err == errProviderTestInternal {
-				break
-			}
-			if err != nil {
-				// If we have gotten any error other than
-				// errProviderTestInternal, we should bail out.
-				errCh <- err
-				return
-			}
-		}
-		proceedCh <- struct{}{}
-
-		// This call to KeyMaterial() should eventually return errProviderClosed
-		// error.
-		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		for {
-			if _, err := dist.KeyMaterial(ctx); err == errProviderClosed {
-				break
-			}
-			time.Sleep(100 * time.Millisecond)
-		}
-	}()
 
-	waitAndDo(t, proceedCh, errCh, func() {
-		dist.Set(&wantKM1, nil)
-	})
+	// Push new key material into the distributor and make sure that a call to
+	// KeyMaterial() returns the expected key material, with only root certs.
+	dist.Set(km2, nil)
+	if err := readAndVerifyKeyMaterial(ctx, dist, km2); err != nil {
+		t.Fatal(err)
+	}
 
-	waitAndDo(t, proceedCh, errCh, func() {
-		dist.Set(&wantKM2, nil)
-	})
+	// Push an error into the distributor and make sure that a call to
+	// KeyMaterial() returns that error and nil keyMaterial.
+	dist.Set(km2, errProviderTestInternal)
+	if gotKM, err := dist.KeyMaterial(ctx); gotKM != nil || !errors.Is(err, errProviderTestInternal) {
+		t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", gotKM, err, errProviderTestInternal)
+	}
 
-	waitAndDo(t, proceedCh, errCh, func() {
-		dist.Set(&wantKM2, errProviderTestInternal)
-	})
+	// Stop the distributor and KeyMaterial() should return errProviderClosed.
+	dist.Stop()
+	if km, err := dist.KeyMaterial(ctx); !errors.Is(err, errProviderClosed) {
+		t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", km, err, errProviderClosed)
+	}
+}
 
-	waitAndDo(t, proceedCh, errCh, func() {
-		dist.Stop()
-	})
+// TestDistributorConcurrency invokes methods on the distributor in parallel. It
+// exercises that the scenario where a distributor's KeyMaterial() method is
+// blocked waiting for keyMaterial, while the Set() method is called from
+// another goroutine. It verifies that the KeyMaterial() method eventually
+// returns with expected keyMaterial.
+func (s) TestDistributorConcurrency(t *testing.T) {
+	dist := NewDistributor()
 
-}
+	// Read cert/key files from testdata.
+	km := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
 
-func waitAndDo(t *testing.T, proceedCh chan struct{}, errCh chan error, do func()) {
-	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		t.Fatalf("test timed out when waiting for event from distributor")
-	case <-proceedCh:
-		do()
-	case err := <-errCh:
+	// Push key material into the distributor from a goroutine and read from
+	// here to verify that the distributor returns the expected keyMaterial.
+	go func() {
+		// Add a small sleep here to make sure that the call to KeyMaterial()
+		// happens before the call to Set(), thereby the former is blocked till
+		// the latter happens.
+		time.Sleep(100 * time.Microsecond)
+		dist.Set(km, nil)
+	}()
+	if err := readAndVerifyKeyMaterial(ctx, dist, km); err != nil {
 		t.Fatal(err)
 	}
 }
diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go
index 58c93c1e731f..204e55686e91 100644
--- a/credentials/tls/certprovider/provider.go
+++ b/credentials/tls/certprovider/provider.go
@@ -16,7 +16,7 @@
  *
  */
 
-// Package certprovider defines APIs for certificate providers in gRPC.
+// Package certprovider defines APIs for Certificate Providers in gRPC.
 //
 // Experimental
 //
@@ -36,18 +36,18 @@ var (
 	// closed.
 	errProviderClosed = errors.New("provider instance is closed")
 
-	// m is a map from name to provider builder.
+	// m is a map from name to Provider builder.
 	m = make(map[string]Builder)
 )
 
-// Register registers the provider builder, whose name as returned by its Name()
+// Register registers the Provider builder, whose name as returned by its Name()
 // method will be used as the name registered with this builder. Registered
 // Builders are used by the Store to create Providers.
 func Register(b Builder) {
 	m[b.Name()] = b
 }
 
-// getBuilder returns the provider builder registered with the given name.
+// getBuilder returns the Provider builder registered with the given name.
 // If no builder is registered with the provided name, nil will be returned.
 func getBuilder(name string) Builder {
 	if b, ok := m[name]; ok {
@@ -58,8 +58,9 @@ func getBuilder(name string) Builder {
 
 // Builder creates a Provider.
 type Builder interface {
-	// Build creates a new provider with the provided config.
-	Build(StableConfig) Provider
+	// Build creates a new Provider and initializes it with the given config and
+	// options combination.
+	Build(StableConfig, Options) Provider
 
 	// ParseConfig converts config input in a format specific to individual
 	// implementations and returns an implementation of the StableConfig
@@ -72,9 +73,9 @@ type Builder interface {
 	Name() string
 }
 
-// StableConfig wraps the method to return a stable provider configuration.
+// StableConfig wraps the method to return a stable Provider configuration.
 type StableConfig interface {
-	// Canonical returns provider config as an arbitrary byte slice.
+	// Canonical returns Provider config as an arbitrary byte slice.
 	// Equivalent configurations must return the same output.
 	Canonical() []byte
 }
@@ -87,18 +88,30 @@ type StableConfig interface {
 // the latest secrets, and free to share any state between different
 // instantiations as they deem fit.
 type Provider interface {
-	// KeyMaterial returns the key material sourced by the provider.
+	// KeyMaterial returns the key material sourced by the Provider.
 	// Callers are expected to use the returned value as read-only.
 	KeyMaterial(ctx context.Context) (*KeyMaterial, error)
 
-	// Close cleans up resources allocated by the provider.
+	// Close cleans up resources allocated by the Provider.
 	Close()
 }
 
-// KeyMaterial wraps the certificates and keys returned by a provider instance.
+// KeyMaterial wraps the certificates and keys returned by a Provider instance.
 type KeyMaterial struct {
 	// Certs contains a slice of cert/key pairs used to prove local identity.
 	Certs []tls.Certificate
 	// Roots contains the set of trusted roots to validate the peer's identity.
 	Roots *x509.CertPool
 }
+
+// Options contains configuration knobs passed to a Provider at creation time.
+type Options struct {
+	// CertName holds the certificate name, whose key material is of interest to
+	// the caller.
+	CertName string
+	// WantRoot indicates if the caller is interested in the root certificate.
+	WantRoot bool
+	// WantIdentity indicates if the caller is interested in the identity
+	// certificate.
+	WantIdentity bool
+}
diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go
index 0d2b29b675b1..990e6ab06c34 100644
--- a/credentials/tls/certprovider/store.go
+++ b/credentials/tls/certprovider/store.go
@@ -38,6 +38,8 @@ type storeKey struct {
 	name string
 	// configuration of the certificate provider in string form.
 	config string
+	// opts contains the certificate name and other keyMaterial options.
+	opts Options
 }
 
 // wrappedProvider wraps a provider instance with a reference count.
@@ -57,17 +59,20 @@ type store struct {
 	providers map[storeKey]*wrappedProvider
 }
 
-// GetProvider returns a provider instance corresponding to name and config.
-// name is the registered name of the provider and config is the
-// provider-specific configuration. Implementations of the Builder interface
-// should clearly document the type of configuration accepted by them.
+// GetProvider returns a provider instance from which keyMaterial can be read.
 //
-// If a provider exists for the (name+config) combination, its reference count
-// is incremented before returning. If no provider exists for the (name+config)
-// combination, a new one is created using the registered builder. If no
-// registered builder is found, or the provider configuration is rejected by it,
-// a non-nil error is returned.
-func GetProvider(name string, config interface{}) (Provider, error) {
+// name is the registered name of the provider, config is the provider-specific
+// configuration, opts contains extra information that controls the keyMaterial
+// returned by the provider.
+//
+// Implementations of the Builder interface should clearly document the type of
+// configuration accepted by them.
+//
+// If a provider exists for passed arguments, its reference count is incremented
+// before returning. If no provider exists for the passed arguments, a new one
+// is created using the registered builder. If no registered builder is found,
+// or the provider configuration is rejected by it, a non-nil error is returned.
+func GetProvider(name string, config interface{}, opts Options) (Provider, error) {
 	provStore.mu.Lock()
 	defer provStore.mu.Unlock()
 
@@ -83,13 +88,14 @@ func GetProvider(name string, config interface{}) (Provider, error) {
 	sk := storeKey{
 		name:   name,
 		config: string(stableConfig.Canonical()),
+		opts:   opts,
 	}
 	if wp, ok := provStore.providers[sk]; ok {
 		wp.refCount++
 		return wp, nil
 	}
 
-	provider := builder.Build(stableConfig)
+	provider := builder.Build(stableConfig, opts)
 	if provider == nil {
 		return nil, fmt.Errorf("certprovider.Build(%v) failed", sk)
 	}
diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go
index 33135d41c854..a116148945eb 100644
--- a/credentials/tls/certprovider/store_test.go
+++ b/credentials/tls/certprovider/store_test.go
@@ -1,3 +1,5 @@
+// +build go1.13
+
 /*
  *
  * Copyright 2020 gRPC authors.
@@ -22,6 +24,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"reflect"
@@ -29,6 +32,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/testdata"
 )
 
@@ -39,9 +43,19 @@ const (
 	defaultTestTimeout = 1 * time.Second
 )
 
+var fpb1, fpb2 *fakeProviderBuilder
+
 func init() {
-	Register(&fakeProviderBuilder{name: fakeProvider1Name})
-	Register(&fakeProviderBuilder{name: fakeProvider2Name})
+	fpb1 = &fakeProviderBuilder{
+		name:         fakeProvider1Name,
+		providerChan: testutils.NewChannel(),
+	}
+	fpb2 = &fakeProviderBuilder{
+		name:         fakeProvider2Name,
+		providerChan: testutils.NewChannel(),
+	}
+	Register(fpb1)
+	Register(fpb2)
 }
 
 type s struct {
@@ -55,25 +69,20 @@ func Test(t *testing.T) {
 // fakeProviderBuilder builds new instances of fakeProvider and interprets the
 // config provided to it as a string.
 type fakeProviderBuilder struct {
-	name string
+	name         string
+	providerChan *testutils.Channel
 }
 
-func (b *fakeProviderBuilder) Build(StableConfig) Provider {
-	ctx, cancel := context.WithCancel(context.Background())
-	p := &fakeProvider{
-		Distributor: NewDistributor(),
-		cancel:      cancel,
-		done:        make(chan struct{}),
-		kmCh:        make(chan *KeyMaterial, 2),
-	}
-	go p.run(ctx)
+func (b *fakeProviderBuilder) Build(StableConfig, Options) Provider {
+	p := &fakeProvider{Distributor: NewDistributor()}
+	b.providerChan.Send(p)
 	return p
 }
 
 func (b *fakeProviderBuilder) ParseConfig(config interface{}) (StableConfig, error) {
 	s, ok := config.(string)
 	if !ok {
-		return nil, fmt.Errorf("provider %s received bad config %v", b.name, config)
+		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
 	}
 	return &fakeStableConfig{config: s}, nil
 }
@@ -90,262 +99,309 @@ func (c *fakeStableConfig) Canonical() []byte {
 	return []byte(c.config)
 }
 
-// fakeProvider is an implementation of the Provider interface which embeds a
-// Distributor and exposes two channels for the user:
-// 1. to be notified when the provider is closed
-// 2. to push new key material into the provider
+// fakeProvider is an implementation of the Provider interface which provides a
+// method for tests to invoke to push new key materials.
 type fakeProvider struct {
 	*Distributor
-
-	// Used to cancel the run goroutine when the provider is closed.
-	cancel context.CancelFunc
-	// This channel is closed when the provider is closed. Tests should block on
-	// this to make sure the provider is closed.
-	done chan struct{}
-	// Tests can push new key material on this channel, and the provider will
-	// return this on subsequent calls to KeyMaterial().
-	kmCh chan *KeyMaterial
 }
 
-func (p *fakeProvider) run(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case km := <-p.kmCh:
-			p.Distributor.Set(km, nil)
-		}
-	}
+// newKeyMaterial allows tests to push new key material to the fake provider
+// which will be made available to users of this provider.
+func (p *fakeProvider) newKeyMaterial(km *KeyMaterial, err error) {
+	p.Distributor.Set(km, err)
 }
 
+// Close helps implement the Provider interface.
 func (p *fakeProvider) Close() {
-	p.cancel()
 	p.Distributor.Stop()
 }
 
 // loadKeyMaterials is a helper to read cert/key files from testdata and convert
-// them into a KeyMaterial struct.
-func loadKeyMaterials() (*KeyMaterial, error) {
-	certs, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+// them into a KeyMaterialReader struct.
+func loadKeyMaterials(t *testing.T, cert, key, ca string) *KeyMaterial {
+	t.Helper()
+
+	certs, err := tls.LoadX509KeyPair(testdata.Path(cert), testdata.Path(key))
 	if err != nil {
-		return nil, err
+		t.Fatalf("Failed to load keyPair: %v", err)
 	}
 
-	pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
+	pemData, err := ioutil.ReadFile(testdata.Path(ca))
 	if err != nil {
-		return nil, err
+		t.Fatal(err)
 	}
 	roots := x509.NewCertPool()
 	roots.AppendCertsFromPEM(pemData)
-	return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots}, nil
+	return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots}
 }
 
-func makeProvider(t *testing.T, name, config string) (Provider, *fakeProvider) {
-	t.Helper()
+// kmReader wraps the KeyMaterial method exposed by Provider and Distributor
+// implementations. Defining the interface here makes it possible to use the
+// same helper from both provider and distributor tests.
+type kmReader interface {
+	KeyMaterial(context.Context) (*KeyMaterial, error)
+}
 
-	prov, err := GetProvider(name, config)
+// readAndVerifyKeyMaterial attempts to read key material from the given
+// provider and compares it against the expected key material.
+func readAndVerifyKeyMaterial(ctx context.Context, kmr kmReader, wantKM *KeyMaterial) error {
+	gotKM, err := kmr.KeyMaterial(ctx)
 	if err != nil {
-		t.Fatal(err)
+		return fmt.Errorf("KeyMaterial(ctx) failed: %w", err)
 	}
+	return compareKeyMaterial(gotKM, wantKM)
+}
 
-	// The store returns a wrappedProvider, which holds a reference to the
-	// actual provider, which in our case in the fakeProvider.
-	wp := prov.(*wrappedProvider)
-	fp := wp.Provider.(*fakeProvider)
-	return prov, fp
+func compareKeyMaterial(got, want *KeyMaterial) error {
+	// TODO(easwars): Remove all references to reflect.DeepEqual and use
+	// cmp.Equal instead. Currently, the later panics because x509.Certificate
+	// type defines an Equal method, but does not check for nil. This has been
+	// fixed in
+	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
+	// but this is only available starting go1.14. So, once we remove support
+	// for go1.13, we can make the switch.
+	if !reflect.DeepEqual(got, want) {
+		return fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", got, want)
+	}
+	return nil
 }
 
-// TestStoreWithSingleProvider creates a single provider through the store and
-// calls methods on it.
-func (s) TestStoreWithSingleProvider(t *testing.T) {
-	prov, fp := makeProvider(t, fakeProvider1Name, fakeConfig)
+// TestStoreSingleProvider creates a single provider through the store and calls
+// methods on them.
+func (s) TestStoreSingleProvider(t *testing.T) {
+	// Create a Provider through the store.
+	kmOpts := Options{CertName: "default"}
+	prov, err := GetProvider(fakeProvider1Name, fakeConfig, kmOpts)
+	if err != nil {
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, kmOpts, err)
+	}
+	defer prov.Close()
 
-	// Push key materials into the provider.
-	wantKM, err := loadKeyMaterials()
+	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
+	// the fake provider from that channel.
+	p, err := fpb1.providerChan.Receive()
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
-	fp.kmCh <- wantKM
+	fakeProv := p.(*fakeProvider)
 
-	// Get key materials from the provider and compare it to the ones we pushed
-	// above.
+	// Attempt to read from key material from the Provider returned by the
+	// store. This will fail because we have not pushed any key material into
+	// our fake provider.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	gotKM, err := prov.KeyMaterial(ctx)
-	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+	if err := readAndVerifyKeyMaterial(ctx, prov, nil); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal(err)
 	}
-	// TODO(easwars): Remove all references to reflect.DeepEqual and use
-	// cmp.Equal instead. Currently, the later panics because x509.Certificate
-	// type defines an Equal method, but does not check for nil. This has been
-	// fixed in
-	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
-	// but this is only available starting go1.14. So, once we remove support
-	// for go1.13, we can make the switch.
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+
+	// Load key material from testdata directory, push it into out fakeProvider
+	// and attempt to read from the Provider returned by the store.
+	testKM1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProv.newKeyMaterial(testKM1, nil)
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, prov, testKM1); err != nil {
+		t.Fatal(err)
 	}
 
-	// Close the provider and retry the KeyMaterial() call, and expect it to
-	// fail with a known error.
-	prov.Close()
-	if _, err := prov.KeyMaterial(ctx); err != errProviderClosed {
-		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	// Push new key material and read from the Provider. This should returned
+	// updated key material.
+	testKM2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProv.newKeyMaterial(testKM2, nil)
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, prov, testKM2); err != nil {
+		t.Fatal(err)
 	}
 }
 
-// TestStoreWithSingleProviderWithSharing creates multiple instances of the same
-// type of provider through the store (and expects the store's sharing mechanism
-// to kick in) and calls methods on it.
-func (s) TestStoreWithSingleProviderWithSharing(t *testing.T) {
-	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig)
-	prov2, _ := makeProvider(t, fakeProvider1Name, fakeConfig)
+// TestStoreSingleProviderSameConfigDifferentOpts creates multiple providers of
+// same type, for same configs but different keyMaterial options through the
+// store (and expects the store's sharing mechanism to kick in) and calls
+// methods on them.
+func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
+	// Create three readers on the same fake provider. Two of these readers use
+	// certName `foo`, while the third one uses certName `bar`.
+	optsFoo := Options{CertName: "foo"}
+	optsBar := Options{CertName: "bar"}
+	provFoo1, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo)
+	if err != nil {
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err)
+	}
+	defer provFoo1.Close()
+	provFoo2, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo)
+	if err != nil {
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err)
+	}
+	defer provFoo2.Close()
+	// Our fakeProviderBuilder pushes newly created providers on a channel.
+	// Grab the fake provider for optsFoo.
+	p, err := fpb1.providerChan.Receive()
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProvFoo := p.(*fakeProvider)
 
-	// Push key materials into the fake provider1.
-	wantKM, err := loadKeyMaterials()
+	provBar1, err := GetProvider(fakeProvider1Name, fakeConfig, optsBar)
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsBar, err)
 	}
-	fp1.kmCh <- wantKM
+	defer provBar1.Close()
+	// Grab the fake provider for optsBar.
+	p, err = fpb1.providerChan.Receive()
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProvBar := p.(*fakeProvider)
 
-	// Get key materials from the fake provider2 and compare it to the ones we
-	// pushed above.
+	// Push key material for optsFoo, and make sure the foo providers return
+	// appropriate key material and the bar provider times out.
+	fooKM := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProvFoo.newKeyMaterial(fooKM, nil)
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	gotKM, err := prov2.KeyMaterial(ctx)
-	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+	if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, provFoo2, fooKM); err != nil {
+		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	if err := readAndVerifyKeyMaterial(ctx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal(err)
 	}
 
-	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
-	// it to succeed.
-	prov1.Close()
-	if _, err := prov2.KeyMaterial(ctx); err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+	// Push key material for optsBar, and make sure the bar provider returns
+	// appropriate key material.
+	barKM := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProvBar.newKeyMaterial(barKM, nil)
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, provBar1, barKM); err != nil {
+		t.Fatal(err)
 	}
 
-	prov2.Close()
-	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
-		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	// Make sure the above push of new key material does not affect foo readers.
+	if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil {
+		t.Fatal(err)
 	}
 }
 
-// TestStoreWithSingleProviderWithoutSharing creates multiple instances of the
+// TestStoreSingleProviderDifferentConfigs creates multiple instances of the
 // same type of provider through the store with different configs. The store
 // would end up creating different provider instances for these and no sharing
 // would take place.
-func (s) TestStoreWithSingleProviderWithoutSharing(t *testing.T) {
-	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig+"1111")
-	prov2, fp2 := makeProvider(t, fakeProvider1Name, fakeConfig+"2222")
-
-	// Push the same key materials into the two providers.
-	wantKM, err := loadKeyMaterials()
+func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
+	// Create two providers of the same type, but with different configs.
+	opts := Options{CertName: "foo"}
+	cfg1 := fakeConfig + "1111"
+	cfg2 := fakeConfig + "2222"
+	prov1, err := GetProvider(fakeProvider1Name, cfg1, opts)
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg1, opts, err)
 	}
-	fp1.kmCh <- wantKM
-	fp2.kmCh <- wantKM
-
-	// Get key materials from the fake provider1 and compare it to the ones we
-	// pushed above.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	gotKM, err := prov1.KeyMaterial(ctx)
+	defer prov1.Close()
+	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
+	// the fake provider from that channel.
+	p1, err := fpb1.providerChan.Receive()
 	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
-	}
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
+	fakeProv1 := p1.(*fakeProvider)
 
-	// Get key materials from the fake provider2 and compare it to the ones we
-	// pushed above.
-	gotKM, err = prov2.KeyMaterial(ctx)
+	prov2, err := GetProvider(fakeProvider1Name, cfg2, opts)
 	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg2, opts, err)
 	}
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	defer prov2.Close()
+	// Grab the second provider from the channel.
+	p2, err := fpb1.providerChan.Receive()
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
+	fakeProv2 := p2.(*fakeProvider)
 
-	// Update the key materials used by provider1, and make sure provider2 is
-	// not affected.
-	newKM, err := loadKeyMaterials()
-	if err != nil {
+	// Push the same key material into both fake providers and verify that the
+	// providers returned by the store return the appropriate key material.
+	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProv1.newKeyMaterial(km1, nil)
+	fakeProv2.newKeyMaterial(km1, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km1); err != nil {
 		t.Fatal(err)
 	}
-	newKM.Roots = nil
-	fp1.kmCh <- newKM
 
-	gotKM, err = prov2.KeyMaterial(ctx)
-	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+	// Push new key material into only one of the fake providers and and verify
+	// that the providers returned by the store return the appropriate key
+	// material.
+	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProv2.newKeyMaterial(km2, nil)
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
+		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
 	}
 
-	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
-	// it to succeed.
+	// Close one of the providers and verify that the other one is not affected.
 	prov1.Close()
-	if _, err := prov2.KeyMaterial(ctx); err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
-	}
-
-	prov2.Close()
-	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
-		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
 	}
 }
 
-// TestStoreWithMultipleProviders creates multiple providers of different types
-// and make sure closing of one does not affect the other.
-func (s) TestStoreWithMultipleProviders(t *testing.T) {
-	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig)
-	prov2, fp2 := makeProvider(t, fakeProvider2Name, fakeConfig)
-
-	// Push key materials into the fake providers.
-	wantKM, err := loadKeyMaterials()
+// TestStoreMultipleProviders creates providers of different types and makes
+// sure closing of one does not affect the other.
+func (s) TestStoreMultipleProviders(t *testing.T) {
+	opts := Options{CertName: "foo"}
+	prov1, err := GetProvider(fakeProvider1Name, fakeConfig, opts)
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err)
 	}
-	fp1.kmCh <- wantKM
-	fp2.kmCh <- wantKM
-
-	// Get key materials from the fake provider1 and compare it to the ones we
-	// pushed above.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	gotKM, err := prov1.KeyMaterial(ctx)
+	defer prov1.Close()
+	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
+	// the fake provider from that channel.
+	p1, err := fpb1.providerChan.Receive()
 	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
-	}
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
+	fakeProv1 := p1.(*fakeProvider)
 
-	// Get key materials from the fake provider2 and compare it to the ones we
-	// pushed above.
-	gotKM, err = prov2.KeyMaterial(ctx)
+	prov2, err := GetProvider(fakeProvider2Name, fakeConfig, opts)
 	if err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err)
 	}
-	if !reflect.DeepEqual(gotKM, wantKM) {
-		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	defer prov2.Close()
+	// Grab the second provider from the channel.
+	p2, err := fpb2.providerChan.Receive()
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider2Name)
 	}
+	fakeProv2 := p2.(*fakeProvider)
 
-	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
-	// it to succeed.
-	prov1.Close()
-	if _, err := prov2.KeyMaterial(ctx); err != nil {
-		t.Fatalf("provider.KeyMaterial() = %v", err)
+	// Push the key material into both providers and verify that the
+	// readers return the appropriate key material.
+	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProv1.newKeyMaterial(km1, nil)
+	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProv2.newKeyMaterial(km2, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
 	}
 
-	prov2.Close()
-	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
-		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	// Close one of the providers and verify that the other one is not affected.
+	prov1.Close()
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
 	}
 }

From b9bc8e75a675b85e57b14635f0d1aca6ac01ba82 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Fri, 21 Aug 2020 14:22:17 -0700
Subject: [PATCH 170/481] End stream flag bugfix (#3803)

---
 internal/transport/http2_server.go |   4 ++
 test/end2end_test.go               | 110 +++++++++++++++++++++++++++--
 test/servertester.go               |  19 ++++-
 3 files changed, 126 insertions(+), 7 deletions(-)

diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index 04cbedf7945f..3be22fee426c 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -611,6 +611,10 @@ func (t *http2Server) handleData(f *http2.DataFrame) {
 	if !ok {
 		return
 	}
+	if s.getState() == streamReadDone {
+		t.closeStream(s, true, http2.ErrCodeStreamClosed, false)
+		return
+	}
 	if size > 0 {
 		if err := s.fc.onData(size); err != nil {
 			t.closeStream(s, true, http2.ErrCodeFlowControl, false)
diff --git a/test/end2end_test.go b/test/end2end_test.go
index d1a2cdf68612..0842dccaad01 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -4535,7 +4535,7 @@ func testClientRequestBodyErrorUnexpectedEOF(t *testing.T, e env) {
 	te.startServer(ts)
 	defer te.tearDown()
 	te.withServerTester(func(st *serverTester) {
-		st.writeHeadersGRPC(1, "/grpc.testing.TestService/UnaryCall")
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/UnaryCall", false)
 		// Say we have 5 bytes coming, but set END_STREAM flag:
 		st.writeData(1, true, []byte{0, 0, 0, 0, 5})
 		st.wantAnyFrame() // wait for server to crash (it used to crash)
@@ -4559,7 +4559,7 @@ func testClientRequestBodyErrorCloseAfterLength(t *testing.T, e env) {
 	te.startServer(ts)
 	defer te.tearDown()
 	te.withServerTester(func(st *serverTester) {
-		st.writeHeadersGRPC(1, "/grpc.testing.TestService/UnaryCall")
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/UnaryCall", false)
 		// say we're sending 5 bytes, but then close the connection instead.
 		st.writeData(1, false, []byte{0, 0, 0, 0, 5})
 		st.cc.Close()
@@ -4582,7 +4582,7 @@ func testClientRequestBodyErrorCancel(t *testing.T, e env) {
 	te.startServer(ts)
 	defer te.tearDown()
 	te.withServerTester(func(st *serverTester) {
-		st.writeHeadersGRPC(1, "/grpc.testing.TestService/UnaryCall")
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/UnaryCall", false)
 		// Say we have 5 bytes coming, but cancel it instead.
 		st.writeRSTStream(1, http2.ErrCodeCancel)
 		st.writeData(1, false, []byte{0, 0, 0, 0, 5})
@@ -4595,7 +4595,7 @@ func testClientRequestBodyErrorCancel(t *testing.T, e env) {
 		}
 
 		// And now send an uncanceled (but still invalid), just to get a response.
-		st.writeHeadersGRPC(3, "/grpc.testing.TestService/UnaryCall")
+		st.writeHeadersGRPC(3, "/grpc.testing.TestService/UnaryCall", false)
 		st.writeData(3, true, []byte{0, 0, 0, 0, 0})
 		<-gotCall
 		st.wantAnyFrame()
@@ -4619,7 +4619,7 @@ func testClientRequestBodyErrorCancelStreamingInput(t *testing.T, e env) {
 	te.startServer(ts)
 	defer te.tearDown()
 	te.withServerTester(func(st *serverTester) {
-		st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall")
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall", false)
 		// Say we have 5 bytes coming, but cancel it instead.
 		st.writeData(1, false, []byte{0, 0, 0, 0, 5})
 		st.writeRSTStream(1, http2.ErrCodeCancel)
@@ -4636,6 +4636,106 @@ func testClientRequestBodyErrorCancelStreamingInput(t *testing.T, e env) {
 	})
 }
 
+func (s) TestClientInitialHeaderEndStream(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.httpHandler {
+			continue
+		}
+		testClientInitialHeaderEndStream(t, e)
+	}
+}
+
+func testClientInitialHeaderEndStream(t *testing.T, e env) {
+	// To ensure RST_STREAM is sent for illegal data write and not normal stream
+	// close.
+	frameCheckingDone := make(chan struct{})
+	// To ensure goroutine for test does not end before RPC handler performs error
+	// checking.
+	handlerDone := make(chan struct{})
+	te := newTest(t, e)
+	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+		defer close(handlerDone)
+		// Block on serverTester receiving RST_STREAM. This ensures server has closed
+		// stream before stream.Recv().
+		<-frameCheckingDone
+		data, err := stream.Recv()
+		if err == nil {
+			t.Errorf("unexpected data received in func server method: '%v'", data)
+		} else if status.Code(err) != codes.Canceled {
+			t.Errorf("expected canceled error, instead received '%v'", err)
+		}
+		return nil
+	}}
+	te.startServer(ts)
+	defer te.tearDown()
+	te.withServerTester(func(st *serverTester) {
+		// Send a headers with END_STREAM flag, but then write data.
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall", true)
+		st.writeData(1, false, []byte{0, 0, 0, 0, 0})
+		st.wantAnyFrame()
+		st.wantAnyFrame()
+		st.wantRSTStream(http2.ErrCodeStreamClosed)
+		close(frameCheckingDone)
+		<-handlerDone
+	})
+}
+
+func (s) TestClientSendDataAfterCloseSend(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.httpHandler {
+			continue
+		}
+		testClientSendDataAfterCloseSend(t, e)
+	}
+}
+
+func testClientSendDataAfterCloseSend(t *testing.T, e env) {
+	// To ensure RST_STREAM is sent for illegal data write prior to execution of RPC
+	// handler.
+	frameCheckingDone := make(chan struct{})
+	// To ensure goroutine for test does not end before RPC handler performs error
+	// checking.
+	handlerDone := make(chan struct{})
+	te := newTest(t, e)
+	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+		defer close(handlerDone)
+		// Block on serverTester receiving RST_STREAM. This ensures server has closed
+		// stream before stream.Recv().
+		<-frameCheckingDone
+		for {
+			_, err := stream.Recv()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				if status.Code(err) != codes.Canceled {
+					t.Errorf("expected canceled error, instead received '%v'", err)
+				}
+				break
+			}
+		}
+		if err := stream.SendMsg(nil); err == nil {
+			t.Error("expected error sending message on stream after stream closed due to illegal data")
+		} else if status.Code(err) != codes.Internal {
+			t.Errorf("expected internal error, instead received '%v'", err)
+		}
+		return nil
+	}}
+	te.startServer(ts)
+	defer te.tearDown()
+	te.withServerTester(func(st *serverTester) {
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall", false)
+		// Send data with END_STREAM flag, but then write more data.
+		st.writeData(1, true, []byte{0, 0, 0, 0, 0})
+		st.writeData(1, false, []byte{0, 0, 0, 0, 0})
+		st.wantAnyFrame()
+		st.wantAnyFrame()
+		st.wantRSTStream(http2.ErrCodeStreamClosed)
+		close(frameCheckingDone)
+		<-handlerDone
+	})
+}
+
 func (s) TestClientResourceExhaustedCancelFullDuplex(t *testing.T) {
 	for _, e := range listTestEnv() {
 		if e.httpHandler {
diff --git a/test/servertester.go b/test/servertester.go
index ff4fa0b3c6bc..9758e8eb6cf8 100644
--- a/test/servertester.go
+++ b/test/servertester.go
@@ -138,6 +138,21 @@ func (st *serverTester) writeSettingsAck() {
 	}
 }
 
+func (st *serverTester) wantRSTStream(errCode http2.ErrCode) *http2.RSTStreamFrame {
+	f, err := st.readFrame()
+	if err != nil {
+		st.t.Fatalf("Error while expecting an RST frame: %v", err)
+	}
+	sf, ok := f.(*http2.RSTStreamFrame)
+	if !ok {
+		st.t.Fatalf("got a %T; want *http2.RSTStreamFrame", f)
+	}
+	if sf.ErrCode != errCode {
+		st.t.Fatalf("expected RST error code '%v', got '%v'", errCode.String(), sf.ErrCode.String())
+	}
+	return sf
+}
+
 func (st *serverTester) wantSettings() *http2.SettingsFrame {
 	f, err := st.readFrame()
 	if err != nil {
@@ -227,7 +242,7 @@ func (st *serverTester) encodeHeader(headers ...string) []byte {
 	return st.headerBuf.Bytes()
 }
 
-func (st *serverTester) writeHeadersGRPC(streamID uint32, path string) {
+func (st *serverTester) writeHeadersGRPC(streamID uint32, path string, endStream bool) {
 	st.writeHeaders(http2.HeadersFrameParam{
 		StreamID: streamID,
 		BlockFragment: st.encodeHeader(
@@ -236,7 +251,7 @@ func (st *serverTester) writeHeadersGRPC(streamID uint32, path string) {
 			"content-type", "application/grpc",
 			"te", "trailers",
 		),
-		EndStream:  false,
+		EndStream:  endStream,
 		EndHeaders: true,
 	})
 }

From 0e72e09474d6f277fe497cded849ae1c609c27c6 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Fri, 21 Aug 2020 18:04:04 -0700
Subject: [PATCH 171/481] server: prevent hang in Go HTTP transport in some
 error cases (#3833)

---
 server.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/server.go b/server.go
index 85beec413974..ff71435fad72 100644
--- a/server.go
+++ b/server.go
@@ -1156,10 +1156,8 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 	}
 	d, err := recvAndDecompress(&parser{r: stream}, stream, dc, s.opts.maxReceiveMessageSize, payInfo, decomp)
 	if err != nil {
-		if st, ok := status.FromError(err); ok {
-			if e := t.WriteStatus(stream, st); e != nil {
-				channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status %v", e)
-			}
+		if e := t.WriteStatus(stream, status.Convert(err)); e != nil {
+			channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status %v", e)
 		}
 		return err
 	}

From 410880dd7d91952bbc9dfb756f3e03a90c87f8ed Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 24 Aug 2020 11:09:31 -0700
Subject: [PATCH 172/481] eds: fix priority timeout failure when EDS removes
 all priorities (#3830)

Without this fix, when the EDS response removes all priorities, after the timeout, the priority check panics because priority is unset.
---
 .../balancer/edsbalancer/eds_impl_priority.go | 15 ++++-
 .../edsbalancer/eds_impl_priority_test.go     | 65 +++++++++++++++++++
 2 files changed, 78 insertions(+), 2 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
index f21d64f60a71..53ac6ef5e873 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority.go
@@ -48,6 +48,12 @@ func (edsImpl *edsBalancerImpl) handlePriorityChange() {
 	// Everything was removed by EDS.
 	if !edsImpl.priorityLowest.isSet() {
 		edsImpl.priorityInUse = newPriorityTypeUnset()
+		// Stop the init timer. This can happen if the only priority is removed
+		// shortly after it's added.
+		if timer := edsImpl.priorityInitTimer; timer != nil {
+			timer.Stop()
+			edsImpl.priorityInitTimer = nil
+		}
 		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(errAllPrioritiesRemoved)})
 		return
 	}
@@ -116,7 +122,7 @@ func (edsImpl *edsBalancerImpl) startPriority(priority priorityType) {
 	edsImpl.priorityInitTimer = time.AfterFunc(defaultPriorityInitTimeout, func() {
 		edsImpl.priorityMu.Lock()
 		defer edsImpl.priorityMu.Unlock()
-		if !edsImpl.priorityInUse.equal(priority) {
+		if !edsImpl.priorityInUse.isSet() || !edsImpl.priorityInUse.equal(priority) {
 			return
 		}
 		edsImpl.priorityInitTimer = nil
@@ -309,14 +315,18 @@ func (p priorityType) isSet() bool {
 }
 
 func (p priorityType) equal(p2 priorityType) bool {
+	if !p.isSet() && !p2.isSet() {
+		return true
+	}
 	if !p.isSet() || !p2.isSet() {
-		panic("priority unset")
+		return false
 	}
 	return p == p2
 }
 
 func (p priorityType) higherThan(p2 priorityType) bool {
 	if !p.isSet() || !p2.isSet() {
+		// TODO(menghanl): return an appropriate value instead of panic.
 		panic("priority unset")
 	}
 	return p.p < p2.p
@@ -324,6 +334,7 @@ func (p priorityType) higherThan(p2 priorityType) bool {
 
 func (p priorityType) lowerThan(p2 priorityType) bool {
 	if !p.isSet() || !p2.isSet() {
+		// TODO(menghanl): return an appropriate value instead of panic.
 		panic("priority unset")
 	}
 	return p.p > p2.p
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index a82e64431420..255665e3d4c1 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -655,6 +655,46 @@ func (s) TestPriorityType(t *testing.T) {
 	}
 }
 
+func (s) TestPriorityTypeEqual(t *testing.T) {
+	tests := []struct {
+		name   string
+		p1, p2 priorityType
+		want   bool
+	}{
+		{
+			name: "equal",
+			p1:   newPriorityType(12),
+			p2:   newPriorityType(12),
+			want: true,
+		},
+		{
+			name: "not equal",
+			p1:   newPriorityType(12),
+			p2:   newPriorityType(34),
+			want: false,
+		},
+		{
+			name: "one not set",
+			p1:   newPriorityType(1),
+			p2:   newPriorityTypeUnset(),
+			want: false,
+		},
+		{
+			name: "both not set",
+			p1:   newPriorityTypeUnset(),
+			p2:   newPriorityTypeUnset(),
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.p1.equal(tt.p2); got != tt.want {
+				t.Errorf("equal() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 // Test the case where the high priority contains no backends. The low priority
 // will be used.
 func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
@@ -774,3 +814,28 @@ func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
+
+// Test the case where the first and only priority is removed.
+func (s) TestEDSPriority_FirstPriorityUnavailable(t *testing.T) {
+	const testPriorityInitTimeout = time.Second
+	defer func(t time.Duration) {
+		defaultPriorityInitTimeout = t
+	}(defaultPriorityInitTimeout)
+	defaultPriorityInitTimeout = testPriorityInitTimeout
+
+	cc := testutils.NewTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+
+	// One localities, with priorities [0], each with one backend.
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
+
+	// Remove the only localities.
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
+
+	// Wait after double the init timer timeout, to ensure it doesn't fail.
+	time.Sleep(testPriorityInitTimeout * 2)
+}

From 44d73dff99bfd2aff2cb587bb942d6f8bf8148f7 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 25 Aug 2020 09:28:01 -0700
Subject: [PATCH 173/481] cmd/protoc-gen-go-grpc: rework service registration
 (#3828)

---
 Documentation/server-reflection-tutorial.md   |   2 +-
 .../grpclb/grpc_lb_v1/load_balancer.pb.go     | 118 +++++
 .../grpc_lb_v1/load_balancer_grpc.pb.go       | 147 ++----
 balancer/grpclb/grpclb_test.go                |   8 +-
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  86 ++++
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       | 108 ++--
 balancer/roundrobin/roundrobin_test.go        |  13 +-
 benchmark/benchmark.go                        |  13 +-
 benchmark/grpc_testing/services_grpc.pb.go    | 423 ++++++++++------
 benchmark/worker/main.go                      |   7 +-
 channelz/grpc_channelz_v1/channelz.pb.go      | 316 ++++++++++++
 channelz/grpc_channelz_v1/channelz_grpc.pb.go | 368 +++++++-------
 cmd/protoc-gen-go-grpc/README.md              | 142 +++++-
 cmd/protoc-gen-go-grpc/grpc.go                | 354 ++++++++-----
 cmd/protoc-gen-go-grpc/main.go                |   4 +-
 .../internal/proto/grpc_gcp/handshaker.pb.go  | 128 +++++
 .../proto/grpc_gcp/handshaker_grpc.pb.go      | 147 ++----
 .../meshca/internal/v1/meshca.pb.go           |  88 ++++
 .../meshca/internal/v1/meshca_grpc.pb.go      | 110 ++---
 .../features/authentication/server/main.go    |   8 +-
 examples/features/cancellation/server/main.go |   8 +-
 examples/features/compression/server/main.go  |   8 +-
 examples/features/deadline/server/main.go     |   8 +-
 examples/features/debugging/server/main.go    |  24 +-
 .../features/encryption/ALTS/server/main.go   |  10 +-
 .../features/encryption/TLS/server/main.go    |   8 +-
 examples/features/errors/server/main.go       |   4 +-
 examples/features/health/server/main.go       |  10 +-
 examples/features/interceptor/server/main.go  |  15 +-
 examples/features/keepalive/server/main.go    |   9 +-
 .../features/load_balancing/server/main.go    |   4 +-
 examples/features/metadata/server/main.go     |  19 +-
 examples/features/multiplex/server/main.go    |  22 +-
 .../features/name_resolving/server/main.go    |  11 +-
 examples/features/profiling/README.md         |   2 +-
 examples/features/profiling/server/main.go    |   8 +-
 examples/features/proto/echo/echo_grpc.pb.go  | 207 +++++---
 examples/features/reflection/server/main.go   |  20 +-
 examples/features/retry/server/main.go        |   3 +-
 examples/features/wait_for_ready/main.go      |   9 +-
 examples/features/xds/server/main.go          |   5 +-
 examples/go.mod                               |   7 +-
 examples/go.sum                               | 310 +++++++++++-
 examples/gotutorial.md                        |   8 +-
 examples/helloworld/greeter_server/main.go    |  11 +-
 .../helloworld/helloworld_grpc.pb.go          |  89 ++--
 .../routeguide/route_guide_grpc.pb.go         | 223 ++++++---
 examples/route_guide/server/server.go         |   3 +-
 health/grpc_health_v1/health.pb.go            | 182 +++++++
 health/grpc_health_v1/health_grpc.pb.go       | 201 +++-----
 internal/binarylog/binarylog_end2end_test.go  |  15 +-
 interop/alts/server/server.go                 |   2 +-
 interop/grpc_testing/test_grpc.pb.go          | 467 ++++++++++++------
 interop/server/server.go                      |   2 +-
 interop/test_utils.go                         |   8 +-
 interop/xds/client/client.go                  |   8 +-
 interop/xds/server/server.go                  |  10 +-
 profiling/proto/service.pb.go                 | 126 +++++
 profiling/proto/service_grpc.pb.go            | 153 +++---
 .../grpc_reflection_v1alpha/reflection.pb.go  | 120 +++++
 .../reflection_grpc.pb.go                     | 147 ++----
 reflection/grpc_testing/test_grpc.pb.go       | 127 +++--
 reflection/serverreflection_test.go           |   8 +-
 regenerate.sh                                 |  20 +-
 server.go                                     |  13 +-
 stats/grpc_testing/test_grpc.pb.go            | 218 +++++---
 stats/stats_test.go                           |  20 +-
 stress/client/main.go                         |   5 +-
 stress/grpc_testing/metrics_grpc.pb.go        | 147 +++---
 test/balancer_test.go                         |   8 +-
 test/end2end_test.go                          |  48 +-
 test/goaway_test.go                           |   2 +-
 test/gracefulstop_test.go                     |   2 +-
 test/grpc_testing/test_grpc.pb.go             | 293 +++++++----
 test/healthcheck_test.go                      |   2 +-
 test/local_creds_test.go                      |   4 +-
 76 files changed, 3994 insertions(+), 2019 deletions(-)

diff --git a/Documentation/server-reflection-tutorial.md b/Documentation/server-reflection-tutorial.md
index 78117ff5745a..212d4b3771a6 100644
--- a/Documentation/server-reflection-tutorial.md
+++ b/Documentation/server-reflection-tutorial.md
@@ -28,7 +28,7 @@ make the following changes:
 @@ -61,6 +62,8 @@ func main() {
         }
         s := grpc.NewServer()
-        pb.RegisterGreeterServer(s, &server{})
+        pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
 +       // Register reflection service on gRPC server.
 +       reflection.Register(s)
         if err := s.Serve(lis); err != nil {
diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
index 7381dfc1ae4f..b59191ac5825 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
@@ -4,10 +4,14 @@
 package grpc_lb_v1
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -638,3 +642,117 @@ var fileDescriptor_7cd3f6d792743fdf = []byte{
 	0x6d, 0xe1, 0xbe, 0xfb, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x47, 0x55, 0xac, 0xab, 0x06, 0x00,
 	0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// LoadBalancerClient is the client API for LoadBalancer service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type LoadBalancerClient interface {
+	// Bidirectional rpc to get a list of servers.
+	BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error)
+}
+
+type loadBalancerClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
+	return &loadBalancerClient{cc}
+}
+
+func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &loadBalancerBalanceLoadClient{stream}
+	return x, nil
+}
+
+type LoadBalancer_BalanceLoadClient interface {
+	Send(*LoadBalanceRequest) error
+	Recv() (*LoadBalanceResponse, error)
+	grpc.ClientStream
+}
+
+type loadBalancerBalanceLoadClient struct {
+	grpc.ClientStream
+}
+
+func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
+	m := new(LoadBalanceResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// LoadBalancerServer is the server API for LoadBalancer service.
+type LoadBalancerServer interface {
+	// Bidirectional rpc to get a list of servers.
+	BalanceLoad(LoadBalancer_BalanceLoadServer) error
+}
+
+// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations.
+type UnimplementedLoadBalancerServer struct {
+}
+
+func (*UnimplementedLoadBalancerServer) BalanceLoad(srv LoadBalancer_BalanceLoadServer) error {
+	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
+}
+
+func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
+	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
+}
+
+func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream})
+}
+
+type LoadBalancer_BalanceLoadServer interface {
+	Send(*LoadBalanceResponse) error
+	Recv() (*LoadBalanceRequest, error)
+	grpc.ServerStream
+}
+
+type loadBalancerBalanceLoadServer struct {
+	grpc.ServerStream
+}
+
+func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
+	m := new(LoadBalanceRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.lb.v1.LoadBalancer",
+	HandlerType: (*LoadBalancerServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "BalanceLoad",
+			Handler:       _LoadBalancer_BalanceLoad_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "grpc/lb/v1/load_balancer.proto",
+}
diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index 5a3a2ec57641..0079d0fad8df 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -3,7 +3,6 @@
 package grpc_lb_v1
 
 import (
-	context "context"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
@@ -11,112 +10,64 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// LoadBalancerClient is the client API for LoadBalancer service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type LoadBalancerClient interface {
+// LoadBalancerService is the service API for LoadBalancer service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterLoadBalancerService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type LoadBalancerService struct {
 	// Bidirectional rpc to get a list of servers.
-	BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error)
+	BalanceLoad func(LoadBalancer_BalanceLoadServer) error
 }
 
-type loadBalancerClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
-	return &loadBalancerClient{cc}
-}
-
-func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
-	if err != nil {
-		return nil, err
+func (s *LoadBalancerService) balanceLoad(_ interface{}, stream grpc.ServerStream) error {
+	if s.BalanceLoad == nil {
+		return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
+	}
+	return s.BalanceLoad(&loadBalancerBalanceLoadServer{stream})
+}
+
+// RegisterLoadBalancerService registers a service implementation with a gRPC server.
+func RegisterLoadBalancerService(s grpc.ServiceRegistrar, srv *LoadBalancerService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.lb.v1.LoadBalancer",
+		Methods:     []grpc.MethodDesc{},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "BalanceLoad",
+				Handler:       srv.balanceLoad,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+		},
+		Metadata: "grpc/lb/v1/load_balancer.proto",
 	}
-	x := &loadBalancerBalanceLoadClient{stream}
-	return x, nil
-}
-
-type LoadBalancer_BalanceLoadClient interface {
-	Send(*LoadBalanceRequest) error
-	Recv() (*LoadBalanceResponse, error)
-	grpc.ClientStream
-}
-
-type loadBalancerBalanceLoadClient struct {
-	grpc.ClientStream
-}
-
-func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
 
-func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
-	m := new(LoadBalanceResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
+	s.RegisterService(&sd, nil)
+}
+
+// NewLoadBalancerService creates a new LoadBalancerService containing the
+// implemented methods of the LoadBalancer service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewLoadBalancerService(s interface{}) *LoadBalancerService {
+	ns := &LoadBalancerService{}
+	if h, ok := s.(interface {
+		BalanceLoad(LoadBalancer_BalanceLoadServer) error
+	}); ok {
+		ns.BalanceLoad = h.BalanceLoad
 	}
-	return m, nil
+	return ns
 }
 
-// LoadBalancerServer is the server API for LoadBalancer service.
-// All implementations should embed UnimplementedLoadBalancerServer
-// for forward compatibility
-type LoadBalancerServer interface {
+// UnstableLoadBalancerService is the service API for LoadBalancer service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableLoadBalancerService interface {
 	// Bidirectional rpc to get a list of servers.
 	BalanceLoad(LoadBalancer_BalanceLoadServer) error
 }
-
-// UnimplementedLoadBalancerServer should be embedded to have forward compatible implementations.
-type UnimplementedLoadBalancerServer struct {
-}
-
-func (*UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
-	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
-}
-
-func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
-	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
-}
-
-func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream})
-}
-
-type LoadBalancer_BalanceLoadServer interface {
-	Send(*LoadBalanceResponse) error
-	Recv() (*LoadBalanceRequest, error)
-	grpc.ServerStream
-}
-
-type loadBalancerBalanceLoadServer struct {
-	grpc.ServerStream
-}
-
-func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
-	m := new(LoadBalanceRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.lb.v1.LoadBalancer",
-	HandlerType: (*LoadBalancerServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "BalanceLoad",
-			Handler:       _LoadBalancer_BalanceLoad_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "grpc/lb/v1/load_balancer.proto",
-}
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index 48082e2069fa..5a2297d43ec8 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -276,8 +276,6 @@ func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServe
 }
 
 type testServer struct {
-	testpb.UnimplementedTestServiceServer
-
 	addr     string
 	fallback bool
 }
@@ -306,7 +304,11 @@ func startBackends(sn string, fallback bool, lis ...net.Listener) (servers []*gr
 			sn: sn,
 		}
 		s := grpc.NewServer(grpc.Creds(creds))
-		testpb.RegisterTestServiceServer(s, &testServer{addr: l.Addr().String(), fallback: fallback})
+		ts := &testServer{addr: l.Addr().String(), fallback: fallback}
+		testpb.RegisterTestServiceService(s, &testpb.TestServiceService{
+			EmptyCall:      ts.EmptyCall,
+			FullDuplexCall: ts.FullDuplexCall,
+		})
 		servers = append(servers, s)
 		go func(s *grpc.Server, l net.Listener) {
 			s.Serve(l)
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index 7ec14279109f..a3a3f8118ce9 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -4,8 +4,12 @@
 package grpc_lookup_v1
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -176,3 +180,85 @@ var fileDescriptor_3bab962d3362f3ca = []byte{
 	0x29, 0xa7, 0xcf, 0x5e, 0xb3, 0x1d, 0x27, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0xca, 0x8d, 0x5c,
 	0xc7, 0x39, 0x02, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// RouteLookupServiceClient is the client API for RouteLookupService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type RouteLookupServiceClient interface {
+	// Lookup returns a target for a single key.
+	RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error)
+}
+
+type routeLookupServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
+	return &routeLookupServiceClient{cc}
+}
+
+func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
+	out := new(RouteLookupResponse)
+	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// RouteLookupServiceServer is the server API for RouteLookupService service.
+type RouteLookupServiceServer interface {
+	// Lookup returns a target for a single key.
+	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
+}
+
+// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedRouteLookupServiceServer struct {
+}
+
+func (*UnimplementedRouteLookupServiceServer) RouteLookup(ctx context.Context, req *RouteLookupRequest) (*RouteLookupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
+}
+
+func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
+	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
+}
+
+func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RouteLookupRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RouteLookupServiceServer).RouteLookup(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.lookup.v1.RouteLookupService",
+	HandlerType: (*RouteLookupServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "RouteLookup",
+			Handler:    _RouteLookupService_RouteLookup_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/lookup/v1/rls.proto",
+}
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index 79fa758f7e97..fe1027c71ec7 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -11,80 +11,76 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// RouteLookupServiceClient is the client API for RouteLookupService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type RouteLookupServiceClient interface {
+// RouteLookupServiceService is the service API for RouteLookupService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterRouteLookupServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type RouteLookupServiceService struct {
 	// Lookup returns a target for a single key.
-	RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error)
+	RouteLookup func(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
 }
 
-type routeLookupServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
-	return &routeLookupServiceClient{cc}
-}
-
-func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
-	out := new(RouteLookupResponse)
-	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
-	if err != nil {
-		return nil, err
+func (s *RouteLookupServiceService) routeLookup(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.RouteLookup == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
 	}
-	return out, nil
-}
-
-// RouteLookupServiceServer is the server API for RouteLookupService service.
-// All implementations should embed UnimplementedRouteLookupServiceServer
-// for forward compatibility
-type RouteLookupServiceServer interface {
-	// Lookup returns a target for a single key.
-	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
-}
-
-// UnimplementedRouteLookupServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedRouteLookupServiceServer struct {
-}
-
-func (*UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
-}
-
-func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
-	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
-}
-
-func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(RouteLookupRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(RouteLookupServiceServer).RouteLookup(ctx, in)
+		return s.RouteLookup(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest))
+		return s.RouteLookup(ctx, req.(*RouteLookupRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.lookup.v1.RouteLookupService",
-	HandlerType: (*RouteLookupServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "RouteLookup",
-			Handler:    _RouteLookupService_RouteLookup_Handler,
+// RegisterRouteLookupServiceService registers a service implementation with a gRPC server.
+func RegisterRouteLookupServiceService(s grpc.ServiceRegistrar, srv *RouteLookupServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.lookup.v1.RouteLookupService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "RouteLookup",
+				Handler:    srv.routeLookup,
+			},
 		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/lookup/v1/rls.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "grpc/lookup/v1/rls.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewRouteLookupServiceService creates a new RouteLookupServiceService containing the
+// implemented methods of the RouteLookupService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewRouteLookupServiceService(s interface{}) *RouteLookupServiceService {
+	ns := &RouteLookupServiceService{}
+	if h, ok := s.(interface {
+		RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
+	}); ok {
+		ns.RouteLookup = h.RouteLookup
+	}
+	return ns
+}
+
+// UnstableRouteLookupServiceService is the service API for RouteLookupService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableRouteLookupServiceService interface {
+	// Lookup returns a target for a single key.
+	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
 }
diff --git a/balancer/roundrobin/roundrobin_test.go b/balancer/roundrobin/roundrobin_test.go
index 5a8ba481c9f0..0c54465a6af2 100644
--- a/balancer/roundrobin/roundrobin_test.go
+++ b/balancer/roundrobin/roundrobin_test.go
@@ -47,15 +47,11 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-type testServer struct {
-	testpb.UnimplementedTestServiceServer
-}
-
-func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+func emptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 	return &testpb.Empty{}, nil
 }
 
-func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+func fullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
 	return nil
 }
 
@@ -85,7 +81,10 @@ func startTestServers(count int) (_ *test, err error) {
 		}
 
 		s := grpc.NewServer()
-		testpb.RegisterTestServiceServer(s, &testServer{})
+		testpb.RegisterTestServiceService(s, &testpb.TestServiceService{
+			EmptyCall:      emptyCall,
+			FullDuplexCall: fullDuplexCall,
+		})
 		t.servers = append(t.servers, s)
 		t.addresses = append(t.addresses, lis.Addr().String())
 
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index 56841e172366..d2783a87e00a 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -61,9 +61,9 @@ func NewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 	return p
 }
 
-type testServer struct {
-	testpb.UnimplementedBenchmarkServiceServer
-}
+type testServer struct{}
+
+var _ testpb.UnstableBenchmarkServiceService = (*testServer)(nil)
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	return &testpb.SimpleResponse{
@@ -144,10 +144,11 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U
 // byteBufServer is a gRPC server that sends and receives byte buffer.
 // The purpose is to benchmark the gRPC performance without protobuf serialization/deserialization overhead.
 type byteBufServer struct {
-	testpb.UnimplementedBenchmarkServiceServer
 	respSize int32
 }
 
+var _ testpb.UnstableBenchmarkServiceService = (*byteBufServer)(nil)
+
 // UnaryCall is an empty function and is not used for benchmark.
 // If bytebuf UnaryCall benchmark is needed later, the function body needs to be updated.
 func (s *byteBufServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -211,13 +212,13 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() {
 	s := grpc.NewServer(opts...)
 	switch info.Type {
 	case "protobuf":
-		testpb.RegisterBenchmarkServiceServer(s, &testServer{})
+		testpb.RegisterBenchmarkServiceService(s, testpb.NewBenchmarkServiceService(&testServer{}))
 	case "bytebuf":
 		respSize, ok := info.Metadata.(int32)
 		if !ok {
 			logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type)
 		}
-		testpb.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize})
+		testpb.RegisterBenchmarkServiceService(s, testpb.NewBenchmarkServiceService(&byteBufServer{respSize: respSize}))
 	default:
 		logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type)
 	}
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index 385870ae5959..60be6cd689a2 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // BenchmarkServiceClient is the client API for BenchmarkService service.
 //
@@ -36,6 +36,10 @@ func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClie
 	return &benchmarkServiceClient{cc}
 }
 
+var benchmarkServiceUnaryCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "UnaryCall",
+}
+
 func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...)
@@ -45,8 +49,14 @@ func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleReques
 	return out, nil
 }
 
+var benchmarkServiceStreamingCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "StreamingCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
+	stream, err := c.cc.NewStream(ctx, benchmarkServiceStreamingCallStreamDesc, "/grpc.testing.BenchmarkService/StreamingCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -76,8 +86,14 @@ func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
 	return m, nil
 }
 
+var benchmarkServiceUnconstrainedStreamingCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "UnconstrainedStreamingCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
+	stream, err := c.cc.NewStream(ctx, benchmarkServiceUnconstrainedStreamingCallStreamDesc, "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -107,59 +123,53 @@ func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleRespon
 	return m, nil
 }
 
-// BenchmarkServiceServer is the server API for BenchmarkService service.
-// All implementations should embed UnimplementedBenchmarkServiceServer
-// for forward compatibility
-type BenchmarkServiceServer interface {
+// BenchmarkServiceService is the service API for BenchmarkService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterBenchmarkServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type BenchmarkServiceService struct {
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	StreamingCall(BenchmarkService_StreamingCallServer) error
+	StreamingCall func(BenchmarkService_StreamingCallServer) error
 	// Unconstrainted streaming.
 	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
-}
-
-// UnimplementedBenchmarkServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedBenchmarkServiceServer struct {
-}
-
-func (*UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedBenchmarkServiceServer) StreamingCall(BenchmarkService_StreamingCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
-}
-func (*UnimplementedBenchmarkServiceServer) UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
-}
-
-func RegisterBenchmarkServiceServer(s *grpc.Server, srv BenchmarkServiceServer) {
-	s.RegisterService(&_BenchmarkService_serviceDesc, srv)
+	UnconstrainedStreamingCall func(BenchmarkService_UnconstrainedStreamingCallServer) error
 }
 
-func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *BenchmarkServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.UnaryCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(BenchmarkServiceServer).UnaryCall(ctx, in)
+		return s.UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.BenchmarkService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+		return s.UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream})
+func (s *BenchmarkServiceService) streamingCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.StreamingCall == nil {
+		return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
+	}
+	return s.StreamingCall(&benchmarkServiceStreamingCallServer{stream})
+}
+func (s *BenchmarkServiceService) unconstrainedStreamingCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.UnconstrainedStreamingCall == nil {
+		return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
+	}
+	return s.UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
 }
 
 type BenchmarkService_StreamingCallServer interface {
@@ -184,10 +194,6 @@ func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) {
 	return m, nil
 }
 
-func _BenchmarkService_UnconstrainedStreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(BenchmarkServiceServer).UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
-}
-
 type BenchmarkService_UnconstrainedStreamingCallServer interface {
 	Send(*SimpleResponse) error
 	Recv() (*SimpleRequest, error)
@@ -210,30 +216,76 @@ func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleReques
 	return m, nil
 }
 
-var _BenchmarkService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.BenchmarkService",
-	HandlerType: (*BenchmarkServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryCall",
-			Handler:    _BenchmarkService_UnaryCall_Handler,
+// RegisterBenchmarkServiceService registers a service implementation with a gRPC server.
+func RegisterBenchmarkServiceService(s grpc.ServiceRegistrar, srv *BenchmarkServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.BenchmarkService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "UnaryCall",
+				Handler:    srv.unaryCall,
+			},
 		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingCall",
-			Handler:       _BenchmarkService_StreamingCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "StreamingCall",
+				Handler:       srv.streamingCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "UnconstrainedStreamingCall",
+				Handler:       srv.unconstrainedStreamingCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
 		},
-		{
-			StreamName:    "UnconstrainedStreamingCall",
-			Handler:       _BenchmarkService_UnconstrainedStreamingCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "benchmark/grpc_testing/services.proto",
+		Metadata: "benchmark/grpc_testing/services.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewBenchmarkServiceService creates a new BenchmarkServiceService containing the
+// implemented methods of the BenchmarkService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewBenchmarkServiceService(s interface{}) *BenchmarkServiceService {
+	ns := &BenchmarkServiceService{}
+	if h, ok := s.(interface {
+		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	}); ok {
+		ns.UnaryCall = h.UnaryCall
+	}
+	if h, ok := s.(interface {
+		StreamingCall(BenchmarkService_StreamingCallServer) error
+	}); ok {
+		ns.StreamingCall = h.StreamingCall
+	}
+	if h, ok := s.(interface {
+		UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
+	}); ok {
+		ns.UnconstrainedStreamingCall = h.UnconstrainedStreamingCall
+	}
+	return ns
+}
+
+// UnstableBenchmarkServiceService is the service API for BenchmarkService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableBenchmarkServiceService interface {
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	StreamingCall(BenchmarkService_StreamingCallServer) error
+	// Unconstrainted streaming.
+	// Both server and client keep sending & receiving simultaneously.
+	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
 }
 
 // WorkerServiceClient is the client API for WorkerService service.
@@ -268,8 +320,14 @@ func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient {
 	return &workerServiceClient{cc}
 }
 
+var workerServiceRunServerStreamDesc = &grpc.StreamDesc{
+	StreamName:    "RunServer",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[0], "/grpc.testing.WorkerService/RunServer", opts...)
+	stream, err := c.cc.NewStream(ctx, workerServiceRunServerStreamDesc, "/grpc.testing.WorkerService/RunServer", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -299,8 +357,14 @@ func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) {
 	return m, nil
 }
 
+var workerServiceRunClientStreamDesc = &grpc.StreamDesc{
+	StreamName:    "RunClient",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[1], "/grpc.testing.WorkerService/RunClient", opts...)
+	stream, err := c.cc.NewStream(ctx, workerServiceRunClientStreamDesc, "/grpc.testing.WorkerService/RunClient", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -330,6 +394,10 @@ func (x *workerServiceRunClientClient) Recv() (*ClientStatus, error) {
 	return m, nil
 }
 
+var workerServiceCoreCountStreamDesc = &grpc.StreamDesc{
+	StreamName: "CoreCount",
+}
+
 func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) {
 	out := new(CoreResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/CoreCount", in, out, opts...)
@@ -339,6 +407,10 @@ func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, op
 	return out, nil
 }
 
+var workerServiceQuitWorkerStreamDesc = &grpc.StreamDesc{
+	StreamName: "QuitWorker",
+}
+
 func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) {
 	out := new(Void)
 	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/QuitWorker", in, out, opts...)
@@ -348,53 +420,82 @@ func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...
 	return out, nil
 }
 
-// WorkerServiceServer is the server API for WorkerService service.
-// All implementations should embed UnimplementedWorkerServiceServer
-// for forward compatibility
-type WorkerServiceServer interface {
+// WorkerServiceService is the service API for WorkerService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterWorkerServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type WorkerServiceService struct {
 	// Start server with specified workload.
 	// First request sent specifies the ServerConfig followed by ServerStatus
 	// response. After that, a "Mark" can be sent anytime to request the latest
 	// stats. Closing the stream will initiate shutdown of the test server
 	// and once the shutdown has finished, the OK status is sent to terminate
 	// this RPC.
-	RunServer(WorkerService_RunServerServer) error
+	RunServer func(WorkerService_RunServerServer) error
 	// Start client with specified workload.
 	// First request sent specifies the ClientConfig followed by ClientStatus
 	// response. After that, a "Mark" can be sent anytime to request the latest
 	// stats. Closing the stream will initiate shutdown of the test client
 	// and once the shutdown has finished, the OK status is sent to terminate
 	// this RPC.
-	RunClient(WorkerService_RunClientServer) error
+	RunClient func(WorkerService_RunClientServer) error
 	// Just return the core count - unary call
-	CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
+	CoreCount func(context.Context, *CoreRequest) (*CoreResponse, error)
 	// Quit this worker
-	QuitWorker(context.Context, *Void) (*Void, error)
-}
-
-// UnimplementedWorkerServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedWorkerServiceServer struct {
+	QuitWorker func(context.Context, *Void) (*Void, error)
 }
 
-func (*UnimplementedWorkerServiceServer) RunServer(WorkerService_RunServerServer) error {
-	return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
-}
-func (*UnimplementedWorkerServiceServer) RunClient(WorkerService_RunClientServer) error {
-	return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
-}
-func (*UnimplementedWorkerServiceServer) CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
+func (s *WorkerServiceService) runServer(_ interface{}, stream grpc.ServerStream) error {
+	if s.RunServer == nil {
+		return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
+	}
+	return s.RunServer(&workerServiceRunServerServer{stream})
 }
-func (*UnimplementedWorkerServiceServer) QuitWorker(context.Context, *Void) (*Void, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
+func (s *WorkerServiceService) runClient(_ interface{}, stream grpc.ServerStream) error {
+	if s.RunClient == nil {
+		return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
+	}
+	return s.RunClient(&workerServiceRunClientServer{stream})
 }
-
-func RegisterWorkerServiceServer(s *grpc.Server, srv WorkerServiceServer) {
-	s.RegisterService(&_WorkerService_serviceDesc, srv)
+func (s *WorkerServiceService) coreCount(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.CoreCount == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
+	}
+	in := new(CoreRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return s.CoreCount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     s,
+		FullMethod: "/grpc.testing.WorkerService/CoreCount",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return s.CoreCount(ctx, req.(*CoreRequest))
+	}
+	return interceptor(ctx, in, info, handler)
 }
-
-func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(WorkerServiceServer).RunServer(&workerServiceRunServerServer{stream})
+func (s *WorkerServiceService) quitWorker(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.QuitWorker == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
+	}
+	in := new(Void)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return s.QuitWorker(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     s,
+		FullMethod: "/grpc.testing.WorkerService/QuitWorker",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return s.QuitWorker(ctx, req.(*Void))
+	}
+	return interceptor(ctx, in, info, handler)
 }
 
 type WorkerService_RunServerServer interface {
@@ -419,10 +520,6 @@ func (x *workerServiceRunServerServer) Recv() (*ServerArgs, error) {
 	return m, nil
 }
 
-func _WorkerService_RunClient_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(WorkerServiceServer).RunClient(&workerServiceRunClientServer{stream})
-}
-
 type WorkerService_RunClientServer interface {
 	Send(*ClientStatus) error
 	Recv() (*ClientArgs, error)
@@ -445,68 +542,92 @@ func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) {
 	return m, nil
 }
 
-func _WorkerService_CoreCount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(CoreRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(WorkerServiceServer).CoreCount(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.WorkerService/CoreCount",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(WorkerServiceServer).CoreCount(ctx, req.(*CoreRequest))
+// RegisterWorkerServiceService registers a service implementation with a gRPC server.
+func RegisterWorkerServiceService(s grpc.ServiceRegistrar, srv *WorkerServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.WorkerService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "CoreCount",
+				Handler:    srv.coreCount,
+			},
+			{
+				MethodName: "QuitWorker",
+				Handler:    srv.quitWorker,
+			},
+		},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "RunServer",
+				Handler:       srv.runServer,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "RunClient",
+				Handler:       srv.runClient,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+		},
+		Metadata: "benchmark/grpc_testing/services.proto",
 	}
-	return interceptor(ctx, in, info, handler)
-}
 
-func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Void)
-	if err := dec(in); err != nil {
-		return nil, err
+	s.RegisterService(&sd, nil)
+}
+
+// NewWorkerServiceService creates a new WorkerServiceService containing the
+// implemented methods of the WorkerService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewWorkerServiceService(s interface{}) *WorkerServiceService {
+	ns := &WorkerServiceService{}
+	if h, ok := s.(interface {
+		RunServer(WorkerService_RunServerServer) error
+	}); ok {
+		ns.RunServer = h.RunServer
 	}
-	if interceptor == nil {
-		return srv.(WorkerServiceServer).QuitWorker(ctx, in)
+	if h, ok := s.(interface {
+		RunClient(WorkerService_RunClientServer) error
+	}); ok {
+		ns.RunClient = h.RunClient
 	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.WorkerService/QuitWorker",
+	if h, ok := s.(interface {
+		CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
+	}); ok {
+		ns.CoreCount = h.CoreCount
 	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(WorkerServiceServer).QuitWorker(ctx, req.(*Void))
+	if h, ok := s.(interface {
+		QuitWorker(context.Context, *Void) (*Void, error)
+	}); ok {
+		ns.QuitWorker = h.QuitWorker
 	}
-	return interceptor(ctx, in, info, handler)
+	return ns
 }
 
-var _WorkerService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.WorkerService",
-	HandlerType: (*WorkerServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "CoreCount",
-			Handler:    _WorkerService_CoreCount_Handler,
-		},
-		{
-			MethodName: "QuitWorker",
-			Handler:    _WorkerService_QuitWorker_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "RunServer",
-			Handler:       _WorkerService_RunServer_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "RunClient",
-			Handler:       _WorkerService_RunClient_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "benchmark/grpc_testing/services.proto",
+// UnstableWorkerServiceService is the service API for WorkerService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableWorkerServiceService interface {
+	// Start server with specified workload.
+	// First request sent specifies the ServerConfig followed by ServerStatus
+	// response. After that, a "Mark" can be sent anytime to request the latest
+	// stats. Closing the stream will initiate shutdown of the test server
+	// and once the shutdown has finished, the OK status is sent to terminate
+	// this RPC.
+	RunServer(WorkerService_RunServerServer) error
+	// Start client with specified workload.
+	// First request sent specifies the ClientConfig followed by ClientStatus
+	// response. After that, a "Mark" can be sent anytime to request the latest
+	// stats. Closing the stream will initiate shutdown of the test client
+	// and once the shutdown has finished, the OK status is sent to terminate
+	// this RPC.
+	RunClient(WorkerService_RunClientServer) error
+	// Just return the core count - unary call
+	CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
+	// Quit this worker
+	QuitWorker(context.Context, *Void) (*Void, error)
 }
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 634f09e658c5..891cd01fc92b 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -75,11 +75,12 @@ func (byteBufCodec) String() string {
 // workerServer implements WorkerService rpc handlers.
 // It can create benchmarkServer or benchmarkClient on demand.
 type workerServer struct {
-	testpb.UnimplementedWorkerServiceServer
 	stop       chan<- bool
 	serverPort int
 }
 
+var _ testpb.UnstableWorkerServiceService = (*workerServer)(nil)
+
 func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) error {
 	var bs *benchmarkServer
 	defer func() {
@@ -209,10 +210,10 @@ func main() {
 
 	s := grpc.NewServer()
 	stop := make(chan bool)
-	testpb.RegisterWorkerServiceServer(s, &workerServer{
+	testpb.RegisterWorkerServiceService(s, testpb.NewWorkerServiceService(&workerServer{
 		stop:       stop,
 		serverPort: *serverPort,
-	})
+	}))
 
 	go func() {
 		<-stop
diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go
index 9c364e1ebd9c..34bfa5ab8f50 100644
--- a/channelz/grpc_channelz_v1/channelz.pb.go
+++ b/channelz/grpc_channelz_v1/channelz.pb.go
@@ -4,12 +4,16 @@
 package grpc_channelz_v1
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	any "github.com/golang/protobuf/ptypes/any"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	wrappers "github.com/golang/protobuf/ptypes/wrappers"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -2921,3 +2925,315 @@ var fileDescriptor_6ee37dfd35a8ab00 = []byte{
 	0xd3, 0x77, 0xc6, 0x68, 0xe7, 0x3c, 0xcf, 0x4f, 0xf3, 0x5f, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff,
 	0x54, 0xae, 0x0b, 0x93, 0xdf, 0x1f, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ChannelzClient is the client API for Channelz service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type ChannelzClient interface {
+	// Gets all root channels (i.e. channels the application has directly
+	// created). This does not include subchannels nor non-top level channels.
+	GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error)
+	// Gets all servers that exist in the process.
+	GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error)
+	// Returns a single Server, or else a NOT_FOUND code.
+	GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error)
+	// Gets all server sockets that exist in the process.
+	GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error)
+	// Returns a single Channel, or else a NOT_FOUND code.
+	GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error)
+	// Returns a single Subchannel, or else a NOT_FOUND code.
+	GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error)
+	// Returns a single Socket or else a NOT_FOUND code.
+	GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error)
+}
+
+type channelzClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
+	return &channelzClient{cc}
+}
+
+func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
+	out := new(GetTopChannelsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
+	out := new(GetServersResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
+	out := new(GetServerResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
+	out := new(GetServerSocketsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
+	out := new(GetChannelResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
+	out := new(GetSubchannelResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
+	out := new(GetSocketResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ChannelzServer is the server API for Channelz service.
+type ChannelzServer interface {
+	// Gets all root channels (i.e. channels the application has directly
+	// created). This does not include subchannels nor non-top level channels.
+	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
+	// Gets all servers that exist in the process.
+	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
+	// Returns a single Server, or else a NOT_FOUND code.
+	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
+	// Gets all server sockets that exist in the process.
+	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
+	// Returns a single Channel, or else a NOT_FOUND code.
+	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
+	// Returns a single Subchannel, or else a NOT_FOUND code.
+	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
+	// Returns a single Socket or else a NOT_FOUND code.
+	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
+}
+
+// UnimplementedChannelzServer can be embedded to have forward compatible implementations.
+type UnimplementedChannelzServer struct {
+}
+
+func (*UnimplementedChannelzServer) GetTopChannels(ctx context.Context, req *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
+}
+func (*UnimplementedChannelzServer) GetServers(ctx context.Context, req *GetServersRequest) (*GetServersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
+}
+func (*UnimplementedChannelzServer) GetServer(ctx context.Context, req *GetServerRequest) (*GetServerResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
+}
+func (*UnimplementedChannelzServer) GetServerSockets(ctx context.Context, req *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
+}
+func (*UnimplementedChannelzServer) GetChannel(ctx context.Context, req *GetChannelRequest) (*GetChannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
+}
+func (*UnimplementedChannelzServer) GetSubchannel(ctx context.Context, req *GetSubchannelRequest) (*GetSubchannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
+}
+func (*UnimplementedChannelzServer) GetSocket(ctx context.Context, req *GetSocketRequest) (*GetSocketResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
+}
+
+func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
+	s.RegisterService(&_Channelz_serviceDesc, srv)
+}
+
+func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopChannelsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetTopChannels(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetServers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetServers",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServerRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetServer(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetServer",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServerSocketsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetServerSockets(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetChannelRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetChannel(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetChannel",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetSubchannelRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetSubchannel(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetSocketRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ChannelzServer).GetSocket(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.channelz.v1.Channelz/GetSocket",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Channelz_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.channelz.v1.Channelz",
+	HandlerType: (*ChannelzServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetTopChannels",
+			Handler:    _Channelz_GetTopChannels_Handler,
+		},
+		{
+			MethodName: "GetServers",
+			Handler:    _Channelz_GetServers_Handler,
+		},
+		{
+			MethodName: "GetServer",
+			Handler:    _Channelz_GetServer_Handler,
+		},
+		{
+			MethodName: "GetServerSockets",
+			Handler:    _Channelz_GetServerSockets_Handler,
+		},
+		{
+			MethodName: "GetChannel",
+			Handler:    _Channelz_GetChannel_Handler,
+		},
+		{
+			MethodName: "GetSubchannel",
+			Handler:    _Channelz_GetSubchannel_Handler,
+		},
+		{
+			MethodName: "GetSocket",
+			Handler:    _Channelz_GetSocket_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/channelz/v1/channelz.proto",
+}
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index 20e1fe709796..e4815814853e 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -11,310 +11,276 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// ChannelzClient is the client API for Channelz service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type ChannelzClient interface {
+// ChannelzService is the service API for Channelz service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterChannelzService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type ChannelzService struct {
 	// Gets all root channels (i.e. channels the application has directly
 	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error)
+	GetTopChannels func(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
 	// Gets all servers that exist in the process.
-	GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error)
+	GetServers func(context.Context, *GetServersRequest) (*GetServersResponse, error)
 	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error)
+	GetServer func(context.Context, *GetServerRequest) (*GetServerResponse, error)
 	// Gets all server sockets that exist in the process.
-	GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error)
+	GetServerSockets func(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
 	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error)
+	GetChannel func(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
 	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error)
+	GetSubchannel func(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
 	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error)
+	GetSocket func(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
 }
 
-type channelzClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
-	return &channelzClient{cc}
-}
-
-func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
-	out := new(GetTopChannelsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
-	out := new(GetServersResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
-	out := new(GetServerResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
-	out := new(GetServerSocketsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
-	out := new(GetChannelResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
-	out := new(GetSubchannelResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
-	if err != nil {
-		return nil, err
+func (s *ChannelzService) getTopChannels(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetTopChannels == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
 	}
-	return out, nil
-}
-
-func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
-	out := new(GetSocketResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ChannelzServer is the server API for Channelz service.
-// All implementations should embed UnimplementedChannelzServer
-// for forward compatibility
-type ChannelzServer interface {
-	// Gets all root channels (i.e. channels the application has directly
-	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
-	// Gets all servers that exist in the process.
-	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
-	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
-	// Gets all server sockets that exist in the process.
-	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
-	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
-	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
-	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
-}
-
-// UnimplementedChannelzServer should be embedded to have forward compatible implementations.
-type UnimplementedChannelzServer struct {
-}
-
-func (*UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
-}
-func (*UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
-}
-func (*UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
-}
-func (*UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
-}
-func (*UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
-}
-func (*UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
-}
-func (*UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
-}
-
-func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
-	s.RegisterService(&_Channelz_serviceDesc, srv)
-}
-
-func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetTopChannelsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetTopChannels(ctx, in)
+		return s.GetTopChannels(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest))
+		return s.GetTopChannels(ctx, req.(*GetTopChannelsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ChannelzService) getServers(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetServers == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
+	}
 	in := new(GetServersRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetServers(ctx, in)
+		return s.GetServers(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetServers",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest))
+		return s.GetServers(ctx, req.(*GetServersRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ChannelzService) getServer(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetServer == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
+	}
 	in := new(GetServerRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetServer(ctx, in)
+		return s.GetServer(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetServer",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest))
+		return s.GetServer(ctx, req.(*GetServerRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ChannelzService) getServerSockets(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetServerSockets == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
+	}
 	in := new(GetServerSocketsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetServerSockets(ctx, in)
+		return s.GetServerSockets(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest))
+		return s.GetServerSockets(ctx, req.(*GetServerSocketsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ChannelzService) getChannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetChannel == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
+	}
 	in := new(GetChannelRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetChannel(ctx, in)
+		return s.GetChannel(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetChannel",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest))
+		return s.GetChannel(ctx, req.(*GetChannelRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ChannelzService) getSubchannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetSubchannel == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
+	}
 	in := new(GetSubchannelRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetSubchannel(ctx, in)
+		return s.GetSubchannel(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest))
+		return s.GetSubchannel(ctx, req.(*GetSubchannelRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ChannelzService) getSocket(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetSocket == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
+	}
 	in := new(GetSocketRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ChannelzServer).GetSocket(ctx, in)
+		return s.GetSocket(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetSocket",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest))
+		return s.GetSocket(ctx, req.(*GetSocketRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _Channelz_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.channelz.v1.Channelz",
-	HandlerType: (*ChannelzServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetTopChannels",
-			Handler:    _Channelz_GetTopChannels_Handler,
-		},
-		{
-			MethodName: "GetServers",
-			Handler:    _Channelz_GetServers_Handler,
-		},
-		{
-			MethodName: "GetServer",
-			Handler:    _Channelz_GetServer_Handler,
-		},
-		{
-			MethodName: "GetServerSockets",
-			Handler:    _Channelz_GetServerSockets_Handler,
+// RegisterChannelzService registers a service implementation with a gRPC server.
+func RegisterChannelzService(s grpc.ServiceRegistrar, srv *ChannelzService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.channelz.v1.Channelz",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "GetTopChannels",
+				Handler:    srv.getTopChannels,
+			},
+			{
+				MethodName: "GetServers",
+				Handler:    srv.getServers,
+			},
+			{
+				MethodName: "GetServer",
+				Handler:    srv.getServer,
+			},
+			{
+				MethodName: "GetServerSockets",
+				Handler:    srv.getServerSockets,
+			},
+			{
+				MethodName: "GetChannel",
+				Handler:    srv.getChannel,
+			},
+			{
+				MethodName: "GetSubchannel",
+				Handler:    srv.getSubchannel,
+			},
+			{
+				MethodName: "GetSocket",
+				Handler:    srv.getSocket,
+			},
 		},
-		{
-			MethodName: "GetChannel",
-			Handler:    _Channelz_GetChannel_Handler,
-		},
-		{
-			MethodName: "GetSubchannel",
-			Handler:    _Channelz_GetSubchannel_Handler,
-		},
-		{
-			MethodName: "GetSocket",
-			Handler:    _Channelz_GetSocket_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/channelz/v1/channelz.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "grpc/channelz/v1/channelz.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewChannelzService creates a new ChannelzService containing the
+// implemented methods of the Channelz service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewChannelzService(s interface{}) *ChannelzService {
+	ns := &ChannelzService{}
+	if h, ok := s.(interface {
+		GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
+	}); ok {
+		ns.GetTopChannels = h.GetTopChannels
+	}
+	if h, ok := s.(interface {
+		GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
+	}); ok {
+		ns.GetServers = h.GetServers
+	}
+	if h, ok := s.(interface {
+		GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
+	}); ok {
+		ns.GetServer = h.GetServer
+	}
+	if h, ok := s.(interface {
+		GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
+	}); ok {
+		ns.GetServerSockets = h.GetServerSockets
+	}
+	if h, ok := s.(interface {
+		GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
+	}); ok {
+		ns.GetChannel = h.GetChannel
+	}
+	if h, ok := s.(interface {
+		GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
+	}); ok {
+		ns.GetSubchannel = h.GetSubchannel
+	}
+	if h, ok := s.(interface {
+		GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
+	}); ok {
+		ns.GetSocket = h.GetSocket
+	}
+	return ns
+}
+
+// UnstableChannelzService is the service API for Channelz service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableChannelzService interface {
+	// Gets all root channels (i.e. channels the application has directly
+	// created). This does not include subchannels nor non-top level channels.
+	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
+	// Gets all servers that exist in the process.
+	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
+	// Returns a single Server, or else a NOT_FOUND code.
+	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
+	// Gets all server sockets that exist in the process.
+	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
+	// Returns a single Channel, or else a NOT_FOUND code.
+	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
+	// Returns a single Subchannel, or else a NOT_FOUND code.
+	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
+	// Returns a single Socket or else a NOT_FOUND code.
+	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
 }
diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md
index f76e419a26c7..6d047f1e514c 100644
--- a/cmd/protoc-gen-go-grpc/README.md
+++ b/cmd/protoc-gen-go-grpc/README.md
@@ -4,18 +4,140 @@ This tool generates Go language bindings of `service`s in protobuf definition
 files for gRPC.  For usage information, please see our [quick start
 guide](https://grpc.io/docs/languages/go/quickstart/).
 
-## Future-proofing services
+## Service implementation and registration
 
-By default, to register services using the methods generated by this tool, the
-service implementations must embed the corresponding
-`Unimplemented<ServiceName>Server` for future compatibility.  This is a behavior
-change from the grpc code generator previously included with `protoc-gen-go`.
-To restore this behavior, set the option `requireUnimplementedServers=false`.
-E.g.:
+**NOTE:** service registration has changed from the previous version of the
+code generator.  Please read this section carefully if you are migrating.
 
+To register your service handlers with a gRPC server, first implement the
+methods as either functions or methods on a struct.  Examples:
+
+```go
+// As a function:
+
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+     // Echo.UnaryEcho implementation
+}
+
+// As a struct + method:
+
+type myEchoService struct {
+     // ...fields used by this service...
+}
+
+func (s *myEchoService) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+     // Echo.UnaryEcho implementation
+}
 ```
-  protoc --go-grpc_out=requireUnimplementedServers=false[,other options...]:. \
+
+Then create an instance of the generated `Service` struct type and initialize
+the handlers which have been implemented:
+
+```go
+func main() {
+     // ...
+
+     // As a function:
+     echoService := pb.EchoService{
+          UnaryEcho: unaryEcho,
+          // etc
+     }
+
+     // As a struct+method:
+     mes := &myEchoService{...}
+     echoService := pb.EchoService{
+          UnaryEcho: mes.UnaryEcho,
+          // etc
+     }
+
+     // ...
+ }
+```
+
+Finally, pass this `Service` instance to the generated `Register` function:
+
+```go
+     pb.RegisterEchoService(grpcServer, echoService)
+```
+
+### Migration from legacy version
+
+#### Updating existing code
+
+The previous version of protoc-gen-go-grpc used a different method to register
+services.  In that version, it was only possible to register a service
+implementation that was a complete implementation of the service.  It was also
+possible to embed an `Unimplemented` implementation of the service, which was
+also generated and returned an UNIMPLEMENTED status for all methods.  To make
+it easier to migrate from the previous version, two new symbols are generated:
+`New<Service>Service` and `Unstable<Service>Service`.
+
+`New<Service>Service` allows for easy wrapping of a service implementation into
+an instance of the new generated `Service` struct type.  *This has drawbacks,
+however: `New<Service>Service` accepts any parameter, and does not panic or
+error if any methods are missing, are misspelled, or have the wrong signature.*
+These methods will return an UNIMPLEMENTED status if called by a client.
+
+`Unstable<Service>Service` allows for asserting that a type properly implements
+all methods of a service.  It is generated with the name `Unstable` to denote
+that it will be extended whenever new methods are added to the service
+definition.  This kind of change to the service definition is considered
+backward-compatible in protobuf, but is not backward-compatible in Go, because
+adding methods to an interface invalidates any existing implementations.  *Use
+of this symbol could result in future compilation errors.*
+
+To convert your existing code using the previous code generator, please refer
+to the following example:
+
+```go
+type myEchoService{
+     // ...fields used by this service...
+}
+// ... method handler implementation ...
+
+
+// Optional; not recommended: to guarantee myEchoService fully implements
+// EchoService:
+var _ pb.UnstableEchoService = &myEchoService{}
+
+func main() {
+     // ...
+
+     // OLD:
+     pb.RegisterEchoServer(grpcServer, &myEchoService{})
+
+     // NEW:
+     pb.RegisterEchoService(grpcServer, pb.NewEchoService(&myEchoService{}))
+
+
+     // Optional: to gracefully detect missing methods:
+     if _, ok := &myEchoService{}.(pb.UnstableEchoService); !ok {
+        fmt.Println("myEchoService does not implement all methods of EchoService.")
+     }
+
+
+     // ...
+}
+```
+
+#### Updating generated code to support both legacy and new code
+
+`protoc-gen-go-grpc` supports a flag, `migration_mode`, which enables it to be
+run in tandem with the previous tool (`protoc-gen-go` with the grpc plugin).
+It can be used as follows:
+
+```sh
+go install github.com/golang/protobuf/protoc-gen-go
+
+# Example 1: with OPTS set to common options for protoc-gen-go and
+# protoc-gen-go-grpc
+protoc --go_out=${OPTS},plugins=grpc:. --go-grpc_out=${OPTS},migration_mode=true:. *.proto
+
+# Example 2: if no special options are needed
+protoc --go_out=plugins=grpc:. --go-grpc_out=migration_mode=true:. *.proto
 ```
 
-Note that this is not recommended, and the option is only provided to restore
-backward compatibility with previously-generated code.
+This is recommended for temporary use only to ease migration from the legacy
+version.  The `Register<Service>Server` and `<Service>Server` symbols it
+produced are not backward compatible in the presence of newly added methods to
+a service.
\ No newline at end of file
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 4f78bdd46018..47503bdc0f42 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -58,14 +58,19 @@ func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.
 
 	g.P("// This is a compile-time assertion to ensure that this generated file")
 	g.P("// is compatible with the grpc package it is being compiled against.")
-	g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion6"))
+	g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion7"))
 	g.P()
 	for _, service := range file.Services {
+		genClient(gen, file, g, service)
 		genService(gen, file, g, service)
+		genUnstableServiceInterface(gen, file, g, service)
 	}
 }
 
-func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+func genClient(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	if *migrationMode {
+		return
+	}
 	clientName := service.GoName + "Client"
 
 	g.P("// ", clientName, " is the client API for ", service.GoName, " service.")
@@ -105,121 +110,10 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 	g.P("}")
 	g.P()
 
-	var methodIndex, streamIndex int
 	// Client method implementations.
 	for _, method := range service.Methods {
-		if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() {
-			// Unary RPC method
-			genClientMethod(gen, file, g, method, methodIndex)
-			methodIndex++
-		} else {
-			// Streaming RPC method
-			genClientMethod(gen, file, g, method, streamIndex)
-			streamIndex++
-		}
-	}
-
-	mustOrShould := "must"
-	if !*requireUnimplemented {
-		mustOrShould = "should"
-	}
-
-	// Server interface.
-	serverType := service.GoName + "Server"
-	g.P("// ", serverType, " is the server API for ", service.GoName, " service.")
-	g.P("// All implementations ", mustOrShould, " embed Unimplemented", serverType)
-	g.P("// for forward compatibility")
-	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
-		g.P("//")
-		g.P(deprecationComment)
-	}
-	g.Annotate(serverType, service.Location)
-	g.P("type ", serverType, " interface {")
-	for _, method := range service.Methods {
-		g.Annotate(serverType+"."+method.GoName, method.Location)
-		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
-			g.P(deprecationComment)
-		}
-		g.P(method.Comments.Leading,
-			serverSignature(g, method))
-	}
-	if *requireUnimplemented {
-		g.P("mustEmbedUnimplemented", serverType, "()")
-	}
-	g.P("}")
-	g.P()
-
-	// Server Unimplemented struct for forward compatibility.
-	g.P("// Unimplemented", serverType, " ", mustOrShould, " be embedded to have forward compatible implementations.")
-	g.P("type Unimplemented", serverType, " struct {")
-	g.P("}")
-	g.P()
-	for _, method := range service.Methods {
-		nilArg := ""
-		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-			nilArg = "nil,"
-		}
-		g.P("func (*Unimplemented", serverType, ") ", serverSignature(g, method), "{")
-		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
-		g.P("}")
-	}
-	if *requireUnimplemented {
-		g.P("func (*Unimplemented", serverType, ") mustEmbedUnimplemented", serverType, "() {}")
-	}
-	g.P()
-
-	// Server registration.
-	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
-		g.P(deprecationComment)
-	}
-	serviceDescVar := "_" + service.GoName + "_serviceDesc"
-	g.P("func Register", service.GoName, "Server(s *", grpcPackage.Ident("Server"), ", srv ", serverType, ") {")
-	g.P("s.RegisterService(&", serviceDescVar, `, srv)`)
-	g.P("}")
-	g.P()
-
-	// Server handler implementations.
-	var handlerNames []string
-	for _, method := range service.Methods {
-		hname := genServerMethod(gen, file, g, method)
-		handlerNames = append(handlerNames, hname)
-	}
-
-	// Service descriptor.
-	g.P("var ", serviceDescVar, " = ", grpcPackage.Ident("ServiceDesc"), " {")
-	g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",")
-	g.P("HandlerType: (*", serverType, ")(nil),")
-	g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{")
-	for i, method := range service.Methods {
-		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
-			continue
-		}
-		g.P("{")
-		g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",")
-		g.P("Handler: ", handlerNames[i], ",")
-		g.P("},")
+		genClientMethod(gen, g, method)
 	}
-	g.P("},")
-	g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{")
-	for i, method := range service.Methods {
-		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-			continue
-		}
-		g.P("{")
-		g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
-		g.P("Handler: ", handlerNames[i], ",")
-		if method.Desc.IsStreamingServer() {
-			g.P("ServerStreams: true,")
-		}
-		if method.Desc.IsStreamingClient() {
-			g.P("ClientStreams: true,")
-		}
-		g.P("},")
-	}
-	g.P("},")
-	g.P("Metadata: \"", file.Desc.Path(), "\",")
-	g.P("}")
-	g.P()
 }
 
 func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
@@ -237,13 +131,25 @@ func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string
 	return s
 }
 
-func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, index int) {
+func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
 	service := method.Parent
 	sname := fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name())
 
 	if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
 		g.P(deprecationComment)
 	}
+
+	streamDescName := unexport(service.GoName) + method.GoName + "StreamDesc"
+	g.P("var ", streamDescName, " = &", grpcPackage.Ident("StreamDesc"), "{")
+	g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
+	if method.Desc.IsStreamingServer() {
+		g.P("ServerStreams: true,")
+	}
+	if method.Desc.IsStreamingClient() {
+		g.P("ClientStreams: true,")
+	}
+	g.P("}")
+
 	g.P("func (c *", unexport(service.GoName), "Client) ", clientSignature(g, method), "{")
 	if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() {
 		g.P("out := new(", method.Output.GoIdent, ")")
@@ -255,8 +161,8 @@ func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.Gene
 		return
 	}
 	streamType := unexport(service.GoName) + method.GoName + "Client"
-	serviceDescVar := "_" + service.GoName + "_serviceDesc"
-	g.P("stream, err := c.cc.NewStream(ctx, &", serviceDescVar, ".Streams[", index, `], "`, sname, `", opts...)`)
+
+	g.P(`stream, err := c.cc.NewStream(ctx, `, streamDescName, `, "`, sname, `", opts...)`)
 	g.P("if err != nil { return nil, err }")
 	g.P("x := &", streamType, "{stream}")
 	if !method.Desc.IsStreamingClient() {
@@ -316,7 +222,140 @@ func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.Gene
 	}
 }
 
-func serverSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
+func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	// Server struct.
+	serviceType := service.GoName + "Service"
+	g.P("// ", serviceType, " is the service API for ", service.GoName, " service.")
+	g.P("// Fields should be assigned to their respective handler implementations only before")
+	g.P("// Register", serviceType, " is called.  Any unassigned fields will result in the")
+	g.P("// handler for that method returning an Unimplemented error.")
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.Annotate(serviceType, service.Location)
+	g.P("type ", serviceType, " struct {")
+	for _, method := range service.Methods {
+		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
+			g.P(deprecationComment)
+		}
+		g.Annotate(serviceType+"."+method.GoName, method.Location)
+		g.P(method.Comments.Leading,
+			handlerSignature(g, method))
+	}
+	g.P("}")
+	g.P()
+
+	// Method handler implementations.
+	for _, method := range service.Methods {
+		genMethodHandler(gen, g, method)
+	}
+
+	// Stream interfaces and implementations.
+	for _, method := range service.Methods {
+		genServerStreamTypes(gen, g, method)
+	}
+
+	// Service registration.
+	genRegisterFunction(gen, file, g, service)
+
+	// Short-cut service constructor.
+	genServiceConstructor(gen, g, service)
+}
+
+func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	g.P("// Register", service.GoName, "Service registers a service implementation with a gRPC server.")
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.P("func Register", service.GoName, "Service(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv *", service.GoName, "Service) {")
+
+	// Service descriptor.
+	g.P("sd := ", grpcPackage.Ident("ServiceDesc"), " {")
+	g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",")
+	g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{")
+	for _, method := range service.Methods {
+		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+			continue
+		}
+		g.P("{")
+		g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",")
+		g.P("Handler: srv.", unexport(method.GoName), ",")
+		g.P("},")
+	}
+	g.P("},")
+	g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{")
+	for _, method := range service.Methods {
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			continue
+		}
+		g.P("{")
+		g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
+		g.P("Handler: srv.", unexport(method.GoName), ",")
+		if method.Desc.IsStreamingServer() {
+			g.P("ServerStreams: true,")
+		}
+		if method.Desc.IsStreamingClient() {
+			g.P("ClientStreams: true,")
+		}
+		g.P("},")
+	}
+	g.P("},")
+	g.P("Metadata: \"", file.Desc.Path(), "\",")
+	g.P("}")
+	g.P()
+
+	g.P("s.RegisterService(&sd, nil)")
+	g.P("}")
+	g.P()
+}
+
+func genServiceConstructor(gen *protogen.Plugin, g *protogen.GeneratedFile, service *protogen.Service) {
+	g.P("// New", service.GoName, "Service creates a new ", service.GoName, "Service containing the")
+	g.P("// implemented methods of the ", service.GoName, " service in s.  Any unimplemented")
+	g.P("// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.")
+	g.P("// This includes situations where the method handler is misspelled or has the wrong")
+	g.P("// signature.  For this reason, this function should be used with great care and")
+	g.P("// is not recommended to be used by most users.")
+	g.P("func New", service.GoName, "Service(s interface{}) *", service.GoName, "Service {")
+	g.P("ns := &", service.GoName, "Service{}")
+	for _, method := range service.Methods {
+		g.P("if h, ok := s.(interface {", methodSignature(g, method), "}); ok {")
+		g.P("ns.", method.GoName, " = h.", method.GoName)
+		g.P("}")
+	}
+	g.P("return ns")
+	g.P("}")
+	g.P()
+}
+
+func genUnstableServiceInterface(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	// Service interface.
+	serviceType := service.GoName + "Service"
+	g.P("// Unstable", serviceType, " is the service API for ", service.GoName, " service.")
+	g.P("// New methods may be added to this interface if they are added to the service")
+	g.P("// definition, which is not a backward-compatible change.  For this reason, ")
+	g.P("// use of this type is not recommended.")
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.Annotate("Unstable"+serviceType, service.Location)
+	g.P("type Unstable", serviceType, " interface {")
+	for _, method := range service.Methods {
+		g.Annotate("Unstable"+serviceType+"."+method.GoName, method.Location)
+		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
+			g.P(deprecationComment)
+		}
+		g.P(method.Comments.Leading,
+			methodSignature(g, method))
+	}
+	g.P("}")
+	g.P()
+}
+
+func methodSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
 	var reqArgs []string
 	ret := "error"
 	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
@@ -332,39 +371,88 @@ func serverSignature(g *protogen.GeneratedFile, method *protogen.Method) string
 	return method.GoName + "(" + strings.Join(reqArgs, ", ") + ") " + ret
 }
 
-func genServerMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method) string {
+func handlerSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
+	var reqArgs []string
+	ret := "error"
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context")))
+		ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)"
+	}
+	if !method.Desc.IsStreamingClient() {
+		reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent))
+	}
+	if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+		reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server")
+	}
+	return method.GoName + " func(" + strings.Join(reqArgs, ", ") + ") " + ret
+}
+
+func unaryHandlerSignature(g *protogen.GeneratedFile) string {
+	return "(_ interface{}, ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context")) +
+		", dec func(interface{}) error, interceptor " + g.QualifiedGoIdent(grpcPackage.Ident("UnaryServerInterceptor")) + ") (interface{}, error)"
+}
+
+func streamHandlerSignature(g *protogen.GeneratedFile) string {
+	return "(_ interface{}, stream " + g.QualifiedGoIdent(grpcPackage.Ident("ServerStream")) + ") error"
+}
+
+func genMethodHandler(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
 	service := method.Parent
-	hname := fmt.Sprintf("_%s_%s_Handler", service.GoName, method.GoName)
 
+	nilArg := ""
+	signature := streamHandlerSignature(g)
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		nilArg = "nil,"
+		signature = unaryHandlerSignature(g)
+	}
+	g.P("func (s *", service.GoName, "Service) ", unexport(method.GoName), signature, " {")
+
+	g.P("if s.", method.GoName, " == nil {")
+	g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
+	g.P("}")
+	genHandlerBody(gen, g, method)
+
+	g.P("}")
+}
+
+func genHandlerBody(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
+	service := method.Parent
 	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-		g.P("func ", hname, "(srv interface{}, ctx ", contextPackage.Ident("Context"), ", dec func(interface{}) error, interceptor ", grpcPackage.Ident("UnaryServerInterceptor"), ") (interface{}, error) {")
 		g.P("in := new(", method.Input.GoIdent, ")")
 		g.P("if err := dec(in); err != nil { return nil, err }")
-		g.P("if interceptor == nil { return srv.(", service.GoName, "Server).", method.GoName, "(ctx, in) }")
+		g.P("if interceptor == nil { return s.", method.GoName, "(ctx, in) }")
 		g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{")
-		g.P("Server: srv,")
+		g.P("Server: s,")
 		g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.GoName)), ",")
 		g.P("}")
 		g.P("handler := func(ctx ", contextPackage.Ident("Context"), ", req interface{}) (interface{}, error) {")
-		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")
+		g.P("return s.", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")
 		g.P("}")
 		g.P("return interceptor(ctx, in, info, handler)")
-		g.P("}")
-		g.P()
-		return hname
+		return
 	}
 	streamType := unexport(service.GoName) + method.GoName + "Server"
-	g.P("func ", hname, "(srv interface{}, stream ", grpcPackage.Ident("ServerStream"), ") error {")
 	if !method.Desc.IsStreamingClient() {
+		// Server-streaming
 		g.P("m := new(", method.Input.GoIdent, ")")
 		g.P("if err := stream.RecvMsg(m); err != nil { return err }")
-		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(m, &", streamType, "{stream})")
+		g.P("return s.", method.GoName, "(m, &", streamType, "{stream})")
 	} else {
-		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(&", streamType, "{stream})")
+		// Bidi-streaming
+		g.P("return s.", method.GoName, "(&", streamType, "{stream})")
 	}
-	g.P("}")
-	g.P()
+}
 
+func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
+	if *migrationMode {
+		return
+	}
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		// Unary method
+		return
+	}
+	service := method.Parent
+	streamType := unexport(service.GoName) + method.GoName + "Server"
 	genSend := method.Desc.IsStreamingServer()
 	genSendAndClose := !method.Desc.IsStreamingServer()
 	genRecv := method.Desc.IsStreamingClient()
@@ -410,7 +498,7 @@ func genServerMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.Gene
 		g.P()
 	}
 
-	return hname
+	return
 }
 
 const deprecationComment = "// Deprecated: Do not use."
diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index 9165761da6ff..0d7e76c64c4e 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -37,11 +37,11 @@ import (
 	"google.golang.org/protobuf/types/pluginpb"
 )
 
-var requireUnimplemented *bool
+var migrationMode *bool
 
 func main() {
 	var flags flag.FlagSet
-	requireUnimplemented = flags.Bool("requireUnimplementedServers", true, "unset to match legacy behavior")
+	migrationMode = flags.Bool("migration_mode", false, "set to generate new symbols only; requires symbols produced by legacy protoc-gen-go")
 
 	protogen.Options{
 		ParamFunc: flags.Set,
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
index 6d9c304e7981..a2060de402bc 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
@@ -4,8 +4,12 @@
 package grpc_gcp
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -975,3 +979,127 @@ var fileDescriptor_54c074f40c7c7e99 = []byte{
 	0x5f, 0xef, 0xa8, 0xf5, 0x83, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc1, 0xf9, 0x9d, 0xf2, 0xd9,
 	0x0b, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// HandshakerServiceClient is the client API for HandshakerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type HandshakerServiceClient interface {
+	// Handshaker service accepts a stream of handshaker request, returning a
+	// stream of handshaker response. Client is expected to send exactly one
+	// message with either client_start or server_start followed by one or more
+	// messages with next. Each time client sends a request, the handshaker
+	// service expects to respond. Client does not have to wait for service's
+	// response before sending next request.
+	DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error)
+}
+
+type handshakerServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient {
+	return &handshakerServiceClient{cc}
+}
+
+func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &handshakerServiceDoHandshakeClient{stream}
+	return x, nil
+}
+
+type HandshakerService_DoHandshakeClient interface {
+	Send(*HandshakerReq) error
+	Recv() (*HandshakerResp, error)
+	grpc.ClientStream
+}
+
+type handshakerServiceDoHandshakeClient struct {
+	grpc.ClientStream
+}
+
+func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
+	m := new(HandshakerResp)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// HandshakerServiceServer is the server API for HandshakerService service.
+type HandshakerServiceServer interface {
+	// Handshaker service accepts a stream of handshaker request, returning a
+	// stream of handshaker response. Client is expected to send exactly one
+	// message with either client_start or server_start followed by one or more
+	// messages with next. Each time client sends a request, the handshaker
+	// service expects to respond. Client does not have to wait for service's
+	// response before sending next request.
+	DoHandshake(HandshakerService_DoHandshakeServer) error
+}
+
+// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedHandshakerServiceServer struct {
+}
+
+func (*UnimplementedHandshakerServiceServer) DoHandshake(srv HandshakerService_DoHandshakeServer) error {
+	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
+}
+
+func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
+	s.RegisterService(&_HandshakerService_serviceDesc, srv)
+}
+
+func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream})
+}
+
+type HandshakerService_DoHandshakeServer interface {
+	Send(*HandshakerResp) error
+	Recv() (*HandshakerReq, error)
+	grpc.ServerStream
+}
+
+type handshakerServiceDoHandshakeServer struct {
+	grpc.ServerStream
+}
+
+func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
+	m := new(HandshakerReq)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _HandshakerService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.gcp.HandshakerService",
+	HandlerType: (*HandshakerServiceServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "DoHandshake",
+			Handler:       _HandshakerService_DoHandshake_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "grpc/gcp/handshaker.proto",
+}
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index 0e973b8250e3..bf7ff0a7bb76 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -3,7 +3,6 @@
 package grpc_gcp
 
 import (
-	context "context"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
@@ -11,64 +10,69 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// HandshakerServiceClient is the client API for HandshakerService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type HandshakerServiceClient interface {
+// HandshakerServiceService is the service API for HandshakerService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterHandshakerServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type HandshakerServiceService struct {
 	// Handshaker service accepts a stream of handshaker request, returning a
 	// stream of handshaker response. Client is expected to send exactly one
 	// message with either client_start or server_start followed by one or more
 	// messages with next. Each time client sends a request, the handshaker
 	// service expects to respond. Client does not have to wait for service's
 	// response before sending next request.
-	DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error)
+	DoHandshake func(HandshakerService_DoHandshakeServer) error
 }
 
-type handshakerServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient {
-	return &handshakerServiceClient{cc}
-}
-
-func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
-	if err != nil {
-		return nil, err
+func (s *HandshakerServiceService) doHandshake(_ interface{}, stream grpc.ServerStream) error {
+	if s.DoHandshake == nil {
+		return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
+	}
+	return s.DoHandshake(&handshakerServiceDoHandshakeServer{stream})
+}
+
+// RegisterHandshakerServiceService registers a service implementation with a gRPC server.
+func RegisterHandshakerServiceService(s grpc.ServiceRegistrar, srv *HandshakerServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.gcp.HandshakerService",
+		Methods:     []grpc.MethodDesc{},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "DoHandshake",
+				Handler:       srv.doHandshake,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+		},
+		Metadata: "grpc/gcp/handshaker.proto",
 	}
-	x := &handshakerServiceDoHandshakeClient{stream}
-	return x, nil
-}
-
-type HandshakerService_DoHandshakeClient interface {
-	Send(*HandshakerReq) error
-	Recv() (*HandshakerResp, error)
-	grpc.ClientStream
-}
-
-type handshakerServiceDoHandshakeClient struct {
-	grpc.ClientStream
-}
-
-func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error {
-	return x.ClientStream.SendMsg(m)
-}
 
-func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
-	m := new(HandshakerResp)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
+	s.RegisterService(&sd, nil)
+}
+
+// NewHandshakerServiceService creates a new HandshakerServiceService containing the
+// implemented methods of the HandshakerService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewHandshakerServiceService(s interface{}) *HandshakerServiceService {
+	ns := &HandshakerServiceService{}
+	if h, ok := s.(interface {
+		DoHandshake(HandshakerService_DoHandshakeServer) error
+	}); ok {
+		ns.DoHandshake = h.DoHandshake
 	}
-	return m, nil
+	return ns
 }
 
-// HandshakerServiceServer is the server API for HandshakerService service.
-// All implementations should embed UnimplementedHandshakerServiceServer
-// for forward compatibility
-type HandshakerServiceServer interface {
+// UnstableHandshakerServiceService is the service API for HandshakerService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableHandshakerServiceService interface {
 	// Handshaker service accepts a stream of handshaker request, returning a
 	// stream of handshaker response. Client is expected to send exactly one
 	// message with either client_start or server_start followed by one or more
@@ -77,56 +81,3 @@ type HandshakerServiceServer interface {
 	// response before sending next request.
 	DoHandshake(HandshakerService_DoHandshakeServer) error
 }
-
-// UnimplementedHandshakerServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedHandshakerServiceServer struct {
-}
-
-func (*UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
-	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
-}
-
-func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
-	s.RegisterService(&_HandshakerService_serviceDesc, srv)
-}
-
-func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream})
-}
-
-type HandshakerService_DoHandshakeServer interface {
-	Send(*HandshakerResp) error
-	Recv() (*HandshakerReq, error)
-	grpc.ServerStream
-}
-
-type handshakerServiceDoHandshakeServer struct {
-	grpc.ServerStream
-}
-
-func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
-	m := new(HandshakerReq)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _HandshakerService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.gcp.HandshakerService",
-	HandlerType: (*HandshakerServiceServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "DoHandshake",
-			Handler:       _HandshakerService_DoHandshake_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "grpc/gcp/handshaker.proto",
-}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
index 16122f768488..b09de93beac6 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
@@ -4,9 +4,13 @@
 package google_security_meshca_v1
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -153,3 +157,87 @@ var fileDescriptor_f72841047b94fe5e = []byte{
 	0x75, 0x36, 0xec, 0xba, 0x62, 0xee, 0x66, 0x99, 0x93, 0xe5, 0x45, 0xb7, 0xcf, 0xc3, 0x77, 0x00,
 	0x00, 0x00, 0xff, 0xff, 0xb7, 0x0d, 0xfd, 0xff, 0xf7, 0x01, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// MeshCertificateServiceClient is the client API for MeshCertificateService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type MeshCertificateServiceClient interface {
+	// Using provided CSR, returns a signed certificate that represents a GCP
+	// service account identity.
+	CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error)
+}
+
+type meshCertificateServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient {
+	return &meshCertificateServiceClient{cc}
+}
+
+func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
+	out := new(MeshCertificateResponse)
+	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// MeshCertificateServiceServer is the server API for MeshCertificateService service.
+type MeshCertificateServiceServer interface {
+	// Using provided CSR, returns a signed certificate that represents a GCP
+	// service account identity.
+	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
+}
+
+// UnimplementedMeshCertificateServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedMeshCertificateServiceServer struct {
+}
+
+func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(ctx context.Context, req *MeshCertificateRequest) (*MeshCertificateResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
+}
+
+func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) {
+	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
+}
+
+func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(MeshCertificateRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "google.security.meshca.v1.MeshCertificateService",
+	HandlerType: (*MeshCertificateServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "CreateCertificate",
+			Handler:    _MeshCertificateService_CreateCertificate_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "istio/google/security/meshca/v1/meshca.proto",
+}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index 2361ac632705..a063519dab8c 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -11,82 +11,78 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// MeshCertificateServiceClient is the client API for MeshCertificateService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type MeshCertificateServiceClient interface {
+// MeshCertificateServiceService is the service API for MeshCertificateService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterMeshCertificateServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type MeshCertificateServiceService struct {
 	// Using provided CSR, returns a signed certificate that represents a GCP
 	// service account identity.
-	CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error)
+	CreateCertificate func(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
 }
 
-type meshCertificateServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient {
-	return &meshCertificateServiceClient{cc}
-}
-
-func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
-	out := new(MeshCertificateResponse)
-	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
-	if err != nil {
-		return nil, err
+func (s *MeshCertificateServiceService) createCertificate(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.CreateCertificate == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
 	}
-	return out, nil
-}
-
-// MeshCertificateServiceServer is the server API for MeshCertificateService service.
-// All implementations should embed UnimplementedMeshCertificateServiceServer
-// for forward compatibility
-type MeshCertificateServiceServer interface {
-	// Using provided CSR, returns a signed certificate that represents a GCP
-	// service account identity.
-	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
-}
-
-// UnimplementedMeshCertificateServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedMeshCertificateServiceServer struct {
-}
-
-func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
-}
-
-func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) {
-	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
-}
-
-func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(MeshCertificateRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in)
+		return s.CreateCertificate(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest))
+		return s.CreateCertificate(ctx, req.(*MeshCertificateRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "google.security.meshca.v1.MeshCertificateService",
-	HandlerType: (*MeshCertificateServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "CreateCertificate",
-			Handler:    _MeshCertificateService_CreateCertificate_Handler,
+// RegisterMeshCertificateServiceService registers a service implementation with a gRPC server.
+func RegisterMeshCertificateServiceService(s grpc.ServiceRegistrar, srv *MeshCertificateServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "google.security.meshca.v1.MeshCertificateService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "CreateCertificate",
+				Handler:    srv.createCertificate,
+			},
 		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "istio/google/security/meshca/v1/meshca.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "istio/google/security/meshca/v1/meshca.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewMeshCertificateServiceService creates a new MeshCertificateServiceService containing the
+// implemented methods of the MeshCertificateService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewMeshCertificateServiceService(s interface{}) *MeshCertificateServiceService {
+	ns := &MeshCertificateServiceService{}
+	if h, ok := s.(interface {
+		CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
+	}); ok {
+		ns.CreateCertificate = h.CreateCertificate
+	}
+	return ns
+}
+
+// UnstableMeshCertificateServiceService is the service API for MeshCertificateService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableMeshCertificateServiceService interface {
+	// Using provided CSR, returns a signed certificate that represents a GCP
+	// service account identity.
+	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
 }
diff --git a/examples/features/authentication/server/main.go b/examples/features/authentication/server/main.go
index 5163fc31671b..505e8cb09847 100644
--- a/examples/features/authentication/server/main.go
+++ b/examples/features/authentication/server/main.go
@@ -63,7 +63,7 @@ func main() {
 		grpc.Creds(credentials.NewServerTLSFromCert(&cert)),
 	}
 	s := grpc.NewServer(opts...)
-	pb.RegisterEchoServer(s, &ecServer{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
@@ -73,11 +73,7 @@ func main() {
 	}
 }
 
-type ecServer struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
diff --git a/examples/features/cancellation/server/main.go b/examples/features/cancellation/server/main.go
index 520286bf193e..1896828f2c99 100644
--- a/examples/features/cancellation/server/main.go
+++ b/examples/features/cancellation/server/main.go
@@ -33,11 +33,7 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
+func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
 	for {
 		in, err := stream.Recv()
 		if err != nil {
@@ -61,6 +57,6 @@ func main() {
 	}
 	fmt.Printf("server listening at port %v\n", lis.Addr())
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{BidirectionalStreamingEcho: bidirectionalStreamingEcho})
 	s.Serve(lis)
 }
diff --git a/examples/features/compression/server/main.go b/examples/features/compression/server/main.go
index e495cc89106b..442d196085c9 100644
--- a/examples/features/compression/server/main.go
+++ b/examples/features/compression/server/main.go
@@ -34,11 +34,7 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage())
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
@@ -53,6 +49,6 @@ func main() {
 	fmt.Printf("server listening at %v\n", lis.Addr())
 
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 	s.Serve(lis)
 }
diff --git a/examples/features/deadline/server/main.go b/examples/features/deadline/server/main.go
index 11cd47a6b5b3..f5a033db3d9c 100644
--- a/examples/features/deadline/server/main.go
+++ b/examples/features/deadline/server/main.go
@@ -40,7 +40,6 @@ var port = flag.Int("port", 50052, "port number")
 
 // server is used to implement EchoServer.
 type server struct {
-	pb.UnimplementedEchoServer
 	client pb.EchoClient
 	cc     *grpc.ClientConn
 }
@@ -112,9 +111,12 @@ func main() {
 
 	echoServer := newEchoServer()
 	defer echoServer.Close()
-
 	grpcServer := grpc.NewServer()
-	pb.RegisterEchoServer(grpcServer, echoServer)
+
+	pb.RegisterEchoService(grpcServer, &pb.EchoService{
+		UnaryEcho:                  echoServer.UnaryEcho,
+		BidirectionalStreamingEcho: echoServer.BidirectionalStreamingEcho,
+	})
 
 	if err := grpcServer.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/debugging/server/main.go b/examples/features/debugging/server/main.go
index 397cb0c781e6..839212c99fbc 100644
--- a/examples/features/debugging/server/main.go
+++ b/examples/features/debugging/server/main.go
@@ -36,23 +36,13 @@ var (
 	ports = []string{":10001", ":10002", ":10003"}
 )
 
-// server is used to implement helloworld.GreeterServer.
-type server struct {
-	pb.UnimplementedGreeterServer
-}
-
-// SayHello implements helloworld.GreeterServer
-func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// sayHello implements helloworld.GreeterServer.SayHello
+func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
-// slow server is used to simulate a server that has a variable delay in its response.
-type slowServer struct {
-	pb.UnimplementedGreeterServer
-}
-
-// SayHello implements helloworld.GreeterServer
-func (s *slowServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// sayHelloSlow implements helloworld.GreeterServer.SayHello
+func sayHelloSlow(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	// Delay 100ms ~ 200ms before replying
 	time.Sleep(time.Duration(100+grpcrand.Intn(100)) * time.Millisecond)
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
@@ -70,7 +60,7 @@ func main() {
 	go s.Serve(lis)
 	defer s.Stop()
 
-	/***** Start three GreeterServers(with one of them to be the slowServer). *****/
+	/***** Start three GreeterServers(with one of them to be the slow server). *****/
 	for i := 0; i < 3; i++ {
 		lis, err := net.Listen("tcp", ports[i])
 		if err != nil {
@@ -79,9 +69,9 @@ func main() {
 		defer lis.Close()
 		s := grpc.NewServer()
 		if i == 2 {
-			pb.RegisterGreeterServer(s, &slowServer{})
+			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHelloSlow})
 		} else {
-			pb.RegisterGreeterServer(s, &server{})
+			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
 		}
 		go s.Serve(lis)
 	}
diff --git a/examples/features/encryption/ALTS/server/main.go b/examples/features/encryption/ALTS/server/main.go
index 87bedd810f47..a72ccd11e7fe 100644
--- a/examples/features/encryption/ALTS/server/main.go
+++ b/examples/features/encryption/ALTS/server/main.go
@@ -34,11 +34,7 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-type ecServer struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -54,8 +50,8 @@ func main() {
 
 	s := grpc.NewServer(grpc.Creds(altsTC))
 
-	// Register EchoServer on the server.
-	pb.RegisterEchoServer(s, &ecServer{})
+	// Register EchoService on the server.
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/encryption/TLS/server/main.go b/examples/features/encryption/TLS/server/main.go
index 81bf1f3acc3e..d5ea6ccfcaf3 100644
--- a/examples/features/encryption/TLS/server/main.go
+++ b/examples/features/encryption/TLS/server/main.go
@@ -35,11 +35,7 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-type ecServer struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -60,7 +56,7 @@ func main() {
 	s := grpc.NewServer(grpc.Creds(creds))
 
 	// Register EchoServer on the server.
-	pb.RegisterEchoServer(s, &ecServer{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/errors/server/main.go b/examples/features/errors/server/main.go
index dc337741ad02..c6c06a2bf2cd 100644
--- a/examples/features/errors/server/main.go
+++ b/examples/features/errors/server/main.go
@@ -39,7 +39,6 @@ var port = flag.Int("port", 50052, "port number")
 
 // server is used to implement helloworld.GreeterServer.
 type server struct {
-	pb.UnimplementedGreeterServer
 	mu    sync.Mutex
 	count map[string]int
 }
@@ -78,7 +77,8 @@ func main() {
 	}
 
 	s := grpc.NewServer()
-	pb.RegisterGreeterServer(s, &server{count: make(map[string]int)})
+	hw := &server{count: make(map[string]int)}
+	pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: hw.SayHello})
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
diff --git a/examples/features/health/server/main.go b/examples/features/health/server/main.go
index 3f79c8ba3470..670518da9d08 100644
--- a/examples/features/health/server/main.go
+++ b/examples/features/health/server/main.go
@@ -40,18 +40,12 @@ var (
 	system = "" // empty string represents the health of the system
 )
 
-type echoServer struct {
-	pb.UnimplementedEchoServer
-}
-
-func (e *echoServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{
 		Message: fmt.Sprintf("hello from localhost:%d", *port),
 	}, nil
 }
 
-var _ pb.EchoServer = &echoServer{}
-
 func main() {
 	flag.Parse()
 
@@ -63,7 +57,7 @@ func main() {
 	s := grpc.NewServer()
 	healthcheck := health.NewServer()
 	healthpb.RegisterHealthServer(s, healthcheck)
-	pb.RegisterEchoServer(s, &echoServer{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 
 	go func() {
 		// asynchronously inspect dependencies and toggle serving status as needed
diff --git a/examples/features/interceptor/server/main.go b/examples/features/interceptor/server/main.go
index 1b07cdecd6ca..5ca5a8432517 100644
--- a/examples/features/interceptor/server/main.go
+++ b/examples/features/interceptor/server/main.go
@@ -51,16 +51,12 @@ func logger(format string, a ...interface{}) {
 	fmt.Printf("LOG:\t"+format+"\n", a...)
 }
 
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("unary echoing message %q\n", in.Message)
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
 
-func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
+func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
 	for {
 		in, err := stream.Recv()
 		if err != nil {
@@ -156,8 +152,11 @@ func main() {
 
 	s := grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor))
 
-	// Register EchoServer on the server.
-	pb.RegisterEchoServer(s, &server{})
+	// Register EchoService on the server.
+	pb.RegisterEchoService(s, &pb.EchoService{
+		UnaryEcho:                  unaryEcho,
+		BidirectionalStreamingEcho: bidirectionalStreamingEcho,
+	})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/keepalive/server/main.go b/examples/features/keepalive/server/main.go
index beaa8f710885..e3ec8ae0679a 100644
--- a/examples/features/keepalive/server/main.go
+++ b/examples/features/keepalive/server/main.go
@@ -48,12 +48,7 @@ var kasp = keepalive.ServerParameters{
 	Timeout:               1 * time.Second,  // Wait 1 second for the ping ack before assuming the connection is dead
 }
 
-// server implements EchoServer.
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -67,7 +62,7 @@ func main() {
 	}
 
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/load_balancing/server/main.go b/examples/features/load_balancing/server/main.go
index 9d179579ed41..680895fe50f6 100644
--- a/examples/features/load_balancing/server/main.go
+++ b/examples/features/load_balancing/server/main.go
@@ -36,7 +36,6 @@ var (
 )
 
 type ecServer struct {
-	pb.UnimplementedEchoServer
 	addr string
 }
 
@@ -50,7 +49,8 @@ func startServer(addr string) {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &ecServer{addr: addr})
+	e := &ecServer{addr: addr}
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: e.UnaryEcho})
 	log.Printf("serving on %s\n", addr)
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/metadata/server/main.go b/examples/features/metadata/server/main.go
index cda3b4978697..af12a3a65d17 100644
--- a/examples/features/metadata/server/main.go
+++ b/examples/features/metadata/server/main.go
@@ -44,11 +44,7 @@ const (
 	streamingCount  = 10
 )
 
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("--- UnaryEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -77,7 +73,7 @@ func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoRes
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
 
-func (s *server) ServerStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error {
+func serverStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error {
 	fmt.Printf("--- ServerStreamingEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -114,7 +110,7 @@ func (s *server) ServerStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerSt
 	return nil
 }
 
-func (s *server) ClientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error {
+func clientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error {
 	fmt.Printf("--- ClientStreamingEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -154,7 +150,7 @@ func (s *server) ClientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) e
 	}
 }
 
-func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
+func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
 	fmt.Printf("--- BidirectionalStreamingEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -205,6 +201,11 @@ func main() {
 	fmt.Printf("server listening at %v\n", lis.Addr())
 
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{
+		UnaryEcho:                  unaryEcho,
+		ServerStreamingEcho:        serverStreamingEcho,
+		ClientStreamingEcho:        clientStreamingEcho,
+		BidirectionalStreamingEcho: bidirectionalStreamingEcho,
+	})
 	s.Serve(lis)
 }
diff --git a/examples/features/multiplex/server/main.go b/examples/features/multiplex/server/main.go
index 18da09adda3f..d8b13305da25 100644
--- a/examples/features/multiplex/server/main.go
+++ b/examples/features/multiplex/server/main.go
@@ -34,21 +34,13 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-// hwServer is used to implement helloworld.GreeterServer.
-type hwServer struct {
-	hwpb.UnimplementedGreeterServer
-}
-
-// SayHello implements helloworld.GreeterServer
-func (s *hwServer) SayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
+// sayHello implements helloworld.GreeterServer.SayHello
+func sayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
 	return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
-type ecServer struct {
-	ecpb.UnimplementedEchoServer
-}
-
-func (s *ecServer) UnaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
+// unaryEcho implements echo.Echo.UnaryEcho
+func unaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
 	return &ecpb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -63,10 +55,10 @@ func main() {
 	s := grpc.NewServer()
 
 	// Register Greeter on the server.
-	hwpb.RegisterGreeterServer(s, &hwServer{})
+	hwpb.RegisterGreeterService(s, &hwpb.GreeterService{SayHello: sayHello})
 
-	// Register RouteGuide on the same server.
-	ecpb.RegisterEchoServer(s, &ecServer{})
+	// Register Echo on the same server.
+	ecpb.RegisterEchoService(s, &ecpb.EchoService{UnaryEcho: unaryEcho})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/name_resolving/server/main.go b/examples/features/name_resolving/server/main.go
index 1977f44d0c5e..77d35eb2adac 100644
--- a/examples/features/name_resolving/server/main.go
+++ b/examples/features/name_resolving/server/main.go
@@ -32,13 +32,8 @@ import (
 
 const addr = "localhost:50051"
 
-type ecServer struct {
-	pb.UnimplementedEchoServer
-	addr string
-}
-
-func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
-	return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+	return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, addr)}, nil
 }
 
 func main() {
@@ -47,7 +42,7 @@ func main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &ecServer{addr: addr})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 	log.Printf("serving on %s\n", addr)
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/profiling/README.md b/examples/features/profiling/README.md
index 1fd80e9ada2d..8c30a3dbcac4 100644
--- a/examples/features/profiling/README.md
+++ b/examples/features/profiling/README.md
@@ -41,7 +41,7 @@ type server struct{}
 
 func main() error {
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{...})
 
 	// Include this to register a profiling-specific service within your server.
 	if err := profsvc.Init(&profsvc.ProfilingConfig{Server: s}); err != nil {
diff --git a/examples/features/profiling/server/main.go b/examples/features/profiling/server/main.go
index ee0a4f55ab2e..aa0bf11b2403 100644
--- a/examples/features/profiling/server/main.go
+++ b/examples/features/profiling/server/main.go
@@ -33,11 +33,7 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage())
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
@@ -52,7 +48,7 @@ func main() {
 	fmt.Printf("server listening at %v\n", lis.Addr())
 
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 
 	// Register your grpc.Server with profiling.
 	pc := &profsvc.ProfilingConfig{
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index ff36643530ef..a3597048577a 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // EchoClient is the client API for Echo service.
 //
@@ -35,6 +35,10 @@ func NewEchoClient(cc grpc.ClientConnInterface) EchoClient {
 	return &echoClient{cc}
 }
 
+var echoUnaryEchoStreamDesc = &grpc.StreamDesc{
+	StreamName: "UnaryEcho",
+}
+
 func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) {
 	out := new(EchoResponse)
 	err := c.cc.Invoke(ctx, "/grpc.examples.echo.Echo/UnaryEcho", in, out, opts...)
@@ -44,8 +48,13 @@ func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grp
 	return out, nil
 }
 
+var echoServerStreamingEchoStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ServerStreamingEcho",
+	ServerStreams: true,
+}
+
 func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[0], "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, echoServerStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -76,8 +85,13 @@ func (x *echoServerStreamingEchoClient) Recv() (*EchoResponse, error) {
 	return m, nil
 }
 
+var echoClientStreamingEchoStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ClientStreamingEcho",
+	ClientStreams: true,
+}
+
 func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[1], "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, echoClientStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -110,8 +124,14 @@ func (x *echoClientStreamingEchoClient) CloseAndRecv() (*EchoResponse, error) {
 	return m, nil
 }
 
+var echoBidirectionalStreamingEchoStreamDesc = &grpc.StreamDesc{
+	StreamName:    "BidirectionalStreamingEcho",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[2], "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, echoBidirectionalStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -141,65 +161,62 @@ func (x *echoBidirectionalStreamingEchoClient) Recv() (*EchoResponse, error) {
 	return m, nil
 }
 
-// EchoServer is the server API for Echo service.
-// All implementations should embed UnimplementedEchoServer
-// for forward compatibility
-type EchoServer interface {
+// EchoService is the service API for Echo service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterEchoService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type EchoService struct {
 	// UnaryEcho is unary echo.
-	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
+	UnaryEcho func(context.Context, *EchoRequest) (*EchoResponse, error)
 	// ServerStreamingEcho is server side streaming.
-	ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
+	ServerStreamingEcho func(*EchoRequest, Echo_ServerStreamingEchoServer) error
 	// ClientStreamingEcho is client side streaming.
-	ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
+	ClientStreamingEcho func(Echo_ClientStreamingEchoServer) error
 	// BidirectionalStreamingEcho is bidi streaming.
-	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
+	BidirectionalStreamingEcho func(Echo_BidirectionalStreamingEchoServer) error
 }
 
-// UnimplementedEchoServer should be embedded to have forward compatible implementations.
-type UnimplementedEchoServer struct {
-}
-
-func (*UnimplementedEchoServer) UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
-}
-func (*UnimplementedEchoServer) ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
-}
-func (*UnimplementedEchoServer) ClientStreamingEcho(Echo_ClientStreamingEchoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
-}
-func (*UnimplementedEchoServer) BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error {
-	return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
-}
-
-func RegisterEchoServer(s *grpc.Server, srv EchoServer) {
-	s.RegisterService(&_Echo_serviceDesc, srv)
-}
-
-func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *EchoService) unaryEcho(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.UnaryEcho == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
+	}
 	in := new(EchoRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(EchoServer).UnaryEcho(ctx, in)
+		return s.UnaryEcho(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.examples.echo.Echo/UnaryEcho",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(EchoServer).UnaryEcho(ctx, req.(*EchoRequest))
+		return s.UnaryEcho(ctx, req.(*EchoRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Echo_ServerStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
+func (s *EchoService) serverStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
+	if s.ServerStreamingEcho == nil {
+		return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
+	}
 	m := new(EchoRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(EchoServer).ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
+	return s.ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
+}
+func (s *EchoService) clientStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
+	if s.ClientStreamingEcho == nil {
+		return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
+	}
+	return s.ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
+}
+func (s *EchoService) bidirectionalStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
+	if s.BidirectionalStreamingEcho == nil {
+		return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
+	}
+	return s.BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
 }
 
 type Echo_ServerStreamingEchoServer interface {
@@ -215,10 +232,6 @@ func (x *echoServerStreamingEchoServer) Send(m *EchoResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-func _Echo_ClientStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(EchoServer).ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
-}
-
 type Echo_ClientStreamingEchoServer interface {
 	SendAndClose(*EchoResponse) error
 	Recv() (*EchoRequest, error)
@@ -241,10 +254,6 @@ func (x *echoClientStreamingEchoServer) Recv() (*EchoRequest, error) {
 	return m, nil
 }
 
-func _Echo_BidirectionalStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(EchoServer).BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
-}
-
 type Echo_BidirectionalStreamingEchoServer interface {
 	Send(*EchoResponse) error
 	Recv() (*EchoRequest, error)
@@ -267,32 +276,82 @@ func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) {
 	return m, nil
 }
 
-var _Echo_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.examples.echo.Echo",
-	HandlerType: (*EchoServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryEcho",
-			Handler:    _Echo_UnaryEcho_Handler,
+// RegisterEchoService registers a service implementation with a gRPC server.
+func RegisterEchoService(s grpc.ServiceRegistrar, srv *EchoService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.examples.echo.Echo",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "UnaryEcho",
+				Handler:    srv.unaryEcho,
+			},
 		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ServerStreamingEcho",
-			Handler:       _Echo_ServerStreamingEcho_Handler,
-			ServerStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "ServerStreamingEcho",
+				Handler:       srv.serverStreamingEcho,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "ClientStreamingEcho",
+				Handler:       srv.clientStreamingEcho,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "BidirectionalStreamingEcho",
+				Handler:       srv.bidirectionalStreamingEcho,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
 		},
-		{
-			StreamName:    "ClientStreamingEcho",
-			Handler:       _Echo_ClientStreamingEcho_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "BidirectionalStreamingEcho",
-			Handler:       _Echo_BidirectionalStreamingEcho_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "examples/features/proto/echo/echo.proto",
+		Metadata: "examples/features/proto/echo/echo.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewEchoService creates a new EchoService containing the
+// implemented methods of the Echo service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewEchoService(s interface{}) *EchoService {
+	ns := &EchoService{}
+	if h, ok := s.(interface {
+		UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
+	}); ok {
+		ns.UnaryEcho = h.UnaryEcho
+	}
+	if h, ok := s.(interface {
+		ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
+	}); ok {
+		ns.ServerStreamingEcho = h.ServerStreamingEcho
+	}
+	if h, ok := s.(interface {
+		ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
+	}); ok {
+		ns.ClientStreamingEcho = h.ClientStreamingEcho
+	}
+	if h, ok := s.(interface {
+		BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
+	}); ok {
+		ns.BidirectionalStreamingEcho = h.BidirectionalStreamingEcho
+	}
+	return ns
+}
+
+// UnstableEchoService is the service API for Echo service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableEchoService interface {
+	// UnaryEcho is unary echo.
+	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
+	// ServerStreamingEcho is server side streaming.
+	ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
+	// ClientStreamingEcho is client side streaming.
+	ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
+	// BidirectionalStreamingEcho is bidi streaming.
+	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
 }
diff --git a/examples/features/reflection/server/main.go b/examples/features/reflection/server/main.go
index 569273dfdd3e..1501c01f33aa 100644
--- a/examples/features/reflection/server/main.go
+++ b/examples/features/reflection/server/main.go
@@ -35,21 +35,13 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-// hwServer is used to implement helloworld.GreeterServer.
-type hwServer struct {
-	hwpb.UnimplementedGreeterServer
-}
-
-// SayHello implements helloworld.GreeterServer
-func (s *hwServer) SayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
+// sayHello implements helloworld.GreeterServer.SayHello
+func sayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
 	return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
-type ecServer struct {
-	ecpb.UnimplementedEchoServer
-}
-
-func (s *ecServer) UnaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
+// unaryEcho implements echo.Echo.UnaryEcho
+func unaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
 	return &ecpb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -64,10 +56,10 @@ func main() {
 	s := grpc.NewServer()
 
 	// Register Greeter on the server.
-	hwpb.RegisterGreeterServer(s, &hwServer{})
+	hwpb.RegisterGreeterService(s, &hwpb.GreeterService{SayHello: sayHello})
 
 	// Register RouteGuide on the same server.
-	ecpb.RegisterEchoServer(s, &ecServer{})
+	ecpb.RegisterEchoService(s, &ecpb.EchoService{UnaryEcho: unaryEcho})
 
 	// Register reflection service on gRPC server.
 	reflection.Register(s)
diff --git a/examples/features/retry/server/main.go b/examples/features/retry/server/main.go
index fdec2f052e27..08dd40c93f1c 100644
--- a/examples/features/retry/server/main.go
+++ b/examples/features/retry/server/main.go
@@ -37,7 +37,6 @@ import (
 var port = flag.Int("port", 50052, "port number")
 
 type failingServer struct {
-	pb.UnimplementedEchoServer
 	mu sync.Mutex
 
 	reqCounter uint
@@ -86,7 +85,7 @@ func main() {
 		reqModulo:  4,
 	}
 
-	pb.RegisterEchoServer(s, failingservice)
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: failingservice.UnaryEcho})
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
diff --git a/examples/features/wait_for_ready/main.go b/examples/features/wait_for_ready/main.go
index f865410f1aa2..f3669e986576 100644
--- a/examples/features/wait_for_ready/main.go
+++ b/examples/features/wait_for_ready/main.go
@@ -34,12 +34,7 @@ import (
 	pb "google.golang.org/grpc/examples/features/proto/echo"
 )
 
-// server is used to implement EchoServer.
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -50,7 +45,7 @@ func serve() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
+	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/xds/server/main.go b/examples/features/xds/server/main.go
index 7e0815645e5a..96cd6e468cf5 100644
--- a/examples/features/xds/server/main.go
+++ b/examples/features/xds/server/main.go
@@ -45,8 +45,6 @@ const (
 
 // server is used to implement helloworld.GreeterServer.
 type server struct {
-	pb.UnimplementedGreeterServer
-
 	serverName string
 }
 
@@ -124,7 +122,8 @@ func main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterGreeterServer(s, newServer(hostname))
+	hw := newServer(hostname)
+	pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: hw.SayHello})
 
 	reflection.Register(s)
 	healthServer := health.NewServer()
diff --git a/examples/go.mod b/examples/go.mod
index 5235f543f576..d4e2b7c34a0f 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -3,8 +3,11 @@ module google.golang.org/grpc/examples
 go 1.11
 
 require (
+	cloud.google.com/go v0.63.0 // indirect
 	github.com/golang/protobuf v1.4.2
-	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
-	google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98
 	google.golang.org/grpc v1.31.0
 )
+
+replace google.golang.org/grpc => ../
diff --git a/examples/go.sum b/examples/go.sum
index f55a46171240..1ef62013c1a0 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -1,22 +1,68 @@
-cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.63.0 h1:A+DfAZQ/eWca7gvu42CS6FNSDX4R8cghF+XfWLn4R6g=
+cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -26,53 +72,259 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad h1:uAwc13+y0Y8QZLTYhLCu6lHhnG99ecQU5FYTj8zxAng=
-google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -84,5 +336,19 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/examples/gotutorial.md b/examples/gotutorial.md
index 86993562e909..2862b0a28b98 100644
--- a/examples/gotutorial.md
+++ b/examples/gotutorial.md
@@ -268,7 +268,13 @@ if err != nil {
         log.Fatalf("failed to listen: %v", err)
 }
 grpcServer := grpc.NewServer()
-pb.RegisterRouteGuideServer(grpcServer, &routeGuideServer{})
+rgs := &routeGuideServer{}
+pb.RegisterRouteGuideService(grpcServer, pb.RouteGuideService{
+        GetFeature:   rgs.GetFeature,
+        ListFeatures: rgs.ListFeatures,
+        RecordRoute:  rgs.RecordRoute,
+        RouteChat:    rgs.RouteChat,
+})
 ... // determine whether to use TLS
 grpcServer.Serve(lis)
 ```
diff --git a/examples/helloworld/greeter_server/main.go b/examples/helloworld/greeter_server/main.go
index 15604f9fc1f4..4406f42011e4 100644
--- a/examples/helloworld/greeter_server/main.go
+++ b/examples/helloworld/greeter_server/main.go
@@ -32,13 +32,8 @@ const (
 	port = ":50051"
 )
 
-// server is used to implement helloworld.GreeterServer.
-type server struct {
-	pb.UnimplementedGreeterServer
-}
-
-// SayHello implements helloworld.GreeterServer
-func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// sayHello implements helloworld.GreeterServer.SayHello
+func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	log.Printf("Received: %v", in.GetName())
 	return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil
 }
@@ -49,7 +44,7 @@ func main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterGreeterServer(s, &server{})
+	pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 435df1a0e5eb..b45a600cead2 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // GreeterClient is the client API for Greeter service.
 //
@@ -29,6 +29,10 @@ func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient {
 	return &greeterClient{cc}
 }
 
+var greeterSayHelloStreamDesc = &grpc.StreamDesc{
+	StreamName: "SayHello",
+}
+
 func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) {
 	out := new(HelloReply)
 	err := c.cc.Invoke(ctx, "/helloworld.Greeter/SayHello", in, out, opts...)
@@ -38,53 +42,74 @@ func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...
 	return out, nil
 }
 
-// GreeterServer is the server API for Greeter service.
-// All implementations should embed UnimplementedGreeterServer
-// for forward compatibility
-type GreeterServer interface {
+// GreeterService is the service API for Greeter service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterGreeterService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type GreeterService struct {
 	// Sends a greeting
-	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
-}
-
-// UnimplementedGreeterServer should be embedded to have forward compatible implementations.
-type UnimplementedGreeterServer struct {
+	SayHello func(context.Context, *HelloRequest) (*HelloReply, error)
 }
 
-func (*UnimplementedGreeterServer) SayHello(context.Context, *HelloRequest) (*HelloReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
-}
-
-func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) {
-	s.RegisterService(&_Greeter_serviceDesc, srv)
-}
-
-func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *GreeterService) sayHello(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.SayHello == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
+	}
 	in := new(HelloRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(GreeterServer).SayHello(ctx, in)
+		return s.SayHello(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/helloworld.Greeter/SayHello",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest))
+		return s.SayHello(ctx, req.(*HelloRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _Greeter_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "helloworld.Greeter",
-	HandlerType: (*GreeterServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "SayHello",
-			Handler:    _Greeter_SayHello_Handler,
+// RegisterGreeterService registers a service implementation with a gRPC server.
+func RegisterGreeterService(s grpc.ServiceRegistrar, srv *GreeterService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "helloworld.Greeter",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "SayHello",
+				Handler:    srv.sayHello,
+			},
 		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "examples/helloworld/helloworld/helloworld.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "examples/helloworld/helloworld/helloworld.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewGreeterService creates a new GreeterService containing the
+// implemented methods of the Greeter service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewGreeterService(s interface{}) *GreeterService {
+	ns := &GreeterService{}
+	if h, ok := s.(interface {
+		SayHello(context.Context, *HelloRequest) (*HelloReply, error)
+	}); ok {
+		ns.SayHello = h.SayHello
+	}
+	return ns
+}
+
+// UnstableGreeterService is the service API for Greeter service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableGreeterService interface {
+	// Sends a greeting
+	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
 }
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index 9ae8b4483bfb..f9c59a7f1d0f 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // RouteGuideClient is the client API for RouteGuide service.
 //
@@ -51,6 +51,10 @@ func NewRouteGuideClient(cc grpc.ClientConnInterface) RouteGuideClient {
 	return &routeGuideClient{cc}
 }
 
+var routeGuideGetFeatureStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetFeature",
+}
+
 func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) {
 	out := new(Feature)
 	err := c.cc.Invoke(ctx, "/routeguide.RouteGuide/GetFeature", in, out, opts...)
@@ -60,8 +64,13 @@ func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...gr
 	return out, nil
 }
 
+var routeGuideListFeaturesStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ListFeatures",
+	ServerStreams: true,
+}
+
 func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[0], "/routeguide.RouteGuide/ListFeatures", opts...)
+	stream, err := c.cc.NewStream(ctx, routeGuideListFeaturesStreamDesc, "/routeguide.RouteGuide/ListFeatures", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -92,8 +101,13 @@ func (x *routeGuideListFeaturesClient) Recv() (*Feature, error) {
 	return m, nil
 }
 
+var routeGuideRecordRouteStreamDesc = &grpc.StreamDesc{
+	StreamName:    "RecordRoute",
+	ClientStreams: true,
+}
+
 func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[1], "/routeguide.RouteGuide/RecordRoute", opts...)
+	stream, err := c.cc.NewStream(ctx, routeGuideRecordRouteStreamDesc, "/routeguide.RouteGuide/RecordRoute", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -126,8 +140,14 @@ func (x *routeGuideRecordRouteClient) CloseAndRecv() (*RouteSummary, error) {
 	return m, nil
 }
 
+var routeGuideRouteChatStreamDesc = &grpc.StreamDesc{
+	StreamName:    "RouteChat",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[2], "/routeguide.RouteGuide/RouteChat", opts...)
+	stream, err := c.cc.NewStream(ctx, routeGuideRouteChatStreamDesc, "/routeguide.RouteGuide/RouteChat", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -157,81 +177,78 @@ func (x *routeGuideRouteChatClient) Recv() (*RouteNote, error) {
 	return m, nil
 }
 
-// RouteGuideServer is the server API for RouteGuide service.
-// All implementations should embed UnimplementedRouteGuideServer
-// for forward compatibility
-type RouteGuideServer interface {
+// RouteGuideService is the service API for RouteGuide service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterRouteGuideService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type RouteGuideService struct {
 	// A simple RPC.
 	//
 	// Obtains the feature at a given position.
 	//
 	// A feature with an empty name is returned if there's no feature at the given
 	// position.
-	GetFeature(context.Context, *Point) (*Feature, error)
+	GetFeature func(context.Context, *Point) (*Feature, error)
 	// A server-to-client streaming RPC.
 	//
 	// Obtains the Features available within the given Rectangle.  Results are
 	// streamed rather than returned at once (e.g. in a response message with a
 	// repeated field), as the rectangle may cover a large area and contain a
 	// huge number of features.
-	ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
+	ListFeatures func(*Rectangle, RouteGuide_ListFeaturesServer) error
 	// A client-to-server streaming RPC.
 	//
 	// Accepts a stream of Points on a route being traversed, returning a
 	// RouteSummary when traversal is completed.
-	RecordRoute(RouteGuide_RecordRouteServer) error
+	RecordRoute func(RouteGuide_RecordRouteServer) error
 	// A Bidirectional streaming RPC.
 	//
 	// Accepts a stream of RouteNotes sent while a route is being traversed,
 	// while receiving other RouteNotes (e.g. from other users).
-	RouteChat(RouteGuide_RouteChatServer) error
-}
-
-// UnimplementedRouteGuideServer should be embedded to have forward compatible implementations.
-type UnimplementedRouteGuideServer struct {
+	RouteChat func(RouteGuide_RouteChatServer) error
 }
 
-func (*UnimplementedRouteGuideServer) GetFeature(context.Context, *Point) (*Feature, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
-}
-func (*UnimplementedRouteGuideServer) ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error {
-	return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
-}
-func (*UnimplementedRouteGuideServer) RecordRoute(RouteGuide_RecordRouteServer) error {
-	return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
-}
-func (*UnimplementedRouteGuideServer) RouteChat(RouteGuide_RouteChatServer) error {
-	return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
-}
-
-func RegisterRouteGuideServer(s *grpc.Server, srv RouteGuideServer) {
-	s.RegisterService(&_RouteGuide_serviceDesc, srv)
-}
-
-func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *RouteGuideService) getFeature(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetFeature == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
+	}
 	in := new(Point)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(RouteGuideServer).GetFeature(ctx, in)
+		return s.GetFeature(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/routeguide.RouteGuide/GetFeature",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(RouteGuideServer).GetFeature(ctx, req.(*Point))
+		return s.GetFeature(ctx, req.(*Point))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _RouteGuide_ListFeatures_Handler(srv interface{}, stream grpc.ServerStream) error {
+func (s *RouteGuideService) listFeatures(_ interface{}, stream grpc.ServerStream) error {
+	if s.ListFeatures == nil {
+		return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
+	}
 	m := new(Rectangle)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(RouteGuideServer).ListFeatures(m, &routeGuideListFeaturesServer{stream})
+	return s.ListFeatures(m, &routeGuideListFeaturesServer{stream})
+}
+func (s *RouteGuideService) recordRoute(_ interface{}, stream grpc.ServerStream) error {
+	if s.RecordRoute == nil {
+		return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
+	}
+	return s.RecordRoute(&routeGuideRecordRouteServer{stream})
+}
+func (s *RouteGuideService) routeChat(_ interface{}, stream grpc.ServerStream) error {
+	if s.RouteChat == nil {
+		return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
+	}
+	return s.RouteChat(&routeGuideRouteChatServer{stream})
 }
 
 type RouteGuide_ListFeaturesServer interface {
@@ -247,10 +264,6 @@ func (x *routeGuideListFeaturesServer) Send(m *Feature) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-func _RouteGuide_RecordRoute_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(RouteGuideServer).RecordRoute(&routeGuideRecordRouteServer{stream})
-}
-
 type RouteGuide_RecordRouteServer interface {
 	SendAndClose(*RouteSummary) error
 	Recv() (*Point, error)
@@ -273,10 +286,6 @@ func (x *routeGuideRecordRouteServer) Recv() (*Point, error) {
 	return m, nil
 }
 
-func _RouteGuide_RouteChat_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(RouteGuideServer).RouteChat(&routeGuideRouteChatServer{stream})
-}
-
 type RouteGuide_RouteChatServer interface {
 	Send(*RouteNote) error
 	Recv() (*RouteNote, error)
@@ -299,32 +308,98 @@ func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) {
 	return m, nil
 }
 
-var _RouteGuide_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "routeguide.RouteGuide",
-	HandlerType: (*RouteGuideServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetFeature",
-			Handler:    _RouteGuide_GetFeature_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ListFeatures",
-			Handler:       _RouteGuide_ListFeatures_Handler,
-			ServerStreams: true,
+// RegisterRouteGuideService registers a service implementation with a gRPC server.
+func RegisterRouteGuideService(s grpc.ServiceRegistrar, srv *RouteGuideService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "routeguide.RouteGuide",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "GetFeature",
+				Handler:    srv.getFeature,
+			},
 		},
-		{
-			StreamName:    "RecordRoute",
-			Handler:       _RouteGuide_RecordRoute_Handler,
-			ClientStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "ListFeatures",
+				Handler:       srv.listFeatures,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "RecordRoute",
+				Handler:       srv.recordRoute,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "RouteChat",
+				Handler:       srv.routeChat,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
 		},
-		{
-			StreamName:    "RouteChat",
-			Handler:       _RouteGuide_RouteChat_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "examples/route_guide/routeguide/route_guide.proto",
+		Metadata: "examples/route_guide/routeguide/route_guide.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewRouteGuideService creates a new RouteGuideService containing the
+// implemented methods of the RouteGuide service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewRouteGuideService(s interface{}) *RouteGuideService {
+	ns := &RouteGuideService{}
+	if h, ok := s.(interface {
+		GetFeature(context.Context, *Point) (*Feature, error)
+	}); ok {
+		ns.GetFeature = h.GetFeature
+	}
+	if h, ok := s.(interface {
+		ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
+	}); ok {
+		ns.ListFeatures = h.ListFeatures
+	}
+	if h, ok := s.(interface {
+		RecordRoute(RouteGuide_RecordRouteServer) error
+	}); ok {
+		ns.RecordRoute = h.RecordRoute
+	}
+	if h, ok := s.(interface {
+		RouteChat(RouteGuide_RouteChatServer) error
+	}); ok {
+		ns.RouteChat = h.RouteChat
+	}
+	return ns
+}
+
+// UnstableRouteGuideService is the service API for RouteGuide service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableRouteGuideService interface {
+	// A simple RPC.
+	//
+	// Obtains the feature at a given position.
+	//
+	// A feature with an empty name is returned if there's no feature at the given
+	// position.
+	GetFeature(context.Context, *Point) (*Feature, error)
+	// A server-to-client streaming RPC.
+	//
+	// Obtains the Features available within the given Rectangle.  Results are
+	// streamed rather than returned at once (e.g. in a response message with a
+	// repeated field), as the rectangle may cover a large area and contain a
+	// huge number of features.
+	ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
+	// A client-to-server streaming RPC.
+	//
+	// Accepts a stream of Points on a route being traversed, returning a
+	// RouteSummary when traversal is completed.
+	RecordRoute(RouteGuide_RecordRouteServer) error
+	// A Bidirectional streaming RPC.
+	//
+	// Accepts a stream of RouteNotes sent while a route is being traversed,
+	// while receiving other RouteNotes (e.g. from other users).
+	RouteChat(RouteGuide_RouteChatServer) error
 }
diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go
index dd804406afcd..7c959dde06b1 100644
--- a/examples/route_guide/server/server.go
+++ b/examples/route_guide/server/server.go
@@ -54,7 +54,6 @@ var (
 )
 
 type routeGuideServer struct {
-	pb.UnimplementedRouteGuideServer
 	savedFeatures []*pb.Feature // read-only after initialized
 
 	mu         sync.Mutex // protects routeNotes
@@ -238,7 +237,7 @@ func main() {
 		opts = []grpc.ServerOption{grpc.Creds(creds)}
 	}
 	grpcServer := grpc.NewServer(opts...)
-	pb.RegisterRouteGuideServer(grpcServer, newServer())
+	pb.RegisterRouteGuideService(grpcServer, pb.NewRouteGuideService(newServer()))
 	grpcServer.Serve(lis)
 }
 
diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go
index e9919c0073b6..4c2a527ec597 100644
--- a/health/grpc_health_v1/health.pb.go
+++ b/health/grpc_health_v1/health.pb.go
@@ -4,8 +4,12 @@
 package grpc_health_v1
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -159,3 +163,181 @@ var fileDescriptor_e265fd9d4e077217 = []byte{
 	0xd3, 0x20, 0x46, 0xe8, 0x85, 0x19, 0x26, 0xb1, 0x81, 0x93, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff,
 	0xff, 0x12, 0x7d, 0x96, 0xcb, 0x2d, 0x02, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// HealthClient is the client API for Health service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type HealthClient interface {
+	// If the requested service is unknown, the call will fail with status
+	// NOT_FOUND.
+	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
+	// Performs a watch for the serving status of the requested service.
+	// The server will immediately send back a message indicating the current
+	// serving status.  It will then subsequently send a new message whenever
+	// the service's serving status changes.
+	//
+	// If the requested service is unknown when the call is received, the
+	// server will send a message setting the serving status to
+	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
+	// future point, the serving status of the service becomes known, the
+	// server will send a new message with the service's serving status.
+	//
+	// If the call terminates with status UNIMPLEMENTED, then clients
+	// should assume this method is not supported and should not retry the
+	// call.  If the call terminates with any other status (including OK),
+	// clients should retry the call with appropriate exponential backoff.
+	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
+}
+
+type healthClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
+	return &healthClient{cc}
+}
+
+func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
+	out := new(HealthCheckResponse)
+	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &healthWatchClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Health_WatchClient interface {
+	Recv() (*HealthCheckResponse, error)
+	grpc.ClientStream
+}
+
+type healthWatchClient struct {
+	grpc.ClientStream
+}
+
+func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
+	m := new(HealthCheckResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// HealthServer is the server API for Health service.
+type HealthServer interface {
+	// If the requested service is unknown, the call will fail with status
+	// NOT_FOUND.
+	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
+	// Performs a watch for the serving status of the requested service.
+	// The server will immediately send back a message indicating the current
+	// serving status.  It will then subsequently send a new message whenever
+	// the service's serving status changes.
+	//
+	// If the requested service is unknown when the call is received, the
+	// server will send a message setting the serving status to
+	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
+	// future point, the serving status of the service becomes known, the
+	// server will send a new message with the service's serving status.
+	//
+	// If the call terminates with status UNIMPLEMENTED, then clients
+	// should assume this method is not supported and should not retry the
+	// call.  If the call terminates with any other status (including OK),
+	// clients should retry the call with appropriate exponential backoff.
+	Watch(*HealthCheckRequest, Health_WatchServer) error
+}
+
+// UnimplementedHealthServer can be embedded to have forward compatible implementations.
+type UnimplementedHealthServer struct {
+}
+
+func (*UnimplementedHealthServer) Check(ctx context.Context, req *HealthCheckRequest) (*HealthCheckResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+}
+func (*UnimplementedHealthServer) Watch(req *HealthCheckRequest, srv Health_WatchServer) error {
+	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+}
+
+func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
+	s.RegisterService(&_Health_serviceDesc, srv)
+}
+
+func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HealthCheckRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(HealthServer).Check(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.health.v1.Health/Check",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(HealthCheckRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
+}
+
+type Health_WatchServer interface {
+	Send(*HealthCheckResponse) error
+	grpc.ServerStream
+}
+
+type healthWatchServer struct {
+	grpc.ServerStream
+}
+
+func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+var _Health_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.health.v1.Health",
+	HandlerType: (*HealthServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Check",
+			Handler:    _Health_Check_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Watch",
+			Handler:       _Health_Watch_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "grpc/health/v1/health.proto",
+}
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index f87e3c92adbe..716f8c0153a4 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -11,15 +11,16 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// HealthClient is the client API for Health service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type HealthClient interface {
+// HealthService is the service API for Health service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterHealthService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type HealthService struct {
 	// If the requested service is unknown, the call will fail with status
 	// NOT_FOUND.
-	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
+	Check func(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
 	// Performs a watch for the serving status of the requested service.
 	// The server will immediately send back a message indicating the current
 	// serving status.  It will then subsequently send a new message whenever
@@ -35,62 +36,89 @@ type HealthClient interface {
 	// should assume this method is not supported and should not retry the
 	// call.  If the call terminates with any other status (including OK),
 	// clients should retry the call with appropriate exponential backoff.
-	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
+	Watch func(*HealthCheckRequest, Health_WatchServer) error
 }
 
-type healthClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
-	return &healthClient{cc}
-}
-
-func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
-	out := new(HealthCheckResponse)
-	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
-	if err != nil {
-		return nil, err
+func (s *HealthService) check(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.Check == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
 	}
-	return out, nil
-}
-
-func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
-	if err != nil {
+	in := new(HealthCheckRequest)
+	if err := dec(in); err != nil {
 		return nil, err
 	}
-	x := &healthWatchClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
+	if interceptor == nil {
+		return s.Check(ctx, in)
 	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
+	info := &grpc.UnaryServerInfo{
+		Server:     s,
+		FullMethod: "/grpc.health.v1.Health/Check",
 	}
-	return x, nil
-}
-
-type Health_WatchClient interface {
-	Recv() (*HealthCheckResponse, error)
-	grpc.ClientStream
-}
-
-type healthWatchClient struct {
-	grpc.ClientStream
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return s.Check(ctx, req.(*HealthCheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
 }
+func (s *HealthService) watch(_ interface{}, stream grpc.ServerStream) error {
+	if s.Watch == nil {
+		return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+	}
+	m := new(HealthCheckRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return s.Watch(m, &healthWatchServer{stream})
+}
+
+// RegisterHealthService registers a service implementation with a gRPC server.
+func RegisterHealthService(s grpc.ServiceRegistrar, srv *HealthService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.health.v1.Health",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "Check",
+				Handler:    srv.check,
+			},
+		},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "Watch",
+				Handler:       srv.watch,
+				ServerStreams: true,
+			},
+		},
+		Metadata: "grpc/health/v1/health.proto",
+	}
 
-func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
-	m := new(HealthCheckResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
+	s.RegisterService(&sd, nil)
+}
+
+// NewHealthService creates a new HealthService containing the
+// implemented methods of the Health service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewHealthService(s interface{}) *HealthService {
+	ns := &HealthService{}
+	if h, ok := s.(interface {
+		Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
+	}); ok {
+		ns.Check = h.Check
 	}
-	return m, nil
+	if h, ok := s.(interface {
+		Watch(*HealthCheckRequest, Health_WatchServer) error
+	}); ok {
+		ns.Watch = h.Watch
+	}
+	return ns
 }
 
-// HealthServer is the server API for Health service.
-// All implementations should embed UnimplementedHealthServer
-// for forward compatibility
-type HealthServer interface {
+// UnstableHealthService is the service API for Health service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableHealthService interface {
 	// If the requested service is unknown, the call will fail with status
 	// NOT_FOUND.
 	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
@@ -111,76 +139,3 @@ type HealthServer interface {
 	// clients should retry the call with appropriate exponential backoff.
 	Watch(*HealthCheckRequest, Health_WatchServer) error
 }
-
-// UnimplementedHealthServer should be embedded to have forward compatible implementations.
-type UnimplementedHealthServer struct {
-}
-
-func (*UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
-}
-func (*UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
-	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
-}
-
-func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
-	s.RegisterService(&_Health_serviceDesc, srv)
-}
-
-func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(HealthCheckRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(HealthServer).Check(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.health.v1.Health/Check",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(HealthCheckRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
-}
-
-type Health_WatchServer interface {
-	Send(*HealthCheckResponse) error
-	grpc.ServerStream
-}
-
-type healthWatchServer struct {
-	grpc.ServerStream
-}
-
-func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-var _Health_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.health.v1.Health",
-	HandlerType: (*HealthServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Check",
-			Handler:    _Health_Check_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "Watch",
-			Handler:       _Health_Watch_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "grpc/health/v1/health.proto",
-}
diff --git a/internal/binarylog/binarylog_end2end_test.go b/internal/binarylog/binarylog_end2end_test.go
index 999bd90fb052..bdce754a57d5 100644
--- a/internal/binarylog/binarylog_end2end_test.go
+++ b/internal/binarylog/binarylog_end2end_test.go
@@ -115,10 +115,11 @@ var (
 )
 
 type testServer struct {
-	testpb.UnimplementedTestServiceServer
 	te *test
 }
 
+var _ testpb.UnstableTestServiceService = (*testServer)(nil)
+
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	md, ok := metadata.FromIncomingContext(ctx)
 	if ok {
@@ -216,7 +217,7 @@ func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.Te
 type test struct {
 	t *testing.T
 
-	testServer testpb.TestServiceServer // nil means none
+	testService *testpb.TestServiceService // nil means none
 	// srv and srvAddr are set once startServer is called.
 	srv     *grpc.Server
 	srvAddr string // Server IP without port.
@@ -271,8 +272,8 @@ func (lw *listenerWrapper) Accept() (net.Conn, error) {
 
 // startServer starts a gRPC server listening. Callers should defer a
 // call to te.tearDown to clean up.
-func (te *test) startServer(ts testpb.TestServiceServer) {
-	te.testServer = ts
+func (te *test) startServer(ts *testpb.TestServiceService) {
+	te.testService = ts
 	lis, err := net.Listen("tcp", "localhost:0")
 
 	lis = &listenerWrapper{
@@ -286,8 +287,8 @@ func (te *test) startServer(ts testpb.TestServiceServer) {
 	var opts []grpc.ServerOption
 	s := grpc.NewServer(opts...)
 	te.srv = s
-	if te.testServer != nil {
-		testpb.RegisterTestServiceServer(s, te.testServer)
+	if te.testService != nil {
+		testpb.RegisterTestServiceService(s, te.testService)
 	}
 
 	go s.Serve(lis)
@@ -783,7 +784,7 @@ func (ed *expectedData) toServerLogEntries() []*pb.GrpcLogEntry {
 
 func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData {
 	te := newTest(t, tc)
-	te.startServer(&testServer{te: te})
+	te.startServer(testpb.NewTestServiceService(&testServer{te: te}))
 	defer te.tearDown()
 
 	expect := &expectedData{
diff --git a/interop/alts/server/server.go b/interop/alts/server/server.go
index 0d0f375a0d1a..0ac7678a51b5 100644
--- a/interop/alts/server/server.go
+++ b/interop/alts/server/server.go
@@ -64,7 +64,7 @@ func main() {
 	}
 	altsTC := alts.NewServerCreds(opts)
 	grpcServer := grpc.NewServer(grpc.Creds(altsTC), grpc.InTapHandle(authz))
-	testpb.RegisterTestServiceServer(grpcServer, interop.NewTestServer())
+	testpb.RegisterTestServiceService(grpcServer, interop.NewTestServer())
 	grpcServer.Serve(lis)
 }
 
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index 07b555a3f690..d71f6d2a46b9 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // TestServiceClient is the client API for TestService service.
 //
@@ -47,6 +47,10 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
 	return &testServiceClient{cc}
 }
 
+var testServiceEmptyCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "EmptyCall",
+}
+
 func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
 	out := new(Empty)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
@@ -56,6 +60,10 @@ func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...gr
 	return out, nil
 }
 
+var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "UnaryCall",
+}
+
 func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
@@ -65,8 +73,13 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
+var testServiceStreamingOutputCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "StreamingOutputCall",
+	ServerStreams: true,
+}
+
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceStreamingOutputCallStreamDesc, "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -97,8 +110,13 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo
 	return m, nil
 }
 
+var testServiceStreamingInputCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "StreamingInputCall",
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceStreamingInputCallStreamDesc, "/grpc.testing.TestService/StreamingInputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -131,8 +149,14 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal
 	return m, nil
 }
 
+var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "FullDuplexCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -162,8 +186,14 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
+var testServiceHalfDuplexCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "HalfDuplexCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceHalfDuplexCallStreamDesc, "/grpc.testing.TestService/HalfDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -193,101 +223,100 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
-// TestServiceServer is the server API for TestService service.
-// All implementations should embed UnimplementedTestServiceServer
-// for forward compatibility
-type TestServiceServer interface {
+// TestServiceService is the service API for TestService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterTestServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type TestServiceService struct {
 	// One empty request followed by one empty response.
-	EmptyCall(context.Context, *Empty) (*Empty, error)
+	EmptyCall func(context.Context, *Empty) (*Empty, error)
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by a sequence of responses (streamed download).
 	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	StreamingOutputCall func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
 	// A sequence of requests followed by one response (streamed upload).
 	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(TestService_StreamingInputCallServer) error
+	StreamingInputCall func(TestService_StreamingInputCallServer) error
 	// A sequence of requests with each request served by the server immediately.
 	// As one request could lead to multiple responses, this interface
 	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
+	FullDuplexCall func(TestService_FullDuplexCallServer) error
 	// A sequence of requests followed by a sequence of responses.
 	// The server buffers all the client requests and then serves them in order. A
 	// stream of responses are returned to the client when the server starts with
 	// first request.
-	HalfDuplexCall(TestService_HalfDuplexCallServer) error
-}
-
-// UnimplementedTestServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
-}
-
-func (*UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-}
-func (*UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (*UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+	HalfDuplexCall func(TestService_HalfDuplexCallServer) error
 }
 
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
-}
-
-func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.EmptyCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+	}
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(TestServiceServer).EmptyCall(ctx, in)
+		return s.EmptyCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.TestService/EmptyCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
+		return s.EmptyCall(ctx, req.(*Empty))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.UnaryCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
+		return s.UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.TestService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+		return s.UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.StreamingOutputCall == nil {
+		return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+	}
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
+	return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
+}
+func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.StreamingInputCall == nil {
+		return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+	}
+	return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream})
+}
+func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.FullDuplexCall == nil {
+		return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+	}
+	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.HalfDuplexCall == nil {
+		return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+	}
+	return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
 }
 
 type TestService_StreamingOutputCallServer interface {
@@ -303,10 +332,6 @@ func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallRespon
 	return x.ServerStream.SendMsg(m)
 }
 
-func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
-}
-
 type TestService_StreamingInputCallServer interface {
 	SendAndClose(*StreamingInputCallResponse) error
 	Recv() (*StreamingInputCallRequest, error)
@@ -329,10 +354,6 @@ func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest
 	return m, nil
 }
 
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-
 type TestService_FullDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -355,10 +376,6 @@ func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
-}
-
 type TestService_HalfDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -381,44 +398,116 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-var _TestService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "EmptyCall",
-			Handler:    _TestService_EmptyCall_Handler,
-		},
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingOutputCall",
-			Handler:       _TestService_StreamingOutputCall_Handler,
-			ServerStreams: true,
+// RegisterTestServiceService registers a service implementation with a gRPC server.
+func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.TestService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "EmptyCall",
+				Handler:    srv.emptyCall,
+			},
+			{
+				MethodName: "UnaryCall",
+				Handler:    srv.unaryCall,
+			},
 		},
-		{
-			StreamName:    "StreamingInputCall",
-			Handler:       _TestService_StreamingInputCall_Handler,
-			ClientStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "StreamingOutputCall",
+				Handler:       srv.streamingOutputCall,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "StreamingInputCall",
+				Handler:       srv.streamingInputCall,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "FullDuplexCall",
+				Handler:       srv.fullDuplexCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "HalfDuplexCall",
+				Handler:       srv.halfDuplexCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
 		},
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "HalfDuplexCall",
-			Handler:       _TestService_HalfDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "interop/grpc_testing/test.proto",
+		Metadata: "interop/grpc_testing/test.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewTestServiceService creates a new TestServiceService containing the
+// implemented methods of the TestService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewTestServiceService(s interface{}) *TestServiceService {
+	ns := &TestServiceService{}
+	if h, ok := s.(interface {
+		EmptyCall(context.Context, *Empty) (*Empty, error)
+	}); ok {
+		ns.EmptyCall = h.EmptyCall
+	}
+	if h, ok := s.(interface {
+		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	}); ok {
+		ns.UnaryCall = h.UnaryCall
+	}
+	if h, ok := s.(interface {
+		StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	}); ok {
+		ns.StreamingOutputCall = h.StreamingOutputCall
+	}
+	if h, ok := s.(interface {
+		StreamingInputCall(TestService_StreamingInputCallServer) error
+	}); ok {
+		ns.StreamingInputCall = h.StreamingInputCall
+	}
+	if h, ok := s.(interface {
+		FullDuplexCall(TestService_FullDuplexCallServer) error
+	}); ok {
+		ns.FullDuplexCall = h.FullDuplexCall
+	}
+	if h, ok := s.(interface {
+		HalfDuplexCall(TestService_HalfDuplexCallServer) error
+	}); ok {
+		ns.HalfDuplexCall = h.HalfDuplexCall
+	}
+	return ns
+}
+
+// UnstableTestServiceService is the service API for TestService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableTestServiceService interface {
+	// One empty request followed by one empty response.
+	EmptyCall(context.Context, *Empty) (*Empty, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by a sequence of responses (streamed download).
+	// The server returns the payload with client desired type and sizes.
+	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	// A sequence of requests followed by one response (streamed upload).
+	// The server returns the aggregated size of client payload as the result.
+	StreamingInputCall(TestService_StreamingInputCallServer) error
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(TestService_FullDuplexCallServer) error
+	// A sequence of requests followed by a sequence of responses.
+	// The server buffers all the client requests and then serves them in order. A
+	// stream of responses are returned to the client when the server starts with
+	// first request.
+	HalfDuplexCall(TestService_HalfDuplexCallServer) error
 }
 
 // UnimplementedServiceClient is the client API for UnimplementedService service.
@@ -437,6 +526,10 @@ func NewUnimplementedServiceClient(cc grpc.ClientConnInterface) UnimplementedSer
 	return &unimplementedServiceClient{cc}
 }
 
+var unimplementedServiceUnimplementedCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "UnimplementedCall",
+}
+
 func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
 	out := new(Empty)
 	err := c.cc.Invoke(ctx, "/grpc.testing.UnimplementedService/UnimplementedCall", in, out, opts...)
@@ -446,55 +539,76 @@ func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *
 	return out, nil
 }
 
-// UnimplementedServiceServer is the server API for UnimplementedService service.
-// All implementations should embed UnimplementedUnimplementedServiceServer
-// for forward compatibility
-type UnimplementedServiceServer interface {
+// UnimplementedServiceService is the service API for UnimplementedService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterUnimplementedServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type UnimplementedServiceService struct {
 	// A call that no server should implement
-	UnimplementedCall(context.Context, *Empty) (*Empty, error)
-}
-
-// UnimplementedUnimplementedServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedUnimplementedServiceServer struct {
+	UnimplementedCall func(context.Context, *Empty) (*Empty, error)
 }
 
-func (*UnimplementedUnimplementedServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
-}
-
-func RegisterUnimplementedServiceServer(s *grpc.Server, srv UnimplementedServiceServer) {
-	s.RegisterService(&_UnimplementedService_serviceDesc, srv)
-}
-
-func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *UnimplementedServiceService) unimplementedCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.UnimplementedCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
+	}
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, in)
+		return s.UnimplementedCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.UnimplementedService/UnimplementedCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, req.(*Empty))
+		return s.UnimplementedCall(ctx, req.(*Empty))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _UnimplementedService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.UnimplementedService",
-	HandlerType: (*UnimplementedServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnimplementedCall",
-			Handler:    _UnimplementedService_UnimplementedCall_Handler,
+// RegisterUnimplementedServiceService registers a service implementation with a gRPC server.
+func RegisterUnimplementedServiceService(s grpc.ServiceRegistrar, srv *UnimplementedServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.UnimplementedService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "UnimplementedCall",
+				Handler:    srv.unimplementedCall,
+			},
 		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "interop/grpc_testing/test.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewUnimplementedServiceService creates a new UnimplementedServiceService containing the
+// implemented methods of the UnimplementedService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewUnimplementedServiceService(s interface{}) *UnimplementedServiceService {
+	ns := &UnimplementedServiceService{}
+	if h, ok := s.(interface {
+		UnimplementedCall(context.Context, *Empty) (*Empty, error)
+	}); ok {
+		ns.UnimplementedCall = h.UnimplementedCall
+	}
+	return ns
+}
+
+// UnstableUnimplementedServiceService is the service API for UnimplementedService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableUnimplementedServiceService interface {
+	// A call that no server should implement
+	UnimplementedCall(context.Context, *Empty) (*Empty, error)
 }
 
 // LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
@@ -513,6 +627,10 @@ func NewLoadBalancerStatsServiceClient(cc grpc.ClientConnInterface) LoadBalancer
 	return &loadBalancerStatsServiceClient{cc}
 }
 
+var loadBalancerStatsServiceGetClientStatsStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetClientStats",
+}
+
 func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) {
 	out := new(LoadBalancerStatsResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.LoadBalancerStatsService/GetClientStats", in, out, opts...)
@@ -522,53 +640,74 @@ func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in
 	return out, nil
 }
 
-// LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service.
-// All implementations should embed UnimplementedLoadBalancerStatsServiceServer
-// for forward compatibility
-type LoadBalancerStatsServiceServer interface {
+// LoadBalancerStatsServiceService is the service API for LoadBalancerStatsService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterLoadBalancerStatsServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type LoadBalancerStatsServiceService struct {
 	// Gets the backend distribution for RPCs sent by a test client.
-	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
+	GetClientStats func(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
 }
 
-// UnimplementedLoadBalancerStatsServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedLoadBalancerStatsServiceServer struct {
-}
-
-func (*UnimplementedLoadBalancerStatsServiceServer) GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
-}
-
-func RegisterLoadBalancerStatsServiceServer(s *grpc.Server, srv LoadBalancerStatsServiceServer) {
-	s.RegisterService(&_LoadBalancerStatsService_serviceDesc, srv)
-}
-
-func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *LoadBalancerStatsServiceService) getClientStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetClientStats == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
+	}
 	in := new(LoadBalancerStatsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, in)
+		return s.GetClientStats(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.LoadBalancerStatsService/GetClientStats",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, req.(*LoadBalancerStatsRequest))
+		return s.GetClientStats(ctx, req.(*LoadBalancerStatsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _LoadBalancerStatsService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.LoadBalancerStatsService",
-	HandlerType: (*LoadBalancerStatsServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetClientStats",
-			Handler:    _LoadBalancerStatsService_GetClientStats_Handler,
+// RegisterLoadBalancerStatsServiceService registers a service implementation with a gRPC server.
+func RegisterLoadBalancerStatsServiceService(s grpc.ServiceRegistrar, srv *LoadBalancerStatsServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.LoadBalancerStatsService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "GetClientStats",
+				Handler:    srv.getClientStats,
+			},
 		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "interop/grpc_testing/test.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewLoadBalancerStatsServiceService creates a new LoadBalancerStatsServiceService containing the
+// implemented methods of the LoadBalancerStatsService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewLoadBalancerStatsServiceService(s interface{}) *LoadBalancerStatsServiceService {
+	ns := &LoadBalancerStatsServiceService{}
+	if h, ok := s.(interface {
+		GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
+	}); ok {
+		ns.GetClientStats = h.GetClientStats
+	}
+	return ns
+}
+
+// UnstableLoadBalancerStatsServiceService is the service API for LoadBalancerStatsService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableLoadBalancerStatsServiceService interface {
+	// Gets the backend distribution for RPCs sent by a test client.
+	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
 }
diff --git a/interop/server/server.go b/interop/server/server.go
index c70e450bb108..662fb2963090 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -76,6 +76,6 @@ func main() {
 		opts = append(opts, grpc.Creds(altsTC))
 	}
 	server := grpc.NewServer(opts...)
-	testpb.RegisterTestServiceServer(server, interop.NewTestServer())
+	testpb.RegisterTestServiceService(server, interop.NewTestServer())
 	server.Serve(lis)
 }
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 7e3aaa95f0ca..7a42116e74b3 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -673,13 +673,11 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) {
 	}
 }
 
-type testServer struct {
-	testpb.UnimplementedTestServiceServer
-}
+type testServer struct{}
 
 // NewTestServer creates a test server for test service.
-func NewTestServer() testpb.TestServiceServer {
-	return &testServer{}
+func NewTestServer() *testpb.TestServiceService {
+	return testpb.NewTestServiceService(testpb.UnstableTestServiceService(&testServer{}))
 }
 
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index b119bcd2b865..466219063881 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -95,10 +95,6 @@ var (
 	logger = grpclog.Component("interop")
 )
 
-type statsService struct {
-	testpb.UnimplementedLoadBalancerStatsServiceServer
-}
-
 func hasRPCSucceeded() bool {
 	return atomic.LoadUint32(&rpcSucceeded) > 0
 }
@@ -111,7 +107,7 @@ func setRPCSucceeded() {
 // and return the distribution of remote peers. This is essentially a clientside
 // LB reporting mechanism that is designed to be queried by an external test
 // driver when verifying that the client is distributing RPCs as expected.
-func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) {
+func getClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) {
 	mu.Lock()
 	watcherKey := statsWatcherKey{currentRequestID, currentRequestID + in.GetNumRpcs()}
 	watcher, ok := watchers[watcherKey]
@@ -226,7 +222,7 @@ func main() {
 	}
 	s := grpc.NewServer()
 	defer s.Stop()
-	testpb.RegisterLoadBalancerStatsServiceServer(s, &statsService{})
+	testpb.RegisterLoadBalancerStatsServiceService(s, &testpb.LoadBalancerStatsServiceService{GetClientStats: getClientStats})
 	go s.Serve(lis)
 
 	clients := make([]testpb.TestServiceClient, *numChannels)
diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go
index 45b8448822b2..8d2a62487a88 100644
--- a/interop/xds/server/server.go
+++ b/interop/xds/server/server.go
@@ -49,16 +49,12 @@ func getHostname() string {
 	return hostname
 }
 
-type server struct {
-	testpb.UnimplementedTestServiceServer
-}
-
-func (s *server) EmptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+func emptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 	grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname))
 	return &testpb.Empty{}, nil
 }
 
-func (s *server) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+func unaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname))
 	return &testpb.SimpleResponse{ServerId: *serverID, Hostname: hostname}, nil
 }
@@ -71,6 +67,6 @@ func main() {
 		logger.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceServer(s, &server{})
+	testpb.RegisterTestServiceService(s, &testpb.TestServiceService{EmptyCall: emptyCall, UnaryCall: unaryCall})
 	s.Serve(lis)
 }
diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go
index 831a62721383..90f02824ef69 100644
--- a/profiling/proto/service.pb.go
+++ b/profiling/proto/service.pb.go
@@ -4,8 +4,12 @@
 package proto
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -362,3 +366,125 @@ var fileDescriptor_e1ab2aa17b47c6fb = []byte{
 	0x95, 0xbb, 0x2e, 0xf9, 0xc9, 0xfd, 0xaf, 0x42, 0xfa, 0xbc, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff,
 	0x5d, 0x47, 0x09, 0xa9, 0x19, 0x03, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ProfilingClient is the client API for Profiling service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type ProfilingClient interface {
+	// Enable allows users to toggle profiling on and off remotely.
+	Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error)
+	// GetStreamStats is used to retrieve an array of stream-level stats from a
+	// gRPC client/server.
+	GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error)
+}
+
+type profilingClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
+	return &profilingClient{cc}
+}
+
+func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
+	out := new(EnableResponse)
+	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
+	out := new(GetStreamStatsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ProfilingServer is the server API for Profiling service.
+type ProfilingServer interface {
+	// Enable allows users to toggle profiling on and off remotely.
+	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
+	// GetStreamStats is used to retrieve an array of stream-level stats from a
+	// gRPC client/server.
+	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
+}
+
+// UnimplementedProfilingServer can be embedded to have forward compatible implementations.
+type UnimplementedProfilingServer struct {
+}
+
+func (*UnimplementedProfilingServer) Enable(ctx context.Context, req *EnableRequest) (*EnableResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
+}
+func (*UnimplementedProfilingServer) GetStreamStats(ctx context.Context, req *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
+}
+
+func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
+	s.RegisterService(&_Profiling_serviceDesc, srv)
+}
+
+func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EnableRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ProfilingServer).Enable(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetStreamStatsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ProfilingServer).GetStreamStats(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Profiling_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
+	HandlerType: (*ProfilingServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Enable",
+			Handler:    _Profiling_Enable_Handler,
+		},
+		{
+			MethodName: "GetStreamStats",
+			Handler:    _Profiling_GetStreamStats_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "profiling/proto/service.proto",
+}
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index aed8d58762d3..e32cbb405db4 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -11,120 +11,111 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// ProfilingClient is the client API for Profiling service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type ProfilingClient interface {
+// ProfilingService is the service API for Profiling service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterProfilingService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type ProfilingService struct {
 	// Enable allows users to toggle profiling on and off remotely.
-	Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error)
+	Enable func(context.Context, *EnableRequest) (*EnableResponse, error)
 	// GetStreamStats is used to retrieve an array of stream-level stats from a
 	// gRPC client/server.
-	GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error)
+	GetStreamStats func(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
 }
 
-type profilingClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
-	return &profilingClient{cc}
-}
-
-func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
-	out := new(EnableResponse)
-	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
-	if err != nil {
-		return nil, err
+func (s *ProfilingService) enable(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.Enable == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
 	}
-	return out, nil
-}
-
-func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
-	out := new(GetStreamStatsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ProfilingServer is the server API for Profiling service.
-// All implementations should embed UnimplementedProfilingServer
-// for forward compatibility
-type ProfilingServer interface {
-	// Enable allows users to toggle profiling on and off remotely.
-	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
-	// GetStreamStats is used to retrieve an array of stream-level stats from a
-	// gRPC client/server.
-	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
-}
-
-// UnimplementedProfilingServer should be embedded to have forward compatible implementations.
-type UnimplementedProfilingServer struct {
-}
-
-func (*UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
-}
-func (*UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
-}
-
-func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
-	s.RegisterService(&_Profiling_serviceDesc, srv)
-}
-
-func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(EnableRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ProfilingServer).Enable(ctx, in)
+		return s.Enable(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest))
+		return s.Enable(ctx, req.(*EnableRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *ProfilingService) getStreamStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetStreamStats == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
+	}
 	in := new(GetStreamStatsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ProfilingServer).GetStreamStats(ctx, in)
+		return s.GetStreamStats(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest))
+		return s.GetStreamStats(ctx, req.(*GetStreamStatsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-var _Profiling_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
-	HandlerType: (*ProfilingServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Enable",
-			Handler:    _Profiling_Enable_Handler,
-		},
-		{
-			MethodName: "GetStreamStats",
-			Handler:    _Profiling_GetStreamStats_Handler,
+// RegisterProfilingService registers a service implementation with a gRPC server.
+func RegisterProfilingService(s grpc.ServiceRegistrar, srv *ProfilingService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.go.profiling.v1alpha.Profiling",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "Enable",
+				Handler:    srv.enable,
+			},
+			{
+				MethodName: "GetStreamStats",
+				Handler:    srv.getStreamStats,
+			},
 		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "profiling/proto/service.proto",
+		Streams:  []grpc.StreamDesc{},
+		Metadata: "profiling/proto/service.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewProfilingService creates a new ProfilingService containing the
+// implemented methods of the Profiling service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewProfilingService(s interface{}) *ProfilingService {
+	ns := &ProfilingService{}
+	if h, ok := s.(interface {
+		Enable(context.Context, *EnableRequest) (*EnableResponse, error)
+	}); ok {
+		ns.Enable = h.Enable
+	}
+	if h, ok := s.(interface {
+		GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
+	}); ok {
+		ns.GetStreamStats = h.GetStreamStats
+	}
+	return ns
+}
+
+// UnstableProfilingService is the service API for Profiling service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableProfilingService interface {
+	// Enable allows users to toggle profiling on and off remotely.
+	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
+	// GetStreamStats is used to retrieve an array of stream-level stats from a
+	// gRPC client/server.
+	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
 }
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 382612d520dd..900bd6c05c78 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -4,8 +4,12 @@
 package grpc_reflection_v1alpha
 
 import (
+	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -632,3 +636,119 @@ var fileDescriptor_e8cf9f2921ad6c95 = []byte{
 	0xcb, 0xb3, 0xdb, 0x8c, 0xdb, 0xea, 0x53, 0xd5, 0xb9, 0xfd, 0xd3, 0x35, 0xdc, 0x54, 0xbe, 0x39,
 	0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x74, 0x3a, 0x67, 0xe7, 0x06, 0x00, 0x00,
 }
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ServerReflectionClient is the client API for ServerReflection service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type ServerReflectionClient interface {
+	// The reflection service is structured as a bidirectional stream, ensuring
+	// all related requests go to a single server.
+	ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error)
+}
+
+type serverReflectionClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient {
+	return &serverReflectionClient{cc}
+}
+
+func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &serverReflectionServerReflectionInfoClient{stream}
+	return x, nil
+}
+
+type ServerReflection_ServerReflectionInfoClient interface {
+	Send(*ServerReflectionRequest) error
+	Recv() (*ServerReflectionResponse, error)
+	grpc.ClientStream
+}
+
+type serverReflectionServerReflectionInfoClient struct {
+	grpc.ClientStream
+}
+
+func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) {
+	m := new(ServerReflectionResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// ServerReflectionServer is the server API for ServerReflection service.
+type ServerReflectionServer interface {
+	// The reflection service is structured as a bidirectional stream, ensuring
+	// all related requests go to a single server.
+	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
+}
+
+// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations.
+type UnimplementedServerReflectionServer struct {
+}
+
+func (*UnimplementedServerReflectionServer) ServerReflectionInfo(srv ServerReflection_ServerReflectionInfoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
+}
+
+func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
+	s.RegisterService(&_ServerReflection_serviceDesc, srv)
+}
+
+func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
+}
+
+type ServerReflection_ServerReflectionInfoServer interface {
+	Send(*ServerReflectionResponse) error
+	Recv() (*ServerReflectionRequest, error)
+	grpc.ServerStream
+}
+
+type serverReflectionServerReflectionInfoServer struct {
+	grpc.ServerStream
+}
+
+func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) {
+	m := new(ServerReflectionRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _ServerReflection_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
+	HandlerType: (*ServerReflectionServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ServerReflectionInfo",
+			Handler:       _ServerReflection_ServerReflectionInfo_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
+}
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index 2294b2c6c9e5..95f9d0cb62d9 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -3,7 +3,6 @@
 package grpc_reflection_v1alpha
 
 import (
-	context "context"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
@@ -11,114 +10,66 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
-// ServerReflectionClient is the client API for ServerReflection service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type ServerReflectionClient interface {
+// ServerReflectionService is the service API for ServerReflection service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterServerReflectionService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type ServerReflectionService struct {
 	// The reflection service is structured as a bidirectional stream, ensuring
 	// all related requests go to a single server.
-	ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error)
+	ServerReflectionInfo func(ServerReflection_ServerReflectionInfoServer) error
 }
 
-type serverReflectionClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient {
-	return &serverReflectionClient{cc}
-}
-
-func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
-	if err != nil {
-		return nil, err
+func (s *ServerReflectionService) serverReflectionInfo(_ interface{}, stream grpc.ServerStream) error {
+	if s.ServerReflectionInfo == nil {
+		return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
+	}
+	return s.ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
+}
+
+// RegisterServerReflectionService registers a service implementation with a gRPC server.
+func RegisterServerReflectionService(s grpc.ServiceRegistrar, srv *ServerReflectionService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.reflection.v1alpha.ServerReflection",
+		Methods:     []grpc.MethodDesc{},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "ServerReflectionInfo",
+				Handler:       srv.serverReflectionInfo,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+		},
+		Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
 	}
-	x := &serverReflectionServerReflectionInfoClient{stream}
-	return x, nil
-}
-
-type ServerReflection_ServerReflectionInfoClient interface {
-	Send(*ServerReflectionRequest) error
-	Recv() (*ServerReflectionResponse, error)
-	grpc.ClientStream
-}
-
-type serverReflectionServerReflectionInfoClient struct {
-	grpc.ClientStream
-}
-
-func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
 
-func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) {
-	m := new(ServerReflectionResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
+	s.RegisterService(&sd, nil)
+}
+
+// NewServerReflectionService creates a new ServerReflectionService containing the
+// implemented methods of the ServerReflection service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewServerReflectionService(s interface{}) *ServerReflectionService {
+	ns := &ServerReflectionService{}
+	if h, ok := s.(interface {
+		ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
+	}); ok {
+		ns.ServerReflectionInfo = h.ServerReflectionInfo
 	}
-	return m, nil
+	return ns
 }
 
-// ServerReflectionServer is the server API for ServerReflection service.
-// All implementations should embed UnimplementedServerReflectionServer
-// for forward compatibility
-type ServerReflectionServer interface {
+// UnstableServerReflectionService is the service API for ServerReflection service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableServerReflectionService interface {
 	// The reflection service is structured as a bidirectional stream, ensuring
 	// all related requests go to a single server.
 	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
 }
-
-// UnimplementedServerReflectionServer should be embedded to have forward compatible implementations.
-type UnimplementedServerReflectionServer struct {
-}
-
-func (*UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
-}
-
-func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
-	s.RegisterService(&_ServerReflection_serviceDesc, srv)
-}
-
-func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
-}
-
-type ServerReflection_ServerReflectionInfoServer interface {
-	Send(*ServerReflectionResponse) error
-	Recv() (*ServerReflectionRequest, error)
-	grpc.ServerStream
-}
-
-type serverReflectionServerReflectionInfoServer struct {
-	grpc.ServerStream
-}
-
-func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) {
-	m := new(ServerReflectionRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _ServerReflection_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
-	HandlerType: (*ServerReflectionServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ServerReflectionInfo",
-			Handler:       _ServerReflection_ServerReflectionInfo_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
-}
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 95cd2bab1c22..285cdbc85ace 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // SearchServiceClient is the client API for SearchService service.
 //
@@ -29,6 +29,10 @@ func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient {
 	return &searchServiceClient{cc}
 }
 
+var searchServiceSearchStreamDesc = &grpc.StreamDesc{
+	StreamName: "Search",
+}
+
 func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) {
 	out := new(SearchResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.SearchService/Search", in, out, opts...)
@@ -38,8 +42,14 @@ func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opt
 	return out, nil
 }
 
+var searchServiceStreamingSearchStreamDesc = &grpc.StreamDesc{
+	StreamName:    "StreamingSearch",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_SearchService_serviceDesc.Streams[0], "/grpc.testing.SearchService/StreamingSearch", opts...)
+	stream, err := c.cc.NewStream(ctx, searchServiceStreamingSearchStreamDesc, "/grpc.testing.SearchService/StreamingSearch", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -69,49 +79,40 @@ func (x *searchServiceStreamingSearchClient) Recv() (*SearchResponse, error) {
 	return m, nil
 }
 
-// SearchServiceServer is the server API for SearchService service.
-// All implementations should embed UnimplementedSearchServiceServer
-// for forward compatibility
-type SearchServiceServer interface {
-	Search(context.Context, *SearchRequest) (*SearchResponse, error)
-	StreamingSearch(SearchService_StreamingSearchServer) error
-}
-
-// UnimplementedSearchServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedSearchServiceServer struct {
-}
-
-func (*UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
-}
-func (*UnimplementedSearchServiceServer) StreamingSearch(SearchService_StreamingSearchServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
-}
-
-func RegisterSearchServiceServer(s *grpc.Server, srv SearchServiceServer) {
-	s.RegisterService(&_SearchService_serviceDesc, srv)
+// SearchServiceService is the service API for SearchService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterSearchServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type SearchServiceService struct {
+	Search          func(context.Context, *SearchRequest) (*SearchResponse, error)
+	StreamingSearch func(SearchService_StreamingSearchServer) error
 }
 
-func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *SearchServiceService) search(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.Search == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
+	}
 	in := new(SearchRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(SearchServiceServer).Search(ctx, in)
+		return s.Search(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.SearchService/Search",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest))
+		return s.Search(ctx, req.(*SearchRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _SearchService_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(SearchServiceServer).StreamingSearch(&searchServiceStreamingSearchServer{stream})
+func (s *SearchServiceService) streamingSearch(_ interface{}, stream grpc.ServerStream) error {
+	if s.StreamingSearch == nil {
+		return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
+	}
+	return s.StreamingSearch(&searchServiceStreamingSearchServer{stream})
 }
 
 type SearchService_StreamingSearchServer interface {
@@ -136,22 +137,56 @@ func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) {
 	return m, nil
 }
 
-var _SearchService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.SearchService",
-	HandlerType: (*SearchServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Search",
-			Handler:    _SearchService_Search_Handler,
+// RegisterSearchServiceService registers a service implementation with a gRPC server.
+func RegisterSearchServiceService(s grpc.ServiceRegistrar, srv *SearchServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.SearchService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "Search",
+				Handler:    srv.search,
+			},
 		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingSearch",
-			Handler:       _SearchService_StreamingSearch_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "StreamingSearch",
+				Handler:       srv.streamingSearch,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
 		},
-	},
-	Metadata: "reflection/grpc_testing/test.proto",
+		Metadata: "reflection/grpc_testing/test.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewSearchServiceService creates a new SearchServiceService containing the
+// implemented methods of the SearchService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewSearchServiceService(s interface{}) *SearchServiceService {
+	ns := &SearchServiceService{}
+	if h, ok := s.(interface {
+		Search(context.Context, *SearchRequest) (*SearchResponse, error)
+	}); ok {
+		ns.Search = h.Search
+	}
+	if h, ok := s.(interface {
+		StreamingSearch(SearchService_StreamingSearchServer) error
+	}); ok {
+		ns.StreamingSearch = h.StreamingSearch
+	}
+	return ns
+}
+
+// UnstableSearchServiceService is the service API for SearchService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableSearchServiceService interface {
+	Search(context.Context, *SearchRequest) (*SearchResponse, error)
+	StreamingSearch(SearchService_StreamingSearchServer) error
 }
diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go
index 4cf6717feafa..db5fce2d8939 100644
--- a/reflection/serverreflection_test.go
+++ b/reflection/serverreflection_test.go
@@ -166,9 +166,9 @@ func (x) TestAllExtensionNumbersForType(t *testing.T) {
 
 // Do end2end tests.
 
-type server struct {
-	pb.UnimplementedSearchServiceServer
-}
+type server struct{}
+
+var _ pb.UnstableSearchServiceService = (*server)(nil)
 
 func (s *server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.SearchResponse, error) {
 	return &pb.SearchResponse{}, nil
@@ -195,7 +195,7 @@ func (x) TestReflectionEnd2end(t *testing.T) {
 		t.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterSearchServiceServer(s, &server{})
+	pb.RegisterSearchServiceService(s, pb.NewSearchServiceService(&server{}))
 	pbv3.RegisterSearchServiceV3Server(s, &serverV3{})
 	// Register reflection service on s.
 	Register(s)
diff --git a/regenerate.sh b/regenerate.sh
index 7e33b45e0862..750e3500e529 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -59,7 +59,8 @@ curl --silent https://raw.githubusercontent.com/istio/istio/master/security/prot
 
 mkdir -p ${WORKDIR}/out
 
-SOURCES=(
+# Generates legacy gRPC Server symbols in addition to the newer Service symbols
+LEGACY_SOURCES=(
   ${WORKDIR}/googleapis/google/rpc/code.proto
   ${WORKDIR}/grpc-proto/grpc/binlog/v1/binarylog.proto
   ${WORKDIR}/grpc-proto/grpc/channelz/v1/channelz.proto
@@ -73,16 +74,29 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
   ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
   ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
-  $(git ls-files --exclude-standard --cached --others "*.proto")
+  profiling/proto/service.proto
+  reflection/grpc_reflection_v1alpha/reflection.proto
 )
+
+# Generates only the new gRPC Service symbols
+SOURCES=(
+  $(git ls-files --exclude-standard --cached --others "*.proto" | grep -v '^\(profiling/proto/service.proto\|reflection/grpc_reflection_v1alpha/reflection.proto\)$')
+)
+
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
 # import path of 'bar' in the generated code when 'foo.proto' is imported in
 # one of the sources.
 OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config,\
 Menvoy/config/core/v3/config_source.proto=github.com/envoyproxy/go-control-plane/envoy/config/core/v3
+
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
-  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},requireUnimplementedServers=false:${WORKDIR}/out \
+  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out ${src}
+done
+
+for src in ${LEGACY_SOURCES[@]}; do
+  echo "protoc ${src}"
+  protoc --go_out=${OPTS},plugins=grpc:${WORKDIR}/out --go-grpc_out=${OPTS},migration_mode=true:${WORKDIR}/out \
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
diff --git a/server.go b/server.go
index ff71435fad72..33fc3240bcc6 100644
--- a/server.go
+++ b/server.go
@@ -554,12 +554,15 @@ type ServiceRegistrar interface {
 
 // RegisterService registers a service and its implementation to the gRPC
 // server. It is called from the IDL generated code. This must be called before
-// invoking Serve.
+// invoking Serve. If ss is non-nil (for legacy code), its type is checked to
+// ensure it implements sd.HandlerType.
 func (s *Server) RegisterService(sd *ServiceDesc, ss interface{}) {
-	ht := reflect.TypeOf(sd.HandlerType).Elem()
-	st := reflect.TypeOf(ss)
-	if !st.Implements(ht) {
-		logger.Fatalf("grpc: Server.RegisterService found the handler of type %v that does not satisfy %v", st, ht)
+	if ss != nil {
+		ht := reflect.TypeOf(sd.HandlerType).Elem()
+		st := reflect.TypeOf(ss)
+		if !st.Implements(ht) {
+			logger.Fatalf("grpc: Server.RegisterService found the handler of type %v that does not satisfy %v", st, ht)
+		}
 	}
 	s.register(sd, ss)
 }
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index 98da3340b76a..1fff30c631d6 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // TestServiceClient is the client API for TestService service.
 //
@@ -38,6 +38,10 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
 	return &testServiceClient{cc}
 }
 
+var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "UnaryCall",
+}
+
 func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
@@ -47,8 +51,14 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
+var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "FullDuplexCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -78,8 +88,13 @@ func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) {
 	return m, nil
 }
 
+var testServiceClientStreamCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ClientStreamCall",
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceClientStreamCallStreamDesc, "/grpc.testing.TestService/ClientStreamCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -112,8 +127,13 @@ func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, err
 	return m, nil
 }
 
+var testServiceServerStreamCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ServerStreamCall",
+	ServerStreams: true,
+}
+
 func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceServerStreamCallStreamDesc, "/grpc.testing.TestService/ServerStreamCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -144,64 +164,65 @@ func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) {
 	return m, nil
 }
 
-// TestServiceServer is the server API for TestService service.
-// All implementations should embed UnimplementedTestServiceServer
-// for forward compatibility
-type TestServiceServer interface {
+// TestServiceService is the service API for TestService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterTestServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type TestServiceService struct {
 	// One request followed by one response.
 	// The server returns the client id as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// A sequence of requests with each request served by the server immediately.
 	// As one request could lead to multiple responses, this interface
 	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
+	FullDuplexCall func(TestService_FullDuplexCallServer) error
 	// Client stream
-	ClientStreamCall(TestService_ClientStreamCallServer) error
+	ClientStreamCall func(TestService_ClientStreamCallServer) error
 	// Server stream
-	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
+	ServerStreamCall func(*SimpleRequest, TestService_ServerStreamCallServer) error
 }
 
-// UnimplementedTestServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
-}
-
-func (*UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (*UnimplementedTestServiceServer) ClientStreamCall(TestService_ClientStreamCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
-}
-func (*UnimplementedTestServiceServer) ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
-}
-
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
-}
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.UnaryCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
+		return s.UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.TestService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+		return s.UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.FullDuplexCall == nil {
+		return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+	}
+	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+func (s *TestServiceService) clientStreamCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.ClientStreamCall == nil {
+		return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
+	}
+	return s.ClientStreamCall(&testServiceClientStreamCallServer{stream})
+}
+func (s *TestServiceService) serverStreamCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.ServerStreamCall == nil {
+		return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
+	}
+	m := new(SimpleRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return s.ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
 }
 
 type TestService_FullDuplexCallServer interface {
@@ -226,10 +247,6 @@ func (x *testServiceFullDuplexCallServer) Recv() (*SimpleRequest, error) {
 	return m, nil
 }
 
-func _TestService_ClientStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).ClientStreamCall(&testServiceClientStreamCallServer{stream})
-}
-
 type TestService_ClientStreamCallServer interface {
 	SendAndClose(*SimpleResponse) error
 	Recv() (*SimpleRequest, error)
@@ -252,14 +269,6 @@ func (x *testServiceClientStreamCallServer) Recv() (*SimpleRequest, error) {
 	return m, nil
 }
 
-func _TestService_ServerStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(SimpleRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(TestServiceServer).ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
-}
-
 type TestService_ServerStreamCallServer interface {
 	Send(*SimpleResponse) error
 	grpc.ServerStream
@@ -273,32 +282,85 @@ func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-var _TestService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
+// RegisterTestServiceService registers a service implementation with a gRPC server.
+func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.TestService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "UnaryCall",
+				Handler:    srv.unaryCall,
+			},
 		},
-		{
-			StreamName:    "ClientStreamCall",
-			Handler:       _TestService_ClientStreamCall_Handler,
-			ClientStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "FullDuplexCall",
+				Handler:       srv.fullDuplexCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "ClientStreamCall",
+				Handler:       srv.clientStreamCall,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "ServerStreamCall",
+				Handler:       srv.serverStreamCall,
+				ServerStreams: true,
+			},
 		},
-		{
-			StreamName:    "ServerStreamCall",
-			Handler:       _TestService_ServerStreamCall_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "stats/grpc_testing/test.proto",
+		Metadata: "stats/grpc_testing/test.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewTestServiceService creates a new TestServiceService containing the
+// implemented methods of the TestService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewTestServiceService(s interface{}) *TestServiceService {
+	ns := &TestServiceService{}
+	if h, ok := s.(interface {
+		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	}); ok {
+		ns.UnaryCall = h.UnaryCall
+	}
+	if h, ok := s.(interface {
+		FullDuplexCall(TestService_FullDuplexCallServer) error
+	}); ok {
+		ns.FullDuplexCall = h.FullDuplexCall
+	}
+	if h, ok := s.(interface {
+		ClientStreamCall(TestService_ClientStreamCallServer) error
+	}); ok {
+		ns.ClientStreamCall = h.ClientStreamCall
+	}
+	if h, ok := s.(interface {
+		ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
+	}); ok {
+		ns.ServerStreamCall = h.ServerStreamCall
+	}
+	return ns
+}
+
+// UnstableTestServiceService is the service API for TestService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableTestServiceService interface {
+	// One request followed by one response.
+	// The server returns the client id as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(TestService_FullDuplexCallServer) error
+	// Client stream
+	ClientStreamCall(TestService_ClientStreamCallServer) error
+	// Server stream
+	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
 }
diff --git a/stats/stats_test.go b/stats/stats_test.go
index d047d48bc5e2..628533edbe2c 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -73,9 +73,9 @@ var (
 	errorID int32 = 32202
 )
 
-type testServer struct {
-	testpb.UnimplementedTestServiceServer
-}
+type testServer struct{}
+
+var _ testpb.UnstableTestServiceService = (*testServer)(nil)
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	if err := grpc.SendHeader(ctx, testHeaderMetadata); err != nil {
@@ -165,7 +165,7 @@ type test struct {
 	clientStatsHandler stats.Handler
 	serverStatsHandler stats.Handler
 
-	testServer testpb.TestServiceServer // nil means none
+	testService *testpb.TestServiceService // nil means none
 	// srv and srvAddr are set once startServer is called.
 	srv     *grpc.Server
 	srvAddr string
@@ -200,8 +200,8 @@ func newTest(t *testing.T, tc *testConfig, ch stats.Handler, sh stats.Handler) *
 
 // startServer starts a gRPC server listening. Callers should defer a
 // call to te.tearDown to clean up.
-func (te *test) startServer(ts testpb.TestServiceServer) {
-	te.testServer = ts
+func (te *test) startServer(ts *testpb.TestServiceService) {
+	te.testService = ts
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		te.t.Fatalf("Failed to listen: %v", err)
@@ -218,8 +218,8 @@ func (te *test) startServer(ts testpb.TestServiceServer) {
 	}
 	s := grpc.NewServer(opts...)
 	te.srv = s
-	if te.testServer != nil {
-		testpb.RegisterTestServiceServer(s, te.testServer)
+	if te.testService != nil {
+		testpb.RegisterTestServiceService(s, te.testService)
 	}
 
 	go s.Serve(lis)
@@ -815,7 +815,7 @@ func checkServerStats(t *testing.T, got []*gotData, expect *expectedData, checkF
 func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) {
 	h := &statshandler{}
 	te := newTest(t, tc, nil, h)
-	te.startServer(&testServer{})
+	te.startServer(testpb.NewTestServiceService(&testServer{}))
 	defer te.tearDown()
 
 	var (
@@ -1106,7 +1106,7 @@ func checkClientStats(t *testing.T, got []*gotData, expect *expectedData, checkF
 func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map[int]*checkFuncWithCount) {
 	h := &statshandler{}
 	te := newTest(t, tc, h, nil)
-	te.startServer(&testServer{})
+	te.startServer(testpb.NewTestServiceService(&testServer{}))
 	defer te.tearDown()
 
 	var (
diff --git a/stress/client/main.go b/stress/client/main.go
index c5bfffa4e519..0353476d8f39 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -146,12 +146,13 @@ func (g *gauge) get() int64 {
 
 // server implements metrics server functions.
 type server struct {
-	metricspb.UnimplementedMetricsServiceServer
 	mutex sync.RWMutex
 	// gauges is a map from /stress_test/server_<n>/channel_<n>/stub_<n>/qps to its qps gauge.
 	gauges map[string]*gauge
 }
 
+var _ metricspb.UnstableMetricsServiceService = (*server)(nil)
+
 // newMetricsServer returns a new metrics server.
 func newMetricsServer() *server {
 	return &server{gauges: make(map[string]*gauge)}
@@ -202,7 +203,7 @@ func startServer(server *server, port int) {
 	}
 
 	s := grpc.NewServer()
-	metricspb.RegisterMetricsServiceServer(s, server)
+	metricspb.RegisterMetricsServiceService(s, metricspb.NewMetricsServiceService(server))
 	s.Serve(lis)
 
 }
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 3a232ff8501b..3f0a83584b2d 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // MetricsServiceClient is the client API for MetricsService service.
 //
@@ -32,8 +32,13 @@ func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient {
 	return &metricsServiceClient{cc}
 }
 
+var metricsServiceGetAllGaugesStreamDesc = &grpc.StreamDesc{
+	StreamName:    "GetAllGauges",
+	ServerStreams: true,
+}
+
 func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_MetricsService_serviceDesc.Streams[0], "/grpc.testing.MetricsService/GetAllGauges", opts...)
+	stream, err := c.cc.NewStream(ctx, metricsServiceGetAllGaugesStreamDesc, "/grpc.testing.MetricsService/GetAllGauges", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -64,6 +69,10 @@ func (x *metricsServiceGetAllGaugesClient) Recv() (*GaugeResponse, error) {
 	return m, nil
 }
 
+var metricsServiceGetGaugeStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetGauge",
+}
+
 func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) {
 	out := new(GaugeResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.MetricsService/GetGauge", in, out, opts...)
@@ -73,38 +82,47 @@ func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, o
 	return out, nil
 }
 
-// MetricsServiceServer is the server API for MetricsService service.
-// All implementations should embed UnimplementedMetricsServiceServer
-// for forward compatibility
-type MetricsServiceServer interface {
+// MetricsServiceService is the service API for MetricsService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterMetricsServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type MetricsServiceService struct {
 	// Returns the values of all the gauges that are currently being maintained by
 	// the service
-	GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
+	GetAllGauges func(*EmptyMessage, MetricsService_GetAllGaugesServer) error
 	// Returns the value of one gauge
-	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
-}
-
-// UnimplementedMetricsServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedMetricsServiceServer struct {
-}
-
-func (*UnimplementedMetricsServiceServer) GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error {
-	return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
-}
-func (*UnimplementedMetricsServiceServer) GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
-}
-
-func RegisterMetricsServiceServer(s *grpc.Server, srv MetricsServiceServer) {
-	s.RegisterService(&_MetricsService_serviceDesc, srv)
+	GetGauge func(context.Context, *GaugeRequest) (*GaugeResponse, error)
 }
 
-func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error {
+func (s *MetricsServiceService) getAllGauges(_ interface{}, stream grpc.ServerStream) error {
+	if s.GetAllGauges == nil {
+		return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
+	}
 	m := new(EmptyMessage)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(MetricsServiceServer).GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
+	return s.GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
+}
+func (s *MetricsServiceService) getGauge(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.GetGauge == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
+	}
+	in := new(GaugeRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return s.GetGauge(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     s,
+		FullMethod: "/grpc.testing.MetricsService/GetGauge",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return s.GetGauge(ctx, req.(*GaugeRequest))
+	}
+	return interceptor(ctx, in, info, handler)
 }
 
 type MetricsService_GetAllGaugesServer interface {
@@ -120,39 +138,58 @@ func (x *metricsServiceGetAllGaugesServer) Send(m *GaugeResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GaugeRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(MetricsServiceServer).GetGauge(ctx, in)
+// RegisterMetricsServiceService registers a service implementation with a gRPC server.
+func RegisterMetricsServiceService(s grpc.ServiceRegistrar, srv *MetricsServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.MetricsService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "GetGauge",
+				Handler:    srv.getGauge,
+			},
+		},
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "GetAllGauges",
+				Handler:       srv.getAllGauges,
+				ServerStreams: true,
+			},
+		},
+		Metadata: "stress/grpc_testing/metrics.proto",
 	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.MetricsService/GetGauge",
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewMetricsServiceService creates a new MetricsServiceService containing the
+// implemented methods of the MetricsService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewMetricsServiceService(s interface{}) *MetricsServiceService {
+	ns := &MetricsServiceService{}
+	if h, ok := s.(interface {
+		GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
+	}); ok {
+		ns.GetAllGauges = h.GetAllGauges
 	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(MetricsServiceServer).GetGauge(ctx, req.(*GaugeRequest))
+	if h, ok := s.(interface {
+		GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
+	}); ok {
+		ns.GetGauge = h.GetGauge
 	}
-	return interceptor(ctx, in, info, handler)
+	return ns
 }
 
-var _MetricsService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.MetricsService",
-	HandlerType: (*MetricsServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetGauge",
-			Handler:    _MetricsService_GetGauge_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "GetAllGauges",
-			Handler:       _MetricsService_GetAllGauges_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "stress/grpc_testing/metrics.proto",
+// UnstableMetricsServiceService is the service API for MetricsService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableMetricsServiceService interface {
+	// Returns the values of all the gauges that are currently being maintained by
+	// the service
+	GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
+	// Returns the value of one gauge
+	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
 }
diff --git a/test/balancer_test.go b/test/balancer_test.go
index f0189cf26444..c45a380115b3 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -498,7 +498,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	}
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceServer(s, &testServer{})
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&testServer{}))
 	go s.Serve(lis)
 	defer s.Stop()
 	t.Logf("Started gRPC server at %s...", lis.Addr().String())
@@ -561,7 +561,7 @@ func (s) TestServersSwap(t *testing.T) {
 				return &testpb.SimpleResponse{Username: username}, nil
 			},
 		}
-		testpb.RegisterTestServiceServer(s, ts)
+		testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
 		go s.Serve(lis)
 		return lis.Addr().String(), s.Stop
 	}
@@ -621,7 +621,7 @@ func (s) TestEmptyAddrs(t *testing.T) {
 			return &testpb.SimpleResponse{Username: one}, nil
 		},
 	}
-	testpb.RegisterTestServiceServer(s, ts)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
 	go s.Serve(lis)
 
 	// Initialize pickfirst client
@@ -710,7 +710,7 @@ func (s) TestWaitForReady(t *testing.T) {
 			return &testpb.SimpleResponse{Username: one}, nil
 		},
 	}
-	testpb.RegisterTestServiceServer(s, ts)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
 	go s.Serve(lis)
 
 	// Initialize client
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 0842dccaad01..f26997ea808b 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -126,8 +126,6 @@ var (
 var raceMode bool // set by race.go in race mode
 
 type testServer struct {
-	testpb.UnimplementedTestServiceServer
-
 	security           string // indicate the authentication protocol used by this server.
 	earlyFail          bool   // whether to error out the execution of a service handler prematurely.
 	setAndSendHeader   bool   // whether to call setHeader and sendHeader.
@@ -136,6 +134,8 @@ type testServer struct {
 	unaryCallSleepTime time.Duration
 }
 
+var _ testpb.UnstableTestServiceService = (*testServer)(nil)
+
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 	if md, ok := metadata.FromIncomingContext(ctx); ok {
 		// For testing purpose, returns an error if user-agent is failAppUA.
@@ -566,7 +566,7 @@ func newTest(t *testing.T, e env) *test {
 	return te
 }
 
-func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network, address string) (net.Listener, error)) net.Listener {
+func (te *test) listenAndServe(ts interface{}, listen func(network, address string) (net.Listener, error)) net.Listener {
 	te.t.Helper()
 	te.t.Logf("Running test in %s environment...", te.e.name)
 	sopts := []grpc.ServerOption{grpc.MaxConcurrentStreams(te.maxStream)}
@@ -626,7 +626,7 @@ func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network,
 	sopts = append(sopts, te.customServerOptions...)
 	s := grpc.NewServer(sopts...)
 	if ts != nil {
-		testpb.RegisterTestServiceServer(s, ts)
+		testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
 	}
 
 	// Create a new default health server if enableHealthServer is set, or use
@@ -691,20 +691,20 @@ func (w wrapHS) Stop() {
 	w.s.Close()
 }
 
-func (te *test) startServerWithConnControl(ts testpb.TestServiceServer) *listenerWrapper {
+func (te *test) startServerWithConnControl(ts interface{}) *listenerWrapper {
 	l := te.listenAndServe(ts, listenWithConnControl)
 	return l.(*listenerWrapper)
 }
 
 // startServer starts a gRPC server exposing the provided TestService
 // implementation. Callers should defer a call to te.tearDown to clean up
-func (te *test) startServer(ts testpb.TestServiceServer) {
+func (te *test) startServer(ts interface{}) {
 	te.t.Helper()
 	te.listenAndServe(ts, net.Listen)
 }
 
 // startServers starts 'num' gRPC servers exposing the provided TestService.
-func (te *test) startServers(ts testpb.TestServiceServer, num int) {
+func (te *test) startServers(ts interface{}, num int) {
 	for i := 0; i < num; i++ {
 		te.startServer(ts)
 		te.srvs = append(te.srvs, te.srv.(*grpc.Server))
@@ -3892,15 +3892,13 @@ func equalError(x, y error) bool {
 	return x == y || (x != nil && y != nil && x.Error() == y.Error())
 }
 
-// concurrentSendServer is a TestServiceServer whose
+// concurrentSendServer is a TestServiceService whose
 // StreamingOutputCall makes ten serial Send calls, sending payloads
 // "0".."9", inclusive.  TestServerStreamingConcurrent verifies they
 // were received in the correct order, and that there were no races.
 //
-// All other TestServiceServer methods crash if called.
-type concurrentSendServer struct {
-	testpb.TestServiceServer
-}
+// All other TestServiceService methods return unimplemented if called.
+type concurrentSendServer struct{}
 
 func (s concurrentSendServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
 	for i := 0; i < 10; i++ {
@@ -4496,12 +4494,11 @@ func testStreamServerInterceptor(t *testing.T, e env) {
 	}
 }
 
-// funcServer implements methods of TestServiceServer using funcs,
+// funcServer implements methods of TestServiceService using funcs,
 // similar to an http.HandlerFunc.
-// Any unimplemented method will crash. Tests implement the method(s)
+// Any unimplemented method will return unimplemented. Tests implement the method(s)
 // they need.
 type funcServer struct {
-	testpb.TestServiceServer
 	unaryCall          func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
 	streamingInputCall func(stream testpb.TestService_StreamingInputCallServer) error
 	fullDuplexCall     func(stream testpb.TestService_FullDuplexCallServer) error
@@ -4872,10 +4869,10 @@ func (s) TestFlowControlLogicalRace(t *testing.T) {
 	defer lis.Close()
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceServer(s, &flowControlLogicalRaceServer{
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&flowControlLogicalRaceServer{
 		itemCount: itemCount,
 		itemSize:  itemSize,
-	})
+	}))
 	defer s.Stop()
 
 	go s.Serve(lis)
@@ -4929,8 +4926,6 @@ func (s) TestFlowControlLogicalRace(t *testing.T) {
 }
 
 type flowControlLogicalRaceServer struct {
-	testpb.TestServiceServer
-
 	itemSize  int
 	itemCount int
 }
@@ -5077,9 +5072,6 @@ func (fw *filterWriter) Write(p []byte) (n int, err error) {
 // stubServer is a server that is easy to customize within individual test
 // cases.
 type stubServer struct {
-	// Guarantees we satisfy this interface; panics if unimplemented methods are called.
-	testpb.TestServiceServer
-
 	// Customizable implementations of server handlers.
 	emptyCall      func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error)
 	unaryCall      func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
@@ -5133,7 +5125,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption)
 	ss.cleanups = append(ss.cleanups, func() { lis.Close() })
 
 	s := grpc.NewServer(sopts...)
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 	go s.Serve(lis)
 	ss.cleanups = append(ss.cleanups, s.Stop)
 	ss.s = s
@@ -6276,7 +6268,7 @@ func (s) TestServeExitsWhenListenerClosed(t *testing.T) {
 
 	s := grpc.NewServer()
 	defer s.Stop()
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -6495,7 +6487,7 @@ func (s) TestDisabledIOBuffers(t *testing.T) {
 	}
 
 	s := grpc.NewServer(grpc.WriteBufferSize(0), grpc.ReadBufferSize(0))
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -6708,7 +6700,7 @@ func (s) TestNetPipeConn(t *testing.T) {
 	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		return &testpb.SimpleResponse{}, nil
 	}}
-	testpb.RegisterTestServiceServer(s, ts)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
 	go s.Serve(pl)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -6791,7 +6783,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 			return err
 		},
 	}
-	testpb.RegisterTestServiceServer(s1, ts)
+	testpb.RegisterTestServiceService(s1, testpb.NewTestServiceService(ts))
 	go s1.Serve(lis1)
 
 	conn2Established := grpcsync.NewEvent()
@@ -6801,7 +6793,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	}
 	s2 := grpc.NewServer()
 	defer s2.Stop()
-	testpb.RegisterTestServiceServer(s2, ts)
+	testpb.RegisterTestServiceService(s2, testpb.NewTestServiceService(ts))
 	go s2.Serve(lis2)
 
 	r := manual.NewBuilderWithScheme("whatever")
diff --git a/test/goaway_test.go b/test/goaway_test.go
index 55f79ebc8548..93d964803c42 100644
--- a/test/goaway_test.go
+++ b/test/goaway_test.go
@@ -48,7 +48,7 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) {
 
 	s := grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge}))
 	defer s.Stop()
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
diff --git a/test/gracefulstop_test.go b/test/gracefulstop_test.go
index 3da75ea1b51b..f7659524d1d8 100644
--- a/test/gracefulstop_test.go
+++ b/test/gracefulstop_test.go
@@ -117,7 +117,7 @@ func (s) TestGracefulStop(t *testing.T) {
 		},
 	}
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 
 	// 1. Start Server
 	wg := sync.WaitGroup{}
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index 2340ac05e890..2ab12ac94210 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -11,7 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
+const _ = grpc.SupportPackageIsVersion7
 
 // TestServiceClient is the client API for TestService service.
 //
@@ -47,6 +47,10 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
 	return &testServiceClient{cc}
 }
 
+var testServiceEmptyCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "EmptyCall",
+}
+
 func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
 	out := new(Empty)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
@@ -56,6 +60,10 @@ func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...gr
 	return out, nil
 }
 
+var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{
+	StreamName: "UnaryCall",
+}
+
 func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
@@ -65,8 +73,13 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
+var testServiceStreamingOutputCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "StreamingOutputCall",
+	ServerStreams: true,
+}
+
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceStreamingOutputCallStreamDesc, "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -97,8 +110,13 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo
 	return m, nil
 }
 
+var testServiceStreamingInputCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "StreamingInputCall",
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceStreamingInputCallStreamDesc, "/grpc.testing.TestService/StreamingInputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -131,8 +149,14 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal
 	return m, nil
 }
 
+var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "FullDuplexCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -162,8 +186,14 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
+var testServiceHalfDuplexCallStreamDesc = &grpc.StreamDesc{
+	StreamName:    "HalfDuplexCall",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
 func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, testServiceHalfDuplexCallStreamDesc, "/grpc.testing.TestService/HalfDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -193,101 +223,100 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
-// TestServiceServer is the server API for TestService service.
-// All implementations should embed UnimplementedTestServiceServer
-// for forward compatibility
-type TestServiceServer interface {
+// TestServiceService is the service API for TestService service.
+// Fields should be assigned to their respective handler implementations only before
+// RegisterTestServiceService is called.  Any unassigned fields will result in the
+// handler for that method returning an Unimplemented error.
+type TestServiceService struct {
 	// One empty request followed by one empty response.
-	EmptyCall(context.Context, *Empty) (*Empty, error)
+	EmptyCall func(context.Context, *Empty) (*Empty, error)
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by a sequence of responses (streamed download).
 	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	StreamingOutputCall func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
 	// A sequence of requests followed by one response (streamed upload).
 	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(TestService_StreamingInputCallServer) error
+	StreamingInputCall func(TestService_StreamingInputCallServer) error
 	// A sequence of requests with each request served by the server immediately.
 	// As one request could lead to multiple responses, this interface
 	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
+	FullDuplexCall func(TestService_FullDuplexCallServer) error
 	// A sequence of requests followed by a sequence of responses.
 	// The server buffers all the client requests and then serves them in order. A
 	// stream of responses are returned to the client when the server starts with
 	// first request.
-	HalfDuplexCall(TestService_HalfDuplexCallServer) error
-}
-
-// UnimplementedTestServiceServer should be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
+	HalfDuplexCall func(TestService_HalfDuplexCallServer) error
 }
 
-func (*UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-}
-func (*UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-}
-func (*UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (*UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-}
-
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
-}
-
-func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.EmptyCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+	}
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(TestServiceServer).EmptyCall(ctx, in)
+		return s.EmptyCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.TestService/EmptyCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
+		return s.EmptyCall(ctx, req.(*Empty))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	if s.UnaryCall == nil {
+		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
+		return s.UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     srv,
+		Server:     s,
 		FullMethod: "/grpc.testing.TestService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+		return s.UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-
-func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.StreamingOutputCall == nil {
+		return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+	}
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
+	return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
+}
+func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.StreamingInputCall == nil {
+		return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+	}
+	return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream})
+}
+func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.FullDuplexCall == nil {
+		return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+	}
+	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error {
+	if s.HalfDuplexCall == nil {
+		return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+	}
+	return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
 }
 
 type TestService_StreamingOutputCallServer interface {
@@ -303,10 +332,6 @@ func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallRespon
 	return x.ServerStream.SendMsg(m)
 }
 
-func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
-}
-
 type TestService_StreamingInputCallServer interface {
 	SendAndClose(*StreamingInputCallResponse) error
 	Recv() (*StreamingInputCallRequest, error)
@@ -329,10 +354,6 @@ func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest
 	return m, nil
 }
 
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-
 type TestService_FullDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -355,10 +376,6 @@ func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
-}
-
 type TestService_HalfDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -381,42 +398,114 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-var _TestService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "EmptyCall",
-			Handler:    _TestService_EmptyCall_Handler,
-		},
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingOutputCall",
-			Handler:       _TestService_StreamingOutputCall_Handler,
-			ServerStreams: true,
-		},
-		{
-			StreamName:    "StreamingInputCall",
-			Handler:       _TestService_StreamingInputCall_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
+// RegisterTestServiceService registers a service implementation with a gRPC server.
+func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
+	sd := grpc.ServiceDesc{
+		ServiceName: "grpc.testing.TestService",
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: "EmptyCall",
+				Handler:    srv.emptyCall,
+			},
+			{
+				MethodName: "UnaryCall",
+				Handler:    srv.unaryCall,
+			},
 		},
-		{
-			StreamName:    "HalfDuplexCall",
-			Handler:       _TestService_HalfDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
+		Streams: []grpc.StreamDesc{
+			{
+				StreamName:    "StreamingOutputCall",
+				Handler:       srv.streamingOutputCall,
+				ServerStreams: true,
+			},
+			{
+				StreamName:    "StreamingInputCall",
+				Handler:       srv.streamingInputCall,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "FullDuplexCall",
+				Handler:       srv.fullDuplexCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
+			{
+				StreamName:    "HalfDuplexCall",
+				Handler:       srv.halfDuplexCall,
+				ServerStreams: true,
+				ClientStreams: true,
+			},
 		},
-	},
-	Metadata: "test/grpc_testing/test.proto",
+		Metadata: "test/grpc_testing/test.proto",
+	}
+
+	s.RegisterService(&sd, nil)
+}
+
+// NewTestServiceService creates a new TestServiceService containing the
+// implemented methods of the TestService service in s.  Any unimplemented
+// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
+// This includes situations where the method handler is misspelled or has the wrong
+// signature.  For this reason, this function should be used with great care and
+// is not recommended to be used by most users.
+func NewTestServiceService(s interface{}) *TestServiceService {
+	ns := &TestServiceService{}
+	if h, ok := s.(interface {
+		EmptyCall(context.Context, *Empty) (*Empty, error)
+	}); ok {
+		ns.EmptyCall = h.EmptyCall
+	}
+	if h, ok := s.(interface {
+		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	}); ok {
+		ns.UnaryCall = h.UnaryCall
+	}
+	if h, ok := s.(interface {
+		StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	}); ok {
+		ns.StreamingOutputCall = h.StreamingOutputCall
+	}
+	if h, ok := s.(interface {
+		StreamingInputCall(TestService_StreamingInputCallServer) error
+	}); ok {
+		ns.StreamingInputCall = h.StreamingInputCall
+	}
+	if h, ok := s.(interface {
+		FullDuplexCall(TestService_FullDuplexCallServer) error
+	}); ok {
+		ns.FullDuplexCall = h.FullDuplexCall
+	}
+	if h, ok := s.(interface {
+		HalfDuplexCall(TestService_HalfDuplexCallServer) error
+	}); ok {
+		ns.HalfDuplexCall = h.HalfDuplexCall
+	}
+	return ns
+}
+
+// UnstableTestServiceService is the service API for TestService service.
+// New methods may be added to this interface if they are added to the service
+// definition, which is not a backward-compatible change.  For this reason,
+// use of this type is not recommended.
+type UnstableTestServiceService interface {
+	// One empty request followed by one empty response.
+	EmptyCall(context.Context, *Empty) (*Empty, error)
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by a sequence of responses (streamed download).
+	// The server returns the payload with client desired type and sizes.
+	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	// A sequence of requests followed by one response (streamed upload).
+	// The server returns the aggregated size of client payload as the result.
+	StreamingInputCall(TestService_StreamingInputCallServer) error
+	// A sequence of requests with each request served by the server immediately.
+	// As one request could lead to multiple responses, this interface
+	// demonstrates the idea of full duplexing.
+	FullDuplexCall(TestService_FullDuplexCallServer) error
+	// A sequence of requests followed by a sequence of responses.
+	// The server buffers all the client requests and then serves them in order. A
+	// stream of responses are returned to the client when the server starts with
+	// first request.
+	HalfDuplexCall(TestService_HalfDuplexCallServer) error
 }
diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go
index 0a60f8c927ac..0137494efb8f 100644
--- a/test/healthcheck_test.go
+++ b/test/healthcheck_test.go
@@ -140,7 +140,7 @@ func setupServer(sc *svrConfig) (s *grpc.Server, lis net.Listener, ts *testHealt
 		ts = newTestHealthServer()
 	}
 	healthgrpc.RegisterHealthServer(s, ts)
-	testpb.RegisterTestServiceServer(s, &testServer{})
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&testServer{}))
 	go s.Serve(lis)
 	return s, lis, ts, s.Stop, nil
 }
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index b55b73bdcbce..12af20b2b34c 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -64,7 +64,7 @@ func testLocalCredsE2ESucceed(network, address string) error {
 	s := grpc.NewServer(sopts...)
 	defer s.Stop()
 
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 
 	lis, err := net.Listen(network, address)
 	if err != nil {
@@ -162,7 +162,7 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
 	s := grpc.NewServer(sopts...)
 	defer s.Stop()
 
-	testpb.RegisterTestServiceServer(s, ss)
+	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {

From ff1fc890e43ac77e922f53a2cef396b3c6a8f2a1 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Tue, 25 Aug 2020 13:49:57 -0700
Subject: [PATCH 174/481] internal: reintroduce legacy appengine build
 constraints (#3838)

---
 internal/channelz/types_linux.go              |  2 +
 internal/channelz/types_nonlinux.go           |  2 +-
 internal/channelz/util_linux.go               |  2 +-
 internal/channelz/util_test.go                |  2 +-
 internal/credentials/spiffe.go                |  2 +
 internal/credentials/syscallconn.go           |  2 +
 internal/credentials/syscallconn_test.go      |  2 +
 internal/profiling/buffer/buffer.go           |  2 +
 internal/profiling/buffer/buffer_appengine.go | 43 +++++++++++++++
 internal/profiling/buffer/buffer_test.go      |  2 +
 internal/syscall/syscall_linux.go             |  2 +
 internal/syscall/syscall_nonlinux.go          | 14 ++---
 interop/grpclb_fallback/client.go             |  2 +-
 test/go_vet/vet.go                            | 53 +++++++++++++++++++
 vet.sh                                        |  4 ++
 15 files changed, 125 insertions(+), 11 deletions(-)
 create mode 100644 internal/profiling/buffer/buffer_appengine.go
 create mode 100644 test/go_vet/vet.go

diff --git a/internal/channelz/types_linux.go b/internal/channelz/types_linux.go
index 1b1c4cce34a9..692dd6181778 100644
--- a/internal/channelz/types_linux.go
+++ b/internal/channelz/types_linux.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/internal/channelz/types_nonlinux.go b/internal/channelz/types_nonlinux.go
index da7351e9709a..19c2fc521dcf 100644
--- a/internal/channelz/types_nonlinux.go
+++ b/internal/channelz/types_nonlinux.go
@@ -37,6 +37,6 @@ type SocketOptionData struct {
 // Windows OS doesn't support Socket Option
 func (s *SocketOptionData) Getsockopt(fd uintptr) {
 	once.Do(func() {
-		logger.Warning("Channelz: socket options are not supported on non-linux os.")
+		logger.Warning("Channelz: socket options are not supported on non-linux os and appengine.")
 	})
 }
diff --git a/internal/channelz/util_linux.go b/internal/channelz/util_linux.go
index 49432bebf5f4..fdf409d55de3 100644
--- a/internal/channelz/util_linux.go
+++ b/internal/channelz/util_linux.go
@@ -1,4 +1,4 @@
-// +build linux
+// +build linux,!appengine
 
 /*
  *
diff --git a/internal/channelz/util_test.go b/internal/channelz/util_test.go
index 2621e7410d61..3d1a1183fa41 100644
--- a/internal/channelz/util_test.go
+++ b/internal/channelz/util_test.go
@@ -1,4 +1,4 @@
-// +build linux
+// +build linux,!appengine
 
 /*
  *
diff --git a/internal/credentials/spiffe.go b/internal/credentials/spiffe.go
index 406023035288..b66e3dbb5ac2 100644
--- a/internal/credentials/spiffe.go
+++ b/internal/credentials/spiffe.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/internal/credentials/syscallconn.go b/internal/credentials/syscallconn.go
index 2919632d657e..f499a614c20e 100644
--- a/internal/credentials/syscallconn.go
+++ b/internal/credentials/syscallconn.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/internal/credentials/syscallconn_test.go b/internal/credentials/syscallconn_test.go
index b229a47d116e..ee17a0ca67bc 100644
--- a/internal/credentials/syscallconn_test.go
+++ b/internal/credentials/syscallconn_test.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/internal/profiling/buffer/buffer.go b/internal/profiling/buffer/buffer.go
index f4cd4201de11..45745cd09197 100644
--- a/internal/profiling/buffer/buffer.go
+++ b/internal/profiling/buffer/buffer.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/internal/profiling/buffer/buffer_appengine.go b/internal/profiling/buffer/buffer_appengine.go
new file mode 100644
index 000000000000..c92599e5b9c0
--- /dev/null
+++ b/internal/profiling/buffer/buffer_appengine.go
@@ -0,0 +1,43 @@
+// +build appengine
+
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package buffer
+
+// CircularBuffer is a no-op implementation for appengine builds.
+//
+// Appengine does not support stats because of lack of the support for unsafe
+// pointers, which are necessary to efficiently store and retrieve things into
+// and from a circular buffer. As a result, Push does not do anything and Drain
+// returns an empty slice.
+type CircularBuffer struct{}
+
+// NewCircularBuffer returns a no-op for appengine builds.
+func NewCircularBuffer(size uint32) (*CircularBuffer, error) {
+	return nil, nil
+}
+
+// Push returns a no-op for appengine builds.
+func (cb *CircularBuffer) Push(x interface{}) {
+}
+
+// Drain returns a no-op for appengine builds.
+func (cb *CircularBuffer) Drain() []interface{} {
+	return nil
+}
diff --git a/internal/profiling/buffer/buffer_test.go b/internal/profiling/buffer/buffer_test.go
index a7f3b61e4afa..86bd77d4a2e6 100644
--- a/internal/profiling/buffer/buffer_test.go
+++ b/internal/profiling/buffer/buffer_test.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go
index 3743bd45afbc..2fdcb76e6c74 100644
--- a/internal/syscall/syscall_linux.go
+++ b/internal/syscall/syscall_linux.go
@@ -1,3 +1,5 @@
+// +build !appengine
+
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go
index 51159e638d77..adae60d65188 100644
--- a/internal/syscall/syscall_nonlinux.go
+++ b/internal/syscall/syscall_nonlinux.go
@@ -35,40 +35,40 @@ var logger = grpclog.Component("core")
 
 func log() {
 	once.Do(func() {
-		logger.Info("CPU time info is unavailable on non-linux environment.")
+		logger.Info("CPU time info is unavailable on non-linux or appengine environment.")
 	})
 }
 
 // GetCPUTime returns the how much CPU time has passed since the start of this process.
-// It always returns 0 under non-linux environment.
+// It always returns 0 under non-linux or appengine environment.
 func GetCPUTime() int64 {
 	log()
 	return 0
 }
 
-// Rusage is an empty struct under non-linux environment.
+// Rusage is an empty struct under non-linux or appengine environment.
 type Rusage struct{}
 
-// GetRusage is a no-op function under non-linux environment.
+// GetRusage is a no-op function under non-linux or appengine environment.
 func GetRusage() (rusage *Rusage) {
 	log()
 	return nil
 }
 
 // CPUTimeDiff returns the differences of user CPU time and system CPU time used
-// between two Rusage structs. It a no-op function for non-linux environment.
+// between two Rusage structs. It a no-op function for non-linux or appengine environment.
 func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) {
 	log()
 	return 0, 0
 }
 
-// SetTCPUserTimeout is a no-op function under non-linux environments
+// SetTCPUserTimeout is a no-op function under non-linux or appengine environments
 func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error {
 	log()
 	return nil
 }
 
-// GetTCPUserTimeout is a no-op function under non-linux environments
+// GetTCPUserTimeout is a no-op function under non-linux or appengine environments
 // a negative return value indicates the operation is not supported
 func GetTCPUserTimeout(conn net.Conn) (int, error) {
 	log()
diff --git a/interop/grpclb_fallback/client.go b/interop/grpclb_fallback/client.go
index 853c9a4353f1..262d7458aba5 100644
--- a/interop/grpclb_fallback/client.go
+++ b/interop/grpclb_fallback/client.go
@@ -1,4 +1,4 @@
-// +build linux
+// +build linux,!appengine
 
 /*
  *
diff --git a/test/go_vet/vet.go b/test/go_vet/vet.go
new file mode 100644
index 000000000000..475e8d683fc3
--- /dev/null
+++ b/test/go_vet/vet.go
@@ -0,0 +1,53 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// vet checks whether files that are supposed to be built on appengine running
+// Go 1.10 or earlier import an unsupported package (e.g. "unsafe", "syscall").
+package main
+
+import (
+	"fmt"
+	"go/build"
+	"os"
+)
+
+func main() {
+	fail := false
+	b := build.Default
+	b.BuildTags = []string{"appengine", "appenginevm"}
+	argsWithoutProg := os.Args[1:]
+	for _, dir := range argsWithoutProg {
+		p, err := b.Import(".", dir, 0)
+		if _, ok := err.(*build.NoGoError); ok {
+			continue
+		} else if err != nil {
+			fmt.Printf("build.Import failed due to %v\n", err)
+			fail = true
+			continue
+		}
+		for _, pkg := range p.Imports {
+			if pkg == "syscall" || pkg == "unsafe" {
+				fmt.Printf("Package %s/%s importing %s package without appengine build tag is NOT ALLOWED!\n", p.Dir, p.Name, pkg)
+				fail = true
+			}
+		}
+	}
+	if fail {
+		os.Exit(1)
+	}
+}
diff --git a/vet.sh b/vet.sh
index c52c1bdbf4c9..f644d832d232 100755
--- a/vet.sh
+++ b/vet.sh
@@ -92,6 +92,10 @@ not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go"
 # - Ensure all xds proto imports are renamed to *pb or *grpc.
 git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' ':(exclude)*.pb.go' | not grep -v 'pb "\|grpc "'
 
+# - Check imports that are illegal in appengine (until Go 1.11).
+# TODO: Remove when we drop Go 1.10 support
+go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go
+
 # - gofmt, goimports, golint (with exceptions for generated code), go vet.
 gofmt -s -d -l . 2>&1 | fail_on_output
 goimports -l . 2>&1 | not grep -vE "\.pb\.go"

From c4ba4cc6af4a2000d2891592bd8a309a9cb46790 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 25 Aug 2020 14:41:59 -0700
Subject: [PATCH 175/481] Change version to 1.33.0-dev (#3846)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index 674c12f9a7e1..e5ce470f0c6b 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.32.0-dev"
+const Version = "1.33.0-dev"

From cc8e63cae1d5fb57a2beff5fa75feb665901f490 Mon Sep 17 00:00:00 2001
From: Elizabeth Zou <ezou@google.com>
Date: Wed, 26 Aug 2020 13:44:05 -0500
Subject: [PATCH 176/481] security/authorization: add integration tests (#3809)

* security/authorization: integration tests for using CEL engine library
---
 security/authorization/engine/engine.go       |   7 +-
 security/authorization/engine/engine_test.go  | 242 ++++++++++++++----
 .../authorization/{util => engine}/util.go    |   8 +
 .../{util => engine}/util_test.go             |   9 -
 security/authorization/go.mod                 |   1 +
 5 files changed, 205 insertions(+), 62 deletions(-)
 rename security/authorization/{util => engine}/util.go (86%)
 rename security/authorization/{util => engine}/util_test.go (96%)

diff --git a/security/authorization/engine/engine.go b/security/authorization/engine/engine.go
index d598ff409fa6..596382e49b8d 100644
--- a/security/authorization/engine/engine.go
+++ b/security/authorization/engine/engine.go
@@ -22,10 +22,10 @@ import (
 	"strconv"
 
 	pb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2"
-	cel "github.com/google/cel-go/cel"
+	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/checker/decls"
-	types "github.com/google/cel-go/common/types"
-	interpreter "github.com/google/cel-go/interpreter"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/interpreter"
 	expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/metadata"
@@ -324,6 +324,7 @@ func NewAuthorizationEngine(allow, deny *pb.RBAC) (*AuthorizationEngine, error)
 			decls.NewVar("destination.address", decls.String),
 			decls.NewVar("destination.port", decls.Int),
 			decls.NewVar("connection.uri_san_peer_certificate", decls.String),
+			decls.NewVar("source.principal", decls.String),
 		),
 	)
 	if err != nil {
diff --git a/security/authorization/engine/engine_test.go b/security/authorization/engine/engine_test.go
index a87a7f53d858..c159c4bd5c21 100644
--- a/security/authorization/engine/engine_test.go
+++ b/security/authorization/engine/engine_test.go
@@ -22,21 +22,30 @@ import (
 	"testing"
 
 	pb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2"
-	cel "github.com/google/cel-go/cel"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
-	interpreter "github.com/google/cel-go/interpreter"
+	"github.com/google/cel-go/interpreter"
+	"github.com/google/go-cmp/cmp"
+	expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 )
 
-type programMock struct {
+type s struct {
+	grpctest.Tester
+}
+
+type fakeProgram struct {
 	out ref.Val
 	err error
 }
 
-func (mock programMock) Eval(vars interface{}) (ref.Val, *cel.EvalDetails, error) {
-	return mock.out, nil, mock.err
+func (fake fakeProgram) Eval(vars interface{}) (ref.Val, *cel.EvalDetails, error) {
+	return fake.out, nil, fake.err
 }
 
 type valMock struct {
@@ -66,12 +75,24 @@ func (mock valMock) Value() interface{} {
 	return mock.val
 }
 
+type addrMock struct {
+	addr string
+}
+
+func (mock addrMock) Network() string {
+	return "tcp"
+}
+
+func (mock addrMock) String() string {
+	return mock.addr
+}
+
 var (
 	emptyActivation     = interpreter.EmptyActivation()
-	unsuccessfulProgram = programMock{out: nil, err: status.Errorf(codes.InvalidArgument, "Unsuccessful program evaluation")}
-	errProgram          = programMock{out: valMock{"missing attributes"}, err: status.Errorf(codes.InvalidArgument, "Successful program evaluation to an error result -- missing attributes")}
-	trueProgram         = programMock{out: valMock{true}, err: nil}
-	falseProgram        = programMock{out: valMock{false}, err: nil}
+	unsuccessfulProgram = fakeProgram{out: nil, err: status.Errorf(codes.InvalidArgument, "Unsuccessful program evaluation")}
+	errProgram          = fakeProgram{out: valMock{"missing attributes"}, err: status.Errorf(codes.InvalidArgument, "Successful program evaluation to an error result -- missing attributes")}
+	trueProgram         = fakeProgram{out: valMock{true}, err: nil}
+	falseProgram        = fakeProgram{out: valMock{false}, err: nil}
 
 	allowMatchEngine = &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{
 		"allow match policy1": unsuccessfulProgram,
@@ -92,41 +113,37 @@ var (
 	}}
 )
 
-func TestNewAuthorizationEngine(t *testing.T) {
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestNewAuthorizationEngine(t *testing.T) {
 	tests := map[string]struct {
 		allow   *pb.RBAC
 		deny    *pb.RBAC
-		wantErr string
+		wantErr bool
 		errStr  string
 	}{
 		"too few rbacs": {
-			allow:   nil,
-			deny:    nil,
-			wantErr: "at least one of allow, deny must be non-nil",
+			wantErr: true,
 			errStr:  "Expected error: at least one of allow, deny must be non-nil",
 		},
 		"one rbac allow": {
-			allow:   &pb.RBAC{Action: pb.RBAC_ALLOW},
-			deny:    nil,
-			wantErr: "",
-			errStr:  "Expected 1 ALLOW RBAC to be successful",
+			allow:  &pb.RBAC{Action: pb.RBAC_ALLOW},
+			errStr: "Expected 1 ALLOW RBAC to be successful",
 		},
 		"one rbac deny": {
-			allow:   nil,
-			deny:    &pb.RBAC{Action: pb.RBAC_DENY},
-			wantErr: "",
-			errStr:  "Expected 1 DENY RBAC to be successful",
+			deny:   &pb.RBAC{Action: pb.RBAC_DENY},
+			errStr: "Expected 1 DENY RBAC to be successful",
 		},
 		"two rbacs": {
-			allow:   &pb.RBAC{Action: pb.RBAC_ALLOW},
-			deny:    &pb.RBAC{Action: pb.RBAC_DENY},
-			wantErr: "",
-			errStr:  "Expected 2 RBACs (DENY + ALLOW) to be successful",
+			allow:  &pb.RBAC{Action: pb.RBAC_ALLOW},
+			deny:   &pb.RBAC{Action: pb.RBAC_DENY},
+			errStr: "Expected 2 RBACs (DENY + ALLOW) to be successful",
 		},
 		"wrong rbac actions": {
 			allow:   &pb.RBAC{Action: pb.RBAC_DENY},
-			deny:    nil,
-			wantErr: "allow must have action ALLOW, deny must have action DENY",
+			wantErr: true,
 			errStr:  "Expected error: allow must have action ALLOW, deny must have action DENY",
 		},
 	}
@@ -134,17 +151,14 @@ func TestNewAuthorizationEngine(t *testing.T) {
 	for name, tc := range tests {
 		t.Run(name, func(t *testing.T) {
 			_, gotErr := NewAuthorizationEngine(tc.allow, tc.deny)
-			if tc.wantErr == "" && gotErr == nil {
-				return
-			}
-			if gotErr == nil || gotErr.Error() != tc.wantErr {
-				t.Errorf(tc.errStr)
+			if (gotErr != nil) != tc.wantErr {
+				t.Fatal(tc.errStr)
 			}
 		})
 	}
 }
 
-func TestGetDecision(t *testing.T) {
+func (s) TestGetDecision(t *testing.T) {
 	tests := map[string]struct {
 		engine *policyEngine
 		match  bool
@@ -157,7 +171,6 @@ func TestGetDecision(t *testing.T) {
 		},
 		"ALLOW engine fail": {
 			engine: &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{}},
-			match:  false,
 			want:   DecisionDeny,
 		},
 		"DENY engine match": {
@@ -167,7 +180,6 @@ func TestGetDecision(t *testing.T) {
 		},
 		"DENY engine fail": {
 			engine: &policyEngine{action: pb.RBAC_DENY, programs: map[string]cel.Program{}},
-			match:  false,
 			want:   DecisionAllow,
 		},
 	}
@@ -175,13 +187,13 @@ func TestGetDecision(t *testing.T) {
 	for name, tc := range tests {
 		t.Run(name, func(t *testing.T) {
 			if got := getDecision(tc.engine, tc.match); got != tc.want {
-				t.Errorf("Expected %v, instead got %v", tc.want, got)
+				t.Fatalf("getDecision(%v, %v) = (%v), want (%v)", tc.engine, tc.match, got, tc.want)
 			}
 		})
 	}
 }
 
-func TestPolicyEngineEvaluate(t *testing.T) {
+func (s) TestPolicyEngineEvaluate(t *testing.T) {
 	tests := map[string]struct {
 		engine          *policyEngine
 		activation      interpreter.Activation
@@ -218,43 +230,38 @@ func TestPolicyEngineEvaluate(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			gotDecision, gotPolicyNames := tc.engine.evaluate(tc.activation)
 			sort.Strings(gotPolicyNames)
-			if gotDecision != tc.wantDecision || !reflect.DeepEqual(gotPolicyNames, tc.wantPolicyNames) {
-				t.Errorf("Expected (%v, %v), instead got (%v, %v)", tc.wantDecision, tc.wantPolicyNames, gotDecision, gotPolicyNames)
+			if gotDecision != tc.wantDecision || !cmp.Equal(gotPolicyNames, tc.wantPolicyNames) {
+				t.Fatalf("policyEngine.evaluate(%v, %v) = (%v, %v), want (%v, %v)", tc.engine, tc.activation, gotDecision, gotPolicyNames, tc.wantDecision, tc.wantPolicyNames)
 			}
 		})
 	}
 }
 
-func TestAuthorizationEngineEvaluate(t *testing.T) {
+func (s) TestAuthorizationEngineEvaluate(t *testing.T) {
 	tests := map[string]struct {
 		engine           *AuthorizationEngine
 		authArgs         *AuthorizationArgs
 		wantAuthDecision *AuthorizationDecision
-		wantErr          error
 	}{
 		"allow match": {
 			engine:           &AuthorizationEngine{allow: allowMatchEngine},
 			authArgs:         &AuthorizationArgs{},
 			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"allow match policy2"}},
-			wantErr:          nil,
 		},
 		"deny fail": {
 			engine:           &AuthorizationEngine{deny: denyFailEngine},
 			authArgs:         &AuthorizationArgs{},
 			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{}},
-			wantErr:          nil,
 		},
 		"first engine unknown": {
 			engine:           &AuthorizationEngine{allow: allowMatchEngine, deny: denyUnknownEngine},
 			authArgs:         &AuthorizationArgs{},
 			wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"deny unknown policy2", "deny unknown policy3"}},
-			wantErr:          nil,
 		},
 		"second engine match": {
 			engine:           &AuthorizationEngine{allow: allowMatchEngine, deny: denyFailEngine},
 			authArgs:         &AuthorizationArgs{},
 			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"allow match policy2"}},
-			wantErr:          nil,
 		},
 	}
 
@@ -262,10 +269,145 @@ func TestAuthorizationEngineEvaluate(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			gotAuthDecision, gotErr := tc.engine.Evaluate(tc.authArgs)
 			sort.Strings(gotAuthDecision.policyNames)
-			if tc.wantErr != nil && (gotErr == nil || gotErr.Error() != tc.wantErr.Error()) {
-				t.Errorf("Expected error to be %v, instead got %v", tc.wantErr, gotErr)
-			} else if tc.wantErr == nil && (gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !reflect.DeepEqual(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames)) {
-				t.Errorf("Expected authorization decision to be (%v, %v), instead got (%v, %v)", tc.wantAuthDecision.decision, tc.wantAuthDecision.policyNames, gotAuthDecision.decision, gotAuthDecision.policyNames)
+			if gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !cmp.Equal(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames) {
+				t.Fatalf("AuthorizationEngine.Evaluate(%v, %v) = (%v, %v), want (%v, %v)", tc.engine, tc.authArgs, gotAuthDecision, gotErr, tc.wantAuthDecision, nil)
+			}
+		})
+	}
+}
+
+func (s) TestIntegration(t *testing.T) {
+	declarations := []*expr.Decl{
+		decls.NewVar("request.url_path", decls.String),
+		decls.NewVar("request.host", decls.String),
+		decls.NewVar("request.method", decls.String),
+		decls.NewVar("request.headers", decls.NewMapType(decls.String, decls.String)),
+		decls.NewVar("source.address", decls.String),
+		decls.NewVar("source.port", decls.Int),
+		decls.NewVar("destination.address", decls.String),
+		decls.NewVar("destination.port", decls.Int),
+		decls.NewVar("connection.uri_san_peer_certificate", decls.String),
+		decls.NewVar("source.principal", decls.String),
+	}
+
+	tests := map[string]struct {
+		allow            *pb.RBAC
+		deny             *pb.RBAC
+		authArgs         *AuthorizationArgs
+		wantAuthDecision *AuthorizationDecision
+	}{
+		"ALLOW engine: DecisionAllow": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path starts with": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test')", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"url_path starts with"}},
+		},
+		"ALLOW engine: DecisionUnknown": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path and uri_san_peer_certificate": {Condition: compileStringToExpr("request.url_path == '/pkg.service/test' && connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'", declarations)},
+				"source port":                           {Condition: compileStringToExpr("source.port == 8080", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:25"}}, fullMethod: "/pkg.service/test"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"url_path and uri_san_peer_certificate"}},
+		},
+		"ALLOW engine: DecisionDeny": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path": {Condition: compileStringToExpr("request.url_path == '/pkg.service/test'", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{}},
+		},
+		"DENY engine: DecisionAllow": {
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"url_path and uri_san_peer_certificate": {Condition: compileStringToExpr("request.url_path == '/pkg.service/test' && connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{}},
+		},
+		"DENY engine: DecisionUnknown": {
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"destination address": {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)},
+				"source port":         {Condition: compileStringToExpr("source.port == 8080", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:25"}}, fullMethod: "/pkg.service/test"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"destination address"}},
+		},
+		"DENY engine: DecisionDeny": {
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"destination address":           {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)},
+				"source address or source port": {Condition: compileStringToExpr("source.address == '192.0.4.1' || source.port == 8080", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{"source address or source port"}},
+		},
+		"DENY ALLOW engine: DecisionDeny from DENY policy": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path starts with": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test')", declarations)},
+			}},
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"destination address":           {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)},
+				"source address or source port": {Condition: compileStringToExpr("source.address == '192.0.4.1' || source.port == 8080", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{"source address or source port"}},
+		},
+		"DENY ALLOW engine: DecisionUnknown from DENY policy": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path starts with": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test')", declarations)},
+			}},
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"destination address":              {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)},
+				"source port and destination port": {Condition: compileStringToExpr("source.port == 8080 && destination.port == 1234", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"destination address", "source port and destination port"}},
+		},
+		"DENY ALLOW engine: DecisionAllow from ALLOW policy": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"method or url_path starts with": {Condition: compileStringToExpr("request.method == 'POST' || request.url_path.startsWith('/pkg.service/test')", declarations)},
+			}},
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"source address":           {Condition: compileStringToExpr("source.address == '192.0.3.1'", declarations)},
+				"source port and url_path": {Condition: compileStringToExpr("source.port == 8080 && request.url_path == 'pkg.service/test'", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"method or url_path starts with"}},
+		},
+		"DENY ALLOW engine: DecisionUnknown from ALLOW policy": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path starts with and method": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test') && request.method == 'POST'", declarations)},
+			}},
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"source address":           {Condition: compileStringToExpr("source.address == '192.0.3.1'", declarations)},
+				"source port and url_path": {Condition: compileStringToExpr("source.port == 8080 && request.url_path == 'pkg.service/test'", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"url_path starts with and method"}},
+		},
+		"DENY ALLOW engine: DecisionDeny from ALLOW policy": {
+			allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{
+				"url_path starts with and source port": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test') && source.port == 1234", declarations)},
+			}},
+			deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{
+				"source address":           {Condition: compileStringToExpr("source.address == '192.0.3.1'", declarations)},
+				"source port and url_path": {Condition: compileStringToExpr("source.port == 8080 && request.url_path == 'pkg.service/test'", declarations)},
+			}},
+			authArgs:         &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"},
+			wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{}},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			engine, err := NewAuthorizationEngine(tc.allow, tc.deny)
+			if err != nil {
+				t.Fatalf("Error constructing authorization engine: %v", err)
+			}
+			gotAuthDecision, gotErr := engine.Evaluate(tc.authArgs)
+			sort.Strings(gotAuthDecision.policyNames)
+			if gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !cmp.Equal(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames) {
+				t.Fatalf("NewAuthorizationEngine(%v, %v).Evaluate(%v) = (%v, %v), want (%v, %v)", tc.allow, tc.deny, tc.authArgs, gotAuthDecision, gotErr, tc.wantAuthDecision, nil)
 			}
 		})
 	}
diff --git a/security/authorization/util/util.go b/security/authorization/engine/util.go
similarity index 86%
rename from security/authorization/util/util.go
rename to security/authorization/engine/util.go
index 8aadad3862ba..3d1690ac2bff 100644
--- a/security/authorization/util/util.go
+++ b/security/authorization/engine/util.go
@@ -61,3 +61,11 @@ func compileStringToCheckedExpr(expr string, declarations []*expr.Decl) (*expr.C
 	}
 	return checkedExpr, nil
 }
+
+func compileStringToExpr(expr string, declarations []*expr.Decl) *expr.Expr {
+	checkedExpr, err := compileStringToCheckedExpr(expr, declarations)
+	if err != nil {
+		logger.Fatalf("error encountered when compiling string to expression: %v", err)
+	}
+	return checkedExpr.Expr
+}
diff --git a/security/authorization/util/util_test.go b/security/authorization/engine/util_test.go
similarity index 96%
rename from security/authorization/util/util_test.go
rename to security/authorization/engine/util_test.go
index 981a00864d58..e766fbf3ffe0 100644
--- a/security/authorization/util/util_test.go
+++ b/security/authorization/engine/util_test.go
@@ -22,20 +22,11 @@ import (
 	"testing"
 
 	expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
-	"google.golang.org/grpc/internal/grpctest"
 
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/checker/decls"
 )
 
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
 func (s) TestStringConvert(t *testing.T) {
 	declarations := []*expr.Decl{
 		decls.NewIdent("request.url_path", decls.String, nil),
diff --git a/security/authorization/go.mod b/security/authorization/go.mod
index 075cbe9158c0..0581b3401f32 100644
--- a/security/authorization/go.mod
+++ b/security/authorization/go.mod
@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/envoyproxy/go-control-plane v0.9.5
 	github.com/google/cel-go v0.5.1
+	github.com/google/go-cmp v0.5.0
 	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
 	google.golang.org/grpc v1.31.0
 	google.golang.org/protobuf v1.25.0

From d31b6710005da7543c258e246f80a47614657492 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 26 Aug 2020 16:05:36 -0700
Subject: [PATCH 177/481] cmd/protoc-gen-go-grpc: call interceptor even if
 handler is unset (#3849)

---
 .../grpc_lb_v1/load_balancer_grpc.pb.go       | 11 ++-
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       | 11 ++-
 benchmark/grpc_testing/services_grpc.pb.go    | 72 +++++++++-------
 channelz/grpc_channelz_v1/channelz_grpc.pb.go | 71 +++++++++-------
 cmd/protoc-gen-go-grpc/grpc.go                | 64 +++++++-------
 .../proto/grpc_gcp/handshaker_grpc.pb.go      | 11 ++-
 .../meshca/internal/v1/meshca_grpc.pb.go      | 11 ++-
 examples/features/proto/echo/echo_grpc.pb.go  | 41 +++++----
 .../helloworld/helloworld_grpc.pb.go          | 11 ++-
 .../routeguide/route_guide_grpc.pb.go         | 41 +++++----
 health/grpc_health_v1/health_grpc.pb.go       | 21 +++--
 interop/grpc_testing/test_grpc.pb.go          | 83 ++++++++++++-------
 profiling/proto/service_grpc.pb.go            | 21 +++--
 .../reflection_grpc.pb.go                     | 11 ++-
 reflection/grpc_testing/test_grpc.pb.go       | 21 +++--
 stats/grpc_testing/test_grpc.pb.go            | 41 +++++----
 stress/grpc_testing/metrics_grpc.pb.go        | 21 +++--
 test/grpc_testing/test_grpc.pb.go             | 61 ++++++++------
 18 files changed, 380 insertions(+), 244 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index 0079d0fad8df..e59bf309971a 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -22,21 +22,24 @@ type LoadBalancerService struct {
 }
 
 func (s *LoadBalancerService) balanceLoad(_ interface{}, stream grpc.ServerStream) error {
-	if s.BalanceLoad == nil {
-		return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
-	}
 	return s.BalanceLoad(&loadBalancerBalanceLoadServer{stream})
 }
 
 // RegisterLoadBalancerService registers a service implementation with a gRPC server.
 func RegisterLoadBalancerService(s grpc.ServiceRegistrar, srv *LoadBalancerService) {
+	srvCopy := *srv
+	if srvCopy.BalanceLoad == nil {
+		srvCopy.BalanceLoad = func(LoadBalancer_BalanceLoadServer) error {
+			return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.lb.v1.LoadBalancer",
 		Methods:     []grpc.MethodDesc{},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "BalanceLoad",
-				Handler:       srv.balanceLoad,
+				Handler:       srvCopy.balanceLoad,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index fe1027c71ec7..4a9f27067837 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -23,9 +23,6 @@ type RouteLookupServiceService struct {
 }
 
 func (s *RouteLookupServiceService) routeLookup(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.RouteLookup == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
-	}
 	in := new(RouteLookupRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -45,12 +42,18 @@ func (s *RouteLookupServiceService) routeLookup(_ interface{}, ctx context.Conte
 
 // RegisterRouteLookupServiceService registers a service implementation with a gRPC server.
 func RegisterRouteLookupServiceService(s grpc.ServiceRegistrar, srv *RouteLookupServiceService) {
+	srvCopy := *srv
+	if srvCopy.RouteLookup == nil {
+		srvCopy.RouteLookup = func(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.lookup.v1.RouteLookupService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "RouteLookup",
-				Handler:    srv.routeLookup,
+				Handler:    srvCopy.routeLookup,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index 60be6cd689a2..37d637d72148 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -140,9 +140,6 @@ type BenchmarkServiceService struct {
 }
 
 func (s *BenchmarkServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.UnaryCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -160,15 +157,9 @@ func (s *BenchmarkServiceService) unaryCall(_ interface{}, ctx context.Context,
 	return interceptor(ctx, in, info, handler)
 }
 func (s *BenchmarkServiceService) streamingCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.StreamingCall == nil {
-		return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
-	}
 	return s.StreamingCall(&benchmarkServiceStreamingCallServer{stream})
 }
 func (s *BenchmarkServiceService) unconstrainedStreamingCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.UnconstrainedStreamingCall == nil {
-		return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
-	}
 	return s.UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
 }
 
@@ -218,24 +209,40 @@ func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleReques
 
 // RegisterBenchmarkServiceService registers a service implementation with a gRPC server.
 func RegisterBenchmarkServiceService(s grpc.ServiceRegistrar, srv *BenchmarkServiceService) {
+	srvCopy := *srv
+	if srvCopy.UnaryCall == nil {
+		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+		}
+	}
+	if srvCopy.StreamingCall == nil {
+		srvCopy.StreamingCall = func(BenchmarkService_StreamingCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
+		}
+	}
+	if srvCopy.UnconstrainedStreamingCall == nil {
+		srvCopy.UnconstrainedStreamingCall = func(BenchmarkService_UnconstrainedStreamingCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.BenchmarkService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "UnaryCall",
-				Handler:    srv.unaryCall,
+				Handler:    srvCopy.unaryCall,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "StreamingCall",
-				Handler:       srv.streamingCall,
+				Handler:       srvCopy.streamingCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "UnconstrainedStreamingCall",
-				Handler:       srv.unconstrainedStreamingCall,
+				Handler:       srvCopy.unconstrainedStreamingCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
@@ -446,21 +453,12 @@ type WorkerServiceService struct {
 }
 
 func (s *WorkerServiceService) runServer(_ interface{}, stream grpc.ServerStream) error {
-	if s.RunServer == nil {
-		return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
-	}
 	return s.RunServer(&workerServiceRunServerServer{stream})
 }
 func (s *WorkerServiceService) runClient(_ interface{}, stream grpc.ServerStream) error {
-	if s.RunClient == nil {
-		return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
-	}
 	return s.RunClient(&workerServiceRunClientServer{stream})
 }
 func (s *WorkerServiceService) coreCount(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.CoreCount == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
-	}
 	in := new(CoreRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -478,9 +476,6 @@ func (s *WorkerServiceService) coreCount(_ interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 func (s *WorkerServiceService) quitWorker(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.QuitWorker == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
-	}
 	in := new(Void)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -544,28 +539,49 @@ func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) {
 
 // RegisterWorkerServiceService registers a service implementation with a gRPC server.
 func RegisterWorkerServiceService(s grpc.ServiceRegistrar, srv *WorkerServiceService) {
+	srvCopy := *srv
+	if srvCopy.RunServer == nil {
+		srvCopy.RunServer = func(WorkerService_RunServerServer) error {
+			return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
+		}
+	}
+	if srvCopy.RunClient == nil {
+		srvCopy.RunClient = func(WorkerService_RunClientServer) error {
+			return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
+		}
+	}
+	if srvCopy.CoreCount == nil {
+		srvCopy.CoreCount = func(context.Context, *CoreRequest) (*CoreResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
+		}
+	}
+	if srvCopy.QuitWorker == nil {
+		srvCopy.QuitWorker = func(context.Context, *Void) (*Void, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.WorkerService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "CoreCount",
-				Handler:    srv.coreCount,
+				Handler:    srvCopy.coreCount,
 			},
 			{
 				MethodName: "QuitWorker",
-				Handler:    srv.quitWorker,
+				Handler:    srvCopy.quitWorker,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "RunServer",
-				Handler:       srv.runServer,
+				Handler:       srvCopy.runServer,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "RunClient",
-				Handler:       srv.runClient,
+				Handler:       srvCopy.runClient,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index e4815814853e..bc6083b8519f 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -36,9 +36,6 @@ type ChannelzService struct {
 }
 
 func (s *ChannelzService) getTopChannels(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetTopChannels == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
-	}
 	in := new(GetTopChannelsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -56,9 +53,6 @@ func (s *ChannelzService) getTopChannels(_ interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ChannelzService) getServers(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetServers == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
-	}
 	in := new(GetServersRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -76,9 +70,6 @@ func (s *ChannelzService) getServers(_ interface{}, ctx context.Context, dec fun
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ChannelzService) getServer(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetServer == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
-	}
 	in := new(GetServerRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -96,9 +87,6 @@ func (s *ChannelzService) getServer(_ interface{}, ctx context.Context, dec func
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ChannelzService) getServerSockets(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetServerSockets == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
-	}
 	in := new(GetServerSocketsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -116,9 +104,6 @@ func (s *ChannelzService) getServerSockets(_ interface{}, ctx context.Context, d
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ChannelzService) getChannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetChannel == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
-	}
 	in := new(GetChannelRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -136,9 +121,6 @@ func (s *ChannelzService) getChannel(_ interface{}, ctx context.Context, dec fun
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ChannelzService) getSubchannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetSubchannel == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
-	}
 	in := new(GetSubchannelRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -156,9 +138,6 @@ func (s *ChannelzService) getSubchannel(_ interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ChannelzService) getSocket(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetSocket == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
-	}
 	in := new(GetSocketRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -178,36 +157,72 @@ func (s *ChannelzService) getSocket(_ interface{}, ctx context.Context, dec func
 
 // RegisterChannelzService registers a service implementation with a gRPC server.
 func RegisterChannelzService(s grpc.ServiceRegistrar, srv *ChannelzService) {
+	srvCopy := *srv
+	if srvCopy.GetTopChannels == nil {
+		srvCopy.GetTopChannels = func(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
+		}
+	}
+	if srvCopy.GetServers == nil {
+		srvCopy.GetServers = func(context.Context, *GetServersRequest) (*GetServersResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
+		}
+	}
+	if srvCopy.GetServer == nil {
+		srvCopy.GetServer = func(context.Context, *GetServerRequest) (*GetServerResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
+		}
+	}
+	if srvCopy.GetServerSockets == nil {
+		srvCopy.GetServerSockets = func(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
+		}
+	}
+	if srvCopy.GetChannel == nil {
+		srvCopy.GetChannel = func(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
+		}
+	}
+	if srvCopy.GetSubchannel == nil {
+		srvCopy.GetSubchannel = func(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
+		}
+	}
+	if srvCopy.GetSocket == nil {
+		srvCopy.GetSocket = func(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.channelz.v1.Channelz",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "GetTopChannels",
-				Handler:    srv.getTopChannels,
+				Handler:    srvCopy.getTopChannels,
 			},
 			{
 				MethodName: "GetServers",
-				Handler:    srv.getServers,
+				Handler:    srvCopy.getServers,
 			},
 			{
 				MethodName: "GetServer",
-				Handler:    srv.getServer,
+				Handler:    srvCopy.getServer,
 			},
 			{
 				MethodName: "GetServerSockets",
-				Handler:    srv.getServerSockets,
+				Handler:    srvCopy.getServerSockets,
 			},
 			{
 				MethodName: "GetChannel",
-				Handler:    srv.getChannel,
+				Handler:    srvCopy.getChannel,
 			},
 			{
 				MethodName: "GetSubchannel",
-				Handler:    srv.getSubchannel,
+				Handler:    srvCopy.getSubchannel,
 			},
 			{
 				MethodName: "GetSocket",
-				Handler:    srv.getSocket,
+				Handler:    srvCopy.getSocket,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 47503bdc0f42..13bd755e81b8 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -241,7 +241,7 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 		}
 		g.Annotate(serviceType+"."+method.GoName, method.Location)
 		g.P(method.Comments.Leading,
-			handlerSignature(g, method))
+			method.GoName, " func", handlerSignature(g, method))
 	}
 	g.P("}")
 	g.P()
@@ -270,6 +270,19 @@ func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.
 		g.P(deprecationComment)
 	}
 	g.P("func Register", service.GoName, "Service(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv *", service.GoName, "Service) {")
+	g.P("srvCopy := *srv")
+	// Add Unimplemented defaults for unset handlers
+	for _, method := range service.Methods {
+		g.P("if srvCopy.", method.GoName, " == nil {")
+		g.P("srvCopy.", method.GoName, " = func", handlerSignature(g, method), "{")
+		nilArg := ""
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			nilArg = "nil, "
+		}
+		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
+		g.P("}")
+		g.P("}")
+	}
 
 	// Service descriptor.
 	g.P("sd := ", grpcPackage.Ident("ServiceDesc"), " {")
@@ -281,7 +294,7 @@ func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.
 		}
 		g.P("{")
 		g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",")
-		g.P("Handler: srv.", unexport(method.GoName), ",")
+		g.P("Handler: srvCopy.", unexport(method.GoName), ",")
 		g.P("},")
 	}
 	g.P("},")
@@ -292,7 +305,7 @@ func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.
 		}
 		g.P("{")
 		g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
-		g.P("Handler: srv.", unexport(method.GoName), ",")
+		g.P("Handler: srvCopy.", unexport(method.GoName), ",")
 		if method.Desc.IsStreamingServer() {
 			g.P("ServerStreams: true,")
 		}
@@ -384,43 +397,34 @@ func handlerSignature(g *protogen.GeneratedFile, method *protogen.Method) string
 	if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
 		reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server")
 	}
-	return method.GoName + " func(" + strings.Join(reqArgs, ", ") + ") " + ret
+	return "(" + strings.Join(reqArgs, ", ") + ") " + ret
 }
 
-func unaryHandlerSignature(g *protogen.GeneratedFile) string {
-	return "(_ interface{}, ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context")) +
-		", dec func(interface{}) error, interceptor " + g.QualifiedGoIdent(grpcPackage.Ident("UnaryServerInterceptor")) + ") (interface{}, error)"
-}
-
-func streamHandlerSignature(g *protogen.GeneratedFile) string {
+func genericHandlerSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
+	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		// Unary
+		return "(_ interface{}, ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context")) +
+			", dec func(interface{}) error, interceptor " +
+			g.QualifiedGoIdent(grpcPackage.Ident("UnaryServerInterceptor")) + ") (interface{}, error)"
+	}
+	// Streaming
 	return "(_ interface{}, stream " + g.QualifiedGoIdent(grpcPackage.Ident("ServerStream")) + ") error"
 }
 
 func genMethodHandler(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
 	service := method.Parent
 
-	nilArg := ""
-	signature := streamHandlerSignature(g)
-	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-		nilArg = "nil,"
-		signature = unaryHandlerSignature(g)
-	}
-	g.P("func (s *", service.GoName, "Service) ", unexport(method.GoName), signature, " {")
-
-	g.P("if s.", method.GoName, " == nil {")
-	g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
-	g.P("}")
-	genHandlerBody(gen, g, method)
-
-	g.P("}")
-}
+	g.P("func (s *", service.GoName, "Service) ", unexport(method.GoName), genericHandlerSignature(g, method), " {")
 
-func genHandlerBody(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
-	service := method.Parent
+	// Unary
 	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
 		g.P("in := new(", method.Input.GoIdent, ")")
 		g.P("if err := dec(in); err != nil { return nil, err }")
-		g.P("if interceptor == nil { return s.", method.GoName, "(ctx, in) }")
+
+		g.P("if interceptor == nil {")
+		g.P("return s.", method.GoName, "(ctx, in)")
+		g.P("}")
+
 		g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{")
 		g.P("Server: s,")
 		g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.GoName)), ",")
@@ -429,8 +433,11 @@ func genHandlerBody(gen *protogen.Plugin, g *protogen.GeneratedFile, method *pro
 		g.P("return s.", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")
 		g.P("}")
 		g.P("return interceptor(ctx, in, info, handler)")
+		g.P("}")
 		return
 	}
+
+	// Streaming
 	streamType := unexport(service.GoName) + method.GoName + "Server"
 	if !method.Desc.IsStreamingClient() {
 		// Server-streaming
@@ -441,6 +448,7 @@ func genHandlerBody(gen *protogen.Plugin, g *protogen.GeneratedFile, method *pro
 		// Bidi-streaming
 		g.P("return s.", method.GoName, "(&", streamType, "{stream})")
 	}
+	g.P("}")
 }
 
 func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index bf7ff0a7bb76..d6f3af248f08 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -27,21 +27,24 @@ type HandshakerServiceService struct {
 }
 
 func (s *HandshakerServiceService) doHandshake(_ interface{}, stream grpc.ServerStream) error {
-	if s.DoHandshake == nil {
-		return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
-	}
 	return s.DoHandshake(&handshakerServiceDoHandshakeServer{stream})
 }
 
 // RegisterHandshakerServiceService registers a service implementation with a gRPC server.
 func RegisterHandshakerServiceService(s grpc.ServiceRegistrar, srv *HandshakerServiceService) {
+	srvCopy := *srv
+	if srvCopy.DoHandshake == nil {
+		srvCopy.DoHandshake = func(HandshakerService_DoHandshakeServer) error {
+			return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.gcp.HandshakerService",
 		Methods:     []grpc.MethodDesc{},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "DoHandshake",
-				Handler:       srv.doHandshake,
+				Handler:       srvCopy.doHandshake,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index a063519dab8c..3a82e6a31131 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -24,9 +24,6 @@ type MeshCertificateServiceService struct {
 }
 
 func (s *MeshCertificateServiceService) createCertificate(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.CreateCertificate == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
-	}
 	in := new(MeshCertificateRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -46,12 +43,18 @@ func (s *MeshCertificateServiceService) createCertificate(_ interface{}, ctx con
 
 // RegisterMeshCertificateServiceService registers a service implementation with a gRPC server.
 func RegisterMeshCertificateServiceService(s grpc.ServiceRegistrar, srv *MeshCertificateServiceService) {
+	srvCopy := *srv
+	if srvCopy.CreateCertificate == nil {
+		srvCopy.CreateCertificate = func(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "google.security.meshca.v1.MeshCertificateService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "CreateCertificate",
-				Handler:    srv.createCertificate,
+				Handler:    srvCopy.createCertificate,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index a3597048577a..d1d6cf07eb24 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -177,9 +177,6 @@ type EchoService struct {
 }
 
 func (s *EchoService) unaryEcho(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.UnaryEcho == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
-	}
 	in := new(EchoRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -197,9 +194,6 @@ func (s *EchoService) unaryEcho(_ interface{}, ctx context.Context, dec func(int
 	return interceptor(ctx, in, info, handler)
 }
 func (s *EchoService) serverStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
-	if s.ServerStreamingEcho == nil {
-		return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
-	}
 	m := new(EchoRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -207,15 +201,9 @@ func (s *EchoService) serverStreamingEcho(_ interface{}, stream grpc.ServerStrea
 	return s.ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
 }
 func (s *EchoService) clientStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
-	if s.ClientStreamingEcho == nil {
-		return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
-	}
 	return s.ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
 }
 func (s *EchoService) bidirectionalStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
-	if s.BidirectionalStreamingEcho == nil {
-		return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
-	}
 	return s.BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
 }
 
@@ -278,28 +266,49 @@ func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) {
 
 // RegisterEchoService registers a service implementation with a gRPC server.
 func RegisterEchoService(s grpc.ServiceRegistrar, srv *EchoService) {
+	srvCopy := *srv
+	if srvCopy.UnaryEcho == nil {
+		srvCopy.UnaryEcho = func(context.Context, *EchoRequest) (*EchoResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
+		}
+	}
+	if srvCopy.ServerStreamingEcho == nil {
+		srvCopy.ServerStreamingEcho = func(*EchoRequest, Echo_ServerStreamingEchoServer) error {
+			return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
+		}
+	}
+	if srvCopy.ClientStreamingEcho == nil {
+		srvCopy.ClientStreamingEcho = func(Echo_ClientStreamingEchoServer) error {
+			return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
+		}
+	}
+	if srvCopy.BidirectionalStreamingEcho == nil {
+		srvCopy.BidirectionalStreamingEcho = func(Echo_BidirectionalStreamingEchoServer) error {
+			return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.examples.echo.Echo",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "UnaryEcho",
-				Handler:    srv.unaryEcho,
+				Handler:    srvCopy.unaryEcho,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "ServerStreamingEcho",
-				Handler:       srv.serverStreamingEcho,
+				Handler:       srvCopy.serverStreamingEcho,
 				ServerStreams: true,
 			},
 			{
 				StreamName:    "ClientStreamingEcho",
-				Handler:       srv.clientStreamingEcho,
+				Handler:       srvCopy.clientStreamingEcho,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "BidirectionalStreamingEcho",
-				Handler:       srv.bidirectionalStreamingEcho,
+				Handler:       srvCopy.bidirectionalStreamingEcho,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index b45a600cead2..3a88bb1f2f3f 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -52,9 +52,6 @@ type GreeterService struct {
 }
 
 func (s *GreeterService) sayHello(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.SayHello == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
-	}
 	in := new(HelloRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -74,12 +71,18 @@ func (s *GreeterService) sayHello(_ interface{}, ctx context.Context, dec func(i
 
 // RegisterGreeterService registers a service implementation with a gRPC server.
 func RegisterGreeterService(s grpc.ServiceRegistrar, srv *GreeterService) {
+	srvCopy := *srv
+	if srvCopy.SayHello == nil {
+		srvCopy.SayHello = func(context.Context, *HelloRequest) (*HelloReply, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "helloworld.Greeter",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "SayHello",
-				Handler:    srv.sayHello,
+				Handler:    srvCopy.sayHello,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index f9c59a7f1d0f..eb406ec838cc 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -209,9 +209,6 @@ type RouteGuideService struct {
 }
 
 func (s *RouteGuideService) getFeature(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetFeature == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
-	}
 	in := new(Point)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -229,9 +226,6 @@ func (s *RouteGuideService) getFeature(_ interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 func (s *RouteGuideService) listFeatures(_ interface{}, stream grpc.ServerStream) error {
-	if s.ListFeatures == nil {
-		return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
-	}
 	m := new(Rectangle)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -239,15 +233,9 @@ func (s *RouteGuideService) listFeatures(_ interface{}, stream grpc.ServerStream
 	return s.ListFeatures(m, &routeGuideListFeaturesServer{stream})
 }
 func (s *RouteGuideService) recordRoute(_ interface{}, stream grpc.ServerStream) error {
-	if s.RecordRoute == nil {
-		return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
-	}
 	return s.RecordRoute(&routeGuideRecordRouteServer{stream})
 }
 func (s *RouteGuideService) routeChat(_ interface{}, stream grpc.ServerStream) error {
-	if s.RouteChat == nil {
-		return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
-	}
 	return s.RouteChat(&routeGuideRouteChatServer{stream})
 }
 
@@ -310,28 +298,49 @@ func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) {
 
 // RegisterRouteGuideService registers a service implementation with a gRPC server.
 func RegisterRouteGuideService(s grpc.ServiceRegistrar, srv *RouteGuideService) {
+	srvCopy := *srv
+	if srvCopy.GetFeature == nil {
+		srvCopy.GetFeature = func(context.Context, *Point) (*Feature, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
+		}
+	}
+	if srvCopy.ListFeatures == nil {
+		srvCopy.ListFeatures = func(*Rectangle, RouteGuide_ListFeaturesServer) error {
+			return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
+		}
+	}
+	if srvCopy.RecordRoute == nil {
+		srvCopy.RecordRoute = func(RouteGuide_RecordRouteServer) error {
+			return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
+		}
+	}
+	if srvCopy.RouteChat == nil {
+		srvCopy.RouteChat = func(RouteGuide_RouteChatServer) error {
+			return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "routeguide.RouteGuide",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "GetFeature",
-				Handler:    srv.getFeature,
+				Handler:    srvCopy.getFeature,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "ListFeatures",
-				Handler:       srv.listFeatures,
+				Handler:       srvCopy.listFeatures,
 				ServerStreams: true,
 			},
 			{
 				StreamName:    "RecordRoute",
-				Handler:       srv.recordRoute,
+				Handler:       srvCopy.recordRoute,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "RouteChat",
-				Handler:       srv.routeChat,
+				Handler:       srvCopy.routeChat,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index 716f8c0153a4..f848b4188b28 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -40,9 +40,6 @@ type HealthService struct {
 }
 
 func (s *HealthService) check(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.Check == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
-	}
 	in := new(HealthCheckRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -60,9 +57,6 @@ func (s *HealthService) check(_ interface{}, ctx context.Context, dec func(inter
 	return interceptor(ctx, in, info, handler)
 }
 func (s *HealthService) watch(_ interface{}, stream grpc.ServerStream) error {
-	if s.Watch == nil {
-		return status.Errorf(codes.Unimplemented, "method Watch not implemented")
-	}
 	m := new(HealthCheckRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -72,18 +66,29 @@ func (s *HealthService) watch(_ interface{}, stream grpc.ServerStream) error {
 
 // RegisterHealthService registers a service implementation with a gRPC server.
 func RegisterHealthService(s grpc.ServiceRegistrar, srv *HealthService) {
+	srvCopy := *srv
+	if srvCopy.Check == nil {
+		srvCopy.Check = func(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+		}
+	}
+	if srvCopy.Watch == nil {
+		srvCopy.Watch = func(*HealthCheckRequest, Health_WatchServer) error {
+			return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.health.v1.Health",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "Check",
-				Handler:    srv.check,
+				Handler:    srvCopy.check,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "Watch",
-				Handler:       srv.watch,
+				Handler:       srvCopy.watch,
 				ServerStreams: true,
 			},
 		},
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index d71f6d2a46b9..9496348fbe4d 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -251,9 +251,6 @@ type TestServiceService struct {
 }
 
 func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.EmptyCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-	}
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -271,9 +268,6 @@ func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.UnaryCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -291,9 +285,6 @@ func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.StreamingOutputCall == nil {
-		return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-	}
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -301,21 +292,12 @@ func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.Serv
 	return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
 }
 func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.StreamingInputCall == nil {
-		return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-	}
 	return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream})
 }
 func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.FullDuplexCall == nil {
-		return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-	}
 	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
 }
 func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.HalfDuplexCall == nil {
-		return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-	}
 	return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
 }
 
@@ -400,38 +382,69 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 
 // RegisterTestServiceService registers a service implementation with a gRPC server.
 func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
+	srvCopy := *srv
+	if srvCopy.EmptyCall == nil {
+		srvCopy.EmptyCall = func(context.Context, *Empty) (*Empty, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+		}
+	}
+	if srvCopy.UnaryCall == nil {
+		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+		}
+	}
+	if srvCopy.StreamingOutputCall == nil {
+		srvCopy.StreamingOutputCall = func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+		}
+	}
+	if srvCopy.StreamingInputCall == nil {
+		srvCopy.StreamingInputCall = func(TestService_StreamingInputCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+		}
+	}
+	if srvCopy.FullDuplexCall == nil {
+		srvCopy.FullDuplexCall = func(TestService_FullDuplexCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+		}
+	}
+	if srvCopy.HalfDuplexCall == nil {
+		srvCopy.HalfDuplexCall = func(TestService_HalfDuplexCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.TestService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "EmptyCall",
-				Handler:    srv.emptyCall,
+				Handler:    srvCopy.emptyCall,
 			},
 			{
 				MethodName: "UnaryCall",
-				Handler:    srv.unaryCall,
+				Handler:    srvCopy.unaryCall,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "StreamingOutputCall",
-				Handler:       srv.streamingOutputCall,
+				Handler:       srvCopy.streamingOutputCall,
 				ServerStreams: true,
 			},
 			{
 				StreamName:    "StreamingInputCall",
-				Handler:       srv.streamingInputCall,
+				Handler:       srvCopy.streamingInputCall,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "FullDuplexCall",
-				Handler:       srv.fullDuplexCall,
+				Handler:       srvCopy.fullDuplexCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "HalfDuplexCall",
-				Handler:       srv.halfDuplexCall,
+				Handler:       srvCopy.halfDuplexCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
@@ -549,9 +562,6 @@ type UnimplementedServiceService struct {
 }
 
 func (s *UnimplementedServiceService) unimplementedCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.UnimplementedCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
-	}
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -571,12 +581,18 @@ func (s *UnimplementedServiceService) unimplementedCall(_ interface{}, ctx conte
 
 // RegisterUnimplementedServiceService registers a service implementation with a gRPC server.
 func RegisterUnimplementedServiceService(s grpc.ServiceRegistrar, srv *UnimplementedServiceService) {
+	srvCopy := *srv
+	if srvCopy.UnimplementedCall == nil {
+		srvCopy.UnimplementedCall = func(context.Context, *Empty) (*Empty, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.UnimplementedService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "UnimplementedCall",
-				Handler:    srv.unimplementedCall,
+				Handler:    srvCopy.unimplementedCall,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
@@ -650,9 +666,6 @@ type LoadBalancerStatsServiceService struct {
 }
 
 func (s *LoadBalancerStatsServiceService) getClientStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetClientStats == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
-	}
 	in := new(LoadBalancerStatsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -672,12 +685,18 @@ func (s *LoadBalancerStatsServiceService) getClientStats(_ interface{}, ctx cont
 
 // RegisterLoadBalancerStatsServiceService registers a service implementation with a gRPC server.
 func RegisterLoadBalancerStatsServiceService(s grpc.ServiceRegistrar, srv *LoadBalancerStatsServiceService) {
+	srvCopy := *srv
+	if srvCopy.GetClientStats == nil {
+		srvCopy.GetClientStats = func(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.LoadBalancerStatsService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "GetClientStats",
-				Handler:    srv.getClientStats,
+				Handler:    srvCopy.getClientStats,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index e32cbb405db4..41450878f8b5 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -26,9 +26,6 @@ type ProfilingService struct {
 }
 
 func (s *ProfilingService) enable(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.Enable == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
-	}
 	in := new(EnableRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -46,9 +43,6 @@ func (s *ProfilingService) enable(_ interface{}, ctx context.Context, dec func(i
 	return interceptor(ctx, in, info, handler)
 }
 func (s *ProfilingService) getStreamStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetStreamStats == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
-	}
 	in := new(GetStreamStatsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -68,16 +62,27 @@ func (s *ProfilingService) getStreamStats(_ interface{}, ctx context.Context, de
 
 // RegisterProfilingService registers a service implementation with a gRPC server.
 func RegisterProfilingService(s grpc.ServiceRegistrar, srv *ProfilingService) {
+	srvCopy := *srv
+	if srvCopy.Enable == nil {
+		srvCopy.Enable = func(context.Context, *EnableRequest) (*EnableResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
+		}
+	}
+	if srvCopy.GetStreamStats == nil {
+		srvCopy.GetStreamStats = func(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.go.profiling.v1alpha.Profiling",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "Enable",
-				Handler:    srv.enable,
+				Handler:    srvCopy.enable,
 			},
 			{
 				MethodName: "GetStreamStats",
-				Handler:    srv.getStreamStats,
+				Handler:    srvCopy.getStreamStats,
 			},
 		},
 		Streams:  []grpc.StreamDesc{},
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index 95f9d0cb62d9..55241526bf80 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -23,21 +23,24 @@ type ServerReflectionService struct {
 }
 
 func (s *ServerReflectionService) serverReflectionInfo(_ interface{}, stream grpc.ServerStream) error {
-	if s.ServerReflectionInfo == nil {
-		return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
-	}
 	return s.ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
 }
 
 // RegisterServerReflectionService registers a service implementation with a gRPC server.
 func RegisterServerReflectionService(s grpc.ServiceRegistrar, srv *ServerReflectionService) {
+	srvCopy := *srv
+	if srvCopy.ServerReflectionInfo == nil {
+		srvCopy.ServerReflectionInfo = func(ServerReflection_ServerReflectionInfoServer) error {
+			return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.reflection.v1alpha.ServerReflection",
 		Methods:     []grpc.MethodDesc{},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "ServerReflectionInfo",
-				Handler:       srv.serverReflectionInfo,
+				Handler:       srvCopy.serverReflectionInfo,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 285cdbc85ace..2749e160f4e0 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -89,9 +89,6 @@ type SearchServiceService struct {
 }
 
 func (s *SearchServiceService) search(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.Search == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
-	}
 	in := new(SearchRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -109,9 +106,6 @@ func (s *SearchServiceService) search(_ interface{}, ctx context.Context, dec fu
 	return interceptor(ctx, in, info, handler)
 }
 func (s *SearchServiceService) streamingSearch(_ interface{}, stream grpc.ServerStream) error {
-	if s.StreamingSearch == nil {
-		return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
-	}
 	return s.StreamingSearch(&searchServiceStreamingSearchServer{stream})
 }
 
@@ -139,18 +133,29 @@ func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) {
 
 // RegisterSearchServiceService registers a service implementation with a gRPC server.
 func RegisterSearchServiceService(s grpc.ServiceRegistrar, srv *SearchServiceService) {
+	srvCopy := *srv
+	if srvCopy.Search == nil {
+		srvCopy.Search = func(context.Context, *SearchRequest) (*SearchResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
+		}
+	}
+	if srvCopy.StreamingSearch == nil {
+		srvCopy.StreamingSearch = func(SearchService_StreamingSearchServer) error {
+			return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.SearchService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "Search",
-				Handler:    srv.search,
+				Handler:    srvCopy.search,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "StreamingSearch",
-				Handler:       srv.streamingSearch,
+				Handler:       srvCopy.streamingSearch,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index 1fff30c631d6..c131efebec3e 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -183,9 +183,6 @@ type TestServiceService struct {
 }
 
 func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.UnaryCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -203,21 +200,12 @@ func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.FullDuplexCall == nil {
-		return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-	}
 	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
 }
 func (s *TestServiceService) clientStreamCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.ClientStreamCall == nil {
-		return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
-	}
 	return s.ClientStreamCall(&testServiceClientStreamCallServer{stream})
 }
 func (s *TestServiceService) serverStreamCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.ServerStreamCall == nil {
-		return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
-	}
 	m := new(SimpleRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -284,29 +272,50 @@ func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
 
 // RegisterTestServiceService registers a service implementation with a gRPC server.
 func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
+	srvCopy := *srv
+	if srvCopy.UnaryCall == nil {
+		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+		}
+	}
+	if srvCopy.FullDuplexCall == nil {
+		srvCopy.FullDuplexCall = func(TestService_FullDuplexCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+		}
+	}
+	if srvCopy.ClientStreamCall == nil {
+		srvCopy.ClientStreamCall = func(TestService_ClientStreamCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
+		}
+	}
+	if srvCopy.ServerStreamCall == nil {
+		srvCopy.ServerStreamCall = func(*SimpleRequest, TestService_ServerStreamCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.TestService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "UnaryCall",
-				Handler:    srv.unaryCall,
+				Handler:    srvCopy.unaryCall,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "FullDuplexCall",
-				Handler:       srv.fullDuplexCall,
+				Handler:       srvCopy.fullDuplexCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "ClientStreamCall",
-				Handler:       srv.clientStreamCall,
+				Handler:       srvCopy.clientStreamCall,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "ServerStreamCall",
-				Handler:       srv.serverStreamCall,
+				Handler:       srvCopy.serverStreamCall,
 				ServerStreams: true,
 			},
 		},
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 3f0a83584b2d..6ae1c80d263b 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -95,9 +95,6 @@ type MetricsServiceService struct {
 }
 
 func (s *MetricsServiceService) getAllGauges(_ interface{}, stream grpc.ServerStream) error {
-	if s.GetAllGauges == nil {
-		return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
-	}
 	m := new(EmptyMessage)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -105,9 +102,6 @@ func (s *MetricsServiceService) getAllGauges(_ interface{}, stream grpc.ServerSt
 	return s.GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
 }
 func (s *MetricsServiceService) getGauge(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.GetGauge == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
-	}
 	in := new(GaugeRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -140,18 +134,29 @@ func (x *metricsServiceGetAllGaugesServer) Send(m *GaugeResponse) error {
 
 // RegisterMetricsServiceService registers a service implementation with a gRPC server.
 func RegisterMetricsServiceService(s grpc.ServiceRegistrar, srv *MetricsServiceService) {
+	srvCopy := *srv
+	if srvCopy.GetAllGauges == nil {
+		srvCopy.GetAllGauges = func(*EmptyMessage, MetricsService_GetAllGaugesServer) error {
+			return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
+		}
+	}
+	if srvCopy.GetGauge == nil {
+		srvCopy.GetGauge = func(context.Context, *GaugeRequest) (*GaugeResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.MetricsService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "GetGauge",
-				Handler:    srv.getGauge,
+				Handler:    srvCopy.getGauge,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "GetAllGauges",
-				Handler:       srv.getAllGauges,
+				Handler:       srvCopy.getAllGauges,
 				ServerStreams: true,
 			},
 		},
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index 2ab12ac94210..7bcc9b06dd92 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -251,9 +251,6 @@ type TestServiceService struct {
 }
 
 func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.EmptyCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-	}
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -271,9 +268,6 @@ func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	if s.UnaryCall == nil {
-		return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-	}
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
@@ -291,9 +285,6 @@ func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.StreamingOutputCall == nil {
-		return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-	}
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
@@ -301,21 +292,12 @@ func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.Serv
 	return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
 }
 func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.StreamingInputCall == nil {
-		return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-	}
 	return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream})
 }
 func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.FullDuplexCall == nil {
-		return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-	}
 	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
 }
 func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	if s.HalfDuplexCall == nil {
-		return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-	}
 	return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
 }
 
@@ -400,38 +382,69 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 
 // RegisterTestServiceService registers a service implementation with a gRPC server.
 func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
+	srvCopy := *srv
+	if srvCopy.EmptyCall == nil {
+		srvCopy.EmptyCall = func(context.Context, *Empty) (*Empty, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+		}
+	}
+	if srvCopy.UnaryCall == nil {
+		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+		}
+	}
+	if srvCopy.StreamingOutputCall == nil {
+		srvCopy.StreamingOutputCall = func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+		}
+	}
+	if srvCopy.StreamingInputCall == nil {
+		srvCopy.StreamingInputCall = func(TestService_StreamingInputCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+		}
+	}
+	if srvCopy.FullDuplexCall == nil {
+		srvCopy.FullDuplexCall = func(TestService_FullDuplexCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+		}
+	}
+	if srvCopy.HalfDuplexCall == nil {
+		srvCopy.HalfDuplexCall = func(TestService_HalfDuplexCallServer) error {
+			return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+		}
+	}
 	sd := grpc.ServiceDesc{
 		ServiceName: "grpc.testing.TestService",
 		Methods: []grpc.MethodDesc{
 			{
 				MethodName: "EmptyCall",
-				Handler:    srv.emptyCall,
+				Handler:    srvCopy.emptyCall,
 			},
 			{
 				MethodName: "UnaryCall",
-				Handler:    srv.unaryCall,
+				Handler:    srvCopy.unaryCall,
 			},
 		},
 		Streams: []grpc.StreamDesc{
 			{
 				StreamName:    "StreamingOutputCall",
-				Handler:       srv.streamingOutputCall,
+				Handler:       srvCopy.streamingOutputCall,
 				ServerStreams: true,
 			},
 			{
 				StreamName:    "StreamingInputCall",
-				Handler:       srv.streamingInputCall,
+				Handler:       srvCopy.streamingInputCall,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "FullDuplexCall",
-				Handler:       srv.fullDuplexCall,
+				Handler:       srvCopy.fullDuplexCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},
 			{
 				StreamName:    "HalfDuplexCall",
-				Handler:       srv.halfDuplexCall,
+				Handler:       srvCopy.halfDuplexCall,
 				ServerStreams: true,
 				ClientStreams: true,
 			},

From 35afeb6efe9b0d62407464a8ff1c7aabb3ee49b8 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Thu, 27 Aug 2020 11:04:57 -0700
Subject: [PATCH 178/481] internal: reintroduce legacy appengine build
 constraints (#3850)

---
 .travis.yml                                   |  2 ++
 Makefile                                      |  8 +++++
 install_gae.sh                                |  6 ++++
 internal/credentials/syscallconn_appengine.go | 30 +++++++++++++++++++
 4 files changed, 46 insertions(+)
 create mode 100755 install_gae.sh
 create mode 100644 internal/credentials/syscallconn_appengine.go

diff --git a/.travis.yml b/.travis.yml
index 9097478f856d..3e495fa23b9d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,11 +30,13 @@ before_install:
 install:
   - try3() { eval "$*" || eval "$*" || eval "$*"; }
   - try3 'if [[ "${GO111MODULE}" = "on" ]]; then go mod download; else make testdeps; fi'
+  - if [[ -n "${GAE}" ]]; then source ./install_gae.sh; make testappenginedeps; fi
   - if [[ -n "${VET}" ]]; then ./vet.sh -install; fi
 
 script:
   - set -e
   - if [[ -n "${TESTEXTRAS}" ]]; then examples/examples_test.sh; interop/interop_test.sh; make testsubmodule; exit 0; fi
   - if [[ -n "${VET}" ]]; then ./vet.sh; fi
+  - if [[ -n "${GAE}" ]]; then make testappengine; exit 0; fi
   - if [[ -n "${RACE}" ]]; then make testrace; exit 0; fi
   - make test
diff --git a/Makefile b/Makefile
index cf474ae2fb97..3f661a7879d6 100644
--- a/Makefile
+++ b/Makefile
@@ -23,6 +23,12 @@ testsubmodule: testdeps
 	cd security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
 	cd security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/...
 
+testappengine: testappenginedeps
+	goapp test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
+
+testappenginedeps:
+	goapp get -d -v -t -tags 'appengine appenginevm' google.golang.org/grpc/...
+
 testdeps:
 	go get -d -v -t google.golang.org/grpc/...
 
@@ -48,6 +54,8 @@ vetdeps:
 	deps \
 	proto \
 	test \
+	testappengine \
+	testappenginedeps \
 	testdeps \
 	testrace \
 	updatedeps \
diff --git a/install_gae.sh b/install_gae.sh
new file mode 100755
index 000000000000..15ff9facdd78
--- /dev/null
+++ b/install_gae.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+TMP=$(mktemp -d /tmp/sdk.XXX) \
+&& curl -o $TMP.zip "https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_linux_amd64-1.9.68.zip" \
+&& unzip -q $TMP.zip -d $TMP \
+&& export PATH="$PATH:$TMP/go_appengine"
\ No newline at end of file
diff --git a/internal/credentials/syscallconn_appengine.go b/internal/credentials/syscallconn_appengine.go
new file mode 100644
index 000000000000..a6144cd661c2
--- /dev/null
+++ b/internal/credentials/syscallconn_appengine.go
@@ -0,0 +1,30 @@
+// +build appengine
+
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package credentials
+
+import (
+	"net"
+)
+
+// WrapSyscallConn returns newConn on appengine.
+func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn {
+	return newConn
+}

From d25c71b54334380ff1febd25d88064b36de44b3c Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 27 Aug 2020 13:55:15 -0700
Subject: [PATCH 179/481] testutils: Add a context parameter to the Receive()
 method. (#3835)

---
 balancer/rls/internal/balancer_test.go        |  24 ++-
 balancer/rls/internal/client_test.go          |  15 +-
 balancer/rls/internal/picker_test.go          |  16 +-
 credentials/sts/sts_test.go                   |  17 +-
 credentials/tls/certprovider/store_test.go    |  34 ++--
 internal/testutils/channel.go                 |  43 ++---
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 149 ++++++++++--------
 xds/internal/balancer/edsbalancer/eds_test.go |  64 ++++++--
 .../edsbalancer/xds_client_wrapper_test.go    |  38 +++--
 .../balancer/edsbalancer/xds_lrs_test.go      |  13 +-
 xds/internal/client/client_test.go            |  21 ++-
 .../client/client_watchers_cluster_test.go    | 103 ++++++++----
 .../client/client_watchers_endpoints_test.go  |  71 ++++++---
 .../client/client_watchers_lds_test.go        |  84 ++++++----
 .../client/client_watchers_rds_test.go        |  63 +++++---
 .../client/client_watchers_service_test.go    | 100 +++++++-----
 xds/internal/client/v2/client_ack_test.go     |  24 ++-
 xds/internal/client/v2/client_rds_test.go     |   5 +-
 xds/internal/client/v2/client_test.go         |  43 +++--
 xds/internal/resolver/xds_resolver_test.go    |  53 +++++--
 xds/internal/testutils/balancer.go            |   1 +
 xds/internal/testutils/channel.go             |  87 ----------
 xds/internal/testutils/fakeclient/client.go   |  59 +++----
 xds/internal/testutils/fakeserver/server.go   |   2 +-
 24 files changed, 674 insertions(+), 455 deletions(-)
 delete mode 100644 xds/internal/testutils/channel.go

diff --git a/balancer/rls/internal/balancer_test.go b/balancer/rls/internal/balancer_test.go
index d6a98aa9ab00..2378a86fff10 100644
--- a/balancer/rls/internal/balancer_test.go
+++ b/balancer/rls/internal/balancer_test.go
@@ -19,6 +19,7 @@
 package rls
 
 import (
+	"context"
 	"net"
 	"testing"
 	"time"
@@ -32,6 +33,8 @@ import (
 	"google.golang.org/grpc/testdata"
 )
 
+const defaultTestTimeout = 1 * time.Second
+
 type s struct {
 	grpctest.Tester
 }
@@ -99,7 +102,9 @@ func (s) TestUpdateControlChannelFirstConfig(t *testing.T) {
 	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
 	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
 
-	if _, err := lis.connCh.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := lis.connCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
 	}
 
@@ -132,7 +137,9 @@ func (s) TestUpdateControlChannelSwitch(t *testing.T) {
 	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
 	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
 
-	if _, err := lis1.connCh.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := lis1.connCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
 	}
 
@@ -140,7 +147,7 @@ func (s) TestUpdateControlChannelSwitch(t *testing.T) {
 	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
 	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
 
-	if _, err := lis2.connCh.Receive(); err != nil {
+	if _, err := lis2.connCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
 	}
 
@@ -169,14 +176,17 @@ func (s) TestUpdateControlChannelTimeout(t *testing.T) {
 	lbCfg := &lbConfig{lookupService: server.Address, lookupServiceTimeout: 1 * time.Second}
 	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
 	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
-	if _, err := lis.connCh.Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := lis.connCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
 	}
 
 	lbCfg = &lbConfig{lookupService: server.Address, lookupServiceTimeout: 2 * time.Second}
 	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
 	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
-	if _, err := lis.connCh.Receive(); err != testutils.ErrRecvTimeout {
+	if _, err := lis.connCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatal("LB policy created new control channel when only lookupServiceTimeout changed")
 	}
 
@@ -215,7 +225,9 @@ func (s) TestUpdateControlChannelWithCreds(t *testing.T) {
 	t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg)
 	rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg})
 
-	if _, err := lis.connCh.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := lis.connCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout expired when waiting for LB policy to create control channel")
 	}
 
diff --git a/balancer/rls/internal/client_test.go b/balancer/rls/internal/client_test.go
index a45850dce11e..4bf0303a2769 100644
--- a/balancer/rls/internal/client_test.go
+++ b/balancer/rls/internal/client_test.go
@@ -19,6 +19,7 @@
 package rls
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"testing"
@@ -82,7 +83,9 @@ func (s) TestLookupFailure(t *testing.T) {
 		errCh.Send(nil)
 	})
 
-	if e, err := errCh.Receive(); err != nil || e != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if e, err := errCh.Receive(ctx); err != nil || e != nil {
 		t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err)
 	}
 }
@@ -106,7 +109,9 @@ func (s) TestLookupDeadlineExceeded(t *testing.T) {
 		errCh.Send(nil)
 	})
 
-	if e, err := errCh.Receive(); err != nil || e != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if e, err := errCh.Receive(ctx); err != nil || e != nil {
 		t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err)
 	}
 }
@@ -150,7 +155,9 @@ func (s) TestLookupSuccess(t *testing.T) {
 
 	// Make sure that the fake server received the expected RouteLookupRequest
 	// proto.
-	req, err := server.RequestChan.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	req, err := server.RequestChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timed out wile waiting for a RouteLookupRequest")
 	}
@@ -168,7 +175,7 @@ func (s) TestLookupSuccess(t *testing.T) {
 		},
 	}
 
-	if e, err := errCh.Receive(); err != nil || e != nil {
+	if e, err := errCh.Receive(ctx); err != nil || e != nil {
 		t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err)
 	}
 }
diff --git a/balancer/rls/internal/picker_test.go b/balancer/rls/internal/picker_test.go
index 1397bf8085d8..762eb5fd80e9 100644
--- a/balancer/rls/internal/picker_test.go
+++ b/balancer/rls/internal/picker_test.go
@@ -260,7 +260,9 @@ func TestPick_DataCacheMiss_PendingCacheMiss(t *testing.T) {
 			// If the test specified that a new RLS request should be made,
 			// verify it.
 			if test.wantRLSRequest {
-				if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil {
+				ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+				defer cancel()
+				if rlsErr, err := rlsCh.Receive(ctx); err != nil || rlsErr != nil {
 					t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err)
 				}
 			}
@@ -339,7 +341,9 @@ func TestPick_DataCacheMiss_PendingCacheHit(t *testing.T) {
 			}
 
 			// Make sure that no RLS request was sent out.
-			if _, err := rlsCh.Receive(); err != testutils.ErrRecvTimeout {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if _, err := rlsCh.Receive(ctx); err != context.DeadlineExceeded {
 				t.Fatalf("RLS request sent out when pending entry exists")
 			}
 		})
@@ -483,7 +487,9 @@ func TestPick_DataCacheHit_PendingCacheMiss(t *testing.T) {
 			// If the test specified that a new RLS request should be made,
 			// verify it.
 			if test.wantRLSRequest {
-				if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil {
+				ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+				defer cancel()
+				if rlsErr, err := rlsCh.Receive(ctx); err != nil || rlsErr != nil {
 					t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err)
 				}
 			}
@@ -590,7 +596,9 @@ func TestPick_DataCacheHit_PendingCacheHit(t *testing.T) {
 				t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr)
 			}
 			// Make sure that no RLS request was sent out.
-			if _, err := rlsCh.Receive(); err != testutils.ErrRecvTimeout {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if _, err := rlsCh.Receive(ctx); err != context.DeadlineExceeded {
 				t.Fatalf("RLS request sent out when pending entry exists")
 			}
 			if test.wantErr != nil {
diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go
index 641bad5820bb..cc8b08e5d933 100644
--- a/credentials/sts/sts_test.go
+++ b/credentials/sts/sts_test.go
@@ -55,6 +55,7 @@ const (
 	exampleResource      = "https://backend.example.com/api"
 	exampleAudience      = "example-backend-service"
 	testScope            = "https://www.googleapis.com/auth/monitoring"
+	defaultTestTimeout   = 1 * time.Second
 )
 
 var (
@@ -142,7 +143,11 @@ type fakeHTTPDoer struct {
 
 func (fc *fakeHTTPDoer) Do(req *http.Request) (*http.Response, error) {
 	fc.reqCh.Send(req)
-	val, err := fc.respCh.Receive()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	val, err := fc.respCh.Receive(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -240,7 +245,10 @@ func compareRequest(gotRequest *http.Request, wantReqParams *requestParameters)
 // by the tests. So, any errors encountered are pushed to an error channel
 // which is monitored by the test.
 func receiveAndCompareRequest(reqCh *testutils.Channel, errCh chan error) {
-	val, err := reqCh.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	val, err := reqCh.Receive(ctx)
 	if err != nil {
 		errCh <- err
 		return
@@ -430,7 +438,10 @@ func (s) TestGetRequestMetadataBadSubjectTokenRead(t *testing.T) {
 
 	errCh := make(chan error, 1)
 	go func() {
-		if _, err := fc.reqCh.Receive(); err != testutils.ErrRecvTimeout {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+
+		if _, err := fc.reqCh.Receive(ctx); err != context.DeadlineExceeded {
 			errCh <- err
 			return
 		}
diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go
index a116148945eb..618c2d7d9b00 100644
--- a/credentials/tls/certprovider/store_test.go
+++ b/credentials/tls/certprovider/store_test.go
@@ -179,7 +179,9 @@ func (s) TestStoreSingleProvider(t *testing.T) {
 
 	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
 	// the fake provider from that channel.
-	p, err := fpb1.providerChan.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p, err := fpb1.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
@@ -188,8 +190,6 @@ func (s) TestStoreSingleProvider(t *testing.T) {
 	// Attempt to read from key material from the Provider returned by the
 	// store. This will fail because we have not pushed any key material into
 	// our fake provider.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, prov, nil); !errors.Is(err, context.DeadlineExceeded) {
 		t.Fatal(err)
 	}
@@ -208,8 +208,6 @@ func (s) TestStoreSingleProvider(t *testing.T) {
 	// updated key material.
 	testKM2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
 	fakeProv.newKeyMaterial(testKM2, nil)
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, prov, testKM2); err != nil {
 		t.Fatal(err)
 	}
@@ -236,7 +234,9 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	defer provFoo2.Close()
 	// Our fakeProviderBuilder pushes newly created providers on a channel.
 	// Grab the fake provider for optsFoo.
-	p, err := fpb1.providerChan.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p, err := fpb1.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
@@ -248,7 +248,7 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	}
 	defer provBar1.Close()
 	// Grab the fake provider for optsBar.
-	p, err = fpb1.providerChan.Receive()
+	p, err = fpb1.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
@@ -258,8 +258,6 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	// appropriate key material and the bar provider times out.
 	fooKM := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
 	fakeProvFoo.newKeyMaterial(fooKM, nil)
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil {
 		t.Fatal(err)
 	}
@@ -302,7 +300,9 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 	defer prov1.Close()
 	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
 	// the fake provider from that channel.
-	p1, err := fpb1.providerChan.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1, err := fpb1.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
@@ -314,7 +314,7 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 	}
 	defer prov2.Close()
 	// Grab the second provider from the channel.
-	p2, err := fpb1.providerChan.Receive()
+	p2, err := fpb1.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
@@ -325,8 +325,6 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
 	fakeProv1.newKeyMaterial(km1, nil)
 	fakeProv2.newKeyMaterial(km1, nil)
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
 		t.Fatal(err)
 	}
@@ -339,8 +337,6 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 	// material.
 	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
 	fakeProv2.newKeyMaterial(km2, nil)
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
 		t.Fatal(err)
 	}
@@ -366,7 +362,9 @@ func (s) TestStoreMultipleProviders(t *testing.T) {
 	defer prov1.Close()
 	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
 	// the fake provider from that channel.
-	p1, err := fpb1.providerChan.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1, err := fpb1.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
 	}
@@ -378,7 +376,7 @@ func (s) TestStoreMultipleProviders(t *testing.T) {
 	}
 	defer prov2.Close()
 	// Grab the second provider from the channel.
-	p2, err := fpb2.providerChan.Receive()
+	p2, err := fpb2.providerChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider2Name)
 	}
@@ -390,8 +388,6 @@ func (s) TestStoreMultipleProviders(t *testing.T) {
 	fakeProv1.newKeyMaterial(km1, nil)
 	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
 	fakeProv2.newKeyMaterial(km2, nil)
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/testutils/channel.go b/internal/testutils/channel.go
index 35f67ea285c1..c92b1311c0ed 100644
--- a/internal/testutils/channel.go
+++ b/internal/testutils/channel.go
@@ -18,21 +18,11 @@
 package testutils
 
 import (
-	"errors"
-	"time"
+	"context"
 )
 
-// ErrRecvTimeout is an error to indicate that a receive operation on the
-// channel timed out.
-var ErrRecvTimeout = errors.New("timed out when waiting for value on channel")
-
-const (
-	// DefaultChanRecvTimeout is the default timeout for receive operations on the
-	// underlying channel.
-	DefaultChanRecvTimeout = 1 * time.Second
-	// DefaultChanBufferSize is the default buffer size of the underlying channel.
-	DefaultChanBufferSize = 1
-)
+// DefaultChanBufferSize is the default buffer size of the underlying channel.
+const DefaultChanBufferSize = 1
 
 // Channel wraps a generic channel and provides a timed receive operation.
 type Channel struct {
@@ -44,19 +34,32 @@ func (cwt *Channel) Send(value interface{}) {
 	cwt.ch <- value
 }
 
-// Receive returns the value received on the underlying channel, or
-// ErrRecvTimeout if DefaultChanRecvTimeout amount of time elapses.
-func (cwt *Channel) Receive() (interface{}, error) {
-	timer := time.NewTimer(DefaultChanRecvTimeout)
+// Receive returns the value received on the underlying channel, or the error
+// returned by ctx if it is closed or cancelled.
+func (cwt *Channel) Receive(ctx context.Context) (interface{}, error) {
 	select {
-	case <-timer.C:
-		return nil, ErrRecvTimeout
+	case <-ctx.Done():
+		return nil, ctx.Err()
 	case got := <-cwt.ch:
-		timer.Stop()
 		return got, nil
 	}
 }
 
+// Replace clears the value on the underlying channel, and sends the new value.
+//
+// It's expected to be used with a size-1 channel, to only keep the most
+// up-to-date item. This method is inherently racy when invoked concurrently
+// from multiple goroutines.
+func (cwt *Channel) Replace(value interface{}) {
+	for {
+		select {
+		case cwt.ch <- value:
+			return
+		case <-cwt.ch:
+		}
+	}
+}
+
 // NewChannel returns a new Channel.
 func NewChannel() *Channel {
 	return NewChannelWithSize(DefaultChanBufferSize)
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 358e6c3895ef..689a39e47fb8 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -17,6 +17,7 @@
 package cdsbalancer
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -30,19 +31,19 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
 const (
 	clusterName        = "cluster1"
 	serviceName        = "service1"
-	defaultTestTimeout = 2 * time.Second
+	defaultTestTimeout = 1 * time.Second
 )
 
 type s struct {
@@ -90,13 +91,13 @@ func invokeWatchCbAndWait(xdsC *fakeclient.Client, cdsW cdsWatchInfo, wantCCS ba
 // to the test.
 type testEDSBalancer struct {
 	// ccsCh is a channel used to signal the receipt of a ClientConn update.
-	ccsCh chan balancer.ClientConnState
+	ccsCh *testutils.Channel
 	// scStateCh is a channel used to signal the receipt of a SubConn update.
-	scStateCh chan subConnWithState
+	scStateCh *testutils.Channel
 	// resolverErrCh is a channel used to signal a resolver error.
-	resolverErrCh chan error
+	resolverErrCh *testutils.Channel
 	// closeCh is a channel used to signal the closing of this balancer.
-	closeCh chan struct{}
+	closeCh *testutils.Channel
 }
 
 type subConnWithState struct {
@@ -106,89 +107,86 @@ type subConnWithState struct {
 
 func newTestEDSBalancer() *testEDSBalancer {
 	return &testEDSBalancer{
-		ccsCh:         make(chan balancer.ClientConnState, 1),
-		scStateCh:     make(chan subConnWithState, 1),
-		resolverErrCh: make(chan error, 1),
-		closeCh:       make(chan struct{}, 1),
+		ccsCh:         testutils.NewChannel(),
+		scStateCh:     testutils.NewChannel(),
+		resolverErrCh: testutils.NewChannel(),
+		closeCh:       testutils.NewChannel(),
 	}
 }
 
 func (tb *testEDSBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error {
-	tb.ccsCh <- ccs
+	tb.ccsCh.Send(ccs)
 	return nil
 }
 
 func (tb *testEDSBalancer) ResolverError(err error) {
-	tb.resolverErrCh <- err
+	tb.resolverErrCh.Send(err)
 }
 
 func (tb *testEDSBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
-	tb.scStateCh <- subConnWithState{sc: sc, state: state}
+	tb.scStateCh.Send(subConnWithState{sc: sc, state: state})
 }
 
 func (tb *testEDSBalancer) Close() {
-	tb.closeCh <- struct{}{}
+	tb.closeCh.Send(struct{}{})
 }
 
 // waitForClientConnUpdate verifies if the testEDSBalancer receives the
 // provided ClientConnState within a reasonable amount of time.
 func (tb *testEDSBalancer) waitForClientConnUpdate(wantCCS balancer.ClientConnState) error {
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		return errors.New("Timeout when expecting ClientConn update on EDS balancer")
-	case gotCCS := <-tb.ccsCh:
-		timer.Stop()
-		if !cmp.Equal(gotCCS, wantCCS, cmpopts.IgnoreUnexported(attributes.Attributes{})) {
-			return fmt.Errorf("received ClientConnState: %+v, want %+v", gotCCS, wantCCS)
-		}
-		return nil
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ccs, err := tb.ccsCh.Receive(ctx)
+	if err != nil {
+		return err
 	}
+	gotCCS := ccs.(balancer.ClientConnState)
+	if !cmp.Equal(gotCCS, wantCCS, cmpopts.IgnoreUnexported(attributes.Attributes{})) {
+		return fmt.Errorf("received ClientConnState: %+v, want %+v", gotCCS, wantCCS)
+	}
+	return nil
 }
 
 // waitForSubConnUpdate verifies if the testEDSBalancer receives the provided
 // SubConn update within a reasonable amount of time.
 func (tb *testEDSBalancer) waitForSubConnUpdate(wantSCS subConnWithState) error {
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		return errors.New("Timeout when expecting SubConn update on EDS balancer")
-	case gotSCS := <-tb.scStateCh:
-		timer.Stop()
-		if !cmp.Equal(gotSCS, wantSCS, cmp.AllowUnexported(subConnWithState{})) {
-			return fmt.Errorf("received SubConnState: %+v, want %+v", gotSCS, wantSCS)
-		}
-		return nil
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	scs, err := tb.scStateCh.Receive(ctx)
+	if err != nil {
+		return err
 	}
+	gotSCS := scs.(subConnWithState)
+	if !cmp.Equal(gotSCS, wantSCS, cmp.AllowUnexported(subConnWithState{})) {
+		return fmt.Errorf("received SubConnState: %+v, want %+v", gotSCS, wantSCS)
+	}
+	return nil
 }
 
 // waitForResolverError verifies if the testEDSBalancer receives the
 // provided resolver error within a reasonable amount of time.
 func (tb *testEDSBalancer) waitForResolverError(wantErr error) error {
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		return errors.New("Timeout when expecting a resolver error")
-	case gotErr := <-tb.resolverErrCh:
-		timer.Stop()
-		if gotErr != wantErr {
-			return fmt.Errorf("received resolver error: %v, want %v", gotErr, wantErr)
-		}
-		return nil
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotErr, err := tb.resolverErrCh.Receive(ctx)
+	if err != nil {
+		return err
 	}
+	if gotErr != wantErr {
+		return fmt.Errorf("received resolver error: %v, want %v", gotErr, wantErr)
+	}
+	return nil
 }
 
 // waitForClose verifies that the edsBalancer is closed with a reasonable
 // amount of time.
 func (tb *testEDSBalancer) waitForClose() error {
-	timer := time.NewTimer(defaultTestTimeout)
-	select {
-	case <-timer.C:
-		return errors.New("Timeout when expecting a close")
-	case <-tb.closeCh:
-		timer.Stop()
-		return nil
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tb.closeCh.Receive(ctx); err != nil {
+		return err
 	}
+	return nil
 }
 
 // cdsCCS is a helper function to construct a good update passed from the
@@ -254,7 +252,10 @@ func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBal
 	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
-	gotCluster, err := xdsC.WaitForWatchCluster()
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	gotCluster, err := xdsC.WaitForWatchCluster(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
 	}
@@ -328,7 +329,9 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 				// When we wanted an error and got it, we should return early.
 				return
 			}
-			gotCluster, err := xdsC.WaitForWatchCluster()
+			ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer ctxCancel()
+			gotCluster, err := xdsC.WaitForWatchCluster(ctx)
 			if err != nil {
 				t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
 			}
@@ -364,7 +367,9 @@ func (s) TestUpdateClientConnStateWithSameState(t *testing.T) {
 	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
-	if _, err := xdsC.WaitForWatchCluster(); err != testutils.ErrRecvTimeout {
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if _, err := xdsC.WaitForWatchCluster(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("waiting for WatchCluster() should have timed out, but returned error: %v", err)
 	}
 }
@@ -422,13 +427,17 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// And this is not a resource not found error, watch shouldn't be canceled.
 	err1 := errors.New("cdsBalancer resolver error 1")
 	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err1)
-	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsB.waitForResolverError(err1); err == nil {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
-	state, err := tcc.newPickerCh.Receive()
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	state, err := tcc.newPickerCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get picker, expect an error picker")
 	}
@@ -447,7 +456,9 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// is not a resource not found error, watch shouldn't be canceled
 	err2 := errors.New("cdsBalancer resolver error 2")
 	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err2)
-	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsB.waitForResolverError(err2); err != nil {
@@ -458,7 +469,9 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// means CDS resource is removed, and eds should receive the error.
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
 	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, resourceErr)
-	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
 		t.Fatalf("want watch to be not canceled, watchForCancel should timeout")
 	}
 	if err := edsB.waitForResolverError(resourceErr); err != nil {
@@ -479,13 +492,17 @@ func (s) TestResolverError(t *testing.T) {
 	// Not a resource not found error, watch shouldn't be canceled.
 	err1 := errors.New("cdsBalancer resolver error 1")
 	cdsB.ResolverError(err1)
-	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsB.waitForResolverError(err1); err == nil {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
-	state, err := tcc.newPickerCh.Receive()
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	state, err := tcc.newPickerCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get picker, expect an error picker")
 	}
@@ -504,7 +521,9 @@ func (s) TestResolverError(t *testing.T) {
 	// should receive the error.
 	err2 := errors.New("cdsBalancer resolver error 2")
 	cdsB.ResolverError(err2)
-	if err := xdsC.WaitForCancelClusterWatch(); err == nil {
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsB.waitForResolverError(err2); err != nil {
@@ -515,7 +534,9 @@ func (s) TestResolverError(t *testing.T) {
 	// receive the error.
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
 	cdsB.ResolverError(resourceErr)
-	if err := xdsC.WaitForCancelClusterWatch(); err != nil {
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, watchForCancel failed: %v", err)
 	}
 	if err := edsB.waitForResolverError(resourceErr); err != nil {
@@ -559,7 +580,9 @@ func (s) TestClose(t *testing.T) {
 	}
 
 	cdsB.Close()
-	if err := xdsC.WaitForCancelClusterWatch(); err != nil {
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatal(err)
 	}
 	if err := edsB.waitForClose(); err != nil {
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 312ad9efb030..7ace7cc2c170 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -20,10 +20,12 @@ package edsbalancer
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/golang/protobuf/jsonpb"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
@@ -34,17 +36,20 @@ import (
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpctest"
 	scpb "google.golang.org/grpc/internal/proto/grpc_service_config"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 
 	_ "google.golang.org/grpc/xds/internal/client/v2" // V2 client registration.
 )
 
+const defaultTestTimeout = 1 * time.Second
+
 func init() {
 	balancer.Register(&edsBalancerBuilder{})
 
@@ -52,7 +57,7 @@ func init() {
 		return &bootstrap.Config{
 			BalancerName: testBalancerNameFooBar,
 			Creds:        grpc.WithInsecure(),
-			NodeProto:    testutils.EmptyNodeProtoV2,
+			NodeProto:    xdstestutils.EmptyNodeProtoV2,
 		}, nil
 	}
 }
@@ -120,7 +125,10 @@ func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {
 func (f *fakeEDSBalancer) close() {}
 
 func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) error {
-	val, err := f.childPolicy.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	val, err := f.childPolicy.Receive(ctx)
 	if err != nil {
 		return fmt.Errorf("error waiting for childPolicy: %v", err)
 	}
@@ -132,7 +140,10 @@ func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) er
 }
 
 func (f *fakeEDSBalancer) waitForSubConnStateChange(wantState *scStateChange) error {
-	val, err := f.subconnStateChange.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	val, err := f.subconnStateChange.Receive(ctx)
 	if err != nil {
 		return fmt.Errorf("error waiting for subconnStateChange: %v", err)
 	}
@@ -144,7 +155,10 @@ func (f *fakeEDSBalancer) waitForSubConnStateChange(wantState *scStateChange) er
 }
 
 func (f *fakeEDSBalancer) waitForEDSResponse(wantUpdate xdsclient.EndpointsUpdate) error {
-	val, err := f.edsUpdate.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	val, err := f.edsUpdate.Receive(ctx)
 	if err != nil {
 		return fmt.Errorf("error waiting for edsUpdate: %v", err)
 	}
@@ -176,7 +190,9 @@ func (*fakeSubConn) Connect()                           { panic("implement me")
 func waitForNewXDSClientWithEDSWatch(t *testing.T, ch *testutils.Channel, wantName string) *fakeclient.Client {
 	t.Helper()
 
-	val, err := ch.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ch.Receive(ctx)
 	if err != nil {
 		t.Fatalf("error when waiting for a new xds client: %v", err)
 		return nil
@@ -186,7 +202,7 @@ func waitForNewXDSClientWithEDSWatch(t *testing.T, ch *testutils.Channel, wantNa
 		t.Fatalf("xdsClient created to balancer: %v, want %v", xdsC.Name(), wantName)
 		return nil
 	}
-	_, err = xdsC.WaitForWatchEDS()
+	_, err = xdsC.WaitForWatchEDS(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 		return nil
@@ -199,7 +215,9 @@ func waitForNewXDSClientWithEDSWatch(t *testing.T, ch *testutils.Channel, wantNa
 func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
 	t.Helper()
 
-	val, err := ch.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ch.Receive(ctx)
 	if err != nil {
 		t.Fatalf("error when waiting for a new edsLB: %v", err)
 		return nil
@@ -439,12 +457,14 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	}
 	defer edsB.Close()
 
-	edsB.UpdateClientConnState(balancer.ClientConnState{
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			BalancerName:   testBalancerNameFooBar,
 			EDSServiceName: testEDSClusterName,
 		},
-	})
+	}); err != nil {
+		t.Fatal(err)
+	}
 
 	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
@@ -455,7 +475,10 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 
 	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, connectionErr)
-	if err := xdsC.WaitForCancelEDSWatch(); err == nil {
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil {
@@ -467,7 +490,9 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	// Even if error is resource not found, watch shouldn't be canceled, because
 	// this is an EDS resource removed (and xds client actually never sends this
 	// error, but we still handles it).
-	if err := xdsC.WaitForCancelEDSWatch(); err == nil {
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
@@ -496,12 +521,14 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	}
 	defer edsB.Close()
 
-	edsB.UpdateClientConnState(balancer.ClientConnState{
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			BalancerName:   testBalancerNameFooBar,
 			EDSServiceName: testEDSClusterName,
 		},
-	})
+	}); err != nil {
+		t.Fatal(err)
+	}
 
 	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
@@ -512,7 +539,10 @@ func (s) TestErrorFromResolver(t *testing.T) {
 
 	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
 	edsB.ResolverError(connectionErr)
-	if err := xdsC.WaitForCancelEDSWatch(); err == nil {
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
 	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil {
@@ -521,7 +551,9 @@ func (s) TestErrorFromResolver(t *testing.T) {
 
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error")
 	edsB.ResolverError(resourceErr)
-	if err := xdsC.WaitForCancelEDSWatch(); err != nil {
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := xdsC.WaitForCancelEDSWatch(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, waitForCancel failed: %v", err)
 	}
 	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index 6ae65dc7c94b..a230ca9ca49d 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -19,10 +19,10 @@
 package edsbalancer
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"testing"
-	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	"github.com/golang/protobuf/proto"
@@ -31,11 +31,12 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
@@ -51,13 +52,16 @@ var (
 func verifyExpectedRequests(fs *fakeserver.Server, resourceNames ...string) error {
 	wantReq := &xdspb.DiscoveryRequest{
 		TypeUrl: version.V2EndpointsURL,
-		Node:    testutils.EmptyNodeProtoV2,
+		Node:    xdstestutils.EmptyNodeProtoV2,
 	}
 	for _, name := range resourceNames {
 		if name != "" {
 			wantReq.ResourceNames = []string{name}
 		}
-		req, err := fs.XDSRequestChan.TimedReceive(time.Millisecond * 100)
+
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		req, err := fs.XDSRequestChan.Receive(ctx)
 		if err != nil {
 			return fmt.Errorf("timed out when expecting request {%+v} at fake server", wantReq)
 		}
@@ -98,7 +102,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 		return &bootstrap.Config{
 			BalancerName: fakeServer.Address,
 			Creds:        grpc.WithInsecure(),
-			NodeProto:    testutils.EmptyNodeProtoV2,
+			NodeProto:    xdstestutils.EmptyNodeProtoV2,
 		}, nil
 	}
 	defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
@@ -109,7 +113,9 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 		BalancerName:   fakeServer.Address,
 		EDSServiceName: "",
 	}, nil)
-	if _, err := fakeServer.NewConnChan.TimedReceive(1 * time.Second); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fakeServer.NewConnChan.Receive(ctx); err != nil {
 		t.Fatal("Failed to connect to fake server")
 	}
 	t.Log("Client connection established to fake server...")
@@ -161,7 +167,10 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 
 	xdsC := fakeclient.NewClient()
 	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC))
-	gotCluster, err := xdsC.WaitForWatchEDS()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotCluster, err := xdsC.WaitForWatchEDS(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
@@ -175,7 +184,9 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 	//
 	// TODO: check for loseContact() when errors indicating "lose contact" are
 	// handled correctly.
-	gotUpdate, err := edsRespChan.Receive()
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotUpdate, err := edsRespChan.Receive(ctx)
 	if err != nil {
 		t.Fatalf("edsBalancer failed to get edsUpdate %v", err)
 	}
@@ -199,10 +210,13 @@ func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
 	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
 	defer cw.close()
 
-	// Verify that the eds watch is registered for the expected resource name.
 	xdsC1 := fakeclient.NewClient()
 	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC1))
-	gotCluster, err := xdsC1.WaitForWatchEDS()
+
+	// Verify that the eds watch is registered for the expected resource name.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotCluster, err := xdsC1.WaitForWatchEDS(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
@@ -216,7 +230,7 @@ func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
 	// close client that are passed through attributes).
 	xdsC2 := fakeclient.NewClient()
 	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC2))
-	gotCluster, err = xdsC2.WaitForWatchEDS()
+	gotCluster, err = xdsC2.WaitForWatchEDS(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
@@ -224,7 +238,7 @@ func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
 		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
 	}
 
-	if err := xdsC1.WaitForClose(); err != testutils.ErrRecvTimeout {
+	if err := xdsC1.WaitForClose(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("clientWrapper closed xdsClient received in attributes")
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
index 7e2b28ba387a..92de46360160 100644
--- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
@@ -19,6 +19,7 @@
 package edsbalancer
 
 import (
+	"context"
 	"testing"
 
 	"google.golang.org/grpc/attributes"
@@ -41,12 +42,16 @@ func (s) TestXDSLoadReporting(t *testing.T) {
 	defer edsB.Close()
 
 	xdsC := fakeclient.NewClient()
-	edsB.UpdateClientConnState(balancer.ClientConnState{
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{LrsLoadReportingServerName: new(string)},
-	})
+	}); err != nil {
+		t.Fatal(err)
+	}
 
-	gotCluster, err := xdsC.WaitForWatchEDS()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotCluster, err := xdsC.WaitForWatchEDS(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
@@ -54,7 +59,7 @@ func (s) TestXDSLoadReporting(t *testing.T) {
 		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
 	}
 
-	got, err := xdsC.WaitForReportLoad()
+	got, err := xdsC.WaitForReportLoad(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
 	}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 729be4cc0726..34c26479fa94 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -19,13 +19,15 @@
 package client
 
 import (
+	"context"
 	"testing"
 	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -47,6 +49,7 @@ const (
 	testEDSName = "test-eds"
 
 	defaultTestWatchExpiryTimeout = 500 * time.Millisecond
+	defaultTestTimeout            = 1 * time.Second
 )
 
 func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
@@ -58,7 +61,7 @@ func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
 		Config: bootstrap.Config{
 			BalancerName: balancerName,
 			Creds:        grpc.WithInsecure(),
-			NodeProto:    testutils.EmptyNodeProtoV2,
+			NodeProto:    xdstestutils.EmptyNodeProtoV2,
 		},
 		WatchExpiryTimeout: watchExpiryTimeout,
 	}
@@ -132,12 +135,18 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		// Calls another watch inline, to ensure there's deadlock.
 		c.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
-		if _, err := v2Client.addWatches[ClusterResource].Receive(); firstTime && err != nil {
+
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); firstTime && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 		firstTime = false
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -146,7 +155,7 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -155,7 +164,7 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		testCDSName: wantUpdate2,
 	})
 
-	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index eedda3e0a2d5..3c816f9c71c5 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -19,9 +19,10 @@
 package client
 
 import (
+	"context"
 	"testing"
 
-	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/internal/testutils"
 )
 
 type clusterUpdateErr struct {
@@ -52,7 +53,10 @@ func (s) TestClusterWatch(t *testing.T) {
 	cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -67,7 +71,7 @@ func (s) TestClusterWatch(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -77,7 +81,7 @@ func (s) TestClusterWatch(t *testing.T) {
 		"randomName": {},
 	})
 
-	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected clusterUpdate: %+v, %v, want channel recv timeout", u, err)
 	}
 
@@ -87,7 +91,7 @@ func (s) TestClusterWatch(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := clusterUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -109,6 +113,9 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	var clusterUpdateChs []*testutils.Channel
 	var cancelLastWatch func()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	const count = 2
 	for i := 0; i < count; i++ {
 		clusterUpdateCh := testutils.NewChannel()
@@ -116,8 +123,13 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[ClusterResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -127,7 +139,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		if u, err := clusterUpdateChs[i].Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -139,12 +151,12 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+		if u, err := clusterUpdateChs[i].Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := clusterUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := clusterUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -167,14 +179,21 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	const count = 2
 
 	// Two watches for the same name.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		clusterUpdateCh := testutils.NewChannel()
 		clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh)
 		c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[ClusterResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -183,7 +202,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -195,12 +214,12 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
+		if u, err := clusterUpdateChs[i].Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
 			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -223,7 +242,10 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -232,7 +254,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -241,17 +263,19 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[ClusterResource].Receive(); err == nil {
+	if n, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := clusterUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -275,11 +299,14 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	u, err := clusterUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
+	u, err := clusterUpdateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get clusterUpdate: %v", err)
 	}
@@ -311,7 +338,10 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -320,13 +350,13 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
+	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Wait for an error, the error should never happen.
-	u, err := clusterUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
-	if err != testutils.ErrRecvTimeout {
+	u, err := clusterUpdateCh.Receive(ctx)
+	if err != context.DeadlineExceeded {
 		t.Fatalf("got unexpected: %v, %v, want recv timeout", u.(clusterUpdateErr).u, u.(clusterUpdateErr).err)
 	}
 }
@@ -353,7 +383,10 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 		clusterUpdateCh1.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
@@ -361,7 +394,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -372,11 +405,11 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 		testCDSName + "2": wantUpdate2,
 	})
 
-	if u, err := clusterUpdateCh1.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
+	if u, err := clusterUpdateCh1.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
-	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -386,12 +419,12 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	})
 
 	// watcher 1 should get an error.
-	if u, err := clusterUpdateCh1.Receive(); err != nil || ErrType(u.(clusterUpdateErr).err) != ErrorTypeResourceNotFound {
+	if u, err := clusterUpdateCh1.Receive(ctx); err != nil || ErrType(u.(clusterUpdateErr).err) != ErrorTypeResourceNotFound {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
 	}
 
 	// watcher 2 should get the same update again.
-	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -401,12 +434,14 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	})
 
 	// watcher 1 should get an error.
-	if u, err := clusterUpdateCh1.Receive(); err != testutils.ErrRecvTimeout {
+	if u, err := clusterUpdateCh1.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, want receiving from channel timeout", u)
 	}
 
 	// watcher 2 should get the same update again.
-	if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index 09dc80817284..c414e3a27bb4 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -19,13 +19,14 @@
 package client
 
 import (
+	"context"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 var (
@@ -71,7 +72,10 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -80,7 +84,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
+	if u, err := endpointsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -89,7 +93,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 		"randomName": {},
 	})
 
-	if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := endpointsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
@@ -99,7 +103,9 @@ func (s) TestEndpointsWatch(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := endpointsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -123,14 +129,21 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 
 	var cancelLastWatch func()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		endpointsUpdateCh := testutils.NewChannel()
 		endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh)
 		cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[EndpointsResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -140,7 +153,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
+		if u, err := endpointsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -152,12 +165,12 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
+		if u, err := endpointsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := endpointsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := endpointsUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -180,14 +193,21 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	const count = 2
 
 	// Two watches for the same name.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		endpointsUpdateCh := testutils.NewChannel()
 		endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh)
 		c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[EndpointsResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -196,7 +216,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -208,12 +228,12 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, endpointsCmpOpts...) {
+		if u, err := endpointsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, endpointsCmpOpts...) {
 			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, endpointsCmpOpts...) {
+	if u, err := endpointsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -236,7 +256,10 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -245,7 +268,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 		testCDSName: wantUpdate,
 	})
 
-	if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
+	if u, err := endpointsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -254,17 +277,19 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[EndpointsResource].Receive(); err == nil {
+	if n, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := endpointsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
 		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := endpointsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -288,11 +313,13 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	u, err := endpointsUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
+	u, err := endpointsUpdateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get endpointsUpdate: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index b2bcef8c7237..b3a25be15621 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -19,9 +19,10 @@
 package client
 
 import (
+	"context"
 	"testing"
 
-	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/internal/testutils"
 )
 
 type ldsUpdateErr struct {
@@ -49,7 +50,10 @@ func (s) TestLDSWatch(t *testing.T) {
 	cancelWatch := c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -58,7 +62,7 @@ func (s) TestLDSWatch(t *testing.T) {
 		testLDSName: wantUpdate,
 	})
 
-	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+	if u, err := ldsUpdateCh.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -68,7 +72,7 @@ func (s) TestLDSWatch(t *testing.T) {
 		"randomName": {},
 	})
 
-	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+	if u, err := ldsUpdateCh.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
@@ -78,7 +82,7 @@ func (s) TestLDSWatch(t *testing.T) {
 		testLDSName: wantUpdate,
 	})
 
-	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := ldsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -102,14 +106,21 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 
 	var cancelLastWatch func()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
 		cancelLastWatch = c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[ListenerResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -119,7 +130,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+		if u, err := ldsUpdateChs[i].Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -131,12 +142,12 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+		if u, err := ldsUpdateChs[i].Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := ldsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := ldsUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -159,14 +170,21 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	const count = 2
 
 	// Two watches for the same name.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
 		c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[ListenerResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -175,7 +193,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -187,12 +205,12 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
+		if u, err := ldsUpdateChs[i].Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
 			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -215,7 +233,10 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -224,7 +245,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 		testLDSName: wantUpdate,
 	})
 
-	if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+	if u, err := ldsUpdateCh.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -233,17 +254,19 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[ListenerResource].Receive(); err == nil {
+	if n, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := ldsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -270,7 +293,10 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 		ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
@@ -278,7 +304,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -289,11 +315,11 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 		testLDSName + "2": wantUpdate2,
 	})
 
-	if u, err := ldsUpdateCh1.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
+	if u, err := ldsUpdateCh1.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
-	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -303,12 +329,12 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	})
 
 	// watcher 1 should get an error.
-	if u, err := ldsUpdateCh1.Receive(); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound {
+	if u, err := ldsUpdateCh1.Receive(ctx); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
 	}
 
 	// watcher 2 should get the same update again.
-	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -318,12 +344,14 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	})
 
 	// watcher 1 should get an error.
-	if u, err := ldsUpdateCh1.Receive(); err != testutils.ErrRecvTimeout {
+	if u, err := ldsUpdateCh1.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, want receiving from channel timeout", u)
 	}
 
 	// watcher 2 should get the same update again.
-	if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index f9bb30d559f4..abaec80a4c9c 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -19,10 +19,11 @@
 package client
 
 import (
+	"context"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/internal/testutils"
 )
 
 type rdsUpdateErr struct {
@@ -50,7 +51,9 @@ func (s) TestRDSWatch(t *testing.T) {
 	cancelWatch := c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -59,7 +62,7 @@ func (s) TestRDSWatch(t *testing.T) {
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+	if u, err := rdsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -68,7 +71,7 @@ func (s) TestRDSWatch(t *testing.T) {
 		"randomName": {},
 	})
 
-	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := rdsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
@@ -78,7 +81,9 @@ func (s) TestRDSWatch(t *testing.T) {
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := rdsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -102,14 +107,21 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 
 	var cancelLastWatch func()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
 		cancelLastWatch = c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[RouteConfigResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -119,7 +131,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		if u, err := rdsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
@@ -131,12 +143,12 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count-1; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		if u, err := rdsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := rdsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := rdsUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -159,14 +171,21 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	const count = 2
 
 	// Two watches for the same name.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
 		c.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
-		if _, err := v2Client.addWatches[RouteConfigResource].Receive(); i == 0 && err != nil {
-			t.Fatalf("want new watch to start, got error %v", err)
+
+		if i == 0 {
+			// A new watch is registered on the underlying API client only for
+			// the first iteration because we are using the same resource name.
+			if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+				t.Fatalf("want new watch to start, got error %v", err)
+			}
 		}
 	}
 
@@ -175,7 +194,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	c.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -187,12 +206,12 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+		if u, err := rdsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
 		}
 	}
 
-	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+	if u, err := rdsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -215,7 +234,9 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -224,7 +245,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 		testRDSName: wantUpdate,
 	})
 
-	if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+	if u, err := rdsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -233,17 +254,19 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[RouteConfigResource].Receive(); err == nil {
+	if n, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := rdsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout {
+	if u, err := rdsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 6f994686b8e4..5a6a401216ac 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -19,12 +19,13 @@
 package client
 
 import (
+	"context"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 
-	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/internal/testutils"
 )
 
 type serviceUpdateErr struct {
@@ -56,20 +57,22 @@ func (s) TestServiceWatch(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -87,7 +90,7 @@ func (s) TestServiceWatch(t *testing.T) {
 			}},
 		},
 	})
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -114,20 +117,22 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -135,7 +140,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName + "2"},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
@@ -144,7 +149,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
+	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
@@ -154,7 +159,9 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		testRDSName + "2": {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
@@ -181,20 +188,22 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -204,7 +213,7 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		serviceUpdateCh2.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	u, err := serviceUpdateCh2.Receive()
+	u, err := serviceUpdateCh2.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
 	}
@@ -225,11 +234,11 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
-	if u, err := serviceUpdateCh2.Receive(); err != testutils.ErrRecvTimeout {
+	if u, err := serviceUpdateCh2.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -253,10 +262,13 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
+	u, err := serviceUpdateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
 	}
@@ -288,17 +300,19 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{})
-	u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2)
+	u, err := serviceUpdateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
 	}
@@ -331,18 +345,20 @@ func (s) TestServiceWatchWithClientClose(t *testing.T) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Client is closed before it receives the RDS response.
 	c.Close()
-	if u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2); err != testutils.ErrRecvTimeout {
+	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -369,20 +385,22 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
@@ -390,7 +408,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if v, err := v2Client.removeWatches[RouteConfigResource].Receive(); err == nil {
+	if v, err := v2Client.removeWatches[RouteConfigResource].Receive(ctx); err == nil {
 		t.Fatalf("unexpected rds watch cancel: %v", v)
 	}
 }
@@ -420,30 +438,32 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 
-	if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Remove LDS resource, should cancel the RDS watch, and trigger resource
 	// removed error.
 	v2Client.r.NewListeners(map[string]ListenerUpdate{})
-	if _, err := v2Client.removeWatches[RouteConfigResource].Receive(); err != nil {
+	if _, err := v2Client.removeWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, got error %v", err)
 	}
-	if u, err := serviceUpdateCh.Receive(); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
 	}
 
@@ -452,7 +472,7 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}},
 	})
-	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
+	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
 	}
 
@@ -462,17 +482,21 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	v2Client.r.NewListeners(map[string]ListenerUpdate{
 		testLDSName: {RouteConfigName: testRDSName},
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout {
+	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
 	}
 
 	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}},
 	})
-	if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, nil}, serviceCmpOpts...) {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, nil}, serviceCmpOpts...) {
 		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
 	}
 }
diff --git a/xds/internal/client/v2/client_ack_test.go b/xds/internal/client/v2/client_ack_test.go
index 40f5668297e2..2b38d3aa3709 100644
--- a/xds/internal/client/v2/client_ack_test.go
+++ b/xds/internal/client/v2/client_ack_test.go
@@ -18,6 +18,7 @@
 package v2
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"testing"
@@ -28,12 +29,14 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
+const defaultTestTimeout = 1 * time.Second
+
 func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cbRDS, cbCDS, cbEDS *testutils.Channel, cleanup func()) {
 	cbLDS = testutils.NewChannel()
 	cbRDS = testutils.NewChannel()
@@ -71,7 +74,9 @@ func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cb
 
 // compareXDSRequest reads requests from channel, compare it with want.
 func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string) error {
-	val, err := ch.Receive()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ch.Receive(ctx)
 	if err != nil {
 		return err
 	}
@@ -133,7 +138,9 @@ func sendGoodResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakese
 	}
 	t.Logf("Good %v response acked", rType)
 
-	if _, err := callbackCh.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := callbackCh.Receive(ctx); err != nil {
 		return "", fmt.Errorf("timeout when expecting %v update", rType)
 	}
 	t.Logf("Good %v response callback executed", rType)
@@ -408,7 +415,9 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	}
 	versionCDS++
 
-	if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if req, err := fakeServer.XDSRequestChan.Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
 	}
 
@@ -417,11 +426,14 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	t.Logf("Good %v response pushed to fakeServer...", xdsclient.ClusterResource)
 
 	// Expect no ACK because watch was canceled.
-	if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout {
+	if req, err := fakeServer.XDSRequestChan.Receive(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
 	}
+
 	// Still expected an callback update, because response was good.
-	if _, err := cbCDS.Receive(); err != nil {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := cbCDS.Receive(ctx); err != nil {
 		t.Fatalf("Timeout when expecting %v update", xdsclient.ClusterResource)
 	}
 
diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/client_rds_test.go
index 59e63d83e287..09a73335a015 100644
--- a/xds/internal/client/v2/client_rds_test.go
+++ b/xds/internal/client/v2/client_rds_test.go
@@ -19,6 +19,7 @@
 package v2
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -35,7 +36,9 @@ import (
 // watch.
 func doLDS(t *testing.T, v2c xdsclient.APIClient, fakeServer *fakeserver.Server) {
 	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout waiting for LDS request: %v", err)
 	}
 }
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index fecda6ad06d9..51ebcf547519 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -19,6 +19,7 @@
 package v2
 
 import (
+	"context"
 	"errors"
 	"reflect"
 	"testing"
@@ -29,10 +30,10 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
 
@@ -421,7 +422,9 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 
 	// Wait till the request makes it to the fakeServer. This ensures that
 	// the watch request has been processed by the v2Client.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout waiting for an xDS request: %v", err)
 	}
 
@@ -452,16 +455,16 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	// Cannot directly compare test.wantUpdate with nil (typed vs non-typed nil:
 	// https://golang.org/doc/faq#nil_error).
 	if c := test.wantUpdate; c == nil || (reflect.ValueOf(c).Kind() == reflect.Ptr && reflect.ValueOf(c).IsNil()) {
-		update, err := gotUpdateCh.Receive()
-		if err == testutils.ErrRecvTimeout {
+		update, err := gotUpdateCh.Receive(ctx)
+		if err == context.DeadlineExceeded {
 			return
 		}
 		t.Fatalf("Unexpected update: +%v", update)
 	}
 
 	wantUpdate := reflect.ValueOf(test.wantUpdate).Elem().Interface()
-	uErr, err := gotUpdateCh.Receive()
-	if err == testutils.ErrRecvTimeout {
+	uErr, err := gotUpdateCh.Receive(ctx)
+	if err == context.DeadlineExceeded {
 		t.Fatal("Timeout expecting xDS update")
 	}
 	gotUpdate := uErr.(updateErr).u
@@ -533,7 +536,9 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	t.Log("Started xds v2Client...")
 
 	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
 	t.Log("FakeServer received request...")
@@ -552,7 +557,7 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 		t.Fatal("Received unexpected LDS callback")
 	}
 
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
 	t.Log("FakeServer received request after backoff...")
@@ -583,7 +588,9 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	t.Log("Started xds v2Client...")
 
 	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
 	t.Log("FakeServer received request...")
@@ -591,20 +598,20 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
 	t.Log("Good LDS response pushed to fakeServer...")
 
-	if _, err := callbackCh.Receive(); err != nil {
+	if _, err := callbackCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout when expecting LDS update")
 	}
 
 	// Read the ack, so the next request is sent after stream re-creation.
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS ACK")
 	}
 
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")}
 	t.Log("Bad LDS response pushed to fakeServer...")
 
-	val, err := fakeServer.XDSRequestChan.Receive()
-	if err == testutils.ErrRecvTimeout {
+	val, err := fakeServer.XDSRequestChan.Receive(ctx)
+	if err == context.DeadlineExceeded {
 		t.Fatalf("Timeout expired when expecting LDS update")
 	}
 	gotRequest := val.(*fakeserver.Request)
@@ -657,7 +664,9 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
 
 	// The watcher should receive an update, with a timeout error in it.
-	if v, err := callbackCh.TimedReceive(100 * time.Millisecond); err == nil {
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	if v, err := callbackCh.Receive(ctx); err == nil {
 		t.Fatalf("Expect an timeout error from watcher, got %v", v)
 	}
 
@@ -667,7 +676,9 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 		Addresses: []resolver.Address{{Addr: fakeServer.Address}},
 	})
 
-	if _, err := fakeServer.XDSRequestChan.Receive(); err != nil {
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")
 	}
 	t.Log("FakeServer received request...")
@@ -675,7 +686,7 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1}
 	t.Log("Good LDS response pushed to fakeServer...")
 
-	if v, err := callbackCh.Receive(); err != nil {
+	if v, err := callbackCh.Receive(ctx); err != nil {
 		t.Fatal("Timeout when expecting LDS update")
 	} else if _, ok := v.(xdsclient.ListenerUpdate); !ok {
 		t.Fatalf("Expect an LDS update from watcher, got %v", v)
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 33f2e2695a0c..98c6d6a2e5b1 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -24,11 +24,13 @@ import (
 	"fmt"
 	"net"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	xdsinternal "google.golang.org/grpc/xds/internal"
@@ -36,21 +38,22 @@ import (
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
 const (
-	targetStr    = "target"
-	cluster      = "cluster"
-	balancerName = "dummyBalancer"
+	targetStr          = "target"
+	cluster            = "cluster"
+	balancerName       = "dummyBalancer"
+	defaultTestTimeout = 1 * time.Second
 )
 
 var (
 	validConfig = bootstrap.Config{
 		BalancerName: balancerName,
 		Creds:        grpc.WithInsecure(),
-		NodeProto:    testutils.EmptyNodeProtoV2,
+		NodeProto:    xdstestutils.EmptyNodeProtoV2,
 	}
 	target = resolver.Target{Endpoint: targetStr}
 )
@@ -135,7 +138,7 @@ func TestResolverBuilder(t *testing.T) {
 			rbo:  resolver.BuildOptions{},
 			config: bootstrap.Config{
 				Creds:     grpc.WithInsecure(),
-				NodeProto: testutils.EmptyNodeProtoV2,
+				NodeProto: xdstestutils.EmptyNodeProtoV2,
 			},
 			wantErr: true,
 		},
@@ -144,7 +147,7 @@ func TestResolverBuilder(t *testing.T) {
 			rbo:  resolver.BuildOptions{},
 			config: bootstrap.Config{
 				BalancerName: balancerName,
-				NodeProto:    testutils.EmptyNodeProtoV2,
+				NodeProto:    xdstestutils.EmptyNodeProtoV2,
 			},
 			xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}),
 			wantErr:       false,
@@ -248,7 +251,7 @@ func testSetup(t *testing.T, opts setupOpts) (*xdsResolver, *testClientConn, fun
 func waitForWatchService(t *testing.T, xdsC *fakeclient.Client, wantTarget string) {
 	t.Helper()
 
-	gotTarget, err := xdsC.WaitForWatchService()
+	gotTarget, err := xdsC.WaitForWatchService(context.Background())
 	if err != nil {
 		t.Fatalf("xdsClient.WatchService failed with error: %v", err)
 	}
@@ -273,7 +276,10 @@ func TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	// update is triggerred on the ClientConn.
 	xdsR.Close()
 	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil)
-	if gotVal, gotErr := tcc.stateCh.Receive(); gotErr != testutils.ErrRecvTimeout {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if gotVal, gotErr := tcc.stateCh.Receive(ctx); gotErr != context.DeadlineExceeded {
 		t.Fatalf("ClientConn.UpdateState called after xdsResolver is closed: %v", gotVal)
 	}
 }
@@ -297,7 +303,10 @@ func TestXDSResolverBadServiceUpdate(t *testing.T) {
 	// ReportError method to be called on the ClientConn.
 	suErr := errors.New("bad serviceupdate")
 	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
-	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != nil || gotErrVal != suErr {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr)
 	}
 }
@@ -337,7 +346,10 @@ func TestXDSResolverGoodServiceUpdate(t *testing.T) {
 		// Invoke the watchAPI callback with a good service update and wait for the
 		// UpdateState method to be called on the ClientConn.
 		xdsC.InvokeWatchServiceCallback(tt.su, nil)
-		gotState, err := tcc.stateCh.Receive()
+
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		gotState, err := tcc.stateCh.Receive(ctx)
 		if err != nil {
 			t.Fatalf("ClientConn.UpdateState returned error: %v", err)
 		}
@@ -377,14 +389,17 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	// ReportError method to be called on the ClientConn.
 	suErr := errors.New("bad serviceupdate")
 	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
-	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != nil || gotErrVal != suErr {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr)
 	}
 
 	// Invoke the watchAPI callback with a good service update and wait for the
 	// UpdateState method to be called on the ClientConn.
 	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil)
-	gotState, err := tcc.stateCh.Receive()
+	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
 	}
@@ -400,7 +415,7 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	// ReportError method to be called on the ClientConn.
 	suErr2 := errors.New("bad serviceupdate 2")
 	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr2)
-	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr2 {
+	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != nil || gotErrVal != suErr2 {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr2)
 	}
 }
@@ -425,10 +440,16 @@ func TestXDSResolverResourceNotFoundError(t *testing.T) {
 	// ReportError method to be called on the ClientConn.
 	suErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "resource removed error")
 	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
-	if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != testutils.ErrRecvTimeout {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != context.DeadlineExceeded {
 		t.Fatalf("ClientConn.ReportError() received %v, %v, want channel recv timeout", gotErrVal, gotErr)
 	}
-	gotState, err := tcc.stateCh.Receive()
+
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
 	}
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index ec3c4b5c6e91..ecb21e6202e7 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -16,6 +16,7 @@
  *
  */
 
+// Package testutils provides utility types, for use in xds tests.
 package testutils
 
 import (
diff --git a/xds/internal/testutils/channel.go b/xds/internal/testutils/channel.go
deleted file mode 100644
index 1715a01ff68f..000000000000
--- a/xds/internal/testutils/channel.go
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Package testutils provides utility types, for use in xds tests.
-package testutils
-
-import (
-	"errors"
-	"time"
-)
-
-// ErrRecvTimeout is an error to indicate that a receive operation on the
-// channel timed out.
-var ErrRecvTimeout = errors.New("timed out when waiting for value on channel")
-
-const (
-	// DefaultChanRecvTimeout is the default timeout for receive operations on the
-	// underlying channel.
-	DefaultChanRecvTimeout = 1 * time.Second
-	// DefaultChanBufferSize is the default buffer size of the underlying channel.
-	DefaultChanBufferSize = 1
-)
-
-// Channel wraps a generic channel and provides a timed receive operation.
-type Channel struct {
-	ch chan interface{}
-}
-
-// Send sends value on the underlying channel.
-func (cwt *Channel) Send(value interface{}) {
-	cwt.ch <- value
-}
-
-// Replace clears the value on the underlying channel, and sends the new value.
-//
-// It's expected to be used with a size-1 channel, to only keep the most
-// up-to-date item.
-func (cwt *Channel) Replace(value interface{}) {
-	select {
-	case <-cwt.ch:
-	default:
-	}
-	cwt.ch <- value
-}
-
-// TimedReceive returns the value received on the underlying channel, or
-// ErrRecvTimeout if timeout amount of time elapsed.
-func (cwt *Channel) TimedReceive(timeout time.Duration) (interface{}, error) {
-	timer := time.NewTimer(timeout)
-	select {
-	case <-timer.C:
-		return nil, ErrRecvTimeout
-	case got := <-cwt.ch:
-		timer.Stop()
-		return got, nil
-	}
-}
-
-// Receive returns the value received on the underlying channel, or
-// ErrRecvTimeout if DefaultChanRecvTimeout amount of time elapses.
-func (cwt *Channel) Receive() (interface{}, error) {
-	return cwt.TimedReceive(DefaultChanRecvTimeout)
-}
-
-// NewChannel returns a new Channel.
-func NewChannel() *Channel {
-	return NewChannelWithSize(DefaultChanBufferSize)
-}
-
-// NewChannelWithSize returns a new Channel with a buffer of bufSize.
-func NewChannelWithSize(bufSize int) *Channel {
-	return &Channel{ch: make(chan interface{}, bufSize)}
-}
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index aacade2fbd0a..79dabc00b56b 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -20,11 +20,12 @@
 package fakeclient
 
 import (
+	"context"
 	"sync"
 
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/testutils"
 )
 
 // Client is a fake implementation of an xds client. It exposes a bunch of
@@ -58,10 +59,10 @@ func (xdsC *Client) WatchService(target string, callback func(xdsclient.ServiceU
 	}
 }
 
-// WaitForWatchService waits for WatchService to be invoked on this client
-// within a reasonable timeout, and returns the serviceName being watched.
-func (xdsC *Client) WaitForWatchService() (string, error) {
-	val, err := xdsC.suWatchCh.Receive()
+// WaitForWatchService waits for WatchService to be invoked on this client and
+// returns the serviceName being watched.
+func (xdsC *Client) WaitForWatchService(ctx context.Context) (string, error) {
+	val, err := xdsC.suWatchCh.Receive(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -88,10 +89,10 @@ func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsclient.Clu
 	}
 }
 
-// WaitForWatchCluster waits for WatchCluster to be invoked on this client
-// within a reasonable timeout, and returns the clusterName being watched.
-func (xdsC *Client) WaitForWatchCluster() (string, error) {
-	val, err := xdsC.cdsWatchCh.Receive()
+// WaitForWatchCluster waits for WatchCluster to be invoked on this client and
+// returns the clusterName being watched.
+func (xdsC *Client) WaitForWatchCluster(ctx context.Context) (string, error) {
+	val, err := xdsC.cdsWatchCh.Receive(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -106,10 +107,10 @@ func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.ClusterUpdate, e
 	xdsC.cdsCb(update, err)
 }
 
-// WaitForCancelClusterWatch waits for a CDS watch to be cancelled within a
-// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise.
-func (xdsC *Client) WaitForCancelClusterWatch() error {
-	_, err := xdsC.cdsCancelCh.Receive()
+// WaitForCancelClusterWatch waits for a CDS watch to be cancelled  and returns
+// context.DeadlineExceeded otherwise.
+func (xdsC *Client) WaitForCancelClusterWatch(ctx context.Context) error {
+	_, err := xdsC.cdsCancelCh.Receive(ctx)
 	return err
 }
 
@@ -125,10 +126,10 @@ func (xdsC *Client) WatchEndpoints(clusterName string, callback func(xdsclient.E
 	}
 }
 
-// WaitForWatchEDS waits for WatchEndpoints to be invoked on this client within a
-// reasonable timeout, and returns the clusterName being watched.
-func (xdsC *Client) WaitForWatchEDS() (string, error) {
-	val, err := xdsC.edsWatchCh.Receive()
+// WaitForWatchEDS waits for WatchEndpoints to be invoked on this client and
+// returns the clusterName being watched.
+func (xdsC *Client) WaitForWatchEDS(ctx context.Context) (string, error) {
+	val, err := xdsC.edsWatchCh.Receive(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -143,10 +144,10 @@ func (xdsC *Client) InvokeWatchEDSCallback(update xdsclient.EndpointsUpdate, err
 	xdsC.edsCb(update, err)
 }
 
-// WaitForCancelEDSWatch waits for a EDS watch to be cancelled within a
-// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise.
-func (xdsC *Client) WaitForCancelEDSWatch() error {
-	_, err := xdsC.edsCancelCh.Receive()
+// WaitForCancelEDSWatch waits for a EDS watch to be cancelled and returns
+// context.DeadlineExceeded otherwise.
+func (xdsC *Client) WaitForCancelEDSWatch(ctx context.Context) error {
+	_, err := xdsC.edsCancelCh.Receive(ctx)
 	return err
 }
 
@@ -164,10 +165,10 @@ func (xdsC *Client) ReportLoad(server string, clusterName string, loadStore lrs.
 	return func() {}
 }
 
-// WaitForReportLoad waits for ReportLoad to be invoked on this client within a
-// reasonable timeout, and returns the arguments passed to it.
-func (xdsC *Client) WaitForReportLoad() (ReportLoadArgs, error) {
-	val, err := xdsC.loadReportCh.Receive()
+// WaitForReportLoad waits for ReportLoad to be invoked on this client and
+// returns the arguments passed to it.
+func (xdsC *Client) WaitForReportLoad(ctx context.Context) (ReportLoadArgs, error) {
+	val, err := xdsC.loadReportCh.Receive(ctx)
 	return val.(ReportLoadArgs), err
 }
 
@@ -176,10 +177,10 @@ func (xdsC *Client) Close() {
 	xdsC.closeCh.Send(nil)
 }
 
-// WaitForClose waits for Close to be invoked on this client within a
-// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise.
-func (xdsC *Client) WaitForClose() error {
-	_, err := xdsC.closeCh.Receive()
+// WaitForClose waits for Close to be invoked on this client and returns
+// context.DeadlineExceeded otherwise.
+func (xdsC *Client) WaitForClose(ctx context.Context) error {
+	_, err := xdsC.closeCh.Receive(ctx)
 	return err
 }
 
diff --git a/xds/internal/testutils/fakeserver/server.go b/xds/internal/testutils/fakeserver/server.go
index e1663c60444c..4cff72087cec 100644
--- a/xds/internal/testutils/fakeserver/server.go
+++ b/xds/internal/testutils/fakeserver/server.go
@@ -29,8 +29,8 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/xds/internal/testutils"
 
 	discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"

From d8ef479ab79a776d354d28e24ba6022fdeddfacf Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 28 Aug 2020 09:59:40 -0700
Subject: [PATCH 180/481] internal: fix build for gae (#3852)

---
 security/advancedtls/advancedtls.go           |  27 --
 .../advancedtls_integration_test.go           |   4 +-
 security/advancedtls/go.mod                   |   9 +-
 security/advancedtls/go.sum                   | 311 ++++++++++++++++--
 security/advancedtls/sni.go                   |  53 +++
 5 files changed, 341 insertions(+), 63 deletions(-)
 create mode 100644 security/advancedtls/sni.go

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index c21ff753aab6..40a9fd5f735e 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -506,30 +506,3 @@ func cloneTLSConfig(cfg *tls.Config) *tls.Config {
 	}
 	return cfg.Clone()
 }
-
-// buildGetCertificates returns the certificate that matches the SNI field
-// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates.
-func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
-	if o.GetCertificates == nil {
-		return nil, fmt.Errorf("function GetCertificates must be specified")
-	}
-	certificates, err := o.GetCertificates(clientHello)
-	if err != nil {
-		return nil, err
-	}
-	if len(certificates) == 0 {
-		return nil, fmt.Errorf("no certificates configured")
-	}
-	// If users pass in only one certificate, return that certificate.
-	if len(certificates) == 1 {
-		return certificates[0], nil
-	}
-	// Choose the SNI certificate using SupportsCertificate.
-	for _, cert := range certificates {
-		if err := clientHello.SupportsCertificate(cert); err == nil {
-			return cert, nil
-		}
-	}
-	// If nothing matches, return the first certificate.
-	return certificates[0], nil
-}
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 195da18385cb..a794e0d00370 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -409,7 +409,7 @@ func (s) TestEnd2End(t *testing.T) {
 				t.Fatalf("failed to listen: %v", err)
 			}
 			defer lis.Close()
-			pb.RegisterGreeterServer(s, &serverImpl{})
+			pb.RegisterGreeterService(s, pb.NewGreeterService(&serverImpl{}))
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{
 				Certificates:         test.clientCert,
@@ -451,7 +451,7 @@ func (s) TestEnd2End(t *testing.T) {
 				t.Fatal(err)
 			}
 			defer conn2.Close()
-			//// --------------------------------------------------------------------
+			// ----------------------------------------------------------------------
 			stage.increase()
 			// ------------------------Scenario 4------------------------------------
 			// stage = 2,  new connection should succeed
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index 58823b03e877..b8fe4c9cadce 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -3,10 +3,11 @@ module google.golang.org/grpc/security/advancedtls
 go 1.13
 
 require (
-	github.com/google/go-cmp v0.4.0
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
-	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
-	golang.org/x/text v0.3.3 // indirect
+	github.com/google/go-cmp v0.5.1
 	google.golang.org/grpc v1.31.0
 	google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f
 )
+
+replace google.golang.org/grpc => ../../
+
+replace google.golang.org/grpc/examples => ../../examples
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 8283cf07e736..b1759c051067 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,17 +1,64 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -20,58 +67,249 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
-golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad h1:uAwc13+y0Y8QZLTYhLCu6lHhnG99ecQU5FYTj8zxAng=
-google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f h1:HNNmM2dnxUknBEJDvuCRZcUzhGbhkz00ckV2ha2gFJY=
-google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f/go.mod h1:TGiSRL2BBv2WqzfsFNWYp/pkWdtf5kbZS/DQ9Ee3mWk=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -80,7 +318,20 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/security/advancedtls/sni.go b/security/advancedtls/sni.go
new file mode 100644
index 000000000000..6c8623283cf5
--- /dev/null
+++ b/security/advancedtls/sni.go
@@ -0,0 +1,53 @@
+// +build !appengine
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+	"fmt"
+)
+
+// buildGetCertificates returns the certificate that matches the SNI field
+// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates.
+func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
+	if o.GetCertificates == nil {
+		return nil, fmt.Errorf("function GetCertificates must be specified")
+	}
+	certificates, err := o.GetCertificates(clientHello)
+	if err != nil {
+		return nil, err
+	}
+	if len(certificates) == 0 {
+		return nil, fmt.Errorf("no certificates configured")
+	}
+	// If users pass in only one certificate, return that certificate.
+	if len(certificates) == 1 {
+		return certificates[0], nil
+	}
+	// Choose the SNI certificate using SupportsCertificate.
+	for _, cert := range certificates {
+		if err := clientHello.SupportsCertificate(cert); err == nil {
+			return cert, nil
+		}
+	}
+	// If nothing matches, return the first certificate.
+	return certificates[0], nil
+}

From 48bf772d0c298ad151783cecd54ca84bd4e1c2f4 Mon Sep 17 00:00:00 2001
From: Anton Nep <antonbn@gmail.com>
Date: Tue, 1 Sep 2020 16:23:56 -0700
Subject: [PATCH 181/481] credentials/alts: ClientAuthorizationCheck to
 case-fold compare of peer SA (#3792)

---
 credentials/alts/utils.go      | 7 ++++---
 credentials/alts/utils_test.go | 7 +++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/credentials/alts/utils.go b/credentials/alts/utils.go
index e46280ad5f58..9a300bc19aa3 100644
--- a/credentials/alts/utils.go
+++ b/credentials/alts/utils.go
@@ -152,12 +152,13 @@ func AuthInfoFromPeer(p *peer.Peer) (AuthInfo, error) {
 func ClientAuthorizationCheck(ctx context.Context, expectedServiceAccounts []string) error {
 	authInfo, err := AuthInfoFromContext(ctx)
 	if err != nil {
-		return status.Newf(codes.PermissionDenied, "The context is not an ALTS-compatible context: %v", err).Err()
+		return status.Errorf(codes.PermissionDenied, "The context is not an ALTS-compatible context: %v", err)
 	}
+	peer := authInfo.PeerServiceAccount()
 	for _, sa := range expectedServiceAccounts {
-		if authInfo.PeerServiceAccount() == sa {
+		if strings.EqualFold(peer, sa) {
 			return nil
 		}
 	}
-	return status.Newf(codes.PermissionDenied, "Client %v is not authorized", authInfo.PeerServiceAccount()).Err()
+	return status.Errorf(codes.PermissionDenied, "Client %v is not authorized", peer)
 }
diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go
index f0ab2377c953..7c84c3bc9f92 100644
--- a/credentials/alts/utils_test.go
+++ b/credentials/alts/utils_test.go
@@ -175,6 +175,13 @@ func (s) TestClientAuthorizationCheck(t *testing.T) {
 			true,
 			codes.OK, // err is nil, code is OK.
 		},
+		{
+			"working case (case ignored)",
+			peer.NewContext(ctx, p),
+			[]string{strings.ToUpper(testServiceAccount1), testServiceAccount2},
+			true,
+			codes.OK, // err is nil, code is OK.
+		},
 		{
 			"context does not have AuthInfo",
 			ctx,

From b5802b5f3b8cee27916ac26293a82f203c61be40 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 1 Sep 2020 16:56:52 -0700
Subject: [PATCH 182/481] xds: v3 support for LRS. (#3856)

---
 .../balancer/balancergroup/balancergroup.go   |  62 ++-
 .../balancergroup/balancergroup_test.go       |  63 ++-
 xds/internal/balancer/edsbalancer/eds.go      |  10 +-
 xds/internal/balancer/edsbalancer/eds_impl.go |  28 +-
 .../balancer/edsbalancer/eds_impl_test.go     |  57 +-
 xds/internal/balancer/edsbalancer/eds_test.go |  12 +-
 .../edsbalancer/xds_client_wrapper.go         |  74 +--
 .../edsbalancer/xds_client_wrapper_test.go    |   6 +-
 xds/internal/balancer/lrs/balancer.go         |  42 +-
 xds/internal/balancer/lrs/balancer_test.go    |  92 +---
 xds/internal/balancer/lrs/picker.go           |  31 +-
 xds/internal/balancer/lrs/store.go            | 376 -------------
 xds/internal/balancer/lrs/store_test.go       | 517 ------------------
 xds/internal/client/client.go                 |  27 +-
 xds/internal/client/client_loadreport.go      |  41 +-
 xds/internal/client/client_test.go            |   3 +
 xds/internal/client/load/store.go             | 299 ++++++++++
 xds/internal/client/load/store_test.go        | 256 +++++++++
 xds/internal/client/transport_helper.go       |  87 ++-
 xds/internal/client/v2/client.go              |   6 +
 xds/internal/client/v2/loadreport.go          | 173 ++++++
 xds/internal/client/v3/client.go              |   6 +
 xds/internal/client/v3/loadreport.go          | 173 ++++++
 xds/internal/internal.go                      |  20 +-
 xds/internal/testutils/fakeclient/client.go   |  11 +-
 25 files changed, 1317 insertions(+), 1155 deletions(-)
 delete mode 100644 xds/internal/balancer/lrs/store.go
 delete mode 100644 xds/internal/balancer/lrs/store_test.go
 create mode 100644 xds/internal/client/load/store.go
 create mode 100644 xds/internal/client/load/store_test.go
 create mode 100644 xds/internal/client/v2/loadreport.go
 create mode 100644 xds/internal/client/v3/loadreport.go

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index 84de4b63ecfe..f8f2e0e9a25b 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -24,15 +24,22 @@ import (
 	"time"
 
 	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
+
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/cache"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 )
 
+// loadReporter wraps the methods from the loadStore that are used here.
+type loadReporter interface {
+	CallStarted(locality string)
+	CallFinished(locality string, err error)
+	CallServerLoad(locality, name string, val float64)
+}
+
 // subBalancerWrapper is used to keep the configurations that will be used to start
 // the underlying balancer. It can be called to start/stop the underlying
 // balancer.
@@ -180,7 +187,7 @@ func (sbc *subBalancerWrapper) stopBalancer() {
 type BalancerGroup struct {
 	cc        balancer.ClientConn
 	logger    *grpclog.PrefixLogger
-	loadStore lrs.Store
+	loadStore loadReporter
 
 	// stateAggregator is where the state/picker updates will be sent to. It's
 	// provided by the parent balancer, to build a picker with all the
@@ -235,7 +242,7 @@ var DefaultSubBalancerCloseTimeout = 15 * time.Minute
 
 // New creates a new BalancerGroup. Note that the BalancerGroup
 // needs to be started to work.
-func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore lrs.Store, logger *grpclog.PrefixLogger) *BalancerGroup {
+func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore loadReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
 	return &BalancerGroup{
 		cc:        cc,
 		logger:    logger,
@@ -493,38 +500,43 @@ const (
 type loadReportPicker struct {
 	p balancer.Picker
 
-	id        internal.LocalityID
-	loadStore lrs.Store
+	locality  string
+	loadStore loadReporter
 }
 
-func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore lrs.Store) *loadReportPicker {
+func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore loadReporter) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
-		id:        id,
+		locality:  id.String(),
 		loadStore: loadStore,
 	}
 }
 
 func (lrp *loadReportPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 	res, err := lrp.p.Pick(info)
-	if lrp.loadStore != nil && err == nil {
-		lrp.loadStore.CallStarted(lrp.id)
-		td := res.Done
-		res.Done = func(info balancer.DoneInfo) {
-			lrp.loadStore.CallFinished(lrp.id, info.Err)
-			if load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport); ok {
-				lrp.loadStore.CallServerLoad(lrp.id, serverLoadCPUName, load.CpuUtilization)
-				lrp.loadStore.CallServerLoad(lrp.id, serverLoadMemoryName, load.MemUtilization)
-				for n, d := range load.RequestCost {
-					lrp.loadStore.CallServerLoad(lrp.id, n, d)
-				}
-				for n, d := range load.Utilization {
-					lrp.loadStore.CallServerLoad(lrp.id, n, d)
-				}
-			}
-			if td != nil {
-				td(info)
-			}
+	if err != nil {
+		return res, err
+	}
+
+	lrp.loadStore.CallStarted(lrp.locality)
+	oldDone := res.Done
+	res.Done = func(info balancer.DoneInfo) {
+		if oldDone != nil {
+			oldDone(info)
+		}
+		lrp.loadStore.CallFinished(lrp.locality, info.Err)
+
+		load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport)
+		if !ok {
+			return
+		}
+		lrp.loadStore.CallServerLoad(lrp.locality, serverLoadCPUName, load.CpuUtilization)
+		lrp.loadStore.CallServerLoad(lrp.locality, serverLoadMemoryName, load.MemUtilization)
+		for n, d := range load.RequestCost {
+			lrp.loadStore.CallServerLoad(lrp.locality, n, d)
+		}
+		for n, d := range load.Utilization {
+			lrp.loadStore.CallServerLoad(lrp.locality, n, d)
 		}
 	}
 	return res, err
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 347404989d01..813ee8a5e7fc 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -32,14 +32,16 @@ import (
 
 	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
+	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -69,7 +71,7 @@ func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
 	}
 }
 
-func newTestBalancerGroup(t *testing.T, loadStore lrs.Store) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
+func newTestBalancerGroup(t *testing.T, loadStore *load.Store) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
@@ -399,8 +401,8 @@ func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
 }
 
 func (s) TestBalancerGroup_LoadReport(t *testing.T) {
-	testLoadStore := testutils.NewTestLoadStore()
-	cc, gator, bg := newTestBalancerGroup(t, testLoadStore)
+	loadStore := &load.Store{}
+	cc, gator, bg := newTestBalancerGroup(t, loadStore)
 
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
@@ -434,15 +436,35 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 
 	// Test roundrobin on the last picker.
 	p1 := <-cc.NewPickerCh
-	var (
-		wantStart []internal.LocalityID
-		wantEnd   []internal.LocalityID
-		wantCost  []testutils.TestServerLoad
-	)
-	for i := 0; i < 10; i++ {
+	// bg1 has a weight of 2, while bg2 has a weight of 1. So, we expect 20 of
+	// these picks to go to bg1 and 10 of them to bg2. And since there are two
+	// subConns in each group, we expect the picks to be equally split between
+	// the subConns. We do not call Done() on picks routed to sc1, so we expect
+	// these to show up as pending rpcs.
+	wantStoreData := &load.Data{
+		LocalityStats: map[string]load.LocalityData{
+			testBalancerIDs[0].String(): {
+				RequestStats: load.RequestData{Succeeded: 10, InProgress: 10},
+				LoadStats: map[string]load.ServerLoadData{
+					"cpu_utilization": {Count: 10, Sum: 100},
+					"mem_utilization": {Count: 10, Sum: 50},
+					"pic":             {Count: 10, Sum: 31.4},
+					"piu":             {Count: 10, Sum: 31.4},
+				},
+			},
+			testBalancerIDs[1].String(): {
+				RequestStats: load.RequestData{Succeeded: 10},
+				LoadStats: map[string]load.ServerLoadData{
+					"cpu_utilization": {Count: 10, Sum: 100},
+					"mem_utilization": {Count: 10, Sum: 50},
+					"pic":             {Count: 10, Sum: 31.4},
+					"piu":             {Count: 10, Sum: 31.4},
+				},
+			},
+		},
+	}
+	for i := 0; i < 30; i++ {
 		scst, _ := p1.Pick(balancer.PickInfo{})
-		locality := backendToBalancerID[scst.SubConn]
-		wantStart = append(wantStart, locality)
 		if scst.Done != nil && scst.SubConn != sc1 {
 			scst.Done(balancer.DoneInfo{
 				ServerLoad: &orcapb.OrcaLoadReport{
@@ -452,23 +474,12 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 					Utilization:    map[string]float64{"piu": 3.14},
 				},
 			})
-			wantEnd = append(wantEnd, locality)
-			wantCost = append(wantCost,
-				testutils.TestServerLoad{Name: serverLoadCPUName, D: 10},
-				testutils.TestServerLoad{Name: serverLoadMemoryName, D: 5},
-				testutils.TestServerLoad{Name: "pic", D: 3.14},
-				testutils.TestServerLoad{Name: "piu", D: 3.14})
 		}
 	}
 
-	if !cmp.Equal(testLoadStore.CallsStarted, wantStart) {
-		t.Fatalf("want started: %v, got: %v", testLoadStore.CallsStarted, wantStart)
-	}
-	if !cmp.Equal(testLoadStore.CallsEnded, wantEnd) {
-		t.Fatalf("want ended: %v, got: %v", testLoadStore.CallsEnded, wantEnd)
-	}
-	if !cmp.Equal(testLoadStore.CallsCost, wantCost, cmp.AllowUnexported(testutils.TestServerLoad{})) {
-		t.Fatalf("want cost: %v, got: %v", testLoadStore.CallsCost, wantCost)
+	gotStoreData := loadStore.Stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.EquateApprox(0, 0.1)); diff != "" {
+		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
 
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 5dc4a6a458ac..ba87c4de8f3c 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -31,7 +31,6 @@ import (
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/serviceconfig"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
@@ -40,8 +39,8 @@ const (
 )
 
 var (
-	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), loadStore lrs.Store, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
-		return newEDSBalancerImpl(cc, enqueueState, loadStore, logger)
+	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsclientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+		return newEDSBalancerImpl(cc, enqueueState, xdsClient, logger)
 	}
 )
 
@@ -63,10 +62,9 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
 	}
-	loadStore := lrs.NewStore()
 	x.logger = prefixLogger((x))
-	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, loadStore, x.logger)
-	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.buildOpts, loadStore, x.logger)
+	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.buildOpts, x.logger)
+	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.client, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 95172cfb0ab9..c00c58cfe695 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -34,7 +34,6 @@ import (
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
@@ -62,13 +61,13 @@ type balancerGroupWithConfig struct {
 // The localities are picked as weighted round robin. A configurable child
 // policy is used to manage endpoints in each locality.
 type edsBalancerImpl struct {
-	cc     balancer.ClientConn
-	logger *grpclog.PrefixLogger
+	cc        balancer.ClientConn
+	logger    *grpclog.PrefixLogger
+	xdsClient *xdsclientWrapper // To fetch the load.Store from.
 
 	enqueueChildBalancerStateUpdate func(priorityType, balancer.State)
 
 	subBalancerBuilder   balancer.Builder
-	loadStore            lrs.Store
 	priorityToLocalities map[priorityType]*balancerGroupWithConfig
 	respReceived         bool
 
@@ -99,18 +98,18 @@ type edsBalancerImpl struct {
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
-func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), loadStore lrs.Store, logger *grpclog.PrefixLogger) *edsBalancerImpl {
+func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsclientWrapper, logger *grpclog.PrefixLogger) *edsBalancerImpl {
 	edsImpl := &edsBalancerImpl{
 		cc:                 cc,
 		logger:             logger,
 		subBalancerBuilder: balancer.Get(roundrobin.Name),
+		xdsClient:          xdsClient,
 
 		enqueueChildBalancerStateUpdate: enqueueState,
 
 		priorityToLocalities: make(map[priorityType]*balancerGroupWithConfig),
 		priorityToState:      make(map[priorityType]*balancer.State),
 		subConnToPriority:    make(map[balancer.SubConn]priorityType),
-		loadStore:            loadStore,
 	}
 	// Don't start balancer group here. Start it when handling the first EDS
 	// response. Otherwise the balancer group will be started with round-robin,
@@ -170,7 +169,7 @@ func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropC
 		// Update picker with old inner picker, new drops.
 		edsImpl.cc.UpdateState(balancer.State{
 			ConnectivityState: edsImpl.innerState.ConnectivityState,
-			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadStore)},
+			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.xdsClient.loadStore())},
 		)
 	}
 	edsImpl.pickerMu.Unlock()
@@ -240,7 +239,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
 			ccPriorityWrapper := edsImpl.ccWrapperWithPriority(priority)
 			stateAggregator := weightedaggregator.New(ccPriorityWrapper, edsImpl.logger, newRandomWRR)
 			bgwc = &balancerGroupWithConfig{
-				bg:              balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.loadStore, edsImpl.logger),
+				bg:              balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.xdsClient.loadStore(), edsImpl.logger),
 				stateAggregator: stateAggregator,
 				configs:         make(map[internal.LocalityID]*localityConfig),
 			}
@@ -403,7 +402,7 @@ func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.St
 		defer edsImpl.pickerMu.Unlock()
 		edsImpl.innerState = s
 		// Don't reset drops when it's a state change.
-		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadStore)})
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.xdsClient.loadStore())})
 	}
 }
 
@@ -451,13 +450,20 @@ func (edsImpl *edsBalancerImpl) close() {
 	}
 }
 
+// dropReporter wraps the single method used by the dropPicker to report dropped
+// calls to the load store.
+type dropReporter interface {
+	// CallDropped reports the drop of one RPC with the given category.
+	CallDropped(category string)
+}
+
 type dropPicker struct {
 	drops     []*dropper
 	p         balancer.Picker
-	loadStore lrs.Store
+	loadStore dropReporter
 }
 
-func newDropPicker(p balancer.Picker, drops []*dropper, loadStore lrs.Store) *dropPicker {
+func newDropPicker(p balancer.Picker, drops []*dropper, loadStore dropReporter) *dropPicker {
 	return &dropPicker{
 		drops:     drops,
 		p:         p,
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index c0961e46c949..525da1ba645a 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -25,12 +25,15 @@ import (
 
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -679,11 +682,26 @@ func (s) TestDropPicker(t *testing.T) {
 	}
 }
 
+type loadStoreWrapper struct {
+	xdsClientInterface
+	ls *load.Store
+}
+
+func (l *loadStoreWrapper) LoadStore() *load.Store {
+	return l.ls
+}
+
 func (s) TestEDS_LoadReport(t *testing.T) {
-	testLoadStore := testutils.NewTestLoadStore()
+	// We create an xdsClientWrapper with a dummy xdsClientInterface which only
+	// implements the LoadStore() method to return the underlying load.Store to
+	// be used.
+	loadStore := &load.Store{}
+	cw := &xdsclientWrapper{
+		xdsClient: &loadStoreWrapper{ls: loadStore},
+	}
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, testLoadStore, nil)
+	edsb := newEDSBalancerImpl(cc, nil, cw, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
@@ -695,9 +713,8 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	sc1 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
-	backendToBalancerID[sc1] = internal.LocalityID{
-		SubZone: testSubZones[0],
-	}
+	locality1 := internal.LocalityID{SubZone: testSubZones[0]}
+	backendToBalancerID[sc1] = locality1
 
 	// Add the second locality later to make sure sc2 belongs to the second
 	// locality. Otherwise the test is flaky because of a map is used in EDS to
@@ -707,31 +724,29 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	sc2 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
-	backendToBalancerID[sc2] = internal.LocalityID{
-		SubZone: testSubZones[1],
-	}
+	locality2 := internal.LocalityID{SubZone: testSubZones[1]}
+	backendToBalancerID[sc2] = locality2
 
 	// Test roundrobin with two subconns.
 	p1 := <-cc.NewPickerCh
-	var (
-		wantStart []internal.LocalityID
-		wantEnd   []internal.LocalityID
-	)
-
+	// We expect the 10 picks to be split between the localities since they are
+	// of equal weight. And since we only mark the picks routed to sc2 as done,
+	// the picks on sc1 should show up as inProgress.
+	wantStoreData := &load.Data{
+		LocalityStats: map[string]load.LocalityData{
+			locality1.String(): {RequestStats: load.RequestData{InProgress: 5}},
+			locality2.String(): {RequestStats: load.RequestData{Succeeded: 5}},
+		},
+	}
 	for i := 0; i < 10; i++ {
 		scst, _ := p1.Pick(balancer.PickInfo{})
-		locality := backendToBalancerID[scst.SubConn]
-		wantStart = append(wantStart, locality)
 		if scst.Done != nil && scst.SubConn != sc1 {
 			scst.Done(balancer.DoneInfo{})
-			wantEnd = append(wantEnd, backendToBalancerID[scst.SubConn])
 		}
 	}
 
-	if !cmp.Equal(testLoadStore.CallsStarted, wantStart) {
-		t.Fatalf("want started: %v, got: %v", testLoadStore.CallsStarted, wantStart)
-	}
-	if !cmp.Equal(testLoadStore.CallsEnded, wantEnd) {
-		t.Fatalf("want ended: %v, got: %v", testLoadStore.CallsEnded, wantEnd)
+	gotStoreData := loadStore.Stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
+		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 7ace7cc2c170..0b989bed8320 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -30,6 +30,7 @@ import (
 	"github.com/golang/protobuf/jsonpb"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
@@ -39,7 +40,6 @@ import (
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
@@ -105,7 +105,6 @@ type fakeEDSBalancer struct {
 	childPolicy        *testutils.Channel
 	subconnStateChange *testutils.Channel
 	edsUpdate          *testutils.Channel
-	loadStore          lrs.Store
 }
 
 func (f *fakeEDSBalancer) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
@@ -169,13 +168,12 @@ func (f *fakeEDSBalancer) waitForEDSResponse(wantUpdate xdsclient.EndpointsUpdat
 	return nil
 }
 
-func newFakeEDSBalancer(cc balancer.ClientConn, loadStore lrs.Store) edsBalancerImplInterface {
+func newFakeEDSBalancer(cc balancer.ClientConn) edsBalancerImplInterface {
 	return &fakeEDSBalancer{
 		cc:                 cc,
 		childPolicy:        testutils.NewChannelWithSize(10),
 		subconnStateChange: testutils.NewChannelWithSize(10),
 		edsUpdate:          testutils.NewChannelWithSize(10),
-		loadStore:          loadStore,
 	}
 }
 
@@ -231,8 +229,8 @@ func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
 // cleanup.
 func setup(edsLBCh *testutils.Channel, xdsClientCh *testutils.Channel) func() {
 	origNewEDSBalancer := newEDSBalancer
-	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), loadStore lrs.Store, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
-		edsLB := newFakeEDSBalancer(cc, loadStore)
+	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ *xdsclientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+		edsLB := newFakeEDSBalancer(cc)
 		defer func() { edsLBCh.Send(edsLB) }()
 		return edsLB
 	}
@@ -435,7 +433,7 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 	edsLB.waitForSubConnStateChange(&scStateChange{sc: fsc, state: state})
 }
 
-// TestErrorFromXDSClientUpdate verifies that errros from xdsclient update are
+// TestErrorFromXDSClientUpdate verifies that errros from xdsClient update are
 // handled correctly.
 //
 // If it's resource-not-found, watch will NOT be canceled, the EDS impl will
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index 0f987a948fb9..06b45847e0b5 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -24,16 +24,17 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsinternal "google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/client/load"
 )
 
 // xdsClientInterface contains only the xds_client methods needed by EDS
 // balancer. It's defined so we can override xdsclientNew function in tests.
 type xdsClientInterface interface {
 	WatchEndpoints(clusterName string, edsCb func(xdsclient.EndpointsUpdate, error)) (cancel func())
-	ReportLoad(server string, clusterName string, loadStore lrs.Store) (cancel func())
+	LoadStore() *load.Store
+	ReportLoad(server string, clusterName string) (cancel func())
 	Close()
 }
 
@@ -52,11 +53,10 @@ type xdsclientWrapper struct {
 
 	newEDSUpdate func(xdsclient.EndpointsUpdate, error)
 	bbo          balancer.BuildOptions
-	loadStore    lrs.Store
 
 	balancerName string
-	// xdsclient could come from attributes, or created with balancerName.
-	xdsclient xdsClientInterface
+	// xdsClient could come from attributes, or created with balancerName.
+	xdsClient xdsClientInterface
 
 	// edsServiceName is the edsServiceName currently being watched, not
 	// necessary the edsServiceName from service config.
@@ -77,30 +77,29 @@ type xdsclientWrapper struct {
 //
 // The given callbacks won't be called until the underlying xds_client is
 // working and sends updates.
-func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, loadStore lrs.Store, logger *grpclog.PrefixLogger) *xdsclientWrapper {
+func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, logger *grpclog.PrefixLogger) *xdsclientWrapper {
 	return &xdsclientWrapper{
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
 		bbo:          bbo,
-		loadStore:    loadStore,
 	}
 }
 
-// replaceXDSClient replaces xdsclient fields to the newClient if they are
-// different. If xdsclient is replaced, the balancerName field will also be
+// replaceXDSClient replaces xdsClient fields to the newClient if they are
+// different. If xdsClient is replaced, the balancerName field will also be
 // updated to newBalancerName.
 //
-// If the old xdsclient is replaced, and was created locally (not from
+// If the old xdsClient is replaced, and was created locally (not from
 // attributes), it will be closed.
 //
-// It returns whether xdsclient is replaced.
+// It returns whether xdsClient is replaced.
 func (c *xdsclientWrapper) replaceXDSClient(newClient xdsClientInterface, newBalancerName string) bool {
-	if c.xdsclient == newClient {
+	if c.xdsClient == newClient {
 		return false
 	}
-	oldClient := c.xdsclient
+	oldClient := c.xdsClient
 	oldBalancerName := c.balancerName
-	c.xdsclient = newClient
+	c.xdsClient = newClient
 	c.balancerName = newBalancerName
 	if oldBalancerName != "" {
 		// OldBalancerName!="" means if the old client was not from attributes.
@@ -109,7 +108,7 @@ func (c *xdsclientWrapper) replaceXDSClient(newClient xdsClientInterface, newBal
 	return true
 }
 
-// updateXDSClient sets xdsclient in wrapper to the correct one based on the
+// updateXDSClient sets xdsClient in wrapper to the correct one based on the
 // attributes and service config.
 //
 // If client is found in attributes, it will be used, but we also need to decide
@@ -160,8 +159,8 @@ func (c *xdsclientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 		// This should never fail. xdsclientnew does a non-blocking dial, and
 		// all the config passed in should be validated.
 		//
-		// This could leave c.xdsclient as nil if this is the first update.
-		c.logger.Warningf("eds: failed to create xdsclient, error: %v", err)
+		// This could leave c.xdsClient as nil if this is the first update.
+		c.logger.Warningf("eds: failed to create xdsClient, error: %v", err)
 		return false
 	}
 	return c.replaceXDSClient(newClient, clientConfig.BalancerName)
@@ -177,7 +176,7 @@ func (c *xdsclientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 // This usually means load report needs to be restarted, but this function does
 // NOT do that. Caller needs to call startLoadReport separately.
 func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
-	if c.xdsclient == nil {
+	if c.xdsClient == nil {
 		return
 	}
 
@@ -185,14 +184,14 @@ func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
 	if c.cancelEndpointsWatch != nil {
 		c.cancelEndpointsWatch()
 	}
-	cancelEDSWatch := c.xdsclient.WatchEndpoints(c.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
-		c.logger.Infof("Watch update from xds-client %p, content: %+v", c.xdsclient, update)
+	cancelEDSWatch := c.xdsClient.WatchEndpoints(c.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
+		c.logger.Infof("Watch update from xds-client %p, content: %+v", c.xdsClient, update)
 		c.newEDSUpdate(update, err)
 	})
-	c.logger.Infof("Watch started on resource name %v with xds-client %p", c.edsServiceName, c.xdsclient)
+	c.logger.Infof("Watch started on resource name %v with xds-client %p", c.edsServiceName, c.xdsClient)
 	c.cancelEndpointsWatch = func() {
 		cancelEDSWatch()
-		c.logger.Infof("Watch cancelled on resource name %v with xds-client %p", c.edsServiceName, c.xdsclient)
+		c.logger.Infof("Watch cancelled on resource name %v with xds-client %p", c.edsServiceName, c.xdsClient)
 	}
 }
 
@@ -203,19 +202,24 @@ func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
 // edsServiceName doesn't (so we only need to restart load reporting, not EDS
 // watch).
 func (c *xdsclientWrapper) startLoadReport(edsServiceNameBeingWatched string, loadReportServer *string) {
-	if c.xdsclient == nil {
-		c.logger.Warningf("xds: xdsclient is nil when trying to start load reporting. This means xdsclient wasn't passed in from the resolver, and xdsclient.New failed")
+	if c.xdsClient == nil {
+		c.logger.Warningf("xds: xdsClient is nil when trying to start load reporting. This means xdsClient wasn't passed in from the resolver, and xdsClient.New failed")
 		return
 	}
-	if c.loadStore != nil {
-		if c.cancelLoadReport != nil {
-			c.cancelLoadReport()
-		}
-		c.loadReportServer = loadReportServer
-		if c.loadReportServer != nil {
-			c.cancelLoadReport = c.xdsclient.ReportLoad(*c.loadReportServer, edsServiceNameBeingWatched, c.loadStore)
-		}
+	if c.cancelLoadReport != nil {
+		c.cancelLoadReport()
+	}
+	c.loadReportServer = loadReportServer
+	if c.loadReportServer != nil {
+		c.cancelLoadReport = c.xdsClient.ReportLoad(*c.loadReportServer, edsServiceNameBeingWatched)
+	}
+}
+
+func (c *xdsclientWrapper) loadStore() *load.Store {
+	if c == nil || c.xdsClient == nil {
+		return nil
 	}
+	return c.xdsClient.LoadStore()
 }
 
 // handleUpdate applies the service config and attributes updates to the client,
@@ -271,9 +275,9 @@ func (c *xdsclientWrapper) cancelWatch() {
 
 func (c *xdsclientWrapper) close() {
 	c.cancelWatch()
-	if c.xdsclient != nil && c.balancerName != "" {
-		// Only close xdsclient if it's not from attributes.
-		c.xdsclient.Close()
+	if c.xdsClient != nil && c.balancerName != "" {
+		// Only close xdsClient if it's not from attributes.
+		c.xdsClient.Close()
 	}
 }
 
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index a230ca9ca49d..cf39597c53d8 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -93,7 +93,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 	defer cleanup()
 	t.Logf("Started fake xDS server at %s...", fakeServer.Address)
 
-	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
+	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil)
 	defer cw.close()
 	t.Logf("Started xDS client wrapper for endpoint %s...", testServiceName)
 
@@ -162,7 +162,7 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 		edsRespChan.Send(&edsUpdate{resp: update, err: err})
 	}
 
-	cw := newXDSClientWrapper(newEDS, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
+	cw := newXDSClientWrapper(newEDS, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil)
 	defer cw.close()
 
 	xdsC := fakeclient.NewClient()
@@ -207,7 +207,7 @@ func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
 	}
 	defer func() { xdsclientNew = oldxdsclientNew }()
 
-	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil)
+	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil)
 	defer cw.close()
 
 	xdsC1 := fakeclient.NewClient()
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index 4af91c76498a..e214c4667046 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal"
 	xdsinternal "google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/client/load"
 )
 
 func init() {
@@ -44,8 +45,7 @@ func (l *lrsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balanc
 		cc:        cc,
 		buildOpts: opts,
 	}
-	b.loadStore = NewStore()
-	b.client = newXDSClientWrapper(b.loadStore)
+	b.client = &xdsClientWrapper{}
 	b.logger = prefixLogger(b)
 	b.logger.Infof("Created")
 	return b
@@ -63,9 +63,8 @@ type lrsBalancer struct {
 	cc        balancer.ClientConn
 	buildOpts balancer.BuildOptions
 
-	logger    *grpclog.PrefixLogger
-	loadStore Store
-	client    *xdsClientWrapper
+	logger *grpclog.PrefixLogger
+	client *xdsClientWrapper
 
 	config *lbConfig
 	lb     balancer.Balancer // The sub balancer.
@@ -77,6 +76,11 @@ func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
 	}
 
+	// Update load reporting config or xds client. This needs to be done before
+	// updating the child policy because we need the loadStore from the updated
+	// client to be passed to the ccWrapper.
+	b.client.update(newConfig, s.ResolverState.Attributes)
+
 	// If child policy is a different type, recreate the sub-balancer.
 	if b.config == nil || b.config.ChildPolicy.Name != newConfig.ChildPolicy.Name {
 		bb := balancer.Get(newConfig.ChildPolicy.Name)
@@ -86,10 +90,8 @@ func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 		if b.lb != nil {
 			b.lb.Close()
 		}
-		b.lb = bb.Build(newCCWrapper(b.cc, b.loadStore, newConfig.Locality), b.buildOpts)
+		b.lb = bb.Build(newCCWrapper(b.cc, b.client.loadStore(), newConfig.Locality), b.buildOpts)
 	}
-	// Update load reporting config or xds client.
-	b.client.update(newConfig, s.ResolverState.Attributes)
 	b.config = newConfig
 
 	// Addresses and sub-balancer config are sent to sub-balancer.
@@ -121,11 +123,11 @@ func (b *lrsBalancer) Close() {
 
 type ccWrapper struct {
 	balancer.ClientConn
-	loadStore  Store
+	loadStore  *load.Store
 	localityID *internal.LocalityID
 }
 
-func newCCWrapper(cc balancer.ClientConn, loadStore Store, localityID *internal.LocalityID) *ccWrapper {
+func newCCWrapper(cc balancer.ClientConn, loadStore *load.Store, localityID *internal.LocalityID) *ccWrapper {
 	return &ccWrapper{
 		ClientConn: cc,
 		loadStore:  loadStore,
@@ -141,25 +143,18 @@ func (ccw *ccWrapper) UpdateState(s balancer.State) {
 // xdsClientInterface contains only the xds_client methods needed by LRS
 // balancer. It's defined so we can override xdsclient in tests.
 type xdsClientInterface interface {
-	ReportLoad(server string, clusterName string, loadStore Store) (cancel func())
+	LoadStore() *load.Store
+	ReportLoad(server string, clusterName string) func()
 	Close()
 }
 
 type xdsClientWrapper struct {
-	loadStore Store
-
 	c                xdsClientInterface
 	cancelLoadReport func()
 	clusterName      string
 	lrsServerName    string
 }
 
-func newXDSClientWrapper(loadStore Store) *xdsClientWrapper {
-	return &xdsClientWrapper{
-		loadStore: loadStore,
-	}
-}
-
 // update checks the config and xdsclient, and decides whether it needs to
 // restart the load reporting stream.
 //
@@ -203,11 +198,18 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attribut
 			w.cancelLoadReport = nil
 		}
 		if w.c != nil {
-			w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName, w.clusterName, w.loadStore)
+			w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName, w.clusterName)
 		}
 	}
 }
 
+func (w *xdsClientWrapper) loadStore() *load.Store {
+	if w.c == nil {
+		return nil
+	}
+	return w.c.LoadStore()
+}
+
 func (w *xdsClientWrapper) close() {
 	if w.cancelLoadReport != nil {
 		w.cancelLoadReport()
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index c57b5e1a127f..772dcb2656b1 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -19,6 +19,7 @@
 package lrs
 
 import (
+	"context"
 	"fmt"
 	"testing"
 	"time"
@@ -32,8 +33,11 @@ import (
 	"google.golang.org/grpc/resolver"
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
+const defaultTestTimeout = 1 * time.Second
+
 var (
 	testBackendAddrs = []resolver.Address{
 		{Addr: "1.1.1.1:1"},
@@ -45,53 +49,6 @@ var (
 	}
 )
 
-// This is a subset of testutils.fakeclient. Cannot use testutils.fakeclient
-// because testutils imports package lrs.
-//
-// TODO: after refactoring xdsclient to support load reporting, the testutils
-// package won't need to depend on lrs package for the store. And we can use the
-// testutils for this.
-type fakeXDSClient struct {
-	loadReportCh chan *reportLoadArgs
-}
-
-func newFakeXDSClient() *fakeXDSClient {
-	return &fakeXDSClient{
-		loadReportCh: make(chan *reportLoadArgs, 10),
-	}
-}
-
-// reportLoadArgs wraps the arguments passed to ReportLoad.
-type reportLoadArgs struct {
-	// server is the name of the server to which the load is reported.
-	server string
-	// cluster is the name of the cluster for which load is reported.
-	cluster string
-	// loadStore is the store where loads are stored.
-	loadStore interface{}
-}
-
-// ReportLoad starts reporting load about clusterName to server.
-func (xdsC *fakeXDSClient) ReportLoad(server string, clusterName string, loadStore Store) (cancel func()) {
-	xdsC.loadReportCh <- &reportLoadArgs{server: server, cluster: clusterName, loadStore: loadStore}
-	return func() {}
-}
-
-// waitForReportLoad waits for ReportLoad to be invoked on this client within a
-// reasonable timeout, and returns the arguments passed to it.
-func (xdsC *fakeXDSClient) waitForReportLoad() (*reportLoadArgs, error) {
-	select {
-	case <-time.After(time.Second):
-		return nil, fmt.Errorf("timeout")
-	case a := <-xdsC.loadReportCh:
-		return a, nil
-	}
-}
-
-// Close closes the xds client.
-func (xdsC *fakeXDSClient) Close() {
-}
-
 // TestLoadReporting verifies that the lrs balancer starts the loadReport
 // stream when the lbConfig passed to it contains a valid value for the LRS
 // server (empty string).
@@ -101,7 +58,7 @@ func TestLoadReporting(t *testing.T) {
 	lrsB := builder.Build(cc, balancer.BuildOptions{})
 	defer lrsB.Close()
 
-	xdsC := newFakeXDSClient()
+	xdsC := fakeclient.NewClient()
 	if err := lrsB.UpdateClientConnState(balancer.ClientConnState{
 		ResolverState: resolver.State{
 			Addresses:  testBackendAddrs,
@@ -119,12 +76,15 @@ func TestLoadReporting(t *testing.T) {
 		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
 	}
 
-	got, err := xdsC.waitForReportLoad()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	got, err := xdsC.WaitForReportLoad(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
 	}
-	if got.server != testLRSServerName || got.cluster != testClusterName {
-		t.Fatalf("xdsClient.ReportLoad called with {%q, %q}: want {%q, %q}", got.server, got.cluster, testLRSServerName, testClusterName)
+	if got.Server != testLRSServerName || got.Cluster != testClusterName {
+		t.Fatalf("xdsClient.ReportLoad called with {%q, %q}: want {%q, %q}", got.Server, got.Cluster, testLRSServerName, testClusterName)
 	}
 
 	sc1 := <-cc.NewSubConnCh
@@ -150,26 +110,24 @@ func TestLoadReporting(t *testing.T) {
 		gotSCSt.Done(balancer.DoneInfo{Err: fmt.Errorf("error")})
 	}
 
-	loads := make(map[xdsinternal.LocalityID]*rpcCountData)
-
-	got.loadStore.(*lrsStore).localityRPCCount.Range(
-		func(key, value interface{}) bool {
-			loads[key.(xdsinternal.LocalityID)] = value.(*rpcCountData)
-			return true
-		},
-	)
-
-	countData, ok := loads[*testLocality]
+	// Dump load data from the store and compare with expected counts.
+	loadStore := xdsC.LoadStore()
+	if loadStore == nil {
+		t.Fatal("loadStore is nil in xdsClient")
+	}
+	sd := loadStore.Stats()
+	localityData, ok := sd.LocalityStats[testLocality.String()]
 	if !ok {
 		t.Fatalf("loads for %v not found in store", testLocality)
 	}
-	if *countData.succeeded != successCount {
-		t.Errorf("got succeeded %v, want %v", *countData.succeeded, successCount)
+	reqStats := localityData.RequestStats
+	if reqStats.Succeeded != successCount {
+		t.Errorf("got succeeded %v, want %v", reqStats.Succeeded, successCount)
 	}
-	if *countData.errored != errorCount {
-		t.Errorf("got errord %v, want %v", *countData.errored, errorCount)
+	if reqStats.Errored != errorCount {
+		t.Errorf("got errord %v, want %v", reqStats.Errored, errorCount)
 	}
-	if *countData.inProgress != 0 {
-		t.Errorf("got inProgress %v, want %v", *countData.inProgress, 0)
+	if reqStats.InProgress != 0 {
+		t.Errorf("got inProgress %v, want %v", reqStats.InProgress, 0)
 	}
 }
diff --git a/xds/internal/balancer/lrs/picker.go b/xds/internal/balancer/lrs/picker.go
index 1fcc6e9e5b31..8ca72b4fe57c 100644
--- a/xds/internal/balancer/lrs/picker.go
+++ b/xds/internal/balancer/lrs/picker.go
@@ -29,17 +29,24 @@ const (
 	serverLoadMemoryName = "mem_utilization"
 )
 
+// loadReporter wraps the methods from the loadStore that are used here.
+type loadReporter interface {
+	CallStarted(locality string)
+	CallFinished(locality string, err error)
+	CallServerLoad(locality, name string, val float64)
+}
+
 type loadReportPicker struct {
 	p balancer.Picker
 
-	id        internal.LocalityID
-	loadStore Store
+	locality  string
+	loadStore loadReporter
 }
 
-func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore Store) *loadReportPicker {
+func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore loadReporter) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
-		id:        id,
+		locality:  id.String(),
 		loadStore: loadStore,
 	}
 }
@@ -50,25 +57,29 @@ func (lrp *loadReportPicker) Pick(info balancer.PickInfo) (balancer.PickResult,
 		return res, err
 	}
 
-	lrp.loadStore.CallStarted(lrp.id)
+	if lrp.loadStore == nil {
+		return res, err
+	}
+
+	lrp.loadStore.CallStarted(lrp.locality)
 	oldDone := res.Done
 	res.Done = func(info balancer.DoneInfo) {
 		if oldDone != nil {
 			oldDone(info)
 		}
-		lrp.loadStore.CallFinished(lrp.id, info.Err)
+		lrp.loadStore.CallFinished(lrp.locality, info.Err)
 
 		load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport)
 		if !ok {
 			return
 		}
-		lrp.loadStore.CallServerLoad(lrp.id, serverLoadCPUName, load.CpuUtilization)
-		lrp.loadStore.CallServerLoad(lrp.id, serverLoadMemoryName, load.MemUtilization)
+		lrp.loadStore.CallServerLoad(lrp.locality, serverLoadCPUName, load.CpuUtilization)
+		lrp.loadStore.CallServerLoad(lrp.locality, serverLoadMemoryName, load.MemUtilization)
 		for n, d := range load.RequestCost {
-			lrp.loadStore.CallServerLoad(lrp.id, n, d)
+			lrp.loadStore.CallServerLoad(lrp.locality, n, d)
 		}
 		for n, d := range load.Utilization {
-			lrp.loadStore.CallServerLoad(lrp.id, n, d)
+			lrp.loadStore.CallServerLoad(lrp.locality, n, d)
 		}
 	}
 	return res, err
diff --git a/xds/internal/balancer/lrs/store.go b/xds/internal/balancer/lrs/store.go
deleted file mode 100644
index 96c85f9cc9cd..000000000000
--- a/xds/internal/balancer/lrs/store.go
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package lrs
-
-import (
-	"context"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
-	lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
-	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
-	"github.com/golang/protobuf/ptypes"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/internal/backoff"
-	"google.golang.org/grpc/xds/internal"
-)
-
-const negativeOneUInt64 = ^uint64(0)
-
-// Store defines the interface for a load store. It keeps loads and can report
-// them to a server when requested.
-type Store interface {
-	CallDropped(category string)
-	CallStarted(l internal.LocalityID)
-	CallFinished(l internal.LocalityID, err error)
-	CallServerLoad(l internal.LocalityID, name string, d float64)
-	// Report the load of clusterName to cc.
-	ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node)
-}
-
-type rpcCountData struct {
-	// Only atomic accesses are allowed for the fields.
-	succeeded  *uint64
-	errored    *uint64
-	inProgress *uint64
-
-	// Map from load name to load data (sum+count). Loading data from map is
-	// atomic, but updating data takes a lock, which could cause contention when
-	// multiple RPCs try to report loads for the same name.
-	//
-	// To fix the contention, shard this map.
-	serverLoads sync.Map // map[string]*rpcLoadData
-}
-
-func newRPCCountData() *rpcCountData {
-	return &rpcCountData{
-		succeeded:  new(uint64),
-		errored:    new(uint64),
-		inProgress: new(uint64),
-	}
-}
-
-func (rcd *rpcCountData) incrSucceeded() {
-	atomic.AddUint64(rcd.succeeded, 1)
-}
-
-func (rcd *rpcCountData) loadAndClearSucceeded() uint64 {
-	return atomic.SwapUint64(rcd.succeeded, 0)
-}
-
-func (rcd *rpcCountData) incrErrored() {
-	atomic.AddUint64(rcd.errored, 1)
-}
-
-func (rcd *rpcCountData) loadAndClearErrored() uint64 {
-	return atomic.SwapUint64(rcd.errored, 0)
-}
-
-func (rcd *rpcCountData) incrInProgress() {
-	atomic.AddUint64(rcd.inProgress, 1)
-}
-
-func (rcd *rpcCountData) decrInProgress() {
-	atomic.AddUint64(rcd.inProgress, negativeOneUInt64) // atomic.Add(x, -1)
-}
-
-func (rcd *rpcCountData) loadInProgress() uint64 {
-	return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading.
-}
-
-func (rcd *rpcCountData) addServerLoad(name string, d float64) {
-	loads, ok := rcd.serverLoads.Load(name)
-	if !ok {
-		tl := newRPCLoadData()
-		loads, _ = rcd.serverLoads.LoadOrStore(name, tl)
-	}
-	loads.(*rpcLoadData).add(d)
-}
-
-// Data for server loads (from trailers or oob). Fields in this struct must be
-// updated consistently.
-//
-// The current solution is to hold a lock, which could cause contention. To fix,
-// shard serverLoads map in rpcCountData.
-type rpcLoadData struct {
-	mu    sync.Mutex
-	sum   float64
-	count uint64
-}
-
-func newRPCLoadData() *rpcLoadData {
-	return &rpcLoadData{}
-}
-
-func (rld *rpcLoadData) add(v float64) {
-	rld.mu.Lock()
-	rld.sum += v
-	rld.count++
-	rld.mu.Unlock()
-}
-
-func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) {
-	rld.mu.Lock()
-	s = rld.sum
-	rld.sum = 0
-	c = rld.count
-	rld.count = 0
-	rld.mu.Unlock()
-	return
-}
-
-// lrsStore collects loads from xds balancer, and periodically sends load to the
-// server.
-type lrsStore struct {
-	backoff      backoff.Strategy
-	lastReported time.Time
-
-	drops            sync.Map // map[string]*uint64
-	localityRPCCount sync.Map // map[internal.LocalityID]*rpcCountData
-}
-
-// NewStore creates a store for load reports.
-func NewStore() Store {
-	return &lrsStore{
-		backoff:      backoff.DefaultExponential,
-		lastReported: time.Now(),
-	}
-}
-
-// Update functions are called by picker for each RPC. To avoid contention, all
-// updates are done atomically.
-
-// CallDropped adds one drop record with the given category to store.
-func (ls *lrsStore) CallDropped(category string) {
-	p, ok := ls.drops.Load(category)
-	if !ok {
-		tp := new(uint64)
-		p, _ = ls.drops.LoadOrStore(category, tp)
-	}
-	atomic.AddUint64(p.(*uint64), 1)
-}
-
-func (ls *lrsStore) CallStarted(l internal.LocalityID) {
-	p, ok := ls.localityRPCCount.Load(l)
-	if !ok {
-		tp := newRPCCountData()
-		p, _ = ls.localityRPCCount.LoadOrStore(l, tp)
-	}
-	p.(*rpcCountData).incrInProgress()
-}
-
-func (ls *lrsStore) CallFinished(l internal.LocalityID, err error) {
-	p, ok := ls.localityRPCCount.Load(l)
-	if !ok {
-		// The map is never cleared, only values in the map are reset. So the
-		// case where entry for call-finish is not found should never happen.
-		return
-	}
-	p.(*rpcCountData).decrInProgress()
-	if err == nil {
-		p.(*rpcCountData).incrSucceeded()
-	} else {
-		p.(*rpcCountData).incrErrored()
-	}
-}
-
-func (ls *lrsStore) CallServerLoad(l internal.LocalityID, name string, d float64) {
-	p, ok := ls.localityRPCCount.Load(l)
-	if !ok {
-		// The map is never cleared, only values in the map are reset. So the
-		// case where entry for CallServerLoad is not found should never happen.
-		return
-	}
-	p.(*rpcCountData).addServerLoad(name, d)
-}
-
-func (ls *lrsStore) buildStats(clusterName string) []*endpointpb.ClusterStats {
-	var (
-		totalDropped  uint64
-		droppedReqs   []*endpointpb.ClusterStats_DroppedRequests
-		localityStats []*endpointpb.UpstreamLocalityStats
-	)
-	ls.drops.Range(func(category, countP interface{}) bool {
-		tempCount := atomic.SwapUint64(countP.(*uint64), 0)
-		if tempCount == 0 {
-			return true
-		}
-		totalDropped += tempCount
-		droppedReqs = append(droppedReqs, &endpointpb.ClusterStats_DroppedRequests{
-			Category:     category.(string),
-			DroppedCount: tempCount,
-		})
-		return true
-	})
-	ls.localityRPCCount.Range(func(locality, countP interface{}) bool {
-		tempLocality := locality.(internal.LocalityID)
-		tempCount := countP.(*rpcCountData)
-
-		tempSucceeded := tempCount.loadAndClearSucceeded()
-		tempInProgress := tempCount.loadInProgress()
-		tempErrored := tempCount.loadAndClearErrored()
-		if tempSucceeded == 0 && tempInProgress == 0 && tempErrored == 0 {
-			return true
-		}
-
-		var loadMetricStats []*endpointpb.EndpointLoadMetricStats
-		tempCount.serverLoads.Range(func(name, data interface{}) bool {
-			tempName := name.(string)
-			tempSum, tempCount := data.(*rpcLoadData).loadAndClear()
-			if tempCount == 0 {
-				return true
-			}
-			loadMetricStats = append(loadMetricStats,
-				&endpointpb.EndpointLoadMetricStats{
-					MetricName:                    tempName,
-					NumRequestsFinishedWithMetric: tempCount,
-					TotalMetricValue:              tempSum,
-				},
-			)
-			return true
-		})
-
-		localityStats = append(localityStats, &endpointpb.UpstreamLocalityStats{
-			Locality: &corepb.Locality{
-				Region:  tempLocality.Region,
-				Zone:    tempLocality.Zone,
-				SubZone: tempLocality.SubZone,
-			},
-			TotalSuccessfulRequests: tempSucceeded,
-			TotalRequestsInProgress: tempInProgress,
-			TotalErrorRequests:      tempErrored,
-			LoadMetricStats:         loadMetricStats,
-			UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
-		})
-		return true
-	})
-
-	dur := time.Since(ls.lastReported)
-	ls.lastReported = time.Now()
-
-	var ret []*endpointpb.ClusterStats
-	ret = append(ret, &endpointpb.ClusterStats{
-		ClusterName:           clusterName,
-		UpstreamLocalityStats: localityStats,
-
-		TotalDroppedRequests: totalDropped,
-		DroppedRequests:      droppedReqs,
-		LoadReportInterval:   ptypes.DurationProto(dur),
-	})
-
-	return ret
-}
-
-// ReportTo makes a streaming lrs call to cc and blocks.
-//
-// It retries the call (with backoff) until ctx is canceled.
-func (ls *lrsStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) {
-	c := lrsgrpc.NewLoadReportingServiceClient(cc)
-	var (
-		retryCount int
-		doBackoff  bool
-	)
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-
-		if doBackoff {
-			backoffTimer := time.NewTimer(ls.backoff.Backoff(retryCount))
-			select {
-			case <-backoffTimer.C:
-			case <-ctx.Done():
-				backoffTimer.Stop()
-				return
-			}
-			retryCount++
-		}
-
-		doBackoff = true
-		stream, err := c.StreamLoadStats(ctx)
-		if err != nil {
-			logger.Warningf("lrs: failed to create stream: %v", err)
-			continue
-		}
-		logger.Infof("lrs: created LRS stream")
-		req := &lrspb.LoadStatsRequest{Node: node}
-		logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
-		if err := stream.Send(req); err != nil {
-			logger.Warningf("lrs: failed to send first request: %v", err)
-			continue
-		}
-		first, err := stream.Recv()
-		if err != nil {
-			logger.Warningf("lrs: failed to receive first response: %v", err)
-			continue
-		}
-		logger.Infof("lrs: received first LoadStatsResponse: %+v", first)
-		interval, err := ptypes.Duration(first.LoadReportingInterval)
-		if err != nil {
-			logger.Warningf("lrs: failed to convert report interval: %v", err)
-			continue
-		}
-		// The LRS client should join the clusters it knows with the cluster
-		// list from response, and send loads for them.
-		//
-		// But the LRS client now only supports one cluster. TODO: extend it to
-		// support multiple clusters.
-		var clusterFoundInResponse bool
-		for _, c := range first.Clusters {
-			if c == clusterName {
-				clusterFoundInResponse = true
-			}
-		}
-		if !clusterFoundInResponse {
-			logger.Warningf("lrs: received clusters %v does not contain expected {%v}", first.Clusters, clusterName)
-			continue
-		}
-		if first.ReportEndpointGranularity {
-			// TODO: fixme to support per endpoint loads.
-			logger.Warningf("lrs: endpoint loads requested, but not supported by current implementation")
-			continue
-		}
-
-		// No backoff afterwards.
-		doBackoff = false
-		retryCount = 0
-		ls.sendLoads(ctx, stream, clusterName, interval)
-	}
-}
-
-func (ls *lrsStore) sendLoads(ctx context.Context, stream lrsgrpc.LoadReportingService_StreamLoadStatsClient, clusterName string, interval time.Duration) {
-	tick := time.NewTicker(interval)
-	defer tick.Stop()
-	for {
-		select {
-		case <-tick.C:
-		case <-ctx.Done():
-			return
-		}
-		req := &lrspb.LoadStatsRequest{ClusterStats: ls.buildStats(clusterName)}
-		logger.Infof("lrs: sending LRS loads: %+v", req)
-		if err := stream.Send(req); err != nil {
-			logger.Warningf("lrs: failed to send report: %v", err)
-			return
-		}
-	}
-}
diff --git a/xds/internal/balancer/lrs/store_test.go b/xds/internal/balancer/lrs/store_test.go
deleted file mode 100644
index b18c3d7e218d..000000000000
--- a/xds/internal/balancer/lrs/store_test.go
+++ /dev/null
@@ -1,517 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package lrs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"sort"
-	"sync"
-	"testing"
-	"time"
-
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
-	lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
-	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
-	"github.com/golang/protobuf/proto"
-	durationpb "github.com/golang/protobuf/ptypes/duration"
-	structpb "github.com/golang/protobuf/ptypes/struct"
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/testutils"
-)
-
-const (
-	testService             = "grpc.service.test"
-	testHostname            = "grpc.server.name"
-	nodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME"
-)
-
-var (
-	dropCategories = []string{"drop_for_real", "drop_for_fun"}
-	localities     = []internal.LocalityID{{Region: "a"}, {Region: "b"}}
-	errTest        = fmt.Errorf("test error")
-)
-
-type rpcCountDataForTest struct {
-	succeeded   uint64
-	errored     uint64
-	inProgress  uint64
-	serverLoads map[string]float64
-}
-
-func newRPCCountDataForTest(succeeded, errored, inprogress uint64, serverLoads map[string]float64) *rpcCountDataForTest {
-	return &rpcCountDataForTest{
-		succeeded:   succeeded,
-		errored:     errored,
-		inProgress:  inprogress,
-		serverLoads: serverLoads,
-	}
-}
-
-// Equal() is needed to compare unexported fields.
-func (rcd *rpcCountDataForTest) Equal(b *rpcCountDataForTest) bool {
-	return rcd.inProgress == b.inProgress &&
-		rcd.errored == b.errored &&
-		rcd.succeeded == b.succeeded &&
-		cmp.Equal(rcd.serverLoads, b.serverLoads)
-}
-
-// equalClusterStats sorts requests and clear report internal before comparing.
-func equalClusterStats(a, b []*endpointpb.ClusterStats) bool {
-	for _, t := range [][]*endpointpb.ClusterStats{a, b} {
-		for _, s := range t {
-			sort.Slice(s.DroppedRequests, func(i, j int) bool {
-				return s.DroppedRequests[i].Category < s.DroppedRequests[j].Category
-			})
-			sort.Slice(s.UpstreamLocalityStats, func(i, j int) bool {
-				return s.UpstreamLocalityStats[i].Locality.String() < s.UpstreamLocalityStats[j].Locality.String()
-			})
-			for _, us := range s.UpstreamLocalityStats {
-				sort.Slice(us.LoadMetricStats, func(i, j int) bool {
-					return us.LoadMetricStats[i].MetricName < us.LoadMetricStats[j].MetricName
-				})
-			}
-			s.LoadReportInterval = nil
-		}
-	}
-	return cmp.Equal(a, b, cmp.Comparer(proto.Equal))
-}
-
-func Test_lrsStore_buildStats_drops(t *testing.T) {
-	tests := []struct {
-		name  string
-		drops []map[string]uint64
-	}{
-		{
-			name: "one drop report",
-			drops: []map[string]uint64{{
-				dropCategories[0]: 31,
-				dropCategories[1]: 41,
-			}},
-		},
-		{
-			name: "two drop reports",
-			drops: []map[string]uint64{{
-				dropCategories[0]: 31,
-				dropCategories[1]: 41,
-			}, {
-				dropCategories[0]: 59,
-				dropCategories[1]: 26,
-			}},
-		},
-		{
-			name: "no empty report",
-			drops: []map[string]uint64{{
-				dropCategories[0]: 31,
-				dropCategories[1]: 41,
-			}, {
-				dropCategories[0]: 0, // This shouldn't cause an empty report for category[0].
-				dropCategories[1]: 26,
-			}},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ls := NewStore().(*lrsStore)
-
-			for _, ds := range tt.drops {
-				var (
-					totalDropped uint64
-					droppedReqs  []*endpointpb.ClusterStats_DroppedRequests
-				)
-				for cat, count := range ds {
-					if count == 0 {
-						continue
-					}
-					totalDropped += count
-					droppedReqs = append(droppedReqs, &endpointpb.ClusterStats_DroppedRequests{
-						Category:     cat,
-						DroppedCount: count,
-					})
-				}
-				want := []*endpointpb.ClusterStats{
-					{
-						ClusterName:          testService,
-						TotalDroppedRequests: totalDropped,
-						DroppedRequests:      droppedReqs,
-					},
-				}
-
-				var wg sync.WaitGroup
-				for c, count := range ds {
-					for i := 0; i < int(count); i++ {
-						wg.Add(1)
-						go func(i int, c string) {
-							ls.CallDropped(c)
-							wg.Done()
-						}(i, c)
-					}
-				}
-				wg.Wait()
-
-				if got := ls.buildStats(testService); !equalClusterStats(got, want) {
-					t.Errorf("lrsStore.buildStats() = %v, want %v", got, want)
-					t.Errorf("%s", cmp.Diff(got, want))
-				}
-			}
-		})
-	}
-}
-
-func Test_lrsStore_buildStats_rpcCounts(t *testing.T) {
-	tests := []struct {
-		name string
-		rpcs []map[internal.LocalityID]struct {
-			start, success, failure uint64
-			serverData              map[string]float64 // Will be reported with successful RPCs.
-		}
-	}{
-		{
-			name: "one rpcCount report",
-			rpcs: []map[internal.LocalityID]struct {
-				start, success, failure uint64
-				serverData              map[string]float64
-			}{{
-				localities[0]: {8, 3, 1, nil},
-			}},
-		},
-		{
-			name: "two localities one rpcCount report",
-			rpcs: []map[internal.LocalityID]struct {
-				start, success, failure uint64
-				serverData              map[string]float64
-			}{{
-				localities[0]: {8, 3, 1, nil},
-				localities[1]: {15, 1, 5, nil},
-			}},
-		},
-		{
-			name: "three rpcCount reports",
-			rpcs: []map[internal.LocalityID]struct {
-				start, success, failure uint64
-				serverData              map[string]float64
-			}{{
-				localities[0]: {8, 3, 1, nil},
-				localities[1]: {15, 1, 5, nil},
-			}, {
-				localities[0]: {8, 3, 1, nil},
-			}, {
-				localities[1]: {15, 1, 5, nil},
-			}},
-		},
-		{
-			name: "no empty report",
-			rpcs: []map[internal.LocalityID]struct {
-				start, success, failure uint64
-				serverData              map[string]float64
-			}{{
-				localities[0]: {4, 3, 1, nil},
-				localities[1]: {7, 1, 5, nil},
-			}, {
-				localities[0]: {0, 0, 0, nil}, // This shouldn't cause an empty report for locality[0].
-				localities[1]: {1, 1, 0, nil},
-			}},
-		},
-		{
-			name: "two localities one report with server loads",
-			rpcs: []map[internal.LocalityID]struct {
-				start, success, failure uint64
-				serverData              map[string]float64
-			}{{
-				localities[0]: {8, 3, 1, map[string]float64{"cpu": 15, "mem": 20}},
-				localities[1]: {15, 4, 5, map[string]float64{"net": 5, "disk": 0.8}},
-			}},
-		},
-		{
-			name: "three reports with server loads",
-			rpcs: []map[internal.LocalityID]struct {
-				start, success, failure uint64
-				serverData              map[string]float64
-			}{{
-				localities[0]: {8, 3, 1, map[string]float64{"cpu": 15, "mem": 20}},
-				localities[1]: {15, 4, 5, map[string]float64{"net": 5, "disk": 0.8}},
-			}, {
-				localities[0]: {8, 3, 1, map[string]float64{"cpu": 1, "mem": 2}},
-			}, {
-				localities[1]: {15, 4, 5, map[string]float64{"net": 13, "disk": 1.4}},
-			}},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ls := NewStore().(*lrsStore)
-
-			// InProgress count doesn't get cleared at each buildStats, keep
-			// them to carry over.
-			inProgressCounts := make(map[internal.LocalityID]uint64)
-
-			for _, counts := range tt.rpcs {
-				var upstreamLocalityStats []*endpointpb.UpstreamLocalityStats
-
-				for l, count := range counts {
-					tempInProgress := count.start - count.success - count.failure + inProgressCounts[l]
-					inProgressCounts[l] = tempInProgress
-					if count.success == 0 && tempInProgress == 0 && count.failure == 0 {
-						continue
-					}
-					var loadMetricStats []*endpointpb.EndpointLoadMetricStats
-					for n, d := range count.serverData {
-						loadMetricStats = append(loadMetricStats,
-							&endpointpb.EndpointLoadMetricStats{
-								MetricName:                    n,
-								NumRequestsFinishedWithMetric: count.success,
-								TotalMetricValue:              d * float64(count.success),
-							},
-						)
-					}
-					upstreamLocalityStats = append(upstreamLocalityStats, &endpointpb.UpstreamLocalityStats{
-						Locality:                testutils.LocalityIDToProto(l),
-						TotalSuccessfulRequests: count.success,
-						TotalRequestsInProgress: tempInProgress,
-						TotalErrorRequests:      count.failure,
-						LoadMetricStats:         loadMetricStats,
-					})
-				}
-				// InProgress count doesn't get cleared at each buildStats, and
-				// needs to be carried over to the next result.
-				for l, c := range inProgressCounts {
-					if _, ok := counts[l]; !ok {
-						upstreamLocalityStats = append(upstreamLocalityStats, &endpointpb.UpstreamLocalityStats{
-							Locality:                testutils.LocalityIDToProto(l),
-							TotalRequestsInProgress: c,
-						})
-					}
-				}
-				want := []*endpointpb.ClusterStats{
-					{
-						ClusterName:           testService,
-						UpstreamLocalityStats: upstreamLocalityStats,
-					},
-				}
-
-				var wg sync.WaitGroup
-				for l, count := range counts {
-					for i := 0; i < int(count.success); i++ {
-						wg.Add(1)
-						go func(l internal.LocalityID, serverData map[string]float64) {
-							ls.CallStarted(l)
-							ls.CallFinished(l, nil)
-							for n, d := range serverData {
-								ls.CallServerLoad(l, n, d)
-							}
-							wg.Done()
-						}(l, count.serverData)
-					}
-					for i := 0; i < int(count.failure); i++ {
-						wg.Add(1)
-						go func(l internal.LocalityID) {
-							ls.CallStarted(l)
-							ls.CallFinished(l, errTest)
-							wg.Done()
-						}(l)
-					}
-					for i := 0; i < int(count.start-count.success-count.failure); i++ {
-						wg.Add(1)
-						go func(l internal.LocalityID) {
-							ls.CallStarted(l)
-							wg.Done()
-						}(l)
-					}
-				}
-				wg.Wait()
-
-				if got := ls.buildStats(testService); !equalClusterStats(got, want) {
-					t.Errorf("lrsStore.buildStats() = %v, want %v", got, want)
-					t.Errorf("%s", cmp.Diff(got, want))
-				}
-			}
-		})
-	}
-}
-
-type lrsServer struct {
-	reportingInterval *durationpb.Duration
-
-	mu        sync.Mutex
-	dropTotal uint64
-	drops     map[string]uint64
-	rpcs      map[internal.LocalityID]*rpcCountDataForTest
-}
-
-func (lrss *lrsServer) StreamLoadStats(stream lrsgrpc.LoadReportingService_StreamLoadStatsServer) error {
-	req, err := stream.Recv()
-	if err != nil {
-		return err
-	}
-
-	if req.GetNode().GetMetadata().GetFields()[nodeMetadataHostnameKey].GetStringValue() != testHostname {
-		return status.Errorf(codes.FailedPrecondition, "unexpected req: %+v", req)
-	}
-	if err := stream.Send(&lrspb.LoadStatsResponse{
-		Clusters:              []string{testService, "another-cluster"},
-		LoadReportingInterval: lrss.reportingInterval,
-	}); err != nil {
-		return err
-	}
-
-	for {
-		req, err := stream.Recv()
-		if err != nil {
-			if err == io.EOF {
-				return nil
-			}
-			return err
-		}
-		stats := req.ClusterStats[0]
-		lrss.mu.Lock()
-		lrss.dropTotal += stats.TotalDroppedRequests
-		for _, d := range stats.DroppedRequests {
-			lrss.drops[d.Category] += d.DroppedCount
-		}
-		for _, ss := range stats.UpstreamLocalityStats {
-			l := internal.LocalityID{
-				Region:  ss.Locality.Region,
-				Zone:    ss.Locality.Zone,
-				SubZone: ss.Locality.SubZone,
-			}
-			counts, ok := lrss.rpcs[l]
-			if !ok {
-				counts = newRPCCountDataForTest(0, 0, 0, nil)
-				lrss.rpcs[l] = counts
-			}
-			counts.succeeded += ss.TotalSuccessfulRequests
-			counts.inProgress = ss.TotalRequestsInProgress
-			counts.errored += ss.TotalErrorRequests
-			for _, ts := range ss.LoadMetricStats {
-				if counts.serverLoads == nil {
-					counts.serverLoads = make(map[string]float64)
-				}
-				counts.serverLoads[ts.MetricName] = ts.TotalMetricValue / float64(ts.NumRequestsFinishedWithMetric)
-			}
-		}
-		lrss.mu.Unlock()
-	}
-}
-
-func setupServer(t *testing.T, reportingInterval *durationpb.Duration) (addr string, lrss *lrsServer, cleanup func()) {
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("listen failed due to: %v", err)
-	}
-	svr := grpc.NewServer()
-	lrss = &lrsServer{
-		reportingInterval: reportingInterval,
-		drops:             make(map[string]uint64),
-		rpcs:              make(map[internal.LocalityID]*rpcCountDataForTest),
-	}
-	lrsgrpc.RegisterLoadReportingServiceServer(svr, lrss)
-	go svr.Serve(lis)
-	return lis.Addr().String(), lrss, func() {
-		svr.Stop()
-		lis.Close()
-	}
-}
-
-func Test_lrsStore_ReportTo(t *testing.T) {
-	const intervalNano = 1000 * 1000 * 50
-	addr, lrss, cleanup := setupServer(t, &durationpb.Duration{
-		Seconds: 0,
-		Nanos:   intervalNano,
-	})
-	defer cleanup()
-
-	ls := NewStore()
-	cc, err := grpc.Dial(addr, grpc.WithInsecure())
-	if err != nil {
-		t.Fatalf("failed to dial: %v", err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	done := make(chan struct{})
-	go func() {
-		node := &corepb.Node{
-			Metadata: &structpb.Struct{
-				Fields: map[string]*structpb.Value{
-					nodeMetadataHostnameKey: {
-						Kind: &structpb.Value_StringValue{StringValue: testHostname},
-					},
-				},
-			},
-		}
-		ls.ReportTo(ctx, cc, testService, node)
-		close(done)
-	}()
-
-	drops := map[string]uint64{
-		dropCategories[0]: 13,
-		dropCategories[1]: 14,
-	}
-	for c, d := range drops {
-		for i := 0; i < int(d); i++ {
-			ls.CallDropped(c)
-			time.Sleep(time.Nanosecond * intervalNano / 10)
-		}
-	}
-
-	rpcs := map[internal.LocalityID]*rpcCountDataForTest{
-		localities[0]: newRPCCountDataForTest(3, 1, 4, nil),
-		localities[1]: newRPCCountDataForTest(1, 5, 9, map[string]float64{"pi": 3.14, "e": 2.71}),
-	}
-	for l, count := range rpcs {
-		for i := 0; i < int(count.succeeded); i++ {
-			go func(i int, l internal.LocalityID, count *rpcCountDataForTest) {
-				ls.CallStarted(l)
-				ls.CallFinished(l, nil)
-				for n, d := range count.serverLoads {
-					ls.CallServerLoad(l, n, d)
-				}
-			}(i, l, count)
-		}
-		for i := 0; i < int(count.inProgress); i++ {
-			go func(i int, l internal.LocalityID) {
-				ls.CallStarted(l)
-			}(i, l)
-		}
-		for i := 0; i < int(count.errored); i++ {
-			go func(i int, l internal.LocalityID) {
-				ls.CallStarted(l)
-				ls.CallFinished(l, errTest)
-			}(i, l)
-		}
-	}
-
-	time.Sleep(time.Nanosecond * intervalNano * 2)
-	cancel()
-	<-done
-
-	lrss.mu.Lock()
-	defer lrss.mu.Unlock()
-	if !cmp.Equal(lrss.drops, drops) {
-		t.Errorf("different: %v", cmp.Diff(lrss.drops, drops))
-	}
-	if !cmp.Equal(lrss.rpcs, rpcs) {
-		t.Errorf("different: %v", cmp.Diff(lrss.rpcs, rpcs))
-	}
-}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 2f260643777b..0f093c20b9e8 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -21,6 +21,7 @@
 package client
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
@@ -38,6 +39,7 @@ import (
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -76,6 +78,9 @@ type BuildOptions struct {
 	// Backoff returns the amount of time to backoff before retrying broken
 	// streams.
 	Backoff func(int) time.Duration
+	// LoadStore contains load reports which need to be pushed to the management
+	// server.
+	LoadStore *load.Store
 	// Logger provides enhanced logging capabilities.
 	Logger *grpclog.PrefixLogger
 }
@@ -97,13 +102,28 @@ type APIClientBuilder interface {
 type APIClient interface {
 	// AddWatch adds a watch for an xDS resource given its type and name.
 	AddWatch(ResourceType, string)
+
 	// RemoveWatch cancels an already registered watch for an xDS resource
 	// given its type and name.
 	RemoveWatch(ResourceType, string)
+
+	// ReportLoad starts an LRS stream to periodically report load using the
+	// provided ClientConn, which represent a connection to the management
+	// server.
+	ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions)
+
 	// Close cleans up resources allocated by the API client.
 	Close()
 }
 
+// LoadReportingOptions contains configuration knobs for reporting load data.
+type LoadReportingOptions struct {
+	// ClusterName is the cluster name for which load is being reported.
+	ClusterName string
+	// TargetName is the target of the parent ClientConn.
+	TargetName string
+}
+
 // UpdateHandler receives and processes (by taking appropriate actions) xDS
 // resource updates from an APIClient for a specific version.
 type UpdateHandler interface {
@@ -272,6 +292,7 @@ type Client struct {
 	opts      Options
 	cc        *grpc.ClientConn // Connection to the xDS server
 	apiClient APIClient
+	loadStore *load.Store
 
 	logger *grpclog.PrefixLogger
 
@@ -325,8 +346,9 @@ func New(opts Options) (*Client, error) {
 	}
 
 	c := &Client{
-		done: grpcsync.NewEvent(),
-		opts: opts,
+		done:      grpcsync.NewEvent(),
+		opts:      opts,
+		loadStore: &load.Store{},
 
 		updateCh:    buffer.NewUnbounded(),
 		ldsWatchers: make(map[string]map[*watchInfo]bool),
@@ -352,6 +374,7 @@ func New(opts Options) (*Client, error) {
 		Parent:    c,
 		NodeProto: opts.Config.NodeProto,
 		Backoff:   backoff.DefaultExponential.Backoff,
+		LoadStore: c.loadStore,
 		Logger:    c.logger,
 	})
 	if err != nil {
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go
index 8539e973511c..e52c3b93f90e 100644
--- a/xds/internal/client/client_loadreport.go
+++ b/xds/internal/client/client_loadreport.go
@@ -20,25 +20,29 @@ package client
 import (
 	"context"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	"github.com/golang/protobuf/proto"
-	structpb "github.com/golang/protobuf/ptypes/struct"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
+	"google.golang.org/grpc/xds/internal/client/load"
 )
 
-const nodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME"
+// NodeMetadataHostnameKey is the metadata key for specifying the target name in
+// the node proto of an LRS request.
+const NodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME"
 
-// ReportLoad sends the load of the given clusterName from loadStore to the
-// given server. If the server is not an empty string, and is different from the
-// xds server, a new ClientConn will be created.
+// LoadStore returns the underlying load data store used by the xDS client.
+func (c *Client) LoadStore() *load.Store {
+	return c.loadStore
+}
+
+// ReportLoad sends the load of the given clusterName to the given server. If
+// the server is not an empty string, and is different from the xds server, a
+// new ClientConn will be created.
 //
 // The same options used for creating the Client will be used (including
 // NodeProto, and dial options if necessary).
 //
 // It returns a function to cancel the load reporting stream. If server is
 // different from xds server, the ClientConn will also be closed.
-func (c *Client) ReportLoad(server string, clusterName string, loadStore lrs.Store) func() {
+func (c *Client) ReportLoad(server string, clusterName string) func() {
 	var (
 		cc      *grpc.ClientConn
 		closeCC bool
@@ -59,21 +63,10 @@ func (c *Client) ReportLoad(server string, clusterName string, loadStore lrs.Sto
 		closeCC = true
 	}
 	ctx, cancel := context.WithCancel(context.Background())
-
-	nodeTemp := proto.Clone(c.opts.Config.NodeProto).(*corepb.Node)
-	if nodeTemp == nil {
-		nodeTemp = &corepb.Node{}
-	}
-	if nodeTemp.Metadata == nil {
-		nodeTemp.Metadata = &structpb.Struct{}
-	}
-	if nodeTemp.Metadata.Fields == nil {
-		nodeTemp.Metadata.Fields = make(map[string]*structpb.Value)
-	}
-	nodeTemp.Metadata.Fields[nodeMetadataHostnameKey] = &structpb.Value{
-		Kind: &structpb.Value_StringValue{StringValue: c.opts.TargetName},
-	}
-	go loadStore.ReportTo(ctx, c.cc, clusterName, nodeTemp)
+	go c.apiClient.ReportLoad(ctx, c.cc, LoadReportingOptions{
+		ClusterName: clusterName,
+		TargetName:  c.opts.TargetName,
+	})
 	return func() {
 		cancel()
 		if closeCC {
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 34c26479fa94..b36a5af7ee26 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -113,6 +113,9 @@ func (c *testAPIClient) RemoveWatch(resourceType ResourceType, resourceName stri
 	c.removeWatches[resourceType].Send(resourceName)
 }
 
+func (c *testAPIClient) ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions) {
+}
+
 func (c *testAPIClient) Close() {}
 
 // TestWatchCallAnotherWatch covers the case where watch() is called inline by a
diff --git a/xds/internal/client/load/store.go b/xds/internal/client/load/store.go
new file mode 100644
index 000000000000..034f47f8ee29
--- /dev/null
+++ b/xds/internal/client/load/store.go
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package load provides functionality to record and maintain load data.
+package load
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+const negativeOneUInt64 = ^uint64(0)
+
+// Store is a repository for LB policy implementations to report store load
+// data. It is safe for concurrent use.
+//
+// A zero Store is empty and ready for use.
+//
+// TODO(easwars): Use regular maps with mutexes instead of sync.Map here. The
+// latter is optimized for two common use cases: (1) when the entry for a given
+// key is only ever written once but read many times, as in caches that only
+// grow, or (2) when multiple goroutines read, write, and overwrite entries for
+// disjoint sets of keys. In these two cases, use of a Map may significantly
+// reduce lock contention compared to a Go map paired with a separate Mutex or
+// RWMutex.
+// Neither of these conditions are met here, and we should transition to a
+// regular map with a mutex for better type safety.
+type Store struct {
+	drops            sync.Map // map[string]*uint64
+	localityRPCCount sync.Map // map[string]*rpcCountData
+}
+
+// Update functions are called by picker for each RPC. To avoid contention, all
+// updates are done atomically.
+
+// CallDropped adds one drop record with the given category to store.
+func (ls *Store) CallDropped(category string) {
+	if ls == nil {
+		return
+	}
+
+	p, ok := ls.drops.Load(category)
+	if !ok {
+		tp := new(uint64)
+		p, _ = ls.drops.LoadOrStore(category, tp)
+	}
+	atomic.AddUint64(p.(*uint64), 1)
+}
+
+// CallStarted adds one call started record for the given locality.
+func (ls *Store) CallStarted(locality string) {
+	if ls == nil {
+		return
+	}
+
+	p, ok := ls.localityRPCCount.Load(locality)
+	if !ok {
+		tp := newRPCCountData()
+		p, _ = ls.localityRPCCount.LoadOrStore(locality, tp)
+	}
+	p.(*rpcCountData).incrInProgress()
+}
+
+// CallFinished adds one call finished record for the given locality.
+// For successful calls, err needs to be nil.
+func (ls *Store) CallFinished(locality string, err error) {
+	if ls == nil {
+		return
+	}
+
+	p, ok := ls.localityRPCCount.Load(locality)
+	if !ok {
+		// The map is never cleared, only values in the map are reset. So the
+		// case where entry for call-finish is not found should never happen.
+		return
+	}
+	p.(*rpcCountData).decrInProgress()
+	if err == nil {
+		p.(*rpcCountData).incrSucceeded()
+	} else {
+		p.(*rpcCountData).incrErrored()
+	}
+}
+
+// CallServerLoad adds one server load record for the given locality. The
+// load type is specified by desc, and its value by val.
+func (ls *Store) CallServerLoad(locality, name string, d float64) {
+	if ls == nil {
+		return
+	}
+
+	p, ok := ls.localityRPCCount.Load(locality)
+	if !ok {
+		// The map is never cleared, only values in the map are reset. So the
+		// case where entry for CallServerLoad is not found should never happen.
+		return
+	}
+	p.(*rpcCountData).addServerLoad(name, d)
+}
+
+// Data contains all load data reported to the Store since the most recent call
+// to Stats().
+type Data struct {
+	// TotalDrops is the total number of dropped requests.
+	TotalDrops uint64
+	// Drops is the number of dropped requests per category.
+	Drops map[string]uint64
+	// LocalityStats contains load reports per locality.
+	LocalityStats map[string]LocalityData
+}
+
+// LocalityData contains load data for a single locality.
+type LocalityData struct {
+	// RequestStats contains counts of requests made to the locality.
+	RequestStats RequestData
+	// LoadStats contains server load data for requests made to the locality,
+	// indexed by the load type.
+	LoadStats map[string]ServerLoadData
+}
+
+// RequestData contains request counts.
+type RequestData struct {
+	// Succeeded is the number of succeeded requests.
+	Succeeded uint64
+	// Errored is the number of requests which ran into errors.
+	Errored uint64
+	// InProgress is the number of requests in flight.
+	InProgress uint64
+}
+
+// ServerLoadData contains server load data.
+type ServerLoadData struct {
+	// Count is the number of load reports.
+	Count uint64
+	// Sum is the total value of all load reports.
+	Sum float64
+}
+
+func newStoreData() *Data {
+	return &Data{
+		Drops:         make(map[string]uint64),
+		LocalityStats: make(map[string]LocalityData),
+	}
+}
+
+// Stats returns and resets all loads reported to the store, except inProgress
+// rpc counts.
+func (ls *Store) Stats() *Data {
+	if ls == nil {
+		return nil
+	}
+
+	sd := newStoreData()
+	ls.drops.Range(func(key, val interface{}) bool {
+		d := atomic.SwapUint64(val.(*uint64), 0)
+		if d == 0 {
+			return true
+		}
+		sd.TotalDrops += d
+		sd.Drops[key.(string)] = d
+		return true
+	})
+	ls.localityRPCCount.Range(func(key, val interface{}) bool {
+		countData := val.(*rpcCountData)
+		succeeded := countData.loadAndClearSucceeded()
+		inProgress := countData.loadInProgress()
+		errored := countData.loadAndClearErrored()
+		if succeeded == 0 && inProgress == 0 && errored == 0 {
+			return true
+		}
+
+		ld := LocalityData{
+			RequestStats: RequestData{
+				Succeeded:  succeeded,
+				Errored:    errored,
+				InProgress: inProgress,
+			},
+			LoadStats: make(map[string]ServerLoadData),
+		}
+		countData.serverLoads.Range(func(key, val interface{}) bool {
+			sum, count := val.(*rpcLoadData).loadAndClear()
+			if count == 0 {
+				return true
+			}
+			ld.LoadStats[key.(string)] = ServerLoadData{
+				Count: count,
+				Sum:   sum,
+			}
+			return true
+		})
+		sd.LocalityStats[key.(string)] = ld
+		return true
+	})
+	return sd
+}
+
+type rpcCountData struct {
+	// Only atomic accesses are allowed for the fields.
+	succeeded  *uint64
+	errored    *uint64
+	inProgress *uint64
+
+	// Map from load desc to load data (sum+count). Loading data from map is
+	// atomic, but updating data takes a lock, which could cause contention when
+	// multiple RPCs try to report loads for the same desc.
+	//
+	// To fix the contention, shard this map.
+	serverLoads sync.Map // map[string]*rpcLoadData
+}
+
+func newRPCCountData() *rpcCountData {
+	return &rpcCountData{
+		succeeded:  new(uint64),
+		errored:    new(uint64),
+		inProgress: new(uint64),
+	}
+}
+
+func (rcd *rpcCountData) incrSucceeded() {
+	atomic.AddUint64(rcd.succeeded, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearSucceeded() uint64 {
+	return atomic.SwapUint64(rcd.succeeded, 0)
+}
+
+func (rcd *rpcCountData) incrErrored() {
+	atomic.AddUint64(rcd.errored, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearErrored() uint64 {
+	return atomic.SwapUint64(rcd.errored, 0)
+}
+
+func (rcd *rpcCountData) incrInProgress() {
+	atomic.AddUint64(rcd.inProgress, 1)
+}
+
+func (rcd *rpcCountData) decrInProgress() {
+	atomic.AddUint64(rcd.inProgress, negativeOneUInt64) // atomic.Add(x, -1)
+}
+
+func (rcd *rpcCountData) loadInProgress() uint64 {
+	return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading.
+}
+
+func (rcd *rpcCountData) addServerLoad(name string, d float64) {
+	loads, ok := rcd.serverLoads.Load(name)
+	if !ok {
+		tl := newRPCLoadData()
+		loads, _ = rcd.serverLoads.LoadOrStore(name, tl)
+	}
+	loads.(*rpcLoadData).add(d)
+}
+
+// Data for server loads (from trailers or oob). Fields in this struct must be
+// updated consistently.
+//
+// The current solution is to hold a lock, which could cause contention. To fix,
+// shard serverLoads map in rpcCountData.
+type rpcLoadData struct {
+	mu    sync.Mutex
+	sum   float64
+	count uint64
+}
+
+func newRPCLoadData() *rpcLoadData {
+	return &rpcLoadData{}
+}
+
+func (rld *rpcLoadData) add(v float64) {
+	rld.mu.Lock()
+	rld.sum += v
+	rld.count++
+	rld.mu.Unlock()
+}
+
+func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) {
+	rld.mu.Lock()
+	s = rld.sum
+	rld.sum = 0
+	c = rld.count
+	rld.count = 0
+	rld.mu.Unlock()
+	return
+}
diff --git a/xds/internal/client/load/store_test.go b/xds/internal/client/load/store_test.go
new file mode 100644
index 000000000000..190b826a64e5
--- /dev/null
+++ b/xds/internal/client/load/store_test.go
@@ -0,0 +1,256 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package load
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+var (
+	dropCategories = []string{"drop_for_real", "drop_for_fun"}
+	localities     = []string{"locality-A", "locality-B"}
+	errTest        = fmt.Errorf("test error")
+)
+
+// rpcData wraps the rpc counts and load data to be pushed to the store.
+type rpcData struct {
+	start, success, failure int
+	serverData              map[string]float64 // Will be reported with successful RPCs.
+}
+
+// TestDrops spawns a bunch of goroutines which report drop data. After the
+// goroutines have exited, the test dumps the stats from the Store and makes
+// sure they are as expected.
+func TestDrops(t *testing.T) {
+	var (
+		drops = map[string]int{
+			dropCategories[0]: 30,
+			dropCategories[1]: 40,
+		}
+		wantStoreData = &Data{
+			TotalDrops: 70,
+			Drops: map[string]uint64{
+				dropCategories[0]: 30,
+				dropCategories[1]: 40,
+			},
+		}
+	)
+
+	ls := Store{}
+	var wg sync.WaitGroup
+	for category, count := range drops {
+		for i := 0; i < count; i++ {
+			wg.Add(1)
+			go func(c string) {
+				ls.CallDropped(c)
+				wg.Done()
+			}(category)
+		}
+	}
+	wg.Wait()
+
+	gotStoreData := ls.Stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
+		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+}
+
+// TestLocalityStats spawns a bunch of goroutines which report rpc and load
+// data. After the goroutines have exited, the test dumps the stats from the
+// Store and makes sure they are as expected.
+func TestLocalityStats(t *testing.T) {
+	var (
+		localityData = map[string]rpcData{
+			localities[0]: {
+				start:      40,
+				success:    20,
+				failure:    10,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+			localities[1]: {
+				start:      80,
+				success:    40,
+				failure:    20,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+		}
+		wantStoreData = &Data{
+			LocalityStats: map[string]LocalityData{
+				localities[0]: {
+					RequestStats: RequestData{Succeeded: 20, Errored: 10, InProgress: 10},
+					LoadStats: map[string]ServerLoadData{
+						"net":  {Count: 20, Sum: 20},
+						"disk": {Count: 20, Sum: 40},
+						"cpu":  {Count: 20, Sum: 60},
+						"mem":  {Count: 20, Sum: 80},
+					},
+				},
+				localities[1]: {
+					RequestStats: RequestData{Succeeded: 40, Errored: 20, InProgress: 20},
+					LoadStats: map[string]ServerLoadData{
+						"net":  {Count: 40, Sum: 40},
+						"disk": {Count: 40, Sum: 80},
+						"cpu":  {Count: 40, Sum: 120},
+						"mem":  {Count: 40, Sum: 160},
+					},
+				},
+			},
+		}
+	)
+
+	ls := Store{}
+	var wg sync.WaitGroup
+	for locality, data := range localityData {
+		wg.Add(data.start)
+		for i := 0; i < data.start; i++ {
+			go func(l string) {
+				ls.CallStarted(l)
+				wg.Done()
+			}(locality)
+		}
+		// The calls to CallStarted() need to happen before the other calls are
+		// made. Hence the wait here.
+		wg.Wait()
+
+		wg.Add(data.success)
+		for i := 0; i < data.success; i++ {
+			go func(l string, serverData map[string]float64) {
+				ls.CallFinished(l, nil)
+				for n, d := range serverData {
+					ls.CallServerLoad(l, n, d)
+				}
+				wg.Done()
+			}(locality, data.serverData)
+		}
+		wg.Add(data.failure)
+		for i := 0; i < data.failure; i++ {
+			go func(l string) {
+				ls.CallFinished(l, errTest)
+				wg.Done()
+			}(locality)
+		}
+		wg.Wait()
+	}
+
+	gotStoreData := ls.Stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
+		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+}
+
+func TestResetAfterStats(t *testing.T) {
+	// Push a bunch of drops, call stats and load stats, and leave inProgress to be non-zero.
+	// Dump the stats. Verify expexted
+	// Push the same set of loads as before
+	// Now dump and verify the newly expected ones.
+	var (
+		drops = map[string]int{
+			dropCategories[0]: 30,
+			dropCategories[1]: 40,
+		}
+		localityData = map[string]rpcData{
+			localities[0]: {
+				start:      40,
+				success:    20,
+				failure:    10,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+			localities[1]: {
+				start:      80,
+				success:    40,
+				failure:    20,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+		}
+		wantStoreData = &Data{
+			TotalDrops: 70,
+			Drops: map[string]uint64{
+				dropCategories[0]: 30,
+				dropCategories[1]: 40,
+			},
+			LocalityStats: map[string]LocalityData{
+				localities[0]: {
+					RequestStats: RequestData{Succeeded: 20, Errored: 10, InProgress: 10},
+					LoadStats: map[string]ServerLoadData{
+						"net":  {Count: 20, Sum: 20},
+						"disk": {Count: 20, Sum: 40},
+						"cpu":  {Count: 20, Sum: 60},
+						"mem":  {Count: 20, Sum: 80},
+					},
+				},
+				localities[1]: {
+					RequestStats: RequestData{Succeeded: 40, Errored: 20, InProgress: 20},
+					LoadStats: map[string]ServerLoadData{
+						"net":  {Count: 40, Sum: 40},
+						"disk": {Count: 40, Sum: 80},
+						"cpu":  {Count: 40, Sum: 120},
+						"mem":  {Count: 40, Sum: 160},
+					},
+				},
+			},
+		}
+	)
+
+	reportLoad := func(ls *Store) {
+		for category, count := range drops {
+			for i := 0; i < count; i++ {
+				ls.CallDropped(category)
+			}
+		}
+		for locality, data := range localityData {
+			for i := 0; i < data.start; i++ {
+				ls.CallStarted(locality)
+			}
+			for i := 0; i < data.success; i++ {
+				ls.CallFinished(locality, nil)
+				for n, d := range data.serverData {
+					ls.CallServerLoad(locality, n, d)
+				}
+			}
+			for i := 0; i < data.failure; i++ {
+				ls.CallFinished(locality, errTest)
+			}
+		}
+	}
+
+	ls := Store{}
+	reportLoad(&ls)
+	gotStoreData := ls.Stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
+		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+
+	// The above call to Stats() should have reset all load reports except the
+	// inProgress rpc count. We are now going to push the same load data into
+	// the store. So, we should expect to see twice the count for inProgress.
+	for _, l := range localities {
+		ls := wantStoreData.LocalityStats[l]
+		ls.RequestStats.InProgress *= 2
+		wantStoreData.LocalityStats[l] = ls
+	}
+	reportLoad(&ls)
+	gotStoreData = ls.Stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
+		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+}
diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
index 976003605041..4f5779956d73 100644
--- a/xds/internal/client/transport_helper.go
+++ b/xds/internal/client/transport_helper.go
@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
@@ -44,7 +45,7 @@ func (e ErrResourceTypeUnsupported) Error() string {
 // specific client implementations. This mainly deals with the actual sending
 // and receiving of messages.
 type VersionedClient interface {
-	// NewStream returns a new grpc.ClientStream specific to the underlying
+	// NewStream returns a new xDS client stream specific to the underlying
 	// transport protocol version.
 	NewStream(ctx context.Context) (grpc.ClientStream, error)
 
@@ -54,7 +55,7 @@ type VersionedClient interface {
 
 	// RecvResponse uses the provided stream to receive a response specific to
 	// the underlying transport protocol version.
-	RecvResponse(stream grpc.ClientStream) (proto.Message, error)
+	RecvResponse(s grpc.ClientStream) (proto.Message, error)
 
 	// HandleResponse parses and validates the received response and notifies
 	// the top-level client which in turn notifies the registered watchers.
@@ -64,6 +65,25 @@ type VersionedClient interface {
 	// supported, implementations must return an error of type
 	// ErrResourceTypeUnsupported.
 	HandleResponse(proto.Message) (ResourceType, string, string, error)
+
+	// NewLoadStatsStream returns a new LRS client stream specific to the underlying
+	// transport protocol version.
+	NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error)
+
+	// SendFirstLoadStatsRequest constructs and sends the first request on the
+	// LRS stream. This contains the node proto with appropriate metadata
+	// fields.
+	SendFirstLoadStatsRequest(s grpc.ClientStream, targetName string) error
+
+	// HandleLoadStatsResponse receives the first response from the server which
+	// contains the load reporting interval and the clusters for which the
+	// server asks the client to report load for.
+	HandleLoadStatsResponse(s grpc.ClientStream, clusterName string) (time.Duration, error)
+
+	// SendLoadStatsRequest will be invoked at regular intervals to send load
+	// report with load data reported since the last time this method was
+	// invoked.
+	SendLoadStatsRequest(s grpc.ClientStream, clusterName string) error
 }
 
 // TransportHelper contains all xDS transport protocol related functionality
@@ -419,3 +439,66 @@ func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStrea
 	}
 	return target, rType, version, nonce, send
 }
+
+// ReportLoad starts an LRS stream to report load data to the management server.
+// It blocks until the context is cancelled.
+func (t *TransportHelper) ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions) {
+	retries := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if retries != 0 {
+			timer := time.NewTimer(t.backoff(retries))
+			select {
+			case <-timer.C:
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			}
+		}
+
+		retries++
+		stream, err := t.vClient.NewLoadStatsStream(ctx, cc)
+		if err != nil {
+			logger.Warningf("lrs: failed to create stream: %v", err)
+			continue
+		}
+		logger.Infof("lrs: created LRS stream")
+
+		if err := t.vClient.SendFirstLoadStatsRequest(stream, opts.TargetName); err != nil {
+			logger.Warningf("lrs: failed to send first request: %v", err)
+			continue
+		}
+
+		interval, err := t.vClient.HandleLoadStatsResponse(stream, opts.ClusterName)
+		if err != nil {
+			logger.Warning(err)
+			continue
+		}
+
+		retries = 0
+		t.sendLoads(ctx, stream, opts.ClusterName, interval)
+	}
+}
+
+func (t *TransportHelper) sendLoads(ctx context.Context, stream grpc.ClientStream, clusterName string, interval time.Duration) {
+	tick := time.NewTicker(interval)
+	defer tick.Stop()
+	for {
+		select {
+		case <-tick.C:
+		case <-ctx.Done():
+			return
+		}
+		if err := t.vClient.SendLoadStatsRequest(stream, clusterName); err != nil {
+			logger.Warning(err)
+			return
+		}
+	}
+}
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index 96bd5e9b5686..f838b5fed355 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -23,11 +23,13 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/version"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
@@ -67,6 +69,7 @@ func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIC
 		cc:        cc,
 		parent:    opts.Parent,
 		nodeProto: nodeProto,
+		loadStore: opts.LoadStore,
 		logger:    opts.Logger,
 	}
 	v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background())
@@ -85,6 +88,7 @@ type client struct {
 	ctx       context.Context
 	cancelCtx context.CancelFunc
 	parent    xdsclient.UpdateHandler
+	loadStore *load.Store
 	logger    *grpclog.PrefixLogger
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
@@ -99,6 +103,8 @@ type client struct {
 	// processing needs this to do the host matching.
 	ldsResourceName string
 	ldsWatchCount   int
+
+	lastLoadReportAt time.Time
 }
 
 // AddWatch overrides the transport helper's AddWatch to save the LDS
diff --git a/xds/internal/client/v2/loadreport.go b/xds/internal/client/v2/loadreport.go
new file mode 100644
index 000000000000..d56ed3349ee8
--- /dev/null
+++ b/xds/internal/client/v2/loadreport.go
@@ -0,0 +1,173 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package v2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
+	lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
+	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
+	structpb "github.com/golang/protobuf/ptypes/struct"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/xds/internal"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient
+
+func (v2c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) {
+	c := lrsgrpc.NewLoadReportingServiceClient(cc)
+	return c.StreamLoadStats(ctx)
+}
+
+func (v2c *client) SendFirstLoadStatsRequest(s grpc.ClientStream, targetName string) error {
+	stream, ok := s.(lrsStream)
+	if !ok {
+		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
+	}
+	node := proto.Clone(v2c.nodeProto).(*v2corepb.Node)
+	if node == nil {
+		node = &v2corepb.Node{}
+	}
+	if node.Metadata == nil {
+		node.Metadata = &structpb.Struct{}
+	}
+	if node.Metadata.Fields == nil {
+		node.Metadata.Fields = make(map[string]*structpb.Value)
+	}
+	node.Metadata.Fields[xdsclient.NodeMetadataHostnameKey] = &structpb.Value{
+		Kind: &structpb.Value_StringValue{StringValue: targetName},
+	}
+
+	req := &lrspb.LoadStatsRequest{Node: node}
+	v2c.logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
+	return stream.Send(req)
+}
+
+func (v2c *client) HandleLoadStatsResponse(s grpc.ClientStream, clusterName string) (time.Duration, error) {
+	stream, ok := s.(lrsStream)
+	if !ok {
+		return 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s)
+	}
+
+	resp, err := stream.Recv()
+	if err != nil {
+		return 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
+	}
+	v2c.logger.Infof("lrs: received first LoadStatsResponse: %+v", resp)
+
+	interval, err := ptypes.Duration(resp.GetLoadReportingInterval())
+	if err != nil {
+		return 0, fmt.Errorf("lrs: failed to convert report interval: %v", err)
+	}
+
+	// The LRS client should join the clusters it knows with the cluster
+	// list from response, and send loads for them.
+	//
+	// But the LRS client now only supports one cluster. TODO: extend it to
+	// support multiple clusters.
+	var clusterFoundInResponse bool
+	for _, c := range resp.Clusters {
+		if c == clusterName {
+			clusterFoundInResponse = true
+		}
+	}
+	if !clusterFoundInResponse {
+		return 0, fmt.Errorf("lrs: received clusters %v does not contain expected {%v}", resp.Clusters, clusterName)
+	}
+	if resp.ReportEndpointGranularity {
+		// TODO: fixme to support per endpoint loads.
+		return 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation")
+	}
+
+	return interval, nil
+}
+
+func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string) error {
+	stream, ok := s.(lrsStream)
+	if !ok {
+		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
+	}
+	if v2c.loadStore == nil {
+		return errors.New("lrs: LoadStore is not initialized")
+	}
+
+	var (
+		droppedReqs   []*v2endpointpb.ClusterStats_DroppedRequests
+		localityStats []*v2endpointpb.UpstreamLocalityStats
+	)
+
+	sd := v2c.loadStore.Stats()
+	for category, count := range sd.Drops {
+		droppedReqs = append(droppedReqs, &v2endpointpb.ClusterStats_DroppedRequests{
+			Category:     category,
+			DroppedCount: count,
+		})
+	}
+	for l, localityData := range sd.LocalityStats {
+		lid, err := internal.LocalityIDFromString(l)
+		if err != nil {
+			return err
+		}
+		var loadMetricStats []*v2endpointpb.EndpointLoadMetricStats
+		for name, loadData := range localityData.LoadStats {
+			loadMetricStats = append(loadMetricStats, &v2endpointpb.EndpointLoadMetricStats{
+				MetricName:                    name,
+				NumRequestsFinishedWithMetric: loadData.Count,
+				TotalMetricValue:              loadData.Sum,
+			})
+		}
+		localityStats = append(localityStats, &v2endpointpb.UpstreamLocalityStats{
+			Locality: &v2corepb.Locality{
+				Region:  lid.Region,
+				Zone:    lid.Zone,
+				SubZone: lid.SubZone,
+			},
+			TotalSuccessfulRequests: localityData.RequestStats.Succeeded,
+			TotalRequestsInProgress: localityData.RequestStats.InProgress,
+			TotalErrorRequests:      localityData.RequestStats.Errored,
+			LoadMetricStats:         loadMetricStats,
+			UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
+		})
+	}
+
+	dur := time.Since(v2c.lastLoadReportAt)
+	v2c.lastLoadReportAt = time.Now()
+
+	cs := []*v2endpointpb.ClusterStats{
+		{
+			ClusterName:           clusterName,
+			UpstreamLocalityStats: localityStats,
+			TotalDroppedRequests:  sd.TotalDrops,
+			DroppedRequests:       droppedReqs,
+			LoadReportInterval:    ptypes.DurationProto(dur),
+		},
+	}
+	req := &lrspb.LoadStatsRequest{ClusterStats: cs}
+	v2c.logger.Infof("lrs: sending LRS loads: %+v", req)
+	return stream.Send(req)
+}
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index a1103f8f8b85..f3daaecd43b5 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -23,11 +23,13 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/version"
 
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -67,6 +69,7 @@ func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIC
 		cc:        cc,
 		parent:    opts.Parent,
 		nodeProto: nodeProto,
+		loadStore: opts.LoadStore,
 		logger:    opts.Logger,
 	}
 	v3c.ctx, v3c.cancelCtx = context.WithCancel(context.Background())
@@ -85,6 +88,7 @@ type client struct {
 	ctx       context.Context
 	cancelCtx context.CancelFunc
 	parent    xdsclient.UpdateHandler
+	loadStore *load.Store
 	logger    *grpclog.PrefixLogger
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
@@ -99,6 +103,8 @@ type client struct {
 	// processing needs this to do the host matching.
 	ldsResourceName string
 	ldsWatchCount   int
+
+	lastLoadReportAt time.Time
 }
 
 // AddWatch overrides the transport helper's AddWatch to save the LDS
diff --git a/xds/internal/client/v3/loadreport.go b/xds/internal/client/v3/loadreport.go
new file mode 100644
index 000000000000..f2dc77855a3b
--- /dev/null
+++ b/xds/internal/client/v3/loadreport.go
@@ -0,0 +1,173 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package v3
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+	structpb "github.com/golang/protobuf/ptypes/struct"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/xds/internal"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient
+
+func (v3c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) {
+	c := lrsgrpc.NewLoadReportingServiceClient(cc)
+	return c.StreamLoadStats(ctx)
+}
+
+func (v3c *client) SendFirstLoadStatsRequest(s grpc.ClientStream, targetName string) error {
+	stream, ok := s.(lrsStream)
+	if !ok {
+		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
+	}
+	node := proto.Clone(v3c.nodeProto).(*v3corepb.Node)
+	if node == nil {
+		node = &v3corepb.Node{}
+	}
+	if node.Metadata == nil {
+		node.Metadata = &structpb.Struct{}
+	}
+	if node.Metadata.Fields == nil {
+		node.Metadata.Fields = make(map[string]*structpb.Value)
+	}
+	node.Metadata.Fields[xdsclient.NodeMetadataHostnameKey] = &structpb.Value{
+		Kind: &structpb.Value_StringValue{StringValue: targetName},
+	}
+
+	req := &lrspb.LoadStatsRequest{Node: node}
+	v3c.logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
+	return stream.Send(req)
+}
+
+func (v3c *client) HandleLoadStatsResponse(s grpc.ClientStream, clusterName string) (time.Duration, error) {
+	stream, ok := s.(lrsStream)
+	if !ok {
+		return 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s)
+	}
+
+	resp, err := stream.Recv()
+	if err != nil {
+		return 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
+	}
+	v3c.logger.Infof("lrs: received first LoadStatsResponse: %+v", resp)
+
+	interval, err := ptypes.Duration(resp.GetLoadReportingInterval())
+	if err != nil {
+		return 0, fmt.Errorf("lrs: failed to convert report interval: %v", err)
+	}
+
+	// The LRS client should join the clusters it knows with the cluster
+	// list from response, and send loads for them.
+	//
+	// But the LRS client now only supports one cluster. TODO: extend it to
+	// support multiple clusters.
+	var clusterFoundInResponse bool
+	for _, c := range resp.Clusters {
+		if c == clusterName {
+			clusterFoundInResponse = true
+		}
+	}
+	if !clusterFoundInResponse {
+		return 0, fmt.Errorf("lrs: received clusters %v does not contain expected {%v}", resp.Clusters, clusterName)
+	}
+	if resp.ReportEndpointGranularity {
+		// TODO: fixme to support per endpoint loads.
+		return 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation")
+	}
+
+	return interval, nil
+}
+
+func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string) error {
+	stream, ok := s.(lrsStream)
+	if !ok {
+		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
+	}
+	if v3c.loadStore == nil {
+		return errors.New("lrs: LoadStore is not initialized")
+	}
+
+	var (
+		droppedReqs   []*v3endpointpb.ClusterStats_DroppedRequests
+		localityStats []*v3endpointpb.UpstreamLocalityStats
+	)
+
+	sd := v3c.loadStore.Stats()
+	for category, count := range sd.Drops {
+		droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{
+			Category:     category,
+			DroppedCount: count,
+		})
+	}
+	for l, localityData := range sd.LocalityStats {
+		lid, err := internal.LocalityIDFromString(l)
+		if err != nil {
+			return err
+		}
+		var loadMetricStats []*v3endpointpb.EndpointLoadMetricStats
+		for name, loadData := range localityData.LoadStats {
+			loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
+				MetricName:                    name,
+				NumRequestsFinishedWithMetric: loadData.Count,
+				TotalMetricValue:              loadData.Sum,
+			})
+		}
+		localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{
+			Locality: &v3corepb.Locality{
+				Region:  lid.Region,
+				Zone:    lid.Zone,
+				SubZone: lid.SubZone,
+			},
+			TotalSuccessfulRequests: localityData.RequestStats.Succeeded,
+			TotalRequestsInProgress: localityData.RequestStats.InProgress,
+			TotalErrorRequests:      localityData.RequestStats.Errored,
+			LoadMetricStats:         loadMetricStats,
+			UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
+		})
+	}
+
+	dur := time.Since(v3c.lastLoadReportAt)
+	v3c.lastLoadReportAt = time.Now()
+
+	cs := []*v3endpointpb.ClusterStats{
+		{
+			ClusterName:           clusterName,
+			UpstreamLocalityStats: localityStats,
+			TotalDroppedRequests:  sd.TotalDrops,
+			DroppedRequests:       droppedReqs,
+			LoadReportInterval:    ptypes.DurationProto(dur),
+		},
+	}
+	req := &lrspb.LoadStatsRequest{ClusterStats: cs}
+	v3c.logger.Infof("lrs: sending LRS loads: %+v", req)
+	return stream.Send(req)
+}
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index 462a8bac59b2..69243147a35a 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -21,6 +21,7 @@ package internal
 
 import (
 	"fmt"
+	"strings"
 )
 
 type clientID string
@@ -40,6 +41,23 @@ type LocalityID struct {
 	SubZone string `json:"subZone,omitempty"`
 }
 
+// String generates a string representation of LocalityID by adding ":" between
+// the components of the LocalityID.
 func (l LocalityID) String() string {
-	return fmt.Sprintf("%s-%s-%s", l.Region, l.Zone, l.SubZone)
+	return fmt.Sprintf("%s:%s:%s", l.Region, l.Zone, l.SubZone)
+}
+
+// LocalityIDFromString converts a string representation of locality, of the
+// form region:zone:sub-zone (as generated by the above String() method), into a
+// LocalityID struct.
+func LocalityIDFromString(l string) (LocalityID, error) {
+	parts := strings.Split(l, ":")
+	if len(parts) != 3 {
+		return LocalityID{}, fmt.Errorf("%s is not a well formatted locality ID", l)
+	}
+	return LocalityID{
+		Region:  parts[0],
+		Zone:    parts[1],
+		SubZone: parts[2],
+	}, nil
 }
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 79dabc00b56b..972dd21a9dc4 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -24,8 +24,8 @@ import (
 	"sync"
 
 	"google.golang.org/grpc/internal/testutils"
-	"google.golang.org/grpc/xds/internal/balancer/lrs"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 )
 
 // Client is a fake implementation of an xds client. It exposes a bunch of
@@ -40,6 +40,7 @@ type Client struct {
 	edsCancelCh  *testutils.Channel
 	loadReportCh *testutils.Channel
 	closeCh      *testutils.Channel
+	loadStore    *load.Store
 
 	mu        sync.Mutex
 	serviceCb func(xdsclient.ServiceUpdate, error)
@@ -160,11 +161,16 @@ type ReportLoadArgs struct {
 }
 
 // ReportLoad starts reporting load about clusterName to server.
-func (xdsC *Client) ReportLoad(server string, clusterName string, loadStore lrs.Store) (cancel func()) {
+func (xdsC *Client) ReportLoad(server string, clusterName string) (cancel func()) {
 	xdsC.loadReportCh.Send(ReportLoadArgs{Server: server, Cluster: clusterName})
 	return func() {}
 }
 
+// LoadStore returns the underlying load data store.
+func (xdsC *Client) LoadStore() *load.Store {
+	return xdsC.loadStore
+}
+
 // WaitForReportLoad waits for ReportLoad to be invoked on this client and
 // returns the arguments passed to it.
 func (xdsC *Client) WaitForReportLoad(ctx context.Context) (ReportLoadArgs, error) {
@@ -208,5 +214,6 @@ func NewClientWithName(name string) *Client {
 		edsCancelCh:  testutils.NewChannel(),
 		loadReportCh: testutils.NewChannel(),
 		closeCh:      testutils.NewChannel(),
+		loadStore:    &load.Store{},
 	}
 }

From 0b575750a90c43c6763154333a6f065756b3b25e Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 1 Sep 2020 17:43:01 -0700
Subject: [PATCH 183/481] credentials/alts: Skip ALTS tests on darwin. (#3859)

Anyways, only linux and windows are supported platforms. Running these
tests on darwin causes a top level `make test` to fail, and one has to
scroll all the way up to realize that it is only these alts tests which
have failed, and not something that one is actively working on.
---
 credentials/alts/alts_test.go  | 2 ++
 credentials/alts/utils_test.go | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/credentials/alts/alts_test.go b/credentials/alts/alts_test.go
index 61666696942f..ed13ee447f3e 100644
--- a/credentials/alts/alts_test.go
+++ b/credentials/alts/alts_test.go
@@ -1,3 +1,5 @@
+// +build linux,windows
+
 /*
  *
  * Copyright 2018 gRPC authors.
diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go
index 7c84c3bc9f92..d33f189ee624 100644
--- a/credentials/alts/utils_test.go
+++ b/credentials/alts/utils_test.go
@@ -1,3 +1,5 @@
+// +build linux,windows
+
 /*
  *
  * Copyright 2018 gRPC authors.

From 8c8305f8d5f24afb8b636700c633a97e36404d27 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 1 Sep 2020 17:59:14 -0700
Subject: [PATCH 184/481] benchmark: Revert to use old certs. (#3860)

---
 benchmark/worker/benchmark_client.go | 2 +-
 benchmark/worker/benchmark_server.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go
index 3fa682d5a77e..9253c0b48737 100644
--- a/benchmark/worker/benchmark_client.go
+++ b/benchmark/worker/benchmark_client.go
@@ -123,7 +123,7 @@ func createConns(config *testpb.ClientConfig) ([]*grpc.ClientConn, func(), error
 	// Check and set security options.
 	if config.SecurityParams != nil {
 		if *caFile == "" {
-			*caFile = testdata.Path("x509/server_ca.pem")
+			*caFile = testdata.Path("ca.pem")
 		}
 		creds, err := credentials.NewClientTLSFromFile(*caFile, config.SecurityParams.ServerHostOverride)
 		if err != nil {
diff --git a/benchmark/worker/benchmark_server.go b/benchmark/worker/benchmark_server.go
index c2e5e2609786..60d52d75da38 100644
--- a/benchmark/worker/benchmark_server.go
+++ b/benchmark/worker/benchmark_server.go
@@ -94,10 +94,10 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma
 	// Set security options.
 	if config.SecurityParams != nil {
 		if *certFile == "" {
-			*certFile = testdata.Path("x509/server1_cert.pem")
+			*certFile = testdata.Path("server1.pem")
 		}
 		if *keyFile == "" {
-			*keyFile = testdata.Path("x509/server1_key.pem")
+			*keyFile = testdata.Path("server1.key")
 		}
 		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
 		if err != nil {

From 9a132e444fbe917d98a11fa42480b10113a861e6 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Wed, 2 Sep 2020 00:01:40 -0700
Subject: [PATCH 185/481] advancedtls: add PEMFileProvider implementation for
 on-file-change credential reloading (#3826)

* add provider for pem file reloading
---
 security/advancedtls/pemfile_provider.go      | 190 +++++++++++++++
 security/advancedtls/pemfile_provider_test.go | 219 ++++++++++++++++++
 2 files changed, 409 insertions(+)
 create mode 100644 security/advancedtls/pemfile_provider.go
 create mode 100644 security/advancedtls/pemfile_provider_test.go

diff --git a/security/advancedtls/pemfile_provider.go b/security/advancedtls/pemfile_provider.go
new file mode 100644
index 000000000000..3dbb52440d45
--- /dev/null
+++ b/security/advancedtls/pemfile_provider.go
@@ -0,0 +1,190 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/grpclog"
+)
+
+const defaultIdentityInterval = 1 * time.Hour
+const defaultRootInterval = 2 * time.Hour
+
+// readKeyCertPairFunc will be overridden from unit tests.
+var readKeyCertPairFunc = tls.LoadX509KeyPair
+
+// readTrustCertFunc will be overridden from unit tests.
+var readTrustCertFunc = func(trustFile string) (*x509.CertPool, error) {
+	trustData, err := ioutil.ReadFile(trustFile)
+	if err != nil {
+		return nil, err
+	}
+	trustPool := x509.NewCertPool()
+	if !trustPool.AppendCertsFromPEM(trustData) {
+		return nil, fmt.Errorf("AppendCertsFromPEM failed to parse certificates")
+	}
+	return trustPool, nil
+}
+
+var logger = grpclog.Component("advancedtls")
+
+// PEMFileProviderOptions contains options to configure a PEMFileProvider.
+// Note that these fields will only take effect during construction. Once the
+// PEMFileProvider starts, changing fields in PEMFileProviderOptions will have
+// no effect.
+type PEMFileProviderOptions struct {
+	// CertFile is the file path that holds identity certificate whose updates
+	// will be captured by a watching goroutine.
+	// Optional. If this is set, KeyFile must also be set.
+	CertFile string
+	// KeyFile is the file path that holds identity private key whose updates
+	// will be captured by a watching goroutine.
+	// Optional. If this is set, CertFile must also be set.
+	KeyFile string
+	// TrustFile is the file path that holds trust certificate whose updates will
+	// be captured by a watching goroutine.
+	// Optional.
+	TrustFile string
+	// IdentityInterval is the time duration between two credential update checks
+	// for identity certs.
+	// Optional. If not set, we will use the default interval(1 hour).
+	IdentityInterval time.Duration
+	// RootInterval is the time duration between two credential update checks
+	// for root certs.
+	// Optional. If not set, we will use the default interval(2 hours).
+	RootInterval time.Duration
+}
+
+// PEMFileProvider implements certprovider.Provider.
+// It provides the most up-to-date identity private key-cert pairs and/or
+// root certificates.
+type PEMFileProvider struct {
+	identityDistributor *certprovider.Distributor
+	rootDistributor     *certprovider.Distributor
+	cancel              context.CancelFunc
+}
+
+// NewPEMFileProvider returns a new PEMFileProvider constructed using the
+// provided options.
+func NewPEMFileProvider(o PEMFileProviderOptions) (*PEMFileProvider, error) {
+	if o.CertFile == "" && o.KeyFile == "" && o.TrustFile == "" {
+		return nil, fmt.Errorf("at least one credential file needs to be specified")
+	}
+	if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified {
+		return nil, fmt.Errorf("private key file and identity cert file should be both specified or not specified")
+	}
+	if o.IdentityInterval == 0 {
+		logger.Warningf("heyheyhey")
+		o.IdentityInterval = defaultIdentityInterval
+	}
+	if o.RootInterval == 0 {
+		o.RootInterval = defaultRootInterval
+	}
+	provider := &PEMFileProvider{}
+	if o.CertFile != "" && o.KeyFile != "" {
+		provider.identityDistributor = certprovider.NewDistributor()
+	}
+	if o.TrustFile != "" {
+		provider.rootDistributor = certprovider.NewDistributor()
+	}
+	// A goroutine to pull file changes.
+	identityTicker := time.NewTicker(o.IdentityInterval)
+	rootTicker := time.NewTicker(o.RootInterval)
+	ctx, cancel := context.WithCancel(context.Background())
+	// We pass a copy of PEMFileProviderOptions to the goroutine in case users
+	// change it after we start reloading.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				identityTicker.Stop()
+				rootTicker.Stop()
+				return
+			case <-identityTicker.C:
+				if provider.identityDistributor == nil {
+					continue
+				}
+				// Read identity certs from PEM files.
+				identityCert, err := readKeyCertPairFunc(o.CertFile, o.KeyFile)
+				if err != nil {
+					// If the reading produces an error, we will skip the update for this
+					// round and log the error.
+					logger.Warningf("tls.LoadX509KeyPair reads %s and %s failed: %v", o.CertFile, o.KeyFile, err)
+					continue
+				}
+				provider.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{identityCert}}, nil)
+			case <-rootTicker.C:
+				if provider.rootDistributor == nil {
+					continue
+				}
+				// Read root certs from PEM files.
+				trustPool, err := readTrustCertFunc(o.TrustFile)
+				if err != nil {
+					// If the reading produces an error, we will skip the update for this
+					// round and log the error.
+					logger.Warningf("readTrustCertFunc reads %v failed: %v", o.TrustFile, err)
+					continue
+				}
+				provider.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil)
+			default:
+			}
+		}
+	}()
+	provider.cancel = cancel
+	return provider, nil
+}
+
+// KeyMaterial returns the key material sourced by the PEMFileProvider.
+// Callers are expected to use the returned value as read-only.
+func (p *PEMFileProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
+	km := &certprovider.KeyMaterial{}
+	if p.identityDistributor != nil {
+		identityKM, err := p.identityDistributor.KeyMaterial(ctx)
+		if err != nil {
+			return nil, err
+		}
+		km.Certs = identityKM.Certs
+	}
+	if p.rootDistributor != nil {
+		rootKM, err := p.rootDistributor.KeyMaterial(ctx)
+		if err != nil {
+			return nil, err
+		}
+		km.Roots = rootKM.Roots
+	}
+	return km, nil
+}
+
+// Close cleans up resources allocated by the PEMFileProvider.
+func (p *PEMFileProvider) Close() {
+	p.cancel()
+	if p.identityDistributor != nil {
+		p.identityDistributor.Stop()
+	}
+	if p.rootDistributor != nil {
+		p.rootDistributor.Stop()
+	}
+}
diff --git a/security/advancedtls/pemfile_provider_test.go b/security/advancedtls/pemfile_provider_test.go
new file mode 100644
index 000000000000..abc494bcbbcb
--- /dev/null
+++ b/security/advancedtls/pemfile_provider_test.go
@@ -0,0 +1,219 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/security/advancedtls/testdata"
+)
+
+func (s) TestNewPEMFileProvider(t *testing.T) {
+	tests := []struct {
+		desc      string
+		options   PEMFileProviderOptions
+		certFile  string
+		keyFile   string
+		trustFile string
+		wantError bool
+	}{
+		{
+			desc:      "Expect error if no credential files specified",
+			options:   PEMFileProviderOptions{},
+			wantError: true,
+		},
+		{
+			desc: "Expect error if only certFile is specified",
+			options: PEMFileProviderOptions{
+				CertFile: testdata.Path("client_cert_1.pem"),
+			},
+			wantError: true,
+		},
+		{
+			desc: "Should be good if only identity key cert pairs are specified",
+			options: PEMFileProviderOptions{
+				KeyFile:  testdata.Path("client_key_1.pem"),
+				CertFile: testdata.Path("client_cert_1.pem"),
+			},
+			wantError: false,
+		},
+		{
+			desc: "Should be good if only root certs are specified",
+			options: PEMFileProviderOptions{
+				TrustFile: testdata.Path("client_trust_cert_1.pem"),
+			},
+			wantError: false,
+		},
+		{
+			desc: "Should be good if both identity pairs and root certs are specified",
+			options: PEMFileProviderOptions{
+				KeyFile:   testdata.Path("client_key_1.pem"),
+				CertFile:  testdata.Path("client_cert_1.pem"),
+				TrustFile: testdata.Path("client_trust_cert_1.pem"),
+			},
+			wantError: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			provider, err := NewPEMFileProvider(test.options)
+			if (err != nil) != test.wantError {
+				t.Fatalf("NewPEMFileProvider(%v) = %v, want %v", test.options, err, test.wantError)
+			}
+			if err != nil {
+				return
+			}
+			provider.Close()
+		})
+	}
+
+}
+
+// This test overwrites the credential reading function used by the watching
+// goroutine. It is tested under different stages:
+// At stage 0, we force reading function to load clientPeer1 and serverTrust1,
+// and see if the credentials are picked up by the watching go routine.
+// At stage 1, we force reading function to cause an error. The watching go
+// routine should log the error while leaving the credentials unchanged.
+// At stage 2, we force reading function to load clientPeer2 and serverTrust2,
+// and see if the new credentials are picked up.
+func (s) TestWatchingRoutineUpdates(t *testing.T) {
+	// Load certificates.
+	cs := &certStore{}
+	if err := cs.loadCerts(); err != nil {
+		t.Fatalf("cs.loadCerts() failed: %v", err)
+	}
+	tests := []struct {
+		desc         string
+		options      PEMFileProviderOptions
+		wantKmStage0 certprovider.KeyMaterial
+		wantKmStage1 certprovider.KeyMaterial
+		wantKmStage2 certprovider.KeyMaterial
+	}{
+		{
+			desc: "use identity certs and root certs",
+			options: PEMFileProviderOptions{
+				CertFile:  "not_empty_cert_file",
+				KeyFile:   "not_empty_key_file",
+				TrustFile: "not_empty_trust_file",
+			},
+			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}, Roots: cs.serverTrust1},
+			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}, Roots: cs.serverTrust1},
+			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer2}, Roots: cs.serverTrust2},
+		},
+		{
+			desc: "use identity certs only",
+			options: PEMFileProviderOptions{
+				CertFile: "not_empty_cert_file",
+				KeyFile:  "not_empty_key_file",
+			},
+			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}},
+			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}},
+			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer2}},
+		},
+		{
+			desc: "use trust certs only",
+			options: PEMFileProviderOptions{
+				TrustFile: "not_empty_trust_file",
+			},
+			wantKmStage0: certprovider.KeyMaterial{Roots: cs.serverTrust1},
+			wantKmStage1: certprovider.KeyMaterial{Roots: cs.serverTrust1},
+			wantKmStage2: certprovider.KeyMaterial{Roots: cs.serverTrust2},
+		},
+	}
+	for _, test := range tests {
+		testInterval := 200 * time.Millisecond
+		test.options.IdentityInterval = testInterval
+		test.options.RootInterval = testInterval
+		t.Run(test.desc, func(t *testing.T) {
+			stage := &stageInfo{}
+			oldReadKeyCertPairFunc := readKeyCertPairFunc
+			readKeyCertPairFunc = func(certFile, keyFile string) (tls.Certificate, error) {
+				switch stage.read() {
+				case 0:
+					return cs.clientPeer1, nil
+				case 1:
+					return tls.Certificate{}, fmt.Errorf("error occurred while reloading")
+				case 2:
+					return cs.clientPeer2, nil
+				default:
+					return tls.Certificate{}, fmt.Errorf("test stage not supported")
+				}
+			}
+			defer func() {
+				readKeyCertPairFunc = oldReadKeyCertPairFunc
+			}()
+			oldReadTrustCertFunc := readTrustCertFunc
+			readTrustCertFunc = func(trustFile string) (*x509.CertPool, error) {
+				switch stage.read() {
+				case 0:
+					return cs.serverTrust1, nil
+				case 1:
+					return nil, fmt.Errorf("error occurred while reloading")
+				case 2:
+					return cs.serverTrust2, nil
+				default:
+					return nil, fmt.Errorf("test stage not supported")
+				}
+			}
+			defer func() {
+				readTrustCertFunc = oldReadTrustCertFunc
+			}()
+			provider, err := NewPEMFileProvider(test.options)
+			if err != nil {
+				t.Fatalf("NewPEMFileProvider failed: %v", err)
+			}
+			defer provider.Close()
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			//// ------------------------Stage 0------------------------------------
+			// Wait for the refreshing go-routine to pick up the changes.
+			time.Sleep(1 * time.Second)
+			gotKM, err := provider.KeyMaterial(ctx)
+			if !cmp.Equal(*gotKM, test.wantKmStage0, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+				t.Fatalf("provider.KeyMaterial() = %+v, want %+v", *gotKM, test.wantKmStage0)
+			}
+			// ------------------------Stage 1------------------------------------
+			stage.increase()
+			// Wait for the refreshing go-routine to pick up the changes.
+			time.Sleep(1 * time.Second)
+			gotKM, err = provider.KeyMaterial(ctx)
+			if !cmp.Equal(*gotKM, test.wantKmStage1, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+				t.Fatalf("provider.KeyMaterial() = %+v, want %+v", *gotKM, test.wantKmStage1)
+			}
+			//// ------------------------Stage 2------------------------------------
+			// Wait for the refreshing go-routine to pick up the changes.
+			stage.increase()
+			time.Sleep(1 * time.Second)
+			gotKM, err = provider.KeyMaterial(ctx)
+			if !cmp.Equal(*gotKM, test.wantKmStage2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+				t.Fatalf("provider.KeyMaterial() = %+v, want %+v", *gotKM, test.wantKmStage2)
+			}
+			stage.reset()
+		})
+	}
+}

From 8630cac324bf02ea0edfba758c2b2f3344af7bf7 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 2 Sep 2020 14:02:33 -0700
Subject: [PATCH 186/481] xds: update balancer group ID type to string (#3862)

---
 .../balancer/balancergroup/balancergroup.go   | 19 +++++-----
 .../balancergroup/balancergroup_test.go       |  9 +++--
 .../balancergroup/balancerstateaggregator.go  |  3 +-
 xds/internal/balancer/edsbalancer/eds_impl.go | 20 +++++------
 .../weightedaggregator/aggregator.go          | 13 ++++---
 .../balancer/weightedtarget/weightedtarget.go | 21 ++++-------
 .../xdsrouting/balancerstateaggregator.go     | 11 +++---
 xds/internal/balancer/xdsrouting/routing.go   | 23 +++---------
 .../balancer/xdsrouting/routing_picker.go     |  5 ++-
 .../xdsrouting/routing_picker_test.go         | 35 +++++++++----------
 10 files changed, 65 insertions(+), 94 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index f8f2e0e9a25b..6923abb0e488 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -30,7 +30,6 @@ import (
 	"google.golang.org/grpc/internal/cache"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/xds/internal"
 )
 
 // loadReporter wraps the methods from the loadStore that are used here.
@@ -58,7 +57,7 @@ type subBalancerWrapper struct {
 	// of the actions are forwarded to the parent ClientConn with no change.
 	// Some are forward to balancer group with the sub-balancer ID.
 	balancer.ClientConn
-	id    internal.LocalityID
+	id    string
 	group *BalancerGroup
 
 	mu    sync.Mutex
@@ -202,7 +201,7 @@ type BalancerGroup struct {
 	// to sub-balancers after they are closed.
 	outgoingMu         sync.Mutex
 	outgoingStarted    bool
-	idToBalancerConfig map[internal.LocalityID]*subBalancerWrapper
+	idToBalancerConfig map[string]*subBalancerWrapper
 	// Cache for sub-balancers when they are removed.
 	balancerCache *cache.TimeoutCache
 
@@ -250,7 +249,7 @@ func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadSt
 
 		stateAggregator: stateAggregator,
 
-		idToBalancerConfig: make(map[internal.LocalityID]*subBalancerWrapper),
+		idToBalancerConfig: make(map[string]*subBalancerWrapper),
 		balancerCache:      cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout),
 		scToSubBalancer:    make(map[balancer.SubConn]*subBalancerWrapper),
 	}
@@ -281,7 +280,7 @@ func (bg *BalancerGroup) Start() {
 }
 
 // Add adds a balancer built by builder to the group, with given id.
-func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) {
+func (bg *BalancerGroup) Add(id string, builder balancer.Builder) {
 	// Store data in static map, and then check to see if bg is started.
 	bg.outgoingMu.Lock()
 	var sbc *subBalancerWrapper
@@ -332,7 +331,7 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) {
 // But doesn't close the balancer. The balancer is kept in a cache, and will be
 // closed after timeout. Cleanup work (closing sub-balancer and removing
 // subconns) will be done after timeout.
-func (bg *BalancerGroup) Remove(id internal.LocalityID) {
+func (bg *BalancerGroup) Remove(id string) {
 	bg.outgoingMu.Lock()
 	if sbToRemove, ok := bg.idToBalancerConfig[id]; ok {
 		if bg.outgoingStarted {
@@ -400,7 +399,7 @@ func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.
 
 // UpdateClientConnState handles ClientState (including balancer config and
 // addresses) from resolver. It finds the balancer and forwards the update.
-func (bg *BalancerGroup) UpdateClientConnState(id internal.LocalityID, s balancer.ClientConnState) error {
+func (bg *BalancerGroup) UpdateClientConnState(id string, s balancer.ClientConnState) error {
 	bg.outgoingMu.Lock()
 	defer bg.outgoingMu.Unlock()
 	if config, ok := bg.idToBalancerConfig[id]; ok {
@@ -449,7 +448,7 @@ func (bg *BalancerGroup) newSubConn(config *subBalancerWrapper, addrs []resolver
 // updateBalancerState: forward the new state to balancer state aggregator. The
 // aggregator will create an aggregated picker and an aggregated connectivity
 // state, then forward to ClientConn.
-func (bg *BalancerGroup) updateBalancerState(id internal.LocalityID, state balancer.State) {
+func (bg *BalancerGroup) updateBalancerState(id string, state balancer.State) {
 	bg.logger.Infof("Balancer state update from locality %v, new state: %+v", id, state)
 	if bg.loadStore != nil {
 		// Only wrap the picker to do load reporting if loadStore was set.
@@ -504,10 +503,10 @@ type loadReportPicker struct {
 	loadStore loadReporter
 }
 
-func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore loadReporter) *loadReportPicker {
+func newLoadReportPicker(p balancer.Picker, id string, loadStore loadReporter) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
-		locality:  id.String(),
+		locality:  id,
 		loadStore: loadStore,
 	}
 }
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 813ee8a5e7fc..ecfa97c9645b 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -39,7 +39,6 @@ import (
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/testutils"
@@ -48,7 +47,7 @@ import (
 var (
 	rrBuilder        = balancer.Get(roundrobin.Name)
 	pfBuilder        = balancer.Get(grpc.PickFirstBalancerName)
-	testBalancerIDs  = []internal.LocalityID{{Region: "b1"}, {Region: "b2"}, {Region: "b3"}}
+	testBalancerIDs  = []string{"b1", "b2", "b3"}
 	testBackendAddrs []resolver.Address
 )
 
@@ -404,7 +403,7 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	loadStore := &load.Store{}
 	cc, gator, bg := newTestBalancerGroup(t, loadStore)
 
-	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
+	backendToBalancerID := make(map[balancer.SubConn]string)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
@@ -443,7 +442,7 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	// these to show up as pending rpcs.
 	wantStoreData := &load.Data{
 		LocalityStats: map[string]load.LocalityData{
-			testBalancerIDs[0].String(): {
+			testBalancerIDs[0]: {
 				RequestStats: load.RequestData{Succeeded: 10, InProgress: 10},
 				LoadStats: map[string]load.ServerLoadData{
 					"cpu_utilization": {Count: 10, Sum: 100},
@@ -452,7 +451,7 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 					"piu":             {Count: 10, Sum: 31.4},
 				},
 			},
-			testBalancerIDs[1].String(): {
+			testBalancerIDs[1]: {
 				RequestStats: load.RequestData{Succeeded: 10},
 				LoadStats: map[string]load.ServerLoadData{
 					"cpu_utilization": {Count: 10, Sum: 100},
diff --git a/xds/internal/balancer/balancergroup/balancerstateaggregator.go b/xds/internal/balancer/balancergroup/balancerstateaggregator.go
index 0a555f3d9ba2..116394385059 100644
--- a/xds/internal/balancer/balancergroup/balancerstateaggregator.go
+++ b/xds/internal/balancer/balancergroup/balancerstateaggregator.go
@@ -20,7 +20,6 @@ package balancergroup
 
 import (
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/xds/internal"
 )
 
 // BalancerStateAggregator aggregates sub-picker and connectivity states into a
@@ -34,5 +33,5 @@ type BalancerStateAggregator interface {
 	//
 	// It's up to the implementation whether this will trigger an update to the
 	// parent ClientConn.
-	UpdateState(id internal.LocalityID, state balancer.State)
+	UpdateState(id string, state balancer.State)
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index c00c58cfe695..4aaf20925a46 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -137,13 +137,13 @@ func (edsImpl *edsBalancerImpl) handleChildPolicy(name string, config json.RawMe
 		if bgwc == nil {
 			continue
 		}
-		for id, config := range bgwc.configs {
+		for lid, config := range bgwc.configs {
 			// TODO: (eds) add support to balancer group to support smoothly
 			//  switching sub-balancers (keep old balancer around until new
 			//  balancer becomes ready).
-			bgwc.bg.Remove(id)
-			bgwc.bg.Add(id, edsImpl.subBalancerBuilder)
-			bgwc.bg.UpdateClientConnState(id, balancer.ClientConnState{
+			bgwc.bg.Remove(lid.String())
+			bgwc.bg.Add(lid.String(), edsImpl.subBalancerBuilder)
+			bgwc.bg.UpdateClientConnState(lid.String(), balancer.ClientConnState{
 				ResolverState: resolver.State{Addresses: config.addrs},
 			})
 			// This doesn't need to manually update picker, because the new
@@ -315,8 +315,8 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 		config, ok := bgwc.configs[lid]
 		if !ok {
 			// A new balancer, add it to balancer group and balancer map.
-			bgwc.stateAggregator.Add(lid, newWeight)
-			bgwc.bg.Add(lid, edsImpl.subBalancerBuilder)
+			bgwc.stateAggregator.Add(lid.String(), newWeight)
+			bgwc.bg.Add(lid.String(), edsImpl.subBalancerBuilder)
 			config = &localityConfig{
 				weight: newWeight,
 			}
@@ -339,13 +339,13 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 
 		if weightChanged {
 			config.weight = newWeight
-			bgwc.stateAggregator.UpdateWeight(lid, newWeight)
+			bgwc.stateAggregator.UpdateWeight(lid.String(), newWeight)
 			rebuildStateAndPicker = true
 		}
 
 		if addrsChanged {
 			config.addrs = newAddrs
-			bgwc.bg.UpdateClientConnState(lid, balancer.ClientConnState{
+			bgwc.bg.UpdateClientConnState(lid.String(), balancer.ClientConnState{
 				ResolverState: resolver.State{Addresses: newAddrs},
 			})
 		}
@@ -354,8 +354,8 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 	// Delete localities that are removed in the latest response.
 	for lid := range bgwc.configs {
 		if _, ok := newLocalitiesSet[lid]; !ok {
-			bgwc.stateAggregator.Remove(lid)
-			bgwc.bg.Remove(lid)
+			bgwc.stateAggregator.Remove(lid.String())
+			bgwc.bg.Remove(lid.String())
 			delete(bgwc.configs, lid)
 			edsImpl.logger.Infof("Locality %v deleted", lid)
 			rebuildStateAndPicker = true
diff --git a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go b/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
index a77a7c3f495f..6c36e2a69cd9 100644
--- a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
+++ b/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go
@@ -34,7 +34,6 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/wrr"
-	"google.golang.org/grpc/xds/internal"
 )
 
 type weightedPickerState struct {
@@ -68,7 +67,7 @@ type Aggregator struct {
 	// started.
 	//
 	// If an ID is not in map, it's either removed or never added.
-	idToPickerState map[internal.LocalityID]*weightedPickerState
+	idToPickerState map[string]*weightedPickerState
 }
 
 // New creates a new weighted balancer state aggregator.
@@ -77,7 +76,7 @@ func New(cc balancer.ClientConn, logger *grpclog.PrefixLogger, newWRR func() wrr
 		cc:              cc,
 		logger:          logger,
 		newWRR:          newWRR,
-		idToPickerState: make(map[internal.LocalityID]*weightedPickerState),
+		idToPickerState: make(map[string]*weightedPickerState),
 	}
 }
 
@@ -100,7 +99,7 @@ func (wbsa *Aggregator) Stop() {
 
 // Add adds a sub-balancer state with weight. It adds a place holder, and waits for
 // the real sub-balancer to update state.
-func (wbsa *Aggregator) Add(id internal.LocalityID, weight uint32) {
+func (wbsa *Aggregator) Add(id string, weight uint32) {
 	wbsa.mu.Lock()
 	defer wbsa.mu.Unlock()
 	wbsa.idToPickerState[id] = &weightedPickerState{
@@ -118,7 +117,7 @@ func (wbsa *Aggregator) Add(id internal.LocalityID, weight uint32) {
 
 // Remove removes the sub-balancer state. Future updates from this sub-balancer,
 // if any, will be ignored.
-func (wbsa *Aggregator) Remove(id internal.LocalityID) {
+func (wbsa *Aggregator) Remove(id string) {
 	wbsa.mu.Lock()
 	defer wbsa.mu.Unlock()
 	if _, ok := wbsa.idToPickerState[id]; !ok {
@@ -132,7 +131,7 @@ func (wbsa *Aggregator) Remove(id internal.LocalityID) {
 // UpdateWeight updates the weight for the given id. Note that this doesn't
 // trigger an update to the parent ClientConn. The caller should decide when
 // it's necessary, and call BuildAndUpdate.
-func (wbsa *Aggregator) UpdateWeight(id internal.LocalityID, newWeight uint32) {
+func (wbsa *Aggregator) UpdateWeight(id string, newWeight uint32) {
 	wbsa.mu.Lock()
 	defer wbsa.mu.Unlock()
 	pState, ok := wbsa.idToPickerState[id]
@@ -146,7 +145,7 @@ func (wbsa *Aggregator) UpdateWeight(id internal.LocalityID, newWeight uint32) {
 // It's usually called by the balancer group.
 //
 // It calls parent ClientConn's UpdateState with the new aggregated state.
-func (wbsa *Aggregator) UpdateState(id internal.LocalityID, newState balancer.State) {
+func (wbsa *Aggregator) UpdateState(id string, newState balancer.State) {
 	wbsa.mu.Lock()
 	defer wbsa.mu.Unlock()
 	oldState, ok := wbsa.idToPickerState[id]
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index 4172a4033641..b0310660fb29 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -29,7 +29,6 @@ import (
 	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
-	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 )
@@ -79,11 +78,6 @@ type weightedTargetBalancer struct {
 	targets map[string]target
 }
 
-// TODO: remove this and use strings directly as keys for balancer group.
-func makeLocalityFromName(name string) internal.LocalityID {
-	return internal.LocalityID{Region: name}
-}
-
 // UpdateClientConnState takes the new targets in balancer group,
 // creates/deletes sub-balancers and sends them update. Addresses are split into
 // groups based on hierarchy path.
@@ -99,9 +93,8 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 	// Remove sub-pickers and sub-balancers that are not in the new config.
 	for name := range w.targets {
 		if _, ok := newConfig.Targets[name]; !ok {
-			l := makeLocalityFromName(name)
-			w.stateAggregator.Remove(l)
-			w.bg.Remove(l)
+			w.stateAggregator.Remove(name)
+			w.bg.Remove(name)
 			// Trigger a state/picker update, because we don't want `ClientConn`
 			// to pick this sub-balancer anymore.
 			rebuildStateAndPicker = true
@@ -114,19 +107,17 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 	//
 	// For all sub-balancers, forward the address/balancer config update.
 	for name, newT := range newConfig.Targets {
-		l := makeLocalityFromName(name)
-
 		oldT, ok := w.targets[name]
 		if !ok {
 			// If this is a new sub-balancer, add weights to the picker map.
-			w.stateAggregator.Add(l, newT.Weight)
+			w.stateAggregator.Add(name, newT.Weight)
 			// Then add to the balancer group.
-			w.bg.Add(l, balancer.Get(newT.ChildPolicy.Name))
+			w.bg.Add(name, balancer.Get(newT.ChildPolicy.Name))
 			// Not trigger a state/picker update. Wait for the new sub-balancer
 			// to send its updates.
 		} else if newT.Weight != oldT.Weight {
 			// If this is an existing sub-balancer, update weight if necessary.
-			w.stateAggregator.UpdateWeight(l, newT.Weight)
+			w.stateAggregator.UpdateWeight(name, newT.Weight)
 			// Trigger a state/picker update, because we don't want `ClientConn`
 			// should do picks with the new weights now.
 			rebuildStateAndPicker = true
@@ -138,7 +129,7 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 		// - Balancer config comes from the targets map.
 		//
 		// TODO: handle error? How to aggregate errors and return?
-		_ = w.bg.UpdateClientConnState(l, balancer.ClientConnState{
+		_ = w.bg.UpdateClientConnState(name, balancer.ClientConnState{
 			ResolverState: resolver.State{
 				Addresses:     addressesSplit[name],
 				ServiceConfig: s.ResolverState.ServiceConfig,
diff --git a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go b/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
index 9f6250e5c804..dd613993629b 100644
--- a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
+++ b/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
@@ -26,7 +26,6 @@ import (
 	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/xds/internal"
 )
 
 type subBalancerState struct {
@@ -59,14 +58,14 @@ type balancerStateAggregator struct {
 	// started.
 	//
 	// If an ID is not in map, it's either removed or never added.
-	idToPickerState map[internal.LocalityID]*subBalancerState
+	idToPickerState map[string]*subBalancerState
 }
 
 func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator {
 	return &balancerStateAggregator{
 		cc:              cc,
 		logger:          logger,
-		idToPickerState: make(map[internal.LocalityID]*subBalancerState),
+		idToPickerState: make(map[string]*subBalancerState),
 	}
 }
 
@@ -91,7 +90,7 @@ func (rbsa *balancerStateAggregator) close() {
 // for the real sub-balancer to update state.
 //
 // This is called when there's a new action.
-func (rbsa *balancerStateAggregator) add(id internal.LocalityID) {
+func (rbsa *balancerStateAggregator) add(id string) {
 	rbsa.mu.Lock()
 	defer rbsa.mu.Unlock()
 	rbsa.idToPickerState[id] = &subBalancerState{
@@ -110,7 +109,7 @@ func (rbsa *balancerStateAggregator) add(id internal.LocalityID) {
 // if any, will be ignored.
 //
 // This is called when an action is removed.
-func (rbsa *balancerStateAggregator) remove(id internal.LocalityID) {
+func (rbsa *balancerStateAggregator) remove(id string) {
 	rbsa.mu.Lock()
 	defer rbsa.mu.Unlock()
 	if _, ok := rbsa.idToPickerState[id]; !ok {
@@ -134,7 +133,7 @@ func (rbsa *balancerStateAggregator) updateRoutes(newRoutes []route) {
 // It's usually called by the balancer group.
 //
 // It calls parent ClientConn's UpdateState with the new aggregated state.
-func (rbsa *balancerStateAggregator) UpdateState(id internal.LocalityID, state balancer.State) {
+func (rbsa *balancerStateAggregator) UpdateState(id string, state balancer.State) {
 	rbsa.mu.Lock()
 	defer rbsa.mu.Unlock()
 	pickerSt, ok := rbsa.idToPickerState[id]
diff --git a/xds/internal/balancer/xdsrouting/routing.go b/xds/internal/balancer/xdsrouting/routing.go
index ebbb2b8cf9d7..ad5373a2f773 100644
--- a/xds/internal/balancer/xdsrouting/routing.go
+++ b/xds/internal/balancer/xdsrouting/routing.go
@@ -28,7 +28,6 @@ import (
 	"google.golang.org/grpc/internal/hierarchy"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
-	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 )
 
@@ -80,16 +79,6 @@ type routingBalancer struct {
 	routes  []route
 }
 
-// TODO: remove this and use strings directly as keys for balancer group.
-func makeLocalityFromName(name string) internal.LocalityID {
-	return internal.LocalityID{Region: name}
-}
-
-// TODO: remove this and use strings directly as keys for balancer group.
-func getNameFromLocality(id internal.LocalityID) string {
-	return id.Region
-}
-
 func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *lbConfig) (needRebuild bool) {
 	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
 	var rebuildStateAndPicker bool
@@ -97,9 +86,8 @@ func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *
 	// Remove sub-pickers and sub-balancers that are not in the new action list.
 	for name := range rb.actions {
 		if _, ok := newConfig.actions[name]; !ok {
-			l := makeLocalityFromName(name)
-			rb.stateAggregator.remove(l)
-			rb.bg.Remove(l)
+			rb.stateAggregator.remove(name)
+			rb.bg.Remove(name)
 			// Trigger a state/picker update, because we don't want `ClientConn`
 			// to pick this sub-balancer anymore.
 			rebuildStateAndPicker = true
@@ -110,12 +98,11 @@ func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *
 	// - add to balancer group if it's new,
 	// - forward the address/balancer config update.
 	for name, newT := range newConfig.actions {
-		l := makeLocalityFromName(name)
 		if _, ok := rb.actions[name]; !ok {
 			// If this is a new sub-balancer, add weights to the picker map.
-			rb.stateAggregator.add(l)
+			rb.stateAggregator.add(name)
 			// Then add to the balancer group.
-			rb.bg.Add(l, balancer.Get(newT.ChildPolicy.Name))
+			rb.bg.Add(name, balancer.Get(newT.ChildPolicy.Name))
 			// Not trigger a state/picker update. Wait for the new sub-balancer
 			// to send its updates.
 		}
@@ -125,7 +112,7 @@ func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *
 		// - Balancer config comes from the targets map.
 		//
 		// TODO: handle error? How to aggregate errors and return?
-		_ = rb.bg.UpdateClientConnState(l, balancer.ClientConnState{
+		_ = rb.bg.UpdateClientConnState(name, balancer.ClientConnState{
 			ResolverState: resolver.State{
 				Addresses:     addressesSplit[name],
 				ServiceConfig: s.ResolverState.ServiceConfig,
diff --git a/xds/internal/balancer/xdsrouting/routing_picker.go b/xds/internal/balancer/xdsrouting/routing_picker.go
index 36fdebd4ad2a..2309f144802c 100644
--- a/xds/internal/balancer/xdsrouting/routing_picker.go
+++ b/xds/internal/balancer/xdsrouting/routing_picker.go
@@ -22,7 +22,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/xds/internal"
 )
 
 // pickerGroup contains a list of route matchers and their corresponding
@@ -33,10 +32,10 @@ type pickerGroup struct {
 	pickers map[string]balancer.Picker
 }
 
-func newPickerGroup(routes []route, idToPickerState map[internal.LocalityID]*subBalancerState) *pickerGroup {
+func newPickerGroup(routes []route, idToPickerState map[string]*subBalancerState) *pickerGroup {
 	pickers := make(map[string]balancer.Picker)
 	for id, st := range idToPickerState {
-		pickers[getNameFromLocality(id)] = st.state.Picker
+		pickers[id] = st.state.Picker
 	}
 	return &pickerGroup{
 		routes:  routes,
diff --git a/xds/internal/balancer/xdsrouting/routing_picker_test.go b/xds/internal/balancer/xdsrouting/routing_picker_test.go
index 2de40757a4f7..83e494a4bf8d 100644
--- a/xds/internal/balancer/xdsrouting/routing_picker_test.go
+++ b/xds/internal/balancer/xdsrouting/routing_picker_test.go
@@ -24,7 +24,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -40,7 +39,7 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		name string
 
 		routes  []route
-		pickers map[internal.LocalityID]*subBalancerState
+		pickers map[string]*subBalancerState
 		info    balancer.PickInfo
 
 		want    balancer.PickResult
@@ -55,8 +54,8 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 			routes: []route{
 				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
 			},
-			pickers: map[internal.LocalityID]*subBalancerState{
-				makeLocalityFromName("action-0"): {state: balancer.State{
+			pickers: map[string]*subBalancerState{
+				"action-0": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[0],
 				}},
@@ -69,8 +68,8 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 			routes: []route{
 				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
 			},
-			pickers: map[internal.LocalityID]*subBalancerState{
-				makeLocalityFromName("action-0"): {state: balancer.State{
+			pickers: map[string]*subBalancerState{
+				"action-0": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[0],
 				}},
@@ -84,12 +83,12 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
 				{m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"},
 			},
-			pickers: map[internal.LocalityID]*subBalancerState{
-				makeLocalityFromName("action-0"): {state: balancer.State{
+			pickers: map[string]*subBalancerState{
+				"action-0": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[0],
 				}},
-				makeLocalityFromName("action-1"): {state: balancer.State{
+				"action-1": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[1],
 				}},
@@ -103,12 +102,12 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
 				{m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"},
 			},
-			pickers: map[internal.LocalityID]*subBalancerState{
-				makeLocalityFromName("action-0"): {state: balancer.State{
+			pickers: map[string]*subBalancerState{
+				"action-0": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[0],
 				}},
-				makeLocalityFromName("action-1"): {state: balancer.State{
+				"action-1": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[1],
 				}},
@@ -122,12 +121,12 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 				{m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-0"},
 				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-1"},
 			},
-			pickers: map[internal.LocalityID]*subBalancerState{
-				makeLocalityFromName("action-0"): {state: balancer.State{
+			pickers: map[string]*subBalancerState{
+				"action-0": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[0],
 				}},
-				makeLocalityFromName("action-1"): {state: balancer.State{
+				"action-1": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[1],
 				}},
@@ -142,12 +141,12 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
 				{m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-1"},
 			},
-			pickers: map[internal.LocalityID]*subBalancerState{
-				makeLocalityFromName("action-0"): {state: balancer.State{
+			pickers: map[string]*subBalancerState{
+				"action-0": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[0],
 				}},
-				makeLocalityFromName("action-1"): {state: balancer.State{
+				"action-1": {state: balancer.State{
 					ConnectivityState: connectivity.Ready,
 					Picker:            testPickers[1],
 				}},

From 15157e2664af976a817a4256506dbb4576aee974 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 9 Sep 2020 13:23:46 -0700
Subject: [PATCH 187/481] test: remove funcServer and some uses of
 NewTestServiceService (#3867)

---
 test/balancer_test.go             |  26 +--
 test/channelz_linux_go110_test.go |   2 +-
 test/channelz_test.go             |  40 ++---
 test/creds_test.go                |  20 +--
 test/end2end_test.go              | 279 +++++++++++++++---------------
 test/healthcheck_test.go          |   2 +-
 6 files changed, 180 insertions(+), 189 deletions(-)

diff --git a/test/balancer_test.go b/test/balancer_test.go
index c45a380115b3..a94e25a8faf9 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -150,7 +150,7 @@ func (s) TestCredsBundleFromBalancer(t *testing.T) {
 	te.customServerOptions = []grpc.ServerOption{
 		grpc.Creds(creds),
 	}
-	te.startServer(&testServer{})
+	te.startServer((&testServer{}).Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -179,7 +179,7 @@ func testPickExtraMetadata(t *testing.T, e env) {
 		grpc.WithBalancerName(testBalancerName),
 		grpc.WithUserAgent(testUserAgent),
 	}
-	te.startServer(&testServer{security: e.security})
+	te.startServer((&testServer{security: e.security}).Svc())
 	defer te.tearDown()
 
 	// Set resolver to xds to trigger the extra metadata code path.
@@ -228,7 +228,7 @@ func testDoneInfo(t *testing.T, e env) {
 		grpc.WithBalancerName(testBalancerName),
 	}
 	te.userAgent = failAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer((&testServer{security: e.security}).Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -498,7 +498,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	}
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&testServer{}))
+	testpb.RegisterTestServiceService(s, testServer{}.Svc())
 	go s.Serve(lis)
 	defer s.Stop()
 	t.Logf("Started gRPC server at %s...", lis.Addr().String())
@@ -556,12 +556,12 @@ func (s) TestServersSwap(t *testing.T) {
 			t.Fatalf("Error while listening. Err: %v", err)
 		}
 		s := grpc.NewServer()
-		ts := &funcServer{
-			unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+		ts := &testpb.TestServiceService{
+			UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 				return &testpb.SimpleResponse{Username: username}, nil
 			},
 		}
-		testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
+		testpb.RegisterTestServiceService(s, ts)
 		go s.Serve(lis)
 		return lis.Addr().String(), s.Stop
 	}
@@ -616,12 +616,12 @@ func (s) TestEmptyAddrs(t *testing.T) {
 	s := grpc.NewServer()
 	defer s.Stop()
 	const one = "1"
-	ts := &funcServer{
-		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{
+		UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{Username: one}, nil
 		},
 	}
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
+	testpb.RegisterTestServiceService(s, ts)
 	go s.Serve(lis)
 
 	// Initialize pickfirst client
@@ -705,12 +705,12 @@ func (s) TestWaitForReady(t *testing.T) {
 	s := grpc.NewServer()
 	defer s.Stop()
 	const one = "1"
-	ts := &funcServer{
-		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{
+		UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{Username: one}, nil
 		},
 	}
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
+	testpb.RegisterTestServiceService(s, ts)
 	go s.Serve(lis)
 
 	// Initialize client
diff --git a/test/channelz_linux_go110_test.go b/test/channelz_linux_go110_test.go
index dea374bfc08b..35fd2e826802 100644
--- a/test/channelz_linux_go110_test.go
+++ b/test/channelz_linux_go110_test.go
@@ -43,7 +43,7 @@ func testCZSocketMetricsSocketOption(t *testing.T, e env) {
 	czCleanup := channelz.NewChannelzStorage()
 	defer czCleanupWrapper(czCleanup, t)
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer((&testServer{security: e.security}).Svc())
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
diff --git a/test/channelz_test.go b/test/channelz_test.go
index db510d4c6df5..1f2197849e6f 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -85,7 +85,7 @@ func (s) TestCZServerRegistrationAndDeletion(t *testing.T) {
 		defer czCleanupWrapper(czCleanup, t)
 		e := tcpClearRREnv
 		te := newTest(t, e)
-		te.startServers(&testServer{security: e.security}, c.total)
+		te.startServers(testServer{security: e.security}.Svc(), c.total)
 
 		ss, end := channelz.GetServers(c.start, c.max)
 		if int64(len(ss)) != c.length || end != c.end {
@@ -104,7 +104,7 @@ func (s) TestCZGetServer(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	ss, _ := channelz.GetServers(0, 0)
@@ -253,7 +253,7 @@ func (s) TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) {
 	num := 3 // number of backends
 	te := newTest(t, e)
 	var svrAddrs []resolver.Address
-	te.startServers(&testServer{security: e.security}, num)
+	te.startServers(testServer{security: e.security}.Svc(), num)
 	r := manual.NewBuilderWithScheme("whatever")
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
@@ -339,7 +339,7 @@ func (s) TestCZServerSocketRegistrationAndDeletion(t *testing.T) {
 		defer czCleanupWrapper(czCleanup, t)
 		e := tcpClearRREnv
 		te := newTest(t, e)
-		te.startServer(&testServer{security: e.security})
+		te.startServer(testServer{security: e.security}.Svc())
 		var ccs []*grpc.ClientConn
 		for i := 0; i < c.total; i++ {
 			cc := te.clientConn()
@@ -504,7 +504,7 @@ func (s) TestCZChannelMetrics(t *testing.T) {
 	te := newTest(t, e)
 	te.maxClientSendMsgSize = newInt(8)
 	var svrAddrs []resolver.Address
-	te.startServers(&testServer{security: e.security}, num)
+	te.startServers(testServer{security: e.security}.Svc(), num)
 	r := manual.NewBuilderWithScheme("whatever")
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
@@ -590,7 +590,7 @@ func (s) TestCZServerMetrics(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(8)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -861,7 +861,7 @@ func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) {
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(20)
 	te.maxClientReceiveMsgSize = newInt(20)
-	rcw := te.startServerWithConnControl(&testServer{security: e.security})
+	rcw := te.startServerWithConnControl(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)}
@@ -963,7 +963,7 @@ func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *t
 	// Avoid overflowing connection level flow control window, which will lead to
 	// transport being closed.
 	te.serverInitialConnWindowSize = 65536 * 2
-	ts := &funcServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ts := &testpb.TestServiceService{FullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
 		stream.Send(&testpb.StreamingOutputCallResponse{})
 		<-stream.Context().Done()
 		return status.Errorf(codes.DeadlineExceeded, "deadline exceeded or cancelled")
@@ -1048,7 +1048,7 @@ func (s) TestCZClientAndServerSocketMetricsFlowControl(t *testing.T) {
 	te.serverInitialConnWindowSize = 65536
 	te.clientInitialWindowSize = 65536
 	te.clientInitialConnWindowSize = 65536
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -1169,7 +1169,7 @@ func (s) TestCZClientSocketMetricsKeepAlive(t *testing.T) {
 			MinTime:             500 * time.Millisecond,
 			PermitWithoutStream: true,
 		}))
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	te.clientConn() // Dial the server
 	defer te.tearDown()
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -1211,7 +1211,7 @@ func (s) TestCZServerSocketMetricsStreamsAndMessagesCount(t *testing.T) {
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(20)
 	te.maxClientReceiveMsgSize = newInt(20)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	cc, _ := te.clientConnWithConnControl()
 	tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)}
@@ -1282,7 +1282,7 @@ func (s) TestCZServerSocketMetricsKeepAlive(t *testing.T) {
 		Timeout: 100 * time.Millisecond,
 	})
 	te.customServerOptions = append(te.customServerOptions, kpOption)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -1342,7 +1342,7 @@ func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpTLSRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	te.clientConn()
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -1467,7 +1467,7 @@ func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1560,7 +1560,7 @@ func (s) TestCZChannelAddressResolutionChange(t *testing.T) {
 	e := tcpClearRREnv
 	e.balancer = ""
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	r := manual.NewBuilderWithScheme("whatever")
 	addrs := []resolver.Address{{Addr: te.srvAddr}}
 	r.InitialState(resolver.State{Addresses: addrs})
@@ -1663,7 +1663,7 @@ func (s) TestCZSubChannelPickedNewAddress(t *testing.T) {
 	e := tcpClearRREnv
 	e.balancer = ""
 	te := newTest(t, e)
-	te.startServers(&testServer{security: e.security}, 3)
+	te.startServers(testServer{security: e.security}.Svc(), 3)
 	r := manual.NewBuilderWithScheme("whatever")
 	var svrAddrs []resolver.Address
 	for _, a := range te.srvAddrs {
@@ -1722,7 +1722,7 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1816,7 +1816,7 @@ func (s) TestCZChannelConnectivityState(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1939,7 +1939,7 @@ func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) {
 	te := newTest(t, e)
 	channelz.SetMaxTraceEntry(1)
 	defer channelz.ResetMaxTraceEntryToDefault()
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1997,7 +1997,7 @@ func (s) TestCZTraceTopChannelDeletionTraceClear(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
diff --git a/test/creds_test.go b/test/creds_test.go
index 46bdd30dc85e..a4d1e6cda0cb 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -87,7 +87,7 @@ func (s) TestCredsBundleBoth(t *testing.T) {
 	te.customServerOptions = []grpc.ServerOption{
 		grpc.Creds(creds),
 	}
-	te.startServer(&testServer{})
+	te.startServer(testServer{}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -109,7 +109,7 @@ func (s) TestCredsBundleTransportCredentials(t *testing.T) {
 	te.customServerOptions = []grpc.ServerOption{
 		grpc.Creds(creds),
 	}
-	te.startServer(&testServer{})
+	te.startServer(testServer{}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -125,7 +125,7 @@ func (s) TestCredsBundlePerRPCCredentials(t *testing.T) {
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundlePerRPCOnly}),
 	}
-	te.startServer(&testServer{})
+	te.startServer(testServer{}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -159,7 +159,7 @@ func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials {
 func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
 	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty"})
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: te.e.security})
+	te.startServer(testServer{security: te.e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{}))
@@ -183,7 +183,7 @@ func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
 	const wantMethod = "/grpc.testing.TestService/EmptyCall"
 	te := newTest(t, env{name: "context-request-info", network: "tcp"})
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: te.e.security})
+	te.startServer(testServer{security: te.e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{}))
@@ -218,7 +218,7 @@ func (c clientAlwaysFailCred) Clone() credentials.TransportCredentials {
 
 func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
 	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
-	te.startServer(&testServer{security: te.e.security})
+	te.startServer(testServer{security: te.e.security}.Svc())
 	defer te.tearDown()
 
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
@@ -246,7 +246,7 @@ func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
 
 func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) {
 	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
-	te.startServer(&testServer{security: te.e.security})
+	te.startServer(testServer{security: te.e.security}.Svc())
 	defer te.tearDown()
 
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
@@ -312,7 +312,7 @@ func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.tapHandle = authHandle
 	te.perRPCCreds = testPerRPCCredentials{}
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -331,7 +331,7 @@ func (s) TestPerRPCCredentialsViaCallOptions(t *testing.T) {
 func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.tapHandle = authHandle
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -371,7 +371,7 @@ func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) {
 		}
 		return ctx, nil
 	}
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
diff --git a/test/end2end_test.go b/test/end2end_test.go
index f26997ea808b..b0fa10b7872a 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -136,6 +136,20 @@ type testServer struct {
 
 var _ testpb.UnstableTestServiceService = (*testServer)(nil)
 
+// Svc returns a registerable TestService for this testServer instances.
+// Because `s` is passed by value for convenience, any subsequent changes to
+// `s` are not recognized.
+func (s testServer) Svc() *testpb.TestServiceService {
+	return &testpb.TestServiceService{
+		EmptyCall:           s.EmptyCall,
+		UnaryCall:           s.UnaryCall,
+		StreamingOutputCall: s.StreamingOutputCall,
+		StreamingInputCall:  s.StreamingInputCall,
+		FullDuplexCall:      s.FullDuplexCall,
+		HalfDuplexCall:      s.HalfDuplexCall,
+	}
+}
+
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 	if md, ok := metadata.FromIncomingContext(ctx); ok {
 		// For testing purpose, returns an error if user-agent is failAppUA.
@@ -566,7 +580,7 @@ func newTest(t *testing.T, e env) *test {
 	return te
 }
 
-func (te *test) listenAndServe(ts interface{}, listen func(network, address string) (net.Listener, error)) net.Listener {
+func (te *test) listenAndServe(ts *testpb.TestServiceService, listen func(network, address string) (net.Listener, error)) net.Listener {
 	te.t.Helper()
 	te.t.Logf("Running test in %s environment...", te.e.name)
 	sopts := []grpc.ServerOption{grpc.MaxConcurrentStreams(te.maxStream)}
@@ -626,7 +640,7 @@ func (te *test) listenAndServe(ts interface{}, listen func(network, address stri
 	sopts = append(sopts, te.customServerOptions...)
 	s := grpc.NewServer(sopts...)
 	if ts != nil {
-		testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
+		testpb.RegisterTestServiceService(s, ts)
 	}
 
 	// Create a new default health server if enableHealthServer is set, or use
@@ -691,20 +705,20 @@ func (w wrapHS) Stop() {
 	w.s.Close()
 }
 
-func (te *test) startServerWithConnControl(ts interface{}) *listenerWrapper {
+func (te *test) startServerWithConnControl(ts *testpb.TestServiceService) *listenerWrapper {
 	l := te.listenAndServe(ts, listenWithConnControl)
 	return l.(*listenerWrapper)
 }
 
 // startServer starts a gRPC server exposing the provided TestService
 // implementation. Callers should defer a call to te.tearDown to clean up
-func (te *test) startServer(ts interface{}) {
+func (te *test) startServer(ts *testpb.TestServiceService) {
 	te.t.Helper()
 	te.listenAndServe(ts, net.Listen)
 }
 
 // startServers starts 'num' gRPC servers exposing the provided TestService.
-func (te *test) startServers(ts interface{}, num int) {
+func (te *test) startServers(ts *testpb.TestServiceService, num int) {
 	for i := 0; i < num; i++ {
 		te.startServer(ts)
 		te.srvs = append(te.srvs, te.srv.(*grpc.Server))
@@ -912,7 +926,7 @@ func (s) TestContextDeadlineNotIgnored(t *testing.T) {
 	}
 
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -946,7 +960,7 @@ func testTimeoutOnDeadServer(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -988,7 +1002,7 @@ func (s) TestServerGracefulStopIdempotent(t *testing.T) {
 func testServerGracefulStopIdempotent(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	for i := 0; i < 3; i++ {
@@ -1008,7 +1022,7 @@ func (s) TestServerGoAway(t *testing.T) {
 func testServerGoAway(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1060,7 +1074,7 @@ func testServerGoAwayPendingRPC(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1134,7 +1148,7 @@ func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1221,7 +1235,7 @@ func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1258,7 +1272,7 @@ func testConcurrentServerStopAndGoAway(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1333,7 +1347,7 @@ func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) {
 
 func testClientConnCloseAfterGoAwayWithActiveStream(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -1372,7 +1386,7 @@ func testFailFast(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1731,7 +1745,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 
 	te1.resolverScheme = r.Scheme()
 	te1.nonBlockingDial = true
-	te1.startServer(&testServer{security: e.security})
+	te1.startServer(testServer{security: e.security}.Svc())
 	cc1 := te1.clientConn(grpc.WithResolvers(r))
 
 	addrs := []resolver.Address{{Addr: te1.srvAddr}}
@@ -1821,7 +1835,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	te2.maxClientReceiveMsgSize = newInt(1024)
 	te2.maxClientSendMsgSize = newInt(1024)
 
-	te2.startServer(&testServer{security: e.security})
+	te2.startServer(testServer{security: e.security}.Svc())
 	defer te2.tearDown()
 	cc2 := te2.clientConn(grpc.WithResolvers(r))
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: te2.srvAddr}}, ServiceConfig: sc})
@@ -1881,7 +1895,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	te3.maxClientReceiveMsgSize = newInt(4096)
 	te3.maxClientSendMsgSize = newInt(4096)
 
-	te3.startServer(&testServer{security: e.security})
+	te3.startServer(testServer{security: e.security}.Svc())
 	defer te3.tearDown()
 
 	cc3 := te3.clientConn(grpc.WithResolvers(r))
@@ -1965,7 +1979,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 // test makes sure read from streaming RPC doesn't fail in this case.
 func (s) TestStreamingRPCWithTimeoutInServiceConfigRecv(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
-	te.startServer(&testServer{security: tcpClearRREnv.security})
+	te.startServer(testServer{security: tcpClearRREnv.security}.Svc())
 	defer te.tearDown()
 	r := manual.NewBuilderWithScheme("whatever")
 
@@ -2047,7 +2061,7 @@ func testPreloaderClientSend(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2121,7 +2135,7 @@ func testMaxMsgSizeClientDefault(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2185,7 +2199,7 @@ func testMaxMsgSizeClientAPI(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2270,7 +2284,7 @@ func testMaxMsgSizeServerAPI(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2375,7 +2389,7 @@ func testTap(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2470,7 +2484,7 @@ func (s) TestHealthCheckSuccess(t *testing.T) {
 func testHealthCheckSuccess(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
 	defer te.tearDown()
 
@@ -2492,7 +2506,7 @@ func testHealthCheckFailure(t *testing.T, e env) {
 		"grpc: the client connection is closing; please retry",
 	)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
 	defer te.tearDown()
 
@@ -2516,7 +2530,7 @@ func (s) TestHealthCheckOff(t *testing.T) {
 func testHealthCheckOff(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.NotFound)
@@ -2533,7 +2547,7 @@ func (s) TestHealthWatchMultipleClients(t *testing.T) {
 func testHealthWatchMultipleClients(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2562,7 +2576,7 @@ func (s) TestHealthWatchSameStatus(t *testing.T) {
 func testHealthWatchSameStatus(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService)
@@ -2590,7 +2604,7 @@ func testHealthWatchSetServiceStatusBeforeStartingServer(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.healthServer = hs
 	hs.SetServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService)
@@ -2611,7 +2625,7 @@ func (s) TestHealthWatchDefaultStatusChange(t *testing.T) {
 func testHealthWatchDefaultStatusChange(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService)
@@ -2632,7 +2646,7 @@ func (s) TestHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T) {
 func testHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
 	defer te.tearDown()
 
@@ -2652,7 +2666,7 @@ func (s) TestHealthWatchOverallServerHealthChange(t *testing.T) {
 func testHealthWatchOverallServerHealthChange(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), "")
@@ -2684,7 +2698,7 @@ func (s) TestUnknownHandler(t *testing.T) {
 func testUnknownHandler(t *testing.T, e env, unknownHandler grpc.StreamHandler) {
 	te := newTest(t, e)
 	te.unknownHandler = unknownHandler
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), "", codes.Unauthenticated)
 }
@@ -2700,7 +2714,7 @@ func (s) TestHealthCheckServingStatus(t *testing.T) {
 func testHealthCheckServingStatus(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2721,7 +2735,7 @@ func (s) TestEmptyUnaryWithUserAgent(t *testing.T) {
 func testEmptyUnaryWithUserAgent(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2752,7 +2766,7 @@ func (s) TestFailedEmptyUnary(t *testing.T) {
 func testFailedEmptyUnary(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = failAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -2771,7 +2785,7 @@ func (s) TestLargeUnary(t *testing.T) {
 
 func testLargeUnary(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -2810,7 +2824,7 @@ func testExceedMsgLimit(t *testing.T, e env) {
 	te := newTest(t, e)
 	maxMsgSize := 1024
 	te.maxServerMsgSize, te.maxClientMsgSize = newInt(maxMsgSize), newInt(maxMsgSize)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -2889,7 +2903,7 @@ func (s) TestPeerClientSide(t *testing.T) {
 func testPeerClientSide(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	peer := new(peer.Peer)
@@ -2927,7 +2941,7 @@ func (s) TestPeerNegative(t *testing.T) {
 
 func testPeerNegative(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2947,7 +2961,7 @@ func (s) TestPeerFailedRPC(t *testing.T) {
 func testPeerFailedRPC(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(1 * 1024)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -3001,7 +3015,7 @@ func (s) TestMetadataUnaryRPC(t *testing.T) {
 
 func testMetadataUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3046,7 +3060,7 @@ func (s) TestMetadataOrderUnaryRPC(t *testing.T) {
 
 func testMetadataOrderUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3083,7 +3097,7 @@ func (s) TestMultipleSetTrailerUnaryRPC(t *testing.T) {
 
 func testMultipleSetTrailerUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, multipleSetTrailer: true})
+	te.startServer(testServer{security: e.security, multipleSetTrailer: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3120,7 +3134,7 @@ func (s) TestMultipleSetTrailerStreamingRPC(t *testing.T) {
 
 func testMultipleSetTrailerStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, multipleSetTrailer: true})
+	te.startServer(testServer{security: e.security, multipleSetTrailer: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3155,7 +3169,7 @@ func (s) TestSetAndSendHeaderUnaryRPC(t *testing.T) {
 // To test header metadata is sent on SendHeader().
 func testSetAndSendHeaderUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, setAndSendHeader: true})
+	te.startServer(testServer{security: e.security, setAndSendHeader: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3198,7 +3212,7 @@ func (s) TestMultipleSetHeaderUnaryRPC(t *testing.T) {
 // To test header metadata is sent when sending response.
 func testMultipleSetHeaderUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
+	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3242,7 +3256,7 @@ func (s) TestMultipleSetHeaderUnaryRPCError(t *testing.T) {
 // To test header metadata is sent when sending status.
 func testMultipleSetHeaderUnaryRPCError(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
+	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3285,7 +3299,7 @@ func (s) TestSetAndSendHeaderStreamingRPC(t *testing.T) {
 // To test header metadata is sent on SendHeader().
 func testSetAndSendHeaderStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, setAndSendHeader: true})
+	te.startServer(testServer{security: e.security, setAndSendHeader: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3325,7 +3339,7 @@ func (s) TestMultipleSetHeaderStreamingRPC(t *testing.T) {
 // To test header metadata is sent when sending response.
 func testMultipleSetHeaderStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
+	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3389,7 +3403,7 @@ func (s) TestMultipleSetHeaderStreamingRPCError(t *testing.T) {
 // To test header metadata is sent when sending status.
 func testMultipleSetHeaderStreamingRPCError(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
+	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3454,7 +3468,7 @@ func (s) TestMalformedHTTP2Metadata(t *testing.T) {
 
 func testMalformedHTTP2Metadata(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3497,7 +3511,7 @@ func testTransparentRetry(t *testing.T, e env) {
 		}
 		return ctx, nil
 	}
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -3546,7 +3560,7 @@ func (s) TestCancel(t *testing.T) {
 func testCancel(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("grpc: the client connection is closing; please retry")
-	te.startServer(&testServer{security: e.security, unaryCallSleepTime: time.Second})
+	te.startServer(testServer{security: e.security, unaryCallSleepTime: time.Second}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -3583,7 +3597,7 @@ func testCancelNoIO(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("http2Client.notifyError got notified that the client transport was broken")
 	te.maxStream = 1 // Only allows 1 live stream per server transport.
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -3592,7 +3606,7 @@ func testCancelNoIO(t *testing.T, e env) {
 	// Start one blocked RPC for which we'll never send streaming
 	// input. This will consume the 1 maximum concurrent streams,
 	// causing future RPCs to hang.
-	ctx, cancelFirst := context.WithCancel(context.Background())
+	ctx, cancelFirst := context.WithTimeout(context.Background(), 5*time.Second)
 	_, err := tc.StreamingInputCall(ctx)
 	if err != nil {
 		t.Fatalf("%v.StreamingInputCall(_) = _, %v, want _, <nil>", tc, err)
@@ -3605,6 +3619,9 @@ func testCancelNoIO(t *testing.T, e env) {
 	// succeeding.
 	// TODO(bradfitz): add internal test hook for this (Issue 534)
 	for {
+		if ctx.Err() != nil {
+			t.Fatal("timed out waiting to get deadline exceeded error")
+		}
 		ctx, cancelSecond := context.WithTimeout(context.Background(), 50*time.Millisecond)
 		_, err := tc.StreamingInputCall(ctx)
 		cancelSecond()
@@ -3672,7 +3689,7 @@ func (s) TestPingPong(t *testing.T) {
 
 func testPingPong(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3731,7 +3748,7 @@ func (s) TestMetadataStreamingRPC(t *testing.T) {
 
 func testMetadataStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3808,7 +3825,7 @@ func (s) TestServerStreaming(t *testing.T) {
 
 func testServerStreaming(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3863,7 +3880,7 @@ func (s) TestFailedServerStreaming(t *testing.T) {
 func testFailedServerStreaming(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = failAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3892,15 +3909,10 @@ func equalError(x, y error) bool {
 	return x == y || (x != nil && y != nil && x.Error() == y.Error())
 }
 
-// concurrentSendServer is a TestServiceService whose
-// StreamingOutputCall makes ten serial Send calls, sending payloads
+// concurrentSendStreamingOutputCall makes ten serial Send calls, sending payloads
 // "0".."9", inclusive.  TestServerStreamingConcurrent verifies they
 // were received in the correct order, and that there were no races.
-//
-// All other TestServiceService methods return unimplemented if called.
-type concurrentSendServer struct{}
-
-func (s concurrentSendServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
+func concurrentSendStreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
 	for i := 0; i < 10; i++ {
 		stream.Send(&testpb.StreamingOutputCallResponse{
 			Payload: &testpb.Payload{
@@ -3920,7 +3932,7 @@ func (s) TestServerStreamingConcurrent(t *testing.T) {
 
 func testServerStreamingConcurrent(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(concurrentSendServer{})
+	te.startServer(&testpb.TestServiceService{StreamingOutputCall: concurrentSendStreamingOutputCall})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -4001,7 +4013,7 @@ func (s) TestClientStreaming(t *testing.T) {
 
 func testClientStreaming(t *testing.T, e env, sizes []int) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4047,7 +4059,7 @@ func (s) TestClientStreamingError(t *testing.T) {
 
 func testClientStreamingError(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, earlyFail: true})
+	te.startServer(testServer{security: e.security, earlyFail: true}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4092,7 +4104,7 @@ func testExceedMaxStreamsLimit(t *testing.T, e env) {
 		"grpc: the connection is closing",
 	)
 	te.maxStream = 1 // Only allows 1 live stream per server transport.
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -4132,7 +4144,7 @@ func testStreamsQuotaRecovery(t *testing.T, e env) {
 		"grpc: the connection is closing",
 	)
 	te.maxStream = 1 // Allows 1 live stream.
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -4202,7 +4214,7 @@ func testCompressServerHasNoSupport(t *testing.T, e env) {
 	te.serverCompression = false
 	te.clientCompression = false
 	te.clientNopCompression = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4240,7 +4252,7 @@ func testCompressOK(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.serverCompression = true
 	te.clientCompression = true
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4301,7 +4313,7 @@ func (s) TestIdentityEncoding(t *testing.T) {
 
 func testIdentityEncoding(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4365,7 +4377,7 @@ func testUnaryClientInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
 	te.unaryClientInt = failOkayRPC
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4391,7 +4403,7 @@ func failOkayStream(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientC
 func testStreamClientInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.streamClientInt = failOkayStream
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4427,7 +4439,7 @@ func errInjector(ctx context.Context, req interface{}, info *grpc.UnaryServerInf
 func testUnaryServerInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.unaryServerInt = errInjector
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4457,7 +4469,7 @@ func fullDuplexOnly(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServ
 func testStreamServerInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.streamServerInt = fullDuplexOnly
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4494,28 +4506,6 @@ func testStreamServerInterceptor(t *testing.T, e env) {
 	}
 }
 
-// funcServer implements methods of TestServiceService using funcs,
-// similar to an http.HandlerFunc.
-// Any unimplemented method will return unimplemented. Tests implement the method(s)
-// they need.
-type funcServer struct {
-	unaryCall          func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
-	streamingInputCall func(stream testpb.TestService_StreamingInputCallServer) error
-	fullDuplexCall     func(stream testpb.TestService_FullDuplexCallServer) error
-}
-
-func (s *funcServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
-	return s.unaryCall(ctx, in)
-}
-
-func (s *funcServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
-	return s.streamingInputCall(stream)
-}
-
-func (s *funcServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
-	return s.fullDuplexCall(stream)
-}
-
 func (s) TestClientRequestBodyErrorUnexpectedEOF(t *testing.T) {
 	for _, e := range listTestEnv() {
 		testClientRequestBodyErrorUnexpectedEOF(t, e)
@@ -4524,7 +4514,7 @@ func (s) TestClientRequestBodyErrorUnexpectedEOF(t *testing.T) {
 
 func testClientRequestBodyErrorUnexpectedEOF(t *testing.T, e env) {
 	te := newTest(t, e)
-	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		errUnexpectedCall := errors.New("unexpected call func server method")
 		t.Error(errUnexpectedCall)
 		return nil, errUnexpectedCall
@@ -4548,7 +4538,7 @@ func (s) TestClientRequestBodyErrorCloseAfterLength(t *testing.T) {
 func testClientRequestBodyErrorCloseAfterLength(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("Server.processUnaryRPC failed to write status")
-	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		errUnexpectedCall := errors.New("unexpected call func server method")
 		t.Error(errUnexpectedCall)
 		return nil, errUnexpectedCall
@@ -4572,7 +4562,7 @@ func (s) TestClientRequestBodyErrorCancel(t *testing.T) {
 func testClientRequestBodyErrorCancel(t *testing.T, e env) {
 	te := newTest(t, e)
 	gotCall := make(chan bool, 1)
-	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		gotCall <- true
 		return new(testpb.SimpleResponse), nil
 	}}
@@ -4608,7 +4598,7 @@ func (s) TestClientRequestBodyErrorCancelStreamingInput(t *testing.T) {
 func testClientRequestBodyErrorCancelStreamingInput(t *testing.T, e env) {
 	te := newTest(t, e)
 	recvErr := make(chan error, 1)
-	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+	ts := &testpb.TestServiceService{StreamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
 		_, err := stream.Recv()
 		recvErr <- err
 		return nil
@@ -4650,7 +4640,7 @@ func testClientInitialHeaderEndStream(t *testing.T, e env) {
 	// checking.
 	handlerDone := make(chan struct{})
 	te := newTest(t, e)
-	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+	ts := &testpb.TestServiceService{StreamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
 		defer close(handlerDone)
 		// Block on serverTester receiving RST_STREAM. This ensures server has closed
 		// stream before stream.Recv().
@@ -4694,7 +4684,7 @@ func testClientSendDataAfterCloseSend(t *testing.T, e env) {
 	// checking.
 	handlerDone := make(chan struct{})
 	te := newTest(t, e)
-	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+	ts := &testpb.TestServiceService{StreamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
 		defer close(handlerDone)
 		// Block on serverTester receiving RST_STREAM. This ensures server has closed
 		// stream before stream.Recv().
@@ -4746,7 +4736,7 @@ func (s) TestClientResourceExhaustedCancelFullDuplex(t *testing.T) {
 func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) {
 	te := newTest(t, e)
 	recvErr := make(chan error, 1)
-	ts := &funcServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ts := &testpb.TestServiceService{FullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
 		defer close(recvErr)
 		_, err := stream.Recv()
 		if err != nil {
@@ -5484,7 +5474,7 @@ func testConfigurableWindowSize(t *testing.T, e env, wc windowSizeConfig) {
 	te.clientInitialWindowSize = wc.clientStream
 	te.clientInitialConnWindowSize = wc.clientConn
 
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -5534,7 +5524,7 @@ func (s) TestWaitForReadyConnection(t *testing.T) {
 func testWaitForReadyConnection(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn() // Non-blocking dial.
@@ -5585,7 +5575,7 @@ func testEncodeDoesntPanic(t *testing.T, e env) {
 	te := newTest(t, e)
 	erc := &errCodec{}
 	te.customCodec = erc
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	te.customCodec = nil
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -5619,7 +5609,7 @@ func testSvrWriteStatusEarlyWrite(t *testing.T, e env) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	respParam := []*testpb.ResponseParameters{
@@ -5898,7 +5888,7 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	}
 	// Case1: sc set maxReqSize to 2048 (send), maxRespSize to 2048 (recv).
 	te1, ch1 := testServiceConfigSetupTD(t, e)
-	te1.startServer(&testServer{security: e.security})
+	te1.startServer(testServer{security: e.security}.Svc())
 	defer te1.tearDown()
 
 	ch1 <- sc
@@ -5958,7 +5948,7 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	te2, ch2 := testServiceConfigSetupTD(t, e)
 	te2.maxClientReceiveMsgSize = newInt(1024)
 	te2.maxClientSendMsgSize = newInt(1024)
-	te2.startServer(&testServer{security: e.security})
+	te2.startServer(testServer{security: e.security}.Svc())
 	defer te2.tearDown()
 	ch2 <- sc
 	tc = testpb.NewTestServiceClient(te2.clientConn())
@@ -6007,7 +5997,7 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	te3, ch3 := testServiceConfigSetupTD(t, e)
 	te3.maxClientReceiveMsgSize = newInt(4096)
 	te3.maxClientSendMsgSize = newInt(4096)
-	te3.startServer(&testServer{security: e.security})
+	te3.startServer(testServer{security: e.security}.Svc())
 	defer te3.tearDown()
 	ch3 <- sc
 	tc = testpb.NewTestServiceClient(te3.clientConn())
@@ -6099,7 +6089,7 @@ func (s) TestMethodFromServerStream(t *testing.T) {
 func (s) TestInterceptorCanAccessCallOptions(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	type observedOptions struct {
@@ -6210,7 +6200,7 @@ func testCompressorRegister(t *testing.T, e env) {
 	te.serverCompression = false
 	te.clientUseCompression = true
 
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -6385,7 +6375,7 @@ func testClientDoesntDeadlockWhileWritingErrornousLargeMessages(t *testing.T, e
 	te.userAgent = testAppUA
 	smallSize := 1024
 	te.maxServerReceiveMsgSize = &smallSize
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 1048576)
@@ -6422,7 +6412,7 @@ func (s) TestRPCTimeout(t *testing.T) {
 
 func testRPCTimeout(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testServer{security: e.security, unaryCallSleepTime: 500 * time.Millisecond})
+	te.startServer(testServer{security: e.security, unaryCallSleepTime: 500 * time.Millisecond}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -6545,7 +6535,7 @@ func testServerMaxHeaderListSizeClientUserViolation(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.maxServerHeaderListSize = new(uint32)
 	*te.maxServerHeaderListSize = 216
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -6577,7 +6567,7 @@ func testClientMaxHeaderListSizeServerUserViolation(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.maxClientHeaderListSize = new(uint32)
 	*te.maxClientHeaderListSize = 1 // any header server sends will violate
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -6608,7 +6598,7 @@ func testServerMaxHeaderListSizeClientIntentionalViolation(t *testing.T, e env)
 	te := newTest(t, e)
 	te.maxServerHeaderListSize = new(uint32)
 	*te.maxServerHeaderListSize = 512
-	te.startServer(&testServer{security: e.security})
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc, dw := te.clientConnWithConnControl()
@@ -6650,7 +6640,7 @@ func testClientMaxHeaderListSizeServerIntentionalViolation(t *testing.T, e env)
 	te := newTest(t, e)
 	te.maxClientHeaderListSize = new(uint32)
 	*te.maxClientHeaderListSize = 200
-	lw := te.startServerWithConnControl(&testServer{security: e.security, setHeaderOnly: true})
+	lw := te.startServerWithConnControl(testServer{security: e.security, setHeaderOnly: true}.Svc())
 	defer te.tearDown()
 	cc, _ := te.clientConnWithConnControl()
 	tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)}
@@ -6697,10 +6687,10 @@ func (s) TestNetPipeConn(t *testing.T) {
 	pl := testutils.NewPipeListener()
 	s := grpc.NewServer()
 	defer s.Stop()
-	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		return &testpb.SimpleResponse{}, nil
 	}}
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts))
+	testpb.RegisterTestServiceService(s, ts)
 	go s.Serve(pl)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -6725,7 +6715,18 @@ func testLargeTimeout(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("Server.processUnaryRPC failed to write status")
 
-	ts := &funcServer{}
+	maxTimeoutChan := make(chan time.Duration, 1)
+	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+		maxTimeout := <-maxTimeoutChan
+		deadline, ok := ctx.Deadline()
+		timeout := time.Until(deadline)
+		minTimeout := maxTimeout - 5*time.Second
+		if !ok || timeout < minTimeout || timeout > maxTimeout {
+			t.Errorf("ctx.Deadline() = (now+%v), %v; want [%v, %v], true", timeout, ok, minTimeout, maxTimeout)
+			return nil, status.Error(codes.OutOfRange, "deadline error")
+		}
+		return &testpb.SimpleResponse{}, nil
+	}}
 	te.startServer(ts)
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -6737,17 +6738,7 @@ func testLargeTimeout(t *testing.T, e env) {
 	}
 
 	for i, maxTimeout := range timeouts {
-		ts.unaryCall = func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
-			deadline, ok := ctx.Deadline()
-			timeout := time.Until(deadline)
-			minTimeout := maxTimeout - 5*time.Second
-			if !ok || timeout < minTimeout || timeout > maxTimeout {
-				t.Errorf("ctx.Deadline() = (now+%v), %v; want [%v, %v], true", timeout, ok, minTimeout, maxTimeout)
-				return nil, status.Error(codes.OutOfRange, "deadline error")
-			}
-			return &testpb.SimpleResponse{}, nil
-		}
-
+		maxTimeoutChan <- maxTimeout
 		ctx, cancel := context.WithTimeout(context.Background(), maxTimeout)
 		defer cancel()
 
@@ -6770,11 +6761,11 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	}
 	s1 := grpc.NewServer()
 	defer s1.Stop()
-	ts := &funcServer{
-		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &testpb.TestServiceService{
+		UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{}, nil
 		},
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+		FullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
 			// Wait forever.
 			_, err := stream.Recv()
 			if err == nil {
@@ -6783,7 +6774,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 			return err
 		},
 	}
-	testpb.RegisterTestServiceService(s1, testpb.NewTestServiceService(ts))
+	testpb.RegisterTestServiceService(s1, ts)
 	go s1.Serve(lis1)
 
 	conn2Established := grpcsync.NewEvent()
@@ -6793,7 +6784,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	}
 	s2 := grpc.NewServer()
 	defer s2.Stop()
-	testpb.RegisterTestServiceService(s2, testpb.NewTestServiceService(ts))
+	testpb.RegisterTestServiceService(s2, ts)
 	go s2.Serve(lis2)
 
 	r := manual.NewBuilderWithScheme("whatever")
@@ -6863,7 +6854,7 @@ func (lis notifyingListener) Accept() (net.Conn, error) {
 
 func (s) TestRPCWaitsForResolver(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
-	te.startServer(&testServer{security: tcpClearRREnv.security})
+	te.startServer(testServer{security: tcpClearRREnv.security}.Svc())
 	defer te.tearDown()
 	r := manual.NewBuilderWithScheme("whatever")
 
diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go
index 0137494efb8f..c8a3796048b7 100644
--- a/test/healthcheck_test.go
+++ b/test/healthcheck_test.go
@@ -140,7 +140,7 @@ func setupServer(sc *svrConfig) (s *grpc.Server, lis net.Listener, ts *testHealt
 		ts = newTestHealthServer()
 	}
 	healthgrpc.RegisterHealthServer(s, ts)
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&testServer{}))
+	testpb.RegisterTestServiceService(s, testServer{}.Svc())
 	go s.Serve(lis)
 	return s, lis, ts, s.Stop, nil
 }

From 52029da148228fa9714c47f1de7ea718c7d29102 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Wed, 9 Sep 2020 15:37:11 -0700
Subject: [PATCH 188/481] service reflection: include transitive closure for a
 file (#3851)

---
 reflection/serverreflection.go      | 69 +++++++++++++++++++++--------
 reflection/serverreflection_test.go | 35 +++++++++++++++
 2 files changed, 86 insertions(+), 18 deletions(-)

diff --git a/reflection/serverreflection.go b/reflection/serverreflection.go
index 7b6dd414a275..d2696168b10c 100644
--- a/reflection/serverreflection.go
+++ b/reflection/serverreflection.go
@@ -269,9 +269,39 @@ func (s *serverReflectionServer) allExtensionNumbersForType(st reflect.Type) ([]
 	return out, nil
 }
 
+// fileDescWithDependencies returns a slice of serialized fileDescriptors in
+// wire format ([]byte). The fileDescriptors will include fd and all the
+// transitive dependencies of fd with names not in sentFileDescriptors.
+func fileDescWithDependencies(fd *dpb.FileDescriptorProto, sentFileDescriptors map[string]bool) ([][]byte, error) {
+	r := [][]byte{}
+	queue := []*dpb.FileDescriptorProto{fd}
+	for len(queue) > 0 {
+		currentfd := queue[0]
+		queue = queue[1:]
+		if sent := sentFileDescriptors[currentfd.GetName()]; len(r) == 0 || !sent {
+			sentFileDescriptors[currentfd.GetName()] = true
+			currentfdEncoded, err := proto.Marshal(currentfd)
+			if err != nil {
+				return nil, err
+			}
+			r = append(r, currentfdEncoded)
+		}
+		for _, dep := range currentfd.Dependency {
+			fdenc := proto.FileDescriptor(dep)
+			fdDep, err := decodeFileDesc(fdenc)
+			if err != nil {
+				continue
+			}
+			queue = append(queue, fdDep)
+		}
+	}
+	return r, nil
+}
+
 // fileDescEncodingByFilename finds the file descriptor for given filename,
-// does marshalling on it and returns the marshalled result.
-func (s *serverReflectionServer) fileDescEncodingByFilename(name string) ([]byte, error) {
+// finds all of its previously unsent transitive dependencies, does marshalling
+// on them, and returns the marshalled result.
+func (s *serverReflectionServer) fileDescEncodingByFilename(name string, sentFileDescriptors map[string]bool) ([][]byte, error) {
 	enc := proto.FileDescriptor(name)
 	if enc == nil {
 		return nil, fmt.Errorf("unknown file: %v", name)
@@ -280,7 +310,7 @@ func (s *serverReflectionServer) fileDescEncodingByFilename(name string) ([]byte
 	if err != nil {
 		return nil, err
 	}
-	return proto.Marshal(fd)
+	return fileDescWithDependencies(fd, sentFileDescriptors)
 }
 
 // parseMetadata finds the file descriptor bytes specified meta.
@@ -301,10 +331,11 @@ func parseMetadata(meta interface{}) ([]byte, bool) {
 	return nil, false
 }
 
-// fileDescEncodingContainingSymbol finds the file descriptor containing the given symbol,
-// does marshalling on it and returns the marshalled result.
-// The given symbol can be a type, a service or a method.
-func (s *serverReflectionServer) fileDescEncodingContainingSymbol(name string) ([]byte, error) {
+// fileDescEncodingContainingSymbol finds the file descriptor containing the
+// given symbol, finds all of its previously unsent transitive dependencies,
+// does marshalling on them, and returns the marshalled result. The given symbol
+// can be a type, a service or a method.
+func (s *serverReflectionServer) fileDescEncodingContainingSymbol(name string, sentFileDescriptors map[string]bool) ([][]byte, error) {
 	_, symbols := s.getSymbols()
 	fd := symbols[name]
 	if fd == nil {
@@ -322,12 +353,13 @@ func (s *serverReflectionServer) fileDescEncodingContainingSymbol(name string) (
 		return nil, fmt.Errorf("unknown symbol: %v", name)
 	}
 
-	return proto.Marshal(fd)
+	return fileDescWithDependencies(fd, sentFileDescriptors)
 }
 
-// fileDescEncodingContainingExtension finds the file descriptor containing given extension,
-// does marshalling on it and returns the marshalled result.
-func (s *serverReflectionServer) fileDescEncodingContainingExtension(typeName string, extNum int32) ([]byte, error) {
+// fileDescEncodingContainingExtension finds the file descriptor containing
+// given extension, finds all of its previously unsent transitive dependencies,
+// does marshalling on them, and returns the marshalled result.
+func (s *serverReflectionServer) fileDescEncodingContainingExtension(typeName string, extNum int32, sentFileDescriptors map[string]bool) ([][]byte, error) {
 	st, err := typeForName(typeName)
 	if err != nil {
 		return nil, err
@@ -336,7 +368,7 @@ func (s *serverReflectionServer) fileDescEncodingContainingExtension(typeName st
 	if err != nil {
 		return nil, err
 	}
-	return proto.Marshal(fd)
+	return fileDescWithDependencies(fd, sentFileDescriptors)
 }
 
 // allExtensionNumbersForTypeName returns all extension numbers for the given type.
@@ -354,6 +386,7 @@ func (s *serverReflectionServer) allExtensionNumbersForTypeName(name string) ([]
 
 // ServerReflectionInfo is the reflection service handler.
 func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflection_ServerReflectionInfoServer) error {
+	sentFileDescriptors := make(map[string]bool)
 	for {
 		in, err := stream.Recv()
 		if err == io.EOF {
@@ -369,7 +402,7 @@ func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflectio
 		}
 		switch req := in.MessageRequest.(type) {
 		case *rpb.ServerReflectionRequest_FileByFilename:
-			b, err := s.fileDescEncodingByFilename(req.FileByFilename)
+			b, err := s.fileDescEncodingByFilename(req.FileByFilename, sentFileDescriptors)
 			if err != nil {
 				out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{
 					ErrorResponse: &rpb.ErrorResponse{
@@ -379,11 +412,11 @@ func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflectio
 				}
 			} else {
 				out.MessageResponse = &rpb.ServerReflectionResponse_FileDescriptorResponse{
-					FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: [][]byte{b}},
+					FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: b},
 				}
 			}
 		case *rpb.ServerReflectionRequest_FileContainingSymbol:
-			b, err := s.fileDescEncodingContainingSymbol(req.FileContainingSymbol)
+			b, err := s.fileDescEncodingContainingSymbol(req.FileContainingSymbol, sentFileDescriptors)
 			if err != nil {
 				out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{
 					ErrorResponse: &rpb.ErrorResponse{
@@ -393,13 +426,13 @@ func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflectio
 				}
 			} else {
 				out.MessageResponse = &rpb.ServerReflectionResponse_FileDescriptorResponse{
-					FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: [][]byte{b}},
+					FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: b},
 				}
 			}
 		case *rpb.ServerReflectionRequest_FileContainingExtension:
 			typeName := req.FileContainingExtension.ContainingType
 			extNum := req.FileContainingExtension.ExtensionNumber
-			b, err := s.fileDescEncodingContainingExtension(typeName, extNum)
+			b, err := s.fileDescEncodingContainingExtension(typeName, extNum, sentFileDescriptors)
 			if err != nil {
 				out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{
 					ErrorResponse: &rpb.ErrorResponse{
@@ -409,7 +442,7 @@ func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflectio
 				}
 			} else {
 				out.MessageResponse = &rpb.ServerReflectionResponse_FileDescriptorResponse{
-					FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: [][]byte{b}},
+					FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: b},
 				}
 			}
 		case *rpb.ServerReflectionRequest_AllExtensionNumbersOfType:
diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go
index db5fce2d8939..9f252d778b94 100644
--- a/reflection/serverreflection_test.go
+++ b/reflection/serverreflection_test.go
@@ -214,6 +214,8 @@ func (x) TestReflectionEnd2end(t *testing.T) {
 		t.Fatalf("cannot get ServerReflectionInfo: %v", err)
 	}
 
+	testFileByFilenameTransitiveClosure(t, stream, true)
+	testFileByFilenameTransitiveClosure(t, stream, false)
 	testFileByFilename(t, stream)
 	testFileByFilenameError(t, stream)
 	testFileContainingSymbol(t, stream)
@@ -227,6 +229,39 @@ func (x) TestReflectionEnd2end(t *testing.T) {
 	s.Stop()
 }
 
+func testFileByFilenameTransitiveClosure(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient, expectClosure bool) {
+	filename := "reflection/grpc_testing/proto2_ext2.proto"
+	if err := stream.Send(&rpb.ServerReflectionRequest{
+		MessageRequest: &rpb.ServerReflectionRequest_FileByFilename{
+			FileByFilename: filename,
+		},
+	}); err != nil {
+		t.Fatalf("failed to send request: %v", err)
+	}
+	r, err := stream.Recv()
+	if err != nil {
+		// io.EOF is not ok.
+		t.Fatalf("failed to recv response: %v", err)
+	}
+	switch r.MessageResponse.(type) {
+	case *rpb.ServerReflectionResponse_FileDescriptorResponse:
+		if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], fdProto2Ext2Byte) {
+			t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", filename, r.GetFileDescriptorResponse().FileDescriptorProto[0], fdProto2Ext2Byte)
+		}
+		if expectClosure {
+			if len(r.GetFileDescriptorResponse().FileDescriptorProto) != 2 {
+				t.Errorf("FileByFilename(%v) returned %v file descriptors, expected 2", filename, len(r.GetFileDescriptorResponse().FileDescriptorProto))
+			} else if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[1], fdProto2Byte) {
+				t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", filename, r.GetFileDescriptorResponse().FileDescriptorProto[1], fdProto2Byte)
+			}
+		} else if len(r.GetFileDescriptorResponse().FileDescriptorProto) != 1 {
+			t.Errorf("FileByFilename(%v) returned %v file descriptors, expected 1", filename, len(r.GetFileDescriptorResponse().FileDescriptorProto))
+		}
+	default:
+		t.Errorf("FileByFilename(%v) = %v, want type <ServerReflectionResponse_FileDescriptorResponse>", filename, r.MessageResponse)
+	}
+}
+
 func testFileByFilename(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) {
 	for _, test := range []struct {
 		filename string

From 4a2248a42bc2b2410b7214ef7020eb05ae87124d Mon Sep 17 00:00:00 2001
From: "Sean R. Abraham" <seanabraham@gmail.com>
Date: Thu, 10 Sep 2020 11:18:02 -0600
Subject: [PATCH 189/481] Remove grpc_cli build instructions (#3868)

The instructions didn't work, as the build process has changed a bit. It's probably best to just link to the official build instructions to avoid future skew.
---
 Documentation/server-reflection-tutorial.md | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/Documentation/server-reflection-tutorial.md b/Documentation/server-reflection-tutorial.md
index 212d4b3771a6..b1781fa68dc9 100644
--- a/Documentation/server-reflection-tutorial.md
+++ b/Documentation/server-reflection-tutorial.md
@@ -43,19 +43,9 @@ An example server with reflection registered can be found at
 
 After enabling Server Reflection in a server application, you can use gRPC CLI
 to check its services. gRPC CLI is only available in c++. Instructions on how to
-use gRPC CLI can be found at
+build and use gRPC CLI can be found at
 [command_line_tool.md](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md).
 
-To build gRPC CLI:
-
-```sh
-git clone https://github.com/grpc/grpc
-cd grpc
-git submodule update --init
-make grpc_cli
-cd bins/opt # grpc_cli is in directory bins/opt/
-```
-
 ## Use gRPC CLI to check services
 
 First, start the helloworld server in grpc-go directory:

From 6591123024b3370443c8c5836087412cae51cde3 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 10 Sep 2020 13:10:57 -0700
Subject: [PATCH 190/481] internal: rename package for test alone file (#3855)

Otherwise glaze complains.
---
 xds/internal/client/tests/client_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xds/internal/client/tests/client_test.go b/xds/internal/client/tests/client_test.go
index 3c96175b0d89..7887835cbadc 100644
--- a/xds/internal/client/tests/client_test.go
+++ b/xds/internal/client/tests/client_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package tests
+package tests_test
 
 import (
 	"testing"

From 86d33e463b4b57cc901488ea6cec3ed164815a63 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 14 Sep 2020 16:52:49 -0700
Subject: [PATCH 191/481] connectivity: Get rid of unused Reporter interface.
 (#3875)

---
 connectivity/connectivity.go | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/connectivity/connectivity.go b/connectivity/connectivity.go
index 0a8d682ac6af..010156261505 100644
--- a/connectivity/connectivity.go
+++ b/connectivity/connectivity.go
@@ -22,8 +22,6 @@
 package connectivity
 
 import (
-	"context"
-
 	"google.golang.org/grpc/grpclog"
 )
 
@@ -63,13 +61,3 @@ const (
 	// Shutdown indicates the ClientConn has started shutting down.
 	Shutdown
 )
-
-// Reporter reports the connectivity states.
-type Reporter interface {
-	// CurrentState returns the current state of the reporter.
-	CurrentState() State
-	// WaitForStateChange blocks until the reporter's state is different from the given state,
-	// and returns true.
-	// It returns false if <-ctx.Done() can proceed (ctx got timeout or got canceled).
-	WaitForStateChange(context.Context, State) bool
-}

From 32e7099cccac274f650ec3dc9427deefa3d54d25 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 14 Sep 2020 17:05:51 -0700
Subject: [PATCH 192/481] grpctest: Minor improvements to tlogger. (#3874)

* Simplify the logic to get the file and line number or call site
* Add a monotonic clock value as a suffix. This helps with debugging of test failures due to timing issues.
---
 internal/grpctest/tlogger.go | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/internal/grpctest/tlogger.go b/internal/grpctest/tlogger.go
index 4558fc10176b..a074fbfa82ac 100644
--- a/internal/grpctest/tlogger.go
+++ b/internal/grpctest/tlogger.go
@@ -22,12 +22,14 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"regexp"
-	"runtime/debug"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"google.golang.org/grpc/grpclog"
 )
@@ -63,25 +65,33 @@ func init() {
 	}
 }
 
-// getStackFrame gets, from the stack byte string, the appropriate stack frame.
-func getStackFrame(stack []byte, frame int) (string, error) {
-	s := strings.Split(string(stack), "\n")
-	if frame >= (len(s)-1)/2 {
+// getCallingPrefix returns the <file:line> at the given depth from the stack.
+func getCallingPrefix(depth int) (string, error) {
+	_, file, line, ok := runtime.Caller(depth)
+	if !ok {
 		return "", errors.New("frame request out-of-bounds")
 	}
-	split := strings.Split(strings.Fields(s[(frame*2)+2][1:])[0], "/")
-	return fmt.Sprintf("%v:", split[len(split)-1]), nil
+	return fmt.Sprintf("%s:%d", path.Base(file), line), nil
+}
+
+// Returns the last component of the stringified current time, which is of the
+// format "m=±<value>", where value is the monotonic clock reading formatted as
+// a decimal number of seconds.
+func getTimeSuffix() string {
+	parts := strings.Split(time.Now().String(), " ")
+	return fmt.Sprintf(" (%s)", parts[len(parts)-1])
 }
 
 // log logs the message with the specified parameters to the tLogger.
 func (g *tLogger) log(ltype logType, depth int, format string, args ...interface{}) {
-	s := debug.Stack()
-	prefix, err := getStackFrame(s, callingFrame+depth)
-	args = append([]interface{}{prefix}, args...)
+	prefix, err := getCallingPrefix(callingFrame + depth)
 	if err != nil {
 		g.t.Error(err)
 		return
 	}
+	args = append([]interface{}{prefix}, args...)
+	args = append(args, getTimeSuffix())
+
 	if format == "" {
 		switch ltype {
 		case errorLog:

From a3e63e87bd7ef17a4fbc3e11cbe8a0a2c11519e3 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 15 Sep 2020 09:53:04 -0700
Subject: [PATCH 193/481] alts: Specify the build constraints correctly.
 (#3877)

From the official docs:
A build constraint is evaluated as the OR of space-separated options.
Each option evaluates as the AND of its comma-separated terms.
---
 credentials/alts/alts_test.go  | 2 +-
 credentials/alts/utils_test.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/credentials/alts/alts_test.go b/credentials/alts/alts_test.go
index ed13ee447f3e..cbb1656d20c3 100644
--- a/credentials/alts/alts_test.go
+++ b/credentials/alts/alts_test.go
@@ -1,4 +1,4 @@
-// +build linux,windows
+// +build linux windows
 
 /*
  *
diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go
index d33f189ee624..b9e752ebbac9 100644
--- a/credentials/alts/utils_test.go
+++ b/credentials/alts/utils_test.go
@@ -1,4 +1,4 @@
-// +build linux,windows
+// +build linux windows
 
 /*
  *

From ff9dd65c90004251e1d11bab1a425f270cb2fd80 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 16 Sep 2020 10:15:56 -0700
Subject: [PATCH 194/481] protoc-gen-go-grpc: generate interfaces optionally;
 remove NewFooService (#3876)

---
 .../grpclb/grpc_lb_v1/load_balancer.pb.go     | 118 -------
 .../grpc_lb_v1/load_balancer_grpc.pb.go       | 115 +++++--
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  86 -----
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       |  68 ++--
 benchmark/benchmark.go                        |  20 +-
 benchmark/grpc_testing/services_grpc.pb.go    |  98 ------
 benchmark/worker/main.go                      |  13 +-
 channelz/grpc_channelz_v1/channelz.pb.go      | 316 ------------------
 channelz/grpc_channelz_v1/channelz_grpc.pb.go | 213 +++++++++---
 cmd/protoc-gen-go-grpc/README.md              |   4 +-
 cmd/protoc-gen-go-grpc/grpc.go                |  90 +++--
 cmd/protoc-gen-go-grpc/main.go                |   4 +-
 .../internal/proto/grpc_gcp/handshaker.pb.go  | 128 -------
 .../proto/grpc_gcp/handshaker_grpc.pb.go      | 120 +++++--
 .../meshca/internal/v1/meshca.pb.go           |  88 -----
 .../meshca/internal/v1/meshca_grpc.pb.go      |  69 ++--
 examples/features/proto/echo/echo_grpc.pb.go  |  46 ---
 .../helloworld/helloworld_grpc.pb.go          |  25 --
 .../routeguide/route_guide_grpc.pb.go         |  62 ----
 examples/route_guide/server/server.go         |  11 +-
 health/grpc_health_v1/health.pb.go            | 182 ----------
 health/grpc_health_v1/health_grpc.pb.go       | 144 ++++++--
 internal/binarylog/binarylog_end2end_test.go  |  11 +-
 interop/grpc_testing/test_grpc.pb.go          | 118 -------
 interop/test_utils.go                         |  10 +-
 profiling/proto/service.pb.go                 | 126 -------
 profiling/proto/service_grpc.pb.go            |  93 ++++--
 .../grpc_reflection_v1alpha/reflection.pb.go  | 120 -------
 .../reflection_grpc.pb.go                     | 116 +++++--
 reflection/grpc_testing/test_grpc.pb.go       |  30 --
 reflection/serverreflection_test.go           |   9 +-
 regenerate.sh                                 |   2 +-
 .../advancedtls_integration_test.go           |   9 +-
 stats/grpc_testing/test_grpc.pb.go            |  49 ---
 stats/stats_test.go                           |  13 +-
 stress/client/main.go                         |  12 +-
 stress/grpc_testing/metrics_grpc.pb.go        |  33 --
 test/balancer_test.go                         |   6 +-
 test/end2end_test.go                          |  34 +-
 test/goaway_test.go                           |   2 +-
 test/gracefulstop_test.go                     |   2 +-
 test/grpc_testing/test_grpc.pb.go             |  68 ----
 test/local_creds_test.go                      |   4 +-
 43 files changed, 911 insertions(+), 1976 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
index b59191ac5825..7381dfc1ae4f 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
@@ -4,14 +4,10 @@
 package grpc_lb_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -642,117 +638,3 @@ var fileDescriptor_7cd3f6d792743fdf = []byte{
 	0x6d, 0xe1, 0xbe, 0xfb, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x47, 0x55, 0xac, 0xab, 0x06, 0x00,
 	0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// LoadBalancerClient is the client API for LoadBalancer service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type LoadBalancerClient interface {
-	// Bidirectional rpc to get a list of servers.
-	BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error)
-}
-
-type loadBalancerClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
-	return &loadBalancerClient{cc}
-}
-
-func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &loadBalancerBalanceLoadClient{stream}
-	return x, nil
-}
-
-type LoadBalancer_BalanceLoadClient interface {
-	Send(*LoadBalanceRequest) error
-	Recv() (*LoadBalanceResponse, error)
-	grpc.ClientStream
-}
-
-type loadBalancerBalanceLoadClient struct {
-	grpc.ClientStream
-}
-
-func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
-	m := new(LoadBalanceResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// LoadBalancerServer is the server API for LoadBalancer service.
-type LoadBalancerServer interface {
-	// Bidirectional rpc to get a list of servers.
-	BalanceLoad(LoadBalancer_BalanceLoadServer) error
-}
-
-// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations.
-type UnimplementedLoadBalancerServer struct {
-}
-
-func (*UnimplementedLoadBalancerServer) BalanceLoad(srv LoadBalancer_BalanceLoadServer) error {
-	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
-}
-
-func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
-	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
-}
-
-func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream})
-}
-
-type LoadBalancer_BalanceLoadServer interface {
-	Send(*LoadBalanceResponse) error
-	Recv() (*LoadBalanceRequest, error)
-	grpc.ServerStream
-}
-
-type loadBalancerBalanceLoadServer struct {
-	grpc.ServerStream
-}
-
-func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
-	m := new(LoadBalanceRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.lb.v1.LoadBalancer",
-	HandlerType: (*LoadBalancerServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "BalanceLoad",
-			Handler:       _LoadBalancer_BalanceLoad_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "grpc/lb/v1/load_balancer.proto",
-}
diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index e59bf309971a..62bd5bef8a04 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -3,6 +3,7 @@
 package grpc_lb_v1
 
 import (
+	context "context"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
@@ -12,6 +13,59 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// LoadBalancerClient is the client API for LoadBalancer service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type LoadBalancerClient interface {
+	// Bidirectional rpc to get a list of servers.
+	BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error)
+}
+
+type loadBalancerClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
+	return &loadBalancerClient{cc}
+}
+
+var loadBalancerBalanceLoadStreamDesc = &grpc.StreamDesc{
+	StreamName:    "BalanceLoad",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
+func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
+	stream, err := c.cc.NewStream(ctx, loadBalancerBalanceLoadStreamDesc, "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &loadBalancerBalanceLoadClient{stream}
+	return x, nil
+}
+
+type LoadBalancer_BalanceLoadClient interface {
+	Send(*LoadBalanceRequest) error
+	Recv() (*LoadBalanceResponse, error)
+	grpc.ClientStream
+}
+
+type loadBalancerBalanceLoadClient struct {
+	grpc.ClientStream
+}
+
+func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
+	m := new(LoadBalanceResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // LoadBalancerService is the service API for LoadBalancer service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterLoadBalancerService is called.  Any unassigned fields will result in the
@@ -25,6 +79,28 @@ func (s *LoadBalancerService) balanceLoad(_ interface{}, stream grpc.ServerStrea
 	return s.BalanceLoad(&loadBalancerBalanceLoadServer{stream})
 }
 
+type LoadBalancer_BalanceLoadServer interface {
+	Send(*LoadBalanceResponse) error
+	Recv() (*LoadBalanceRequest, error)
+	grpc.ServerStream
+}
+
+type loadBalancerBalanceLoadServer struct {
+	grpc.ServerStream
+}
+
+func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
+	m := new(LoadBalanceRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // RegisterLoadBalancerService registers a service implementation with a gRPC server.
 func RegisterLoadBalancerService(s grpc.ServiceRegistrar, srv *LoadBalancerService) {
 	srvCopy := *srv
@@ -50,27 +126,28 @@ func RegisterLoadBalancerService(s grpc.ServiceRegistrar, srv *LoadBalancerServi
 	s.RegisterService(&sd, nil)
 }
 
-// NewLoadBalancerService creates a new LoadBalancerService containing the
-// implemented methods of the LoadBalancer service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewLoadBalancerService(s interface{}) *LoadBalancerService {
-	ns := &LoadBalancerService{}
-	if h, ok := s.(interface {
-		BalanceLoad(LoadBalancer_BalanceLoadServer) error
-	}); ok {
-		ns.BalanceLoad = h.BalanceLoad
-	}
-	return ns
-}
-
-// UnstableLoadBalancerService is the service API for LoadBalancer service.
+// LoadBalancerServer is the service API for LoadBalancer service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableLoadBalancerService interface {
+// use of this type is not recommended unless you own the service definition.
+type LoadBalancerServer interface {
 	// Bidirectional rpc to get a list of servers.
 	BalanceLoad(LoadBalancer_BalanceLoadServer) error
 }
+
+// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations of
+// LoadBalancerServer
+type UnimplementedLoadBalancerServer struct {
+}
+
+func (*UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
+	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
+}
+
+// RegisterLoadBalancerServer registers a service implementation with a gRPC server.
+func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) {
+	str := &LoadBalancerService{
+		BalanceLoad: srv.BalanceLoad,
+	}
+	RegisterLoadBalancerService(s, str)
+}
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index a3a3f8118ce9..7ec14279109f 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -4,12 +4,8 @@
 package grpc_lookup_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -180,85 +176,3 @@ var fileDescriptor_3bab962d3362f3ca = []byte{
 	0x29, 0xa7, 0xcf, 0x5e, 0xb3, 0x1d, 0x27, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0xca, 0x8d, 0x5c,
 	0xc7, 0x39, 0x02, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// RouteLookupServiceClient is the client API for RouteLookupService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type RouteLookupServiceClient interface {
-	// Lookup returns a target for a single key.
-	RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error)
-}
-
-type routeLookupServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
-	return &routeLookupServiceClient{cc}
-}
-
-func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
-	out := new(RouteLookupResponse)
-	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// RouteLookupServiceServer is the server API for RouteLookupService service.
-type RouteLookupServiceServer interface {
-	// Lookup returns a target for a single key.
-	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
-}
-
-// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedRouteLookupServiceServer struct {
-}
-
-func (*UnimplementedRouteLookupServiceServer) RouteLookup(ctx context.Context, req *RouteLookupRequest) (*RouteLookupResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
-}
-
-func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
-	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
-}
-
-func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(RouteLookupRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(RouteLookupServiceServer).RouteLookup(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.lookup.v1.RouteLookupService",
-	HandlerType: (*RouteLookupServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "RouteLookup",
-			Handler:    _RouteLookupService_RouteLookup_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/lookup/v1/rls.proto",
-}
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index 4a9f27067837..6952ce62c923 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -13,6 +13,35 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// RouteLookupServiceClient is the client API for RouteLookupService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type RouteLookupServiceClient interface {
+	// Lookup returns a target for a single key.
+	RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error)
+}
+
+type routeLookupServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient {
+	return &routeLookupServiceClient{cc}
+}
+
+var routeLookupServiceRouteLookupStreamDesc = &grpc.StreamDesc{
+	StreamName: "RouteLookup",
+}
+
+func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
+	out := new(RouteLookupResponse)
+	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // RouteLookupServiceService is the service API for RouteLookupService service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterRouteLookupServiceService is called.  Any unassigned fields will result in the
@@ -63,27 +92,28 @@ func RegisterRouteLookupServiceService(s grpc.ServiceRegistrar, srv *RouteLookup
 	s.RegisterService(&sd, nil)
 }
 
-// NewRouteLookupServiceService creates a new RouteLookupServiceService containing the
-// implemented methods of the RouteLookupService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewRouteLookupServiceService(s interface{}) *RouteLookupServiceService {
-	ns := &RouteLookupServiceService{}
-	if h, ok := s.(interface {
-		RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
-	}); ok {
-		ns.RouteLookup = h.RouteLookup
-	}
-	return ns
-}
-
-// UnstableRouteLookupServiceService is the service API for RouteLookupService service.
+// RouteLookupServiceServer is the service API for RouteLookupService service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableRouteLookupServiceService interface {
+// use of this type is not recommended unless you own the service definition.
+type RouteLookupServiceServer interface {
 	// Lookup returns a target for a single key.
 	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
 }
+
+// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations of
+// RouteLookupServiceServer
+type UnimplementedRouteLookupServiceServer struct {
+}
+
+func (*UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
+}
+
+// RegisterRouteLookupServiceServer registers a service implementation with a gRPC server.
+func RegisterRouteLookupServiceServer(s grpc.ServiceRegistrar, srv RouteLookupServiceServer) {
+	str := &RouteLookupServiceService{
+		RouteLookup: srv.RouteLookup,
+	}
+	RegisterRouteLookupServiceService(s, str)
+}
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index d2783a87e00a..5794aebcc559 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -63,7 +63,13 @@ func NewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 
 type testServer struct{}
 
-var _ testpb.UnstableBenchmarkServiceService = (*testServer)(nil)
+func (s *testServer) Svc() *testpb.BenchmarkServiceService {
+	return &testpb.BenchmarkServiceService{
+		UnaryCall:                  s.UnaryCall,
+		StreamingCall:              s.StreamingCall,
+		UnconstrainedStreamingCall: s.UnconstrainedStreamingCall,
+	}
+}
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	return &testpb.SimpleResponse{
@@ -147,7 +153,13 @@ type byteBufServer struct {
 	respSize int32
 }
 
-var _ testpb.UnstableBenchmarkServiceService = (*byteBufServer)(nil)
+func (s *byteBufServer) Svc() *testpb.BenchmarkServiceService {
+	return &testpb.BenchmarkServiceService{
+		UnaryCall:                  s.UnaryCall,
+		StreamingCall:              s.StreamingCall,
+		UnconstrainedStreamingCall: s.UnconstrainedStreamingCall,
+	}
+}
 
 // UnaryCall is an empty function and is not used for benchmark.
 // If bytebuf UnaryCall benchmark is needed later, the function body needs to be updated.
@@ -212,13 +224,13 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() {
 	s := grpc.NewServer(opts...)
 	switch info.Type {
 	case "protobuf":
-		testpb.RegisterBenchmarkServiceService(s, testpb.NewBenchmarkServiceService(&testServer{}))
+		testpb.RegisterBenchmarkServiceService(s, (&testServer{}).Svc())
 	case "bytebuf":
 		respSize, ok := info.Metadata.(int32)
 		if !ok {
 			logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type)
 		}
-		testpb.RegisterBenchmarkServiceService(s, testpb.NewBenchmarkServiceService(&byteBufServer{respSize: respSize}))
+		testpb.RegisterBenchmarkServiceService(s, (&byteBufServer{respSize: respSize}).Svc())
 	default:
 		logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type)
 	}
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index 37d637d72148..81e9ed6a2ac0 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -253,48 +253,6 @@ func RegisterBenchmarkServiceService(s grpc.ServiceRegistrar, srv *BenchmarkServ
 	s.RegisterService(&sd, nil)
 }
 
-// NewBenchmarkServiceService creates a new BenchmarkServiceService containing the
-// implemented methods of the BenchmarkService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewBenchmarkServiceService(s interface{}) *BenchmarkServiceService {
-	ns := &BenchmarkServiceService{}
-	if h, ok := s.(interface {
-		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	}); ok {
-		ns.UnaryCall = h.UnaryCall
-	}
-	if h, ok := s.(interface {
-		StreamingCall(BenchmarkService_StreamingCallServer) error
-	}); ok {
-		ns.StreamingCall = h.StreamingCall
-	}
-	if h, ok := s.(interface {
-		UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
-	}); ok {
-		ns.UnconstrainedStreamingCall = h.UnconstrainedStreamingCall
-	}
-	return ns
-}
-
-// UnstableBenchmarkServiceService is the service API for BenchmarkService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableBenchmarkServiceService interface {
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	StreamingCall(BenchmarkService_StreamingCallServer) error
-	// Unconstrainted streaming.
-	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
-}
-
 // WorkerServiceClient is the client API for WorkerService service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@@ -591,59 +549,3 @@ func RegisterWorkerServiceService(s grpc.ServiceRegistrar, srv *WorkerServiceSer
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewWorkerServiceService creates a new WorkerServiceService containing the
-// implemented methods of the WorkerService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewWorkerServiceService(s interface{}) *WorkerServiceService {
-	ns := &WorkerServiceService{}
-	if h, ok := s.(interface {
-		RunServer(WorkerService_RunServerServer) error
-	}); ok {
-		ns.RunServer = h.RunServer
-	}
-	if h, ok := s.(interface {
-		RunClient(WorkerService_RunClientServer) error
-	}); ok {
-		ns.RunClient = h.RunClient
-	}
-	if h, ok := s.(interface {
-		CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
-	}); ok {
-		ns.CoreCount = h.CoreCount
-	}
-	if h, ok := s.(interface {
-		QuitWorker(context.Context, *Void) (*Void, error)
-	}); ok {
-		ns.QuitWorker = h.QuitWorker
-	}
-	return ns
-}
-
-// UnstableWorkerServiceService is the service API for WorkerService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableWorkerServiceService interface {
-	// Start server with specified workload.
-	// First request sent specifies the ServerConfig followed by ServerStatus
-	// response. After that, a "Mark" can be sent anytime to request the latest
-	// stats. Closing the stream will initiate shutdown of the test server
-	// and once the shutdown has finished, the OK status is sent to terminate
-	// this RPC.
-	RunServer(WorkerService_RunServerServer) error
-	// Start client with specified workload.
-	// First request sent specifies the ClientConfig followed by ClientStatus
-	// response. After that, a "Mark" can be sent anytime to request the latest
-	// stats. Closing the stream will initiate shutdown of the test client
-	// and once the shutdown has finished, the OK status is sent to terminate
-	// this RPC.
-	RunClient(WorkerService_RunClientServer) error
-	// Just return the core count - unary call
-	CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
-	// Quit this worker
-	QuitWorker(context.Context, *Void) (*Void, error)
-}
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 891cd01fc92b..71a1fb6d294a 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -79,7 +79,14 @@ type workerServer struct {
 	serverPort int
 }
 
-var _ testpb.UnstableWorkerServiceService = (*workerServer)(nil)
+func (s *workerServer) Svc() *testpb.WorkerServiceService {
+	return &testpb.WorkerServiceService{
+		RunServer:  s.RunServer,
+		RunClient:  s.RunClient,
+		CoreCount:  s.CoreCount,
+		QuitWorker: s.QuitWorker,
+	}
+}
 
 func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) error {
 	var bs *benchmarkServer
@@ -210,10 +217,10 @@ func main() {
 
 	s := grpc.NewServer()
 	stop := make(chan bool)
-	testpb.RegisterWorkerServiceService(s, testpb.NewWorkerServiceService(&workerServer{
+	testpb.RegisterWorkerServiceService(s, (&workerServer{
 		stop:       stop,
 		serverPort: *serverPort,
-	}))
+	}).Svc())
 
 	go func() {
 		<-stop
diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go
index 34bfa5ab8f50..9c364e1ebd9c 100644
--- a/channelz/grpc_channelz_v1/channelz.pb.go
+++ b/channelz/grpc_channelz_v1/channelz.pb.go
@@ -4,16 +4,12 @@
 package grpc_channelz_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	any "github.com/golang/protobuf/ptypes/any"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	wrappers "github.com/golang/protobuf/ptypes/wrappers"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -2925,315 +2921,3 @@ var fileDescriptor_6ee37dfd35a8ab00 = []byte{
 	0xd3, 0x77, 0xc6, 0x68, 0xe7, 0x3c, 0xcf, 0x4f, 0xf3, 0x5f, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff,
 	0x54, 0xae, 0x0b, 0x93, 0xdf, 0x1f, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// ChannelzClient is the client API for Channelz service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ChannelzClient interface {
-	// Gets all root channels (i.e. channels the application has directly
-	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error)
-	// Gets all servers that exist in the process.
-	GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error)
-	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error)
-	// Gets all server sockets that exist in the process.
-	GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error)
-	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error)
-	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error)
-	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error)
-}
-
-type channelzClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
-	return &channelzClient{cc}
-}
-
-func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
-	out := new(GetTopChannelsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
-	out := new(GetServersResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
-	out := new(GetServerResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
-	out := new(GetServerSocketsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
-	out := new(GetChannelResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
-	out := new(GetSubchannelResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
-	out := new(GetSocketResponse)
-	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ChannelzServer is the server API for Channelz service.
-type ChannelzServer interface {
-	// Gets all root channels (i.e. channels the application has directly
-	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
-	// Gets all servers that exist in the process.
-	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
-	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
-	// Gets all server sockets that exist in the process.
-	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
-	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
-	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
-	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
-}
-
-// UnimplementedChannelzServer can be embedded to have forward compatible implementations.
-type UnimplementedChannelzServer struct {
-}
-
-func (*UnimplementedChannelzServer) GetTopChannels(ctx context.Context, req *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
-}
-func (*UnimplementedChannelzServer) GetServers(ctx context.Context, req *GetServersRequest) (*GetServersResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
-}
-func (*UnimplementedChannelzServer) GetServer(ctx context.Context, req *GetServerRequest) (*GetServerResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
-}
-func (*UnimplementedChannelzServer) GetServerSockets(ctx context.Context, req *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
-}
-func (*UnimplementedChannelzServer) GetChannel(ctx context.Context, req *GetChannelRequest) (*GetChannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
-}
-func (*UnimplementedChannelzServer) GetSubchannel(ctx context.Context, req *GetSubchannelRequest) (*GetSubchannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
-}
-func (*UnimplementedChannelzServer) GetSocket(ctx context.Context, req *GetSocketRequest) (*GetSocketResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
-}
-
-func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
-	s.RegisterService(&_Channelz_serviceDesc, srv)
-}
-
-func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetTopChannelsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetTopChannels(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetServersRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetServers(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetServers",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetServerRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetServer(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetServer",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetServerSocketsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetServerSockets(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetChannelRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetChannel(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetChannel",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetSubchannelRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetSubchannel(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetSocketRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ChannelzServer).GetSocket(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.channelz.v1.Channelz/GetSocket",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Channelz_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.channelz.v1.Channelz",
-	HandlerType: (*ChannelzServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetTopChannels",
-			Handler:    _Channelz_GetTopChannels_Handler,
-		},
-		{
-			MethodName: "GetServers",
-			Handler:    _Channelz_GetServers_Handler,
-		},
-		{
-			MethodName: "GetServer",
-			Handler:    _Channelz_GetServer_Handler,
-		},
-		{
-			MethodName: "GetServerSockets",
-			Handler:    _Channelz_GetServerSockets_Handler,
-		},
-		{
-			MethodName: "GetChannel",
-			Handler:    _Channelz_GetChannel_Handler,
-		},
-		{
-			MethodName: "GetSubchannel",
-			Handler:    _Channelz_GetSubchannel_Handler,
-		},
-		{
-			MethodName: "GetSocket",
-			Handler:    _Channelz_GetSocket_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "grpc/channelz/v1/channelz.proto",
-}
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index bc6083b8519f..791151fdd7f8 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -13,6 +13,126 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// ChannelzClient is the client API for Channelz service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ChannelzClient interface {
+	// Gets all root channels (i.e. channels the application has directly
+	// created). This does not include subchannels nor non-top level channels.
+	GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error)
+	// Gets all servers that exist in the process.
+	GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error)
+	// Returns a single Server, or else a NOT_FOUND code.
+	GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error)
+	// Gets all server sockets that exist in the process.
+	GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error)
+	// Returns a single Channel, or else a NOT_FOUND code.
+	GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error)
+	// Returns a single Subchannel, or else a NOT_FOUND code.
+	GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error)
+	// Returns a single Socket or else a NOT_FOUND code.
+	GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error)
+}
+
+type channelzClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
+	return &channelzClient{cc}
+}
+
+var channelzGetTopChannelsStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetTopChannels",
+}
+
+func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
+	out := new(GetTopChannelsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var channelzGetServersStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetServers",
+}
+
+func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
+	out := new(GetServersResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var channelzGetServerStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetServer",
+}
+
+func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
+	out := new(GetServerResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var channelzGetServerSocketsStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetServerSockets",
+}
+
+func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
+	out := new(GetServerSocketsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var channelzGetChannelStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetChannel",
+}
+
+func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
+	out := new(GetChannelResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var channelzGetSubchannelStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetSubchannel",
+}
+
+func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
+	out := new(GetSubchannelResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var channelzGetSocketStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetSocket",
+}
+
+func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
+	out := new(GetSocketResponse)
+	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ChannelzService is the service API for Channelz service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterChannelzService is called.  Any unassigned fields will result in the
@@ -232,57 +352,11 @@ func RegisterChannelzService(s grpc.ServiceRegistrar, srv *ChannelzService) {
 	s.RegisterService(&sd, nil)
 }
 
-// NewChannelzService creates a new ChannelzService containing the
-// implemented methods of the Channelz service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewChannelzService(s interface{}) *ChannelzService {
-	ns := &ChannelzService{}
-	if h, ok := s.(interface {
-		GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
-	}); ok {
-		ns.GetTopChannels = h.GetTopChannels
-	}
-	if h, ok := s.(interface {
-		GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
-	}); ok {
-		ns.GetServers = h.GetServers
-	}
-	if h, ok := s.(interface {
-		GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
-	}); ok {
-		ns.GetServer = h.GetServer
-	}
-	if h, ok := s.(interface {
-		GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
-	}); ok {
-		ns.GetServerSockets = h.GetServerSockets
-	}
-	if h, ok := s.(interface {
-		GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
-	}); ok {
-		ns.GetChannel = h.GetChannel
-	}
-	if h, ok := s.(interface {
-		GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
-	}); ok {
-		ns.GetSubchannel = h.GetSubchannel
-	}
-	if h, ok := s.(interface {
-		GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
-	}); ok {
-		ns.GetSocket = h.GetSocket
-	}
-	return ns
-}
-
-// UnstableChannelzService is the service API for Channelz service.
+// ChannelzServer is the service API for Channelz service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableChannelzService interface {
+// use of this type is not recommended unless you own the service definition.
+type ChannelzServer interface {
 	// Gets all root channels (i.e. channels the application has directly
 	// created). This does not include subchannels nor non-top level channels.
 	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
@@ -299,3 +373,44 @@ type UnstableChannelzService interface {
 	// Returns a single Socket or else a NOT_FOUND code.
 	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
 }
+
+// UnimplementedChannelzServer can be embedded to have forward compatible implementations of
+// ChannelzServer
+type UnimplementedChannelzServer struct {
+}
+
+func (*UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
+}
+func (*UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
+}
+func (*UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
+}
+func (*UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
+}
+func (*UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
+}
+func (*UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
+}
+func (*UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
+}
+
+// RegisterChannelzServer registers a service implementation with a gRPC server.
+func RegisterChannelzServer(s grpc.ServiceRegistrar, srv ChannelzServer) {
+	str := &ChannelzService{
+		GetTopChannels:   srv.GetTopChannels,
+		GetServers:       srv.GetServers,
+		GetServer:        srv.GetServer,
+		GetServerSockets: srv.GetServerSockets,
+		GetChannel:       srv.GetChannel,
+		GetSubchannel:    srv.GetSubchannel,
+		GetSocket:        srv.GetSocket,
+	}
+	RegisterChannelzService(s, str)
+}
diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md
index 6d047f1e514c..d5b0a3bbc6a0 100644
--- a/cmd/protoc-gen-go-grpc/README.md
+++ b/cmd/protoc-gen-go-grpc/README.md
@@ -98,7 +98,7 @@ type myEchoService{
 
 // Optional; not recommended: to guarantee myEchoService fully implements
 // EchoService:
-var _ pb.UnstableEchoService = &myEchoService{}
+var _ pb.EchoServer = &myEchoService{}
 
 func main() {
      // ...
@@ -111,7 +111,7 @@ func main() {
 
 
      // Optional: to gracefully detect missing methods:
-     if _, ok := &myEchoService{}.(pb.UnstableEchoService); !ok {
+     if _, ok := &myEchoService{}.(pb.EchoServer); !ok {
         fmt.Println("myEchoService does not implement all methods of EchoService.")
      }
 
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 13bd755e81b8..358bae2fe53b 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -63,14 +63,13 @@ func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.
 	for _, service := range file.Services {
 		genClient(gen, file, g, service)
 		genService(gen, file, g, service)
-		genUnstableServiceInterface(gen, file, g, service)
+		if *genUnstableServerInterfaces {
+			genUnstableServerInterface(gen, file, g, service)
+		}
 	}
 }
 
 func genClient(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
-	if *migrationMode {
-		return
-	}
 	clientName := service.GoName + "Client"
 
 	g.P("// ", clientName, " is the client API for ", service.GoName, " service.")
@@ -258,9 +257,6 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 
 	// Service registration.
 	genRegisterFunction(gen, file, g, service)
-
-	// Short-cut service constructor.
-	genServiceConstructor(gen, g, service)
 }
 
 func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
@@ -324,40 +320,21 @@ func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.
 	g.P()
 }
 
-func genServiceConstructor(gen *protogen.Plugin, g *protogen.GeneratedFile, service *protogen.Service) {
-	g.P("// New", service.GoName, "Service creates a new ", service.GoName, "Service containing the")
-	g.P("// implemented methods of the ", service.GoName, " service in s.  Any unimplemented")
-	g.P("// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.")
-	g.P("// This includes situations where the method handler is misspelled or has the wrong")
-	g.P("// signature.  For this reason, this function should be used with great care and")
-	g.P("// is not recommended to be used by most users.")
-	g.P("func New", service.GoName, "Service(s interface{}) *", service.GoName, "Service {")
-	g.P("ns := &", service.GoName, "Service{}")
-	for _, method := range service.Methods {
-		g.P("if h, ok := s.(interface {", methodSignature(g, method), "}); ok {")
-		g.P("ns.", method.GoName, " = h.", method.GoName)
-		g.P("}")
-	}
-	g.P("return ns")
-	g.P("}")
-	g.P()
-}
-
-func genUnstableServiceInterface(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+func genUnstableServerInterface(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
 	// Service interface.
-	serviceType := service.GoName + "Service"
-	g.P("// Unstable", serviceType, " is the service API for ", service.GoName, " service.")
+	serverType := service.GoName + "Server"
+	g.P("// ", serverType, " is the service API for ", service.GoName, " service.")
 	g.P("// New methods may be added to this interface if they are added to the service")
 	g.P("// definition, which is not a backward-compatible change.  For this reason, ")
-	g.P("// use of this type is not recommended.")
+	g.P("// use of this type is not recommended unless you own the service definition.")
 	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
 		g.P("//")
 		g.P(deprecationComment)
 	}
-	g.Annotate("Unstable"+serviceType, service.Location)
-	g.P("type Unstable", serviceType, " interface {")
+	g.Annotate(serverType, service.Location)
+	g.P("type ", serverType, " interface {")
 	for _, method := range service.Methods {
-		g.Annotate("Unstable"+serviceType+"."+method.GoName, method.Location)
+		g.Annotate(serverType+"."+method.GoName, method.Location)
 		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
 			g.P(deprecationComment)
 		}
@@ -366,6 +343,50 @@ func genUnstableServiceInterface(gen *protogen.Plugin, file *protogen.File, g *p
 	}
 	g.P("}")
 	g.P()
+
+	// Unimplemented implementation.
+	genUnimplementedServer(gen, file, g, service)
+
+	// Service registration.
+	genUnstableRegisterFunction(gen, file, g, service)
+}
+
+func genUnimplementedServer(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	// Server Unimplemented struct for forward compatibility.
+	serverType := service.GoName + "Server"
+	g.P("// Unimplemented", serverType, " can be embedded to have forward compatible implementations of")
+	g.P("// ", serverType)
+	g.P("type Unimplemented", serverType, " struct {")
+	g.P("}")
+	g.P()
+	for _, method := range service.Methods {
+		nilArg := ""
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			nilArg = "nil,"
+		}
+		g.P("func (Unimplemented", serverType, ") ", methodSignature(g, method), "{")
+		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
+		g.P("}")
+	}
+	g.P()
+}
+
+func genUnstableRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+	serverType := service.GoName + "Server"
+	g.P("// Register", serverType, " registers a service implementation with a gRPC server.")
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.P("func Register", serverType, "(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv ", serverType, ") {")
+	g.P("str := &", service.GoName, "Service{")
+	for _, method := range service.Methods {
+		g.P(method.GoName, ": srv.", method.GoName, ",")
+	}
+	g.P("}")
+	g.P("Register", service.GoName, "Service(s, str)")
+	g.P("}")
+	g.P()
 }
 
 func methodSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
@@ -452,9 +473,6 @@ func genMethodHandler(gen *protogen.Plugin, g *protogen.GeneratedFile, method *p
 }
 
 func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
-	if *migrationMode {
-		return
-	}
 	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
 		// Unary method
 		return
diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index 0d7e76c64c4e..65896f13188d 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -37,11 +37,11 @@ import (
 	"google.golang.org/protobuf/types/pluginpb"
 )
 
-var migrationMode *bool
+var genUnstableServerInterfaces *bool
 
 func main() {
 	var flags flag.FlagSet
-	migrationMode = flags.Bool("migration_mode", false, "set to generate new symbols only; requires symbols produced by legacy protoc-gen-go")
+	genUnstableServerInterfaces = flags.Bool("gen_unstable_server_interfaces", false, `set to generate legacy "Server" interfaces which do not guarantee backward compatibility`)
 
 	protogen.Options{
 		ParamFunc: flags.Set,
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
index a2060de402bc..6d9c304e7981 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
@@ -4,12 +4,8 @@
 package grpc_gcp
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -979,127 +975,3 @@ var fileDescriptor_54c074f40c7c7e99 = []byte{
 	0x5f, 0xef, 0xa8, 0xf5, 0x83, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc1, 0xf9, 0x9d, 0xf2, 0xd9,
 	0x0b, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// HandshakerServiceClient is the client API for HandshakerService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type HandshakerServiceClient interface {
-	// Handshaker service accepts a stream of handshaker request, returning a
-	// stream of handshaker response. Client is expected to send exactly one
-	// message with either client_start or server_start followed by one or more
-	// messages with next. Each time client sends a request, the handshaker
-	// service expects to respond. Client does not have to wait for service's
-	// response before sending next request.
-	DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error)
-}
-
-type handshakerServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient {
-	return &handshakerServiceClient{cc}
-}
-
-func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &handshakerServiceDoHandshakeClient{stream}
-	return x, nil
-}
-
-type HandshakerService_DoHandshakeClient interface {
-	Send(*HandshakerReq) error
-	Recv() (*HandshakerResp, error)
-	grpc.ClientStream
-}
-
-type handshakerServiceDoHandshakeClient struct {
-	grpc.ClientStream
-}
-
-func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
-	m := new(HandshakerResp)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// HandshakerServiceServer is the server API for HandshakerService service.
-type HandshakerServiceServer interface {
-	// Handshaker service accepts a stream of handshaker request, returning a
-	// stream of handshaker response. Client is expected to send exactly one
-	// message with either client_start or server_start followed by one or more
-	// messages with next. Each time client sends a request, the handshaker
-	// service expects to respond. Client does not have to wait for service's
-	// response before sending next request.
-	DoHandshake(HandshakerService_DoHandshakeServer) error
-}
-
-// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedHandshakerServiceServer struct {
-}
-
-func (*UnimplementedHandshakerServiceServer) DoHandshake(srv HandshakerService_DoHandshakeServer) error {
-	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
-}
-
-func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
-	s.RegisterService(&_HandshakerService_serviceDesc, srv)
-}
-
-func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream})
-}
-
-type HandshakerService_DoHandshakeServer interface {
-	Send(*HandshakerResp) error
-	Recv() (*HandshakerReq, error)
-	grpc.ServerStream
-}
-
-type handshakerServiceDoHandshakeServer struct {
-	grpc.ServerStream
-}
-
-func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
-	m := new(HandshakerReq)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _HandshakerService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.gcp.HandshakerService",
-	HandlerType: (*HandshakerServiceServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "DoHandshake",
-			Handler:       _HandshakerService_DoHandshake_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "grpc/gcp/handshaker.proto",
-}
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index d6f3af248f08..d2f2971e2b3e 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -3,6 +3,7 @@
 package grpc_gcp
 
 import (
+	context "context"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
@@ -12,6 +13,64 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// HandshakerServiceClient is the client API for HandshakerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type HandshakerServiceClient interface {
+	// Handshaker service accepts a stream of handshaker request, returning a
+	// stream of handshaker response. Client is expected to send exactly one
+	// message with either client_start or server_start followed by one or more
+	// messages with next. Each time client sends a request, the handshaker
+	// service expects to respond. Client does not have to wait for service's
+	// response before sending next request.
+	DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error)
+}
+
+type handshakerServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient {
+	return &handshakerServiceClient{cc}
+}
+
+var handshakerServiceDoHandshakeStreamDesc = &grpc.StreamDesc{
+	StreamName:    "DoHandshake",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
+func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
+	stream, err := c.cc.NewStream(ctx, handshakerServiceDoHandshakeStreamDesc, "/grpc.gcp.HandshakerService/DoHandshake", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &handshakerServiceDoHandshakeClient{stream}
+	return x, nil
+}
+
+type HandshakerService_DoHandshakeClient interface {
+	Send(*HandshakerReq) error
+	Recv() (*HandshakerResp, error)
+	grpc.ClientStream
+}
+
+type handshakerServiceDoHandshakeClient struct {
+	grpc.ClientStream
+}
+
+func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
+	m := new(HandshakerResp)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // HandshakerServiceService is the service API for HandshakerService service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterHandshakerServiceService is called.  Any unassigned fields will result in the
@@ -30,6 +89,28 @@ func (s *HandshakerServiceService) doHandshake(_ interface{}, stream grpc.Server
 	return s.DoHandshake(&handshakerServiceDoHandshakeServer{stream})
 }
 
+type HandshakerService_DoHandshakeServer interface {
+	Send(*HandshakerResp) error
+	Recv() (*HandshakerReq, error)
+	grpc.ServerStream
+}
+
+type handshakerServiceDoHandshakeServer struct {
+	grpc.ServerStream
+}
+
+func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
+	m := new(HandshakerReq)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // RegisterHandshakerServiceService registers a service implementation with a gRPC server.
 func RegisterHandshakerServiceService(s grpc.ServiceRegistrar, srv *HandshakerServiceService) {
 	srvCopy := *srv
@@ -55,27 +136,11 @@ func RegisterHandshakerServiceService(s grpc.ServiceRegistrar, srv *HandshakerSe
 	s.RegisterService(&sd, nil)
 }
 
-// NewHandshakerServiceService creates a new HandshakerServiceService containing the
-// implemented methods of the HandshakerService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewHandshakerServiceService(s interface{}) *HandshakerServiceService {
-	ns := &HandshakerServiceService{}
-	if h, ok := s.(interface {
-		DoHandshake(HandshakerService_DoHandshakeServer) error
-	}); ok {
-		ns.DoHandshake = h.DoHandshake
-	}
-	return ns
-}
-
-// UnstableHandshakerServiceService is the service API for HandshakerService service.
+// HandshakerServiceServer is the service API for HandshakerService service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableHandshakerServiceService interface {
+// use of this type is not recommended unless you own the service definition.
+type HandshakerServiceServer interface {
 	// Handshaker service accepts a stream of handshaker request, returning a
 	// stream of handshaker response. Client is expected to send exactly one
 	// message with either client_start or server_start followed by one or more
@@ -84,3 +149,20 @@ type UnstableHandshakerServiceService interface {
 	// response before sending next request.
 	DoHandshake(HandshakerService_DoHandshakeServer) error
 }
+
+// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations of
+// HandshakerServiceServer
+type UnimplementedHandshakerServiceServer struct {
+}
+
+func (*UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
+	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
+}
+
+// RegisterHandshakerServiceServer registers a service implementation with a gRPC server.
+func RegisterHandshakerServiceServer(s grpc.ServiceRegistrar, srv HandshakerServiceServer) {
+	str := &HandshakerServiceService{
+		DoHandshake: srv.DoHandshake,
+	}
+	RegisterHandshakerServiceService(s, str)
+}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
index b09de93beac6..16122f768488 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
@@ -4,13 +4,9 @@
 package google_security_meshca_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -157,87 +153,3 @@ var fileDescriptor_f72841047b94fe5e = []byte{
 	0x75, 0x36, 0xec, 0xba, 0x62, 0xee, 0x66, 0x99, 0x93, 0xe5, 0x45, 0xb7, 0xcf, 0xc3, 0x77, 0x00,
 	0x00, 0x00, 0xff, 0xff, 0xb7, 0x0d, 0xfd, 0xff, 0xf7, 0x01, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// MeshCertificateServiceClient is the client API for MeshCertificateService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type MeshCertificateServiceClient interface {
-	// Using provided CSR, returns a signed certificate that represents a GCP
-	// service account identity.
-	CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error)
-}
-
-type meshCertificateServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient {
-	return &meshCertificateServiceClient{cc}
-}
-
-func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
-	out := new(MeshCertificateResponse)
-	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// MeshCertificateServiceServer is the server API for MeshCertificateService service.
-type MeshCertificateServiceServer interface {
-	// Using provided CSR, returns a signed certificate that represents a GCP
-	// service account identity.
-	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
-}
-
-// UnimplementedMeshCertificateServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedMeshCertificateServiceServer struct {
-}
-
-func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(ctx context.Context, req *MeshCertificateRequest) (*MeshCertificateResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
-}
-
-func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) {
-	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
-}
-
-func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(MeshCertificateRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "google.security.meshca.v1.MeshCertificateService",
-	HandlerType: (*MeshCertificateServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "CreateCertificate",
-			Handler:    _MeshCertificateService_CreateCertificate_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "istio/google/security/meshca/v1/meshca.proto",
-}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index 3a82e6a31131..4dbb2ac05454 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -13,6 +13,36 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// MeshCertificateServiceClient is the client API for MeshCertificateService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type MeshCertificateServiceClient interface {
+	// Using provided CSR, returns a signed certificate that represents a GCP
+	// service account identity.
+	CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error)
+}
+
+type meshCertificateServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient {
+	return &meshCertificateServiceClient{cc}
+}
+
+var meshCertificateServiceCreateCertificateStreamDesc = &grpc.StreamDesc{
+	StreamName: "CreateCertificate",
+}
+
+func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
+	out := new(MeshCertificateResponse)
+	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // MeshCertificateServiceService is the service API for MeshCertificateService service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterMeshCertificateServiceService is called.  Any unassigned fields will result in the
@@ -64,28 +94,29 @@ func RegisterMeshCertificateServiceService(s grpc.ServiceRegistrar, srv *MeshCer
 	s.RegisterService(&sd, nil)
 }
 
-// NewMeshCertificateServiceService creates a new MeshCertificateServiceService containing the
-// implemented methods of the MeshCertificateService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewMeshCertificateServiceService(s interface{}) *MeshCertificateServiceService {
-	ns := &MeshCertificateServiceService{}
-	if h, ok := s.(interface {
-		CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
-	}); ok {
-		ns.CreateCertificate = h.CreateCertificate
-	}
-	return ns
-}
-
-// UnstableMeshCertificateServiceService is the service API for MeshCertificateService service.
+// MeshCertificateServiceServer is the service API for MeshCertificateService service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableMeshCertificateServiceService interface {
+// use of this type is not recommended unless you own the service definition.
+type MeshCertificateServiceServer interface {
 	// Using provided CSR, returns a signed certificate that represents a GCP
 	// service account identity.
 	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
 }
+
+// UnimplementedMeshCertificateServiceServer can be embedded to have forward compatible implementations of
+// MeshCertificateServiceServer
+type UnimplementedMeshCertificateServiceServer struct {
+}
+
+func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
+}
+
+// RegisterMeshCertificateServiceServer registers a service implementation with a gRPC server.
+func RegisterMeshCertificateServiceServer(s grpc.ServiceRegistrar, srv MeshCertificateServiceServer) {
+	str := &MeshCertificateServiceService{
+		CreateCertificate: srv.CreateCertificate,
+	}
+	RegisterMeshCertificateServiceService(s, str)
+}
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index d1d6cf07eb24..ff30c6524b4a 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -318,49 +318,3 @@ func RegisterEchoService(s grpc.ServiceRegistrar, srv *EchoService) {
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewEchoService creates a new EchoService containing the
-// implemented methods of the Echo service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewEchoService(s interface{}) *EchoService {
-	ns := &EchoService{}
-	if h, ok := s.(interface {
-		UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
-	}); ok {
-		ns.UnaryEcho = h.UnaryEcho
-	}
-	if h, ok := s.(interface {
-		ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
-	}); ok {
-		ns.ServerStreamingEcho = h.ServerStreamingEcho
-	}
-	if h, ok := s.(interface {
-		ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
-	}); ok {
-		ns.ClientStreamingEcho = h.ClientStreamingEcho
-	}
-	if h, ok := s.(interface {
-		BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
-	}); ok {
-		ns.BidirectionalStreamingEcho = h.BidirectionalStreamingEcho
-	}
-	return ns
-}
-
-// UnstableEchoService is the service API for Echo service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableEchoService interface {
-	// UnaryEcho is unary echo.
-	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
-	// ServerStreamingEcho is server side streaming.
-	ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
-	// ClientStreamingEcho is client side streaming.
-	ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
-	// BidirectionalStreamingEcho is bidi streaming.
-	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
-}
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 3a88bb1f2f3f..78d60253a6d8 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -91,28 +91,3 @@ func RegisterGreeterService(s grpc.ServiceRegistrar, srv *GreeterService) {
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewGreeterService creates a new GreeterService containing the
-// implemented methods of the Greeter service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewGreeterService(s interface{}) *GreeterService {
-	ns := &GreeterService{}
-	if h, ok := s.(interface {
-		SayHello(context.Context, *HelloRequest) (*HelloReply, error)
-	}); ok {
-		ns.SayHello = h.SayHello
-	}
-	return ns
-}
-
-// UnstableGreeterService is the service API for Greeter service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableGreeterService interface {
-	// Sends a greeting
-	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
-}
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index eb406ec838cc..bbcfe7daea35 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -350,65 +350,3 @@ func RegisterRouteGuideService(s grpc.ServiceRegistrar, srv *RouteGuideService)
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewRouteGuideService creates a new RouteGuideService containing the
-// implemented methods of the RouteGuide service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewRouteGuideService(s interface{}) *RouteGuideService {
-	ns := &RouteGuideService{}
-	if h, ok := s.(interface {
-		GetFeature(context.Context, *Point) (*Feature, error)
-	}); ok {
-		ns.GetFeature = h.GetFeature
-	}
-	if h, ok := s.(interface {
-		ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
-	}); ok {
-		ns.ListFeatures = h.ListFeatures
-	}
-	if h, ok := s.(interface {
-		RecordRoute(RouteGuide_RecordRouteServer) error
-	}); ok {
-		ns.RecordRoute = h.RecordRoute
-	}
-	if h, ok := s.(interface {
-		RouteChat(RouteGuide_RouteChatServer) error
-	}); ok {
-		ns.RouteChat = h.RouteChat
-	}
-	return ns
-}
-
-// UnstableRouteGuideService is the service API for RouteGuide service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableRouteGuideService interface {
-	// A simple RPC.
-	//
-	// Obtains the feature at a given position.
-	//
-	// A feature with an empty name is returned if there's no feature at the given
-	// position.
-	GetFeature(context.Context, *Point) (*Feature, error)
-	// A server-to-client streaming RPC.
-	//
-	// Obtains the Features available within the given Rectangle.  Results are
-	// streamed rather than returned at once (e.g. in a response message with a
-	// repeated field), as the rectangle may cover a large area and contain a
-	// huge number of features.
-	ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
-	// A client-to-server streaming RPC.
-	//
-	// Accepts a stream of Points on a route being traversed, returning a
-	// RouteSummary when traversal is completed.
-	RecordRoute(RouteGuide_RecordRouteServer) error
-	// A Bidirectional streaming RPC.
-	//
-	// Accepts a stream of RouteNotes sent while a route is being traversed,
-	// while receiving other RouteNotes (e.g. from other users).
-	RouteChat(RouteGuide_RouteChatServer) error
-}
diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go
index 7c959dde06b1..3c0775e76f50 100644
--- a/examples/route_guide/server/server.go
+++ b/examples/route_guide/server/server.go
@@ -60,6 +60,15 @@ type routeGuideServer struct {
 	routeNotes map[string][]*pb.RouteNote
 }
 
+func (s *routeGuideServer) Svc() *pb.RouteGuideService {
+	return &pb.RouteGuideService{
+		GetFeature:   s.GetFeature,
+		ListFeatures: s.ListFeatures,
+		RecordRoute:  s.RecordRoute,
+		RouteChat:    s.RouteChat,
+	}
+}
+
 // GetFeature returns the feature at the given point.
 func (s *routeGuideServer) GetFeature(ctx context.Context, point *pb.Point) (*pb.Feature, error) {
 	for _, feature := range s.savedFeatures {
@@ -237,7 +246,7 @@ func main() {
 		opts = []grpc.ServerOption{grpc.Creds(creds)}
 	}
 	grpcServer := grpc.NewServer(opts...)
-	pb.RegisterRouteGuideService(grpcServer, pb.NewRouteGuideService(newServer()))
+	pb.RegisterRouteGuideService(grpcServer, newServer().Svc())
 	grpcServer.Serve(lis)
 }
 
diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go
index 4c2a527ec597..e9919c0073b6 100644
--- a/health/grpc_health_v1/health.pb.go
+++ b/health/grpc_health_v1/health.pb.go
@@ -4,12 +4,8 @@
 package grpc_health_v1
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -163,181 +159,3 @@ var fileDescriptor_e265fd9d4e077217 = []byte{
 	0xd3, 0x20, 0x46, 0xe8, 0x85, 0x19, 0x26, 0xb1, 0x81, 0x93, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff,
 	0xff, 0x12, 0x7d, 0x96, 0xcb, 0x2d, 0x02, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// HealthClient is the client API for Health service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type HealthClient interface {
-	// If the requested service is unknown, the call will fail with status
-	// NOT_FOUND.
-	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
-	// Performs a watch for the serving status of the requested service.
-	// The server will immediately send back a message indicating the current
-	// serving status.  It will then subsequently send a new message whenever
-	// the service's serving status changes.
-	//
-	// If the requested service is unknown when the call is received, the
-	// server will send a message setting the serving status to
-	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
-	// future point, the serving status of the service becomes known, the
-	// server will send a new message with the service's serving status.
-	//
-	// If the call terminates with status UNIMPLEMENTED, then clients
-	// should assume this method is not supported and should not retry the
-	// call.  If the call terminates with any other status (including OK),
-	// clients should retry the call with appropriate exponential backoff.
-	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
-}
-
-type healthClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
-	return &healthClient{cc}
-}
-
-func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
-	out := new(HealthCheckResponse)
-	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &healthWatchClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type Health_WatchClient interface {
-	Recv() (*HealthCheckResponse, error)
-	grpc.ClientStream
-}
-
-type healthWatchClient struct {
-	grpc.ClientStream
-}
-
-func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
-	m := new(HealthCheckResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// HealthServer is the server API for Health service.
-type HealthServer interface {
-	// If the requested service is unknown, the call will fail with status
-	// NOT_FOUND.
-	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
-	// Performs a watch for the serving status of the requested service.
-	// The server will immediately send back a message indicating the current
-	// serving status.  It will then subsequently send a new message whenever
-	// the service's serving status changes.
-	//
-	// If the requested service is unknown when the call is received, the
-	// server will send a message setting the serving status to
-	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
-	// future point, the serving status of the service becomes known, the
-	// server will send a new message with the service's serving status.
-	//
-	// If the call terminates with status UNIMPLEMENTED, then clients
-	// should assume this method is not supported and should not retry the
-	// call.  If the call terminates with any other status (including OK),
-	// clients should retry the call with appropriate exponential backoff.
-	Watch(*HealthCheckRequest, Health_WatchServer) error
-}
-
-// UnimplementedHealthServer can be embedded to have forward compatible implementations.
-type UnimplementedHealthServer struct {
-}
-
-func (*UnimplementedHealthServer) Check(ctx context.Context, req *HealthCheckRequest) (*HealthCheckResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
-}
-func (*UnimplementedHealthServer) Watch(req *HealthCheckRequest, srv Health_WatchServer) error {
-	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
-}
-
-func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
-	s.RegisterService(&_Health_serviceDesc, srv)
-}
-
-func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(HealthCheckRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(HealthServer).Check(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.health.v1.Health/Check",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(HealthCheckRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
-}
-
-type Health_WatchServer interface {
-	Send(*HealthCheckResponse) error
-	grpc.ServerStream
-}
-
-type healthWatchServer struct {
-	grpc.ServerStream
-}
-
-func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-var _Health_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.health.v1.Health",
-	HandlerType: (*HealthServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Check",
-			Handler:    _Health_Check_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "Watch",
-			Handler:       _Health_Watch_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "grpc/health/v1/health.proto",
-}
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index f848b4188b28..fa3a6a775749 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -13,6 +13,89 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// HealthClient is the client API for Health service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type HealthClient interface {
+	// If the requested service is unknown, the call will fail with status
+	// NOT_FOUND.
+	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
+	// Performs a watch for the serving status of the requested service.
+	// The server will immediately send back a message indicating the current
+	// serving status.  It will then subsequently send a new message whenever
+	// the service's serving status changes.
+	//
+	// If the requested service is unknown when the call is received, the
+	// server will send a message setting the serving status to
+	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
+	// future point, the serving status of the service becomes known, the
+	// server will send a new message with the service's serving status.
+	//
+	// If the call terminates with status UNIMPLEMENTED, then clients
+	// should assume this method is not supported and should not retry the
+	// call.  If the call terminates with any other status (including OK),
+	// clients should retry the call with appropriate exponential backoff.
+	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
+}
+
+type healthClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
+	return &healthClient{cc}
+}
+
+var healthCheckStreamDesc = &grpc.StreamDesc{
+	StreamName: "Check",
+}
+
+func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
+	out := new(HealthCheckResponse)
+	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var healthWatchStreamDesc = &grpc.StreamDesc{
+	StreamName:    "Watch",
+	ServerStreams: true,
+}
+
+func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
+	stream, err := c.cc.NewStream(ctx, healthWatchStreamDesc, "/grpc.health.v1.Health/Watch", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &healthWatchClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Health_WatchClient interface {
+	Recv() (*HealthCheckResponse, error)
+	grpc.ClientStream
+}
+
+type healthWatchClient struct {
+	grpc.ClientStream
+}
+
+func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
+	m := new(HealthCheckResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // HealthService is the service API for Health service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterHealthService is called.  Any unassigned fields will result in the
@@ -64,6 +147,19 @@ func (s *HealthService) watch(_ interface{}, stream grpc.ServerStream) error {
 	return s.Watch(m, &healthWatchServer{stream})
 }
 
+type Health_WatchServer interface {
+	Send(*HealthCheckResponse) error
+	grpc.ServerStream
+}
+
+type healthWatchServer struct {
+	grpc.ServerStream
+}
+
+func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
 // RegisterHealthService registers a service implementation with a gRPC server.
 func RegisterHealthService(s grpc.ServiceRegistrar, srv *HealthService) {
 	srvCopy := *srv
@@ -98,32 +194,11 @@ func RegisterHealthService(s grpc.ServiceRegistrar, srv *HealthService) {
 	s.RegisterService(&sd, nil)
 }
 
-// NewHealthService creates a new HealthService containing the
-// implemented methods of the Health service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewHealthService(s interface{}) *HealthService {
-	ns := &HealthService{}
-	if h, ok := s.(interface {
-		Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
-	}); ok {
-		ns.Check = h.Check
-	}
-	if h, ok := s.(interface {
-		Watch(*HealthCheckRequest, Health_WatchServer) error
-	}); ok {
-		ns.Watch = h.Watch
-	}
-	return ns
-}
-
-// UnstableHealthService is the service API for Health service.
+// HealthServer is the service API for Health service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableHealthService interface {
+// use of this type is not recommended unless you own the service definition.
+type HealthServer interface {
 	// If the requested service is unknown, the call will fail with status
 	// NOT_FOUND.
 	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
@@ -144,3 +219,24 @@ type UnstableHealthService interface {
 	// clients should retry the call with appropriate exponential backoff.
 	Watch(*HealthCheckRequest, Health_WatchServer) error
 }
+
+// UnimplementedHealthServer can be embedded to have forward compatible implementations of
+// HealthServer
+type UnimplementedHealthServer struct {
+}
+
+func (*UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+}
+func (*UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
+	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+}
+
+// RegisterHealthServer registers a service implementation with a gRPC server.
+func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) {
+	str := &HealthService{
+		Check: srv.Check,
+		Watch: srv.Watch,
+	}
+	RegisterHealthService(s, str)
+}
diff --git a/internal/binarylog/binarylog_end2end_test.go b/internal/binarylog/binarylog_end2end_test.go
index bdce754a57d5..54dd3da74196 100644
--- a/internal/binarylog/binarylog_end2end_test.go
+++ b/internal/binarylog/binarylog_end2end_test.go
@@ -118,7 +118,14 @@ type testServer struct {
 	te *test
 }
 
-var _ testpb.UnstableTestServiceService = (*testServer)(nil)
+func (s *testServer) Svc() *testpb.TestServiceService {
+	return &testpb.TestServiceService{
+		UnaryCall:        s.UnaryCall,
+		FullDuplexCall:   s.FullDuplexCall,
+		ClientStreamCall: s.ClientStreamCall,
+		ServerStreamCall: s.ServerStreamCall,
+	}
+}
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	md, ok := metadata.FromIncomingContext(ctx)
@@ -784,7 +791,7 @@ func (ed *expectedData) toServerLogEntries() []*pb.GrpcLogEntry {
 
 func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData {
 	te := newTest(t, tc)
-	te.startServer(testpb.NewTestServiceService(&testServer{te: te}))
+	te.startServer((&testServer{te: te}).Svc())
 	defer te.tearDown()
 
 	expect := &expectedData{
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index 9496348fbe4d..e158bb4b20dd 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -455,74 +455,6 @@ func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService
 	s.RegisterService(&sd, nil)
 }
 
-// NewTestServiceService creates a new TestServiceService containing the
-// implemented methods of the TestService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewTestServiceService(s interface{}) *TestServiceService {
-	ns := &TestServiceService{}
-	if h, ok := s.(interface {
-		EmptyCall(context.Context, *Empty) (*Empty, error)
-	}); ok {
-		ns.EmptyCall = h.EmptyCall
-	}
-	if h, ok := s.(interface {
-		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	}); ok {
-		ns.UnaryCall = h.UnaryCall
-	}
-	if h, ok := s.(interface {
-		StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
-	}); ok {
-		ns.StreamingOutputCall = h.StreamingOutputCall
-	}
-	if h, ok := s.(interface {
-		StreamingInputCall(TestService_StreamingInputCallServer) error
-	}); ok {
-		ns.StreamingInputCall = h.StreamingInputCall
-	}
-	if h, ok := s.(interface {
-		FullDuplexCall(TestService_FullDuplexCallServer) error
-	}); ok {
-		ns.FullDuplexCall = h.FullDuplexCall
-	}
-	if h, ok := s.(interface {
-		HalfDuplexCall(TestService_HalfDuplexCallServer) error
-	}); ok {
-		ns.HalfDuplexCall = h.HalfDuplexCall
-	}
-	return ns
-}
-
-// UnstableTestServiceService is the service API for TestService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableTestServiceService interface {
-	// One empty request followed by one empty response.
-	EmptyCall(context.Context, *Empty) (*Empty, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by a sequence of responses (streamed download).
-	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
-	// A sequence of requests followed by one response (streamed upload).
-	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(TestService_StreamingInputCallServer) error
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// A sequence of requests followed by a sequence of responses.
-	// The server buffers all the client requests and then serves them in order. A
-	// stream of responses are returned to the client when the server starts with
-	// first request.
-	HalfDuplexCall(TestService_HalfDuplexCallServer) error
-}
-
 // UnimplementedServiceClient is the client API for UnimplementedService service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@@ -602,31 +534,6 @@ func RegisterUnimplementedServiceService(s grpc.ServiceRegistrar, srv *Unimpleme
 	s.RegisterService(&sd, nil)
 }
 
-// NewUnimplementedServiceService creates a new UnimplementedServiceService containing the
-// implemented methods of the UnimplementedService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewUnimplementedServiceService(s interface{}) *UnimplementedServiceService {
-	ns := &UnimplementedServiceService{}
-	if h, ok := s.(interface {
-		UnimplementedCall(context.Context, *Empty) (*Empty, error)
-	}); ok {
-		ns.UnimplementedCall = h.UnimplementedCall
-	}
-	return ns
-}
-
-// UnstableUnimplementedServiceService is the service API for UnimplementedService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableUnimplementedServiceService interface {
-	// A call that no server should implement
-	UnimplementedCall(context.Context, *Empty) (*Empty, error)
-}
-
 // LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@@ -705,28 +612,3 @@ func RegisterLoadBalancerStatsServiceService(s grpc.ServiceRegistrar, srv *LoadB
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewLoadBalancerStatsServiceService creates a new LoadBalancerStatsServiceService containing the
-// implemented methods of the LoadBalancerStatsService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewLoadBalancerStatsServiceService(s interface{}) *LoadBalancerStatsServiceService {
-	ns := &LoadBalancerStatsServiceService{}
-	if h, ok := s.(interface {
-		GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
-	}); ok {
-		ns.GetClientStats = h.GetClientStats
-	}
-	return ns
-}
-
-// UnstableLoadBalancerStatsServiceService is the service API for LoadBalancerStatsService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableLoadBalancerStatsServiceService interface {
-	// Gets the backend distribution for RPCs sent by a test client.
-	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
-}
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 7a42116e74b3..dd2a73155e6a 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -677,7 +677,15 @@ type testServer struct{}
 
 // NewTestServer creates a test server for test service.
 func NewTestServer() *testpb.TestServiceService {
-	return testpb.NewTestServiceService(testpb.UnstableTestServiceService(&testServer{}))
+	s := testServer{}
+	return &testpb.TestServiceService{
+		EmptyCall:           s.EmptyCall,
+		UnaryCall:           s.UnaryCall,
+		StreamingOutputCall: s.StreamingOutputCall,
+		StreamingInputCall:  s.StreamingInputCall,
+		FullDuplexCall:      s.FullDuplexCall,
+		HalfDuplexCall:      s.HalfDuplexCall,
+	}
 }
 
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go
index 90f02824ef69..831a62721383 100644
--- a/profiling/proto/service.pb.go
+++ b/profiling/proto/service.pb.go
@@ -4,12 +4,8 @@
 package proto
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -366,125 +362,3 @@ var fileDescriptor_e1ab2aa17b47c6fb = []byte{
 	0x95, 0xbb, 0x2e, 0xf9, 0xc9, 0xfd, 0xaf, 0x42, 0xfa, 0xbc, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff,
 	0x5d, 0x47, 0x09, 0xa9, 0x19, 0x03, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// ProfilingClient is the client API for Profiling service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ProfilingClient interface {
-	// Enable allows users to toggle profiling on and off remotely.
-	Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error)
-	// GetStreamStats is used to retrieve an array of stream-level stats from a
-	// gRPC client/server.
-	GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error)
-}
-
-type profilingClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
-	return &profilingClient{cc}
-}
-
-func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
-	out := new(EnableResponse)
-	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
-	out := new(GetStreamStatsResponse)
-	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ProfilingServer is the server API for Profiling service.
-type ProfilingServer interface {
-	// Enable allows users to toggle profiling on and off remotely.
-	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
-	// GetStreamStats is used to retrieve an array of stream-level stats from a
-	// gRPC client/server.
-	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
-}
-
-// UnimplementedProfilingServer can be embedded to have forward compatible implementations.
-type UnimplementedProfilingServer struct {
-}
-
-func (*UnimplementedProfilingServer) Enable(ctx context.Context, req *EnableRequest) (*EnableResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
-}
-func (*UnimplementedProfilingServer) GetStreamStats(ctx context.Context, req *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
-}
-
-func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
-	s.RegisterService(&_Profiling_serviceDesc, srv)
-}
-
-func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(EnableRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ProfilingServer).Enable(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetStreamStatsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ProfilingServer).GetStreamStats(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Profiling_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
-	HandlerType: (*ProfilingServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Enable",
-			Handler:    _Profiling_Enable_Handler,
-		},
-		{
-			MethodName: "GetStreamStats",
-			Handler:    _Profiling_GetStreamStats_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "profiling/proto/service.proto",
-}
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index 41450878f8b5..02783c9b864c 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -13,6 +13,51 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// ProfilingClient is the client API for Profiling service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ProfilingClient interface {
+	// Enable allows users to toggle profiling on and off remotely.
+	Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error)
+	// GetStreamStats is used to retrieve an array of stream-level stats from a
+	// gRPC client/server.
+	GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error)
+}
+
+type profilingClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
+	return &profilingClient{cc}
+}
+
+var profilingEnableStreamDesc = &grpc.StreamDesc{
+	StreamName: "Enable",
+}
+
+func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
+	out := new(EnableResponse)
+	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+var profilingGetStreamStatsStreamDesc = &grpc.StreamDesc{
+	StreamName: "GetStreamStats",
+}
+
+func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
+	out := new(GetStreamStatsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ProfilingService is the service API for Profiling service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterProfilingService is called.  Any unassigned fields will result in the
@@ -92,35 +137,35 @@ func RegisterProfilingService(s grpc.ServiceRegistrar, srv *ProfilingService) {
 	s.RegisterService(&sd, nil)
 }
 
-// NewProfilingService creates a new ProfilingService containing the
-// implemented methods of the Profiling service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewProfilingService(s interface{}) *ProfilingService {
-	ns := &ProfilingService{}
-	if h, ok := s.(interface {
-		Enable(context.Context, *EnableRequest) (*EnableResponse, error)
-	}); ok {
-		ns.Enable = h.Enable
-	}
-	if h, ok := s.(interface {
-		GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
-	}); ok {
-		ns.GetStreamStats = h.GetStreamStats
-	}
-	return ns
-}
-
-// UnstableProfilingService is the service API for Profiling service.
+// ProfilingServer is the service API for Profiling service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableProfilingService interface {
+// use of this type is not recommended unless you own the service definition.
+type ProfilingServer interface {
 	// Enable allows users to toggle profiling on and off remotely.
 	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
 	// GetStreamStats is used to retrieve an array of stream-level stats from a
 	// gRPC client/server.
 	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
 }
+
+// UnimplementedProfilingServer can be embedded to have forward compatible implementations of
+// ProfilingServer
+type UnimplementedProfilingServer struct {
+}
+
+func (*UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
+}
+func (*UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
+}
+
+// RegisterProfilingServer registers a service implementation with a gRPC server.
+func RegisterProfilingServer(s grpc.ServiceRegistrar, srv ProfilingServer) {
+	str := &ProfilingService{
+		Enable:         srv.Enable,
+		GetStreamStats: srv.GetStreamStats,
+	}
+	RegisterProfilingService(s, str)
+}
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 900bd6c05c78..382612d520dd 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -4,12 +4,8 @@
 package grpc_reflection_v1alpha
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -636,119 +632,3 @@ var fileDescriptor_e8cf9f2921ad6c95 = []byte{
 	0xcb, 0xb3, 0xdb, 0x8c, 0xdb, 0xea, 0x53, 0xd5, 0xb9, 0xfd, 0xd3, 0x35, 0xdc, 0x54, 0xbe, 0x39,
 	0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x74, 0x3a, 0x67, 0xe7, 0x06, 0x00, 0x00,
 }
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConnInterface
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion6
-
-// ServerReflectionClient is the client API for ServerReflection service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ServerReflectionClient interface {
-	// The reflection service is structured as a bidirectional stream, ensuring
-	// all related requests go to a single server.
-	ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error)
-}
-
-type serverReflectionClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient {
-	return &serverReflectionClient{cc}
-}
-
-func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &serverReflectionServerReflectionInfoClient{stream}
-	return x, nil
-}
-
-type ServerReflection_ServerReflectionInfoClient interface {
-	Send(*ServerReflectionRequest) error
-	Recv() (*ServerReflectionResponse, error)
-	grpc.ClientStream
-}
-
-type serverReflectionServerReflectionInfoClient struct {
-	grpc.ClientStream
-}
-
-func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) {
-	m := new(ServerReflectionResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// ServerReflectionServer is the server API for ServerReflection service.
-type ServerReflectionServer interface {
-	// The reflection service is structured as a bidirectional stream, ensuring
-	// all related requests go to a single server.
-	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
-}
-
-// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations.
-type UnimplementedServerReflectionServer struct {
-}
-
-func (*UnimplementedServerReflectionServer) ServerReflectionInfo(srv ServerReflection_ServerReflectionInfoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
-}
-
-func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
-	s.RegisterService(&_ServerReflection_serviceDesc, srv)
-}
-
-func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
-}
-
-type ServerReflection_ServerReflectionInfoServer interface {
-	Send(*ServerReflectionResponse) error
-	Recv() (*ServerReflectionRequest, error)
-	grpc.ServerStream
-}
-
-type serverReflectionServerReflectionInfoServer struct {
-	grpc.ServerStream
-}
-
-func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) {
-	m := new(ServerReflectionRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _ServerReflection_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
-	HandlerType: (*ServerReflectionServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "ServerReflectionInfo",
-			Handler:       _ServerReflection_ServerReflectionInfo_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
-}
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index 55241526bf80..eface9e3262b 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -3,6 +3,7 @@
 package grpc_reflection_v1alpha
 
 import (
+	context "context"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
@@ -12,6 +13,60 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
+// ServerReflectionClient is the client API for ServerReflection service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ServerReflectionClient interface {
+	// The reflection service is structured as a bidirectional stream, ensuring
+	// all related requests go to a single server.
+	ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error)
+}
+
+type serverReflectionClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient {
+	return &serverReflectionClient{cc}
+}
+
+var serverReflectionServerReflectionInfoStreamDesc = &grpc.StreamDesc{
+	StreamName:    "ServerReflectionInfo",
+	ServerStreams: true,
+	ClientStreams: true,
+}
+
+func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
+	stream, err := c.cc.NewStream(ctx, serverReflectionServerReflectionInfoStreamDesc, "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &serverReflectionServerReflectionInfoClient{stream}
+	return x, nil
+}
+
+type ServerReflection_ServerReflectionInfoClient interface {
+	Send(*ServerReflectionRequest) error
+	Recv() (*ServerReflectionResponse, error)
+	grpc.ClientStream
+}
+
+type serverReflectionServerReflectionInfoClient struct {
+	grpc.ClientStream
+}
+
+func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) {
+	m := new(ServerReflectionResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // ServerReflectionService is the service API for ServerReflection service.
 // Fields should be assigned to their respective handler implementations only before
 // RegisterServerReflectionService is called.  Any unassigned fields will result in the
@@ -26,6 +81,28 @@ func (s *ServerReflectionService) serverReflectionInfo(_ interface{}, stream grp
 	return s.ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
 }
 
+type ServerReflection_ServerReflectionInfoServer interface {
+	Send(*ServerReflectionResponse) error
+	Recv() (*ServerReflectionRequest, error)
+	grpc.ServerStream
+}
+
+type serverReflectionServerReflectionInfoServer struct {
+	grpc.ServerStream
+}
+
+func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) {
+	m := new(ServerReflectionRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // RegisterServerReflectionService registers a service implementation with a gRPC server.
 func RegisterServerReflectionService(s grpc.ServiceRegistrar, srv *ServerReflectionService) {
 	srvCopy := *srv
@@ -51,28 +128,29 @@ func RegisterServerReflectionService(s grpc.ServiceRegistrar, srv *ServerReflect
 	s.RegisterService(&sd, nil)
 }
 
-// NewServerReflectionService creates a new ServerReflectionService containing the
-// implemented methods of the ServerReflection service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewServerReflectionService(s interface{}) *ServerReflectionService {
-	ns := &ServerReflectionService{}
-	if h, ok := s.(interface {
-		ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
-	}); ok {
-		ns.ServerReflectionInfo = h.ServerReflectionInfo
-	}
-	return ns
-}
-
-// UnstableServerReflectionService is the service API for ServerReflection service.
+// ServerReflectionServer is the service API for ServerReflection service.
 // New methods may be added to this interface if they are added to the service
 // definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableServerReflectionService interface {
+// use of this type is not recommended unless you own the service definition.
+type ServerReflectionServer interface {
 	// The reflection service is structured as a bidirectional stream, ensuring
 	// all related requests go to a single server.
 	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
 }
+
+// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations of
+// ServerReflectionServer
+type UnimplementedServerReflectionServer struct {
+}
+
+func (*UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
+}
+
+// RegisterServerReflectionServer registers a service implementation with a gRPC server.
+func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) {
+	str := &ServerReflectionService{
+		ServerReflectionInfo: srv.ServerReflectionInfo,
+	}
+	RegisterServerReflectionService(s, str)
+}
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 2749e160f4e0..503c8569368f 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -165,33 +165,3 @@ func RegisterSearchServiceService(s grpc.ServiceRegistrar, srv *SearchServiceSer
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewSearchServiceService creates a new SearchServiceService containing the
-// implemented methods of the SearchService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewSearchServiceService(s interface{}) *SearchServiceService {
-	ns := &SearchServiceService{}
-	if h, ok := s.(interface {
-		Search(context.Context, *SearchRequest) (*SearchResponse, error)
-	}); ok {
-		ns.Search = h.Search
-	}
-	if h, ok := s.(interface {
-		StreamingSearch(SearchService_StreamingSearchServer) error
-	}); ok {
-		ns.StreamingSearch = h.StreamingSearch
-	}
-	return ns
-}
-
-// UnstableSearchServiceService is the service API for SearchService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableSearchServiceService interface {
-	Search(context.Context, *SearchRequest) (*SearchResponse, error)
-	StreamingSearch(SearchService_StreamingSearchServer) error
-}
diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go
index 9f252d778b94..95b8269b4bd0 100644
--- a/reflection/serverreflection_test.go
+++ b/reflection/serverreflection_test.go
@@ -168,7 +168,12 @@ func (x) TestAllExtensionNumbersForType(t *testing.T) {
 
 type server struct{}
 
-var _ pb.UnstableSearchServiceService = (*server)(nil)
+func (s *server) Svc() *pb.SearchServiceService {
+	return &pb.SearchServiceService{
+		Search:          s.Search,
+		StreamingSearch: s.StreamingSearch,
+	}
+}
 
 func (s *server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.SearchResponse, error) {
 	return &pb.SearchResponse{}, nil
@@ -195,7 +200,7 @@ func (x) TestReflectionEnd2end(t *testing.T) {
 		t.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterSearchServiceService(s, pb.NewSearchServiceService(&server{}))
+	pb.RegisterSearchServiceService(s, (&server{}).Svc())
 	pbv3.RegisterSearchServiceV3Server(s, &serverV3{})
 	// Register reflection service on s.
 	Register(s)
diff --git a/regenerate.sh b/regenerate.sh
index 750e3500e529..c10bc99a9692 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -96,7 +96,7 @@ done
 
 for src in ${LEGACY_SOURCES[@]}; do
   echo "protoc ${src}"
-  protoc --go_out=${OPTS},plugins=grpc:${WORKDIR}/out --go-grpc_out=${OPTS},migration_mode=true:${WORKDIR}/out \
+  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},gen_unstable_server_interfaces=true:${WORKDIR}/out \
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index a794e0d00370..dc4f513f25bd 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -130,11 +130,8 @@ func (cs *certStore) loadCerts() error {
 	return nil
 }
 
-// serverImpl is used to implement pb.GreeterServer.
-type serverImpl struct{}
-
-// SayHello is a simple implementation of pb.GreeterServer.
-func (s *serverImpl) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// sayHello is a simple implementation of the pb.GreeterServer SayHello method.
+func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
@@ -409,7 +406,7 @@ func (s) TestEnd2End(t *testing.T) {
 				t.Fatalf("failed to listen: %v", err)
 			}
 			defer lis.Close()
-			pb.RegisterGreeterService(s, pb.NewGreeterService(&serverImpl{}))
+			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{
 				Certificates:         test.clientCert,
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index c131efebec3e..32f93dde8db4 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -324,52 +324,3 @@ func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewTestServiceService creates a new TestServiceService containing the
-// implemented methods of the TestService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewTestServiceService(s interface{}) *TestServiceService {
-	ns := &TestServiceService{}
-	if h, ok := s.(interface {
-		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	}); ok {
-		ns.UnaryCall = h.UnaryCall
-	}
-	if h, ok := s.(interface {
-		FullDuplexCall(TestService_FullDuplexCallServer) error
-	}); ok {
-		ns.FullDuplexCall = h.FullDuplexCall
-	}
-	if h, ok := s.(interface {
-		ClientStreamCall(TestService_ClientStreamCallServer) error
-	}); ok {
-		ns.ClientStreamCall = h.ClientStreamCall
-	}
-	if h, ok := s.(interface {
-		ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
-	}); ok {
-		ns.ServerStreamCall = h.ServerStreamCall
-	}
-	return ns
-}
-
-// UnstableTestServiceService is the service API for TestService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableTestServiceService interface {
-	// One request followed by one response.
-	// The server returns the client id as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// Client stream
-	ClientStreamCall(TestService_ClientStreamCallServer) error
-	// Server stream
-	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
-}
diff --git a/stats/stats_test.go b/stats/stats_test.go
index 628533edbe2c..90b9cdbee9ca 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -75,7 +75,14 @@ var (
 
 type testServer struct{}
 
-var _ testpb.UnstableTestServiceService = (*testServer)(nil)
+func (s *testServer) Svc() *testpb.TestServiceService {
+	return &testpb.TestServiceService{
+		UnaryCall:        s.UnaryCall,
+		FullDuplexCall:   s.FullDuplexCall,
+		ClientStreamCall: s.ClientStreamCall,
+		ServerStreamCall: s.ServerStreamCall,
+	}
+}
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	if err := grpc.SendHeader(ctx, testHeaderMetadata); err != nil {
@@ -815,7 +822,7 @@ func checkServerStats(t *testing.T, got []*gotData, expect *expectedData, checkF
 func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) {
 	h := &statshandler{}
 	te := newTest(t, tc, nil, h)
-	te.startServer(testpb.NewTestServiceService(&testServer{}))
+	te.startServer((&testServer{}).Svc())
 	defer te.tearDown()
 
 	var (
@@ -1106,7 +1113,7 @@ func checkClientStats(t *testing.T, got []*gotData, expect *expectedData, checkF
 func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map[int]*checkFuncWithCount) {
 	h := &statshandler{}
 	te := newTest(t, tc, h, nil)
-	te.startServer(testpb.NewTestServiceService(&testServer{}))
+	te.startServer((&testServer{}).Svc())
 	defer te.tearDown()
 
 	var (
diff --git a/stress/client/main.go b/stress/client/main.go
index 0353476d8f39..c91aeb0a4180 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -151,13 +151,18 @@ type server struct {
 	gauges map[string]*gauge
 }
 
-var _ metricspb.UnstableMetricsServiceService = (*server)(nil)
-
 // newMetricsServer returns a new metrics server.
 func newMetricsServer() *server {
 	return &server{gauges: make(map[string]*gauge)}
 }
 
+func (s *server) Svc() *metricspb.MetricsServiceService {
+	return &metricspb.MetricsServiceService{
+		GetAllGauges: s.GetAllGauges,
+		GetGauge:     s.GetGauge,
+	}
+}
+
 // GetAllGauges returns all gauges.
 func (s *server) GetAllGauges(in *metricspb.EmptyMessage, stream metricspb.MetricsService_GetAllGaugesServer) error {
 	s.mutex.RLock()
@@ -203,9 +208,8 @@ func startServer(server *server, port int) {
 	}
 
 	s := grpc.NewServer()
-	metricspb.RegisterMetricsServiceService(s, metricspb.NewMetricsServiceService(server))
+	metricspb.RegisterMetricsServiceService(s, server.Svc())
 	s.Serve(lis)
-
 }
 
 // performRPCs uses weightedRandomTestSelector to select test case and runs the tests.
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 6ae1c80d263b..b28a52e79f71 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -165,36 +165,3 @@ func RegisterMetricsServiceService(s grpc.ServiceRegistrar, srv *MetricsServiceS
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewMetricsServiceService creates a new MetricsServiceService containing the
-// implemented methods of the MetricsService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewMetricsServiceService(s interface{}) *MetricsServiceService {
-	ns := &MetricsServiceService{}
-	if h, ok := s.(interface {
-		GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
-	}); ok {
-		ns.GetAllGauges = h.GetAllGauges
-	}
-	if h, ok := s.(interface {
-		GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
-	}); ok {
-		ns.GetGauge = h.GetGauge
-	}
-	return ns
-}
-
-// UnstableMetricsServiceService is the service API for MetricsService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableMetricsServiceService interface {
-	// Returns the values of all the gauges that are currently being maintained by
-	// the service
-	GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
-	// Returns the value of one gauge
-	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
-}
diff --git a/test/balancer_test.go b/test/balancer_test.go
index a94e25a8faf9..43e50210349e 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -179,7 +179,7 @@ func testPickExtraMetadata(t *testing.T, e env) {
 		grpc.WithBalancerName(testBalancerName),
 		grpc.WithUserAgent(testUserAgent),
 	}
-	te.startServer((&testServer{security: e.security}).Svc())
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	// Set resolver to xds to trigger the extra metadata code path.
@@ -228,7 +228,7 @@ func testDoneInfo(t *testing.T, e env) {
 		grpc.WithBalancerName(testBalancerName),
 	}
 	te.userAgent = failAppUA
-	te.startServer((&testServer{security: e.security}).Svc())
+	te.startServer(testServer{security: e.security}.Svc())
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -498,7 +498,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	}
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, testServer{}.Svc())
+	testpb.RegisterTestServiceService(s, (&testServer{}).Svc())
 	go s.Serve(lis)
 	defer s.Stop()
 	t.Logf("Started gRPC server at %s...", lis.Addr().String())
diff --git a/test/end2end_test.go b/test/end2end_test.go
index b0fa10b7872a..01f99cb29971 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -134,8 +134,6 @@ type testServer struct {
 	unaryCallSleepTime time.Duration
 }
 
-var _ testpb.UnstableTestServiceService = (*testServer)(nil)
-
 // Svc returns a registerable TestService for this testServer instances.
 // Because `s` is passed by value for convenience, any subsequent changes to
 // `s` are not recognized.
@@ -4859,10 +4857,10 @@ func (s) TestFlowControlLogicalRace(t *testing.T) {
 	defer lis.Close()
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&flowControlLogicalRaceServer{
+	testpb.RegisterTestServiceService(s, (&flowControlLogicalRaceServer{
 		itemCount: itemCount,
 		itemSize:  itemSize,
-	}))
+	}).Svc())
 	defer s.Stop()
 
 	go s.Serve(lis)
@@ -4920,6 +4918,12 @@ type flowControlLogicalRaceServer struct {
 	itemCount int
 }
 
+func (s *flowControlLogicalRaceServer) Svc() *testpb.TestServiceService {
+	return &testpb.TestServiceService{
+		StreamingOutputCall: s.StreamingOutputCall,
+	}
+}
+
 func (s *flowControlLogicalRaceServer) StreamingOutputCall(req *testpb.StreamingOutputCallRequest, srv testpb.TestService_StreamingOutputCallServer) error {
 	for i := 0; i < s.itemCount; i++ {
 		err := srv.Send(&testpb.StreamingOutputCallResponse{
@@ -5083,16 +5087,12 @@ type stubServer struct {
 	r *manual.Resolver
 }
 
-func (ss *stubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-	return ss.emptyCall(ctx, in)
-}
-
-func (ss *stubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
-	return ss.unaryCall(ctx, in)
-}
-
-func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
-	return ss.fullDuplexCall(stream)
+func (ss *stubServer) Svc() *testpb.TestServiceService {
+	return &testpb.TestServiceService{
+		EmptyCall:      ss.emptyCall,
+		UnaryCall:      ss.unaryCall,
+		FullDuplexCall: ss.fullDuplexCall,
+	}
 }
 
 // Start starts the server and creates a client connected to it.
@@ -5115,7 +5115,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption)
 	ss.cleanups = append(ss.cleanups, func() { lis.Close() })
 
 	s := grpc.NewServer(sopts...)
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 	go s.Serve(lis)
 	ss.cleanups = append(ss.cleanups, s.Stop)
 	ss.s = s
@@ -6258,7 +6258,7 @@ func (s) TestServeExitsWhenListenerClosed(t *testing.T) {
 
 	s := grpc.NewServer()
 	defer s.Stop()
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -6477,7 +6477,7 @@ func (s) TestDisabledIOBuffers(t *testing.T) {
 	}
 
 	s := grpc.NewServer(grpc.WriteBufferSize(0), grpc.ReadBufferSize(0))
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
diff --git a/test/goaway_test.go b/test/goaway_test.go
index 93d964803c42..1e00976bc144 100644
--- a/test/goaway_test.go
+++ b/test/goaway_test.go
@@ -48,7 +48,7 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) {
 
 	s := grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge}))
 	defer s.Stop()
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
diff --git a/test/gracefulstop_test.go b/test/gracefulstop_test.go
index f7659524d1d8..1f5f1e705cb5 100644
--- a/test/gracefulstop_test.go
+++ b/test/gracefulstop_test.go
@@ -117,7 +117,7 @@ func (s) TestGracefulStop(t *testing.T) {
 		},
 	}
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 
 	// 1. Start Server
 	wg := sync.WaitGroup{}
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index 7bcc9b06dd92..45ae65525134 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -454,71 +454,3 @@ func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService
 
 	s.RegisterService(&sd, nil)
 }
-
-// NewTestServiceService creates a new TestServiceService containing the
-// implemented methods of the TestService service in s.  Any unimplemented
-// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.
-// This includes situations where the method handler is misspelled or has the wrong
-// signature.  For this reason, this function should be used with great care and
-// is not recommended to be used by most users.
-func NewTestServiceService(s interface{}) *TestServiceService {
-	ns := &TestServiceService{}
-	if h, ok := s.(interface {
-		EmptyCall(context.Context, *Empty) (*Empty, error)
-	}); ok {
-		ns.EmptyCall = h.EmptyCall
-	}
-	if h, ok := s.(interface {
-		UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	}); ok {
-		ns.UnaryCall = h.UnaryCall
-	}
-	if h, ok := s.(interface {
-		StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
-	}); ok {
-		ns.StreamingOutputCall = h.StreamingOutputCall
-	}
-	if h, ok := s.(interface {
-		StreamingInputCall(TestService_StreamingInputCallServer) error
-	}); ok {
-		ns.StreamingInputCall = h.StreamingInputCall
-	}
-	if h, ok := s.(interface {
-		FullDuplexCall(TestService_FullDuplexCallServer) error
-	}); ok {
-		ns.FullDuplexCall = h.FullDuplexCall
-	}
-	if h, ok := s.(interface {
-		HalfDuplexCall(TestService_HalfDuplexCallServer) error
-	}); ok {
-		ns.HalfDuplexCall = h.HalfDuplexCall
-	}
-	return ns
-}
-
-// UnstableTestServiceService is the service API for TestService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended.
-type UnstableTestServiceService interface {
-	// One empty request followed by one empty response.
-	EmptyCall(context.Context, *Empty) (*Empty, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by a sequence of responses (streamed download).
-	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
-	// A sequence of requests followed by one response (streamed upload).
-	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall(TestService_StreamingInputCallServer) error
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// A sequence of requests followed by a sequence of responses.
-	// The server buffers all the client requests and then serves them in order. A
-	// stream of responses are returned to the client when the server starts with
-	// first request.
-	HalfDuplexCall(TestService_HalfDuplexCallServer) error
-}
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index 12af20b2b34c..86969c15d33d 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -64,7 +64,7 @@ func testLocalCredsE2ESucceed(network, address string) error {
 	s := grpc.NewServer(sopts...)
 	defer s.Stop()
 
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 
 	lis, err := net.Listen(network, address)
 	if err != nil {
@@ -162,7 +162,7 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
 	s := grpc.NewServer(sopts...)
 	defer s.Stop()
 
-	testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss))
+	testpb.RegisterTestServiceService(s, ss.Svc())
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {

From 4bee4dbc7b1c03006e69d0e3a6eee1235bbd020a Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 16 Sep 2020 14:28:16 -0700
Subject: [PATCH 195/481] protoc-gen-go-grpc: update README.md to reflect
 recent changes (#3883)

---
 cmd/protoc-gen-go-grpc/README.md | 87 ++++++++++++--------------------
 1 file changed, 31 insertions(+), 56 deletions(-)

diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md
index d5b0a3bbc6a0..152c68a191d4 100644
--- a/cmd/protoc-gen-go-grpc/README.md
+++ b/cmd/protoc-gen-go-grpc/README.md
@@ -62,29 +62,33 @@ Finally, pass this `Service` instance to the generated `Register` function:
 
 ### Migration from legacy version
 
-#### Updating existing code
+Older versions of `protoc-gen-go-grpc` and `protoc-gen-go` with the grpc plugin
+used a different method to register services.  With that method, it was only
+possible to register a service implementation that was a complete
+implementation of the service.  It was also possible to embed an
+`Unimplemented` implementation of the service, which was also generated and
+returned an UNIMPLEMENTED status for all methods.
+
+#### Generating the legacy API
+
+To avoid the need to update existing code, an option has been added to the code
+generator to produce the legacy API alongside the new API.  To use it:
+
+```sh
+# Example 1: with OPTS set to common options for protoc-gen-go and
+# protoc-gen-go-grpc
+protoc --go_out=${OPTS}:. --go-grpc_out=${OPTS},gen_unstable_server_interfaces=true:. *.proto
 
-The previous version of protoc-gen-go-grpc used a different method to register
-services.  In that version, it was only possible to register a service
-implementation that was a complete implementation of the service.  It was also
-possible to embed an `Unimplemented` implementation of the service, which was
-also generated and returned an UNIMPLEMENTED status for all methods.  To make
-it easier to migrate from the previous version, two new symbols are generated:
-`New<Service>Service` and `Unstable<Service>Service`.
-
-`New<Service>Service` allows for easy wrapping of a service implementation into
-an instance of the new generated `Service` struct type.  *This has drawbacks,
-however: `New<Service>Service` accepts any parameter, and does not panic or
-error if any methods are missing, are misspelled, or have the wrong signature.*
-These methods will return an UNIMPLEMENTED status if called by a client.
-
-`Unstable<Service>Service` allows for asserting that a type properly implements
-all methods of a service.  It is generated with the name `Unstable` to denote
-that it will be extended whenever new methods are added to the service
-definition.  This kind of change to the service definition is considered
-backward-compatible in protobuf, but is not backward-compatible in Go, because
-adding methods to an interface invalidates any existing implementations.  *Use
-of this symbol could result in future compilation errors.*
+# Example 2: if no special options are needed
+protoc --go_out=:. --go-grpc_out=gen_unstable_server_interfaces=true:. *.proto
+```
+
+**The use of this legacy API is NOT recommended.** It was discontinued as it
+results in compilation breakages when new methods are added to services, which
+is a backward-compatible change in all other languages supported by gRPC.  With
+the newer API, newly-added methods will return an UNIMPLEMENTED status.
+
+#### Updating existing code
 
 To convert your existing code using the previous code generator, please refer
 to the following example:
@@ -96,10 +100,6 @@ type myEchoService{
 // ... method handler implementation ...
 
 
-// Optional; not recommended: to guarantee myEchoService fully implements
-// EchoService:
-var _ pb.EchoServer = &myEchoService{}
-
 func main() {
      // ...
 
@@ -107,37 +107,12 @@ func main() {
      pb.RegisterEchoServer(grpcServer, &myEchoService{})
 
      // NEW:
-     pb.RegisterEchoService(grpcServer, pb.NewEchoService(&myEchoService{}))
-
-
-     // Optional: to gracefully detect missing methods:
-     if _, ok := &myEchoService{}.(pb.EchoServer); !ok {
-        fmt.Println("myEchoService does not implement all methods of EchoService.")
-     }
-
+     es := &myEchoService{}
+     pb.RegisterEchoService(grpcServer, pb.EchoService{
+        UnaryEcho: es.UnaryEcho,
+        // enumerate all methods in EchoService implemented by myEchoService...
+     })
 
      // ...
 }
 ```
-
-#### Updating generated code to support both legacy and new code
-
-`protoc-gen-go-grpc` supports a flag, `migration_mode`, which enables it to be
-run in tandem with the previous tool (`protoc-gen-go` with the grpc plugin).
-It can be used as follows:
-
-```sh
-go install github.com/golang/protobuf/protoc-gen-go
-
-# Example 1: with OPTS set to common options for protoc-gen-go and
-# protoc-gen-go-grpc
-protoc --go_out=${OPTS},plugins=grpc:. --go-grpc_out=${OPTS},migration_mode=true:. *.proto
-
-# Example 2: if no special options are needed
-protoc --go_out=plugins=grpc:. --go-grpc_out=migration_mode=true:. *.proto
-```
-
-This is recommended for temporary use only to ease migration from the legacy
-version.  The `Register<Service>Server` and `<Service>Server` symbols it
-produced are not backward compatible in the presence of newly added methods to
-a service.
\ No newline at end of file

From 9ec6f11015bc54a780bcd89e6413b81cf0bb6d7f Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 16 Sep 2020 14:53:49 -0700
Subject: [PATCH 196/481] xds: Exit from run() goroutine when resolver is
 closed. (#3882)

---
 xds/internal/resolver/xds_resolver.go      | 20 ++++++++++---------
 xds/internal/resolver/xds_resolver_test.go | 23 +++++++++++++++-------
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index cdd103ef7dc3..02bf19c7651c 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -20,12 +20,12 @@
 package resolver
 
 import (
-	"context"
 	"fmt"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
 
 	xdsinternal "google.golang.org/grpc/xds/internal"
@@ -62,6 +62,7 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 	r := &xdsResolver{
 		target:   t,
 		cc:       cc,
+		closed:   grpcsync.NewEvent(),
 		updateCh: make(chan suWithError, 1),
 	}
 	r.logger = prefixLogger((r))
@@ -86,7 +87,8 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 		return nil, fmt.Errorf("xds: failed to create xds-client: %v", err)
 	}
 	r.client = client
-	r.ctx, r.cancelCtx = context.WithCancel(context.Background())
+
+	// Register a watch on the xdsClient for the user's dial target.
 	cancelWatch := r.client.WatchService(r.target.Endpoint, r.handleServiceUpdate)
 	r.logger.Infof("Watch started on resource name %v with xds-client %p", r.target.Endpoint, r.client)
 	r.cancelWatch = func() {
@@ -145,10 +147,9 @@ type suWithError struct {
 // (which performs LDS/RDS queries for the same), and passes the received
 // updates to the ClientConn.
 type xdsResolver struct {
-	ctx       context.Context
-	cancelCtx context.CancelFunc
-	target    resolver.Target
-	cc        resolver.ClientConn
+	target resolver.Target
+	cc     resolver.ClientConn
+	closed *grpcsync.Event
 
 	logger *grpclog.PrefixLogger
 
@@ -176,7 +177,8 @@ type xdsResolver struct {
 func (r *xdsResolver) run() {
 	for {
 		select {
-		case <-r.ctx.Done():
+		case <-r.closed.Done():
+			return
 		case update := <-r.updateCh:
 			if update.err != nil {
 				r.logger.Warningf("Watch error on resource %v from xds-client %p, %v", r.target.Endpoint, r.client, update.err)
@@ -214,7 +216,7 @@ func (r *xdsResolver) run() {
 // the received update to the update channel, which is picked by the run
 // goroutine.
 func (r *xdsResolver) handleServiceUpdate(su xdsclient.ServiceUpdate, err error) {
-	if r.ctx.Err() != nil {
+	if r.closed.HasFired() {
 		// Do not pass updates to the ClientConn once the resolver is closed.
 		return
 	}
@@ -228,6 +230,6 @@ func (*xdsResolver) ResolveNow(o resolver.ResolveNowOptions) {}
 func (r *xdsResolver) Close() {
 	r.cancelWatch()
 	r.client.Close()
-	r.cancelCtx()
+	r.closed.Fire()
 	r.logger.Infof("Shutdown")
 }
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 98c6d6a2e5b1..9d04e3c95462 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
@@ -58,7 +59,15 @@ var (
 	target = resolver.Target{Endpoint: targetStr}
 )
 
-func TestRegister(t *testing.T) {
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestRegister(t *testing.T) {
 	b := resolver.Get(xdsScheme)
 	if b == nil {
 		t.Errorf("scheme %v is not registered", xdsScheme)
@@ -119,7 +128,7 @@ func errorDialer(_ context.Context, _ string) (net.Conn, error) {
 
 // TestResolverBuilder tests the xdsResolverBuilder's Build method with
 // different parameters.
-func TestResolverBuilder(t *testing.T) {
+func (s) TestResolverBuilder(t *testing.T) {
 	tests := []struct {
 		name          string
 		rbo           resolver.BuildOptions
@@ -262,7 +271,7 @@ func waitForWatchService(t *testing.T, xdsC *fakeclient.Client, wantTarget strin
 
 // TestXDSResolverWatchCallbackAfterClose tests the case where a service update
 // from the underlying xdsClient is received after the resolver is closed.
-func TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
+func (s) TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		config:        &validConfig,
@@ -286,7 +295,7 @@ func TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 
 // TestXDSResolverBadServiceUpdate tests the case the xdsClient returns a bad
 // service update.
-func TestXDSResolverBadServiceUpdate(t *testing.T) {
+func (s) TestXDSResolverBadServiceUpdate(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		config:        &validConfig,
@@ -313,7 +322,7 @@ func TestXDSResolverBadServiceUpdate(t *testing.T) {
 
 // TestXDSResolverGoodServiceUpdate tests the happy case where the resolver
 // gets a good service update from the xdsClient.
-func TestXDSResolverGoodServiceUpdate(t *testing.T) {
+func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		config:        &validConfig,
@@ -372,7 +381,7 @@ func TestXDSResolverGoodServiceUpdate(t *testing.T) {
 
 // TestXDSResolverUpdates tests the cases where the resolver gets a good update
 // after an error, and an error after the good update.
-func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
+func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		config:        &validConfig,
@@ -423,7 +432,7 @@ func TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 // TestXDSResolverResourceNotFoundError tests the cases where the resolver gets
 // a ResourceNotFoundError. It should generate a service config picking
 // weighted_target, but no child balancers.
-func TestXDSResolverResourceNotFoundError(t *testing.T) {
+func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		config:        &validConfig,

From bf9584e105730bef3051a9f4a8e4339a8564058f Mon Sep 17 00:00:00 2001
From: Tzu-Chiao Yeh <su3g4284zo6y7@gmail.com>
Date: Fri, 18 Sep 2020 01:34:27 +0800
Subject: [PATCH 197/481] internal/transport: handle h2 errcode on header
 decoding (#3872)

Handles HTTP2 error code when malformed request/response header appears.
Fixes: #3819
---
 internal/transport/http2_client.go   |  4 +-
 internal/transport/http2_server.go   |  6 +--
 internal/transport/http_util.go      | 21 +++------
 internal/transport/http_util_test.go | 65 ++++++++++++++++++++++++++++
 4 files changed, 77 insertions(+), 19 deletions(-)

diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index e7f2321131e4..a12d6b89f991 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -1206,8 +1206,8 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 	state := &decodeState{}
 	// Initialize isGRPC value to be !initialHeader, since if a gRPC Response-Headers has already been received, then it means that the peer is speaking gRPC and we are in gRPC mode.
 	state.data.isGRPC = !initialHeader
-	if err := state.decodeHeader(frame); err != nil {
-		t.closeStream(s, err, true, http2.ErrCodeProtocol, status.Convert(err), nil, endStream)
+	if h2code, err := state.decodeHeader(frame); err != nil {
+		t.closeStream(s, err, true, h2code, status.Convert(err), nil, endStream)
 		return
 	}
 
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index 3be22fee426c..0cf1cc320bb2 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -306,12 +306,12 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 	state := &decodeState{
 		serverSide: true,
 	}
-	if err := state.decodeHeader(frame); err != nil {
-		if se, ok := status.FromError(err); ok {
+	if h2code, err := state.decodeHeader(frame); err != nil {
+		if _, ok := status.FromError(err); ok {
 			t.controlBuf.put(&cleanupStream{
 				streamID: streamID,
 				rst:      true,
-				rstCode:  statusCodeConvTab[se.Code()],
+				rstCode:  h2code,
 				onWrite:  func() {},
 			})
 		}
diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go
index 5e1e7a65da2b..4d15afbf73f1 100644
--- a/internal/transport/http_util.go
+++ b/internal/transport/http_util.go
@@ -73,13 +73,6 @@ var (
 		http2.ErrCodeInadequateSecurity: codes.PermissionDenied,
 		http2.ErrCodeHTTP11Required:     codes.Internal,
 	}
-	statusCodeConvTab = map[codes.Code]http2.ErrCode{
-		codes.Internal:          http2.ErrCodeInternal,
-		codes.Canceled:          http2.ErrCodeCancel,
-		codes.Unavailable:       http2.ErrCodeRefusedStream,
-		codes.ResourceExhausted: http2.ErrCodeEnhanceYourCalm,
-		codes.PermissionDenied:  http2.ErrCodeInadequateSecurity,
-	}
 	// HTTPStatusConvTab is the HTTP status code to gRPC error code conversion table.
 	HTTPStatusConvTab = map[int]codes.Code{
 		// 400 Bad Request - INTERNAL.
@@ -222,11 +215,11 @@ func decodeMetadataHeader(k, v string) (string, error) {
 	return v, nil
 }
 
-func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) error {
+func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) (http2.ErrCode, error) {
 	// frame.Truncated is set to true when framer detects that the current header
 	// list size hits MaxHeaderListSize limit.
 	if frame.Truncated {
-		return status.Error(codes.Internal, "peer header list size exceeded limit")
+		return http2.ErrCodeFrameSize, status.Error(codes.Internal, "peer header list size exceeded limit")
 	}
 
 	for _, hf := range frame.Fields {
@@ -235,10 +228,10 @@ func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) error {
 
 	if d.data.isGRPC {
 		if d.data.grpcErr != nil {
-			return d.data.grpcErr
+			return http2.ErrCodeProtocol, d.data.grpcErr
 		}
 		if d.serverSide {
-			return nil
+			return http2.ErrCodeNo, nil
 		}
 		if d.data.rawStatusCode == nil && d.data.statusGen == nil {
 			// gRPC status doesn't exist.
@@ -250,12 +243,12 @@ func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) error {
 			code := int(codes.Unknown)
 			d.data.rawStatusCode = &code
 		}
-		return nil
+		return http2.ErrCodeNo, nil
 	}
 
 	// HTTP fallback mode
 	if d.data.httpErr != nil {
-		return d.data.httpErr
+		return http2.ErrCodeProtocol, d.data.httpErr
 	}
 
 	var (
@@ -270,7 +263,7 @@ func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) error {
 		}
 	}
 
-	return status.Error(code, d.constructHTTPErrMsg())
+	return http2.ErrCodeProtocol, status.Error(code, d.constructHTTPErrMsg())
 }
 
 // constructErrMsg constructs error message to be returned in HTTP fallback mode.
diff --git a/internal/transport/http_util_test.go b/internal/transport/http_util_test.go
index a3616f7389f5..85a083f6c8a8 100644
--- a/internal/transport/http_util_test.go
+++ b/internal/transport/http_util_test.go
@@ -23,6 +23,9 @@ import (
 	"reflect"
 	"testing"
 	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/hpack"
 )
 
 func (s) TestTimeoutDecode(t *testing.T) {
@@ -185,3 +188,65 @@ func (s) TestDecodeMetadataHeader(t *testing.T) {
 		}
 	}
 }
+
+func (s) TestDecodeHeaderH2ErrCode(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		// input
+		metaHeaderFrame *http2.MetaHeadersFrame
+		serverSide      bool
+		// output
+		wantCode http2.ErrCode
+	}{
+		{
+			name: "valid header",
+			metaHeaderFrame: &http2.MetaHeadersFrame{Fields: []hpack.HeaderField{
+				{Name: "content-type", Value: "application/grpc"},
+			}},
+			wantCode: http2.ErrCodeNo,
+		},
+		{
+			name: "valid header serverSide",
+			metaHeaderFrame: &http2.MetaHeadersFrame{Fields: []hpack.HeaderField{
+				{Name: "content-type", Value: "application/grpc"},
+			}},
+			serverSide: true,
+			wantCode:   http2.ErrCodeNo,
+		},
+		{
+			name: "invalid grpc status header field",
+			metaHeaderFrame: &http2.MetaHeadersFrame{Fields: []hpack.HeaderField{
+				{Name: "content-type", Value: "application/grpc"},
+				{Name: "grpc-status", Value: "xxxx"},
+			}},
+			wantCode: http2.ErrCodeProtocol,
+		},
+		{
+			name: "invalid http content type",
+			metaHeaderFrame: &http2.MetaHeadersFrame{Fields: []hpack.HeaderField{
+				{Name: "content-type", Value: "application/json"},
+			}},
+			wantCode: http2.ErrCodeProtocol,
+		},
+		{
+			name: "http fallback and invalid http status",
+			metaHeaderFrame: &http2.MetaHeadersFrame{Fields: []hpack.HeaderField{
+				// No content type provided then fallback into handling http error.
+				{Name: ":status", Value: "xxxx"},
+			}},
+			wantCode: http2.ErrCodeProtocol,
+		},
+		{
+			name:            "http2 frame size exceeds",
+			metaHeaderFrame: &http2.MetaHeadersFrame{Fields: nil, Truncated: true},
+			wantCode:        http2.ErrCodeFrameSize,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			state := &decodeState{serverSide: test.serverSide}
+			if h2code, _ := state.decodeHeader(test.metaHeaderFrame); h2code != test.wantCode {
+				t.Fatalf("decodeState.decodeHeader(%v) = %v, want %v", test.metaHeaderFrame, h2code, test.wantCode)
+			}
+		})
+	}
+}

From 4270c3cfce29c699e3139df4b2213772b8eb9500 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 17 Sep 2020 11:05:45 -0700
Subject: [PATCH 198/481] xds bootstrap: support insecure and make Creds
 required (#3881)

---
 .../edsbalancer/xds_client_wrapper.go         | 30 --------
 xds/internal/client/bootstrap/bootstrap.go    | 15 +++-
 .../client/bootstrap/bootstrap_test.go        | 73 ++++++++++---------
 xds/internal/resolver/xds_resolver.go         | 31 --------
 xds/internal/resolver/xds_resolver_test.go    |  4 +-
 5 files changed, 52 insertions(+), 101 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index 06b45847e0b5..dd66a4160a7a 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -141,14 +141,6 @@ func (c *xdsclientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 		return false
 	}
 
-	if clientConfig.Creds == nil {
-		// TODO: Once we start supporting a mechanism to register credential
-		// types, a failure to find the credential type mentioned in the
-		// bootstrap file should result in a failure, and not in using
-		// credentials from the parent channel (passed through the
-		// resolver.BuildOptions).
-		clientConfig.Creds = c.defaultDialCreds(clientConfig.BalancerName)
-	}
 	var dopts []grpc.DialOption
 	if dialer := c.bbo.Dialer; dialer != nil {
 		dopts = []grpc.DialOption{grpc.WithContextDialer(dialer)}
@@ -281,28 +273,6 @@ func (c *xdsclientWrapper) close() {
 	}
 }
 
-// defaultDialCreds builds a DialOption containing the credentials to be used
-// while talking to the xDS server (this is done only if the xds bootstrap
-// process does not return any credentials to use). If the parent channel
-// contains DialCreds, we use it as is. If it contains a CredsBundle, we use
-// just the transport credentials from the bundle. If we don't find any
-// credentials on the parent channel, we resort to using an insecure channel.
-func (c *xdsclientWrapper) defaultDialCreds(balancerName string) grpc.DialOption {
-	switch {
-	case c.bbo.DialCreds != nil:
-		if err := c.bbo.DialCreds.OverrideServerName(balancerName); err != nil {
-			c.logger.Warningf("xds: failed to override server name in credentials: %v, using Insecure", err)
-			return grpc.WithInsecure()
-		}
-		return grpc.WithTransportCredentials(c.bbo.DialCreds)
-	case c.bbo.CredsBundle != nil:
-		return grpc.WithTransportCredentials(c.bbo.CredsBundle.TransportCredentials())
-	default:
-		c.logger.Warningf("xds: no credentials available, using Insecure")
-		return grpc.WithInsecure()
-	}
-}
-
 // equalStringPointers returns true if
 // - a and b are both nil OR
 // - *a == *b (and a and b are both non-nil)
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index b2805bf73720..e3f2ce59dfb3 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -47,7 +47,8 @@ const (
 	serverFeaturesV3 = "xds_v3"
 
 	// Type name for Google default credentials.
-	googleDefaultCreds              = "google_default"
+	credsGoogleDefault              = "google_default"
+	credsInsecure                   = "insecure"
 	gRPCUserAgentName               = "gRPC Go"
 	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
 )
@@ -161,9 +162,12 @@ func NewConfig() (*Config, error) {
 			xs := servers[0]
 			config.BalancerName = xs.ServerURI
 			for _, cc := range xs.ChannelCreds {
-				if cc.Type == googleDefaultCreds {
+				// We stop at the first credential type that we support.
+				if cc.Type == credsGoogleDefault {
 					config.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials())
-					// We stop at the first credential type that we support.
+					break
+				} else if cc.Type == credsInsecure {
+					config.Creds = grpc.WithInsecure()
 					break
 				}
 			}
@@ -185,7 +189,10 @@ func NewConfig() (*Config, error) {
 	}
 
 	if config.BalancerName == "" {
-		return nil, fmt.Errorf("xds: Required field %q not found in bootstrap", "xds_servers.server_uri")
+		return nil, fmt.Errorf("xds: Required field %q not found in bootstrap %s", "xds_servers.server_uri", jsonData["xds_servers"])
+	}
+	if config.Creds == nil {
+		return nil, fmt.Errorf("xds: Required field %q doesn't contain valid value in bootstrap %s", "xds_servers.channel_creds", jsonData["xds_servers"])
 	}
 
 	// We end up using v3 transport protocol version only if the following
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 18c28db346be..a7734f03436e 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -38,7 +38,10 @@ var (
 		"emptyNodeProto": `
 		{
 			"xds_servers" : [{
-				"server_uri": "trafficdirector.googleapis.com:443"
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "insecure" }
+				]
 			}]
 		}`,
 		"unknownTopLevelFieldInFile": `
@@ -52,7 +55,7 @@ var (
 			"xds_servers" : [{
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
-					{ "type": "not-google-default" }
+					{ "type": "insecure" }
 				]
 			}],
 			"unknownField": "foobar"
@@ -67,7 +70,10 @@ var (
 			    }
 			},
 			"xds_servers" : [{
-				"server_uri": "trafficdirector.googleapis.com:443"
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "insecure" }
+				]
 			}]
 		}`,
 		"unknownFieldInXdsServer": `
@@ -81,38 +87,11 @@ var (
 			"xds_servers" : [{
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
-					{ "type": "not-google-default" }
+					{ "type": "insecure" }
 				],
 				"unknownField": "foobar"
 			}]
 		}`,
-		"emptyChannelCreds": `
-		{
-			"node": {
-				"id": "ENVOY_NODE_ID",
-				"metadata": {
-				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
-			    }
-			},
-			"xds_servers" : [{
-				"server_uri": "trafficdirector.googleapis.com:443"
-			}]
-		}`,
-		"nonGoogleDefaultCreds": `
-		{
-			"node": {
-				"id": "ENVOY_NODE_ID",
-				"metadata": {
-				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
-			    }
-			},
-			"xds_servers" : [{
-				"server_uri": "trafficdirector.googleapis.com:443",
-				"channel_creds": [
-					{ "type": "not-google-default" }
-				]
-			}]
-		}`,
 		"multipleChannelCreds": `
 		{
 			"node": {
@@ -222,7 +201,7 @@ var (
 	}
 	nilCredsConfigV2 = &Config{
 		BalancerName: "trafficdirector.googleapis.com:443",
-		Creds:        nil,
+		Creds:        grpc.WithInsecure(),
 		NodeProto:    v2NodeProto,
 	}
 	nonNilCredsConfigV2 = &Config{
@@ -290,6 +269,33 @@ func TestNewConfigV2ProtoFailure(t *testing.T) {
 			    }
 			}
 		}`,
+		"emptyChannelCreds": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443"
+			}]
+		}`,
+		"nonGoogleDefaultCreds": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "not-google-default" }
+				]
+			}]
+		}`,
 	}
 	cancel := setupBootstrapOverride(bootstrapFileMap)
 	defer cancel()
@@ -331,6 +337,7 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 		{
 			"emptyNodeProto", &Config{
 				BalancerName: "trafficdirector.googleapis.com:443",
+				Creds:        grpc.WithInsecure(),
 				NodeProto: &v2corepb.Node{
 					BuildVersion:         gRPCVersion,
 					UserAgentName:        gRPCUserAgentName,
@@ -342,8 +349,6 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 		{"unknownTopLevelFieldInFile", nilCredsConfigV2},
 		{"unknownFieldInNodeProto", nilCredsConfigV2},
 		{"unknownFieldInXdsServer", nilCredsConfigV2},
-		{"emptyChannelCreds", nilCredsConfigV2},
-		{"nonGoogleDefaultCreds", nilCredsConfigV2},
 		{"multipleChannelCreds", nonNilCredsConfigV2},
 		{"goodBootstrap", nonNilCredsConfigV2},
 		{"multipleXDSServers", nonNilCredsConfigV2},
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 02bf19c7651c..359ee129fb2f 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -68,15 +68,6 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 	r.logger = prefixLogger((r))
 	r.logger.Infof("Creating resolver for target: %+v", t)
 
-	if config.Creds == nil {
-		// TODO: Once we start supporting a mechanism to register credential
-		// types, a failure to find the credential type mentioned in the
-		// bootstrap file should result in a failure, and not in using
-		// credentials from the parent channel (passed through the
-		// resolver.BuildOptions).
-		config.Creds = r.defaultDialCreds(config.BalancerName, rbo)
-	}
-
 	var dopts []grpc.DialOption
 	if rbo.Dialer != nil {
 		dopts = []grpc.DialOption{grpc.WithContextDialer(rbo.Dialer)}
@@ -100,28 +91,6 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 	return r, nil
 }
 
-// defaultDialCreds builds a DialOption containing the credentials to be used
-// while talking to the xDS server (this is done only if the xds bootstrap
-// process does not return any credentials to use). If the parent channel
-// contains DialCreds, we use it as is. If it contains a CredsBundle, we use
-// just the transport credentials from the bundle. If we don't find any
-// credentials on the parent channel, we resort to using an insecure channel.
-func (r *xdsResolver) defaultDialCreds(balancerName string, rbo resolver.BuildOptions) grpc.DialOption {
-	switch {
-	case rbo.DialCreds != nil:
-		if err := rbo.DialCreds.OverrideServerName(balancerName); err != nil {
-			r.logger.Errorf("Failed to override server name in credentials: %v, using Insecure", err)
-			return grpc.WithInsecure()
-		}
-		return grpc.WithTransportCredentials(rbo.DialCreds)
-	case rbo.CredsBundle != nil:
-		return grpc.WithTransportCredentials(rbo.CredsBundle.TransportCredentials())
-	default:
-		r.logger.Warningf("No credentials available, using Insecure")
-		return grpc.WithInsecure()
-	}
-}
-
 // Name helps implement the resolver.Builder interface.
 func (*xdsResolverBuilder) Scheme() string {
 	return xdsScheme
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 9d04e3c95462..7eb0629a560a 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -113,7 +113,7 @@ func getXDSClientMakerFunc(wantOpts xdsclient.Options) func(xdsclient.Options) (
 		// what the want option is. We should be able to do extensive
 		// credential testing in e2e tests.
 		if (gotOpts.Config.Creds != nil) != (wantOpts.Config.Creds != nil) {
-			return nil, fmt.Errorf("got len(creds): %s, want: %s", gotOpts.Config.Creds, wantOpts.Config.Creds)
+			return nil, fmt.Errorf("got len(creds): %v, want: %v", gotOpts.Config.Creds, wantOpts.Config.Creds)
 		}
 		if len(gotOpts.DialOpts) != len(wantOpts.DialOpts) {
 			return nil, fmt.Errorf("got len(DialOpts): %v, want: %v", len(gotOpts.DialOpts), len(wantOpts.DialOpts))
@@ -159,7 +159,7 @@ func (s) TestResolverBuilder(t *testing.T) {
 				NodeProto:    xdstestutils.EmptyNodeProtoV2,
 			},
 			xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}),
-			wantErr:       false,
+			wantErr:       true,
 		},
 		{
 			name:   "error-dialer-in-rbo",

From 0f7e218c2cf49c7b0ca8247711b0daed2a07e79a Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Thu, 17 Sep 2020 12:08:03 -0700
Subject: [PATCH 199/481] advancedtls: add fields for root and identity
 providers in API (#3863)

* add provider in advancedtls API for pem file reloading
---
 security/advancedtls/advancedtls.go           | 293 ++++++---
 .../advancedtls_integration_test.go           |  26 +-
 security/advancedtls/advancedtls_test.go      | 568 ++++++++++++------
 security/advancedtls/pemfile_provider.go      |   1 -
 security/advancedtls/sni.go                   |   4 +-
 5 files changed, 612 insertions(+), 280 deletions(-)

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index 40a9fd5f735e..74564632660f 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -27,10 +27,12 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net"
+	"reflect"
 	"syscall"
 	"time"
 
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
 	credinternal "google.golang.org/grpc/internal/credentials"
 )
 
@@ -79,21 +81,67 @@ type GetRootCAsResults struct {
 	TrustCerts *x509.CertPool
 }
 
-// RootCertificateOptions contains a field and a function for obtaining root
-// trust certificates.
-// It is used by both ClientOptions and ServerOptions.
-// If users want to use default verification, but did not provide a valid
-// RootCertificateOptions, we use the system default trust certificates.
+// RootCertificateOptions contains options to obtain root trust certificates
+// for both the client and the server.
+// At most one option could be set. If none of them are set, we
+// use the system default trust certificates.
 type RootCertificateOptions struct {
-	// If field RootCACerts is set, field GetRootCAs will be ignored. RootCACerts
-	// will be used every time when verifying the peer certificates, without
-	// performing root certificate reloading.
+	// If RootCACerts is set, it will be used every time when verifying
+	// the peer certificates, without performing root certificate reloading.
 	RootCACerts *x509.CertPool
-	// If GetRootCAs is set and RootCACerts is nil, GetRootCAs will be invoked
-	// every time asked to check certificates sent from the server when a new
-	// connection is established.
-	// This is known as root CA certificate reloading.
-	GetRootCAs func(params *GetRootCAsParams) (*GetRootCAsResults, error)
+	// If GetRootCertificates is set, it will be invoked to obtain root certs for
+	// every new connection.
+	GetRootCertificates func(params *GetRootCAsParams) (*GetRootCAsResults, error)
+	// If RootProvider is set, we will use the root certs from the Provider's
+	// KeyMaterial() call in the new connections. The Provider must have initial
+	// credentials if specified. Otherwise, KeyMaterial() will block forever.
+	RootProvider certprovider.Provider
+}
+
+// nonNilFieldCount returns the number of set fields in RootCertificateOptions.
+func (o RootCertificateOptions) nonNilFieldCount() int {
+	cnt := 0
+	rv := reflect.ValueOf(o)
+	for i := 0; i < rv.NumField(); i++ {
+		if !rv.Field(i).IsNil() {
+			cnt++
+		}
+	}
+	return cnt
+}
+
+// IdentityCertificateOptions contains options to obtain identity certificates
+// for both the client and the server.
+// At most one option could be set.
+type IdentityCertificateOptions struct {
+	// If Certificates is set, it will be used every time when needed to present
+	//identity certificates, without performing identity certificate reloading.
+	Certificates []tls.Certificate
+	// If GetIdentityCertificatesForClient is set, it will be invoked to obtain
+	// identity certs for every new connection.
+	// This field MUST be set on client side.
+	GetIdentityCertificatesForClient func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+	// If GetIdentityCertificatesForServer is set, it will be invoked to obtain
+	// identity certs for every new connection.
+	// This field MUST be set on server side.
+	GetIdentityCertificatesForServer func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
+	// If IdentityProvider is set, we will use the identity certs from the
+	// Provider's KeyMaterial() call in the new connections. The Provider must
+	// have initial credentials if specified. Otherwise, KeyMaterial() will block
+	// forever.
+	IdentityProvider certprovider.Provider
+}
+
+// nonNilFieldCount returns the number of set fields in IdentityCertificateOptions.
+func (o IdentityCertificateOptions) nonNilFieldCount() int {
+	cnt := 0
+	rv := reflect.ValueOf(o)
+	for i := 0; i < rv.NumField(); i++ {
+		if !rv.Field(i).IsNil() {
+			cnt++
+		}
+	}
+	return cnt
 }
 
 // VerificationType is the enum type that represents different levels of
@@ -115,27 +163,11 @@ const (
 	SkipVerification
 )
 
-// ClientOptions contains all the fields and functions needed to be filled by
-// the client.
-// General rules for certificate setting on client side:
-// Certificates or GetClientCertificate indicates the certificates sent from
-// the client to the server to prove client's identities. The rules for setting
-// these two fields are:
-// If requiring mutual authentication on server side:
-//     Either Certificates or GetClientCertificate must be set; the other will
-//     be ignored.
-// Otherwise:
-//     Nothing needed(the two fields will be ignored).
+// ClientOptions contains the fields needed to be filled by the client.
 type ClientOptions struct {
-	// If field Certificates is set, field GetClientCertificate will be ignored.
-	// The client will use Certificates every time when asked for a certificate,
-	// without performing certificate reloading.
-	Certificates []tls.Certificate
-	// If GetClientCertificate is set and Certificates is nil, the client will
-	// invoke this function every time asked to present certificates to the
-	// server when a new connection is established. This is known as peer
-	// certificate reloading.
-	GetClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+	// IdentityOptions is OPTIONAL on client side. This field only needs to be
+	// set if mutual authentication is required on server side.
+	IdentityOptions IdentityCertificateOptions
 	// VerifyPeer is a custom verification check after certificate signature
 	// check.
 	// If this is set, we will perform this customized check after doing the
@@ -145,37 +177,25 @@ type ClientOptions struct {
 	// it will override the virtual host name of authority (e.g. :authority
 	// header field) in requests.
 	ServerNameOverride string
-	// RootCertificateOptions is REQUIRED to be correctly set on client side.
-	RootCertificateOptions
+	// RootOptions is OPTIONAL on client side. If not set, we will try to use the
+	// default trust certificates in users' OS system.
+	RootOptions RootCertificateOptions
 	// VType is the verification type on the client side.
 	VType VerificationType
 }
 
-// ServerOptions contains all the fields and functions needed to be filled by
-// the client.
-// General rules for certificate setting on server side:
-// Certificates or GetClientCertificate indicates the certificates sent from
-// the server to the client to prove server's identities. The rules for setting
-// these two fields are:
-// Either Certificates or GetCertificates must be set; the other will be ignored.
+// ServerOptions contains the fields needed to be filled by the server.
 type ServerOptions struct {
-	// If field Certificates is set, field GetClientCertificate will be ignored.
-	// The server will use Certificates every time when asked for a certificate,
-	// without performing certificate reloading.
-	Certificates []tls.Certificate
-	// If GetClientCertificate is set and Certificates is nil, the server will
-	// invoke this function every time asked to present certificates to the
-	// client when a new connection is established. This is known as peer
-	// certificate reloading.
-	GetCertificates func(*tls.ClientHelloInfo) ([]*tls.Certificate, error)
+	// IdentityOptions is REQUIRED on server side.
+	IdentityOptions IdentityCertificateOptions
 	// VerifyPeer is a custom verification check after certificate signature
 	// check.
 	// If this is set, we will perform this customized check after doing the
 	// normal check(s) indicated by setting VType.
 	VerifyPeer CustomVerificationFunc
-	// RootCertificateOptions is only required when mutual TLS is
-	// enabled(RequireClientCert is true).
-	RootCertificateOptions
+	// RootOptions is OPTIONAL on server side. This field only needs to be set if
+	// mutual authentication is required(RequireClientCert is true).
+	RootOptions RootCertificateOptions
 	// If the server want the client to send certificates.
 	RequireClientCert bool
 	// VType is the verification type on the server side.
@@ -184,48 +204,89 @@ type ServerOptions struct {
 
 func (o *ClientOptions) config() (*tls.Config, error) {
 	if o.VType == SkipVerification && o.VerifyPeer == nil {
-		return nil, fmt.Errorf(
-			"client needs to provide custom verification mechanism if choose to skip default verification")
+		return nil, fmt.Errorf("client needs to provide custom verification mechanism if choose to skip default verification")
 	}
-	rootCAs := o.RootCACerts
-	if o.VType != SkipVerification && o.RootCACerts == nil && o.GetRootCAs == nil {
-		// Set rootCAs to system default.
-		systemRootCAs, err := x509.SystemCertPool()
-		if err != nil {
-			return nil, err
-		}
-		rootCAs = systemRootCAs
+	// Make sure users didn't specify more than one fields in
+	// RootCertificateOptions and IdentityCertificateOptions.
+	if num := o.RootOptions.nonNilFieldCount(); num > 1 {
+		return nil, fmt.Errorf("at most one field in RootCertificateOptions could be specified")
+	}
+	if num := o.IdentityOptions.nonNilFieldCount(); num > 1 {
+		return nil, fmt.Errorf("at most one field in IdentityCertificateOptions could be specified")
+	}
+	if o.IdentityOptions.GetIdentityCertificatesForServer != nil {
+		return nil, fmt.Errorf("GetIdentityCertificatesForServer cannot be specified on the client side")
 	}
-	// We have to set InsecureSkipVerify to true to skip the default checks and
-	// use the verification function we built from buildVerifyFunc.
 	config := &tls.Config{
-		ServerName:           o.ServerNameOverride,
-		Certificates:         o.Certificates,
-		GetClientCertificate: o.GetClientCertificate,
-		InsecureSkipVerify:   true,
+		ServerName: o.ServerNameOverride,
+		// We have to set InsecureSkipVerify to true to skip the default checks and
+		// use the verification function we built from buildVerifyFunc.
+		InsecureSkipVerify: true,
+	}
+	// Propagate root-certificate-related fields in tls.Config.
+	switch {
+	case o.RootOptions.RootCACerts != nil:
+		config.RootCAs = o.RootOptions.RootCACerts
+	case o.RootOptions.GetRootCertificates != nil:
+		// In cases when users provide GetRootCertificates callback, since this
+		// callback is not contained in tls.Config, we have nothing to set here.
+		// We will invoke the callback in ClientHandshake.
+	case o.RootOptions.RootProvider != nil:
+		o.RootOptions.GetRootCertificates = func(*GetRootCAsParams) (*GetRootCAsResults, error) {
+			km, err := o.RootOptions.RootProvider.KeyMaterial(context.Background())
+			if err != nil {
+				return nil, err
+			}
+			return &GetRootCAsResults{TrustCerts: km.Roots}, nil
+		}
+	default:
+		// No root certificate options specified by user. Use the certificates
+		// stored in system default path as the last resort.
+		if o.VType != SkipVerification {
+			systemRootCAs, err := x509.SystemCertPool()
+			if err != nil {
+				return nil, err
+			}
+			config.RootCAs = systemRootCAs
+		}
 	}
-	if rootCAs != nil {
-		config.RootCAs = rootCAs
+	// Propagate identity-certificate-related fields in tls.Config.
+	switch {
+	case o.IdentityOptions.Certificates != nil:
+		config.Certificates = o.IdentityOptions.Certificates
+	case o.IdentityOptions.GetIdentityCertificatesForClient != nil:
+		config.GetClientCertificate = o.IdentityOptions.GetIdentityCertificatesForClient
+	case o.IdentityOptions.IdentityProvider != nil:
+		config.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			km, err := o.IdentityOptions.IdentityProvider.KeyMaterial(context.Background())
+			if err != nil {
+				return nil, err
+			}
+			if len(km.Certs) != 1 {
+				return nil, fmt.Errorf("there should always be only one identity cert chain on the client side in IdentityProvider")
+			}
+			return &km.Certs[0], nil
+		}
+	default:
+		// It's fine for users to not specify identity certificate options here.
 	}
 	return config, nil
 }
 
 func (o *ServerOptions) config() (*tls.Config, error) {
-	if o.Certificates == nil && o.GetCertificates == nil {
-		return nil, fmt.Errorf("either Certificates or GetCertificates must be specified")
-	}
 	if o.RequireClientCert && o.VType == SkipVerification && o.VerifyPeer == nil {
-		return nil, fmt.Errorf(
-			"server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)")
+		return nil, fmt.Errorf("server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)")
 	}
-	clientCAs := o.RootCACerts
-	if o.VType != SkipVerification && o.RootCACerts == nil && o.GetRootCAs == nil && o.RequireClientCert {
-		// Set clientCAs to system default.
-		systemRootCAs, err := x509.SystemCertPool()
-		if err != nil {
-			return nil, err
-		}
-		clientCAs = systemRootCAs
+	// Make sure users didn't specify more than one fields in
+	// RootCertificateOptions and IdentityCertificateOptions.
+	if num := o.RootOptions.nonNilFieldCount(); num > 1 {
+		return nil, fmt.Errorf("at most one field in RootCertificateOptions could be specified")
+	}
+	if num := o.IdentityOptions.nonNilFieldCount(); num > 1 {
+		return nil, fmt.Errorf("at most one field in IdentityCertificateOptions could be specified")
+	}
+	if o.IdentityOptions.GetIdentityCertificatesForClient != nil {
+		return nil, fmt.Errorf("GetIdentityCertificatesForClient cannot be specified on the server side")
 	}
 	clientAuth := tls.NoClientCert
 	if o.RequireClientCert {
@@ -235,18 +296,60 @@ func (o *ServerOptions) config() (*tls.Config, error) {
 		clientAuth = tls.RequireAnyClientCert
 	}
 	config := &tls.Config{
-		ClientAuth:   clientAuth,
-		Certificates: o.Certificates,
+		ClientAuth: clientAuth,
+	}
+	// Propagate root-certificate-related fields in tls.Config.
+	switch {
+	case o.RootOptions.RootCACerts != nil:
+		config.ClientCAs = o.RootOptions.RootCACerts
+	case o.RootOptions.GetRootCertificates != nil:
+		// In cases when users provide GetRootCertificates callback, since this
+		// callback is not contained in tls.Config, we have nothing to set here.
+		// We will invoke the callback in ServerHandshake.
+	case o.RootOptions.RootProvider != nil:
+		o.RootOptions.GetRootCertificates = func(*GetRootCAsParams) (*GetRootCAsResults, error) {
+			km, err := o.RootOptions.RootProvider.KeyMaterial(context.Background())
+			if err != nil {
+				return nil, err
+			}
+			return &GetRootCAsResults{TrustCerts: km.Roots}, nil
+		}
+	default:
+		// No root certificate options specified by user. Use the certificates
+		// stored in system default path as the last resort.
+		if o.VType != SkipVerification && o.RequireClientCert {
+			systemRootCAs, err := x509.SystemCertPool()
+			if err != nil {
+				return nil, err
+			}
+			config.ClientCAs = systemRootCAs
+		}
 	}
-	if o.GetCertificates != nil {
-		// GetCertificate is only able to perform SNI logic for go1.10 and above.
-		// It will return the first certificate in o.GetCertificates for go1.9.
+	// Propagate identity-certificate-related fields in tls.Config.
+	switch {
+	case o.IdentityOptions.Certificates != nil:
+		config.Certificates = o.IdentityOptions.Certificates
+	case o.IdentityOptions.GetIdentityCertificatesForServer != nil:
 		config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
 			return buildGetCertificates(clientHello, o)
 		}
-	}
-	if clientCAs != nil {
-		config.ClientCAs = clientCAs
+	case o.IdentityOptions.IdentityProvider != nil:
+		o.IdentityOptions.GetIdentityCertificatesForServer = func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+			km, err := o.IdentityOptions.IdentityProvider.KeyMaterial(context.Background())
+			if err != nil {
+				return nil, err
+			}
+			var certChains []*tls.Certificate
+			for i := 0; i < len(km.Certs); i++ {
+				certChains = append(certChains, &km.Certs[i])
+			}
+			return certChains, nil
+		}
+		config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			return buildGetCertificates(clientHello, o)
+		}
+	default:
+		return nil, fmt.Errorf("needs to specify at least one field in IdentityCertificateOptions")
 	}
 	return config, nil
 }
@@ -423,7 +526,7 @@ func NewClientCreds(o *ClientOptions) (credentials.TransportCredentials, error)
 	tc := &advancedTLSCreds{
 		config:     conf,
 		isClient:   true,
-		getRootCAs: o.GetRootCAs,
+		getRootCAs: o.RootOptions.GetRootCertificates,
 		verifyFunc: o.VerifyPeer,
 		vType:      o.VType,
 	}
@@ -441,7 +544,7 @@ func NewServerCreds(o *ServerOptions) (credentials.TransportCredentials, error)
 	tc := &advancedTLSCreds{
 		config:     conf,
 		isClient:   false,
-		getRootCAs: o.GetRootCAs,
+		getRootCAs: o.RootOptions.GetRootCertificates,
 		verifyFunc: o.VerifyPeer,
 		vType:      o.VType,
 	}
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index dc4f513f25bd..3398644471e8 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -385,11 +385,13 @@ func (s) TestEnd2End(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			// Start a server using ServerOptions in another goroutine.
 			serverOptions := &ServerOptions{
-				Certificates:    test.serverCert,
-				GetCertificates: test.serverGetCert,
-				RootCertificateOptions: RootCertificateOptions{
-					RootCACerts: test.serverRoot,
-					GetRootCAs:  test.serverGetRoot,
+				IdentityOptions: IdentityCertificateOptions{
+					Certificates:                     test.serverCert,
+					GetIdentityCertificatesForServer: test.serverGetCert,
+				},
+				RootOptions: RootCertificateOptions{
+					RootCACerts:         test.serverRoot,
+					GetRootCertificates: test.serverGetRoot,
 				},
 				RequireClientCert: true,
 				VerifyPeer:        test.serverVerifyFunc,
@@ -409,12 +411,14 @@ func (s) TestEnd2End(t *testing.T) {
 			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{
-				Certificates:         test.clientCert,
-				GetClientCertificate: test.clientGetCert,
-				VerifyPeer:           test.clientVerifyFunc,
-				RootCertificateOptions: RootCertificateOptions{
-					RootCACerts: test.clientRoot,
-					GetRootCAs:  test.clientGetRoot,
+				IdentityOptions: IdentityCertificateOptions{
+					Certificates:                     test.clientCert,
+					GetIdentityCertificatesForClient: test.clientGetCert,
+				},
+				VerifyPeer: test.clientVerifyFunc,
+				RootOptions: RootCertificateOptions{
+					RootCACerts:         test.clientRoot,
+					GetRootCertificates: test.clientGetRoot,
 				},
 				VType: test.clientVType,
 			}
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 8800a51a94d6..a631ee465d3d 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -34,6 +34,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/security/advancedtls/testdata"
 )
@@ -46,14 +47,274 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
+type provType int
+
+const (
+	provTypeRoot provType = iota
+	provTypeIdentity
+)
+
+type fakeProvider struct {
+	pt            provType
+	isClient      bool
+	wantMultiCert bool
+	wantError     bool
+}
+
+func (f fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
+	if f.wantError {
+		return nil, fmt.Errorf("bad fakeProvider")
+	}
+	cs := &certStore{}
+	err := cs.loadCerts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load certs: %v", err)
+	}
+	if f.pt == provTypeRoot && f.isClient {
+		return &certprovider.KeyMaterial{Roots: cs.clientTrust1}, nil
+	}
+	if f.pt == provTypeRoot && !f.isClient {
+		return &certprovider.KeyMaterial{Roots: cs.serverTrust1}, nil
+	}
+	if f.pt == provTypeIdentity && f.isClient {
+		if f.wantMultiCert {
+			return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1, cs.clientPeer2}}, nil
+		}
+		return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}}, nil
+	}
+	if f.wantMultiCert {
+		return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.serverPeer1, cs.serverPeer2}}, nil
+	}
+	return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.serverPeer1}}, nil
+}
+
+func (f fakeProvider) Close() {}
+
+func (s) TestClientOptionsConfigErrorCases(t *testing.T) {
+	tests := []struct {
+		desc            string
+		clientVType     VerificationType
+		IdentityOptions IdentityCertificateOptions
+		RootOptions     RootCertificateOptions
+	}{
+		{
+			desc:        "Skip default verification and provide no root credentials",
+			clientVType: SkipVerification,
+		},
+		{
+			desc:        "More than one fields in RootCertificateOptions is specified",
+			clientVType: CertVerification,
+			RootOptions: RootCertificateOptions{
+				RootCACerts:  x509.NewCertPool(),
+				RootProvider: fakeProvider{},
+			},
+		},
+		{
+			desc:        "More than one fields in IdentityCertificateOptions is specified",
+			clientVType: CertVerification,
+			IdentityOptions: IdentityCertificateOptions{
+				GetIdentityCertificatesForClient: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					return nil, nil
+				},
+				IdentityProvider: fakeProvider{pt: provTypeIdentity},
+			},
+		},
+		{
+			desc: "Specify GetIdentityCertificatesForServer",
+			IdentityOptions: IdentityCertificateOptions{
+				GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+					return nil, nil
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			clientOptions := &ClientOptions{
+				VType:           test.clientVType,
+				IdentityOptions: test.IdentityOptions,
+				RootOptions:     test.RootOptions,
+			}
+			_, err := clientOptions.config()
+			if err == nil {
+				t.Fatalf("ClientOptions{%v}.config() returns no err, wantErr != nil", clientOptions)
+			}
+		})
+	}
+}
+
+func (s) TestClientOptionsConfigSuccessCases(t *testing.T) {
+	tests := []struct {
+		desc            string
+		clientVType     VerificationType
+		IdentityOptions IdentityCertificateOptions
+		RootOptions     RootCertificateOptions
+	}{
+		{
+			desc:        "Use system default if no fields in RootCertificateOptions is specified",
+			clientVType: CertVerification,
+		},
+		{
+			desc:        "Good case with mutual TLS",
+			clientVType: CertVerification,
+			RootOptions: RootCertificateOptions{
+				RootProvider: fakeProvider{},
+			},
+			IdentityOptions: IdentityCertificateOptions{
+				IdentityProvider: fakeProvider{pt: provTypeIdentity},
+			},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			clientOptions := &ClientOptions{
+				VType:           test.clientVType,
+				IdentityOptions: test.IdentityOptions,
+				RootOptions:     test.RootOptions,
+			}
+			clientConfig, err := clientOptions.config()
+			if err != nil {
+				t.Fatalf("ClientOptions{%v}.config() = %v, wantErr == nil", clientOptions, err)
+			}
+			// Verify that the system-provided certificates would be used
+			// when no verification method was set in clientOptions.
+			if clientOptions.RootOptions.RootCACerts == nil &&
+				clientOptions.RootOptions.GetRootCertificates == nil && clientOptions.RootOptions.RootProvider == nil {
+				if clientConfig.RootCAs == nil {
+					t.Fatalf("Failed to assign system-provided certificates on the client side.")
+				}
+			}
+		})
+	}
+}
+
+func (s) TestServerOptionsConfigErrorCases(t *testing.T) {
+	tests := []struct {
+		desc              string
+		requireClientCert bool
+		serverVType       VerificationType
+		IdentityOptions   IdentityCertificateOptions
+		RootOptions       RootCertificateOptions
+	}{
+		{
+			desc:              "Skip default verification and provide no root credentials",
+			requireClientCert: true,
+			serverVType:       SkipVerification,
+		},
+		{
+			desc:              "More than one fields in RootCertificateOptions is specified",
+			requireClientCert: true,
+			serverVType:       CertVerification,
+			RootOptions: RootCertificateOptions{
+				RootCACerts: x509.NewCertPool(),
+				GetRootCertificates: func(*GetRootCAsParams) (*GetRootCAsResults, error) {
+					return nil, nil
+				},
+			},
+		},
+		{
+			desc:        "More than one fields in IdentityCertificateOptions is specified",
+			serverVType: CertVerification,
+			IdentityOptions: IdentityCertificateOptions{
+				Certificates:     []tls.Certificate{},
+				IdentityProvider: fakeProvider{pt: provTypeIdentity},
+			},
+		},
+		{
+			desc:        "no field in IdentityCertificateOptions is specified",
+			serverVType: CertVerification,
+		},
+		{
+			desc: "Specify GetIdentityCertificatesForClient",
+			IdentityOptions: IdentityCertificateOptions{
+				GetIdentityCertificatesForClient: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					return nil, nil
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			serverOptions := &ServerOptions{
+				VType:             test.serverVType,
+				RequireClientCert: test.requireClientCert,
+				IdentityOptions:   test.IdentityOptions,
+				RootOptions:       test.RootOptions,
+			}
+			_, err := serverOptions.config()
+			if err == nil {
+				t.Fatalf("ServerOptions{%v}.config() returns no err, wantErr != nil", serverOptions)
+			}
+		})
+	}
+}
+
+func (s) TestServerOptionsConfigSuccessCases(t *testing.T) {
+	tests := []struct {
+		desc              string
+		requireClientCert bool
+		serverVType       VerificationType
+		IdentityOptions   IdentityCertificateOptions
+		RootOptions       RootCertificateOptions
+	}{
+		{
+			desc:              "Use system default if no fields in RootCertificateOptions is specified",
+			requireClientCert: true,
+			serverVType:       CertVerification,
+			IdentityOptions: IdentityCertificateOptions{
+				Certificates: []tls.Certificate{},
+			},
+		},
+		{
+			desc:              "Good case with mutual TLS",
+			requireClientCert: true,
+			serverVType:       CertVerification,
+			RootOptions: RootCertificateOptions{
+				RootProvider: fakeProvider{},
+			},
+			IdentityOptions: IdentityCertificateOptions{
+				GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+					return nil, nil
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			serverOptions := &ServerOptions{
+				VType:             test.serverVType,
+				RequireClientCert: test.requireClientCert,
+				IdentityOptions:   test.IdentityOptions,
+				RootOptions:       test.RootOptions,
+			}
+			serverConfig, err := serverOptions.config()
+			if err != nil {
+				t.Fatalf("ServerOptions{%v}.config() = %v, wantErr == nil", serverOptions, err)
+			}
+			// Verify that the system-provided certificates would be used
+			// when no verification method was set in serverOptions.
+			if serverOptions.RootOptions.RootCACerts == nil &&
+				serverOptions.RootOptions.GetRootCertificates == nil && serverOptions.RootOptions.RootProvider == nil {
+				if serverConfig.ClientCAs == nil {
+					t.Fatalf("Failed to assign system-provided certificates on the server side.")
+				}
+			}
+		})
+	}
+}
+
 func (s) TestClientServerHandshake(t *testing.T) {
-	// ------------------Load Client Trust Cert and Peer Cert-------------------
-	clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem"))
+	cs := &certStore{}
+	err := cs.loadCerts()
 	if err != nil {
-		t.Fatalf("Client is unable to load trust certs. Error: %v", err)
+		t.Fatalf("Failed to load certs: %v", err)
 	}
 	getRootCAsForClient := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
-		return &GetRootCAsResults{TrustCerts: clientTrustPool}, nil
+		return &GetRootCAsResults{TrustCerts: cs.clientTrust1}, nil
 	}
 	clientVerifyFuncGood := func(params *VerificationFuncParams) (*VerificationResults, error) {
 		if params.ServerName == "" {
@@ -69,18 +330,8 @@ func (s) TestClientServerHandshake(t *testing.T) {
 	verifyFuncBad := func(params *VerificationFuncParams) (*VerificationResults, error) {
 		return nil, fmt.Errorf("custom verification function failed")
 	}
-	clientPeerCert, err := tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"),
-		testdata.Path("client_key_1.pem"))
-	if err != nil {
-		t.Fatalf("Client is unable to parse peer certificates. Error: %v", err)
-	}
-	// ------------------Load Server Trust Cert and Peer Cert-------------------
-	serverTrustPool, err := readTrustCert(testdata.Path("server_trust_cert_1.pem"))
-	if err != nil {
-		t.Fatalf("Server is unable to load trust certs. Error: %v", err)
-	}
 	getRootCAsForServer := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
-		return &GetRootCAsResults{TrustCerts: serverTrustPool}, nil
+		return &GetRootCAsResults{TrustCerts: cs.serverTrust1}, nil
 	}
 	serverVerifyFunc := func(params *VerificationFuncParams) (*VerificationResults, error) {
 		if params.ServerName != "" {
@@ -93,11 +344,6 @@ func (s) TestClientServerHandshake(t *testing.T) {
 
 		return &VerificationResults{}, nil
 	}
-	serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"),
-		testdata.Path("server_key_1.pem"))
-	if err != nil {
-		t.Fatalf("Server is unable to parse peer certificates. Error: %v", err)
-	}
 	getRootCAsForServerBad := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 		return nil, fmt.Errorf("bad root certificate reloading")
 	}
@@ -109,7 +355,8 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		clientGetRoot              func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		clientVerifyFunc           CustomVerificationFunc
 		clientVType                VerificationType
-		clientExpectCreateError    bool
+		clientRootProvider         certprovider.Provider
+		clientIdentityProvider     certprovider.Provider
 		clientExpectHandshakeError bool
 		serverMutualTLS            bool
 		serverCert                 []tls.Certificate
@@ -118,23 +365,10 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		serverGetRoot              func(params *GetRootCAsParams) (*GetRootCAsResults, error)
 		serverVerifyFunc           CustomVerificationFunc
 		serverVType                VerificationType
+		serverRootProvider         certprovider.Provider
+		serverIdentityProvider     certprovider.Provider
 		serverExpectError          bool
 	}{
-		// Client: nil setting
-		// Server: only set serverCert with mutual TLS off
-		// Expected Behavior: server side failure
-		// Reason: if clientRoot, clientGetRoot and verifyFunc is not set, client
-		// side doesn't provide any verification mechanism. We don't allow this
-		// even setting vType to SkipVerification. Clients should at least provide
-		// their own verification logic.
-		{
-			desc:                    "Client has no trust cert; server sends peer cert",
-			clientVType:             SkipVerification,
-			clientExpectCreateError: true,
-			serverCert:              []tls.Certificate{serverPeerCert},
-			serverVType:             CertAndHostVerification,
-			serverExpectError:       true,
-		},
 		// Client: nil setting except verifyFuncGood
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: success
@@ -144,7 +378,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			desc:             "Client has no trust cert with verifyFuncGood; server sends peer cert",
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      SkipVerification,
-			serverCert:       []tls.Certificate{serverPeerCert},
+			serverCert:       []tls.Certificate{cs.serverPeer1},
 			serverVType:      CertAndHostVerification,
 		},
 		// Client: only set clientRoot
@@ -155,10 +389,10 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// this test suites.
 		{
 			desc:                       "Client has root cert; server sends peer cert",
-			clientRoot:                 clientTrustPool,
+			clientRoot:                 cs.clientTrust1,
 			clientVType:                CertAndHostVerification,
 			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverCert:                 []tls.Certificate{cs.serverPeer1},
 			serverVType:                CertAndHostVerification,
 			serverExpectError:          true,
 		},
@@ -173,7 +407,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientGetRoot:              getRootCAsForClient,
 			clientVType:                CertAndHostVerification,
 			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverCert:                 []tls.Certificate{cs.serverPeer1},
 			serverVType:                CertAndHostVerification,
 			serverExpectError:          true,
 		},
@@ -185,7 +419,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
-			serverCert:       []tls.Certificate{serverPeerCert},
+			serverCert:       []tls.Certificate{cs.serverPeer1},
 			serverVType:      CertAndHostVerification,
 		},
 		// Client: set clientGetRoot and bad clientVerifyFunc function
@@ -198,66 +432,35 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientVerifyFunc:           verifyFuncBad,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverCert:                 []tls.Certificate{cs.serverPeer1},
 			serverVType:                CertVerification,
 			serverExpectError:          true,
 		},
-		// Client: set clientGetRoot and clientVerifyFunc
-		// Server: nil setting
-		// Expected Behavior: server side failure
-		// Reason: server side must either set serverCert or serverGetCert
-		{
-			desc:              "Client sets reload root function with verifyFuncGood; server sets nil",
-			clientGetRoot:     getRootCAsForClient,
-			clientVerifyFunc:  clientVerifyFuncGood,
-			clientVType:       CertVerification,
-			serverVType:       CertVerification,
-			serverExpectError: true,
-		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverRoot and serverCert with mutual TLS on
 		// Expected Behavior: success
 		{
 			desc:             "Client sets peer cert, reload root function with verifyFuncGood; server sets peer cert and root cert; mutualTLS",
-			clientCert:       []tls.Certificate{clientPeerCert},
+			clientCert:       []tls.Certificate{cs.clientPeer1},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverCert:       []tls.Certificate{serverPeerCert},
-			serverRoot:       serverTrustPool,
+			serverCert:       []tls.Certificate{cs.serverPeer1},
+			serverRoot:       cs.serverTrust1,
 			serverVType:      CertVerification,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
-		// Server: set serverCert, but not setting any of serverRoot, serverGetRoot
-		// or serverVerifyFunc, with mutual TLS on
-		// Expected Behavior: server side failure
-		// Reason: server side needs to provide any verification mechanism when
-		// mTLS in on, even setting vType to SkipVerification. Servers should at
-		// least provide their own verification logic.
-		{
-			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; server sets no verification; mutualTLS",
-			clientCert:                 []tls.Certificate{clientPeerCert},
-			clientGetRoot:              getRootCAsForClient,
-			clientVerifyFunc:           clientVerifyFuncGood,
-			clientVType:                CertVerification,
-			clientExpectHandshakeError: true,
-			serverMutualTLS:            true,
-			serverCert:                 []tls.Certificate{serverPeerCert},
-			serverVType:                SkipVerification,
-			serverExpectError:          true,
-		},
-		// Client: set clientGetRoot, clientVerifyFunc and clientCert
 		// Server: set serverGetRoot and serverCert with mutual TLS on
 		// Expected Behavior: success
 		{
 			desc:             "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS",
-			clientCert:       []tls.Certificate{clientPeerCert},
+			clientCert:       []tls.Certificate{cs.clientPeer1},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverCert:       []tls.Certificate{serverPeerCert},
+			serverCert:       []tls.Certificate{cs.serverPeer1},
 			serverGetRoot:    getRootCAsForServer,
 			serverVType:      CertVerification,
 		},
@@ -268,12 +471,12 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// Reason: server side reloading returns failure
 		{
 			desc:              "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, bad reload root function; mutualTLS",
-			clientCert:        []tls.Certificate{clientPeerCert},
+			clientCert:        []tls.Certificate{cs.clientPeer1},
 			clientGetRoot:     getRootCAsForClient,
 			clientVerifyFunc:  clientVerifyFuncGood,
 			clientVType:       CertVerification,
 			serverMutualTLS:   true,
-			serverCert:        []tls.Certificate{serverPeerCert},
+			serverCert:        []tls.Certificate{cs.serverPeer1},
 			serverGetRoot:     getRootCAsForServerBad,
 			serverVType:       CertVerification,
 			serverExpectError: true,
@@ -284,14 +487,14 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sets reload peer/root function with verifyFuncGood; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &clientPeerCert, nil
+				return &cs.clientPeer1, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverPeerCert}, nil
+				return []*tls.Certificate{&cs.serverPeer1}, nil
 			},
 			serverGetRoot:    getRootCAsForServer,
 			serverVerifyFunc: serverVerifyFunc,
@@ -305,14 +508,14 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sends wrong peer cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &serverPeerCert, nil
+				return &cs.serverPeer1, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverPeerCert}, nil
+				return []*tls.Certificate{&cs.serverPeer1}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -326,7 +529,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client has wrong trust cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &clientPeerCert, nil
+				return &cs.clientPeer1, nil
 			},
 			clientGetRoot:              getRootCAsForServer,
 			clientVerifyFunc:           clientVerifyFuncGood,
@@ -334,7 +537,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverPeerCert}, nil
+				return []*tls.Certificate{&cs.serverPeer1}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -349,14 +552,14 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sets reload peer/root function with verifyFuncGood; Server sends wrong peer cert; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &clientPeerCert, nil
+				return &cs.clientPeer1, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&clientPeerCert}, nil
+				return []*tls.Certificate{&cs.clientPeer1}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -370,7 +573,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sets reload peer/root function with verifyFuncGood; Server has wrong trust cert; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &clientPeerCert, nil
+				return &cs.clientPeer1, nil
 			},
 			clientGetRoot:              getRootCAsForClient,
 			clientVerifyFunc:           clientVerifyFuncGood,
@@ -378,7 +581,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&serverPeerCert}, nil
+				return []*tls.Certificate{&cs.serverPeer1}, nil
 			},
 			serverGetRoot:     getRootCAsForClient,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -391,18 +594,92 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// server custom check fails
 		{
 			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; Server sets bad custom check; mutualTLS",
-			clientCert:                 []tls.Certificate{clientPeerCert},
+			clientCert:                 []tls.Certificate{cs.clientPeer1},
 			clientGetRoot:              getRootCAsForClient,
 			clientVerifyFunc:           clientVerifyFuncGood,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
-			serverCert:                 []tls.Certificate{serverPeerCert},
+			serverCert:                 []tls.Certificate{cs.serverPeer1},
 			serverGetRoot:              getRootCAsForServer,
 			serverVerifyFunc:           verifyFuncBad,
 			serverVType:                CertVerification,
 			serverExpectError:          true,
 		},
+		// Client: set a clientIdentityProvider which will get multiple cert chains
+		// Server: set serverIdentityProvider and serverRootProvider with mutual TLS on
+		// Expected Behavior: server side failure due to multiple cert chains in
+		// clientIdentityProvider
+		{
+			desc:                   "Client sets multiple certs in clientIdentityProvider; Server sets root and identity provider; mutualTLS",
+			clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true, wantMultiCert: true},
+			clientRootProvider:     fakeProvider{isClient: true},
+			clientVerifyFunc:       clientVerifyFuncGood,
+			clientVType:            CertVerification,
+			serverMutualTLS:        true,
+			serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false},
+			serverRootProvider:     fakeProvider{isClient: false},
+			serverVType:            CertVerification,
+			serverExpectError:      true,
+		},
+		// Client: set a bad clientIdentityProvider
+		// Server: set serverIdentityProvider and serverRootProvider with mutual TLS on
+		// Expected Behavior: server side failure due to bad clientIdentityProvider
+		{
+			desc:                   "Client sets bad clientIdentityProvider; Server sets root and identity provider; mutualTLS",
+			clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true, wantError: true},
+			clientRootProvider:     fakeProvider{isClient: true},
+			clientVerifyFunc:       clientVerifyFuncGood,
+			clientVType:            CertVerification,
+			serverMutualTLS:        true,
+			serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false},
+			serverRootProvider:     fakeProvider{isClient: false},
+			serverVType:            CertVerification,
+			serverExpectError:      true,
+		},
+		// Client: set clientIdentityProvider and clientRootProvider
+		// Server: set bad serverRootProvider with mutual TLS on
+		// Expected Behavior: server side failure due to bad serverRootProvider
+		{
+			desc:                   "Client sets root and identity provider; Server sets bad root provider; mutualTLS",
+			clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true},
+			clientRootProvider:     fakeProvider{isClient: true},
+			clientVerifyFunc:       clientVerifyFuncGood,
+			clientVType:            CertVerification,
+			serverMutualTLS:        true,
+			serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false},
+			serverRootProvider:     fakeProvider{isClient: false, wantError: true},
+			serverVType:            CertVerification,
+			serverExpectError:      true,
+		},
+		// Client: set clientIdentityProvider and clientRootProvider
+		// Server: set serverIdentityProvider and serverRootProvider with mutual TLS on
+		// Expected Behavior: success
+		{
+			desc:                   "Client sets root and identity provider; Server sets root and identity provider; mutualTLS",
+			clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true},
+			clientRootProvider:     fakeProvider{isClient: true},
+			clientVerifyFunc:       clientVerifyFuncGood,
+			clientVType:            CertVerification,
+			serverMutualTLS:        true,
+			serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false},
+			serverRootProvider:     fakeProvider{isClient: false},
+			serverVType:            CertVerification,
+		},
+		// Client: set clientIdentityProvider and clientRootProvider
+		// Server: set serverIdentityProvider getting multiple cert chains and serverRootProvider with mutual TLS on
+		// Expected Behavior: success, because server side has SNI
+		{
+			desc:                   "Client sets root and identity provider; Server sets multiple certs in serverIdentityProvider; mutualTLS",
+			clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true},
+			clientRootProvider:     fakeProvider{isClient: true},
+			clientVerifyFunc:       clientVerifyFuncGood,
+			clientVType:            CertVerification,
+			serverMutualTLS:        true,
+			serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false, wantMultiCert: true},
+			serverRootProvider:     fakeProvider{isClient: false},
+			serverVType:            CertVerification,
+		},
 	} {
 		test := test
 		t.Run(test.desc, func(t *testing.T) {
@@ -413,11 +690,15 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			}
 			// Start a server using ServerOptions in another goroutine.
 			serverOptions := &ServerOptions{
-				Certificates:    test.serverCert,
-				GetCertificates: test.serverGetCert,
-				RootCertificateOptions: RootCertificateOptions{
-					RootCACerts: test.serverRoot,
-					GetRootCAs:  test.serverGetRoot,
+				IdentityOptions: IdentityCertificateOptions{
+					Certificates:                     test.serverCert,
+					GetIdentityCertificatesForServer: test.serverGetCert,
+					IdentityProvider:                 test.serverIdentityProvider,
+				},
+				RootOptions: RootCertificateOptions{
+					RootCACerts:         test.serverRoot,
+					GetRootCertificates: test.serverGetRoot,
+					RootProvider:        test.serverRootProvider,
 				},
 				RequireClientCert: test.serverMutualTLS,
 				VerifyPeer:        test.serverVerifyFunc,
@@ -452,23 +733,22 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			}
 			defer conn.Close()
 			clientOptions := &ClientOptions{
-				Certificates:         test.clientCert,
-				GetClientCertificate: test.clientGetCert,
-				VerifyPeer:           test.clientVerifyFunc,
-				RootCertificateOptions: RootCertificateOptions{
-					RootCACerts: test.clientRoot,
-					GetRootCAs:  test.clientGetRoot,
+				IdentityOptions: IdentityCertificateOptions{
+					Certificates:                     test.clientCert,
+					GetIdentityCertificatesForClient: test.clientGetCert,
+					IdentityProvider:                 test.clientIdentityProvider,
+				},
+				VerifyPeer: test.clientVerifyFunc,
+				RootOptions: RootCertificateOptions{
+					RootCACerts:         test.clientRoot,
+					GetRootCertificates: test.clientGetRoot,
+					RootProvider:        test.clientRootProvider,
 				},
 				VType: test.clientVType,
 			}
-			clientTLS, newClientErr := NewClientCreds(clientOptions)
-			if newClientErr != nil && test.clientExpectCreateError {
-				return
-			}
-			if newClientErr != nil && !test.clientExpectCreateError ||
-				newClientErr == nil && test.clientExpectCreateError {
-				t.Fatalf("Expect error: %v, but err is %v",
-					test.clientExpectCreateError, newClientErr)
+			clientTLS, err := NewClientCreds(clientOptions)
+			if err != nil {
+				t.Fatalf("NewClientCreds failed: %v", err)
 			}
 			_, clientAuthInfo, handshakeErr := clientTLS.ClientHandshake(context.Background(),
 				lisAddr, conn)
@@ -541,7 +821,7 @@ func (s) TestAdvancedTLSOverrideServerName(t *testing.T) {
 		t.Fatalf("Client is unable to load trust certs. Error: %v", err)
 	}
 	clientOptions := &ClientOptions{
-		RootCertificateOptions: RootCertificateOptions{
+		RootOptions: RootCertificateOptions{
 			RootCACerts: clientTrustPool,
 		},
 		ServerNameOverride: expectedServerName,
@@ -563,7 +843,7 @@ func (s) TestTLSClone(t *testing.T) {
 		t.Fatalf("Client is unable to load trust certs. Error: %v", err)
 	}
 	clientOptions := &ClientOptions{
-		RootCertificateOptions: RootCertificateOptions{
+		RootOptions: RootCertificateOptions{
 			RootCACerts: clientTrustPool,
 		},
 		ServerNameOverride: expectedServerName,
@@ -635,62 +915,6 @@ func (s) TestWrapSyscallConn(t *testing.T) {
 	}
 }
 
-func (s) TestOptionsConfig(t *testing.T) {
-	serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"),
-		testdata.Path("server_key_1.pem"))
-	if err != nil {
-		t.Fatalf("Server is unable to parse peer certificates. Error: %v", err)
-	}
-	tests := []struct {
-		desc            string
-		clientVType     VerificationType
-		serverMutualTLS bool
-		serverCert      []tls.Certificate
-		serverVType     VerificationType
-	}{
-		{
-			desc:            "Client uses system-provided RootCAs; server uses system-provided ClientCAs",
-			clientVType:     CertVerification,
-			serverMutualTLS: true,
-			serverCert:      []tls.Certificate{serverPeerCert},
-			serverVType:     CertAndHostVerification,
-		},
-	}
-	for _, test := range tests {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			serverOptions := &ServerOptions{
-				Certificates:      test.serverCert,
-				RequireClientCert: test.serverMutualTLS,
-				VType:             test.serverVType,
-			}
-			serverConfig, err := serverOptions.config()
-			if err != nil {
-				t.Fatalf("Unable to generate serverConfig. Error: %v", err)
-			}
-			// Verify that the system-provided certificates would be used
-			// when no verification method was set in serverOptions.
-			if serverOptions.RootCACerts == nil && serverOptions.GetRootCAs == nil &&
-				serverOptions.RequireClientCert && serverConfig.ClientCAs == nil {
-				t.Fatalf("Failed to assign system-provided certificates on the server side.")
-			}
-			clientOptions := &ClientOptions{
-				VType: test.clientVType,
-			}
-			clientConfig, err := clientOptions.config()
-			if err != nil {
-				t.Fatalf("Unable to generate clientConfig. Error: %v", err)
-			}
-			// Verify that the system-provided certificates would be used
-			// when no verification method was set in clientOptions.
-			if clientOptions.RootCACerts == nil && clientOptions.GetRootCAs == nil &&
-				clientConfig.RootCAs == nil {
-				t.Fatalf("Failed to assign system-provided certificates on the client side.")
-			}
-		})
-	}
-}
-
 func (s) TestGetCertificatesSNI(t *testing.T) {
 	// Load server certificates for setting the serverGetCert callback function.
 	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
@@ -734,8 +958,10 @@ func (s) TestGetCertificatesSNI(t *testing.T) {
 		test := test
 		t.Run(test.desc, func(t *testing.T) {
 			serverOptions := &ServerOptions{
-				GetCertificates: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-					return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+				IdentityOptions: IdentityCertificateOptions{
+					GetIdentityCertificatesForServer: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
+						return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+					},
 				},
 			}
 			serverConfig, err := serverOptions.config()
diff --git a/security/advancedtls/pemfile_provider.go b/security/advancedtls/pemfile_provider.go
index 3dbb52440d45..7e03bb835282 100644
--- a/security/advancedtls/pemfile_provider.go
+++ b/security/advancedtls/pemfile_provider.go
@@ -97,7 +97,6 @@ func NewPEMFileProvider(o PEMFileProviderOptions) (*PEMFileProvider, error) {
 		return nil, fmt.Errorf("private key file and identity cert file should be both specified or not specified")
 	}
 	if o.IdentityInterval == 0 {
-		logger.Warningf("heyheyhey")
 		o.IdentityInterval = defaultIdentityInterval
 	}
 	if o.RootInterval == 0 {
diff --git a/security/advancedtls/sni.go b/security/advancedtls/sni.go
index 6c8623283cf5..7fef1990c566 100644
--- a/security/advancedtls/sni.go
+++ b/security/advancedtls/sni.go
@@ -28,10 +28,10 @@ import (
 // buildGetCertificates returns the certificate that matches the SNI field
 // for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates.
 func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
-	if o.GetCertificates == nil {
+	if o.IdentityOptions.GetIdentityCertificatesForServer == nil {
 		return nil, fmt.Errorf("function GetCertificates must be specified")
 	}
-	certificates, err := o.GetCertificates(clientHello)
+	certificates, err := o.IdentityOptions.GetIdentityCertificatesForServer(clientHello)
 	if err != nil {
 		return nil, err
 	}

From 924b48b046be7c65d28f8de9b83e4394c8e54ffe Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 21 Sep 2020 16:58:50 -0700
Subject: [PATCH 200/481] xds: Use a grpcsync.Event to represent close in EDS
 LB policy (#3895)

---
 xds/internal/balancer/edsbalancer/eds.go | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index ba87c4de8f3c..468c6c5826e9 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -20,7 +20,6 @@
 package edsbalancer
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 
@@ -30,6 +29,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/serviceconfig"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
@@ -52,12 +52,10 @@ type edsBalancerBuilder struct{}
 
 // Build helps implement the balancer.Builder interface.
 func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
-	ctx, cancel := context.WithCancel(context.Background())
 	x := &edsBalancer{
-		ctx:               ctx,
-		cancel:            cancel,
 		cc:                cc,
 		buildOpts:         opts,
+		closed:            grpcsync.NewEvent(),
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
@@ -106,10 +104,8 @@ type edsBalancerImplInterface interface {
 type edsBalancer struct {
 	cc        balancer.ClientConn // *xdsClientConn
 	buildOpts balancer.BuildOptions
-	ctx       context.Context
-	cancel    context.CancelFunc
-
-	logger *grpclog.PrefixLogger
+	closed    *grpcsync.Event
+	logger    *grpclog.PrefixLogger
 
 	// edsBalancer continuously monitor the channels below, and will handle events from them in sync.
 	grpcUpdate        chan interface{}
@@ -135,7 +131,7 @@ func (x *edsBalancer) run() {
 			x.childPolicyUpdate.Load()
 			u := update.(*balancerStateWithPriority)
 			x.edsImpl.updateState(u.priority, u.s)
-		case <-x.ctx.Done():
+		case <-x.closed.Done():
 			x.client.close()
 			x.edsImpl.close()
 			return
@@ -227,21 +223,21 @@ func (x *edsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 	}
 	select {
 	case x.grpcUpdate <- update:
-	case <-x.ctx.Done():
+	case <-x.closed.Done():
 	}
 }
 
 func (x *edsBalancer) ResolverError(err error) {
 	select {
 	case x.grpcUpdate <- err:
-	case <-x.ctx.Done():
+	case <-x.closed.Done():
 	}
 }
 
 func (x *edsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	select {
 	case x.grpcUpdate <- &s:
-	case <-x.ctx.Done():
+	case <-x.closed.Done():
 	}
 	return nil
 }
@@ -254,7 +250,7 @@ type edsUpdate struct {
 func (x *edsBalancer) handleEDSUpdate(resp xdsclient.EndpointsUpdate, err error) {
 	select {
 	case x.xdsClientUpdate <- &edsUpdate{resp: resp, err: err}:
-	case <-x.ctx.Done():
+	case <-x.closed.Done():
 	}
 }
 
@@ -271,6 +267,6 @@ func (x *edsBalancer) enqueueChildBalancerState(p priorityType, s balancer.State
 }
 
 func (x *edsBalancer) Close() {
-	x.cancel()
+	x.closed.Fire()
 	x.logger.Infof("Shutdown")
 }

From d81def4352bca5fad5d6c330236c9e4a8f5abe4e Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 21 Sep 2020 16:59:02 -0700
Subject: [PATCH 201/481] xds: Rename xdsclientWrapper type to xdsClientWrapper
 in EDS. (#3896)

---
 xds/internal/balancer/edsbalancer/eds.go      |  4 ++--
 xds/internal/balancer/edsbalancer/eds_impl.go |  4 ++--
 .../balancer/edsbalancer/eds_impl_test.go     |  2 +-
 xds/internal/balancer/edsbalancer/eds_test.go |  2 +-
 .../edsbalancer/xds_client_wrapper.go         | 24 +++++++++----------
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 468c6c5826e9..6b124c27071d 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -39,7 +39,7 @@ const (
 )
 
 var (
-	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsclientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
 		return newEDSBalancerImpl(cc, enqueueState, xdsClient, logger)
 	}
 )
@@ -112,7 +112,7 @@ type edsBalancer struct {
 	xdsClientUpdate   chan *edsUpdate
 	childPolicyUpdate *buffer.Unbounded
 
-	client  *xdsclientWrapper // may change when passed a different service config
+	client  *xdsClientWrapper // may change when passed a different service config
 	config  *EDSConfig        // may change when passed a different service config
 	edsImpl edsBalancerImplInterface
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 4aaf20925a46..7cbd783aba53 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -63,7 +63,7 @@ type balancerGroupWithConfig struct {
 type edsBalancerImpl struct {
 	cc        balancer.ClientConn
 	logger    *grpclog.PrefixLogger
-	xdsClient *xdsclientWrapper // To fetch the load.Store from.
+	xdsClient *xdsClientWrapper // To fetch the load.Store from.
 
 	enqueueChildBalancerStateUpdate func(priorityType, balancer.State)
 
@@ -98,7 +98,7 @@ type edsBalancerImpl struct {
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
-func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsclientWrapper, logger *grpclog.PrefixLogger) *edsBalancerImpl {
+func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsClientWrapper, logger *grpclog.PrefixLogger) *edsBalancerImpl {
 	edsImpl := &edsBalancerImpl{
 		cc:                 cc,
 		logger:             logger,
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 525da1ba645a..a2c5e8ce358c 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -696,7 +696,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
 	loadStore := &load.Store{}
-	cw := &xdsclientWrapper{
+	cw := &xdsClientWrapper{
 		xdsClient: &loadStoreWrapper{ls: loadStore},
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 0b989bed8320..62075aed29bb 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -229,7 +229,7 @@ func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
 // cleanup.
 func setup(edsLBCh *testutils.Channel, xdsClientCh *testutils.Channel) func() {
 	origNewEDSBalancer := newEDSBalancer
-	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ *xdsclientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
 		edsLB := newFakeEDSBalancer(cc)
 		defer func() { edsLBCh.Send(edsLB) }()
 		return edsLB
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index dd66a4160a7a..0319cde90960 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -45,10 +45,10 @@ var (
 	bootstrapConfigNew = bootstrap.NewConfig
 )
 
-// xdsclientWrapper is responsible for getting the xds client from attributes or
+// xdsClientWrapper is responsible for getting the xds client from attributes or
 // creating a new xds client, and start watching EDS. The given callbacks will
 // be called with EDS updates or errors.
-type xdsclientWrapper struct {
+type xdsClientWrapper struct {
 	logger *grpclog.PrefixLogger
 
 	newEDSUpdate func(xdsclient.EndpointsUpdate, error)
@@ -77,8 +77,8 @@ type xdsclientWrapper struct {
 //
 // The given callbacks won't be called until the underlying xds_client is
 // working and sends updates.
-func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, logger *grpclog.PrefixLogger) *xdsclientWrapper {
-	return &xdsclientWrapper{
+func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, logger *grpclog.PrefixLogger) *xdsClientWrapper {
+	return &xdsClientWrapper{
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
 		bbo:          bbo,
@@ -93,7 +93,7 @@ func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bb
 // attributes), it will be closed.
 //
 // It returns whether xdsClient is replaced.
-func (c *xdsclientWrapper) replaceXDSClient(newClient xdsClientInterface, newBalancerName string) bool {
+func (c *xdsClientWrapper) replaceXDSClient(newClient xdsClientInterface, newBalancerName string) bool {
 	if c.xdsClient == newClient {
 		return false
 	}
@@ -122,7 +122,7 @@ func (c *xdsclientWrapper) replaceXDSClient(newClient xdsClientInterface, newBal
 // the balancerName (from bootstrap file or from service config) changed.
 // - if balancer names are the same, do nothing, and return false
 // - if balancer names are different, create new one, and return true
-func (c *xdsclientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.Attributes) bool {
+func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.Attributes) bool {
 	if attr != nil {
 		if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil {
 			// This will also clear balancerName, to indicate that client is
@@ -167,7 +167,7 @@ func (c *xdsclientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 //
 // This usually means load report needs to be restarted, but this function does
 // NOT do that. Caller needs to call startLoadReport separately.
-func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
+func (c *xdsClientWrapper) startEndpointsWatch(nameToWatch string) {
 	if c.xdsClient == nil {
 		return
 	}
@@ -193,7 +193,7 @@ func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) {
 // Caller can cal this when the loadReportServer name changes, but
 // edsServiceName doesn't (so we only need to restart load reporting, not EDS
 // watch).
-func (c *xdsclientWrapper) startLoadReport(edsServiceNameBeingWatched string, loadReportServer *string) {
+func (c *xdsClientWrapper) startLoadReport(edsServiceNameBeingWatched string, loadReportServer *string) {
 	if c.xdsClient == nil {
 		c.logger.Warningf("xds: xdsClient is nil when trying to start load reporting. This means xdsClient wasn't passed in from the resolver, and xdsClient.New failed")
 		return
@@ -207,7 +207,7 @@ func (c *xdsclientWrapper) startLoadReport(edsServiceNameBeingWatched string, lo
 	}
 }
 
-func (c *xdsclientWrapper) loadStore() *load.Store {
+func (c *xdsClientWrapper) loadStore() *load.Store {
 	if c == nil || c.xdsClient == nil {
 		return nil
 	}
@@ -216,7 +216,7 @@ func (c *xdsclientWrapper) loadStore() *load.Store {
 
 // handleUpdate applies the service config and attributes updates to the client,
 // including updating the xds_client to use, and updating the EDS name to watch.
-func (c *xdsclientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) {
+func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) {
 	clientChanged := c.updateXDSClient(config, attr)
 
 	var (
@@ -254,7 +254,7 @@ func (c *xdsclientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attr
 	}
 }
 
-func (c *xdsclientWrapper) cancelWatch() {
+func (c *xdsClientWrapper) cancelWatch() {
 	c.loadReportServer = nil
 	if c.cancelLoadReport != nil {
 		c.cancelLoadReport()
@@ -265,7 +265,7 @@ func (c *xdsclientWrapper) cancelWatch() {
 	}
 }
 
-func (c *xdsclientWrapper) close() {
+func (c *xdsClientWrapper) close() {
 	c.cancelWatch()
 	if c.xdsClient != nil && c.balancerName != "" {
 		// Only close xdsClient if it's not from attributes.

From 400b4a0a6d5a2f8f2c917a9fb46174a4705015ae Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 22 Sep 2020 09:52:20 -0700
Subject: [PATCH 202/481] binarylog: export Sink (#3879)

---
 .../binarylog_end2end_test.go                 | 14 ++--
 binarylog/sink.go                             | 68 +++++++++++++++++++
 internal/binarylog/method_logger.go           |  2 +-
 internal/binarylog/sink.go                    | 68 +++++++++----------
 4 files changed, 109 insertions(+), 43 deletions(-)
 rename {internal/binarylog => binarylog}/binarylog_end2end_test.go (98%)
 create mode 100644 binarylog/sink.go

diff --git a/internal/binarylog/binarylog_end2end_test.go b/binarylog/binarylog_end2end_test.go
similarity index 98%
rename from internal/binarylog/binarylog_end2end_test.go
rename to binarylog/binarylog_end2end_test.go
index 54dd3da74196..b8caa828a687 100644
--- a/internal/binarylog/binarylog_end2end_test.go
+++ b/binarylog/binarylog_end2end_test.go
@@ -29,11 +29,11 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/binarylog"
 	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
 	"google.golang.org/grpc/grpclog"
-	"google.golang.org/grpc/internal/binarylog"
+	iblog "google.golang.org/grpc/internal/binarylog"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/metadata"
 	testpb "google.golang.org/grpc/stats/grpc_testing"
@@ -53,8 +53,8 @@ func Test(t *testing.T) {
 func init() {
 	// Setting environment variable in tests doesn't work because of the init
 	// orders. Set the loggers directly here.
-	binarylog.SetLogger(binarylog.AllLogger)
-	binarylog.SetDefaultSink(testSink)
+	iblog.SetLogger(iblog.AllLogger)
+	binarylog.SetSink(testSink)
 }
 
 var testSink = &testBinLogSink{}
@@ -503,7 +503,7 @@ func (ed *expectedData) newClientHeaderEntry(client bool, rpcID, inRPCID uint64)
 		Logger:               logger,
 		Payload: &pb.GrpcLogEntry_ClientHeader{
 			ClientHeader: &pb.ClientHeader{
-				Metadata:   binarylog.MdToMetadataProto(testMetadata),
+				Metadata:   iblog.MdToMetadataProto(testMetadata),
 				MethodName: ed.method,
 				Authority:  ed.te.srvAddr,
 			},
@@ -535,7 +535,7 @@ func (ed *expectedData) newServerHeaderEntry(client bool, rpcID, inRPCID uint64)
 		Logger:               logger,
 		Payload: &pb.GrpcLogEntry_ServerHeader{
 			ServerHeader: &pb.ServerHeader{
-				Metadata: binarylog.MdToMetadataProto(testMetadata),
+				Metadata: iblog.MdToMetadataProto(testMetadata),
 			},
 		},
 		Peer: peer,
@@ -643,7 +643,7 @@ func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64
 		Logger:               logger,
 		Payload: &pb.GrpcLogEntry_Trailer{
 			Trailer: &pb.Trailer{
-				Metadata: binarylog.MdToMetadataProto(testTrailerMetadata),
+				Metadata: iblog.MdToMetadataProto(testTrailerMetadata),
 				// st will be nil if err was not a status error, but nil is ok.
 				StatusCode:    uint32(st.Code()),
 				StatusMessage: st.Message(),
diff --git a/binarylog/sink.go b/binarylog/sink.go
new file mode 100644
index 000000000000..db79346a2917
--- /dev/null
+++ b/binarylog/sink.go
@@ -0,0 +1,68 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package binarylog implementation binary logging as defined in
+// https://github.com/grpc/proposal/blob/master/A16-binary-logging.md.
+//
+// Notice: All APIs in this package are experimental.
+package binarylog
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
+	iblog "google.golang.org/grpc/internal/binarylog"
+)
+
+// SetSink sets the destination for the binary log entries.
+//
+// NOTE: this function must only be called during initialization time (i.e. in
+// an init() function), and is not thread-safe.
+func SetSink(s Sink) {
+	if iblog.DefaultSink != nil {
+		iblog.DefaultSink.Close()
+	}
+	iblog.DefaultSink = s
+}
+
+// Sink represents the destination for the binary log entries.
+type Sink interface {
+	// Write marshals the log entry and writes it to the destination. The format
+	// is not specified, but should have sufficient information to rebuild the
+	// entry. Some options are: proto bytes, or proto json.
+	//
+	// Note this function needs to be thread-safe.
+	Write(*pb.GrpcLogEntry) error
+	// Close closes this sink and cleans up resources (e.g. the flushing
+	// goroutine).
+	Close() error
+}
+
+// NewTempFileSink creates a temp file and returns a Sink that writes to this
+// file.
+func NewTempFileSink() (Sink, error) {
+	// Two other options to replace this function:
+	// 1. take filename as input.
+	// 2. export NewBufferedSink().
+	tempFile, err := ioutil.TempFile("/tmp", "grpcgo_binarylog_*.txt")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp file: %v", err)
+	}
+	return iblog.NewBufferedSink(tempFile), nil
+}
diff --git a/internal/binarylog/method_logger.go b/internal/binarylog/method_logger.go
index 5e1083539b49..0cdb41831509 100644
--- a/internal/binarylog/method_logger.go
+++ b/internal/binarylog/method_logger.go
@@ -65,7 +65,7 @@ func newMethodLogger(h, m uint64) *MethodLogger {
 		callID:          idGen.next(),
 		idWithinCallGen: &callIDGenerator{},
 
-		sink: defaultSink, // TODO(blog): make it plugable.
+		sink: DefaultSink, // TODO(blog): make it plugable.
 	}
 }
 
diff --git a/internal/binarylog/sink.go b/internal/binarylog/sink.go
index 835f51040cb0..7d7a3056b71e 100644
--- a/internal/binarylog/sink.go
+++ b/internal/binarylog/sink.go
@@ -21,9 +21,7 @@ package binarylog
 import (
 	"bufio"
 	"encoding/binary"
-	"fmt"
 	"io"
-	"io/ioutil"
 	"sync"
 	"time"
 
@@ -32,20 +30,14 @@ import (
 )
 
 var (
-	defaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp).
+	// DefaultSink is the sink where the logs will be written to. It's exported
+	// for the binarylog package to update.
+	DefaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp).
 )
 
-// SetDefaultSink sets the sink where binary logs will be written to.
-//
-// Not thread safe. Only set during initialization.
-func SetDefaultSink(s Sink) {
-	if defaultSink != nil {
-		defaultSink.Close()
-	}
-	defaultSink = s
-}
-
 // Sink writes log entry into the binary log sink.
+//
+// sink is a copy of the exported binarylog.Sink, to avoid circular dependency.
 type Sink interface {
 	// Write will be called to write the log entry into the sink.
 	//
@@ -66,7 +58,7 @@ func (ns *noopSink) Close() error                 { return nil }
 // message is prefixed with a 4 byte big endian unsigned integer as the length.
 //
 // No buffer is done, Close() doesn't try to close the writer.
-func newWriterSink(w io.Writer) *writerSink {
+func newWriterSink(w io.Writer) Sink {
 	return &writerSink{out: w}
 }
 
@@ -92,17 +84,17 @@ func (ws *writerSink) Write(e *pb.GrpcLogEntry) error {
 
 func (ws *writerSink) Close() error { return nil }
 
-type bufWriteCloserSink struct {
+type bufferedSink struct {
 	mu     sync.Mutex
 	closer io.Closer
-	out    *writerSink   // out is built on buf.
+	out    Sink          // out is built on buf.
 	buf    *bufio.Writer // buf is kept for flush.
 
 	writeStartOnce sync.Once
 	writeTicker    *time.Ticker
 }
 
-func (fs *bufWriteCloserSink) Write(e *pb.GrpcLogEntry) error {
+func (fs *bufferedSink) Write(e *pb.GrpcLogEntry) error {
 	// Start the write loop when Write is called.
 	fs.writeStartOnce.Do(fs.startFlushGoroutine)
 	fs.mu.Lock()
@@ -118,44 +110,50 @@ const (
 	bufFlushDuration = 60 * time.Second
 )
 
-func (fs *bufWriteCloserSink) startFlushGoroutine() {
+func (fs *bufferedSink) startFlushGoroutine() {
 	fs.writeTicker = time.NewTicker(bufFlushDuration)
 	go func() {
 		for range fs.writeTicker.C {
 			fs.mu.Lock()
-			fs.buf.Flush()
+			if err := fs.buf.Flush(); err != nil {
+				grpclogLogger.Warningf("failed to flush to Sink: %v", err)
+			}
 			fs.mu.Unlock()
 		}
 	}()
 }
 
-func (fs *bufWriteCloserSink) Close() error {
+func (fs *bufferedSink) Close() error {
 	if fs.writeTicker != nil {
 		fs.writeTicker.Stop()
 	}
 	fs.mu.Lock()
-	fs.buf.Flush()
-	fs.closer.Close()
-	fs.out.Close()
+	if err := fs.buf.Flush(); err != nil {
+		grpclogLogger.Warningf("failed to flush to Sink: %v", err)
+	}
+	if err := fs.closer.Close(); err != nil {
+		grpclogLogger.Warningf("failed to close the underlying WriterCloser: %v", err)
+	}
+	if err := fs.out.Close(); err != nil {
+		grpclogLogger.Warningf("failed to close the Sink: %v", err)
+	}
 	fs.mu.Unlock()
 	return nil
 }
 
-func newBufWriteCloserSink(o io.WriteCloser) Sink {
+// NewBufferedSink creates a binary log sink with the given WriteCloser.
+//
+// Write() marshals the proto message and writes it to the given writer. Each
+// message is prefixed with a 4 byte big endian unsigned integer as the length.
+//
+// Content is kept in a buffer, and is flushed every 60 seconds.
+//
+// Close closes the WriteCloser.
+func NewBufferedSink(o io.WriteCloser) Sink {
 	bufW := bufio.NewWriter(o)
-	return &bufWriteCloserSink{
+	return &bufferedSink{
 		closer: o,
 		out:    newWriterSink(bufW),
 		buf:    bufW,
 	}
 }
-
-// NewTempFileSink creates a temp file and returns a Sink that writes to this
-// file.
-func NewTempFileSink() (Sink, error) {
-	tempFile, err := ioutil.TempFile("/tmp", "grpcgo_binarylog_*.txt")
-	if err != nil {
-		return nil, fmt.Errorf("failed to create temp file: %v", err)
-	}
-	return newBufWriteCloserSink(tempFile), nil
-}

From 64c4c373506b5b6e25da57e347c11073ee2dd68a Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 22 Sep 2020 10:00:31 -0700
Subject: [PATCH 203/481] xds: Use a grpcsync.Event to represent close in CDS
 LB policy. (#3894)

---
 .../balancer/cdsbalancer/cdsbalancer.go       | 168 ++++++++----------
 1 file changed, 76 insertions(+), 92 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index c6ab764763aa..830b8b33b1b8 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"sync"
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
@@ -29,6 +28,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
@@ -75,6 +75,7 @@ func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.
 		cc:       cc,
 		bOpts:    opts,
 		updateCh: buffer.NewUnbounded(),
+		closed:   grpcsync.NewEvent(),
 	}
 	b.logger = prefixLogger((b))
 	b.logger.Infof("Created")
@@ -137,10 +138,6 @@ type watchUpdate struct {
 	err error
 }
 
-// closeUpdate is an empty struct used to notify the run() goroutine that a
-// Close has been called on the balancer.
-type closeUpdate struct{}
-
 // cdsBalancer implements a CDS based LB policy. It instantiates an EDS based
 // LB policy to further resolve the serviceName received from CDS, into
 // localities and endpoints. Implements the balancer.Balancer interface which
@@ -154,13 +151,8 @@ type cdsBalancer struct {
 	cancelWatch    func()
 	edsLB          balancer.Balancer
 	clusterToWatch string
-
-	logger *grpclog.PrefixLogger
-
-	// The only thing protected by this mutex is the closed boolean. This is
-	// checked by all methods before acting on updates.
-	mu     sync.Mutex
-	closed bool
+	logger         *grpclog.PrefixLogger
+	closed         *grpcsync.Event
 }
 
 // run is a long-running goroutine which handles all updates from gRPC. All
@@ -178,78 +170,80 @@ type cdsBalancer struct {
 //    underlying edsBalancer and is the only way to exit this goroutine.
 func (b *cdsBalancer) run() {
 	for {
-		u := <-b.updateCh.Get()
-		b.updateCh.Load()
-		switch update := u.(type) {
-		case *ccUpdate:
-			// We first handle errors, if any, and then proceed with handling
-			// the update, only if the status quo has changed.
-			if err := update.err; err != nil {
-				b.handleErrorFromUpdate(err, true)
-			}
-			if b.client == update.client && b.clusterToWatch == update.clusterName {
-				break
-			}
-			if update.client != nil {
-				// Since the cdsBalancer doesn't own the xdsClient object, we
-				// don't have to bother about closing the old client here, but
-				// we still need to cancel the watch on the old client.
-				if b.cancelWatch != nil {
-					b.cancelWatch()
+		select {
+		case u := <-b.updateCh.Get():
+			b.updateCh.Load()
+			switch update := u.(type) {
+			case *ccUpdate:
+				// We first handle errors, if any, and then proceed with handling
+				// the update, only if the status quo has changed.
+				if err := update.err; err != nil {
+					b.handleErrorFromUpdate(err, true)
 				}
-				b.client = update.client
-			}
-			if update.clusterName != "" {
-				cancelWatch := b.client.WatchCluster(update.clusterName, b.handleClusterUpdate)
-				b.logger.Infof("Watch started on resource name %v with xds-client %p", update.clusterName, b.client)
-				b.cancelWatch = func() {
-					cancelWatch()
-					b.logger.Infof("Watch cancelled on resource name %v with xds-client %p", update.clusterName, b.client)
+				if b.client == update.client && b.clusterToWatch == update.clusterName {
+					break
 				}
-				b.clusterToWatch = update.clusterName
-			}
-		case *scUpdate:
-			if b.edsLB == nil {
-				b.logger.Errorf("xds: received scUpdate {%+v} with no edsBalancer", update)
-				break
-			}
-			b.edsLB.UpdateSubConnState(update.subConn, update.state)
-		case *watchUpdate:
-			if err := update.err; err != nil {
-				b.logger.Warningf("Watch error from xds-client %p: %v", b.client, err)
-				b.handleErrorFromUpdate(err, false)
-				break
-			}
-
-			b.logger.Infof("Watch update from xds-client %p, content: %+v", b.client, update.cds)
-			// The first good update from the watch API leads to the
-			// instantiation of an edsBalancer. Further updates/errors are
-			// propagated to the existing edsBalancer.
-			if b.edsLB == nil {
-				var err error
-				b.edsLB, err = newEDSBalancer(b.cc, b.bOpts)
+				if update.client != nil {
+					// Since the cdsBalancer doesn't own the xdsClient object, we
+					// don't have to bother about closing the old client here, but
+					// we still need to cancel the watch on the old client.
+					if b.cancelWatch != nil {
+						b.cancelWatch()
+					}
+					b.client = update.client
+				}
+				if update.clusterName != "" {
+					cancelWatch := b.client.WatchCluster(update.clusterName, b.handleClusterUpdate)
+					b.logger.Infof("Watch started on resource name %v with xds-client %p", update.clusterName, b.client)
+					b.cancelWatch = func() {
+						cancelWatch()
+						b.logger.Infof("Watch cancelled on resource name %v with xds-client %p", update.clusterName, b.client)
+					}
+					b.clusterToWatch = update.clusterName
+				}
+			case *scUpdate:
 				if b.edsLB == nil {
-					b.logger.Errorf("Failed to create child policy of type %s, %v", edsName, err)
+					b.logger.Errorf("xds: received scUpdate {%+v} with no edsBalancer", update)
+					break
+				}
+				b.edsLB.UpdateSubConnState(update.subConn, update.state)
+			case *watchUpdate:
+				if err := update.err; err != nil {
+					b.logger.Warningf("Watch error from xds-client %p: %v", b.client, err)
+					b.handleErrorFromUpdate(err, false)
 					break
 				}
-				b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
-			}
-			lbCfg := &edsbalancer.EDSConfig{EDSServiceName: update.cds.ServiceName}
-			if update.cds.EnableLRS {
-				// An empty string here indicates that the edsBalancer
-				// should use the same xDS server for load reporting as
-				// it does for EDS requests/responses.
-				lbCfg.LrsLoadReportingServerName = new(string)
 
+				b.logger.Infof("Watch update from xds-client %p, content: %+v", b.client, update.cds)
+				// The first good update from the watch API leads to the
+				// instantiation of an edsBalancer. Further updates/errors are
+				// propagated to the existing edsBalancer.
+				if b.edsLB == nil {
+					var err error
+					b.edsLB, err = newEDSBalancer(b.cc, b.bOpts)
+					if b.edsLB == nil {
+						b.logger.Errorf("Failed to create child policy of type %s, %v", edsName, err)
+						break
+					}
+					b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
+				}
+				lbCfg := &edsbalancer.EDSConfig{EDSServiceName: update.cds.ServiceName}
+				if update.cds.EnableLRS {
+					// An empty string here indicates that the edsBalancer
+					// should use the same xDS server for load reporting as
+					// it does for EDS requests/responses.
+					lbCfg.LrsLoadReportingServerName = new(string)
+
+				}
+				ccState := balancer.ClientConnState{
+					ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, b.client)},
+					BalancerConfig: lbCfg,
+				}
+				if err := b.edsLB.UpdateClientConnState(ccState); err != nil {
+					b.logger.Errorf("xds: edsBalancer.UpdateClientConnState(%+v) returned error: %v", ccState, err)
+				}
 			}
-			ccState := balancer.ClientConnState{
-				ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, b.client)},
-				BalancerConfig: lbCfg,
-			}
-			if err := b.edsLB.UpdateClientConnState(ccState); err != nil {
-				b.logger.Errorf("xds: edsBalancer.UpdateClientConnState(%+v) returned error: %v", ccState, err)
-			}
-		case *closeUpdate:
+		case <-b.closed.Done():
 			if b.cancelWatch != nil {
 				b.cancelWatch()
 				b.cancelWatch = nil
@@ -309,7 +303,7 @@ func (b *cdsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
 // handleClusterUpdate is the CDS watch API callback. It simply pushes the
 // received information on to the update channel for run() to pick it up.
 func (b *cdsBalancer) handleClusterUpdate(cu xdsclient.ClusterUpdate, err error) {
-	if b.isClosed() {
+	if b.closed.HasFired() {
 		b.logger.Warningf("xds: received cluster update {%+v} after cdsBalancer was closed", cu)
 		return
 	}
@@ -320,7 +314,7 @@ func (b *cdsBalancer) handleClusterUpdate(cu xdsclient.ClusterUpdate, err error)
 // clusterName to watch for in CDS) and the xdsClient object from the
 // xdsResolver.
 func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) error {
-	if b.isClosed() {
+	if b.closed.HasFired() {
 		b.logger.Warningf("xds: received ClientConnState {%+v} after cdsBalancer was closed", state)
 		return errBalancerClosed
 	}
@@ -354,7 +348,7 @@ func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) erro
 
 // ResolverError handles errors reported by the xdsResolver.
 func (b *cdsBalancer) ResolverError(err error) {
-	if b.isClosed() {
+	if b.closed.HasFired() {
 		b.logger.Warningf("xds: received resolver error {%v} after cdsBalancer was closed", err)
 		return
 	}
@@ -364,7 +358,7 @@ func (b *cdsBalancer) ResolverError(err error) {
 
 // UpdateSubConnState handles subConn updates from gRPC.
 func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
-	if b.isClosed() {
+	if b.closed.HasFired() {
 		b.logger.Warningf("xds: received subConn update {%v, %v} after cdsBalancer was closed", sc, state)
 		return
 	}
@@ -373,15 +367,5 @@ func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 
 // Close closes the cdsBalancer and the underlying edsBalancer.
 func (b *cdsBalancer) Close() {
-	b.mu.Lock()
-	b.closed = true
-	b.mu.Unlock()
-	b.updateCh.Put(&closeUpdate{})
-}
-
-func (b *cdsBalancer) isClosed() bool {
-	b.mu.Lock()
-	closed := b.closed
-	b.mu.Unlock()
-	return closed
+	b.closed.Fire()
 }

From 0dc9986169d8a75d8fb5ec837155b701b132839a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 22 Sep 2020 14:26:20 -0700
Subject: [PATCH 204/481] lrs: add a layer for clusters in load store (#3880)

---
 .../balancer/balancergroup/balancergroup.go   | 16 +--
 .../balancergroup/balancergroup_test.go       |  4 +-
 xds/internal/balancer/edsbalancer/eds_impl.go | 12 +--
 .../balancer/edsbalancer/eds_impl_test.go     | 15 +--
 .../edsbalancer/xds_client_wrapper.go         | 98 ++++++++++++-------
 .../edsbalancer/xds_client_wrapper_test.go    | 28 ++----
 .../balancer/edsbalancer/xds_lrs_test.go      |  9 +-
 xds/internal/balancer/lrs/balancer.go         | 95 ++++++++++++++----
 xds/internal/balancer/lrs/balancer_test.go    |  5 +-
 xds/internal/client/client.go                 |  2 +-
 xds/internal/client/load/reporter.go          | 27 +++++
 xds/internal/client/load/store.go             | 69 +++++++++++--
 xds/internal/client/load/store_test.go        |  8 +-
 xds/internal/client/v2/loadreport.go          |  2 +-
 xds/internal/client/v3/loadreport.go          |  2 +-
 xds/internal/testutils/fakeclient/client.go   |  2 +-
 16 files changed, 262 insertions(+), 132 deletions(-)
 create mode 100644 xds/internal/client/load/reporter.go

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index 6923abb0e488..d646355191c3 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -24,6 +24,7 @@ import (
 	"time"
 
 	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
+	"google.golang.org/grpc/xds/internal/client/load"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
@@ -32,13 +33,6 @@ import (
 	"google.golang.org/grpc/resolver"
 )
 
-// loadReporter wraps the methods from the loadStore that are used here.
-type loadReporter interface {
-	CallStarted(locality string)
-	CallFinished(locality string, err error)
-	CallServerLoad(locality, name string, val float64)
-}
-
 // subBalancerWrapper is used to keep the configurations that will be used to start
 // the underlying balancer. It can be called to start/stop the underlying
 // balancer.
@@ -186,7 +180,7 @@ func (sbc *subBalancerWrapper) stopBalancer() {
 type BalancerGroup struct {
 	cc        balancer.ClientConn
 	logger    *grpclog.PrefixLogger
-	loadStore loadReporter
+	loadStore load.PerClusterReporter
 
 	// stateAggregator is where the state/picker updates will be sent to. It's
 	// provided by the parent balancer, to build a picker with all the
@@ -241,7 +235,7 @@ var DefaultSubBalancerCloseTimeout = 15 * time.Minute
 
 // New creates a new BalancerGroup. Note that the BalancerGroup
 // needs to be started to work.
-func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore loadReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
+func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore load.PerClusterReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
 	return &BalancerGroup{
 		cc:        cc,
 		logger:    logger,
@@ -500,10 +494,10 @@ type loadReportPicker struct {
 	p balancer.Picker
 
 	locality  string
-	loadStore loadReporter
+	loadStore load.PerClusterReporter
 }
 
-func newLoadReportPicker(p balancer.Picker, id string, loadStore loadReporter) *loadReportPicker {
+func newLoadReportPicker(p balancer.Picker, id string, loadStore load.PerClusterReporter) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
 		locality:  id,
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index ecfa97c9645b..daf8f8824085 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -70,7 +70,7 @@ func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
 	}
 }
 
-func newTestBalancerGroup(t *testing.T, loadStore *load.Store) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
+func newTestBalancerGroup(t *testing.T, loadStore *load.PerClusterStore) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
@@ -400,7 +400,7 @@ func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
 }
 
 func (s) TestBalancerGroup_LoadReport(t *testing.T) {
-	loadStore := &load.Store{}
+	loadStore := &load.PerClusterStore{}
 	cc, gator, bg := newTestBalancerGroup(t, loadStore)
 
 	backendToBalancerID := make(map[balancer.SubConn]string)
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 7cbd783aba53..1ce79bf61bb6 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -36,6 +36,7 @@ import (
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 )
 
 // TODO: make this a environment variable?
@@ -450,20 +451,13 @@ func (edsImpl *edsBalancerImpl) close() {
 	}
 }
 
-// dropReporter wraps the single method used by the dropPicker to report dropped
-// calls to the load store.
-type dropReporter interface {
-	// CallDropped reports the drop of one RPC with the given category.
-	CallDropped(category string)
-}
-
 type dropPicker struct {
 	drops     []*dropper
 	p         balancer.Picker
-	loadStore dropReporter
+	loadStore load.PerClusterReporter
 }
 
-func newDropPicker(p balancer.Picker, drops []*dropper, loadStore dropReporter) *dropPicker {
+func newDropPicker(p balancer.Picker, drops []*dropper, loadStore load.PerClusterReporter) *dropPicker {
 	return &dropPicker{
 		drops:     drops,
 		p:         p,
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index a2c5e8ce358c..0febc1be3e57 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -682,22 +682,13 @@ func (s) TestDropPicker(t *testing.T) {
 	}
 }
 
-type loadStoreWrapper struct {
-	xdsClientInterface
-	ls *load.Store
-}
-
-func (l *loadStoreWrapper) LoadStore() *load.Store {
-	return l.ls
-}
-
 func (s) TestEDS_LoadReport(t *testing.T) {
 	// We create an xdsClientWrapper with a dummy xdsClientInterface which only
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
-	loadStore := &load.Store{}
+	loadStore := load.NewStore()
 	cw := &xdsClientWrapper{
-		xdsClient: &loadStoreWrapper{ls: loadStore},
+		load: &loadStoreWrapper{store: loadStore, service: testClusterNames[0]},
 	}
 
 	cc := testutils.NewTestClientConn(t)
@@ -745,7 +736,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 		}
 	}
 
-	gotStoreData := loadStore.Stats()
+	gotStoreData := loadStore.PerCluster(testClusterNames[0], "").Stats()
 	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
 		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index 0319cde90960..99d986f59c77 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -19,6 +19,8 @@
 package edsbalancer
 
 import (
+	"sync"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
@@ -45,7 +47,44 @@ var (
 	bootstrapConfigNew = bootstrap.NewConfig
 )
 
-// xdsClientWrapper is responsible for getting the xds client from attributes or
+type loadStoreWrapper struct {
+	mu      sync.RWMutex
+	store   *load.Store
+	service string
+}
+
+func (lsw *loadStoreWrapper) update(store *load.Store, service string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	lsw.store = store
+	lsw.service = service
+}
+
+func (lsw *loadStoreWrapper) CallStarted(locality string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.service, "").CallStarted(locality)
+}
+
+func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.service, "").CallFinished(locality, err)
+}
+
+func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.service, "").CallServerLoad(locality, name, val)
+}
+
+func (lsw *loadStoreWrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.service, "").CallDropped(category)
+}
+
+// xdsclientWrapper is responsible for getting the xds client from attributes or
 // creating a new xds client, and start watching EDS. The given callbacks will
 // be called with EDS updates or errors.
 type xdsClientWrapper struct {
@@ -58,6 +97,7 @@ type xdsClientWrapper struct {
 	// xdsClient could come from attributes, or created with balancerName.
 	xdsClient xdsClientInterface
 
+	load *loadStoreWrapper
 	// edsServiceName is the edsServiceName currently being watched, not
 	// necessary the edsServiceName from service config.
 	//
@@ -82,6 +122,7 @@ func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bb
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
 		bbo:          bbo,
+		load:         &loadStoreWrapper{},
 	}
 }
 
@@ -167,12 +208,11 @@ func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 //
 // This usually means load report needs to be restarted, but this function does
 // NOT do that. Caller needs to call startLoadReport separately.
-func (c *xdsClientWrapper) startEndpointsWatch(nameToWatch string) {
+func (c *xdsClientWrapper) startEndpointsWatch() {
 	if c.xdsClient == nil {
 		return
 	}
 
-	c.edsServiceName = nameToWatch
 	if c.cancelEndpointsWatch != nil {
 		c.cancelEndpointsWatch()
 	}
@@ -193,7 +233,7 @@ func (c *xdsClientWrapper) startEndpointsWatch(nameToWatch string) {
 // Caller can cal this when the loadReportServer name changes, but
 // edsServiceName doesn't (so we only need to restart load reporting, not EDS
 // watch).
-func (c *xdsClientWrapper) startLoadReport(edsServiceNameBeingWatched string, loadReportServer *string) {
+func (c *xdsClientWrapper) startLoadReport(loadReportServer *string) {
 	if c.xdsClient == nil {
 		c.logger.Warningf("xds: xdsClient is nil when trying to start load reporting. This means xdsClient wasn't passed in from the resolver, and xdsClient.New failed")
 		return
@@ -203,15 +243,15 @@ func (c *xdsClientWrapper) startLoadReport(edsServiceNameBeingWatched string, lo
 	}
 	c.loadReportServer = loadReportServer
 	if c.loadReportServer != nil {
-		c.cancelLoadReport = c.xdsClient.ReportLoad(*c.loadReportServer, edsServiceNameBeingWatched)
+		c.cancelLoadReport = c.xdsClient.ReportLoad(*c.loadReportServer, c.edsServiceName)
 	}
 }
 
-func (c *xdsClientWrapper) loadStore() *load.Store {
-	if c == nil || c.xdsClient == nil {
+func (c *xdsClientWrapper) loadStore() load.PerClusterReporter {
+	if c == nil || c.load.store == nil {
 		return nil
 	}
-	return c.xdsClient.LoadStore()
+	return c.load
 }
 
 // handleUpdate applies the service config and attributes updates to the client,
@@ -219,38 +259,26 @@ func (c *xdsClientWrapper) loadStore() *load.Store {
 func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) {
 	clientChanged := c.updateXDSClient(config, attr)
 
-	var (
-		restartEndpointsWatch bool
-		restartLoadReport     bool
-	)
-
-	// The clusterName to watch should come from CDS response, via service
-	// config. If it's an empty string, fallback user's dial target.
-	nameToWatch := config.EDSServiceName
-	if nameToWatch == "" {
-		c.logger.Warningf("eds: cluster name to watch is an empty string. Fallback to user's dial target")
-		nameToWatch = c.bbo.Target.Endpoint
-	}
-
 	// Need to restart EDS watch when one of the following happens:
 	// - the xds_client is updated
 	// - the xds_client didn't change, but the edsServiceName changed
-	//
-	// Only need to restart load reporting when:
-	// - no need to restart EDS, but loadReportServer name changed
-	if clientChanged || c.edsServiceName != nameToWatch {
-		restartEndpointsWatch = true
-		restartLoadReport = true
-	} else if !equalStringPointers(c.loadReportServer, config.LrsLoadReportingServerName) {
-		restartLoadReport = true
-	}
-
-	if restartEndpointsWatch {
-		c.startEndpointsWatch(nameToWatch)
+	if clientChanged || c.edsServiceName != config.EDSServiceName {
+		c.edsServiceName = config.EDSServiceName
+		c.startEndpointsWatch()
+		// TODO: this update for the LRS service name is too early. It should
+		// only apply to the new EDS response. But this is applied to the RPCs
+		// before the new EDS response. To fully fix this, the EDS balancer
+		// needs to do a graceful switch to another EDS implementation.
+		//
+		// This is OK for now, because we don't actually expect edsServiceName
+		// to change. Fix this (a bigger change) will happen later.
+		c.load.update(c.xdsClient.LoadStore(), c.edsServiceName)
 	}
 
-	if restartLoadReport {
-		c.startLoadReport(nameToWatch, config.LrsLoadReportingServerName)
+	// Only need to restart load reporting when:
+	// - the loadReportServer name changed
+	if !equalStringPointers(c.loadReportServer, config.LrsLoadReportingServerName) {
+		c.startLoadReport(config.LrsLoadReportingServerName)
 	}
 }
 
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index cf39597c53d8..c8dac9c9925e 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -107,36 +107,20 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 	}
 	defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
 
-	// Update with an empty edsServiceName should trigger an EDS watch
-	// for the user's dial target.
-	cw.handleUpdate(&EDSConfig{
-		BalancerName:   fakeServer.Address,
-		EDSServiceName: "",
-	}, nil)
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := fakeServer.NewConnChan.Receive(ctx); err != nil {
-		t.Fatal("Failed to connect to fake server")
-	}
-	t.Log("Client connection established to fake server...")
-	if err := verifyExpectedRequests(fakeServer, testServiceName); err != nil {
-		t.Fatal(err)
-	}
-
 	// Update with an non-empty edsServiceName should trigger an EDS watch for
-	// the same. The previously registered watch will be cancelled, which will
-	// result in an EDS request with no resource names being sent to the server.
+	// the same.
 	cw.handleUpdate(&EDSConfig{
 		BalancerName:   fakeServer.Address,
 		EDSServiceName: "foobar-1",
 	}, nil)
-	if err := verifyExpectedRequests(fakeServer, "", "foobar-1"); err != nil {
+	if err := verifyExpectedRequests(fakeServer, "foobar-1"); err != nil {
 		t.Fatal(err)
 	}
 
-	// Also test the case where the edsServerName changes from one
-	// non-empty name to another, and make sure a new watch is
-	// registered.
+	// Also test the case where the edsServerName changes from one non-empty
+	// name to another, and make sure a new watch is registered. The previously
+	// registered watch will be cancelled, which will result in an EDS request
+	// with no resource names being sent to the server.
 	cw.handleUpdate(&EDSConfig{
 		BalancerName:   fakeServer.Address,
 		EDSServiceName: "foobar-2",
diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
index 92de46360160..8d888ec6f3b0 100644
--- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
@@ -35,7 +35,7 @@ import (
 func (s) TestXDSLoadReporting(t *testing.T) {
 	builder := balancer.Get(edsName)
 	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
+	edsB, ok := builder.Build(cc, balancer.BuildOptions{}).(*edsBalancer)
 	if !ok {
 		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
 	}
@@ -43,8 +43,11 @@ func (s) TestXDSLoadReporting(t *testing.T) {
 
 	xdsC := fakeclient.NewClient()
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
-		BalancerConfig: &EDSConfig{LrsLoadReportingServerName: new(string)},
+		ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
+		BalancerConfig: &EDSConfig{
+			EDSServiceName:             testEDSClusterName,
+			LrsLoadReportingServerName: new(string),
+		},
 	}); err != nil {
 		t.Fatal(err)
 	}
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index e214c4667046..b5bf3001fccf 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -22,6 +22,7 @@ package lrs
 import (
 	"encoding/json"
 	"fmt"
+	"sync"
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
@@ -45,7 +46,7 @@ func (l *lrsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balanc
 		cc:        cc,
 		buildOpts: opts,
 	}
-	b.client = &xdsClientWrapper{}
+	b.client = newXDSClientWrapper()
 	b.logger = prefixLogger(b)
 	b.logger.Infof("Created")
 	return b
@@ -123,11 +124,11 @@ func (b *lrsBalancer) Close() {
 
 type ccWrapper struct {
 	balancer.ClientConn
-	loadStore  *load.Store
+	loadStore  load.PerClusterReporter
 	localityID *internal.LocalityID
 }
 
-func newCCWrapper(cc balancer.ClientConn, loadStore *load.Store, localityID *internal.LocalityID) *ccWrapper {
+func newCCWrapper(cc balancer.ClientConn, loadStore load.PerClusterReporter, localityID *internal.LocalityID) *ccWrapper {
 	return &ccWrapper{
 		ClientConn: cc,
 		loadStore:  loadStore,
@@ -148,24 +149,73 @@ type xdsClientInterface interface {
 	Close()
 }
 
+type loadStoreWrapper struct {
+	mu         sync.RWMutex
+	store      *load.Store
+	cluster    string
+	edsService string
+}
+
+func (lsw *loadStoreWrapper) update(store *load.Store, cluster, edsService string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	lsw.store = store
+	lsw.cluster = cluster
+	lsw.edsService = edsService
+}
+
+func (lsw *loadStoreWrapper) CallStarted(locality string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallStarted(locality)
+}
+
+func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallFinished(locality, err)
+}
+
+func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallServerLoad(locality, name, val)
+}
+
+func (lsw *loadStoreWrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallDropped(category)
+}
+
 type xdsClientWrapper struct {
 	c                xdsClientInterface
 	cancelLoadReport func()
 	clusterName      string
+	edsServiceName   string
 	lrsServerName    string
+	load             *loadStoreWrapper
+}
+
+func newXDSClientWrapper() *xdsClientWrapper {
+	return &xdsClientWrapper{
+		load: &loadStoreWrapper{},
+	}
 }
 
 // update checks the config and xdsclient, and decides whether it needs to
 // restart the load reporting stream.
-//
-// TODO: refactor lrs to share one stream instead of one per EDS.
 func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attributes) {
-	var restartLoadReport bool
+	var (
+		restartLoadReport bool
+		updateLoadStore   bool
+	)
 	if attr != nil {
 		if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil {
 			if w.c != clientFromAttr {
 				// xds client is different, restart.
 				restartLoadReport = true
+				updateLoadStore = true
 				w.c = clientFromAttr
 			}
 		}
@@ -173,16 +223,13 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attribut
 
 	// ClusterName is different, restart. ClusterName is from ClusterName and
 	// EdsServiceName.
-	//
-	// TODO: LRS request actually has separate fields from these two values.
-	// Update lrs.Store to set both.
-	newClusterName := newConfig.EdsServiceName
-	if newClusterName == "" {
-		newClusterName = newConfig.ClusterName
+	if w.clusterName != newConfig.ClusterName {
+		updateLoadStore = true
+		w.clusterName = newConfig.ClusterName
 	}
-	if w.clusterName != newClusterName {
-		restartLoadReport = true
-		w.clusterName = newClusterName
+	if w.edsServiceName != newConfig.EdsServiceName {
+		updateLoadStore = true
+		w.edsServiceName = newConfig.EdsServiceName
 	}
 
 	if w.lrsServerName != newConfig.LrsLoadReportingServerName {
@@ -192,6 +239,18 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attribut
 		w.lrsServerName = newConfig.LrsLoadReportingServerName
 	}
 
+	// This updates the clusterName and serviceName that will reported for the
+	// loads. The update here is too early, the perfect timing is when the
+	// picker is updated with the new connection. But from this balancer's point
+	// of view, it's impossible to tell.
+	//
+	// On the other hand, this will almost never happen. Each LRS policy
+	// shouldn't get updated config. The parent should do a graceful switch when
+	// the clusterName or serviceName is changed.
+	if updateLoadStore {
+		w.load.update(w.c.LoadStore(), w.clusterName, w.edsServiceName)
+	}
+
 	if restartLoadReport {
 		if w.cancelLoadReport != nil {
 			w.cancelLoadReport()
@@ -203,11 +262,11 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attribut
 	}
 }
 
-func (w *xdsClientWrapper) loadStore() *load.Store {
-	if w.c == nil {
+func (w *xdsClientWrapper) loadStore() load.PerClusterReporter {
+	if w.load.store == nil {
 		return nil
 	}
-	return w.c.LoadStore()
+	return w.load
 }
 
 func (w *xdsClientWrapper) close() {
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index 772dcb2656b1..386b1431eaa6 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -65,7 +65,8 @@ func TestLoadReporting(t *testing.T) {
 			Attributes: attributes.New(xdsinternal.XDSClientID, xdsC),
 		},
 		BalancerConfig: &lbConfig{
-			EdsServiceName:             testClusterName,
+			ClusterName:                testClusterName,
+			EdsServiceName:             testServiceName,
 			LrsLoadReportingServerName: testLRSServerName,
 			Locality:                   testLocality,
 			ChildPolicy: &internalserviceconfig.BalancerConfig{
@@ -115,7 +116,7 @@ func TestLoadReporting(t *testing.T) {
 	if loadStore == nil {
 		t.Fatal("loadStore is nil in xdsClient")
 	}
-	sd := loadStore.Stats()
+	sd := loadStore.PerCluster(testClusterName, testServiceName).Stats()
 	localityData, ok := sd.LocalityStats[testLocality.String()]
 	if !ok {
 		t.Fatalf("loads for %v not found in store", testLocality)
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 0f093c20b9e8..b2afed0c0140 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -348,7 +348,7 @@ func New(opts Options) (*Client, error) {
 	c := &Client{
 		done:      grpcsync.NewEvent(),
 		opts:      opts,
-		loadStore: &load.Store{},
+		loadStore: load.NewStore(),
 
 		updateCh:    buffer.NewUnbounded(),
 		ldsWatchers: make(map[string]map[*watchInfo]bool),
diff --git a/xds/internal/client/load/reporter.go b/xds/internal/client/load/reporter.go
new file mode 100644
index 000000000000..67e29e5bae13
--- /dev/null
+++ b/xds/internal/client/load/reporter.go
@@ -0,0 +1,27 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package load
+
+// PerClusterReporter wraps the methods from the loadStore that are used here.
+type PerClusterReporter interface {
+	CallStarted(locality string)
+	CallFinished(locality string, err error)
+	CallServerLoad(locality, name string, val float64)
+	CallDropped(category string)
+}
diff --git a/xds/internal/client/load/store.go b/xds/internal/client/load/store.go
index 034f47f8ee29..c0c741f670f8 100644
--- a/xds/internal/client/load/store.go
+++ b/xds/internal/client/load/store.go
@@ -24,10 +24,59 @@ import (
 
 const negativeOneUInt64 = ^uint64(0)
 
-// Store is a repository for LB policy implementations to report store load
-// data. It is safe for concurrent use.
-//
-// A zero Store is empty and ready for use.
+// A pair of cluster and service name. The same cluster can be used by multiple
+// services, and one service can use multiple clusters. So we need a pair with
+// both name to accurately indicate where the load belongs.
+type storeKey struct {
+	cluster string
+	service string
+}
+
+// Store keeps the loads for multiple clusters and services to be reported via
+// LRS. It is safe for concurrent use.
+type Store struct {
+	mu       sync.RWMutex
+	clusters map[storeKey]*PerClusterStore
+}
+
+// NewStore creates a Store.
+func NewStore() *Store {
+	return &Store{
+		clusters: make(map[storeKey]*PerClusterStore),
+	}
+}
+
+// PerCluster returns the PerClusterStore for the given clusterName +
+// serviceName.
+func (ls *Store) PerCluster(clusterName, serviceName string) *PerClusterStore {
+	if ls == nil {
+		return nil
+	}
+
+	k := storeKey{
+		cluster: clusterName,
+		service: serviceName,
+	}
+
+	ls.mu.RLock()
+	if p, ok := ls.clusters[k]; ok {
+		ls.mu.RUnlock()
+		return p
+	}
+	ls.mu.RUnlock()
+
+	ls.mu.Lock()
+	defer ls.mu.Unlock()
+	p, ok := ls.clusters[k]
+	if !ok {
+		p = &PerClusterStore{}
+		ls.clusters[k] = p
+	}
+	return p
+}
+
+// PerClusterStore is a repository for LB policy implementations to report store
+// load data. It is safe for concurrent use.
 //
 // TODO(easwars): Use regular maps with mutexes instead of sync.Map here. The
 // latter is optimized for two common use cases: (1) when the entry for a given
@@ -38,7 +87,7 @@ const negativeOneUInt64 = ^uint64(0)
 // RWMutex.
 // Neither of these conditions are met here, and we should transition to a
 // regular map with a mutex for better type safety.
-type Store struct {
+type PerClusterStore struct {
 	drops            sync.Map // map[string]*uint64
 	localityRPCCount sync.Map // map[string]*rpcCountData
 }
@@ -47,7 +96,7 @@ type Store struct {
 // updates are done atomically.
 
 // CallDropped adds one drop record with the given category to store.
-func (ls *Store) CallDropped(category string) {
+func (ls *PerClusterStore) CallDropped(category string) {
 	if ls == nil {
 		return
 	}
@@ -61,7 +110,7 @@ func (ls *Store) CallDropped(category string) {
 }
 
 // CallStarted adds one call started record for the given locality.
-func (ls *Store) CallStarted(locality string) {
+func (ls *PerClusterStore) CallStarted(locality string) {
 	if ls == nil {
 		return
 	}
@@ -76,7 +125,7 @@ func (ls *Store) CallStarted(locality string) {
 
 // CallFinished adds one call finished record for the given locality.
 // For successful calls, err needs to be nil.
-func (ls *Store) CallFinished(locality string, err error) {
+func (ls *PerClusterStore) CallFinished(locality string, err error) {
 	if ls == nil {
 		return
 	}
@@ -97,7 +146,7 @@ func (ls *Store) CallFinished(locality string, err error) {
 
 // CallServerLoad adds one server load record for the given locality. The
 // load type is specified by desc, and its value by val.
-func (ls *Store) CallServerLoad(locality, name string, d float64) {
+func (ls *PerClusterStore) CallServerLoad(locality, name string, d float64) {
 	if ls == nil {
 		return
 	}
@@ -158,7 +207,7 @@ func newStoreData() *Data {
 
 // Stats returns and resets all loads reported to the store, except inProgress
 // rpc counts.
-func (ls *Store) Stats() *Data {
+func (ls *PerClusterStore) Stats() *Data {
 	if ls == nil {
 		return nil
 	}
diff --git a/xds/internal/client/load/store_test.go b/xds/internal/client/load/store_test.go
index 190b826a64e5..df9aed7c031a 100644
--- a/xds/internal/client/load/store_test.go
+++ b/xds/internal/client/load/store_test.go
@@ -56,7 +56,7 @@ func TestDrops(t *testing.T) {
 		}
 	)
 
-	ls := Store{}
+	ls := PerClusterStore{}
 	var wg sync.WaitGroup
 	for category, count := range drops {
 		for i := 0; i < count; i++ {
@@ -118,7 +118,7 @@ func TestLocalityStats(t *testing.T) {
 		}
 	)
 
-	ls := Store{}
+	ls := PerClusterStore{}
 	var wg sync.WaitGroup
 	for locality, data := range localityData {
 		wg.Add(data.start)
@@ -211,7 +211,7 @@ func TestResetAfterStats(t *testing.T) {
 		}
 	)
 
-	reportLoad := func(ls *Store) {
+	reportLoad := func(ls *PerClusterStore) {
 		for category, count := range drops {
 			for i := 0; i < count; i++ {
 				ls.CallDropped(category)
@@ -233,7 +233,7 @@ func TestResetAfterStats(t *testing.T) {
 		}
 	}
 
-	ls := Store{}
+	ls := PerClusterStore{}
 	reportLoad(&ls)
 	gotStoreData := ls.Stats()
 	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
diff --git a/xds/internal/client/v2/loadreport.go b/xds/internal/client/v2/loadreport.go
index d56ed3349ee8..899cb1b5543b 100644
--- a/xds/internal/client/v2/loadreport.go
+++ b/xds/internal/client/v2/loadreport.go
@@ -121,7 +121,7 @@ func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string)
 		localityStats []*v2endpointpb.UpstreamLocalityStats
 	)
 
-	sd := v2c.loadStore.Stats()
+	sd := v2c.loadStore.PerCluster(clusterName, "").Stats()
 	for category, count := range sd.Drops {
 		droppedReqs = append(droppedReqs, &v2endpointpb.ClusterStats_DroppedRequests{
 			Category:     category,
diff --git a/xds/internal/client/v3/loadreport.go b/xds/internal/client/v3/loadreport.go
index f2dc77855a3b..1997277886b7 100644
--- a/xds/internal/client/v3/loadreport.go
+++ b/xds/internal/client/v3/loadreport.go
@@ -121,7 +121,7 @@ func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string)
 		localityStats []*v3endpointpb.UpstreamLocalityStats
 	)
 
-	sd := v3c.loadStore.Stats()
+	sd := v3c.loadStore.PerCluster(clusterName, "").Stats()
 	for category, count := range sd.Drops {
 		droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{
 			Category:     category,
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 972dd21a9dc4..cd2710e612aa 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -214,6 +214,6 @@ func NewClientWithName(name string) *Client {
 		edsCancelCh:  testutils.NewChannel(),
 		loadReportCh: testutils.NewChannel(),
 		closeCh:      testutils.NewChannel(),
-		loadStore:    &load.Store{},
+		loadStore:    load.NewStore(),
 	}
 }

From e4ae8742c7e7cd8caaeb872b29a0b5a43576e4ae Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 22 Sep 2020 15:01:00 -0700
Subject: [PATCH 205/481] xds/cdsbalancer: Use testutils.TestClientConn.
 (#3899)

---
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 53 ++++++++-----------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 689a39e47fb8..3044826c0691 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -37,6 +37,7 @@ import (
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
@@ -54,22 +55,6 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-type testClientConn struct {
-	balancer.ClientConn
-
-	newPickerCh *testutils.Channel // The last picker updated.
-}
-
-func newTestClientConn() *testClientConn {
-	return &testClientConn{
-		newPickerCh: testutils.NewChannelWithSize(1),
-	}
-}
-
-func (tcc *testClientConn) UpdateState(bs balancer.State) {
-	tcc.newPickerCh.Replace(bs)
-}
-
 // cdsWatchInfo wraps the update and the error sent in a CDS watch callback.
 type cdsWatchInfo struct {
 	update xdsclient.ClusterUpdate
@@ -226,9 +211,9 @@ func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.Clie
 
 // setup creates a cdsBalancer and an edsBalancer (and overrides the
 // newEDSBalancer function to return it), and also returns a cleanup function.
-func setup() (*cdsBalancer, *testEDSBalancer, *testClientConn, func()) {
+func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
 	builder := cdsBB{}
-	tcc := newTestClientConn()
+	tcc := xdstestutils.NewTestClientConn(t)
 	cdsB := builder.Build(tcc, balancer.BuildOptions{})
 
 	edsB := newTestEDSBalancer()
@@ -244,11 +229,11 @@ func setup() (*cdsBalancer, *testEDSBalancer, *testClientConn, func()) {
 
 // setupWithWatch does everything that setup does, and also pushes a ClientConn
 // update to the cdsBalancer and waits for a CDS watch call to be registered.
-func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *testClientConn, func()) {
+func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
 	t.Helper()
 
 	xdsC := fakeclient.NewClient()
-	cdsB, edsB, tcc, cancel := setup()
+	cdsB, edsB, tcc, cancel := setup(t)
 	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
@@ -316,7 +301,7 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			cdsB, _, _, cancel := setup()
+			cdsB, _, _, cancel := setup(t)
 			defer func() {
 				cancel()
 				cdsB.Close()
@@ -345,7 +330,7 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 // TestUpdateClientConnStateAfterClose invokes the UpdateClientConnState method
 // on the cdsBalancer after close and verifies that it returns an error.
 func (s) TestUpdateClientConnStateAfterClose(t *testing.T) {
-	cdsB, _, _, cancel := setup()
+	cdsB, _, _, cancel := setup(t)
 	defer cancel()
 	cdsB.Close()
 
@@ -435,13 +420,15 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	if err := edsB.waitForResolverError(err1); err == nil {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
-	state, err := tcc.newPickerCh.Receive(ctx)
-	if err != nil {
+
+	var picker balancer.Picker
+	timer := time.NewTimer(defaultTestTimeout)
+	select {
+	case <-timer.C:
 		t.Fatalf("failed to get picker, expect an error picker")
+	case picker = <-tcc.NewPickerCh:
+		timer.Stop()
 	}
-	picker := state.(balancer.State).Picker
 	if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
 		t.Fatalf("want picker to always fail, got nil")
 	}
@@ -500,13 +487,15 @@ func (s) TestResolverError(t *testing.T) {
 	if err := edsB.waitForResolverError(err1); err == nil {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
-	state, err := tcc.newPickerCh.Receive(ctx)
-	if err != nil {
+
+	var picker balancer.Picker
+	timer := time.NewTimer(defaultTestTimeout)
+	select {
+	case <-timer.C:
 		t.Fatalf("failed to get picker, expect an error picker")
+	case picker = <-tcc.NewPickerCh:
+		timer.Stop()
 	}
-	picker := state.(balancer.State).Picker
 	if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
 		t.Fatalf("want picker to always fail, got nil")
 	}

From 659b82911da86d34b883807efe1092ba80d7a274 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 22 Sep 2020 15:17:47 -0700
Subject: [PATCH 206/481] internal: regenerate pb.go (#3893)

---
 .../grpclb/grpc_lb_v1/load_balancer_grpc.pb.go     |  2 +-
 .../internal/proto/grpc_lookup_v1/rls_grpc.pb.go   |  2 +-
 channelz/grpc_channelz_v1/channelz_grpc.pb.go      | 14 +++++++-------
 .../internal/proto/grpc_gcp/handshaker_grpc.pb.go  |  2 +-
 .../meshca/internal/v1/meshca_grpc.pb.go           |  2 +-
 health/grpc_health_v1/health_grpc.pb.go            |  4 ++--
 profiling/proto/service_grpc.pb.go                 |  4 ++--
 .../grpc_reflection_v1alpha/reflection_grpc.pb.go  |  2 +-
 8 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index 62bd5bef8a04..ecbb77509ea3 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -140,7 +140,7 @@ type LoadBalancerServer interface {
 type UnimplementedLoadBalancerServer struct {
 }
 
-func (*UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
+func (UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
 	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
 }
 
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index 6952ce62c923..93c95132b520 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -106,7 +106,7 @@ type RouteLookupServiceServer interface {
 type UnimplementedRouteLookupServiceServer struct {
 }
 
-func (*UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
+func (UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
 }
 
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index 791151fdd7f8..e56d7353d8f3 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -379,25 +379,25 @@ type ChannelzServer interface {
 type UnimplementedChannelzServer struct {
 }
 
-func (*UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
+func (UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
 }
-func (*UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
+func (UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
 }
-func (*UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
+func (UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
 }
-func (*UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
+func (UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
 }
-func (*UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
+func (UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
 }
-func (*UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
+func (UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
 }
-func (*UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
+func (UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
 }
 
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index d2f2971e2b3e..2b06da50df2f 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -155,7 +155,7 @@ type HandshakerServiceServer interface {
 type UnimplementedHandshakerServiceServer struct {
 }
 
-func (*UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
+func (UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
 	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
 }
 
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index 4dbb2ac05454..35b3681491de 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -109,7 +109,7 @@ type MeshCertificateServiceServer interface {
 type UnimplementedMeshCertificateServiceServer struct {
 }
 
-func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
+func (UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
 }
 
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index fa3a6a775749..e7fda8735b22 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -225,10 +225,10 @@ type HealthServer interface {
 type UnimplementedHealthServer struct {
 }
 
-func (*UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
+func (UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
 }
-func (*UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
+func (UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
 	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
 }
 
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index 02783c9b864c..a1993dc3b816 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -154,10 +154,10 @@ type ProfilingServer interface {
 type UnimplementedProfilingServer struct {
 }
 
-func (*UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
+func (UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
 }
-func (*UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
+func (UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
 }
 
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index eface9e3262b..803e3a45ff64 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -143,7 +143,7 @@ type ServerReflectionServer interface {
 type UnimplementedServerReflectionServer struct {
 }
 
-func (*UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
+func (UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
 	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
 }
 

From 4e932bbcb079d1fc8cdfe8a62adca81fe1371165 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 22 Sep 2020 16:00:38 -0700
Subject: [PATCH 207/481] xds/testutils: Minor cleanup. (#3898)

---
 xds/internal/testutils/balancer.go | 116 ++++-------------------------
 xds/internal/testutils/wrr.go      |  68 +++++++++++++++++
 2 files changed, 84 insertions(+), 100 deletions(-)
 create mode 100644 xds/internal/testutils/wrr.go

diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index ecb21e6202e7..e655ffec8c51 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -20,27 +20,29 @@
 package testutils
 
 import (
-	"context"
 	"fmt"
-	"sync"
 	"testing"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/xds/internal"
 )
 
-const testSubConnsCount = 16
+// TestSubConnsCount is the number of TestSubConns initialized as part of
+// package init.
+const TestSubConnsCount = 16
+
+// testingLogger wraps the logging methods from testing.T.
+type testingLogger interface {
+	Log(args ...interface{})
+	Logf(format string, args ...interface{})
+}
 
 // TestSubConns contains a list of SubConns to be used in tests.
 var TestSubConns []*TestSubConn
 
 func init() {
-	for i := 0; i < testSubConnsCount; i++ {
+	for i := 0; i < TestSubConnsCount; i++ {
 		TestSubConns = append(TestSubConns, &TestSubConn{
 			id: fmt.Sprintf("sc%d", i),
 		})
@@ -65,7 +67,7 @@ func (tsc *TestSubConn) String() string {
 
 // TestClientConn is a mock balancer.ClientConn used in tests.
 type TestClientConn struct {
-	t *testing.T // For logging only.
+	logger testingLogger
 
 	NewSubConnAddrsCh chan []resolver.Address // the last 10 []Address to create subconn.
 	NewSubConnCh      chan balancer.SubConn   // the last 10 subconn created.
@@ -80,7 +82,7 @@ type TestClientConn struct {
 // NewTestClientConn creates a TestClientConn.
 func NewTestClientConn(t *testing.T) *TestClientConn {
 	return &TestClientConn{
-		t: t,
+		logger: t,
 
 		NewSubConnAddrsCh: make(chan []resolver.Address, 10),
 		NewSubConnCh:      make(chan balancer.SubConn, 10),
@@ -96,7 +98,7 @@ func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubCon
 	sc := TestSubConns[tcc.subConnIdx]
 	tcc.subConnIdx++
 
-	tcc.t.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc)
+	tcc.logger.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc)
 	select {
 	case tcc.NewSubConnAddrsCh <- a:
 	default:
@@ -112,7 +114,7 @@ func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubCon
 
 // RemoveSubConn removes the SubConn.
 func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
-	tcc.t.Logf("testClientCOnn: RemoveSubConn(%p)", sc)
+	tcc.logger.Logf("testClientCOnn: RemoveSubConn(%p)", sc)
 	select {
 	case tcc.RemoveSubConnCh <- sc:
 	default:
@@ -122,12 +124,12 @@ func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
 // UpdateBalancerState implements balancer.Balancer API. It will be removed when
 // switching to the new balancer interface.
 func (tcc *TestClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) {
-	tcc.t.Fatal("not implemented")
+	panic("not implemented")
 }
 
 // UpdateState updates connectivity state and picker.
 func (tcc *TestClientConn) UpdateState(bs balancer.State) {
-	tcc.t.Logf("testClientConn: UpdateState(%v)", bs)
+	tcc.logger.Logf("testClientConn: UpdateState(%v)", bs)
 	select {
 	case <-tcc.NewStateCh:
 	default:
@@ -151,49 +153,6 @@ func (tcc *TestClientConn) Target() string {
 	panic("not implemented")
 }
 
-// TestServerLoad is testing Load for testing LRS.
-type TestServerLoad struct {
-	Name string
-	D    float64
-}
-
-// TestLoadStore is a load store to be used in tests.
-type TestLoadStore struct {
-	CallsStarted []internal.LocalityID
-	CallsEnded   []internal.LocalityID
-	CallsCost    []TestServerLoad
-}
-
-// NewTestLoadStore creates a new TestLoadStore.
-func NewTestLoadStore() *TestLoadStore {
-	return &TestLoadStore{}
-}
-
-// CallDropped records a call dropped.
-func (*TestLoadStore) CallDropped(category string) {
-	panic("not implemented")
-}
-
-// CallStarted records a call started.
-func (tls *TestLoadStore) CallStarted(l internal.LocalityID) {
-	tls.CallsStarted = append(tls.CallsStarted, l)
-}
-
-// CallFinished records a call finished.
-func (tls *TestLoadStore) CallFinished(l internal.LocalityID, err error) {
-	tls.CallsEnded = append(tls.CallsEnded, l)
-}
-
-// CallServerLoad records a call server load.
-func (tls *TestLoadStore) CallServerLoad(l internal.LocalityID, name string, d float64) {
-	tls.CallsCost = append(tls.CallsCost, TestServerLoad{Name: name, D: d})
-}
-
-// ReportTo panics.
-func (*TestLoadStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) {
-	panic("not implemented")
-}
-
 // IsRoundRobin checks whether f's return value is roundrobin of elements from
 // want. But it doesn't check for the order. Note that want can contain
 // duplicate items, which makes it weight-round-robin.
@@ -318,46 +277,3 @@ func (tcp *TestConstPicker) Pick(info balancer.PickInfo) (balancer.PickResult, e
 	}
 	return balancer.PickResult{SubConn: tcp.SC}, nil
 }
-
-// testWRR is a deterministic WRR implementation.
-//
-// The real implementation does random WRR. testWRR makes the balancer behavior
-// deterministic and easier to test.
-//
-// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}.
-type testWRR struct {
-	itemsWithWeight []struct {
-		item   interface{}
-		weight int64
-	}
-	length int
-
-	mu    sync.Mutex
-	idx   int   // The index of the item that will be picked
-	count int64 // The number of times the current item has been picked.
-}
-
-// NewTestWRR return a WRR for testing. It's deterministic instead random.
-func NewTestWRR() wrr.WRR {
-	return &testWRR{}
-}
-
-func (twrr *testWRR) Add(item interface{}, weight int64) {
-	twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct {
-		item   interface{}
-		weight int64
-	}{item: item, weight: weight})
-	twrr.length++
-}
-
-func (twrr *testWRR) Next() interface{} {
-	twrr.mu.Lock()
-	iww := twrr.itemsWithWeight[twrr.idx]
-	twrr.count++
-	if twrr.count >= iww.weight {
-		twrr.idx = (twrr.idx + 1) % twrr.length
-		twrr.count = 0
-	}
-	twrr.mu.Unlock()
-	return iww.item
-}
diff --git a/xds/internal/testutils/wrr.go b/xds/internal/testutils/wrr.go
new file mode 100644
index 000000000000..a4df5fc6050e
--- /dev/null
+++ b/xds/internal/testutils/wrr.go
@@ -0,0 +1,68 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import (
+	"sync"
+
+	"google.golang.org/grpc/internal/wrr"
+)
+
+// testWRR is a deterministic WRR implementation.
+//
+// The real implementation does random WRR. testWRR makes the balancer behavior
+// deterministic and easier to test.
+//
+// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}.
+type testWRR struct {
+	itemsWithWeight []struct {
+		item   interface{}
+		weight int64
+	}
+	length int
+
+	mu    sync.Mutex
+	idx   int   // The index of the item that will be picked
+	count int64 // The number of times the current item has been picked.
+}
+
+// NewTestWRR return a WRR for testing. It's deterministic instead of random.
+func NewTestWRR() wrr.WRR {
+	return &testWRR{}
+}
+
+func (twrr *testWRR) Add(item interface{}, weight int64) {
+	twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct {
+		item   interface{}
+		weight int64
+	}{item: item, weight: weight})
+	twrr.length++
+}
+
+func (twrr *testWRR) Next() interface{} {
+	twrr.mu.Lock()
+	iww := twrr.itemsWithWeight[twrr.idx]
+	twrr.count++
+	if twrr.count >= iww.weight {
+		twrr.idx = (twrr.idx + 1) % twrr.length
+		twrr.count = 0
+	}
+	twrr.mu.Unlock()
+	return iww.item
+}

From 21f897eab8af2d88578555864320bc409a0bcca1 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 24 Sep 2020 11:29:52 -0700
Subject: [PATCH 208/481] xds: Add bootstrap support for certificate providers.
 (#3901)

---
 credentials/tls/certprovider/provider.go      |   6 +
 internal/internal.go                          |   5 +
 xds/internal/client/bootstrap/bootstrap.go    |  47 ++++
 .../client/bootstrap/bootstrap_test.go        | 251 ++++++++++++++++++
 4 files changed, 309 insertions(+)

diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go
index 204e55686e91..8d8ae80a053b 100644
--- a/credentials/tls/certprovider/provider.go
+++ b/credentials/tls/certprovider/provider.go
@@ -29,8 +29,14 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+
+	"google.golang.org/grpc/internal"
 )
 
+func init() {
+	internal.GetCertificateProviderBuilder = getBuilder
+}
+
 var (
 	// errProviderClosed is returned by Distributor.KeyMaterial when it is
 	// closed.
diff --git a/internal/internal.go b/internal/internal.go
index 818ca857998b..716d92800b9a 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -52,6 +52,11 @@ var (
 	// This function compares the config without rawJSON stripped, in case the
 	// there's difference in white space.
 	EqualServiceConfigForTesting func(a, b serviceconfig.Config) bool
+	// GetCertificateProviderBuilder returns the registered builder for the
+	// given name. This is set by package certprovider for use from xDS
+	// bootstrap code while parsing certificate provider configs in the
+	// bootstrap file.
+	GetCertificateProviderBuilder interface{} // func(string) certprovider.Builder
 )
 
 // HealthChecker defines the signature of the client-side LB channel health checking function.
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index e3f2ce59dfb3..93e6d3e94cb1 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -33,6 +33,8 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -77,6 +79,18 @@ type Config struct {
 	// NodeProto contains the Node proto to be used in xDS requests. The actual
 	// type depends on the transport protocol version used.
 	NodeProto proto.Message
+	// CertProviderConfigs contain parsed configs for supported certificate
+	// provider plugins found in the bootstrap file.
+	CertProviderConfigs map[string]CertProviderConfig
+}
+
+// CertProviderConfig wraps the certificate provider plugin name and config
+// (corresponding to one plugin instance) found in the bootstrap file.
+type CertProviderConfig struct {
+	// Name is the registered name of the certificate provider.
+	Name string
+	// Config is the parsed config to be passed to the certificate provider.
+	Config certprovider.StableConfig
 }
 
 type channelCreds struct {
@@ -103,6 +117,10 @@ type xdsServer struct {
 //        }
 //      ],
 //      "server_features": [ ... ]
+//		"certificate_providers" : {
+//			"default": { default cert provider config },
+//			"foo": { config for provider foo }
+//		}
 //    },
 //    "node": <JSON form of Node proto>
 // }
@@ -182,6 +200,35 @@ func NewConfig() (*Config, error) {
 					serverSupportsV3 = true
 				}
 			}
+		case "certificate_providers":
+			var providerInstances map[string]json.RawMessage
+			if err := json.Unmarshal(v, &providerInstances); err != nil {
+				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
+			}
+			configs := make(map[string]CertProviderConfig)
+			getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
+			for instance, data := range providerInstances {
+				var providerConfigs map[string]json.RawMessage
+				if err := json.Unmarshal(data, &providerConfigs); err != nil {
+					return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err)
+				}
+				for name, cfg := range providerConfigs {
+					parser := getBuilder(name)
+					if parser == nil {
+						// We ignore plugins that we do not know about.
+						continue
+					}
+					c, err := parser.ParseConfig(cfg)
+					if err != nil {
+						return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
+					}
+					configs[instance] = CertProviderConfig{
+						Name:   name,
+						Config: c,
+					}
+				}
+			}
+			config.CertProviderConfigs = configs
 		}
 		// Do not fail the xDS bootstrap when an unknown field is seen. This can
 		// happen when an older version client reads a newer version bootstrap
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index a7734f03436e..353bcd9eb5bf 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -19,6 +19,8 @@
 package bootstrap
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"testing"
@@ -28,8 +30,11 @@ import (
 	"github.com/golang/protobuf/proto"
 	structpb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -233,6 +238,24 @@ func (c *Config) compare(want *Config) error {
 	if diff := cmp.Diff(want.NodeProto, c.NodeProto, cmp.Comparer(proto.Equal)); diff != "" {
 		return fmt.Errorf("config.NodeProto diff (-want, +got):\n%s", diff)
 	}
+
+	// A vanilla cmp.Equal or cmp.Diff will not produce useful error message
+	// here. So, we iterate through the list of configs and compare them one at
+	// a time.
+	gotCfgs := c.CertProviderConfigs
+	wantCfgs := want.CertProviderConfigs
+	if len(gotCfgs) != len(wantCfgs) {
+		return fmt.Errorf("config.CertProviderConfigs is %d entries, want %d", len(gotCfgs), len(wantCfgs))
+	}
+	for instance, gotCfg := range gotCfgs {
+		wantCfg, ok := wantCfgs[instance]
+		if !ok {
+			return fmt.Errorf("config.CertProviderConfigs has unexpected plugin instance %q with config %q", instance, string(gotCfg.Config.Canonical()))
+		}
+		if gotCfg.Name != wantCfg.Name || !cmp.Equal(gotCfg.Config.Canonical(), wantCfg.Config.Canonical()) {
+			return fmt.Errorf("config.CertProviderConfigs for plugin instance %q has config {%s, %s, want {%s, %s}", instance, gotCfg.Name, string(gotCfg.Config.Canonical()), wantCfg.Name, string(wantCfg.Config.Canonical()))
+		}
+	}
 	return nil
 }
 
@@ -452,3 +475,231 @@ func TestNewConfigBootstrapFileEnvNotSet(t *testing.T) {
 		t.Errorf("NewConfig() returned nil error, expected to fail")
 	}
 }
+
+func init() {
+	certprovider.Register(&fakeCertProviderBuilder{})
+}
+
+const fakeCertProviderName = "fake-certificate-provider"
+
+// fakeCertProviderBuilder builds new instances of fakeCertProvider and
+// interprets the config provided to it as JSON with a single key and value.
+type fakeCertProviderBuilder struct{}
+
+func (b *fakeCertProviderBuilder) Build(certprovider.StableConfig, certprovider.Options) certprovider.Provider {
+	return &fakeCertProvider{}
+}
+
+// ParseConfig expects input in JSON format containing a map from string to
+// string, with a single entry and mapKey being "configKey".
+func (b *fakeCertProviderBuilder) ParseConfig(cfg interface{}) (certprovider.StableConfig, error) {
+	config, ok := cfg.(json.RawMessage)
+	if !ok {
+		return nil, fmt.Errorf("fakeCertProviderBuilder received config of type %T, want []byte", config)
+	}
+	var cfgData map[string]string
+	if err := json.Unmarshal(config, &cfgData); err != nil {
+		return nil, fmt.Errorf("fakeCertProviderBuilder config parsing failed: %v", err)
+	}
+	if len(cfgData) != 1 || cfgData["configKey"] == "" {
+		return nil, errors.New("fakeCertProviderBuilder received invalid config")
+	}
+	return &fakeStableConfig{config: cfgData}, nil
+}
+
+func (b *fakeCertProviderBuilder) Name() string {
+	return fakeCertProviderName
+}
+
+type fakeStableConfig struct {
+	config map[string]string
+}
+
+func (c *fakeStableConfig) Canonical() []byte {
+	var cfg string
+	for k, v := range c.config {
+		cfg = fmt.Sprintf("%s:%s", k, v)
+	}
+	return []byte(cfg)
+}
+
+// fakeCertProvider is an empty implementation of the Provider interface.
+type fakeCertProvider struct {
+	certprovider.Provider
+}
+
+func TestNewConfigWithCertificateProviders(t *testing.T) {
+	bootstrapFileMap := map[string]string{
+		"badJSONCertProviderConfig": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": "bad JSON"
+		}`,
+		"allUnknownCertProviders": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": {
+				"unknownProviderInstance1": {
+					"foo1": "bar1"
+				},
+				"unknownProviderInstance2": {
+					"foo2": "bar2"
+				}
+			}
+		}`,
+		"badCertProviderConfig": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": {
+				"unknownProviderInstance": {
+					"foo": "bar"
+				},
+				"fakeProviderInstance": {
+					"fake-certificate-provider": {
+						"configKey": "configValue"
+					}
+				},
+				"fakeProviderInstanceBad": {
+					"fake-certificate-provider": {
+						"configKey": 666
+					}
+				}
+			}
+		}`,
+		"goodCertProviderConfig": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": {
+				"unknownProviderInstance": {
+					"foo": "bar"
+				},
+				"fakeProviderInstance": {
+					"fake-certificate-provider": {
+						"configKey": "configValue"
+					}
+				}
+			}
+		}`,
+	}
+
+	getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
+	parser := getBuilder(fakeCertProviderName)
+	if parser == nil {
+		t.Fatalf("missing certprovider plugin %q", fakeCertProviderName)
+	}
+	wantCfg, err := parser.ParseConfig(json.RawMessage(`{"configKey": "configValue"}`))
+	if err != nil {
+		t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err)
+	}
+
+	if err := os.Setenv(v3SupportEnv, "true"); err != nil {
+		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
+	}
+	defer os.Unsetenv(v3SupportEnv)
+
+	cancel := setupBootstrapOverride(bootstrapFileMap)
+	defer cancel()
+
+	goodConfig := &Config{
+		BalancerName: "trafficdirector.googleapis.com:443",
+		Creds:        grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
+		TransportAPI: version.TransportV3,
+		NodeProto:    v3NodeProto,
+		CertProviderConfigs: map[string]CertProviderConfig{
+			"fakeProviderInstance": {
+				Name:   fakeCertProviderName,
+				Config: wantCfg,
+			},
+		},
+	}
+	tests := []struct {
+		name       string
+		wantConfig *Config
+		wantErr    bool
+	}{
+		{
+			name:    "badJSONCertProviderConfig",
+			wantErr: true,
+		},
+		{
+
+			name:    "badCertProviderConfig",
+			wantErr: true,
+		},
+		{
+
+			name:       "allUnknownCertProviders",
+			wantConfig: nonNilCredsConfigV3,
+		},
+		{
+			name:       "goodCertProviderConfig",
+			wantConfig: goodConfig,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
+				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
+			}
+			c, err := NewConfig()
+			if (err != nil) != test.wantErr {
+				t.Fatalf("NewConfig() returned: %v, wantErr: %v", err, test.wantErr)
+			}
+			if test.wantErr {
+				return
+			}
+			if err := c.compare(test.wantConfig); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

From e6c98a478e62a717b945eb60edb115faf65215d3 Mon Sep 17 00:00:00 2001
From: "Stephen L. White" <stephen95@gmail.com>
Date: Fri, 25 Sep 2020 13:06:54 -0400
Subject: [PATCH 209/481] stats: include message header in
 stats.InPayload.WireLength (#3886)

---
 server.go | 2 +-
 stream.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/server.go b/server.go
index 33fc3240bcc6..c10a5c0a3a14 100644
--- a/server.go
+++ b/server.go
@@ -1175,7 +1175,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 			sh.HandleRPC(stream.Context(), &stats.InPayload{
 				RecvTime:   time.Now(),
 				Payload:    v,
-				WireLength: payInfo.wireLength,
+				WireLength: payInfo.wireLength + headerLen,
 				Data:       d,
 				Length:     len(d),
 			})
diff --git a/stream.go b/stream.go
index fbc3fb11cb4d..5fd856a38212 100644
--- a/stream.go
+++ b/stream.go
@@ -929,7 +929,7 @@ func (a *csAttempt) recvMsg(m interface{}, payInfo *payloadInfo) (err error) {
 			Payload:  m,
 			// TODO truncate large payload.
 			Data:       payInfo.uncompressedBytes,
-			WireLength: payInfo.wireLength,
+			WireLength: payInfo.wireLength + headerLen,
 			Length:     len(payInfo.uncompressedBytes),
 		})
 	}
@@ -1511,7 +1511,7 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) {
 			Payload:  m,
 			// TODO truncate large payload.
 			Data:       payInfo.uncompressedBytes,
-			WireLength: payInfo.wireLength,
+			WireLength: payInfo.wireLength + headerLen,
 			Length:     len(payInfo.uncompressedBytes),
 		})
 	}

From 02cd07d9bb5609fe76f8bb8d948037e79a8b29c7 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 29 Sep 2020 15:17:06 -0700
Subject: [PATCH 210/481] cmd/protoc-gen-go-grpc: revert to interface-based
 service registration (#3911)

---
 .../grpc_lb_v1/load_balancer_grpc.pb.go       | 102 ++---
 balancer/grpclb/grpclb_test.go                |   8 +-
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       | 100 ++---
 balancer/roundrobin/roundrobin_test.go        |  13 +-
 benchmark/benchmark.go                        |  23 +-
 benchmark/grpc_testing/services_grpc.pb.go    | 361 ++++++++--------
 benchmark/worker/main.go                      |  14 +-
 binarylog/binarylog_end2end_test.go           |  18 +-
 channelz/grpc_channelz_v1/channelz_grpc.pb.go | 321 ++++++--------
 cmd/protoc-gen-go-grpc/README.md              | 117 +-----
 cmd/protoc-gen-go-grpc/grpc.go                | 382 ++++++-----------
 cmd/protoc-gen-go-grpc/main.go                |   6 +-
 .../proto/grpc_gcp/handshaker_grpc.pb.go      | 109 ++---
 .../meshca/internal/v1/meshca_grpc.pb.go      | 102 ++---
 .../features/authentication/server/main.go    |   8 +-
 examples/features/cancellation/server/main.go |   8 +-
 examples/features/compression/server/main.go  |   8 +-
 examples/features/deadline/server/main.go     |   8 +-
 examples/features/debugging/server/main.go    |  24 +-
 .../features/encryption/ALTS/server/main.go   |  10 +-
 .../features/encryption/TLS/server/main.go    |   8 +-
 examples/features/errors/server/main.go       |   4 +-
 examples/features/health/server/main.go       |  10 +-
 examples/features/interceptor/server/main.go  |  15 +-
 examples/features/keepalive/server/main.go    |   9 +-
 .../features/load_balancing/server/main.go    |   4 +-
 examples/features/metadata/server/main.go     |  19 +-
 examples/features/multiplex/server/main.go    |  22 +-
 .../features/name_resolving/server/main.go    |  11 +-
 examples/features/profiling/README.md         |   2 +-
 examples/features/profiling/server/main.go    |   8 +-
 examples/features/proto/echo/echo_grpc.pb.go  | 177 ++++----
 examples/features/reflection/server/main.go   |  20 +-
 examples/features/retry/server/main.go        |   3 +-
 examples/features/wait_for_ready/main.go      |   9 +-
 examples/features/xds/server/main.go          |   5 +-
 examples/gotutorial.md                        |   8 +-
 examples/helloworld/greeter_server/main.go    |  11 +-
 .../helloworld/helloworld_grpc.pb.go          |  74 ++--
 .../routeguide/route_guide_grpc.pb.go         | 177 ++++----
 examples/route_guide/server/server.go         |  12 +-
 health/grpc_health_v1/health_grpc.pb.go       | 151 +++----
 interop/alts/server/server.go                 |   2 +-
 interop/grpc_testing/test_grpc.pb.go          | 394 +++++++++---------
 interop/server/server.go                      |   2 +-
 interop/test_utils.go                         |  16 +-
 interop/xds/client/client.go                  |   8 +-
 interop/xds/server/server.go                  |  10 +-
 profiling/proto/service_grpc.pb.go            | 136 +++---
 .../reflection_grpc.pb.go                     | 103 ++---
 reflection/grpc_testing/test_grpc.pb.go       | 109 +++--
 reflection/serverreflection_test.go           |  11 +-
 regenerate.sh                                 |  37 +-
 .../advancedtls_integration_test.go           |   8 +-
 stats/grpc_testing/test_grpc.pb.go            | 185 ++++----
 stats/stats_test.go                           |  25 +-
 stress/client/main.go                         |  11 +-
 stress/grpc_testing/metrics_grpc.pb.go        | 128 +++---
 test/balancer_test.go                         |  26 +-
 test/channelz_linux_go110_test.go             |   2 +-
 test/channelz_test.go                         |  40 +-
 test/creds_test.go                            |  20 +-
 test/end2end_test.go                          | 321 +++++++-------
 test/goaway_test.go                           |   2 +-
 test/gracefulstop_test.go                     |   2 +-
 test/grpc_testing/test_grpc.pb.go             | 245 +++++------
 test/healthcheck_test.go                      |   2 +-
 test/local_creds_test.go                      |   4 +-
 68 files changed, 1908 insertions(+), 2442 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index ecbb77509ea3..b28b7341dbaf 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -29,14 +29,8 @@ func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
 	return &loadBalancerClient{cc}
 }
 
-var loadBalancerBalanceLoadStreamDesc = &grpc.StreamDesc{
-	StreamName:    "BalanceLoad",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
-	stream, err := c.cc.NewStream(ctx, loadBalancerBalanceLoadStreamDesc, "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
+	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -66,17 +60,35 @@ func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) {
 	return m, nil
 }
 
-// LoadBalancerService is the service API for LoadBalancer service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterLoadBalancerService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type LoadBalancerService struct {
+// LoadBalancerServer is the server API for LoadBalancer service.
+// All implementations should embed UnimplementedLoadBalancerServer
+// for forward compatibility
+type LoadBalancerServer interface {
 	// Bidirectional rpc to get a list of servers.
-	BalanceLoad func(LoadBalancer_BalanceLoadServer) error
+	BalanceLoad(LoadBalancer_BalanceLoadServer) error
 }
 
-func (s *LoadBalancerService) balanceLoad(_ interface{}, stream grpc.ServerStream) error {
-	return s.BalanceLoad(&loadBalancerBalanceLoadServer{stream})
+// UnimplementedLoadBalancerServer should be embedded to have forward compatible implementations.
+type UnimplementedLoadBalancerServer struct {
+}
+
+func (UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
+	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
+}
+
+// UnsafeLoadBalancerServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to LoadBalancerServer will
+// result in compilation errors.
+type UnsafeLoadBalancerServer interface {
+	mustEmbedUnimplementedLoadBalancerServer()
+}
+
+func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
+	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
+}
+
+func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream})
 }
 
 type LoadBalancer_BalanceLoadServer interface {
@@ -101,53 +113,17 @@ func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
 	return m, nil
 }
 
-// RegisterLoadBalancerService registers a service implementation with a gRPC server.
-func RegisterLoadBalancerService(s grpc.ServiceRegistrar, srv *LoadBalancerService) {
-	srvCopy := *srv
-	if srvCopy.BalanceLoad == nil {
-		srvCopy.BalanceLoad = func(LoadBalancer_BalanceLoadServer) error {
-			return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.lb.v1.LoadBalancer",
-		Methods:     []grpc.MethodDesc{},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "BalanceLoad",
-				Handler:       srvCopy.balanceLoad,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.lb.v1.LoadBalancer",
+	HandlerType: (*LoadBalancerServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "BalanceLoad",
+			Handler:       _LoadBalancer_BalanceLoad_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
-		Metadata: "grpc/lb/v1/load_balancer.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// LoadBalancerServer is the service API for LoadBalancer service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type LoadBalancerServer interface {
-	// Bidirectional rpc to get a list of servers.
-	BalanceLoad(LoadBalancer_BalanceLoadServer) error
-}
-
-// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations of
-// LoadBalancerServer
-type UnimplementedLoadBalancerServer struct {
-}
-
-func (UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error {
-	return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented")
-}
-
-// RegisterLoadBalancerServer registers a service implementation with a gRPC server.
-func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) {
-	str := &LoadBalancerService{
-		BalanceLoad: srv.BalanceLoad,
-	}
-	RegisterLoadBalancerService(s, str)
+	},
+	Metadata: "grpc/lb/v1/load_balancer.proto",
 }
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index 5a2297d43ec8..48082e2069fa 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -276,6 +276,8 @@ func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServe
 }
 
 type testServer struct {
+	testpb.UnimplementedTestServiceServer
+
 	addr     string
 	fallback bool
 }
@@ -304,11 +306,7 @@ func startBackends(sn string, fallback bool, lis ...net.Listener) (servers []*gr
 			sn: sn,
 		}
 		s := grpc.NewServer(grpc.Creds(creds))
-		ts := &testServer{addr: l.Addr().String(), fallback: fallback}
-		testpb.RegisterTestServiceService(s, &testpb.TestServiceService{
-			EmptyCall:      ts.EmptyCall,
-			FullDuplexCall: ts.FullDuplexCall,
-		})
+		testpb.RegisterTestServiceServer(s, &testServer{addr: l.Addr().String(), fallback: fallback})
 		servers = append(servers, s)
 		go func(s *grpc.Server, l net.Listener) {
 			s.Serve(l)
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index 93c95132b520..f3147189337f 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -29,10 +29,6 @@ func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupService
 	return &routeLookupServiceClient{cc}
 }
 
-var routeLookupServiceRouteLookupStreamDesc = &grpc.StreamDesc{
-	StreamName: "RouteLookup",
-}
-
 func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) {
 	out := new(RouteLookupResponse)
 	err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...)
@@ -42,78 +38,62 @@ func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLoo
 	return out, nil
 }
 
-// RouteLookupServiceService is the service API for RouteLookupService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterRouteLookupServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type RouteLookupServiceService struct {
+// RouteLookupServiceServer is the server API for RouteLookupService service.
+// All implementations must embed UnimplementedRouteLookupServiceServer
+// for forward compatibility
+type RouteLookupServiceServer interface {
 	// Lookup returns a target for a single key.
-	RouteLookup func(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
+	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
+	mustEmbedUnimplementedRouteLookupServiceServer()
 }
 
-func (s *RouteLookupServiceService) routeLookup(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+// UnimplementedRouteLookupServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedRouteLookupServiceServer struct {
+}
+
+func (UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
+}
+func (UnimplementedRouteLookupServiceServer) mustEmbedUnimplementedRouteLookupServiceServer() {}
+
+// UnsafeRouteLookupServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to RouteLookupServiceServer will
+// result in compilation errors.
+type UnsafeRouteLookupServiceServer interface {
+	mustEmbedUnimplementedRouteLookupServiceServer()
+}
+
+func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
+	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
+}
+
+func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(RouteLookupRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.RouteLookup(ctx, in)
+		return srv.(RouteLookupServiceServer).RouteLookup(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.RouteLookup(ctx, req.(*RouteLookupRequest))
+		return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterRouteLookupServiceService registers a service implementation with a gRPC server.
-func RegisterRouteLookupServiceService(s grpc.ServiceRegistrar, srv *RouteLookupServiceService) {
-	srvCopy := *srv
-	if srvCopy.RouteLookup == nil {
-		srvCopy.RouteLookup = func(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.lookup.v1.RouteLookupService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "RouteLookup",
-				Handler:    srvCopy.routeLookup,
-			},
+var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.lookup.v1.RouteLookupService",
+	HandlerType: (*RouteLookupServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "RouteLookup",
+			Handler:    _RouteLookupService_RouteLookup_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "grpc/lookup/v1/rls.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// RouteLookupServiceServer is the service API for RouteLookupService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type RouteLookupServiceServer interface {
-	// Lookup returns a target for a single key.
-	RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error)
-}
-
-// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations of
-// RouteLookupServiceServer
-type UnimplementedRouteLookupServiceServer struct {
-}
-
-func (UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented")
-}
-
-// RegisterRouteLookupServiceServer registers a service implementation with a gRPC server.
-func RegisterRouteLookupServiceServer(s grpc.ServiceRegistrar, srv RouteLookupServiceServer) {
-	str := &RouteLookupServiceService{
-		RouteLookup: srv.RouteLookup,
-	}
-	RegisterRouteLookupServiceService(s, str)
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/lookup/v1/rls.proto",
 }
diff --git a/balancer/roundrobin/roundrobin_test.go b/balancer/roundrobin/roundrobin_test.go
index 0c54465a6af2..5a8ba481c9f0 100644
--- a/balancer/roundrobin/roundrobin_test.go
+++ b/balancer/roundrobin/roundrobin_test.go
@@ -47,11 +47,15 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-func emptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+type testServer struct {
+	testpb.UnimplementedTestServiceServer
+}
+
+func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 	return &testpb.Empty{}, nil
 }
 
-func fullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
 	return nil
 }
 
@@ -81,10 +85,7 @@ func startTestServers(count int) (_ *test, err error) {
 		}
 
 		s := grpc.NewServer()
-		testpb.RegisterTestServiceService(s, &testpb.TestServiceService{
-			EmptyCall:      emptyCall,
-			FullDuplexCall: fullDuplexCall,
-		})
+		testpb.RegisterTestServiceServer(s, &testServer{})
 		t.servers = append(t.servers, s)
 		t.addresses = append(t.addresses, lis.Addr().String())
 
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index 5794aebcc559..56841e172366 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -61,14 +61,8 @@ func NewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 	return p
 }
 
-type testServer struct{}
-
-func (s *testServer) Svc() *testpb.BenchmarkServiceService {
-	return &testpb.BenchmarkServiceService{
-		UnaryCall:                  s.UnaryCall,
-		StreamingCall:              s.StreamingCall,
-		UnconstrainedStreamingCall: s.UnconstrainedStreamingCall,
-	}
+type testServer struct {
+	testpb.UnimplementedBenchmarkServiceServer
 }
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -150,17 +144,10 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U
 // byteBufServer is a gRPC server that sends and receives byte buffer.
 // The purpose is to benchmark the gRPC performance without protobuf serialization/deserialization overhead.
 type byteBufServer struct {
+	testpb.UnimplementedBenchmarkServiceServer
 	respSize int32
 }
 
-func (s *byteBufServer) Svc() *testpb.BenchmarkServiceService {
-	return &testpb.BenchmarkServiceService{
-		UnaryCall:                  s.UnaryCall,
-		StreamingCall:              s.StreamingCall,
-		UnconstrainedStreamingCall: s.UnconstrainedStreamingCall,
-	}
-}
-
 // UnaryCall is an empty function and is not used for benchmark.
 // If bytebuf UnaryCall benchmark is needed later, the function body needs to be updated.
 func (s *byteBufServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -224,13 +211,13 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() {
 	s := grpc.NewServer(opts...)
 	switch info.Type {
 	case "protobuf":
-		testpb.RegisterBenchmarkServiceService(s, (&testServer{}).Svc())
+		testpb.RegisterBenchmarkServiceServer(s, &testServer{})
 	case "bytebuf":
 		respSize, ok := info.Metadata.(int32)
 		if !ok {
 			logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type)
 		}
-		testpb.RegisterBenchmarkServiceService(s, (&byteBufServer{respSize: respSize}).Svc())
+		testpb.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize})
 	default:
 		logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type)
 	}
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index 81e9ed6a2ac0..64937d1e4609 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -36,10 +36,6 @@ func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClie
 	return &benchmarkServiceClient{cc}
 }
 
-var benchmarkServiceUnaryCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "UnaryCall",
-}
-
 func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...)
@@ -49,14 +45,8 @@ func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleReques
 	return out, nil
 }
 
-var benchmarkServiceStreamingCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "StreamingCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, benchmarkServiceStreamingCallStreamDesc, "/grpc.testing.BenchmarkService/StreamingCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -86,14 +76,8 @@ func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
 	return m, nil
 }
 
-var benchmarkServiceUnconstrainedStreamingCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "UnconstrainedStreamingCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, benchmarkServiceUnconstrainedStreamingCallStreamDesc, "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -123,44 +107,68 @@ func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleRespon
 	return m, nil
 }
 
-// BenchmarkServiceService is the service API for BenchmarkService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterBenchmarkServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type BenchmarkServiceService struct {
+// BenchmarkServiceServer is the server API for BenchmarkService service.
+// All implementations must embed UnimplementedBenchmarkServiceServer
+// for forward compatibility
+type BenchmarkServiceServer interface {
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	StreamingCall func(BenchmarkService_StreamingCallServer) error
+	StreamingCall(BenchmarkService_StreamingCallServer) error
 	// Unconstrainted streaming.
 	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall func(BenchmarkService_UnconstrainedStreamingCallServer) error
+	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
+	mustEmbedUnimplementedBenchmarkServiceServer()
+}
+
+// UnimplementedBenchmarkServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedBenchmarkServiceServer struct {
+}
+
+func (UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) StreamingCall(BenchmarkService_StreamingCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
 }
+func (UnimplementedBenchmarkServiceServer) mustEmbedUnimplementedBenchmarkServiceServer() {}
 
-func (s *BenchmarkServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+// UnsafeBenchmarkServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to BenchmarkServiceServer will
+// result in compilation errors.
+type UnsafeBenchmarkServiceServer interface {
+	mustEmbedUnimplementedBenchmarkServiceServer()
+}
+
+func RegisterBenchmarkServiceServer(s *grpc.Server, srv BenchmarkServiceServer) {
+	s.RegisterService(&_BenchmarkService_serviceDesc, srv)
+}
+
+func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.UnaryCall(ctx, in)
+		return srv.(BenchmarkServiceServer).UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.BenchmarkService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.UnaryCall(ctx, req.(*SimpleRequest))
+		return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *BenchmarkServiceService) streamingCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.StreamingCall(&benchmarkServiceStreamingCallServer{stream})
-}
-func (s *BenchmarkServiceService) unconstrainedStreamingCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
+
+func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream})
 }
 
 type BenchmarkService_StreamingCallServer interface {
@@ -185,6 +193,10 @@ func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) {
 	return m, nil
 }
 
+func _BenchmarkService_UnconstrainedStreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
+}
+
 type BenchmarkService_UnconstrainedStreamingCallServer interface {
 	Send(*SimpleResponse) error
 	Recv() (*SimpleRequest, error)
@@ -207,50 +219,30 @@ func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleReques
 	return m, nil
 }
 
-// RegisterBenchmarkServiceService registers a service implementation with a gRPC server.
-func RegisterBenchmarkServiceService(s grpc.ServiceRegistrar, srv *BenchmarkServiceService) {
-	srvCopy := *srv
-	if srvCopy.UnaryCall == nil {
-		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-		}
-	}
-	if srvCopy.StreamingCall == nil {
-		srvCopy.StreamingCall = func(BenchmarkService_StreamingCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
-		}
-	}
-	if srvCopy.UnconstrainedStreamingCall == nil {
-		srvCopy.UnconstrainedStreamingCall = func(BenchmarkService_UnconstrainedStreamingCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.BenchmarkService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "UnaryCall",
-				Handler:    srvCopy.unaryCall,
-			},
+var _BenchmarkService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.BenchmarkService",
+	HandlerType: (*BenchmarkServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryCall",
+			Handler:    _BenchmarkService_UnaryCall_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "StreamingCall",
-				Handler:       srvCopy.streamingCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "UnconstrainedStreamingCall",
-				Handler:       srvCopy.unconstrainedStreamingCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingCall",
+			Handler:       _BenchmarkService_StreamingCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
-		Metadata: "benchmark/grpc_testing/services.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+		{
+			StreamName:    "UnconstrainedStreamingCall",
+			Handler:       _BenchmarkService_UnconstrainedStreamingCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "benchmark/grpc_testing/services.proto",
 }
 
 // WorkerServiceClient is the client API for WorkerService service.
@@ -285,14 +277,8 @@ func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient {
 	return &workerServiceClient{cc}
 }
 
-var workerServiceRunServerStreamDesc = &grpc.StreamDesc{
-	StreamName:    "RunServer",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) {
-	stream, err := c.cc.NewStream(ctx, workerServiceRunServerStreamDesc, "/grpc.testing.WorkerService/RunServer", opts...)
+	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[0], "/grpc.testing.WorkerService/RunServer", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -322,14 +308,8 @@ func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) {
 	return m, nil
 }
 
-var workerServiceRunClientStreamDesc = &grpc.StreamDesc{
-	StreamName:    "RunClient",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) {
-	stream, err := c.cc.NewStream(ctx, workerServiceRunClientStreamDesc, "/grpc.testing.WorkerService/RunClient", opts...)
+	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[1], "/grpc.testing.WorkerService/RunClient", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -359,10 +339,6 @@ func (x *workerServiceRunClientClient) Recv() (*ClientStatus, error) {
 	return m, nil
 }
 
-var workerServiceCoreCountStreamDesc = &grpc.StreamDesc{
-	StreamName: "CoreCount",
-}
-
 func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) {
 	out := new(CoreResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/CoreCount", in, out, opts...)
@@ -372,10 +348,6 @@ func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, op
 	return out, nil
 }
 
-var workerServiceQuitWorkerStreamDesc = &grpc.StreamDesc{
-	StreamName: "QuitWorker",
-}
-
 func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) {
 	out := new(Void)
 	err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/QuitWorker", in, out, opts...)
@@ -385,70 +357,62 @@ func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...
 	return out, nil
 }
 
-// WorkerServiceService is the service API for WorkerService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterWorkerServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type WorkerServiceService struct {
+// WorkerServiceServer is the server API for WorkerService service.
+// All implementations must embed UnimplementedWorkerServiceServer
+// for forward compatibility
+type WorkerServiceServer interface {
 	// Start server with specified workload.
 	// First request sent specifies the ServerConfig followed by ServerStatus
 	// response. After that, a "Mark" can be sent anytime to request the latest
 	// stats. Closing the stream will initiate shutdown of the test server
 	// and once the shutdown has finished, the OK status is sent to terminate
 	// this RPC.
-	RunServer func(WorkerService_RunServerServer) error
+	RunServer(WorkerService_RunServerServer) error
 	// Start client with specified workload.
 	// First request sent specifies the ClientConfig followed by ClientStatus
 	// response. After that, a "Mark" can be sent anytime to request the latest
 	// stats. Closing the stream will initiate shutdown of the test client
 	// and once the shutdown has finished, the OK status is sent to terminate
 	// this RPC.
-	RunClient func(WorkerService_RunClientServer) error
+	RunClient(WorkerService_RunClientServer) error
 	// Just return the core count - unary call
-	CoreCount func(context.Context, *CoreRequest) (*CoreResponse, error)
+	CoreCount(context.Context, *CoreRequest) (*CoreResponse, error)
 	// Quit this worker
-	QuitWorker func(context.Context, *Void) (*Void, error)
+	QuitWorker(context.Context, *Void) (*Void, error)
+	mustEmbedUnimplementedWorkerServiceServer()
 }
 
-func (s *WorkerServiceService) runServer(_ interface{}, stream grpc.ServerStream) error {
-	return s.RunServer(&workerServiceRunServerServer{stream})
+// UnimplementedWorkerServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedWorkerServiceServer struct {
 }
-func (s *WorkerServiceService) runClient(_ interface{}, stream grpc.ServerStream) error {
-	return s.RunClient(&workerServiceRunClientServer{stream})
+
+func (UnimplementedWorkerServiceServer) RunServer(WorkerService_RunServerServer) error {
+	return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
 }
-func (s *WorkerServiceService) coreCount(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(CoreRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return s.CoreCount(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     s,
-		FullMethod: "/grpc.testing.WorkerService/CoreCount",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.CoreCount(ctx, req.(*CoreRequest))
-	}
-	return interceptor(ctx, in, info, handler)
+func (UnimplementedWorkerServiceServer) RunClient(WorkerService_RunClientServer) error {
+	return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
 }
-func (s *WorkerServiceService) quitWorker(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Void)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return s.QuitWorker(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     s,
-		FullMethod: "/grpc.testing.WorkerService/QuitWorker",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.QuitWorker(ctx, req.(*Void))
-	}
-	return interceptor(ctx, in, info, handler)
+func (UnimplementedWorkerServiceServer) CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
+}
+func (UnimplementedWorkerServiceServer) QuitWorker(context.Context, *Void) (*Void, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
+}
+func (UnimplementedWorkerServiceServer) mustEmbedUnimplementedWorkerServiceServer() {}
+
+// UnsafeWorkerServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to WorkerServiceServer will
+// result in compilation errors.
+type UnsafeWorkerServiceServer interface {
+	mustEmbedUnimplementedWorkerServiceServer()
+}
+
+func RegisterWorkerServiceServer(s *grpc.Server, srv WorkerServiceServer) {
+	s.RegisterService(&_WorkerService_serviceDesc, srv)
+}
+
+func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(WorkerServiceServer).RunServer(&workerServiceRunServerServer{stream})
 }
 
 type WorkerService_RunServerServer interface {
@@ -473,6 +437,10 @@ func (x *workerServiceRunServerServer) Recv() (*ServerArgs, error) {
 	return m, nil
 }
 
+func _WorkerService_RunClient_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(WorkerServiceServer).RunClient(&workerServiceRunClientServer{stream})
+}
+
 type WorkerService_RunClientServer interface {
 	Send(*ClientStatus) error
 	Recv() (*ClientArgs, error)
@@ -495,57 +463,68 @@ func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) {
 	return m, nil
 }
 
-// RegisterWorkerServiceService registers a service implementation with a gRPC server.
-func RegisterWorkerServiceService(s grpc.ServiceRegistrar, srv *WorkerServiceService) {
-	srvCopy := *srv
-	if srvCopy.RunServer == nil {
-		srvCopy.RunServer = func(WorkerService_RunServerServer) error {
-			return status.Errorf(codes.Unimplemented, "method RunServer not implemented")
-		}
+func _WorkerService_CoreCount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CoreRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(WorkerServiceServer).CoreCount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.WorkerService/CoreCount",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(WorkerServiceServer).CoreCount(ctx, req.(*CoreRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Void)
+	if err := dec(in); err != nil {
+		return nil, err
 	}
-	if srvCopy.RunClient == nil {
-		srvCopy.RunClient = func(WorkerService_RunClientServer) error {
-			return status.Errorf(codes.Unimplemented, "method RunClient not implemented")
-		}
+	if interceptor == nil {
+		return srv.(WorkerServiceServer).QuitWorker(ctx, in)
 	}
-	if srvCopy.CoreCount == nil {
-		srvCopy.CoreCount = func(context.Context, *CoreRequest) (*CoreResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented")
-		}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.WorkerService/QuitWorker",
 	}
-	if srvCopy.QuitWorker == nil {
-		srvCopy.QuitWorker = func(context.Context, *Void) (*Void, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented")
-		}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(WorkerServiceServer).QuitWorker(ctx, req.(*Void))
 	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.WorkerService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "CoreCount",
-				Handler:    srvCopy.coreCount,
-			},
-			{
-				MethodName: "QuitWorker",
-				Handler:    srvCopy.quitWorker,
-			},
+	return interceptor(ctx, in, info, handler)
+}
+
+var _WorkerService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.WorkerService",
+	HandlerType: (*WorkerServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "CoreCount",
+			Handler:    _WorkerService_CoreCount_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "RunServer",
-				Handler:       srvCopy.runServer,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "RunClient",
-				Handler:       srvCopy.runClient,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+		{
+			MethodName: "QuitWorker",
+			Handler:    _WorkerService_QuitWorker_Handler,
 		},
-		Metadata: "benchmark/grpc_testing/services.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "RunServer",
+			Handler:       _WorkerService_RunServer_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "RunClient",
+			Handler:       _WorkerService_RunClient_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "benchmark/grpc_testing/services.proto",
 }
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 71a1fb6d294a..634f09e658c5 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -75,19 +75,11 @@ func (byteBufCodec) String() string {
 // workerServer implements WorkerService rpc handlers.
 // It can create benchmarkServer or benchmarkClient on demand.
 type workerServer struct {
+	testpb.UnimplementedWorkerServiceServer
 	stop       chan<- bool
 	serverPort int
 }
 
-func (s *workerServer) Svc() *testpb.WorkerServiceService {
-	return &testpb.WorkerServiceService{
-		RunServer:  s.RunServer,
-		RunClient:  s.RunClient,
-		CoreCount:  s.CoreCount,
-		QuitWorker: s.QuitWorker,
-	}
-}
-
 func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) error {
 	var bs *benchmarkServer
 	defer func() {
@@ -217,10 +209,10 @@ func main() {
 
 	s := grpc.NewServer()
 	stop := make(chan bool)
-	testpb.RegisterWorkerServiceService(s, (&workerServer{
+	testpb.RegisterWorkerServiceServer(s, &workerServer{
 		stop:       stop,
 		serverPort: *serverPort,
-	}).Svc())
+	})
 
 	go func() {
 		<-stop
diff --git a/binarylog/binarylog_end2end_test.go b/binarylog/binarylog_end2end_test.go
index b8caa828a687..89685c65a26b 100644
--- a/binarylog/binarylog_end2end_test.go
+++ b/binarylog/binarylog_end2end_test.go
@@ -115,18 +115,10 @@ var (
 )
 
 type testServer struct {
+	testpb.UnimplementedTestServiceServer
 	te *test
 }
 
-func (s *testServer) Svc() *testpb.TestServiceService {
-	return &testpb.TestServiceService{
-		UnaryCall:        s.UnaryCall,
-		FullDuplexCall:   s.FullDuplexCall,
-		ClientStreamCall: s.ClientStreamCall,
-		ServerStreamCall: s.ServerStreamCall,
-	}
-}
-
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	md, ok := metadata.FromIncomingContext(ctx)
 	if ok {
@@ -224,7 +216,7 @@ func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.Te
 type test struct {
 	t *testing.T
 
-	testService *testpb.TestServiceService // nil means none
+	testService testpb.TestServiceServer // nil means none
 	// srv and srvAddr are set once startServer is called.
 	srv     *grpc.Server
 	srvAddr string // Server IP without port.
@@ -279,7 +271,7 @@ func (lw *listenerWrapper) Accept() (net.Conn, error) {
 
 // startServer starts a gRPC server listening. Callers should defer a
 // call to te.tearDown to clean up.
-func (te *test) startServer(ts *testpb.TestServiceService) {
+func (te *test) startServer(ts testpb.TestServiceServer) {
 	te.testService = ts
 	lis, err := net.Listen("tcp", "localhost:0")
 
@@ -295,7 +287,7 @@ func (te *test) startServer(ts *testpb.TestServiceService) {
 	s := grpc.NewServer(opts...)
 	te.srv = s
 	if te.testService != nil {
-		testpb.RegisterTestServiceService(s, te.testService)
+		testpb.RegisterTestServiceServer(s, te.testService)
 	}
 
 	go s.Serve(lis)
@@ -791,7 +783,7 @@ func (ed *expectedData) toServerLogEntries() []*pb.GrpcLogEntry {
 
 func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData {
 	te := newTest(t, tc)
-	te.startServer((&testServer{te: te}).Svc())
+	te.startServer(&testServer{te: te})
 	defer te.tearDown()
 
 	expect := &expectedData{
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index e56d7353d8f3..8fba59abe569 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -42,10 +42,6 @@ func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient {
 	return &channelzClient{cc}
 }
 
-var channelzGetTopChannelsStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetTopChannels",
-}
-
 func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) {
 	out := new(GetTopChannelsResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...)
@@ -55,10 +51,6 @@ func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsR
 	return out, nil
 }
 
-var channelzGetServersStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetServers",
-}
-
 func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) {
 	out := new(GetServersResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...)
@@ -68,10 +60,6 @@ func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest,
 	return out, nil
 }
 
-var channelzGetServerStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetServer",
-}
-
 func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) {
 	out := new(GetServerResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...)
@@ -81,10 +69,6 @@ func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, op
 	return out, nil
 }
 
-var channelzGetServerSocketsStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetServerSockets",
-}
-
 func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) {
 	out := new(GetServerSocketsResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...)
@@ -94,10 +78,6 @@ func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSock
 	return out, nil
 }
 
-var channelzGetChannelStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetChannel",
-}
-
 func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) {
 	out := new(GetChannelResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...)
@@ -107,10 +87,6 @@ func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest,
 	return out, nil
 }
 
-var channelzGetSubchannelStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetSubchannel",
-}
-
 func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) {
 	out := new(GetSubchannelResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...)
@@ -120,10 +96,6 @@ func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelReq
 	return out, nil
 }
 
-var channelzGetSocketStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetSocket",
-}
-
 func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) {
 	out := new(GetSocketResponse)
 	err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...)
@@ -133,284 +105,223 @@ func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, op
 	return out, nil
 }
 
-// ChannelzService is the service API for Channelz service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterChannelzService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type ChannelzService struct {
+// ChannelzServer is the server API for Channelz service.
+// All implementations should embed UnimplementedChannelzServer
+// for forward compatibility
+type ChannelzServer interface {
 	// Gets all root channels (i.e. channels the application has directly
 	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels func(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
+	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
 	// Gets all servers that exist in the process.
-	GetServers func(context.Context, *GetServersRequest) (*GetServersResponse, error)
+	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
 	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer func(context.Context, *GetServerRequest) (*GetServerResponse, error)
+	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
 	// Gets all server sockets that exist in the process.
-	GetServerSockets func(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
+	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
 	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel func(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
+	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
 	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel func(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
+	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
 	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket func(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
+	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
+}
+
+// UnimplementedChannelzServer should be embedded to have forward compatible implementations.
+type UnimplementedChannelzServer struct {
 }
 
-func (s *ChannelzService) getTopChannels(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
+}
+func (UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
+}
+func (UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
+}
+func (UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
+}
+func (UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
+}
+func (UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
+}
+func (UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
+}
+
+// UnsafeChannelzServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to ChannelzServer will
+// result in compilation errors.
+type UnsafeChannelzServer interface {
+	mustEmbedUnimplementedChannelzServer()
+}
+
+func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
+	s.RegisterService(&_Channelz_serviceDesc, srv)
+}
+
+func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetTopChannelsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetTopChannels(ctx, in)
+		return srv.(ChannelzServer).GetTopChannels(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetTopChannels(ctx, req.(*GetTopChannelsRequest))
+		return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ChannelzService) getServers(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetServersRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetServers(ctx, in)
+		return srv.(ChannelzServer).GetServers(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetServers",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetServers(ctx, req.(*GetServersRequest))
+		return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ChannelzService) getServer(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetServerRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetServer(ctx, in)
+		return srv.(ChannelzServer).GetServer(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetServer",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetServer(ctx, req.(*GetServerRequest))
+		return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ChannelzService) getServerSockets(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetServerSocketsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetServerSockets(ctx, in)
+		return srv.(ChannelzServer).GetServerSockets(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetServerSockets(ctx, req.(*GetServerSocketsRequest))
+		return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ChannelzService) getChannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetChannelRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetChannel(ctx, in)
+		return srv.(ChannelzServer).GetChannel(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetChannel",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetChannel(ctx, req.(*GetChannelRequest))
+		return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ChannelzService) getSubchannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetSubchannelRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetSubchannel(ctx, in)
+		return srv.(ChannelzServer).GetSubchannel(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetSubchannel(ctx, req.(*GetSubchannelRequest))
+		return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ChannelzService) getSocket(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetSocketRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetSocket(ctx, in)
+		return srv.(ChannelzServer).GetSocket(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.channelz.v1.Channelz/GetSocket",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetSocket(ctx, req.(*GetSocketRequest))
+		return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterChannelzService registers a service implementation with a gRPC server.
-func RegisterChannelzService(s grpc.ServiceRegistrar, srv *ChannelzService) {
-	srvCopy := *srv
-	if srvCopy.GetTopChannels == nil {
-		srvCopy.GetTopChannels = func(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
-		}
-	}
-	if srvCopy.GetServers == nil {
-		srvCopy.GetServers = func(context.Context, *GetServersRequest) (*GetServersResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
-		}
-	}
-	if srvCopy.GetServer == nil {
-		srvCopy.GetServer = func(context.Context, *GetServerRequest) (*GetServerResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
-		}
-	}
-	if srvCopy.GetServerSockets == nil {
-		srvCopy.GetServerSockets = func(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
-		}
-	}
-	if srvCopy.GetChannel == nil {
-		srvCopy.GetChannel = func(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
-		}
-	}
-	if srvCopy.GetSubchannel == nil {
-		srvCopy.GetSubchannel = func(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
-		}
-	}
-	if srvCopy.GetSocket == nil {
-		srvCopy.GetSocket = func(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.channelz.v1.Channelz",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "GetTopChannels",
-				Handler:    srvCopy.getTopChannels,
-			},
-			{
-				MethodName: "GetServers",
-				Handler:    srvCopy.getServers,
-			},
-			{
-				MethodName: "GetServer",
-				Handler:    srvCopy.getServer,
-			},
-			{
-				MethodName: "GetServerSockets",
-				Handler:    srvCopy.getServerSockets,
-			},
-			{
-				MethodName: "GetChannel",
-				Handler:    srvCopy.getChannel,
-			},
-			{
-				MethodName: "GetSubchannel",
-				Handler:    srvCopy.getSubchannel,
-			},
-			{
-				MethodName: "GetSocket",
-				Handler:    srvCopy.getSocket,
-			},
+var _Channelz_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.channelz.v1.Channelz",
+	HandlerType: (*ChannelzServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetTopChannels",
+			Handler:    _Channelz_GetTopChannels_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "grpc/channelz/v1/channelz.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// ChannelzServer is the service API for Channelz service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type ChannelzServer interface {
-	// Gets all root channels (i.e. channels the application has directly
-	// created). This does not include subchannels nor non-top level channels.
-	GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error)
-	// Gets all servers that exist in the process.
-	GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error)
-	// Returns a single Server, or else a NOT_FOUND code.
-	GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error)
-	// Gets all server sockets that exist in the process.
-	GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error)
-	// Returns a single Channel, or else a NOT_FOUND code.
-	GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error)
-	// Returns a single Subchannel, or else a NOT_FOUND code.
-	GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error)
-	// Returns a single Socket or else a NOT_FOUND code.
-	GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error)
-}
-
-// UnimplementedChannelzServer can be embedded to have forward compatible implementations of
-// ChannelzServer
-type UnimplementedChannelzServer struct {
-}
-
-func (UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented")
-}
-func (UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented")
-}
-func (UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented")
-}
-func (UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented")
-}
-func (UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented")
-}
-func (UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented")
-}
-func (UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented")
-}
-
-// RegisterChannelzServer registers a service implementation with a gRPC server.
-func RegisterChannelzServer(s grpc.ServiceRegistrar, srv ChannelzServer) {
-	str := &ChannelzService{
-		GetTopChannels:   srv.GetTopChannels,
-		GetServers:       srv.GetServers,
-		GetServer:        srv.GetServer,
-		GetServerSockets: srv.GetServerSockets,
-		GetChannel:       srv.GetChannel,
-		GetSubchannel:    srv.GetSubchannel,
-		GetSocket:        srv.GetSocket,
-	}
-	RegisterChannelzService(s, str)
+		{
+			MethodName: "GetServers",
+			Handler:    _Channelz_GetServers_Handler,
+		},
+		{
+			MethodName: "GetServer",
+			Handler:    _Channelz_GetServer_Handler,
+		},
+		{
+			MethodName: "GetServerSockets",
+			Handler:    _Channelz_GetServerSockets_Handler,
+		},
+		{
+			MethodName: "GetChannel",
+			Handler:    _Channelz_GetChannel_Handler,
+		},
+		{
+			MethodName: "GetSubchannel",
+			Handler:    _Channelz_GetSubchannel_Handler,
+		},
+		{
+			MethodName: "GetSocket",
+			Handler:    _Channelz_GetSocket_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/channelz/v1/channelz.proto",
 }
diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md
index 152c68a191d4..4758125de0d2 100644
--- a/cmd/protoc-gen-go-grpc/README.md
+++ b/cmd/protoc-gen-go-grpc/README.md
@@ -4,115 +4,18 @@ This tool generates Go language bindings of `service`s in protobuf definition
 files for gRPC.  For usage information, please see our [quick start
 guide](https://grpc.io/docs/languages/go/quickstart/).
 
-## Service implementation and registration
+## Future-proofing services
 
-**NOTE:** service registration has changed from the previous version of the
-code generator.  Please read this section carefully if you are migrating.
+By default, to register services using the methods generated by this tool, the
+service implementations must embed the corresponding
+`Unimplemented<ServiceName>Server` for future compatibility.  This is a behavior
+change from the grpc code generator previously included with `protoc-gen-go`.
+To restore this behavior, set the option `require_unimplemented_servers=false`.
+E.g.:
 
-To register your service handlers with a gRPC server, first implement the
-methods as either functions or methods on a struct.  Examples:
-
-```go
-// As a function:
-
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
-     // Echo.UnaryEcho implementation
-}
-
-// As a struct + method:
-
-type myEchoService struct {
-     // ...fields used by this service...
-}
-
-func (s *myEchoService) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
-     // Echo.UnaryEcho implementation
-}
 ```
-
-Then create an instance of the generated `Service` struct type and initialize
-the handlers which have been implemented:
-
-```go
-func main() {
-     // ...
-
-     // As a function:
-     echoService := pb.EchoService{
-          UnaryEcho: unaryEcho,
-          // etc
-     }
-
-     // As a struct+method:
-     mes := &myEchoService{...}
-     echoService := pb.EchoService{
-          UnaryEcho: mes.UnaryEcho,
-          // etc
-     }
-
-     // ...
- }
-```
-
-Finally, pass this `Service` instance to the generated `Register` function:
-
-```go
-     pb.RegisterEchoService(grpcServer, echoService)
-```
-
-### Migration from legacy version
-
-Older versions of `protoc-gen-go-grpc` and `protoc-gen-go` with the grpc plugin
-used a different method to register services.  With that method, it was only
-possible to register a service implementation that was a complete
-implementation of the service.  It was also possible to embed an
-`Unimplemented` implementation of the service, which was also generated and
-returned an UNIMPLEMENTED status for all methods.
-
-#### Generating the legacy API
-
-To avoid the need to update existing code, an option has been added to the code
-generator to produce the legacy API alongside the new API.  To use it:
-
-```sh
-# Example 1: with OPTS set to common options for protoc-gen-go and
-# protoc-gen-go-grpc
-protoc --go_out=${OPTS}:. --go-grpc_out=${OPTS},gen_unstable_server_interfaces=true:. *.proto
-
-# Example 2: if no special options are needed
-protoc --go_out=:. --go-grpc_out=gen_unstable_server_interfaces=true:. *.proto
+  protoc --go-grpc_out=require_unimplemented_servers=false[,other options...]:. \
 ```
 
-**The use of this legacy API is NOT recommended.** It was discontinued as it
-results in compilation breakages when new methods are added to services, which
-is a backward-compatible change in all other languages supported by gRPC.  With
-the newer API, newly-added methods will return an UNIMPLEMENTED status.
-
-#### Updating existing code
-
-To convert your existing code using the previous code generator, please refer
-to the following example:
-
-```go
-type myEchoService{
-     // ...fields used by this service...
-}
-// ... method handler implementation ...
-
-
-func main() {
-     // ...
-
-     // OLD:
-     pb.RegisterEchoServer(grpcServer, &myEchoService{})
-
-     // NEW:
-     es := &myEchoService{}
-     pb.RegisterEchoService(grpcServer, pb.EchoService{
-        UnaryEcho: es.UnaryEcho,
-        // enumerate all methods in EchoService implemented by myEchoService...
-     })
-
-     // ...
-}
-```
+Note that this is not recommended, and the option is only provided to restore
+backward compatibility with previously-generated code.
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 358bae2fe53b..919b5d46ae72 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -61,15 +61,11 @@ func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.
 	g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion7"))
 	g.P()
 	for _, service := range file.Services {
-		genClient(gen, file, g, service)
 		genService(gen, file, g, service)
-		if *genUnstableServerInterfaces {
-			genUnstableServerInterface(gen, file, g, service)
-		}
 	}
 }
 
-func genClient(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
+func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
 	clientName := service.GoName + "Client"
 
 	g.P("// ", clientName, " is the client API for ", service.GoName, " service.")
@@ -109,10 +105,129 @@ func genClient(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedF
 	g.P("}")
 	g.P()
 
+	var methodIndex, streamIndex int
 	// Client method implementations.
 	for _, method := range service.Methods {
-		genClientMethod(gen, g, method)
+		if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() {
+			// Unary RPC method
+			genClientMethod(gen, file, g, method, methodIndex)
+			methodIndex++
+		} else {
+			// Streaming RPC method
+			genClientMethod(gen, file, g, method, streamIndex)
+			streamIndex++
+		}
+	}
+
+	mustOrShould := "must"
+	if !*requireUnimplemented {
+		mustOrShould = "should"
+	}
+
+	// Server interface.
+	serverType := service.GoName + "Server"
+	g.P("// ", serverType, " is the server API for ", service.GoName, " service.")
+	g.P("// All implementations ", mustOrShould, " embed Unimplemented", serverType)
+	g.P("// for forward compatibility")
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P("//")
+		g.P(deprecationComment)
+	}
+	g.Annotate(serverType, service.Location)
+	g.P("type ", serverType, " interface {")
+	for _, method := range service.Methods {
+		g.Annotate(serverType+"."+method.GoName, method.Location)
+		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
+			g.P(deprecationComment)
+		}
+		g.P(method.Comments.Leading,
+			serverSignature(g, method))
+	}
+	if *requireUnimplemented {
+		g.P("mustEmbedUnimplemented", serverType, "()")
+	}
+	g.P("}")
+	g.P()
+
+	// Server Unimplemented struct for forward compatibility.
+	g.P("// Unimplemented", serverType, " ", mustOrShould, " be embedded to have forward compatible implementations.")
+	g.P("type Unimplemented", serverType, " struct {")
+	g.P("}")
+	g.P()
+	for _, method := range service.Methods {
+		nilArg := ""
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			nilArg = "nil,"
+		}
+		g.P("func (Unimplemented", serverType, ") ", serverSignature(g, method), "{")
+		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
+		g.P("}")
+	}
+	if *requireUnimplemented {
+		g.P("func (Unimplemented", serverType, ") mustEmbedUnimplemented", serverType, "() {}")
+	}
+	g.P()
+
+	// Unsafe Server interface to opt-out of forward compatibility.
+	g.P("// Unsafe", serverType, " may be embedded to opt out of forward compatibility for this service.")
+	g.P("// Use of this interface is not recommended, as added methods to ", serverType, " will")
+	g.P("// result in compilation errors.")
+	g.P("type Unsafe", serverType, " interface {")
+	g.P("mustEmbedUnimplemented", serverType, "()")
+	g.P("}")
+
+	// Server registration.
+	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
+		g.P(deprecationComment)
+	}
+	serviceDescVar := "_" + service.GoName + "_serviceDesc"
+	g.P("func Register", service.GoName, "Server(s *", grpcPackage.Ident("Server"), ", srv ", serverType, ") {")
+	g.P("s.RegisterService(&", serviceDescVar, `, srv)`)
+	g.P("}")
+	g.P()
+
+	// Server handler implementations.
+	var handlerNames []string
+	for _, method := range service.Methods {
+		hname := genServerMethod(gen, file, g, method)
+		handlerNames = append(handlerNames, hname)
+	}
+
+	// Service descriptor.
+	g.P("var ", serviceDescVar, " = ", grpcPackage.Ident("ServiceDesc"), " {")
+	g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",")
+	g.P("HandlerType: (*", serverType, ")(nil),")
+	g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{")
+	for i, method := range service.Methods {
+		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+			continue
+		}
+		g.P("{")
+		g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",")
+		g.P("Handler: ", handlerNames[i], ",")
+		g.P("},")
+	}
+	g.P("},")
+	g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{")
+	for i, method := range service.Methods {
+		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+			continue
+		}
+		g.P("{")
+		g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
+		g.P("Handler: ", handlerNames[i], ",")
+		if method.Desc.IsStreamingServer() {
+			g.P("ServerStreams: true,")
+		}
+		if method.Desc.IsStreamingClient() {
+			g.P("ClientStreams: true,")
+		}
+		g.P("},")
 	}
+	g.P("},")
+	g.P("Metadata: \"", file.Desc.Path(), "\",")
+	g.P("}")
+	g.P()
 }
 
 func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
@@ -130,25 +245,13 @@ func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string
 	return s
 }
 
-func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
+func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, index int) {
 	service := method.Parent
 	sname := fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name())
 
 	if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
 		g.P(deprecationComment)
 	}
-
-	streamDescName := unexport(service.GoName) + method.GoName + "StreamDesc"
-	g.P("var ", streamDescName, " = &", grpcPackage.Ident("StreamDesc"), "{")
-	g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
-	if method.Desc.IsStreamingServer() {
-		g.P("ServerStreams: true,")
-	}
-	if method.Desc.IsStreamingClient() {
-		g.P("ClientStreams: true,")
-	}
-	g.P("}")
-
 	g.P("func (c *", unexport(service.GoName), "Client) ", clientSignature(g, method), "{")
 	if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() {
 		g.P("out := new(", method.Output.GoIdent, ")")
@@ -160,8 +263,8 @@ func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *pr
 		return
 	}
 	streamType := unexport(service.GoName) + method.GoName + "Client"
-
-	g.P(`stream, err := c.cc.NewStream(ctx, `, streamDescName, `, "`, sname, `", opts...)`)
+	serviceDescVar := "_" + service.GoName + "_serviceDesc"
+	g.P("stream, err := c.cc.NewStream(ctx, &", serviceDescVar, ".Streams[", index, `], "`, sname, `", opts...)`)
 	g.P("if err != nil { return nil, err }")
 	g.P("x := &", streamType, "{stream}")
 	if !method.Desc.IsStreamingClient() {
@@ -221,175 +324,7 @@ func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *pr
 	}
 }
 
-func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
-	// Server struct.
-	serviceType := service.GoName + "Service"
-	g.P("// ", serviceType, " is the service API for ", service.GoName, " service.")
-	g.P("// Fields should be assigned to their respective handler implementations only before")
-	g.P("// Register", serviceType, " is called.  Any unassigned fields will result in the")
-	g.P("// handler for that method returning an Unimplemented error.")
-	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
-		g.P("//")
-		g.P(deprecationComment)
-	}
-	g.Annotate(serviceType, service.Location)
-	g.P("type ", serviceType, " struct {")
-	for _, method := range service.Methods {
-		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
-			g.P(deprecationComment)
-		}
-		g.Annotate(serviceType+"."+method.GoName, method.Location)
-		g.P(method.Comments.Leading,
-			method.GoName, " func", handlerSignature(g, method))
-	}
-	g.P("}")
-	g.P()
-
-	// Method handler implementations.
-	for _, method := range service.Methods {
-		genMethodHandler(gen, g, method)
-	}
-
-	// Stream interfaces and implementations.
-	for _, method := range service.Methods {
-		genServerStreamTypes(gen, g, method)
-	}
-
-	// Service registration.
-	genRegisterFunction(gen, file, g, service)
-}
-
-func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
-	g.P("// Register", service.GoName, "Service registers a service implementation with a gRPC server.")
-	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
-		g.P("//")
-		g.P(deprecationComment)
-	}
-	g.P("func Register", service.GoName, "Service(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv *", service.GoName, "Service) {")
-	g.P("srvCopy := *srv")
-	// Add Unimplemented defaults for unset handlers
-	for _, method := range service.Methods {
-		g.P("if srvCopy.", method.GoName, " == nil {")
-		g.P("srvCopy.", method.GoName, " = func", handlerSignature(g, method), "{")
-		nilArg := ""
-		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-			nilArg = "nil, "
-		}
-		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
-		g.P("}")
-		g.P("}")
-	}
-
-	// Service descriptor.
-	g.P("sd := ", grpcPackage.Ident("ServiceDesc"), " {")
-	g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",")
-	g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{")
-	for _, method := range service.Methods {
-		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
-			continue
-		}
-		g.P("{")
-		g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",")
-		g.P("Handler: srvCopy.", unexport(method.GoName), ",")
-		g.P("},")
-	}
-	g.P("},")
-	g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{")
-	for _, method := range service.Methods {
-		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-			continue
-		}
-		g.P("{")
-		g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",")
-		g.P("Handler: srvCopy.", unexport(method.GoName), ",")
-		if method.Desc.IsStreamingServer() {
-			g.P("ServerStreams: true,")
-		}
-		if method.Desc.IsStreamingClient() {
-			g.P("ClientStreams: true,")
-		}
-		g.P("},")
-	}
-	g.P("},")
-	g.P("Metadata: \"", file.Desc.Path(), "\",")
-	g.P("}")
-	g.P()
-
-	g.P("s.RegisterService(&sd, nil)")
-	g.P("}")
-	g.P()
-}
-
-func genUnstableServerInterface(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
-	// Service interface.
-	serverType := service.GoName + "Server"
-	g.P("// ", serverType, " is the service API for ", service.GoName, " service.")
-	g.P("// New methods may be added to this interface if they are added to the service")
-	g.P("// definition, which is not a backward-compatible change.  For this reason, ")
-	g.P("// use of this type is not recommended unless you own the service definition.")
-	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
-		g.P("//")
-		g.P(deprecationComment)
-	}
-	g.Annotate(serverType, service.Location)
-	g.P("type ", serverType, " interface {")
-	for _, method := range service.Methods {
-		g.Annotate(serverType+"."+method.GoName, method.Location)
-		if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() {
-			g.P(deprecationComment)
-		}
-		g.P(method.Comments.Leading,
-			methodSignature(g, method))
-	}
-	g.P("}")
-	g.P()
-
-	// Unimplemented implementation.
-	genUnimplementedServer(gen, file, g, service)
-
-	// Service registration.
-	genUnstableRegisterFunction(gen, file, g, service)
-}
-
-func genUnimplementedServer(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
-	// Server Unimplemented struct for forward compatibility.
-	serverType := service.GoName + "Server"
-	g.P("// Unimplemented", serverType, " can be embedded to have forward compatible implementations of")
-	g.P("// ", serverType)
-	g.P("type Unimplemented", serverType, " struct {")
-	g.P("}")
-	g.P()
-	for _, method := range service.Methods {
-		nilArg := ""
-		if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-			nilArg = "nil,"
-		}
-		g.P("func (Unimplemented", serverType, ") ", methodSignature(g, method), "{")
-		g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`)
-		g.P("}")
-	}
-	g.P()
-}
-
-func genUnstableRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) {
-	serverType := service.GoName + "Server"
-	g.P("// Register", serverType, " registers a service implementation with a gRPC server.")
-	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
-		g.P("//")
-		g.P(deprecationComment)
-	}
-	g.P("func Register", serverType, "(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv ", serverType, ") {")
-	g.P("str := &", service.GoName, "Service{")
-	for _, method := range service.Methods {
-		g.P(method.GoName, ": srv.", method.GoName, ",")
-	}
-	g.P("}")
-	g.P("Register", service.GoName, "Service(s, str)")
-	g.P("}")
-	g.P()
-}
-
-func methodSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
+func serverSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
 	var reqArgs []string
 	ret := "error"
 	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
@@ -405,80 +340,39 @@ func methodSignature(g *protogen.GeneratedFile, method *protogen.Method) string
 	return method.GoName + "(" + strings.Join(reqArgs, ", ") + ") " + ret
 }
 
-func handlerSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
-	var reqArgs []string
-	ret := "error"
-	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-		reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context")))
-		ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)"
-	}
-	if !method.Desc.IsStreamingClient() {
-		reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent))
-	}
-	if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
-		reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server")
-	}
-	return "(" + strings.Join(reqArgs, ", ") + ") " + ret
-}
-
-func genericHandlerSignature(g *protogen.GeneratedFile, method *protogen.Method) string {
-	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-		// Unary
-		return "(_ interface{}, ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context")) +
-			", dec func(interface{}) error, interceptor " +
-			g.QualifiedGoIdent(grpcPackage.Ident("UnaryServerInterceptor")) + ") (interface{}, error)"
-	}
-	// Streaming
-	return "(_ interface{}, stream " + g.QualifiedGoIdent(grpcPackage.Ident("ServerStream")) + ") error"
-}
-
-func genMethodHandler(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
+func genServerMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method) string {
 	service := method.Parent
+	hname := fmt.Sprintf("_%s_%s_Handler", service.GoName, method.GoName)
 
-	g.P("func (s *", service.GoName, "Service) ", unexport(method.GoName), genericHandlerSignature(g, method), " {")
-
-	// Unary
 	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
+		g.P("func ", hname, "(srv interface{}, ctx ", contextPackage.Ident("Context"), ", dec func(interface{}) error, interceptor ", grpcPackage.Ident("UnaryServerInterceptor"), ") (interface{}, error) {")
 		g.P("in := new(", method.Input.GoIdent, ")")
 		g.P("if err := dec(in); err != nil { return nil, err }")
-
-		g.P("if interceptor == nil {")
-		g.P("return s.", method.GoName, "(ctx, in)")
-		g.P("}")
-
+		g.P("if interceptor == nil { return srv.(", service.GoName, "Server).", method.GoName, "(ctx, in) }")
 		g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{")
-		g.P("Server: s,")
+		g.P("Server: srv,")
 		g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.GoName)), ",")
 		g.P("}")
 		g.P("handler := func(ctx ", contextPackage.Ident("Context"), ", req interface{}) (interface{}, error) {")
-		g.P("return s.", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")
+		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")
 		g.P("}")
 		g.P("return interceptor(ctx, in, info, handler)")
 		g.P("}")
-		return
+		g.P()
+		return hname
 	}
-
-	// Streaming
 	streamType := unexport(service.GoName) + method.GoName + "Server"
+	g.P("func ", hname, "(srv interface{}, stream ", grpcPackage.Ident("ServerStream"), ") error {")
 	if !method.Desc.IsStreamingClient() {
-		// Server-streaming
 		g.P("m := new(", method.Input.GoIdent, ")")
 		g.P("if err := stream.RecvMsg(m); err != nil { return err }")
-		g.P("return s.", method.GoName, "(m, &", streamType, "{stream})")
+		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(m, &", streamType, "{stream})")
 	} else {
-		// Bidi-streaming
-		g.P("return s.", method.GoName, "(&", streamType, "{stream})")
+		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(&", streamType, "{stream})")
 	}
 	g.P("}")
-}
+	g.P()
 
-func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) {
-	if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() {
-		// Unary method
-		return
-	}
-	service := method.Parent
-	streamType := unexport(service.GoName) + method.GoName + "Server"
 	genSend := method.Desc.IsStreamingServer()
 	genSendAndClose := !method.Desc.IsStreamingServer()
 	genRecv := method.Desc.IsStreamingClient()
@@ -524,7 +418,7 @@ func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, metho
 		g.P()
 	}
 
-	return
+	return hname
 }
 
 const deprecationComment = "// Deprecated: Do not use."
diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index 65896f13188d..dcb2e0efe926 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -34,19 +34,17 @@ import (
 	"flag"
 
 	"google.golang.org/protobuf/compiler/protogen"
-	"google.golang.org/protobuf/types/pluginpb"
 )
 
-var genUnstableServerInterfaces *bool
+var requireUnimplemented *bool
 
 func main() {
 	var flags flag.FlagSet
-	genUnstableServerInterfaces = flags.Bool("gen_unstable_server_interfaces", false, `set to generate legacy "Server" interfaces which do not guarantee backward compatibility`)
+	requireUnimplemented = flags.Bool("require_unimplemented_servers", true, "set to false to match legacy behavior")
 
 	protogen.Options{
 		ParamFunc: flags.Set,
 	}.Run(func(gen *protogen.Plugin) error {
-		gen.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL)
 		for _, f := range gen.Files {
 			if !f.Generate {
 				continue
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index 2b06da50df2f..d27bd063ee20 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -34,14 +34,8 @@ func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceCl
 	return &handshakerServiceClient{cc}
 }
 
-var handshakerServiceDoHandshakeStreamDesc = &grpc.StreamDesc{
-	StreamName:    "DoHandshake",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
-	stream, err := c.cc.NewStream(ctx, handshakerServiceDoHandshakeStreamDesc, "/grpc.gcp.HandshakerService/DoHandshake", opts...)
+	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -71,22 +65,42 @@ func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) {
 	return m, nil
 }
 
-// HandshakerServiceService is the service API for HandshakerService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterHandshakerServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type HandshakerServiceService struct {
+// HandshakerServiceServer is the server API for HandshakerService service.
+// All implementations must embed UnimplementedHandshakerServiceServer
+// for forward compatibility
+type HandshakerServiceServer interface {
 	// Handshaker service accepts a stream of handshaker request, returning a
 	// stream of handshaker response. Client is expected to send exactly one
 	// message with either client_start or server_start followed by one or more
 	// messages with next. Each time client sends a request, the handshaker
 	// service expects to respond. Client does not have to wait for service's
 	// response before sending next request.
-	DoHandshake func(HandshakerService_DoHandshakeServer) error
+	DoHandshake(HandshakerService_DoHandshakeServer) error
+	mustEmbedUnimplementedHandshakerServiceServer()
+}
+
+// UnimplementedHandshakerServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedHandshakerServiceServer struct {
 }
 
-func (s *HandshakerServiceService) doHandshake(_ interface{}, stream grpc.ServerStream) error {
-	return s.DoHandshake(&handshakerServiceDoHandshakeServer{stream})
+func (UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
+	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
+}
+func (UnimplementedHandshakerServiceServer) mustEmbedUnimplementedHandshakerServiceServer() {}
+
+// UnsafeHandshakerServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to HandshakerServiceServer will
+// result in compilation errors.
+type UnsafeHandshakerServiceServer interface {
+	mustEmbedUnimplementedHandshakerServiceServer()
+}
+
+func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
+	s.RegisterService(&_HandshakerService_serviceDesc, srv)
+}
+
+func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream})
 }
 
 type HandshakerService_DoHandshakeServer interface {
@@ -111,58 +125,17 @@ func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
 	return m, nil
 }
 
-// RegisterHandshakerServiceService registers a service implementation with a gRPC server.
-func RegisterHandshakerServiceService(s grpc.ServiceRegistrar, srv *HandshakerServiceService) {
-	srvCopy := *srv
-	if srvCopy.DoHandshake == nil {
-		srvCopy.DoHandshake = func(HandshakerService_DoHandshakeServer) error {
-			return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.gcp.HandshakerService",
-		Methods:     []grpc.MethodDesc{},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "DoHandshake",
-				Handler:       srvCopy.doHandshake,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+var _HandshakerService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.gcp.HandshakerService",
+	HandlerType: (*HandshakerServiceServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "DoHandshake",
+			Handler:       _HandshakerService_DoHandshake_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
-		Metadata: "grpc/gcp/handshaker.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// HandshakerServiceServer is the service API for HandshakerService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type HandshakerServiceServer interface {
-	// Handshaker service accepts a stream of handshaker request, returning a
-	// stream of handshaker response. Client is expected to send exactly one
-	// message with either client_start or server_start followed by one or more
-	// messages with next. Each time client sends a request, the handshaker
-	// service expects to respond. Client does not have to wait for service's
-	// response before sending next request.
-	DoHandshake(HandshakerService_DoHandshakeServer) error
-}
-
-// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations of
-// HandshakerServiceServer
-type UnimplementedHandshakerServiceServer struct {
-}
-
-func (UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error {
-	return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented")
-}
-
-// RegisterHandshakerServiceServer registers a service implementation with a gRPC server.
-func RegisterHandshakerServiceServer(s grpc.ServiceRegistrar, srv HandshakerServiceServer) {
-	str := &HandshakerServiceService{
-		DoHandshake: srv.DoHandshake,
-	}
-	RegisterHandshakerServiceService(s, str)
+	},
+	Metadata: "grpc/gcp/handshaker.proto",
 }
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index 35b3681491de..ab5919292784 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -30,10 +30,6 @@ func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificat
 	return &meshCertificateServiceClient{cc}
 }
 
-var meshCertificateServiceCreateCertificateStreamDesc = &grpc.StreamDesc{
-	StreamName: "CreateCertificate",
-}
-
 func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
 	out := new(MeshCertificateResponse)
 	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
@@ -43,80 +39,64 @@ func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in
 	return out, nil
 }
 
-// MeshCertificateServiceService is the service API for MeshCertificateService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterMeshCertificateServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type MeshCertificateServiceService struct {
+// MeshCertificateServiceServer is the server API for MeshCertificateService service.
+// All implementations must embed UnimplementedMeshCertificateServiceServer
+// for forward compatibility
+type MeshCertificateServiceServer interface {
 	// Using provided CSR, returns a signed certificate that represents a GCP
 	// service account identity.
-	CreateCertificate func(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
+	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
+	mustEmbedUnimplementedMeshCertificateServiceServer()
+}
+
+// UnimplementedMeshCertificateServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedMeshCertificateServiceServer struct {
+}
+
+func (UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
+}
+func (UnimplementedMeshCertificateServiceServer) mustEmbedUnimplementedMeshCertificateServiceServer() {
+}
+
+// UnsafeMeshCertificateServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to MeshCertificateServiceServer will
+// result in compilation errors.
+type UnsafeMeshCertificateServiceServer interface {
+	mustEmbedUnimplementedMeshCertificateServiceServer()
 }
 
-func (s *MeshCertificateServiceService) createCertificate(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) {
+	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
+}
+
+func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(MeshCertificateRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.CreateCertificate(ctx, in)
+		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.CreateCertificate(ctx, req.(*MeshCertificateRequest))
+		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterMeshCertificateServiceService registers a service implementation with a gRPC server.
-func RegisterMeshCertificateServiceService(s grpc.ServiceRegistrar, srv *MeshCertificateServiceService) {
-	srvCopy := *srv
-	if srvCopy.CreateCertificate == nil {
-		srvCopy.CreateCertificate = func(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "google.security.meshca.v1.MeshCertificateService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "CreateCertificate",
-				Handler:    srvCopy.createCertificate,
-			},
+var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "google.security.meshca.v1.MeshCertificateService",
+	HandlerType: (*MeshCertificateServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "CreateCertificate",
+			Handler:    _MeshCertificateService_CreateCertificate_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "istio/google/security/meshca/v1/meshca.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// MeshCertificateServiceServer is the service API for MeshCertificateService service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type MeshCertificateServiceServer interface {
-	// Using provided CSR, returns a signed certificate that represents a GCP
-	// service account identity.
-	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
-}
-
-// UnimplementedMeshCertificateServiceServer can be embedded to have forward compatible implementations of
-// MeshCertificateServiceServer
-type UnimplementedMeshCertificateServiceServer struct {
-}
-
-func (UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
-}
-
-// RegisterMeshCertificateServiceServer registers a service implementation with a gRPC server.
-func RegisterMeshCertificateServiceServer(s grpc.ServiceRegistrar, srv MeshCertificateServiceServer) {
-	str := &MeshCertificateServiceService{
-		CreateCertificate: srv.CreateCertificate,
-	}
-	RegisterMeshCertificateServiceService(s, str)
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "istio/google/security/meshca/v1/meshca.proto",
 }
diff --git a/examples/features/authentication/server/main.go b/examples/features/authentication/server/main.go
index 505e8cb09847..5163fc31671b 100644
--- a/examples/features/authentication/server/main.go
+++ b/examples/features/authentication/server/main.go
@@ -63,7 +63,7 @@ func main() {
 		grpc.Creds(credentials.NewServerTLSFromCert(&cert)),
 	}
 	s := grpc.NewServer(opts...)
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &ecServer{})
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
@@ -73,7 +73,11 @@ func main() {
 	}
 }
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+type ecServer struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
diff --git a/examples/features/cancellation/server/main.go b/examples/features/cancellation/server/main.go
index 1896828f2c99..520286bf193e 100644
--- a/examples/features/cancellation/server/main.go
+++ b/examples/features/cancellation/server/main.go
@@ -33,7 +33,11 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
 	for {
 		in, err := stream.Recv()
 		if err != nil {
@@ -57,6 +61,6 @@ func main() {
 	}
 	fmt.Printf("server listening at port %v\n", lis.Addr())
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{BidirectionalStreamingEcho: bidirectionalStreamingEcho})
+	pb.RegisterEchoServer(s, &server{})
 	s.Serve(lis)
 }
diff --git a/examples/features/compression/server/main.go b/examples/features/compression/server/main.go
index 442d196085c9..e495cc89106b 100644
--- a/examples/features/compression/server/main.go
+++ b/examples/features/compression/server/main.go
@@ -34,7 +34,11 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage())
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
@@ -49,6 +53,6 @@ func main() {
 	fmt.Printf("server listening at %v\n", lis.Addr())
 
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &server{})
 	s.Serve(lis)
 }
diff --git a/examples/features/deadline/server/main.go b/examples/features/deadline/server/main.go
index f5a033db3d9c..11cd47a6b5b3 100644
--- a/examples/features/deadline/server/main.go
+++ b/examples/features/deadline/server/main.go
@@ -40,6 +40,7 @@ var port = flag.Int("port", 50052, "port number")
 
 // server is used to implement EchoServer.
 type server struct {
+	pb.UnimplementedEchoServer
 	client pb.EchoClient
 	cc     *grpc.ClientConn
 }
@@ -111,12 +112,9 @@ func main() {
 
 	echoServer := newEchoServer()
 	defer echoServer.Close()
-	grpcServer := grpc.NewServer()
 
-	pb.RegisterEchoService(grpcServer, &pb.EchoService{
-		UnaryEcho:                  echoServer.UnaryEcho,
-		BidirectionalStreamingEcho: echoServer.BidirectionalStreamingEcho,
-	})
+	grpcServer := grpc.NewServer()
+	pb.RegisterEchoServer(grpcServer, echoServer)
 
 	if err := grpcServer.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/debugging/server/main.go b/examples/features/debugging/server/main.go
index 839212c99fbc..397cb0c781e6 100644
--- a/examples/features/debugging/server/main.go
+++ b/examples/features/debugging/server/main.go
@@ -36,13 +36,23 @@ var (
 	ports = []string{":10001", ":10002", ":10003"}
 )
 
-// sayHello implements helloworld.GreeterServer.SayHello
-func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// server is used to implement helloworld.GreeterServer.
+type server struct {
+	pb.UnimplementedGreeterServer
+}
+
+// SayHello implements helloworld.GreeterServer
+func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
-// sayHelloSlow implements helloworld.GreeterServer.SayHello
-func sayHelloSlow(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// slow server is used to simulate a server that has a variable delay in its response.
+type slowServer struct {
+	pb.UnimplementedGreeterServer
+}
+
+// SayHello implements helloworld.GreeterServer
+func (s *slowServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	// Delay 100ms ~ 200ms before replying
 	time.Sleep(time.Duration(100+grpcrand.Intn(100)) * time.Millisecond)
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
@@ -60,7 +70,7 @@ func main() {
 	go s.Serve(lis)
 	defer s.Stop()
 
-	/***** Start three GreeterServers(with one of them to be the slow server). *****/
+	/***** Start three GreeterServers(with one of them to be the slowServer). *****/
 	for i := 0; i < 3; i++ {
 		lis, err := net.Listen("tcp", ports[i])
 		if err != nil {
@@ -69,9 +79,9 @@ func main() {
 		defer lis.Close()
 		s := grpc.NewServer()
 		if i == 2 {
-			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHelloSlow})
+			pb.RegisterGreeterServer(s, &slowServer{})
 		} else {
-			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
+			pb.RegisterGreeterServer(s, &server{})
 		}
 		go s.Serve(lis)
 	}
diff --git a/examples/features/encryption/ALTS/server/main.go b/examples/features/encryption/ALTS/server/main.go
index a72ccd11e7fe..87bedd810f47 100644
--- a/examples/features/encryption/ALTS/server/main.go
+++ b/examples/features/encryption/ALTS/server/main.go
@@ -34,7 +34,11 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+type ecServer struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -50,8 +54,8 @@ func main() {
 
 	s := grpc.NewServer(grpc.Creds(altsTC))
 
-	// Register EchoService on the server.
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	// Register EchoServer on the server.
+	pb.RegisterEchoServer(s, &ecServer{})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/encryption/TLS/server/main.go b/examples/features/encryption/TLS/server/main.go
index d5ea6ccfcaf3..81bf1f3acc3e 100644
--- a/examples/features/encryption/TLS/server/main.go
+++ b/examples/features/encryption/TLS/server/main.go
@@ -35,7 +35,11 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+type ecServer struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -56,7 +60,7 @@ func main() {
 	s := grpc.NewServer(grpc.Creds(creds))
 
 	// Register EchoServer on the server.
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &ecServer{})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/errors/server/main.go b/examples/features/errors/server/main.go
index c6c06a2bf2cd..dc337741ad02 100644
--- a/examples/features/errors/server/main.go
+++ b/examples/features/errors/server/main.go
@@ -39,6 +39,7 @@ var port = flag.Int("port", 50052, "port number")
 
 // server is used to implement helloworld.GreeterServer.
 type server struct {
+	pb.UnimplementedGreeterServer
 	mu    sync.Mutex
 	count map[string]int
 }
@@ -77,8 +78,7 @@ func main() {
 	}
 
 	s := grpc.NewServer()
-	hw := &server{count: make(map[string]int)}
-	pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: hw.SayHello})
+	pb.RegisterGreeterServer(s, &server{count: make(map[string]int)})
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
diff --git a/examples/features/health/server/main.go b/examples/features/health/server/main.go
index 670518da9d08..3f79c8ba3470 100644
--- a/examples/features/health/server/main.go
+++ b/examples/features/health/server/main.go
@@ -40,12 +40,18 @@ var (
 	system = "" // empty string represents the health of the system
 )
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+type echoServer struct {
+	pb.UnimplementedEchoServer
+}
+
+func (e *echoServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{
 		Message: fmt.Sprintf("hello from localhost:%d", *port),
 	}, nil
 }
 
+var _ pb.EchoServer = &echoServer{}
+
 func main() {
 	flag.Parse()
 
@@ -57,7 +63,7 @@ func main() {
 	s := grpc.NewServer()
 	healthcheck := health.NewServer()
 	healthpb.RegisterHealthServer(s, healthcheck)
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &echoServer{})
 
 	go func() {
 		// asynchronously inspect dependencies and toggle serving status as needed
diff --git a/examples/features/interceptor/server/main.go b/examples/features/interceptor/server/main.go
index 5ca5a8432517..1b07cdecd6ca 100644
--- a/examples/features/interceptor/server/main.go
+++ b/examples/features/interceptor/server/main.go
@@ -51,12 +51,16 @@ func logger(format string, a ...interface{}) {
 	fmt.Printf("LOG:\t"+format+"\n", a...)
 }
 
-func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("unary echoing message %q\n", in.Message)
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
 
-func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
+func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
 	for {
 		in, err := stream.Recv()
 		if err != nil {
@@ -152,11 +156,8 @@ func main() {
 
 	s := grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor))
 
-	// Register EchoService on the server.
-	pb.RegisterEchoService(s, &pb.EchoService{
-		UnaryEcho:                  unaryEcho,
-		BidirectionalStreamingEcho: bidirectionalStreamingEcho,
-	})
+	// Register EchoServer on the server.
+	pb.RegisterEchoServer(s, &server{})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/keepalive/server/main.go b/examples/features/keepalive/server/main.go
index e3ec8ae0679a..beaa8f710885 100644
--- a/examples/features/keepalive/server/main.go
+++ b/examples/features/keepalive/server/main.go
@@ -48,7 +48,12 @@ var kasp = keepalive.ServerParameters{
 	Timeout:               1 * time.Second,  // Wait 1 second for the ping ack before assuming the connection is dead
 }
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+// server implements EchoServer.
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -62,7 +67,7 @@ func main() {
 	}
 
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &server{})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/load_balancing/server/main.go b/examples/features/load_balancing/server/main.go
index 680895fe50f6..9d179579ed41 100644
--- a/examples/features/load_balancing/server/main.go
+++ b/examples/features/load_balancing/server/main.go
@@ -36,6 +36,7 @@ var (
 )
 
 type ecServer struct {
+	pb.UnimplementedEchoServer
 	addr string
 }
 
@@ -49,8 +50,7 @@ func startServer(addr string) {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	e := &ecServer{addr: addr}
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: e.UnaryEcho})
+	pb.RegisterEchoServer(s, &ecServer{addr: addr})
 	log.Printf("serving on %s\n", addr)
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/metadata/server/main.go b/examples/features/metadata/server/main.go
index af12a3a65d17..cda3b4978697 100644
--- a/examples/features/metadata/server/main.go
+++ b/examples/features/metadata/server/main.go
@@ -44,7 +44,11 @@ const (
 	streamingCount  = 10
 )
 
-func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("--- UnaryEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -73,7 +77,7 @@ func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
 
-func serverStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error {
+func (s *server) ServerStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error {
 	fmt.Printf("--- ServerStreamingEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -110,7 +114,7 @@ func serverStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoS
 	return nil
 }
 
-func clientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error {
+func (s *server) ClientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error {
 	fmt.Printf("--- ClientStreamingEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -150,7 +154,7 @@ func clientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error {
 	}
 }
 
-func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
+func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error {
 	fmt.Printf("--- BidirectionalStreamingEcho ---\n")
 	// Create trailer in defer to record function return time.
 	defer func() {
@@ -201,11 +205,6 @@ func main() {
 	fmt.Printf("server listening at %v\n", lis.Addr())
 
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{
-		UnaryEcho:                  unaryEcho,
-		ServerStreamingEcho:        serverStreamingEcho,
-		ClientStreamingEcho:        clientStreamingEcho,
-		BidirectionalStreamingEcho: bidirectionalStreamingEcho,
-	})
+	pb.RegisterEchoServer(s, &server{})
 	s.Serve(lis)
 }
diff --git a/examples/features/multiplex/server/main.go b/examples/features/multiplex/server/main.go
index d8b13305da25..18da09adda3f 100644
--- a/examples/features/multiplex/server/main.go
+++ b/examples/features/multiplex/server/main.go
@@ -34,13 +34,21 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-// sayHello implements helloworld.GreeterServer.SayHello
-func sayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
+// hwServer is used to implement helloworld.GreeterServer.
+type hwServer struct {
+	hwpb.UnimplementedGreeterServer
+}
+
+// SayHello implements helloworld.GreeterServer
+func (s *hwServer) SayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
 	return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
-// unaryEcho implements echo.Echo.UnaryEcho
-func unaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
+type ecServer struct {
+	ecpb.UnimplementedEchoServer
+}
+
+func (s *ecServer) UnaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
 	return &ecpb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -55,10 +63,10 @@ func main() {
 	s := grpc.NewServer()
 
 	// Register Greeter on the server.
-	hwpb.RegisterGreeterService(s, &hwpb.GreeterService{SayHello: sayHello})
+	hwpb.RegisterGreeterServer(s, &hwServer{})
 
-	// Register Echo on the same server.
-	ecpb.RegisterEchoService(s, &ecpb.EchoService{UnaryEcho: unaryEcho})
+	// Register RouteGuide on the same server.
+	ecpb.RegisterEchoServer(s, &ecServer{})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/name_resolving/server/main.go b/examples/features/name_resolving/server/main.go
index 77d35eb2adac..1977f44d0c5e 100644
--- a/examples/features/name_resolving/server/main.go
+++ b/examples/features/name_resolving/server/main.go
@@ -32,8 +32,13 @@ import (
 
 const addr = "localhost:50051"
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
-	return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, addr)}, nil
+type ecServer struct {
+	pb.UnimplementedEchoServer
+	addr string
+}
+
+func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+	return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil
 }
 
 func main() {
@@ -42,7 +47,7 @@ func main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &ecServer{addr: addr})
 	log.Printf("serving on %s\n", addr)
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/profiling/README.md b/examples/features/profiling/README.md
index 8c30a3dbcac4..1fd80e9ada2d 100644
--- a/examples/features/profiling/README.md
+++ b/examples/features/profiling/README.md
@@ -41,7 +41,7 @@ type server struct{}
 
 func main() error {
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{...})
+	pb.RegisterEchoServer(s, &server{})
 
 	// Include this to register a profiling-specific service within your server.
 	if err := profsvc.Init(&profsvc.ProfilingConfig{Server: s}); err != nil {
diff --git a/examples/features/profiling/server/main.go b/examples/features/profiling/server/main.go
index aa0bf11b2403..ee0a4f55ab2e 100644
--- a/examples/features/profiling/server/main.go
+++ b/examples/features/profiling/server/main.go
@@ -33,7 +33,11 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
 	fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage())
 	return &pb.EchoResponse{Message: in.Message}, nil
 }
@@ -48,7 +52,7 @@ func main() {
 	fmt.Printf("server listening at %v\n", lis.Addr())
 
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &server{})
 
 	// Register your grpc.Server with profiling.
 	pc := &profsvc.ProfilingConfig{
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index ff30c6524b4a..2da0bfeb371c 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -35,10 +35,6 @@ func NewEchoClient(cc grpc.ClientConnInterface) EchoClient {
 	return &echoClient{cc}
 }
 
-var echoUnaryEchoStreamDesc = &grpc.StreamDesc{
-	StreamName: "UnaryEcho",
-}
-
 func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) {
 	out := new(EchoResponse)
 	err := c.cc.Invoke(ctx, "/grpc.examples.echo.Echo/UnaryEcho", in, out, opts...)
@@ -48,13 +44,8 @@ func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grp
 	return out, nil
 }
 
-var echoServerStreamingEchoStreamDesc = &grpc.StreamDesc{
-	StreamName:    "ServerStreamingEcho",
-	ServerStreams: true,
-}
-
 func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, echoServerStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[0], "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -85,13 +76,8 @@ func (x *echoServerStreamingEchoClient) Recv() (*EchoResponse, error) {
 	return m, nil
 }
 
-var echoClientStreamingEchoStreamDesc = &grpc.StreamDesc{
-	StreamName:    "ClientStreamingEcho",
-	ClientStreams: true,
-}
-
 func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, echoClientStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[1], "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -124,14 +110,8 @@ func (x *echoClientStreamingEchoClient) CloseAndRecv() (*EchoResponse, error) {
 	return m, nil
 }
 
-var echoBidirectionalStreamingEchoStreamDesc = &grpc.StreamDesc{
-	StreamName:    "BidirectionalStreamingEcho",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, echoBidirectionalStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[2], "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -161,50 +141,74 @@ func (x *echoBidirectionalStreamingEchoClient) Recv() (*EchoResponse, error) {
 	return m, nil
 }
 
-// EchoService is the service API for Echo service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterEchoService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type EchoService struct {
+// EchoServer is the server API for Echo service.
+// All implementations must embed UnimplementedEchoServer
+// for forward compatibility
+type EchoServer interface {
 	// UnaryEcho is unary echo.
-	UnaryEcho func(context.Context, *EchoRequest) (*EchoResponse, error)
+	UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error)
 	// ServerStreamingEcho is server side streaming.
-	ServerStreamingEcho func(*EchoRequest, Echo_ServerStreamingEchoServer) error
+	ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error
 	// ClientStreamingEcho is client side streaming.
-	ClientStreamingEcho func(Echo_ClientStreamingEchoServer) error
+	ClientStreamingEcho(Echo_ClientStreamingEchoServer) error
 	// BidirectionalStreamingEcho is bidi streaming.
-	BidirectionalStreamingEcho func(Echo_BidirectionalStreamingEchoServer) error
+	BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error
+	mustEmbedUnimplementedEchoServer()
+}
+
+// UnimplementedEchoServer must be embedded to have forward compatible implementations.
+type UnimplementedEchoServer struct {
+}
+
+func (UnimplementedEchoServer) UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
+}
+func (UnimplementedEchoServer) ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
+}
+func (UnimplementedEchoServer) ClientStreamingEcho(Echo_ClientStreamingEchoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
+}
+func (UnimplementedEchoServer) BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error {
+	return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
+}
+func (UnimplementedEchoServer) mustEmbedUnimplementedEchoServer() {}
+
+// UnsafeEchoServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to EchoServer will
+// result in compilation errors.
+type UnsafeEchoServer interface {
+	mustEmbedUnimplementedEchoServer()
 }
 
-func (s *EchoService) unaryEcho(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func RegisterEchoServer(s *grpc.Server, srv EchoServer) {
+	s.RegisterService(&_Echo_serviceDesc, srv)
+}
+
+func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(EchoRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.UnaryEcho(ctx, in)
+		return srv.(EchoServer).UnaryEcho(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.examples.echo.Echo/UnaryEcho",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.UnaryEcho(ctx, req.(*EchoRequest))
+		return srv.(EchoServer).UnaryEcho(ctx, req.(*EchoRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *EchoService) serverStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
+
+func _Echo_ServerStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(EchoRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return s.ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
-}
-func (s *EchoService) clientStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
-	return s.ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
-}
-func (s *EchoService) bidirectionalStreamingEcho(_ interface{}, stream grpc.ServerStream) error {
-	return s.BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
+	return srv.(EchoServer).ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream})
 }
 
 type Echo_ServerStreamingEchoServer interface {
@@ -220,6 +224,10 @@ func (x *echoServerStreamingEchoServer) Send(m *EchoResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
+func _Echo_ClientStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(EchoServer).ClientStreamingEcho(&echoClientStreamingEchoServer{stream})
+}
+
 type Echo_ClientStreamingEchoServer interface {
 	SendAndClose(*EchoResponse) error
 	Recv() (*EchoRequest, error)
@@ -242,6 +250,10 @@ func (x *echoClientStreamingEchoServer) Recv() (*EchoRequest, error) {
 	return m, nil
 }
 
+func _Echo_BidirectionalStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(EchoServer).BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream})
+}
+
 type Echo_BidirectionalStreamingEchoServer interface {
 	Send(*EchoResponse) error
 	Recv() (*EchoRequest, error)
@@ -264,57 +276,32 @@ func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) {
 	return m, nil
 }
 
-// RegisterEchoService registers a service implementation with a gRPC server.
-func RegisterEchoService(s grpc.ServiceRegistrar, srv *EchoService) {
-	srvCopy := *srv
-	if srvCopy.UnaryEcho == nil {
-		srvCopy.UnaryEcho = func(context.Context, *EchoRequest) (*EchoResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented")
-		}
-	}
-	if srvCopy.ServerStreamingEcho == nil {
-		srvCopy.ServerStreamingEcho = func(*EchoRequest, Echo_ServerStreamingEchoServer) error {
-			return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented")
-		}
-	}
-	if srvCopy.ClientStreamingEcho == nil {
-		srvCopy.ClientStreamingEcho = func(Echo_ClientStreamingEchoServer) error {
-			return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented")
-		}
-	}
-	if srvCopy.BidirectionalStreamingEcho == nil {
-		srvCopy.BidirectionalStreamingEcho = func(Echo_BidirectionalStreamingEchoServer) error {
-			return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.examples.echo.Echo",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "UnaryEcho",
-				Handler:    srvCopy.unaryEcho,
-			},
+var _Echo_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.examples.echo.Echo",
+	HandlerType: (*EchoServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryEcho",
+			Handler:    _Echo_UnaryEcho_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "ServerStreamingEcho",
-				Handler:       srvCopy.serverStreamingEcho,
-				ServerStreams: true,
-			},
-			{
-				StreamName:    "ClientStreamingEcho",
-				Handler:       srvCopy.clientStreamingEcho,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "BidirectionalStreamingEcho",
-				Handler:       srvCopy.bidirectionalStreamingEcho,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ServerStreamingEcho",
+			Handler:       _Echo_ServerStreamingEcho_Handler,
+			ServerStreams: true,
 		},
-		Metadata: "examples/features/proto/echo/echo.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+		{
+			StreamName:    "ClientStreamingEcho",
+			Handler:       _Echo_ClientStreamingEcho_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "BidirectionalStreamingEcho",
+			Handler:       _Echo_BidirectionalStreamingEcho_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "examples/features/proto/echo/echo.proto",
 }
diff --git a/examples/features/reflection/server/main.go b/examples/features/reflection/server/main.go
index 1501c01f33aa..569273dfdd3e 100644
--- a/examples/features/reflection/server/main.go
+++ b/examples/features/reflection/server/main.go
@@ -35,13 +35,21 @@ import (
 
 var port = flag.Int("port", 50051, "the port to serve on")
 
-// sayHello implements helloworld.GreeterServer.SayHello
-func sayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
+// hwServer is used to implement helloworld.GreeterServer.
+type hwServer struct {
+	hwpb.UnimplementedGreeterServer
+}
+
+// SayHello implements helloworld.GreeterServer
+func (s *hwServer) SayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) {
 	return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
-// unaryEcho implements echo.Echo.UnaryEcho
-func unaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
+type ecServer struct {
+	ecpb.UnimplementedEchoServer
+}
+
+func (s *ecServer) UnaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) {
 	return &ecpb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -56,10 +64,10 @@ func main() {
 	s := grpc.NewServer()
 
 	// Register Greeter on the server.
-	hwpb.RegisterGreeterService(s, &hwpb.GreeterService{SayHello: sayHello})
+	hwpb.RegisterGreeterServer(s, &hwServer{})
 
 	// Register RouteGuide on the same server.
-	ecpb.RegisterEchoService(s, &ecpb.EchoService{UnaryEcho: unaryEcho})
+	ecpb.RegisterEchoServer(s, &ecServer{})
 
 	// Register reflection service on gRPC server.
 	reflection.Register(s)
diff --git a/examples/features/retry/server/main.go b/examples/features/retry/server/main.go
index 08dd40c93f1c..fdec2f052e27 100644
--- a/examples/features/retry/server/main.go
+++ b/examples/features/retry/server/main.go
@@ -37,6 +37,7 @@ import (
 var port = flag.Int("port", 50052, "port number")
 
 type failingServer struct {
+	pb.UnimplementedEchoServer
 	mu sync.Mutex
 
 	reqCounter uint
@@ -85,7 +86,7 @@ func main() {
 		reqModulo:  4,
 	}
 
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: failingservice.UnaryEcho})
+	pb.RegisterEchoServer(s, failingservice)
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
diff --git a/examples/features/wait_for_ready/main.go b/examples/features/wait_for_ready/main.go
index f3669e986576..f865410f1aa2 100644
--- a/examples/features/wait_for_ready/main.go
+++ b/examples/features/wait_for_ready/main.go
@@ -34,7 +34,12 @@ import (
 	pb "google.golang.org/grpc/examples/features/proto/echo"
 )
 
-func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
+// server is used to implement EchoServer.
+type server struct {
+	pb.UnimplementedEchoServer
+}
+
+func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) {
 	return &pb.EchoResponse{Message: req.Message}, nil
 }
 
@@ -45,7 +50,7 @@ func serve() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho})
+	pb.RegisterEchoServer(s, &server{})
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
diff --git a/examples/features/xds/server/main.go b/examples/features/xds/server/main.go
index 96cd6e468cf5..7e0815645e5a 100644
--- a/examples/features/xds/server/main.go
+++ b/examples/features/xds/server/main.go
@@ -45,6 +45,8 @@ const (
 
 // server is used to implement helloworld.GreeterServer.
 type server struct {
+	pb.UnimplementedGreeterServer
+
 	serverName string
 }
 
@@ -122,8 +124,7 @@ func main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	hw := newServer(hostname)
-	pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: hw.SayHello})
+	pb.RegisterGreeterServer(s, newServer(hostname))
 
 	reflection.Register(s)
 	healthServer := health.NewServer()
diff --git a/examples/gotutorial.md b/examples/gotutorial.md
index 2862b0a28b98..86993562e909 100644
--- a/examples/gotutorial.md
+++ b/examples/gotutorial.md
@@ -268,13 +268,7 @@ if err != nil {
         log.Fatalf("failed to listen: %v", err)
 }
 grpcServer := grpc.NewServer()
-rgs := &routeGuideServer{}
-pb.RegisterRouteGuideService(grpcServer, pb.RouteGuideService{
-        GetFeature:   rgs.GetFeature,
-        ListFeatures: rgs.ListFeatures,
-        RecordRoute:  rgs.RecordRoute,
-        RouteChat:    rgs.RouteChat,
-})
+pb.RegisterRouteGuideServer(grpcServer, &routeGuideServer{})
 ... // determine whether to use TLS
 grpcServer.Serve(lis)
 ```
diff --git a/examples/helloworld/greeter_server/main.go b/examples/helloworld/greeter_server/main.go
index 4406f42011e4..15604f9fc1f4 100644
--- a/examples/helloworld/greeter_server/main.go
+++ b/examples/helloworld/greeter_server/main.go
@@ -32,8 +32,13 @@ const (
 	port = ":50051"
 )
 
-// sayHello implements helloworld.GreeterServer.SayHello
-func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+// server is used to implement helloworld.GreeterServer.
+type server struct {
+	pb.UnimplementedGreeterServer
+}
+
+// SayHello implements helloworld.GreeterServer
+func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	log.Printf("Received: %v", in.GetName())
 	return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil
 }
@@ -44,7 +49,7 @@ func main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
+	pb.RegisterGreeterServer(s, &server{})
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 78d60253a6d8..2608e4acda99 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -29,10 +29,6 @@ func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient {
 	return &greeterClient{cc}
 }
 
-var greeterSayHelloStreamDesc = &grpc.StreamDesc{
-	StreamName: "SayHello",
-}
-
 func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) {
 	out := new(HelloReply)
 	err := c.cc.Invoke(ctx, "/helloworld.Greeter/SayHello", in, out, opts...)
@@ -42,52 +38,62 @@ func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...
 	return out, nil
 }
 
-// GreeterService is the service API for Greeter service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterGreeterService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type GreeterService struct {
+// GreeterServer is the server API for Greeter service.
+// All implementations must embed UnimplementedGreeterServer
+// for forward compatibility
+type GreeterServer interface {
 	// Sends a greeting
-	SayHello func(context.Context, *HelloRequest) (*HelloReply, error)
+	SayHello(context.Context, *HelloRequest) (*HelloReply, error)
+	mustEmbedUnimplementedGreeterServer()
+}
+
+// UnimplementedGreeterServer must be embedded to have forward compatible implementations.
+type UnimplementedGreeterServer struct {
 }
 
-func (s *GreeterService) sayHello(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (UnimplementedGreeterServer) SayHello(context.Context, *HelloRequest) (*HelloReply, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
+}
+func (UnimplementedGreeterServer) mustEmbedUnimplementedGreeterServer() {}
+
+// UnsafeGreeterServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to GreeterServer will
+// result in compilation errors.
+type UnsafeGreeterServer interface {
+	mustEmbedUnimplementedGreeterServer()
+}
+
+func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) {
+	s.RegisterService(&_Greeter_serviceDesc, srv)
+}
+
+func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(HelloRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.SayHello(ctx, in)
+		return srv.(GreeterServer).SayHello(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/helloworld.Greeter/SayHello",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.SayHello(ctx, req.(*HelloRequest))
+		return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterGreeterService registers a service implementation with a gRPC server.
-func RegisterGreeterService(s grpc.ServiceRegistrar, srv *GreeterService) {
-	srvCopy := *srv
-	if srvCopy.SayHello == nil {
-		srvCopy.SayHello = func(context.Context, *HelloRequest) (*HelloReply, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "helloworld.Greeter",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "SayHello",
-				Handler:    srvCopy.sayHello,
-			},
+var _Greeter_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "helloworld.Greeter",
+	HandlerType: (*GreeterServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "SayHello",
+			Handler:    _Greeter_SayHello_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "examples/helloworld/helloworld/helloworld.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "examples/helloworld/helloworld/helloworld.proto",
 }
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index bbcfe7daea35..3c11b121a54e 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -51,10 +51,6 @@ func NewRouteGuideClient(cc grpc.ClientConnInterface) RouteGuideClient {
 	return &routeGuideClient{cc}
 }
 
-var routeGuideGetFeatureStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetFeature",
-}
-
 func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) {
 	out := new(Feature)
 	err := c.cc.Invoke(ctx, "/routeguide.RouteGuide/GetFeature", in, out, opts...)
@@ -64,13 +60,8 @@ func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...gr
 	return out, nil
 }
 
-var routeGuideListFeaturesStreamDesc = &grpc.StreamDesc{
-	StreamName:    "ListFeatures",
-	ServerStreams: true,
-}
-
 func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error) {
-	stream, err := c.cc.NewStream(ctx, routeGuideListFeaturesStreamDesc, "/routeguide.RouteGuide/ListFeatures", opts...)
+	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[0], "/routeguide.RouteGuide/ListFeatures", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -101,13 +92,8 @@ func (x *routeGuideListFeaturesClient) Recv() (*Feature, error) {
 	return m, nil
 }
 
-var routeGuideRecordRouteStreamDesc = &grpc.StreamDesc{
-	StreamName:    "RecordRoute",
-	ClientStreams: true,
-}
-
 func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error) {
-	stream, err := c.cc.NewStream(ctx, routeGuideRecordRouteStreamDesc, "/routeguide.RouteGuide/RecordRoute", opts...)
+	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[1], "/routeguide.RouteGuide/RecordRoute", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -140,14 +126,8 @@ func (x *routeGuideRecordRouteClient) CloseAndRecv() (*RouteSummary, error) {
 	return m, nil
 }
 
-var routeGuideRouteChatStreamDesc = &grpc.StreamDesc{
-	StreamName:    "RouteChat",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error) {
-	stream, err := c.cc.NewStream(ctx, routeGuideRouteChatStreamDesc, "/routeguide.RouteGuide/RouteChat", opts...)
+	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[2], "/routeguide.RouteGuide/RouteChat", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -177,66 +157,90 @@ func (x *routeGuideRouteChatClient) Recv() (*RouteNote, error) {
 	return m, nil
 }
 
-// RouteGuideService is the service API for RouteGuide service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterRouteGuideService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type RouteGuideService struct {
+// RouteGuideServer is the server API for RouteGuide service.
+// All implementations must embed UnimplementedRouteGuideServer
+// for forward compatibility
+type RouteGuideServer interface {
 	// A simple RPC.
 	//
 	// Obtains the feature at a given position.
 	//
 	// A feature with an empty name is returned if there's no feature at the given
 	// position.
-	GetFeature func(context.Context, *Point) (*Feature, error)
+	GetFeature(context.Context, *Point) (*Feature, error)
 	// A server-to-client streaming RPC.
 	//
 	// Obtains the Features available within the given Rectangle.  Results are
 	// streamed rather than returned at once (e.g. in a response message with a
 	// repeated field), as the rectangle may cover a large area and contain a
 	// huge number of features.
-	ListFeatures func(*Rectangle, RouteGuide_ListFeaturesServer) error
+	ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error
 	// A client-to-server streaming RPC.
 	//
 	// Accepts a stream of Points on a route being traversed, returning a
 	// RouteSummary when traversal is completed.
-	RecordRoute func(RouteGuide_RecordRouteServer) error
+	RecordRoute(RouteGuide_RecordRouteServer) error
 	// A Bidirectional streaming RPC.
 	//
 	// Accepts a stream of RouteNotes sent while a route is being traversed,
 	// while receiving other RouteNotes (e.g. from other users).
-	RouteChat func(RouteGuide_RouteChatServer) error
+	RouteChat(RouteGuide_RouteChatServer) error
+	mustEmbedUnimplementedRouteGuideServer()
+}
+
+// UnimplementedRouteGuideServer must be embedded to have forward compatible implementations.
+type UnimplementedRouteGuideServer struct {
+}
+
+func (UnimplementedRouteGuideServer) GetFeature(context.Context, *Point) (*Feature, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
+}
+func (UnimplementedRouteGuideServer) ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error {
+	return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
+}
+func (UnimplementedRouteGuideServer) RecordRoute(RouteGuide_RecordRouteServer) error {
+	return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
+}
+func (UnimplementedRouteGuideServer) RouteChat(RouteGuide_RouteChatServer) error {
+	return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
+}
+func (UnimplementedRouteGuideServer) mustEmbedUnimplementedRouteGuideServer() {}
+
+// UnsafeRouteGuideServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to RouteGuideServer will
+// result in compilation errors.
+type UnsafeRouteGuideServer interface {
+	mustEmbedUnimplementedRouteGuideServer()
 }
 
-func (s *RouteGuideService) getFeature(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func RegisterRouteGuideServer(s *grpc.Server, srv RouteGuideServer) {
+	s.RegisterService(&_RouteGuide_serviceDesc, srv)
+}
+
+func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(Point)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetFeature(ctx, in)
+		return srv.(RouteGuideServer).GetFeature(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/routeguide.RouteGuide/GetFeature",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetFeature(ctx, req.(*Point))
+		return srv.(RouteGuideServer).GetFeature(ctx, req.(*Point))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *RouteGuideService) listFeatures(_ interface{}, stream grpc.ServerStream) error {
+
+func _RouteGuide_ListFeatures_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(Rectangle)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return s.ListFeatures(m, &routeGuideListFeaturesServer{stream})
-}
-func (s *RouteGuideService) recordRoute(_ interface{}, stream grpc.ServerStream) error {
-	return s.RecordRoute(&routeGuideRecordRouteServer{stream})
-}
-func (s *RouteGuideService) routeChat(_ interface{}, stream grpc.ServerStream) error {
-	return s.RouteChat(&routeGuideRouteChatServer{stream})
+	return srv.(RouteGuideServer).ListFeatures(m, &routeGuideListFeaturesServer{stream})
 }
 
 type RouteGuide_ListFeaturesServer interface {
@@ -252,6 +256,10 @@ func (x *routeGuideListFeaturesServer) Send(m *Feature) error {
 	return x.ServerStream.SendMsg(m)
 }
 
+func _RouteGuide_RecordRoute_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(RouteGuideServer).RecordRoute(&routeGuideRecordRouteServer{stream})
+}
+
 type RouteGuide_RecordRouteServer interface {
 	SendAndClose(*RouteSummary) error
 	Recv() (*Point, error)
@@ -274,6 +282,10 @@ func (x *routeGuideRecordRouteServer) Recv() (*Point, error) {
 	return m, nil
 }
 
+func _RouteGuide_RouteChat_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(RouteGuideServer).RouteChat(&routeGuideRouteChatServer{stream})
+}
+
 type RouteGuide_RouteChatServer interface {
 	Send(*RouteNote) error
 	Recv() (*RouteNote, error)
@@ -296,57 +308,32 @@ func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) {
 	return m, nil
 }
 
-// RegisterRouteGuideService registers a service implementation with a gRPC server.
-func RegisterRouteGuideService(s grpc.ServiceRegistrar, srv *RouteGuideService) {
-	srvCopy := *srv
-	if srvCopy.GetFeature == nil {
-		srvCopy.GetFeature = func(context.Context, *Point) (*Feature, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented")
-		}
-	}
-	if srvCopy.ListFeatures == nil {
-		srvCopy.ListFeatures = func(*Rectangle, RouteGuide_ListFeaturesServer) error {
-			return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented")
-		}
-	}
-	if srvCopy.RecordRoute == nil {
-		srvCopy.RecordRoute = func(RouteGuide_RecordRouteServer) error {
-			return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented")
-		}
-	}
-	if srvCopy.RouteChat == nil {
-		srvCopy.RouteChat = func(RouteGuide_RouteChatServer) error {
-			return status.Errorf(codes.Unimplemented, "method RouteChat not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "routeguide.RouteGuide",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "GetFeature",
-				Handler:    srvCopy.getFeature,
-			},
+var _RouteGuide_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "routeguide.RouteGuide",
+	HandlerType: (*RouteGuideServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetFeature",
+			Handler:    _RouteGuide_GetFeature_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "ListFeatures",
-				Handler:       srvCopy.listFeatures,
-				ServerStreams: true,
-			},
-			{
-				StreamName:    "RecordRoute",
-				Handler:       srvCopy.recordRoute,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "RouteChat",
-				Handler:       srvCopy.routeChat,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ListFeatures",
+			Handler:       _RouteGuide_ListFeatures_Handler,
+			ServerStreams: true,
 		},
-		Metadata: "examples/route_guide/routeguide/route_guide.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+		{
+			StreamName:    "RecordRoute",
+			Handler:       _RouteGuide_RecordRoute_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "RouteChat",
+			Handler:       _RouteGuide_RouteChat_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "examples/route_guide/routeguide/route_guide.proto",
 }
diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go
index 3c0775e76f50..dd804406afcd 100644
--- a/examples/route_guide/server/server.go
+++ b/examples/route_guide/server/server.go
@@ -54,21 +54,13 @@ var (
 )
 
 type routeGuideServer struct {
+	pb.UnimplementedRouteGuideServer
 	savedFeatures []*pb.Feature // read-only after initialized
 
 	mu         sync.Mutex // protects routeNotes
 	routeNotes map[string][]*pb.RouteNote
 }
 
-func (s *routeGuideServer) Svc() *pb.RouteGuideService {
-	return &pb.RouteGuideService{
-		GetFeature:   s.GetFeature,
-		ListFeatures: s.ListFeatures,
-		RecordRoute:  s.RecordRoute,
-		RouteChat:    s.RouteChat,
-	}
-}
-
 // GetFeature returns the feature at the given point.
 func (s *routeGuideServer) GetFeature(ctx context.Context, point *pb.Point) (*pb.Feature, error) {
 	for _, feature := range s.savedFeatures {
@@ -246,7 +238,7 @@ func main() {
 		opts = []grpc.ServerOption{grpc.Creds(creds)}
 	}
 	grpcServer := grpc.NewServer(opts...)
-	pb.RegisterRouteGuideService(grpcServer, newServer().Svc())
+	pb.RegisterRouteGuideServer(grpcServer, newServer())
 	grpcServer.Serve(lis)
 }
 
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index e7fda8735b22..463c8734ade9 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -46,10 +46,6 @@ func NewHealthClient(cc grpc.ClientConnInterface) HealthClient {
 	return &healthClient{cc}
 }
 
-var healthCheckStreamDesc = &grpc.StreamDesc{
-	StreamName: "Check",
-}
-
 func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
 	out := new(HealthCheckResponse)
 	err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...)
@@ -59,13 +55,8 @@ func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts .
 	return out, nil
 }
 
-var healthWatchStreamDesc = &grpc.StreamDesc{
-	StreamName:    "Watch",
-	ServerStreams: true,
-}
-
 func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
-	stream, err := c.cc.NewStream(ctx, healthWatchStreamDesc, "/grpc.health.v1.Health/Watch", opts...)
+	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -96,14 +87,13 @@ func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
 	return m, nil
 }
 
-// HealthService is the service API for Health service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterHealthService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type HealthService struct {
+// HealthServer is the server API for Health service.
+// All implementations should embed UnimplementedHealthServer
+// for forward compatibility
+type HealthServer interface {
 	// If the requested service is unknown, the call will fail with status
 	// NOT_FOUND.
-	Check func(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
+	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
 	// Performs a watch for the serving status of the requested service.
 	// The server will immediately send back a message indicating the current
 	// serving status.  It will then subsequently send a new message whenever
@@ -119,32 +109,55 @@ type HealthService struct {
 	// should assume this method is not supported and should not retry the
 	// call.  If the call terminates with any other status (including OK),
 	// clients should retry the call with appropriate exponential backoff.
-	Watch func(*HealthCheckRequest, Health_WatchServer) error
+	Watch(*HealthCheckRequest, Health_WatchServer) error
+}
+
+// UnimplementedHealthServer should be embedded to have forward compatible implementations.
+type UnimplementedHealthServer struct {
 }
 
-func (s *HealthService) check(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func (UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+}
+func (UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
+	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+}
+
+// UnsafeHealthServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to HealthServer will
+// result in compilation errors.
+type UnsafeHealthServer interface {
+	mustEmbedUnimplementedHealthServer()
+}
+
+func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
+	s.RegisterService(&_Health_serviceDesc, srv)
+}
+
+func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(HealthCheckRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.Check(ctx, in)
+		return srv.(HealthServer).Check(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.health.v1.Health/Check",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.Check(ctx, req.(*HealthCheckRequest))
+		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *HealthService) watch(_ interface{}, stream grpc.ServerStream) error {
+
+func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(HealthCheckRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return s.Watch(m, &healthWatchServer{stream})
+	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
 }
 
 type Health_WatchServer interface {
@@ -160,83 +173,21 @@ func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-// RegisterHealthService registers a service implementation with a gRPC server.
-func RegisterHealthService(s grpc.ServiceRegistrar, srv *HealthService) {
-	srvCopy := *srv
-	if srvCopy.Check == nil {
-		srvCopy.Check = func(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
-		}
-	}
-	if srvCopy.Watch == nil {
-		srvCopy.Watch = func(*HealthCheckRequest, Health_WatchServer) error {
-			return status.Errorf(codes.Unimplemented, "method Watch not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.health.v1.Health",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "Check",
-				Handler:    srvCopy.check,
-			},
+var _Health_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.health.v1.Health",
+	HandlerType: (*HealthServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Check",
+			Handler:    _Health_Check_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "Watch",
-				Handler:       srvCopy.watch,
-				ServerStreams: true,
-			},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Watch",
+			Handler:       _Health_Watch_Handler,
+			ServerStreams: true,
 		},
-		Metadata: "grpc/health/v1/health.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// HealthServer is the service API for Health service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type HealthServer interface {
-	// If the requested service is unknown, the call will fail with status
-	// NOT_FOUND.
-	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
-	// Performs a watch for the serving status of the requested service.
-	// The server will immediately send back a message indicating the current
-	// serving status.  It will then subsequently send a new message whenever
-	// the service's serving status changes.
-	//
-	// If the requested service is unknown when the call is received, the
-	// server will send a message setting the serving status to
-	// SERVICE_UNKNOWN but will *not* terminate the call.  If at some
-	// future point, the serving status of the service becomes known, the
-	// server will send a new message with the service's serving status.
-	//
-	// If the call terminates with status UNIMPLEMENTED, then clients
-	// should assume this method is not supported and should not retry the
-	// call.  If the call terminates with any other status (including OK),
-	// clients should retry the call with appropriate exponential backoff.
-	Watch(*HealthCheckRequest, Health_WatchServer) error
-}
-
-// UnimplementedHealthServer can be embedded to have forward compatible implementations of
-// HealthServer
-type UnimplementedHealthServer struct {
-}
-
-func (UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
-}
-func (UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error {
-	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
-}
-
-// RegisterHealthServer registers a service implementation with a gRPC server.
-func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) {
-	str := &HealthService{
-		Check: srv.Check,
-		Watch: srv.Watch,
-	}
-	RegisterHealthService(s, str)
+	},
+	Metadata: "grpc/health/v1/health.proto",
 }
diff --git a/interop/alts/server/server.go b/interop/alts/server/server.go
index 0ac7678a51b5..0d0f375a0d1a 100644
--- a/interop/alts/server/server.go
+++ b/interop/alts/server/server.go
@@ -64,7 +64,7 @@ func main() {
 	}
 	altsTC := alts.NewServerCreds(opts)
 	grpcServer := grpc.NewServer(grpc.Creds(altsTC), grpc.InTapHandle(authz))
-	testpb.RegisterTestServiceService(grpcServer, interop.NewTestServer())
+	testpb.RegisterTestServiceServer(grpcServer, interop.NewTestServer())
 	grpcServer.Serve(lis)
 }
 
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index e158bb4b20dd..f1b3bfb6c8b3 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -47,10 +47,6 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
 	return &testServiceClient{cc}
 }
 
-var testServiceEmptyCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "EmptyCall",
-}
-
 func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
 	out := new(Empty)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
@@ -60,10 +56,6 @@ func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...gr
 	return out, nil
 }
 
-var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "UnaryCall",
-}
-
 func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
@@ -73,13 +65,8 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
-var testServiceStreamingOutputCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "StreamingOutputCall",
-	ServerStreams: true,
-}
-
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceStreamingOutputCallStreamDesc, "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -110,13 +97,8 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo
 	return m, nil
 }
 
-var testServiceStreamingInputCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "StreamingInputCall",
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceStreamingInputCallStreamDesc, "/grpc.testing.TestService/StreamingInputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -149,14 +131,8 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal
 	return m, nil
 }
 
-var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "FullDuplexCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -186,14 +162,8 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
-var testServiceHalfDuplexCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "HalfDuplexCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceHalfDuplexCallStreamDesc, "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -223,82 +193,110 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
-// TestServiceService is the service API for TestService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterTestServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type TestServiceService struct {
+// TestServiceServer is the server API for TestService service.
+// All implementations must embed UnimplementedTestServiceServer
+// for forward compatibility
+type TestServiceServer interface {
 	// One empty request followed by one empty response.
-	EmptyCall func(context.Context, *Empty) (*Empty, error)
+	EmptyCall(context.Context, *Empty) (*Empty, error)
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by a sequence of responses (streamed download).
 	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
 	// A sequence of requests followed by one response (streamed upload).
 	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall func(TestService_StreamingInputCallServer) error
+	StreamingInputCall(TestService_StreamingInputCallServer) error
 	// A sequence of requests with each request served by the server immediately.
 	// As one request could lead to multiple responses, this interface
 	// demonstrates the idea of full duplexing.
-	FullDuplexCall func(TestService_FullDuplexCallServer) error
+	FullDuplexCall(TestService_FullDuplexCallServer) error
 	// A sequence of requests followed by a sequence of responses.
 	// The server buffers all the client requests and then serves them in order. A
 	// stream of responses are returned to the client when the server starts with
 	// first request.
-	HalfDuplexCall func(TestService_HalfDuplexCallServer) error
+	HalfDuplexCall(TestService_HalfDuplexCallServer) error
+	mustEmbedUnimplementedTestServiceServer()
+}
+
+// UnimplementedTestServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedTestServiceServer struct {
+}
+
+func (UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+}
+func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
 }
+func (UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+}
+func (UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+}
+func (UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+}
+func (UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+}
+func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {}
 
-func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to TestServiceServer will
+// result in compilation errors.
+type UnsafeTestServiceServer interface {
+	mustEmbedUnimplementedTestServiceServer()
+}
+
+func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+	s.RegisterService(&_TestService_serviceDesc, srv)
+}
+
+func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.EmptyCall(ctx, in)
+		return srv.(TestServiceServer).EmptyCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.TestService/EmptyCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.EmptyCall(ctx, req.(*Empty))
+		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.UnaryCall(ctx, in)
+		return srv.(TestServiceServer).UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.TestService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.UnaryCall(ctx, req.(*SimpleRequest))
+		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error {
+
+func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
-}
-func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream})
-}
-func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
+	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
 }
 
 type TestService_StreamingOutputCallServer interface {
@@ -314,6 +312,10 @@ func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallRespon
 	return x.ServerStream.SendMsg(m)
 }
 
+func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
+}
+
 type TestService_StreamingInputCallServer interface {
 	SendAndClose(*StreamingInputCallResponse) error
 	Recv() (*StreamingInputCallRequest, error)
@@ -336,6 +338,10 @@ func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest
 	return m, nil
 }
 
+func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+
 type TestService_FullDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -358,6 +364,10 @@ func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
+func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
+}
+
 type TestService_HalfDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -380,79 +390,44 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-// RegisterTestServiceService registers a service implementation with a gRPC server.
-func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
-	srvCopy := *srv
-	if srvCopy.EmptyCall == nil {
-		srvCopy.EmptyCall = func(context.Context, *Empty) (*Empty, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-		}
-	}
-	if srvCopy.UnaryCall == nil {
-		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-		}
-	}
-	if srvCopy.StreamingOutputCall == nil {
-		srvCopy.StreamingOutputCall = func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-		}
-	}
-	if srvCopy.StreamingInputCall == nil {
-		srvCopy.StreamingInputCall = func(TestService_StreamingInputCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-		}
-	}
-	if srvCopy.FullDuplexCall == nil {
-		srvCopy.FullDuplexCall = func(TestService_FullDuplexCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-		}
-	}
-	if srvCopy.HalfDuplexCall == nil {
-		srvCopy.HalfDuplexCall = func(TestService_HalfDuplexCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.TestService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "EmptyCall",
-				Handler:    srvCopy.emptyCall,
-			},
-			{
-				MethodName: "UnaryCall",
-				Handler:    srvCopy.unaryCall,
-			},
+var _TestService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.TestService",
+	HandlerType: (*TestServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "EmptyCall",
+			Handler:    _TestService_EmptyCall_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "StreamingOutputCall",
-				Handler:       srvCopy.streamingOutputCall,
-				ServerStreams: true,
-			},
-			{
-				StreamName:    "StreamingInputCall",
-				Handler:       srvCopy.streamingInputCall,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "FullDuplexCall",
-				Handler:       srvCopy.fullDuplexCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "HalfDuplexCall",
-				Handler:       srvCopy.halfDuplexCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+		{
+			MethodName: "UnaryCall",
+			Handler:    _TestService_UnaryCall_Handler,
 		},
-		Metadata: "interop/grpc_testing/test.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingOutputCall",
+			Handler:       _TestService_StreamingOutputCall_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "StreamingInputCall",
+			Handler:       _TestService_StreamingInputCall_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "FullDuplexCall",
+			Handler:       _TestService_FullDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "HalfDuplexCall",
+			Handler:       _TestService_HalfDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "interop/grpc_testing/test.proto",
 }
 
 // UnimplementedServiceClient is the client API for UnimplementedService service.
@@ -471,10 +446,6 @@ func NewUnimplementedServiceClient(cc grpc.ClientConnInterface) UnimplementedSer
 	return &unimplementedServiceClient{cc}
 }
 
-var unimplementedServiceUnimplementedCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "UnimplementedCall",
-}
-
 func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
 	out := new(Empty)
 	err := c.cc.Invoke(ctx, "/grpc.testing.UnimplementedService/UnimplementedCall", in, out, opts...)
@@ -484,54 +455,64 @@ func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *
 	return out, nil
 }
 
-// UnimplementedServiceService is the service API for UnimplementedService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterUnimplementedServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type UnimplementedServiceService struct {
+// UnimplementedServiceServer is the server API for UnimplementedService service.
+// All implementations must embed UnimplementedUnimplementedServiceServer
+// for forward compatibility
+type UnimplementedServiceServer interface {
 	// A call that no server should implement
-	UnimplementedCall func(context.Context, *Empty) (*Empty, error)
+	UnimplementedCall(context.Context, *Empty) (*Empty, error)
+	mustEmbedUnimplementedUnimplementedServiceServer()
+}
+
+// UnimplementedUnimplementedServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedUnimplementedServiceServer struct {
+}
+
+func (UnimplementedUnimplementedServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
+}
+func (UnimplementedUnimplementedServiceServer) mustEmbedUnimplementedUnimplementedServiceServer() {}
+
+// UnsafeUnimplementedServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to UnimplementedServiceServer will
+// result in compilation errors.
+type UnsafeUnimplementedServiceServer interface {
+	mustEmbedUnimplementedUnimplementedServiceServer()
 }
 
-func (s *UnimplementedServiceService) unimplementedCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func RegisterUnimplementedServiceServer(s *grpc.Server, srv UnimplementedServiceServer) {
+	s.RegisterService(&_UnimplementedService_serviceDesc, srv)
+}
+
+func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.UnimplementedCall(ctx, in)
+		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.UnimplementedService/UnimplementedCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.UnimplementedCall(ctx, req.(*Empty))
+		return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, req.(*Empty))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterUnimplementedServiceService registers a service implementation with a gRPC server.
-func RegisterUnimplementedServiceService(s grpc.ServiceRegistrar, srv *UnimplementedServiceService) {
-	srvCopy := *srv
-	if srvCopy.UnimplementedCall == nil {
-		srvCopy.UnimplementedCall = func(context.Context, *Empty) (*Empty, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.UnimplementedService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "UnimplementedCall",
-				Handler:    srvCopy.unimplementedCall,
-			},
+var _UnimplementedService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.UnimplementedService",
+	HandlerType: (*UnimplementedServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnimplementedCall",
+			Handler:    _UnimplementedService_UnimplementedCall_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "interop/grpc_testing/test.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "interop/grpc_testing/test.proto",
 }
 
 // LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
@@ -550,10 +531,6 @@ func NewLoadBalancerStatsServiceClient(cc grpc.ClientConnInterface) LoadBalancer
 	return &loadBalancerStatsServiceClient{cc}
 }
 
-var loadBalancerStatsServiceGetClientStatsStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetClientStats",
-}
-
 func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) {
 	out := new(LoadBalancerStatsResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.LoadBalancerStatsService/GetClientStats", in, out, opts...)
@@ -563,52 +540,63 @@ func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in
 	return out, nil
 }
 
-// LoadBalancerStatsServiceService is the service API for LoadBalancerStatsService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterLoadBalancerStatsServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type LoadBalancerStatsServiceService struct {
+// LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service.
+// All implementations must embed UnimplementedLoadBalancerStatsServiceServer
+// for forward compatibility
+type LoadBalancerStatsServiceServer interface {
 	// Gets the backend distribution for RPCs sent by a test client.
-	GetClientStats func(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
+	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
+	mustEmbedUnimplementedLoadBalancerStatsServiceServer()
+}
+
+// UnimplementedLoadBalancerStatsServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedLoadBalancerStatsServiceServer struct {
+}
+
+func (UnimplementedLoadBalancerStatsServiceServer) GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
+}
+func (UnimplementedLoadBalancerStatsServiceServer) mustEmbedUnimplementedLoadBalancerStatsServiceServer() {
+}
+
+// UnsafeLoadBalancerStatsServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to LoadBalancerStatsServiceServer will
+// result in compilation errors.
+type UnsafeLoadBalancerStatsServiceServer interface {
+	mustEmbedUnimplementedLoadBalancerStatsServiceServer()
 }
 
-func (s *LoadBalancerStatsServiceService) getClientStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func RegisterLoadBalancerStatsServiceServer(s *grpc.Server, srv LoadBalancerStatsServiceServer) {
+	s.RegisterService(&_LoadBalancerStatsService_serviceDesc, srv)
+}
+
+func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(LoadBalancerStatsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetClientStats(ctx, in)
+		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.LoadBalancerStatsService/GetClientStats",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetClientStats(ctx, req.(*LoadBalancerStatsRequest))
+		return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, req.(*LoadBalancerStatsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterLoadBalancerStatsServiceService registers a service implementation with a gRPC server.
-func RegisterLoadBalancerStatsServiceService(s grpc.ServiceRegistrar, srv *LoadBalancerStatsServiceService) {
-	srvCopy := *srv
-	if srvCopy.GetClientStats == nil {
-		srvCopy.GetClientStats = func(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.LoadBalancerStatsService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "GetClientStats",
-				Handler:    srvCopy.getClientStats,
-			},
+var _LoadBalancerStatsService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.LoadBalancerStatsService",
+	HandlerType: (*LoadBalancerStatsServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetClientStats",
+			Handler:    _LoadBalancerStatsService_GetClientStats_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "interop/grpc_testing/test.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "interop/grpc_testing/test.proto",
 }
diff --git a/interop/server/server.go b/interop/server/server.go
index 662fb2963090..c70e450bb108 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -76,6 +76,6 @@ func main() {
 		opts = append(opts, grpc.Creds(altsTC))
 	}
 	server := grpc.NewServer(opts...)
-	testpb.RegisterTestServiceService(server, interop.NewTestServer())
+	testpb.RegisterTestServiceServer(server, interop.NewTestServer())
 	server.Serve(lis)
 }
diff --git a/interop/test_utils.go b/interop/test_utils.go
index dd2a73155e6a..7e3aaa95f0ca 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -673,19 +673,13 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) {
 	}
 }
 
-type testServer struct{}
+type testServer struct {
+	testpb.UnimplementedTestServiceServer
+}
 
 // NewTestServer creates a test server for test service.
-func NewTestServer() *testpb.TestServiceService {
-	s := testServer{}
-	return &testpb.TestServiceService{
-		EmptyCall:           s.EmptyCall,
-		UnaryCall:           s.UnaryCall,
-		StreamingOutputCall: s.StreamingOutputCall,
-		StreamingInputCall:  s.StreamingInputCall,
-		FullDuplexCall:      s.FullDuplexCall,
-		HalfDuplexCall:      s.HalfDuplexCall,
-	}
+func NewTestServer() testpb.TestServiceServer {
+	return &testServer{}
 }
 
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 466219063881..b119bcd2b865 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -95,6 +95,10 @@ var (
 	logger = grpclog.Component("interop")
 )
 
+type statsService struct {
+	testpb.UnimplementedLoadBalancerStatsServiceServer
+}
+
 func hasRPCSucceeded() bool {
 	return atomic.LoadUint32(&rpcSucceeded) > 0
 }
@@ -107,7 +111,7 @@ func setRPCSucceeded() {
 // and return the distribution of remote peers. This is essentially a clientside
 // LB reporting mechanism that is designed to be queried by an external test
 // driver when verifying that the client is distributing RPCs as expected.
-func getClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) {
+func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) {
 	mu.Lock()
 	watcherKey := statsWatcherKey{currentRequestID, currentRequestID + in.GetNumRpcs()}
 	watcher, ok := watchers[watcherKey]
@@ -222,7 +226,7 @@ func main() {
 	}
 	s := grpc.NewServer()
 	defer s.Stop()
-	testpb.RegisterLoadBalancerStatsServiceService(s, &testpb.LoadBalancerStatsServiceService{GetClientStats: getClientStats})
+	testpb.RegisterLoadBalancerStatsServiceServer(s, &statsService{})
 	go s.Serve(lis)
 
 	clients := make([]testpb.TestServiceClient, *numChannels)
diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go
index 8d2a62487a88..45b8448822b2 100644
--- a/interop/xds/server/server.go
+++ b/interop/xds/server/server.go
@@ -49,12 +49,16 @@ func getHostname() string {
 	return hostname
 }
 
-func emptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+type server struct {
+	testpb.UnimplementedTestServiceServer
+}
+
+func (s *server) EmptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 	grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname))
 	return &testpb.Empty{}, nil
 }
 
-func unaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+func (s *server) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 	grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname))
 	return &testpb.SimpleResponse{ServerId: *serverID, Hostname: hostname}, nil
 }
@@ -67,6 +71,6 @@ func main() {
 		logger.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, &testpb.TestServiceService{EmptyCall: emptyCall, UnaryCall: unaryCall})
+	testpb.RegisterTestServiceServer(s, &server{})
 	s.Serve(lis)
 }
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index a1993dc3b816..a23453e009ac 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -32,10 +32,6 @@ func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient {
 	return &profilingClient{cc}
 }
 
-var profilingEnableStreamDesc = &grpc.StreamDesc{
-	StreamName: "Enable",
-}
-
 func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) {
 	out := new(EnableResponse)
 	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...)
@@ -45,10 +41,6 @@ func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ..
 	return out, nil
 }
 
-var profilingGetStreamStatsStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetStreamStats",
-}
-
 func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) {
 	out := new(GetStreamStatsResponse)
 	err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...)
@@ -58,114 +50,88 @@ func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStats
 	return out, nil
 }
 
-// ProfilingService is the service API for Profiling service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterProfilingService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type ProfilingService struct {
+// ProfilingServer is the server API for Profiling service.
+// All implementations should embed UnimplementedProfilingServer
+// for forward compatibility
+type ProfilingServer interface {
 	// Enable allows users to toggle profiling on and off remotely.
-	Enable func(context.Context, *EnableRequest) (*EnableResponse, error)
+	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
 	// GetStreamStats is used to retrieve an array of stream-level stats from a
 	// gRPC client/server.
-	GetStreamStats func(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
+	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
+}
+
+// UnimplementedProfilingServer should be embedded to have forward compatible implementations.
+type UnimplementedProfilingServer struct {
+}
+
+func (UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
+}
+func (UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
+}
+
+// UnsafeProfilingServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to ProfilingServer will
+// result in compilation errors.
+type UnsafeProfilingServer interface {
+	mustEmbedUnimplementedProfilingServer()
+}
+
+func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
+	s.RegisterService(&_Profiling_serviceDesc, srv)
 }
 
-func (s *ProfilingService) enable(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(EnableRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.Enable(ctx, in)
+		return srv.(ProfilingServer).Enable(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.Enable(ctx, req.(*EnableRequest))
+		return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *ProfilingService) getStreamStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetStreamStatsRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.GetStreamStats(ctx, in)
+		return srv.(ProfilingServer).GetStreamStats(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetStreamStats(ctx, req.(*GetStreamStatsRequest))
+		return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-// RegisterProfilingService registers a service implementation with a gRPC server.
-func RegisterProfilingService(s grpc.ServiceRegistrar, srv *ProfilingService) {
-	srvCopy := *srv
-	if srvCopy.Enable == nil {
-		srvCopy.Enable = func(context.Context, *EnableRequest) (*EnableResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
-		}
-	}
-	if srvCopy.GetStreamStats == nil {
-		srvCopy.GetStreamStats = func(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.go.profiling.v1alpha.Profiling",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "Enable",
-				Handler:    srvCopy.enable,
-			},
-			{
-				MethodName: "GetStreamStats",
-				Handler:    srvCopy.getStreamStats,
-			},
+var _Profiling_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
+	HandlerType: (*ProfilingServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Enable",
+			Handler:    _Profiling_Enable_Handler,
 		},
-		Streams:  []grpc.StreamDesc{},
-		Metadata: "profiling/proto/service.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// ProfilingServer is the service API for Profiling service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type ProfilingServer interface {
-	// Enable allows users to toggle profiling on and off remotely.
-	Enable(context.Context, *EnableRequest) (*EnableResponse, error)
-	// GetStreamStats is used to retrieve an array of stream-level stats from a
-	// gRPC client/server.
-	GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error)
-}
-
-// UnimplementedProfilingServer can be embedded to have forward compatible implementations of
-// ProfilingServer
-type UnimplementedProfilingServer struct {
-}
-
-func (UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented")
-}
-func (UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented")
-}
-
-// RegisterProfilingServer registers a service implementation with a gRPC server.
-func RegisterProfilingServer(s grpc.ServiceRegistrar, srv ProfilingServer) {
-	str := &ProfilingService{
-		Enable:         srv.Enable,
-		GetStreamStats: srv.GetStreamStats,
-	}
-	RegisterProfilingService(s, str)
+		{
+			MethodName: "GetStreamStats",
+			Handler:    _Profiling_GetStreamStats_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "profiling/proto/service.proto",
 }
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index 803e3a45ff64..47f2ea55298c 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -30,14 +30,8 @@ func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClie
 	return &serverReflectionClient{cc}
 }
 
-var serverReflectionServerReflectionInfoStreamDesc = &grpc.StreamDesc{
-	StreamName:    "ServerReflectionInfo",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
-	stream, err := c.cc.NewStream(ctx, serverReflectionServerReflectionInfoStreamDesc, "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
+	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +61,36 @@ func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionRe
 	return m, nil
 }
 
-// ServerReflectionService is the service API for ServerReflection service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterServerReflectionService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type ServerReflectionService struct {
+// ServerReflectionServer is the server API for ServerReflection service.
+// All implementations should embed UnimplementedServerReflectionServer
+// for forward compatibility
+type ServerReflectionServer interface {
 	// The reflection service is structured as a bidirectional stream, ensuring
 	// all related requests go to a single server.
-	ServerReflectionInfo func(ServerReflection_ServerReflectionInfoServer) error
+	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
 }
 
-func (s *ServerReflectionService) serverReflectionInfo(_ interface{}, stream grpc.ServerStream) error {
-	return s.ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
+// UnimplementedServerReflectionServer should be embedded to have forward compatible implementations.
+type UnimplementedServerReflectionServer struct {
+}
+
+func (UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
+}
+
+// UnsafeServerReflectionServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to ServerReflectionServer will
+// result in compilation errors.
+type UnsafeServerReflectionServer interface {
+	mustEmbedUnimplementedServerReflectionServer()
+}
+
+func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
+	s.RegisterService(&_ServerReflection_serviceDesc, srv)
+}
+
+func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream})
 }
 
 type ServerReflection_ServerReflectionInfoServer interface {
@@ -103,54 +115,17 @@ func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRe
 	return m, nil
 }
 
-// RegisterServerReflectionService registers a service implementation with a gRPC server.
-func RegisterServerReflectionService(s grpc.ServiceRegistrar, srv *ServerReflectionService) {
-	srvCopy := *srv
-	if srvCopy.ServerReflectionInfo == nil {
-		srvCopy.ServerReflectionInfo = func(ServerReflection_ServerReflectionInfoServer) error {
-			return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.reflection.v1alpha.ServerReflection",
-		Methods:     []grpc.MethodDesc{},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "ServerReflectionInfo",
-				Handler:       srvCopy.serverReflectionInfo,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+var _ServerReflection_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
+	HandlerType: (*ServerReflectionServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ServerReflectionInfo",
+			Handler:       _ServerReflection_ServerReflectionInfo_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
-		Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
-	}
-
-	s.RegisterService(&sd, nil)
-}
-
-// ServerReflectionServer is the service API for ServerReflection service.
-// New methods may be added to this interface if they are added to the service
-// definition, which is not a backward-compatible change.  For this reason,
-// use of this type is not recommended unless you own the service definition.
-type ServerReflectionServer interface {
-	// The reflection service is structured as a bidirectional stream, ensuring
-	// all related requests go to a single server.
-	ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error
-}
-
-// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations of
-// ServerReflectionServer
-type UnimplementedServerReflectionServer struct {
-}
-
-func (UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented")
-}
-
-// RegisterServerReflectionServer registers a service implementation with a gRPC server.
-func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) {
-	str := &ServerReflectionService{
-		ServerReflectionInfo: srv.ServerReflectionInfo,
-	}
-	RegisterServerReflectionService(s, str)
+	},
+	Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto",
 }
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 503c8569368f..095d10ae4263 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -29,10 +29,6 @@ func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient {
 	return &searchServiceClient{cc}
 }
 
-var searchServiceSearchStreamDesc = &grpc.StreamDesc{
-	StreamName: "Search",
-}
-
 func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) {
 	out := new(SearchResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.SearchService/Search", in, out, opts...)
@@ -42,14 +38,8 @@ func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opt
 	return out, nil
 }
 
-var searchServiceStreamingSearchStreamDesc = &grpc.StreamDesc{
-	StreamName:    "StreamingSearch",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error) {
-	stream, err := c.cc.NewStream(ctx, searchServiceStreamingSearchStreamDesc, "/grpc.testing.SearchService/StreamingSearch", opts...)
+	stream, err := c.cc.NewStream(ctx, &_SearchService_serviceDesc.Streams[0], "/grpc.testing.SearchService/StreamingSearch", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -79,34 +69,58 @@ func (x *searchServiceStreamingSearchClient) Recv() (*SearchResponse, error) {
 	return m, nil
 }
 
-// SearchServiceService is the service API for SearchService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterSearchServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type SearchServiceService struct {
-	Search          func(context.Context, *SearchRequest) (*SearchResponse, error)
-	StreamingSearch func(SearchService_StreamingSearchServer) error
+// SearchServiceServer is the server API for SearchService service.
+// All implementations must embed UnimplementedSearchServiceServer
+// for forward compatibility
+type SearchServiceServer interface {
+	Search(context.Context, *SearchRequest) (*SearchResponse, error)
+	StreamingSearch(SearchService_StreamingSearchServer) error
+	mustEmbedUnimplementedSearchServiceServer()
+}
+
+// UnimplementedSearchServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedSearchServiceServer struct {
+}
+
+func (UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
+}
+func (UnimplementedSearchServiceServer) StreamingSearch(SearchService_StreamingSearchServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
+}
+func (UnimplementedSearchServiceServer) mustEmbedUnimplementedSearchServiceServer() {}
+
+// UnsafeSearchServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SearchServiceServer will
+// result in compilation errors.
+type UnsafeSearchServiceServer interface {
+	mustEmbedUnimplementedSearchServiceServer()
 }
 
-func (s *SearchServiceService) search(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func RegisterSearchServiceServer(s *grpc.Server, srv SearchServiceServer) {
+	s.RegisterService(&_SearchService_serviceDesc, srv)
+}
+
+func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(SearchRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.Search(ctx, in)
+		return srv.(SearchServiceServer).Search(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.SearchService/Search",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.Search(ctx, req.(*SearchRequest))
+		return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *SearchServiceService) streamingSearch(_ interface{}, stream grpc.ServerStream) error {
-	return s.StreamingSearch(&searchServiceStreamingSearchServer{stream})
+
+func _SearchService_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SearchServiceServer).StreamingSearch(&searchServiceStreamingSearchServer{stream})
 }
 
 type SearchService_StreamingSearchServer interface {
@@ -131,37 +145,22 @@ func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) {
 	return m, nil
 }
 
-// RegisterSearchServiceService registers a service implementation with a gRPC server.
-func RegisterSearchServiceService(s grpc.ServiceRegistrar, srv *SearchServiceService) {
-	srvCopy := *srv
-	if srvCopy.Search == nil {
-		srvCopy.Search = func(context.Context, *SearchRequest) (*SearchResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
-		}
-	}
-	if srvCopy.StreamingSearch == nil {
-		srvCopy.StreamingSearch = func(SearchService_StreamingSearchServer) error {
-			return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.SearchService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "Search",
-				Handler:    srvCopy.search,
-			},
+var _SearchService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.SearchService",
+	HandlerType: (*SearchServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Search",
+			Handler:    _SearchService_Search_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "StreamingSearch",
-				Handler:       srvCopy.streamingSearch,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingSearch",
+			Handler:       _SearchService_StreamingSearch_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
-		Metadata: "reflection/grpc_testing/test.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Metadata: "reflection/grpc_testing/test.proto",
 }
diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go
index 95b8269b4bd0..55d1840fdc3a 100644
--- a/reflection/serverreflection_test.go
+++ b/reflection/serverreflection_test.go
@@ -166,13 +166,8 @@ func (x) TestAllExtensionNumbersForType(t *testing.T) {
 
 // Do end2end tests.
 
-type server struct{}
-
-func (s *server) Svc() *pb.SearchServiceService {
-	return &pb.SearchServiceService{
-		Search:          s.Search,
-		StreamingSearch: s.StreamingSearch,
-	}
+type server struct {
+	pb.UnimplementedSearchServiceServer
 }
 
 func (s *server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.SearchResponse, error) {
@@ -200,7 +195,7 @@ func (x) TestReflectionEnd2end(t *testing.T) {
 		t.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	pb.RegisterSearchServiceService(s, (&server{}).Svc())
+	pb.RegisterSearchServiceServer(s, &server{})
 	pbv3.RegisterSearchServiceV3Server(s, &serverV3{})
 	// Register reflection service on s.
 	Register(s)
diff --git a/regenerate.sh b/regenerate.sh
index c10bc99a9692..647a55afbbfc 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -26,6 +26,11 @@ export GOBIN=${WORKDIR}/bin
 export PATH=${GOBIN}:${PATH}
 mkdir -p ${GOBIN}
 
+echo "remove existing generated files"
+# grpc_testingv3/testv3.pb.go is not re-generated because it was
+# intentionally generated by an older version of protoc-gen-go.
+rm -f $(find . -name '*.pb.go' | grep -v 'grpc_testingv3/testv3.pb.go')
+
 echo "go install github.com/golang/protobuf/protoc-gen-go"
 (cd test/tools && go install github.com/golang/protobuf/protoc-gen-go)
 
@@ -35,6 +40,7 @@ echo "go install cmd/protoc-gen-go-grpc"
 echo "git clone https://github.com/grpc/grpc-proto"
 git clone --quiet https://github.com/grpc/grpc-proto ${WORKDIR}/grpc-proto
 
+# Pull in code.proto as a proto dependency
 mkdir -p ${WORKDIR}/googleapis/google/rpc
 echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
 curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > ${WORKDIR}/googleapis/google/rpc/code.proto
@@ -59,21 +65,12 @@ curl --silent https://raw.githubusercontent.com/istio/istio/master/security/prot
 
 mkdir -p ${WORKDIR}/out
 
-# Generates legacy gRPC Server symbols in addition to the newer Service symbols
+# Generates sources without the embed requirement
 LEGACY_SOURCES=(
-  ${WORKDIR}/googleapis/google/rpc/code.proto
   ${WORKDIR}/grpc-proto/grpc/binlog/v1/binarylog.proto
   ${WORKDIR}/grpc-proto/grpc/channelz/v1/channelz.proto
-  ${WORKDIR}/grpc-proto/grpc/gcp/altscontext.proto
-  ${WORKDIR}/grpc-proto/grpc/gcp/handshaker.proto
-  ${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto
   ${WORKDIR}/grpc-proto/grpc/health/v1/health.proto
   ${WORKDIR}/grpc-proto/grpc/lb/v1/load_balancer.proto
-  ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
-  ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto
-  ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
-  ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
-  ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
   profiling/proto/service.proto
   reflection/grpc_reflection_v1alpha/reflection.proto
 )
@@ -81,6 +78,14 @@ LEGACY_SOURCES=(
 # Generates only the new gRPC Service symbols
 SOURCES=(
   $(git ls-files --exclude-standard --cached --others "*.proto" | grep -v '^\(profiling/proto/service.proto\|reflection/grpc_reflection_v1alpha/reflection.proto\)$')
+  ${WORKDIR}/grpc-proto/grpc/gcp/altscontext.proto
+  ${WORKDIR}/grpc-proto/grpc/gcp/handshaker.proto
+  ${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto
+  ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
+  ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto
+  ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
+  ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
+  ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
 )
 
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
@@ -91,12 +96,20 @@ Menvoy/config/core/v3/config_source.proto=github.com/envoyproxy/go-control-plane
 
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
-  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out ${src}
+  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out \
+    -I"." \
+    -I${WORKDIR}/grpc-proto \
+    -I${WORKDIR}/googleapis \
+    -I${WORKDIR}/data-plane-api \
+    -I${WORKDIR}/udpa \
+    -I${WORKDIR}/protoc-gen-validate \
+    -I${WORKDIR}/istio \
+    ${src}
 done
 
 for src in ${LEGACY_SOURCES[@]}; do
   echo "protoc ${src}"
-  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},gen_unstable_server_interfaces=true:${WORKDIR}/out \
+  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},require_unimplemented_servers=false:${WORKDIR}/out \
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 3398644471e8..a95fa56a6d6f 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -130,8 +130,12 @@ func (cs *certStore) loadCerts() error {
 	return nil
 }
 
+type greeterServer struct {
+	pb.UnimplementedGreeterServer
+}
+
 // sayHello is a simple implementation of the pb.GreeterServer SayHello method.
-func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+func (greeterServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
@@ -408,7 +412,7 @@ func (s) TestEnd2End(t *testing.T) {
 				t.Fatalf("failed to listen: %v", err)
 			}
 			defer lis.Close()
-			pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello})
+			pb.RegisterGreeterServer(s, greeterServer{})
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{
 				IdentityOptions: IdentityCertificateOptions{
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index 32f93dde8db4..89fe0aba5d60 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -38,10 +38,6 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
 	return &testServiceClient{cc}
 }
 
-var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "UnaryCall",
-}
-
 func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
@@ -51,14 +47,8 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
-var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "FullDuplexCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -88,13 +78,8 @@ func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) {
 	return m, nil
 }
 
-var testServiceClientStreamCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "ClientStreamCall",
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceClientStreamCallStreamDesc, "/grpc.testing.TestService/ClientStreamCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -127,13 +112,8 @@ func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, err
 	return m, nil
 }
 
-var testServiceServerStreamCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "ServerStreamCall",
-	ServerStreams: true,
-}
-
 func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceServerStreamCallStreamDesc, "/grpc.testing.TestService/ServerStreamCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -164,53 +144,73 @@ func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) {
 	return m, nil
 }
 
-// TestServiceService is the service API for TestService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterTestServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type TestServiceService struct {
+// TestServiceServer is the server API for TestService service.
+// All implementations must embed UnimplementedTestServiceServer
+// for forward compatibility
+type TestServiceServer interface {
 	// One request followed by one response.
 	// The server returns the client id as-is.
-	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// A sequence of requests with each request served by the server immediately.
 	// As one request could lead to multiple responses, this interface
 	// demonstrates the idea of full duplexing.
-	FullDuplexCall func(TestService_FullDuplexCallServer) error
+	FullDuplexCall(TestService_FullDuplexCallServer) error
 	// Client stream
-	ClientStreamCall func(TestService_ClientStreamCallServer) error
+	ClientStreamCall(TestService_ClientStreamCallServer) error
 	// Server stream
-	ServerStreamCall func(*SimpleRequest, TestService_ServerStreamCallServer) error
+	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
+	mustEmbedUnimplementedTestServiceServer()
 }
 
-func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+// UnimplementedTestServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedTestServiceServer struct {
+}
+
+func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+}
+func (UnimplementedTestServiceServer) ClientStreamCall(TestService_ClientStreamCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
+}
+func (UnimplementedTestServiceServer) ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
+}
+func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {}
+
+// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to TestServiceServer will
+// result in compilation errors.
+type UnsafeTestServiceServer interface {
+	mustEmbedUnimplementedTestServiceServer()
+}
+
+func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+	s.RegisterService(&_TestService_serviceDesc, srv)
+}
+
+func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.UnaryCall(ctx, in)
+		return srv.(TestServiceServer).UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.TestService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.UnaryCall(ctx, req.(*SimpleRequest))
+		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-func (s *TestServiceService) clientStreamCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.ClientStreamCall(&testServiceClientStreamCallServer{stream})
-}
-func (s *TestServiceService) serverStreamCall(_ interface{}, stream grpc.ServerStream) error {
-	m := new(SimpleRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return s.ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
+
+func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
 }
 
 type TestService_FullDuplexCallServer interface {
@@ -235,6 +235,10 @@ func (x *testServiceFullDuplexCallServer) Recv() (*SimpleRequest, error) {
 	return m, nil
 }
 
+func _TestService_ClientStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).ClientStreamCall(&testServiceClientStreamCallServer{stream})
+}
+
 type TestService_ClientStreamCallServer interface {
 	SendAndClose(*SimpleResponse) error
 	Recv() (*SimpleRequest, error)
@@ -257,6 +261,14 @@ func (x *testServiceClientStreamCallServer) Recv() (*SimpleRequest, error) {
 	return m, nil
 }
 
+func _TestService_ServerStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(SimpleRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(TestServiceServer).ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
+}
+
 type TestService_ServerStreamCallServer interface {
 	Send(*SimpleResponse) error
 	grpc.ServerStream
@@ -270,57 +282,32 @@ func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-// RegisterTestServiceService registers a service implementation with a gRPC server.
-func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
-	srvCopy := *srv
-	if srvCopy.UnaryCall == nil {
-		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-		}
-	}
-	if srvCopy.FullDuplexCall == nil {
-		srvCopy.FullDuplexCall = func(TestService_FullDuplexCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-		}
-	}
-	if srvCopy.ClientStreamCall == nil {
-		srvCopy.ClientStreamCall = func(TestService_ClientStreamCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
-		}
-	}
-	if srvCopy.ServerStreamCall == nil {
-		srvCopy.ServerStreamCall = func(*SimpleRequest, TestService_ServerStreamCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.TestService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "UnaryCall",
-				Handler:    srvCopy.unaryCall,
-			},
+var _TestService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.TestService",
+	HandlerType: (*TestServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryCall",
+			Handler:    _TestService_UnaryCall_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "FullDuplexCall",
-				Handler:       srvCopy.fullDuplexCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "ClientStreamCall",
-				Handler:       srvCopy.clientStreamCall,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "ServerStreamCall",
-				Handler:       srvCopy.serverStreamCall,
-				ServerStreams: true,
-			},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "FullDuplexCall",
+			Handler:       _TestService_FullDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
-		Metadata: "stats/grpc_testing/test.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+		{
+			StreamName:    "ClientStreamCall",
+			Handler:       _TestService_ClientStreamCall_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "ServerStreamCall",
+			Handler:       _TestService_ServerStreamCall_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "stats/grpc_testing/test.proto",
 }
diff --git a/stats/stats_test.go b/stats/stats_test.go
index 90b9cdbee9ca..d047d48bc5e2 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -73,15 +73,8 @@ var (
 	errorID int32 = 32202
 )
 
-type testServer struct{}
-
-func (s *testServer) Svc() *testpb.TestServiceService {
-	return &testpb.TestServiceService{
-		UnaryCall:        s.UnaryCall,
-		FullDuplexCall:   s.FullDuplexCall,
-		ClientStreamCall: s.ClientStreamCall,
-		ServerStreamCall: s.ServerStreamCall,
-	}
+type testServer struct {
+	testpb.UnimplementedTestServiceServer
 }
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -172,7 +165,7 @@ type test struct {
 	clientStatsHandler stats.Handler
 	serverStatsHandler stats.Handler
 
-	testService *testpb.TestServiceService // nil means none
+	testServer testpb.TestServiceServer // nil means none
 	// srv and srvAddr are set once startServer is called.
 	srv     *grpc.Server
 	srvAddr string
@@ -207,8 +200,8 @@ func newTest(t *testing.T, tc *testConfig, ch stats.Handler, sh stats.Handler) *
 
 // startServer starts a gRPC server listening. Callers should defer a
 // call to te.tearDown to clean up.
-func (te *test) startServer(ts *testpb.TestServiceService) {
-	te.testService = ts
+func (te *test) startServer(ts testpb.TestServiceServer) {
+	te.testServer = ts
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		te.t.Fatalf("Failed to listen: %v", err)
@@ -225,8 +218,8 @@ func (te *test) startServer(ts *testpb.TestServiceService) {
 	}
 	s := grpc.NewServer(opts...)
 	te.srv = s
-	if te.testService != nil {
-		testpb.RegisterTestServiceService(s, te.testService)
+	if te.testServer != nil {
+		testpb.RegisterTestServiceServer(s, te.testServer)
 	}
 
 	go s.Serve(lis)
@@ -822,7 +815,7 @@ func checkServerStats(t *testing.T, got []*gotData, expect *expectedData, checkF
 func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) {
 	h := &statshandler{}
 	te := newTest(t, tc, nil, h)
-	te.startServer((&testServer{}).Svc())
+	te.startServer(&testServer{})
 	defer te.tearDown()
 
 	var (
@@ -1113,7 +1106,7 @@ func checkClientStats(t *testing.T, got []*gotData, expect *expectedData, checkF
 func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map[int]*checkFuncWithCount) {
 	h := &statshandler{}
 	te := newTest(t, tc, h, nil)
-	te.startServer((&testServer{}).Svc())
+	te.startServer(&testServer{})
 	defer te.tearDown()
 
 	var (
diff --git a/stress/client/main.go b/stress/client/main.go
index c91aeb0a4180..c5bfffa4e519 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -146,6 +146,7 @@ func (g *gauge) get() int64 {
 
 // server implements metrics server functions.
 type server struct {
+	metricspb.UnimplementedMetricsServiceServer
 	mutex sync.RWMutex
 	// gauges is a map from /stress_test/server_<n>/channel_<n>/stub_<n>/qps to its qps gauge.
 	gauges map[string]*gauge
@@ -156,13 +157,6 @@ func newMetricsServer() *server {
 	return &server{gauges: make(map[string]*gauge)}
 }
 
-func (s *server) Svc() *metricspb.MetricsServiceService {
-	return &metricspb.MetricsServiceService{
-		GetAllGauges: s.GetAllGauges,
-		GetGauge:     s.GetGauge,
-	}
-}
-
 // GetAllGauges returns all gauges.
 func (s *server) GetAllGauges(in *metricspb.EmptyMessage, stream metricspb.MetricsService_GetAllGaugesServer) error {
 	s.mutex.RLock()
@@ -208,8 +202,9 @@ func startServer(server *server, port int) {
 	}
 
 	s := grpc.NewServer()
-	metricspb.RegisterMetricsServiceService(s, server.Svc())
+	metricspb.RegisterMetricsServiceServer(s, server)
 	s.Serve(lis)
+
 }
 
 // performRPCs uses weightedRandomTestSelector to select test case and runs the tests.
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index b28a52e79f71..07d480b0b6a5 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -32,13 +32,8 @@ func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient {
 	return &metricsServiceClient{cc}
 }
 
-var metricsServiceGetAllGaugesStreamDesc = &grpc.StreamDesc{
-	StreamName:    "GetAllGauges",
-	ServerStreams: true,
-}
-
 func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error) {
-	stream, err := c.cc.NewStream(ctx, metricsServiceGetAllGaugesStreamDesc, "/grpc.testing.MetricsService/GetAllGauges", opts...)
+	stream, err := c.cc.NewStream(ctx, &_MetricsService_serviceDesc.Streams[0], "/grpc.testing.MetricsService/GetAllGauges", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -69,10 +64,6 @@ func (x *metricsServiceGetAllGaugesClient) Recv() (*GaugeResponse, error) {
 	return m, nil
 }
 
-var metricsServiceGetGaugeStreamDesc = &grpc.StreamDesc{
-	StreamName: "GetGauge",
-}
-
 func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) {
 	out := new(GaugeResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.MetricsService/GetGauge", in, out, opts...)
@@ -82,41 +73,47 @@ func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, o
 	return out, nil
 }
 
-// MetricsServiceService is the service API for MetricsService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterMetricsServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type MetricsServiceService struct {
+// MetricsServiceServer is the server API for MetricsService service.
+// All implementations must embed UnimplementedMetricsServiceServer
+// for forward compatibility
+type MetricsServiceServer interface {
 	// Returns the values of all the gauges that are currently being maintained by
 	// the service
-	GetAllGauges func(*EmptyMessage, MetricsService_GetAllGaugesServer) error
+	GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error
 	// Returns the value of one gauge
-	GetGauge func(context.Context, *GaugeRequest) (*GaugeResponse, error)
+	GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error)
+	mustEmbedUnimplementedMetricsServiceServer()
+}
+
+// UnimplementedMetricsServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedMetricsServiceServer struct {
+}
+
+func (UnimplementedMetricsServiceServer) GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error {
+	return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
+}
+func (UnimplementedMetricsServiceServer) GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
+}
+func (UnimplementedMetricsServiceServer) mustEmbedUnimplementedMetricsServiceServer() {}
+
+// UnsafeMetricsServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to MetricsServiceServer will
+// result in compilation errors.
+type UnsafeMetricsServiceServer interface {
+	mustEmbedUnimplementedMetricsServiceServer()
+}
+
+func RegisterMetricsServiceServer(s *grpc.Server, srv MetricsServiceServer) {
+	s.RegisterService(&_MetricsService_serviceDesc, srv)
 }
 
-func (s *MetricsServiceService) getAllGauges(_ interface{}, stream grpc.ServerStream) error {
+func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(EmptyMessage)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return s.GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
-}
-func (s *MetricsServiceService) getGauge(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GaugeRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return s.GetGauge(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     s,
-		FullMethod: "/grpc.testing.MetricsService/GetGauge",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.GetGauge(ctx, req.(*GaugeRequest))
-	}
-	return interceptor(ctx, in, info, handler)
+	return srv.(MetricsServiceServer).GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream})
 }
 
 type MetricsService_GetAllGaugesServer interface {
@@ -132,36 +129,39 @@ func (x *metricsServiceGetAllGaugesServer) Send(m *GaugeResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-// RegisterMetricsServiceService registers a service implementation with a gRPC server.
-func RegisterMetricsServiceService(s grpc.ServiceRegistrar, srv *MetricsServiceService) {
-	srvCopy := *srv
-	if srvCopy.GetAllGauges == nil {
-		srvCopy.GetAllGauges = func(*EmptyMessage, MetricsService_GetAllGaugesServer) error {
-			return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented")
-		}
+func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GaugeRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MetricsServiceServer).GetGauge(ctx, in)
 	}
-	if srvCopy.GetGauge == nil {
-		srvCopy.GetGauge = func(context.Context, *GaugeRequest) (*GaugeResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented")
-		}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.MetricsService/GetGauge",
 	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.MetricsService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "GetGauge",
-				Handler:    srvCopy.getGauge,
-			},
-		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "GetAllGauges",
-				Handler:       srvCopy.getAllGauges,
-				ServerStreams: true,
-			},
-		},
-		Metadata: "stress/grpc_testing/metrics.proto",
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MetricsServiceServer).GetGauge(ctx, req.(*GaugeRequest))
 	}
+	return interceptor(ctx, in, info, handler)
+}
 
-	s.RegisterService(&sd, nil)
+var _MetricsService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.MetricsService",
+	HandlerType: (*MetricsServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetGauge",
+			Handler:    _MetricsService_GetGauge_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "GetAllGauges",
+			Handler:       _MetricsService_GetAllGauges_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "stress/grpc_testing/metrics.proto",
 }
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 43e50210349e..f0189cf26444 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -150,7 +150,7 @@ func (s) TestCredsBundleFromBalancer(t *testing.T) {
 	te.customServerOptions = []grpc.ServerOption{
 		grpc.Creds(creds),
 	}
-	te.startServer((&testServer{}).Svc())
+	te.startServer(&testServer{})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -179,7 +179,7 @@ func testPickExtraMetadata(t *testing.T, e env) {
 		grpc.WithBalancerName(testBalancerName),
 		grpc.WithUserAgent(testUserAgent),
 	}
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	// Set resolver to xds to trigger the extra metadata code path.
@@ -228,7 +228,7 @@ func testDoneInfo(t *testing.T, e env) {
 		grpc.WithBalancerName(testBalancerName),
 	}
 	te.userAgent = failAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -498,7 +498,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	}
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, (&testServer{}).Svc())
+	testpb.RegisterTestServiceServer(s, &testServer{})
 	go s.Serve(lis)
 	defer s.Stop()
 	t.Logf("Started gRPC server at %s...", lis.Addr().String())
@@ -556,12 +556,12 @@ func (s) TestServersSwap(t *testing.T) {
 			t.Fatalf("Error while listening. Err: %v", err)
 		}
 		s := grpc.NewServer()
-		ts := &testpb.TestServiceService{
-			UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+		ts := &funcServer{
+			unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 				return &testpb.SimpleResponse{Username: username}, nil
 			},
 		}
-		testpb.RegisterTestServiceService(s, ts)
+		testpb.RegisterTestServiceServer(s, ts)
 		go s.Serve(lis)
 		return lis.Addr().String(), s.Stop
 	}
@@ -616,12 +616,12 @@ func (s) TestEmptyAddrs(t *testing.T) {
 	s := grpc.NewServer()
 	defer s.Stop()
 	const one = "1"
-	ts := &testpb.TestServiceService{
-		UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{
+		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{Username: one}, nil
 		},
 	}
-	testpb.RegisterTestServiceService(s, ts)
+	testpb.RegisterTestServiceServer(s, ts)
 	go s.Serve(lis)
 
 	// Initialize pickfirst client
@@ -705,12 +705,12 @@ func (s) TestWaitForReady(t *testing.T) {
 	s := grpc.NewServer()
 	defer s.Stop()
 	const one = "1"
-	ts := &testpb.TestServiceService{
-		UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{
+		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{Username: one}, nil
 		},
 	}
-	testpb.RegisterTestServiceService(s, ts)
+	testpb.RegisterTestServiceServer(s, ts)
 	go s.Serve(lis)
 
 	// Initialize client
diff --git a/test/channelz_linux_go110_test.go b/test/channelz_linux_go110_test.go
index 35fd2e826802..dea374bfc08b 100644
--- a/test/channelz_linux_go110_test.go
+++ b/test/channelz_linux_go110_test.go
@@ -43,7 +43,7 @@ func testCZSocketMetricsSocketOption(t *testing.T, e env) {
 	czCleanup := channelz.NewChannelzStorage()
 	defer czCleanupWrapper(czCleanup, t)
 	te := newTest(t, e)
-	te.startServer((&testServer{security: e.security}).Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
diff --git a/test/channelz_test.go b/test/channelz_test.go
index 1f2197849e6f..9f8af01e7c74 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -85,7 +85,7 @@ func (s) TestCZServerRegistrationAndDeletion(t *testing.T) {
 		defer czCleanupWrapper(czCleanup, t)
 		e := tcpClearRREnv
 		te := newTest(t, e)
-		te.startServers(testServer{security: e.security}.Svc(), c.total)
+		te.startServers(&testServer{security: e.security}, c.total)
 
 		ss, end := channelz.GetServers(c.start, c.max)
 		if int64(len(ss)) != c.length || end != c.end {
@@ -104,7 +104,7 @@ func (s) TestCZGetServer(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	ss, _ := channelz.GetServers(0, 0)
@@ -253,7 +253,7 @@ func (s) TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) {
 	num := 3 // number of backends
 	te := newTest(t, e)
 	var svrAddrs []resolver.Address
-	te.startServers(testServer{security: e.security}.Svc(), num)
+	te.startServers(&testServer{security: e.security}, num)
 	r := manual.NewBuilderWithScheme("whatever")
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
@@ -339,7 +339,7 @@ func (s) TestCZServerSocketRegistrationAndDeletion(t *testing.T) {
 		defer czCleanupWrapper(czCleanup, t)
 		e := tcpClearRREnv
 		te := newTest(t, e)
-		te.startServer(testServer{security: e.security}.Svc())
+		te.startServer(&testServer{security: e.security})
 		var ccs []*grpc.ClientConn
 		for i := 0; i < c.total; i++ {
 			cc := te.clientConn()
@@ -504,7 +504,7 @@ func (s) TestCZChannelMetrics(t *testing.T) {
 	te := newTest(t, e)
 	te.maxClientSendMsgSize = newInt(8)
 	var svrAddrs []resolver.Address
-	te.startServers(testServer{security: e.security}.Svc(), num)
+	te.startServers(&testServer{security: e.security}, num)
 	r := manual.NewBuilderWithScheme("whatever")
 	for _, a := range te.srvAddrs {
 		svrAddrs = append(svrAddrs, resolver.Address{Addr: a})
@@ -590,7 +590,7 @@ func (s) TestCZServerMetrics(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(8)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -861,7 +861,7 @@ func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) {
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(20)
 	te.maxClientReceiveMsgSize = newInt(20)
-	rcw := te.startServerWithConnControl(testServer{security: e.security}.Svc())
+	rcw := te.startServerWithConnControl(&testServer{security: e.security})
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)}
@@ -963,7 +963,7 @@ func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *t
 	// Avoid overflowing connection level flow control window, which will lead to
 	// transport being closed.
 	te.serverInitialConnWindowSize = 65536 * 2
-	ts := &testpb.TestServiceService{FullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ts := &stubServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
 		stream.Send(&testpb.StreamingOutputCallResponse{})
 		<-stream.Context().Done()
 		return status.Errorf(codes.DeadlineExceeded, "deadline exceeded or cancelled")
@@ -1048,7 +1048,7 @@ func (s) TestCZClientAndServerSocketMetricsFlowControl(t *testing.T) {
 	te.serverInitialConnWindowSize = 65536
 	te.clientInitialWindowSize = 65536
 	te.clientInitialConnWindowSize = 65536
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -1169,7 +1169,7 @@ func (s) TestCZClientSocketMetricsKeepAlive(t *testing.T) {
 			MinTime:             500 * time.Millisecond,
 			PermitWithoutStream: true,
 		}))
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	te.clientConn() // Dial the server
 	defer te.tearDown()
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -1211,7 +1211,7 @@ func (s) TestCZServerSocketMetricsStreamsAndMessagesCount(t *testing.T) {
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(20)
 	te.maxClientReceiveMsgSize = newInt(20)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	cc, _ := te.clientConnWithConnControl()
 	tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)}
@@ -1282,7 +1282,7 @@ func (s) TestCZServerSocketMetricsKeepAlive(t *testing.T) {
 		Timeout: 100 * time.Millisecond,
 	})
 	te.customServerOptions = append(te.customServerOptions, kpOption)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -1342,7 +1342,7 @@ func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpTLSRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	te.clientConn()
 	if err := verifyResultWithDelay(func() (bool, error) {
@@ -1467,7 +1467,7 @@ func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1560,7 +1560,7 @@ func (s) TestCZChannelAddressResolutionChange(t *testing.T) {
 	e := tcpClearRREnv
 	e.balancer = ""
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	r := manual.NewBuilderWithScheme("whatever")
 	addrs := []resolver.Address{{Addr: te.srvAddr}}
 	r.InitialState(resolver.State{Addresses: addrs})
@@ -1663,7 +1663,7 @@ func (s) TestCZSubChannelPickedNewAddress(t *testing.T) {
 	e := tcpClearRREnv
 	e.balancer = ""
 	te := newTest(t, e)
-	te.startServers(testServer{security: e.security}.Svc(), 3)
+	te.startServers(&testServer{security: e.security}, 3)
 	r := manual.NewBuilderWithScheme("whatever")
 	var svrAddrs []resolver.Address
 	for _, a := range te.srvAddrs {
@@ -1722,7 +1722,7 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1816,7 +1816,7 @@ func (s) TestCZChannelConnectivityState(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1939,7 +1939,7 @@ func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) {
 	te := newTest(t, e)
 	channelz.SetMaxTraceEntry(1)
 	defer channelz.ResetMaxTraceEntryToDefault()
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
@@ -1997,7 +1997,7 @@ func (s) TestCZTraceTopChannelDeletionTraceClear(t *testing.T) {
 	defer czCleanupWrapper(czCleanup, t)
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}})
 	te.resolverScheme = r.Scheme()
diff --git a/test/creds_test.go b/test/creds_test.go
index a4d1e6cda0cb..46bdd30dc85e 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -87,7 +87,7 @@ func (s) TestCredsBundleBoth(t *testing.T) {
 	te.customServerOptions = []grpc.ServerOption{
 		grpc.Creds(creds),
 	}
-	te.startServer(testServer{}.Svc())
+	te.startServer(&testServer{})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -109,7 +109,7 @@ func (s) TestCredsBundleTransportCredentials(t *testing.T) {
 	te.customServerOptions = []grpc.ServerOption{
 		grpc.Creds(creds),
 	}
-	te.startServer(testServer{}.Svc())
+	te.startServer(&testServer{})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -125,7 +125,7 @@ func (s) TestCredsBundlePerRPCCredentials(t *testing.T) {
 	te.customDialOptions = []grpc.DialOption{
 		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundlePerRPCOnly}),
 	}
-	te.startServer(testServer{}.Svc())
+	te.startServer(&testServer{})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -159,7 +159,7 @@ func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials {
 func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
 	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty"})
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: te.e.security}.Svc())
+	te.startServer(&testServer{security: te.e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{}))
@@ -183,7 +183,7 @@ func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
 	const wantMethod = "/grpc.testing.TestService/EmptyCall"
 	te := newTest(t, env{name: "context-request-info", network: "tcp"})
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: te.e.security}.Svc())
+	te.startServer(&testServer{security: te.e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{}))
@@ -218,7 +218,7 @@ func (c clientAlwaysFailCred) Clone() credentials.TransportCredentials {
 
 func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
 	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
-	te.startServer(testServer{security: te.e.security}.Svc())
+	te.startServer(&testServer{security: te.e.security})
 	defer te.tearDown()
 
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
@@ -246,7 +246,7 @@ func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
 
 func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) {
 	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
-	te.startServer(testServer{security: te.e.security}.Svc())
+	te.startServer(&testServer{security: te.e.security})
 	defer te.tearDown()
 
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
@@ -312,7 +312,7 @@ func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.tapHandle = authHandle
 	te.perRPCCreds = testPerRPCCredentials{}
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -331,7 +331,7 @@ func (s) TestPerRPCCredentialsViaCallOptions(t *testing.T) {
 func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.tapHandle = authHandle
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -371,7 +371,7 @@ func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) {
 		}
 		return ctx, nil
 	}
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 01f99cb29971..0842dccaad01 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -126,6 +126,8 @@ var (
 var raceMode bool // set by race.go in race mode
 
 type testServer struct {
+	testpb.UnimplementedTestServiceServer
+
 	security           string // indicate the authentication protocol used by this server.
 	earlyFail          bool   // whether to error out the execution of a service handler prematurely.
 	setAndSendHeader   bool   // whether to call setHeader and sendHeader.
@@ -134,20 +136,6 @@ type testServer struct {
 	unaryCallSleepTime time.Duration
 }
 
-// Svc returns a registerable TestService for this testServer instances.
-// Because `s` is passed by value for convenience, any subsequent changes to
-// `s` are not recognized.
-func (s testServer) Svc() *testpb.TestServiceService {
-	return &testpb.TestServiceService{
-		EmptyCall:           s.EmptyCall,
-		UnaryCall:           s.UnaryCall,
-		StreamingOutputCall: s.StreamingOutputCall,
-		StreamingInputCall:  s.StreamingInputCall,
-		FullDuplexCall:      s.FullDuplexCall,
-		HalfDuplexCall:      s.HalfDuplexCall,
-	}
-}
-
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 	if md, ok := metadata.FromIncomingContext(ctx); ok {
 		// For testing purpose, returns an error if user-agent is failAppUA.
@@ -578,7 +566,7 @@ func newTest(t *testing.T, e env) *test {
 	return te
 }
 
-func (te *test) listenAndServe(ts *testpb.TestServiceService, listen func(network, address string) (net.Listener, error)) net.Listener {
+func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network, address string) (net.Listener, error)) net.Listener {
 	te.t.Helper()
 	te.t.Logf("Running test in %s environment...", te.e.name)
 	sopts := []grpc.ServerOption{grpc.MaxConcurrentStreams(te.maxStream)}
@@ -638,7 +626,7 @@ func (te *test) listenAndServe(ts *testpb.TestServiceService, listen func(networ
 	sopts = append(sopts, te.customServerOptions...)
 	s := grpc.NewServer(sopts...)
 	if ts != nil {
-		testpb.RegisterTestServiceService(s, ts)
+		testpb.RegisterTestServiceServer(s, ts)
 	}
 
 	// Create a new default health server if enableHealthServer is set, or use
@@ -703,20 +691,20 @@ func (w wrapHS) Stop() {
 	w.s.Close()
 }
 
-func (te *test) startServerWithConnControl(ts *testpb.TestServiceService) *listenerWrapper {
+func (te *test) startServerWithConnControl(ts testpb.TestServiceServer) *listenerWrapper {
 	l := te.listenAndServe(ts, listenWithConnControl)
 	return l.(*listenerWrapper)
 }
 
 // startServer starts a gRPC server exposing the provided TestService
 // implementation. Callers should defer a call to te.tearDown to clean up
-func (te *test) startServer(ts *testpb.TestServiceService) {
+func (te *test) startServer(ts testpb.TestServiceServer) {
 	te.t.Helper()
 	te.listenAndServe(ts, net.Listen)
 }
 
 // startServers starts 'num' gRPC servers exposing the provided TestService.
-func (te *test) startServers(ts *testpb.TestServiceService, num int) {
+func (te *test) startServers(ts testpb.TestServiceServer, num int) {
 	for i := 0; i < num; i++ {
 		te.startServer(ts)
 		te.srvs = append(te.srvs, te.srv.(*grpc.Server))
@@ -924,7 +912,7 @@ func (s) TestContextDeadlineNotIgnored(t *testing.T) {
 	}
 
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -958,7 +946,7 @@ func testTimeoutOnDeadServer(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1000,7 +988,7 @@ func (s) TestServerGracefulStopIdempotent(t *testing.T) {
 func testServerGracefulStopIdempotent(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	for i := 0; i < 3; i++ {
@@ -1020,7 +1008,7 @@ func (s) TestServerGoAway(t *testing.T) {
 func testServerGoAway(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1072,7 +1060,7 @@ func testServerGoAwayPendingRPC(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1146,7 +1134,7 @@ func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1233,7 +1221,7 @@ func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1270,7 +1258,7 @@ func testConcurrentServerStopAndGoAway(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1345,7 +1333,7 @@ func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) {
 
 func testClientConnCloseAfterGoAwayWithActiveStream(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
@@ -1384,7 +1372,7 @@ func testFailFast(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -1743,7 +1731,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 
 	te1.resolverScheme = r.Scheme()
 	te1.nonBlockingDial = true
-	te1.startServer(testServer{security: e.security}.Svc())
+	te1.startServer(&testServer{security: e.security})
 	cc1 := te1.clientConn(grpc.WithResolvers(r))
 
 	addrs := []resolver.Address{{Addr: te1.srvAddr}}
@@ -1833,7 +1821,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	te2.maxClientReceiveMsgSize = newInt(1024)
 	te2.maxClientSendMsgSize = newInt(1024)
 
-	te2.startServer(testServer{security: e.security}.Svc())
+	te2.startServer(&testServer{security: e.security})
 	defer te2.tearDown()
 	cc2 := te2.clientConn(grpc.WithResolvers(r))
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: te2.srvAddr}}, ServiceConfig: sc})
@@ -1893,7 +1881,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	te3.maxClientReceiveMsgSize = newInt(4096)
 	te3.maxClientSendMsgSize = newInt(4096)
 
-	te3.startServer(testServer{security: e.security}.Svc())
+	te3.startServer(&testServer{security: e.security})
 	defer te3.tearDown()
 
 	cc3 := te3.clientConn(grpc.WithResolvers(r))
@@ -1977,7 +1965,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 // test makes sure read from streaming RPC doesn't fail in this case.
 func (s) TestStreamingRPCWithTimeoutInServiceConfigRecv(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
-	te.startServer(testServer{security: tcpClearRREnv.security}.Svc())
+	te.startServer(&testServer{security: tcpClearRREnv.security})
 	defer te.tearDown()
 	r := manual.NewBuilderWithScheme("whatever")
 
@@ -2059,7 +2047,7 @@ func testPreloaderClientSend(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2133,7 +2121,7 @@ func testMaxMsgSizeClientDefault(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2197,7 +2185,7 @@ func testMaxMsgSizeClientAPI(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2282,7 +2270,7 @@ func testMaxMsgSizeServerAPI(t *testing.T, e env) {
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 		"Failed to dial : context canceled; please retry.",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -2387,7 +2375,7 @@ func testTap(t *testing.T, e env) {
 		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
 		"grpc: addrConn.resetTransport failed to create client transport: connection error",
 	)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2482,7 +2470,7 @@ func (s) TestHealthCheckSuccess(t *testing.T) {
 func testHealthCheckSuccess(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
 	defer te.tearDown()
 
@@ -2504,7 +2492,7 @@ func testHealthCheckFailure(t *testing.T, e env) {
 		"grpc: the client connection is closing; please retry",
 	)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
 	defer te.tearDown()
 
@@ -2528,7 +2516,7 @@ func (s) TestHealthCheckOff(t *testing.T) {
 func testHealthCheckOff(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.NotFound)
@@ -2545,7 +2533,7 @@ func (s) TestHealthWatchMultipleClients(t *testing.T) {
 func testHealthWatchMultipleClients(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2574,7 +2562,7 @@ func (s) TestHealthWatchSameStatus(t *testing.T) {
 func testHealthWatchSameStatus(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService)
@@ -2602,7 +2590,7 @@ func testHealthWatchSetServiceStatusBeforeStartingServer(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.healthServer = hs
 	hs.SetServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService)
@@ -2623,7 +2611,7 @@ func (s) TestHealthWatchDefaultStatusChange(t *testing.T) {
 func testHealthWatchDefaultStatusChange(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService)
@@ -2644,7 +2632,7 @@ func (s) TestHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T) {
 func testHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING)
 	defer te.tearDown()
 
@@ -2664,7 +2652,7 @@ func (s) TestHealthWatchOverallServerHealthChange(t *testing.T) {
 func testHealthWatchOverallServerHealthChange(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	stream, cf := newHealthCheckStream(t, te.clientConn(), "")
@@ -2696,7 +2684,7 @@ func (s) TestUnknownHandler(t *testing.T) {
 func testUnknownHandler(t *testing.T, e env, unknownHandler grpc.StreamHandler) {
 	te := newTest(t, e)
 	te.unknownHandler = unknownHandler
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), "", codes.Unauthenticated)
 }
@@ -2712,7 +2700,7 @@ func (s) TestHealthCheckServingStatus(t *testing.T) {
 func testHealthCheckServingStatus(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.enableHealthServer = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2733,7 +2721,7 @@ func (s) TestEmptyUnaryWithUserAgent(t *testing.T) {
 func testEmptyUnaryWithUserAgent(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2764,7 +2752,7 @@ func (s) TestFailedEmptyUnary(t *testing.T) {
 func testFailedEmptyUnary(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = failAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -2783,7 +2771,7 @@ func (s) TestLargeUnary(t *testing.T) {
 
 func testLargeUnary(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -2822,7 +2810,7 @@ func testExceedMsgLimit(t *testing.T, e env) {
 	te := newTest(t, e)
 	maxMsgSize := 1024
 	te.maxServerMsgSize, te.maxClientMsgSize = newInt(maxMsgSize), newInt(maxMsgSize)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -2901,7 +2889,7 @@ func (s) TestPeerClientSide(t *testing.T) {
 func testPeerClientSide(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	peer := new(peer.Peer)
@@ -2939,7 +2927,7 @@ func (s) TestPeerNegative(t *testing.T) {
 
 func testPeerNegative(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -2959,7 +2947,7 @@ func (s) TestPeerFailedRPC(t *testing.T) {
 func testPeerFailedRPC(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.maxServerReceiveMsgSize = newInt(1 * 1024)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -3013,7 +3001,7 @@ func (s) TestMetadataUnaryRPC(t *testing.T) {
 
 func testMetadataUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3058,7 +3046,7 @@ func (s) TestMetadataOrderUnaryRPC(t *testing.T) {
 
 func testMetadataOrderUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3095,7 +3083,7 @@ func (s) TestMultipleSetTrailerUnaryRPC(t *testing.T) {
 
 func testMultipleSetTrailerUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, multipleSetTrailer: true}.Svc())
+	te.startServer(&testServer{security: e.security, multipleSetTrailer: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3132,7 +3120,7 @@ func (s) TestMultipleSetTrailerStreamingRPC(t *testing.T) {
 
 func testMultipleSetTrailerStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, multipleSetTrailer: true}.Svc())
+	te.startServer(&testServer{security: e.security, multipleSetTrailer: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3167,7 +3155,7 @@ func (s) TestSetAndSendHeaderUnaryRPC(t *testing.T) {
 // To test header metadata is sent on SendHeader().
 func testSetAndSendHeaderUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, setAndSendHeader: true}.Svc())
+	te.startServer(&testServer{security: e.security, setAndSendHeader: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3210,7 +3198,7 @@ func (s) TestMultipleSetHeaderUnaryRPC(t *testing.T) {
 // To test header metadata is sent when sending response.
 func testMultipleSetHeaderUnaryRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
+	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3254,7 +3242,7 @@ func (s) TestMultipleSetHeaderUnaryRPCError(t *testing.T) {
 // To test header metadata is sent when sending status.
 func testMultipleSetHeaderUnaryRPCError(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
+	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3297,7 +3285,7 @@ func (s) TestSetAndSendHeaderStreamingRPC(t *testing.T) {
 // To test header metadata is sent on SendHeader().
 func testSetAndSendHeaderStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, setAndSendHeader: true}.Svc())
+	te.startServer(&testServer{security: e.security, setAndSendHeader: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3337,7 +3325,7 @@ func (s) TestMultipleSetHeaderStreamingRPC(t *testing.T) {
 // To test header metadata is sent when sending response.
 func testMultipleSetHeaderStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
+	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3401,7 +3389,7 @@ func (s) TestMultipleSetHeaderStreamingRPCError(t *testing.T) {
 // To test header metadata is sent when sending status.
 func testMultipleSetHeaderStreamingRPCError(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, setHeaderOnly: true}.Svc())
+	te.startServer(&testServer{security: e.security, setHeaderOnly: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3466,7 +3454,7 @@ func (s) TestMalformedHTTP2Metadata(t *testing.T) {
 
 func testMalformedHTTP2Metadata(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3509,7 +3497,7 @@ func testTransparentRetry(t *testing.T, e env) {
 		}
 		return ctx, nil
 	}
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -3558,7 +3546,7 @@ func (s) TestCancel(t *testing.T) {
 func testCancel(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("grpc: the client connection is closing; please retry")
-	te.startServer(testServer{security: e.security, unaryCallSleepTime: time.Second}.Svc())
+	te.startServer(&testServer{security: e.security, unaryCallSleepTime: time.Second})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -3595,7 +3583,7 @@ func testCancelNoIO(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("http2Client.notifyError got notified that the client transport was broken")
 	te.maxStream = 1 // Only allows 1 live stream per server transport.
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -3604,7 +3592,7 @@ func testCancelNoIO(t *testing.T, e env) {
 	// Start one blocked RPC for which we'll never send streaming
 	// input. This will consume the 1 maximum concurrent streams,
 	// causing future RPCs to hang.
-	ctx, cancelFirst := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancelFirst := context.WithCancel(context.Background())
 	_, err := tc.StreamingInputCall(ctx)
 	if err != nil {
 		t.Fatalf("%v.StreamingInputCall(_) = _, %v, want _, <nil>", tc, err)
@@ -3617,9 +3605,6 @@ func testCancelNoIO(t *testing.T, e env) {
 	// succeeding.
 	// TODO(bradfitz): add internal test hook for this (Issue 534)
 	for {
-		if ctx.Err() != nil {
-			t.Fatal("timed out waiting to get deadline exceeded error")
-		}
 		ctx, cancelSecond := context.WithTimeout(context.Background(), 50*time.Millisecond)
 		_, err := tc.StreamingInputCall(ctx)
 		cancelSecond()
@@ -3687,7 +3672,7 @@ func (s) TestPingPong(t *testing.T) {
 
 func testPingPong(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3746,7 +3731,7 @@ func (s) TestMetadataStreamingRPC(t *testing.T) {
 
 func testMetadataStreamingRPC(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3823,7 +3808,7 @@ func (s) TestServerStreaming(t *testing.T) {
 
 func testServerStreaming(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3878,7 +3863,7 @@ func (s) TestFailedServerStreaming(t *testing.T) {
 func testFailedServerStreaming(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = failAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -3907,10 +3892,17 @@ func equalError(x, y error) bool {
 	return x == y || (x != nil && y != nil && x.Error() == y.Error())
 }
 
-// concurrentSendStreamingOutputCall makes ten serial Send calls, sending payloads
+// concurrentSendServer is a TestServiceServer whose
+// StreamingOutputCall makes ten serial Send calls, sending payloads
 // "0".."9", inclusive.  TestServerStreamingConcurrent verifies they
 // were received in the correct order, and that there were no races.
-func concurrentSendStreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
+//
+// All other TestServiceServer methods crash if called.
+type concurrentSendServer struct {
+	testpb.TestServiceServer
+}
+
+func (s concurrentSendServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
 	for i := 0; i < 10; i++ {
 		stream.Send(&testpb.StreamingOutputCallResponse{
 			Payload: &testpb.Payload{
@@ -3930,7 +3922,7 @@ func (s) TestServerStreamingConcurrent(t *testing.T) {
 
 func testServerStreamingConcurrent(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(&testpb.TestServiceService{StreamingOutputCall: concurrentSendStreamingOutputCall})
+	te.startServer(concurrentSendServer{})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -4011,7 +4003,7 @@ func (s) TestClientStreaming(t *testing.T) {
 
 func testClientStreaming(t *testing.T, e env, sizes []int) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4057,7 +4049,7 @@ func (s) TestClientStreamingError(t *testing.T) {
 
 func testClientStreamingError(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, earlyFail: true}.Svc())
+	te.startServer(&testServer{security: e.security, earlyFail: true})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4102,7 +4094,7 @@ func testExceedMaxStreamsLimit(t *testing.T, e env) {
 		"grpc: the connection is closing",
 	)
 	te.maxStream = 1 // Only allows 1 live stream per server transport.
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -4142,7 +4134,7 @@ func testStreamsQuotaRecovery(t *testing.T, e env) {
 		"grpc: the connection is closing",
 	)
 	te.maxStream = 1 // Allows 1 live stream.
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -4212,7 +4204,7 @@ func testCompressServerHasNoSupport(t *testing.T, e env) {
 	te.serverCompression = false
 	te.clientCompression = false
 	te.clientNopCompression = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4250,7 +4242,7 @@ func testCompressOK(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.serverCompression = true
 	te.clientCompression = true
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4311,7 +4303,7 @@ func (s) TestIdentityEncoding(t *testing.T) {
 
 func testIdentityEncoding(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -4375,7 +4367,7 @@ func testUnaryClientInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
 	te.unaryClientInt = failOkayRPC
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4401,7 +4393,7 @@ func failOkayStream(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientC
 func testStreamClientInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.streamClientInt = failOkayStream
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4437,7 +4429,7 @@ func errInjector(ctx context.Context, req interface{}, info *grpc.UnaryServerInf
 func testUnaryServerInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.unaryServerInt = errInjector
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4467,7 +4459,7 @@ func fullDuplexOnly(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServ
 func testStreamServerInterceptor(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.streamServerInt = fullDuplexOnly
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -4504,6 +4496,29 @@ func testStreamServerInterceptor(t *testing.T, e env) {
 	}
 }
 
+// funcServer implements methods of TestServiceServer using funcs,
+// similar to an http.HandlerFunc.
+// Any unimplemented method will crash. Tests implement the method(s)
+// they need.
+type funcServer struct {
+	testpb.TestServiceServer
+	unaryCall          func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
+	streamingInputCall func(stream testpb.TestService_StreamingInputCallServer) error
+	fullDuplexCall     func(stream testpb.TestService_FullDuplexCallServer) error
+}
+
+func (s *funcServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	return s.unaryCall(ctx, in)
+}
+
+func (s *funcServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
+	return s.streamingInputCall(stream)
+}
+
+func (s *funcServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+	return s.fullDuplexCall(stream)
+}
+
 func (s) TestClientRequestBodyErrorUnexpectedEOF(t *testing.T) {
 	for _, e := range listTestEnv() {
 		testClientRequestBodyErrorUnexpectedEOF(t, e)
@@ -4512,7 +4527,7 @@ func (s) TestClientRequestBodyErrorUnexpectedEOF(t *testing.T) {
 
 func testClientRequestBodyErrorUnexpectedEOF(t *testing.T, e env) {
 	te := newTest(t, e)
-	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		errUnexpectedCall := errors.New("unexpected call func server method")
 		t.Error(errUnexpectedCall)
 		return nil, errUnexpectedCall
@@ -4536,7 +4551,7 @@ func (s) TestClientRequestBodyErrorCloseAfterLength(t *testing.T) {
 func testClientRequestBodyErrorCloseAfterLength(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("Server.processUnaryRPC failed to write status")
-	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		errUnexpectedCall := errors.New("unexpected call func server method")
 		t.Error(errUnexpectedCall)
 		return nil, errUnexpectedCall
@@ -4560,7 +4575,7 @@ func (s) TestClientRequestBodyErrorCancel(t *testing.T) {
 func testClientRequestBodyErrorCancel(t *testing.T, e env) {
 	te := newTest(t, e)
 	gotCall := make(chan bool, 1)
-	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		gotCall <- true
 		return new(testpb.SimpleResponse), nil
 	}}
@@ -4596,7 +4611,7 @@ func (s) TestClientRequestBodyErrorCancelStreamingInput(t *testing.T) {
 func testClientRequestBodyErrorCancelStreamingInput(t *testing.T, e env) {
 	te := newTest(t, e)
 	recvErr := make(chan error, 1)
-	ts := &testpb.TestServiceService{StreamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
 		_, err := stream.Recv()
 		recvErr <- err
 		return nil
@@ -4638,7 +4653,7 @@ func testClientInitialHeaderEndStream(t *testing.T, e env) {
 	// checking.
 	handlerDone := make(chan struct{})
 	te := newTest(t, e)
-	ts := &testpb.TestServiceService{StreamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
 		defer close(handlerDone)
 		// Block on serverTester receiving RST_STREAM. This ensures server has closed
 		// stream before stream.Recv().
@@ -4682,7 +4697,7 @@ func testClientSendDataAfterCloseSend(t *testing.T, e env) {
 	// checking.
 	handlerDone := make(chan struct{})
 	te := newTest(t, e)
-	ts := &testpb.TestServiceService{StreamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
+	ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error {
 		defer close(handlerDone)
 		// Block on serverTester receiving RST_STREAM. This ensures server has closed
 		// stream before stream.Recv().
@@ -4734,7 +4749,7 @@ func (s) TestClientResourceExhaustedCancelFullDuplex(t *testing.T) {
 func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) {
 	te := newTest(t, e)
 	recvErr := make(chan error, 1)
-	ts := &testpb.TestServiceService{FullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ts := &funcServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
 		defer close(recvErr)
 		_, err := stream.Recv()
 		if err != nil {
@@ -4857,10 +4872,10 @@ func (s) TestFlowControlLogicalRace(t *testing.T) {
 	defer lis.Close()
 
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, (&flowControlLogicalRaceServer{
+	testpb.RegisterTestServiceServer(s, &flowControlLogicalRaceServer{
 		itemCount: itemCount,
 		itemSize:  itemSize,
-	}).Svc())
+	})
 	defer s.Stop()
 
 	go s.Serve(lis)
@@ -4914,16 +4929,12 @@ func (s) TestFlowControlLogicalRace(t *testing.T) {
 }
 
 type flowControlLogicalRaceServer struct {
+	testpb.TestServiceServer
+
 	itemSize  int
 	itemCount int
 }
 
-func (s *flowControlLogicalRaceServer) Svc() *testpb.TestServiceService {
-	return &testpb.TestServiceService{
-		StreamingOutputCall: s.StreamingOutputCall,
-	}
-}
-
 func (s *flowControlLogicalRaceServer) StreamingOutputCall(req *testpb.StreamingOutputCallRequest, srv testpb.TestService_StreamingOutputCallServer) error {
 	for i := 0; i < s.itemCount; i++ {
 		err := srv.Send(&testpb.StreamingOutputCallResponse{
@@ -5066,6 +5077,9 @@ func (fw *filterWriter) Write(p []byte) (n int, err error) {
 // stubServer is a server that is easy to customize within individual test
 // cases.
 type stubServer struct {
+	// Guarantees we satisfy this interface; panics if unimplemented methods are called.
+	testpb.TestServiceServer
+
 	// Customizable implementations of server handlers.
 	emptyCall      func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error)
 	unaryCall      func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
@@ -5087,12 +5101,16 @@ type stubServer struct {
 	r *manual.Resolver
 }
 
-func (ss *stubServer) Svc() *testpb.TestServiceService {
-	return &testpb.TestServiceService{
-		EmptyCall:      ss.emptyCall,
-		UnaryCall:      ss.unaryCall,
-		FullDuplexCall: ss.fullDuplexCall,
-	}
+func (ss *stubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	return ss.emptyCall(ctx, in)
+}
+
+func (ss *stubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	return ss.unaryCall(ctx, in)
+}
+
+func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+	return ss.fullDuplexCall(stream)
 }
 
 // Start starts the server and creates a client connected to it.
@@ -5115,7 +5133,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption)
 	ss.cleanups = append(ss.cleanups, func() { lis.Close() })
 
 	s := grpc.NewServer(sopts...)
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 	go s.Serve(lis)
 	ss.cleanups = append(ss.cleanups, s.Stop)
 	ss.s = s
@@ -5474,7 +5492,7 @@ func testConfigurableWindowSize(t *testing.T, e env, wc windowSizeConfig) {
 	te.clientInitialWindowSize = wc.clientStream
 	te.clientInitialConnWindowSize = wc.clientConn
 
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -5524,7 +5542,7 @@ func (s) TestWaitForReadyConnection(t *testing.T) {
 func testWaitForReadyConnection(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.userAgent = testAppUA
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn() // Non-blocking dial.
@@ -5575,7 +5593,7 @@ func testEncodeDoesntPanic(t *testing.T, e env) {
 	te := newTest(t, e)
 	erc := &errCodec{}
 	te.customCodec = erc
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	te.customCodec = nil
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -5609,7 +5627,7 @@ func testSvrWriteStatusEarlyWrite(t *testing.T, e env) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	respParam := []*testpb.ResponseParameters{
@@ -5888,7 +5906,7 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	}
 	// Case1: sc set maxReqSize to 2048 (send), maxRespSize to 2048 (recv).
 	te1, ch1 := testServiceConfigSetupTD(t, e)
-	te1.startServer(testServer{security: e.security}.Svc())
+	te1.startServer(&testServer{security: e.security})
 	defer te1.tearDown()
 
 	ch1 <- sc
@@ -5948,7 +5966,7 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	te2, ch2 := testServiceConfigSetupTD(t, e)
 	te2.maxClientReceiveMsgSize = newInt(1024)
 	te2.maxClientSendMsgSize = newInt(1024)
-	te2.startServer(testServer{security: e.security}.Svc())
+	te2.startServer(&testServer{security: e.security})
 	defer te2.tearDown()
 	ch2 <- sc
 	tc = testpb.NewTestServiceClient(te2.clientConn())
@@ -5997,7 +6015,7 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	te3, ch3 := testServiceConfigSetupTD(t, e)
 	te3.maxClientReceiveMsgSize = newInt(4096)
 	te3.maxClientSendMsgSize = newInt(4096)
-	te3.startServer(testServer{security: e.security}.Svc())
+	te3.startServer(&testServer{security: e.security})
 	defer te3.tearDown()
 	ch3 <- sc
 	tc = testpb.NewTestServiceClient(te3.clientConn())
@@ -6089,7 +6107,7 @@ func (s) TestMethodFromServerStream(t *testing.T) {
 func (s) TestInterceptorCanAccessCallOptions(t *testing.T) {
 	e := tcpClearRREnv
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	type observedOptions struct {
@@ -6200,7 +6218,7 @@ func testCompressorRegister(t *testing.T, e env) {
 	te.serverCompression = false
 	te.clientUseCompression = true
 
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
@@ -6258,7 +6276,7 @@ func (s) TestServeExitsWhenListenerClosed(t *testing.T) {
 
 	s := grpc.NewServer()
 	defer s.Stop()
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -6375,7 +6393,7 @@ func testClientDoesntDeadlockWhileWritingErrornousLargeMessages(t *testing.T, e
 	te.userAgent = testAppUA
 	smallSize := 1024
 	te.maxServerReceiveMsgSize = &smallSize
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 1048576)
@@ -6412,7 +6430,7 @@ func (s) TestRPCTimeout(t *testing.T) {
 
 func testRPCTimeout(t *testing.T, e env) {
 	te := newTest(t, e)
-	te.startServer(testServer{security: e.security, unaryCallSleepTime: 500 * time.Millisecond}.Svc())
+	te.startServer(&testServer{security: e.security, unaryCallSleepTime: 500 * time.Millisecond})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -6477,7 +6495,7 @@ func (s) TestDisabledIOBuffers(t *testing.T) {
 	}
 
 	s := grpc.NewServer(grpc.WriteBufferSize(0), grpc.ReadBufferSize(0))
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -6535,7 +6553,7 @@ func testServerMaxHeaderListSizeClientUserViolation(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.maxServerHeaderListSize = new(uint32)
 	*te.maxServerHeaderListSize = 216
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -6567,7 +6585,7 @@ func testClientMaxHeaderListSizeServerUserViolation(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.maxClientHeaderListSize = new(uint32)
 	*te.maxClientHeaderListSize = 1 // any header server sends will violate
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc := te.clientConn()
@@ -6598,7 +6616,7 @@ func testServerMaxHeaderListSizeClientIntentionalViolation(t *testing.T, e env)
 	te := newTest(t, e)
 	te.maxServerHeaderListSize = new(uint32)
 	*te.maxServerHeaderListSize = 512
-	te.startServer(testServer{security: e.security}.Svc())
+	te.startServer(&testServer{security: e.security})
 	defer te.tearDown()
 
 	cc, dw := te.clientConnWithConnControl()
@@ -6640,7 +6658,7 @@ func testClientMaxHeaderListSizeServerIntentionalViolation(t *testing.T, e env)
 	te := newTest(t, e)
 	te.maxClientHeaderListSize = new(uint32)
 	*te.maxClientHeaderListSize = 200
-	lw := te.startServerWithConnControl(testServer{security: e.security, setHeaderOnly: true}.Svc())
+	lw := te.startServerWithConnControl(&testServer{security: e.security, setHeaderOnly: true})
 	defer te.tearDown()
 	cc, _ := te.clientConnWithConnControl()
 	tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)}
@@ -6687,10 +6705,10 @@ func (s) TestNetPipeConn(t *testing.T) {
 	pl := testutils.NewPipeListener()
 	s := grpc.NewServer()
 	defer s.Stop()
-	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 		return &testpb.SimpleResponse{}, nil
 	}}
-	testpb.RegisterTestServiceService(s, ts)
+	testpb.RegisterTestServiceServer(s, ts)
 	go s.Serve(pl)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -6715,18 +6733,7 @@ func testLargeTimeout(t *testing.T, e env) {
 	te := newTest(t, e)
 	te.declareLogNoise("Server.processUnaryRPC failed to write status")
 
-	maxTimeoutChan := make(chan time.Duration, 1)
-	ts := &testpb.TestServiceService{UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
-		maxTimeout := <-maxTimeoutChan
-		deadline, ok := ctx.Deadline()
-		timeout := time.Until(deadline)
-		minTimeout := maxTimeout - 5*time.Second
-		if !ok || timeout < minTimeout || timeout > maxTimeout {
-			t.Errorf("ctx.Deadline() = (now+%v), %v; want [%v, %v], true", timeout, ok, minTimeout, maxTimeout)
-			return nil, status.Error(codes.OutOfRange, "deadline error")
-		}
-		return &testpb.SimpleResponse{}, nil
-	}}
+	ts := &funcServer{}
 	te.startServer(ts)
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -6738,7 +6745,17 @@ func testLargeTimeout(t *testing.T, e env) {
 	}
 
 	for i, maxTimeout := range timeouts {
-		maxTimeoutChan <- maxTimeout
+		ts.unaryCall = func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			deadline, ok := ctx.Deadline()
+			timeout := time.Until(deadline)
+			minTimeout := maxTimeout - 5*time.Second
+			if !ok || timeout < minTimeout || timeout > maxTimeout {
+				t.Errorf("ctx.Deadline() = (now+%v), %v; want [%v, %v], true", timeout, ok, minTimeout, maxTimeout)
+				return nil, status.Error(codes.OutOfRange, "deadline error")
+			}
+			return &testpb.SimpleResponse{}, nil
+		}
+
 		ctx, cancel := context.WithTimeout(context.Background(), maxTimeout)
 		defer cancel()
 
@@ -6761,11 +6778,11 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	}
 	s1 := grpc.NewServer()
 	defer s1.Stop()
-	ts := &testpb.TestServiceService{
-		UnaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ts := &funcServer{
+		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{}, nil
 		},
-		FullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
 			// Wait forever.
 			_, err := stream.Recv()
 			if err == nil {
@@ -6774,7 +6791,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 			return err
 		},
 	}
-	testpb.RegisterTestServiceService(s1, ts)
+	testpb.RegisterTestServiceServer(s1, ts)
 	go s1.Serve(lis1)
 
 	conn2Established := grpcsync.NewEvent()
@@ -6784,7 +6801,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	}
 	s2 := grpc.NewServer()
 	defer s2.Stop()
-	testpb.RegisterTestServiceService(s2, ts)
+	testpb.RegisterTestServiceServer(s2, ts)
 	go s2.Serve(lis2)
 
 	r := manual.NewBuilderWithScheme("whatever")
@@ -6854,7 +6871,7 @@ func (lis notifyingListener) Accept() (net.Conn, error) {
 
 func (s) TestRPCWaitsForResolver(t *testing.T) {
 	te := testServiceConfigSetup(t, tcpClearRREnv)
-	te.startServer(testServer{security: tcpClearRREnv.security}.Svc())
+	te.startServer(&testServer{security: tcpClearRREnv.security})
 	defer te.tearDown()
 	r := manual.NewBuilderWithScheme("whatever")
 
diff --git a/test/goaway_test.go b/test/goaway_test.go
index 1e00976bc144..55f79ebc8548 100644
--- a/test/goaway_test.go
+++ b/test/goaway_test.go
@@ -48,7 +48,7 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) {
 
 	s := grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge}))
 	defer s.Stop()
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
diff --git a/test/gracefulstop_test.go b/test/gracefulstop_test.go
index 1f5f1e705cb5..3da75ea1b51b 100644
--- a/test/gracefulstop_test.go
+++ b/test/gracefulstop_test.go
@@ -117,7 +117,7 @@ func (s) TestGracefulStop(t *testing.T) {
 		},
 	}
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 
 	// 1. Start Server
 	wg := sync.WaitGroup{}
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index 45ae65525134..915eafd82f94 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -47,10 +47,6 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
 	return &testServiceClient{cc}
 }
 
-var testServiceEmptyCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "EmptyCall",
-}
-
 func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
 	out := new(Empty)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...)
@@ -60,10 +56,6 @@ func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...gr
 	return out, nil
 }
 
-var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{
-	StreamName: "UnaryCall",
-}
-
 func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
 	out := new(SimpleResponse)
 	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
@@ -73,13 +65,8 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
-var testServiceStreamingOutputCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "StreamingOutputCall",
-	ServerStreams: true,
-}
-
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceStreamingOutputCallStreamDesc, "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -110,13 +97,8 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo
 	return m, nil
 }
 
-var testServiceStreamingInputCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "StreamingInputCall",
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceStreamingInputCallStreamDesc, "/grpc.testing.TestService/StreamingInputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -149,14 +131,8 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal
 	return m, nil
 }
 
-var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "FullDuplexCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -186,14 +162,8 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
-var testServiceHalfDuplexCallStreamDesc = &grpc.StreamDesc{
-	StreamName:    "HalfDuplexCall",
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
 func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, testServiceHalfDuplexCallStreamDesc, "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -223,82 +193,110 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
-// TestServiceService is the service API for TestService service.
-// Fields should be assigned to their respective handler implementations only before
-// RegisterTestServiceService is called.  Any unassigned fields will result in the
-// handler for that method returning an Unimplemented error.
-type TestServiceService struct {
+// TestServiceServer is the server API for TestService service.
+// All implementations must embed UnimplementedTestServiceServer
+// for forward compatibility
+type TestServiceServer interface {
 	// One empty request followed by one empty response.
-	EmptyCall func(context.Context, *Empty) (*Empty, error)
+	EmptyCall(context.Context, *Empty) (*Empty, error)
 	// One request followed by one response.
 	// The server returns the client payload as-is.
-	UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by a sequence of responses (streamed download).
 	// The server returns the payload with client desired type and sizes.
-	StreamingOutputCall func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
+	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
 	// A sequence of requests followed by one response (streamed upload).
 	// The server returns the aggregated size of client payload as the result.
-	StreamingInputCall func(TestService_StreamingInputCallServer) error
+	StreamingInputCall(TestService_StreamingInputCallServer) error
 	// A sequence of requests with each request served by the server immediately.
 	// As one request could lead to multiple responses, this interface
 	// demonstrates the idea of full duplexing.
-	FullDuplexCall func(TestService_FullDuplexCallServer) error
+	FullDuplexCall(TestService_FullDuplexCallServer) error
 	// A sequence of requests followed by a sequence of responses.
 	// The server buffers all the client requests and then serves them in order. A
 	// stream of responses are returned to the client when the server starts with
 	// first request.
-	HalfDuplexCall func(TestService_HalfDuplexCallServer) error
+	HalfDuplexCall(TestService_HalfDuplexCallServer) error
+	mustEmbedUnimplementedTestServiceServer()
+}
+
+// UnimplementedTestServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedTestServiceServer struct {
+}
+
+func (UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
+}
+func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
+}
+func (UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
+}
+func (UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
+}
+func (UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
+}
+func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {}
+
+// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to TestServiceServer will
+// result in compilation errors.
+type UnsafeTestServiceServer interface {
+	mustEmbedUnimplementedTestServiceServer()
+}
+
+func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+	s.RegisterService(&_TestService_serviceDesc, srv)
 }
 
-func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(Empty)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.EmptyCall(ctx, in)
+		return srv.(TestServiceServer).EmptyCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.TestService/EmptyCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.EmptyCall(ctx, req.(*Empty))
+		return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+
+func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(SimpleRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return s.UnaryCall(ctx, in)
+		return srv.(TestServiceServer).UnaryCall(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
-		Server:     s,
+		Server:     srv,
 		FullMethod: "/grpc.testing.TestService/UnaryCall",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return s.UnaryCall(ctx, req.(*SimpleRequest))
+		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
-func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error {
+
+func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
-}
-func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream})
-}
-func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error {
-	return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
+	return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream})
 }
 
 type TestService_StreamingOutputCallServer interface {
@@ -314,6 +312,10 @@ func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallRespon
 	return x.ServerStream.SendMsg(m)
 }
 
+func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream})
+}
+
 type TestService_StreamingInputCallServer interface {
 	SendAndClose(*StreamingInputCallResponse) error
 	Recv() (*StreamingInputCallRequest, error)
@@ -336,6 +338,10 @@ func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest
 	return m, nil
 }
 
+func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
+}
+
 type TestService_FullDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -358,6 +364,10 @@ func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
+func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream})
+}
+
 type TestService_HalfDuplexCallServer interface {
 	Send(*StreamingOutputCallResponse) error
 	Recv() (*StreamingOutputCallRequest, error)
@@ -380,77 +390,42 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-// RegisterTestServiceService registers a service implementation with a gRPC server.
-func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) {
-	srvCopy := *srv
-	if srvCopy.EmptyCall == nil {
-		srvCopy.EmptyCall = func(context.Context, *Empty) (*Empty, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented")
-		}
-	}
-	if srvCopy.UnaryCall == nil {
-		srvCopy.UnaryCall = func(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-			return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-		}
-	}
-	if srvCopy.StreamingOutputCall == nil {
-		srvCopy.StreamingOutputCall = func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
-		}
-	}
-	if srvCopy.StreamingInputCall == nil {
-		srvCopy.StreamingInputCall = func(TestService_StreamingInputCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented")
-		}
-	}
-	if srvCopy.FullDuplexCall == nil {
-		srvCopy.FullDuplexCall = func(TestService_FullDuplexCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-		}
-	}
-	if srvCopy.HalfDuplexCall == nil {
-		srvCopy.HalfDuplexCall = func(TestService_HalfDuplexCallServer) error {
-			return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
-		}
-	}
-	sd := grpc.ServiceDesc{
-		ServiceName: "grpc.testing.TestService",
-		Methods: []grpc.MethodDesc{
-			{
-				MethodName: "EmptyCall",
-				Handler:    srvCopy.emptyCall,
-			},
-			{
-				MethodName: "UnaryCall",
-				Handler:    srvCopy.unaryCall,
-			},
+var _TestService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.TestService",
+	HandlerType: (*TestServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "EmptyCall",
+			Handler:    _TestService_EmptyCall_Handler,
 		},
-		Streams: []grpc.StreamDesc{
-			{
-				StreamName:    "StreamingOutputCall",
-				Handler:       srvCopy.streamingOutputCall,
-				ServerStreams: true,
-			},
-			{
-				StreamName:    "StreamingInputCall",
-				Handler:       srvCopy.streamingInputCall,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "FullDuplexCall",
-				Handler:       srvCopy.fullDuplexCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
-			{
-				StreamName:    "HalfDuplexCall",
-				Handler:       srvCopy.halfDuplexCall,
-				ServerStreams: true,
-				ClientStreams: true,
-			},
+		{
+			MethodName: "UnaryCall",
+			Handler:    _TestService_UnaryCall_Handler,
 		},
-		Metadata: "test/grpc_testing/test.proto",
-	}
-
-	s.RegisterService(&sd, nil)
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingOutputCall",
+			Handler:       _TestService_StreamingOutputCall_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "StreamingInputCall",
+			Handler:       _TestService_StreamingInputCall_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "FullDuplexCall",
+			Handler:       _TestService_FullDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "HalfDuplexCall",
+			Handler:       _TestService_HalfDuplexCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "test/grpc_testing/test.proto",
 }
diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go
index c8a3796048b7..0a60f8c927ac 100644
--- a/test/healthcheck_test.go
+++ b/test/healthcheck_test.go
@@ -140,7 +140,7 @@ func setupServer(sc *svrConfig) (s *grpc.Server, lis net.Listener, ts *testHealt
 		ts = newTestHealthServer()
 	}
 	healthgrpc.RegisterHealthServer(s, ts)
-	testpb.RegisterTestServiceService(s, testServer{}.Svc())
+	testpb.RegisterTestServiceServer(s, &testServer{})
 	go s.Serve(lis)
 	return s, lis, ts, s.Stop, nil
 }
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index 86969c15d33d..b55b73bdcbce 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -64,7 +64,7 @@ func testLocalCredsE2ESucceed(network, address string) error {
 	s := grpc.NewServer(sopts...)
 	defer s.Stop()
 
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 
 	lis, err := net.Listen(network, address)
 	if err != nil {
@@ -162,7 +162,7 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
 	s := grpc.NewServer(sopts...)
 	defer s.Stop()
 
-	testpb.RegisterTestServiceService(s, ss.Svc())
+	testpb.RegisterTestServiceServer(s, ss)
 
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {

From 6f47205d8c16696c698460a707959a0e82417595 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 29 Sep 2020 16:36:05 -0700
Subject: [PATCH 211/481] credentials/xds: Implementation of client-side xDS
 credentials. (#3888)

---
 credentials/xds/xds.go      | 352 +++++++++++++++++++++++
 credentials/xds/xds_test.go | 553 ++++++++++++++++++++++++++++++++++++
 2 files changed, 905 insertions(+)
 create mode 100644 credentials/xds/xds.go
 create mode 100644 credentials/xds/xds_test.go

diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go
new file mode 100644
index 000000000000..cecc27d14559
--- /dev/null
+++ b/credentials/xds/xds.go
@@ -0,0 +1,352 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds provides a transport credentials implementation where the
+// security configuration is pushed by a management server using xDS APIs.
+//
+// Experimental
+//
+// Notice: All APIs in this package are EXPERIMENTAL and may be removed in a
+// later release.
+package xds
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"sync"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	credinternal "google.golang.org/grpc/internal/credentials"
+	"google.golang.org/grpc/resolver"
+)
+
+// ClientOptions contains parameters to configure a new client-side xDS
+// credentials implementation.
+type ClientOptions struct {
+	// FallbackCreds specifies the fallback credentials to be used when either
+	// the `xds` scheme is not used in the user's dial target or when the xDS
+	// server does not return any security configuration. Attempts to create
+	// client credentials without a fallback credentials will fail.
+	FallbackCreds credentials.TransportCredentials
+}
+
+// NewClientCredentials returns a new client-side transport credentials
+// implementation which uses xDS APIs to fetch its security configuration.
+func NewClientCredentials(opts ClientOptions) (credentials.TransportCredentials, error) {
+	if opts.FallbackCreds == nil {
+		return nil, errors.New("missing fallback credentials")
+	}
+	return &credsImpl{
+		isClient: true,
+		fallback: opts.FallbackCreds,
+	}, nil
+}
+
+// credsImpl is an implementation of the credentials.TransportCredentials
+// interface which uses xDS APIs to fetch its security configuration.
+type credsImpl struct {
+	isClient bool
+	fallback credentials.TransportCredentials
+}
+
+// handshakeAttrKey is the type used as the key to store HandshakeInfo in
+// the Attributes field of resolver.Address.
+type handshakeAttrKey struct{}
+
+// SetHandshakeInfo returns a copy of addr in which the Attributes field is
+// updated with hInfo.
+func SetHandshakeInfo(addr resolver.Address, hInfo *HandshakeInfo) resolver.Address {
+	addr.Attributes = addr.Attributes.WithValues(handshakeAttrKey{}, hInfo)
+	return addr
+}
+
+// getHandshakeInfo returns a pointer to the HandshakeInfo stored in attr.
+func getHandshakeInfo(attr *attributes.Attributes) *HandshakeInfo {
+	v := attr.Value(handshakeAttrKey{})
+	hi, _ := v.(*HandshakeInfo)
+	return hi
+}
+
+// HandshakeInfo wraps all the security configuration required by client and
+// server handshake methods in credsImpl. The xDS implementation will be
+// responsible for populating these fields.
+//
+// Safe for concurrent access.
+type HandshakeInfo struct {
+	mu               sync.Mutex
+	rootProvider     certprovider.Provider
+	identityProvider certprovider.Provider
+	acceptedSANs     map[string]bool // Only on the client side.
+}
+
+// SetRootCertProvider updates the root certificate provider.
+func (hi *HandshakeInfo) SetRootCertProvider(root certprovider.Provider) {
+	hi.mu.Lock()
+	hi.rootProvider = root
+	hi.mu.Unlock()
+}
+
+// SetIdentityCertProvider updates the identity certificate provider.
+func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider) {
+	hi.mu.Lock()
+	hi.identityProvider = identity
+	hi.mu.Unlock()
+}
+
+// SetAcceptedSANs updates the list of accepted SANs.
+func (hi *HandshakeInfo) SetAcceptedSANs(sans []string) {
+	hi.mu.Lock()
+	hi.acceptedSANs = make(map[string]bool, len(sans))
+	for _, san := range sans {
+		hi.acceptedSANs[san] = true
+	}
+	hi.mu.Unlock()
+}
+
+func (hi *HandshakeInfo) validate(isClient bool) error {
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+
+	// On the client side, rootProvider is mandatory. IdentityProvider is
+	// optional based on whether the client is doing TLS or mTLS.
+	if isClient && hi.rootProvider == nil {
+		return errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server")
+	}
+
+	// On the server side, identityProvider is mandatory. RootProvider is
+	// optional based on whether the server is doing TLS or mTLS.
+	if !isClient && hi.identityProvider == nil {
+		return errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server")
+	}
+
+	return nil
+}
+
+func (hi *HandshakeInfo) makeTLSConfig(ctx context.Context) (*tls.Config, error) {
+	hi.mu.Lock()
+	// Since the call to KeyMaterial() can block, we read the providers under
+	// the lock but call the actual function after releasing the lock.
+	rootProv, idProv := hi.rootProvider, hi.identityProvider
+	hi.mu.Unlock()
+
+	// InsecureSkipVerify needs to be set to true because we need to perform
+	// custom verification to check the SAN on the received certificate.
+	// Currently the Go stdlib does complete verification of the cert (which
+	// includes hostname verification) or none. We are forced to go with the
+	// latter and perform the normal cert validation ourselves.
+	cfg := &tls.Config{InsecureSkipVerify: true}
+	if rootProv != nil {
+		km, err := rootProv.KeyMaterial(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
+		}
+		cfg.RootCAs = km.Roots
+	}
+	if idProv != nil {
+		km, err := idProv.KeyMaterial(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err)
+		}
+		cfg.Certificates = km.Certs
+	}
+	return cfg, nil
+}
+
+func (hi *HandshakeInfo) matchingSANExists(cert *x509.Certificate) bool {
+	var sans []string
+	// SANs can be specified in any of these four fields on the parsed cert.
+	sans = append(sans, cert.DNSNames...)
+	sans = append(sans, cert.EmailAddresses...)
+	for _, ip := range cert.IPAddresses {
+		sans = append(sans, ip.String())
+	}
+	for _, uri := range cert.URIs {
+		sans = append(sans, uri.String())
+	}
+
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+	for _, san := range sans {
+		if hi.acceptedSANs[san] {
+			return true
+		}
+	}
+	return false
+}
+
+// NewHandshakeInfo returns a new instance of HandshakeInfo with the given root
+// and identity certificate providers.
+func NewHandshakeInfo(root, identity certprovider.Provider, sans ...string) *HandshakeInfo {
+	acceptedSANs := make(map[string]bool, len(sans))
+	for _, san := range sans {
+		acceptedSANs[san] = true
+	}
+	return &HandshakeInfo{
+		rootProvider:     root,
+		identityProvider: identity,
+		acceptedSANs:     acceptedSANs,
+	}
+}
+
+// ClientHandshake performs the TLS handshake on the client-side.
+//
+// It looks for the presence of a HandshakeInfo value in the passed in context
+// (added using a call to NewContextWithHandshakeInfo()), and retrieves identity
+// and root certificates from there. It also retrieves a list of acceptable SANs
+// and uses a custom verification function to validate the certificate presented
+// by the peer. It uses fallback credentials if no HandshakeInfo is present in
+// the passed in context.
+func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	if !c.isClient {
+		return nil, nil, errors.New("ClientHandshake() is not supported for server credentials")
+	}
+
+	// The CDS balancer constructs a new HandshakeInfo using a call to
+	// NewHandshakeInfo(), and then adds it to the attributes field of the
+	// resolver.Address when handling calls to NewSubConn(). The transport layer
+	// takes care of shipping these attributes in the context to this handshake
+	// function. We first read the credentials.ClientHandshakeInfo type from the
+	// context, which contains the attributes added by the CDS balancer. We then
+	// read the HandshakeInfo from the attributes to get to the actual data that
+	// we need here for the handshake.
+	chi := credentials.ClientHandshakeInfoFromContext(ctx)
+	// If there are no attributes in the received context or the attributes does
+	// not contain a HandshakeInfo, it could either mean that the user did not
+	// specify an `xds` scheme in their dial target or that the xDS server did
+	// not provide any security configuration. In both of these cases, we use
+	// the fallback credentials specified by the user.
+	if chi.Attributes == nil {
+		return c.fallback.ClientHandshake(ctx, authority, rawConn)
+	}
+	hi := getHandshakeInfo(chi.Attributes)
+	if hi == nil {
+		return c.fallback.ClientHandshake(ctx, authority, rawConn)
+	}
+
+	if err := hi.validate(c.isClient); err != nil {
+		return nil, nil, err
+	}
+
+	// We build the tls.Config with the following values
+	// 1. Root certificate as returned by the root provider.
+	// 2. Identity certificate as returned by the identity provider. This may be
+	//    empty on the client side, if the client is not doing mTLS.
+	// 3. InsecureSkipVerify to true. Certificates used in Mesh environments
+	//    usually contains the identity of the workload presenting the
+	//    certificate as a SAN (instead of a hostname in the CommonName field).
+	//    This means that normal certificate verification as done by the
+	//    standard library will fail.
+	// 4. Key usage to match whether client/server usage.
+	// 5. A `VerifyPeerCertificate` function which performs normal peer
+	// 	  cert verification using configured roots, and the custom SAN checks.
+	cfg, err := hi.makeTLSConfig(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+		// Parse all raw certificates presented by the peer.
+		var certs []*x509.Certificate
+		for _, rc := range rawCerts {
+			cert, err := x509.ParseCertificate(rc)
+			if err != nil {
+				return err
+			}
+			certs = append(certs, cert)
+		}
+
+		// Build the intermediates list and verify that the leaf certificate
+		// is signed by one of the root certificates.
+		intermediates := x509.NewCertPool()
+		for _, cert := range certs[1:] {
+			intermediates.AddCert(cert)
+		}
+		opts := x509.VerifyOptions{
+			Roots:         cfg.RootCAs,
+			Intermediates: intermediates,
+			KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		}
+		if _, err := certs[0].Verify(opts); err != nil {
+			return err
+		}
+		// The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to
+		// only look at the SANs on the leaf cert.
+		if !hi.matchingSANExists(certs[0]) {
+			return fmt.Errorf("SANs received in leaf certificate %+v does not match any of the accepted SANs", certs[0])
+		}
+		return nil
+	}
+
+	// Perform the TLS handshake with the tls.Config that we have. We run the
+	// actual Handshake() function in a goroutine because we need to respect the
+	// deadline specified on the passed in context, and we need a way to cancel
+	// the handshake if the context is cancelled.
+	conn := tls.Client(rawConn, cfg)
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- conn.Handshake()
+		close(errCh)
+	}()
+	select {
+	case err := <-errCh:
+		if err != nil {
+			conn.Close()
+			return nil, nil, err
+		}
+	case <-ctx.Done():
+		conn.Close()
+		return nil, nil, ctx.Err()
+	}
+	info := credentials.TLSInfo{
+		State: conn.ConnectionState(),
+		CommonAuthInfo: credentials.CommonAuthInfo{
+			SecurityLevel: credentials.PrivacyAndIntegrity,
+		},
+		SPIFFEID: credinternal.SPIFFEIDFromState(conn.ConnectionState()),
+	}
+	return credinternal.WrapSyscallConn(rawConn, conn), info, nil
+}
+
+// ServerHandshake performs the TLS handshake on the server-side.
+func (c *credsImpl) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	if c.isClient {
+		return nil, nil, errors.New("ServerHandshake is not supported for client credentials")
+	}
+	// TODO(easwars): Implement along with server side xDS implementation.
+	return nil, nil, errors.New("not implemented")
+}
+
+// Info provides the ProtocolInfo of this TransportCredentials.
+func (c *credsImpl) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{SecurityProtocol: "tls"}
+}
+
+// Clone makes a copy of this TransportCredentials.
+func (c *credsImpl) Clone() credentials.TransportCredentials {
+	clone := *c
+	return &clone
+}
+
+func (c *credsImpl) OverrideServerName(_ string) error {
+	return errors.New("serverName for peer validation must be configured as a list of acceptable SANs")
+}
diff --git a/credentials/xds/xds_test.go b/credentials/xds/xds_test.go
new file mode 100644
index 000000000000..a2adcf4558ea
--- /dev/null
+++ b/credentials/xds/xds_test.go
@@ -0,0 +1,553 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/testdata"
+)
+
+const (
+	defaultTestTimeout = 1 * time.Second
+	defaultTestCertSAN = "*.test.example.com"
+	authority          = "authority"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// Helper function to create a real TLS client credentials which is used as
+// fallback credentials from multiple tests.
+func makeFallbackClientCreds(t *testing.T) credentials.TransportCredentials {
+	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return creds
+}
+
+// testServer is a no-op server which listens on a local TCP port for incoming
+// connections, and performs a manual TLS handshake on the received raw
+// connection using a user specified handshake function. It then makes the
+// result of the handshake operation available through a channel for tests to
+// inspect. Tests should stop the testServer as part of their cleanup.
+type testServer struct {
+	lis           net.Listener
+	address       string             // Listening address of the test server.
+	handshakeFunc testHandshakeFunc  // Test specified handshake function.
+	hsResult      *testutils.Channel // Channel to deliver handshake results.
+}
+
+// handshakeResult wraps the result of the handshake operation on the test
+// server. It consists of TLS connection state and an error, if the handshake
+// failed. This result is delivered on the `hsResult` channel on the testServer.
+type handshakeResult struct {
+	connState tls.ConnectionState
+	err       error
+}
+
+// Configurable handshake function for the testServer. Tests can set this to
+// simulate different conditions like handshake success, failure, timeout etc.
+type testHandshakeFunc func(net.Conn) handshakeResult
+
+// newTestServerWithHandshakeFunc starts a new testServer which listens for
+// connections on a local TCP port, and uses the provided custom handshake
+// function to perform TLS handshake.
+func newTestServerWithHandshakeFunc(f testHandshakeFunc) *testServer {
+	ts := &testServer{
+		handshakeFunc: f,
+		hsResult:      testutils.NewChannel(),
+	}
+	ts.start()
+	return ts
+}
+
+// starts actually starts listening on a local TCP port, and spawns a goroutine
+// to handle new connections.
+func (ts *testServer) start() error {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return err
+	}
+	ts.lis = lis
+	ts.address = lis.Addr().String()
+	go ts.handleConn()
+	return nil
+}
+
+// handleconn accepts a new raw connection, and invokes the test provided
+// handshake function to perform TLS handshake, and returns the result on the
+// `hsResult` channel.
+func (ts *testServer) handleConn() {
+	for {
+		rawConn, err := ts.lis.Accept()
+		if err != nil {
+			// Once the listeners closed, Accept() will return with an error.
+			return
+		}
+		hsr := ts.handshakeFunc(rawConn)
+		ts.hsResult.Send(hsr)
+	}
+}
+
+// stop closes the associated listener which causes the connection handling
+// goroutine to exit.
+func (ts *testServer) stop() {
+	ts.lis.Close()
+}
+
+// A handshake function which simulates a handshake timeout. Tests usually pass
+// `defaultTestTimeout` to the ClientHandshake() method. This function just
+// hangs around for twice that duration, thus making sure that the context
+// passes to the credentials code times out.
+func testServerTLSHandshakeTimeout(_ net.Conn) handshakeResult {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*defaultTestTimeout)
+	<-ctx.Done()
+	cancel()
+	return handshakeResult{err: ctx.Err()}
+}
+
+// A handshake function which simulates a successful handshake without client
+// authentication (server does not request for client certificate during the
+// handshake here).
+func testServerTLSHandshake(rawConn net.Conn) handshakeResult {
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		return handshakeResult{err: err}
+	}
+	cfg := &tls.Config{Certificates: []tls.Certificate{cert}}
+	conn := tls.Server(rawConn, cfg)
+	if err := conn.Handshake(); err != nil {
+		return handshakeResult{err: err}
+	}
+	return handshakeResult{connState: conn.ConnectionState()}
+}
+
+// A handshake function which simulates a successful handshake with mutual
+// authentication.
+func testServerMutualTLSHandshake(rawConn net.Conn) handshakeResult {
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		return handshakeResult{err: err}
+	}
+	pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
+	if err != nil {
+		return handshakeResult{err: err}
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	cfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    roots,
+	}
+	conn := tls.Server(rawConn, cfg)
+	if err := conn.Handshake(); err != nil {
+		return handshakeResult{err: err}
+	}
+	return handshakeResult{connState: conn.ConnectionState()}
+}
+
+// fakeProvider is an implementation of the certprovider.Provider interface
+// which returns the configured key material and error in calls to
+// KeyMaterial().
+type fakeProvider struct {
+	km  *certprovider.KeyMaterial
+	err error
+}
+
+func (f *fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
+	return f.km, f.err
+}
+
+func (f *fakeProvider) Close() {}
+
+// makeIdentityProvider creates a new instance of the fakeProvider returning the
+// identity key material specified in the provider file paths.
+func makeIdentityProvider(t *testing.T, certPath, keyPath string) certprovider.Provider {
+	t.Helper()
+	cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &fakeProvider{km: &certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}}
+}
+
+// makeRootProvider creates a new instance of the fakeProvider returning the
+// root key material specified in the provider file paths.
+func makeRootProvider(t *testing.T, caPath string) *fakeProvider {
+	pemData, err := ioutil.ReadFile(testdata.Path(caPath))
+	if err != nil {
+		t.Fatal(err)
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	return &fakeProvider{km: &certprovider.KeyMaterial{Roots: roots}}
+}
+
+// newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo
+// context value added to it.
+func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sans ...string) context.Context {
+	// Creating the HandshakeInfo and adding it to the attributes is very
+	// similar to what the CDS balancer would do when it intercepts calls to
+	// NewSubConn().
+	info := NewHandshakeInfo(root, identity, sans...)
+	addr := SetHandshakeInfo(resolver.Address{}, info)
+
+	// Moving the attributes from the resolver.Address to the context passed to
+	// the handshaker is done in the transport layer. Since we directly call the
+	// handshaker in these tests, we need to do the same here.
+	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
+	return contextWithHandshakeInfo(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
+}
+
+// compareAuthInfo compares the AuthInfo received on the client side after a
+// successful handshake with the authInfo available on the testServer.
+func compareAuthInfo(ts *testServer, ai credentials.AuthInfo) error {
+	if ai.AuthType() != "tls" {
+		return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls")
+	}
+	info, ok := ai.(credentials.TLSInfo)
+	if !ok {
+		return fmt.Errorf("ClientHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})
+	}
+	gotState := info.State
+
+	// Read the handshake result from the testServer which contains the TLS
+	// connection state and compare it with the one received on the client-side.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ts.hsResult.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("testServer failed to return handshake result: %v", err)
+	}
+	hsr := val.(handshakeResult)
+	if hsr.err != nil {
+		return fmt.Errorf("testServer handshake failure: %v", hsr.err)
+	}
+	// AuthInfo contains a variety of information. We only verify a subset here.
+	// This is the same subset which is verified in TLS credentials tests.
+	if err := compareConnState(gotState, hsr.connState); err != nil {
+		return err
+	}
+	return nil
+}
+
+func compareConnState(got, want tls.ConnectionState) error {
+	switch {
+	case got.Version != want.Version:
+		return fmt.Errorf("TLS.ConnectionState got Version: %v, want: %v", got.Version, want.Version)
+	case got.HandshakeComplete != want.HandshakeComplete:
+		return fmt.Errorf("TLS.ConnectionState got HandshakeComplete: %v, want: %v", got.HandshakeComplete, want.HandshakeComplete)
+	case got.CipherSuite != want.CipherSuite:
+		return fmt.Errorf("TLS.ConnectionState got CipherSuite: %v, want: %v", got.CipherSuite, want.CipherSuite)
+	case got.NegotiatedProtocol != want.NegotiatedProtocol:
+		return fmt.Errorf("TLS.ConnectionState got NegotiatedProtocol: %v, want: %v", got.NegotiatedProtocol, want.NegotiatedProtocol)
+	}
+	return nil
+}
+
+// TestClientCredsWithoutFallback verifies that the call to
+// NewClientCredentials() fails when no fallback is specified.
+func (s) TestClientCredsWithoutFallback(t *testing.T) {
+	if _, err := NewClientCredentials(ClientOptions{}); err == nil {
+		t.Fatal("NewClientCredentials() succeeded without specifying fallback")
+	}
+}
+
+// TestClientCredsInvalidHandshakeInfo verifies scenarios where the passed in
+// HandshakeInfo is invalid because it does not contain the expected certificate
+// providers.
+func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) {
+	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+	creds, err := NewClientCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+	}
+
+	pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx := newTestContextWithHandshakeInfo(pCtx, nil, nil)
+	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
+		t.Fatal("ClientHandshake succeeded without certificate providers in HandshakeInfo")
+	}
+
+	ctx = newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{})
+	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
+		t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo")
+	}
+}
+
+// TestClientCredsProviderFailure verifies the cases where an expected
+// certificate provider is missing in the HandshakeInfo value in the context.
+func (s) TestClientCredsProviderFailure(t *testing.T) {
+	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+	creds, err := NewClientCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+	}
+
+	tests := []struct {
+		desc             string
+		rootProvider     certprovider.Provider
+		identityProvider certprovider.Provider
+		wantErr          string
+	}{
+		{
+			desc:         "erroring root provider",
+			rootProvider: &fakeProvider{err: errors.New("root provider error")},
+			wantErr:      "root provider error",
+		},
+		{
+			desc:             "erroring identity provider",
+			rootProvider:     &fakeProvider{km: &certprovider.KeyMaterial{}},
+			identityProvider: &fakeProvider{err: errors.New("identity provider error")},
+			wantErr:          "identity provider error",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider)
+			if _, _, err := creds.ClientHandshake(ctx, authority, nil); !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
+			}
+		})
+	}
+}
+
+// TestClientCredsSuccess verifies successful client handshake cases.
+func (s) TestClientCredsSuccess(t *testing.T) {
+	tests := []struct {
+		desc             string
+		handshakeFunc    testHandshakeFunc
+		rootProvider     certprovider.Provider
+		identityProvider certprovider.Provider
+	}{
+		{
+			// Since we don't specify rootProvider and identityProvider here,
+			// the test does not add a HandshakeInfo context value, and thereby
+			// the ClientHandshake() method will delegate to the fallback.
+			desc:          "fallback",
+			handshakeFunc: testServerTLSHandshake,
+		},
+		{
+			desc:          "TLS",
+			handshakeFunc: testServerTLSHandshake,
+			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
+		},
+		{
+			desc:             "mTLS",
+			handshakeFunc:    testServerMutualTLSHandshake,
+			rootProvider:     makeRootProvider(t, "x509/server_ca_cert.pem"),
+			identityProvider: makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
+			defer ts.stop()
+
+			opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+			creds, err := NewClientCredentials(opts)
+			if err != nil {
+				t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+			}
+
+			conn, err := net.Dial("tcp", ts.address)
+			if err != nil {
+				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+			}
+			defer conn.Close()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if test.rootProvider != nil || test.identityProvider != nil {
+				ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider, defaultTestCertSAN)
+			}
+			_, ai, err := creds.ClientHandshake(ctx, authority, conn)
+			if err != nil {
+				t.Fatalf("ClientHandshake() returned failed: %q", err)
+			}
+			if err := compareAuthInfo(ts, ai); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// TestClientCredsHandshakeFailure verifies different handshake failure cases.
+func (s) TestClientCredsHandshakeFailure(t *testing.T) {
+	tests := []struct {
+		desc          string
+		handshakeFunc testHandshakeFunc
+		rootProvider  certprovider.Provider
+		san           string
+		wantErr       string
+	}{
+		{
+			desc:          "cert validation failure",
+			handshakeFunc: testServerTLSHandshake,
+			rootProvider:  makeRootProvider(t, "x509/client_ca_cert.pem"),
+			san:           defaultTestCertSAN,
+			wantErr:       "x509: certificate signed by unknown authority",
+		},
+		{
+			desc:          "handshake times out",
+			handshakeFunc: testServerTLSHandshakeTimeout,
+			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
+			san:           defaultTestCertSAN,
+			wantErr:       "context deadline exceeded",
+		},
+		{
+			desc:          "SAN mismatch",
+			handshakeFunc: testServerTLSHandshake,
+			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
+			san:           "bad-san",
+			wantErr:       "does not match any of the accepted SANs",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
+			defer ts.stop()
+
+			opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+			creds, err := NewClientCredentials(opts)
+			if err != nil {
+				t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+			}
+
+			conn, err := net.Dial("tcp", ts.address)
+			if err != nil {
+				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+			}
+			defer conn.Close()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, nil, test.san)
+			if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil || !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("ClientHandshake() returned %q, wantErr %q", err, test.wantErr)
+			}
+		})
+	}
+}
+
+// TestClientCredsProviderSwitch verifies the case where the first attempt of
+// ClientHandshake fails because of a handshake failure. Then we update the
+// certificate provider and the second attempt succeeds. This is an
+// approximation of the flow of events when the control plane specifies new
+// security config which results in new certificate providers being used.
+func (s) TestClientCredsProviderSwitch(t *testing.T) {
+	ts := newTestServerWithHandshakeFunc(testServerTLSHandshake)
+	defer ts.stop()
+
+	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+	creds, err := NewClientCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+	}
+
+	conn, err := net.Dial("tcp", ts.address)
+	if err != nil {
+		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+	}
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Create a root provider which will fail the handshake because it does not
+	// use the correct trust roots.
+	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
+	handshakeInfo := NewHandshakeInfo(root1, nil, defaultTestCertSAN)
+
+	// We need to repeat most of what newTestContextWithHandshakeInfo() does
+	// here because we need access to the underlying HandshakeInfo so that we
+	// can update it before the next call to ClientHandshake().
+	addr := SetHandshakeInfo(resolver.Address{}, handshakeInfo)
+	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
+	ctx = contextWithHandshakeInfo(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
+	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
+		t.Fatal("ClientHandshake() succeeded when expected to fail")
+	}
+	// Drain the result channel on the test server so that we can inspect the
+	// result for the next handshake.
+	_, err = ts.hsResult.Receive(ctx)
+	if err != nil {
+		t.Errorf("testServer failed to return handshake result: %v", err)
+	}
+
+	conn, err = net.Dial("tcp", ts.address)
+	if err != nil {
+		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+	}
+	defer conn.Close()
+
+	// Create a new root provider which uses the correct trust roots. And update
+	// the HandshakeInfo with the new provider.
+	root2 := makeRootProvider(t, "x509/server_ca_cert.pem")
+	handshakeInfo.SetRootCertProvider(root2)
+	_, ai, err := creds.ClientHandshake(ctx, authority, conn)
+	if err != nil {
+		t.Fatalf("ClientHandshake() returned failed: %q", err)
+	}
+	if err := compareAuthInfo(ts, ai); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestClone verifies the Clone() method.
+func (s) TestClone(t *testing.T) {
+	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+	orig, err := NewClientCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+	}
+
+	// The credsImpl does not have any exported fields, and it does not make
+	// sense to use any cmp options to look deep into. So, all we make sure here
+	// is that the cloned object points to a different locaiton in memory.
+	if clone := orig.Clone(); clone == orig {
+		t.Fatal("return value from Clone() doesn't point to new credentials instance")
+	}
+}

From 866de13d56a0887d262009befb084c247f6c8e16 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 29 Sep 2020 17:40:30 -0700
Subject: [PATCH 212/481] meshca: CertificateProvider plugin implementation.
 (#3871)

---
 credentials/sts/sts.go                        |   6 +-
 credentials/sts/sts_test.go                   | 102 ++--
 .../tls/certprovider/meshca/builder.go        | 169 +++++++
 .../tls/certprovider/meshca/builder_test.go   | 182 +++++++
 credentials/tls/certprovider/meshca/config.go | 281 +++++++++++
 .../tls/certprovider/meshca/config_test.go    | 410 ++++++++++++++++
 .../tls/certprovider/meshca/logging.go        |  36 ++
 credentials/tls/certprovider/meshca/plugin.go | 288 +++++++++++
 .../tls/certprovider/meshca/plugin_test.go    | 464 ++++++++++++++++++
 examples/go.sum                               |   1 +
 go.mod                                        |   1 +
 go.sum                                        |   2 +
 internal/grpctest/tlogger.go                  |  16 +-
 internal/leakcheck/leakcheck.go               |   1 +
 internal/testutils/http_client.go             |  63 +++
 security/advancedtls/go.sum                   |   1 +
 16 files changed, 1949 insertions(+), 74 deletions(-)
 create mode 100644 credentials/tls/certprovider/meshca/builder.go
 create mode 100644 credentials/tls/certprovider/meshca/builder_test.go
 create mode 100644 credentials/tls/certprovider/meshca/config.go
 create mode 100644 credentials/tls/certprovider/meshca/config_test.go
 create mode 100644 credentials/tls/certprovider/meshca/logging.go
 create mode 100644 credentials/tls/certprovider/meshca/plugin.go
 create mode 100644 credentials/tls/certprovider/meshca/plugin_test.go
 create mode 100644 internal/testutils/http_client.go

diff --git a/credentials/sts/sts.go b/credentials/sts/sts.go
index a97b20a9eb72..f4d58011be86 100644
--- a/credentials/sts/sts.go
+++ b/credentials/sts/sts.go
@@ -111,6 +111,10 @@ type Options struct {
 	ActorTokenType string // Optional.
 }
 
+func (o Options) String() string {
+	return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", o.TokenExchangeServiceURI, o.Resource, o.Audience, o.Scope, o.RequestedTokenType, o.SubjectTokenPath, o.SubjectTokenType, o.ActorTokenPath, o.ActorTokenType)
+}
+
 // NewCredentials returns a new PerRPCCredentials implementation, configured
 // using opts, which performs token exchange using STS.
 func NewCredentials(opts Options) (credentials.PerRPCCredentials, error) {
@@ -213,7 +217,7 @@ func validateOptions(opts Options) error {
 		return err
 	}
 	if u.Scheme != "http" && u.Scheme != "https" {
-		return fmt.Errorf("scheme is not supported: %s. Only http(s) is supported", u.Scheme)
+		return fmt.Errorf("scheme is not supported: %q. Only http(s) is supported", u.Scheme)
 	}
 
 	if opts.SubjectTokenPath == "" {
diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go
index cc8b08e5d933..9cfa12097d60 100644
--- a/credentials/sts/sts_test.go
+++ b/credentials/sts/sts_test.go
@@ -43,19 +43,20 @@ import (
 )
 
 const (
-	requestedTokenType   = "urn:ietf:params:oauth:token-type:access-token"
-	actorTokenPath       = "/var/run/secrets/token.jwt"
-	actorTokenType       = "urn:ietf:params:oauth:token-type:refresh_token"
-	actorTokenContents   = "actorToken.jwt.contents"
-	accessTokenContents  = "access_token"
-	subjectTokenPath     = "/var/run/secrets/token.jwt"
-	subjectTokenType     = "urn:ietf:params:oauth:token-type:id_token"
-	subjectTokenContents = "subjectToken.jwt.contents"
-	serviceURI           = "http://localhost"
-	exampleResource      = "https://backend.example.com/api"
-	exampleAudience      = "example-backend-service"
-	testScope            = "https://www.googleapis.com/auth/monitoring"
-	defaultTestTimeout   = 1 * time.Second
+	requestedTokenType      = "urn:ietf:params:oauth:token-type:access-token"
+	actorTokenPath          = "/var/run/secrets/token.jwt"
+	actorTokenType          = "urn:ietf:params:oauth:token-type:refresh_token"
+	actorTokenContents      = "actorToken.jwt.contents"
+	accessTokenContents     = "access_token"
+	subjectTokenPath        = "/var/run/secrets/token.jwt"
+	subjectTokenType        = "urn:ietf:params:oauth:token-type:id_token"
+	subjectTokenContents    = "subjectToken.jwt.contents"
+	serviceURI              = "http://localhost"
+	exampleResource         = "https://backend.example.com/api"
+	exampleAudience         = "example-backend-service"
+	testScope               = "https://www.googleapis.com/auth/monitoring"
+	defaultTestTimeout      = 1 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
 )
 
 var (
@@ -132,35 +133,13 @@ func makeGoodResponse() *http.Response {
 	}
 }
 
-// fakeHTTPDoer helps mock out the http.Client.Do calls made by the credentials
-// code under test. It makes the http.Request made by the credentials available
-// through a channel, and makes it possible to inject various responses.
-type fakeHTTPDoer struct {
-	reqCh  *testutils.Channel
-	respCh *testutils.Channel
-	err    error
-}
-
-func (fc *fakeHTTPDoer) Do(req *http.Request) (*http.Response, error) {
-	fc.reqCh.Send(req)
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-
-	val, err := fc.respCh.Receive(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return val.(*http.Response), fc.err
-}
-
 // Overrides the http.Client with a fakeClient which sends a good response.
-func overrideHTTPClientGood() (*fakeHTTPDoer, func()) {
-	fc := &fakeHTTPDoer{
-		reqCh:  testutils.NewChannel(),
-		respCh: testutils.NewChannel(),
+func overrideHTTPClientGood() (*testutils.FakeHTTPClient, func()) {
+	fc := &testutils.FakeHTTPClient{
+		ReqChan:  testutils.NewChannel(),
+		RespChan: testutils.NewChannel(),
 	}
-	fc.respCh.Send(makeGoodResponse())
+	fc.RespChan.Send(makeGoodResponse())
 
 	origMakeHTTPDoer := makeHTTPDoer
 	makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc }
@@ -168,7 +147,7 @@ func overrideHTTPClientGood() (*fakeHTTPDoer, func()) {
 }
 
 // Overrides the http.Client with the provided fakeClient.
-func overrideHTTPClient(fc *fakeHTTPDoer) func() {
+func overrideHTTPClient(fc *testutils.FakeHTTPClient) func() {
 	origMakeHTTPDoer := makeHTTPDoer
 	makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc }
 	return func() { makeHTTPDoer = origMakeHTTPDoer }
@@ -244,11 +223,11 @@ func compareRequest(gotRequest *http.Request, wantReqParams *requestParameters)
 // expected goodRequest. This is expected to be called in a separate goroutine
 // by the tests. So, any errors encountered are pushed to an error channel
 // which is monitored by the test.
-func receiveAndCompareRequest(reqCh *testutils.Channel, errCh chan error) {
+func receiveAndCompareRequest(ReqChan *testutils.Channel, errCh chan error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 
-	val, err := reqCh.Receive(ctx)
+	val, err := ReqChan.Receive(ctx)
 	if err != nil {
 		errCh <- err
 		return
@@ -274,7 +253,7 @@ func (s) TestGetRequestMetadataSuccess(t *testing.T) {
 	}
 
 	errCh := make(chan error, 1)
-	go receiveAndCompareRequest(fc.reqCh, errCh)
+	go receiveAndCompareRequest(fc.ReqChan, errCh)
 
 	gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
 	if err != nil {
@@ -323,9 +302,9 @@ func (s) TestGetRequestMetadataBadSecurityLevel(t *testing.T) {
 func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) {
 	const expiresInSecs = 1
 	defer overrideSubjectTokenGood()()
-	fc := &fakeHTTPDoer{
-		reqCh:  testutils.NewChannel(),
-		respCh: testutils.NewChannel(),
+	fc := &testutils.FakeHTTPClient{
+		ReqChan:  testutils.NewChannel(),
+		RespChan: testutils.NewChannel(),
 	}
 	defer overrideHTTPClient(fc)()
 
@@ -340,7 +319,7 @@ func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) {
 	// out a fresh request.
 	for i := 0; i < 2; i++ {
 		errCh := make(chan error, 1)
-		go receiveAndCompareRequest(fc.reqCh, errCh)
+		go receiveAndCompareRequest(fc.ReqChan, errCh)
 
 		respJSON, _ := json.Marshal(responseParameters{
 			AccessToken:     accessTokenContents,
@@ -354,7 +333,7 @@ func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) {
 			StatusCode: http.StatusOK,
 			Body:       respBody,
 		}
-		fc.respCh.Send(resp)
+		fc.RespChan.Send(resp)
 
 		gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
 		if err != nil {
@@ -399,9 +378,9 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			defer overrideSubjectTokenGood()()
 
-			fc := &fakeHTTPDoer{
-				reqCh:  testutils.NewChannel(),
-				respCh: testutils.NewChannel(),
+			fc := &testutils.FakeHTTPClient{
+				ReqChan:  testutils.NewChannel(),
+				RespChan: testutils.NewChannel(),
 			}
 			defer overrideHTTPClient(fc)()
 
@@ -411,9 +390,9 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) {
 			}
 
 			errCh := make(chan error, 1)
-			go receiveAndCompareRequest(fc.reqCh, errCh)
+			go receiveAndCompareRequest(fc.ReqChan, errCh)
 
-			fc.respCh.Send(test.response)
+			fc.RespChan.Send(test.response)
 			if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil {
 				t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail")
 			}
@@ -438,10 +417,9 @@ func (s) TestGetRequestMetadataBadSubjectTokenRead(t *testing.T) {
 
 	errCh := make(chan error, 1)
 	go func() {
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 		defer cancel()
-
-		if _, err := fc.reqCh.Receive(ctx); err != context.DeadlineExceeded {
+		if _, err := fc.ReqChan.Receive(ctx); err != context.DeadlineExceeded {
 			errCh <- err
 			return
 		}
@@ -698,12 +676,12 @@ func (s) TestSendRequest(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			client := &fakeHTTPDoer{
-				reqCh:  testutils.NewChannel(),
-				respCh: testutils.NewChannel(),
-				err:    test.respErr,
+			client := &testutils.FakeHTTPClient{
+				ReqChan:  testutils.NewChannel(),
+				RespChan: testutils.NewChannel(),
+				Err:      test.respErr,
 			}
-			client.respCh.Send(test.resp)
+			client.RespChan.Send(test.resp)
 			_, err := sendRequest(client, req)
 			if (err != nil) != test.wantErr {
 				t.Errorf("sendRequest(%v) = %v, wantErr: %v", req, err, test.wantErr)
diff --git a/credentials/tls/certprovider/meshca/builder.go b/credentials/tls/certprovider/meshca/builder.go
new file mode 100644
index 000000000000..3544a1647f67
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/builder.go
@@ -0,0 +1,169 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package meshca
+
+import (
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/sts"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/backoff"
+)
+
+const pluginName = "mesh_ca"
+
+// For overriding in unit tests.
+var (
+	grpcDialFunc = grpc.Dial
+	backoffFunc  = backoff.DefaultExponential.Backoff
+)
+
+func init() {
+	certprovider.Register(newPluginBuilder())
+}
+
+func newPluginBuilder() *pluginBuilder {
+	return &pluginBuilder{clients: make(map[ccMapKey]*refCountedCC)}
+}
+
+// Key for the map containing ClientConns to the MeshCA server. Only the server
+// name and the STS options (which is used to create call creds) from the plugin
+// configuration determine if two configs can share the same ClientConn. Hence
+// only those form the key to this map.
+type ccMapKey struct {
+	name    string
+	stsOpts sts.Options
+}
+
+// refCountedCC wraps a grpc.ClientConn to MeshCA along with a reference count.
+type refCountedCC struct {
+	cc     *grpc.ClientConn
+	refCnt int
+}
+
+// pluginBuilder is an implementation of the certprovider.Builder interface,
+// which build certificate provider instances which get certificates signed from
+// the MeshCA.
+type pluginBuilder struct {
+	// A collection of ClientConns to the MeshCA server along with a reference
+	// count. Provider instances whose config point to the same server name will
+	// end up sharing the ClientConn.
+	mu      sync.Mutex
+	clients map[ccMapKey]*refCountedCC
+}
+
+// Build returns a MeshCA certificate provider for the passed in configuration
+// and options.
+//
+// This builder takes care of sharing the ClientConn to the MeshCA server among
+// different plugin instantiations.
+func (b *pluginBuilder) Build(c certprovider.StableConfig, opts certprovider.Options) certprovider.Provider {
+	cfg, ok := c.(*pluginConfig)
+	if !ok {
+		// This is not expected when passing config returned by ParseConfig().
+		// This could indicate a bug in the certprovider.Store implementation or
+		// in cases where the user is directly using these APIs, could be a user
+		// error.
+		logger.Errorf("unsupported config type: %T", c)
+		return nil
+	}
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	ccmk := ccMapKey{
+		name:    cfg.serverURI,
+		stsOpts: cfg.stsOpts,
+	}
+	rcc, ok := b.clients[ccmk]
+	if !ok {
+		// STS call credentials take care of exchanging a locally provisioned
+		// JWT token for an access token which will be accepted by the MeshCA.
+		callCreds, err := sts.NewCredentials(cfg.stsOpts)
+		if err != nil {
+			logger.Errorf("sts.NewCredentials() failed: %v", err)
+			return nil
+		}
+
+		// MeshCA is a public endpoint whose certificate is Web-PKI compliant.
+		// So, we just need to use the system roots to authenticate the MeshCA.
+		cp, err := x509.SystemCertPool()
+		if err != nil {
+			logger.Errorf("x509.SystemCertPool() failed: %v", err)
+			return nil
+		}
+		transportCreds := credentials.NewClientTLSFromCert(cp, "")
+
+		cc, err := grpcDialFunc(cfg.serverURI, grpc.WithTransportCredentials(transportCreds), grpc.WithPerRPCCredentials(callCreds))
+		if err != nil {
+			logger.Errorf("grpc.Dial(%s) failed: %v", cfg.serverURI, err)
+			return nil
+		}
+
+		rcc = &refCountedCC{cc: cc}
+		b.clients[ccmk] = rcc
+	}
+	rcc.refCnt++
+
+	p := newProviderPlugin(providerParams{
+		cc:      rcc.cc,
+		cfg:     cfg,
+		opts:    opts,
+		backoff: backoffFunc,
+		doneFunc: func() {
+			// The plugin implementation will invoke this function when it is
+			// being closed, and here we take care of closing the ClientConn
+			// when there are no more plugins using it. We need to acquire the
+			// lock before accessing the rcc from the enclosing function.
+			b.mu.Lock()
+			defer b.mu.Unlock()
+			rcc.refCnt--
+			if rcc.refCnt == 0 {
+				logger.Infof("Closing grpc.ClientConn to %s", ccmk.name)
+				rcc.cc.Close()
+				delete(b.clients, ccmk)
+			}
+		},
+	})
+	return p
+}
+
+// ParseConfig parses the configuration to be passed to the MeshCA plugin
+// implementation. Expects the config to be a json.RawMessage which contains a
+// serialized JSON representation of the meshca_experimental.GoogleMeshCaConfig
+// proto message.
+func (b *pluginBuilder) ParseConfig(c interface{}) (certprovider.StableConfig, error) {
+	data, ok := c.(json.RawMessage)
+	if !ok {
+		return nil, fmt.Errorf("meshca: unsupported config type: %T", c)
+	}
+	return pluginConfigFromJSON(data)
+}
+
+// Name returns the MeshCA plugin name.
+func (b *pluginBuilder) Name() string {
+	return pluginName
+}
diff --git a/credentials/tls/certprovider/meshca/builder_test.go b/credentials/tls/certprovider/meshca/builder_test.go
new file mode 100644
index 000000000000..b395f4f4b91d
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/builder_test.go
@@ -0,0 +1,182 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package meshca
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
+	"google.golang.org/grpc/internal/testutils"
+)
+
+func overrideHTTPFuncs() func() {
+	// Directly override the functions which are used to read the zone and
+	// audience instead of overriding the http.Client.
+	origReadZone := readZoneFunc
+	readZoneFunc = func(httpDoer) string { return "test-zone" }
+	origReadAudience := readAudienceFunc
+	readAudienceFunc = func(httpDoer) string { return "test-audience" }
+	return func() {
+		readZoneFunc = origReadZone
+		readAudienceFunc = origReadAudience
+	}
+}
+
+func (s) TestBuildSameConfig(t *testing.T) {
+	defer overrideHTTPFuncs()()
+
+	// We will attempt to create `cnt` number of providers. So we create a
+	// channel of the same size here, even though we expect only one ClientConn
+	// to be pushed into this channel. This makes sure that even if more than
+	// one ClientConn ends up being created, the Build() call does not block.
+	const cnt = 5
+	ccChan := testutils.NewChannelWithSize(cnt)
+
+	// Override the dial func to dial a dummy MeshCA endpoint, and also push the
+	// returned ClientConn on a channel to be inspected by the test.
+	origDialFunc := grpcDialFunc
+	grpcDialFunc = func(string, ...grpc.DialOption) (*grpc.ClientConn, error) {
+		cc, err := grpc.Dial("dummy-meshca-endpoint", grpc.WithInsecure())
+		ccChan.Send(cc)
+		return cc, err
+	}
+	defer func() { grpcDialFunc = origDialFunc }()
+
+	// Parse a good config to generate a stable config which will be passed to
+	// invocations of Build().
+	inputConfig := makeJSONConfig(t, goodConfigFullySpecified)
+	builder := newPluginBuilder()
+	stableConfig, err := builder.ParseConfig(inputConfig)
+	if err != nil {
+		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
+	}
+
+	// Create multiple providers with the same config. All these providers must
+	// end up sharing the same ClientConn.
+	providers := []certprovider.Provider{}
+	for i := 0; i < cnt; i++ {
+		p := builder.Build(stableConfig, certprovider.Options{})
+		if p == nil {
+			t.Fatalf("builder.Build(%s) failed: %v", string(stableConfig.Canonical()), err)
+		}
+		providers = append(providers, p)
+	}
+
+	// Make sure only one ClientConn is created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ccChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failed to create ClientConn: %v", err)
+	}
+	testCC := val.(*grpc.ClientConn)
+
+	// Attempt to read the second ClientConn should timeout.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer cancel()
+	if _, err := ccChan.Receive(ctx); err != context.DeadlineExceeded {
+		t.Fatal("Builder created more than one ClientConn")
+	}
+
+	for _, p := range providers {
+		p.Close()
+	}
+
+	for {
+		state := testCC.GetState()
+		if state == connectivity.Shutdown {
+			break
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		if !testCC.WaitForStateChange(ctx, state) {
+			t.Fatalf("timeout waiting for clientConn state to change from %s", state)
+		}
+	}
+}
+
+func (s) TestBuildDifferentConfig(t *testing.T) {
+	defer overrideHTTPFuncs()()
+
+	// We will attempt to create two providers with different configs. So we
+	// expect two ClientConns to be pushed on to this channel.
+	const cnt = 2
+	ccChan := testutils.NewChannelWithSize(cnt)
+
+	// Override the dial func to dial a dummy MeshCA endpoint, and also push the
+	// returned ClientConn on a channel to be inspected by the test.
+	origDialFunc := grpcDialFunc
+	grpcDialFunc = func(string, ...grpc.DialOption) (*grpc.ClientConn, error) {
+		cc, err := grpc.Dial("dummy-meshca-endpoint", grpc.WithInsecure())
+		ccChan.Send(cc)
+		return cc, err
+	}
+	defer func() { grpcDialFunc = origDialFunc }()
+
+	builder := newPluginBuilder()
+	providers := []certprovider.Provider{}
+	for i := 0; i < cnt; i++ {
+		// Copy the good test config and modify the serverURI to make sure that
+		// a new provider is created for the config.
+		cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
+		cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = fmt.Sprintf("test-mesh-ca:%d", i)
+		inputConfig := makeJSONConfig(t, cfg)
+		stableConfig, err := builder.ParseConfig(inputConfig)
+		if err != nil {
+			t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
+		}
+
+		p := builder.Build(stableConfig, certprovider.Options{})
+		if p == nil {
+			t.Fatalf("builder.Build(%s) failed: %v", string(stableConfig.Canonical()), err)
+		}
+		providers = append(providers, p)
+	}
+
+	// Make sure two ClientConns are created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for i := 0; i < cnt; i++ {
+		if _, err := ccChan.Receive(ctx); err != nil {
+			t.Fatalf("Failed to create ClientConn: %v", err)
+		}
+	}
+
+	// Close the first provider, and attempt to read key material from the
+	// second provider. The call to read key material should timeout, but it
+	// should not return certprovider.errProviderClosed.
+	providers[0].Close()
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer cancel()
+	if _, err := providers[1].KeyMaterial(ctx); err != context.DeadlineExceeded {
+		t.Fatalf("provider.KeyMaterial(ctx) = %v, want contextDeadlineExceeded", err)
+	}
+
+	// Close the second provider to make sure that the leakchecker is happy.
+	providers[1].Close()
+}
diff --git a/credentials/tls/certprovider/meshca/config.go b/credentials/tls/certprovider/meshca/config.go
new file mode 100644
index 000000000000..38186fa84477
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/config.go
@@ -0,0 +1,281 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package meshca
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httputil"
+	"path"
+	"strings"
+	"time"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/ptypes"
+
+	"google.golang.org/grpc/credentials/sts"
+	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
+)
+
+const (
+	// GKE metadata server endpoint.
+	mdsBaseURI        = "http://metadata.google.internal/"
+	mdsRequestTimeout = 5 * time.Second
+
+	// The following are default values used in the interaction with MeshCA.
+	defaultMeshCaEndpoint = "meshca.googleapis.com"
+	defaultCallTimeout    = 10 * time.Second
+	defaultCertLifetime   = 24 * time.Hour
+	defaultCertGraceTime  = 12 * time.Hour
+	defaultKeyTypeRSA     = "RSA"
+	defaultKeySize        = 2048
+
+	// The following are default values used in the interaction with STS or
+	// Secure Token Service, which is used to exchange the JWT token for an
+	// access token.
+	defaultSTSEndpoint        = "securetoken.googleapis.com"
+	defaultCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"
+	defaultRequestedTokenType = "urn:ietf:params:oauth:token-type:access_token"
+	defaultSubjectTokenType   = "urn:ietf:params:oauth:token-type:jwt"
+)
+
+// For overriding in unit tests.
+var (
+	makeHTTPDoer     = makeHTTPClient
+	readZoneFunc     = readZone
+	readAudienceFunc = readAudience
+)
+
+// Implements the certprovider.StableConfig interface.
+type pluginConfig struct {
+	serverURI     string
+	stsOpts       sts.Options
+	callTimeout   time.Duration
+	certLifetime  time.Duration
+	certGraceTime time.Duration
+	keyType       string
+	keySize       int
+	location      string
+}
+
+// pluginConfigFromJSON parses the provided config in JSON.
+//
+// For certain values missing in the config, we use default values defined at
+// the top of this file.
+//
+// If the location field or STS audience field is missing, we try talking to the
+// GKE Metadata server and try to infer these values. If this attempt does not
+// succeed, we let those fields have empty values.
+func pluginConfigFromJSON(data json.RawMessage) (*pluginConfig, error) {
+	cfgProto := &configpb.GoogleMeshCaConfig{}
+	m := jsonpb.Unmarshaler{AllowUnknownFields: true}
+	if err := m.Unmarshal(bytes.NewReader(data), cfgProto); err != nil {
+		return nil, fmt.Errorf("meshca: failed to unmarshal config: %v", err)
+	}
+
+	if api := cfgProto.GetServer().GetApiType(); api != v3corepb.ApiConfigSource_GRPC {
+		return nil, fmt.Errorf("meshca: server has apiType %s, want %s", api, v3corepb.ApiConfigSource_GRPC)
+	}
+
+	pc := &pluginConfig{}
+	gs := cfgProto.GetServer().GetGrpcServices()
+	if l := len(gs); l != 1 {
+		return nil, fmt.Errorf("meshca: number of gRPC services in config is %d, expected 1", l)
+	}
+	grpcService := gs[0]
+	googGRPC := grpcService.GetGoogleGrpc()
+	if googGRPC == nil {
+		return nil, errors.New("meshca: missing google gRPC service in config")
+	}
+	pc.serverURI = googGRPC.GetTargetUri()
+	if pc.serverURI == "" {
+		pc.serverURI = defaultMeshCaEndpoint
+	}
+
+	callCreds := googGRPC.GetCallCredentials()
+	if len(callCreds) == 0 {
+		return nil, errors.New("meshca: missing call credentials in config")
+	}
+	var stsCallCreds *v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService
+	for _, cc := range callCreds {
+		if stsCallCreds = cc.GetStsService(); stsCallCreds != nil {
+			break
+		}
+	}
+	if stsCallCreds == nil {
+		return nil, errors.New("meshca: missing STS call credentials in config")
+	}
+	if stsCallCreds.GetSubjectTokenPath() == "" {
+		return nil, errors.New("meshca: missing subjectTokenPath in STS call credentials config")
+	}
+	pc.stsOpts = makeStsOptsWithDefaults(stsCallCreds)
+
+	var err error
+	if pc.callTimeout, err = ptypes.Duration(grpcService.GetTimeout()); err != nil {
+		pc.callTimeout = defaultCallTimeout
+	}
+	if pc.certLifetime, err = ptypes.Duration(cfgProto.GetCertificateLifetime()); err != nil {
+		pc.certLifetime = defaultCertLifetime
+	}
+	if pc.certGraceTime, err = ptypes.Duration(cfgProto.GetRenewalGracePeriod()); err != nil {
+		pc.certGraceTime = defaultCertGraceTime
+	}
+	switch cfgProto.GetKeyType() {
+	case configpb.GoogleMeshCaConfig_KEY_TYPE_UNKNOWN, configpb.GoogleMeshCaConfig_KEY_TYPE_RSA:
+		pc.keyType = defaultKeyTypeRSA
+	default:
+		return nil, fmt.Errorf("meshca: unsupported key type: %s, only support RSA keys", pc.keyType)
+	}
+	pc.keySize = int(cfgProto.GetKeySize())
+	if pc.keySize == 0 {
+		pc.keySize = defaultKeySize
+	}
+	pc.location = cfgProto.GetLocation()
+	if pc.location == "" {
+		pc.location = readZoneFunc(makeHTTPDoer())
+	}
+
+	return pc, nil
+}
+
+func (pc *pluginConfig) Canonical() []byte {
+	return []byte(fmt.Sprintf("%s:%s:%s:%s:%s:%s:%d:%s", pc.serverURI, pc.stsOpts, pc.callTimeout, pc.certLifetime, pc.certGraceTime, pc.keyType, pc.keySize, pc.location))
+}
+
+func makeStsOptsWithDefaults(stsCallCreds *v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService) sts.Options {
+	opts := sts.Options{
+		TokenExchangeServiceURI: stsCallCreds.GetTokenExchangeServiceUri(),
+		Resource:                stsCallCreds.GetResource(),
+		Audience:                stsCallCreds.GetAudience(),
+		Scope:                   stsCallCreds.GetScope(),
+		RequestedTokenType:      stsCallCreds.GetRequestedTokenType(),
+		SubjectTokenPath:        stsCallCreds.GetSubjectTokenPath(),
+		SubjectTokenType:        stsCallCreds.GetSubjectTokenType(),
+		ActorTokenPath:          stsCallCreds.GetActorTokenPath(),
+		ActorTokenType:          stsCallCreds.GetActorTokenType(),
+	}
+
+	// Use sane defaults for unspecified fields.
+	if opts.TokenExchangeServiceURI == "" {
+		opts.TokenExchangeServiceURI = defaultSTSEndpoint
+	}
+	if opts.Audience == "" {
+		opts.Audience = readAudienceFunc(makeHTTPDoer())
+	}
+	if opts.Scope == "" {
+		opts.Scope = defaultCloudPlatformScope
+	}
+	if opts.RequestedTokenType == "" {
+		opts.RequestedTokenType = defaultRequestedTokenType
+	}
+	if opts.SubjectTokenType == "" {
+		opts.SubjectTokenType = defaultSubjectTokenType
+	}
+	return opts
+}
+
+// httpDoer wraps the single method on the http.Client type that we use. This
+// helps with overriding in unit tests.
+type httpDoer interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+func makeHTTPClient() httpDoer {
+	return &http.Client{Timeout: mdsRequestTimeout}
+}
+
+func readMetadata(client httpDoer, uriPath string) (string, error) {
+	req, err := http.NewRequest("GET", mdsBaseURI+uriPath, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Metadata-Flavor", "Google")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		dump, err := httputil.DumpRequestOut(req, false)
+		if err != nil {
+			logger.Warningf("Failed to dump HTTP request: %v", err)
+		}
+		logger.Warningf("Request %q returned status %v", dump, resp.StatusCode)
+	}
+	return string(body), err
+}
+
+func readZone(client httpDoer) string {
+	zoneURI := "computeMetadata/v1/instance/zone"
+	data, err := readMetadata(client, zoneURI)
+	if err != nil {
+		logger.Warningf("GET %s failed: %v", path.Join(mdsBaseURI, zoneURI), err)
+		return ""
+	}
+
+	// The output returned by the metadata server looks like this:
+	// projects/<PROJECT-NUMBER>/zones/<ZONE>
+	parts := strings.Split(data, "/")
+	if len(parts) == 0 {
+		logger.Warningf("GET %s returned {%s}, does not match expected format {projects/<PROJECT-NUMBER>/zones/<ZONE>}", path.Join(mdsBaseURI, zoneURI))
+		return ""
+	}
+	return parts[len(parts)-1]
+}
+
+// readAudience constructs the audience field to be used in the STS request, if
+// it is not specified in the plugin configuration.
+//
+// "identitynamespace:{TRUST_DOMAIN}:{GKE_CLUSTER_URL}" is the format of the
+// audience field. When workload identity is enabled on a GCP project, a default
+// trust domain is created whose value is "{PROJECT_ID}.svc.id.goog". The format
+// of the GKE_CLUSTER_URL is:
+// https://container.googleapis.com/v1/projects/{PROJECT_ID}/zones/{ZONE}/clusters/{CLUSTER_NAME}.
+func readAudience(client httpDoer) string {
+	projURI := "computeMetadata/v1/project/project-id"
+	project, err := readMetadata(client, projURI)
+	if err != nil {
+		logger.Warningf("GET %s failed: %v", path.Join(mdsBaseURI, projURI), err)
+		return ""
+	}
+	trustDomain := fmt.Sprintf("%s.svc.id.goog", project)
+
+	clusterURI := "computeMetadata/v1/instance/attributes/cluster-name"
+	cluster, err := readMetadata(client, clusterURI)
+	if err != nil {
+		logger.Warningf("GET %s failed: %v", path.Join(mdsBaseURI, clusterURI), err)
+		return ""
+	}
+	zone := readZoneFunc(client)
+	clusterURL := fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/zones/%s/clusters/%s", project, zone, cluster)
+	audience := fmt.Sprintf("identitynamespace:%s:%s", trustDomain, clusterURL)
+	return audience
+}
diff --git a/credentials/tls/certprovider/meshca/config_test.go b/credentials/tls/certprovider/meshca/config_test.go
new file mode 100644
index 000000000000..34dd9f75f558
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/config_test.go
@@ -0,0 +1,410 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package meshca
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"github.com/golang/protobuf/jsonpb"
+	durationpb "github.com/golang/protobuf/ptypes/duration"
+	"github.com/google/go-cmp/cmp"
+
+	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+)
+
+const (
+	testProjectID  = "test-project-id"
+	testGKECluster = "test-gke-cluster"
+	testGCEZone    = "test-zone"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+var (
+	goodConfigFullySpecified = &configpb.GoogleMeshCaConfig{
+		Server: &v3corepb.ApiConfigSource{
+			ApiType: v3corepb.ApiConfigSource_GRPC,
+			GrpcServices: []*v3corepb.GrpcService{
+				{
+					TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
+						GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
+							TargetUri: "test-meshca",
+							CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
+								// This call creds should be ignored.
+								{
+									CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_AccessToken{},
+								},
+								{
+									CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService_{
+										StsService: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService{
+											TokenExchangeServiceUri: "http://test-sts",
+											Resource:                "test-resource",
+											Audience:                "test-audience",
+											Scope:                   "test-scope",
+											RequestedTokenType:      "test-requested-token-type",
+											SubjectTokenPath:        "test-subject-token-path",
+											SubjectTokenType:        "test-subject-token-type",
+											ActorTokenPath:          "test-actor-token-path",
+											ActorTokenType:          "test-actor-token-type",
+										},
+									},
+								},
+							},
+						},
+					},
+					Timeout: &durationpb.Duration{Seconds: 10}, // 10s
+				},
+			},
+		},
+		CertificateLifetime: &durationpb.Duration{Seconds: 86400}, // 1d
+		RenewalGracePeriod:  &durationpb.Duration{Seconds: 43200}, //12h
+		KeyType:             configpb.GoogleMeshCaConfig_KEY_TYPE_RSA,
+		KeySize:             uint32(2048),
+		Location:            "us-west1-b",
+	}
+	goodConfigWithDefaults = &configpb.GoogleMeshCaConfig{
+		Server: &v3corepb.ApiConfigSource{
+			ApiType: v3corepb.ApiConfigSource_GRPC,
+			GrpcServices: []*v3corepb.GrpcService{
+				{
+					TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
+						GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
+							CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
+								{
+									CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService_{
+										StsService: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService{
+											SubjectTokenPath: "test-subject-token-path",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+)
+
+// makeJSONConfig marshals the provided config proto into JSON. This makes it
+// possible for tests to specify the config in proto form, which is much easier
+// than specifying the config in JSON form.
+func makeJSONConfig(t *testing.T, cfg *configpb.GoogleMeshCaConfig) json.RawMessage {
+	t.Helper()
+
+	b := &bytes.Buffer{}
+	m := &jsonpb.Marshaler{EnumsAsInts: true}
+	if err := m.Marshal(b, cfg); err != nil {
+		t.Fatalf("jsonpb.Marshal(%+v) failed: %v", cfg, err)
+	}
+	return json.RawMessage(b.Bytes())
+}
+
+// verifyReceivedRequest reads the HTTP request received by the fake client
+// (exposed through a channel), and verifies that it matches the expected
+// request.
+func verifyReceivedRequest(fc *testutils.FakeHTTPClient, wantURI string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := fc.ReqChan.Receive(ctx)
+	if err != nil {
+		return err
+	}
+	gotReq := val.(*http.Request)
+	if gotURI := gotReq.URL.String(); gotURI != wantURI {
+		return fmt.Errorf("request contains URL %q want %q", gotURI, wantURI)
+	}
+	if got, want := gotReq.Header.Get("Metadata-Flavor"), "Google"; got != want {
+		return fmt.Errorf("request contains flavor %q want %q", got, want)
+	}
+	return nil
+}
+
+// TestParseConfigSuccessFullySpecified tests the case where the config is fully
+// specified and no defaults are required.
+func (s) TestParseConfigSuccessFullySpecified(t *testing.T) {
+	inputConfig := makeJSONConfig(t, goodConfigFullySpecified)
+	wantConfig := "test-meshca:http://test-sts:test-resource:test-audience:test-scope:test-requested-token-type:test-subject-token-path:test-subject-token-type:test-actor-token-path:test-actor-token-type:10s:24h0m0s:12h0m0s:RSA:2048:us-west1-b"
+
+	builder := newPluginBuilder()
+	gotConfig, err := builder.ParseConfig(inputConfig)
+	if err != nil {
+		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
+	}
+	if diff := cmp.Diff(wantConfig, string(gotConfig.Canonical())); diff != "" {
+		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
+	}
+}
+
+// TestParseConfigSuccessWithDefaults tests cases where the config is not fully
+// specified, and we end up using some sane defaults.
+func (s) TestParseConfigSuccessWithDefaults(t *testing.T) {
+	inputConfig := makeJSONConfig(t, goodConfigWithDefaults)
+	wantConfig := fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s",
+		"meshca.googleapis.com",      // Mesh CA Server URI.
+		"securetoken.googleapis.com", // STS Server URI.
+		"",                           // STS Resource Name.
+		"identitynamespace:test-project-id.svc.id.goog:https://container.googleapis.com/v1/projects/test-project-id/zones/test-zone/clusters/test-gke-cluster", // STS Audience.
+		"https://www.googleapis.com/auth/cloud-platform", // STS Scope.
+		"urn:ietf:params:oauth:token-type:access_token",  // STS requested token type.
+		"test-subject-token-path",                        // STS subject token path.
+		"urn:ietf:params:oauth:token-type:jwt",           // STS subject token type.
+		"",                                               // STS actor token path.
+		"",                                               // STS actor token type.
+		"10s",                                            // Call timeout.
+		"24h0m0s",                                        // Cert life time.
+		"12h0m0s",                                        // Cert grace time.
+		"RSA",                                            // Key type
+		"2048",                                           // Key size
+		"test-zone",                                      // Zone
+	)
+
+	// We expect the config parser to make four HTTP requests and receive four
+	// responses. Hence we setup the request and response channels in the fake
+	// client with appropriate buffer size.
+	fc := &testutils.FakeHTTPClient{
+		ReqChan:  testutils.NewChannelWithSize(4),
+		RespChan: testutils.NewChannelWithSize(4),
+	}
+	// Set up the responses to be delivered to the config parser by the fake
+	// client. The config parser expects responses with project_id,
+	// gke_cluster_id and gce_zone. The zone is read twice, once as part of
+	// reading the STS audience and once to get location metadata.
+	fc.RespChan.Send(&http.Response{
+		Status:     "200 OK",
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(testProjectID))),
+	})
+	fc.RespChan.Send(&http.Response{
+		Status:     "200 OK",
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(testGKECluster))),
+	})
+	fc.RespChan.Send(&http.Response{
+		Status:     "200 OK",
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("projects/%s/zones/%s", testProjectID, testGCEZone)))),
+	})
+	fc.RespChan.Send(&http.Response{
+		Status:     "200 OK",
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("projects/%s/zones/%s", testProjectID, testGCEZone)))),
+	})
+	// Override the http.Client with our fakeClient.
+	origMakeHTTPDoer := makeHTTPDoer
+	makeHTTPDoer = func() httpDoer { return fc }
+	defer func() { makeHTTPDoer = origMakeHTTPDoer }()
+
+	// Spawn a goroutine to verify the HTTP requests sent out as part of the
+	// config parsing.
+	errCh := make(chan error, 1)
+	go func() {
+		if err := verifyReceivedRequest(fc, "http://metadata.google.internal/computeMetadata/v1/project/project-id"); err != nil {
+			errCh <- err
+			return
+		}
+		if err := verifyReceivedRequest(fc, "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name"); err != nil {
+			errCh <- err
+			return
+		}
+		if err := verifyReceivedRequest(fc, "http://metadata.google.internal/computeMetadata/v1/instance/zone"); err != nil {
+			errCh <- err
+			return
+		}
+		errCh <- nil
+	}()
+
+	builder := newPluginBuilder()
+	gotConfig, err := builder.ParseConfig(inputConfig)
+	if err != nil {
+		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
+
+	}
+	if diff := cmp.Diff(wantConfig, string(gotConfig.Canonical())); diff != "" {
+		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
+	}
+
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestParseConfigFailureCases tests several invalid configs which all result in
+// config parsing failures.
+func (s) TestParseConfigFailureCases(t *testing.T) {
+	tests := []struct {
+		desc        string
+		inputConfig interface{}
+		wantErr     string
+	}{
+		{
+			desc:        "bad config type",
+			inputConfig: struct{ foo string }{foo: "bar"},
+			wantErr:     "unsupported config type",
+		},
+		{
+			desc:        "invalid JSON",
+			inputConfig: json.RawMessage(`bad bad json`),
+			wantErr:     "failed to unmarshal config",
+		},
+		{
+			desc: "bad apiType",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType: v3corepb.ApiConfigSource_REST,
+				},
+			}),
+			wantErr: "server has apiType REST, want GRPC",
+		},
+		{
+			desc: "no grpc services",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType: v3corepb.ApiConfigSource_GRPC,
+				},
+			}),
+			wantErr: "number of gRPC services in config is 0, expected 1",
+		},
+		{
+			desc: "too many grpc services",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType:      v3corepb.ApiConfigSource_GRPC,
+					GrpcServices: []*v3corepb.GrpcService{nil, nil},
+				},
+			}),
+			wantErr: "number of gRPC services in config is 2, expected 1",
+		},
+		{
+			desc: "missing google grpc service",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType: v3corepb.ApiConfigSource_GRPC,
+					GrpcServices: []*v3corepb.GrpcService{
+						{
+							TargetSpecifier: &v3corepb.GrpcService_EnvoyGrpc_{
+								EnvoyGrpc: &v3corepb.GrpcService_EnvoyGrpc{
+									ClusterName: "foo",
+								},
+							},
+						},
+					},
+				},
+			}),
+			wantErr: "missing google gRPC service in config",
+		},
+		{
+			desc: "missing call credentials",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType: v3corepb.ApiConfigSource_GRPC,
+					GrpcServices: []*v3corepb.GrpcService{
+						{
+							TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
+								GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
+									TargetUri: "foo",
+								},
+							},
+						},
+					},
+				},
+			}),
+			wantErr: "missing call credentials in config",
+		},
+		{
+			desc: "missing STS call credentials",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType: v3corepb.ApiConfigSource_GRPC,
+					GrpcServices: []*v3corepb.GrpcService{
+						{
+							TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
+								GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
+									TargetUri: "foo",
+									CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
+										{
+											CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_AccessToken{},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}),
+			wantErr: "missing STS call credentials in config",
+		},
+		{
+			desc: "with no defaults",
+			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
+				Server: &v3corepb.ApiConfigSource{
+					ApiType: v3corepb.ApiConfigSource_GRPC,
+					GrpcServices: []*v3corepb.GrpcService{
+						{
+							TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
+								GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
+									CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
+										{
+											CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService_{
+												StsService: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService{},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}),
+			wantErr: "missing subjectTokenPath in STS call credentials config",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			builder := newPluginBuilder()
+			sc, err := builder.ParseConfig(test.inputConfig)
+			if err == nil {
+				t.Fatalf("builder.ParseConfig(%q) = %v, expected to return error (%v)", test.inputConfig, string(sc.Canonical()), test.wantErr)
+
+			}
+			if !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("builder.ParseConfig(%q) = (%v), want error (%v)", test.inputConfig, err, test.wantErr)
+			}
+		})
+	}
+}
diff --git a/credentials/tls/certprovider/meshca/logging.go b/credentials/tls/certprovider/meshca/logging.go
new file mode 100644
index 000000000000..ae20059c4f72
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/logging.go
@@ -0,0 +1,36 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package meshca
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[%p] "
+
+var logger = grpclog.Component("meshca")
+
+func prefixLogger(p *providerPlugin) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}
diff --git a/credentials/tls/certprovider/meshca/plugin.go b/credentials/tls/certprovider/meshca/plugin.go
new file mode 100644
index 000000000000..b00ad5f08f6f
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/plugin.go
@@ -0,0 +1,288 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package meshca provides an implementation of the Provider interface which
+// communicates with MeshCA to get certificates signed.
+package meshca
+
+import (
+	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"time"
+
+	durationpb "github.com/golang/protobuf/ptypes/duration"
+	"github.com/google/uuid"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	meshpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/metadata"
+)
+
+// In requests sent to the MeshCA, we add a metadata header with this key and
+// the value being the GCE zone in which the workload is running in.
+const locationMetadataKey = "x-goog-request-params"
+
+// For overriding from unit tests.
+var newDistributorFunc = func() distributor { return certprovider.NewDistributor() }
+
+// distributor wraps the methods on certprovider.Distributor which are used by
+// the plugin. This is very useful in tests which need to know exactly when the
+// plugin updates its key material.
+type distributor interface {
+	KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error)
+	Set(km *certprovider.KeyMaterial, err error)
+	Stop()
+}
+
+// providerPlugin is an implementation of the certprovider.Provider interface,
+// which gets certificates signed by communicating with the MeshCA.
+type providerPlugin struct {
+	distributor // Holds the key material.
+	cancel      context.CancelFunc
+	cc          *grpc.ClientConn        // Connection to MeshCA server.
+	cfg         *pluginConfig           // Plugin configuration.
+	opts        certprovider.Options    // Key material options.
+	logger      *grpclog.PrefixLogger   // Plugin instance specific prefix.
+	backoff     func(int) time.Duration // Exponential backoff.
+	doneFunc    func()                  // Notify the builder when done.
+}
+
+// providerParams wraps params passed to the provider plugin at creation time.
+type providerParams struct {
+	// This ClientConn to the MeshCA server is owned by the builder.
+	cc       *grpc.ClientConn
+	cfg      *pluginConfig
+	opts     certprovider.Options
+	backoff  func(int) time.Duration
+	doneFunc func()
+}
+
+func newProviderPlugin(params providerParams) *providerPlugin {
+	ctx, cancel := context.WithCancel(context.Background())
+	p := &providerPlugin{
+		cancel:      cancel,
+		cc:          params.cc,
+		cfg:         params.cfg,
+		opts:        params.opts,
+		backoff:     params.backoff,
+		doneFunc:    params.doneFunc,
+		distributor: newDistributorFunc(),
+	}
+	p.logger = prefixLogger((p))
+	p.logger.Infof("plugin created")
+	go p.run(ctx)
+	return p
+}
+
+func (p *providerPlugin) Close() {
+	p.logger.Infof("plugin closed")
+	p.Stop() // Stop the embedded distributor.
+	p.cancel()
+	p.doneFunc()
+}
+
+// run is a long running goroutine which periodically sends out CSRs to the
+// MeshCA, and updates the underlying Distributor with the new key material.
+func (p *providerPlugin) run(ctx context.Context) {
+	// We need to start fetching key material right away. The next attempt will
+	// be triggered by the timer firing.
+	for {
+		certValidity, err := p.updateKeyMaterial(ctx)
+		if err != nil {
+			return
+		}
+
+		// We request a certificate with the configured validity duration (which
+		// is usually twice as much as the grace period). But the server is free
+		// to return a certificate with whatever validity time it deems right.
+		refreshAfter := p.cfg.certGraceTime
+		if refreshAfter > certValidity {
+			// The default value of cert grace time is half that of the default
+			// cert validity time. So here, when we have to use a non-default
+			// cert life time, we will set the grace time again to half that of
+			// the validity time.
+			refreshAfter = certValidity / 2
+		}
+		timer := time.NewTimer(refreshAfter)
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+		}
+	}
+}
+
+// updateKeyMaterial generates a CSR and attempts to get it signed from the
+// MeshCA. It retries with an exponential backoff till it succeeds or the
+// deadline specified in ctx expires. Once it gets the CSR signed from the
+// MeshCA, it updates the Distributor with the new key material.
+//
+// It returns the amount of time the new certificate is valid for.
+func (p *providerPlugin) updateKeyMaterial(ctx context.Context) (time.Duration, error) {
+	client := meshpb.NewMeshCertificateServiceClient(p.cc)
+	retries := 0
+	for {
+		if ctx.Err() != nil {
+			return 0, ctx.Err()
+		}
+
+		if retries != 0 {
+			bi := p.backoff(retries)
+			p.logger.Warningf("Backing off for %s before attempting the next CreateCertificate() request", bi)
+			timer := time.NewTimer(bi)
+			select {
+			case <-timer.C:
+			case <-ctx.Done():
+				return 0, ctx.Err()
+			}
+		}
+		retries++
+
+		privKey, err := rsa.GenerateKey(rand.Reader, p.cfg.keySize)
+		if err != nil {
+			p.logger.Warningf("RSA key generation failed: %v", err)
+			continue
+		}
+		// We do not set any fields in the CSR (we use an empty
+		// x509.CertificateRequest as the template) because the MeshCA discards
+		// them anyways, and uses the workload identity from the access token
+		// that we present (as part of the STS call creds).
+		csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, crypto.PrivateKey(privKey))
+		if err != nil {
+			p.logger.Warningf("CSR creation failed: %v", err)
+			continue
+		}
+		csrPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})
+
+		// Send out the CSR with a call timeout and location metadata, as
+		// specified in the plugin configuration.
+		req := &meshpb.MeshCertificateRequest{
+			RequestId: uuid.New().String(),
+			Csr:       string(csrPEM),
+			Validity:  &durationpb.Duration{Seconds: int64(p.cfg.certLifetime / time.Second)},
+		}
+		p.logger.Debugf("Sending CreateCertificate() request: %v", req)
+
+		callCtx, ctxCancel := context.WithTimeout(context.Background(), p.cfg.callTimeout)
+		callCtx = metadata.NewOutgoingContext(callCtx, metadata.Pairs(locationMetadataKey, p.cfg.location))
+		resp, err := client.CreateCertificate(callCtx, req)
+		if err != nil {
+			p.logger.Warningf("CreateCertificate request failed: %v", err)
+			ctxCancel()
+			continue
+		}
+		ctxCancel()
+
+		// The returned cert chain must contain more than one cert. Leaf cert is
+		// element '0', while root cert is element 'n', and the intermediate
+		// entries form the chain from the root to the leaf.
+		certChain := resp.GetCertChain()
+		if l := len(certChain); l <= 1 {
+			p.logger.Errorf("Received certificate chain contains %d certificates, need more than one", l)
+			continue
+		}
+
+		// We need to explicitly parse the PEM cert contents as an
+		// x509.Certificate to read the certificate validity period. We use this
+		// to decide when to refresh the cert. Even though the call to
+		// tls.X509KeyPair actually parses the PEM contents into an
+		// x509.Certificate, it does not store that in the `Leaf` field. See:
+		// https://golang.org/pkg/crypto/tls/#X509KeyPair.
+		identity, intermediates, roots, err := parseCertChain(certChain)
+		if err != nil {
+			p.logger.Errorf(err.Error())
+			continue
+		}
+		_, err = identity.Verify(x509.VerifyOptions{
+			Intermediates: intermediates,
+			Roots:         roots,
+		})
+		if err != nil {
+			p.logger.Errorf("Certificate verification failed for return certChain: %v", err)
+			continue
+		}
+
+		key := x509.MarshalPKCS1PrivateKey(privKey)
+		keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: key})
+		certPair, err := tls.X509KeyPair([]byte(certChain[0]), keyPEM)
+		if err != nil {
+			p.logger.Errorf("Failed to create x509 key pair: %v", err)
+			continue
+		}
+
+		// At this point, the received response has been deemed good.
+		retries = 0
+
+		// All certs signed by the MeshCA roll up to the same root. And treating
+		// the last element of the returned chain as the root is the only
+		// supported option to get the root certificate. So, we ignore the
+		// options specified in the call to Build(), which contain certificate
+		// name and whether the caller is interested in identity or root cert.
+		p.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{certPair}, Roots: roots}, nil)
+		return time.Until(identity.NotAfter), nil
+	}
+}
+
+// ParseCertChain parses the result returned by the MeshCA which consists of a
+// list of PEM encoded certs. The first element in the list is the leaf or
+// identity cert, while the last element is the root, and everything in between
+// form the chain of trust.
+//
+// Caller needs to make sure that certChain has at least two elements.
+func parseCertChain(certChain []string) (*x509.Certificate, *x509.CertPool, *x509.CertPool, error) {
+	identity, err := parseCert([]byte(certChain[0]))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	intermediates := x509.NewCertPool()
+	for _, cert := range certChain[1 : len(certChain)-1] {
+		i, err := parseCert([]byte(cert))
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		intermediates.AddCert(i)
+	}
+
+	roots := x509.NewCertPool()
+	root, err := parseCert([]byte(certChain[len(certChain)-1]))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	roots.AddCert(root)
+
+	return identity, intermediates, roots, nil
+}
+
+func parseCert(certPEM []byte) (*x509.Certificate, error) {
+	block, _ := pem.Decode(certPEM)
+	if block == nil {
+		return nil, fmt.Errorf("failed to decode received PEM data: %v", certPEM)
+	}
+	return x509.ParseCertificate(block.Bytes)
+}
diff --git a/credentials/tls/certprovider/meshca/plugin_test.go b/credentials/tls/certprovider/meshca/plugin_test.go
new file mode 100644
index 000000000000..48740c140c15
--- /dev/null
+++ b/credentials/tls/certprovider/meshca/plugin_test.go
@@ -0,0 +1,464 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package meshca
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
+	meshgrpc "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
+	meshpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
+	"google.golang.org/grpc/internal/testutils"
+)
+
+const (
+	// Used when waiting for something that is expected to *not* happen.
+	defaultTestShortTimeout = 10 * time.Millisecond
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestCertLife     = time.Hour
+	shortTestCertLife       = 2 * time.Second
+	maxErrCount             = 2
+)
+
+// fakeCA provides a very simple fake implementation of the certificate signing
+// service as exported by the MeshCA.
+type fakeCA struct {
+	meshgrpc.UnimplementedMeshCertificateServiceServer
+
+	withErrors    bool // Whether the CA returns errors to begin with.
+	withShortLife bool // Whether to create certs with short lifetime
+
+	ccChan  *testutils.Channel // Channel to get notified about CreateCertificate calls.
+	errors  int                // Error count.
+	key     *rsa.PrivateKey    // Private key of CA.
+	cert    *x509.Certificate  // Signing certificate.
+	certPEM []byte             // PEM encoding of signing certificate.
+}
+
+// Returns a new instance of the fake Mesh CA. It generates a new RSA key and a
+// self-signed certificate which will be used to sign CSRs received in incoming
+// requests.
+// withErrors controls whether the fake returns errors before succeeding, while
+// withShortLife controls whether the fake returns certs with very small
+// lifetimes (to test plugin refresh behavior). Every time a CreateCertificate()
+// call succeeds, an event is pushed on the ccChan.
+func newFakeMeshCA(ccChan *testutils.Channel, withErrors, withShortLife bool) (*fakeCA, error) {
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, fmt.Errorf("RSA key generation failed: %v", err)
+	}
+
+	now := time.Now()
+	tmpl := &x509.Certificate{
+		Subject:               pkix.Name{CommonName: "my-fake-ca"},
+		SerialNumber:          big.NewInt(10),
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.Add(time.Hour),
+		KeyUsage:              x509.KeyUsageCertSign,
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		return nil, fmt.Errorf("x509.CreateCertificate(%v) failed: %v", tmpl, err)
+	}
+	// The PEM encoding of the self-signed certificate is stored because we need
+	// to return a chain of certificates in the response, starting with the
+	// client certificate and ending in the root.
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+	cert, err := x509.ParseCertificate(certDER)
+	if err != nil {
+		return nil, fmt.Errorf("x509.ParseCertificate(%v) failed: %v", certDER, err)
+	}
+
+	return &fakeCA{
+		withErrors:    withErrors,
+		withShortLife: withShortLife,
+		ccChan:        ccChan,
+		key:           key,
+		cert:          cert,
+		certPEM:       certPEM,
+	}, nil
+}
+
+// CreateCertificate helps implement the MeshCA service.
+//
+// If the fakeMeshCA was created with `withErrors` set to true, the first
+// `maxErrCount` number of RPC return errors. Subsequent requests are signed and
+// returned without error.
+func (f *fakeCA) CreateCertificate(ctx context.Context, req *meshpb.MeshCertificateRequest) (*meshpb.MeshCertificateResponse, error) {
+	if f.withErrors {
+		if f.errors < maxErrCount {
+			f.errors++
+			return nil, errors.New("fake Mesh CA error")
+
+		}
+	}
+
+	csrPEM := []byte(req.GetCsr())
+	block, _ := pem.Decode(csrPEM)
+	if block == nil {
+		return nil, fmt.Errorf("failed to decode received CSR: %v", csrPEM)
+	}
+	csr, err := x509.ParseCertificateRequest(block.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse received CSR: %v", csrPEM)
+	}
+
+	// By default, we create certs which are valid for an hour. But if
+	// `withShortLife` is set, we create certs which are valid only for a couple
+	// of seconds.
+	now := time.Now()
+	notBefore, notAfter := now.Add(-defaultTestCertLife), now.Add(defaultTestCertLife)
+	if f.withShortLife {
+		notBefore, notAfter = now.Add(-shortTestCertLife), now.Add(shortTestCertLife)
+	}
+	tmpl := &x509.Certificate{
+		Subject:      pkix.Name{CommonName: "signed-cert"},
+		SerialNumber: big.NewInt(10),
+		NotBefore:    notBefore,
+		NotAfter:     notAfter,
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, tmpl, f.cert, csr.PublicKey, f.key)
+	if err != nil {
+		return nil, fmt.Errorf("x509.CreateCertificate(%v) failed: %v", tmpl, err)
+	}
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	// Push to ccChan to indicate that the RPC is processed.
+	f.ccChan.Send(nil)
+
+	certChain := []string{
+		string(certPEM),   // Signed certificate corresponding to CSR
+		string(f.certPEM), // Root certificate
+	}
+	return &meshpb.MeshCertificateResponse{CertChain: certChain}, nil
+}
+
+// opts wraps the options to be passed to setup.
+type opts struct {
+	// Whether the CA returns certs with short lifetime. Used to test client refresh.
+	withShortLife bool
+	// Whether the CA returns errors to begin with. Used to test client backoff.
+	withbackoff bool
+}
+
+// events wraps channels which indicate different events.
+type events struct {
+	// Pushed to when the plugin dials the MeshCA.
+	dialDone *testutils.Channel
+	// Pushed to when CreateCertifcate() succeeds on the MeshCA.
+	createCertDone *testutils.Channel
+	// Pushed to when the plugin updates the distributor with new key material.
+	keyMaterialDone *testutils.Channel
+	// Pushed to when the client backs off after a failed CreateCertificate().
+	backoffDone *testutils.Channel
+}
+
+// setup performs tasks common to all tests in this file.
+func setup(t *testing.T, o opts) (events, string, func()) {
+	t.Helper()
+
+	// Create a fake MeshCA which pushes events on the passed channel for
+	// successful RPCs.
+	createCertDone := testutils.NewChannel()
+	fs, err := newFakeMeshCA(createCertDone, o.withbackoff, o.withShortLife)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a gRPC server and register the fake MeshCA on it.
+	server := grpc.NewServer()
+	meshgrpc.RegisterMeshCertificateServiceServer(server, fs)
+
+	// Start a net.Listener on a local port, and pass it to the gRPC server
+	// created above and start serving.
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := lis.Addr().String()
+	go server.Serve(lis)
+
+	// Override the plugin's dial function and perform a blocking dial. Also
+	// push on dialDone once the dial is complete so that test can block on this
+	// event before verifying other things.
+	dialDone := testutils.NewChannel()
+	origDialFunc := grpcDialFunc
+	grpcDialFunc = func(uri string, _ ...grpc.DialOption) (*grpc.ClientConn, error) {
+		if uri != addr {
+			t.Fatalf("plugin dialing MeshCA at %s, want %s", uri, addr)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		cc, err := grpc.DialContext(ctx, uri, grpc.WithInsecure(), grpc.WithBlock())
+		if err != nil {
+			t.Fatalf("grpc.DialContext(%s) failed: %v", addr, err)
+		}
+		dialDone.Send(nil)
+		return cc, nil
+	}
+
+	// Override the plugin's newDistributorFunc and return a wrappedDistributor
+	// which allows the test to be notified whenever the plugin pushes new key
+	// material into the distributor.
+	origDistributorFunc := newDistributorFunc
+	keyMaterialDone := testutils.NewChannel()
+	d := newWrappedDistributor(keyMaterialDone)
+	newDistributorFunc = func() distributor { return d }
+
+	// Override the plugin's backoff function to perform no real backoff, but
+	// push on a channel so that the test can verifiy that backoff actually
+	// happened.
+	backoffDone := testutils.NewChannelWithSize(maxErrCount)
+	origBackoffFunc := backoffFunc
+	if o.withbackoff {
+		// Override the plugin's backoff function with this, so that we can verify
+		// that a backoff actually was triggered.
+		backoffFunc = func(v int) time.Duration {
+			backoffDone.Send(v)
+			return 0
+		}
+	}
+
+	// Return all the channels, and a cancel function to undo all the overrides.
+	e := events{
+		dialDone:        dialDone,
+		createCertDone:  createCertDone,
+		keyMaterialDone: keyMaterialDone,
+		backoffDone:     backoffDone,
+	}
+	done := func() {
+		server.Stop()
+		grpcDialFunc = origDialFunc
+		newDistributorFunc = origDistributorFunc
+		backoffFunc = origBackoffFunc
+	}
+	return e, addr, done
+}
+
+// wrappedDistributor wraps a distributor and pushes on a channel whenever new
+// key material is pushed to the distributor.
+type wrappedDistributor struct {
+	*certprovider.Distributor
+	kmChan *testutils.Channel
+}
+
+func newWrappedDistributor(kmChan *testutils.Channel) *wrappedDistributor {
+	return &wrappedDistributor{
+		kmChan:      kmChan,
+		Distributor: certprovider.NewDistributor(),
+	}
+}
+
+func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) {
+	wd.Distributor.Set(km, err)
+	wd.kmChan.Send(nil)
+}
+
+// TestCreateCertificate verifies the simple case where the MeshCA server
+// returns a good certificate.
+func (s) TestCreateCertificate(t *testing.T) {
+	e, addr, cancel := setup(t, opts{})
+	defer cancel()
+
+	// Set the MeshCA targetURI in the plugin configuration to point to our fake
+	// MeshCA.
+	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
+	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
+	inputConfig := makeJSONConfig(t, cfg)
+	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.Options{})
+	if err != nil {
+		t.Fatalf("certprovider.GetProvider(%s, %s) failed: %v", pluginName, cfg, err)
+	}
+	defer prov.Close()
+
+	// Wait till the plugin dials the MeshCA server.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := e.dialDone.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for plugin to dial MeshCA")
+	}
+
+	// Wait till the plugin makes a CreateCertificate() call.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := e.createCertDone.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for plugin to make CreateCertificate RPC")
+	}
+
+	// We don't really care about the exact key material returned here. All we
+	// care about is whether we get any key material at all, and that we don't
+	// get any errors.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = prov.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
+	}
+}
+
+// TestCreateCertificateWithBackoff verifies the case where the MeshCA server
+// returns errors initially and then returns a good certificate. The test makes
+// sure that the client backs off when the server returns errors.
+func (s) TestCreateCertificateWithBackoff(t *testing.T) {
+	e, addr, cancel := setup(t, opts{withbackoff: true})
+	defer cancel()
+
+	// Set the MeshCA targetURI in the plugin configuration to point to our fake
+	// MeshCA.
+	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
+	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
+	inputConfig := makeJSONConfig(t, cfg)
+	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.Options{})
+	if err != nil {
+		t.Fatalf("certprovider.GetProvider(%s, %s) failed: %v", pluginName, cfg, err)
+	}
+	defer prov.Close()
+
+	// Wait till the plugin dials the MeshCA server.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := e.dialDone.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for plugin to dial MeshCA")
+	}
+
+	// Making the CreateCertificateRPC involves generating the keys, creating
+	// the CSR etc which seem to take reasonable amount of time. And in this
+	// test, the first two attempts will fail. Hence we give it a reasonable
+	// deadline here.
+	ctx, cancel = context.WithTimeout(context.Background(), 3*defaultTestTimeout)
+	defer cancel()
+	if _, err := e.createCertDone.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for plugin to make CreateCertificate RPC")
+	}
+
+	// The first `maxErrCount` calls to CreateCertificate end in failure, and
+	// should lead to a backoff.
+	for i := 0; i < maxErrCount; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		if _, err := e.backoffDone.Receive(ctx); err != nil {
+			t.Fatalf("plugin failed to backoff after error from fake server: %v", err)
+		}
+	}
+
+	// We don't really care about the exact key material returned here. All we
+	// care about is whether we get any key material at all, and that we don't
+	// get any errors.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = prov.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
+	}
+}
+
+// TestCreateCertificateWithRefresh verifies the case where the MeshCA returns a
+// certificate with a really short lifetime, and makes sure that the plugin
+// refreshes the cert in time.
+func (s) TestCreateCertificateWithRefresh(t *testing.T) {
+	e, addr, cancel := setup(t, opts{withShortLife: true})
+	defer cancel()
+
+	// Set the MeshCA targetURI in the plugin configuration to point to our fake
+	// MeshCA.
+	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
+	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
+	inputConfig := makeJSONConfig(t, cfg)
+	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.Options{})
+	if err != nil {
+		t.Fatalf("certprovider.GetProvider(%s, %s) failed: %v", pluginName, cfg, err)
+	}
+	defer prov.Close()
+
+	// Wait till the plugin dials the MeshCA server.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := e.dialDone.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for plugin to dial MeshCA")
+	}
+
+	// Wait till the plugin makes a CreateCertificate() call.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := e.createCertDone.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for plugin to make CreateCertificate RPC")
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
+	}
+
+	// At this point, we have read the first key material, and since the
+	// returned key material has a really short validity period, we expect the
+	// key material to be refreshed quite soon. We drain the channel on which
+	// the event corresponding to setting of new key material is pushed. This
+	// enables us to block on the same channel, waiting for refreshed key
+	// material.
+	// Since we do not expect this call to block, it is OK to pass the
+	// background context.
+	e.keyMaterialDone.Receive(context.Background())
+
+	// Wait for the next call to CreateCertificate() to refresh the certificate
+	// returned earlier.
+	ctx, cancel = context.WithTimeout(context.Background(), 2*shortTestCertLife)
+	defer cancel()
+	if _, err := e.keyMaterialDone.Receive(ctx); err != nil {
+		t.Fatalf("CreateCertificate() RPC not made: %v", err)
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
+	}
+
+	// TODO(easwars): Remove all references to reflect.DeepEqual and use
+	// cmp.Equal instead. Currently, the later panics because x509.Certificate
+	// type defines an Equal method, but does not check for nil. This has been
+	// fixed in
+	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
+	// but this is only available starting go1.14. So, once we remove support
+	// for go1.13, we can make the switch.
+	if reflect.DeepEqual(km1, km2) {
+		t.Error("certificate refresh did not happen in the background")
+	}
+}
diff --git a/examples/go.sum b/examples/go.sum
index 1ef62013c1a0..fae9a831ae02 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -93,6 +93,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
diff --git a/go.mod b/go.mod
index 31f2b01f64e8..0bcae7362db8 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.3.3
 	github.com/google/go-cmp v0.4.0
+	github.com/google/uuid v1.1.2
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
diff --git a/go.sum b/go.sum
index be8078eace22..bab616e439f1 100644
--- a/go.sum
+++ b/go.sum
@@ -25,6 +25,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
diff --git a/internal/grpctest/tlogger.go b/internal/grpctest/tlogger.go
index a074fbfa82ac..95c3598d1d5d 100644
--- a/internal/grpctest/tlogger.go
+++ b/internal/grpctest/tlogger.go
@@ -26,7 +26,6 @@ import (
 	"regexp"
 	"runtime"
 	"strconv"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -51,6 +50,7 @@ const (
 type tLogger struct {
 	v           int
 	t           *testing.T
+	start       time.Time
 	initialized bool
 
 	m      sync.Mutex // protects errors
@@ -74,14 +74,6 @@ func getCallingPrefix(depth int) (string, error) {
 	return fmt.Sprintf("%s:%d", path.Base(file), line), nil
 }
 
-// Returns the last component of the stringified current time, which is of the
-// format "m=±<value>", where value is the monotonic clock reading formatted as
-// a decimal number of seconds.
-func getTimeSuffix() string {
-	parts := strings.Split(time.Now().String(), " ")
-	return fmt.Sprintf(" (%s)", parts[len(parts)-1])
-}
-
 // log logs the message with the specified parameters to the tLogger.
 func (g *tLogger) log(ltype logType, depth int, format string, args ...interface{}) {
 	prefix, err := getCallingPrefix(callingFrame + depth)
@@ -90,7 +82,7 @@ func (g *tLogger) log(ltype logType, depth int, format string, args ...interface
 		return
 	}
 	args = append([]interface{}{prefix}, args...)
-	args = append(args, getTimeSuffix())
+	args = append(args, fmt.Sprintf(" (t=+%s)", time.Since(g.start)))
 
 	if format == "" {
 		switch ltype {
@@ -107,7 +99,8 @@ func (g *tLogger) log(ltype logType, depth int, format string, args ...interface
 			g.t.Log(args...)
 		}
 	} else {
-		format = "%v " + format
+		// Add formatting directives for the callingPrefix and timeSuffix.
+		format = "%v " + format + "%s"
 		switch ltype {
 		case errorLog:
 			if g.expected(fmt.Sprintf(format, args...)) {
@@ -131,6 +124,7 @@ func (g *tLogger) Update(t *testing.T) {
 		g.initialized = true
 	}
 	g.t = t
+	g.start = time.Now()
 	g.m.Lock()
 	defer g.m.Unlock()
 	g.errors = map[*regexp.Regexp]int{}
diff --git a/internal/leakcheck/leakcheck.go b/internal/leakcheck/leakcheck.go
index 946c575f140f..1d4fcef994ba 100644
--- a/internal/leakcheck/leakcheck.go
+++ b/internal/leakcheck/leakcheck.go
@@ -42,6 +42,7 @@ var goroutinesToIgnore = []string{
 	"runtime_mcall",
 	"(*loggingT).flushDaemon",
 	"goroutine in C code",
+	"httputil.DumpRequestOut", // TODO: Remove this once Go1.13 support is removed. https://github.com/golang/go/issues/37669.
 }
 
 // RegisterIgnoreGoroutine appends s into the ignore goroutine list. The
diff --git a/internal/testutils/http_client.go b/internal/testutils/http_client.go
new file mode 100644
index 000000000000..9832bf305756
--- /dev/null
+++ b/internal/testutils/http_client.go
@@ -0,0 +1,63 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package testutils
+
+import (
+	"context"
+	"net/http"
+	"time"
+)
+
+// DefaultHTTPRequestTimeout is the default timeout value for the amount of time
+// this client waits for a response to be pushed on RespChan before it fails the
+// Do() call.
+const DefaultHTTPRequestTimeout = 1 * time.Second
+
+// FakeHTTPClient helps mock out HTTP calls made by the code under test. It
+// makes HTTP requests made by the code under test available through a channel,
+// and makes it possible to inject various responses.
+type FakeHTTPClient struct {
+	// ReqChan exposes the HTTP.Request made by the code under test.
+	ReqChan *Channel
+	// RespChan is a channel on which this fake client accepts responses to be
+	// sent to the code under test.
+	RespChan *Channel
+	// Err, if set, is returned by Do().
+	Err error
+	// RecvTimeout is the amount of the time this client waits for a response to
+	// be pushed on RespChan before it fails the Do() call. If this field is
+	// left unspecified, DefaultHTTPRequestTimeout is used.
+	RecvTimeout time.Duration
+}
+
+// Do pushes req on ReqChan and returns the response available on RespChan.
+func (fc *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	fc.ReqChan.Send(req)
+
+	timeout := fc.RecvTimeout
+	if timeout == 0 {
+		timeout = DefaultHTTPRequestTimeout
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	val, err := fc.RespChan.Receive(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return val.(*http.Response), fc.Err
+}
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index b1759c051067..f2ab78d92322 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -87,6 +87,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

From bb64fee312b46ebee26be43364a7a966033521b1 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 29 Sep 2020 17:53:06 -0700
Subject: [PATCH 213/481] xds: Replace a select with an if statement. (#3912)

---
 xds/internal/client/transport_helper.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
index 4f5779956d73..3ce1f8721b3b 100644
--- a/xds/internal/client/transport_helper.go
+++ b/xds/internal/client/transport_helper.go
@@ -445,10 +445,8 @@ func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStrea
 func (t *TransportHelper) ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions) {
 	retries := 0
 	for {
-		select {
-		case <-ctx.Done():
+		if ctx.Err() != nil {
 			return
-		default:
 		}
 
 		if retries != 0 {

From e55dd3e1921bfb4f88132ba2db489641406c801e Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 30 Sep 2020 09:59:26 -0700
Subject: [PATCH 214/481] cmd/protoc-gen-go-grpc: re-add proto3 'optional'
 support (#3914)

---
 cmd/protoc-gen-go-grpc/main.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index dcb2e0efe926..40b876593d82 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -34,6 +34,7 @@ import (
 	"flag"
 
 	"google.golang.org/protobuf/compiler/protogen"
+	"google.golang.org/protobuf/types/pluginpb"
 )
 
 var requireUnimplemented *bool
@@ -45,6 +46,7 @@ func main() {
 	protogen.Options{
 		ParamFunc: flags.Set,
 	}.Run(func(gen *protogen.Plugin) error {
+		gen.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL)
 		for _, f := range gen.Files {
 			if !f.Generate {
 				continue

From 2e2833c718b5e30c5b815ea1f4737f0808842646 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 30 Sep 2020 11:27:50 -0700
Subject: [PATCH 215/481] xds: Fix resource type to URL mapping for v3 client.
 (#3915)

---
 xds/internal/client/v3/client.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index f3daaecd43b5..ba9086146365 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -43,10 +43,10 @@ func init() {
 
 var (
 	resourceTypeToURL = map[xdsclient.ResourceType]string{
-		xdsclient.ListenerResource:    version.V2ListenerURL,
-		xdsclient.RouteConfigResource: version.V2RouteConfigURL,
-		xdsclient.ClusterResource:     version.V2ClusterURL,
-		xdsclient.EndpointsResource:   version.V2EndpointsURL,
+		xdsclient.ListenerResource:    version.V3ListenerURL,
+		xdsclient.RouteConfigResource: version.V3RouteConfigURL,
+		xdsclient.ClusterResource:     version.V3ClusterURL,
+		xdsclient.EndpointsResource:   version.V3EndpointsURL,
 	}
 )
 

From bebda80b05da31cb49ce4a0e13044503f228e279 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 1 Oct 2020 16:12:24 -0700
Subject: [PATCH 216/481] internal/balancer: Remove a TODO in stub balancer.
 (#3919)

---
 internal/balancer/stub/stub.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/internal/balancer/stub/stub.go b/internal/balancer/stub/stub.go
index 2ddaf5f804d2..e3757c1a50bc 100644
--- a/internal/balancer/stub/stub.go
+++ b/internal/balancer/stub/stub.go
@@ -46,10 +46,6 @@ type BalancerData struct {
 }
 
 type bal struct {
-	// TODO: Remove this once the legacy balancer API is removed. See
-	// https://github.com/grpc/grpc-go/pull/3431.
-	balancer.Balancer
-
 	bf BalancerFuncs
 	bd *BalancerData
 }

From d9063e7af3d2b947ad56a84c4fcb50851bdd30fd Mon Sep 17 00:00:00 2001
From: Gaurav Gahlot <gauravgahlot0107@gmail.com>
Date: Fri, 2 Oct 2020 21:41:08 +0530
Subject: [PATCH 217/481] standardized experimental warnings (#3917)

---
 attributes/attributes.go                      |  5 +-
 backoff.go                                    |  5 +-
 .../weightedroundrobin/weightedroundrobin.go  | 10 ++-
 clientconn.go                                 | 23 ++++-
 credentials/local/local.go                    |  5 +-
 credentials/tls.go                            |  5 +-
 dialoptions.go                                | 55 +++++++++---
 encoding/encoding.go                          | 12 ++-
 encoding/gzip/gzip.go                         |  6 +-
 grpclog/loggerv2.go                           |  5 +-
 interceptor.go                                | 12 ++-
 internal/grpclog/grpclog.go                   |  5 +-
 preloader.go                                  |  5 +-
 profiling/profiling.go                        |  5 +-
 profiling/service/service.go                  |  5 +-
 resolver/resolver.go                          |  6 +-
 rpc_util.go                                   | 85 +++++++++++++++----
 server.go                                     | 43 ++++++++--
 serviceconfig/serviceconfig.go                |  5 +-
 tap/tap.go                                    |  7 +-
 20 files changed, 251 insertions(+), 58 deletions(-)

diff --git a/attributes/attributes.go b/attributes/attributes.go
index ee5c51e6cdb0..3220d87be403 100644
--- a/attributes/attributes.go
+++ b/attributes/attributes.go
@@ -19,7 +19,10 @@
 // Package attributes defines a generic key/value store used in various gRPC
 // components.
 //
-// All APIs in this package are EXPERIMENTAL.
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package attributes
 
 import "fmt"
diff --git a/backoff.go b/backoff.go
index ff7c3ee6f482..542594f5cc51 100644
--- a/backoff.go
+++ b/backoff.go
@@ -48,7 +48,10 @@ type BackoffConfig struct {
 // here for more details:
 // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type ConnectParams struct {
 	// Backoff specifies the configuration options for connection backoff.
 	Backoff backoff.Config
diff --git a/balancer/weightedroundrobin/weightedroundrobin.go b/balancer/weightedroundrobin/weightedroundrobin.go
index d232491aef28..4b7d3bfedff2 100644
--- a/balancer/weightedroundrobin/weightedroundrobin.go
+++ b/balancer/weightedroundrobin/weightedroundrobin.go
@@ -39,7 +39,10 @@ type AddrInfo struct {
 // SetAddrInfo returns a copy of addr in which the Attributes field is updated
 // with addrInfo.
 //
-// This is an EXPERIMENTAL API.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func SetAddrInfo(addr resolver.Address, addrInfo AddrInfo) resolver.Address {
 	addr.Attributes = addr.Attributes.WithValues(attributeKey{}, addrInfo)
 	return addr
@@ -47,7 +50,10 @@ func SetAddrInfo(addr resolver.Address, addrInfo AddrInfo) resolver.Address {
 
 // GetAddrInfo returns the AddrInfo stored in the Attributes fields of addr.
 //
-// This is an EXPERIMENTAL API.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func GetAddrInfo(addr resolver.Address) AddrInfo {
 	v := addr.Attributes.Value(attributeKey{})
 	ai, _ := v.(AddrInfo)
diff --git a/clientconn.go b/clientconn.go
index ae5ce4947e2e..6097c8787e70 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -508,7 +508,11 @@ type ClientConn struct {
 
 // WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or
 // ctx expires. A true value is returned in former case and false in latter.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func (cc *ClientConn) WaitForStateChange(ctx context.Context, sourceState connectivity.State) bool {
 	ch := cc.csMgr.getNotifyChan()
 	if cc.csMgr.getState() != sourceState {
@@ -523,7 +527,11 @@ func (cc *ClientConn) WaitForStateChange(ctx context.Context, sourceState connec
 }
 
 // GetState returns the connectivity.State of ClientConn.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func (cc *ClientConn) GetState() connectivity.State {
 	return cc.csMgr.getState()
 }
@@ -773,7 +781,11 @@ func (cc *ClientConn) channelzMetric() *channelz.ChannelInternalMetric {
 }
 
 // Target returns the target string of the ClientConn.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func (cc *ClientConn) Target() string {
 	return cc.target
 }
@@ -974,7 +986,10 @@ func (cc *ClientConn) resolveNow(o resolver.ResolveNowOptions) {
 // However, if a previously unavailable network becomes available, this may be
 // used to trigger an immediate reconnect.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func (cc *ClientConn) ResetConnectBackoff() {
 	cc.mu.Lock()
 	conns := cc.conns
diff --git a/credentials/local/local.go b/credentials/local/local.go
index 23de34cf8a18..a9d446ecaa92 100644
--- a/credentials/local/local.go
+++ b/credentials/local/local.go
@@ -23,7 +23,10 @@
 // reported. If local credentials is not used in local connections
 // (local TCP or UDS), it will fail.
 //
-// This package is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package local
 
 import (
diff --git a/credentials/tls.go b/credentials/tls.go
index 48384f5050e7..8ee7124f2265 100644
--- a/credentials/tls.go
+++ b/credentials/tls.go
@@ -195,7 +195,10 @@ func NewServerTLSFromFile(certFile, keyFile string) (TransportCredentials, error
 // TLSChannelzSecurityValue defines the struct that TLS protocol should return
 // from GetSecurityValue(), containing security info like cipher and certificate used.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type TLSChannelzSecurityValue struct {
 	ChannelzSecurityValue
 	StandardName      string
diff --git a/dialoptions.go b/dialoptions.go
index decb4c5ee891..a93fcab8f341 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -82,7 +82,10 @@ type DialOption interface {
 // EmptyDialOption does not alter the dial configuration. It can be embedded in
 // another structure to build custom dial options.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type EmptyDialOption struct{}
 
 func (EmptyDialOption) apply(*dialOptions) {}
@@ -238,7 +241,10 @@ func WithServiceConfig(c <-chan ServiceConfig) DialOption {
 // using the backoff.DefaultConfig as a base, in cases where you want to
 // override only a subset of the backoff configuration.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithConnectParams(p ConnectParams) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.bs = internalbackoff.Exponential{Config: p.Backoff}
@@ -290,7 +296,10 @@ func WithBlock() DialOption {
 // the context.DeadlineExceeded error.
 // Implies WithBlock()
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithReturnConnectionError() DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.block = true
@@ -310,7 +319,10 @@ func WithInsecure() DialOption {
 // WithNoProxy returns a DialOption which disables the use of proxies for this
 // ClientConn. This is ignored if WithDialer or WithContextDialer are used.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithNoProxy() DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.withProxy = false
@@ -338,7 +350,10 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption {
 // the ClientConn.WithCreds. This should not be used together with
 // WithTransportCredentials.
 //
-// This API is experimental.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithCredentialsBundle(b credentials.Bundle) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.copts.CredsBundle = b
@@ -403,7 +418,10 @@ func WithStatsHandler(h stats.Handler) DialOption {
 // FailOnNonTempDialError only affects the initial dial, and does not do
 // anything useful unless you are also using WithBlock().
 //
-// This is an EXPERIMENTAL API.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func FailOnNonTempDialError(f bool) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.copts.FailOnNonTempDialError = f
@@ -481,7 +499,10 @@ func WithAuthority(a string) DialOption {
 // current ClientConn's parent. This function is used in nested channel creation
 // (e.g. grpclb dial).
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithChannelzParentID(id int64) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.channelzParentID = id
@@ -507,7 +528,10 @@ func WithDisableServiceConfig() DialOption {
 // 2. Resolver does not return a service config or if the resolver returns an
 //    invalid service config.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithDefaultServiceConfig(s string) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.defaultServiceConfigRawJSON = &s
@@ -523,7 +547,10 @@ func WithDefaultServiceConfig(s string) DialOption {
 // default in the future.  Until then, it may be enabled by setting the
 // environment variable "GRPC_GO_RETRY" to "on".
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithDisableRetry() DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.disableRetry = true
@@ -541,7 +568,10 @@ func WithMaxHeaderListSize(s uint32) DialOption {
 // WithDisableHealthCheck disables the LB channel health checking for all
 // SubConns of this ClientConn.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithDisableHealthCheck() DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.disableHealthCheck = true
@@ -597,7 +627,10 @@ func withResolveNowBackoff(f func(int) time.Duration) DialOption {
 // resolver.Register.  They will be matched against the scheme used for the
 // current Dial only, and will take precedence over the global registry.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func WithResolvers(rs ...resolver.Builder) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.resolvers = append(o.resolvers, rs...)
diff --git a/encoding/encoding.go b/encoding/encoding.go
index 195e8448b646..6d84f74c7d08 100644
--- a/encoding/encoding.go
+++ b/encoding/encoding.go
@@ -19,7 +19,10 @@
 // Package encoding defines the interface for the compressor and codec, and
 // functions to register and retrieve compressors and codecs.
 //
-// This package is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package encoding
 
 import (
@@ -46,10 +49,15 @@ type Compressor interface {
 	// coding header.  The result must be static; the result cannot change
 	// between calls.
 	Name() string
-	// EXPERIMENTAL: if a Compressor implements
+	// If a Compressor implements
 	// DecompressedSize(compressedBytes []byte) int, gRPC will call it
 	// to determine the size of the buffer allocated for the result of decompression.
 	// Return -1 to indicate unknown size.
+	//
+	// Experimental
+	//
+	// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+	// later release.
 }
 
 var registeredCompressor = make(map[string]Compressor)
diff --git a/encoding/gzip/gzip.go b/encoding/gzip/gzip.go
index 5f2991a3b377..ce2f15ed288f 100644
--- a/encoding/gzip/gzip.go
+++ b/encoding/gzip/gzip.go
@@ -18,7 +18,11 @@
 
 // Package gzip implements and registers the gzip compressor
 // during the initialization.
-// This package is EXPERIMENTAL.
+//
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package gzip
 
 import (
diff --git a/grpclog/loggerv2.go b/grpclog/loggerv2.go
index 8eba2d0e0eff..4ee33171e008 100644
--- a/grpclog/loggerv2.go
+++ b/grpclog/loggerv2.go
@@ -204,7 +204,10 @@ func (g *loggerT) V(l int) bool {
 // DepthLoggerV2, the below functions will be called with the appropriate stack
 // depth set for trivial functions the logger may ignore.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type DepthLoggerV2 interface {
 	LoggerV2
 	// InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Print.
diff --git a/interceptor.go b/interceptor.go
index 8b7350022ad7..d1a609e4893d 100644
--- a/interceptor.go
+++ b/interceptor.go
@@ -27,7 +27,11 @@ type UnaryInvoker func(ctx context.Context, method string, req, reply interface{
 
 // UnaryClientInterceptor intercepts the execution of a unary RPC on the client. invoker is the handler to complete the RPC
 // and it is the responsibility of the interceptor to call it.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error
 
 // Streamer is called by StreamClientInterceptor to create a ClientStream.
@@ -35,7 +39,11 @@ type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method
 
 // StreamClientInterceptor intercepts the creation of ClientStream. It may return a custom ClientStream to intercept all I/O
 // operations. streamer is the handler to create a ClientStream and it is the responsibility of the interceptor to call it.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error)
 
 // UnaryServerInfo consists of various information about a unary RPC on
diff --git a/internal/grpclog/grpclog.go b/internal/grpclog/grpclog.go
index 745a166f02cf..e6f975cbf6a8 100644
--- a/internal/grpclog/grpclog.go
+++ b/internal/grpclog/grpclog.go
@@ -110,7 +110,10 @@ type LoggerV2 interface {
 // This is a copy of the DepthLoggerV2 defined in the external grpclog package.
 // It is defined here to avoid a circular dependency.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type DepthLoggerV2 interface {
 	// InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Print.
 	InfoDepth(depth int, args ...interface{})
diff --git a/preloader.go b/preloader.go
index 76acbbcc93b9..0a1e975ad916 100644
--- a/preloader.go
+++ b/preloader.go
@@ -25,7 +25,10 @@ import (
 
 // PreparedMsg is responsible for creating a Marshalled and Compressed object.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type PreparedMsg struct {
 	// Struct for preparing msg before sending them
 	encodedData []byte
diff --git a/profiling/profiling.go b/profiling/profiling.go
index 6adcc1fac68a..7112ef2e6a42 100644
--- a/profiling/profiling.go
+++ b/profiling/profiling.go
@@ -18,7 +18,10 @@
 
 // Package profiling exposes methods to manage profiling within gRPC.
 //
-// This package and all its methods are EXPERIMENTAL.
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package profiling
 
 import (
diff --git a/profiling/service/service.go b/profiling/service/service.go
index e2ce8926d6c5..5b034372842e 100644
--- a/profiling/service/service.go
+++ b/profiling/service/service.go
@@ -21,7 +21,10 @@
 // queried by a client to remotely manage the gRPC profiling behaviour of an
 // application.
 //
-// This package and all its methods are EXPERIMENTAL.
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package service
 
 import (
diff --git a/resolver/resolver.go b/resolver/resolver.go
index 379275a2d9b4..e9fa8e33d923 100644
--- a/resolver/resolver.go
+++ b/resolver/resolver.go
@@ -93,7 +93,11 @@ const (
 )
 
 // Address represents a server the client connects to.
-// This is the EXPERIMENTAL API and may be changed or extended in the future.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type Address struct {
 	// Addr is the server address on which a connection will be established.
 	Addr string
diff --git a/rpc_util.go b/rpc_util.go
index d4870ba4a996..f0609f2a4e9e 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -198,7 +198,11 @@ func Header(md *metadata.MD) CallOption {
 
 // HeaderCallOption is a CallOption for collecting response header metadata.
 // The metadata field will be populated *after* the RPC completes.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type HeaderCallOption struct {
 	HeaderAddr *metadata.MD
 }
@@ -216,7 +220,11 @@ func Trailer(md *metadata.MD) CallOption {
 
 // TrailerCallOption is a CallOption for collecting response trailer metadata.
 // The metadata field will be populated *after* the RPC completes.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type TrailerCallOption struct {
 	TrailerAddr *metadata.MD
 }
@@ -234,7 +242,11 @@ func Peer(p *peer.Peer) CallOption {
 
 // PeerCallOption is a CallOption for collecting the identity of the remote
 // peer. The peer field will be populated *after* the RPC completes.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type PeerCallOption struct {
 	PeerAddr *peer.Peer
 }
@@ -269,7 +281,11 @@ func FailFast(failFast bool) CallOption {
 
 // FailFastCallOption is a CallOption for indicating whether an RPC should fail
 // fast or not.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type FailFastCallOption struct {
 	FailFast bool
 }
@@ -288,7 +304,11 @@ func MaxCallRecvMsgSize(bytes int) CallOption {
 
 // MaxRecvMsgSizeCallOption is a CallOption that indicates the maximum message
 // size in bytes the client can receive.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type MaxRecvMsgSizeCallOption struct {
 	MaxRecvMsgSize int
 }
@@ -307,7 +327,11 @@ func MaxCallSendMsgSize(bytes int) CallOption {
 
 // MaxSendMsgSizeCallOption is a CallOption that indicates the maximum message
 // size in bytes the client can send.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type MaxSendMsgSizeCallOption struct {
 	MaxSendMsgSize int
 }
@@ -326,7 +350,11 @@ func PerRPCCredentials(creds credentials.PerRPCCredentials) CallOption {
 
 // PerRPCCredsCallOption is a CallOption that indicates the per-RPC
 // credentials to use for the call.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type PerRPCCredsCallOption struct {
 	Creds credentials.PerRPCCredentials
 }
@@ -341,13 +369,20 @@ func (o PerRPCCredsCallOption) after(c *callInfo, attempt *csAttempt) {}
 // sending the request.  If WithCompressor is also set, UseCompressor has
 // higher priority.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func UseCompressor(name string) CallOption {
 	return CompressorCallOption{CompressorType: name}
 }
 
 // CompressorCallOption is a CallOption that indicates the compressor to use.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type CompressorCallOption struct {
 	CompressorType string
 }
@@ -380,7 +415,11 @@ func CallContentSubtype(contentSubtype string) CallOption {
 
 // ContentSubtypeCallOption is a CallOption that indicates the content-subtype
 // used for marshaling messages.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type ContentSubtypeCallOption struct {
 	ContentSubtype string
 }
@@ -404,7 +443,10 @@ func (o ContentSubtypeCallOption) after(c *callInfo, attempt *csAttempt) {}
 // This function is provided for advanced users; prefer to use only
 // CallContentSubtype to select a registered codec instead.
 //
-// This is an EXPERIMENTAL API.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func ForceCodec(codec encoding.Codec) CallOption {
 	return ForceCodecCallOption{Codec: codec}
 }
@@ -412,7 +454,10 @@ func ForceCodec(codec encoding.Codec) CallOption {
 // ForceCodecCallOption is a CallOption that indicates the codec used for
 // marshaling messages.
 //
-// This is an EXPERIMENTAL API.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type ForceCodecCallOption struct {
 	Codec encoding.Codec
 }
@@ -434,7 +479,10 @@ func CallCustomCodec(codec Codec) CallOption {
 // CustomCodecCallOption is a CallOption that indicates the codec used for
 // marshaling messages.
 //
-// This is an EXPERIMENTAL API.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type CustomCodecCallOption struct {
 	Codec Codec
 }
@@ -448,14 +496,21 @@ func (o CustomCodecCallOption) after(c *callInfo, attempt *csAttempt) {}
 // MaxRetryRPCBufferSize returns a CallOption that limits the amount of memory
 // used for buffering this RPC's requests for retry purposes.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func MaxRetryRPCBufferSize(bytes int) CallOption {
 	return MaxRetryRPCBufferSizeCallOption{bytes}
 }
 
 // MaxRetryRPCBufferSizeCallOption is a CallOption indicating the amount of
 // memory to be used for caching this RPC for retry purposes.
-// This is an EXPERIMENTAL API.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type MaxRetryRPCBufferSizeCallOption struct {
 	MaxRetryRPCBufferSize int
 }
diff --git a/server.go b/server.go
index c10a5c0a3a14..968eb598e1e5 100644
--- a/server.go
+++ b/server.go
@@ -163,7 +163,10 @@ type ServerOption interface {
 // EmptyServerOption does not alter the server configuration. It can be embedded
 // in another structure to build custom server options.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type EmptyServerOption struct{}
 
 func (EmptyServerOption) apply(*serverOptions) {}
@@ -405,7 +408,10 @@ func UnknownServiceHandler(streamHandler StreamHandler) ServerOption {
 // new connections.  If this is not set, the default is 120 seconds.  A zero or
 // negative value will result in an immediate timeout.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func ConnectionTimeout(d time.Duration) ServerOption {
 	return newFuncServerOption(func(o *serverOptions) {
 		o.connectionTimeout = d
@@ -423,7 +429,10 @@ func MaxHeaderListSize(s uint32) ServerOption {
 // HeaderTableSize returns a ServerOption that sets the size of dynamic
 // header table for stream.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func HeaderTableSize(s uint32) ServerOption {
 	return newFuncServerOption(func(o *serverOptions) {
 		o.headerTableSize = &s
@@ -435,7 +444,10 @@ func HeaderTableSize(s uint32) ServerOption {
 // zero (default) will disable workers and spawn a new goroutine for each
 // stream.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func NumStreamWorkers(numServerWorkers uint32) ServerOption {
 	// TODO: If/when this API gets stabilized (i.e. stream workers become the
 	// only way streams are processed), change the behavior of the zero value to
@@ -893,8 +905,12 @@ var _ http.Handler = (*Server)(nil)
 // Note that ServeHTTP uses Go's HTTP/2 server implementation which is totally
 // separate from grpc-go's HTTP/2 server. Performance and features may vary
 // between the two paths. ServeHTTP does not support some gRPC features
-// available through grpc-go's HTTP/2 server, and it is currently EXPERIMENTAL
-// and subject to change.
+// available through grpc-go's HTTP/2 server.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	st, err := transport.NewServerHandlerTransport(w, r, s.opts.statsHandler)
 	if err != nil {
@@ -1555,7 +1571,10 @@ type streamKey struct{}
 // NewContextWithServerTransportStream creates a new context from ctx and
 // attaches stream to it.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func NewContextWithServerTransportStream(ctx context.Context, stream ServerTransportStream) context.Context {
 	return context.WithValue(ctx, streamKey{}, stream)
 }
@@ -1567,7 +1586,10 @@ func NewContextWithServerTransportStream(ctx context.Context, stream ServerTrans
 //
 // See also NewContextWithServerTransportStream.
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
 type ServerTransportStream interface {
 	Method() string
 	SetHeader(md metadata.MD) error
@@ -1579,7 +1601,10 @@ type ServerTransportStream interface {
 // ctx. Returns nil if the given context has no stream associated with it
 // (which implies it is not an RPC invocation context).
 //
-// This API is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func ServerTransportStreamFromContext(ctx context.Context) ServerTransportStream {
 	s, _ := ctx.Value(streamKey{}).(ServerTransportStream)
 	return s
diff --git a/serviceconfig/serviceconfig.go b/serviceconfig/serviceconfig.go
index 187c304421cf..73a2f926613e 100644
--- a/serviceconfig/serviceconfig.go
+++ b/serviceconfig/serviceconfig.go
@@ -19,7 +19,10 @@
 // Package serviceconfig defines types and methods for operating on gRPC
 // service configs.
 //
-// This package is EXPERIMENTAL.
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package serviceconfig
 
 // Config represents an opaque data structure holding a service config.
diff --git a/tap/tap.go b/tap/tap.go
index 584360f681b8..caea1ebed6e3 100644
--- a/tap/tap.go
+++ b/tap/tap.go
@@ -17,7 +17,12 @@
  */
 
 // Package tap defines the function handles which are executed on the transport
-// layer of gRPC-Go and related information. Everything here is EXPERIMENTAL.
+// layer of gRPC-Go and related information.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 package tap
 
 import (

From 8fbea72764339368e5e0b545168cc7aa4102d54c Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 2 Oct 2020 11:27:35 -0700
Subject: [PATCH 218/481] cmd/protoc-gen-go-grpc: add --version flag support
 (#3921)

---
 cmd/protoc-gen-go-grpc/main.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index 40b876593d82..9ef889fa470e 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -32,14 +32,24 @@ package main
 
 import (
 	"flag"
+	"fmt"
 
 	"google.golang.org/protobuf/compiler/protogen"
 	"google.golang.org/protobuf/types/pluginpb"
 )
 
+const version = "1.0"
+
 var requireUnimplemented *bool
 
 func main() {
+	showVersion := flag.Bool("version", false, "print the version and exit")
+	flag.Parse()
+	if *showVersion {
+		fmt.Printf("protoc-gen-go-grpc %v\n", version)
+		return
+	}
+
 	var flags flag.FlagSet
 	requireUnimplemented = flags.Bool("require_unimplemented_servers", true, "set to false to match legacy behavior")
 

From d5280589eb961a1e26dab5d7ec38718ad7be736d Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 2 Oct 2020 12:31:14 -0700
Subject: [PATCH 219/481] xds/bootstrap: Use correct format for
 "certificate_providers" field. (#3922)

---
 xds/internal/client/bootstrap/bootstrap.go    | 46 +++++++++++--------
 .../client/bootstrap/bootstrap_test.go        | 29 ++++++------
 2 files changed, 41 insertions(+), 34 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 93e6d3e94cb1..51ab98dee6f5 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -118,8 +118,14 @@ type xdsServer struct {
 //      ],
 //      "server_features": [ ... ]
 //		"certificate_providers" : {
-//			"default": { default cert provider config },
-//			"foo": { config for provider foo }
+//			"default": {
+//				"plugin_name": "default-plugin-name",
+//				"config": { default plugin config in JSON }
+//			},
+//			"foo": {
+//				"plugin_name": "foo",
+//				"config": { foo plugin config in JSON }
+//			}
 //		}
 //    },
 //    "node": <JSON form of Node proto>
@@ -208,24 +214,28 @@ func NewConfig() (*Config, error) {
 			configs := make(map[string]CertProviderConfig)
 			getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
 			for instance, data := range providerInstances {
-				var providerConfigs map[string]json.RawMessage
-				if err := json.Unmarshal(data, &providerConfigs); err != nil {
+				var nameAndConfig struct {
+					PluginName string          `json:"plugin_name"`
+					Config     json.RawMessage `json:"config"`
+				}
+				if err := json.Unmarshal(data, &nameAndConfig); err != nil {
 					return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err)
 				}
-				for name, cfg := range providerConfigs {
-					parser := getBuilder(name)
-					if parser == nil {
-						// We ignore plugins that we do not know about.
-						continue
-					}
-					c, err := parser.ParseConfig(cfg)
-					if err != nil {
-						return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
-					}
-					configs[instance] = CertProviderConfig{
-						Name:   name,
-						Config: c,
-					}
+
+				name := nameAndConfig.PluginName
+				parser := getBuilder(nameAndConfig.PluginName)
+				if parser == nil {
+					// We ignore plugins that we do not know about.
+					continue
+				}
+				cfg := nameAndConfig.Config
+				c, err := parser.ParseConfig(cfg)
+				if err != nil {
+					return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
+				}
+				configs[instance] = CertProviderConfig{
+					Name:   name,
+					Config: c,
 				}
 			}
 			config.CertProviderConfigs = configs
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 353bcd9eb5bf..22665060b43e 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -564,10 +564,12 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": {
 				"unknownProviderInstance1": {
-					"foo1": "bar1"
+					"plugin_name": "foo",
+					"config": {"foo": "bar"}
 				},
 				"unknownProviderInstance2": {
-					"foo2": "bar2"
+					"plugin_name": "bar",
+					"config": {"foo": "bar"}
 				}
 			}
 		}`,
@@ -588,17 +590,12 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": {
 				"unknownProviderInstance": {
-					"foo": "bar"
-				},
-				"fakeProviderInstance": {
-					"fake-certificate-provider": {
-						"configKey": "configValue"
-					}
+					"plugin_name": "foo",
+					"config": {"foo": "bar"}
 				},
 				"fakeProviderInstanceBad": {
-					"fake-certificate-provider": {
-						"configKey": 666
-					}
+					"plugin_name": "fake-certificate-provider",
+					"config": {"configKey": 666}
 				}
 			}
 		}`,
@@ -619,12 +616,12 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": {
 				"unknownProviderInstance": {
-					"foo": "bar"
+					"plugin_name": "foo",
+					"config": {"foo": "bar"}
 				},
 				"fakeProviderInstance": {
-					"fake-certificate-provider": {
-						"configKey": "configValue"
-					}
+					"plugin_name": "fake-certificate-provider",
+					"config": {"configKey": "configValue"}
 				}
 			}
 		}`,
@@ -692,7 +689,7 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 			}
 			c, err := NewConfig()
 			if (err != nil) != test.wantErr {
-				t.Fatalf("NewConfig() returned: %v, wantErr: %v", err, test.wantErr)
+				t.Fatalf("NewConfig() returned: (%+v, %v), wantErr: %v", c.CertProviderConfigs, err, test.wantErr)
 			}
 			if test.wantErr {
 				return

From b2c5f4a808fd5de543c4e987cd85d356140ed681 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 2 Oct 2020 12:40:53 -0700
Subject: [PATCH 220/481] stalebot: send first ping after 6 days (#3923)

---
 .github/stale.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/stale.yml b/.github/stale.yml
index 04a26a08d2ac..8f69dbc4fe83 100644
--- a/.github/stale.yml
+++ b/.github/stale.yml
@@ -1,7 +1,7 @@
 # Configuration for probot-stale - https://github.com/probot/stale
 
 # Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 7
+daysUntilStale: 6
 
 # Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
 # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
@@ -29,7 +29,7 @@ staleLabel: "stale"
 # Comment to post when marking as stale. Set to `false` to disable
 markComment: >
   This issue is labeled as requiring an update from the reporter, and no update has been received
-  after 7 days.  If no update is provided in the next 7 days, this issue will be automatically closed.
+  after 6 days.  If no update is provided in the next 7 days, this issue will be automatically closed.
 
 # Comment to post when removing the stale label.
 # unmarkComment: >

From dad518ae5e80901b4275f539881d8fe929f7e7d7 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 5 Oct 2020 08:36:34 -0700
Subject: [PATCH 221/481] xds: Refactor/cleanup xds client tests. (#3920)

---
 xds/internal/client/client_cds_test.go        |  39 +-
 xds/internal/client/client_test.go            | 123 ++++--
 .../client/client_watchers_cluster_test.go    | 320 ++++++++-------
 .../client/client_watchers_endpoints_test.go  | 217 +++++------
 .../client/client_watchers_lds_test.go        | 245 ++++++------
 .../client/client_watchers_rds_test.go        | 179 ++++-----
 .../client/client_watchers_service_test.go    | 368 +++++++++---------
 7 files changed, 782 insertions(+), 709 deletions(-)

diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go
index d45baa0e823f..578d6121e214 100644
--- a/xds/internal/client/client_cds_test.go
+++ b/xds/internal/client/client_cds_test.go
@@ -32,15 +32,14 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-func (s) TestValidateCluster(t *testing.T) {
-	const (
-		clusterName = "clusterName"
-		serviceName = "service"
-	)
-	var (
-		emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false}
-	)
+const (
+	clusterName = "clusterName"
+	serviceName = "service"
+)
+
+var emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false}
 
+func (s) TestValidateCluster_Failure(t *testing.T) {
 	tests := []struct {
 		name       string
 		cluster    *v3clusterpb.Cluster
@@ -98,6 +97,23 @@ func (s) TestValidateCluster(t *testing.T) {
 			wantUpdate: emptyUpdate,
 			wantErr:    true,
 		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if update, err := validateCluster(test.cluster); err == nil {
+				t.Errorf("validateCluster(%+v) = %v, wanted error", test.cluster, update)
+			}
+		})
+	}
+}
+
+func (s) TestValidateCluster_Success(t *testing.T) {
+	tests := []struct {
+		name       string
+		cluster    *v3clusterpb.Cluster
+		wantUpdate ClusterUpdate
+	}{
 		{
 			name: "happy-case-no-service-name-no-lrs",
 			cluster: &v3clusterpb.Cluster{
@@ -156,8 +172,11 @@ func (s) TestValidateCluster(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			update, err := validateCluster(test.cluster)
-			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("validateCluster(%+v) = (%v, %v), wantErr: (%v, %v)", test.cluster, update, err, test.wantUpdate, test.wantErr)
+			if err != nil {
+				t.Errorf("validateCluster(%+v) failed: %v", test.cluster, err)
+			}
+			if !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("validateCluster(%+v) = %v, want: %v", test.cluster, update, test.wantUpdate)
 			}
 		})
 	}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index b36a5af7ee26..4a531384fff0 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -20,9 +20,13 @@ package client
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
@@ -49,7 +53,8 @@ const (
 	testEDSName = "test-eds"
 
 	defaultTestWatchExpiryTimeout = 500 * time.Millisecond
-	defaultTestTimeout            = 1 * time.Second
+	defaultTestTimeout            = 5 * time.Second
+	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
 )
 
 func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
@@ -68,24 +73,22 @@ func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
 }
 
 type testAPIClient struct {
-	r UpdateHandler
-
 	addWatches    map[ResourceType]*testutils.Channel
 	removeWatches map[ResourceType]*testutils.Channel
 }
 
-func overrideNewAPIClient() (<-chan *testAPIClient, func()) {
+func overrideNewAPIClient() (*testutils.Channel, func()) {
 	origNewAPIClient := newAPIClient
-	ch := make(chan *testAPIClient, 1)
+	ch := testutils.NewChannel()
 	newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, opts BuildOptions) (APIClient, error) {
-		ret := newTestAPIClient(opts.Parent)
-		ch <- ret
+		ret := newTestAPIClient()
+		ch.Send(ret)
 		return ret, nil
 	}
 	return ch, func() { newAPIClient = origNewAPIClient }
 }
 
-func newTestAPIClient(r UpdateHandler) *testAPIClient {
+func newTestAPIClient() *testAPIClient {
 	addWatches := map[ResourceType]*testutils.Channel{
 		ListenerResource:    testutils.NewChannel(),
 		RouteConfigResource: testutils.NewChannel(),
@@ -99,7 +102,6 @@ func newTestAPIClient(r UpdateHandler) *testAPIClient {
 		EndpointsResource:   testutils.NewChannel(),
 	}
 	return &testAPIClient{
-		r:             r,
 		addWatches:    addWatches,
 		removeWatches: removeWatches,
 	}
@@ -121,53 +123,108 @@ func (c *testAPIClient) Close() {}
 // TestWatchCallAnotherWatch covers the case where watch() is called inline by a
 // callback. It makes sure it doesn't cause a deadlock.
 func (s) TestWatchCallAnotherWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	clusterUpdateCh := testutils.NewChannel()
 	firstTime := true
-	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		// Calls another watch inline, to ensure there's deadlock.
-		c.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
+		client.WatchCluster("another-random-name", func(ClusterUpdate, error) {})
 
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); firstTime && err != nil {
+		if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); firstTime && err != nil {
 			t.Fatalf("want new watch to start, got error %v", err)
 		}
 		firstTime = false
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName: wantUpdate,
-	})
-
-	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName: wantUpdate2,
-	})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate2})
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func verifyListenerUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate ListenerUpdate) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for listener update: %v", err)
+	}
+	gotUpdate := u.(ldsUpdateErr)
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate) {
+		return fmt.Errorf("unexpected endpointsUpdate: (%v, %v), want: (%v, nil)", gotUpdate.u, gotUpdate.err, wantUpdate)
+	}
+	return nil
+}
+
+func verifyRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate RouteConfigUpdate) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for route configuration update: %v", err)
+	}
+	gotUpdate := u.(rdsUpdateErr)
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate) {
+		return fmt.Errorf("unexpected route config update: (%v, %v), want: (%v, nil)", gotUpdate.u, gotUpdate.err, wantUpdate)
+	}
+	return nil
+}
+
+func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate ServiceUpdate) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for service update: %v", err)
+	}
+	gotUpdate := u.(serviceUpdateErr)
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()) {
+		return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil)", gotUpdate.u, gotUpdate.err, wantUpdate)
+	}
+	return nil
+}
 
-	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+func verifyClusterUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate ClusterUpdate) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for cluster update: %v", err)
+	}
+	gotUpdate := u.(clusterUpdateErr)
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate) {
+		return fmt.Errorf("unexpected clusterUpdate: (%v, %v), want: (%v, nil)", gotUpdate.u, gotUpdate.err, wantUpdate)
+	}
+	return nil
+}
+
+func verifyEndpointsUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate EndpointsUpdate) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for endpoints update: %v", err)
+	}
+	gotUpdate := u.(endpointsUpdateErr)
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()) {
+		return fmt.Errorf("unexpected endpointsUpdate: (%v, %v), want: (%v, nil)", gotUpdate.u, gotUpdate.err, wantUpdate)
 	}
+	return nil
 }
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index 3c816f9c71c5..c5126754a401 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -22,6 +22,8 @@ import (
 	"context"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc/internal/testutils"
 )
 
@@ -35,63 +37,52 @@ type clusterUpdateErr struct {
 // - an update for another resource name
 // - an update is received after cancel()
 func (s) TestClusterWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	// TODO: add a timeout to this recv.
-	// Note that this won't be necessary if we finish the TODO below to call
-	// Client directly instead of v2Client.r.
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	clusterUpdateCh := testutils.NewChannel()
-	cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+	cancelWatch := client.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	// This is calling v2Client.r to send the update, but r is set to Client, so
-	// this is same as calling Client to update. The one thing this covers is
-	// that `NewXDSV2Client` is called with the right parent.
-	//
-	// TODO: in a future cleanup, this (and the same thing in other tests) can
-	// be changed call Client directly.
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName: wantUpdate,
-	})
-
-	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another update, with an extra resource for a different resource name.
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
+	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName:  wantUpdate,
 		"randomName": {},
 	})
-
-	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected clusterUpdate: %+v, %v, want channel recv timeout", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName: wantUpdate,
-	})
-
-	if u, err := clusterUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := clusterUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -99,64 +90,62 @@ func (s) TestClusterWatch(t *testing.T) {
 // TestClusterTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
-
-	v2Client := <-v2ClientCh
-
-	var clusterUpdateChs []*testutils.Channel
-	var cancelLastWatch func()
+	defer client.Close()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
+	var clusterUpdateChs []*testutils.Channel
+	var cancelLastWatch func()
 	const count = 2
 	for i := 0; i < count; i++ {
 		clusterUpdateCh := testutils.NewChannel()
 		clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh)
-		cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+		cancelLastWatch = client.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName: wantUpdate,
-	})
-
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
 	for i := 0; i < count; i++ {
-		if u, err := clusterUpdateChs[i].Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName: wantUpdate,
-	})
-
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
 	for i := 0; i < count-1; i++ {
-		if u, err := clusterUpdateChs[i].Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
-	if u, err := clusterUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := clusterUpdateChs[count-1].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -164,34 +153,37 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 // TestClusterThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
+	// Two watches for the same name.
 	var clusterUpdateChs []*testutils.Channel
 	const count = 2
-
-	// Two watches for the same name.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	for i := 0; i < count; i++ {
 		clusterUpdateCh := testutils.NewChannel()
 		clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh)
-		c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
+		client.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 			clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
@@ -199,83 +191,86 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 
 	// Third watch for a different name.
 	clusterUpdateCh2 := testutils.NewChannel()
-	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
+	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := clusterUpdateChs[i].Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
-			t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate1); err != nil {
+			t.Fatal(err)
 		}
 	}
-
-	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
 
 // TestClusterWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestClusterWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	clusterUpdateCh := testutils.NewChannel()
-	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
+	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
-
-	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another watch for the resource in cache.
 	clusterUpdateCh2 := testutils.NewChannel()
-	c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName, func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if n, err := apiClient.addWatches[ClusterResource].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh2, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := clusterUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := clusterUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -284,38 +279,38 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 // an CDS response for the request that it sends out. We want the watch callback
 // to be invoked with an error once the watchExpiryTimer fires.
 func (s) TestClusterWatchExpiryTimer(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, true))
+	client, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	clusterUpdateCh := testutils.NewChannel()
-	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	u, err := clusterUpdateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("failed to get clusterUpdate: %v", err)
+		t.Fatalf("timeout when waiting for cluster update: %v", err)
 	}
-	uu := u.(clusterUpdateErr)
-	if uu.u != (ClusterUpdate{}) {
-		t.Errorf("unexpected clusterUpdate: %v, want %v", uu.u, ClusterUpdate{})
-	}
-	if uu.err == nil {
-		t.Errorf("unexpected clusterError: <nil>, want error watcher timeout")
+	gotUpdate := u.(clusterUpdateErr)
+	if gotUpdate.err == nil || !cmp.Equal(gotUpdate.u, ClusterUpdate{}) {
+		t.Fatalf("unexpected clusterUpdate: (%v, %v), want: (ClusterUpdate{}, nil)", gotUpdate.u, gotUpdate.err)
 	}
 }
 
@@ -323,41 +318,44 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 // an CDS response for the request that it sends out. We want no error even
 // after expiry timeout.
 func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, true))
+	client, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	clusterUpdateCh := testutils.NewChannel()
-	c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName, func(u ClusterUpdate, err error) {
 		clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
+	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	})
-
-	if u, err := clusterUpdateCh.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Wait for an error, the error should never happen.
-	u, err := clusterUpdateCh.Receive(ctx)
-	if err != context.DeadlineExceeded {
-		t.Fatalf("got unexpected: %v, %v, want recv timeout", u.(clusterUpdateErr).u, u.(clusterUpdateErr).err)
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestWatchExpiryTimeout)
+	defer sCancel()
+	if u, err := clusterUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
 
@@ -368,80 +366,78 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 // - one more update without the removed resource
 //   - the callback (above) shouldn't receive any update
 func (s) TestClusterResourceRemoved(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	clusterUpdateCh1 := testutils.NewChannel()
-	c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) {
 		clusterUpdateCh1.Send(clusterUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
+
 	// Another watch for a different name.
 	clusterUpdateCh2 := testutils.NewChannel()
-	c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
+	client.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) {
 		clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ClusterResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ClusterResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
+	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
 	})
-
-	if u, err := clusterUpdateCh1.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh1, wantUpdate1); err != nil {
+		t.Fatal(err)
 	}
-
-	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 
 	// Send another update to remove resource 1.
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName + "2": wantUpdate2,
-	})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName + "2": wantUpdate2})
 
-	// watcher 1 should get an error.
+	// Watcher 1 should get an error.
 	if u, err := clusterUpdateCh1.Receive(ctx); err != nil || ErrType(u.(clusterUpdateErr).err) != ErrorTypeResourceNotFound {
 		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
 	}
 
-	// watcher 2 should get the same update again.
-	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	// Watcher 2 should get the same update again.
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 
 	// Send one more update without resource 1.
-	v2Client.r.NewClusters(map[string]ClusterUpdate{
-		testCDSName + "2": wantUpdate2,
-	})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName + "2": wantUpdate2})
 
-	// watcher 1 should get an error.
-	if u, err := clusterUpdateCh1.Receive(ctx); err != context.DeadlineExceeded {
-		t.Errorf("unexpected clusterUpdate: %v, want receiving from channel timeout", u)
+	// Watcher 1 should not see an update.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := clusterUpdateCh1.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
-	// watcher 2 should get the same update again.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := clusterUpdateCh2.Receive(ctx); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err)
+	// Watcher 2 should get the same update again.
+	if err := verifyClusterUpdate(ctx, clusterUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index c414e3a27bb4..822ee596678e 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -23,7 +23,6 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal"
@@ -44,7 +43,6 @@ var (
 			Weight:    1,
 		},
 	}
-	endpointsCmpOpts = []cmp.Option{cmp.AllowUnexported(endpointsUpdateErr{}), cmpopts.EquateEmpty()}
 )
 
 type endpointsUpdateErr struct {
@@ -57,55 +55,51 @@ type endpointsUpdateErr struct {
 // - an update for another resource name (which doesn't trigger callback)
 // - an update is received after cancel()
 func (s) TestEndpointsWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	endpointsUpdateCh := testutils.NewChannel()
-	cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+	cancelWatch := client.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
-		testCDSName: wantUpdate,
-	})
-
-	if u, err := endpointsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
-		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	if err := verifyEndpointsUpdate(ctx, endpointsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another update for a different resource name.
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
-		"randomName": {},
-	})
-
-	if u, err := endpointsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.NewEndpoints(map[string]EndpointsUpdate{"randomName": {}})
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := endpointsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
-		testCDSName: wantUpdate,
-	})
-
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := endpointsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := endpointsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -113,64 +107,64 @@ func (s) TestEndpointsWatch(t *testing.T) {
 // TestEndpointsTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
-
-	v2Client := <-v2ClientCh
-
-	var endpointsUpdateChs []*testutils.Channel
-	const count = 2
-
-	var cancelLastWatch func()
+	defer client.Close()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
+
+	const count = 2
+	var (
+		endpointsUpdateChs []*testutils.Channel
+		cancelLastWatch    func()
+	)
 	for i := 0; i < count; i++ {
 		endpointsUpdateCh := testutils.NewChannel()
 		endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh)
-		cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+		cancelLastWatch = client.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[EndpointsResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
-		testCDSName: wantUpdate,
-	})
-
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
 	for i := 0; i < count; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
-			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyEndpointsUpdate(ctx, endpointsUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
-		testCDSName: wantUpdate,
-	})
-
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
 	for i := 0; i < count-1; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
-			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyEndpointsUpdate(ctx, endpointsUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
-	if u, err := endpointsUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := endpointsUpdateChs[count-1].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -178,34 +172,37 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 // TestEndpointsThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
+	// Two watches for the same name.
 	var endpointsUpdateChs []*testutils.Channel
 	const count = 2
-
-	// Two watches for the same name.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	for i := 0; i < count; i++ {
 		endpointsUpdateCh := testutils.NewChannel()
 		endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh)
-		c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
+		client.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) {
 			endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[EndpointsResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
@@ -213,83 +210,84 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 
 	// Third watch for a different name.
 	endpointsUpdateCh2 := testutils.NewChannel()
-	c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
+	client.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
 	wantUpdate2 := EndpointsUpdate{Localities: []Locality{testLocalities[1]}}
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
+	client.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := endpointsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, endpointsCmpOpts...) {
-			t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyEndpointsUpdate(ctx, endpointsUpdateChs[i], wantUpdate1); err != nil {
+			t.Fatal(err)
 		}
 	}
-
-	if u, err := endpointsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, endpointsCmpOpts...) {
-		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyEndpointsUpdate(ctx, endpointsUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
 
 // TestEndpointsWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestEndpointsWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	endpointsUpdateCh := testutils.NewChannel()
-	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+	client.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	v2Client.r.NewEndpoints(map[string]EndpointsUpdate{
-		testCDSName: wantUpdate,
-	})
-
-	if u, err := endpointsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
-		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	if err := verifyEndpointsUpdate(ctx, endpointsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another watch for the resource in cache.
 	endpointsUpdateCh2 := testutils.NewChannel()
-	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+	client.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if n, err := apiClient.addWatches[EndpointsResource].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := endpointsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) {
-		t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyEndpointsUpdate(ctx, endpointsUpdateCh2, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := endpointsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := endpointsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -298,36 +296,37 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 // an CDS response for the request that it sends out. We want the watch callback
 // to be invoked with an error once the watchExpiryTimer fires.
 func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, true))
+	client, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	endpointsUpdateCh := testutils.NewChannel()
-	c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
+	client.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) {
 		endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err})
 	})
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[EndpointsResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[EndpointsResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	u, err := endpointsUpdateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("failed to get endpointsUpdate: %v", err)
-	}
-	uu := u.(endpointsUpdateErr)
-	if !cmp.Equal(uu.u, EndpointsUpdate{}, endpointsCmpOpts...) {
-		t.Errorf("unexpected endpointsUpdate: %v, want %v", uu.u, EndpointsUpdate{})
+		t.Fatalf("timeout when waiting for endpoints update: %v", err)
 	}
-	if uu.err == nil {
-		t.Errorf("unexpected endpointsError: <nil>, want error watcher timeout")
+	gotUpdate := u.(endpointsUpdateErr)
+	if gotUpdate.err == nil || !cmp.Equal(gotUpdate.u, EndpointsUpdate{}) {
+		t.Fatalf("unexpected endpointsUpdate: (%v, %v), want: (EndpointsUpdate{}, nil)", gotUpdate.u, gotUpdate.err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go
index b3a25be15621..e922654d45b8 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_lds_test.go
@@ -35,54 +35,52 @@ type ldsUpdateErr struct {
 // - an update for another resource name
 // - an update is received after cancel()
 func (s) TestLDSWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	ldsUpdateCh := testutils.NewChannel()
-	cancelWatch := c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+	cancelWatch := client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: wantUpdate,
-	})
-
-	if u, err := ldsUpdateCh.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another update, with an extra resource for a different resource name.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
+	client.NewListeners(map[string]ListenerUpdate{
 		testLDSName:  wantUpdate,
 		"randomName": {},
 	})
-
-	if u, err := ldsUpdateCh.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: wantUpdate,
-	})
-
-	if u, err := ldsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := ldsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -90,64 +88,65 @@ func (s) TestLDSWatch(t *testing.T) {
 // TestLDSTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
-	var ldsUpdateChs []*testutils.Channel
 	const count = 2
+	var (
+		ldsUpdateChs    []*testutils.Channel
+		cancelLastWatch func()
+	)
 
-	var cancelLastWatch func()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
-		cancelLastWatch = c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+		cancelLastWatch = client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
 	}
 
 	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: wantUpdate,
-	})
-
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
 	for i := 0; i < count; i++ {
-		if u, err := ldsUpdateChs[i].Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyListenerUpdate(ctx, ldsUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: wantUpdate,
-	})
-
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
 	for i := 0; i < count-1; i++ {
-		if u, err := ldsUpdateChs[i].Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyListenerUpdate(ctx, ldsUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
-	if u, err := ldsUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := ldsUpdateChs[count-1].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -155,34 +154,38 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 // TestLDSThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	var ldsUpdateChs []*testutils.Channel
 	const count = 2
 
 	// Two watches for the same name.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
-		c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
+		client.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
@@ -190,83 +193,84 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 
 	// Third watch for a different name.
 	ldsUpdateCh2 := testutils.NewChannel()
-	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
+	client.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := ListenerUpdate{RouteConfigName: testRDSName + "1"}
 	wantUpdate2 := ListenerUpdate{RouteConfigName: testRDSName + "2"}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
+	client.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "1": wantUpdate1,
 		testLDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := ldsUpdateChs[i].Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
-			t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyListenerUpdate(ctx, ldsUpdateChs[i], wantUpdate1); err != nil {
+			t.Fatal(err)
 		}
 	}
-
-	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
 
 // TestLDSWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestLDSWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	ldsUpdateCh := testutils.NewChannel()
-	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+	client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: wantUpdate,
-	})
-
-	if u, err := ldsUpdateCh.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another watch for the resource in cache.
 	ldsUpdateCh2 := testutils.NewChannel()
-	c.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+	client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if n, err := apiClient.addWatches[ListenerResource].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
-	// New watch should receives the update.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	// New watch should receive the update.
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh2, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := ldsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := ldsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -278,80 +282,77 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 // - one more update without the removed resource
 //   - the callback (above) shouldn't receive any update
 func (s) TestLDSResourceRemoved(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	ldsUpdateCh1 := testutils.NewChannel()
-	c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
+	client.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
 		ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	// Another watch for a different name.
 	ldsUpdateCh2 := testutils.NewChannel()
-	c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
+	client.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := ListenerUpdate{RouteConfigName: testEDSName + "1"}
 	wantUpdate2 := ListenerUpdate{RouteConfigName: testEDSName + "2"}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
+	client.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "1": wantUpdate1,
 		testLDSName + "2": wantUpdate2,
 	})
-
-	if u, err := ldsUpdateCh1.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh1, wantUpdate1); err != nil {
+		t.Fatal(err)
 	}
-
-	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 
 	// Send another update to remove resource 1.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName + "2": wantUpdate2,
-	})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName + "2": wantUpdate2})
 
-	// watcher 1 should get an error.
+	// Watcher 1 should get an error.
 	if u, err := ldsUpdateCh1.Receive(ctx); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound {
 		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
 	}
 
-	// watcher 2 should get the same update again.
-	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	// Watcher 2 should get the same update again.
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 
 	// Send one more update without resource 1.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName + "2": wantUpdate2,
-	})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName + "2": wantUpdate2})
 
-	// watcher 1 should get an error.
-	if u, err := ldsUpdateCh1.Receive(ctx); err != context.DeadlineExceeded {
+	// Watcher 1 should not see an update.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := ldsUpdateCh1.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected ListenerUpdate: %v, want receiving from channel timeout", u)
 	}
 
-	// watcher 2 should get the same update again.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := ldsUpdateCh2.Receive(ctx); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) {
-		t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err)
+	// Watcher 2 should get the same update again.
+	if err := verifyListenerUpdate(ctx, ldsUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index abaec80a4c9c..e1f23f3ea76e 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc/internal/testutils"
 )
 
@@ -36,54 +37,51 @@ type rdsUpdateErr struct {
 // - an update for another resource name (which doesn't trigger callback)
 // - an update is received after cancel()
 func (s) TestRDSWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	rdsUpdateCh := testutils.NewChannel()
-	cancelWatch := c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+	cancelWatch := client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: wantUpdate,
-	})
-
-	if u, err := rdsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
-		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another update for a different resource name.
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
-		"randomName": {},
-	})
-
-	if u, err := rdsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{"randomName": {}})
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := rdsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: wantUpdate,
-	})
-
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := rdsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := rdsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -91,64 +89,64 @@ func (s) TestRDSWatch(t *testing.T) {
 // TestRDSTwoWatchSameResourceName covers the case where an update is received
 // after two watch() for the same resource name.
 func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
-
-	v2Client := <-v2ClientCh
-
-	var rdsUpdateChs []*testutils.Channel
-	const count = 2
-
-	var cancelLastWatch func()
+	defer client.Close()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
+
+	const count = 2
+	var (
+		rdsUpdateChs    []*testutils.Channel
+		cancelLastWatch func()
+	)
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
-		cancelLastWatch = c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+		cancelLastWatch = client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
 	}
 
 	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: wantUpdate,
-	})
-
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
-			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: wantUpdate,
-	})
-
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
 	for i := 0; i < count-1; i++ {
-		if u, err := rdsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
-			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate); err != nil {
+			t.Fatal(err)
 		}
 	}
 
-	if u, err := rdsUpdateChs[count-1].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := rdsUpdateChs[count-1].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -156,34 +154,37 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 // TestRDSThreeWatchDifferentResourceName covers the case where an update is
 // received after three watch() for different resource names.
 func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
+	// Two watches for the same name.
 	var rdsUpdateChs []*testutils.Channel
 	const count = 2
-
-	// Two watches for the same name.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
-		c.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) {
+		client.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
 
 		if i == 0 {
 			// A new watch is registered on the underlying API client only for
 			// the first iteration because we are using the same resource name.
-			if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+			if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 				t.Fatalf("want new watch to start, got error %v", err)
 			}
 		}
@@ -191,82 +192,84 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 
 	// Third watch for a different name.
 	rdsUpdateCh2 := testutils.NewChannel()
-	c.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) {
+	client.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate1 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}}}
 	wantUpdate2 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName + "1": wantUpdate1,
 		testRDSName + "2": wantUpdate2,
 	})
 
 	for i := 0; i < count; i++ {
-		if u, err := rdsUpdateChs[i].Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
-			t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err)
+		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate1); err != nil {
+			t.Fatal(err)
 		}
 	}
-
-	if u, err := rdsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
-		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh2, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
 
 // TestRDSWatchAfterCache covers the case where watch is called after the update
 // is in cache.
 func (s) TestRDSWatchAfterCache(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	rdsUpdateCh := testutils.NewChannel()
-	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+	client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: wantUpdate,
-	})
-
-	if u, err := rdsUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
-		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another watch for the resource in cache.
 	rdsUpdateCh2 := testutils.NewChannel()
-	c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+	client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
-	if n, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if n, err := apiClient.addWatches[RouteConfigResource].Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err)
 	}
 
 	// New watch should receives the update.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if u, err := rdsUpdateCh2.Receive(ctx); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) {
 		t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err)
 	}
 
 	// Old watch should see nothing.
-	if u, err := rdsUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := rdsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index 5a6a401216ac..d4f69d5431ae 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -23,7 +23,6 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"google.golang.org/grpc/internal/testutils"
 )
@@ -33,47 +32,45 @@ type serviceUpdateErr struct {
 	err error
 }
 
-var serviceCmpOpts = []cmp.Option{cmp.AllowUnexported(serviceUpdateErr{}), cmpopts.EquateEmpty()}
-
 // TestServiceWatch covers the cases:
 // - an update is received after a watch()
 // - an update with routes received
 func (s) TestServiceWatch(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
-
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	wantUpdate2 := ServiceUpdate{
@@ -82,7 +79,7 @@ func (s) TestServiceWatch(t *testing.T) {
 			Action: map[string]uint32{testCDSName: 1},
 		}},
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {
 			Routes: []*Route{{
 				Prefix: newStringP(""),
@@ -90,8 +87,8 @@ func (s) TestServiceWatch(t *testing.T) {
 			}},
 		},
 	})
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
 
@@ -99,70 +96,66 @@ func (s) TestServiceWatch(t *testing.T) {
 // response, the second LDS response trigger an new RDS watch, and an update of
 // the old RDS watch doesn't trigger update to service callback.
 func (s) TestServiceWatchLDSUpdate(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
-
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another LDS update with a different RDS_name.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName + "2"},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName + "2"}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
 	// Another update for the old name.
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
-
-	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 
-	wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
 	// RDS update for the new name.
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName + "2": {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}},
 	})
-
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
 	}
 }
 
@@ -170,49 +163,48 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 // error (because only one is allowed). But the first watch still receives
 // updates.
 func (s) TestServiceWatchSecond(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
-
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
-	serviceUpdateCh2 := testutils.NewChannel()
 	// Call WatchService() again, with the same or different name.
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	serviceUpdateCh2 := testutils.NewChannel()
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh2.Send(serviceUpdateErr{u: update, err: err})
 	})
-
 	u, err := serviceUpdateCh2.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
@@ -227,18 +219,17 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 
 	// Send update again, first callback should be called, second should
 	// timeout.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
-
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
-	if u, err := serviceUpdateCh2.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := serviceUpdateCh2.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -247,25 +238,28 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 // does not respond to the requests being sent out as part of registering a
 // service update watcher. The callback will get an error.
 func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, true))
+	client, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	u, err := serviceUpdateCh.Receive(ctx)
@@ -281,37 +275,39 @@ func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
 	}
 }
 
-// TestServiceWatchEmptyRDS tests the case where the underlying v2Client
+// TestServiceWatchEmptyRDS tests the case where the underlying apiClient
 // receives an empty RDS response. The callback will get an error.
 func (s) TestServiceWatchEmptyRDS(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, true))
+	client, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{})
+
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{})
 	u, err := serviceUpdateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("failed to get serviceUpdate: %v", err)
@@ -329,36 +325,40 @@ func (s) TestServiceWatchEmptyRDS(t *testing.T) {
 // received after the client is closed, and we make sure that the registered
 // watcher callback is not invoked.
 func (s) TestServiceWatchWithClientClose(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, true))
+	client, err := New(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
+
 	// Client is closed before it receives the RDS response.
-	c.Close()
-	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	client.Close()
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
 	}
 }
@@ -367,49 +367,49 @@ func (s) TestServiceWatchWithClientClose(t *testing.T) {
 // update contains the same RDS name as the previous, the RDS watch isn't
 // canceled and restarted.
 func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
 
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Another LDS update with a the same RDS_name.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if v, err := v2Client.removeWatches[RouteConfigResource].Receive(ctx); err == nil {
-		t.Fatalf("unexpected rds watch cancel: %v", v)
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := apiClient.removeWatches[RouteConfigResource].Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("unexpected rds watch cancel")
 	}
 }
 
@@ -420,47 +420,46 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 // - one more update without the removed resource
 //   - the callback (above) shouldn't receive any update
 func (s) TestServiceResourceRemoved(t *testing.T) {
-	v2ClientCh, cleanup := overrideNewAPIClient()
+	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	c, err := New(clientOpts(testXDSServer, false))
+	client, err := New(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
-	defer c.Close()
+	defer client.Close()
 
-	v2Client := <-v2ClientCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
 
 	serviceUpdateCh := testutils.NewChannel()
-	c.WatchService(testLDSName, func(update ServiceUpdate, err error) {
+	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
 		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
 	})
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[ListenerResource].Receive(ctx); err != nil {
+	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
 	})
-
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 
 	// Remove LDS resource, should cancel the RDS watch, and trigger resource
 	// removed error.
-	v2Client.r.NewListeners(map[string]ListenerUpdate{})
-	if _, err := v2Client.removeWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{})
+	if _, err := apiClient.removeWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, got error %v", err)
 	}
 	if u, err := serviceUpdateCh.Receive(ctx); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
@@ -469,34 +468,33 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 
 	// Send RDS update for the removed LDS resource, expect no updates to
 	// callback, because RDS should be canceled.
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}},
 	})
-	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
 	}
 
 	// Add LDS resource, but not RDS resource, should
 	//  - start a new RDS watch
 	//  - timeout on service channel, because RDS cache was cleared
-	v2Client.r.NewListeners(map[string]ListenerUpdate{
-		testLDSName: {RouteConfigName: testRDSName},
-	})
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := v2Client.addWatches[RouteConfigResource].Receive(ctx); err != nil {
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
+	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
-	if u, err := serviceUpdateCh.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
 	}
 
-	v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}},
 	})
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, nil}, serviceCmpOpts...) {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err)
+	wantUpdate = ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
 	}
 }

From 9a3c02f98356c8998b36bac648f9cdd6cb185bed Mon Sep 17 00:00:00 2001
From: Sorah Fukumori <sora134@gmail.com>
Date: Wed, 7 Oct 2020 06:21:57 +0900
Subject: [PATCH 222/481] http2_client: fix reader segfault on PROTOCOL_ERRORs
 (#3926)

---
 internal/transport/http2_client.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index a12d6b89f991..e73b77a15a8f 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -1306,7 +1306,13 @@ func (t *http2Client) reader() {
 				if s != nil {
 					// use error detail to provide better err message
 					code := http2ErrConvTab[se.Code]
-					msg := t.framer.fr.ErrorDetail().Error()
+					errorDetail := t.framer.fr.ErrorDetail()
+					var msg string
+					if errorDetail != nil {
+						msg = errorDetail.Error()
+					} else {
+						msg = "received invalid frame"
+					}
 					t.closeStream(s, status.Error(code, msg), true, http2.ErrCodeProtocol, status.New(code, msg), nil, false)
 				}
 				continue

From c07366085d694019d8de557ab0ece84141193be5 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 6 Oct 2020 16:16:56 -0700
Subject: [PATCH 223/481] xds: Cleanup CDS balancer code and tests. (#3916)

---
 .../balancer/cdsbalancer/cdsbalancer.go       |  50 ++-
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 326 +++++++++++-------
 2 files changed, 229 insertions(+), 147 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 830b8b33b1b8..c7df3fb10fdc 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -72,10 +72,11 @@ type cdsBB struct{}
 // Build creates a new CDS balancer with the ClientConn.
 func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
 	b := &cdsBalancer{
-		cc:       cc,
-		bOpts:    opts,
-		updateCh: buffer.NewUnbounded(),
-		closed:   grpcsync.NewEvent(),
+		cc:          cc,
+		bOpts:       opts,
+		updateCh:    buffer.NewUnbounded(),
+		closed:      grpcsync.NewEvent(),
+		cancelWatch: func() {}, // No-op at this point.
 	}
 	b.logger = prefixLogger((b))
 	b.logger.Infof("Created")
@@ -158,22 +159,15 @@ type cdsBalancer struct {
 // run is a long-running goroutine which handles all updates from gRPC. All
 // methods which are invoked directly by gRPC or xdsClient simply push an
 // update onto a channel which is read and acted upon right here.
-//
-// 1. Good clientConn updates lead to registration of a CDS watch. Updates with
-//    error lead to cancellation of existing watch and propagation of the same
-//    error to the edsBalancer.
-// 2. SubConn updates are passthrough and are simply handed over to the
-//    underlying edsBalancer.
-// 3. Watch API updates lead to clientConn updates being invoked on the
-//    underlying edsBalancer.
-// 4. Close results in cancellation of the CDS watch and closing of the
-//    underlying edsBalancer and is the only way to exit this goroutine.
 func (b *cdsBalancer) run() {
 	for {
 		select {
 		case u := <-b.updateCh.Get():
 			b.updateCh.Load()
 			switch update := u.(type) {
+			// Good clientConn updates lead to registration of a CDS watch.
+			// Updates with error lead to cancellation of existing watch and
+			// propagation of the same error to the edsBalancer.
 			case *ccUpdate:
 				// We first handle errors, if any, and then proceed with handling
 				// the update, only if the status quo has changed.
@@ -187,9 +181,7 @@ func (b *cdsBalancer) run() {
 					// Since the cdsBalancer doesn't own the xdsClient object, we
 					// don't have to bother about closing the old client here, but
 					// we still need to cancel the watch on the old client.
-					if b.cancelWatch != nil {
-						b.cancelWatch()
-					}
+					b.cancelWatch()
 					b.client = update.client
 				}
 				if update.clusterName != "" {
@@ -201,12 +193,18 @@ func (b *cdsBalancer) run() {
 					}
 					b.clusterToWatch = update.clusterName
 				}
+
+			// SubConn updates are passthrough and are simply handed over to the
+			// underlying edsBalancer.
 			case *scUpdate:
 				if b.edsLB == nil {
 					b.logger.Errorf("xds: received scUpdate {%+v} with no edsBalancer", update)
 					break
 				}
 				b.edsLB.UpdateSubConnState(update.subConn, update.state)
+
+			// Watch API updates lead to clientConn updates being invoked on the
+			// underlying edsBalancer.
 			case *watchUpdate:
 				if err := update.err; err != nil {
 					b.logger.Warningf("Watch error from xds-client %p: %v", b.client, err)
@@ -243,11 +241,13 @@ func (b *cdsBalancer) run() {
 					b.logger.Errorf("xds: edsBalancer.UpdateClientConnState(%+v) returned error: %v", ccState, err)
 				}
 			}
+
+		// Close results in cancellation of the CDS watch and closing of the
+		// underlying edsBalancer and is the only way to exit this goroutine.
 		case <-b.closed.Done():
-			if b.cancelWatch != nil {
-				b.cancelWatch()
-				b.cancelWatch = nil
-			}
+			b.cancelWatch()
+			b.cancelWatch = func() {}
+
 			if b.edsLB != nil {
 				b.edsLB.Close()
 				b.edsLB = nil
@@ -282,11 +282,8 @@ func (b *cdsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
 	//
 	// This is not necessary today, because xds client never sends connection
 	// errors.
-
 	if fromParent && xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound {
-		if b.cancelWatch != nil {
-			b.cancelWatch()
-		}
+		b.cancelWatch()
 	}
 	if b.edsLB != nil {
 		b.edsLB.ResolverError(err)
@@ -319,7 +316,7 @@ func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) erro
 		return errBalancerClosed
 	}
 
-	b.logger.Infof("Receive update from resolver, balancer config: %+v", state.BalancerConfig)
+	b.logger.Infof("Received update from resolver, balancer config: %+v", state.BalancerConfig)
 	// The errors checked here should ideally never happen because the
 	// ServiceConfig in this case is prepared by the xdsResolver and is not
 	// something that is received on the wire.
@@ -352,7 +349,6 @@ func (b *cdsBalancer) ResolverError(err error) {
 		b.logger.Warningf("xds: received resolver error {%v} after cdsBalancer was closed", err)
 		return
 	}
-
 	b.updateCh.Put(&ccUpdate{err: err})
 }
 
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 3044826c0691..1c83845363bb 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -45,6 +45,8 @@ const (
 	clusterName        = "cluster1"
 	serviceName        = "service1"
 	defaultTestTimeout = 1 * time.Second
+	// Used when waiting for something that is expected to *not* happen.
+	defaultTestShortTimeout = 10 * time.Millisecond
 )
 
 type s struct {
@@ -64,11 +66,14 @@ type cdsWatchInfo struct {
 // invokeWatchCb invokes the CDS watch callback registered by the cdsBalancer
 // and waits for appropriate state to be pushed to the provided edsBalancer.
 func invokeWatchCbAndWait(xdsC *fakeclient.Client, cdsW cdsWatchInfo, wantCCS balancer.ClientConnState, edsB *testEDSBalancer) error {
+	// TODO(easwars): Change this func to accept a context.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	xdsC.InvokeWatchClusterCallback(cdsW.update, cdsW.err)
 	if cdsW.err != nil {
-		return edsB.waitForResolverError(cdsW.err)
+		return edsB.waitForResolverError(ctx, cdsW.err)
 	}
-	return edsB.waitForClientConnUpdate(wantCCS)
+	return edsB.waitForClientConnUpdate(ctx, wantCCS)
 }
 
 // testEDSBalancer is a fake edsBalancer used to verify different actions from
@@ -118,9 +123,7 @@ func (tb *testEDSBalancer) Close() {
 
 // waitForClientConnUpdate verifies if the testEDSBalancer receives the
 // provided ClientConnState within a reasonable amount of time.
-func (tb *testEDSBalancer) waitForClientConnUpdate(wantCCS balancer.ClientConnState) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+func (tb *testEDSBalancer) waitForClientConnUpdate(ctx context.Context, wantCCS balancer.ClientConnState) error {
 	ccs, err := tb.ccsCh.Receive(ctx)
 	if err != nil {
 		return err
@@ -133,10 +136,8 @@ func (tb *testEDSBalancer) waitForClientConnUpdate(wantCCS balancer.ClientConnSt
 }
 
 // waitForSubConnUpdate verifies if the testEDSBalancer receives the provided
-// SubConn update within a reasonable amount of time.
-func (tb *testEDSBalancer) waitForSubConnUpdate(wantSCS subConnWithState) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+// SubConn update before the context expires.
+func (tb *testEDSBalancer) waitForSubConnUpdate(ctx context.Context, wantSCS subConnWithState) error {
 	scs, err := tb.scStateCh.Receive(ctx)
 	if err != nil {
 		return err
@@ -148,11 +149,9 @@ func (tb *testEDSBalancer) waitForSubConnUpdate(wantSCS subConnWithState) error
 	return nil
 }
 
-// waitForResolverError verifies if the testEDSBalancer receives the
-// provided resolver error within a reasonable amount of time.
-func (tb *testEDSBalancer) waitForResolverError(wantErr error) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+// waitForResolverError verifies if the testEDSBalancer receives the provided
+// resolver error before the context expires.
+func (tb *testEDSBalancer) waitForResolverError(ctx context.Context, wantErr error) error {
 	gotErr, err := tb.resolverErrCh.Receive(ctx)
 	if err != nil {
 		return err
@@ -163,11 +162,9 @@ func (tb *testEDSBalancer) waitForResolverError(wantErr error) error {
 	return nil
 }
 
-// waitForClose verifies that the edsBalancer is closed with a reasonable
-// amount of time.
-func (tb *testEDSBalancer) waitForClose() error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+// waitForClose verifies that the edsBalancer is closed before the context
+// expires.
+func (tb *testEDSBalancer) waitForClose(ctx context.Context) error {
 	if _, err := tb.closeCh.Receive(ctx); err != nil {
 		return err
 	}
@@ -212,7 +209,10 @@ func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.Clie
 // setup creates a cdsBalancer and an edsBalancer (and overrides the
 // newEDSBalancer function to return it), and also returns a cleanup function.
 func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
-	builder := cdsBB{}
+	builder := balancer.Get(cdsName)
+	if builder == nil {
+		t.Fatalf("balancer.Get(%q) returned nil", cdsName)
+	}
 	tcc := xdstestutils.NewTestClientConn(t)
 	cdsB := builder.Build(tcc, balancer.BuildOptions{})
 
@@ -327,18 +327,6 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 	}
 }
 
-// TestUpdateClientConnStateAfterClose invokes the UpdateClientConnState method
-// on the cdsBalancer after close and verifies that it returns an error.
-func (s) TestUpdateClientConnStateAfterClose(t *testing.T) {
-	cdsB, _, _, cancel := setup(t)
-	defer cancel()
-	cdsB.Close()
-
-	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, fakeclient.NewClient())); err != errBalancerClosed {
-		t.Fatalf("UpdateClientConnState() after close returned %v, want %v", err, errBalancerClosed)
-	}
-}
-
 // TestUpdateClientConnStateWithSameState verifies that a ClientConnState
 // update with the same cluster and xdsClient does not cause the cdsBalancer to
 // create a new watch.
@@ -349,10 +337,12 @@ func (s) TestUpdateClientConnStateWithSameState(t *testing.T) {
 		cdsB.Close()
 	}()
 
+	// This is the same clientConn update sent in setupWithWatch().
 	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	// The above update should not result in a new watch being registered.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer ctxCancel()
 	if _, err := xdsC.WaitForWatchCluster(ctx); err != context.DeadlineExceeded {
 		t.Fatalf("waiting for WatchCluster() should have timed out, but returned error: %v", err)
@@ -398,193 +388,288 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 
 // TestHandleClusterUpdateError covers the cases that an error is returned from
 // the watcher.
-//
-// Includes error with and without a child eds balancer, and whether error is a
-// resource-not-found error.
 func (s) TestHandleClusterUpdateError(t *testing.T) {
+	// This creates a CDS balancer, pushes a ClientConnState update with a fake
+	// xdsClient, and makes sure that the CDS balancer registers a watch on the
+	// provided xdsClient.
 	xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
-	// An error before eds balancer is built. Should result in an error picker.
-	// And this is not a resource not found error, watch shouldn't be canceled.
-	err1 := errors.New("cdsBalancer resolver error 1")
-	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err1)
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	// A watch was registered above, but the watch callback has not been invoked
+	// yet. This means that the watch handler on the CDS balancer has not been
+	// invoked yet, and therefore no EDS balancer has been built so far. A
+	// resolver error at this point should result in the CDS balancer returning
+	// an error picker.
+	watcherErr := errors.New("cdsBalancer watcher error")
+	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, watcherErr)
+
+	// Since the error being pushed here is not a resource-not-found-error, the
+	// registered watch should not be cancelled.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
-		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
-	if err := edsB.waitForResolverError(err1); err == nil {
+	// The CDS balancer has not yet created an EDS balancer. So, this resolver
+	// error should not be forwarded to our fake EDS balancer.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, watcherErr); err != context.DeadlineExceeded {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
-
-	var picker balancer.Picker
+	// Make sure the CDS balancer reports an error picker.
 	timer := time.NewTimer(defaultTestTimeout)
 	select {
 	case <-timer.C:
-		t.Fatalf("failed to get picker, expect an error picker")
-	case picker = <-tcc.NewPickerCh:
-		timer.Stop()
-	}
-	if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
-		t.Fatalf("want picker to always fail, got nil")
-	}
-
+		t.Fatalf("timeout when waiting for an error picker")
+	case picker := <-tcc.NewPickerCh:
+		if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
+			t.Fatalf("CDS balancer returned a picker which is not an error picker")
+		}
+	}
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
-	// An error after eds balancer is build, eds should receive the error. This
-	// is not a resource not found error, watch shouldn't be canceled
-	err2 := errors.New("cdsBalancer resolver error 2")
-	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err2)
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	// Again push a non-resource-not-found-error through the watcher callback.
+	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, watcherErr)
+	// Make sure the registered watch is not cancelled.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
-		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
-	if err := edsB.waitForResolverError(err2); err != nil {
-		t.Fatalf("eds balancer should get error, waitForError failed: %v", err)
+	// Make sure the error is forwarded to the EDS balancer.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, watcherErr); err != nil {
+		t.Fatalf("Watch callback error is not forwarded to EDS balancer")
 	}
 
-	// A resource not found error. Watch should not be canceled because this
-	// means CDS resource is removed, and eds should receive the error.
+	// Push a resource-not-found-error this time around.
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
 	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, resourceErr)
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	// Make sure that the watch is not cancelled. This error indicates that the
+	// request cluster resource is not found. We should continue to watch it.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
-		t.Fatalf("want watch to be not canceled, watchForCancel should timeout")
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+		t.Fatal("cluster watch cancelled for a resource-not-found-error")
 	}
-	if err := edsB.waitForResolverError(resourceErr); err != nil {
-		t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err)
+	// Make sure the error is forwarded to the EDS balancer.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, resourceErr); err != nil {
+		t.Fatalf("Watch callback error is not forwarded to EDS balancer")
 	}
 }
 
-// TestResolverError verifies that resolvers errors (with type
-// resource-not-found or others) are handled correctly.
+// TestResolverError verifies the ResolverError() method in the CDS balancer.
 func (s) TestResolverError(t *testing.T) {
+	// This creates a CDS balancer, pushes a ClientConnState update with a fake
+	// xdsClient, and makes sure that the CDS balancer registers a watch on the
+	// provided xdsClient.
 	xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
-	// An error before eds balancer is built. Should result in an error picker.
-	// Not a resource not found error, watch shouldn't be canceled.
-	err1 := errors.New("cdsBalancer resolver error 1")
-	cdsB.ResolverError(err1)
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	// A watch was registered above, but the watch callback has not been invoked
+	// yet. This means that the watch handler on the CDS balancer has not been
+	// invoked yet, and therefore no EDS balancer has been built so far. A
+	// resolver error at this point should result in the CDS balancer returning
+	// an error picker.
+	resolverErr := errors.New("cdsBalancer resolver error")
+	cdsB.ResolverError(resolverErr)
+
+	// Since the error being pushed here is not a resource-not-found-error, the
+	// registered watch should not be cancelled.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
-		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
-	if err := edsB.waitForResolverError(err1); err == nil {
+	// The CDS balancer has not yet created an EDS balancer. So, this resolver
+	// error should not be forwarded to our fake EDS balancer.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, resolverErr); err != context.DeadlineExceeded {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
-
-	var picker balancer.Picker
+	// Make sure the CDS balancer reports an error picker.
 	timer := time.NewTimer(defaultTestTimeout)
 	select {
 	case <-timer.C:
-		t.Fatalf("failed to get picker, expect an error picker")
-	case picker = <-tcc.NewPickerCh:
-		timer.Stop()
-	}
-	if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
-		t.Fatalf("want picker to always fail, got nil")
-	}
-
+		t.Fatalf("timeout when waiting for an error picker")
+	case picker := <-tcc.NewPickerCh:
+		if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
+			t.Fatalf("CDS balancer returned a picker which is not an error picker")
+		}
+	}
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
-	// Not a resource not found error, watch shouldn't be canceled, and eds
-	// should receive the error.
-	err2 := errors.New("cdsBalancer resolver error 2")
-	cdsB.ResolverError(err2)
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	// Again push a non-resource-not-found-error.
+	cdsB.ResolverError(resolverErr)
+	// Make sure the registered watch is not cancelled.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err == nil {
-		t.Fatal("watch was canceled, want not canceled (timeout error)")
+	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
-	if err := edsB.waitForResolverError(err2); err != nil {
-		t.Fatalf("eds balancer should get error, waitForError failed: %v", err)
+	// Make sure the error is forwarded to the EDS balancer.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, resolverErr); err != nil {
+		t.Fatalf("ResolverError() not forwarded to EDS balancer")
 	}
 
-	// A resource not found error. Watch should be canceled, and eds should
-	// receive the error.
+	// Push a resource-not-found-error this time around.
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
 	cdsB.ResolverError(resourceErr)
+	// Make sure the registered watch is cancelled.
 	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, watchForCancel failed: %v", err)
 	}
-	if err := edsB.waitForResolverError(resourceErr); err != nil {
+	// Make sure the error is forwarded to the EDS balancer.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, resourceErr); err != nil {
 		t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err)
 	}
 }
 
-// TestUpdateSubConnState pushes a SubConn update to the cdsBalancer and
-// verifies that the update is propagated to the edsBalancer.
+// TestUpdateSubConnState verifies the UpdateSubConnState() method in the CDS
+// balancer.
 func (s) TestUpdateSubConnState(t *testing.T) {
+	// This creates a CDS balancer, pushes a ClientConnState update with a fake
+	// xdsClient, and makes sure that the CDS balancer registers a watch on the
+	// provided xdsClient.
 	xdsC, cdsB, edsB, _, cancel := setupWithWatch(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
+	// Push a subConn state change to the CDS balancer.
 	var sc balancer.SubConn
 	state := balancer.SubConnState{ConnectivityState: connectivity.Ready}
 	cdsB.UpdateSubConnState(sc, state)
-	if err := edsB.waitForSubConnUpdate(subConnWithState{sc: sc, state: state}); err != nil {
+
+	// Make sure that the update is forwarded to the EDS balancer.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForSubConnUpdate(ctx, subConnWithState{sc: sc, state: state}); err != nil {
 		t.Fatal(err)
 	}
 }
 
-// TestClose calls Close() on the cdsBalancer, and verifies that the underlying
-// edsBalancer is also closed.
+// TestClose verifies the Close() method in the the CDS balancer.
 func (s) TestClose(t *testing.T) {
+	// This creates a CDS balancer, pushes a ClientConnState update with a fake
+	// xdsClient, and makes sure that the CDS balancer registers a watch on the
+	// provided xdsClient.
 	xdsC, cdsB, edsB, _, cancel := setupWithWatch(t)
 	defer cancel()
 
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
 	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
+	// Close the CDS balancer.
 	cdsB.Close()
+
+	// Make sure that the cluster watch registered by the CDS balancer is
+	// cancelled.
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsB.waitForClose(); err != nil {
+
+	// Make sure that the underlying EDS balancer is closed.
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForClose(ctx); err != nil {
 		t.Fatal(err)
 	}
+
+	// Make sure that the UpdateClientConnState() method on the CDS balancer
+	// returns error.
+	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, fakeclient.NewClient())); err != errBalancerClosed {
+		t.Fatalf("UpdateClientConnState() after close returned %v, want %v", err, errBalancerClosed)
+	}
+
+	// Make sure that the UpdateSubConnState() method on the CDS balancer does
+	// not forward the update to the EDS balancer.
+	cdsB.UpdateSubConnState(&xdstestutils.TestSubConn{}, balancer.SubConnState{})
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForSubConnUpdate(ctx, subConnWithState{}); err != context.DeadlineExceeded {
+		t.Fatal("UpdateSubConnState() forwarded to EDS balancer after Close()")
+	}
+
+	// Make sure that the ResolverErr() method on the CDS balancer does not
+	// forward the update to the EDS balancer.
+	rErr := errors.New("cdsBalancer resolver error")
+	cdsB.ResolverError(rErr)
+	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ctxCancel()
+	if err := edsB.waitForResolverError(ctx, rErr); err != context.DeadlineExceeded {
+		t.Fatal("ResolverError() forwarded to EDS balancer after Close()")
+	}
 }
 
-// TestParseConfig exercises the config parsing functionality in the cds
-// balancer builder.
+// TestParseConfig verifies the ParseConfig() method in the CDS balancer.
 func (s) TestParseConfig(t *testing.T) {
-	bb := cdsBB{}
-	if gotName := bb.Name(); gotName != cdsName {
-		t.Fatalf("cdsBB.Name() = %v, want %v", gotName, cdsName)
+	bb := balancer.Get(cdsName)
+	if bb == nil {
+		t.Fatalf("balancer.Get(%q) returned nil", cdsName)
+	}
+	parser, ok := bb.(balancer.ConfigParser)
+	if !ok {
+		t.Fatalf("balancer %q does not implement the ConfigParser interface", cdsName)
 	}
 
 	tests := []struct {
@@ -596,7 +681,7 @@ func (s) TestParseConfig(t *testing.T) {
 		{
 			name:    "good-lb-config",
 			input:   json.RawMessage(`{"Cluster": "cluster1"}`),
-			wantCfg: &lbConfig{ClusterName: clusterName},
+			wantCfg: &lbConfig{ClusterName: "cluster1"},
 		},
 		{
 			name:    "unknown-fields-in-lb-config",
@@ -612,14 +697,15 @@ func (s) TestParseConfig(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotCfg, gotErr := bb.ParseConfig(test.input)
+			gotCfg, gotErr := parser.ParseConfig(test.input)
 			if (gotErr != nil) != test.wantErr {
-				t.Fatalf("bb.ParseConfig(%v) = %v, wantErr %v", string(test.input), gotErr, test.wantErr)
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", string(test.input), gotErr, test.wantErr)
+			}
+			if test.wantErr {
+				return
 			}
-			if !test.wantErr {
-				if !cmp.Equal(gotCfg, test.wantCfg) {
-					t.Fatalf("bb.ParseConfig(%v) = %v, want %v", string(test.input), gotCfg, test.wantCfg)
-				}
+			if !cmp.Equal(gotCfg, test.wantCfg) {
+				t.Fatalf("ParseConfig(%v) = %v, want %v", string(test.input), gotCfg, test.wantCfg)
 			}
 		})
 	}

From 7464f756aebd7057a912185e3df6d66790c6d589 Mon Sep 17 00:00:00 2001
From: George Kechagias <geo.kechagias@gmail.com>
Date: Wed, 7 Oct 2020 18:50:23 +0300
Subject: [PATCH 224/481] channelz: remove redundant case from AddTraceEvent
 and remove caps from severities (#3925)

Rename UNKNOWN and INFO severities removing caps to match that of Warning and Error severities.
Remove redundant case from AddTraceEvent switch-case.
---
 channelz/service/service_test.go | 26 +++++++++++++-------------
 clientconn.go                    | 16 ++++++++--------
 internal/channelz/funcs.go       |  4 +---
 internal/channelz/logging.go     |  4 ++--
 internal/channelz/types.go       |  8 ++++----
 resolver_conn_wrapper.go         |  2 +-
 6 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go
index 41f4f31a5b94..467ceb4f431c 100644
--- a/channelz/service/service_test.go
+++ b/channelz/service/service_test.go
@@ -470,39 +470,39 @@ func (s) TestGetChannel(t *testing.T) {
 	ids[0] = channelz.RegisterChannel(&dummyChannel{}, 0, refNames[0])
 	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 	})
 	ids[1] = channelz.RegisterChannel(&dummyChannel{}, ids[0], refNames[1])
 	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 		Parent: &channelz.TraceEventDesc{
 			Desc:     fmt.Sprintf("Nested Channel(id:%d) created", ids[1]),
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 		},
 	})
 
 	ids[2] = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[2])
 	channelz.AddTraceEvent(logger, ids[2], 0, &channelz.TraceEventDesc{
 		Desc:     "SubChannel Created",
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 		Parent: &channelz.TraceEventDesc{
 			Desc:     fmt.Sprintf("SubChannel(id:%d) created", ids[2]),
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 		},
 	})
 	ids[3] = channelz.RegisterChannel(&dummyChannel{}, ids[1], refNames[3])
 	channelz.AddTraceEvent(logger, ids[3], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 		Parent: &channelz.TraceEventDesc{
 			Desc:     fmt.Sprintf("Nested Channel(id:%d) created", ids[3]),
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 		},
 	})
 	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready),
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 	})
 	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     "Resolver returns an empty address list",
@@ -573,26 +573,26 @@ func (s) TestGetSubChannel(t *testing.T) {
 	ids[0] = channelz.RegisterChannel(&dummyChannel{}, 0, refNames[0])
 	channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{
 		Desc:     "Channel Created",
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 	})
 	ids[1] = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[1])
 	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     subchanCreated,
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 		Parent: &channelz.TraceEventDesc{
 			Desc:     fmt.Sprintf("Nested Channel(id:%d) created", ids[0]),
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 		},
 	})
 	ids[2] = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[2])
 	ids[3] = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[3])
 	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     subchanConnectivityChange,
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 	})
 	channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{
 		Desc:     subChanPickNewAddress,
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 	})
 	for _, id := range ids {
 		defer channelz.RemoveEntry(id)
diff --git a/clientconn.go b/clientconn.go
index 6097c8787e70..cbd671a8531a 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -151,10 +151,10 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 			cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target)
 			channelz.AddTraceEvent(logger, cc.channelzID, 0, &channelz.TraceEventDesc{
 				Desc:     "Channel Created",
-				Severity: channelz.CtINFO,
+				Severity: channelz.CtInfo,
 				Parent: &channelz.TraceEventDesc{
 					Desc:     fmt.Sprintf("Nested Channel(id:%d) created", cc.channelzID),
-					Severity: channelz.CtINFO,
+					Severity: channelz.CtInfo,
 				},
 			})
 		} else {
@@ -744,10 +744,10 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSub
 		ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "")
 		channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{
 			Desc:     "Subchannel Created",
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 			Parent: &channelz.TraceEventDesc{
 				Desc:     fmt.Sprintf("Subchannel(id:%d) created", ac.channelzID),
-				Severity: channelz.CtINFO,
+				Severity: channelz.CtInfo,
 			},
 		})
 	}
@@ -1033,12 +1033,12 @@ func (cc *ClientConn) Close() error {
 	if channelz.IsOn() {
 		ted := &channelz.TraceEventDesc{
 			Desc:     "Channel Deleted",
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 		}
 		if cc.dopts.channelzParentID != 0 {
 			ted.Parent = &channelz.TraceEventDesc{
 				Desc:     fmt.Sprintf("Nested channel(id:%d) deleted", cc.channelzID),
-				Severity: channelz.CtINFO,
+				Severity: channelz.CtInfo,
 			}
 		}
 		channelz.AddTraceEvent(logger, cc.channelzID, 0, ted)
@@ -1451,10 +1451,10 @@ func (ac *addrConn) tearDown(err error) {
 	if channelz.IsOn() {
 		channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{
 			Desc:     "Subchannel Deleted",
-			Severity: channelz.CtINFO,
+			Severity: channelz.CtInfo,
 			Parent: &channelz.TraceEventDesc{
 				Desc:     fmt.Sprintf("Subchanel(id:%d) deleted", ac.channelzID),
-				Severity: channelz.CtINFO,
+				Severity: channelz.CtInfo,
 			},
 		})
 		// TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to
diff --git a/internal/channelz/funcs.go b/internal/channelz/funcs.go
index 81d3dd33e62c..f7314139303e 100644
--- a/internal/channelz/funcs.go
+++ b/internal/channelz/funcs.go
@@ -297,9 +297,7 @@ type TraceEventDesc struct {
 func AddTraceEvent(l grpclog.DepthLoggerV2, id int64, depth int, desc *TraceEventDesc) {
 	for d := desc; d != nil; d = d.Parent {
 		switch d.Severity {
-		case CtUNKNOWN:
-			l.InfoDepth(depth+1, d.Desc)
-		case CtINFO:
+		case CtUnknown, CtInfo:
 			l.InfoDepth(depth+1, d.Desc)
 		case CtWarning:
 			l.WarningDepth(depth+1, d.Desc)
diff --git a/internal/channelz/logging.go b/internal/channelz/logging.go
index e94039ee20b5..b0013f9c8865 100644
--- a/internal/channelz/logging.go
+++ b/internal/channelz/logging.go
@@ -31,7 +31,7 @@ func Info(l grpclog.DepthLoggerV2, id int64, args ...interface{}) {
 	if IsOn() {
 		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     fmt.Sprint(args...),
-			Severity: CtINFO,
+			Severity: CtInfo,
 		})
 	} else {
 		l.InfoDepth(1, args...)
@@ -44,7 +44,7 @@ func Infof(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}
 	if IsOn() {
 		AddTraceEvent(l, id, 1, &TraceEventDesc{
 			Desc:     msg,
-			Severity: CtINFO,
+			Severity: CtInfo,
 		})
 	} else {
 		l.InfoDepth(1, msg)
diff --git a/internal/channelz/types.go b/internal/channelz/types.go
index 075dc7d16714..3c595d154bd3 100644
--- a/internal/channelz/types.go
+++ b/internal/channelz/types.go
@@ -672,10 +672,10 @@ func (c *channelTrace) clear() {
 type Severity int
 
 const (
-	// CtUNKNOWN indicates unknown severity of a trace event.
-	CtUNKNOWN Severity = iota
-	// CtINFO indicates info level severity of a trace event.
-	CtINFO
+	// CtUnknown indicates unknown severity of a trace event.
+	CtUnknown Severity = iota
+	// CtInfo indicates info level severity of a trace event.
+	CtInfo
 	// CtWarning indicates warning level severity of a trace event.
 	CtWarning
 	// CtError indicates error level severity of a trace event.
diff --git a/resolver_conn_wrapper.go b/resolver_conn_wrapper.go
index 265002a75e00..f2d81968f9ec 100644
--- a/resolver_conn_wrapper.go
+++ b/resolver_conn_wrapper.go
@@ -217,6 +217,6 @@ func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) {
 	}
 	channelz.AddTraceEvent(logger, ccr.cc.channelzID, 0, &channelz.TraceEventDesc{
 		Desc:     fmt.Sprintf("Resolver state updated: %+v (%v)", s, strings.Join(updates, "; ")),
-		Severity: channelz.CtINFO,
+		Severity: channelz.CtInfo,
 	})
 }

From 5af60402cd8c63e4748d711d7a2fd0e99ea3d03f Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 7 Oct 2020 10:39:58 -0700
Subject: [PATCH 225/481] lrs: add Store.Stats() to report loads for multiple
 clusters (#3905)

- unexport `perClusterStore` and it's `stats()`
- add `Store.Stats(clusterNames)` to report loads for the given clusters
  - refactor store's map to a two layer map
- move `lastLoadReportAt` from client ton the load store, because a client can now have multiple clusters, each with a different `lastLoadReportAt`
  - all tests will ignore `ReportInterval` when comparing Data
---
 .../balancergroup/balancergroup_test.go       |  22 +-
 .../balancer/edsbalancer/eds_impl_test.go     |  16 +-
 .../edsbalancer/xds_client_wrapper.go         |  19 +-
 xds/internal/balancer/lrs/balancer.go         |  13 +-
 xds/internal/balancer/lrs/balancer_test.go    |   9 +-
 xds/internal/client/load/store.go             | 157 ++++++++----
 xds/internal/client/load/store_test.go        | 225 ++++++++++++++++--
 xds/internal/client/v2/client.go              |   3 -
 xds/internal/client/v2/loadreport.go          |  88 +++----
 xds/internal/client/v3/client.go              |   3 -
 xds/internal/client/v3/loadreport.go          |  87 ++++---
 11 files changed, 462 insertions(+), 180 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index daf8f8824085..0474e7c722e8 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -70,7 +70,7 @@ func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
 	}
 }
 
-func newTestBalancerGroup(t *testing.T, loadStore *load.PerClusterStore) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
+func newTestBalancerGroup(t *testing.T, loadStore load.PerClusterReporter) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) {
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
@@ -400,8 +400,12 @@ func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
 }
 
 func (s) TestBalancerGroup_LoadReport(t *testing.T) {
-	loadStore := &load.PerClusterStore{}
-	cc, gator, bg := newTestBalancerGroup(t, loadStore)
+	loadStore := load.NewStore()
+	const (
+		testCluster    = "test-cluster"
+		testEDSService = "test-eds-service"
+	)
+	cc, gator, bg := newTestBalancerGroup(t, loadStore.PerCluster(testCluster, testEDSService))
 
 	backendToBalancerID := make(map[balancer.SubConn]string)
 
@@ -440,7 +444,9 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 	// subConns in each group, we expect the picks to be equally split between
 	// the subConns. We do not call Done() on picks routed to sc1, so we expect
 	// these to show up as pending rpcs.
-	wantStoreData := &load.Data{
+	wantStoreData := []*load.Data{{
+		Cluster: testCluster,
+		Service: testEDSService,
 		LocalityStats: map[string]load.LocalityData{
 			testBalancerIDs[0]: {
 				RequestStats: load.RequestData{Succeeded: 10, InProgress: 10},
@@ -461,7 +467,7 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 	for i := 0; i < 30; i++ {
 		scst, _ := p1.Pick(balancer.PickInfo{})
 		if scst.Done != nil && scst.SubConn != sc1 {
@@ -476,9 +482,9 @@ func (s) TestBalancerGroup_LoadReport(t *testing.T) {
 		}
 	}
 
-	gotStoreData := loadStore.Stats()
-	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.EquateApprox(0, 0.1)); diff != "" {
-		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	gotStoreData := loadStore.Stats([]string{testCluster})
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.EquateApprox(0, 0.1), cmpopts.IgnoreFields(load.Data{}, "ReportInterval")); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
 
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 0febc1be3e57..a56acb7e293d 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -687,8 +687,10 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
 	loadStore := load.NewStore()
+	lsWrapper := &loadStoreWrapper{}
+	lsWrapper.update(loadStore, testClusterNames[0])
 	cw := &xdsClientWrapper{
-		load: &loadStoreWrapper{store: loadStore, service: testClusterNames[0]},
+		load: lsWrapper,
 	}
 
 	cc := testutils.NewTestClientConn(t)
@@ -723,12 +725,14 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// We expect the 10 picks to be split between the localities since they are
 	// of equal weight. And since we only mark the picks routed to sc2 as done,
 	// the picks on sc1 should show up as inProgress.
-	wantStoreData := &load.Data{
+	wantStoreData := []*load.Data{{
+		Cluster: testClusterNames[0],
+		Service: "",
 		LocalityStats: map[string]load.LocalityData{
 			locality1.String(): {RequestStats: load.RequestData{InProgress: 5}},
 			locality2.String(): {RequestStats: load.RequestData{Succeeded: 5}},
 		},
-	}
+	}}
 	for i := 0; i < 10; i++ {
 		scst, _ := p1.Pick(balancer.PickInfo{})
 		if scst.Done != nil && scst.SubConn != sc1 {
@@ -736,8 +740,8 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 		}
 	}
 
-	gotStoreData := loadStore.PerCluster(testClusterNames[0], "").Stats()
-	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
-		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	gotStoreData := loadStore.Stats(testClusterNames[0:1])
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(load.Data{}, "ReportInterval")); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index 99d986f59c77..f22c1624a3b6 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -48,40 +48,45 @@ var (
 )
 
 type loadStoreWrapper struct {
-	mu      sync.RWMutex
-	store   *load.Store
-	service string
+	mu         sync.RWMutex
+	store      *load.Store
+	service    string
+	perCluster load.PerClusterReporter
 }
 
 func (lsw *loadStoreWrapper) update(store *load.Store, service string) {
 	lsw.mu.Lock()
 	defer lsw.mu.Unlock()
+	if store == lsw.store && service == lsw.service {
+		return
+	}
 	lsw.store = store
 	lsw.service = service
+	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
 }
 
 func (lsw *loadStoreWrapper) CallStarted(locality string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.service, "").CallStarted(locality)
+	lsw.perCluster.CallStarted(locality)
 }
 
 func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.service, "").CallFinished(locality, err)
+	lsw.perCluster.CallFinished(locality, err)
 }
 
 func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.service, "").CallServerLoad(locality, name, val)
+	lsw.perCluster.CallServerLoad(locality, name, val)
 }
 
 func (lsw *loadStoreWrapper) CallDropped(category string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.service, "").CallDropped(category)
+	lsw.perCluster.CallDropped(category)
 }
 
 // xdsclientWrapper is responsible for getting the xds client from attributes or
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index b5bf3001fccf..1361fb15728f 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -154,38 +154,43 @@ type loadStoreWrapper struct {
 	store      *load.Store
 	cluster    string
 	edsService string
+	perCluster load.PerClusterReporter
 }
 
 func (lsw *loadStoreWrapper) update(store *load.Store, cluster, edsService string) {
 	lsw.mu.Lock()
 	defer lsw.mu.Unlock()
+	if store == lsw.store && cluster == lsw.cluster && edsService == lsw.edsService {
+		return
+	}
 	lsw.store = store
 	lsw.cluster = cluster
 	lsw.edsService = edsService
+	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
 }
 
 func (lsw *loadStoreWrapper) CallStarted(locality string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallStarted(locality)
+	lsw.perCluster.CallStarted(locality)
 }
 
 func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallFinished(locality, err)
+	lsw.perCluster.CallFinished(locality, err)
 }
 
 func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallServerLoad(locality, name, val)
+	lsw.perCluster.CallServerLoad(locality, name, val)
 }
 
 func (lsw *loadStoreWrapper) CallDropped(category string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.store.PerCluster(lsw.cluster, lsw.edsService).CallDropped(category)
+	lsw.perCluster.CallDropped(category)
 }
 
 type xdsClientWrapper struct {
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index 386b1431eaa6..789cfea0c00a 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -116,7 +116,14 @@ func TestLoadReporting(t *testing.T) {
 	if loadStore == nil {
 		t.Fatal("loadStore is nil in xdsClient")
 	}
-	sd := loadStore.PerCluster(testClusterName, testServiceName).Stats()
+	sds := loadStore.Stats([]string{testClusterName})
+	if len(sds) == 0 {
+		t.Fatalf("loads for cluster %v not found in store", testClusterName)
+	}
+	sd := sds[0]
+	if sd.Cluster != testClusterName || sd.Service != testServiceName {
+		t.Fatalf("got unexpected load for %q, %q, want %q, %q", sd.Cluster, sd.Service, testClusterName, testServiceName)
+	}
 	localityData, ok := sd.LocalityStats[testLocality.String()]
 	if !ok {
 		t.Fatalf("loads for %v not found in store", testLocality)
diff --git a/xds/internal/client/load/store.go b/xds/internal/client/load/store.go
index c0c741f670f8..a6ec1ec337cd 100644
--- a/xds/internal/client/load/store.go
+++ b/xds/internal/client/load/store.go
@@ -20,63 +20,112 @@ package load
 import (
 	"sync"
 	"sync/atomic"
+	"time"
 )
 
 const negativeOneUInt64 = ^uint64(0)
 
-// A pair of cluster and service name. The same cluster can be used by multiple
-// services, and one service can use multiple clusters. So we need a pair with
-// both name to accurately indicate where the load belongs.
-type storeKey struct {
-	cluster string
-	service string
-}
-
 // Store keeps the loads for multiple clusters and services to be reported via
-// LRS. It is safe for concurrent use.
+// LRS. It contains loads to reported to one LRS server. Create multiple stores
+// for multiple servers.
+//
+// It is safe for concurrent use.
 type Store struct {
-	mu       sync.RWMutex
-	clusters map[storeKey]*PerClusterStore
+	// mu only protects the map (2 layers). The read/write to *perClusterStore
+	// doesn't need to hold the mu.
+	mu sync.Mutex
+	// clusters is a map with cluster name as the key. The second layer is a map
+	// with service name as the key. Each value (perClusterStore) contains data
+	// for a (cluster, service) pair.
+	//
+	// Note that new entries are added to this map, but never removed. This is
+	// potentially a memory leak. But the memory is allocated for each new
+	// (cluster,service) pair, and the memory allocated is just pointers and
+	// maps. So this shouldn't get too bad.
+	clusters map[string]map[string]*perClusterStore
 }
 
 // NewStore creates a Store.
 func NewStore() *Store {
 	return &Store{
-		clusters: make(map[storeKey]*PerClusterStore),
+		clusters: make(map[string]map[string]*perClusterStore),
+	}
+}
+
+// Stats returns the load data for the given cluster names. Data is returned in
+// a slice with no specific order.
+//
+// If no clusterName is given (an empty slice), all data for all known clusters
+// is returned.
+//
+// If a cluster's Data is empty (no load to report), it's not appended to the
+// returned slice.
+func (s *Store) Stats(clusterNames []string) []*Data {
+	var ret []*Data
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if len(clusterNames) == 0 {
+		for _, c := range s.clusters {
+			ret = appendClusterStats(ret, c)
+		}
+		return ret
+	}
+
+	for _, n := range clusterNames {
+		if c, ok := s.clusters[n]; ok {
+			ret = appendClusterStats(ret, c)
+		}
+	}
+	return ret
+}
+
+// appendClusterStats gets Data for the given cluster, append to ret, and return
+// the new slice.
+//
+// Data is only appended to ret if it's not empty.
+func appendClusterStats(ret []*Data, cluster map[string]*perClusterStore) []*Data {
+	for _, d := range cluster {
+		data := d.stats()
+		if data == nil {
+			// Skip this data if it doesn't contain any information.
+			continue
+		}
+		ret = append(ret, data)
 	}
+	return ret
 }
 
-// PerCluster returns the PerClusterStore for the given clusterName +
+// PerCluster returns the perClusterStore for the given clusterName +
 // serviceName.
-func (ls *Store) PerCluster(clusterName, serviceName string) *PerClusterStore {
-	if ls == nil {
+func (s *Store) PerCluster(clusterName, serviceName string) PerClusterReporter {
+	if s == nil {
 		return nil
 	}
 
-	k := storeKey{
-		cluster: clusterName,
-		service: serviceName,
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	c, ok := s.clusters[clusterName]
+	if !ok {
+		c = make(map[string]*perClusterStore)
+		s.clusters[clusterName] = c
 	}
 
-	ls.mu.RLock()
-	if p, ok := ls.clusters[k]; ok {
-		ls.mu.RUnlock()
+	if p, ok := c[serviceName]; ok {
 		return p
 	}
-	ls.mu.RUnlock()
-
-	ls.mu.Lock()
-	defer ls.mu.Unlock()
-	p, ok := ls.clusters[k]
-	if !ok {
-		p = &PerClusterStore{}
-		ls.clusters[k] = p
+	p := &perClusterStore{
+		cluster: clusterName,
+		service: serviceName,
 	}
+	c[serviceName] = p
 	return p
 }
 
-// PerClusterStore is a repository for LB policy implementations to report store
-// load data. It is safe for concurrent use.
+// perClusterStore is a repository for LB policy implementations to report store
+// load data. It contains load for a (cluster, edsService) pair.
+//
+// It is safe for concurrent use.
 //
 // TODO(easwars): Use regular maps with mutexes instead of sync.Map here. The
 // latter is optimized for two common use cases: (1) when the entry for a given
@@ -87,16 +136,20 @@ func (ls *Store) PerCluster(clusterName, serviceName string) *PerClusterStore {
 // RWMutex.
 // Neither of these conditions are met here, and we should transition to a
 // regular map with a mutex for better type safety.
-type PerClusterStore struct {
+type perClusterStore struct {
+	cluster, service string
 	drops            sync.Map // map[string]*uint64
 	localityRPCCount sync.Map // map[string]*rpcCountData
+
+	mu               sync.Mutex
+	lastLoadReportAt time.Time
 }
 
 // Update functions are called by picker for each RPC. To avoid contention, all
 // updates are done atomically.
 
 // CallDropped adds one drop record with the given category to store.
-func (ls *PerClusterStore) CallDropped(category string) {
+func (ls *perClusterStore) CallDropped(category string) {
 	if ls == nil {
 		return
 	}
@@ -110,7 +163,7 @@ func (ls *PerClusterStore) CallDropped(category string) {
 }
 
 // CallStarted adds one call started record for the given locality.
-func (ls *PerClusterStore) CallStarted(locality string) {
+func (ls *perClusterStore) CallStarted(locality string) {
 	if ls == nil {
 		return
 	}
@@ -125,7 +178,7 @@ func (ls *PerClusterStore) CallStarted(locality string) {
 
 // CallFinished adds one call finished record for the given locality.
 // For successful calls, err needs to be nil.
-func (ls *PerClusterStore) CallFinished(locality string, err error) {
+func (ls *perClusterStore) CallFinished(locality string, err error) {
 	if ls == nil {
 		return
 	}
@@ -146,7 +199,7 @@ func (ls *PerClusterStore) CallFinished(locality string, err error) {
 
 // CallServerLoad adds one server load record for the given locality. The
 // load type is specified by desc, and its value by val.
-func (ls *PerClusterStore) CallServerLoad(locality, name string, d float64) {
+func (ls *perClusterStore) CallServerLoad(locality, name string, d float64) {
 	if ls == nil {
 		return
 	}
@@ -154,21 +207,28 @@ func (ls *PerClusterStore) CallServerLoad(locality, name string, d float64) {
 	p, ok := ls.localityRPCCount.Load(locality)
 	if !ok {
 		// The map is never cleared, only values in the map are reset. So the
-		// case where entry for CallServerLoad is not found should never happen.
+		// case where entry for callServerLoad is not found should never happen.
 		return
 	}
 	p.(*rpcCountData).addServerLoad(name, d)
 }
 
 // Data contains all load data reported to the Store since the most recent call
-// to Stats().
+// to stats().
 type Data struct {
+	// Cluster is the name of the cluster this data is for.
+	Cluster string
+	// Service is the name of the EDS service this data is for.
+	Service string
 	// TotalDrops is the total number of dropped requests.
 	TotalDrops uint64
 	// Drops is the number of dropped requests per category.
 	Drops map[string]uint64
 	// LocalityStats contains load reports per locality.
 	LocalityStats map[string]LocalityData
+	// ReportInternal is the duration since last time load was reported (stats()
+	// was called).
+	ReportInterval time.Duration
 }
 
 // LocalityData contains load data for a single locality.
@@ -198,21 +258,25 @@ type ServerLoadData struct {
 	Sum float64
 }
 
-func newStoreData() *Data {
+func newData(cluster, service string) *Data {
 	return &Data{
+		Cluster:       cluster,
+		Service:       service,
 		Drops:         make(map[string]uint64),
 		LocalityStats: make(map[string]LocalityData),
 	}
 }
 
-// Stats returns and resets all loads reported to the store, except inProgress
+// stats returns and resets all loads reported to the store, except inProgress
 // rpc counts.
-func (ls *PerClusterStore) Stats() *Data {
+//
+// It returns nil if the store doesn't contain any (new) data.
+func (ls *perClusterStore) stats() *Data {
 	if ls == nil {
 		return nil
 	}
 
-	sd := newStoreData()
+	sd := newData(ls.cluster, ls.service)
 	ls.drops.Range(func(key, val interface{}) bool {
 		d := atomic.SwapUint64(val.(*uint64), 0)
 		if d == 0 {
@@ -253,6 +317,15 @@ func (ls *PerClusterStore) Stats() *Data {
 		sd.LocalityStats[key.(string)] = ld
 		return true
 	})
+
+	ls.mu.Lock()
+	sd.ReportInterval = time.Since(ls.lastLoadReportAt)
+	ls.lastLoadReportAt = time.Now()
+	ls.mu.Unlock()
+
+	if sd.TotalDrops == 0 && len(sd.Drops) == 0 && len(sd.LocalityStats) == 0 {
+		return nil
+	}
 	return sd
 }
 
diff --git a/xds/internal/client/load/store_test.go b/xds/internal/client/load/store_test.go
index df9aed7c031a..4d62ebc4c621 100644
--- a/xds/internal/client/load/store_test.go
+++ b/xds/internal/client/load/store_test.go
@@ -19,6 +19,7 @@ package load
 
 import (
 	"fmt"
+	"sort"
 	"sync"
 	"testing"
 
@@ -56,7 +57,7 @@ func TestDrops(t *testing.T) {
 		}
 	)
 
-	ls := PerClusterStore{}
+	ls := perClusterStore{}
 	var wg sync.WaitGroup
 	for category, count := range drops {
 		for i := 0; i < count; i++ {
@@ -69,9 +70,9 @@ func TestDrops(t *testing.T) {
 	}
 	wg.Wait()
 
-	gotStoreData := ls.Stats()
-	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
-		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	gotStoreData := ls.stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
 
@@ -118,7 +119,7 @@ func TestLocalityStats(t *testing.T) {
 		}
 	)
 
-	ls := PerClusterStore{}
+	ls := perClusterStore{}
 	var wg sync.WaitGroup
 	for locality, data := range localityData {
 		wg.Add(data.start)
@@ -128,7 +129,7 @@ func TestLocalityStats(t *testing.T) {
 				wg.Done()
 			}(locality)
 		}
-		// The calls to CallStarted() need to happen before the other calls are
+		// The calls to callStarted() need to happen before the other calls are
 		// made. Hence the wait here.
 		wg.Wait()
 
@@ -152,9 +153,9 @@ func TestLocalityStats(t *testing.T) {
 		wg.Wait()
 	}
 
-	gotStoreData := ls.Stats()
-	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
-		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	gotStoreData := ls.stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
 
@@ -211,7 +212,7 @@ func TestResetAfterStats(t *testing.T) {
 		}
 	)
 
-	reportLoad := func(ls *PerClusterStore) {
+	reportLoad := func(ls *perClusterStore) {
 		for category, count := range drops {
 			for i := 0; i < count; i++ {
 				ls.CallDropped(category)
@@ -233,14 +234,14 @@ func TestResetAfterStats(t *testing.T) {
 		}
 	}
 
-	ls := PerClusterStore{}
+	ls := perClusterStore{}
 	reportLoad(&ls)
-	gotStoreData := ls.Stats()
-	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
-		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	gotStoreData := ls.stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 
-	// The above call to Stats() should have reset all load reports except the
+	// The above call to stats() should have reset all load reports except the
 	// inProgress rpc count. We are now going to push the same load data into
 	// the store. So, we should expect to see twice the count for inProgress.
 	for _, l := range localities {
@@ -249,8 +250,196 @@ func TestResetAfterStats(t *testing.T) {
 		wantStoreData.LocalityStats[l] = ls
 	}
 	reportLoad(&ls)
-	gotStoreData = ls.Stats()
-	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty()); diff != "" {
-		t.Errorf("store.Stats() returned unexpected diff (-want +got):\n%s", diff)
+	gotStoreData = ls.stats()
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+}
+
+var sortDataSlice = cmp.Transformer("SortDataSlice", func(in []*Data) []*Data {
+	out := append([]*Data(nil), in...) // Copy input to avoid mutating it
+	sort.Slice(out,
+		func(i, j int) bool {
+			if out[i].Cluster < out[j].Cluster {
+				return true
+			}
+			if out[i].Cluster == out[j].Cluster {
+				return out[i].Service < out[j].Service
+			}
+			return false
+		},
+	)
+	return out
+})
+
+// Test all load are returned for the given clusters, and all clusters are
+// reported if no cluster is specified.
+func TestStoreStats(t *testing.T) {
+	var (
+		testClusters = []string{"c0", "c1", "c2"}
+		testServices = []string{"s0", "s1"}
+		testLocality = "test-locality"
+	)
+
+	store := NewStore()
+	for _, c := range testClusters {
+		for _, s := range testServices {
+			store.PerCluster(c, s).CallStarted(testLocality)
+			store.PerCluster(c, s).CallServerLoad(testLocality, "abc", 123)
+			store.PerCluster(c, s).CallDropped("dropped")
+			store.PerCluster(c, s).CallFinished(testLocality, nil)
+		}
+	}
+
+	wantC0 := []*Data{
+		{
+			Cluster: "c0", Service: "s0",
+			TotalDrops: 1, Drops: map[string]uint64{"dropped": 1},
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {
+					RequestStats: RequestData{Succeeded: 1},
+					LoadStats:    map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}},
+				},
+			},
+		},
+		{
+			Cluster: "c0", Service: "s1",
+			TotalDrops: 1, Drops: map[string]uint64{"dropped": 1},
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {
+					RequestStats: RequestData{Succeeded: 1},
+					LoadStats:    map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}},
+				},
+			},
+		},
+	}
+	// Call Stats with just "c0", this should return data for "c0", and not
+	// touch data for other clusters.
+	gotC0 := store.Stats([]string{"c0"})
+	if diff := cmp.Diff(wantC0, gotC0, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+
+	wantOther := []*Data{
+		{
+			Cluster: "c1", Service: "s0",
+			TotalDrops: 1, Drops: map[string]uint64{"dropped": 1},
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {
+					RequestStats: RequestData{Succeeded: 1},
+					LoadStats:    map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}},
+				},
+			},
+		},
+		{
+			Cluster: "c1", Service: "s1",
+			TotalDrops: 1, Drops: map[string]uint64{"dropped": 1},
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {
+					RequestStats: RequestData{Succeeded: 1},
+					LoadStats:    map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}},
+				},
+			},
+		},
+		{
+			Cluster: "c2", Service: "s0",
+			TotalDrops: 1, Drops: map[string]uint64{"dropped": 1},
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {
+					RequestStats: RequestData{Succeeded: 1},
+					LoadStats:    map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}},
+				},
+			},
+		},
+		{
+			Cluster: "c2", Service: "s1",
+			TotalDrops: 1, Drops: map[string]uint64{"dropped": 1},
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {
+					RequestStats: RequestData{Succeeded: 1},
+					LoadStats:    map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}},
+				},
+			},
+		},
+	}
+	// Call Stats with empty slice, this should return data for all the
+	// remaining clusters, and not include c0 (because c0 data was cleared).
+	gotOther := store.Stats(nil)
+	if diff := cmp.Diff(wantOther, gotOther, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+}
+
+// Test the cases that if a cluster doesn't have load to report, its data is not
+// appended to the slice returned by Stats().
+func TestStoreStatsEmptyDataNotReported(t *testing.T) {
+	var (
+		testServices = []string{"s0", "s1"}
+		testLocality = "test-locality"
+	)
+
+	store := NewStore()
+	// "c0"'s RPCs all finish with success.
+	for _, s := range testServices {
+		store.PerCluster("c0", s).CallStarted(testLocality)
+		store.PerCluster("c0", s).CallFinished(testLocality, nil)
+	}
+	// "c1"'s RPCs never finish (always inprocess).
+	for _, s := range testServices {
+		store.PerCluster("c1", s).CallStarted(testLocality)
+	}
+
+	want0 := []*Data{
+		{
+			Cluster: "c0", Service: "s0",
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {RequestStats: RequestData{Succeeded: 1}},
+			},
+		},
+		{
+			Cluster: "c0", Service: "s1",
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {RequestStats: RequestData{Succeeded: 1}},
+			},
+		},
+		{
+			Cluster: "c1", Service: "s0",
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {RequestStats: RequestData{InProgress: 1}},
+			},
+		},
+		{
+			Cluster: "c1", Service: "s1",
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {RequestStats: RequestData{InProgress: 1}},
+			},
+		},
+	}
+	// Call Stats with empty slice, this should return data for all the
+	// clusters.
+	got0 := store.Stats(nil)
+	if diff := cmp.Diff(want0, got0, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+
+	want1 := []*Data{
+		{
+			Cluster: "c1", Service: "s0",
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {RequestStats: RequestData{InProgress: 1}},
+			},
+		},
+		{
+			Cluster: "c1", Service: "s1",
+			LocalityStats: map[string]LocalityData{
+				"test-locality": {RequestStats: RequestData{InProgress: 1}},
+			},
+		},
+	}
+	// Call Stats with empty slice again, this should return data only for "c1",
+	// because "c0" data was cleared, but "c1" has in-progress RPCs.
+	got1 := store.Stats(nil)
+	if diff := cmp.Diff(want1, got1, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" {
+		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index f838b5fed355..c3de39c3a853 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -23,7 +23,6 @@ import (
 	"context"
 	"fmt"
 	"sync"
-	"time"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
@@ -103,8 +102,6 @@ type client struct {
 	// processing needs this to do the host matching.
 	ldsResourceName string
 	ldsWatchCount   int
-
-	lastLoadReportAt time.Time
 }
 
 // AddWatch overrides the transport helper's AddWatch to save the LDS
diff --git a/xds/internal/client/v2/loadreport.go b/xds/internal/client/v2/loadreport.go
index 899cb1b5543b..a06dcb8e9f0d 100644
--- a/xds/internal/client/v2/loadreport.go
+++ b/xds/internal/client/v2/loadreport.go
@@ -116,58 +116,58 @@ func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string)
 		return errors.New("lrs: LoadStore is not initialized")
 	}
 
-	var (
-		droppedReqs   []*v2endpointpb.ClusterStats_DroppedRequests
-		localityStats []*v2endpointpb.UpstreamLocalityStats
-	)
-
-	sd := v2c.loadStore.PerCluster(clusterName, "").Stats()
-	for category, count := range sd.Drops {
-		droppedReqs = append(droppedReqs, &v2endpointpb.ClusterStats_DroppedRequests{
-			Category:     category,
-			DroppedCount: count,
-		})
-	}
-	for l, localityData := range sd.LocalityStats {
-		lid, err := internal.LocalityIDFromString(l)
-		if err != nil {
-			return err
+	var clusterStats []*v2endpointpb.ClusterStats
+	sds := v2c.loadStore.Stats([]string{clusterName})
+	for _, sd := range sds {
+		var (
+			droppedReqs   []*v2endpointpb.ClusterStats_DroppedRequests
+			localityStats []*v2endpointpb.UpstreamLocalityStats
+		)
+		for category, count := range sd.Drops {
+			droppedReqs = append(droppedReqs, &v2endpointpb.ClusterStats_DroppedRequests{
+				Category:     category,
+				DroppedCount: count,
+			})
 		}
-		var loadMetricStats []*v2endpointpb.EndpointLoadMetricStats
-		for name, loadData := range localityData.LoadStats {
-			loadMetricStats = append(loadMetricStats, &v2endpointpb.EndpointLoadMetricStats{
-				MetricName:                    name,
-				NumRequestsFinishedWithMetric: loadData.Count,
-				TotalMetricValue:              loadData.Sum,
+		for l, localityData := range sd.LocalityStats {
+			lid, err := internal.LocalityIDFromString(l)
+			if err != nil {
+				return err
+			}
+			var loadMetricStats []*v2endpointpb.EndpointLoadMetricStats
+			for name, loadData := range localityData.LoadStats {
+				loadMetricStats = append(loadMetricStats, &v2endpointpb.EndpointLoadMetricStats{
+					MetricName:                    name,
+					NumRequestsFinishedWithMetric: loadData.Count,
+					TotalMetricValue:              loadData.Sum,
+				})
+			}
+			localityStats = append(localityStats, &v2endpointpb.UpstreamLocalityStats{
+				Locality: &v2corepb.Locality{
+					Region:  lid.Region,
+					Zone:    lid.Zone,
+					SubZone: lid.SubZone,
+				},
+				TotalSuccessfulRequests: localityData.RequestStats.Succeeded,
+				TotalRequestsInProgress: localityData.RequestStats.InProgress,
+				TotalErrorRequests:      localityData.RequestStats.Errored,
+				LoadMetricStats:         loadMetricStats,
+				UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
 			})
 		}
-		localityStats = append(localityStats, &v2endpointpb.UpstreamLocalityStats{
-			Locality: &v2corepb.Locality{
-				Region:  lid.Region,
-				Zone:    lid.Zone,
-				SubZone: lid.SubZone,
-			},
-			TotalSuccessfulRequests: localityData.RequestStats.Succeeded,
-			TotalRequestsInProgress: localityData.RequestStats.InProgress,
-			TotalErrorRequests:      localityData.RequestStats.Errored,
-			LoadMetricStats:         loadMetricStats,
-			UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
-		})
-	}
 
-	dur := time.Since(v2c.lastLoadReportAt)
-	v2c.lastLoadReportAt = time.Now()
-
-	cs := []*v2endpointpb.ClusterStats{
-		{
-			ClusterName:           clusterName,
+		clusterStats = append(clusterStats, &v2endpointpb.ClusterStats{
+			ClusterName:           sd.Cluster,
+			ClusterServiceName:    sd.Service,
 			UpstreamLocalityStats: localityStats,
 			TotalDroppedRequests:  sd.TotalDrops,
 			DroppedRequests:       droppedReqs,
-			LoadReportInterval:    ptypes.DurationProto(dur),
-		},
+			LoadReportInterval:    ptypes.DurationProto(sd.ReportInterval),
+		})
+
 	}
-	req := &lrspb.LoadStatsRequest{ClusterStats: cs}
+
+	req := &lrspb.LoadStatsRequest{ClusterStats: clusterStats}
 	v2c.logger.Infof("lrs: sending LRS loads: %+v", req)
 	return stream.Send(req)
 }
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index ba9086146365..9894280ede32 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -23,7 +23,6 @@ import (
 	"context"
 	"fmt"
 	"sync"
-	"time"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
@@ -103,8 +102,6 @@ type client struct {
 	// processing needs this to do the host matching.
 	ldsResourceName string
 	ldsWatchCount   int
-
-	lastLoadReportAt time.Time
 }
 
 // AddWatch overrides the transport helper's AddWatch to save the LDS
diff --git a/xds/internal/client/v3/loadreport.go b/xds/internal/client/v3/loadreport.go
index 1997277886b7..beca34c49bfd 100644
--- a/xds/internal/client/v3/loadreport.go
+++ b/xds/internal/client/v3/loadreport.go
@@ -116,58 +116,57 @@ func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string)
 		return errors.New("lrs: LoadStore is not initialized")
 	}
 
-	var (
-		droppedReqs   []*v3endpointpb.ClusterStats_DroppedRequests
-		localityStats []*v3endpointpb.UpstreamLocalityStats
-	)
-
-	sd := v3c.loadStore.PerCluster(clusterName, "").Stats()
-	for category, count := range sd.Drops {
-		droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{
-			Category:     category,
-			DroppedCount: count,
-		})
-	}
-	for l, localityData := range sd.LocalityStats {
-		lid, err := internal.LocalityIDFromString(l)
-		if err != nil {
-			return err
+	var clusterStats []*v3endpointpb.ClusterStats
+	sds := v3c.loadStore.Stats([]string{clusterName})
+	for _, sd := range sds {
+		var (
+			droppedReqs   []*v3endpointpb.ClusterStats_DroppedRequests
+			localityStats []*v3endpointpb.UpstreamLocalityStats
+		)
+		for category, count := range sd.Drops {
+			droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{
+				Category:     category,
+				DroppedCount: count,
+			})
 		}
-		var loadMetricStats []*v3endpointpb.EndpointLoadMetricStats
-		for name, loadData := range localityData.LoadStats {
-			loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
-				MetricName:                    name,
-				NumRequestsFinishedWithMetric: loadData.Count,
-				TotalMetricValue:              loadData.Sum,
+		for l, localityData := range sd.LocalityStats {
+			lid, err := internal.LocalityIDFromString(l)
+			if err != nil {
+				return err
+			}
+			var loadMetricStats []*v3endpointpb.EndpointLoadMetricStats
+			for name, loadData := range localityData.LoadStats {
+				loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
+					MetricName:                    name,
+					NumRequestsFinishedWithMetric: loadData.Count,
+					TotalMetricValue:              loadData.Sum,
+				})
+			}
+			localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{
+				Locality: &v3corepb.Locality{
+					Region:  lid.Region,
+					Zone:    lid.Zone,
+					SubZone: lid.SubZone,
+				},
+				TotalSuccessfulRequests: localityData.RequestStats.Succeeded,
+				TotalRequestsInProgress: localityData.RequestStats.InProgress,
+				TotalErrorRequests:      localityData.RequestStats.Errored,
+				LoadMetricStats:         loadMetricStats,
+				UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
 			})
 		}
-		localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{
-			Locality: &v3corepb.Locality{
-				Region:  lid.Region,
-				Zone:    lid.Zone,
-				SubZone: lid.SubZone,
-			},
-			TotalSuccessfulRequests: localityData.RequestStats.Succeeded,
-			TotalRequestsInProgress: localityData.RequestStats.InProgress,
-			TotalErrorRequests:      localityData.RequestStats.Errored,
-			LoadMetricStats:         loadMetricStats,
-			UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
-		})
-	}
-
-	dur := time.Since(v3c.lastLoadReportAt)
-	v3c.lastLoadReportAt = time.Now()
 
-	cs := []*v3endpointpb.ClusterStats{
-		{
-			ClusterName:           clusterName,
+		clusterStats = append(clusterStats, &v3endpointpb.ClusterStats{
+			ClusterName:           sd.Cluster,
+			ClusterServiceName:    sd.Service,
 			UpstreamLocalityStats: localityStats,
 			TotalDroppedRequests:  sd.TotalDrops,
 			DroppedRequests:       droppedReqs,
-			LoadReportInterval:    ptypes.DurationProto(dur),
-		},
+			LoadReportInterval:    ptypes.DurationProto(sd.ReportInterval),
+		})
 	}
-	req := &lrspb.LoadStatsRequest{ClusterStats: cs}
+
+	req := &lrspb.LoadStatsRequest{ClusterStats: clusterStats}
 	v3c.logger.Infof("lrs: sending LRS loads: %+v", req)
 	return stream.Send(req)
 }

From f0db9671c2460192fb61fad5326b9173195a9561 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Thu, 8 Oct 2020 13:41:00 -0700
Subject: [PATCH 226/481] Change version to 1.34.0-dev (#3941)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index e5ce470f0c6b..725093bb882a 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.33.0-dev"
+const Version = "1.34.0-dev"

From 06c094c3ab224883997e33ad8e1649a80f24ee0f Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 8 Oct 2020 15:50:51 -0700
Subject: [PATCH 227/481] xds/client: Process security configuration from
 Cluster resource (#3931)

---
 examples/go.sum                        |  10 +-
 go.mod                                 |   6 +-
 go.sum                                 |  35 +++-
 security/advancedtls/go.sum            |   5 +-
 vet.sh                                 |  21 +-
 xds/internal/client/client.go          |  27 +++
 xds/internal/client/client_cds_test.go | 267 +++++++++++++++++++++++++
 xds/internal/client/client_xds.go      |  70 +++++++
 xds/internal/version/version.go        |  11 +-
 9 files changed, 431 insertions(+), 21 deletions(-)

diff --git a/examples/go.sum b/examples/go.sum
index fae9a831ae02..314ec0703382 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -33,15 +33,16 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7 h1:EARl0OvqMoxq/UMgMSCLnXzkaXbxzskluEBlMQCJPms=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -110,6 +111,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
diff --git a/go.mod b/go.mod
index 0bcae7362db8..f4a728fa21b1 100644
--- a/go.mod
+++ b/go.mod
@@ -3,10 +3,10 @@ module google.golang.org/grpc
 go 1.11
 
 require (
-	github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f
-	github.com/envoyproxy/go-control-plane v0.9.4
+	github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354
+	github.com/envoyproxy/go-control-plane v0.9.7
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/protobuf v1.3.3
+	github.com/golang/protobuf v1.4.2
 	github.com/google/go-cmp v0.4.0
 	github.com/google/uuid v1.1.2
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
diff --git a/go.sum b/go.sum
index bab616e439f1..215a4599a965 100644
--- a/go.sum
+++ b/go.sum
@@ -2,14 +2,17 @@ cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.7 h1:EARl0OvqMoxq/UMgMSCLnXzkaXbxzskluEBlMQCJPms=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@@ -19,15 +22,25 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -65,6 +78,16 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index f2ab78d92322..84e6ee891384 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -36,9 +36,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -104,6 +104,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
diff --git a/vet.sh b/vet.sh
index f644d832d232..7e14befe7e3d 100755
--- a/vet.sh
+++ b/vet.sh
@@ -157,7 +157,26 @@ grpc.WithTimeout
 http.CloseNotifier
 info.SecurityVersion
 resolver.Backend
-resolver.GRPCLB' "${SC_OUT}"
+resolver.GRPCLB
+extDesc.Filename is deprecated
+BuildVersion is deprecated
+github.com/golang/protobuf/jsonpb is deprecated
+proto is deprecated
+xxx_messageInfo_
+proto.InternalMessageInfo is deprecated
+proto.EnumName is deprecated
+proto.ErrInternalBadWireType is deprecated
+proto.FileDescriptor is deprecated
+proto.Marshaler is deprecated
+proto.MessageType is deprecated
+proto.RegisterEnum is deprecated
+proto.RegisterFile is deprecated
+proto.RegisterType is deprecated
+proto.RegisterExtension is deprecated
+proto.RegisteredExtension is deprecated
+proto.RegisteredExtensions is deprecated
+proto.RegisterMapType is deprecated
+proto.Unmarshaler is deprecated' "${SC_OUT}"
 
 # - special golint on package comments.
 lint_package_comment_per_package() {
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index b2afed0c0140..64033227b1f6 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -188,6 +188,31 @@ type ServiceUpdate struct {
 	Routes []*Route
 }
 
+// SecurityConfig contains the security configuration received as part of the
+// Cluster resource.
+type SecurityConfig struct {
+	// RootInstanceName identifies the certProvider plugin to be used to fetch
+	// root certificates. This instance name will be resolved to the plugin name
+	// and its associated configuration from the certificate_providers field of
+	// the bootstrap file.
+	RootInstanceName string
+	// RootCertName is the certificate name to be passed to the plugin (looked
+	// up from the bootstrap file) while fetching root certificates.
+	RootCertName string
+	// IdentityInstanceName identifies the certProvider plugin to be used to
+	// fetch identity certificates. This instance name will be resolved to the
+	// plugin name and its associated configuration from the
+	// certificate_providers field of the bootstrap file.
+	IdentityInstanceName string
+	// IdentityCertName is the certificate name to be passed to the plugin
+	// (looked up from the bootstrap file) while fetching identity certificates.
+	IdentityCertName string
+	// AcceptedSANs is a list of Subject Alternative Names. During the TLS
+	// handshake, the SAN present in the peer certificate is compared against
+	// this list, and the handshake succeeds only if a match is found.
+	AcceptedSANs []string
+}
+
 // ClusterUpdate contains information from a received CDS response, which is of
 // interest to the registered CDS watcher.
 type ClusterUpdate struct {
@@ -196,6 +221,8 @@ type ClusterUpdate struct {
 	ServiceName string
 	// EnableLRS indicates whether or not load should be reported through LRS.
 	EnableLRS bool
+	// SecurityCfg contains security configuration sent by the xDS server.
+	SecurityCfg *SecurityConfig
 }
 
 // OverloadDropConfig contains the config to drop overloads.
diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go
index 578d6121e214..9d36d70be244 100644
--- a/xds/internal/client/client_cds_test.go
+++ b/xds/internal/client/client_cds_test.go
@@ -25,6 +25,8 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
@@ -182,6 +184,271 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 	}
 }
 
+func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
+	const (
+		identityPluginInstance = "identityPluginInstance"
+		identityCertName       = "identityCert"
+		rootPluginInstance     = "rootPluginInstance"
+		rootCertName           = "rootCert"
+		serviceName            = "service"
+		san1                   = "san1"
+		san2                   = "san2"
+	)
+
+	tests := []struct {
+		name       string
+		cluster    *v3clusterpb.Cluster
+		wantUpdate ClusterUpdate
+		wantErr    bool
+	}{
+		{
+			name: "transport-socket-unsupported-typeURL",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3HTTPConnManagerURL,
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "transport-socket-unsupported-type",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value:   []byte{1, 2, 3, 4},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "transport-socket-unsupported-validation-context",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
+											ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
+												Name: "foo-sds-secret",
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "happy-case-with-no-identity-certs",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+												InstanceName:    rootPluginInstance,
+												CertificateName: rootCertName,
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantUpdate: ClusterUpdate{
+				ServiceName: serviceName,
+				EnableLRS:   false,
+				SecurityCfg: &SecurityConfig{
+					RootInstanceName: rootPluginInstance,
+					RootCertName:     rootCertName,
+				},
+			},
+		},
+		{
+			name: "happy-case-with-validation-context-provider-instance",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    identityPluginInstance,
+											CertificateName: identityCertName,
+										},
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+												InstanceName:    rootPluginInstance,
+												CertificateName: rootCertName,
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantUpdate: ClusterUpdate{
+				ServiceName: serviceName,
+				EnableLRS:   false,
+				SecurityCfg: &SecurityConfig{
+					RootInstanceName:     rootPluginInstance,
+					RootCertName:         rootCertName,
+					IdentityInstanceName: identityPluginInstance,
+					IdentityCertName:     identityCertName,
+				},
+			},
+		},
+		{
+			name: "happy-case-with-combined-validation-context",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    identityPluginInstance,
+											CertificateName: identityCertName,
+										},
+										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+														{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: san1}},
+														{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: san2}},
+													},
+												},
+												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    rootPluginInstance,
+													CertificateName: rootCertName,
+												},
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantUpdate: ClusterUpdate{
+				ServiceName: serviceName,
+				EnableLRS:   false,
+				SecurityCfg: &SecurityConfig{
+					RootInstanceName:     rootPluginInstance,
+					RootCertName:         rootCertName,
+					IdentityInstanceName: identityPluginInstance,
+					IdentityCertName:     identityCertName,
+					AcceptedSANs:         []string{san1, san2},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			update, err := validateCluster(test.cluster)
+			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("validateCluster(%+v) = (%+v, %v), want: (%+v, %v)", test.cluster, update, err, test.wantUpdate, test.wantErr)
+			}
+		})
+	}
+}
+
 func (s) TestUnmarshalCluster(t *testing.T) {
 	const (
 		v2ClusterName = "v2clusterName"
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index f5f60a4f9460..a4048c55b600 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -30,12 +30,14 @@ import (
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 // UnmarshalListener processes resources received in an LDS response, validates
@@ -405,12 +407,80 @@ func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 		return emptyUpdate, fmt.Errorf("xds: unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
 
+	sc, err := securityConfigFromCluster(cluster)
+	if err != nil {
+		return emptyUpdate, err
+	}
 	return ClusterUpdate{
 		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
 		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
+		SecurityCfg: sc,
 	}, nil
 }
 
+// securityConfigFromCluster extracts the relevant security configuration from
+// the received Cluster resource.
+func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, error) {
+	// The Cluster resource contains a `transport_socket` field, which contains
+	// a oneof `typed_config` field of type `protobuf.Any`. The any proto
+	// contains a marshaled representation of an `UpstreamTlsContext` message.
+	ts := cluster.GetTransportSocket()
+	if ts == nil {
+		return nil, nil
+	}
+	any := ts.GetTypedConfig()
+	if any == nil || any.TypeUrl != version.V3UpstreamTLSContextURL {
+		return nil, fmt.Errorf("xds: transport_socket field has unexpected typeURL: %s", any.TypeUrl)
+	}
+	upstreamCtx := &v3tlspb.UpstreamTlsContext{}
+	if err := proto.Unmarshal(any.GetValue(), upstreamCtx); err != nil {
+		return nil, fmt.Errorf("xds: failed to unmarshal UpstreamTlsContext in CDS response: %v", err)
+	}
+
+	// The `UpstreamTlsContext` has a `CommonTlsContext` which contains a
+	// `tls_certificate_certificate_provider_instance` field of type
+	// `CertificateProviderInstance`, which contains the provider instance name
+	// and the certificate name to fetch identity certs.
+	sc := &SecurityConfig{}
+	if identity := upstreamCtx.GetCommonTlsContext().GetTlsCertificateCertificateProviderInstance(); identity != nil {
+		sc.IdentityInstanceName = identity.GetInstanceName()
+		sc.IdentityCertName = identity.GetCertificateName()
+	}
+
+	// The `CommonTlsContext` contains a `validation_context_type` field which
+	// is a oneof. We can get the values that we are interested in from two of
+	// those possible values:
+	//  - combined validation context:
+	//    - contains a default validation context which holds the list of
+	//      accepted SANs.
+	//    - contains certificate provider instance configuration
+	//  - certificate provider instance configuration
+	//    - in this case, we do not get a list of accepted SANs.
+	switch t := upstreamCtx.GetCommonTlsContext().GetValidationContextType().(type) {
+	case *v3tlspb.CommonTlsContext_CombinedValidationContext:
+		combined := upstreamCtx.GetCommonTlsContext().GetCombinedValidationContext()
+		if def := combined.GetDefaultValidationContext(); def != nil {
+			for _, matcher := range def.GetMatchSubjectAltNames() {
+				// We only support exact matches for now.
+				if exact := matcher.GetExact(); exact != "" {
+					sc.AcceptedSANs = append(sc.AcceptedSANs, exact)
+				}
+			}
+		}
+		if pi := combined.GetValidationContextCertificateProviderInstance(); pi != nil {
+			sc.RootInstanceName = pi.GetInstanceName()
+			sc.RootCertName = pi.GetCertificateName()
+		}
+	case *v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance:
+		pi := upstreamCtx.GetCommonTlsContext().GetValidationContextCertificateProviderInstance()
+		sc.RootInstanceName = pi.GetInstanceName()
+		sc.RootCertName = pi.GetCertificateName()
+	default:
+		return nil, fmt.Errorf("xds: validation context contains unexpected type: %T", t)
+	}
+	return sc, nil
+}
+
 // UnmarshalEndpoints processes resources received in an EDS response,
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in.
diff --git a/xds/internal/version/version.go b/xds/internal/version/version.go
index d1ca46f658a1..39344187d9e4 100644
--- a/xds/internal/version/version.go
+++ b/xds/internal/version/version.go
@@ -41,9 +41,10 @@ const (
 	V2EndpointsURL       = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
 	V2HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"
 
-	V3ListenerURL        = "type.googleapis.com/envoy.config.listener.v3.Listener"
-	V3RouteConfigURL     = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"
-	V3ClusterURL         = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
-	V3EndpointsURL       = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"
-	V3HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"
+	V3ListenerURL           = "type.googleapis.com/envoy.config.listener.v3.Listener"
+	V3RouteConfigURL        = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"
+	V3ClusterURL            = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
+	V3EndpointsURL          = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"
+	V3HTTPConnManagerURL    = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v3.HttpConnectionManager"
+	V3UpstreamTLSContextURL = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext"
 )

From 9abcdab70a6d67981e38903a7a8d1ec2d03da6d5 Mon Sep 17 00:00:00 2001
From: Piyush Mishra <piyush@codeitout.com>
Date: Fri, 9 Oct 2020 10:03:07 -0700
Subject: [PATCH 228/481] cmd/protoc-gen-go-grpc: fix method name in
 interceptor info (#3889)

---
 cmd/protoc-gen-go-grpc/grpc.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 919b5d46ae72..efc92acf35ce 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -351,7 +351,7 @@ func genServerMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.Gene
 		g.P("if interceptor == nil { return srv.(", service.GoName, "Server).", method.GoName, "(ctx, in) }")
 		g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{")
 		g.P("Server: srv,")
-		g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.GoName)), ",")
+		g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name())), ",")
 		g.P("}")
 		g.P("handler := func(ctx ", contextPackage.Ident("Context"), ", req interface{}) (interface{}, error) {")
 		g.P("return srv.(", service.GoName, "Server).", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))")

From 84e85f71c9dcce9cdf98901b1defaaf9af272382 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Fri, 9 Oct 2020 12:57:50 -0700
Subject: [PATCH 229/481] internal/credentials: fix a bug and add one more
 helper function SPIFFEIDFromCert (#3929)

* internal/credentials: fix a bug and add one more helper function
---
 internal/credentials/spiffe.go      | 16 ++++--
 internal/credentials/spiffe_test.go | 81 ++++++++++++++++++++++++-----
 testdata/x509/create.sh             | 19 +++++++
 testdata/x509/multiple_uri_cert.pem | 34 ++++++++++++
 testdata/x509/multiple_uri_key.pem  | 52 ++++++++++++++++++
 testdata/x509/spiffe_cert.pem       | 33 ++++++++++++
 testdata/x509/spiffe_key.pem        | 52 ++++++++++++++++++
 7 files changed, 270 insertions(+), 17 deletions(-)
 create mode 100644 testdata/x509/multiple_uri_cert.pem
 create mode 100644 testdata/x509/multiple_uri_key.pem
 create mode 100644 testdata/x509/spiffe_cert.pem
 create mode 100644 testdata/x509/spiffe_key.pem

diff --git a/internal/credentials/spiffe.go b/internal/credentials/spiffe.go
index b66e3dbb5ac2..be70b6cdfc31 100644
--- a/internal/credentials/spiffe.go
+++ b/internal/credentials/spiffe.go
@@ -25,6 +25,7 @@ package credentials
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"net/url"
 
 	"google.golang.org/grpc/grpclog"
@@ -38,8 +39,17 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
 	if len(state.PeerCertificates) == 0 || len(state.PeerCertificates[0].URIs) == 0 {
 		return nil
 	}
+	return SPIFFEIDFromCert(state.PeerCertificates[0])
+}
+
+// SPIFFEIDFromCert parses the SPIFFE ID from x509.Certificate. If the SPIFFE
+// ID format is invalid, return nil with warning.
+func SPIFFEIDFromCert(cert *x509.Certificate) *url.URL {
+	if cert == nil || cert.URIs == nil {
+		return nil
+	}
 	var spiffeID *url.URL
-	for _, uri := range state.PeerCertificates[0].URIs {
+	for _, uri := range cert.URIs {
 		if uri == nil || uri.Scheme != "spiffe" || uri.Opaque != "" || (uri.User != nil && uri.User.Username() != "") {
 			continue
 		}
@@ -48,7 +58,7 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
 			logger.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes")
 			return nil
 		}
-		if len(uri.Host) == 0 || len(uri.RawPath) == 0 || len(uri.Path) == 0 {
+		if len(uri.Host) == 0 || len(uri.Path) == 0 {
 			logger.Warning("invalid SPIFFE ID: domain or workload ID is empty")
 			return nil
 		}
@@ -57,7 +67,7 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL {
 			return nil
 		}
 		// A valid SPIFFE certificate can only have exactly one URI SAN field.
-		if len(state.PeerCertificates[0].URIs) > 1 {
+		if len(cert.URIs) > 1 {
 			logger.Warning("invalid SPIFFE ID: multiple URI SANs")
 			return nil
 		}
diff --git a/internal/credentials/spiffe_test.go b/internal/credentials/spiffe_test.go
index 324874e6e396..599481ad0bf9 100644
--- a/internal/credentials/spiffe_test.go
+++ b/internal/credentials/spiffe_test.go
@@ -21,12 +21,17 @@ package credentials
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
 	"net/url"
 	"testing"
 
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/testdata"
 )
 
+const wantURI = "spiffe://foo.bar.com/client/workload/1"
+
 type s struct {
 	grpctest.Tester
 }
@@ -40,12 +45,12 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 		name string
 		urls []*url.URL
 		// If we expect a SPIFFE ID to be returned.
-		expectID bool
+		wantID bool
 	}{
 		{
-			name:     "empty URIs",
-			urls:     []*url.URL{},
-			expectID: false,
+			name:   "empty URIs",
+			urls:   []*url.URL{},
+			wantID: false,
 		},
 		{
 			name: "good SPIFFE ID",
@@ -57,7 +62,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "workload/wl1",
 				},
 			},
-			expectID: true,
+			wantID: true,
 		},
 		{
 			name: "invalid host",
@@ -69,7 +74,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "workload/wl1",
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 		{
 			name: "invalid path",
@@ -81,7 +86,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "",
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 		{
 			name: "large path",
@@ -93,7 +98,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: string(make([]byte, 2050)),
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 		{
 			name: "large host",
@@ -105,7 +110,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "workload/wl1",
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 		{
 			name: "multiple URI SANs",
@@ -129,7 +134,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "workload/wl1",
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 		{
 			name: "multiple URI SANs without SPIFFE ID",
@@ -147,7 +152,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "workload/wl1",
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 		{
 			name: "multiple URI SANs with one SPIFFE ID",
@@ -165,15 +170,63 @@ func (s) TestSPIFFEIDFromState(t *testing.T) {
 					RawPath: "workload/wl1",
 				},
 			},
-			expectID: false,
+			wantID: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: tt.urls}}}
 			id := SPIFFEIDFromState(state)
-			if got, want := id != nil, tt.expectID; got != want {
-				t.Errorf("want expectID = %v, but SPIFFE ID is %v", want, id)
+			if got, want := id != nil, tt.wantID; got != want {
+				t.Errorf("want wantID = %v, but SPIFFE ID is %v", want, id)
+			}
+		})
+	}
+}
+
+func (s) TestSPIFFEIDFromCert(t *testing.T) {
+	tests := []struct {
+		name     string
+		dataPath string
+		// If we expect a SPIFFE ID to be returned.
+		wantID bool
+	}{
+		{
+			name:     "good certificate with SPIFFE ID",
+			dataPath: "x509/spiffe_cert.pem",
+			wantID:   true,
+		},
+		{
+			name:     "bad certificate with SPIFFE ID and another URI",
+			dataPath: "x509/multiple_uri_cert.pem",
+			wantID:   false,
+		},
+		{
+			name:     "certificate without SPIFFE ID",
+			dataPath: "x509/client1_cert.pem",
+			wantID:   false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			data, err := ioutil.ReadFile(testdata.Path(tt.dataPath))
+			if err != nil {
+				t.Fatalf("ioutil.ReadFile(%s) failed: %v", testdata.Path(tt.dataPath), err)
+			}
+			block, _ := pem.Decode(data)
+			if block == nil {
+				t.Fatalf("Failed to parse the certificate: byte block is nil")
+			}
+			cert, err := x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				t.Fatalf("x509.ParseCertificate(%b) failed: %v", block.Bytes, err)
+			}
+			uri := SPIFFEIDFromCert(cert)
+			if (uri != nil) != tt.wantID {
+				t.Fatalf("wantID got and want mismatch, got %t, want %t", uri != nil, tt.wantID)
+			}
+			if uri != nil && uri.String() != wantURI {
+				t.Fatalf("SPIFFE ID not expected, got %s, want %s", uri.String(), wantURI)
 			}
 		})
 	}
diff --git a/testdata/x509/create.sh b/testdata/x509/create.sh
index ecfcfd19f544..5bd3c5801e96 100755
--- a/testdata/x509/create.sh
+++ b/testdata/x509/create.sh
@@ -100,5 +100,24 @@ openssl x509 -req           \
   -extensions test_client
 openssl verify -verbose -CAfile client_ca_cert.pem  client2_cert.pem
 
+# Generate a cert with SPIFFE ID.
+openssl req -x509                                                         \
+  -newkey rsa:4096                                                        \
+  -keyout spiffe_key.pem                                                  \
+  -out spiffe_cert.pem                                                    \
+  -nodes                                                                  \
+  -days 3650                                                              \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/                         \
+  -addext "subjectAltName = URI:spiffe://foo.bar.com/client/workload/1"
+
+# Generate a cert with SPIFFE ID and another SAN URI field(which doesn't meet SPIFFE specs).
+openssl req -x509                                                         \
+  -newkey rsa:4096                                                        \
+  -keyout multiple_uri_key.pem                                            \
+  -out multiple_uri_cert.pem                                              \
+  -nodes                                                                  \
+  -days 3650                                                              \
+  -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/                         \
+  -addext "subjectAltName = URI:spiffe://foo.bar.com/client/workload/1, URI:https://bar.baz.com/client"
 # Cleanup the CSRs.
 rm *_csr.pem
diff --git a/testdata/x509/multiple_uri_cert.pem b/testdata/x509/multiple_uri_cert.pem
new file mode 100644
index 000000000000..97637997e3dd
--- /dev/null
+++ b/testdata/x509/multiple_uri_cert.pem
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIFzjCCA7agAwIBAgIUI8r7b2hX9DRwEQGWuRdk32eU5kowDQYJKoZIhvcNAQEL
+BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMDEwMDcwNjQx
+MTRaFw0zMDEwMDUwNjQxMTRaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM
+MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu
+dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCm/zjNYkfCTcq7tnVf
+qkPEde1+M6s2z05iWDfoBeZfC2NwUxIBqAC6XTXTxqYSjEVRCQUzjVxyWQNiwuz7
+pK/xGZhP/Ih2uSQKTw8vkXay4HCOt9DR0S/XGcQNImdbawKgnGven8Jrg8UZDXrt
+9R9Z0nRajB1eXvXOsEEoEfOnYthc6P+MxWJc0lnfaTlowyEgv84Ha13y1h46W6yC
++WNBT/kWqp/mzDTv/Ima8xcqEft9VUZ82qJ1DVt1064x8KOzm2x7F7QSIcjxr39M
+fbASm8Vdnt10XfhdsDVkxTlBJs8WKGn0uw8MyPNjFG01OpYDHLAfJTL3XlvaUjfF
+yDMFsRDVjfkuYIkqAVWQ7eleFfOFBYzaVf2K+2OvCR+vGAPa5NQ59kwogJYLjV/O
+43axChBizcPM0p7gmRhhO7TQz7LLTea30rBJ/YtXdxFR11y9Jdq+i2KwWi8O50iO
+hzxUBkbcQD/W9Bcn7gOkD/pgEGynWvFSs+UHjLeyL0COk0NiuYIMlOgwtI5BGwzD
+bdLuTU/ZQm4BJBjEIGVHFqKyTqUXcw5t9fWxH8V0XNs8zqj9J7lvNKu9b88GnyaJ
+fKMdDO4rVTJHmvFDHP9MUJHC9SabW8+hK0nuU7n3+Pc07ToCAan+Ych5bQHsRMjI
+9EvxKVNfwIwNrmRr3mhbOU9xIQIDAQABo4GjMIGgMB0GA1UdDgQWBBS6jnt9IccJ
+SOuE1KwP68VCBPB4hTAfBgNVHSMEGDAWgBS6jnt9IccJSOuE1KwP68VCBPB4hTAP
+BgNVHRMBAf8EBTADAQH/ME0GA1UdEQRGMESGJnNwaWZmZTovL2Zvby5iYXIuY29t
+L2NsaWVudC93b3JrbG9hZC8xhhpodHRwczovL2Jhci5iYXouY29tL2NsaWVudDAN
+BgkqhkiG9w0BAQsFAAOCAgEAoR4LbmvtSXLiVg7BRilvSxIWgcG6AI75/afuaM20
+PUTpyDhnrPxEaytb5LP0w42BCMoIHXDLE0Jmbxqbi+ku/Qw1R6723J7gwRSUYIg1
+a2S5Gue4AFp7aSLDUZhl0jPphq7OMKozzH5TrDgjKljYjPURClc/ODSlGdzOqlif
+CbDHwrCorb+BFM3aFDE0pF06pnMDXcn/Ob9QCLIpvZEOWe/fJbPtTUiA5cY3knne
+regyhvfqfVZtU52qg+9o6q5QchVqOt19alAsISK9/H4iVE+S79AiYEAU4yM4S6p5
+VW44idy3KXmr5kyVwJhe3t9f5Ckuswmo6hL32ec6M52ElrS8Er0vFt4bjfNgq996
+lTm4/reL/Anko9chQiGBe7F8J82OfxjLoVH9CbZjIoS4LiZPkey3Ze9HUV1sHhM/
+umkL54jRsVjEwwSCIcF9onzmiD8D7FV3AQ9W/RbBF3wZvVBBs9ZKQCxek2pZX/eZ
+Q+BvXwG7NGArowpqbi+tSrW3O+XZzY7nXbbf23jCBwkBn3jvqn1Kwsr/T/HbXUaz
+dDUvkwgyrX7NfvvZ20svtKLlBZTO5D8P9fy0+cHsS0XkPhw6UbHk396hoOmVZ+OG
+E5uVb2sBy+vx+82IwVzWN0o7380AEmAA5nrA6fMaxTxmo07pOF7avAZ34LgHJIjr
+sTM=
+-----END CERTIFICATE-----
diff --git a/testdata/x509/multiple_uri_key.pem b/testdata/x509/multiple_uri_key.pem
new file mode 100644
index 000000000000..c2918fdd65d0
--- /dev/null
+++ b/testdata/x509/multiple_uri_key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCm/zjNYkfCTcq7
+tnVfqkPEde1+M6s2z05iWDfoBeZfC2NwUxIBqAC6XTXTxqYSjEVRCQUzjVxyWQNi
+wuz7pK/xGZhP/Ih2uSQKTw8vkXay4HCOt9DR0S/XGcQNImdbawKgnGven8Jrg8UZ
+DXrt9R9Z0nRajB1eXvXOsEEoEfOnYthc6P+MxWJc0lnfaTlowyEgv84Ha13y1h46
+W6yC+WNBT/kWqp/mzDTv/Ima8xcqEft9VUZ82qJ1DVt1064x8KOzm2x7F7QSIcjx
+r39MfbASm8Vdnt10XfhdsDVkxTlBJs8WKGn0uw8MyPNjFG01OpYDHLAfJTL3Xlva
+UjfFyDMFsRDVjfkuYIkqAVWQ7eleFfOFBYzaVf2K+2OvCR+vGAPa5NQ59kwogJYL
+jV/O43axChBizcPM0p7gmRhhO7TQz7LLTea30rBJ/YtXdxFR11y9Jdq+i2KwWi8O
+50iOhzxUBkbcQD/W9Bcn7gOkD/pgEGynWvFSs+UHjLeyL0COk0NiuYIMlOgwtI5B
+GwzDbdLuTU/ZQm4BJBjEIGVHFqKyTqUXcw5t9fWxH8V0XNs8zqj9J7lvNKu9b88G
+nyaJfKMdDO4rVTJHmvFDHP9MUJHC9SabW8+hK0nuU7n3+Pc07ToCAan+Ych5bQHs
+RMjI9EvxKVNfwIwNrmRr3mhbOU9xIQIDAQABAoICADn4UuGJAlwC4SN0XR5OXqPu
+Q/kROpgWMqGU+iNDGQtZSrWNQKzugwIupSbUyIWbx9wvg2y336WaHMDF5bodGy5Y
+sjTh9wUvk8E4XI8oscm6e5gvWv/a2/6RZSsiDDsB1LGoWxG256im316o/UlpU+68
+TcO46+D8mdub96JPSQOMHotyHnPheRm7s5MIVfN1+SQDMSQGM2C+z1N2y1XT+I6N
+kmw54rQdoyrDwYjWZe4mu+RwG73vr4Ful5c5WjjfzhPlGi1ItyusKrMrNsd4wgxT
+opmzMjDZBgSPzJkklZF2RWDtuopH/Rt1DngQeTCHG9gMt164bQ7N5JjO/alcq8j4
+TW/IRlZOllqJ0KogOn9nX2ce9Kfxz+H36Yj54sKuOOYvKRsoiTNdTD3D6eB7pwnQ
+KGWAGrpU4llbzotiG5NJ8sDHYUwynmhfmwIeBjq0vuXlITLQplGYQnsQJI29Py3N
+KWBOC9HaiKCKq2gAUacj8BK+BLeGEiV9sxWQb7/MbWRxXnW4KhNI8+ft+PZOuvZZ
+vLxH0wg4/bYQISMaeaqWL4LksKtV7es4MglFdCCZGDMdy1/btIHjRFPQWwIaXxij
+2OtCozfmmzIc76UQ8g506q4rSgzZclDvI3Yd3cm3XFl4cQfr5l8WTQ313wrlmo5U
+DjYdKipOGFRSLHt7aXABAoIBAQDY4KyfuCHFMqKC0FJZUCr3/gGU0aqZqHR4l5jl
+N0TsTuwCRf4lK9BuM1bqumv6Nbi6VwWmp3+BzZCI/Nn7+s6KN5hyulBd56X7owef
+zl9yWW0n6nJxKzutiH06krjmODtO44gLjR7ddcEd+i023hwIQffdAE6IEtYuuoD4
+94pKd+dB9GQmgITwjS5vEP67A0lpFlL6pNMfhhe2QOLUnDPKsgSnKgwJsbBYC19j
+TQUpgFh4iCYKSGAX4ABdKpOUjbKGqNGrZNPQv+4MS9u6s/HWN7yDaSMT/tB9n1MC
+g8m7crWyOuNJ5oO6SPnetkdTbcam7tiCce/auqjx2cMJ2eDBAoIBAQDFHxPHXP6Z
+FHxI2pYBFyUB7j0VipwG3105ujrJJWu2abFU778SrkmM16eaWHVH6tMvuAo24mP1
+6Qfi09uAjdwhRPmIfManxj5wpDafgvG7H5g7+VhY2/IXTahO46JuZAxVoiXUGmct
+WwmOy0vpI2IxoXY8qLvaJv+b9nLpNi1PVJ743BmPMqG3dInoRAIBxMMEu6Drrbj3
+bjPmRNpqhs7/Kn4IahCalD6lgSBkDuz7DaJji8jINw5OhiL9VU1eslXmGrCbZMXv
+1QG0EjAZvGzqWPL88mKYTecndP1k9DMqVBVGhCT2dW1aLypQgDCC5YxyRc5vOnZ+
+2vQpELPeS0hhAoIBABBEqi5A7aeRKMePQN4aOV7o2s2C/L0R+cqh9IIdJzpioSl6
+fpnjM3tQtpBc84SNSxIPPQlHPzVJajIcZW2VXrDXgsP4XdbtbXH2xLekD1zQgHOi
+DnuWtp9JwbsHDn+WcDx2rNnQ+CO8lYPeJE4dUxT7fdBCGaHzZ8WRj+MdDm6Pl/VG
+k8yfj1lL/dOu/qygjn0ng4nxmzSeJmExdNJl9SybNeYkLUr83TF9iOY1/NEkI37H
+F7Nlwm+ICf7zFqbqCh43w6KLqafa/cxGVHEo1lcvTyC8Xjk9v/3sWZmysQsyi5aW
+/D2q4O60Uqn2GluTvHcBK5R9X3SU099wakTu5wECggEAUljjOFu++FA4g27dT2NN
+0HqoBgG7oJtbJKyJtlHtp2yL6kGlfrZUf4PvvmjJxdtxkfO+QKNewvIwmy+J+TBK
+D5Py8nO9wYTtvLy9HPHk7hkKzbMilyx6/AUzFJG/34HoLTXpu6u0ApyPZ5nCAokH
+klgzPq/2mfHEwnC4HHjHgOaG6st32fx61lrW6bLPa9G47pc7aHlQVf0xrTaCUBI1
+Ex+7OuSkPw9DBHzm/SXHFjHh7tgMbqehUGh04YPrKG4zuEbaFHCKx+AiMAmREo9G
+qLez+rt/OMUCldcnrC7f2QT7RlQZ5OO1ZQFjGfITUft3Kp3C2XCA5AmwCh+yJGEq
+wQKCAQANvxxFh6VvjU2+rB8Q4mDzYdr9OFTWMag3SNjBwwWoSXbL2wXPE5gFpzKj
+yvEbjmOgzIRABt6Eytx32p0pC5UFIey5PNu+/4ejxiiQdKSLQbqQavKYdfGgyZ0/
+JVqNKiiEJ0b9VtqhAG+Ye1mHZIBzXncWyBSZtxUGVuLG29uKbBo4ufyKauPd3dDv
+wR+JqEmAg0ICIFR+q81dEWY/gKsyyI5hMYTTsWge3l3FAdwMZEn9Ek0nclSb3dev
+ZiVlFvMZPdp5IwZljClRxnyto7bOTw+X/RMuVLB6p+v3URY4oUSL15+RNODn/tWM
+zJOG+48NgohVKfBhGN7JyxV1dq/X
+-----END PRIVATE KEY-----
diff --git a/testdata/x509/spiffe_cert.pem b/testdata/x509/spiffe_cert.pem
new file mode 100644
index 000000000000..0ce0a1aa190c
--- /dev/null
+++ b/testdata/x509/spiffe_cert.pem
@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFsjCCA5qgAwIBAgIUPxiyjwxoDyMeRvl4g9TSdvLlCA0wDQYJKoZIhvcNAQEL
+BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL
+BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMDEwMDYxNzI2
+MzFaFw0zMDEwMDQxNzI2MzFaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM
+MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu
+dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCV84YR/EV55qfFynHh
+QvWEZW5hUI9q0DeD5kG5CarrkOj11rZuQIBZ7X23CJbeoVbrvbYLghsPYJzxS/n3
+Qlwwzb5k+L0Qt+HrBD836HcSK5k1oh0jGGMaGownap+XCZH9g52s/8iiwfI02CmN
+TbwsNp7wtSEFgNOd2OlzhT6wBLF2Q6uxfBmsDpiChxe2Fs1lyan9RH8fYEf7sxwP
+E+SgBfEs7dSG5ZwFfdF+pd1T3IfrVjIxechKO1MO7HTSxbOTj6eHf1NeErDTGPA7
+VrnDCupRgcDGyAhFd54r62R8TbTjn5MwzMxElO45Ck/Ej7Qw/GWeaBHj/dMa6mhE
+R55PvnKuyj+k9t0Rf0HDZyONtY5/OLqI/xVr27Y1o9v5FysNgjWPkZMRpvuCzkeC
+2RuE6k2TfBDRLiCyYu/Zzw+ZtUyTAKtWtefLdQBjrYpnhrDPpmrnTWomX/e9pylE
+WfkyxCswiPnDw7ypI7uFSTkz0+bUaROmAtlPvR+3SjaQDWigwz3eJsdIaeg5AY9q
+//rWaal6l2iR0Ou9L6A9lLxh5iN/ch+OGk4QPK6pFbOy3IqYfmQ+IpAXG0da9RT2
+EN76cNa3bldEjRRON8oQ3HZmhOQJqVxhQciUz84sTjAqH8WvqqbdG9HKUoZ19T5Z
+9vNldjlQn33Mi5gBxdugqdnmCQIDAQABo4GHMIGEMB0GA1UdDgQWBBT8rr0kPapk
+bGLJ4EU1582sw7WlOTAfBgNVHSMEGDAWgBT8rr0kPapkbGLJ4EU1582sw7WlOTAP
+BgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29t
+L2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA15Ne+Lz5cN1/B
+fkys4QHDWJ0n5Zy9OtwSW6aTyqIIwls6OOSkJn3qJMoT2oFvoHoOxb0swyN+zUoD
+pmPEd7FHkMm8BhRqoyH3UZGR7kOSIIcfvldVZbW9mD88A04qvLsWkkanMyGhkYV4
+0TXyb8USdjeNm1H32iF4k24czSpvoOYo9HOQv+4aFcqTMnGwS7CvwU6O6vVU8gIy
+HYP/oWnkhap6X7acjPxYoW5IDZdN9vdMz9wQlKlc799lWqOCuwl68NSuTNcNNFyn
+TXfFWZaghb7iXsUezGYTY9glsPxY0Egmbcmxut0gz0U2BNVvNGKUUu55MlAS7yXO
+Y7eTfSSf6DJesFQKwTg8qlyNLjzbLSmhvz6EPV55ToUxPPA9CIOrWQwXv4GdySuH
+bwof3U5p/cq2NDtxv8KGisjK04l++s+Ea8AS6T6O8+08nBFGgfNW331eWtU91JoQ
+e6Q4DWipiNzkIvISk48V8CT9eRB2KD7NsigQprePRN3gDZREh+01gwbVUX2gbtHx
+1RGxEjO6H0kUuaoXF5E6+WGwgn8MA47qUy1WXC5QDFpc5LyaoVaMFv8bcoWSNXAS
+Oes+ZDWDXWq6F+9Kt0zWmO651cVquLTjmgt48fgL6m8rU13ikjH7dFnimrwRxfOD
+p+z97N7TvWfgE1HOmYDfsbaHjPFZKg==
+-----END CERTIFICATE-----
diff --git a/testdata/x509/spiffe_key.pem b/testdata/x509/spiffe_key.pem
new file mode 100644
index 000000000000..5462d66326b1
--- /dev/null
+++ b/testdata/x509/spiffe_key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCV84YR/EV55qfF
+ynHhQvWEZW5hUI9q0DeD5kG5CarrkOj11rZuQIBZ7X23CJbeoVbrvbYLghsPYJzx
+S/n3Qlwwzb5k+L0Qt+HrBD836HcSK5k1oh0jGGMaGownap+XCZH9g52s/8iiwfI0
+2CmNTbwsNp7wtSEFgNOd2OlzhT6wBLF2Q6uxfBmsDpiChxe2Fs1lyan9RH8fYEf7
+sxwPE+SgBfEs7dSG5ZwFfdF+pd1T3IfrVjIxechKO1MO7HTSxbOTj6eHf1NeErDT
+GPA7VrnDCupRgcDGyAhFd54r62R8TbTjn5MwzMxElO45Ck/Ej7Qw/GWeaBHj/dMa
+6mhER55PvnKuyj+k9t0Rf0HDZyONtY5/OLqI/xVr27Y1o9v5FysNgjWPkZMRpvuC
+zkeC2RuE6k2TfBDRLiCyYu/Zzw+ZtUyTAKtWtefLdQBjrYpnhrDPpmrnTWomX/e9
+pylEWfkyxCswiPnDw7ypI7uFSTkz0+bUaROmAtlPvR+3SjaQDWigwz3eJsdIaeg5
+AY9q//rWaal6l2iR0Ou9L6A9lLxh5iN/ch+OGk4QPK6pFbOy3IqYfmQ+IpAXG0da
+9RT2EN76cNa3bldEjRRON8oQ3HZmhOQJqVxhQciUz84sTjAqH8WvqqbdG9HKUoZ1
+9T5Z9vNldjlQn33Mi5gBxdugqdnmCQIDAQABAoICADWoJXJsHgRHyAMbtPJRPn94
+uC20YQ1somDdVOk8j1+pw+KsSS1cgVEsjU6gkTPq8ap7gRfPH5W6EY66jCCxK0H/
+bUC+TREda4boRyLfWTQ0S6eIcfqr8FJX64zzN1YZg5b+sL5F7Opokh3ct8mrZkk/
+5lHlzoIknhSemLLQnCTqGQJjpp1k9d6+fk4+vvpWYHsq1VweVYrJrhhf+AthJ+8n
+ESztkZ4PrWu9oOg7u94VTMGmX2Ga3VPKtKbjb844FlEYF2+B3TgNYh63jsb8+o3T
+axNtZaj7zRHmgr/ehF+CgtbstAPDVNi5niDlErQYY/cfadFsFfLKUe8Qr+y23+vG
+1WuVSUmrUcgO/IYMIz2gEOrBOutc9cdKOlCnwrXu3WjSGO6zhcbXCw7WZrSR/Uj5
+1Tatt5QJ5Z3i4vOc6Jj1XKL/9Xa+FEryfVh/HKlQTlHnIuuGXMBpIzyYQ6kY8+cH
+n75FVMo4lB97c48hweupQY6SUQwvWXqXQOAxLJ/eq2k1QpUWJ4GV5kRr3/eQ/AZ1
+y4Kk2ZxM8IWksFdVnomNr65GIk219D1uwDtJQeBrwqrYseGq/2mB2h4llTbwjSez
+GkOPO74tLPh3wkG8wDzbc94nfouxCL6ee9W4XeDGzYXgndSKAPOWWUyFnsIxisVu
+BB2HUkJZotG2Otrgnj/xAoIBAQDF7NjT9JkN+JhrmH4jG+16lI6RDErf/VgheSE8
+G/ayAg1RGY2FsuyTi2bAM3xprXqaZHDSikil8t9G7JJZoKVzesCZR+OJU+9fGZR6
+TCS6mCdv6OEG18GJP9dzDLaqYybJ9VgnSnXT7mRlcCyU8uZ9/2FR/vnLEatbOw99
+2tles0LGdkKT/YIYdxIENIpmaZhzAOWPDLDwDTO8aIXy40DiGzjLWJf/0LgCV7Ub
+2C8aPS4WWXCOdAeYvjcEKeCp9+YSgZaYNT2P8Ns40VcL/yysMtFzvnTOcA9riyzN
+5pu+ppv/KYGt/ENa4zQMCgKFtTUxicr2M9VYqNI/CgJmJPKVAoIBAQDB8x13soMv
+tohfiNwGTjEkVzu/RkTCix8+hERF1C+oupL3ykpJOGvvpOyhFUqwFAYyBQKflrxj
+9lQBKQiPYR0VtgDFJ7UzjYGO8zt8U56gTcYeatpNKY4zvZyGAOBhtWuvuvi8pnpc
+xO8yQjE0jWrwWL3bmf/5lP2jO8j+k1qZfrA2ksTGUWRGEZkFRqqTQxvRrJVr9QiY
+2xpRq/n7fq9UhCfNxm2aLdYgZ/BCVFzNahCEWfFdH3jOcP5N/5Fkiy7IhwfdglO+
+JjydEMqBYHg3ET6MQ3JYM/Gt/GkX0myV9BHd/fYYF2xGBMDIMLVdFKyCivUMGGt1
+pHhcNLzebFylAoIBAQDA4Cr4eh8Az2XxTDx3iEqnLsezr8/zcVYF4J2zjuib1YYW
+pxkT1iXXLnymBkZSUVztwb10Xo+nMAPHgNipgPRakZ/If5bLh8D34tyfRT5xm76q
+vr0zRuPyFQWmtxf2+QKewnjyaQxjx6eMdoDrcb2NwWWcWyYfbwuWrvpMwg0bzQLg
+lfQRdXTm1Hn5IR5R6MtIHvKVsV9nvuXQz+bgp/bdoHt7Jc2R3FrE5aW3Cbf1EPOt
+keEu4QFaJttEMm8eE1bgZ+pST2e7spJfTxlNtpBZCni0G0CGwAs22PyDdhwF8SSJ
+xm/6FZ+pnUlmBgcpN0osCUSBIkfgyzt/dQibc5v1AoIBAF5ZLweQfoLSb+rRf/9N
+QFimWvlEbKSa2vslirTRcNHK2T3TWWnfGZq9hyMhYXDgfNcOWuVZhZG3PcxGstRU
+8LokDKHcHCjU+KaaqmBjqTHgQ7V+U23f/j4rSh5iBMVjZNxavy++aJ4Caz3ut1MS
+TGhZMxrGAqDeGriyl6dH9XXgDEawBStYYsg3PVI0uzviFIFeTF31GFaLl3UNjREL
+4qzhkR9oHN841wZyqY0Kzw5aP2iy/FhJvBHpI7y7y3W2w25nSas3ABfrL+dUSL7B
+OBnJuLyw/snrkvEJbfJZudsEnUB5j6LOmixBmaqJD2EVcooaoPReWMAk3ywzt4EY
+A8UCggEBAJbArCsQ0Q+pFOEce6JlgtYAdcBiu8n5zFYLD2/qZdM2ir/09uj9XDpC
+WbE5YTumgzkt2VLK/wf+HATMAyXObtaAn5AdoA6OJ1AvbNGOwA3F2LOlbJGO2XOW
+TpQlgbDvBBktaKk9PSszDj92W2tdFQPefDP/uBzonymev7BWCERZ/vU2L3HpXQjp
+bxzRyVNWwwg3VBYvbCIz8v2yeviAsiEkOCPRU6+cIr7/VUr0mymzDgfzKaQNoOap
+LqOpnInw0pUA+BhsVr4n/fBXZLbSd7ZG5WU48HUaSLefEI4NuiZLT2K1tZreyBqZ
+Xgln2zbN6APAb+dGDdv27dz4YlasU4s=
+-----END PRIVATE KEY-----

From 3c400e7fcc8719ba3d133563f9c9d126f03a80c0 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Sat, 10 Oct 2020 13:47:49 -0700
Subject: [PATCH 230/481] advancedtls: clean up test files and shared code
 (#3897)

* advancedtls: clean up test files and shared code
---
 security/advancedtls/advancedtls.go           |  73 +-----
 .../advancedtls_integration_test.go           | 174 +++++---------
 security/advancedtls/advancedtls_test.go      | 219 +++++-------------
 .../internal/testutils/testutils.go           | 100 ++++++++
 security/advancedtls/pemfile_provider_test.go |  37 +--
 5 files changed, 234 insertions(+), 369 deletions(-)
 create mode 100644 security/advancedtls/internal/testutils/testutils.go

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index 74564632660f..ea93d64069f5 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -28,7 +28,6 @@ import (
 	"fmt"
 	"net"
 	"reflect"
-	"syscall"
 	"time"
 
 	"google.golang.org/grpc/credentials"
@@ -374,7 +373,7 @@ func (c advancedTLSCreds) Info() credentials.ProtocolInfo {
 
 func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
 	// Use local cfg to avoid clobbering ServerName if using multiple endpoints.
-	cfg := cloneTLSConfig(c.config)
+	cfg := credinternal.CloneTLSConfig(c.config)
 	// We return the full authority name to users if ServerName is empty without
 	// stripping the trailing port.
 	if cfg.ServerName == "" {
@@ -404,11 +403,11 @@ func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string
 		},
 	}
 	info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState())
-	return WrapSyscallConn(rawConn, conn), info, nil
+	return credinternal.WrapSyscallConn(rawConn, conn), info, nil
 }
 
 func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	cfg := cloneTLSConfig(c.config)
+	cfg := credinternal.CloneTLSConfig(c.config)
 	cfg.VerifyPeerCertificate = buildVerifyFunc(c, "", rawConn)
 	conn := tls.Server(rawConn, cfg)
 	if err := conn.Handshake(); err != nil {
@@ -422,12 +421,12 @@ func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credenti
 		},
 	}
 	info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState())
-	return WrapSyscallConn(rawConn, conn), info, nil
+	return credinternal.WrapSyscallConn(rawConn, conn), info, nil
 }
 
 func (c *advancedTLSCreds) Clone() credentials.TransportCredentials {
 	return &advancedTLSCreds{
-		config:     cloneTLSConfig(c.config),
+		config:     credinternal.CloneTLSConfig(c.config),
 		verifyFunc: c.verifyFunc,
 		getRootCAs: c.getRootCAs,
 		isClient:   c.isClient,
@@ -530,7 +529,7 @@ func NewClientCreds(o *ClientOptions) (credentials.TransportCredentials, error)
 		verifyFunc: o.VerifyPeer,
 		vType:      o.VType,
 	}
-	tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos)
+	tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos)
 	return tc, nil
 }
 
@@ -548,64 +547,6 @@ func NewServerCreds(o *ServerOptions) (credentials.TransportCredentials, error)
 		verifyFunc: o.VerifyPeer,
 		vType:      o.VType,
 	}
-	tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos)
+	tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos)
 	return tc, nil
 }
-
-// TODO(ZhenLian): The code below are duplicates with gRPC-Go under
-//                 credentials/internal. Consider refactoring in the future.
-const alpnProtoStrH2 = "h2"
-
-func appendH2ToNextProtos(ps []string) []string {
-	for _, p := range ps {
-		if p == alpnProtoStrH2 {
-			return ps
-		}
-	}
-	ret := make([]string, 0, len(ps)+1)
-	ret = append(ret, ps...)
-	return append(ret, alpnProtoStrH2)
-}
-
-// We give syscall.Conn a new name here since syscall.Conn and net.Conn used
-// below have the same names.
-type sysConn = syscall.Conn
-
-// syscallConn keeps reference of rawConn to support syscall.Conn for channelz.
-// SyscallConn() (the method in interface syscall.Conn) is explicitly
-// implemented on this type,
-//
-// Interface syscall.Conn is implemented by most net.Conn implementations (e.g.
-// TCPConn, UnixConn), but is not part of net.Conn interface. So wrapper conns
-// that embed net.Conn don't implement syscall.Conn. (Side note: tls.Conn
-// doesn't embed net.Conn, so even if syscall.Conn is part of net.Conn, it won't
-// help here).
-type syscallConn struct {
-	net.Conn
-	// sysConn is a type alias of syscall.Conn. It's necessary because the name
-	// `Conn` collides with `net.Conn`.
-	sysConn
-}
-
-// WrapSyscallConn tries to wrap rawConn and newConn into a net.Conn that
-// implements syscall.Conn. rawConn will be used to support syscall, and newConn
-// will be used for read/write.
-//
-// This function returns newConn if rawConn doesn't implement syscall.Conn.
-func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn {
-	sysConn, ok := rawConn.(syscall.Conn)
-	if !ok {
-		return newConn
-	}
-	return &syscallConn{
-		Conn:    newConn,
-		sysConn: sysConn,
-	}
-}
-
-func cloneTLSConfig(cfg *tls.Config) *tls.Config {
-	if cfg == nil {
-		return &tls.Config{}
-	}
-	return cfg.Clone()
-}
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index a95fa56a6d6f..3f4e7059aaf3 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -31,7 +31,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
-	"google.golang.org/grpc/security/advancedtls/testdata"
+	"google.golang.org/grpc/security/advancedtls/internal/testutils"
 )
 
 var (
@@ -67,69 +67,6 @@ func (s *stageInfo) reset() {
 	s.stage = 0
 }
 
-// certStore contains all the certificates used in the integration tests.
-type certStore struct {
-	// clientPeer1 is the certificate sent by client to prove its identity.
-	// It is trusted by serverTrust1.
-	clientPeer1 tls.Certificate
-	// clientPeer2 is the certificate sent by client to prove its identity.
-	// It is trusted by serverTrust2.
-	clientPeer2 tls.Certificate
-	// serverPeer1 is the certificate sent by server to prove its identity.
-	// It is trusted by clientTrust1.
-	serverPeer1 tls.Certificate
-	// serverPeer2 is the certificate sent by server to prove its identity.
-	// It is trusted by clientTrust2.
-	serverPeer2  tls.Certificate
-	clientTrust1 *x509.CertPool
-	clientTrust2 *x509.CertPool
-	serverTrust1 *x509.CertPool
-	serverTrust2 *x509.CertPool
-}
-
-// loadCerts function is used to load test certificates at the beginning of
-// each integration test.
-func (cs *certStore) loadCerts() error {
-	var err error
-	cs.clientPeer1, err = tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"),
-		testdata.Path("client_key_1.pem"))
-	if err != nil {
-		return err
-	}
-	cs.clientPeer2, err = tls.LoadX509KeyPair(testdata.Path("client_cert_2.pem"),
-		testdata.Path("client_key_2.pem"))
-	if err != nil {
-		return err
-	}
-	cs.serverPeer1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"),
-		testdata.Path("server_key_1.pem"))
-	if err != nil {
-		return err
-	}
-	cs.serverPeer2, err = tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"),
-		testdata.Path("server_key_2.pem"))
-	if err != nil {
-		return err
-	}
-	cs.clientTrust1, err = readTrustCert(testdata.Path("client_trust_cert_1.pem"))
-	if err != nil {
-		return err
-	}
-	cs.clientTrust2, err = readTrustCert(testdata.Path("client_trust_cert_2.pem"))
-	if err != nil {
-		return err
-	}
-	cs.serverTrust1, err = readTrustCert(testdata.Path("server_trust_cert_1.pem"))
-	if err != nil {
-		return err
-	}
-	cs.serverTrust2, err = readTrustCert(testdata.Path("server_trust_cert_2.pem"))
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 type greeterServer struct {
 	pb.UnimplementedGreeterServer
 }
@@ -183,10 +120,9 @@ func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds cred
 // (could be change the client's trust certificate, or change custom
 // verification function, etc)
 func (s) TestEnd2End(t *testing.T) {
-	cs := &certStore{}
-	err := cs.loadCerts()
-	if err != nil {
-		t.Fatalf("failed to load certs: %v", err)
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
 	}
 	stage := &stageInfo{}
 	for _, test := range []struct {
@@ -206,38 +142,38 @@ func (s) TestEnd2End(t *testing.T) {
 	}{
 		// Test Scenarios:
 		// At initialization(stage = 0), client will be initialized with cert
-		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1 is
-		// trusted by serverTrust1, and serverPeer1 by clientTrust1.
-		// At stage 1, client changes clientPeer1 to clientPeer2. Since clientPeer2
-		// is not trusted by serverTrust1, following rpc calls are expected to
+		// ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1.
+		// The mutual authentication works at the beginning, since ClientCert1 is
+		// trusted by ServerTrust1, and ServerCert1 by ClientTrust1.
+		// At stage 1, client changes ClientCert1 to ClientCert2. Since ClientCert2
+		// is not trusted by ServerTrust1, following rpc calls are expected to
 		// fail, while the previous rpc calls are still good because those are
 		// already authenticated.
-		// At stage 2, the server changes serverTrust1 to serverTrust2, and we
-		// should see it again accepts the connection, since clientPeer2 is trusted
-		// by serverTrust2.
+		// At stage 2, the server changes ServerTrust1 to ServerTrust2, and we
+		// should see it again accepts the connection, since ClientCert2 is trusted
+		// by ServerTrust2.
 		{
 			desc: "TestClientPeerCertReloadServerTrustCertReload",
 			clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
-					return &cs.clientPeer1, nil
+					return &cs.ClientCert1, nil
 				default:
-					return &cs.clientPeer2, nil
+					return &cs.ClientCert2, nil
 				}
 			},
-			clientRoot: cs.clientTrust1,
+			clientRoot: cs.ClientTrust1,
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
 			clientVType: CertVerification,
-			serverCert:  []tls.Certificate{cs.serverPeer1},
+			serverCert:  []tls.Certificate{cs.ServerCert1},
 			serverGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
 				case 0, 1:
-					return &GetRootCAsResults{TrustCerts: cs.serverTrust1}, nil
+					return &GetRootCAsResults{TrustCerts: cs.ServerTrust1}, nil
 				default:
-					return &GetRootCAsResults{TrustCerts: cs.serverTrust2}, nil
+					return &GetRootCAsResults{TrustCerts: cs.ServerTrust2}, nil
 				}
 			},
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
@@ -247,25 +183,25 @@ func (s) TestEnd2End(t *testing.T) {
 		},
 		// Test Scenarios:
 		// At initialization(stage = 0), client will be initialized with cert
-		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1 is
-		// trusted by serverTrust1, and serverPeer1 by clientTrust1.
-		// At stage 1, server changes serverPeer1 to serverPeer2. Since serverPeer2
-		// is not trusted by clientTrust1, following rpc calls are expected to
+		// ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1.
+		// The mutual authentication works at the beginning, since ClientCert1 is
+		// trusted by ServerTrust1, and ServerCert1 by ClientTrust1.
+		// At stage 1, server changes ServerCert1 to ServerCert2. Since ServerCert2
+		// is not trusted by ClientTrust1, following rpc calls are expected to
 		// fail, while the previous rpc calls are still good because those are
 		// already authenticated.
-		// At stage 2, the client changes clientTrust1 to clientTrust2, and we
-		// should see it again accepts the connection, since serverPeer2 is trusted
-		// by clientTrust2.
+		// At stage 2, the client changes ClientTrust1 to ClientTrust2, and we
+		// should see it again accepts the connection, since ServerCert2 is trusted
+		// by ClientTrust2.
 		{
 			desc:       "TestServerPeerCertReloadClientTrustCertReload",
-			clientCert: []tls.Certificate{cs.clientPeer1},
+			clientCert: []tls.Certificate{cs.ClientCert1},
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
 				case 0, 1:
-					return &GetRootCAsResults{TrustCerts: cs.clientTrust1}, nil
+					return &GetRootCAsResults{TrustCerts: cs.ClientTrust1}, nil
 				default:
-					return &GetRootCAsResults{TrustCerts: cs.clientTrust2}, nil
+					return &GetRootCAsResults{TrustCerts: cs.ClientTrust2}, nil
 				}
 			},
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
@@ -275,12 +211,12 @@ func (s) TestEnd2End(t *testing.T) {
 			serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
-					return []*tls.Certificate{&cs.serverPeer1}, nil
+					return []*tls.Certificate{&cs.ServerCert1}, nil
 				default:
-					return []*tls.Certificate{&cs.serverPeer2}, nil
+					return []*tls.Certificate{&cs.ServerCert2}, nil
 				}
 			},
-			serverRoot: cs.serverTrust1,
+			serverRoot: cs.ServerTrust1,
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
@@ -288,26 +224,26 @@ func (s) TestEnd2End(t *testing.T) {
 		},
 		// Test Scenarios:
 		// At initialization(stage = 0), client will be initialized with cert
-		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1
-		// trusted by serverTrust1, serverPeer1 by clientTrust1, and also the
-		// custom verification check allows the CommonName on serverPeer1.
-		// At stage 1, server changes serverPeer1 to serverPeer2, and client
-		// changes clientTrust1 to clientTrust2. Although serverPeer2 is trusted by
-		// clientTrust2, our authorization check only accepts serverPeer1, and
+		// ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1.
+		// The mutual authentication works at the beginning, since ClientCert1
+		// trusted by ServerTrust1, ServerCert1 by ClientTrust1, and also the
+		// custom verification check allows the CommonName on ServerCert1.
+		// At stage 1, server changes ServerCert1 to ServerCert2, and client
+		// changes ClientTrust1 to ClientTrust2. Although ServerCert2 is trusted by
+		// ClientTrust2, our authorization check only accepts ServerCert1, and
 		// hence the following calls should fail. Previous connections should
 		// not be affected.
 		// At stage 2, the client changes authorization check to only accept
-		// serverPeer2. Now we should see the connection becomes normal again.
+		// ServerCert2. Now we should see the connection becomes normal again.
 		{
 			desc:       "TestClientCustomVerification",
-			clientCert: []tls.Certificate{cs.clientPeer1},
+			clientCert: []tls.Certificate{cs.ClientCert1},
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
 				case 0:
-					return &GetRootCAsResults{TrustCerts: cs.clientTrust1}, nil
+					return &GetRootCAsResults{TrustCerts: cs.ClientTrust1}, nil
 				default:
-					return &GetRootCAsResults{TrustCerts: cs.clientTrust2}, nil
+					return &GetRootCAsResults{TrustCerts: cs.ClientTrust2}, nil
 				}
 			},
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
@@ -321,12 +257,12 @@ func (s) TestEnd2End(t *testing.T) {
 				authzCheck := false
 				switch stage.read() {
 				case 0, 1:
-					// foo.bar.com is the common name on serverPeer1
+					// foo.bar.com is the common name on ServerCert1
 					if cert.Subject.CommonName == "foo.bar.com" {
 						authzCheck = true
 					}
 				default:
-					// foo.bar.server2.com is the common name on serverPeer2
+					// foo.bar.server2.com is the common name on ServerCert2
 					if cert.Subject.CommonName == "foo.bar.server2.com" {
 						authzCheck = true
 					}
@@ -340,12 +276,12 @@ func (s) TestEnd2End(t *testing.T) {
 			serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
-					return []*tls.Certificate{&cs.serverPeer1}, nil
+					return []*tls.Certificate{&cs.ServerCert1}, nil
 				default:
-					return []*tls.Certificate{&cs.serverPeer2}, nil
+					return []*tls.Certificate{&cs.ServerCert2}, nil
 				}
 			},
-			serverRoot: cs.serverTrust1,
+			serverRoot: cs.ServerTrust1,
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
@@ -353,9 +289,9 @@ func (s) TestEnd2End(t *testing.T) {
 		},
 		// Test Scenarios:
 		// At initialization(stage = 0), client will be initialized with cert
-		// clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1.
-		// The mutual authentication works at the beginning, since clientPeer1
-		// trusted by serverTrust1, serverPeer1 by clientTrust1, and also the
+		// ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1.
+		// The mutual authentication works at the beginning, since ClientCert1
+		// trusted by ServerTrust1, ServerCert1 by ClientTrust1, and also the
 		// custom verification check on server side allows all connections.
 		// At stage 1, server disallows the the connections by setting custom
 		// verification check. The following calls should fail. Previous
@@ -364,14 +300,14 @@ func (s) TestEnd2End(t *testing.T) {
 		// authentications should go back to normal.
 		{
 			desc:       "TestServerCustomVerification",
-			clientCert: []tls.Certificate{cs.clientPeer1},
-			clientRoot: cs.clientTrust1,
+			clientCert: []tls.Certificate{cs.ClientCert1},
+			clientRoot: cs.ClientTrust1,
 			clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				return &VerificationResults{}, nil
 			},
 			clientVType: CertVerification,
-			serverCert:  []tls.Certificate{cs.serverPeer1},
-			serverRoot:  cs.serverTrust1,
+			serverCert:  []tls.Certificate{cs.ServerCert1},
+			serverRoot:  cs.ServerTrust1,
 			serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) {
 				switch stage.read() {
 				case 0, 2:
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index a631ee465d3d..a7ecf276da14 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -22,21 +22,17 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/pem"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"net"
-	"reflect"
-	"syscall"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/grpctest"
-	"google.golang.org/grpc/security/advancedtls/testdata"
+	"google.golang.org/grpc/security/advancedtls/internal/testutils"
 )
 
 type s struct {
@@ -65,27 +61,26 @@ func (f fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMateria
 	if f.wantError {
 		return nil, fmt.Errorf("bad fakeProvider")
 	}
-	cs := &certStore{}
-	err := cs.loadCerts()
-	if err != nil {
-		return nil, fmt.Errorf("failed to load certs: %v", err)
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		return nil, fmt.Errorf("cs.LoadCerts() failed, err: %v", err)
 	}
 	if f.pt == provTypeRoot && f.isClient {
-		return &certprovider.KeyMaterial{Roots: cs.clientTrust1}, nil
+		return &certprovider.KeyMaterial{Roots: cs.ClientTrust1}, nil
 	}
 	if f.pt == provTypeRoot && !f.isClient {
-		return &certprovider.KeyMaterial{Roots: cs.serverTrust1}, nil
+		return &certprovider.KeyMaterial{Roots: cs.ServerTrust1}, nil
 	}
 	if f.pt == provTypeIdentity && f.isClient {
 		if f.wantMultiCert {
-			return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1, cs.clientPeer2}}, nil
+			return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1, cs.ClientCert2}}, nil
 		}
-		return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}}, nil
+		return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}}, nil
 	}
 	if f.wantMultiCert {
-		return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.serverPeer1, cs.serverPeer2}}, nil
+		return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ServerCert1, cs.ServerCert2}}, nil
 	}
-	return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.serverPeer1}}, nil
+	return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ServerCert1}}, nil
 }
 
 func (f fakeProvider) Close() {}
@@ -308,13 +303,12 @@ func (s) TestServerOptionsConfigSuccessCases(t *testing.T) {
 }
 
 func (s) TestClientServerHandshake(t *testing.T) {
-	cs := &certStore{}
-	err := cs.loadCerts()
-	if err != nil {
-		t.Fatalf("Failed to load certs: %v", err)
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
 	}
 	getRootCAsForClient := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
-		return &GetRootCAsResults{TrustCerts: cs.clientTrust1}, nil
+		return &GetRootCAsResults{TrustCerts: cs.ClientTrust1}, nil
 	}
 	clientVerifyFuncGood := func(params *VerificationFuncParams) (*VerificationResults, error) {
 		if params.ServerName == "" {
@@ -331,7 +325,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		return nil, fmt.Errorf("custom verification function failed")
 	}
 	getRootCAsForServer := func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
-		return &GetRootCAsResults{TrustCerts: cs.serverTrust1}, nil
+		return &GetRootCAsResults{TrustCerts: cs.ServerTrust1}, nil
 	}
 	serverVerifyFunc := func(params *VerificationFuncParams) (*VerificationResults, error) {
 		if params.ServerName != "" {
@@ -378,7 +372,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			desc:             "Client has no trust cert with verifyFuncGood; server sends peer cert",
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      SkipVerification,
-			serverCert:       []tls.Certificate{cs.serverPeer1},
+			serverCert:       []tls.Certificate{cs.ServerCert1},
 			serverVType:      CertAndHostVerification,
 		},
 		// Client: only set clientRoot
@@ -389,10 +383,10 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// this test suites.
 		{
 			desc:                       "Client has root cert; server sends peer cert",
-			clientRoot:                 cs.clientTrust1,
+			clientRoot:                 cs.ClientTrust1,
 			clientVType:                CertAndHostVerification,
 			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{cs.serverPeer1},
+			serverCert:                 []tls.Certificate{cs.ServerCert1},
 			serverVType:                CertAndHostVerification,
 			serverExpectError:          true,
 		},
@@ -407,7 +401,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientGetRoot:              getRootCAsForClient,
 			clientVType:                CertAndHostVerification,
 			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{cs.serverPeer1},
+			serverCert:                 []tls.Certificate{cs.ServerCert1},
 			serverVType:                CertAndHostVerification,
 			serverExpectError:          true,
 		},
@@ -419,7 +413,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
-			serverCert:       []tls.Certificate{cs.serverPeer1},
+			serverCert:       []tls.Certificate{cs.ServerCert1},
 			serverVType:      CertAndHostVerification,
 		},
 		// Client: set clientGetRoot and bad clientVerifyFunc function
@@ -432,7 +426,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientVerifyFunc:           verifyFuncBad,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{cs.serverPeer1},
+			serverCert:                 []tls.Certificate{cs.ServerCert1},
 			serverVType:                CertVerification,
 			serverExpectError:          true,
 		},
@@ -441,13 +435,13 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// Expected Behavior: success
 		{
 			desc:             "Client sets peer cert, reload root function with verifyFuncGood; server sets peer cert and root cert; mutualTLS",
-			clientCert:       []tls.Certificate{cs.clientPeer1},
+			clientCert:       []tls.Certificate{cs.ClientCert1},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverCert:       []tls.Certificate{cs.serverPeer1},
-			serverRoot:       cs.serverTrust1,
+			serverCert:       []tls.Certificate{cs.ServerCert1},
+			serverRoot:       cs.ServerTrust1,
 			serverVType:      CertVerification,
 		},
 		// Client: set clientGetRoot, clientVerifyFunc and clientCert
@@ -455,12 +449,12 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// Expected Behavior: success
 		{
 			desc:             "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS",
-			clientCert:       []tls.Certificate{cs.clientPeer1},
+			clientCert:       []tls.Certificate{cs.ClientCert1},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
-			serverCert:       []tls.Certificate{cs.serverPeer1},
+			serverCert:       []tls.Certificate{cs.ServerCert1},
 			serverGetRoot:    getRootCAsForServer,
 			serverVType:      CertVerification,
 		},
@@ -471,12 +465,12 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// Reason: server side reloading returns failure
 		{
 			desc:              "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, bad reload root function; mutualTLS",
-			clientCert:        []tls.Certificate{cs.clientPeer1},
+			clientCert:        []tls.Certificate{cs.ClientCert1},
 			clientGetRoot:     getRootCAsForClient,
 			clientVerifyFunc:  clientVerifyFuncGood,
 			clientVType:       CertVerification,
 			serverMutualTLS:   true,
-			serverCert:        []tls.Certificate{cs.serverPeer1},
+			serverCert:        []tls.Certificate{cs.ServerCert1},
 			serverGetRoot:     getRootCAsForServerBad,
 			serverVType:       CertVerification,
 			serverExpectError: true,
@@ -487,14 +481,14 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sets reload peer/root function with verifyFuncGood; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &cs.clientPeer1, nil
+				return &cs.ClientCert1, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&cs.serverPeer1}, nil
+				return []*tls.Certificate{&cs.ServerCert1}, nil
 			},
 			serverGetRoot:    getRootCAsForServer,
 			serverVerifyFunc: serverVerifyFunc,
@@ -508,14 +502,14 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sends wrong peer cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &cs.serverPeer1, nil
+				return &cs.ServerCert1, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&cs.serverPeer1}, nil
+				return []*tls.Certificate{&cs.ServerCert1}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -529,7 +523,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client has wrong trust cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &cs.clientPeer1, nil
+				return &cs.ClientCert1, nil
 			},
 			clientGetRoot:              getRootCAsForServer,
 			clientVerifyFunc:           clientVerifyFuncGood,
@@ -537,7 +531,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&cs.serverPeer1}, nil
+				return []*tls.Certificate{&cs.ServerCert1}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -552,14 +546,14 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sets reload peer/root function with verifyFuncGood; Server sends wrong peer cert; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &cs.clientPeer1, nil
+				return &cs.ClientCert1, nil
 			},
 			clientGetRoot:    getRootCAsForClient,
 			clientVerifyFunc: clientVerifyFuncGood,
 			clientVType:      CertVerification,
 			serverMutualTLS:  true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&cs.clientPeer1}, nil
+				return []*tls.Certificate{&cs.ClientCert1}, nil
 			},
 			serverGetRoot:     getRootCAsForServer,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -573,7 +567,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		{
 			desc: "Client sets reload peer/root function with verifyFuncGood; Server has wrong trust cert; mutualTLS",
 			clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-				return &cs.clientPeer1, nil
+				return &cs.ClientCert1, nil
 			},
 			clientGetRoot:              getRootCAsForClient,
 			clientVerifyFunc:           clientVerifyFuncGood,
@@ -581,7 +575,7 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
 			serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-				return []*tls.Certificate{&cs.serverPeer1}, nil
+				return []*tls.Certificate{&cs.ServerCert1}, nil
 			},
 			serverGetRoot:     getRootCAsForClient,
 			serverVerifyFunc:  serverVerifyFunc,
@@ -594,13 +588,13 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// server custom check fails
 		{
 			desc:                       "Client sets peer cert, reload root function with verifyFuncGood; Server sets bad custom check; mutualTLS",
-			clientCert:                 []tls.Certificate{cs.clientPeer1},
+			clientCert:                 []tls.Certificate{cs.ClientCert1},
 			clientGetRoot:              getRootCAsForClient,
 			clientVerifyFunc:           clientVerifyFuncGood,
 			clientVType:                CertVerification,
 			clientExpectHandshakeError: true,
 			serverMutualTLS:            true,
-			serverCert:                 []tls.Certificate{cs.serverPeer1},
+			serverCert:                 []tls.Certificate{cs.ServerCert1},
 			serverGetRoot:              getRootCAsForServer,
 			serverVerifyFunc:           verifyFuncBad,
 			serverVType:                CertVerification,
@@ -776,24 +770,6 @@ func (s) TestClientServerHandshake(t *testing.T) {
 	}
 }
 
-func readTrustCert(fileName string) (*x509.CertPool, error) {
-	trustData, err := ioutil.ReadFile(fileName)
-	if err != nil {
-		return nil, err
-	}
-	trustBlock, _ := pem.Decode(trustData)
-	if trustBlock == nil {
-		return nil, err
-	}
-	trustCert, err := x509.ParseCertificate(trustBlock.Bytes)
-	if err != nil {
-		return nil, err
-	}
-	trustPool := x509.NewCertPool()
-	trustPool.AddCert(trustCert)
-	return trustPool, nil
-}
-
 func compare(a1, a2 credentials.AuthInfo) bool {
 	if a1.AuthType() != a2.AuthType() {
 		return false
@@ -816,13 +792,13 @@ func compare(a1, a2 credentials.AuthInfo) bool {
 
 func (s) TestAdvancedTLSOverrideServerName(t *testing.T) {
 	expectedServerName := "server.name"
-	clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem"))
-	if err != nil {
-		t.Fatalf("Client is unable to load trust certs. Error: %v", err)
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
 	}
 	clientOptions := &ClientOptions{
 		RootOptions: RootCertificateOptions{
-			RootCACerts: clientTrustPool,
+			RootCACerts: cs.ClientTrust1,
 		},
 		ServerNameOverride: expectedServerName,
 	}
@@ -836,122 +812,33 @@ func (s) TestAdvancedTLSOverrideServerName(t *testing.T) {
 	}
 }
 
-func (s) TestTLSClone(t *testing.T) {
-	expectedServerName := "server.name"
-	clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem"))
-	if err != nil {
-		t.Fatalf("Client is unable to load trust certs. Error: %v", err)
-	}
-	clientOptions := &ClientOptions{
-		RootOptions: RootCertificateOptions{
-			RootCACerts: clientTrustPool,
-		},
-		ServerNameOverride: expectedServerName,
-	}
-	c, err := NewClientCreds(clientOptions)
-	if err != nil {
-		t.Fatalf("Failed to create new client: %v", err)
-	}
-	cc := c.Clone()
-	if cc.Info().ServerName != expectedServerName {
-		t.Fatalf("cc.Info().ServerName = %v, want %v", cc.Info().ServerName, expectedServerName)
-	}
-	cc.OverrideServerName("")
-	if c.Info().ServerName != expectedServerName {
-		t.Fatalf("Change in clone should not affect the original, "+
-			"c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName)
-	}
-
-}
-
-func (s) TestAppendH2ToNextProtos(t *testing.T) {
-	tests := []struct {
-		name string
-		ps   []string
-		want []string
-	}{
-		{
-			name: "empty",
-			ps:   nil,
-			want: []string{"h2"},
-		},
-		{
-			name: "only h2",
-			ps:   []string{"h2"},
-			want: []string{"h2"},
-		},
-		{
-			name: "with h2",
-			ps:   []string{"alpn", "h2"},
-			want: []string{"alpn", "h2"},
-		},
-		{
-			name: "no h2",
-			ps:   []string{"alpn"},
-			want: []string{"alpn", "h2"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := appendH2ToNextProtos(tt.ps); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("appendH2ToNextProtos() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-type nonSyscallConn struct {
-	net.Conn
-}
-
-func (s) TestWrapSyscallConn(t *testing.T) {
-	sc := &syscallConn{}
-	nsc := &nonSyscallConn{}
-
-	wrapConn := WrapSyscallConn(sc, nsc)
-	if _, ok := wrapConn.(syscall.Conn); !ok {
-		t.Errorf("returned conn (type %T) doesn't implement syscall.Conn, want implement",
-			wrapConn)
-	}
-}
-
 func (s) TestGetCertificatesSNI(t *testing.T) {
-	// Load server certificates for setting the serverGetCert callback function.
-	serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err)
-	}
-	serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err)
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
 	}
-	serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err)
-	}
-
 	tests := []struct {
 		desc       string
 		serverName string
 		wantCert   tls.Certificate
 	}{
 		{
-			desc: "Select serverCert1",
+			desc: "Select ServerCert1",
 			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
 			serverName: "foo.bar.com",
-			wantCert:   serverCert1,
+			wantCert:   cs.ServerCert1,
 		},
 		{
-			desc: "Select serverCert2",
+			desc: "Select ServerCert2",
 			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
 			serverName: "foo.bar.server2.com",
-			wantCert:   serverCert2,
+			wantCert:   cs.ServerCert2,
 		},
 		{
 			desc: "Select serverCert3",
 			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
 			serverName: "google.com",
-			wantCert:   serverCert3,
+			wantCert:   cs.ServerPeer3,
 		},
 	}
 	for _, test := range tests {
@@ -960,7 +847,7 @@ func (s) TestGetCertificatesSNI(t *testing.T) {
 			serverOptions := &ServerOptions{
 				IdentityOptions: IdentityCertificateOptions{
 					GetIdentityCertificatesForServer: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) {
-						return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil
+						return []*tls.Certificate{&cs.ServerCert1, &cs.ServerCert2, &cs.ServerPeer3}, nil
 					},
 				},
 			}
diff --git a/security/advancedtls/internal/testutils/testutils.go b/security/advancedtls/internal/testutils/testutils.go
new file mode 100644
index 000000000000..665cc6026389
--- /dev/null
+++ b/security/advancedtls/internal/testutils/testutils.go
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package testutils contains helper functions for advancedtls.
+package testutils
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+
+	"google.golang.org/grpc/security/advancedtls/testdata"
+)
+
+// CertStore contains all the certificates used in the integration tests.
+type CertStore struct {
+	// ClientCert1 is the certificate sent by client to prove its identity.
+	// It is trusted by ServerTrust1.
+	ClientCert1 tls.Certificate
+	// ClientCert2 is the certificate sent by client to prove its identity.
+	// It is trusted by ServerTrust2.
+	ClientCert2 tls.Certificate
+	// ServerCert1 is the certificate sent by server to prove its identity.
+	// It is trusted by ClientTrust1.
+	ServerCert1 tls.Certificate
+	// ServerCert2 is the certificate sent by server to prove its identity.
+	// It is trusted by ClientTrust2.
+	ServerCert2 tls.Certificate
+	// ServerPeer3 is the certificate sent by server to prove its identity.
+	ServerPeer3 tls.Certificate
+	// ClientTrust1 is the root certificate used on the client side.
+	ClientTrust1 *x509.CertPool
+	// ClientTrust2 is the root certificate used on the client side.
+	ClientTrust2 *x509.CertPool
+	// ServerTrust1 is the root certificate used on the server side.
+	ServerTrust1 *x509.CertPool
+	// ServerTrust2 is the root certificate used on the server side.
+	ServerTrust2 *x509.CertPool
+}
+
+func readTrustCert(fileName string) (*x509.CertPool, error) {
+	trustData, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return nil, err
+	}
+	trustPool := x509.NewCertPool()
+	if !trustPool.AppendCertsFromPEM(trustData) {
+		return nil, fmt.Errorf("error loading trust certificates")
+	}
+	return trustPool, nil
+}
+
+// LoadCerts function is used to load test certificates at the beginning of
+// each integration test.
+func (cs *CertStore) LoadCerts() error {
+	var err error
+	if cs.ClientCert1, err = tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"), testdata.Path("client_key_1.pem")); err != nil {
+		return err
+	}
+	if cs.ClientCert2, err = tls.LoadX509KeyPair(testdata.Path("client_cert_2.pem"), testdata.Path("client_key_2.pem")); err != nil {
+		return err
+	}
+	if cs.ServerCert1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem")); err != nil {
+		return err
+	}
+	if cs.ServerCert2, err = tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem")); err != nil {
+		return err
+	}
+	if cs.ServerPeer3, err = tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem")); err != nil {
+		return err
+	}
+	if cs.ClientTrust1, err = readTrustCert(testdata.Path("client_trust_cert_1.pem")); err != nil {
+		return err
+	}
+	if cs.ClientTrust2, err = readTrustCert(testdata.Path("client_trust_cert_2.pem")); err != nil {
+		return err
+	}
+	if cs.ServerTrust1, err = readTrustCert(testdata.Path("server_trust_cert_1.pem")); err != nil {
+		return err
+	}
+	if cs.ServerTrust2, err = readTrustCert(testdata.Path("server_trust_cert_2.pem")); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/security/advancedtls/pemfile_provider_test.go b/security/advancedtls/pemfile_provider_test.go
index abc494bcbbcb..48e0bd2f1c3b 100644
--- a/security/advancedtls/pemfile_provider_test.go
+++ b/security/advancedtls/pemfile_provider_test.go
@@ -29,6 +29,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/security/advancedtls/internal/testutils"
 	"google.golang.org/grpc/security/advancedtls/testdata"
 )
 
@@ -95,17 +96,17 @@ func (s) TestNewPEMFileProvider(t *testing.T) {
 
 // This test overwrites the credential reading function used by the watching
 // goroutine. It is tested under different stages:
-// At stage 0, we force reading function to load clientPeer1 and serverTrust1,
+// At stage 0, we force reading function to load ClientCert1 and ServerTrust1,
 // and see if the credentials are picked up by the watching go routine.
 // At stage 1, we force reading function to cause an error. The watching go
 // routine should log the error while leaving the credentials unchanged.
-// At stage 2, we force reading function to load clientPeer2 and serverTrust2,
+// At stage 2, we force reading function to load ClientCert2 and ServerTrust2,
 // and see if the new credentials are picked up.
 func (s) TestWatchingRoutineUpdates(t *testing.T) {
 	// Load certificates.
-	cs := &certStore{}
-	if err := cs.loadCerts(); err != nil {
-		t.Fatalf("cs.loadCerts() failed: %v", err)
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
 	}
 	tests := []struct {
 		desc         string
@@ -121,9 +122,9 @@ func (s) TestWatchingRoutineUpdates(t *testing.T) {
 				KeyFile:   "not_empty_key_file",
 				TrustFile: "not_empty_trust_file",
 			},
-			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}, Roots: cs.serverTrust1},
-			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}, Roots: cs.serverTrust1},
-			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer2}, Roots: cs.serverTrust2},
+			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}, Roots: cs.ServerTrust1},
+			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}, Roots: cs.ServerTrust1},
+			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert2}, Roots: cs.ServerTrust2},
 		},
 		{
 			desc: "use identity certs only",
@@ -131,18 +132,18 @@ func (s) TestWatchingRoutineUpdates(t *testing.T) {
 				CertFile: "not_empty_cert_file",
 				KeyFile:  "not_empty_key_file",
 			},
-			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}},
-			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer1}},
-			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.clientPeer2}},
+			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}},
+			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}},
+			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert2}},
 		},
 		{
 			desc: "use trust certs only",
 			options: PEMFileProviderOptions{
 				TrustFile: "not_empty_trust_file",
 			},
-			wantKmStage0: certprovider.KeyMaterial{Roots: cs.serverTrust1},
-			wantKmStage1: certprovider.KeyMaterial{Roots: cs.serverTrust1},
-			wantKmStage2: certprovider.KeyMaterial{Roots: cs.serverTrust2},
+			wantKmStage0: certprovider.KeyMaterial{Roots: cs.ServerTrust1},
+			wantKmStage1: certprovider.KeyMaterial{Roots: cs.ServerTrust1},
+			wantKmStage2: certprovider.KeyMaterial{Roots: cs.ServerTrust2},
 		},
 	}
 	for _, test := range tests {
@@ -155,11 +156,11 @@ func (s) TestWatchingRoutineUpdates(t *testing.T) {
 			readKeyCertPairFunc = func(certFile, keyFile string) (tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
-					return cs.clientPeer1, nil
+					return cs.ClientCert1, nil
 				case 1:
 					return tls.Certificate{}, fmt.Errorf("error occurred while reloading")
 				case 2:
-					return cs.clientPeer2, nil
+					return cs.ClientCert2, nil
 				default:
 					return tls.Certificate{}, fmt.Errorf("test stage not supported")
 				}
@@ -171,11 +172,11 @@ func (s) TestWatchingRoutineUpdates(t *testing.T) {
 			readTrustCertFunc = func(trustFile string) (*x509.CertPool, error) {
 				switch stage.read() {
 				case 0:
-					return cs.serverTrust1, nil
+					return cs.ServerTrust1, nil
 				case 1:
 					return nil, fmt.Errorf("error occurred while reloading")
 				case 2:
-					return cs.serverTrust2, nil
+					return cs.ServerTrust2, nil
 				default:
 					return nil, fmt.Errorf("test stage not supported")
 				}

From bc36dc2cbd1d0da552b1acd78bc45139c39f16dc Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Tue, 13 Oct 2020 12:30:16 -0700
Subject: [PATCH 231/481] advancedtls: fix build on version prior to 1.14
 (#3950)

* advancedtls: fix build on version prior to 1.14
---
 security/advancedtls/go.mod             |  2 +-
 security/advancedtls/sni.go             |  2 +-
 security/advancedtls/sni_beforego114.go | 42 +++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 2 deletions(-)
 create mode 100644 security/advancedtls/sni_beforego114.go

diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index b8fe4c9cadce..442735f6fc03 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -1,6 +1,6 @@
 module google.golang.org/grpc/security/advancedtls
 
-go 1.13
+go 1.14
 
 require (
 	github.com/google/go-cmp v0.5.1
diff --git a/security/advancedtls/sni.go b/security/advancedtls/sni.go
index 7fef1990c566..120acf2b376d 100644
--- a/security/advancedtls/sni.go
+++ b/security/advancedtls/sni.go
@@ -1,4 +1,4 @@
-// +build !appengine
+// +build !appengine,go1.14
 
 /*
  *
diff --git a/security/advancedtls/sni_beforego114.go b/security/advancedtls/sni_beforego114.go
new file mode 100644
index 000000000000..26a09b988491
--- /dev/null
+++ b/security/advancedtls/sni_beforego114.go
@@ -0,0 +1,42 @@
+// +build !appengine,!go1.14
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package advancedtls
+
+import (
+	"crypto/tls"
+	"fmt"
+)
+
+// buildGetCertificates returns the first cert contained in ServerOptions for
+// non-appengine builds before version 1.4.
+func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) {
+	if o.IdentityOptions.GetIdentityCertificatesForServer == nil {
+		return nil, fmt.Errorf("function GetCertificates must be specified")
+	}
+	certificates, err := o.IdentityOptions.GetIdentityCertificatesForServer(clientHello)
+	if err != nil {
+		return nil, err
+	}
+	if len(certificates) == 0 {
+		return nil, fmt.Errorf("no certificates configured")
+	}
+	return certificates[0], nil
+}

From 7745e521ff6104ff816d2826de86baa5b8051d03 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 13 Oct 2020 13:51:00 -0700
Subject: [PATCH 232/481] meshca: Use grpc suffix for grpc specific generated
 code. (#3954)

---
 credentials/tls/certprovider/meshca/plugin.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/credentials/tls/certprovider/meshca/plugin.go b/credentials/tls/certprovider/meshca/plugin.go
index b00ad5f08f6f..5ff3e9cf2152 100644
--- a/credentials/tls/certprovider/meshca/plugin.go
+++ b/credentials/tls/certprovider/meshca/plugin.go
@@ -38,6 +38,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/tls/certprovider"
+	meshgrpc "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
 	meshpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/metadata"
@@ -144,7 +145,7 @@ func (p *providerPlugin) run(ctx context.Context) {
 //
 // It returns the amount of time the new certificate is valid for.
 func (p *providerPlugin) updateKeyMaterial(ctx context.Context) (time.Duration, error) {
-	client := meshpb.NewMeshCertificateServiceClient(p.cc)
+	client := meshgrpc.NewMeshCertificateServiceClient(p.cc)
 	retries := 0
 	for {
 		if ctx.Err() != nil {

From 7b167fd6eca1ab8f05ec14085d63197cacd41438 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 14 Oct 2020 14:51:13 -0700
Subject: [PATCH 233/481] encoding/proto: simplify & optimize proto codec
 (#3958)

---
 encoding/proto/proto.go | 66 ++---------------------------------------
 1 file changed, 2 insertions(+), 64 deletions(-)

diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 66b97a6f692a..e1230fdd358a 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -21,9 +21,6 @@
 package proto
 
 import (
-	"math"
-	"sync"
-
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc/encoding"
 )
@@ -38,73 +35,14 @@ func init() {
 // codec is a Codec implementation with protobuf. It is the default codec for gRPC.
 type codec struct{}
 
-type cachedProtoBuffer struct {
-	lastMarshaledSize uint32
-	proto.Buffer
-}
-
-func capToMaxInt32(val int) uint32 {
-	if val > math.MaxInt32 {
-		return uint32(math.MaxInt32)
-	}
-	return uint32(val)
-}
-
-func marshal(v interface{}, cb *cachedProtoBuffer) ([]byte, error) {
-	protoMsg := v.(proto.Message)
-	newSlice := make([]byte, 0, cb.lastMarshaledSize)
-
-	cb.SetBuf(newSlice)
-	cb.Reset()
-	if err := cb.Marshal(protoMsg); err != nil {
-		return nil, err
-	}
-	out := cb.Bytes()
-	cb.lastMarshaledSize = capToMaxInt32(len(out))
-	return out, nil
-}
-
 func (codec) Marshal(v interface{}) ([]byte, error) {
-	if pm, ok := v.(proto.Marshaler); ok {
-		// object can marshal itself, no need for buffer
-		return pm.Marshal()
-	}
-
-	cb := protoBufferPool.Get().(*cachedProtoBuffer)
-	out, err := marshal(v, cb)
-
-	// put back buffer and lose the ref to the slice
-	cb.SetBuf(nil)
-	protoBufferPool.Put(cb)
-	return out, err
+	return proto.Marshal(v.(proto.Message))
 }
 
 func (codec) Unmarshal(data []byte, v interface{}) error {
-	protoMsg := v.(proto.Message)
-	protoMsg.Reset()
-
-	if pu, ok := protoMsg.(proto.Unmarshaler); ok {
-		// object can unmarshal itself, no need for buffer
-		return pu.Unmarshal(data)
-	}
-
-	cb := protoBufferPool.Get().(*cachedProtoBuffer)
-	cb.SetBuf(data)
-	err := cb.Unmarshal(protoMsg)
-	cb.SetBuf(nil)
-	protoBufferPool.Put(cb)
-	return err
+	return proto.Unmarshal(data, v.(proto.Message))
 }
 
 func (codec) Name() string {
 	return Name
 }
-
-var protoBufferPool = &sync.Pool{
-	New: func() interface{} {
-		return &cachedProtoBuffer{
-			Buffer:            proto.Buffer{},
-			lastMarshaledSize: 16,
-		}
-	},
-}

From c6cfaba14db3e4209438d8a8898e7e98be1193d5 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Thu, 15 Oct 2020 11:47:34 -0700
Subject: [PATCH 234/481] advancedtls: Get Instant Updates Before Ticker Runs
 (#3959)

* get instant updates before ticker runs
---
 security/advancedtls/pemfile_provider.go | 62 +++++++++++++-----------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/security/advancedtls/pemfile_provider.go b/security/advancedtls/pemfile_provider.go
index 7e03bb835282..96b3587776e8 100644
--- a/security/advancedtls/pemfile_provider.go
+++ b/security/advancedtls/pemfile_provider.go
@@ -87,6 +87,36 @@ type PEMFileProvider struct {
 	cancel              context.CancelFunc
 }
 
+func updateIdentityDistributor(distributor *certprovider.Distributor, certFile, keyFile string) {
+	if distributor == nil {
+		return
+	}
+	// Read identity certs from PEM files.
+	identityCert, err := readKeyCertPairFunc(certFile, keyFile)
+	if err != nil {
+		// If the reading produces an error, we will skip the update for this
+		// round and log the error.
+		logger.Warningf("tls.LoadX509KeyPair reads %s and %s failed: %v", certFile, keyFile, err)
+		return
+	}
+	distributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{identityCert}}, nil)
+}
+
+func updateRootDistributor(distributor *certprovider.Distributor, trustFile string) {
+	if distributor == nil {
+		return
+	}
+	// Read root certs from PEM files.
+	trustPool, err := readTrustCertFunc(trustFile)
+	if err != nil {
+		// If the reading produces an error, we will skip the update for this
+		// round and log the error.
+		logger.Warningf("readTrustCertFunc reads %v failed: %v", trustFile, err)
+		return
+	}
+	distributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil)
+}
+
 // NewPEMFileProvider returns a new PEMFileProvider constructed using the
 // provided options.
 func NewPEMFileProvider(o PEMFileProviderOptions) (*PEMFileProvider, error) {
@@ -113,42 +143,20 @@ func NewPEMFileProvider(o PEMFileProviderOptions) (*PEMFileProvider, error) {
 	identityTicker := time.NewTicker(o.IdentityInterval)
 	rootTicker := time.NewTicker(o.RootInterval)
 	ctx, cancel := context.WithCancel(context.Background())
-	// We pass a copy of PEMFileProviderOptions to the goroutine in case users
-	// change it after we start reloading.
+
 	go func() {
 		for {
+			updateIdentityDistributor(provider.identityDistributor, o.CertFile, o.KeyFile)
+			updateRootDistributor(provider.rootDistributor, o.TrustFile)
 			select {
 			case <-ctx.Done():
 				identityTicker.Stop()
 				rootTicker.Stop()
 				return
 			case <-identityTicker.C:
-				if provider.identityDistributor == nil {
-					continue
-				}
-				// Read identity certs from PEM files.
-				identityCert, err := readKeyCertPairFunc(o.CertFile, o.KeyFile)
-				if err != nil {
-					// If the reading produces an error, we will skip the update for this
-					// round and log the error.
-					logger.Warningf("tls.LoadX509KeyPair reads %s and %s failed: %v", o.CertFile, o.KeyFile, err)
-					continue
-				}
-				provider.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{identityCert}}, nil)
+				break
 			case <-rootTicker.C:
-				if provider.rootDistributor == nil {
-					continue
-				}
-				// Read root certs from PEM files.
-				trustPool, err := readTrustCertFunc(o.TrustFile)
-				if err != nil {
-					// If the reading produces an error, we will skip the update for this
-					// round and log the error.
-					logger.Warningf("readTrustCertFunc reads %v failed: %v", o.TrustFile, err)
-					continue
-				}
-				provider.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil)
-			default:
+				break
 			}
 		}
 	}()

From ea47aa91b3da3f71e0c079a4c806bdd529efcdf5 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 15 Oct 2020 15:21:24 -0700
Subject: [PATCH 235/481] xdsClient: change WatchRDS to return all virtual
 hosts (#3944)

Instead of finding the best matching domain for the service, and return only
that one virtual host's routes.

This removes the lds request name from the xds client, and makes it xds client
handle multiple RDS watches, so one xds client can be shared by multiple
ClientConns.

This also removes some response validation from the client (e.g. if no virtual
host matches what the client is asking for, the response won't be nack'ed).
---
 xds/internal/client/client.go                 |   9 +
 xds/internal/client/client_rds_test.go        | 159 ++++++++++--------
 xds/internal/client/client_watchers.go        |  23 ++-
 .../client/client_watchers_rds_test.go        |  45 ++++-
 .../client/client_watchers_service_test.go    | 100 +++++++++--
 xds/internal/client/client_xds.go             |  47 ++----
 xds/internal/client/v2/client.go              |  55 +-----
 xds/internal/client/v2/client_rds_test.go     |  55 ++----
 xds/internal/client/v2/client_test.go         |  15 +-
 xds/internal/client/v3/client.go              |  53 +-----
 10 files changed, 283 insertions(+), 278 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 64033227b1f6..0d61963e1b79 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -149,6 +149,15 @@ type ListenerUpdate struct {
 // RouteConfigUpdate contains information received in an RDS response, which is
 // of interest to the registered RDS watcher.
 type RouteConfigUpdate struct {
+	VirtualHosts []*VirtualHost
+}
+
+// VirtualHost contains the routes for a list of Domains.
+//
+// Note that the domains in this slice can be a wildcard, not an exact string.
+// The consumer of this struct needs to find the best match for its hostname.
+type VirtualHost struct {
+	Domains []string
 	// Routes contains a list of routes, each containing matchers and
 	// corresponding action.
 	Routes []*Route
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index 44e2cfc5dde8..286472af8c73 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -195,36 +195,6 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 		wantUpdate RouteConfigUpdate
 		wantError  bool
 	}{
-		{
-			name:      "no-virtual-hosts-in-rc",
-			rc:        &v3routepb.RouteConfiguration{},
-			wantError: true,
-		},
-		{
-			name: "no-domains-in-rc",
-			rc: &v3routepb.RouteConfiguration{
-				VirtualHosts: []*v3routepb.VirtualHost{{}},
-			},
-			wantError: true,
-		},
-		{
-			name: "non-matching-domain-in-rc",
-			rc: &v3routepb.RouteConfiguration{
-				VirtualHosts: []*v3routepb.VirtualHost{
-					{Domains: []string{uninterestingDomain}},
-				},
-			},
-			wantError: true,
-		},
-		{
-			name: "no-routes-in-rc",
-			rc: &v3routepb.RouteConfiguration{
-				VirtualHosts: []*v3routepb.VirtualHost{
-					{Domains: []string{ldsTarget}},
-				},
-			},
-			wantError: true,
-		},
 		{
 			name: "default-route-match-field-is-nil",
 			rc: &v3routepb.RouteConfiguration{
@@ -345,7 +315,18 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{clusterName: 1}}}},
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{uninterestingDomain},
+						Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+					},
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{clusterName: 1}}},
+					},
+				},
+			},
 		},
 		{
 			// default route's match is not empty string, but "/".
@@ -368,7 +349,14 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}}}},
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}}},
+					},
+				},
+			},
 		},
 		{
 			// weights not add up to total-weight.
@@ -431,13 +419,20 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}}},
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}},
+					},
+				},
+			},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, ldsTarget, nil)
+			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, nil)
 			if (gotError != nil) != test.wantError || !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, ldsTarget, gotUpdate, test.wantUpdate)
 			}
@@ -558,24 +553,6 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		{
-			name: "bad routeConfig resource",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3RouteConfigURL,
-					Value: func() []byte {
-						rc := &v3routepb.RouteConfiguration{
-							VirtualHosts: []*v3routepb.VirtualHost{
-								{Domains: []string{uninterestingDomain}},
-							},
-						}
-						m, _ := proto.Marshal(rc)
-						return m
-					}(),
-				},
-			},
-			wantErr: true,
-		},
 		{
 			name: "empty resource list",
 		},
@@ -583,28 +560,72 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 			name:      "v2 routeConfig resource",
 			resources: []*anypb.Any{v2RouteConfig},
 			wantUpdate: map[string]RouteConfigUpdate{
-				v2RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}}},
+				v2RouteConfigName: {
+					VirtualHosts: []*VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+						},
+						{
+							Domains: []string{ldsTarget},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}},
+						},
+					},
+				},
 			},
 		},
 		{
 			name:      "v3 routeConfig resource",
 			resources: []*anypb.Any{v3RouteConfig},
 			wantUpdate: map[string]RouteConfigUpdate{
-				v3RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}}},
+				v3RouteConfigName: {
+					VirtualHosts: []*VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+						},
+						{
+							Domains: []string{ldsTarget},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}},
+						},
+					},
+				},
 			},
 		},
 		{
 			name:      "multiple routeConfig resources",
 			resources: []*anypb.Any{v2RouteConfig, v3RouteConfig},
 			wantUpdate: map[string]RouteConfigUpdate{
-				v3RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}}},
-				v2RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}}},
+				v3RouteConfigName: {
+					VirtualHosts: []*VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+						},
+						{
+							Domains: []string{ldsTarget},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}},
+						},
+					},
+				},
+				v2RouteConfigName: {
+					VirtualHosts: []*VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+						},
+						{
+							Domains: []string{ldsTarget},
+							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}},
+						},
+					},
+				},
 			},
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := UnmarshalRouteConfig(test.resources, ldsTarget, nil)
+			update, err := UnmarshalRouteConfig(test.resources, nil)
 			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("UnmarshalRouteConfig(%v, %v) = (%v, %v) want (%v, %v)", test.resources, ldsTarget, update, err, test.wantUpdate, test.wantErr)
 			}
@@ -660,38 +681,32 @@ func (s) TestMatch(t *testing.T) {
 
 func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
 	var (
-		oneExactMatch = &v3routepb.VirtualHost{
-			Name:    "one-exact-match",
+		oneExactMatch = &VirtualHost{
 			Domains: []string{"foo.bar.com"},
 		}
-		oneSuffixMatch = &v3routepb.VirtualHost{
-			Name:    "one-suffix-match",
+		oneSuffixMatch = &VirtualHost{
 			Domains: []string{"*.bar.com"},
 		}
-		onePrefixMatch = &v3routepb.VirtualHost{
-			Name:    "one-prefix-match",
+		onePrefixMatch = &VirtualHost{
 			Domains: []string{"foo.bar.*"},
 		}
-		oneUniversalMatch = &v3routepb.VirtualHost{
-			Name:    "one-universal-match",
+		oneUniversalMatch = &VirtualHost{
 			Domains: []string{"*"},
 		}
-		longExactMatch = &v3routepb.VirtualHost{
-			Name:    "one-exact-match",
+		longExactMatch = &VirtualHost{
 			Domains: []string{"v2.foo.bar.com"},
 		}
-		multipleMatch = &v3routepb.VirtualHost{
-			Name:    "multiple-match",
+		multipleMatch = &VirtualHost{
 			Domains: []string{"pi.foo.bar.com", "314.*", "*.159"},
 		}
-		vhs = []*v3routepb.VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
+		vhs = []*VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
 	)
 
 	tests := []struct {
 		name   string
 		host   string
-		vHosts []*v3routepb.VirtualHost
-		want   *v3routepb.VirtualHost
+		vHosts []*VirtualHost
+		want   *VirtualHost
 	}{
 		{name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch},
 		{name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch},
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index 7f67be894226..33ac789d3794 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -272,7 +272,7 @@ func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error))
 	}
 	c.mu.Unlock()
 
-	w := &serviceUpdateWatcher{c: c, serviceCb: cb}
+	w := &serviceUpdateWatcher{c: c, serviceName: serviceName, serviceCb: cb}
 	w.ldsCancel = c.watchLDS(serviceName, w.handleLDSResp)
 
 	return w.close
@@ -280,10 +280,15 @@ func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error))
 
 // serviceUpdateWatcher handles LDS and RDS response, and calls the service
 // callback at the right time.
+//
+// TODO: move serviceUpdateWatcher and all its functions into xds resolver. The
+// resolver should be responsible for making WatchListener() and WatchRoute()
+// calls, and finding the best matching virtual host.
 type serviceUpdateWatcher struct {
-	c         *Client
-	ldsCancel func()
-	serviceCb func(ServiceUpdate, error)
+	c           *Client
+	serviceName string
+	ldsCancel   func()
+	serviceCb   func(ServiceUpdate, error)
 
 	mu        sync.Mutex
 	closed    bool
@@ -342,7 +347,15 @@ func (w *serviceUpdateWatcher) handleRDSResp(update RouteConfigUpdate, err error
 		w.serviceCb(ServiceUpdate{}, err)
 		return
 	}
-	w.serviceCb(ServiceUpdate(update), nil)
+
+	matchVh := findBestMatchingVirtualHost(w.serviceName, update.VirtualHosts)
+	if matchVh == nil {
+		// No matching virtual host found.
+		w.serviceCb(ServiceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName))
+		return
+	}
+
+	w.serviceCb(ServiceUpdate{Routes: matchVh.Routes}, nil)
 }
 
 func (w *serviceUpdateWatcher) close() {
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go
index e1f23f3ea76e..63b028d9c1ab 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_rds_test.go
@@ -62,7 +62,14 @@ func (s) TestRDSWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	wantUpdate := RouteConfigUpdate{
+		VirtualHosts: []*VirtualHost{
+			{
+				Domains: []string{testLDSName},
+				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+			},
+		},
+	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
 	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -127,7 +134,14 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 		}
 	}
 
-	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	wantUpdate := RouteConfigUpdate{
+		VirtualHosts: []*VirtualHost{
+			{
+				Domains: []string{testLDSName},
+				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+			},
+		},
+	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
 	for i := 0; i < count; i++ {
 		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate); err != nil {
@@ -199,8 +213,22 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}}}
-	wantUpdate2 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
+	wantUpdate1 := RouteConfigUpdate{
+		VirtualHosts: []*VirtualHost{
+			{
+				Domains: []string{testLDSName},
+				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}},
+			},
+		},
+	}
+	wantUpdate2 := RouteConfigUpdate{
+		VirtualHosts: []*VirtualHost{
+			{
+				Domains: []string{testLDSName},
+				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}},
+			},
+		},
+	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName + "1": wantUpdate1,
 		testRDSName + "2": wantUpdate2,
@@ -244,7 +272,14 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
+	wantUpdate := RouteConfigUpdate{
+		VirtualHosts: []*VirtualHost{
+			{
+				Domains: []string{testLDSName},
+				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+			},
+		},
+	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
 	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
index d4f69d5431ae..1c935c60f31b 100644
--- a/xds/internal/client/client_watchers_service_test.go
+++ b/xds/internal/client/client_watchers_service_test.go
@@ -67,7 +67,14 @@ func (s) TestServiceWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -81,10 +88,12 @@ func (s) TestServiceWatch(t *testing.T) {
 	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName: {
-			Routes: []*Route{{
-				Prefix: newStringP(""),
-				Action: map[string]uint32{testCDSName: 1},
-			}},
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
 		},
 	})
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
@@ -127,7 +136,14 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -141,7 +157,14 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 
 	// Another update for the old name.
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
@@ -152,7 +175,14 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	// RDS update for the new name.
 	wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName + "2": {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}},
+		testRDSName + "2": {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}},
+				},
+			},
+		},
 	})
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
 		t.Fatal(err)
@@ -194,7 +224,14 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -221,7 +258,14 @@ func (s) TestServiceWatchSecond(t *testing.T) {
 	// timeout.
 	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -396,7 +440,14 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
@@ -449,7 +500,14 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				},
+			},
+		},
 	})
 	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
@@ -469,7 +527,14 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	// Send RDS update for the removed LDS resource, expect no updates to
 	// callback, because RDS should be canceled.
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}},
+				},
+			},
+		},
 	})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
@@ -491,7 +556,14 @@ func (s) TestServiceResourceRemoved(t *testing.T) {
 	}
 
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}},
+		testRDSName: {
+			VirtualHosts: []*VirtualHost{
+				{
+					Domains: []string{testLDSName},
+					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}},
+				},
+			},
+		},
 	})
 	wantUpdate = ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index a4048c55b600..e2a7b13f9521 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -104,7 +104,7 @@ func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
 // configuration resources of interest.
-func UnmarshalRouteConfig(resources []*anypb.Any, hostname string, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, error) {
+func UnmarshalRouteConfig(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, error) {
 	update := make(map[string]RouteConfigUpdate)
 	for _, r := range resources {
 		if !IsRouteConfigResource(r.GetTypeUrl()) {
@@ -114,10 +114,10 @@ func UnmarshalRouteConfig(resources []*anypb.Any, hostname string, logger *grpcl
 		if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
 			return nil, fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err)
 		}
-		logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, hostname)
+		logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, rc)
 
 		// Use the hostname (resourceName for LDS) to find the routes.
-		u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname, logger)
+		u, err := generateRDSUpdateFromRouteConfiguration(rc, logger)
 		if err != nil {
 			return nil, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
 		}
@@ -142,30 +142,19 @@ func UnmarshalRouteConfig(resources []*anypb.Any, hostname string, logger *grpcl
 // field must be empty and whose route field must be set.  Inside that route
 // message, the cluster field will contain the clusterName or weighted clusters
 // we are looking for.
-func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, host string, logger *grpclog.PrefixLogger) (RouteConfigUpdate, error) {
-	//
-	// Currently this returns "" on error, and the caller will return an error.
-	// But the error doesn't contain details of why the response is invalid
-	// (mismatch domain or empty route).
-	//
-	// For logging purposes, we can log in line. But if we want to populate
-	// error details for nack, a detailed error needs to be returned.
-	vh := findBestMatchingVirtualHost(host, rc.GetVirtualHosts())
-	if vh == nil {
-		// No matching virtual host found.
-		return RouteConfigUpdate{}, fmt.Errorf("no matching virtual host found")
-	}
-	if len(vh.Routes) == 0 {
-		// The matched virtual host has no routes, this is invalid because there
-		// should be at least one default route.
-		return RouteConfigUpdate{}, fmt.Errorf("matched virtual host has no routes")
-	}
-
-	routes, err := routesProtoToSlice(vh.Routes, logger)
-	if err != nil {
-		return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
+func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, logger *grpclog.PrefixLogger) (RouteConfigUpdate, error) {
+	var vhs []*VirtualHost
+	for _, vh := range rc.GetVirtualHosts() {
+		routes, err := routesProtoToSlice(vh.Routes, logger)
+		if err != nil {
+			return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
+		}
+		vhs = append(vhs, &VirtualHost{
+			Domains: vh.GetDomains(),
+			Routes:  routes,
+		})
 	}
-	return RouteConfigUpdate{Routes: routes}, nil
+	return RouteConfigUpdate{VirtualHosts: vhs}, nil
 }
 
 func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) {
@@ -339,14 +328,14 @@ func match(domain, host string) (domainMatchType, bool) {
 //  - If two matches are of the same pattern type, the longer match is better
 //    - This is to compare the length of the matching pattern, e.g. “*ABCDE” >
 //    “*ABC”
-func findBestMatchingVirtualHost(host string, vHosts []*v3routepb.VirtualHost) *v3routepb.VirtualHost {
+func findBestMatchingVirtualHost(host string, vHosts []*VirtualHost) *VirtualHost {
 	var (
-		matchVh   *v3routepb.VirtualHost
+		matchVh   *VirtualHost
 		matchType = domainMatchTypeInvalid
 		matchLen  int
 	)
 	for _, vh := range vHosts {
-		for _, domain := range vh.GetDomains() {
+		for _, domain := range vh.Domains {
 			typ, matched := match(domain, host)
 			if typ == domainMatchTypeInvalid {
 				// The rds response is invalid.
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index c3de39c3a853..674bba4798f9 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -22,7 +22,6 @@ package v2
 import (
 	"context"
 	"fmt"
-	"sync"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
@@ -93,54 +92,6 @@ type client struct {
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
 	cc        *grpc.ClientConn
 	nodeProto *v2corepb.Node
-
-	mu sync.Mutex
-	// ldsResourceName is the LDS resource_name to watch. It is set to the first
-	// LDS resource_name to watch, and removed when the LDS watch is canceled.
-	//
-	// It's from the dial target of the parent ClientConn. RDS resource
-	// processing needs this to do the host matching.
-	ldsResourceName string
-	ldsWatchCount   int
-}
-
-// AddWatch overrides the transport helper's AddWatch to save the LDS
-// resource_name. This is required when handling an RDS response to perform host
-// matching.
-func (v2c *client) AddWatch(rType xdsclient.ResourceType, rName string) {
-	v2c.mu.Lock()
-	// Special handling for LDS, because RDS needs the LDS resource_name for
-	// response host matching.
-	if rType == xdsclient.ListenerResource {
-		// Set hostname to the first LDS resource_name, and reset it when the
-		// last LDS watch is removed. The upper level Client isn't expected to
-		// watchLDS more than once.
-		v2c.ldsWatchCount++
-		if v2c.ldsWatchCount == 1 {
-			v2c.ldsResourceName = rName
-		}
-	}
-	v2c.mu.Unlock()
-	v2c.TransportHelper.AddWatch(rType, rName)
-}
-
-// RemoveWatch overrides the transport helper's RemoveWatch to clear the LDS
-// resource_name when the last watch is removed.
-func (v2c *client) RemoveWatch(rType xdsclient.ResourceType, rName string) {
-	v2c.mu.Lock()
-	// Special handling for LDS, because RDS needs the LDS resource_name for
-	// response host matching.
-	if rType == xdsclient.ListenerResource {
-		// Set hostname to the first LDS resource_name, and reset it when the
-		// last LDS watch is removed. The upper level Client isn't expected to
-		// watchLDS more than once.
-		v2c.ldsWatchCount--
-		if v2c.ldsWatchCount == 0 {
-			v2c.ldsResourceName = ""
-		}
-	}
-	v2c.mu.Unlock()
-	v2c.TransportHelper.RemoveWatch(rType, rName)
 }
 
 func (v2c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
@@ -242,11 +193,7 @@ func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 // receipt of a good response, it caches validated resources and also invokes
 // the registered watcher callback.
 func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
-	v2c.mu.Lock()
-	hostname := v2c.ldsResourceName
-	v2c.mu.Unlock()
-
-	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v2c.logger)
+	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), v2c.logger)
 	if err != nil {
 		return err
 	}
diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/client_rds_test.go
index 09a73335a015..1d5538ec5492 100644
--- a/xds/internal/client/v2/client_rds_test.go
+++ b/xds/internal/client/v2/client_rds_test.go
@@ -74,10 +74,12 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 		// RouteConfiguration, since the others are covered in
 		// TestGetClusterFromRouteConfiguration.
 		{
-			name:          "no-virtual-hosts-in-response",
-			rdsResponse:   noVirtualHostsInRDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "no-virtual-hosts-in-response",
+			rdsResponse: noVirtualHostsInRDSResponse,
+			wantErr:     false,
+			wantUpdate: &xdsclient.RouteConfigUpdate{
+				VirtualHosts: nil,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one good RouteConfiguration, uninteresting though.
@@ -90,24 +92,20 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 		},
 		// Response contains one good interesting RouteConfiguration.
 		{
-			name:          "one-good-route-config",
-			rdsResponse:   goodRDSResponse1,
-			wantErr:       false,
-			wantUpdate:    &xdsclient.RouteConfigUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}},
-			wantUpdateErr: false,
-		},
-		{
-			name:        "one-good-route-config with routes",
+			name:        "one-good-route-config",
 			rdsResponse: goodRDSResponse1,
 			wantErr:     false,
 			wantUpdate: &xdsclient.RouteConfigUpdate{
-				// Instead of just weighted targets when routing is disabled,
-				// this result contains a route with perfix "", and action as
-				// weighted targets.
-				Routes: []*xdsclient.Route{{
-					Prefix: newStringP(""),
-					Action: map[string]uint32{goodClusterName1: 1},
-				}},
+				VirtualHosts: []*xdsclient.VirtualHost{
+					{
+						Domains: []string{uninterestingDomain},
+						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+					},
+					{
+						Domains: []string{goodLDSTarget1},
+						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}},
+					},
+				},
 			},
 			wantUpdateErr: false,
 		},
@@ -126,25 +124,6 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 	}
 }
 
-// TestRDSHandleResponseWithoutLDSWatch tests the case where the v2Client
-// receives an RDS response without a registered LDS watcher.
-func (s) TestRDSHandleResponseWithoutLDSWatch(t *testing.T) {
-	_, cc, cleanup := startServerAndGetCC(t)
-	defer cleanup()
-
-	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(xdsclient.ResourceType, map[string]interface{}) {},
-	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer v2c.Close()
-
-	if v2c.handleRDSResponse(goodRDSResponse1) == nil {
-		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
-	}
-}
-
 // TestRDSHandleResponseWithoutRDSWatch tests the case where the v2Client
 // receives an RDS response without a registered RDS watcher.
 func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index 51ebcf547519..97634b203069 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -236,13 +236,15 @@ var (
 		},
 		TypeUrl: version.V2RouteConfigURL,
 	}
-	emptyRouteConfig             = &xdspb.RouteConfiguration{}
-	marshaledEmptyRouteConfig, _ = proto.Marshal(emptyRouteConfig)
-	noVirtualHostsInRDSResponse  = &xdspb.DiscoveryResponse{
+	noVirtualHostsRouteConfig = &xdspb.RouteConfiguration{
+		Name: goodRouteName1,
+	}
+	marshaledNoVirtualHostsRouteConfig, _ = proto.Marshal(noVirtualHostsRouteConfig)
+	noVirtualHostsInRDSResponse           = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			{
 				TypeUrl: version.V2RouteConfigURL,
-				Value:   marshaledEmptyRouteConfig,
+				Value:   marshaledNoVirtualHostsRouteConfig,
 			},
 		},
 		TypeUrl: version.V2RouteConfigURL,
@@ -411,11 +413,6 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	}
 	defer v2c.Close()
 
-	// RDS needs an existing LDS watch for the hostname.
-	if test.rType == xdsclient.RouteConfigResource {
-		doLDS(t, v2c, fakeServer)
-	}
-
 	// Register the watcher, this will also trigger the v2Client to send the xDS
 	// request.
 	v2c.AddWatch(test.rType, test.resourceName)
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 9894280ede32..328cd8b9cbea 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -22,7 +22,6 @@ package v3
 import (
 	"context"
 	"fmt"
-	"sync"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
@@ -93,52 +92,6 @@ type client struct {
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
 	cc        *grpc.ClientConn
 	nodeProto *v3corepb.Node
-
-	mu sync.Mutex
-	// ldsResourceName is the LDS resource_name to watch. It is set to the first
-	// LDS resource_name to watch, and removed when the LDS watch is canceled.
-	//
-	// It's from the dial target of the parent ClientConn. RDS resource
-	// processing needs this to do the host matching.
-	ldsResourceName string
-	ldsWatchCount   int
-}
-
-// AddWatch overrides the transport helper's AddWatch to save the LDS
-// resource_name. This is required when handling an RDS response to perform host
-// matching.
-func (v3c *client) AddWatch(rType xdsclient.ResourceType, rName string) {
-	v3c.mu.Lock()
-	// Special handling for LDS, because RDS needs the LDS resource_name for
-	// response host matching.
-	if rType == xdsclient.ListenerResource {
-		// Set hostname to the first LDS resource_name, and reset it when the
-		// last LDS watch is removed. The upper level Client isn't expected to
-		// watchLDS more than once.
-		v3c.ldsWatchCount++
-		if v3c.ldsWatchCount == 1 {
-			v3c.ldsResourceName = rName
-		}
-	}
-	v3c.mu.Unlock()
-	v3c.TransportHelper.AddWatch(rType, rName)
-}
-
-func (v3c *client) RemoveWatch(rType xdsclient.ResourceType, rName string) {
-	v3c.mu.Lock()
-	// Special handling for LDS, because RDS needs the LDS resource_name for
-	// response host matching.
-	if rType == xdsclient.ListenerResource {
-		// Set hostname to the first LDS resource_name, and reset it when the
-		// last LDS watch is removed. The upper level Client isn't expected to
-		// watchLDS more than once.
-		v3c.ldsWatchCount--
-		if v3c.ldsWatchCount == 0 {
-			v3c.ldsResourceName = ""
-		}
-	}
-	v3c.mu.Unlock()
-	v3c.TransportHelper.RemoveWatch(rType, rName)
 }
 
 func (v3c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
@@ -240,11 +193,7 @@ func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 // receipt of a good response, it caches validated resources and also invokes
 // the registered watcher callback.
 func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
-	v3c.mu.Lock()
-	hostname := v3c.ldsResourceName
-	v3c.mu.Unlock()
-
-	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v3c.logger)
+	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), v3c.logger)
 	if err != nil {
 		return err
 	}

From 4be647f7f6dbe8e956014c93a79b9032e5a8b613 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Fri, 16 Oct 2020 11:52:54 -0700
Subject: [PATCH 236/481] internal/resolver/unix: Implemented unix resolver.
 (#3890)

---
 balancer/rls/internal/config.go               |  2 +-
 clientconn.go                                 | 17 +-----
 dialoptions.go                                |  5 +-
 internal/grpcutil/target.go                   | 17 +++++-
 internal/grpcutil/target_test.go              | 31 ++++++++--
 internal/resolver/unix/unix.go                | 49 ++++++++++++++++
 internal/transport/http2_client.go            | 16 ++++--
 internal/transport/networktype/networktype.go | 46 +++++++++++++++
 proxy.go => internal/transport/proxy.go       | 52 +++++++----------
 .../transport/proxy_test.go                   | 16 +-----
 internal/transport/transport.go               |  2 +
 resolver_conn_wrapper_test.go                 |  3 -
 rpc_util.go                                   | 35 ------------
 rpc_util_test.go                              | 21 -------
 test/authority_test.go                        | 57 +++++++++++++++++--
 15 files changed, 229 insertions(+), 140 deletions(-)
 create mode 100644 internal/resolver/unix/unix.go
 create mode 100644 internal/transport/networktype/networktype.go
 rename proxy.go => internal/transport/proxy.go (73%)
 rename proxy_test.go => internal/transport/proxy_test.go (90%)

diff --git a/balancer/rls/internal/config.go b/balancer/rls/internal/config.go
index 305c09106ba3..a3deb8906c9a 100644
--- a/balancer/rls/internal/config.go
+++ b/balancer/rls/internal/config.go
@@ -201,7 +201,7 @@ func (*rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 	if lookupService == "" {
 		return nil, fmt.Errorf("rls: empty lookup_service in service config {%+v}", string(c))
 	}
-	parsedTarget := grpcutil.ParseTarget(lookupService)
+	parsedTarget := grpcutil.ParseTarget(lookupService, false)
 	if parsedTarget.Scheme == "" {
 		parsedTarget.Scheme = resolver.GetDefaultScheme()
 	}
diff --git a/clientconn.go b/clientconn.go
index cbd671a8531a..11ed4af607a6 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"net"
 	"reflect"
 	"strings"
 	"sync"
@@ -48,6 +47,7 @@ import (
 	_ "google.golang.org/grpc/balancer/roundrobin"           // To register roundrobin.
 	_ "google.golang.org/grpc/internal/resolver/dns"         // To register dns resolver.
 	_ "google.golang.org/grpc/internal/resolver/passthrough" // To register passthrough resolver.
+	_ "google.golang.org/grpc/internal/resolver/unix"        // To register unix resolver.
 )
 
 const (
@@ -191,16 +191,6 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 	}
 	cc.mkp = cc.dopts.copts.KeepaliveParams
 
-	if cc.dopts.copts.Dialer == nil {
-		cc.dopts.copts.Dialer = func(ctx context.Context, addr string) (net.Conn, error) {
-			network, addr := parseDialTarget(addr)
-			return (&net.Dialer{}).DialContext(ctx, network, addr)
-		}
-		if cc.dopts.withProxy {
-			cc.dopts.copts.Dialer = newProxyDialer(cc.dopts.copts.Dialer)
-		}
-	}
-
 	if cc.dopts.copts.UserAgent != "" {
 		cc.dopts.copts.UserAgent += " " + grpcUA
 	} else {
@@ -244,8 +234,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 	}
 
 	// Determine the resolver to use.
-	cc.parsedTarget = grpcutil.ParseTarget(cc.target)
-	unixScheme := strings.HasPrefix(cc.target, "unix:")
+	cc.parsedTarget = grpcutil.ParseTarget(cc.target, cc.dopts.copts.Dialer != nil)
 	channelz.Infof(logger, cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme)
 	resolverBuilder := cc.getResolver(cc.parsedTarget.Scheme)
 	if resolverBuilder == nil {
@@ -268,7 +257,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		cc.authority = creds.Info().ServerName
 	} else if cc.dopts.insecure && cc.dopts.authority != "" {
 		cc.authority = cc.dopts.authority
-	} else if unixScheme {
+	} else if strings.HasPrefix(cc.target, "unix:") {
 		cc.authority = "localhost"
 	} else {
 		// Use endpoint from "scheme://authority/endpoint" as the default
diff --git a/dialoptions.go b/dialoptions.go
index a93fcab8f341..e7f86e6d7c81 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -71,7 +71,6 @@ type dialOptions struct {
 	// we need to be able to configure this in tests.
 	resolveNowBackoff func(int) time.Duration
 	resolvers         []resolver.Builder
-	withProxy         bool
 }
 
 // DialOption configures how we set up the connection.
@@ -325,7 +324,7 @@ func WithInsecure() DialOption {
 // later release.
 func WithNoProxy() DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
-		o.withProxy = false
+		o.copts.UseProxy = false
 	})
 }
 
@@ -595,9 +594,9 @@ func defaultDialOptions() dialOptions {
 		copts: transport.ConnectOptions{
 			WriteBufferSize: defaultWriteBufSize,
 			ReadBufferSize:  defaultReadBufSize,
+			UseProxy:        true,
 		},
 		resolveNowBackoff: internalbackoff.DefaultExponential.Backoff,
-		withProxy:         true,
 	}
 }
 
diff --git a/internal/grpcutil/target.go b/internal/grpcutil/target.go
index 80b33cdaf905..3e1b22f5a8c0 100644
--- a/internal/grpcutil/target.go
+++ b/internal/grpcutil/target.go
@@ -37,19 +37,32 @@ func split2(s, sep string) (string, string, bool) {
 }
 
 // ParseTarget splits target into a resolver.Target struct containing scheme,
-// authority and endpoint.
+// authority and endpoint. skipUnixColonParsing indicates that the parse should
+// not parse "unix:[path]" cases. This should be true in cases where a custom
+// dialer is present, to prevent a behavior change.
 //
 // If target is not a valid scheme://authority/endpoint, it returns {Endpoint:
 // target}.
-func ParseTarget(target string) (ret resolver.Target) {
+func ParseTarget(target string, skipUnixColonParsing bool) (ret resolver.Target) {
 	var ok bool
 	ret.Scheme, ret.Endpoint, ok = split2(target, "://")
 	if !ok {
+		if strings.HasPrefix(target, "unix:") && !skipUnixColonParsing {
+			// Handle the "unix:[path]" case, because splitting on :// only
+			// handles the "unix://[/absolute/path]" case. Only handle if the
+			// dialer is nil, to avoid a behavior change with custom dialers.
+			return resolver.Target{Scheme: "unix", Endpoint: target[len("unix:"):]}
+		}
 		return resolver.Target{Endpoint: target}
 	}
 	ret.Authority, ret.Endpoint, ok = split2(ret.Endpoint, "/")
 	if !ok {
 		return resolver.Target{Endpoint: target}
 	}
+	if ret.Scheme == "unix" {
+		// Add the "/" back in the unix case, so the unix resolver receives the
+		// actual endpoint.
+		ret.Endpoint = "/" + ret.Endpoint
+	}
 	return ret
 }
diff --git a/internal/grpcutil/target_test.go b/internal/grpcutil/target_test.go
index 92bdb63d213d..562390bfe381 100644
--- a/internal/grpcutil/target_test.go
+++ b/internal/grpcutil/target_test.go
@@ -32,17 +32,22 @@ func TestParseTarget(t *testing.T) {
 		{Scheme: "passthrough", Authority: "", Endpoint: "/unix/socket/address"},
 	} {
 		str := test.Scheme + "://" + test.Authority + "/" + test.Endpoint
-		got := ParseTarget(str)
+		got := ParseTarget(str, false)
 		if got != test {
-			t.Errorf("ParseTarget(%q) = %+v, want %+v", str, got, test)
+			t.Errorf("ParseTarget(%q, false) = %+v, want %+v", str, got, test)
+		}
+		got = ParseTarget(str, true)
+		if got != test {
+			t.Errorf("ParseTarget(%q, true) = %+v, want %+v", str, got, test)
 		}
 	}
 }
 
 func TestParseTargetString(t *testing.T) {
 	for _, test := range []struct {
-		targetStr string
-		want      resolver.Target
+		targetStr      string
+		want           resolver.Target
+		wantWithDialer resolver.Target
 	}{
 		{targetStr: "", want: resolver.Target{Scheme: "", Authority: "", Endpoint: ""}},
 		{targetStr: ":///", want: resolver.Target{Scheme: "", Authority: "", Endpoint: ""}},
@@ -70,10 +75,24 @@ func TestParseTargetString(t *testing.T) {
 		{targetStr: "a:/b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a:/b"}},
 		{targetStr: "a//b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a//b"}},
 		{targetStr: "a://b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a://b"}},
+
+		// Unix cases without custom dialer.
+		// unix:[local_path] and unix:[/absolute] have different behaviors with
+		// a custom dialer, to prevent behavior changes with custom dialers.
+		{targetStr: "unix:domain", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "domain"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:domain"}},
+		{targetStr: "unix:/domain", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "/domain"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:/domain"}},
 	} {
-		got := ParseTarget(test.targetStr)
+		got := ParseTarget(test.targetStr, false)
 		if got != test.want {
-			t.Errorf("ParseTarget(%q) = %+v, want %+v", test.targetStr, got, test.want)
+			t.Errorf("ParseTarget(%q, false) = %+v, want %+v", test.targetStr, got, test.want)
+		}
+		wantWithDialer := test.wantWithDialer
+		if wantWithDialer == (resolver.Target{}) {
+			wantWithDialer = test.want
+		}
+		got = ParseTarget(test.targetStr, true)
+		if got != wantWithDialer {
+			t.Errorf("ParseTarget(%q, true) = %+v, want %+v", test.targetStr, got, wantWithDialer)
 		}
 	}
 }
diff --git a/internal/resolver/unix/unix.go b/internal/resolver/unix/unix.go
new file mode 100644
index 000000000000..d046e50613d0
--- /dev/null
+++ b/internal/resolver/unix/unix.go
@@ -0,0 +1,49 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package unix implements a resolver for unix targets.
+package unix
+
+import (
+	"google.golang.org/grpc/internal/transport/networktype"
+	"google.golang.org/grpc/resolver"
+)
+
+const scheme = "unix"
+
+type builder struct{}
+
+func (*builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
+	cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(resolver.Address{Addr: target.Endpoint}, "unix")}})
+	return &nopResolver{}, nil
+}
+
+func (*builder) Scheme() string {
+	return scheme
+}
+
+type nopResolver struct {
+}
+
+func (*nopResolver) ResolveNow(resolver.ResolveNowOptions) {}
+
+func (*nopResolver) Close() {}
+
+func init() {
+	resolver.Register(&builder{})
+}
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index e73b77a15a8f..6a4776bb1531 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -33,6 +33,7 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 	"google.golang.org/grpc/internal/grpcutil"
+	"google.golang.org/grpc/internal/transport/networktype"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
@@ -137,11 +138,18 @@ type http2Client struct {
 	connectionID uint64
 }
 
-func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) {
+func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr resolver.Address, useProxy bool, grpcUA string) (net.Conn, error) {
 	if fn != nil {
-		return fn(ctx, addr)
+		return fn(ctx, addr.Addr)
 	}
-	return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
+	networkType := "tcp"
+	if n, ok := networktype.Get(addr); ok {
+		networkType = n
+	}
+	if networkType == "tcp" && useProxy {
+		return proxyDial(ctx, addr.Addr, grpcUA)
+	}
+	return (&net.Dialer{}).DialContext(ctx, networkType, addr.Addr)
 }
 
 func isTemporary(err error) bool {
@@ -172,7 +180,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		}
 	}()
 
-	conn, err := dial(connectCtx, opts.Dialer, addr.Addr)
+	conn, err := dial(connectCtx, opts.Dialer, addr, opts.UseProxy, opts.UserAgent)
 	if err != nil {
 		if opts.FailOnNonTempDialError {
 			return nil, connectionErrorf(isTemporary(err), err, "transport: error while dialing: %v", err)
diff --git a/internal/transport/networktype/networktype.go b/internal/transport/networktype/networktype.go
new file mode 100644
index 000000000000..96967428b515
--- /dev/null
+++ b/internal/transport/networktype/networktype.go
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package networktype declares the network type to be used in the default
+// dailer. Attribute of a resolver.Address.
+package networktype
+
+import (
+	"google.golang.org/grpc/resolver"
+)
+
+// keyType is the key to use for storing State in Attributes.
+type keyType string
+
+const key = keyType("grpc.internal.transport.networktype")
+
+// Set returns a copy of the provided address with attributes containing networkType.
+func Set(address resolver.Address, networkType string) resolver.Address {
+	address.Attributes = address.Attributes.WithValues(key, networkType)
+	return address
+}
+
+// Get returns the network type in the resolver.Address and true, or "", false
+// if not present.
+func Get(address resolver.Address) (string, bool) {
+	v := address.Attributes.Value(key)
+	if v == nil {
+		return "", false
+	}
+	return v.(string), true
+}
diff --git a/proxy.go b/internal/transport/proxy.go
similarity index 73%
rename from proxy.go
rename to internal/transport/proxy.go
index f8f69bfb70fd..a662bf39a6c8 100644
--- a/proxy.go
+++ b/internal/transport/proxy.go
@@ -16,13 +16,12 @@
  *
  */
 
-package grpc
+package transport
 
 import (
 	"bufio"
 	"context"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -34,8 +33,6 @@ import (
 const proxyAuthHeaderKey = "Proxy-Authorization"
 
 var (
-	// errDisabled indicates that proxy is disabled for the address.
-	errDisabled = errors.New("proxy is disabled for the address")
 	// The following variable will be overwritten in the tests.
 	httpProxyFromEnvironment = http.ProxyFromEnvironment
 )
@@ -51,9 +48,6 @@ func mapAddress(ctx context.Context, address string) (*url.URL, error) {
 	if err != nil {
 		return nil, err
 	}
-	if url == nil {
-		return nil, errDisabled
-	}
 	return url, nil
 }
 
@@ -76,7 +70,7 @@ func basicAuth(username, password string) string {
 	return base64.StdEncoding.EncodeToString([]byte(auth))
 }
 
-func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr string, proxyURL *url.URL) (_ net.Conn, err error) {
+func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr string, proxyURL *url.URL, grpcUA string) (_ net.Conn, err error) {
 	defer func() {
 		if err != nil {
 			conn.Close()
@@ -115,32 +109,28 @@ func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr stri
 	return &bufConn{Conn: conn, r: r}, nil
 }
 
-// newProxyDialer returns a dialer that connects to proxy first if necessary.
-// The returned dialer checks if a proxy is necessary, dial to the proxy with the
-// provided dialer, does HTTP CONNECT handshake and returns the connection.
-func newProxyDialer(dialer func(context.Context, string) (net.Conn, error)) func(context.Context, string) (net.Conn, error) {
-	return func(ctx context.Context, addr string) (conn net.Conn, err error) {
-		var newAddr string
-		proxyURL, err := mapAddress(ctx, addr)
-		if err != nil {
-			if err != errDisabled {
-				return nil, err
-			}
-			newAddr = addr
-		} else {
-			newAddr = proxyURL.Host
-		}
+// proxyDial dials, connecting to a proxy first if necessary. Checks if a proxy
+// is necessary, dials, does the HTTP CONNECT handshake, and returns the
+// connection.
+func proxyDial(ctx context.Context, addr string, grpcUA string) (conn net.Conn, err error) {
+	newAddr := addr
+	proxyURL, err := mapAddress(ctx, addr)
+	if err != nil {
+		return nil, err
+	}
+	if proxyURL != nil {
+		newAddr = proxyURL.Host
+	}
 
-		conn, err = dialer(ctx, newAddr)
-		if err != nil {
-			return
-		}
-		if proxyURL != nil {
-			// proxy is disabled if proxyURL is nil.
-			conn, err = doHTTPConnectHandshake(ctx, conn, addr, proxyURL)
-		}
+	conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", newAddr)
+	if err != nil {
 		return
 	}
+	if proxyURL != nil {
+		// proxy is disabled if proxyURL is nil.
+		conn, err = doHTTPConnectHandshake(ctx, conn, addr, proxyURL, grpcUA)
+	}
+	return
 }
 
 func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error {
diff --git a/proxy_test.go b/internal/transport/proxy_test.go
similarity index 90%
rename from proxy_test.go
rename to internal/transport/proxy_test.go
index c9604be62351..628b1fddc494 100644
--- a/proxy_test.go
+++ b/internal/transport/proxy_test.go
@@ -18,7 +18,7 @@
  *
  */
 
-package grpc
+package transport
 
 import (
 	"bufio"
@@ -138,15 +138,9 @@ func testHTTPConnect(t *testing.T, proxyURLModify func(*url.URL) *url.URL, proxy
 	defer overwrite(hpfe)()
 
 	// Dial to proxy server.
-	dialer := newProxyDialer(func(ctx context.Context, addr string) (net.Conn, error) {
-		if deadline, ok := ctx.Deadline(); ok {
-			return net.DialTimeout("tcp", addr, time.Until(deadline))
-		}
-		return net.Dial("tcp", addr)
-	})
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
-	c, err := dialer(ctx, blis.Addr().String())
+	c, err := proxyDial(ctx, blis.Addr().String(), "test")
 	if err != nil {
 		t.Fatalf("http connect Dial failed: %v", err)
 	}
@@ -173,9 +167,6 @@ func (s) TestHTTPConnect(t *testing.T) {
 			if req.Method != http.MethodConnect {
 				return fmt.Errorf("unexpected Method %q, want %q", req.Method, http.MethodConnect)
 			}
-			if req.UserAgent() != grpcUA {
-				return fmt.Errorf("unexpect user agent %q, want %q", req.UserAgent(), grpcUA)
-			}
 			return nil
 		},
 	)
@@ -195,9 +186,6 @@ func (s) TestHTTPConnectBasicAuth(t *testing.T) {
 			if req.Method != http.MethodConnect {
 				return fmt.Errorf("unexpected Method %q, want %q", req.Method, http.MethodConnect)
 			}
-			if req.UserAgent() != grpcUA {
-				return fmt.Errorf("unexpect user agent %q, want %q", req.UserAgent(), grpcUA)
-			}
 			wantProxyAuthStr := "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+password))
 			if got := req.Header.Get(proxyAuthHeaderKey); got != wantProxyAuthStr {
 				gotDecoded, _ := base64.StdEncoding.DecodeString(got)
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index b74030a96878..9c8f79cb4b29 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -569,6 +569,8 @@ type ConnectOptions struct {
 	ChannelzParentID int64
 	// MaxHeaderListSize sets the max (uncompressed) size of header list that is prepared to be received.
 	MaxHeaderListSize *uint32
+	// UseProxy specifies if a proxy should be used.
+	UseProxy bool
 }
 
 // NewClientTransport establishes the transport with the required ConnectOptions
diff --git a/resolver_conn_wrapper_test.go b/resolver_conn_wrapper_test.go
index e125976a5359..f13a408937b1 100644
--- a/resolver_conn_wrapper_test.go
+++ b/resolver_conn_wrapper_test.go
@@ -45,9 +45,6 @@ func (s) TestDialParseTargetUnknownScheme(t *testing.T) {
 	}{
 		{"/unix/socket/address", "/unix/socket/address"},
 
-		// Special test for "unix:///".
-		{"unix:///unix/socket/address", "unix:///unix/socket/address"},
-
 		// For known scheme.
 		{"passthrough://a.server.com/google.com", "google.com"},
 	} {
diff --git a/rpc_util.go b/rpc_util.go
index f0609f2a4e9e..ea5bb8d0c2b9 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -27,7 +27,6 @@ import (
 	"io"
 	"io/ioutil"
 	"math"
-	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -872,40 +871,6 @@ func setCallInfoCodec(c *callInfo) error {
 	return nil
 }
 
-// parseDialTarget returns the network and address to pass to dialer
-func parseDialTarget(target string) (net string, addr string) {
-	net = "tcp"
-
-	m1 := strings.Index(target, ":")
-	m2 := strings.Index(target, ":/")
-
-	// handle unix:addr which will fail with url.Parse
-	if m1 >= 0 && m2 < 0 {
-		if n := target[0:m1]; n == "unix" {
-			net = n
-			addr = target[m1+1:]
-			return net, addr
-		}
-	}
-	if m2 >= 0 {
-		t, err := url.Parse(target)
-		if err != nil {
-			return net, target
-		}
-		scheme := t.Scheme
-		addr = t.Path
-		if scheme == "unix" {
-			net = scheme
-			if addr == "" {
-				addr = t.Host
-			}
-			return net, addr
-		}
-	}
-
-	return net, target
-}
-
 // channelzData is used to store channelz related data for ClientConn, addrConn and Server.
 // These fields cannot be embedded in the original structs (e.g. ClientConn), since to do atomic
 // operation on int64 variable on 32-bit machine, user is responsible to enforce memory alignment.
diff --git a/rpc_util_test.go b/rpc_util_test.go
index 2449c23815ec..90912d52a226 100644
--- a/rpc_util_test.go
+++ b/rpc_util_test.go
@@ -191,27 +191,6 @@ func (s) TestToRPCErr(t *testing.T) {
 	}
 }
 
-func (s) TestParseDialTarget(t *testing.T) {
-	for _, test := range []struct {
-		target, wantNet, wantAddr string
-	}{
-		{"unix:etcd:0", "unix", "etcd:0"},
-		{"unix:///tmp/unix-3", "unix", "/tmp/unix-3"},
-		{"unix://domain", "unix", "domain"},
-		{"unix://etcd:0", "unix", "etcd:0"},
-		{"unix:///etcd:0", "unix", "/etcd:0"},
-		{"passthrough://unix://domain", "tcp", "passthrough://unix://domain"},
-		{"https://google.com:443", "tcp", "https://google.com:443"},
-		{"dns:///google.com", "tcp", "dns:///google.com"},
-		{"/unix/socket/address", "tcp", "/unix/socket/address"},
-	} {
-		gotNet, gotAddr := parseDialTarget(test.target)
-		if gotNet != test.wantNet || gotAddr != test.wantAddr {
-			t.Errorf("parseDialTarget(%q) = %s, %s want %s, %s", test.target, gotNet, gotAddr, test.wantNet, test.wantAddr)
-		}
-	}
-}
-
 // bmEncode benchmarks encoding a Protocol Buffer message containing mSize
 // bytes.
 func bmEncode(b *testing.B, mSize int) {
diff --git a/test/authority_test.go b/test/authority_test.go
index 6cd5d82eec19..11c3c4c1af8d 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -21,17 +21,19 @@ package test
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"testing"
 	"time"
 
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
-func runUnixTest(t *testing.T, address, target, expectedAuthority string) {
+func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer func(context.Context, string) (net.Conn, error)) {
 	if err := os.RemoveAll(address); err != nil {
 		t.Fatalf("Error removing socket file %v: %v\n", address, err)
 	}
@@ -57,7 +59,11 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string) {
 		address: address,
 		target:  target,
 	}
-	if err := us.Start(nil); err != nil {
+	opts := []grpc.DialOption{}
+	if dialer != nil {
+		opts = append(opts, grpc.WithContextDialer(dialer))
+	}
+	if err := us.Start(nil, opts...); err != nil {
 		t.Fatalf("Error starting endpoint server: %v", err)
 		return
 	}
@@ -70,6 +76,8 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string) {
 	}
 }
 
+// TestUnix does end to end tests with the various supported unix target
+// formats, ensuring that the authority is set to localhost in every case.
 func (s) TestUnix(t *testing.T) {
 	tests := []struct {
 		name      string
@@ -78,19 +86,19 @@ func (s) TestUnix(t *testing.T) {
 		authority string
 	}{
 		{
-			name:      "Unix1",
+			name:      "UnixRelative",
 			address:   "sock.sock",
 			target:    "unix:sock.sock",
 			authority: "localhost",
 		},
 		{
-			name:      "Unix2",
+			name:      "UnixAbsolute",
 			address:   "/tmp/sock.sock",
 			target:    "unix:/tmp/sock.sock",
 			authority: "localhost",
 		},
 		{
-			name:      "Unix3",
+			name:      "UnixAbsoluteAlternate",
 			address:   "/tmp/sock.sock",
 			target:    "unix:///tmp/sock.sock",
 			authority: "localhost",
@@ -98,7 +106,44 @@ func (s) TestUnix(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			runUnixTest(t, test.address, test.target, test.authority)
+			runUnixTest(t, test.address, test.target, test.authority, nil)
+		})
+	}
+}
+
+// TestUnixCustomDialer does end to end tests with various supported unix target
+// formats, ensuring that the target sent to the dialer does NOT have the
+// "unix:" prefix stripped.
+func (s) TestUnixCustomDialer(t *testing.T) {
+	tests := []struct {
+		name      string
+		address   string
+		target    string
+		authority string
+	}{
+		{
+			name:      "UnixRelative",
+			address:   "sock.sock",
+			target:    "unix:sock.sock",
+			authority: "localhost",
+		},
+		{
+			name:      "UnixAbsolute",
+			address:   "/tmp/sock.sock",
+			target:    "unix:/tmp/sock.sock",
+			authority: "localhost",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			dialer := func(ctx context.Context, address string) (net.Conn, error) {
+				if address != test.target {
+					return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.target, address)
+				}
+				address = address[len("unix:"):]
+				return (&net.Dialer{}).DialContext(ctx, "unix", address)
+			}
+			runUnixTest(t, test.address, test.target, test.authority, dialer)
 		})
 	}
 }

From ce5e36655676f214220d2fb6ce831f7ba91897fe Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Mon, 19 Oct 2020 13:54:02 -0700
Subject: [PATCH 237/481] advancedtls: add PemFileProvider integration tests
 (#3934)

* advancedtls: add PemFileProvider integration tests
---
 .../advancedtls_integration_test.go           | 342 +++++++++++++++++-
 1 file changed, 330 insertions(+), 12 deletions(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 3f4e7059aaf3..20a9a5857968 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -23,7 +23,9 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -32,6 +34,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 	"google.golang.org/grpc/security/advancedtls/internal/testutils"
+	"google.golang.org/grpc/security/advancedtls/testdata"
 )
 
 var (
@@ -39,6 +42,17 @@ var (
 	port    = ":50051"
 )
 
+const (
+	// Default timeout for normal connections.
+	defaultTestTimeout = 5 * time.Second
+	// Default timeout for failed connections.
+	defaultTestShortTimeout = 10 * time.Millisecond
+	// Intervals that set to monitor the credential updates.
+	credRefreshingInterval = 200 * time.Millisecond
+	// Time we wait for the credential updates to be picked up.
+	sleepInterval = 400 * time.Millisecond
+)
+
 // stageInfo contains a stage number indicating the current phase of each
 // integration test, and a mutex.
 // Based on the stage number of current test, we will use different
@@ -76,6 +90,8 @@ func (greeterServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.Hel
 	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
 }
 
+// TODO(ZhenLian): remove shouldFail to the function signature to provider
+// tests.
 func callAndVerify(msg string, client pb.GreeterClient, shouldFail bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
@@ -86,6 +102,8 @@ func callAndVerify(msg string, client pb.GreeterClient, shouldFail bool) error {
 	return nil
 }
 
+// TODO(ZhenLian): remove shouldFail and add ...DialOption to the function
+// signature to provider cleaner tests.
 func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) {
 	var conn *grpc.ClientConn
 	var err error
@@ -153,7 +171,7 @@ func (s) TestEnd2End(t *testing.T) {
 		// should see it again accepts the connection, since ClientCert2 is trusted
 		// by ServerTrust2.
 		{
-			desc: "TestClientPeerCertReloadServerTrustCertReload",
+			desc: "test the reloading feature for client identity callback and server trust callback",
 			clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				switch stage.read() {
 				case 0:
@@ -194,7 +212,7 @@ func (s) TestEnd2End(t *testing.T) {
 		// should see it again accepts the connection, since ServerCert2 is trusted
 		// by ClientTrust2.
 		{
-			desc:       "TestServerPeerCertReloadClientTrustCertReload",
+			desc:       "test the reloading feature for server identity callback and client trust callback",
 			clientCert: []tls.Certificate{cs.ClientCert1},
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
@@ -236,7 +254,7 @@ func (s) TestEnd2End(t *testing.T) {
 		// At stage 2, the client changes authorization check to only accept
 		// ServerCert2. Now we should see the connection becomes normal again.
 		{
-			desc:       "TestClientCustomVerification",
+			desc:       "test client custom verification",
 			clientCert: []tls.Certificate{cs.ClientCert1},
 			clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) {
 				switch stage.read() {
@@ -368,9 +386,9 @@ func (s) TestEnd2End(t *testing.T) {
 			}
 			// ------------------------Scenario 1------------------------------------
 			// stage = 0, initial connection should succeed
-			ctx1, cancel1 := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel1()
-			conn, greetClient, err := callAndVerifyWithClientConn(ctx1, "rpc call 1", clientTLSCreds, false)
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			conn, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 1", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -385,9 +403,9 @@ func (s) TestEnd2End(t *testing.T) {
 			}
 			// ------------------------Scenario 3------------------------------------
 			// stage = 1, new connection should fail
-			ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel2()
-			conn2, greetClient, err := callAndVerifyWithClientConn(ctx2, "rpc call 3", clientTLSCreds, true)
+			shortCtx, shortCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+			defer shortCancel()
+			conn2, greetClient, err := callAndVerifyWithClientConn(shortCtx, "rpc call 3", clientTLSCreds, true)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -396,9 +414,7 @@ func (s) TestEnd2End(t *testing.T) {
 			stage.increase()
 			// ------------------------Scenario 4------------------------------------
 			// stage = 2,  new connection should succeed
-			ctx3, cancel3 := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel3()
-			conn3, greetClient, err := callAndVerifyWithClientConn(ctx3, "rpc call 4", clientTLSCreds, false)
+			conn3, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 4", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -408,3 +424,305 @@ func (s) TestEnd2End(t *testing.T) {
 		})
 	}
 }
+
+type tmpCredsFiles struct {
+	clientCertTmp  *os.File
+	clientKeyTmp   *os.File
+	clientTrustTmp *os.File
+	serverCertTmp  *os.File
+	serverKeyTmp   *os.File
+	serverTrustTmp *os.File
+}
+
+// Create temp files that are used to hold credentials.
+func createTmpFiles() (*tmpCredsFiles, error) {
+	tmpFiles := &tmpCredsFiles{}
+	var err error
+	tmpFiles.clientCertTmp, err = ioutil.TempFile(os.TempDir(), "pre-")
+	if err != nil {
+		return nil, err
+	}
+	tmpFiles.clientKeyTmp, err = ioutil.TempFile(os.TempDir(), "pre-")
+	if err != nil {
+		return nil, err
+	}
+	tmpFiles.clientTrustTmp, err = ioutil.TempFile(os.TempDir(), "pre-")
+	if err != nil {
+		return nil, err
+	}
+	tmpFiles.serverCertTmp, err = ioutil.TempFile(os.TempDir(), "pre-")
+	if err != nil {
+		return nil, err
+	}
+	tmpFiles.serverKeyTmp, err = ioutil.TempFile(os.TempDir(), "pre-")
+	if err != nil {
+		return nil, err
+	}
+	tmpFiles.serverTrustTmp, err = ioutil.TempFile(os.TempDir(), "pre-")
+	if err != nil {
+		return nil, err
+	}
+	return tmpFiles, nil
+}
+
+// Copy the credential contents to the temporary files.
+func (tmpFiles *tmpCredsFiles) copyCredsToTmpFiles() error {
+	if err := copyFileContents(testdata.Path("client_cert_1.pem"), tmpFiles.clientCertTmp.Name()); err != nil {
+		return err
+	}
+	if err := copyFileContents(testdata.Path("client_key_1.pem"), tmpFiles.clientKeyTmp.Name()); err != nil {
+		return err
+	}
+	if err := copyFileContents(testdata.Path("client_trust_cert_1.pem"), tmpFiles.clientTrustTmp.Name()); err != nil {
+		return err
+	}
+	if err := copyFileContents(testdata.Path("server_cert_1.pem"), tmpFiles.serverCertTmp.Name()); err != nil {
+		return err
+	}
+	if err := copyFileContents(testdata.Path("server_key_1.pem"), tmpFiles.serverKeyTmp.Name()); err != nil {
+		return err
+	}
+	if err := copyFileContents(testdata.Path("server_trust_cert_1.pem"), tmpFiles.serverTrustTmp.Name()); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tmpFiles *tmpCredsFiles) removeFiles() {
+	os.Remove(tmpFiles.clientCertTmp.Name())
+	os.Remove(tmpFiles.clientKeyTmp.Name())
+	os.Remove(tmpFiles.clientTrustTmp.Name())
+	os.Remove(tmpFiles.serverCertTmp.Name())
+	os.Remove(tmpFiles.serverKeyTmp.Name())
+	os.Remove(tmpFiles.serverTrustTmp.Name())
+}
+
+func copyFileContents(sourceFile, destinationFile string) error {
+	input, err := ioutil.ReadFile(sourceFile)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(destinationFile, input, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Create PEMFileProvider(s) watching the content changes of temporary
+// files.
+func createProviders(tmpFiles *tmpCredsFiles) (*PEMFileProvider, *PEMFileProvider, *PEMFileProvider, *PEMFileProvider, error) {
+	clientIdentityOptions := PEMFileProviderOptions{
+		CertFile:         tmpFiles.clientCertTmp.Name(),
+		KeyFile:          tmpFiles.clientKeyTmp.Name(),
+		IdentityInterval: credRefreshingInterval,
+	}
+	clientIdentityProvider, err := NewPEMFileProvider(clientIdentityOptions)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	clientRootOptions := PEMFileProviderOptions{
+		TrustFile:    tmpFiles.clientTrustTmp.Name(),
+		RootInterval: credRefreshingInterval,
+	}
+	clientRootProvider, err := NewPEMFileProvider(clientRootOptions)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	serverIdentityOptions := PEMFileProviderOptions{
+		CertFile:         tmpFiles.serverCertTmp.Name(),
+		KeyFile:          tmpFiles.serverKeyTmp.Name(),
+		IdentityInterval: credRefreshingInterval,
+	}
+	serverIdentityProvider, err := NewPEMFileProvider(serverIdentityOptions)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	serverRootOptions := PEMFileProviderOptions{
+		TrustFile:    tmpFiles.serverTrustTmp.Name(),
+		RootInterval: credRefreshingInterval,
+	}
+	serverRootProvider, err := NewPEMFileProvider(serverRootOptions)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	return clientIdentityProvider, clientRootProvider, serverIdentityProvider, serverRootProvider, nil
+}
+
+// In order to test advanced TLS provider features, we used temporary files to
+// hold credential data, and copy the contents under testdata/ to these tmp
+// files.
+// Initially, we establish a good connection with providers watching contents
+// from tmp files.
+// Next, we change the identity certs that IdentityProvider is watching. Since
+// the identity key is not changed, the IdentityProvider should ignore the
+// update, and the connection should still be good.
+// Then the the identity key is changed. This time IdentityProvider should pick
+// up the update, and the connection should fail, due to the trust certs on the
+// other side is not changed.
+// Finally, the trust certs that other-side's RootProvider is watching get
+// changed. The connection should go back to normal again.
+func (s) TestPEMFileProviderEnd2End(t *testing.T) {
+	tmpFiles, err := createTmpFiles()
+	if err != nil {
+		t.Fatalf("createTmpFiles() failed, error: %v", err)
+	}
+	defer tmpFiles.removeFiles()
+	for _, test := range []struct {
+		desc                string
+		certUpdateFunc      func()
+		keyUpdateFunc       func()
+		trustCertUpdateFunc func()
+	}{
+		{
+			desc: "test the reloading feature for clientIdentityProvider and serverTrustProvider",
+			certUpdateFunc: func() {
+				err = copyFileContents(testdata.Path("client_cert_2.pem"), tmpFiles.clientCertTmp.Name())
+				if err != nil {
+					t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_cert_2.pem"), tmpFiles.clientCertTmp.Name(), err)
+				}
+			},
+			keyUpdateFunc: func() {
+				err = copyFileContents(testdata.Path("client_key_2.pem"), tmpFiles.clientKeyTmp.Name())
+				if err != nil {
+					t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_key_2.pem"), tmpFiles.clientKeyTmp.Name(), err)
+				}
+			},
+			trustCertUpdateFunc: func() {
+				err = copyFileContents(testdata.Path("server_trust_cert_2.pem"), tmpFiles.serverTrustTmp.Name())
+				if err != nil {
+					t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_trust_cert_2.pem"), tmpFiles.serverTrustTmp.Name(), err)
+				}
+			},
+		},
+		{
+			desc: "test the reloading feature for serverIdentityProvider and clientTrustProvider",
+			certUpdateFunc: func() {
+				err = copyFileContents(testdata.Path("server_cert_2.pem"), tmpFiles.serverCertTmp.Name())
+				if err != nil {
+					t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_cert_2.pem"), tmpFiles.serverCertTmp.Name(), err)
+				}
+			},
+			keyUpdateFunc: func() {
+				err = copyFileContents(testdata.Path("server_key_2.pem"), tmpFiles.serverKeyTmp.Name())
+				if err != nil {
+					t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_key_2.pem"), tmpFiles.serverKeyTmp.Name(), err)
+				}
+			},
+			trustCertUpdateFunc: func() {
+				err = copyFileContents(testdata.Path("client_trust_cert_2.pem"), tmpFiles.clientTrustTmp.Name())
+				if err != nil {
+					t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_trust_cert_2.pem"), tmpFiles.clientTrustTmp.Name(), err)
+				}
+			},
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			if err := tmpFiles.copyCredsToTmpFiles(); err != nil {
+				t.Fatalf("tmpFiles.copyCredsToTmpFiles() failed, error: %v", err)
+			}
+			clientIdentityProvider, clientRootProvider, serverIdentityProvider, serverRootProvider, err := createProviders(tmpFiles)
+			if err != nil {
+				t.Fatalf("createProviders(%v) failed, error: %v", tmpFiles, err)
+			}
+			defer clientIdentityProvider.Close()
+			defer clientRootProvider.Close()
+			defer serverIdentityProvider.Close()
+			defer serverRootProvider.Close()
+			// Start a server and create a client using advancedtls API with Provider.
+			serverOptions := &ServerOptions{
+				IdentityOptions: IdentityCertificateOptions{
+					IdentityProvider: serverIdentityProvider,
+				},
+				RootOptions: RootCertificateOptions{
+					RootProvider: serverRootProvider,
+				},
+				RequireClientCert: true,
+				VerifyPeer: func(params *VerificationFuncParams) (*VerificationResults, error) {
+					return &VerificationResults{}, nil
+				},
+				VType: CertVerification,
+			}
+			serverTLSCreds, err := NewServerCreds(serverOptions)
+			if err != nil {
+				t.Fatalf("failed to create server creds: %v", err)
+			}
+			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
+			defer s.Stop()
+			lis, err := net.Listen("tcp", port)
+			if err != nil {
+				t.Fatalf("failed to listen: %v", err)
+			}
+			defer lis.Close()
+			pb.RegisterGreeterServer(s, greeterServer{})
+			go s.Serve(lis)
+			clientOptions := &ClientOptions{
+				IdentityOptions: IdentityCertificateOptions{
+					IdentityProvider: clientIdentityProvider,
+				},
+				VerifyPeer: func(params *VerificationFuncParams) (*VerificationResults, error) {
+					return &VerificationResults{}, nil
+				},
+				RootOptions: RootCertificateOptions{
+					RootProvider: clientRootProvider,
+				},
+				VType: CertVerification,
+			}
+			clientTLSCreds, err := NewClientCreds(clientOptions)
+			if err != nil {
+				t.Fatalf("clientTLSCreds failed to create, error: %v", err)
+			}
+
+			// At initialization, the connection should be good.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			conn, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 1", clientTLSCreds, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer conn.Close()
+			// Make the identity cert change, and wait 1 second for the provider to
+			// pick up the change.
+			test.certUpdateFunc()
+			time.Sleep(sleepInterval)
+			// The already-established connection should not be affected.
+			err = callAndVerify("rpc call 2", greetClient, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// New connections should still be good, because the Provider didn't pick
+			// up the changes due to key-cert mismatch.
+			conn2, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 3", clientTLSCreds, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer conn2.Close()
+			// Make the identity key change, and wait 1 second for the provider to
+			// pick up the change.
+			test.keyUpdateFunc()
+			time.Sleep(sleepInterval)
+			// New connections should fail now, because the Provider picked the
+			// change, and *_cert_2.pem is not trusted by *_trust_cert_1.pem on the
+			// other side.
+			shortCtx, shortCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+			defer shortCancel()
+			conn3, greetClient, err := callAndVerifyWithClientConn(shortCtx, "rpc call 4", clientTLSCreds, true)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer conn3.Close()
+			// Make the trust cert change on the other side, and wait 1 second for
+			// the provider to pick up the change.
+			test.trustCertUpdateFunc()
+			time.Sleep(sleepInterval)
+			// New connections should be good, because the other side is using
+			// *_trust_cert_2.pem now.
+			conn4, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 5", clientTLSCreds, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer conn4.Close()
+		})
+	}
+}

From 9519efffeb5d1897ae8671568871a6d476986524 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 20 Oct 2020 13:02:25 -0700
Subject: [PATCH 238/481] cmd/protoc-gen-go-grpc: use grpc.ServiceRegistrar
 instead of *grpc.Server (#3968)

---
 balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go         | 2 +-
 balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go   | 2 +-
 benchmark/grpc_testing/services_grpc.pb.go                  | 4 ++--
 channelz/grpc_channelz_v1/channelz_grpc.pb.go               | 2 +-
 cmd/protoc-gen-go-grpc/grpc.go                              | 2 +-
 .../alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go      | 2 +-
 .../tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go   | 2 +-
 examples/features/proto/echo/echo_grpc.pb.go                | 2 +-
 examples/helloworld/helloworld/helloworld_grpc.pb.go        | 2 +-
 examples/route_guide/routeguide/route_guide_grpc.pb.go      | 2 +-
 health/grpc_health_v1/health_grpc.pb.go                     | 2 +-
 interop/grpc_testing/test_grpc.pb.go                        | 6 +++---
 profiling/proto/service_grpc.pb.go                          | 2 +-
 reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go    | 2 +-
 reflection/grpc_testing/test_grpc.pb.go                     | 2 +-
 stats/grpc_testing/test_grpc.pb.go                          | 2 +-
 stress/grpc_testing/metrics_grpc.pb.go                      | 2 +-
 test/grpc_testing/test_grpc.pb.go                           | 2 +-
 18 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index b28b7341dbaf..a0061631daa9 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -83,7 +83,7 @@ type UnsafeLoadBalancerServer interface {
 	mustEmbedUnimplementedLoadBalancerServer()
 }
 
-func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) {
+func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) {
 	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
 }
 
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index f3147189337f..9a4f10b784e6 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -63,7 +63,7 @@ type UnsafeRouteLookupServiceServer interface {
 	mustEmbedUnimplementedRouteLookupServiceServer()
 }
 
-func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) {
+func RegisterRouteLookupServiceServer(s grpc.ServiceRegistrar, srv RouteLookupServiceServer) {
 	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
 }
 
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index 64937d1e4609..0afc725ab3f0 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -145,7 +145,7 @@ type UnsafeBenchmarkServiceServer interface {
 	mustEmbedUnimplementedBenchmarkServiceServer()
 }
 
-func RegisterBenchmarkServiceServer(s *grpc.Server, srv BenchmarkServiceServer) {
+func RegisterBenchmarkServiceServer(s grpc.ServiceRegistrar, srv BenchmarkServiceServer) {
 	s.RegisterService(&_BenchmarkService_serviceDesc, srv)
 }
 
@@ -407,7 +407,7 @@ type UnsafeWorkerServiceServer interface {
 	mustEmbedUnimplementedWorkerServiceServer()
 }
 
-func RegisterWorkerServiceServer(s *grpc.Server, srv WorkerServiceServer) {
+func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServer) {
 	s.RegisterService(&_WorkerService_serviceDesc, srv)
 }
 
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index 8fba59abe569..447af4059929 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -159,7 +159,7 @@ type UnsafeChannelzServer interface {
 	mustEmbedUnimplementedChannelzServer()
 }
 
-func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) {
+func RegisterChannelzServer(s grpc.ServiceRegistrar, srv ChannelzServer) {
 	s.RegisterService(&_Channelz_serviceDesc, srv)
 }
 
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index efc92acf35ce..5a3011e1162d 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -181,7 +181,7 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 		g.P(deprecationComment)
 	}
 	serviceDescVar := "_" + service.GoName + "_serviceDesc"
-	g.P("func Register", service.GoName, "Server(s *", grpcPackage.Ident("Server"), ", srv ", serverType, ") {")
+	g.P("func Register", service.GoName, "Server(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv ", serverType, ") {")
 	g.P("s.RegisterService(&", serviceDescVar, `, srv)`)
 	g.P("}")
 	g.P()
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index d27bd063ee20..6dd4158546a7 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -95,7 +95,7 @@ type UnsafeHandshakerServiceServer interface {
 	mustEmbedUnimplementedHandshakerServiceServer()
 }
 
-func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) {
+func RegisterHandshakerServiceServer(s grpc.ServiceRegistrar, srv HandshakerServiceServer) {
 	s.RegisterService(&_HandshakerService_serviceDesc, srv)
 }
 
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index ab5919292784..424a21bf494e 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -66,7 +66,7 @@ type UnsafeMeshCertificateServiceServer interface {
 	mustEmbedUnimplementedMeshCertificateServiceServer()
 }
 
-func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) {
+func RegisterMeshCertificateServiceServer(s grpc.ServiceRegistrar, srv MeshCertificateServiceServer) {
 	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
 }
 
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index 2da0bfeb371c..fb9820f35c98 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -181,7 +181,7 @@ type UnsafeEchoServer interface {
 	mustEmbedUnimplementedEchoServer()
 }
 
-func RegisterEchoServer(s *grpc.Server, srv EchoServer) {
+func RegisterEchoServer(s grpc.ServiceRegistrar, srv EchoServer) {
 	s.RegisterService(&_Echo_serviceDesc, srv)
 }
 
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 2608e4acda99..3c8c7d87df32 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -63,7 +63,7 @@ type UnsafeGreeterServer interface {
 	mustEmbedUnimplementedGreeterServer()
 }
 
-func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) {
+func RegisterGreeterServer(s grpc.ServiceRegistrar, srv GreeterServer) {
 	s.RegisterService(&_Greeter_serviceDesc, srv)
 }
 
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index 3c11b121a54e..fd72beaea51f 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -213,7 +213,7 @@ type UnsafeRouteGuideServer interface {
 	mustEmbedUnimplementedRouteGuideServer()
 }
 
-func RegisterRouteGuideServer(s *grpc.Server, srv RouteGuideServer) {
+func RegisterRouteGuideServer(s grpc.ServiceRegistrar, srv RouteGuideServer) {
 	s.RegisterService(&_RouteGuide_serviceDesc, srv)
 }
 
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index 463c8734ade9..dfa6791a7da6 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -130,7 +130,7 @@ type UnsafeHealthServer interface {
 	mustEmbedUnimplementedHealthServer()
 }
 
-func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
+func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) {
 	s.RegisterService(&_Health_serviceDesc, srv)
 }
 
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index f1b3bfb6c8b3..1a1c910782b6 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -251,7 +251,7 @@ type UnsafeTestServiceServer interface {
 	mustEmbedUnimplementedTestServiceServer()
 }
 
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
 	s.RegisterService(&_TestService_serviceDesc, srv)
 }
 
@@ -480,7 +480,7 @@ type UnsafeUnimplementedServiceServer interface {
 	mustEmbedUnimplementedUnimplementedServiceServer()
 }
 
-func RegisterUnimplementedServiceServer(s *grpc.Server, srv UnimplementedServiceServer) {
+func RegisterUnimplementedServiceServer(s grpc.ServiceRegistrar, srv UnimplementedServiceServer) {
 	s.RegisterService(&_UnimplementedService_serviceDesc, srv)
 }
 
@@ -566,7 +566,7 @@ type UnsafeLoadBalancerStatsServiceServer interface {
 	mustEmbedUnimplementedLoadBalancerStatsServiceServer()
 }
 
-func RegisterLoadBalancerStatsServiceServer(s *grpc.Server, srv LoadBalancerStatsServiceServer) {
+func RegisterLoadBalancerStatsServiceServer(s grpc.ServiceRegistrar, srv LoadBalancerStatsServiceServer) {
 	s.RegisterService(&_LoadBalancerStatsService_serviceDesc, srv)
 }
 
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index a23453e009ac..11f9630db653 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -79,7 +79,7 @@ type UnsafeProfilingServer interface {
 	mustEmbedUnimplementedProfilingServer()
 }
 
-func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) {
+func RegisterProfilingServer(s grpc.ServiceRegistrar, srv ProfilingServer) {
 	s.RegisterService(&_Profiling_serviceDesc, srv)
 }
 
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index 47f2ea55298c..a821e5154e66 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -85,7 +85,7 @@ type UnsafeServerReflectionServer interface {
 	mustEmbedUnimplementedServerReflectionServer()
 }
 
-func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) {
+func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) {
 	s.RegisterService(&_ServerReflection_serviceDesc, srv)
 }
 
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 095d10ae4263..608cf8874c17 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -97,7 +97,7 @@ type UnsafeSearchServiceServer interface {
 	mustEmbedUnimplementedSearchServiceServer()
 }
 
-func RegisterSearchServiceServer(s *grpc.Server, srv SearchServiceServer) {
+func RegisterSearchServiceServer(s grpc.ServiceRegistrar, srv SearchServiceServer) {
 	s.RegisterService(&_SearchService_serviceDesc, srv)
 }
 
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index 89fe0aba5d60..0c825cbff0cd 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -187,7 +187,7 @@ type UnsafeTestServiceServer interface {
 	mustEmbedUnimplementedTestServiceServer()
 }
 
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
 	s.RegisterService(&_TestService_serviceDesc, srv)
 }
 
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 07d480b0b6a5..8262193ea4e4 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -104,7 +104,7 @@ type UnsafeMetricsServiceServer interface {
 	mustEmbedUnimplementedMetricsServiceServer()
 }
 
-func RegisterMetricsServiceServer(s *grpc.Server, srv MetricsServiceServer) {
+func RegisterMetricsServiceServer(s grpc.ServiceRegistrar, srv MetricsServiceServer) {
 	s.RegisterService(&_MetricsService_serviceDesc, srv)
 }
 
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index 915eafd82f94..d68f2cf84a37 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -251,7 +251,7 @@ type UnsafeTestServiceServer interface {
 	mustEmbedUnimplementedTestServiceServer()
 }
 
-func RegisterTestServiceServer(s *grpc.Server, srv TestServiceServer) {
+func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
 	s.RegisterService(&_TestService_serviceDesc, srv)
 }
 

From 8153ecea737122756248d0dbca6f858acb9cfcc6 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 21 Oct 2020 15:07:37 -0700
Subject: [PATCH 239/481] credentials/insecure: Implement insecure credentials.
 (#3964)

---
 credentials/insecure/insecure.go |  74 ++++++++++++++++++
 test/insecure_creds_test.go      | 124 +++++++++++++++++++++++++++++++
 2 files changed, 198 insertions(+)
 create mode 100644 credentials/insecure/insecure.go
 create mode 100644 test/insecure_creds_test.go

diff --git a/credentials/insecure/insecure.go b/credentials/insecure/insecure.go
new file mode 100644
index 000000000000..7fc11717f765
--- /dev/null
+++ b/credentials/insecure/insecure.go
@@ -0,0 +1,74 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package insecure provides an implementation of the
+// credentials.TransportCredentials interface which disables transport security.
+//
+// Experimental
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a
+// later release.
+package insecure
+
+import (
+	"context"
+	"net"
+
+	"google.golang.org/grpc/credentials"
+)
+
+// NewCredentials returns a credentials which disables transport security.
+func NewCredentials() credentials.TransportCredentials {
+	return insecureTC{}
+}
+
+// insecureTC implements the insecure transport credentials. The handshake
+// methods simply return the passed in net.Conn and set the security level to
+// NoSecurity.
+type insecureTC struct{}
+
+func (insecureTC) ClientHandshake(ctx context.Context, _ string, conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil
+}
+
+func (insecureTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil
+}
+
+func (insecureTC) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{SecurityProtocol: "insecure"}
+}
+
+func (insecureTC) Clone() credentials.TransportCredentials {
+	return insecureTC{}
+}
+
+func (insecureTC) OverrideServerName(string) error {
+	return nil
+}
+
+// Info contains the auth information for an insecure connection.
+// It implements the AuthInfo interface.
+type Info struct {
+	credentials.CommonAuthInfo
+}
+
+// AuthType returns the type of Info as a string.
+func (Info) AuthType() string {
+	return "insecure"
+}
diff --git a/test/insecure_creds_test.go b/test/insecure_creds_test.go
new file mode 100644
index 000000000000..81d5a5ba5d0b
--- /dev/null
+++ b/test/insecure_creds_test.go
@@ -0,0 +1,124 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
+
+const defaultTestTimeout = 5 * time.Second
+
+// TestInsecureCreds tests the use of insecure creds on the server and client
+// side, and verifies that expect security level and auth info are returned.
+// Also verifies that this credential can interop with existing `WithInsecure`
+// DialOption.
+func (s) TestInsecureCreds(t *testing.T) {
+	tests := []struct {
+		desc                string
+		clientInsecureCreds bool
+		serverInsecureCreds bool
+	}{
+		{
+			desc:                "client and server insecure creds",
+			clientInsecureCreds: true,
+			serverInsecureCreds: true,
+		},
+		{
+			desc:                "client only insecure creds",
+			clientInsecureCreds: true,
+		},
+		{
+			desc:                "server only insecure creds",
+			serverInsecureCreds: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ss := &stubServer{
+				emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+					if !test.serverInsecureCreds {
+						return &testpb.Empty{}, nil
+					}
+
+					pr, ok := peer.FromContext(ctx)
+					if !ok {
+						return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
+					}
+					// Check security level.
+					info := pr.AuthInfo.(insecure.Info)
+					if at := info.AuthType(); at != "insecure" {
+						return nil, status.Errorf(codes.Unauthenticated, "Wrong AuthType: got %q, want insecure", at)
+					}
+					if secLevel := info.CommonAuthInfo.SecurityLevel; secLevel != credentials.NoSecurity {
+						return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity)
+					}
+					return &testpb.Empty{}, nil
+				},
+			}
+
+			sOpts := []grpc.ServerOption{}
+			if test.serverInsecureCreds {
+				sOpts = append(sOpts, grpc.Creds(insecure.NewCredentials()))
+			}
+			s := grpc.NewServer(sOpts...)
+			defer s.Stop()
+
+			testpb.RegisterTestServiceServer(s, ss)
+
+			lis, err := net.Listen("tcp", "localhost:0")
+			if err != nil {
+				t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+			}
+
+			go s.Serve(lis)
+
+			addr := lis.Addr().String()
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			cOpts := []grpc.DialOption{grpc.WithBlock()}
+			if test.clientInsecureCreds {
+				cOpts = append(cOpts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+			} else {
+				cOpts = append(cOpts, grpc.WithInsecure())
+			}
+			cc, err := grpc.DialContext(ctx, addr, cOpts...)
+			if err != nil {
+				t.Fatalf("grpc.Dial(%q) failed: %v", addr, err)
+			}
+			defer cc.Close()
+
+			c := testpb.NewTestServiceClient(cc)
+			if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("EmptyCall(_, _) = _, %v; want _, <nil>", err)
+			}
+		})
+	}
+}

From 4e8458e5c63864a0216745133b894869c37d9883 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 21 Oct 2020 16:05:44 -0700
Subject: [PATCH 240/481] protobuf: update all generated code to
 google.golang.org/protobuf (#3932)

---
 .../grpclb/grpc_lb_v1/load_balancer.pb.go     | 1074 ++--
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  336 +-
 .../proto/grpc_lookup_v1/rls_config.pb.go     |  727 ++-
 benchmark/grpc_testing/control.pb.go          | 2244 +++++---
 benchmark/grpc_testing/messages.pb.go         | 1277 +++--
 benchmark/grpc_testing/payloads.pb.go         |  508 +-
 benchmark/grpc_testing/services.pb.go         |  173 +-
 benchmark/grpc_testing/stats.pb.go            |  551 +-
 binarylog/grpc_binarylog_v1/binarylog.pb.go   | 1243 ++--
 channelz/grpc_channelz_v1/channelz.pb.go      | 5000 +++++++++++------
 .../internal/proto/grpc_gcp/altscontext.pb.go |  292 +-
 .../internal/proto/grpc_gcp/handshaker.pb.go  | 1511 +++--
 .../grpc_gcp/transport_security_common.pb.go  |  370 +-
 .../internal/meshca_experimental/config.pb.go |  333 +-
 .../meshca/internal/v1/meshca.pb.go           |  315 +-
 encoding/proto/proto_test.go                  |    4 +-
 examples/examples_test.sh                     |    4 +-
 examples/features/proto/echo/echo.pb.go       |  312 +-
 examples/go.mod                               |    1 +
 .../helloworld/helloworld/helloworld.pb.go    |  279 +-
 examples/route_guide/client/client.go         |    3 +-
 .../route_guide/routeguide/route_guide.pb.go  |  614 +-
 go.mod                                        |    5 +-
 go.sum                                        |   18 +-
 health/grpc_health_v1/health.pb.go            |  360 +-
 .../grpc_service_config/service_config.pb.go  | 2855 ++++++----
 interop/grpc_testing/test.pb.go               | 1581 ++++--
 profiling/proto/service.pb.go                 |  650 ++-
 .../grpc_reflection_v1alpha/reflection.pb.go  | 1089 ++--
 reflection/grpc_testing/proto2.pb.go          |  201 +-
 reflection/grpc_testing/proto2_ext.pb.go      |  276 +-
 reflection/grpc_testing/proto2_ext2.pb.go     |  248 +-
 reflection/grpc_testing/test.pb.go            |  397 +-
 regenerate.sh                                 |    4 +-
 stats/grpc_testing/test.pb.go                 |  303 +-
 stress/grpc_testing/metrics.pb.go             |  427 +-
 test/codec_perf/perf.pb.go                    |  188 +-
 test/grpc_testing/test.pb.go                  | 1035 ++--
 test/tools/go.mod                             |    4 +-
 test/tools/go.sum                             |   57 +-
 40 files changed, 17633 insertions(+), 9236 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
index 7381dfc1ae4f..4725858e79d9 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
@@ -1,78 +1,94 @@
+// Copyright 2015 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file defines the GRPCLB LoadBalancing protocol.
+//
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/lb/v1/load_balancer.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/lb/v1/load_balancer.proto
 
 package grpc_lb_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type LoadBalanceRequest struct {
-	// Types that are valid to be assigned to LoadBalanceRequestType:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to LoadBalanceRequestType:
 	//	*LoadBalanceRequest_InitialRequest
 	//	*LoadBalanceRequest_ClientStats
 	LoadBalanceRequestType isLoadBalanceRequest_LoadBalanceRequestType `protobuf_oneof:"load_balance_request_type"`
-	XXX_NoUnkeyedLiteral   struct{}                                    `json:"-"`
-	XXX_unrecognized       []byte                                      `json:"-"`
-	XXX_sizecache          int32                                       `json:"-"`
 }
 
-func (m *LoadBalanceRequest) Reset()         { *m = LoadBalanceRequest{} }
-func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) }
-func (*LoadBalanceRequest) ProtoMessage()    {}
-func (*LoadBalanceRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{0}
+func (x *LoadBalanceRequest) Reset() {
+	*x = LoadBalanceRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadBalanceRequest.Unmarshal(m, b)
-}
-func (m *LoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadBalanceRequest.Marshal(b, m, deterministic)
-}
-func (m *LoadBalanceRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadBalanceRequest.Merge(m, src)
-}
-func (m *LoadBalanceRequest) XXX_Size() int {
-	return xxx_messageInfo_LoadBalanceRequest.Size(m)
-}
-func (m *LoadBalanceRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadBalanceRequest.DiscardUnknown(m)
+func (x *LoadBalanceRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_LoadBalanceRequest proto.InternalMessageInfo
-
-type isLoadBalanceRequest_LoadBalanceRequestType interface {
-	isLoadBalanceRequest_LoadBalanceRequestType()
-}
+func (*LoadBalanceRequest) ProtoMessage() {}
 
-type LoadBalanceRequest_InitialRequest struct {
-	InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,proto3,oneof"`
+func (x *LoadBalanceRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-type LoadBalanceRequest_ClientStats struct {
-	ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,proto3,oneof"`
+// Deprecated: Use LoadBalanceRequest.ProtoReflect.Descriptor instead.
+func (*LoadBalanceRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{0}
 }
 
-func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {}
-
-func (*LoadBalanceRequest_ClientStats) isLoadBalanceRequest_LoadBalanceRequestType() {}
-
 func (m *LoadBalanceRequest) GetLoadBalanceRequestType() isLoadBalanceRequest_LoadBalanceRequestType {
 	if m != nil {
 		return m.LoadBalanceRequestType
@@ -80,117 +96,144 @@ func (m *LoadBalanceRequest) GetLoadBalanceRequestType() isLoadBalanceRequest_Lo
 	return nil
 }
 
-func (m *LoadBalanceRequest) GetInitialRequest() *InitialLoadBalanceRequest {
-	if x, ok := m.GetLoadBalanceRequestType().(*LoadBalanceRequest_InitialRequest); ok {
+func (x *LoadBalanceRequest) GetInitialRequest() *InitialLoadBalanceRequest {
+	if x, ok := x.GetLoadBalanceRequestType().(*LoadBalanceRequest_InitialRequest); ok {
 		return x.InitialRequest
 	}
 	return nil
 }
 
-func (m *LoadBalanceRequest) GetClientStats() *ClientStats {
-	if x, ok := m.GetLoadBalanceRequestType().(*LoadBalanceRequest_ClientStats); ok {
+func (x *LoadBalanceRequest) GetClientStats() *ClientStats {
+	if x, ok := x.GetLoadBalanceRequestType().(*LoadBalanceRequest_ClientStats); ok {
 		return x.ClientStats
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*LoadBalanceRequest) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*LoadBalanceRequest_InitialRequest)(nil),
-		(*LoadBalanceRequest_ClientStats)(nil),
-	}
+type isLoadBalanceRequest_LoadBalanceRequestType interface {
+	isLoadBalanceRequest_LoadBalanceRequestType()
+}
+
+type LoadBalanceRequest_InitialRequest struct {
+	// This message should be sent on the first request to the load balancer.
+	InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,proto3,oneof"`
+}
+
+type LoadBalanceRequest_ClientStats struct {
+	// The client stats should be periodically reported to the load balancer
+	// based on the duration defined in the InitialLoadBalanceResponse.
+	ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,proto3,oneof"`
 }
 
+func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {}
+
+func (*LoadBalanceRequest_ClientStats) isLoadBalanceRequest_LoadBalanceRequestType() {}
+
 type InitialLoadBalanceRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The name of the load balanced service (e.g., service.googleapis.com). Its
 	// length should be less than 256 bytes.
 	// The name might include a port number. How to handle the port number is up
 	// to the balancer.
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *InitialLoadBalanceRequest) Reset()         { *m = InitialLoadBalanceRequest{} }
-func (m *InitialLoadBalanceRequest) String() string { return proto.CompactTextString(m) }
-func (*InitialLoadBalanceRequest) ProtoMessage()    {}
-func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{1}
+func (x *InitialLoadBalanceRequest) Reset() {
+	*x = InitialLoadBalanceRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *InitialLoadBalanceRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_InitialLoadBalanceRequest.Unmarshal(m, b)
-}
-func (m *InitialLoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_InitialLoadBalanceRequest.Marshal(b, m, deterministic)
-}
-func (m *InitialLoadBalanceRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_InitialLoadBalanceRequest.Merge(m, src)
+func (x *InitialLoadBalanceRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *InitialLoadBalanceRequest) XXX_Size() int {
-	return xxx_messageInfo_InitialLoadBalanceRequest.Size(m)
-}
-func (m *InitialLoadBalanceRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_InitialLoadBalanceRequest.DiscardUnknown(m)
+
+func (*InitialLoadBalanceRequest) ProtoMessage() {}
+
+func (x *InitialLoadBalanceRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_InitialLoadBalanceRequest proto.InternalMessageInfo
+// Deprecated: Use InitialLoadBalanceRequest.ProtoReflect.Descriptor instead.
+func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *InitialLoadBalanceRequest) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *InitialLoadBalanceRequest) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 // Contains the number of calls finished for a particular load balance token.
 type ClientStatsPerToken struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// See Server.load_balance_token.
 	LoadBalanceToken string `protobuf:"bytes,1,opt,name=load_balance_token,json=loadBalanceToken,proto3" json:"load_balance_token,omitempty"`
 	// The total number of RPCs that finished associated with the token.
-	NumCalls             int64    `protobuf:"varint,2,opt,name=num_calls,json=numCalls,proto3" json:"num_calls,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	NumCalls int64 `protobuf:"varint,2,opt,name=num_calls,json=numCalls,proto3" json:"num_calls,omitempty"`
 }
 
-func (m *ClientStatsPerToken) Reset()         { *m = ClientStatsPerToken{} }
-func (m *ClientStatsPerToken) String() string { return proto.CompactTextString(m) }
-func (*ClientStatsPerToken) ProtoMessage()    {}
-func (*ClientStatsPerToken) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{2}
+func (x *ClientStatsPerToken) Reset() {
+	*x = ClientStatsPerToken{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClientStatsPerToken) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientStatsPerToken.Unmarshal(m, b)
-}
-func (m *ClientStatsPerToken) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientStatsPerToken.Marshal(b, m, deterministic)
+func (x *ClientStatsPerToken) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientStatsPerToken) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientStatsPerToken.Merge(m, src)
-}
-func (m *ClientStatsPerToken) XXX_Size() int {
-	return xxx_messageInfo_ClientStatsPerToken.Size(m)
-}
-func (m *ClientStatsPerToken) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientStatsPerToken.DiscardUnknown(m)
+
+func (*ClientStatsPerToken) ProtoMessage() {}
+
+func (x *ClientStatsPerToken) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClientStatsPerToken proto.InternalMessageInfo
+// Deprecated: Use ClientStatsPerToken.ProtoReflect.Descriptor instead.
+func (*ClientStatsPerToken) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *ClientStatsPerToken) GetLoadBalanceToken() string {
-	if m != nil {
-		return m.LoadBalanceToken
+func (x *ClientStatsPerToken) GetLoadBalanceToken() string {
+	if x != nil {
+		return x.LoadBalanceToken
 	}
 	return ""
 }
 
-func (m *ClientStatsPerToken) GetNumCalls() int64 {
-	if m != nil {
-		return m.NumCalls
+func (x *ClientStatsPerToken) GetNumCalls() int64 {
+	if x != nil {
+		return x.NumCalls
 	}
 	return 0
 }
@@ -198,6 +241,10 @@ func (m *ClientStatsPerToken) GetNumCalls() int64 {
 // Contains client level statistics that are useful to load balancing. Each
 // count except the timestamp should be reset to zero after reporting the stats.
 type ClientStats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The timestamp of generating the report.
 	Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	// The total number of RPCs that started.
@@ -211,127 +258,172 @@ type ClientStats struct {
 	NumCallsFinishedKnownReceived int64 `protobuf:"varint,7,opt,name=num_calls_finished_known_received,json=numCallsFinishedKnownReceived,proto3" json:"num_calls_finished_known_received,omitempty"`
 	// The list of dropped calls.
 	CallsFinishedWithDrop []*ClientStatsPerToken `protobuf:"bytes,8,rep,name=calls_finished_with_drop,json=callsFinishedWithDrop,proto3" json:"calls_finished_with_drop,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{}               `json:"-"`
-	XXX_unrecognized      []byte                 `json:"-"`
-	XXX_sizecache         int32                  `json:"-"`
 }
 
-func (m *ClientStats) Reset()         { *m = ClientStats{} }
-func (m *ClientStats) String() string { return proto.CompactTextString(m) }
-func (*ClientStats) ProtoMessage()    {}
-func (*ClientStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{3}
+func (x *ClientStats) Reset() {
+	*x = ClientStats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClientStats) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientStats.Unmarshal(m, b)
-}
-func (m *ClientStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientStats.Marshal(b, m, deterministic)
-}
-func (m *ClientStats) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientStats.Merge(m, src)
-}
-func (m *ClientStats) XXX_Size() int {
-	return xxx_messageInfo_ClientStats.Size(m)
+func (x *ClientStats) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientStats) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientStats.DiscardUnknown(m)
+
+func (*ClientStats) ProtoMessage() {}
+
+func (x *ClientStats) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClientStats proto.InternalMessageInfo
+// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead.
+func (*ClientStats) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *ClientStats) GetTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.Timestamp
+func (x *ClientStats) GetTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.Timestamp
 	}
 	return nil
 }
 
-func (m *ClientStats) GetNumCallsStarted() int64 {
-	if m != nil {
-		return m.NumCallsStarted
+func (x *ClientStats) GetNumCallsStarted() int64 {
+	if x != nil {
+		return x.NumCallsStarted
 	}
 	return 0
 }
 
-func (m *ClientStats) GetNumCallsFinished() int64 {
-	if m != nil {
-		return m.NumCallsFinished
+func (x *ClientStats) GetNumCallsFinished() int64 {
+	if x != nil {
+		return x.NumCallsFinished
 	}
 	return 0
 }
 
-func (m *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 {
-	if m != nil {
-		return m.NumCallsFinishedWithClientFailedToSend
+func (x *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 {
+	if x != nil {
+		return x.NumCallsFinishedWithClientFailedToSend
 	}
 	return 0
 }
 
-func (m *ClientStats) GetNumCallsFinishedKnownReceived() int64 {
-	if m != nil {
-		return m.NumCallsFinishedKnownReceived
+func (x *ClientStats) GetNumCallsFinishedKnownReceived() int64 {
+	if x != nil {
+		return x.NumCallsFinishedKnownReceived
 	}
 	return 0
 }
 
-func (m *ClientStats) GetCallsFinishedWithDrop() []*ClientStatsPerToken {
-	if m != nil {
-		return m.CallsFinishedWithDrop
+func (x *ClientStats) GetCallsFinishedWithDrop() []*ClientStatsPerToken {
+	if x != nil {
+		return x.CallsFinishedWithDrop
 	}
 	return nil
 }
 
 type LoadBalanceResponse struct {
-	// Types that are valid to be assigned to LoadBalanceResponseType:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to LoadBalanceResponseType:
 	//	*LoadBalanceResponse_InitialResponse
 	//	*LoadBalanceResponse_ServerList
 	//	*LoadBalanceResponse_FallbackResponse
 	LoadBalanceResponseType isLoadBalanceResponse_LoadBalanceResponseType `protobuf_oneof:"load_balance_response_type"`
-	XXX_NoUnkeyedLiteral    struct{}                                      `json:"-"`
-	XXX_unrecognized        []byte                                        `json:"-"`
-	XXX_sizecache           int32                                         `json:"-"`
 }
 
-func (m *LoadBalanceResponse) Reset()         { *m = LoadBalanceResponse{} }
-func (m *LoadBalanceResponse) String() string { return proto.CompactTextString(m) }
-func (*LoadBalanceResponse) ProtoMessage()    {}
-func (*LoadBalanceResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{4}
+func (x *LoadBalanceResponse) Reset() {
+	*x = LoadBalanceResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *LoadBalanceResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadBalanceResponse.Unmarshal(m, b)
+func (x *LoadBalanceResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *LoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadBalanceResponse.Marshal(b, m, deterministic)
+
+func (*LoadBalanceResponse) ProtoMessage() {}
+
+func (x *LoadBalanceResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *LoadBalanceResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadBalanceResponse.Merge(m, src)
+
+// Deprecated: Use LoadBalanceResponse.ProtoReflect.Descriptor instead.
+func (*LoadBalanceResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{4}
 }
-func (m *LoadBalanceResponse) XXX_Size() int {
-	return xxx_messageInfo_LoadBalanceResponse.Size(m)
+
+func (m *LoadBalanceResponse) GetLoadBalanceResponseType() isLoadBalanceResponse_LoadBalanceResponseType {
+	if m != nil {
+		return m.LoadBalanceResponseType
+	}
+	return nil
+}
+
+func (x *LoadBalanceResponse) GetInitialResponse() *InitialLoadBalanceResponse {
+	if x, ok := x.GetLoadBalanceResponseType().(*LoadBalanceResponse_InitialResponse); ok {
+		return x.InitialResponse
+	}
+	return nil
 }
-func (m *LoadBalanceResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadBalanceResponse.DiscardUnknown(m)
+
+func (x *LoadBalanceResponse) GetServerList() *ServerList {
+	if x, ok := x.GetLoadBalanceResponseType().(*LoadBalanceResponse_ServerList); ok {
+		return x.ServerList
+	}
+	return nil
 }
 
-var xxx_messageInfo_LoadBalanceResponse proto.InternalMessageInfo
+func (x *LoadBalanceResponse) GetFallbackResponse() *FallbackResponse {
+	if x, ok := x.GetLoadBalanceResponseType().(*LoadBalanceResponse_FallbackResponse); ok {
+		return x.FallbackResponse
+	}
+	return nil
+}
 
 type isLoadBalanceResponse_LoadBalanceResponseType interface {
 	isLoadBalanceResponse_LoadBalanceResponseType()
 }
 
 type LoadBalanceResponse_InitialResponse struct {
+	// This message should be sent on the first response to the client.
 	InitialResponse *InitialLoadBalanceResponse `protobuf:"bytes,1,opt,name=initial_response,json=initialResponse,proto3,oneof"`
 }
 
 type LoadBalanceResponse_ServerList struct {
+	// Contains the list of servers selected by the load balancer. The client
+	// should send requests to these servers in the specified order.
 	ServerList *ServerList `protobuf:"bytes,2,opt,name=server_list,json=serverList,proto3,oneof"`
 }
 
 type LoadBalanceResponse_FallbackResponse struct {
+	// If this field is set, then the client should eagerly enter fallback
+	// mode (even if there are existing, healthy connections to backends).
 	FallbackResponse *FallbackResponse `protobuf:"bytes,3,opt,name=fallback_response,json=fallbackResponse,proto3,oneof"`
 }
 
@@ -341,155 +433,141 @@ func (*LoadBalanceResponse_ServerList) isLoadBalanceResponse_LoadBalanceResponse
 
 func (*LoadBalanceResponse_FallbackResponse) isLoadBalanceResponse_LoadBalanceResponseType() {}
 
-func (m *LoadBalanceResponse) GetLoadBalanceResponseType() isLoadBalanceResponse_LoadBalanceResponseType {
-	if m != nil {
-		return m.LoadBalanceResponseType
-	}
-	return nil
+type FallbackResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *LoadBalanceResponse) GetInitialResponse() *InitialLoadBalanceResponse {
-	if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_InitialResponse); ok {
-		return x.InitialResponse
+func (x *FallbackResponse) Reset() {
+	*x = FallbackResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *LoadBalanceResponse) GetServerList() *ServerList {
-	if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_ServerList); ok {
-		return x.ServerList
-	}
-	return nil
+func (x *FallbackResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *LoadBalanceResponse) GetFallbackResponse() *FallbackResponse {
-	if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_FallbackResponse); ok {
-		return x.FallbackResponse
-	}
-	return nil
-}
+func (*FallbackResponse) ProtoMessage() {}
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*LoadBalanceResponse) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*LoadBalanceResponse_InitialResponse)(nil),
-		(*LoadBalanceResponse_ServerList)(nil),
-		(*LoadBalanceResponse_FallbackResponse)(nil),
+func (x *FallbackResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
+	return mi.MessageOf(x)
 }
 
-type FallbackResponse struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *FallbackResponse) Reset()         { *m = FallbackResponse{} }
-func (m *FallbackResponse) String() string { return proto.CompactTextString(m) }
-func (*FallbackResponse) ProtoMessage()    {}
+// Deprecated: Use FallbackResponse.ProtoReflect.Descriptor instead.
 func (*FallbackResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{5}
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{5}
 }
 
-func (m *FallbackResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_FallbackResponse.Unmarshal(m, b)
-}
-func (m *FallbackResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_FallbackResponse.Marshal(b, m, deterministic)
-}
-func (m *FallbackResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_FallbackResponse.Merge(m, src)
-}
-func (m *FallbackResponse) XXX_Size() int {
-	return xxx_messageInfo_FallbackResponse.Size(m)
-}
-func (m *FallbackResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_FallbackResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_FallbackResponse proto.InternalMessageInfo
-
 type InitialLoadBalanceResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// This interval defines how often the client should send the client stats
 	// to the load balancer. Stats should only be reported when the duration is
 	// positive.
 	ClientStatsReportInterval *duration.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"`
-	XXX_NoUnkeyedLiteral      struct{}           `json:"-"`
-	XXX_unrecognized          []byte             `json:"-"`
-	XXX_sizecache             int32              `json:"-"`
 }
 
-func (m *InitialLoadBalanceResponse) Reset()         { *m = InitialLoadBalanceResponse{} }
-func (m *InitialLoadBalanceResponse) String() string { return proto.CompactTextString(m) }
-func (*InitialLoadBalanceResponse) ProtoMessage()    {}
-func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{6}
+func (x *InitialLoadBalanceResponse) Reset() {
+	*x = InitialLoadBalanceResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *InitialLoadBalanceResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_InitialLoadBalanceResponse.Unmarshal(m, b)
-}
-func (m *InitialLoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_InitialLoadBalanceResponse.Marshal(b, m, deterministic)
+func (x *InitialLoadBalanceResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *InitialLoadBalanceResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_InitialLoadBalanceResponse.Merge(m, src)
-}
-func (m *InitialLoadBalanceResponse) XXX_Size() int {
-	return xxx_messageInfo_InitialLoadBalanceResponse.Size(m)
-}
-func (m *InitialLoadBalanceResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_InitialLoadBalanceResponse.DiscardUnknown(m)
+
+func (*InitialLoadBalanceResponse) ProtoMessage() {}
+
+func (x *InitialLoadBalanceResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_InitialLoadBalanceResponse proto.InternalMessageInfo
+// Deprecated: Use InitialLoadBalanceResponse.ProtoReflect.Descriptor instead.
+func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *InitialLoadBalanceResponse) GetClientStatsReportInterval() *duration.Duration {
-	if m != nil {
-		return m.ClientStatsReportInterval
+func (x *InitialLoadBalanceResponse) GetClientStatsReportInterval() *duration.Duration {
+	if x != nil {
+		return x.ClientStatsReportInterval
 	}
 	return nil
 }
 
 type ServerList struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Contains a list of servers selected by the load balancer. The list will
 	// be updated when server resolutions change or as needed to balance load
 	// across more servers. The client should consume the server list in order
 	// unless instructed otherwise via the client_config.
-	Servers              []*Server `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}  `json:"-"`
-	XXX_unrecognized     []byte    `json:"-"`
-	XXX_sizecache        int32     `json:"-"`
+	Servers []*Server `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"`
 }
 
-func (m *ServerList) Reset()         { *m = ServerList{} }
-func (m *ServerList) String() string { return proto.CompactTextString(m) }
-func (*ServerList) ProtoMessage()    {}
-func (*ServerList) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{7}
+func (x *ServerList) Reset() {
+	*x = ServerList{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerList) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerList.Unmarshal(m, b)
-}
-func (m *ServerList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerList.Marshal(b, m, deterministic)
+func (x *ServerList) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerList) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerList.Merge(m, src)
-}
-func (m *ServerList) XXX_Size() int {
-	return xxx_messageInfo_ServerList.Size(m)
-}
-func (m *ServerList) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerList.DiscardUnknown(m)
+
+func (*ServerList) ProtoMessage() {}
+
+func (x *ServerList) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerList proto.InternalMessageInfo
+// Deprecated: Use ServerList.ProtoReflect.Descriptor instead.
+func (*ServerList) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *ServerList) GetServers() []*Server {
-	if m != nil {
-		return m.Servers
+func (x *ServerList) GetServers() []*Server {
+	if x != nil {
+		return x.Servers
 	}
 	return nil
 }
@@ -497,6 +575,10 @@ func (m *ServerList) GetServers() []*Server {
 // Contains server information. When the drop field is not true, use the other
 // fields.
 type Server struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// A resolved address for the server, serialized in network-byte-order. It may
 	// either be an IPv4 or IPv6 address.
 	IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"`
@@ -513,128 +595,366 @@ type Server struct {
 	// Indicates whether this particular request should be dropped by the client.
 	// If the request is dropped, there will be a corresponding entry in
 	// ClientStats.calls_finished_with_drop.
-	Drop                 bool     `protobuf:"varint,4,opt,name=drop,proto3" json:"drop,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Drop bool `protobuf:"varint,4,opt,name=drop,proto3" json:"drop,omitempty"`
 }
 
-func (m *Server) Reset()         { *m = Server{} }
-func (m *Server) String() string { return proto.CompactTextString(m) }
-func (*Server) ProtoMessage()    {}
-func (*Server) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7cd3f6d792743fdf, []int{8}
+func (x *Server) Reset() {
+	*x = Server{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Server) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Server.Unmarshal(m, b)
+func (x *Server) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Server) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Server.Marshal(b, m, deterministic)
-}
-func (m *Server) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Server.Merge(m, src)
-}
-func (m *Server) XXX_Size() int {
-	return xxx_messageInfo_Server.Size(m)
-}
-func (m *Server) XXX_DiscardUnknown() {
-	xxx_messageInfo_Server.DiscardUnknown(m)
+
+func (*Server) ProtoMessage() {}
+
+func (x *Server) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Server proto.InternalMessageInfo
+// Deprecated: Use Server.ProtoReflect.Descriptor instead.
+func (*Server) Descriptor() ([]byte, []int) {
+	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *Server) GetIpAddress() []byte {
-	if m != nil {
-		return m.IpAddress
+func (x *Server) GetIpAddress() []byte {
+	if x != nil {
+		return x.IpAddress
 	}
 	return nil
 }
 
-func (m *Server) GetPort() int32 {
-	if m != nil {
-		return m.Port
+func (x *Server) GetPort() int32 {
+	if x != nil {
+		return x.Port
 	}
 	return 0
 }
 
-func (m *Server) GetLoadBalanceToken() string {
-	if m != nil {
-		return m.LoadBalanceToken
+func (x *Server) GetLoadBalanceToken() string {
+	if x != nil {
+		return x.LoadBalanceToken
 	}
 	return ""
 }
 
-func (m *Server) GetDrop() bool {
-	if m != nil {
-		return m.Drop
+func (x *Server) GetDrop() bool {
+	if x != nil {
+		return x.Drop
 	}
 	return false
 }
 
-func init() {
-	proto.RegisterType((*LoadBalanceRequest)(nil), "grpc.lb.v1.LoadBalanceRequest")
-	proto.RegisterType((*InitialLoadBalanceRequest)(nil), "grpc.lb.v1.InitialLoadBalanceRequest")
-	proto.RegisterType((*ClientStatsPerToken)(nil), "grpc.lb.v1.ClientStatsPerToken")
-	proto.RegisterType((*ClientStats)(nil), "grpc.lb.v1.ClientStats")
-	proto.RegisterType((*LoadBalanceResponse)(nil), "grpc.lb.v1.LoadBalanceResponse")
-	proto.RegisterType((*FallbackResponse)(nil), "grpc.lb.v1.FallbackResponse")
-	proto.RegisterType((*InitialLoadBalanceResponse)(nil), "grpc.lb.v1.InitialLoadBalanceResponse")
-	proto.RegisterType((*ServerList)(nil), "grpc.lb.v1.ServerList")
-	proto.RegisterType((*Server)(nil), "grpc.lb.v1.Server")
-}
-
-func init() { proto.RegisterFile("grpc/lb/v1/load_balancer.proto", fileDescriptor_7cd3f6d792743fdf) }
-
-var fileDescriptor_7cd3f6d792743fdf = []byte{
-	// 769 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xdd, 0x6e, 0xdb, 0x36,
-	0x14, 0x8e, 0x62, 0x25, 0x75, 0x8e, 0xb3, 0x45, 0x61, 0xb1, 0xcd, 0x71, 0xd3, 0x24, 0x13, 0xb0,
-	0x22, 0x18, 0x3a, 0x79, 0xc9, 0x6e, 0x36, 0x60, 0x17, 0x9b, 0x5b, 0x04, 0x69, 0xda, 0x8b, 0x80,
-	0x0e, 0xd0, 0xa1, 0xc0, 0xc0, 0x51, 0x12, 0xed, 0x10, 0xa1, 0x49, 0x8d, 0xa2, 0x5d, 0xec, 0x66,
-	0x37, 0x7b, 0x81, 0x3d, 0xca, 0x5e, 0x61, 0x6f, 0x36, 0x88, 0xa4, 0x2c, 0xd5, 0xae, 0xb1, 0x2b,
-	0x91, 0xe7, 0x7c, 0xfc, 0xce, 0xff, 0x11, 0x9c, 0x4c, 0x75, 0x91, 0x0d, 0x45, 0x3a, 0x5c, 0x5c,
-	0x0c, 0x85, 0xa2, 0x39, 0x49, 0xa9, 0xa0, 0x32, 0x63, 0x3a, 0x29, 0xb4, 0x32, 0x0a, 0x41, 0xa5,
-	0x4f, 0x44, 0x9a, 0x2c, 0x2e, 0x06, 0x27, 0x53, 0xa5, 0xa6, 0x82, 0x0d, 0xad, 0x26, 0x9d, 0x4f,
-	0x86, 0xf9, 0x5c, 0x53, 0xc3, 0x95, 0x74, 0xd8, 0xc1, 0xe9, 0xaa, 0xde, 0xf0, 0x19, 0x2b, 0x0d,
-	0x9d, 0x15, 0x0e, 0x10, 0xff, 0x1b, 0x00, 0x7a, 0xa3, 0x68, 0x3e, 0x72, 0x36, 0x30, 0xfb, 0x7d,
-	0xce, 0x4a, 0x83, 0x6e, 0xe1, 0x80, 0x4b, 0x6e, 0x38, 0x15, 0x44, 0x3b, 0x51, 0x3f, 0x38, 0x0b,
-	0xce, 0x7b, 0x97, 0x5f, 0x25, 0x8d, 0xf5, 0xe4, 0x95, 0x83, 0xac, 0xbf, 0xbf, 0xde, 0xc2, 0x9f,
-	0xfa, 0xf7, 0x35, 0xe3, 0x8f, 0xb0, 0x9f, 0x09, 0xce, 0xa4, 0x21, 0xa5, 0xa1, 0xa6, 0xec, 0x6f,
-	0x5b, 0xba, 0x2f, 0xda, 0x74, 0x2f, 0xac, 0x7e, 0x5c, 0xa9, 0xaf, 0xb7, 0x70, 0x2f, 0x6b, 0xae,
-	0xa3, 0x27, 0x70, 0xd4, 0x4e, 0x45, 0xed, 0x14, 0x31, 0x7f, 0x14, 0x2c, 0x1e, 0xc2, 0xd1, 0x46,
-	0x4f, 0x10, 0x82, 0x50, 0xd2, 0x19, 0xb3, 0xee, 0xef, 0x61, 0x7b, 0x8e, 0x7f, 0x83, 0xc7, 0x2d,
-	0x5b, 0xb7, 0x4c, 0xdf, 0xa9, 0x07, 0x26, 0xd1, 0x73, 0x40, 0x1f, 0x18, 0x31, 0x95, 0xd4, 0x3f,
-	0x8c, 0x44, 0x43, 0xed, 0xd0, 0x4f, 0x60, 0x4f, 0xce, 0x67, 0x24, 0xa3, 0x42, 0xb8, 0x68, 0x3a,
-	0xb8, 0x2b, 0xe7, 0xb3, 0x17, 0xd5, 0x3d, 0xfe, 0xa7, 0x03, 0xbd, 0x96, 0x09, 0xf4, 0x3d, 0xec,
-	0x2d, 0x33, 0xef, 0x33, 0x39, 0x48, 0x5c, 0x6d, 0x92, 0xba, 0x36, 0xc9, 0x5d, 0x8d, 0xc0, 0x0d,
-	0x18, 0x7d, 0x0d, 0x87, 0x4b, 0x33, 0x55, 0xea, 0xb4, 0x61, 0xb9, 0x37, 0x77, 0x50, 0x9b, 0x1b,
-	0x3b, 0x71, 0x15, 0x40, 0x83, 0x9d, 0x70, 0xc9, 0xcb, 0x7b, 0x96, 0xf7, 0x3b, 0x16, 0x1c, 0xd5,
-	0xe0, 0x2b, 0x2f, 0x47, 0xbf, 0xc2, 0x37, 0xeb, 0x68, 0xf2, 0x9e, 0x9b, 0x7b, 0xe2, 0x2b, 0x35,
-	0xa1, 0x5c, 0xb0, 0x9c, 0x18, 0x45, 0x4a, 0x26, 0xf3, 0xfe, 0xae, 0x25, 0x7a, 0xb6, 0x4a, 0xf4,
-	0x96, 0x9b, 0x7b, 0x17, 0xeb, 0x95, 0xc5, 0xdf, 0xa9, 0x31, 0x93, 0x39, 0xba, 0x86, 0x2f, 0x3f,
-	0x42, 0xff, 0x20, 0xd5, 0x7b, 0x49, 0x34, 0xcb, 0x18, 0x5f, 0xb0, 0xbc, 0xff, 0xc8, 0x52, 0x3e,
-	0x5d, 0xa5, 0x7c, 0x5d, 0xa1, 0xb0, 0x07, 0xa1, 0x5f, 0xa0, 0xff, 0x31, 0x27, 0x73, 0xad, 0x8a,
-	0x7e, 0xf7, 0xac, 0x73, 0xde, 0xbb, 0x3c, 0xdd, 0xd0, 0x46, 0x75, 0x69, 0xf1, 0x67, 0xd9, 0xaa,
-	0xc7, 0x2f, 0xb5, 0x2a, 0x6e, 0xc2, 0x6e, 0x18, 0xed, 0xdc, 0x84, 0xdd, 0x9d, 0x68, 0x37, 0xfe,
-	0x7b, 0x1b, 0x1e, 0x7f, 0xd0, 0x3f, 0x65, 0xa1, 0x64, 0xc9, 0xd0, 0x18, 0xa2, 0x66, 0x14, 0x9c,
-	0xcc, 0x57, 0xf0, 0xd9, 0xff, 0xcd, 0x82, 0x43, 0x5f, 0x6f, 0xe1, 0x83, 0xe5, 0x30, 0x78, 0xd2,
-	0x1f, 0xa0, 0x57, 0x32, 0xbd, 0x60, 0x9a, 0x08, 0x5e, 0x1a, 0x3f, 0x0c, 0x9f, 0xb7, 0xf9, 0xc6,
-	0x56, 0xfd, 0x86, 0xdb, 0x61, 0x82, 0x72, 0x79, 0x43, 0xaf, 0xe1, 0x70, 0x42, 0x85, 0x48, 0x69,
-	0xf6, 0xd0, 0x38, 0xd4, 0xb1, 0x04, 0xc7, 0x6d, 0x82, 0x2b, 0x0f, 0x6a, 0xb9, 0x11, 0x4d, 0x56,
-	0x64, 0xa3, 0x63, 0x18, 0xac, 0xcc, 0x95, 0x53, 0xb8, 0xc1, 0x42, 0x10, 0xad, 0xb2, 0xc4, 0x7f,
-	0xc2, 0x60, 0x73, 0xa8, 0xe8, 0x1d, 0x1c, 0xb7, 0xa7, 0x9c, 0x68, 0x56, 0x28, 0x6d, 0x08, 0x97,
-	0x86, 0xe9, 0x05, 0x15, 0x3e, 0xd0, 0xa3, 0xb5, 0xd6, 0x7f, 0xe9, 0xd7, 0x16, 0x3e, 0x6a, 0x4d,
-	0x3d, 0xb6, 0x8f, 0x5f, 0xf9, 0xb7, 0x37, 0x61, 0x37, 0x88, 0xb6, 0xe3, 0x9f, 0x00, 0x9a, 0xd4,
-	0xa0, 0xe7, 0xf0, 0xc8, 0xa5, 0xa6, 0xec, 0x07, 0xb6, 0x13, 0xd0, 0x7a, 0x0e, 0x71, 0x0d, 0xb9,
-	0x09, 0xbb, 0x9d, 0x28, 0x8c, 0xff, 0x0a, 0x60, 0xd7, 0x69, 0xd0, 0x53, 0x00, 0x5e, 0x10, 0x9a,
-	0xe7, 0x9a, 0x95, 0xa5, 0xad, 0xea, 0x3e, 0xde, 0xe3, 0xc5, 0xcf, 0x4e, 0x50, 0xed, 0x8e, 0xca,
-	0x03, 0xeb, 0xf5, 0x0e, 0xb6, 0xe7, 0x0d, 0x4b, 0xa2, 0xb3, 0x61, 0x49, 0x20, 0x08, 0x6d, 0x9b,
-	0x86, 0x67, 0xc1, 0x79, 0x17, 0xdb, 0xb3, 0x6b, 0xb7, 0xcb, 0x14, 0xf6, 0x5b, 0x09, 0xd4, 0x08,
-	0x43, 0xcf, 0x9f, 0x2b, 0x31, 0x3a, 0x69, 0xc7, 0xb1, 0xbe, 0xd6, 0x06, 0xa7, 0x1b, 0xf5, 0xae,
-	0x12, 0xe7, 0xc1, 0xb7, 0xc1, 0xe8, 0x2d, 0x7c, 0xc2, 0x55, 0x0b, 0x38, 0x3a, 0x6c, 0x9b, 0xbc,
-	0xad, 0x92, 0x7f, 0x1b, 0xbc, 0xbb, 0xf0, 0xc5, 0x98, 0x2a, 0x41, 0xe5, 0x34, 0x51, 0x7a, 0x3a,
-	0xb4, 0x7f, 0xa0, 0xfa, 0xb7, 0x63, 0x6f, 0x22, 0xb5, 0x1f, 0x22, 0x52, 0xb2, 0xb8, 0x48, 0x77,
-	0x6d, 0xe1, 0xbe, 0xfb, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x47, 0x55, 0xac, 0xab, 0x06, 0x00,
-	0x00,
+var File_grpc_lb_v1_load_balancer_proto protoreflect.FileDescriptor
+
+var file_grpc_lb_v1_load_balancer_proto_rawDesc = []byte{
+	0x0a, 0x1e, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x62, 0x2f, 0x76, 0x31, 0x2f, 0x6c, 0x6f, 0x61,
+	0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x0a, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69,
+	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x01,
+	0x0a, 0x12, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x50, 0x0a, 0x0f, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f,
+	0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69,
+	0x61, 0x6c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x53, 0x74, 0x61, 0x74, 0x73, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53,
+	0x74, 0x61, 0x74, 0x73, 0x42, 0x1b, 0x0a, 0x19, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c,
+	0x61, 0x6e, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70,
+	0x65, 0x22, 0x2f, 0x0a, 0x19, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4c, 0x6f, 0x61, 0x64,
+	0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12,
+	0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
+	0x6d, 0x65, 0x22, 0x60, 0x0a, 0x13, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x50, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x6c, 0x6f, 0x61,
+	0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e,
+	0x63, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x75, 0x6d, 0x5f, 0x63,
+	0x61, 0x6c, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6e, 0x75, 0x6d, 0x43,
+	0x61, 0x6c, 0x6c, 0x73, 0x22, 0xb0, 0x03, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53,
+	0x74, 0x61, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
+	0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74,
+	0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x2a,
+	0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72,
+	0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x43, 0x61,
+	0x6c, 0x6c, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x6e, 0x75,
+	0x6d, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x73,
+	0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x12, 0x5d, 0x0a, 0x2d, 0x6e, 0x75, 0x6d, 0x5f,
+	0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x77,
+	0x69, 0x74, 0x68, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65,
+	0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x73, 0x65, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x26, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65,
+	0x64, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65,
+	0x64, 0x54, 0x6f, 0x53, 0x65, 0x6e, 0x64, 0x12, 0x48, 0x0a, 0x21, 0x6e, 0x75, 0x6d, 0x5f, 0x63,
+	0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x6b, 0x6e,
+	0x6f, 0x77, 0x6e, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x03, 0x52, 0x1d, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x69, 0x6e, 0x69,
+	0x73, 0x68, 0x65, 0x64, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65,
+	0x64, 0x12, 0x58, 0x0a, 0x18, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73,
+	0x68, 0x65, 0x64, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x5f, 0x64, 0x72, 0x6f, 0x70, 0x18, 0x08, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31,
+	0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, 0x65, 0x72, 0x54,
+	0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x15, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x69, 0x6e, 0x69, 0x73,
+	0x68, 0x65, 0x64, 0x57, 0x69, 0x74, 0x68, 0x44, 0x72, 0x6f, 0x70, 0x4a, 0x04, 0x08, 0x04, 0x10,
+	0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x90, 0x02, 0x0a, 0x13, 0x4c, 0x6f, 0x61, 0x64,
+	0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x53, 0x0a, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x48, 0x00, 0x52, 0x0f, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6c,
+	0x69, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x69, 0x73,
+	0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12,
+	0x4b, 0x0a, 0x11, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x72, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x10, 0x66, 0x61, 0x6c, 0x6c,
+	0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x1c, 0x0a, 0x1a,
+	0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x12, 0x0a, 0x10, 0x46, 0x61,
+	0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x7e,
+	0x0a, 0x1a, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c,
+	0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x1c,
+	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x72, 0x65, 0x70,
+	0x6f, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x19, 0x63,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74,
+	0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x40,
+	0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x07,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04,
+	0x22, 0x83, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x69,
+	0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
+	0x09, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f,
+	0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2c,
+	0x0a, 0x12, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x74,
+	0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x6f, 0x61, 0x64,
+	0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x12, 0x0a, 0x04,
+	0x64, 0x72, 0x6f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x72, 0x6f, 0x70,
+	0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x32, 0x62, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
+	0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x0b, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x65, 0x4c, 0x6f, 0x61, 0x64, 0x12, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e,
+	0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e,
+	0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x57, 0x0a, 0x0d, 0x69, 0x6f,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x42, 0x11, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
+	0x5a, 0x31, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e,
+	0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
+	0x72, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x62,
+	0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_lb_v1_load_balancer_proto_rawDescOnce sync.Once
+	file_grpc_lb_v1_load_balancer_proto_rawDescData = file_grpc_lb_v1_load_balancer_proto_rawDesc
+)
+
+func file_grpc_lb_v1_load_balancer_proto_rawDescGZIP() []byte {
+	file_grpc_lb_v1_load_balancer_proto_rawDescOnce.Do(func() {
+		file_grpc_lb_v1_load_balancer_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_lb_v1_load_balancer_proto_rawDescData)
+	})
+	return file_grpc_lb_v1_load_balancer_proto_rawDescData
+}
+
+var file_grpc_lb_v1_load_balancer_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_grpc_lb_v1_load_balancer_proto_goTypes = []interface{}{
+	(*LoadBalanceRequest)(nil),         // 0: grpc.lb.v1.LoadBalanceRequest
+	(*InitialLoadBalanceRequest)(nil),  // 1: grpc.lb.v1.InitialLoadBalanceRequest
+	(*ClientStatsPerToken)(nil),        // 2: grpc.lb.v1.ClientStatsPerToken
+	(*ClientStats)(nil),                // 3: grpc.lb.v1.ClientStats
+	(*LoadBalanceResponse)(nil),        // 4: grpc.lb.v1.LoadBalanceResponse
+	(*FallbackResponse)(nil),           // 5: grpc.lb.v1.FallbackResponse
+	(*InitialLoadBalanceResponse)(nil), // 6: grpc.lb.v1.InitialLoadBalanceResponse
+	(*ServerList)(nil),                 // 7: grpc.lb.v1.ServerList
+	(*Server)(nil),                     // 8: grpc.lb.v1.Server
+	(*timestamp.Timestamp)(nil),        // 9: google.protobuf.Timestamp
+	(*duration.Duration)(nil),          // 10: google.protobuf.Duration
+}
+var file_grpc_lb_v1_load_balancer_proto_depIdxs = []int32{
+	1,  // 0: grpc.lb.v1.LoadBalanceRequest.initial_request:type_name -> grpc.lb.v1.InitialLoadBalanceRequest
+	3,  // 1: grpc.lb.v1.LoadBalanceRequest.client_stats:type_name -> grpc.lb.v1.ClientStats
+	9,  // 2: grpc.lb.v1.ClientStats.timestamp:type_name -> google.protobuf.Timestamp
+	2,  // 3: grpc.lb.v1.ClientStats.calls_finished_with_drop:type_name -> grpc.lb.v1.ClientStatsPerToken
+	6,  // 4: grpc.lb.v1.LoadBalanceResponse.initial_response:type_name -> grpc.lb.v1.InitialLoadBalanceResponse
+	7,  // 5: grpc.lb.v1.LoadBalanceResponse.server_list:type_name -> grpc.lb.v1.ServerList
+	5,  // 6: grpc.lb.v1.LoadBalanceResponse.fallback_response:type_name -> grpc.lb.v1.FallbackResponse
+	10, // 7: grpc.lb.v1.InitialLoadBalanceResponse.client_stats_report_interval:type_name -> google.protobuf.Duration
+	8,  // 8: grpc.lb.v1.ServerList.servers:type_name -> grpc.lb.v1.Server
+	0,  // 9: grpc.lb.v1.LoadBalancer.BalanceLoad:input_type -> grpc.lb.v1.LoadBalanceRequest
+	4,  // 10: grpc.lb.v1.LoadBalancer.BalanceLoad:output_type -> grpc.lb.v1.LoadBalanceResponse
+	10, // [10:11] is the sub-list for method output_type
+	9,  // [9:10] is the sub-list for method input_type
+	9,  // [9:9] is the sub-list for extension type_name
+	9,  // [9:9] is the sub-list for extension extendee
+	0,  // [0:9] is the sub-list for field type_name
+}
+
+func init() { file_grpc_lb_v1_load_balancer_proto_init() }
+func file_grpc_lb_v1_load_balancer_proto_init() {
+	if File_grpc_lb_v1_load_balancer_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalanceRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*InitialLoadBalanceRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientStatsPerToken); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientStats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalanceResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FallbackResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*InitialLoadBalanceResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerList); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lb_v1_load_balancer_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Server); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_lb_v1_load_balancer_proto_msgTypes[0].OneofWrappers = []interface{}{
+		(*LoadBalanceRequest_InitialRequest)(nil),
+		(*LoadBalanceRequest_ClientStats)(nil),
+	}
+	file_grpc_lb_v1_load_balancer_proto_msgTypes[4].OneofWrappers = []interface{}{
+		(*LoadBalanceResponse_InitialResponse)(nil),
+		(*LoadBalanceResponse_ServerList)(nil),
+		(*LoadBalanceResponse_FallbackResponse)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_lb_v1_load_balancer_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   9,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_lb_v1_load_balancer_proto_goTypes,
+		DependencyIndexes: file_grpc_lb_v1_load_balancer_proto_depIdxs,
+		MessageInfos:      file_grpc_lb_v1_load_balancer_proto_msgTypes,
+	}.Build()
+	File_grpc_lb_v1_load_balancer_proto = out.File
+	file_grpc_lb_v1_load_balancer_proto_rawDesc = nil
+	file_grpc_lb_v1_load_balancer_proto_goTypes = nil
+	file_grpc_lb_v1_load_balancer_proto_depIdxs = nil
 }
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index 7ec14279109f..aad29f104b19 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -1,26 +1,49 @@
+// Copyright 2020 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/lookup/v1/rls.proto
 
 package grpc_lookup_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type RouteLookupRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Full host name of the target server, e.g. firestore.googleapis.com.
 	// Only set for gRPC requests; HTTP requests must use key_map explicitly.
 	Server string `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"`
@@ -31,66 +54,74 @@ type RouteLookupRequest struct {
 	// would like from RLS to allow it to find the regional server, e.g. "grpc".
 	TargetType string `protobuf:"bytes,3,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"`
 	// Map of key values extracted via key builders for the gRPC or HTTP request.
-	KeyMap               map[string]string `protobuf:"bytes,4,rep,name=key_map,json=keyMap,proto3" json:"key_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	KeyMap map[string]string `protobuf:"bytes,4,rep,name=key_map,json=keyMap,proto3" json:"key_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
-func (m *RouteLookupRequest) Reset()         { *m = RouteLookupRequest{} }
-func (m *RouteLookupRequest) String() string { return proto.CompactTextString(m) }
-func (*RouteLookupRequest) ProtoMessage()    {}
-func (*RouteLookupRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_3bab962d3362f3ca, []int{0}
+func (x *RouteLookupRequest) Reset() {
+	*x = RouteLookupRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RouteLookupRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RouteLookupRequest.Unmarshal(m, b)
+func (x *RouteLookupRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RouteLookupRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RouteLookupRequest.Marshal(b, m, deterministic)
-}
-func (m *RouteLookupRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RouteLookupRequest.Merge(m, src)
-}
-func (m *RouteLookupRequest) XXX_Size() int {
-	return xxx_messageInfo_RouteLookupRequest.Size(m)
-}
-func (m *RouteLookupRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_RouteLookupRequest.DiscardUnknown(m)
+
+func (*RouteLookupRequest) ProtoMessage() {}
+
+func (x *RouteLookupRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RouteLookupRequest proto.InternalMessageInfo
+// Deprecated: Use RouteLookupRequest.ProtoReflect.Descriptor instead.
+func (*RouteLookupRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *RouteLookupRequest) GetServer() string {
-	if m != nil {
-		return m.Server
+func (x *RouteLookupRequest) GetServer() string {
+	if x != nil {
+		return x.Server
 	}
 	return ""
 }
 
-func (m *RouteLookupRequest) GetPath() string {
-	if m != nil {
-		return m.Path
+func (x *RouteLookupRequest) GetPath() string {
+	if x != nil {
+		return x.Path
 	}
 	return ""
 }
 
-func (m *RouteLookupRequest) GetTargetType() string {
-	if m != nil {
-		return m.TargetType
+func (x *RouteLookupRequest) GetTargetType() string {
+	if x != nil {
+		return x.TargetType
 	}
 	return ""
 }
 
-func (m *RouteLookupRequest) GetKeyMap() map[string]string {
-	if m != nil {
-		return m.KeyMap
+func (x *RouteLookupRequest) GetKeyMap() map[string]string {
+	if x != nil {
+		return x.KeyMap
 	}
 	return nil
 }
 
 type RouteLookupResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Prioritized list (best one first) of addressable entities to use
 	// for routing, using syntax requested by the request target_type.
 	// The targets will be tried in order until a healthy one is found.
@@ -98,81 +129,172 @@ type RouteLookupResponse struct {
 	// Optional header value to pass along to AFE in the X-Google-RLS-Data header.
 	// Cached with "target" and sent with all requests that match the request key.
 	// Allows the RLS to pass its work product to the eventual target.
-	HeaderData           string   `protobuf:"bytes,2,opt,name=header_data,json=headerData,proto3" json:"header_data,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	HeaderData string `protobuf:"bytes,2,opt,name=header_data,json=headerData,proto3" json:"header_data,omitempty"`
 }
 
-func (m *RouteLookupResponse) Reset()         { *m = RouteLookupResponse{} }
-func (m *RouteLookupResponse) String() string { return proto.CompactTextString(m) }
-func (*RouteLookupResponse) ProtoMessage()    {}
-func (*RouteLookupResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_3bab962d3362f3ca, []int{1}
+func (x *RouteLookupResponse) Reset() {
+	*x = RouteLookupResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RouteLookupResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RouteLookupResponse.Unmarshal(m, b)
-}
-func (m *RouteLookupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RouteLookupResponse.Marshal(b, m, deterministic)
-}
-func (m *RouteLookupResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RouteLookupResponse.Merge(m, src)
+func (x *RouteLookupResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RouteLookupResponse) XXX_Size() int {
-	return xxx_messageInfo_RouteLookupResponse.Size(m)
-}
-func (m *RouteLookupResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_RouteLookupResponse.DiscardUnknown(m)
+
+func (*RouteLookupResponse) ProtoMessage() {}
+
+func (x *RouteLookupResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RouteLookupResponse proto.InternalMessageInfo
+// Deprecated: Use RouteLookupResponse.ProtoReflect.Descriptor instead.
+func (*RouteLookupResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *RouteLookupResponse) GetTargets() []string {
-	if m != nil {
-		return m.Targets
+func (x *RouteLookupResponse) GetTargets() []string {
+	if x != nil {
+		return x.Targets
 	}
 	return nil
 }
 
-func (m *RouteLookupResponse) GetHeaderData() string {
-	if m != nil {
-		return m.HeaderData
+func (x *RouteLookupResponse) GetHeaderData() string {
+	if x != nil {
+		return x.HeaderData
 	}
 	return ""
 }
 
-func init() {
-	proto.RegisterType((*RouteLookupRequest)(nil), "grpc.lookup.v1.RouteLookupRequest")
-	proto.RegisterMapType((map[string]string)(nil), "grpc.lookup.v1.RouteLookupRequest.KeyMapEntry")
-	proto.RegisterType((*RouteLookupResponse)(nil), "grpc.lookup.v1.RouteLookupResponse")
-}
-
-func init() { proto.RegisterFile("grpc/lookup/v1/rls.proto", fileDescriptor_3bab962d3362f3ca) }
-
-var fileDescriptor_3bab962d3362f3ca = []byte{
-	// 341 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xd1, 0x4b, 0xeb, 0x30,
-	0x14, 0xc6, 0x6f, 0xd7, 0xdd, 0x6e, 0x3b, 0x85, 0xcb, 0x6e, 0xee, 0x45, 0xca, 0x5e, 0x1c, 0xf5,
-	0x65, 0x0f, 0x92, 0xb2, 0xf9, 0xa2, 0x3e, 0x0e, 0x45, 0x50, 0x07, 0xa3, 0xfa, 0x20, 0x3e, 0x58,
-	0xe2, 0x76, 0xc8, 0x46, 0x6b, 0x13, 0x93, 0xb4, 0xd0, 0x3f, 0xd8, 0xff, 0x43, 0xda, 0x54, 0xd8,
-	0x14, 0xf4, 0xed, 0xfb, 0xbe, 0x73, 0x48, 0x7e, 0x27, 0x39, 0x10, 0x70, 0x25, 0x57, 0x51, 0x26,
-	0x44, 0x5a, 0xc8, 0xa8, 0x9c, 0x46, 0x2a, 0xd3, 0x54, 0x2a, 0x61, 0x04, 0xf9, 0x53, 0x57, 0xa8,
-	0xad, 0xd0, 0x72, 0x1a, 0xbe, 0x39, 0x40, 0x62, 0x51, 0x18, 0xbc, 0x6d, 0xa2, 0x18, 0x5f, 0x0b,
-	0xd4, 0x86, 0x1c, 0x80, 0xa7, 0x51, 0x95, 0xa8, 0x02, 0x67, 0xec, 0x4c, 0x06, 0x71, 0xeb, 0x08,
-	0x81, 0xae, 0x64, 0x66, 0x13, 0x74, 0x9a, 0xb4, 0xd1, 0xe4, 0x10, 0x7c, 0xc3, 0x14, 0x47, 0x93,
-	0x98, 0x4a, 0x62, 0xe0, 0x36, 0x25, 0xb0, 0xd1, 0x7d, 0x25, 0x91, 0x5c, 0x41, 0x2f, 0xc5, 0x2a,
-	0x79, 0x61, 0x32, 0xe8, 0x8e, 0xdd, 0x89, 0x3f, 0xa3, 0x74, 0x9f, 0x82, 0x7e, 0x25, 0xa0, 0x37,
-	0x58, 0x2d, 0x98, 0xbc, 0xcc, 0x8d, 0xaa, 0x62, 0x2f, 0x6d, 0xcc, 0xe8, 0x0c, 0xfc, 0x9d, 0x98,
-	0x0c, 0xc1, 0x4d, 0xb1, 0x6a, 0x09, 0x6b, 0x49, 0xfe, 0xc3, 0xef, 0x92, 0x65, 0x05, 0xb6, 0x7c,
-	0xd6, 0x9c, 0x77, 0x4e, 0x9d, 0xf0, 0x09, 0xfe, 0xed, 0x5d, 0xa2, 0xa5, 0xc8, 0x35, 0x92, 0x00,
-	0x7a, 0x16, 0x54, 0x07, 0xee, 0xd8, 0x9d, 0x0c, 0xe2, 0x0f, 0x5b, 0x4f, 0xb5, 0x41, 0xb6, 0x46,
-	0x95, 0xac, 0x99, 0x61, 0xed, 0x81, 0x60, 0xa3, 0x0b, 0x66, 0xd8, 0x75, 0xb7, 0xef, 0x0c, 0x3b,
-	0xb1, 0x67, 0xfb, 0x67, 0xf9, 0xde, 0x33, 0xde, 0xa1, 0x2a, 0xb7, 0x2b, 0x24, 0x0f, 0xe0, 0xef,
-	0xa4, 0x24, 0xfc, 0x79, 0xee, 0xd1, 0xd1, 0xb7, 0x3d, 0x16, 0x3b, 0xfc, 0x35, 0x5f, 0xc0, 0xdf,
-	0xad, 0xf8, 0xd4, 0x3a, 0xef, 0xc7, 0x99, 0x5e, 0xd6, 0xdf, 0xbc, 0x74, 0x1e, 0x8f, 0xb9, 0x10,
-	0x3c, 0x43, 0xca, 0x45, 0xc6, 0x72, 0x4e, 0x85, 0xe2, 0xd1, 0xee, 0x52, 0xd4, 0x3a, 0xb1, 0x3a,
-	0x29, 0xa7, 0xcf, 0x5e, 0xb3, 0x1d, 0x27, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0xca, 0x8d, 0x5c,
-	0xc7, 0x39, 0x02, 0x00, 0x00,
+var File_grpc_lookup_v1_rls_proto protoreflect.FileDescriptor
+
+var file_grpc_lookup_v1_rls_proto_rawDesc = []byte{
+	0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x76, 0x31,
+	0x2f, 0x72, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x22, 0xe5, 0x01, 0x0a, 0x12, 0x52,
+	0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74,
+	0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a,
+	0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x47,
+	0x0a, 0x07, 0x6b, 0x65, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x2e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31,
+	0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x2e, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
+	0x06, 0x6b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x1a, 0x39, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x4d, 0x61,
+	0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
+	0x38, 0x01, 0x22, 0x5e, 0x0a, 0x13, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
+	0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72,
+	0x67, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61,
+	0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72,
+	0x44, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x32, 0x6e, 0x0a, 0x12, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
+	0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74,
+	0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f,
+	0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75,
+	0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x00, 0x42, 0x4d, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f,
+	0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42, 0x08, 0x52, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61,
+	0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b,
+	0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76,
+	0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_lookup_v1_rls_proto_rawDescOnce sync.Once
+	file_grpc_lookup_v1_rls_proto_rawDescData = file_grpc_lookup_v1_rls_proto_rawDesc
+)
+
+func file_grpc_lookup_v1_rls_proto_rawDescGZIP() []byte {
+	file_grpc_lookup_v1_rls_proto_rawDescOnce.Do(func() {
+		file_grpc_lookup_v1_rls_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_lookup_v1_rls_proto_rawDescData)
+	})
+	return file_grpc_lookup_v1_rls_proto_rawDescData
+}
+
+var file_grpc_lookup_v1_rls_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_grpc_lookup_v1_rls_proto_goTypes = []interface{}{
+	(*RouteLookupRequest)(nil),  // 0: grpc.lookup.v1.RouteLookupRequest
+	(*RouteLookupResponse)(nil), // 1: grpc.lookup.v1.RouteLookupResponse
+	nil,                         // 2: grpc.lookup.v1.RouteLookupRequest.KeyMapEntry
+}
+var file_grpc_lookup_v1_rls_proto_depIdxs = []int32{
+	2, // 0: grpc.lookup.v1.RouteLookupRequest.key_map:type_name -> grpc.lookup.v1.RouteLookupRequest.KeyMapEntry
+	0, // 1: grpc.lookup.v1.RouteLookupService.RouteLookup:input_type -> grpc.lookup.v1.RouteLookupRequest
+	1, // 2: grpc.lookup.v1.RouteLookupService.RouteLookup:output_type -> grpc.lookup.v1.RouteLookupResponse
+	2, // [2:3] is the sub-list for method output_type
+	1, // [1:2] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_grpc_lookup_v1_rls_proto_init() }
+func file_grpc_lookup_v1_rls_proto_init() {
+	if File_grpc_lookup_v1_rls_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_lookup_v1_rls_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouteLookupRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lookup_v1_rls_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouteLookupResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_lookup_v1_rls_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   3,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_lookup_v1_rls_proto_goTypes,
+		DependencyIndexes: file_grpc_lookup_v1_rls_proto_depIdxs,
+		MessageInfos:      file_grpc_lookup_v1_rls_proto_msgTypes,
+	}.Build()
+	File_grpc_lookup_v1_rls_proto = out.File
+	file_grpc_lookup_v1_rls_proto_rawDesc = nil
+	file_grpc_lookup_v1_rls_proto_goTypes = nil
+	file_grpc_lookup_v1_rls_proto_depIdxs = nil
 }
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
index 01d5b656ebe2..e8a7a6dae544 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
@@ -1,31 +1,54 @@
+// Copyright 2020 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/lookup/v1/rls_config.proto
 
 package grpc_lookup_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Extract a key based on a given name (e.g. header name or query parameter
 // name).  The name must match one of the names listed in the "name" field.  If
 // the "required_match" field is true, one of the specified names must be
 // present for the keybuilder to match.
 type NameMatcher struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The name that will be used in the RLS key_map to refer to this value.
 	Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
 	// Ordered list of names (headers or query parameter names) that can supply
@@ -33,158 +56,119 @@ type NameMatcher struct {
 	Names []string `protobuf:"bytes,2,rep,name=names,proto3" json:"names,omitempty"`
 	// If true, make this extraction required; the key builder will not match
 	// if no value is found.
-	RequiredMatch        bool     `protobuf:"varint,3,opt,name=required_match,json=requiredMatch,proto3" json:"required_match,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	RequiredMatch bool `protobuf:"varint,3,opt,name=required_match,json=requiredMatch,proto3" json:"required_match,omitempty"`
 }
 
-func (m *NameMatcher) Reset()         { *m = NameMatcher{} }
-func (m *NameMatcher) String() string { return proto.CompactTextString(m) }
-func (*NameMatcher) ProtoMessage()    {}
-func (*NameMatcher) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe74d4f6e8f01c1, []int{0}
+func (x *NameMatcher) Reset() {
+	*x = NameMatcher{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *NameMatcher) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_NameMatcher.Unmarshal(m, b)
-}
-func (m *NameMatcher) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_NameMatcher.Marshal(b, m, deterministic)
-}
-func (m *NameMatcher) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_NameMatcher.Merge(m, src)
+func (x *NameMatcher) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *NameMatcher) XXX_Size() int {
-	return xxx_messageInfo_NameMatcher.Size(m)
-}
-func (m *NameMatcher) XXX_DiscardUnknown() {
-	xxx_messageInfo_NameMatcher.DiscardUnknown(m)
+
+func (*NameMatcher) ProtoMessage() {}
+
+func (x *NameMatcher) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_NameMatcher proto.InternalMessageInfo
+// Deprecated: Use NameMatcher.ProtoReflect.Descriptor instead.
+func (*NameMatcher) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *NameMatcher) GetKey() string {
-	if m != nil {
-		return m.Key
+func (x *NameMatcher) GetKey() string {
+	if x != nil {
+		return x.Key
 	}
 	return ""
 }
 
-func (m *NameMatcher) GetNames() []string {
-	if m != nil {
-		return m.Names
+func (x *NameMatcher) GetNames() []string {
+	if x != nil {
+		return x.Names
 	}
 	return nil
 }
 
-func (m *NameMatcher) GetRequiredMatch() bool {
-	if m != nil {
-		return m.RequiredMatch
+func (x *NameMatcher) GetRequiredMatch() bool {
+	if x != nil {
+		return x.RequiredMatch
 	}
 	return false
 }
 
 // A GrpcKeyBuilder applies to a given gRPC service, name, and headers.
 type GrpcKeyBuilder struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	Names []*GrpcKeyBuilder_Name `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"`
 	// Extract keys from all listed headers.
 	// For gRPC, it is an error to specify "required_match" on the NameMatcher
 	// protos.
-	Headers              []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
-	XXX_unrecognized     []byte         `json:"-"`
-	XXX_sizecache        int32          `json:"-"`
-}
-
-func (m *GrpcKeyBuilder) Reset()         { *m = GrpcKeyBuilder{} }
-func (m *GrpcKeyBuilder) String() string { return proto.CompactTextString(m) }
-func (*GrpcKeyBuilder) ProtoMessage()    {}
-func (*GrpcKeyBuilder) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe74d4f6e8f01c1, []int{1}
-}
-
-func (m *GrpcKeyBuilder) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GrpcKeyBuilder.Unmarshal(m, b)
-}
-func (m *GrpcKeyBuilder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GrpcKeyBuilder.Marshal(b, m, deterministic)
-}
-func (m *GrpcKeyBuilder) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GrpcKeyBuilder.Merge(m, src)
-}
-func (m *GrpcKeyBuilder) XXX_Size() int {
-	return xxx_messageInfo_GrpcKeyBuilder.Size(m)
-}
-func (m *GrpcKeyBuilder) XXX_DiscardUnknown() {
-	xxx_messageInfo_GrpcKeyBuilder.DiscardUnknown(m)
+	Headers []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"`
 }
 
-var xxx_messageInfo_GrpcKeyBuilder proto.InternalMessageInfo
-
-func (m *GrpcKeyBuilder) GetNames() []*GrpcKeyBuilder_Name {
-	if m != nil {
-		return m.Names
+func (x *GrpcKeyBuilder) Reset() {
+	*x = GrpcKeyBuilder{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *GrpcKeyBuilder) GetHeaders() []*NameMatcher {
-	if m != nil {
-		return m.Headers
-	}
-	return nil
+func (x *GrpcKeyBuilder) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-// To match, one of the given Name fields must match; the service and method
-// fields are specified as fixed strings.  The service name is required and
-// includes the proto package name.  The method name may be omitted, in
-// which case any method on the given service is matched.
-type GrpcKeyBuilder_Name struct {
-	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
-	Method               string   `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+func (*GrpcKeyBuilder) ProtoMessage() {}
 
-func (m *GrpcKeyBuilder_Name) Reset()         { *m = GrpcKeyBuilder_Name{} }
-func (m *GrpcKeyBuilder_Name) String() string { return proto.CompactTextString(m) }
-func (*GrpcKeyBuilder_Name) ProtoMessage()    {}
-func (*GrpcKeyBuilder_Name) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe74d4f6e8f01c1, []int{1, 0}
+func (x *GrpcKeyBuilder) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-func (m *GrpcKeyBuilder_Name) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GrpcKeyBuilder_Name.Unmarshal(m, b)
-}
-func (m *GrpcKeyBuilder_Name) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GrpcKeyBuilder_Name.Marshal(b, m, deterministic)
-}
-func (m *GrpcKeyBuilder_Name) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GrpcKeyBuilder_Name.Merge(m, src)
-}
-func (m *GrpcKeyBuilder_Name) XXX_Size() int {
-	return xxx_messageInfo_GrpcKeyBuilder_Name.Size(m)
-}
-func (m *GrpcKeyBuilder_Name) XXX_DiscardUnknown() {
-	xxx_messageInfo_GrpcKeyBuilder_Name.DiscardUnknown(m)
+// Deprecated: Use GrpcKeyBuilder.ProtoReflect.Descriptor instead.
+func (*GrpcKeyBuilder) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1}
 }
 
-var xxx_messageInfo_GrpcKeyBuilder_Name proto.InternalMessageInfo
-
-func (m *GrpcKeyBuilder_Name) GetService() string {
-	if m != nil {
-		return m.Service
+func (x *GrpcKeyBuilder) GetNames() []*GrpcKeyBuilder_Name {
+	if x != nil {
+		return x.Names
 	}
-	return ""
+	return nil
 }
 
-func (m *GrpcKeyBuilder_Name) GetMethod() string {
-	if m != nil {
-		return m.Method
+func (x *GrpcKeyBuilder) GetHeaders() []*NameMatcher {
+	if x != nil {
+		return x.Headers
 	}
-	return ""
+	return nil
 }
 
 // An HttpKeyBuilder applies to a given HTTP URL and headers.
@@ -204,6 +188,10 @@ func (m *GrpcKeyBuilder_Name) GetMethod() string {
 // subdomain will be used as the id and the first segment as the object. If
 // neither pattern matches, no keys will be extracted.
 type HttpKeyBuilder struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// host_pattern is an ordered list of host template patterns for the desired
 	// value.  If any host_pattern values are specified, then at least one must
 	// match, and the last one wins and sets any specified variables.  A host
@@ -256,66 +244,74 @@ type HttpKeyBuilder struct {
 	// are marked as "required_match" and are not present, this keybuilder fails
 	// to match.  If a given header appears multiple times in the request we will
 	// report it as a comma-separated string, in standard HTTP fashion.
-	Headers              []*NameMatcher `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
-	XXX_unrecognized     []byte         `json:"-"`
-	XXX_sizecache        int32          `json:"-"`
+	Headers []*NameMatcher `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"`
 }
 
-func (m *HttpKeyBuilder) Reset()         { *m = HttpKeyBuilder{} }
-func (m *HttpKeyBuilder) String() string { return proto.CompactTextString(m) }
-func (*HttpKeyBuilder) ProtoMessage()    {}
-func (*HttpKeyBuilder) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe74d4f6e8f01c1, []int{2}
+func (x *HttpKeyBuilder) Reset() {
+	*x = HttpKeyBuilder{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HttpKeyBuilder) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HttpKeyBuilder.Unmarshal(m, b)
+func (x *HttpKeyBuilder) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HttpKeyBuilder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HttpKeyBuilder.Marshal(b, m, deterministic)
-}
-func (m *HttpKeyBuilder) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HttpKeyBuilder.Merge(m, src)
-}
-func (m *HttpKeyBuilder) XXX_Size() int {
-	return xxx_messageInfo_HttpKeyBuilder.Size(m)
-}
-func (m *HttpKeyBuilder) XXX_DiscardUnknown() {
-	xxx_messageInfo_HttpKeyBuilder.DiscardUnknown(m)
+
+func (*HttpKeyBuilder) ProtoMessage() {}
+
+func (x *HttpKeyBuilder) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HttpKeyBuilder proto.InternalMessageInfo
+// Deprecated: Use HttpKeyBuilder.ProtoReflect.Descriptor instead.
+func (*HttpKeyBuilder) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *HttpKeyBuilder) GetHostPatterns() []string {
-	if m != nil {
-		return m.HostPatterns
+func (x *HttpKeyBuilder) GetHostPatterns() []string {
+	if x != nil {
+		return x.HostPatterns
 	}
 	return nil
 }
 
-func (m *HttpKeyBuilder) GetPathPatterns() []string {
-	if m != nil {
-		return m.PathPatterns
+func (x *HttpKeyBuilder) GetPathPatterns() []string {
+	if x != nil {
+		return x.PathPatterns
 	}
 	return nil
 }
 
-func (m *HttpKeyBuilder) GetQueryParameters() []*NameMatcher {
-	if m != nil {
-		return m.QueryParameters
+func (x *HttpKeyBuilder) GetQueryParameters() []*NameMatcher {
+	if x != nil {
+		return x.QueryParameters
 	}
 	return nil
 }
 
-func (m *HttpKeyBuilder) GetHeaders() []*NameMatcher {
-	if m != nil {
-		return m.Headers
+func (x *HttpKeyBuilder) GetHeaders() []*NameMatcher {
+	if x != nil {
+		return x.Headers
 	}
 	return nil
 }
 
 type RouteLookupConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Ordered specifications for constructing keys for HTTP requests.  Last
 	// match wins.  If no HttpKeyBuilder matches, an empty key_map will be sent to
 	// the lookup service; it should likely reply with a global default route
@@ -357,149 +353,366 @@ type RouteLookupConfig struct {
 	// used if RLS returns an error, times out, or returns an invalid response.
 	// Note that requests can be routed only to a subdomain of the original
 	// target, e.g. "us_east_1.cloudbigtable.googleapis.com".
-	DefaultTarget        string   `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	DefaultTarget string `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"`
 }
 
-func (m *RouteLookupConfig) Reset()         { *m = RouteLookupConfig{} }
-func (m *RouteLookupConfig) String() string { return proto.CompactTextString(m) }
-func (*RouteLookupConfig) ProtoMessage()    {}
-func (*RouteLookupConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_5fe74d4f6e8f01c1, []int{3}
+func (x *RouteLookupConfig) Reset() {
+	*x = RouteLookupConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RouteLookupConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RouteLookupConfig.Unmarshal(m, b)
-}
-func (m *RouteLookupConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RouteLookupConfig.Marshal(b, m, deterministic)
-}
-func (m *RouteLookupConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RouteLookupConfig.Merge(m, src)
+func (x *RouteLookupConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RouteLookupConfig) XXX_Size() int {
-	return xxx_messageInfo_RouteLookupConfig.Size(m)
-}
-func (m *RouteLookupConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_RouteLookupConfig.DiscardUnknown(m)
+
+func (*RouteLookupConfig) ProtoMessage() {}
+
+func (x *RouteLookupConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RouteLookupConfig proto.InternalMessageInfo
+// Deprecated: Use RouteLookupConfig.ProtoReflect.Descriptor instead.
+func (*RouteLookupConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *RouteLookupConfig) GetHttpKeybuilders() []*HttpKeyBuilder {
-	if m != nil {
-		return m.HttpKeybuilders
+func (x *RouteLookupConfig) GetHttpKeybuilders() []*HttpKeyBuilder {
+	if x != nil {
+		return x.HttpKeybuilders
 	}
 	return nil
 }
 
-func (m *RouteLookupConfig) GetGrpcKeybuilders() []*GrpcKeyBuilder {
-	if m != nil {
-		return m.GrpcKeybuilders
+func (x *RouteLookupConfig) GetGrpcKeybuilders() []*GrpcKeyBuilder {
+	if x != nil {
+		return x.GrpcKeybuilders
 	}
 	return nil
 }
 
-func (m *RouteLookupConfig) GetLookupService() string {
-	if m != nil {
-		return m.LookupService
+func (x *RouteLookupConfig) GetLookupService() string {
+	if x != nil {
+		return x.LookupService
 	}
 	return ""
 }
 
-func (m *RouteLookupConfig) GetLookupServiceTimeout() *duration.Duration {
-	if m != nil {
-		return m.LookupServiceTimeout
+func (x *RouteLookupConfig) GetLookupServiceTimeout() *duration.Duration {
+	if x != nil {
+		return x.LookupServiceTimeout
 	}
 	return nil
 }
 
-func (m *RouteLookupConfig) GetMaxAge() *duration.Duration {
-	if m != nil {
-		return m.MaxAge
+func (x *RouteLookupConfig) GetMaxAge() *duration.Duration {
+	if x != nil {
+		return x.MaxAge
 	}
 	return nil
 }
 
-func (m *RouteLookupConfig) GetStaleAge() *duration.Duration {
-	if m != nil {
-		return m.StaleAge
+func (x *RouteLookupConfig) GetStaleAge() *duration.Duration {
+	if x != nil {
+		return x.StaleAge
 	}
 	return nil
 }
 
-func (m *RouteLookupConfig) GetCacheSizeBytes() int64 {
-	if m != nil {
-		return m.CacheSizeBytes
+func (x *RouteLookupConfig) GetCacheSizeBytes() int64 {
+	if x != nil {
+		return x.CacheSizeBytes
 	}
 	return 0
 }
 
-func (m *RouteLookupConfig) GetValidTargets() []string {
-	if m != nil {
-		return m.ValidTargets
+func (x *RouteLookupConfig) GetValidTargets() []string {
+	if x != nil {
+		return x.ValidTargets
 	}
 	return nil
 }
 
-func (m *RouteLookupConfig) GetDefaultTarget() string {
-	if m != nil {
-		return m.DefaultTarget
+func (x *RouteLookupConfig) GetDefaultTarget() string {
+	if x != nil {
+		return x.DefaultTarget
 	}
 	return ""
 }
 
-func init() {
-	proto.RegisterType((*NameMatcher)(nil), "grpc.lookup.v1.NameMatcher")
-	proto.RegisterType((*GrpcKeyBuilder)(nil), "grpc.lookup.v1.GrpcKeyBuilder")
-	proto.RegisterType((*GrpcKeyBuilder_Name)(nil), "grpc.lookup.v1.GrpcKeyBuilder.Name")
-	proto.RegisterType((*HttpKeyBuilder)(nil), "grpc.lookup.v1.HttpKeyBuilder")
-	proto.RegisterType((*RouteLookupConfig)(nil), "grpc.lookup.v1.RouteLookupConfig")
-}
-
-func init() { proto.RegisterFile("grpc/lookup/v1/rls_config.proto", fileDescriptor_5fe74d4f6e8f01c1) }
-
-var fileDescriptor_5fe74d4f6e8f01c1 = []byte{
-	// 615 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xed, 0x6a, 0xdb, 0x30,
-	0x14, 0xc5, 0x75, 0x9a, 0x0f, 0xa5, 0x75, 0x53, 0x53, 0x8a, 0xd7, 0x42, 0x17, 0x52, 0x0a, 0xfe,
-	0x31, 0x1c, 0x9a, 0xb1, 0xb1, 0xfd, 0x5c, 0x36, 0xf6, 0xfd, 0x11, 0xdc, 0xfe, 0x1a, 0x03, 0xa1,
-	0xd8, 0xb7, 0xb2, 0xa9, 0x1d, 0xb9, 0x92, 0x1c, 0x9a, 0x3e, 0xd0, 0x9e, 0x62, 0x2f, 0xb2, 0xb7,
-	0x19, 0x92, 0xec, 0x2e, 0x2e, 0x83, 0xec, 0x5f, 0xee, 0xd1, 0x39, 0x27, 0x3a, 0xf7, 0x5e, 0x19,
-	0x3d, 0xa6, 0xbc, 0x88, 0xc6, 0x19, 0x63, 0xd7, 0x65, 0x31, 0x5e, 0x9e, 0x8f, 0x79, 0x26, 0x70,
-	0xc4, 0x16, 0x57, 0x29, 0x0d, 0x0a, 0xce, 0x24, 0x73, 0x1d, 0x45, 0x08, 0x0c, 0x21, 0x58, 0x9e,
-	0x1f, 0x9d, 0x50, 0xc6, 0x68, 0x06, 0x63, 0x7d, 0x3a, 0x2f, 0xaf, 0xc6, 0x71, 0xc9, 0x89, 0x4c,
-	0xd9, 0xc2, 0xf0, 0x47, 0x3f, 0x50, 0xff, 0x2b, 0xc9, 0xe1, 0x0b, 0x91, 0x51, 0x02, 0xdc, 0x1d,
-	0x20, 0xfb, 0x1a, 0x56, 0x9e, 0x35, 0xb4, 0xfc, 0x5e, 0xa8, 0x7e, 0xba, 0x07, 0x68, 0x7b, 0x41,
-	0x72, 0x10, 0xde, 0xd6, 0xd0, 0xf6, 0x7b, 0xa1, 0x29, 0xdc, 0x33, 0xe4, 0x70, 0xb8, 0x29, 0x53,
-	0x0e, 0x31, 0xce, 0x95, 0xd6, 0xb3, 0x87, 0x96, 0xdf, 0x0d, 0x77, 0x6b, 0x54, 0x1b, 0x8e, 0x7e,
-	0x59, 0xc8, 0x79, 0xc7, 0x8b, 0xe8, 0x13, 0xac, 0xa6, 0x65, 0x9a, 0xc5, 0xc0, 0xdd, 0x97, 0xb5,
-	0x9f, 0x35, 0xb4, 0xfd, 0xfe, 0xe4, 0x34, 0x68, 0x5e, 0x38, 0x68, 0xd2, 0x03, 0x75, 0xb9, 0xfa,
-	0x4f, 0x9f, 0xa1, 0x4e, 0x02, 0x24, 0x06, 0x6e, 0x2e, 0xd3, 0x9f, 0x1c, 0x3f, 0x14, 0xaf, 0x45,
-	0x09, 0x6b, 0xee, 0xd1, 0x0b, 0xd4, 0x52, 0xb8, 0xeb, 0xa1, 0x8e, 0x00, 0xbe, 0x4c, 0x23, 0xa8,
-	0xf2, 0xd5, 0xa5, 0x7b, 0x88, 0xda, 0x39, 0xc8, 0x84, 0xc5, 0xde, 0x96, 0x3e, 0xa8, 0xaa, 0xd1,
-	0x6f, 0x0b, 0x39, 0xef, 0xa5, 0x2c, 0xd6, 0xae, 0x7f, 0x8a, 0x76, 0x13, 0x26, 0x24, 0x2e, 0x88,
-	0x94, 0xc0, 0x17, 0x26, 0x46, 0x2f, 0xdc, 0x51, 0xe0, 0xac, 0xc2, 0x14, 0xa9, 0x20, 0x32, 0xf9,
-	0x4b, 0x32, 0xbd, 0xdb, 0x51, 0xe0, 0x3d, 0xe9, 0x2d, 0x1a, 0xdc, 0x94, 0xc0, 0x57, 0xb8, 0x20,
-	0x9c, 0xe4, 0x20, 0x55, 0x2c, 0x7b, 0x73, 0xac, 0x3d, 0x2d, 0x9a, 0xdd, 0x6b, 0xd6, 0xbb, 0xd2,
-	0xfa, 0xff, 0xae, 0x8c, 0x7e, 0xb6, 0xd0, 0x7e, 0xc8, 0x4a, 0x09, 0x9f, 0x35, 0xef, 0xb5, 0x5e,
-	0x22, 0xf7, 0x03, 0x1a, 0x24, 0x52, 0x16, 0xf8, 0x1a, 0x56, 0x73, 0x93, 0xb8, 0x1e, 0xd4, 0xc9,
-	0x43, 0xd7, 0x66, 0x63, 0xc2, 0xbd, 0xc4, 0xd4, 0xb5, 0x4c, 0x59, 0x29, 0x45, 0xc3, 0x6a, 0xeb,
-	0xdf, 0x56, 0xcd, 0x99, 0x87, 0x7b, 0xd4, 0xd4, 0xf7, 0x56, 0x67, 0xc8, 0x31, 0x64, 0x5c, 0x0f,
-	0xd0, 0xd6, 0x73, 0xda, 0x35, 0xe8, 0x45, 0x35, 0xc6, 0x6f, 0xe8, 0xb0, 0x49, 0xc3, 0x32, 0xcd,
-	0x81, 0x95, 0xd2, 0x6b, 0x0d, 0x2d, 0xbf, 0x3f, 0x79, 0x14, 0x98, 0xc7, 0x10, 0xd4, 0x8f, 0x21,
-	0x78, 0x53, 0x3d, 0x86, 0xf0, 0xa0, 0xe1, 0x74, 0x69, 0x64, 0xee, 0x04, 0x75, 0x72, 0x72, 0x8b,
-	0x09, 0x05, 0x6f, 0x7b, 0x93, 0x43, 0x3b, 0x27, 0xb7, 0xaf, 0x28, 0xb8, 0xcf, 0x51, 0x4f, 0x48,
-	0x92, 0x81, 0x56, 0xb5, 0x37, 0xa9, 0xba, 0x9a, 0xab, 0x74, 0x3e, 0x1a, 0x44, 0x24, 0x4a, 0x00,
-	0x8b, 0xf4, 0x0e, 0xf0, 0x7c, 0x25, 0x41, 0x78, 0x9d, 0xa1, 0xe5, 0xdb, 0xa1, 0xa3, 0xf1, 0x8b,
-	0xf4, 0x0e, 0xa6, 0x0a, 0x55, 0xdb, 0xb5, 0x24, 0x59, 0x1a, 0x63, 0x49, 0x38, 0x05, 0x29, 0xbc,
-	0xae, 0xd9, 0x2e, 0x0d, 0x5e, 0x1a, 0x4c, 0xb5, 0x2c, 0x86, 0x2b, 0x52, 0x66, 0xb2, 0xa2, 0x79,
-	0x3d, 0xd3, 0xb2, 0x0a, 0x35, 0xbc, 0x8f, 0xad, 0x2e, 0x1a, 0xf4, 0xc3, 0x63, 0xf5, 0x6a, 0x41,
-	0x6d, 0x35, 0x67, 0x11, 0x08, 0x91, 0x2e, 0x28, 0x16, 0x92, 0x13, 0x09, 0x74, 0x35, 0xbd, 0x40,
-	0xfb, 0x29, 0x7b, 0x30, 0xb1, 0xa9, 0x13, 0x66, 0xc2, 0xac, 0xcc, 0x4c, 0x65, 0x9a, 0x59, 0xdf,
-	0x9f, 0x54, 0x19, 0x29, 0xcb, 0xc8, 0x82, 0x06, 0x8c, 0xd3, 0xf1, 0xfa, 0xb7, 0x4a, 0xaf, 0x42,
-	0x35, 0x9d, 0xe5, 0xf9, 0xbc, 0xad, 0x5b, 0xf1, 0xf4, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff,
-	0xaa, 0xbd, 0xb2, 0xd0, 0x04, 0x00, 0x00,
+// To match, one of the given Name fields must match; the service and method
+// fields are specified as fixed strings.  The service name is required and
+// includes the proto package name.  The method name may be omitted, in
+// which case any method on the given service is matched.
+type GrpcKeyBuilder_Name struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	Method  string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
+}
+
+func (x *GrpcKeyBuilder_Name) Reset() {
+	*x = GrpcKeyBuilder_Name{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GrpcKeyBuilder_Name) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GrpcKeyBuilder_Name) ProtoMessage() {}
+
+func (x *GrpcKeyBuilder_Name) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GrpcKeyBuilder_Name.ProtoReflect.Descriptor instead.
+func (*GrpcKeyBuilder_Name) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1, 0}
+}
+
+func (x *GrpcKeyBuilder_Name) GetService() string {
+	if x != nil {
+		return x.Service
+	}
+	return ""
+}
+
+func (x *GrpcKeyBuilder_Name) GetMethod() string {
+	if x != nil {
+		return x.Method
+	}
+	return ""
+}
+
+var File_grpc_lookup_v1_rls_config_proto protoreflect.FileDescriptor
+
+var file_grpc_lookup_v1_rls_config_proto_rawDesc = []byte{
+	0x0a, 0x1f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x76, 0x31,
+	0x2f, 0x72, 0x6c, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76,
+	0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
+	0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x22, 0x5c, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72,
+	0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
+	0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x71, 0x75,
+	0x69, 0x72, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
+	0x52, 0x0d, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22,
+	0xbc, 0x01, 0x0a, 0x0e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64,
+	0x65, 0x72, 0x12, 0x39, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e,
+	0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65,
+	0x72, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x35, 0x0a,
+	0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e,
+	0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65, 0x61,
+	0x64, 0x65, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x22, 0xd9,
+	0x01, 0x0a, 0x0e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65,
+	0x72, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72,
+	0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x6f, 0x73, 0x74, 0x50, 0x61,
+	0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70,
+	0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70,
+	0x61, 0x74, 0x68, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x46, 0x0a, 0x10, 0x71,
+	0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18,
+	0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68,
+	0x65, 0x72, 0x52, 0x0f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74,
+	0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b,
+	0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65,
+	0x72, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x22, 0xa6, 0x04, 0x0a, 0x11, 0x52,
+	0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x12, 0x49, 0x0a, 0x10, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c,
+	0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70,
+	0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x0f, 0x68, 0x74, 0x74, 0x70,
+	0x4b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x67,
+	0x72, 0x70, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x18,
+	0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75,
+	0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x62, 0x75,
+	0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
+	0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
+	0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, 0x0a,
+	0x16, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f,
+	0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
+	0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
+	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x32,
+	0x0a, 0x07, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41,
+	0x67, 0x65, 0x12, 0x36, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x61, 0x67, 0x65, 0x18,
+	0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x52, 0x08, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x41, 0x67, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x61,
+	0x63, 0x68, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x42,
+	0x79, 0x74, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x74, 0x61,
+	0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c,
+	0x69, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x66,
+	0x61, 0x75, 0x6c, 0x74, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74,
+	0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x1b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f,
+	0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74,
+	0x65, 0x67, 0x79, 0x42, 0x53, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x52, 0x6c, 0x73, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72,
+	0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_lookup_v1_rls_config_proto_rawDescOnce sync.Once
+	file_grpc_lookup_v1_rls_config_proto_rawDescData = file_grpc_lookup_v1_rls_config_proto_rawDesc
+)
+
+func file_grpc_lookup_v1_rls_config_proto_rawDescGZIP() []byte {
+	file_grpc_lookup_v1_rls_config_proto_rawDescOnce.Do(func() {
+		file_grpc_lookup_v1_rls_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_lookup_v1_rls_config_proto_rawDescData)
+	})
+	return file_grpc_lookup_v1_rls_config_proto_rawDescData
+}
+
+var file_grpc_lookup_v1_rls_config_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_grpc_lookup_v1_rls_config_proto_goTypes = []interface{}{
+	(*NameMatcher)(nil),         // 0: grpc.lookup.v1.NameMatcher
+	(*GrpcKeyBuilder)(nil),      // 1: grpc.lookup.v1.GrpcKeyBuilder
+	(*HttpKeyBuilder)(nil),      // 2: grpc.lookup.v1.HttpKeyBuilder
+	(*RouteLookupConfig)(nil),   // 3: grpc.lookup.v1.RouteLookupConfig
+	(*GrpcKeyBuilder_Name)(nil), // 4: grpc.lookup.v1.GrpcKeyBuilder.Name
+	(*duration.Duration)(nil),   // 5: google.protobuf.Duration
+}
+var file_grpc_lookup_v1_rls_config_proto_depIdxs = []int32{
+	4, // 0: grpc.lookup.v1.GrpcKeyBuilder.names:type_name -> grpc.lookup.v1.GrpcKeyBuilder.Name
+	0, // 1: grpc.lookup.v1.GrpcKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher
+	0, // 2: grpc.lookup.v1.HttpKeyBuilder.query_parameters:type_name -> grpc.lookup.v1.NameMatcher
+	0, // 3: grpc.lookup.v1.HttpKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher
+	2, // 4: grpc.lookup.v1.RouteLookupConfig.http_keybuilders:type_name -> grpc.lookup.v1.HttpKeyBuilder
+	1, // 5: grpc.lookup.v1.RouteLookupConfig.grpc_keybuilders:type_name -> grpc.lookup.v1.GrpcKeyBuilder
+	5, // 6: grpc.lookup.v1.RouteLookupConfig.lookup_service_timeout:type_name -> google.protobuf.Duration
+	5, // 7: grpc.lookup.v1.RouteLookupConfig.max_age:type_name -> google.protobuf.Duration
+	5, // 8: grpc.lookup.v1.RouteLookupConfig.stale_age:type_name -> google.protobuf.Duration
+	9, // [9:9] is the sub-list for method output_type
+	9, // [9:9] is the sub-list for method input_type
+	9, // [9:9] is the sub-list for extension type_name
+	9, // [9:9] is the sub-list for extension extendee
+	0, // [0:9] is the sub-list for field type_name
+}
+
+func init() { file_grpc_lookup_v1_rls_config_proto_init() }
+func file_grpc_lookup_v1_rls_config_proto_init() {
+	if File_grpc_lookup_v1_rls_config_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_lookup_v1_rls_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*NameMatcher); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lookup_v1_rls_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GrpcKeyBuilder); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lookup_v1_rls_config_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HttpKeyBuilder); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lookup_v1_rls_config_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouteLookupConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_lookup_v1_rls_config_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GrpcKeyBuilder_Name); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_lookup_v1_rls_config_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   5,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_lookup_v1_rls_config_proto_goTypes,
+		DependencyIndexes: file_grpc_lookup_v1_rls_config_proto_depIdxs,
+		MessageInfos:      file_grpc_lookup_v1_rls_config_proto_msgTypes,
+	}.Build()
+	File_grpc_lookup_v1_rls_config_proto = out.File
+	file_grpc_lookup_v1_rls_config_proto_rawDesc = nil
+	file_grpc_lookup_v1_rls_config_proto_goTypes = nil
+	file_grpc_lookup_v1_rls_config_proto_depIdxs = nil
 }
diff --git a/benchmark/grpc_testing/control.pb.go b/benchmark/grpc_testing/control.pb.go
index 7a109d8e9cb8..9cb9a2fcde6e 100644
--- a/benchmark/grpc_testing/control.pb.go
+++ b/benchmark/grpc_testing/control.pb.go
@@ -1,24 +1,43 @@
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: benchmark/grpc_testing/control.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type ClientType int32
 
@@ -27,22 +46,43 @@ const (
 	ClientType_ASYNC_CLIENT ClientType = 1
 )
 
-var ClientType_name = map[int32]string{
-	0: "SYNC_CLIENT",
-	1: "ASYNC_CLIENT",
-}
+// Enum value maps for ClientType.
+var (
+	ClientType_name = map[int32]string{
+		0: "SYNC_CLIENT",
+		1: "ASYNC_CLIENT",
+	}
+	ClientType_value = map[string]int32{
+		"SYNC_CLIENT":  0,
+		"ASYNC_CLIENT": 1,
+	}
+)
 
-var ClientType_value = map[string]int32{
-	"SYNC_CLIENT":  0,
-	"ASYNC_CLIENT": 1,
+func (x ClientType) Enum() *ClientType {
+	p := new(ClientType)
+	*p = x
+	return p
 }
 
 func (x ClientType) String() string {
-	return proto.EnumName(ClientType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ClientType) Descriptor() protoreflect.EnumDescriptor {
+	return file_benchmark_grpc_testing_control_proto_enumTypes[0].Descriptor()
+}
+
+func (ClientType) Type() protoreflect.EnumType {
+	return &file_benchmark_grpc_testing_control_proto_enumTypes[0]
+}
+
+func (x ClientType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
+// Deprecated: Use ClientType.Descriptor instead.
 func (ClientType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{0}
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{0}
 }
 
 type ServerType int32
@@ -53,24 +93,45 @@ const (
 	ServerType_ASYNC_GENERIC_SERVER ServerType = 2
 )
 
-var ServerType_name = map[int32]string{
-	0: "SYNC_SERVER",
-	1: "ASYNC_SERVER",
-	2: "ASYNC_GENERIC_SERVER",
-}
+// Enum value maps for ServerType.
+var (
+	ServerType_name = map[int32]string{
+		0: "SYNC_SERVER",
+		1: "ASYNC_SERVER",
+		2: "ASYNC_GENERIC_SERVER",
+	}
+	ServerType_value = map[string]int32{
+		"SYNC_SERVER":          0,
+		"ASYNC_SERVER":         1,
+		"ASYNC_GENERIC_SERVER": 2,
+	}
+)
 
-var ServerType_value = map[string]int32{
-	"SYNC_SERVER":          0,
-	"ASYNC_SERVER":         1,
-	"ASYNC_GENERIC_SERVER": 2,
+func (x ServerType) Enum() *ServerType {
+	p := new(ServerType)
+	*p = x
+	return p
 }
 
 func (x ServerType) String() string {
-	return proto.EnumName(ServerType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
+func (ServerType) Descriptor() protoreflect.EnumDescriptor {
+	return file_benchmark_grpc_testing_control_proto_enumTypes[1].Descriptor()
+}
+
+func (ServerType) Type() protoreflect.EnumType {
+	return &file_benchmark_grpc_testing_control_proto_enumTypes[1]
+}
+
+func (x ServerType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ServerType.Descriptor instead.
 func (ServerType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{1}
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{1}
 }
 
 type RpcType int32
@@ -80,195 +141,248 @@ const (
 	RpcType_STREAMING RpcType = 1
 )
 
-var RpcType_name = map[int32]string{
-	0: "UNARY",
-	1: "STREAMING",
-}
+// Enum value maps for RpcType.
+var (
+	RpcType_name = map[int32]string{
+		0: "UNARY",
+		1: "STREAMING",
+	}
+	RpcType_value = map[string]int32{
+		"UNARY":     0,
+		"STREAMING": 1,
+	}
+)
 
-var RpcType_value = map[string]int32{
-	"UNARY":     0,
-	"STREAMING": 1,
+func (x RpcType) Enum() *RpcType {
+	p := new(RpcType)
+	*p = x
+	return p
 }
 
 func (x RpcType) String() string {
-	return proto.EnumName(RpcType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (RpcType) Descriptor() protoreflect.EnumDescriptor {
+	return file_benchmark_grpc_testing_control_proto_enumTypes[2].Descriptor()
+}
+
+func (RpcType) Type() protoreflect.EnumType {
+	return &file_benchmark_grpc_testing_control_proto_enumTypes[2]
+}
+
+func (x RpcType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
+// Deprecated: Use RpcType.Descriptor instead.
 func (RpcType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{2}
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{2}
 }
 
 // Parameters of poisson process distribution, which is a good representation
 // of activity coming in from independent identical stationary sources.
 type PoissonParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
-	OfferedLoad          float64  `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
 }
 
-func (m *PoissonParams) Reset()         { *m = PoissonParams{} }
-func (m *PoissonParams) String() string { return proto.CompactTextString(m) }
-func (*PoissonParams) ProtoMessage()    {}
-func (*PoissonParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{0}
+func (x *PoissonParams) Reset() {
+	*x = PoissonParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *PoissonParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PoissonParams.Unmarshal(m, b)
-}
-func (m *PoissonParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PoissonParams.Marshal(b, m, deterministic)
-}
-func (m *PoissonParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PoissonParams.Merge(m, src)
-}
-func (m *PoissonParams) XXX_Size() int {
-	return xxx_messageInfo_PoissonParams.Size(m)
+func (x *PoissonParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *PoissonParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_PoissonParams.DiscardUnknown(m)
+
+func (*PoissonParams) ProtoMessage() {}
+
+func (x *PoissonParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_PoissonParams proto.InternalMessageInfo
+// Deprecated: Use PoissonParams.ProtoReflect.Descriptor instead.
+func (*PoissonParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *PoissonParams) GetOfferedLoad() float64 {
-	if m != nil {
-		return m.OfferedLoad
+func (x *PoissonParams) GetOfferedLoad() float64 {
+	if x != nil {
+		return x.OfferedLoad
 	}
 	return 0
 }
 
 type UniformParams struct {
-	InterarrivalLo       float64  `protobuf:"fixed64,1,opt,name=interarrival_lo,json=interarrivalLo,proto3" json:"interarrival_lo,omitempty"`
-	InterarrivalHi       float64  `protobuf:"fixed64,2,opt,name=interarrival_hi,json=interarrivalHi,proto3" json:"interarrival_hi,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *UniformParams) Reset()         { *m = UniformParams{} }
-func (m *UniformParams) String() string { return proto.CompactTextString(m) }
-func (*UniformParams) ProtoMessage()    {}
-func (*UniformParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{1}
+	InterarrivalLo float64 `protobuf:"fixed64,1,opt,name=interarrival_lo,json=interarrivalLo,proto3" json:"interarrival_lo,omitempty"`
+	InterarrivalHi float64 `protobuf:"fixed64,2,opt,name=interarrival_hi,json=interarrivalHi,proto3" json:"interarrival_hi,omitempty"`
 }
 
-func (m *UniformParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_UniformParams.Unmarshal(m, b)
-}
-func (m *UniformParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_UniformParams.Marshal(b, m, deterministic)
-}
-func (m *UniformParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_UniformParams.Merge(m, src)
+func (x *UniformParams) Reset() {
+	*x = UniformParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *UniformParams) XXX_Size() int {
-	return xxx_messageInfo_UniformParams.Size(m)
+
+func (x *UniformParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *UniformParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_UniformParams.DiscardUnknown(m)
+
+func (*UniformParams) ProtoMessage() {}
+
+func (x *UniformParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_UniformParams proto.InternalMessageInfo
+// Deprecated: Use UniformParams.ProtoReflect.Descriptor instead.
+func (*UniformParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *UniformParams) GetInterarrivalLo() float64 {
-	if m != nil {
-		return m.InterarrivalLo
+func (x *UniformParams) GetInterarrivalLo() float64 {
+	if x != nil {
+		return x.InterarrivalLo
 	}
 	return 0
 }
 
-func (m *UniformParams) GetInterarrivalHi() float64 {
-	if m != nil {
-		return m.InterarrivalHi
+func (x *UniformParams) GetInterarrivalHi() float64 {
+	if x != nil {
+		return x.InterarrivalHi
 	}
 	return 0
 }
 
 type DeterministicParams struct {
-	OfferedLoad          float64  `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *DeterministicParams) Reset()         { *m = DeterministicParams{} }
-func (m *DeterministicParams) String() string { return proto.CompactTextString(m) }
-func (*DeterministicParams) ProtoMessage()    {}
-func (*DeterministicParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{2}
+	OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
 }
 
-func (m *DeterministicParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_DeterministicParams.Unmarshal(m, b)
-}
-func (m *DeterministicParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_DeterministicParams.Marshal(b, m, deterministic)
-}
-func (m *DeterministicParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DeterministicParams.Merge(m, src)
+func (x *DeterministicParams) Reset() {
+	*x = DeterministicParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *DeterministicParams) XXX_Size() int {
-	return xxx_messageInfo_DeterministicParams.Size(m)
+
+func (x *DeterministicParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *DeterministicParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_DeterministicParams.DiscardUnknown(m)
+
+func (*DeterministicParams) ProtoMessage() {}
+
+func (x *DeterministicParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_DeterministicParams proto.InternalMessageInfo
+// Deprecated: Use DeterministicParams.ProtoReflect.Descriptor instead.
+func (*DeterministicParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *DeterministicParams) GetOfferedLoad() float64 {
-	if m != nil {
-		return m.OfferedLoad
+func (x *DeterministicParams) GetOfferedLoad() float64 {
+	if x != nil {
+		return x.OfferedLoad
 	}
 	return 0
 }
 
 type ParetoParams struct {
-	InterarrivalBase     float64  `protobuf:"fixed64,1,opt,name=interarrival_base,json=interarrivalBase,proto3" json:"interarrival_base,omitempty"`
-	Alpha                float64  `protobuf:"fixed64,2,opt,name=alpha,proto3" json:"alpha,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ParetoParams) Reset()         { *m = ParetoParams{} }
-func (m *ParetoParams) String() string { return proto.CompactTextString(m) }
-func (*ParetoParams) ProtoMessage()    {}
-func (*ParetoParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{3}
+	InterarrivalBase float64 `protobuf:"fixed64,1,opt,name=interarrival_base,json=interarrivalBase,proto3" json:"interarrival_base,omitempty"`
+	Alpha            float64 `protobuf:"fixed64,2,opt,name=alpha,proto3" json:"alpha,omitempty"`
 }
 
-func (m *ParetoParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ParetoParams.Unmarshal(m, b)
-}
-func (m *ParetoParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ParetoParams.Marshal(b, m, deterministic)
-}
-func (m *ParetoParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ParetoParams.Merge(m, src)
+func (x *ParetoParams) Reset() {
+	*x = ParetoParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ParetoParams) XXX_Size() int {
-	return xxx_messageInfo_ParetoParams.Size(m)
+
+func (x *ParetoParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ParetoParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_ParetoParams.DiscardUnknown(m)
+
+func (*ParetoParams) ProtoMessage() {}
+
+func (x *ParetoParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ParetoParams proto.InternalMessageInfo
+// Deprecated: Use ParetoParams.ProtoReflect.Descriptor instead.
+func (*ParetoParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *ParetoParams) GetInterarrivalBase() float64 {
-	if m != nil {
-		return m.InterarrivalBase
+func (x *ParetoParams) GetInterarrivalBase() float64 {
+	if x != nil {
+		return x.InterarrivalBase
 	}
 	return 0
 }
 
-func (m *ParetoParams) GetAlpha() float64 {
-	if m != nil {
-		return m.Alpha
+func (x *ParetoParams) GetAlpha() float64 {
+	if x != nil {
+		return x.Alpha
 	}
 	return 0
 }
@@ -276,73 +390,130 @@ func (m *ParetoParams) GetAlpha() float64 {
 // Once an RPC finishes, immediately start a new one.
 // No configuration parameters needed.
 type ClosedLoopParams struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *ClosedLoopParams) Reset()         { *m = ClosedLoopParams{} }
-func (m *ClosedLoopParams) String() string { return proto.CompactTextString(m) }
-func (*ClosedLoopParams) ProtoMessage()    {}
-func (*ClosedLoopParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{4}
+func (x *ClosedLoopParams) Reset() {
+	*x = ClosedLoopParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClosedLoopParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClosedLoopParams.Unmarshal(m, b)
-}
-func (m *ClosedLoopParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClosedLoopParams.Marshal(b, m, deterministic)
-}
-func (m *ClosedLoopParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClosedLoopParams.Merge(m, src)
+func (x *ClosedLoopParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClosedLoopParams) XXX_Size() int {
-	return xxx_messageInfo_ClosedLoopParams.Size(m)
-}
-func (m *ClosedLoopParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClosedLoopParams.DiscardUnknown(m)
+
+func (*ClosedLoopParams) ProtoMessage() {}
+
+func (x *ClosedLoopParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClosedLoopParams proto.InternalMessageInfo
+// Deprecated: Use ClosedLoopParams.ProtoReflect.Descriptor instead.
+func (*ClosedLoopParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{4}
+}
 
 type LoadParams struct {
-	// Types that are valid to be assigned to Load:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Load:
 	//	*LoadParams_ClosedLoop
 	//	*LoadParams_Poisson
 	//	*LoadParams_Uniform
 	//	*LoadParams_Determ
 	//	*LoadParams_Pareto
-	Load                 isLoadParams_Load `protobuf_oneof:"load"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	Load isLoadParams_Load `protobuf_oneof:"load"`
+}
+
+func (x *LoadParams) Reset() {
+	*x = LoadParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *LoadParams) Reset()         { *m = LoadParams{} }
-func (m *LoadParams) String() string { return proto.CompactTextString(m) }
-func (*LoadParams) ProtoMessage()    {}
+func (*LoadParams) ProtoMessage() {}
+
+func (x *LoadParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadParams.ProtoReflect.Descriptor instead.
 func (*LoadParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{5}
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{5}
 }
 
-func (m *LoadParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadParams.Unmarshal(m, b)
+func (m *LoadParams) GetLoad() isLoadParams_Load {
+	if m != nil {
+		return m.Load
+	}
+	return nil
 }
-func (m *LoadParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadParams.Marshal(b, m, deterministic)
+
+func (x *LoadParams) GetClosedLoop() *ClosedLoopParams {
+	if x, ok := x.GetLoad().(*LoadParams_ClosedLoop); ok {
+		return x.ClosedLoop
+	}
+	return nil
 }
-func (m *LoadParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadParams.Merge(m, src)
+
+func (x *LoadParams) GetPoisson() *PoissonParams {
+	if x, ok := x.GetLoad().(*LoadParams_Poisson); ok {
+		return x.Poisson
+	}
+	return nil
 }
-func (m *LoadParams) XXX_Size() int {
-	return xxx_messageInfo_LoadParams.Size(m)
+
+func (x *LoadParams) GetUniform() *UniformParams {
+	if x, ok := x.GetLoad().(*LoadParams_Uniform); ok {
+		return x.Uniform
+	}
+	return nil
 }
-func (m *LoadParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadParams.DiscardUnknown(m)
+
+func (x *LoadParams) GetDeterm() *DeterministicParams {
+	if x, ok := x.GetLoad().(*LoadParams_Determ); ok {
+		return x.Determ
+	}
+	return nil
 }
 
-var xxx_messageInfo_LoadParams proto.InternalMessageInfo
+func (x *LoadParams) GetPareto() *ParetoParams {
+	if x, ok := x.GetLoad().(*LoadParams_Pareto); ok {
+		return x.Pareto
+	}
+	return nil
+}
 
 type isLoadParams_Load interface {
 	isLoadParams_Load()
@@ -378,108 +549,67 @@ func (*LoadParams_Determ) isLoadParams_Load() {}
 
 func (*LoadParams_Pareto) isLoadParams_Load() {}
 
-func (m *LoadParams) GetLoad() isLoadParams_Load {
-	if m != nil {
-		return m.Load
-	}
-	return nil
-}
+// presence of SecurityParams implies use of TLS
+type SecurityParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *LoadParams) GetClosedLoop() *ClosedLoopParams {
-	if x, ok := m.GetLoad().(*LoadParams_ClosedLoop); ok {
-		return x.ClosedLoop
-	}
-	return nil
+	UseTestCa          bool   `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"`
+	ServerHostOverride string `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"`
 }
 
-func (m *LoadParams) GetPoisson() *PoissonParams {
-	if x, ok := m.GetLoad().(*LoadParams_Poisson); ok {
-		return x.Poisson
+func (x *SecurityParams) Reset() {
+	*x = SecurityParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *LoadParams) GetUniform() *UniformParams {
-	if x, ok := m.GetLoad().(*LoadParams_Uniform); ok {
-		return x.Uniform
-	}
-	return nil
+func (x *SecurityParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *LoadParams) GetDeterm() *DeterministicParams {
-	if x, ok := m.GetLoad().(*LoadParams_Determ); ok {
-		return x.Determ
-	}
-	return nil
-}
+func (*SecurityParams) ProtoMessage() {}
 
-func (m *LoadParams) GetPareto() *ParetoParams {
-	if x, ok := m.GetLoad().(*LoadParams_Pareto); ok {
-		return x.Pareto
+func (x *SecurityParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*LoadParams) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*LoadParams_ClosedLoop)(nil),
-		(*LoadParams_Poisson)(nil),
-		(*LoadParams_Uniform)(nil),
-		(*LoadParams_Determ)(nil),
-		(*LoadParams_Pareto)(nil),
-	}
-}
-
-// presence of SecurityParams implies use of TLS
-type SecurityParams struct {
-	UseTestCa            bool     `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"`
-	ServerHostOverride   string   `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *SecurityParams) Reset()         { *m = SecurityParams{} }
-func (m *SecurityParams) String() string { return proto.CompactTextString(m) }
-func (*SecurityParams) ProtoMessage()    {}
+// Deprecated: Use SecurityParams.ProtoReflect.Descriptor instead.
 func (*SecurityParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{6}
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{6}
 }
 
-func (m *SecurityParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SecurityParams.Unmarshal(m, b)
-}
-func (m *SecurityParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SecurityParams.Marshal(b, m, deterministic)
-}
-func (m *SecurityParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SecurityParams.Merge(m, src)
-}
-func (m *SecurityParams) XXX_Size() int {
-	return xxx_messageInfo_SecurityParams.Size(m)
-}
-func (m *SecurityParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_SecurityParams.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_SecurityParams proto.InternalMessageInfo
-
-func (m *SecurityParams) GetUseTestCa() bool {
-	if m != nil {
-		return m.UseTestCa
+func (x *SecurityParams) GetUseTestCa() bool {
+	if x != nil {
+		return x.UseTestCa
 	}
 	return false
 }
 
-func (m *SecurityParams) GetServerHostOverride() string {
-	if m != nil {
-		return m.ServerHostOverride
+func (x *SecurityParams) GetServerHostOverride() string {
+	if x != nil {
+		return x.ServerHostOverride
 	}
 	return ""
 }
 
 type ClientConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// List of targets to connect to. At least one target needs to be specified.
 	ServerTargets  []string        `protobuf:"bytes,1,rep,name=server_targets,json=serverTargets,proto3" json:"server_targets,omitempty"`
 	ClientType     ClientType      `protobuf:"varint,2,opt,name=client_type,json=clientType,proto3,enum=grpc.testing.ClientType" json:"client_type,omitempty"`
@@ -498,253 +628,265 @@ type ClientConfig struct {
 	PayloadConfig   *PayloadConfig   `protobuf:"bytes,11,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"`
 	HistogramParams *HistogramParams `protobuf:"bytes,12,opt,name=histogram_params,json=histogramParams,proto3" json:"histogram_params,omitempty"`
 	// Specify the cores we should run the client on, if desired
-	CoreList             []int32  `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
-	CoreLimit            int32    `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	CoreList  []int32 `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
+	CoreLimit int32   `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"`
 }
 
-func (m *ClientConfig) Reset()         { *m = ClientConfig{} }
-func (m *ClientConfig) String() string { return proto.CompactTextString(m) }
-func (*ClientConfig) ProtoMessage()    {}
-func (*ClientConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{7}
+func (x *ClientConfig) Reset() {
+	*x = ClientConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClientConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientConfig.Unmarshal(m, b)
-}
-func (m *ClientConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientConfig.Marshal(b, m, deterministic)
+func (x *ClientConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientConfig.Merge(m, src)
-}
-func (m *ClientConfig) XXX_Size() int {
-	return xxx_messageInfo_ClientConfig.Size(m)
-}
-func (m *ClientConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientConfig.DiscardUnknown(m)
+
+func (*ClientConfig) ProtoMessage() {}
+
+func (x *ClientConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClientConfig proto.InternalMessageInfo
+// Deprecated: Use ClientConfig.ProtoReflect.Descriptor instead.
+func (*ClientConfig) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *ClientConfig) GetServerTargets() []string {
-	if m != nil {
-		return m.ServerTargets
+func (x *ClientConfig) GetServerTargets() []string {
+	if x != nil {
+		return x.ServerTargets
 	}
 	return nil
 }
 
-func (m *ClientConfig) GetClientType() ClientType {
-	if m != nil {
-		return m.ClientType
+func (x *ClientConfig) GetClientType() ClientType {
+	if x != nil {
+		return x.ClientType
 	}
 	return ClientType_SYNC_CLIENT
 }
 
-func (m *ClientConfig) GetSecurityParams() *SecurityParams {
-	if m != nil {
-		return m.SecurityParams
+func (x *ClientConfig) GetSecurityParams() *SecurityParams {
+	if x != nil {
+		return x.SecurityParams
 	}
 	return nil
 }
 
-func (m *ClientConfig) GetOutstandingRpcsPerChannel() int32 {
-	if m != nil {
-		return m.OutstandingRpcsPerChannel
+func (x *ClientConfig) GetOutstandingRpcsPerChannel() int32 {
+	if x != nil {
+		return x.OutstandingRpcsPerChannel
 	}
 	return 0
 }
 
-func (m *ClientConfig) GetClientChannels() int32 {
-	if m != nil {
-		return m.ClientChannels
+func (x *ClientConfig) GetClientChannels() int32 {
+	if x != nil {
+		return x.ClientChannels
 	}
 	return 0
 }
 
-func (m *ClientConfig) GetAsyncClientThreads() int32 {
-	if m != nil {
-		return m.AsyncClientThreads
+func (x *ClientConfig) GetAsyncClientThreads() int32 {
+	if x != nil {
+		return x.AsyncClientThreads
 	}
 	return 0
 }
 
-func (m *ClientConfig) GetRpcType() RpcType {
-	if m != nil {
-		return m.RpcType
+func (x *ClientConfig) GetRpcType() RpcType {
+	if x != nil {
+		return x.RpcType
 	}
 	return RpcType_UNARY
 }
 
-func (m *ClientConfig) GetLoadParams() *LoadParams {
-	if m != nil {
-		return m.LoadParams
+func (x *ClientConfig) GetLoadParams() *LoadParams {
+	if x != nil {
+		return x.LoadParams
 	}
 	return nil
 }
 
-func (m *ClientConfig) GetPayloadConfig() *PayloadConfig {
-	if m != nil {
-		return m.PayloadConfig
+func (x *ClientConfig) GetPayloadConfig() *PayloadConfig {
+	if x != nil {
+		return x.PayloadConfig
 	}
 	return nil
 }
 
-func (m *ClientConfig) GetHistogramParams() *HistogramParams {
-	if m != nil {
-		return m.HistogramParams
+func (x *ClientConfig) GetHistogramParams() *HistogramParams {
+	if x != nil {
+		return x.HistogramParams
 	}
 	return nil
 }
 
-func (m *ClientConfig) GetCoreList() []int32 {
-	if m != nil {
-		return m.CoreList
+func (x *ClientConfig) GetCoreList() []int32 {
+	if x != nil {
+		return x.CoreList
 	}
 	return nil
 }
 
-func (m *ClientConfig) GetCoreLimit() int32 {
-	if m != nil {
-		return m.CoreLimit
+func (x *ClientConfig) GetCoreLimit() int32 {
+	if x != nil {
+		return x.CoreLimit
 	}
 	return 0
 }
 
 type ClientStatus struct {
-	Stats                *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
-	XXX_unrecognized     []byte       `json:"-"`
-	XXX_sizecache        int32        `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ClientStatus) Reset()         { *m = ClientStatus{} }
-func (m *ClientStatus) String() string { return proto.CompactTextString(m) }
-func (*ClientStatus) ProtoMessage()    {}
-func (*ClientStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{8}
+	Stats *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
 }
 
-func (m *ClientStatus) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientStatus.Unmarshal(m, b)
-}
-func (m *ClientStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientStatus.Marshal(b, m, deterministic)
-}
-func (m *ClientStatus) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientStatus.Merge(m, src)
+func (x *ClientStatus) Reset() {
+	*x = ClientStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ClientStatus) XXX_Size() int {
-	return xxx_messageInfo_ClientStatus.Size(m)
+
+func (x *ClientStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientStatus) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientStatus.DiscardUnknown(m)
+
+func (*ClientStatus) ProtoMessage() {}
+
+func (x *ClientStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClientStatus proto.InternalMessageInfo
+// Deprecated: Use ClientStatus.ProtoReflect.Descriptor instead.
+func (*ClientStatus) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *ClientStatus) GetStats() *ClientStats {
-	if m != nil {
-		return m.Stats
+func (x *ClientStatus) GetStats() *ClientStats {
+	if x != nil {
+		return x.Stats
 	}
 	return nil
 }
 
 // Request current stats
 type Mark struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// if true, the stats will be reset after taking their snapshot.
-	Reset_               bool     `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Reset_ bool `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"`
 }
 
-func (m *Mark) Reset()         { *m = Mark{} }
-func (m *Mark) String() string { return proto.CompactTextString(m) }
-func (*Mark) ProtoMessage()    {}
-func (*Mark) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{9}
+func (x *Mark) Reset() {
+	*x = Mark{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Mark) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Mark.Unmarshal(m, b)
-}
-func (m *Mark) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Mark.Marshal(b, m, deterministic)
+func (x *Mark) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Mark) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Mark.Merge(m, src)
-}
-func (m *Mark) XXX_Size() int {
-	return xxx_messageInfo_Mark.Size(m)
-}
-func (m *Mark) XXX_DiscardUnknown() {
-	xxx_messageInfo_Mark.DiscardUnknown(m)
+
+func (*Mark) ProtoMessage() {}
+
+func (x *Mark) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Mark proto.InternalMessageInfo
+// Deprecated: Use Mark.ProtoReflect.Descriptor instead.
+func (*Mark) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{9}
+}
 
-func (m *Mark) GetReset_() bool {
-	if m != nil {
-		return m.Reset_
+func (x *Mark) GetReset_() bool {
+	if x != nil {
+		return x.Reset_
 	}
 	return false
 }
 
 type ClientArgs struct {
-	// Types that are valid to be assigned to Argtype:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Argtype:
 	//	*ClientArgs_Setup
 	//	*ClientArgs_Mark
-	Argtype              isClientArgs_Argtype `protobuf_oneof:"argtype"`
-	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
-	XXX_unrecognized     []byte               `json:"-"`
-	XXX_sizecache        int32                `json:"-"`
+	Argtype isClientArgs_Argtype `protobuf_oneof:"argtype"`
 }
 
-func (m *ClientArgs) Reset()         { *m = ClientArgs{} }
-func (m *ClientArgs) String() string { return proto.CompactTextString(m) }
-func (*ClientArgs) ProtoMessage()    {}
-func (*ClientArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{10}
+func (x *ClientArgs) Reset() {
+	*x = ClientArgs{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClientArgs) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientArgs.Unmarshal(m, b)
-}
-func (m *ClientArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientArgs.Marshal(b, m, deterministic)
-}
-func (m *ClientArgs) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientArgs.Merge(m, src)
+func (x *ClientArgs) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientArgs) XXX_Size() int {
-	return xxx_messageInfo_ClientArgs.Size(m)
-}
-func (m *ClientArgs) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientArgs.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_ClientArgs proto.InternalMessageInfo
 
-type isClientArgs_Argtype interface {
-	isClientArgs_Argtype()
-}
+func (*ClientArgs) ProtoMessage() {}
 
-type ClientArgs_Setup struct {
-	Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
+func (x *ClientArgs) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-type ClientArgs_Mark struct {
-	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
+// Deprecated: Use ClientArgs.ProtoReflect.Descriptor instead.
+func (*ClientArgs) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{10}
 }
 
-func (*ClientArgs_Setup) isClientArgs_Argtype() {}
-
-func (*ClientArgs_Mark) isClientArgs_Argtype() {}
-
 func (m *ClientArgs) GetArgtype() isClientArgs_Argtype {
 	if m != nil {
 		return m.Argtype
@@ -752,29 +894,41 @@ func (m *ClientArgs) GetArgtype() isClientArgs_Argtype {
 	return nil
 }
 
-func (m *ClientArgs) GetSetup() *ClientConfig {
-	if x, ok := m.GetArgtype().(*ClientArgs_Setup); ok {
+func (x *ClientArgs) GetSetup() *ClientConfig {
+	if x, ok := x.GetArgtype().(*ClientArgs_Setup); ok {
 		return x.Setup
 	}
 	return nil
 }
 
-func (m *ClientArgs) GetMark() *Mark {
-	if x, ok := m.GetArgtype().(*ClientArgs_Mark); ok {
+func (x *ClientArgs) GetMark() *Mark {
+	if x, ok := x.GetArgtype().(*ClientArgs_Mark); ok {
 		return x.Mark
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*ClientArgs) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*ClientArgs_Setup)(nil),
-		(*ClientArgs_Mark)(nil),
-	}
+type isClientArgs_Argtype interface {
+	isClientArgs_Argtype()
+}
+
+type ClientArgs_Setup struct {
+	Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
 }
 
+type ClientArgs_Mark struct {
+	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
+}
+
+func (*ClientArgs_Setup) isClientArgs_Argtype() {}
+
+func (*ClientArgs_Mark) isClientArgs_Argtype() {}
+
 type ServerConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	ServerType     ServerType      `protobuf:"varint,1,opt,name=server_type,json=serverType,proto3,enum=grpc.testing.ServerType" json:"server_type,omitempty"`
 	SecurityParams *SecurityParams `protobuf:"bytes,2,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"`
 	// Port on which to listen. Zero means pick unused port.
@@ -786,137 +940,133 @@ type ServerConfig struct {
 	// payload config, used in generic server
 	PayloadConfig *PayloadConfig `protobuf:"bytes,9,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"`
 	// Specify the cores we should run the server on, if desired
-	CoreList             []int32  `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	CoreList []int32 `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
 }
 
-func (m *ServerConfig) Reset()         { *m = ServerConfig{} }
-func (m *ServerConfig) String() string { return proto.CompactTextString(m) }
-func (*ServerConfig) ProtoMessage()    {}
-func (*ServerConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{11}
+func (x *ServerConfig) Reset() {
+	*x = ServerConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerConfig.Unmarshal(m, b)
-}
-func (m *ServerConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerConfig.Marshal(b, m, deterministic)
-}
-func (m *ServerConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerConfig.Merge(m, src)
+func (x *ServerConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerConfig) XXX_Size() int {
-	return xxx_messageInfo_ServerConfig.Size(m)
-}
-func (m *ServerConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerConfig.DiscardUnknown(m)
+
+func (*ServerConfig) ProtoMessage() {}
+
+func (x *ServerConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerConfig proto.InternalMessageInfo
+// Deprecated: Use ServerConfig.ProtoReflect.Descriptor instead.
+func (*ServerConfig) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{11}
+}
 
-func (m *ServerConfig) GetServerType() ServerType {
-	if m != nil {
-		return m.ServerType
+func (x *ServerConfig) GetServerType() ServerType {
+	if x != nil {
+		return x.ServerType
 	}
 	return ServerType_SYNC_SERVER
 }
 
-func (m *ServerConfig) GetSecurityParams() *SecurityParams {
-	if m != nil {
-		return m.SecurityParams
+func (x *ServerConfig) GetSecurityParams() *SecurityParams {
+	if x != nil {
+		return x.SecurityParams
 	}
 	return nil
 }
 
-func (m *ServerConfig) GetPort() int32 {
-	if m != nil {
-		return m.Port
+func (x *ServerConfig) GetPort() int32 {
+	if x != nil {
+		return x.Port
 	}
 	return 0
 }
 
-func (m *ServerConfig) GetAsyncServerThreads() int32 {
-	if m != nil {
-		return m.AsyncServerThreads
+func (x *ServerConfig) GetAsyncServerThreads() int32 {
+	if x != nil {
+		return x.AsyncServerThreads
 	}
 	return 0
 }
 
-func (m *ServerConfig) GetCoreLimit() int32 {
-	if m != nil {
-		return m.CoreLimit
+func (x *ServerConfig) GetCoreLimit() int32 {
+	if x != nil {
+		return x.CoreLimit
 	}
 	return 0
 }
 
-func (m *ServerConfig) GetPayloadConfig() *PayloadConfig {
-	if m != nil {
-		return m.PayloadConfig
+func (x *ServerConfig) GetPayloadConfig() *PayloadConfig {
+	if x != nil {
+		return x.PayloadConfig
 	}
 	return nil
 }
 
-func (m *ServerConfig) GetCoreList() []int32 {
-	if m != nil {
-		return m.CoreList
+func (x *ServerConfig) GetCoreList() []int32 {
+	if x != nil {
+		return x.CoreList
 	}
 	return nil
 }
 
 type ServerArgs struct {
-	// Types that are valid to be assigned to Argtype:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Argtype:
 	//	*ServerArgs_Setup
 	//	*ServerArgs_Mark
-	Argtype              isServerArgs_Argtype `protobuf_oneof:"argtype"`
-	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
-	XXX_unrecognized     []byte               `json:"-"`
-	XXX_sizecache        int32                `json:"-"`
+	Argtype isServerArgs_Argtype `protobuf_oneof:"argtype"`
 }
 
-func (m *ServerArgs) Reset()         { *m = ServerArgs{} }
-func (m *ServerArgs) String() string { return proto.CompactTextString(m) }
-func (*ServerArgs) ProtoMessage()    {}
-func (*ServerArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{12}
+func (x *ServerArgs) Reset() {
+	*x = ServerArgs{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerArgs) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerArgs.Unmarshal(m, b)
-}
-func (m *ServerArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerArgs.Marshal(b, m, deterministic)
-}
-func (m *ServerArgs) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerArgs.Merge(m, src)
-}
-func (m *ServerArgs) XXX_Size() int {
-	return xxx_messageInfo_ServerArgs.Size(m)
-}
-func (m *ServerArgs) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerArgs.DiscardUnknown(m)
+func (x *ServerArgs) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_ServerArgs proto.InternalMessageInfo
+func (*ServerArgs) ProtoMessage() {}
 
-type isServerArgs_Argtype interface {
-	isServerArgs_Argtype()
-}
-
-type ServerArgs_Setup struct {
-	Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
+func (x *ServerArgs) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-type ServerArgs_Mark struct {
-	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
+// Deprecated: Use ServerArgs.ProtoReflect.Descriptor instead.
+func (*ServerArgs) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{12}
 }
 
-func (*ServerArgs_Setup) isServerArgs_Argtype() {}
-
-func (*ServerArgs_Mark) isServerArgs_Argtype() {}
-
 func (m *ServerArgs) GetArgtype() isServerArgs_Argtype {
 	if m != nil {
 		return m.Argtype
@@ -924,189 +1074,231 @@ func (m *ServerArgs) GetArgtype() isServerArgs_Argtype {
 	return nil
 }
 
-func (m *ServerArgs) GetSetup() *ServerConfig {
-	if x, ok := m.GetArgtype().(*ServerArgs_Setup); ok {
+func (x *ServerArgs) GetSetup() *ServerConfig {
+	if x, ok := x.GetArgtype().(*ServerArgs_Setup); ok {
 		return x.Setup
 	}
 	return nil
 }
 
-func (m *ServerArgs) GetMark() *Mark {
-	if x, ok := m.GetArgtype().(*ServerArgs_Mark); ok {
+func (x *ServerArgs) GetMark() *Mark {
+	if x, ok := x.GetArgtype().(*ServerArgs_Mark); ok {
 		return x.Mark
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*ServerArgs) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*ServerArgs_Setup)(nil),
-		(*ServerArgs_Mark)(nil),
-	}
+type isServerArgs_Argtype interface {
+	isServerArgs_Argtype()
+}
+
+type ServerArgs_Setup struct {
+	Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
 }
 
+type ServerArgs_Mark struct {
+	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
+}
+
+func (*ServerArgs_Setup) isServerArgs_Argtype() {}
+
+func (*ServerArgs_Mark) isServerArgs_Argtype() {}
+
 type ServerStatus struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	Stats *ServerStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
 	// the port bound by the server
 	Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
 	// Number of cores available to the server
-	Cores                int32    `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Cores int32 `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"`
 }
 
-func (m *ServerStatus) Reset()         { *m = ServerStatus{} }
-func (m *ServerStatus) String() string { return proto.CompactTextString(m) }
-func (*ServerStatus) ProtoMessage()    {}
-func (*ServerStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{13}
+func (x *ServerStatus) Reset() {
+	*x = ServerStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerStatus) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerStatus.Unmarshal(m, b)
-}
-func (m *ServerStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerStatus.Marshal(b, m, deterministic)
-}
-func (m *ServerStatus) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerStatus.Merge(m, src)
+func (x *ServerStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerStatus) XXX_Size() int {
-	return xxx_messageInfo_ServerStatus.Size(m)
-}
-func (m *ServerStatus) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerStatus.DiscardUnknown(m)
+
+func (*ServerStatus) ProtoMessage() {}
+
+func (x *ServerStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerStatus proto.InternalMessageInfo
+// Deprecated: Use ServerStatus.ProtoReflect.Descriptor instead.
+func (*ServerStatus) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{13}
+}
 
-func (m *ServerStatus) GetStats() *ServerStats {
-	if m != nil {
-		return m.Stats
+func (x *ServerStatus) GetStats() *ServerStats {
+	if x != nil {
+		return x.Stats
 	}
 	return nil
 }
 
-func (m *ServerStatus) GetPort() int32 {
-	if m != nil {
-		return m.Port
+func (x *ServerStatus) GetPort() int32 {
+	if x != nil {
+		return x.Port
 	}
 	return 0
 }
 
-func (m *ServerStatus) GetCores() int32 {
-	if m != nil {
-		return m.Cores
+func (x *ServerStatus) GetCores() int32 {
+	if x != nil {
+		return x.Cores
 	}
 	return 0
 }
 
 type CoreRequest struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *CoreRequest) Reset()         { *m = CoreRequest{} }
-func (m *CoreRequest) String() string { return proto.CompactTextString(m) }
-func (*CoreRequest) ProtoMessage()    {}
-func (*CoreRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{14}
+func (x *CoreRequest) Reset() {
+	*x = CoreRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *CoreRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_CoreRequest.Unmarshal(m, b)
-}
-func (m *CoreRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_CoreRequest.Marshal(b, m, deterministic)
-}
-func (m *CoreRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_CoreRequest.Merge(m, src)
+func (x *CoreRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *CoreRequest) XXX_Size() int {
-	return xxx_messageInfo_CoreRequest.Size(m)
-}
-func (m *CoreRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_CoreRequest.DiscardUnknown(m)
+
+func (*CoreRequest) ProtoMessage() {}
+
+func (x *CoreRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_CoreRequest proto.InternalMessageInfo
+// Deprecated: Use CoreRequest.ProtoReflect.Descriptor instead.
+func (*CoreRequest) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{14}
+}
 
 type CoreResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Number of cores available on the server
-	Cores                int32    `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Cores int32 `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"`
 }
 
-func (m *CoreResponse) Reset()         { *m = CoreResponse{} }
-func (m *CoreResponse) String() string { return proto.CompactTextString(m) }
-func (*CoreResponse) ProtoMessage()    {}
-func (*CoreResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{15}
+func (x *CoreResponse) Reset() {
+	*x = CoreResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *CoreResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_CoreResponse.Unmarshal(m, b)
-}
-func (m *CoreResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_CoreResponse.Marshal(b, m, deterministic)
+func (x *CoreResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *CoreResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_CoreResponse.Merge(m, src)
-}
-func (m *CoreResponse) XXX_Size() int {
-	return xxx_messageInfo_CoreResponse.Size(m)
-}
-func (m *CoreResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_CoreResponse.DiscardUnknown(m)
+
+func (*CoreResponse) ProtoMessage() {}
+
+func (x *CoreResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_CoreResponse proto.InternalMessageInfo
+// Deprecated: Use CoreResponse.ProtoReflect.Descriptor instead.
+func (*CoreResponse) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{15}
+}
 
-func (m *CoreResponse) GetCores() int32 {
-	if m != nil {
-		return m.Cores
+func (x *CoreResponse) GetCores() int32 {
+	if x != nil {
+		return x.Cores
 	}
 	return 0
 }
 
 type Void struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *Void) Reset()         { *m = Void{} }
-func (m *Void) String() string { return proto.CompactTextString(m) }
-func (*Void) ProtoMessage()    {}
-func (*Void) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{16}
+func (x *Void) Reset() {
+	*x = Void{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[16]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Void) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Void.Unmarshal(m, b)
-}
-func (m *Void) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Void.Marshal(b, m, deterministic)
-}
-func (m *Void) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Void.Merge(m, src)
+func (x *Void) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Void) XXX_Size() int {
-	return xxx_messageInfo_Void.Size(m)
-}
-func (m *Void) XXX_DiscardUnknown() {
-	xxx_messageInfo_Void.DiscardUnknown(m)
+
+func (*Void) ProtoMessage() {}
+
+func (x *Void) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[16]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Void proto.InternalMessageInfo
+// Deprecated: Use Void.ProtoReflect.Descriptor instead.
+func (*Void) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{16}
+}
 
 // A single performance scenario: input to qps_json_driver
 type Scenario struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Human readable name for this scenario
 	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	// Client configuration
@@ -1122,239 +1314,681 @@ type Scenario struct {
 	// Benchmark time, in seconds
 	BenchmarkSeconds int32 `protobuf:"varint,7,opt,name=benchmark_seconds,json=benchmarkSeconds,proto3" json:"benchmark_seconds,omitempty"`
 	// Number of workers to spawn locally (usually zero)
-	SpawnLocalWorkerCount int32    `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{} `json:"-"`
-	XXX_unrecognized      []byte   `json:"-"`
-	XXX_sizecache         int32    `json:"-"`
+	SpawnLocalWorkerCount int32 `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"`
 }
 
-func (m *Scenario) Reset()         { *m = Scenario{} }
-func (m *Scenario) String() string { return proto.CompactTextString(m) }
-func (*Scenario) ProtoMessage()    {}
-func (*Scenario) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{17}
+func (x *Scenario) Reset() {
+	*x = Scenario{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[17]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Scenario) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Scenario.Unmarshal(m, b)
-}
-func (m *Scenario) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Scenario.Marshal(b, m, deterministic)
+func (x *Scenario) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Scenario) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Scenario.Merge(m, src)
-}
-func (m *Scenario) XXX_Size() int {
-	return xxx_messageInfo_Scenario.Size(m)
-}
-func (m *Scenario) XXX_DiscardUnknown() {
-	xxx_messageInfo_Scenario.DiscardUnknown(m)
+
+func (*Scenario) ProtoMessage() {}
+
+func (x *Scenario) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[17]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Scenario proto.InternalMessageInfo
+// Deprecated: Use Scenario.ProtoReflect.Descriptor instead.
+func (*Scenario) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{17}
+}
 
-func (m *Scenario) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *Scenario) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
-func (m *Scenario) GetClientConfig() *ClientConfig {
-	if m != nil {
-		return m.ClientConfig
+func (x *Scenario) GetClientConfig() *ClientConfig {
+	if x != nil {
+		return x.ClientConfig
 	}
 	return nil
 }
 
-func (m *Scenario) GetNumClients() int32 {
-	if m != nil {
-		return m.NumClients
+func (x *Scenario) GetNumClients() int32 {
+	if x != nil {
+		return x.NumClients
 	}
 	return 0
 }
 
-func (m *Scenario) GetServerConfig() *ServerConfig {
-	if m != nil {
-		return m.ServerConfig
+func (x *Scenario) GetServerConfig() *ServerConfig {
+	if x != nil {
+		return x.ServerConfig
 	}
 	return nil
 }
 
-func (m *Scenario) GetNumServers() int32 {
-	if m != nil {
-		return m.NumServers
+func (x *Scenario) GetNumServers() int32 {
+	if x != nil {
+		return x.NumServers
 	}
 	return 0
 }
 
-func (m *Scenario) GetWarmupSeconds() int32 {
-	if m != nil {
-		return m.WarmupSeconds
+func (x *Scenario) GetWarmupSeconds() int32 {
+	if x != nil {
+		return x.WarmupSeconds
 	}
 	return 0
 }
 
-func (m *Scenario) GetBenchmarkSeconds() int32 {
-	if m != nil {
-		return m.BenchmarkSeconds
+func (x *Scenario) GetBenchmarkSeconds() int32 {
+	if x != nil {
+		return x.BenchmarkSeconds
 	}
 	return 0
 }
 
-func (m *Scenario) GetSpawnLocalWorkerCount() int32 {
-	if m != nil {
-		return m.SpawnLocalWorkerCount
+func (x *Scenario) GetSpawnLocalWorkerCount() int32 {
+	if x != nil {
+		return x.SpawnLocalWorkerCount
 	}
 	return 0
 }
 
 // A set of scenarios to be run with qps_json_driver
 type Scenarios struct {
-	Scenarios            []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *Scenarios) Reset()         { *m = Scenarios{} }
-func (m *Scenarios) String() string { return proto.CompactTextString(m) }
-func (*Scenarios) ProtoMessage()    {}
-func (*Scenarios) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6f4e2bf9f888bddb, []int{18}
+	Scenarios []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"`
 }
 
-func (m *Scenarios) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Scenarios.Unmarshal(m, b)
-}
-func (m *Scenarios) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Scenarios.Marshal(b, m, deterministic)
-}
-func (m *Scenarios) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Scenarios.Merge(m, src)
+func (x *Scenarios) Reset() {
+	*x = Scenarios{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[18]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Scenarios) XXX_Size() int {
-	return xxx_messageInfo_Scenarios.Size(m)
+
+func (x *Scenarios) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Scenarios) XXX_DiscardUnknown() {
-	xxx_messageInfo_Scenarios.DiscardUnknown(m)
+
+func (*Scenarios) ProtoMessage() {}
+
+func (x *Scenarios) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[18]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Scenarios proto.InternalMessageInfo
+// Deprecated: Use Scenarios.ProtoReflect.Descriptor instead.
+func (*Scenarios) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{18}
+}
 
-func (m *Scenarios) GetScenarios() []*Scenario {
-	if m != nil {
-		return m.Scenarios
+func (x *Scenarios) GetScenarios() []*Scenario {
+	if x != nil {
+		return x.Scenarios
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.testing.ClientType", ClientType_name, ClientType_value)
-	proto.RegisterEnum("grpc.testing.ServerType", ServerType_name, ServerType_value)
-	proto.RegisterEnum("grpc.testing.RpcType", RpcType_name, RpcType_value)
-	proto.RegisterType((*PoissonParams)(nil), "grpc.testing.PoissonParams")
-	proto.RegisterType((*UniformParams)(nil), "grpc.testing.UniformParams")
-	proto.RegisterType((*DeterministicParams)(nil), "grpc.testing.DeterministicParams")
-	proto.RegisterType((*ParetoParams)(nil), "grpc.testing.ParetoParams")
-	proto.RegisterType((*ClosedLoopParams)(nil), "grpc.testing.ClosedLoopParams")
-	proto.RegisterType((*LoadParams)(nil), "grpc.testing.LoadParams")
-	proto.RegisterType((*SecurityParams)(nil), "grpc.testing.SecurityParams")
-	proto.RegisterType((*ClientConfig)(nil), "grpc.testing.ClientConfig")
-	proto.RegisterType((*ClientStatus)(nil), "grpc.testing.ClientStatus")
-	proto.RegisterType((*Mark)(nil), "grpc.testing.Mark")
-	proto.RegisterType((*ClientArgs)(nil), "grpc.testing.ClientArgs")
-	proto.RegisterType((*ServerConfig)(nil), "grpc.testing.ServerConfig")
-	proto.RegisterType((*ServerArgs)(nil), "grpc.testing.ServerArgs")
-	proto.RegisterType((*ServerStatus)(nil), "grpc.testing.ServerStatus")
-	proto.RegisterType((*CoreRequest)(nil), "grpc.testing.CoreRequest")
-	proto.RegisterType((*CoreResponse)(nil), "grpc.testing.CoreResponse")
-	proto.RegisterType((*Void)(nil), "grpc.testing.Void")
-	proto.RegisterType((*Scenario)(nil), "grpc.testing.Scenario")
-	proto.RegisterType((*Scenarios)(nil), "grpc.testing.Scenarios")
-}
-
-func init() {
-	proto.RegisterFile("benchmark/grpc_testing/control.proto", fileDescriptor_6f4e2bf9f888bddb)
-}
-
-var fileDescriptor_6f4e2bf9f888bddb = []byte{
-	// 1219 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xed, 0x6e, 0xdb, 0x36,
-	0x17, 0xb6, 0x5d, 0xdb, 0xb1, 0x8e, 0x3f, 0xe2, 0x97, 0x6f, 0x3a, 0xa8, 0xe9, 0xc7, 0x5a, 0xad,
-	0xc5, 0x82, 0x0e, 0xb3, 0x0b, 0xaf, 0x40, 0x57, 0xec, 0x47, 0xe1, 0x78, 0x46, 0x1d, 0x20, 0xcd,
-	0x32, 0x3a, 0xed, 0xd0, 0xfd, 0x11, 0x18, 0x99, 0x91, 0x85, 0xca, 0xa2, 0x46, 0x52, 0x2d, 0x72,
-	0x0b, 0xbb, 0xa6, 0x5d, 0xc7, 0x6e, 0x63, 0xb7, 0x30, 0xf0, 0x43, 0x8e, 0xe4, 0xba, 0x68, 0xb6,
-	0xfd, 0x13, 0xcf, 0x79, 0x1e, 0x1e, 0xf2, 0x3c, 0xe7, 0x1c, 0x0a, 0x1e, 0x9e, 0xd3, 0x24, 0x58,
-	0xae, 0x08, 0x7f, 0x37, 0x0c, 0x79, 0x1a, 0xf8, 0x92, 0x0a, 0x19, 0x25, 0xe1, 0x30, 0x60, 0x89,
-	0xe4, 0x2c, 0x1e, 0xa4, 0x9c, 0x49, 0x86, 0x3a, 0xca, 0x37, 0xb0, 0xbe, 0xfd, 0x47, 0x9f, 0xe0,
-	0xa4, 0xe4, 0x32, 0x66, 0x64, 0x21, 0x0c, 0x69, 0xdf, 0xfb, 0x04, 0x4c, 0x48, 0x22, 0x2d, 0xc6,
-	0x1b, 0x41, 0xf7, 0x94, 0x45, 0x42, 0xb0, 0xe4, 0x94, 0x70, 0xb2, 0x12, 0xe8, 0x01, 0x74, 0xd8,
-	0xc5, 0x05, 0xe5, 0x74, 0xe1, 0xab, 0xbd, 0xdc, 0xea, 0xfd, 0xea, 0x41, 0x15, 0xb7, 0xad, 0xed,
-	0x98, 0x91, 0x85, 0x47, 0xa0, 0xfb, 0x3a, 0x89, 0x2e, 0x18, 0x5f, 0x59, 0xce, 0xd7, 0xb0, 0x1b,
-	0x25, 0x92, 0x72, 0xc2, 0x79, 0xf4, 0x9e, 0xc4, 0x7e, 0xcc, 0x2c, 0xad, 0x57, 0x34, 0x1f, 0xb3,
-	0x8f, 0x80, 0xcb, 0xc8, 0xad, 0x7d, 0x0c, 0x9c, 0x45, 0xde, 0xf7, 0xf0, 0xff, 0x1f, 0xa9, 0xa4,
-	0x7c, 0x15, 0x25, 0x91, 0x90, 0x51, 0x70, 0xfd, 0xc3, 0xfd, 0x0c, 0x9d, 0x53, 0xc2, 0xa9, 0x64,
-	0x96, 0xf2, 0x0d, 0xfc, 0xaf, 0x14, 0xf2, 0x9c, 0x08, 0x6a, 0x79, 0xfd, 0xa2, 0xe3, 0x90, 0x08,
-	0x8a, 0xf6, 0xa0, 0x41, 0xe2, 0x74, 0x49, 0xec, 0xa9, 0xcc, 0xc2, 0x43, 0xd0, 0x9f, 0xc4, 0x4c,
-	0xa8, 0x00, 0x2c, 0x35, 0xdb, 0x7a, 0x7f, 0xd4, 0x00, 0x54, 0x3c, 0x1b, 0x65, 0x0c, 0xed, 0x40,
-	0x43, 0xfc, 0x98, 0xb1, 0x54, 0xef, 0xdf, 0x1e, 0xdd, 0x1b, 0x14, 0x55, 0x1b, 0x6c, 0xee, 0x31,
-	0xab, 0x60, 0x08, 0xd6, 0x36, 0xf4, 0x0c, 0x76, 0x52, 0xa3, 0x84, 0x8e, 0xde, 0x1e, 0xdd, 0x2e,
-	0xd3, 0x4b, 0x32, 0xcd, 0x2a, 0x38, 0x47, 0x2b, 0x62, 0x66, 0xe4, 0x70, 0x6f, 0x6c, 0x23, 0x96,
-	0xb4, 0x52, 0x44, 0x8b, 0x46, 0x3f, 0x40, 0x73, 0xa1, 0x93, 0xec, 0xd6, 0x35, 0xef, 0x41, 0x99,
-	0xb7, 0x45, 0x80, 0x59, 0x05, 0x5b, 0x0a, 0x7a, 0x0a, 0xcd, 0x54, 0xe7, 0xd9, 0x6d, 0x68, 0xf2,
-	0xfe, 0xc6, 0x69, 0x0b, 0x1a, 0x28, 0x96, 0xc1, 0x1e, 0x36, 0xa1, 0xae, 0x84, 0xf3, 0xce, 0xa1,
-	0x37, 0xa7, 0x41, 0xc6, 0x23, 0x79, 0x69, 0x33, 0x78, 0x0f, 0xda, 0x99, 0xa0, 0xba, 0x46, 0xfd,
-	0x80, 0xe8, 0x0c, 0xb6, 0xb0, 0x93, 0x09, 0x7a, 0x46, 0x85, 0x9c, 0x10, 0xf4, 0x04, 0xf6, 0x04,
-	0xe5, 0xef, 0x29, 0xf7, 0x97, 0x4c, 0x48, 0x9f, 0xbd, 0xa7, 0x9c, 0x47, 0x0b, 0xaa, 0x73, 0xe5,
-	0x60, 0x64, 0x7c, 0x33, 0x26, 0xe4, 0x4f, 0xd6, 0xe3, 0xfd, 0xde, 0x80, 0xce, 0x24, 0x8e, 0x68,
-	0x22, 0x27, 0x2c, 0xb9, 0x88, 0x42, 0xf4, 0x08, 0x7a, 0x76, 0x0b, 0x49, 0x78, 0x48, 0xa5, 0x70,
-	0xab, 0xf7, 0x6f, 0x1c, 0x38, 0xb8, 0x6b, 0xac, 0x67, 0xc6, 0x88, 0x9e, 0x2b, 0x2d, 0x15, 0xcd,
-	0x97, 0x97, 0xa9, 0x09, 0xd0, 0x1b, 0xb9, 0x9b, 0x5a, 0x2a, 0xc0, 0xd9, 0x65, 0x4a, 0x95, 0x86,
-	0xf9, 0x37, 0x9a, 0xc2, 0xae, 0xb0, 0xd7, 0xf2, 0x53, 0x7d, 0x2f, 0x2b, 0xc9, 0x9d, 0x32, 0xbd,
-	0x7c, 0x77, 0xdc, 0x13, 0xe5, 0x5c, 0xbc, 0x80, 0x3b, 0x2c, 0x93, 0x42, 0x92, 0x64, 0x11, 0x25,
-	0xa1, 0xcf, 0xd3, 0x40, 0xf8, 0x29, 0xe5, 0x7e, 0xb0, 0x24, 0x49, 0x42, 0x63, 0x2d, 0x57, 0x03,
-	0xdf, 0x2a, 0x60, 0x70, 0x1a, 0x88, 0x53, 0xca, 0x27, 0x06, 0xa0, 0xfa, 0xcc, 0x5e, 0xc1, 0x52,
-	0x84, 0x56, 0xa9, 0x81, 0x7b, 0xc6, 0x6c, 0x71, 0x42, 0x65, 0x95, 0x88, 0xcb, 0x24, 0xf0, 0xf3,
-	0x1b, 0x2f, 0x39, 0x25, 0x0b, 0xe1, 0xee, 0x68, 0x34, 0xd2, 0x3e, 0x7b, 0x57, 0xe3, 0x41, 0x4f,
-	0xa0, 0xa5, 0x67, 0x89, 0x4a, 0x4d, 0x4b, 0xa7, 0xe6, 0x66, 0xf9, 0x6e, 0x38, 0x0d, 0x74, 0x5e,
-	0x76, 0xb8, 0xf9, 0x50, 0xf9, 0x54, 0x9a, 0xe7, 0x09, 0x01, 0x9d, 0x90, 0x8d, 0x7c, 0x5e, 0xb5,
-	0x12, 0x86, 0xf8, 0xaa, 0xad, 0x0e, 0xa1, 0x67, 0x67, 0x9a, 0x1f, 0x68, 0x0d, 0xdd, 0xf6, 0xd6,
-	0xd6, 0x30, 0x18, 0x23, 0x33, 0xee, 0xa6, 0xc5, 0x25, 0x9a, 0x41, 0x7f, 0x19, 0x09, 0xc9, 0x42,
-	0x4e, 0x56, 0xf9, 0x19, 0x3a, 0x7a, 0x97, 0xbb, 0xe5, 0x5d, 0x66, 0x39, 0xca, 0x1e, 0x64, 0x77,
-	0x59, 0x36, 0xa0, 0xdb, 0xe0, 0x04, 0x8c, 0x53, 0x3f, 0x8e, 0x84, 0x74, 0xbb, 0xf7, 0x6f, 0x1c,
-	0x34, 0x70, 0x4b, 0x19, 0x8e, 0x23, 0x21, 0xd1, 0x5d, 0x00, 0xeb, 0x5c, 0x45, 0xd2, 0xed, 0xe9,
-	0xfc, 0x39, 0xc6, 0xbb, 0x8a, 0xa4, 0xf7, 0x22, 0xaf, 0xc5, 0xb9, 0x24, 0x32, 0x13, 0x68, 0x08,
-	0x0d, 0x3d, 0x86, 0xed, 0xa8, 0xb8, 0xb5, 0xad, 0xbc, 0x14, 0x54, 0x60, 0x83, 0xf3, 0xee, 0x40,
-	0xfd, 0x15, 0xe1, 0xef, 0xd4, 0x88, 0xe2, 0x54, 0x50, 0x69, 0x3b, 0xc4, 0x2c, 0xbc, 0x0c, 0xc0,
-	0x70, 0xc6, 0x3c, 0x14, 0x68, 0x04, 0x0d, 0x41, 0x65, 0x96, 0xcf, 0xa1, 0xfd, 0x6d, 0x9b, 0x9b,
-	0xec, 0xcc, 0x2a, 0xd8, 0x40, 0xd1, 0x01, 0xd4, 0xd5, 0x4b, 0x61, 0x67, 0x0f, 0x2a, 0x53, 0x54,
-	0xe4, 0x59, 0x05, 0x6b, 0xc4, 0xa1, 0x03, 0x3b, 0x84, 0x87, 0xaa, 0x00, 0xbc, 0x3f, 0x6b, 0xd0,
-	0x99, 0xeb, 0xe6, 0xb1, 0xc9, 0x7e, 0x0e, 0xed, 0xbc, 0xc5, 0x54, 0x81, 0x54, 0xb7, 0xf5, 0x8e,
-	0x21, 0x98, 0xde, 0x11, 0xeb, 0xef, 0x6d, 0xbd, 0x53, 0xfb, 0x17, 0xbd, 0x83, 0xa0, 0x9e, 0x32,
-	0x2e, 0x6d, 0x8f, 0xe8, 0xef, 0xab, 0x2a, 0xcf, 0xcf, 0xb6, 0xa5, 0xca, 0xed, 0xa9, 0x6c, 0x95,
-	0x97, 0xd5, 0x6c, 0x6d, 0xa8, 0xb9, 0xa5, 0x2e, 0x9d, 0x7f, 0x5c, 0x97, 0xa5, 0x6a, 0x82, 0x72,
-	0x35, 0x29, 0x3d, 0xcd, 0x81, 0xae, 0xa1, 0x67, 0x51, 0x80, 0xff, 0xa8, 0x67, 0x94, 0xcb, 0x79,
-	0xad, 0x2a, 0xbd, 0x82, 0xe6, 0x55, 0xba, 0xce, 0x7e, 0xad, 0x90, 0xfd, 0x3d, 0x68, 0xa8, 0x7b,
-	0x99, 0x51, 0xd8, 0xc0, 0x66, 0xe1, 0x75, 0xa1, 0x3d, 0x61, 0x9c, 0x62, 0xfa, 0x5b, 0x46, 0x85,
-	0xf4, 0x1e, 0x42, 0xc7, 0x2c, 0x45, 0xca, 0x12, 0xf3, 0x12, 0x1b, 0x52, 0xb5, 0x48, 0x6a, 0x42,
-	0xfd, 0x0d, 0x8b, 0x16, 0xde, 0x5f, 0x35, 0x68, 0xcd, 0x03, 0x9a, 0x10, 0x1e, 0x31, 0x15, 0x33,
-	0x21, 0x2b, 0x53, 0x6c, 0x0e, 0xd6, 0xdf, 0xe8, 0x05, 0x74, 0xf3, 0x01, 0x68, 0xf4, 0xa9, 0x7d,
-	0xae, 0x13, 0x70, 0x27, 0x28, 0xbe, 0x15, 0x5f, 0x42, 0x3b, 0xc9, 0x56, 0x76, 0x2c, 0xe6, 0x47,
-	0x87, 0x24, 0x5b, 0x19, 0x8e, 0x9a, 0xd1, 0xf6, 0xd9, 0xc8, 0x23, 0xd4, 0x3f, 0xa7, 0x0d, 0xee,
-	0x88, 0x62, 0xab, 0xd8, 0x08, 0xc6, 0x96, 0xcf, 0x67, 0x15, 0xc1, 0x70, 0x84, 0x7a, 0xae, 0x3e,
-	0x10, 0xbe, 0xca, 0x52, 0x5f, 0xd0, 0x80, 0x25, 0x0b, 0xe1, 0x36, 0x35, 0xa6, 0x6b, 0xac, 0x73,
-	0x63, 0x54, 0x3f, 0x38, 0xeb, 0xff, 0xbc, 0x35, 0xd2, 0x54, 0x76, 0x7f, 0xed, 0xc8, 0xc1, 0xcf,
-	0xc0, 0x15, 0x29, 0xf9, 0x90, 0xf8, 0x31, 0x0b, 0x48, 0xec, 0x7f, 0x60, 0xfc, 0x9d, 0xbe, 0x41,
-	0x96, 0xe4, 0x55, 0x7e, 0x53, 0xfb, 0x8f, 0x95, 0xfb, 0x17, 0xed, 0x9d, 0x28, 0xa7, 0x37, 0x06,
-	0x27, 0x4f, 0xb8, 0x40, 0x4f, 0xc1, 0x11, 0xf9, 0x42, 0xbf, 0xa1, 0xed, 0xd1, 0x17, 0x1b, 0xf7,
-	0xb6, 0x6e, 0x7c, 0x05, 0x7c, 0x3c, 0xcc, 0x67, 0x94, 0x6e, 0xf7, 0x5d, 0x68, 0xcf, 0xdf, 0x9e,
-	0x4c, 0xfc, 0xc9, 0xf1, 0xd1, 0xf4, 0xe4, 0xac, 0x5f, 0x41, 0x7d, 0xe8, 0x8c, 0x8b, 0x96, 0xea,
-	0xe3, 0xa3, 0xbc, 0x09, 0x4a, 0x84, 0xf9, 0x14, 0xbf, 0x99, 0xe2, 0x22, 0xc1, 0x5a, 0xaa, 0xc8,
-	0x85, 0x3d, 0x63, 0x79, 0x39, 0x3d, 0x99, 0xe2, 0xa3, 0xb5, 0xa7, 0xf6, 0xf8, 0x2b, 0xd8, 0xb1,
-	0xef, 0x12, 0x72, 0xa0, 0xf1, 0xfa, 0x64, 0x8c, 0xdf, 0xf6, 0x2b, 0xa8, 0x0b, 0xce, 0xfc, 0x0c,
-	0x4f, 0xc7, 0xaf, 0x8e, 0x4e, 0x5e, 0xf6, 0xab, 0x87, 0xc3, 0x5f, 0xbf, 0x0d, 0x19, 0x0b, 0x63,
-	0x3a, 0x08, 0x59, 0x4c, 0x92, 0x70, 0xc0, 0x78, 0xa8, 0xff, 0x9c, 0x87, 0xdb, 0x7f, 0xa4, 0xcf,
-	0x9b, 0xfa, 0x1f, 0xfa, 0xbb, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x89, 0xaf, 0xc1, 0xc4,
-	0x0b, 0x00, 0x00,
+var File_benchmark_grpc_testing_control_proto protoreflect.FileDescriptor
+
+var file_benchmark_grpc_testing_control_proto_rawDesc = []byte{
+	0x0a, 0x24, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x1a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f,
+	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22, 0x62, 0x65, 0x6e,
+	0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
+	0x32, 0x0a, 0x0d, 0x50, 0x6f, 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x6c, 0x6f, 0x61, 0x64,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x4c,
+	0x6f, 0x61, 0x64, 0x22, 0x61, 0x0a, 0x0d, 0x55, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72,
+	0x69, 0x76, 0x61, 0x6c, 0x5f, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x69,
+	0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c, 0x4c, 0x6f, 0x12, 0x27, 0x0a,
+	0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c, 0x5f, 0x68, 0x69,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72,
+	0x69, 0x76, 0x61, 0x6c, 0x48, 0x69, 0x22, 0x38, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x65, 0x72, 0x6d,
+	0x69, 0x6e, 0x69, 0x73, 0x74, 0x69, 0x63, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x21, 0x0a,
+	0x0c, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x01, 0x52, 0x0b, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x4c, 0x6f, 0x61, 0x64,
+	0x22, 0x51, 0x0a, 0x0c, 0x50, 0x61, 0x72, 0x65, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x12, 0x2b, 0x0a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c,
+	0x5f, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x69, 0x6e, 0x74,
+	0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c, 0x42, 0x61, 0x73, 0x65, 0x12, 0x14, 0x0a,
+	0x05, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x61, 0x6c,
+	0x70, 0x68, 0x61, 0x22, 0x12, 0x0a, 0x10, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f,
+	0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0xbc, 0x02, 0x0a, 0x0a, 0x4c, 0x6f, 0x61, 0x64,
+	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x64,
+	0x5f, 0x6c, 0x6f, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65,
+	0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x63,
+	0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x12, 0x37, 0x0a, 0x07, 0x70, 0x6f, 0x69,
+	0x73, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x6f, 0x69, 0x73, 0x73, 0x6f,
+	0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x07, 0x70, 0x6f, 0x69, 0x73, 0x73,
+	0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x55, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x48, 0x00, 0x52, 0x07, 0x75, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x3b, 0x0a, 0x06, 0x64,
+	0x65, 0x74, 0x65, 0x72, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x65, 0x74, 0x65, 0x72,
+	0x6d, 0x69, 0x6e, 0x69, 0x73, 0x74, 0x69, 0x63, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00,
+	0x52, 0x06, 0x64, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x12, 0x34, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65,
+	0x74, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x72, 0x65, 0x74, 0x6f, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x74, 0x6f, 0x42, 0x06,
+	0x0a, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x62, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69,
+	0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x5f,
+	0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x75,
+	0x73, 0x65, 0x54, 0x65, 0x73, 0x74, 0x43, 0x61, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x6f,
+	0x73, 0x74, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x22, 0x8a, 0x05, 0x0a, 0x0c, 0x43,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x25, 0x0a, 0x0e, 0x73,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20,
+	0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65,
+	0x74, 0x73, 0x12, 0x39, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70,
+	0x65, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a,
+	0x0f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x12, 0x3f, 0x0a, 0x1c, 0x6f, 0x75, 0x74, 0x73, 0x74, 0x61, 0x6e, 0x64,
+	0x69, 0x6e, 0x67, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x19, 0x6f, 0x75, 0x74, 0x73,
+	0x74, 0x61, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x70, 0x63, 0x73, 0x50, 0x65, 0x72, 0x43, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e,
+	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x30,
+	0x0a, 0x14, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74,
+	0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x73,
+	0x79, 0x6e, 0x63, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73,
+	0x12, 0x30, 0x0a, 0x08, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01,
+	0x28, 0x0e, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x72, 0x70, 0x63, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x39, 0x0a, 0x0b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d,
+	0x73, 0x52, 0x0a, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x42, 0x0a,
+	0x0e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18,
+	0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66,
+	0x69, 0x67, 0x52, 0x0d, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x12, 0x48, 0x0a, 0x10, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x5f, 0x70,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f,
+	0x67, 0x72, 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0f, 0x68, 0x69, 0x73, 0x74,
+	0x6f, 0x67, 0x72, 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63,
+	0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08,
+	0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65,
+	0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f,
+	0x72, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x3f, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0x1c, 0x0a, 0x04, 0x4d, 0x61, 0x72, 0x6b,
+	0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
+	0x05, 0x72, 0x65, 0x73, 0x65, 0x74, 0x22, 0x75, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48,
+	0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61, 0x72, 0x6b,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x61,
+	0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, 0x22, 0xd6, 0x02,
+	0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x39,
+	0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x73,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x73, 0x65, 0x63,
+	0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04,
+	0x70, 0x6f, 0x72, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x73, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x12, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54,
+	0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c,
+	0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65,
+	0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x42, 0x0a, 0x0e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0d, 0x70, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x72,
+	0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6f,
+	0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x75, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48,
+	0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61, 0x72, 0x6b,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x61,
+	0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, 0x22, 0x69, 0x0a,
+	0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a,
+	0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x12, 0x12,
+	0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f,
+	0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22, 0x0d, 0x0a, 0x0b, 0x43, 0x6f, 0x72, 0x65,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x0c, 0x43, 0x6f, 0x72, 0x65, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22, 0x06, 0x0a,
+	0x04, 0x56, 0x6f, 0x69, 0x64, 0x22, 0xef, 0x02, 0x0a, 0x08, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72,
+	0x69, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x63,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6e, 0x75,
+	0x6d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x73, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, 0x6d,
+	0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
+	0x6e, 0x75, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x61,
+	0x72, 0x6d, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x0d, 0x77, 0x61, 0x72, 0x6d, 0x75, 0x70, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64,
+	0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x5f, 0x73,
+	0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x62, 0x65,
+	0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x37,
+	0x0a, 0x18, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x77, 0x6f,
+	0x72, 0x6b, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x15, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x57, 0x6f, 0x72, 0x6b,
+	0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x41, 0x0a, 0x09, 0x53, 0x63, 0x65, 0x6e, 0x61,
+	0x72, 0x69, 0x6f, 0x73, 0x12, 0x34, 0x0a, 0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f,
+	0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52,
+	0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x73, 0x2a, 0x2f, 0x0a, 0x0a, 0x43, 0x6c,
+	0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e, 0x43,
+	0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53, 0x59,
+	0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x2a, 0x49, 0x0a, 0x0a, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e,
+	0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53,
+	0x59, 0x4e, 0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14,
+	0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x5f, 0x53, 0x45,
+	0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x23, 0x0a, 0x07, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09,
+	0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x42, 0x2f, 0x5a, 0x2d, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
+	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f,
+	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_benchmark_grpc_testing_control_proto_rawDescOnce sync.Once
+	file_benchmark_grpc_testing_control_proto_rawDescData = file_benchmark_grpc_testing_control_proto_rawDesc
+)
+
+func file_benchmark_grpc_testing_control_proto_rawDescGZIP() []byte {
+	file_benchmark_grpc_testing_control_proto_rawDescOnce.Do(func() {
+		file_benchmark_grpc_testing_control_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_control_proto_rawDescData)
+	})
+	return file_benchmark_grpc_testing_control_proto_rawDescData
+}
+
+var file_benchmark_grpc_testing_control_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_benchmark_grpc_testing_control_proto_msgTypes = make([]protoimpl.MessageInfo, 19)
+var file_benchmark_grpc_testing_control_proto_goTypes = []interface{}{
+	(ClientType)(0),             // 0: grpc.testing.ClientType
+	(ServerType)(0),             // 1: grpc.testing.ServerType
+	(RpcType)(0),                // 2: grpc.testing.RpcType
+	(*PoissonParams)(nil),       // 3: grpc.testing.PoissonParams
+	(*UniformParams)(nil),       // 4: grpc.testing.UniformParams
+	(*DeterministicParams)(nil), // 5: grpc.testing.DeterministicParams
+	(*ParetoParams)(nil),        // 6: grpc.testing.ParetoParams
+	(*ClosedLoopParams)(nil),    // 7: grpc.testing.ClosedLoopParams
+	(*LoadParams)(nil),          // 8: grpc.testing.LoadParams
+	(*SecurityParams)(nil),      // 9: grpc.testing.SecurityParams
+	(*ClientConfig)(nil),        // 10: grpc.testing.ClientConfig
+	(*ClientStatus)(nil),        // 11: grpc.testing.ClientStatus
+	(*Mark)(nil),                // 12: grpc.testing.Mark
+	(*ClientArgs)(nil),          // 13: grpc.testing.ClientArgs
+	(*ServerConfig)(nil),        // 14: grpc.testing.ServerConfig
+	(*ServerArgs)(nil),          // 15: grpc.testing.ServerArgs
+	(*ServerStatus)(nil),        // 16: grpc.testing.ServerStatus
+	(*CoreRequest)(nil),         // 17: grpc.testing.CoreRequest
+	(*CoreResponse)(nil),        // 18: grpc.testing.CoreResponse
+	(*Void)(nil),                // 19: grpc.testing.Void
+	(*Scenario)(nil),            // 20: grpc.testing.Scenario
+	(*Scenarios)(nil),           // 21: grpc.testing.Scenarios
+	(*PayloadConfig)(nil),       // 22: grpc.testing.PayloadConfig
+	(*HistogramParams)(nil),     // 23: grpc.testing.HistogramParams
+	(*ClientStats)(nil),         // 24: grpc.testing.ClientStats
+	(*ServerStats)(nil),         // 25: grpc.testing.ServerStats
+}
+var file_benchmark_grpc_testing_control_proto_depIdxs = []int32{
+	7,  // 0: grpc.testing.LoadParams.closed_loop:type_name -> grpc.testing.ClosedLoopParams
+	3,  // 1: grpc.testing.LoadParams.poisson:type_name -> grpc.testing.PoissonParams
+	4,  // 2: grpc.testing.LoadParams.uniform:type_name -> grpc.testing.UniformParams
+	5,  // 3: grpc.testing.LoadParams.determ:type_name -> grpc.testing.DeterministicParams
+	6,  // 4: grpc.testing.LoadParams.pareto:type_name -> grpc.testing.ParetoParams
+	0,  // 5: grpc.testing.ClientConfig.client_type:type_name -> grpc.testing.ClientType
+	9,  // 6: grpc.testing.ClientConfig.security_params:type_name -> grpc.testing.SecurityParams
+	2,  // 7: grpc.testing.ClientConfig.rpc_type:type_name -> grpc.testing.RpcType
+	8,  // 8: grpc.testing.ClientConfig.load_params:type_name -> grpc.testing.LoadParams
+	22, // 9: grpc.testing.ClientConfig.payload_config:type_name -> grpc.testing.PayloadConfig
+	23, // 10: grpc.testing.ClientConfig.histogram_params:type_name -> grpc.testing.HistogramParams
+	24, // 11: grpc.testing.ClientStatus.stats:type_name -> grpc.testing.ClientStats
+	10, // 12: grpc.testing.ClientArgs.setup:type_name -> grpc.testing.ClientConfig
+	12, // 13: grpc.testing.ClientArgs.mark:type_name -> grpc.testing.Mark
+	1,  // 14: grpc.testing.ServerConfig.server_type:type_name -> grpc.testing.ServerType
+	9,  // 15: grpc.testing.ServerConfig.security_params:type_name -> grpc.testing.SecurityParams
+	22, // 16: grpc.testing.ServerConfig.payload_config:type_name -> grpc.testing.PayloadConfig
+	14, // 17: grpc.testing.ServerArgs.setup:type_name -> grpc.testing.ServerConfig
+	12, // 18: grpc.testing.ServerArgs.mark:type_name -> grpc.testing.Mark
+	25, // 19: grpc.testing.ServerStatus.stats:type_name -> grpc.testing.ServerStats
+	10, // 20: grpc.testing.Scenario.client_config:type_name -> grpc.testing.ClientConfig
+	14, // 21: grpc.testing.Scenario.server_config:type_name -> grpc.testing.ServerConfig
+	20, // 22: grpc.testing.Scenarios.scenarios:type_name -> grpc.testing.Scenario
+	23, // [23:23] is the sub-list for method output_type
+	23, // [23:23] is the sub-list for method input_type
+	23, // [23:23] is the sub-list for extension type_name
+	23, // [23:23] is the sub-list for extension extendee
+	0,  // [0:23] is the sub-list for field type_name
+}
+
+func init() { file_benchmark_grpc_testing_control_proto_init() }
+func file_benchmark_grpc_testing_control_proto_init() {
+	if File_benchmark_grpc_testing_control_proto != nil {
+		return
+	}
+	file_benchmark_grpc_testing_payloads_proto_init()
+	file_benchmark_grpc_testing_stats_proto_init()
+	if !protoimpl.UnsafeEnabled {
+		file_benchmark_grpc_testing_control_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PoissonParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*UniformParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DeterministicParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ParetoParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClosedLoopParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SecurityParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Mark); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientArgs); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerArgs); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CoreRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CoreResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Void); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Scenario); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_control_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Scenarios); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_benchmark_grpc_testing_control_proto_msgTypes[5].OneofWrappers = []interface{}{
+		(*LoadParams_ClosedLoop)(nil),
+		(*LoadParams_Poisson)(nil),
+		(*LoadParams_Uniform)(nil),
+		(*LoadParams_Determ)(nil),
+		(*LoadParams_Pareto)(nil),
+	}
+	file_benchmark_grpc_testing_control_proto_msgTypes[10].OneofWrappers = []interface{}{
+		(*ClientArgs_Setup)(nil),
+		(*ClientArgs_Mark)(nil),
+	}
+	file_benchmark_grpc_testing_control_proto_msgTypes[12].OneofWrappers = []interface{}{
+		(*ServerArgs_Setup)(nil),
+		(*ServerArgs_Mark)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_benchmark_grpc_testing_control_proto_rawDesc,
+			NumEnums:      3,
+			NumMessages:   19,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_benchmark_grpc_testing_control_proto_goTypes,
+		DependencyIndexes: file_benchmark_grpc_testing_control_proto_depIdxs,
+		EnumInfos:         file_benchmark_grpc_testing_control_proto_enumTypes,
+		MessageInfos:      file_benchmark_grpc_testing_control_proto_msgTypes,
+	}.Build()
+	File_benchmark_grpc_testing_control_proto = out.File
+	file_benchmark_grpc_testing_control_proto_rawDesc = nil
+	file_benchmark_grpc_testing_control_proto_goTypes = nil
+	file_benchmark_grpc_testing_control_proto_depIdxs = nil
 }
diff --git a/benchmark/grpc_testing/messages.pb.go b/benchmark/grpc_testing/messages.pb.go
index 0fddbc137fe2..924e24e1ef9c 100644
--- a/benchmark/grpc_testing/messages.pb.go
+++ b/benchmark/grpc_testing/messages.pb.go
@@ -1,24 +1,45 @@
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Message definitions to be used by integration test service definitions.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: benchmark/grpc_testing/messages.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // The type of payload that should be returned.
 type PayloadType int32
@@ -32,24 +53,45 @@ const (
 	PayloadType_RANDOM PayloadType = 2
 )
 
-var PayloadType_name = map[int32]string{
-	0: "COMPRESSABLE",
-	1: "UNCOMPRESSABLE",
-	2: "RANDOM",
-}
+// Enum value maps for PayloadType.
+var (
+	PayloadType_name = map[int32]string{
+		0: "COMPRESSABLE",
+		1: "UNCOMPRESSABLE",
+		2: "RANDOM",
+	}
+	PayloadType_value = map[string]int32{
+		"COMPRESSABLE":   0,
+		"UNCOMPRESSABLE": 1,
+		"RANDOM":         2,
+	}
+)
 
-var PayloadType_value = map[string]int32{
-	"COMPRESSABLE":   0,
-	"UNCOMPRESSABLE": 1,
-	"RANDOM":         2,
+func (x PayloadType) Enum() *PayloadType {
+	p := new(PayloadType)
+	*p = x
+	return p
 }
 
 func (x PayloadType) String() string {
-	return proto.EnumName(PayloadType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
+func (PayloadType) Descriptor() protoreflect.EnumDescriptor {
+	return file_benchmark_grpc_testing_messages_proto_enumTypes[0].Descriptor()
+}
+
+func (PayloadType) Type() protoreflect.EnumType {
+	return &file_benchmark_grpc_testing_messages_proto_enumTypes[0]
+}
+
+func (x PayloadType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use PayloadType.Descriptor instead.
 func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{0}
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{0}
 }
 
 // Compression algorithms
@@ -62,72 +104,101 @@ const (
 	CompressionType_DEFLATE CompressionType = 2
 )
 
-var CompressionType_name = map[int32]string{
-	0: "NONE",
-	1: "GZIP",
-	2: "DEFLATE",
-}
+// Enum value maps for CompressionType.
+var (
+	CompressionType_name = map[int32]string{
+		0: "NONE",
+		1: "GZIP",
+		2: "DEFLATE",
+	}
+	CompressionType_value = map[string]int32{
+		"NONE":    0,
+		"GZIP":    1,
+		"DEFLATE": 2,
+	}
+)
 
-var CompressionType_value = map[string]int32{
-	"NONE":    0,
-	"GZIP":    1,
-	"DEFLATE": 2,
+func (x CompressionType) Enum() *CompressionType {
+	p := new(CompressionType)
+	*p = x
+	return p
 }
 
 func (x CompressionType) String() string {
-	return proto.EnumName(CompressionType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (CompressionType) Descriptor() protoreflect.EnumDescriptor {
+	return file_benchmark_grpc_testing_messages_proto_enumTypes[1].Descriptor()
 }
 
+func (CompressionType) Type() protoreflect.EnumType {
+	return &file_benchmark_grpc_testing_messages_proto_enumTypes[1]
+}
+
+func (x CompressionType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use CompressionType.Descriptor instead.
 func (CompressionType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{1}
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{1}
 }
 
 // A block of data, to simply increase gRPC message size.
 type Payload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The type of data in body.
 	Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"`
 	// Primary contents of payload.
-	Body                 []byte   `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
 }
 
-func (m *Payload) Reset()         { *m = Payload{} }
-func (m *Payload) String() string { return proto.CompactTextString(m) }
-func (*Payload) ProtoMessage()    {}
-func (*Payload) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{0}
+func (x *Payload) Reset() {
+	*x = Payload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Payload) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Payload.Unmarshal(m, b)
-}
-func (m *Payload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Payload.Marshal(b, m, deterministic)
-}
-func (m *Payload) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Payload.Merge(m, src)
-}
-func (m *Payload) XXX_Size() int {
-	return xxx_messageInfo_Payload.Size(m)
+func (x *Payload) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Payload) XXX_DiscardUnknown() {
-	xxx_messageInfo_Payload.DiscardUnknown(m)
+
+func (*Payload) ProtoMessage() {}
+
+func (x *Payload) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Payload proto.InternalMessageInfo
+// Deprecated: Use Payload.ProtoReflect.Descriptor instead.
+func (*Payload) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *Payload) GetType() PayloadType {
-	if m != nil {
-		return m.Type
+func (x *Payload) GetType() PayloadType {
+	if x != nil {
+		return x.Type
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *Payload) GetBody() []byte {
-	if m != nil {
-		return m.Body
+func (x *Payload) GetBody() []byte {
+	if x != nil {
+		return x.Body
 	}
 	return nil
 }
@@ -135,54 +206,66 @@ func (m *Payload) GetBody() []byte {
 // A protobuf representation for grpc status. This is used by test
 // clients to specify a status that the server should attempt to return.
 type EchoStatus struct {
-	Code                 int32    `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
-	Message              string   `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *EchoStatus) Reset()         { *m = EchoStatus{} }
-func (m *EchoStatus) String() string { return proto.CompactTextString(m) }
-func (*EchoStatus) ProtoMessage()    {}
-func (*EchoStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{1}
+	Code    int32  `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
+	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
 }
 
-func (m *EchoStatus) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EchoStatus.Unmarshal(m, b)
-}
-func (m *EchoStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EchoStatus.Marshal(b, m, deterministic)
-}
-func (m *EchoStatus) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EchoStatus.Merge(m, src)
+func (x *EchoStatus) Reset() {
+	*x = EchoStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *EchoStatus) XXX_Size() int {
-	return xxx_messageInfo_EchoStatus.Size(m)
+
+func (x *EchoStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EchoStatus) XXX_DiscardUnknown() {
-	xxx_messageInfo_EchoStatus.DiscardUnknown(m)
+
+func (*EchoStatus) ProtoMessage() {}
+
+func (x *EchoStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_EchoStatus proto.InternalMessageInfo
+// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead.
+func (*EchoStatus) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *EchoStatus) GetCode() int32 {
-	if m != nil {
-		return m.Code
+func (x *EchoStatus) GetCode() int32 {
+	if x != nil {
+		return x.Code
 	}
 	return 0
 }
 
-func (m *EchoStatus) GetMessage() string {
-	if m != nil {
-		return m.Message
+func (x *EchoStatus) GetMessage() string {
+	if x != nil {
+		return x.Message
 	}
 	return ""
 }
 
 // Unary request.
 type SimpleRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload type in the response from the server.
 	// If response_type is RANDOM, server randomly chooses one from other formats.
 	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
@@ -198,282 +281,322 @@ type SimpleRequest struct {
 	// Compression algorithm to be used by the server for the response (stream)
 	ResponseCompression CompressionType `protobuf:"varint,6,opt,name=response_compression,json=responseCompression,proto3,enum=grpc.testing.CompressionType" json:"response_compression,omitempty"`
 	// Whether server should return a given status
-	ResponseStatus       *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
+	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
 }
 
-func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
-func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
-func (*SimpleRequest) ProtoMessage()    {}
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{2}
+func (x *SimpleRequest) Reset() {
+	*x = SimpleRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleRequest.Unmarshal(m, b)
+func (x *SimpleRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic)
-}
-func (m *SimpleRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleRequest.Merge(m, src)
-}
-func (m *SimpleRequest) XXX_Size() int {
-	return xxx_messageInfo_SimpleRequest.Size(m)
-}
-func (m *SimpleRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleRequest.DiscardUnknown(m)
+
+func (*SimpleRequest) ProtoMessage() {}
+
+func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo
+// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
+func (*SimpleRequest) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *SimpleRequest) GetResponseType() PayloadType {
-	if m != nil {
-		return m.ResponseType
+func (x *SimpleRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *SimpleRequest) GetResponseSize() int32 {
-	if m != nil {
-		return m.ResponseSize
+func (x *SimpleRequest) GetResponseSize() int32 {
+	if x != nil {
+		return x.ResponseSize
 	}
 	return 0
 }
 
-func (m *SimpleRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *SimpleRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *SimpleRequest) GetFillUsername() bool {
-	if m != nil {
-		return m.FillUsername
+func (x *SimpleRequest) GetFillUsername() bool {
+	if x != nil {
+		return x.FillUsername
 	}
 	return false
 }
 
-func (m *SimpleRequest) GetFillOauthScope() bool {
-	if m != nil {
-		return m.FillOauthScope
+func (x *SimpleRequest) GetFillOauthScope() bool {
+	if x != nil {
+		return x.FillOauthScope
 	}
 	return false
 }
 
-func (m *SimpleRequest) GetResponseCompression() CompressionType {
-	if m != nil {
-		return m.ResponseCompression
+func (x *SimpleRequest) GetResponseCompression() CompressionType {
+	if x != nil {
+		return x.ResponseCompression
 	}
 	return CompressionType_NONE
 }
 
-func (m *SimpleRequest) GetResponseStatus() *EchoStatus {
-	if m != nil {
-		return m.ResponseStatus
+func (x *SimpleRequest) GetResponseStatus() *EchoStatus {
+	if x != nil {
+		return x.ResponseStatus
 	}
 	return nil
 }
 
 // Unary response, as configured by the request.
 type SimpleResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Payload to increase message size.
 	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 	// The user the request came from, for verifying authentication was
 	// successful when the client expected it.
 	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
 	// OAuth scope.
-	OauthScope           string   `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
 }
 
-func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
-func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
-func (*SimpleResponse) ProtoMessage()    {}
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{3}
+func (x *SimpleResponse) Reset() {
+	*x = SimpleResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleResponse.Unmarshal(m, b)
-}
-func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic)
-}
-func (m *SimpleResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleResponse.Merge(m, src)
-}
-func (m *SimpleResponse) XXX_Size() int {
-	return xxx_messageInfo_SimpleResponse.Size(m)
+func (x *SimpleResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleResponse.DiscardUnknown(m)
+
+func (*SimpleResponse) ProtoMessage() {}
+
+func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo
+// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
+func (*SimpleResponse) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *SimpleResponse) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *SimpleResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *SimpleResponse) GetUsername() string {
-	if m != nil {
-		return m.Username
+func (x *SimpleResponse) GetUsername() string {
+	if x != nil {
+		return x.Username
 	}
 	return ""
 }
 
-func (m *SimpleResponse) GetOauthScope() string {
-	if m != nil {
-		return m.OauthScope
+func (x *SimpleResponse) GetOauthScope() string {
+	if x != nil {
+		return x.OauthScope
 	}
 	return ""
 }
 
 // Client-streaming request.
 type StreamingInputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Optional input payload sent along with the request.
-	Payload              *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingInputCallRequest) Reset()         { *m = StreamingInputCallRequest{} }
-func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) }
-func (*StreamingInputCallRequest) ProtoMessage()    {}
-func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{4}
+func (x *StreamingInputCallRequest) Reset() {
+	*x = StreamingInputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingInputCallRequest.Unmarshal(m, b)
+func (x *StreamingInputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingInputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingInputCallRequest.Marshal(b, m, deterministic)
-}
-func (m *StreamingInputCallRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingInputCallRequest.Merge(m, src)
-}
-func (m *StreamingInputCallRequest) XXX_Size() int {
-	return xxx_messageInfo_StreamingInputCallRequest.Size(m)
-}
-func (m *StreamingInputCallRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingInputCallRequest.DiscardUnknown(m)
+
+func (*StreamingInputCallRequest) ProtoMessage() {}
+
+func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingInputCallRequest proto.InternalMessageInfo
+// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *StreamingInputCallRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingInputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
 // Client-streaming response.
 type StreamingInputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Aggregated size of payloads received from the client.
-	AggregatedPayloadSize int32    `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{} `json:"-"`
-	XXX_unrecognized      []byte   `json:"-"`
-	XXX_sizecache         int32    `json:"-"`
+	AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
 }
 
-func (m *StreamingInputCallResponse) Reset()         { *m = StreamingInputCallResponse{} }
-func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) }
-func (*StreamingInputCallResponse) ProtoMessage()    {}
-func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{5}
+func (x *StreamingInputCallResponse) Reset() {
+	*x = StreamingInputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingInputCallResponse.Unmarshal(m, b)
-}
-func (m *StreamingInputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingInputCallResponse.Marshal(b, m, deterministic)
-}
-func (m *StreamingInputCallResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingInputCallResponse.Merge(m, src)
-}
-func (m *StreamingInputCallResponse) XXX_Size() int {
-	return xxx_messageInfo_StreamingInputCallResponse.Size(m)
+func (x *StreamingInputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingInputCallResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingInputCallResponse.DiscardUnknown(m)
+
+func (*StreamingInputCallResponse) ProtoMessage() {}
+
+func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingInputCallResponse proto.InternalMessageInfo
+// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
-	if m != nil {
-		return m.AggregatedPayloadSize
+func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
+	if x != nil {
+		return x.AggregatedPayloadSize
 	}
 	return 0
 }
 
 // Configuration for a particular response.
 type ResponseParameters struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload sizes in responses from the server.
 	// If response_type is COMPRESSABLE, this denotes the size before compression.
 	Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"`
 	// Desired interval between consecutive responses in the response stream in
 	// microseconds.
-	IntervalUs           int32    `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
 }
 
-func (m *ResponseParameters) Reset()         { *m = ResponseParameters{} }
-func (m *ResponseParameters) String() string { return proto.CompactTextString(m) }
-func (*ResponseParameters) ProtoMessage()    {}
-func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{6}
+func (x *ResponseParameters) Reset() {
+	*x = ResponseParameters{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ResponseParameters) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ResponseParameters.Unmarshal(m, b)
-}
-func (m *ResponseParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ResponseParameters.Marshal(b, m, deterministic)
+func (x *ResponseParameters) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ResponseParameters) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ResponseParameters.Merge(m, src)
-}
-func (m *ResponseParameters) XXX_Size() int {
-	return xxx_messageInfo_ResponseParameters.Size(m)
-}
-func (m *ResponseParameters) XXX_DiscardUnknown() {
-	xxx_messageInfo_ResponseParameters.DiscardUnknown(m)
+
+func (*ResponseParameters) ProtoMessage() {}
+
+func (x *ResponseParameters) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ResponseParameters proto.InternalMessageInfo
+// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead.
+func (*ResponseParameters) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *ResponseParameters) GetSize() int32 {
-	if m != nil {
-		return m.Size
+func (x *ResponseParameters) GetSize() int32 {
+	if x != nil {
+		return x.Size
 	}
 	return 0
 }
 
-func (m *ResponseParameters) GetIntervalUs() int32 {
-	if m != nil {
-		return m.IntervalUs
+func (x *ResponseParameters) GetIntervalUs() int32 {
+	if x != nil {
+		return x.IntervalUs
 	}
 	return 0
 }
 
 // Server-streaming request.
 type StreamingOutputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload type in the response from the server.
 	// If response_type is RANDOM, the payload from each response in the stream
 	// might be of different types. This is to simulate a mixed type of payload
@@ -486,109 +609,121 @@ type StreamingOutputCallRequest struct {
 	// Compression algorithm to be used by the server for the response (stream)
 	ResponseCompression CompressionType `protobuf:"varint,6,opt,name=response_compression,json=responseCompression,proto3,enum=grpc.testing.CompressionType" json:"response_compression,omitempty"`
 	// Whether server should return a given status
-	ResponseStatus       *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
+	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
 }
 
-func (m *StreamingOutputCallRequest) Reset()         { *m = StreamingOutputCallRequest{} }
-func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) }
-func (*StreamingOutputCallRequest) ProtoMessage()    {}
-func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{7}
+func (x *StreamingOutputCallRequest) Reset() {
+	*x = StreamingOutputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingOutputCallRequest.Unmarshal(m, b)
+func (x *StreamingOutputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingOutputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingOutputCallRequest.Marshal(b, m, deterministic)
-}
-func (m *StreamingOutputCallRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingOutputCallRequest.Merge(m, src)
-}
-func (m *StreamingOutputCallRequest) XXX_Size() int {
-	return xxx_messageInfo_StreamingOutputCallRequest.Size(m)
-}
-func (m *StreamingOutputCallRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingOutputCallRequest.DiscardUnknown(m)
+
+func (*StreamingOutputCallRequest) ProtoMessage() {}
+
+func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingOutputCallRequest proto.InternalMessageInfo
+// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *StreamingOutputCallRequest) GetResponseType() PayloadType {
-	if m != nil {
-		return m.ResponseType
+func (x *StreamingOutputCallRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
-	if m != nil {
-		return m.ResponseParameters
+func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
+	if x != nil {
+		return x.ResponseParameters
 	}
 	return nil
 }
 
-func (m *StreamingOutputCallRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingOutputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *StreamingOutputCallRequest) GetResponseCompression() CompressionType {
-	if m != nil {
-		return m.ResponseCompression
+func (x *StreamingOutputCallRequest) GetResponseCompression() CompressionType {
+	if x != nil {
+		return x.ResponseCompression
 	}
 	return CompressionType_NONE
 }
 
-func (m *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
-	if m != nil {
-		return m.ResponseStatus
+func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
+	if x != nil {
+		return x.ResponseStatus
 	}
 	return nil
 }
 
 // Server-streaming response, as configured by the request and parameters.
 type StreamingOutputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Payload to increase response size.
-	Payload              *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingOutputCallResponse) Reset()         { *m = StreamingOutputCallResponse{} }
-func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) }
-func (*StreamingOutputCallResponse) ProtoMessage()    {}
-func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{8}
+func (x *StreamingOutputCallResponse) Reset() {
+	*x = StreamingOutputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingOutputCallResponse.Unmarshal(m, b)
-}
-func (m *StreamingOutputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingOutputCallResponse.Marshal(b, m, deterministic)
-}
-func (m *StreamingOutputCallResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingOutputCallResponse.Merge(m, src)
+func (x *StreamingOutputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingOutputCallResponse) XXX_Size() int {
-	return xxx_messageInfo_StreamingOutputCallResponse.Size(m)
-}
-func (m *StreamingOutputCallResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingOutputCallResponse.DiscardUnknown(m)
+
+func (*StreamingOutputCallResponse) ProtoMessage() {}
+
+func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingOutputCallResponse proto.InternalMessageInfo
+// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *StreamingOutputCallResponse) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingOutputCallResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
@@ -596,40 +731,48 @@ func (m *StreamingOutputCallResponse) GetPayload() *Payload {
 // For reconnect interop test only.
 // Client tells server what reconnection parameters it used.
 type ReconnectParams struct {
-	MaxReconnectBackoffMs int32    `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{} `json:"-"`
-	XXX_unrecognized      []byte   `json:"-"`
-	XXX_sizecache         int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ReconnectParams) Reset()         { *m = ReconnectParams{} }
-func (m *ReconnectParams) String() string { return proto.CompactTextString(m) }
-func (*ReconnectParams) ProtoMessage()    {}
-func (*ReconnectParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{9}
+	MaxReconnectBackoffMs int32 `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"`
 }
 
-func (m *ReconnectParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ReconnectParams.Unmarshal(m, b)
-}
-func (m *ReconnectParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ReconnectParams.Marshal(b, m, deterministic)
-}
-func (m *ReconnectParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReconnectParams.Merge(m, src)
+func (x *ReconnectParams) Reset() {
+	*x = ReconnectParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ReconnectParams) XXX_Size() int {
-	return xxx_messageInfo_ReconnectParams.Size(m)
+
+func (x *ReconnectParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ReconnectParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_ReconnectParams.DiscardUnknown(m)
+
+func (*ReconnectParams) ProtoMessage() {}
+
+func (x *ReconnectParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ReconnectParams proto.InternalMessageInfo
+// Deprecated: Use ReconnectParams.ProtoReflect.Descriptor instead.
+func (*ReconnectParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{9}
+}
 
-func (m *ReconnectParams) GetMaxReconnectBackoffMs() int32 {
-	if m != nil {
-		return m.MaxReconnectBackoffMs
+func (x *ReconnectParams) GetMaxReconnectBackoffMs() int32 {
+	if x != nil {
+		return x.MaxReconnectBackoffMs
 	}
 	return 0
 }
@@ -638,116 +781,380 @@ func (m *ReconnectParams) GetMaxReconnectBackoffMs() int32 {
 // Server tells client whether its reconnects are following the spec and the
 // reconnect backoffs it saw.
 type ReconnectInfo struct {
-	Passed               bool     `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"`
-	BackoffMs            []int32  `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ReconnectInfo) Reset()         { *m = ReconnectInfo{} }
-func (m *ReconnectInfo) String() string { return proto.CompactTextString(m) }
-func (*ReconnectInfo) ProtoMessage()    {}
-func (*ReconnectInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7e3146bec7fa4346, []int{10}
+	Passed    bool    `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"`
+	BackoffMs []int32 `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"`
 }
 
-func (m *ReconnectInfo) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ReconnectInfo.Unmarshal(m, b)
-}
-func (m *ReconnectInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ReconnectInfo.Marshal(b, m, deterministic)
-}
-func (m *ReconnectInfo) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReconnectInfo.Merge(m, src)
+func (x *ReconnectInfo) Reset() {
+	*x = ReconnectInfo{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ReconnectInfo) XXX_Size() int {
-	return xxx_messageInfo_ReconnectInfo.Size(m)
+
+func (x *ReconnectInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ReconnectInfo) XXX_DiscardUnknown() {
-	xxx_messageInfo_ReconnectInfo.DiscardUnknown(m)
+
+func (*ReconnectInfo) ProtoMessage() {}
+
+func (x *ReconnectInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ReconnectInfo proto.InternalMessageInfo
+// Deprecated: Use ReconnectInfo.ProtoReflect.Descriptor instead.
+func (*ReconnectInfo) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{10}
+}
 
-func (m *ReconnectInfo) GetPassed() bool {
-	if m != nil {
-		return m.Passed
+func (x *ReconnectInfo) GetPassed() bool {
+	if x != nil {
+		return x.Passed
 	}
 	return false
 }
 
-func (m *ReconnectInfo) GetBackoffMs() []int32 {
-	if m != nil {
-		return m.BackoffMs
+func (x *ReconnectInfo) GetBackoffMs() []int32 {
+	if x != nil {
+		return x.BackoffMs
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value)
-	proto.RegisterEnum("grpc.testing.CompressionType", CompressionType_name, CompressionType_value)
-	proto.RegisterType((*Payload)(nil), "grpc.testing.Payload")
-	proto.RegisterType((*EchoStatus)(nil), "grpc.testing.EchoStatus")
-	proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest")
-	proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse")
-	proto.RegisterType((*StreamingInputCallRequest)(nil), "grpc.testing.StreamingInputCallRequest")
-	proto.RegisterType((*StreamingInputCallResponse)(nil), "grpc.testing.StreamingInputCallResponse")
-	proto.RegisterType((*ResponseParameters)(nil), "grpc.testing.ResponseParameters")
-	proto.RegisterType((*StreamingOutputCallRequest)(nil), "grpc.testing.StreamingOutputCallRequest")
-	proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse")
-	proto.RegisterType((*ReconnectParams)(nil), "grpc.testing.ReconnectParams")
-	proto.RegisterType((*ReconnectInfo)(nil), "grpc.testing.ReconnectInfo")
-}
-
-func init() {
-	proto.RegisterFile("benchmark/grpc_testing/messages.proto", fileDescriptor_7e3146bec7fa4346)
-}
-
-var fileDescriptor_7e3146bec7fa4346 = []byte{
-	// 690 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x55, 0x4b, 0x6f, 0xd3, 0x4c,
-	0x14, 0xfd, 0x9c, 0x77, 0x6f, 0xd2, 0x34, 0x9a, 0x8f, 0x82, 0x5b, 0x54, 0x11, 0x19, 0x21, 0x45,
-	0x95, 0x9a, 0x48, 0x41, 0x02, 0x89, 0x05, 0x28, 0x6d, 0x53, 0x14, 0x94, 0x26, 0xc1, 0x6e, 0x37,
-	0xdd, 0x58, 0x13, 0x67, 0xe2, 0x5a, 0xb5, 0x3d, 0x66, 0x66, 0x82, 0x9a, 0x2e, 0xd8, 0xf3, 0x83,
-	0xd9, 0x23, 0x8f, 0x1f, 0x71, 0xda, 0x2c, 0x5a, 0xd8, 0xb0, 0x9b, 0x39, 0xf7, 0xdc, 0xe3, 0xfb,
-	0x38, 0xd6, 0xc0, 0x9b, 0x29, 0xf1, 0xad, 0x6b, 0x0f, 0xb3, 0x9b, 0x8e, 0xcd, 0x02, 0xcb, 0x14,
-	0x84, 0x0b, 0xc7, 0xb7, 0x3b, 0x1e, 0xe1, 0x1c, 0xdb, 0x84, 0xb7, 0x03, 0x46, 0x05, 0x45, 0xb5,
-	0x30, 0xd8, 0x8e, 0x83, 0xda, 0x10, 0xca, 0x13, 0xbc, 0x74, 0x29, 0x9e, 0xa1, 0x23, 0x28, 0x88,
-	0x65, 0x40, 0x54, 0xa5, 0xa9, 0xb4, 0xea, 0xdd, 0xbd, 0x76, 0x96, 0xd7, 0x8e, 0x49, 0x17, 0xcb,
-	0x80, 0xe8, 0x92, 0x86, 0x10, 0x14, 0xa6, 0x74, 0xb6, 0x54, 0x73, 0x4d, 0xa5, 0x55, 0xd3, 0xe5,
-	0x59, 0xfb, 0x00, 0xd0, 0xb7, 0xae, 0xa9, 0x21, 0xb0, 0x58, 0xf0, 0x90, 0x61, 0xd1, 0x59, 0x24,
-	0x58, 0xd4, 0xe5, 0x19, 0xa9, 0x50, 0x8e, 0xeb, 0x91, 0x89, 0x5b, 0x7a, 0x72, 0xd5, 0x7e, 0xe6,
-	0x61, 0xdb, 0x70, 0xbc, 0xc0, 0x25, 0x3a, 0xf9, 0xb6, 0x20, 0x5c, 0xa0, 0x8f, 0xb0, 0xcd, 0x08,
-	0x0f, 0xa8, 0xcf, 0x89, 0xf9, 0xb8, 0xca, 0x6a, 0x09, 0x3f, 0xbc, 0xa1, 0xd7, 0x99, 0x7c, 0xee,
-	0xdc, 0x45, 0x5f, 0x2c, 0xae, 0x48, 0x86, 0x73, 0x47, 0x50, 0x07, 0xca, 0x41, 0xa4, 0xa0, 0xe6,
-	0x9b, 0x4a, 0xab, 0xda, 0xdd, 0xdd, 0x28, 0xaf, 0x27, 0xac, 0x50, 0x75, 0xee, 0xb8, 0xae, 0xb9,
-	0xe0, 0x84, 0xf9, 0xd8, 0x23, 0x6a, 0xa1, 0xa9, 0xb4, 0x2a, 0x7a, 0x2d, 0x04, 0x2f, 0x63, 0x0c,
-	0xb5, 0xa0, 0x21, 0x49, 0x14, 0x2f, 0xc4, 0xb5, 0xc9, 0x2d, 0x1a, 0x10, 0xb5, 0x28, 0x79, 0xf5,
-	0x10, 0x1f, 0x87, 0xb0, 0x11, 0xa2, 0x68, 0x02, 0xcf, 0xd2, 0x22, 0x2d, 0xea, 0x05, 0x8c, 0x70,
-	0xee, 0x50, 0x5f, 0x2d, 0xc9, 0x5e, 0x0f, 0xd6, 0x8b, 0x39, 0x59, 0x11, 0x64, 0xbf, 0xff, 0x27,
-	0xa9, 0x99, 0x00, 0xea, 0xc1, 0xce, 0xaa, 0x6d, 0xb9, 0x09, 0xb5, 0x2c, 0x3b, 0x53, 0xd7, 0xc5,
-	0x56, 0x9b, 0xd2, 0xeb, 0xe9, 0x48, 0xe4, 0x5d, 0xfb, 0x01, 0xf5, 0x64, 0x15, 0x11, 0x9e, 0x1d,
-	0x93, 0xf2, 0xa8, 0x31, 0xed, 0x43, 0x25, 0x9d, 0x50, 0xb4, 0xe9, 0xf4, 0x8e, 0x5e, 0x41, 0x35,
-	0x3b, 0x98, 0xbc, 0x0c, 0x03, 0x4d, 0x87, 0xa2, 0x0d, 0x61, 0xcf, 0x10, 0x8c, 0x60, 0xcf, 0xf1,
-	0xed, 0x81, 0x1f, 0x2c, 0xc4, 0x09, 0x76, 0xdd, 0xc4, 0x16, 0x4f, 0x2d, 0x45, 0xbb, 0x80, 0xfd,
-	0x4d, 0x6a, 0x71, 0x67, 0xef, 0xe0, 0x05, 0xb6, 0x6d, 0x46, 0x6c, 0x2c, 0xc8, 0xcc, 0x8c, 0x73,
-	0x22, 0xbf, 0x44, 0xc6, 0xdd, 0x5d, 0x85, 0x63, 0xe9, 0xd0, 0x38, 0xda, 0x00, 0x50, 0xa2, 0x31,
-	0xc1, 0x0c, 0x7b, 0x44, 0x10, 0x26, 0x3d, 0x9f, 0x49, 0x95, 0xe7, 0xb0, 0x5d, 0xc7, 0x17, 0x84,
-	0x7d, 0xc7, 0xa1, 0x6b, 0x62, 0x17, 0x42, 0x02, 0x5d, 0x72, 0xed, 0x57, 0x2e, 0x53, 0xe1, 0x78,
-	0x21, 0xee, 0x35, 0xfc, 0xb7, 0xff, 0xc1, 0x57, 0x48, 0x7d, 0x62, 0x06, 0x69, 0xa9, 0x6a, 0xae,
-	0x99, 0x6f, 0x55, 0xbb, 0xcd, 0x75, 0x95, 0x87, 0x2d, 0xe9, 0x88, 0x3d, 0x6c, 0xf3, 0xc9, 0x7f,
-	0xcd, 0x3f, 0x69, 0xf3, 0x11, 0xbc, 0xdc, 0x38, 0xf6, 0x3f, 0xf4, 0xbc, 0xf6, 0x05, 0x76, 0x74,
-	0x62, 0x51, 0xdf, 0x27, 0x96, 0x90, 0xc3, 0xe2, 0xe8, 0x3d, 0xa8, 0x1e, 0xbe, 0x35, 0x59, 0x02,
-	0x9b, 0x53, 0x6c, 0xdd, 0xd0, 0xf9, 0xdc, 0xf4, 0x78, 0x62, 0x2f, 0x0f, 0xdf, 0xa6, 0x59, 0xc7,
-	0x51, 0xf4, 0x9c, 0x6b, 0x67, 0xb0, 0x9d, 0xa2, 0x03, 0x7f, 0x4e, 0xd1, 0x73, 0x28, 0x05, 0x98,
-	0x73, 0x12, 0x15, 0x53, 0xd1, 0xe3, 0x1b, 0x3a, 0x00, 0xc8, 0x68, 0x86, 0x4b, 0x2d, 0xea, 0x5b,
-	0xd3, 0x44, 0xe7, 0xf0, 0x13, 0x54, 0x33, 0xce, 0x40, 0x0d, 0xa8, 0x9d, 0x8c, 0xcf, 0x27, 0x7a,
-	0xdf, 0x30, 0x7a, 0xc7, 0xc3, 0x7e, 0xe3, 0x3f, 0x84, 0xa0, 0x7e, 0x39, 0x5a, 0xc3, 0x14, 0x04,
-	0x50, 0xd2, 0x7b, 0xa3, 0xd3, 0xf1, 0x79, 0x23, 0x77, 0xd8, 0x85, 0x9d, 0x7b, 0xfb, 0x40, 0x15,
-	0x28, 0x8c, 0xc6, 0xa3, 0x30, 0xb9, 0x02, 0x85, 0xcf, 0x57, 0x83, 0x49, 0x43, 0x41, 0x55, 0x28,
-	0x9f, 0xf6, 0xcf, 0x86, 0xbd, 0x8b, 0x7e, 0x23, 0x77, 0xdc, 0xb9, 0x3a, 0xb2, 0x29, 0xb5, 0x5d,
-	0xd2, 0xb6, 0xa9, 0x8b, 0x7d, 0xbb, 0x4d, 0x99, 0x2d, 0x1f, 0xa5, 0xce, 0xe6, 0x37, 0x6a, 0x5a,
-	0x92, 0x6f, 0xd3, 0xdb, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xea, 0xde, 0x4b, 0x12, 0xc4, 0x06,
-	0x00, 0x00,
+var File_benchmark_grpc_testing_messages_proto protoreflect.FileDescriptor
+
+var file_benchmark_grpc_testing_messages_proto_rawDesc = []byte{
+	0x0a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x4c, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
+	0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62,
+	0x6f, 0x64, 0x79, 0x22, 0x3a, 0x0a, 0x0a, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22,
+	0x89, 0x03, 0x0a, 0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79,
+	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54,
+	0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x69,
+	0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07,
+	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x6c, 0x5f,
+	0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c,
+	0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10,
+	0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x4f, 0x61, 0x75, 0x74,
+	0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x50, 0x0a, 0x14, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06,
+	0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54,
+	0x79, 0x70, 0x65, 0x52, 0x13, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x6d,
+	0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x7e, 0x0a, 0x0e, 0x53,
+	0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a,
+	0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x61,
+	0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x0a, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x4c, 0x0a, 0x19, 0x53,
+	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c,
+	0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74, 0x72,
+	0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65,
+	0x67, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x69,
+	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67,
+	0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x22,
+	0x49, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d,
+	0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x74,
+	0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
+	0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x22, 0xf5, 0x02, 0x0a, 0x1a, 0x53,
+	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61,
+	0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73,
+	0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x07,
+	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x50, 0x0a,
+	0x14, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65,
+	0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x72,
+	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x13, 0x72, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12,
+	0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74,
+	0x75, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74,
+	0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74,
+	0x75, 0x73, 0x22, 0x4e, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f,
+	0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x22, 0x4a, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x50,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x63,
+	0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d,
+	0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x63, 0x6f,
+	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x22, 0x46,
+	0x0a, 0x0d, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12,
+	0x16, 0x0a, 0x06, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
+	0x06, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x6f,
+	0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x63,
+	0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x2a, 0x3f, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53,
+	0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d,
+	0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52,
+	0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10, 0x02, 0x2a, 0x32, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x72,
+	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f,
+	0x4e, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x47, 0x5a, 0x49, 0x50, 0x10, 0x01, 0x12, 0x0b,
+	0x0a, 0x07, 0x44, 0x45, 0x46, 0x4c, 0x41, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2f, 0x5a, 0x2d, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
+	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f,
+	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_benchmark_grpc_testing_messages_proto_rawDescOnce sync.Once
+	file_benchmark_grpc_testing_messages_proto_rawDescData = file_benchmark_grpc_testing_messages_proto_rawDesc
+)
+
+func file_benchmark_grpc_testing_messages_proto_rawDescGZIP() []byte {
+	file_benchmark_grpc_testing_messages_proto_rawDescOnce.Do(func() {
+		file_benchmark_grpc_testing_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_messages_proto_rawDescData)
+	})
+	return file_benchmark_grpc_testing_messages_proto_rawDescData
+}
+
+var file_benchmark_grpc_testing_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_benchmark_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
+var file_benchmark_grpc_testing_messages_proto_goTypes = []interface{}{
+	(PayloadType)(0),                    // 0: grpc.testing.PayloadType
+	(CompressionType)(0),                // 1: grpc.testing.CompressionType
+	(*Payload)(nil),                     // 2: grpc.testing.Payload
+	(*EchoStatus)(nil),                  // 3: grpc.testing.EchoStatus
+	(*SimpleRequest)(nil),               // 4: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil),              // 5: grpc.testing.SimpleResponse
+	(*StreamingInputCallRequest)(nil),   // 6: grpc.testing.StreamingInputCallRequest
+	(*StreamingInputCallResponse)(nil),  // 7: grpc.testing.StreamingInputCallResponse
+	(*ResponseParameters)(nil),          // 8: grpc.testing.ResponseParameters
+	(*StreamingOutputCallRequest)(nil),  // 9: grpc.testing.StreamingOutputCallRequest
+	(*StreamingOutputCallResponse)(nil), // 10: grpc.testing.StreamingOutputCallResponse
+	(*ReconnectParams)(nil),             // 11: grpc.testing.ReconnectParams
+	(*ReconnectInfo)(nil),               // 12: grpc.testing.ReconnectInfo
+}
+var file_benchmark_grpc_testing_messages_proto_depIdxs = []int32{
+	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
+	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
+	2,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
+	1,  // 3: grpc.testing.SimpleRequest.response_compression:type_name -> grpc.testing.CompressionType
+	3,  // 4: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
+	2,  // 5: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
+	2,  // 6: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
+	0,  // 7: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
+	8,  // 8: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
+	2,  // 9: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
+	1,  // 10: grpc.testing.StreamingOutputCallRequest.response_compression:type_name -> grpc.testing.CompressionType
+	3,  // 11: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
+	2,  // 12: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
+	13, // [13:13] is the sub-list for method output_type
+	13, // [13:13] is the sub-list for method input_type
+	13, // [13:13] is the sub-list for extension type_name
+	13, // [13:13] is the sub-list for extension extendee
+	0,  // [0:13] is the sub-list for field type_name
+}
+
+func init() { file_benchmark_grpc_testing_messages_proto_init() }
+func file_benchmark_grpc_testing_messages_proto_init() {
+	if File_benchmark_grpc_testing_messages_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_benchmark_grpc_testing_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Payload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ResponseParameters); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ReconnectParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_messages_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ReconnectInfo); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_benchmark_grpc_testing_messages_proto_rawDesc,
+			NumEnums:      2,
+			NumMessages:   11,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_benchmark_grpc_testing_messages_proto_goTypes,
+		DependencyIndexes: file_benchmark_grpc_testing_messages_proto_depIdxs,
+		EnumInfos:         file_benchmark_grpc_testing_messages_proto_enumTypes,
+		MessageInfos:      file_benchmark_grpc_testing_messages_proto_msgTypes,
+	}.Build()
+	File_benchmark_grpc_testing_messages_proto = out.File
+	file_benchmark_grpc_testing_messages_proto_rawDesc = nil
+	file_benchmark_grpc_testing_messages_proto_goTypes = nil
+	file_benchmark_grpc_testing_messages_proto_depIdxs = nil
 }
diff --git a/benchmark/grpc_testing/payloads.pb.go b/benchmark/grpc_testing/payloads.pb.go
index c96b7b6c4213..5e6313896009 100644
--- a/benchmark/grpc_testing/payloads.pb.go
+++ b/benchmark/grpc_testing/payloads.pb.go
@@ -1,185 +1,263 @@
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: benchmark/grpc_testing/payloads.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type ByteBufferParams struct {
-	ReqSize              int32    `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"`
-	RespSize             int32    `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ByteBufferParams) Reset()         { *m = ByteBufferParams{} }
-func (m *ByteBufferParams) String() string { return proto.CompactTextString(m) }
-func (*ByteBufferParams) ProtoMessage()    {}
-func (*ByteBufferParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_69438956f5d73a41, []int{0}
+	ReqSize  int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"`
+	RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"`
 }
 
-func (m *ByteBufferParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ByteBufferParams.Unmarshal(m, b)
-}
-func (m *ByteBufferParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ByteBufferParams.Marshal(b, m, deterministic)
-}
-func (m *ByteBufferParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ByteBufferParams.Merge(m, src)
+func (x *ByteBufferParams) Reset() {
+	*x = ByteBufferParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ByteBufferParams) XXX_Size() int {
-	return xxx_messageInfo_ByteBufferParams.Size(m)
+
+func (x *ByteBufferParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ByteBufferParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_ByteBufferParams.DiscardUnknown(m)
+
+func (*ByteBufferParams) ProtoMessage() {}
+
+func (x *ByteBufferParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ByteBufferParams proto.InternalMessageInfo
+// Deprecated: Use ByteBufferParams.ProtoReflect.Descriptor instead.
+func (*ByteBufferParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *ByteBufferParams) GetReqSize() int32 {
-	if m != nil {
-		return m.ReqSize
+func (x *ByteBufferParams) GetReqSize() int32 {
+	if x != nil {
+		return x.ReqSize
 	}
 	return 0
 }
 
-func (m *ByteBufferParams) GetRespSize() int32 {
-	if m != nil {
-		return m.RespSize
+func (x *ByteBufferParams) GetRespSize() int32 {
+	if x != nil {
+		return x.RespSize
 	}
 	return 0
 }
 
 type SimpleProtoParams struct {
-	ReqSize              int32    `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"`
-	RespSize             int32    `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *SimpleProtoParams) Reset()         { *m = SimpleProtoParams{} }
-func (m *SimpleProtoParams) String() string { return proto.CompactTextString(m) }
-func (*SimpleProtoParams) ProtoMessage()    {}
-func (*SimpleProtoParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_69438956f5d73a41, []int{1}
+	ReqSize  int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"`
+	RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"`
 }
 
-func (m *SimpleProtoParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleProtoParams.Unmarshal(m, b)
-}
-func (m *SimpleProtoParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleProtoParams.Marshal(b, m, deterministic)
-}
-func (m *SimpleProtoParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleProtoParams.Merge(m, src)
+func (x *SimpleProtoParams) Reset() {
+	*x = SimpleProtoParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *SimpleProtoParams) XXX_Size() int {
-	return xxx_messageInfo_SimpleProtoParams.Size(m)
+
+func (x *SimpleProtoParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleProtoParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleProtoParams.DiscardUnknown(m)
+
+func (*SimpleProtoParams) ProtoMessage() {}
+
+func (x *SimpleProtoParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleProtoParams proto.InternalMessageInfo
+// Deprecated: Use SimpleProtoParams.ProtoReflect.Descriptor instead.
+func (*SimpleProtoParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *SimpleProtoParams) GetReqSize() int32 {
-	if m != nil {
-		return m.ReqSize
+func (x *SimpleProtoParams) GetReqSize() int32 {
+	if x != nil {
+		return x.ReqSize
 	}
 	return 0
 }
 
-func (m *SimpleProtoParams) GetRespSize() int32 {
-	if m != nil {
-		return m.RespSize
+func (x *SimpleProtoParams) GetRespSize() int32 {
+	if x != nil {
+		return x.RespSize
 	}
 	return 0
 }
 
 type ComplexProtoParams struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *ComplexProtoParams) Reset()         { *m = ComplexProtoParams{} }
-func (m *ComplexProtoParams) String() string { return proto.CompactTextString(m) }
-func (*ComplexProtoParams) ProtoMessage()    {}
-func (*ComplexProtoParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_69438956f5d73a41, []int{2}
+func (x *ComplexProtoParams) Reset() {
+	*x = ComplexProtoParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ComplexProtoParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ComplexProtoParams.Unmarshal(m, b)
-}
-func (m *ComplexProtoParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ComplexProtoParams.Marshal(b, m, deterministic)
-}
-func (m *ComplexProtoParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ComplexProtoParams.Merge(m, src)
-}
-func (m *ComplexProtoParams) XXX_Size() int {
-	return xxx_messageInfo_ComplexProtoParams.Size(m)
+func (x *ComplexProtoParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ComplexProtoParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_ComplexProtoParams.DiscardUnknown(m)
+
+func (*ComplexProtoParams) ProtoMessage() {}
+
+func (x *ComplexProtoParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ComplexProtoParams proto.InternalMessageInfo
+// Deprecated: Use ComplexProtoParams.ProtoReflect.Descriptor instead.
+func (*ComplexProtoParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{2}
+}
 
 type PayloadConfig struct {
-	// Types that are valid to be assigned to Payload:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Payload:
 	//	*PayloadConfig_BytebufParams
 	//	*PayloadConfig_SimpleParams
 	//	*PayloadConfig_ComplexParams
-	Payload              isPayloadConfig_Payload `protobuf_oneof:"payload"`
-	XXX_NoUnkeyedLiteral struct{}                `json:"-"`
-	XXX_unrecognized     []byte                  `json:"-"`
-	XXX_sizecache        int32                   `json:"-"`
+	Payload isPayloadConfig_Payload `protobuf_oneof:"payload"`
 }
 
-func (m *PayloadConfig) Reset()         { *m = PayloadConfig{} }
-func (m *PayloadConfig) String() string { return proto.CompactTextString(m) }
-func (*PayloadConfig) ProtoMessage()    {}
-func (*PayloadConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_69438956f5d73a41, []int{3}
+func (x *PayloadConfig) Reset() {
+	*x = PayloadConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *PayloadConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PayloadConfig.Unmarshal(m, b)
+func (x *PayloadConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *PayloadConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PayloadConfig.Marshal(b, m, deterministic)
+
+func (*PayloadConfig) ProtoMessage() {}
+
+func (x *PayloadConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *PayloadConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PayloadConfig.Merge(m, src)
+
+// Deprecated: Use PayloadConfig.ProtoReflect.Descriptor instead.
+func (*PayloadConfig) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{3}
 }
-func (m *PayloadConfig) XXX_Size() int {
-	return xxx_messageInfo_PayloadConfig.Size(m)
+
+func (m *PayloadConfig) GetPayload() isPayloadConfig_Payload {
+	if m != nil {
+		return m.Payload
+	}
+	return nil
+}
+
+func (x *PayloadConfig) GetBytebufParams() *ByteBufferParams {
+	if x, ok := x.GetPayload().(*PayloadConfig_BytebufParams); ok {
+		return x.BytebufParams
+	}
+	return nil
 }
-func (m *PayloadConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_PayloadConfig.DiscardUnknown(m)
+
+func (x *PayloadConfig) GetSimpleParams() *SimpleProtoParams {
+	if x, ok := x.GetPayload().(*PayloadConfig_SimpleParams); ok {
+		return x.SimpleParams
+	}
+	return nil
 }
 
-var xxx_messageInfo_PayloadConfig proto.InternalMessageInfo
+func (x *PayloadConfig) GetComplexParams() *ComplexProtoParams {
+	if x, ok := x.GetPayload().(*PayloadConfig_ComplexParams); ok {
+		return x.ComplexParams
+	}
+	return nil
+}
 
 type isPayloadConfig_Payload interface {
 	isPayloadConfig_Payload()
@@ -203,73 +281,151 @@ func (*PayloadConfig_SimpleParams) isPayloadConfig_Payload() {}
 
 func (*PayloadConfig_ComplexParams) isPayloadConfig_Payload() {}
 
-func (m *PayloadConfig) GetPayload() isPayloadConfig_Payload {
-	if m != nil {
-		return m.Payload
-	}
-	return nil
-}
+var File_benchmark_grpc_testing_payloads_proto protoreflect.FileDescriptor
+
+var file_benchmark_grpc_testing_payloads_proto_rawDesc = []byte{
+	0x0a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x4a, 0x0a, 0x10, 0x42, 0x79, 0x74, 0x65, 0x42, 0x75, 0x66,
+	0x66, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x65, 0x71,
+	0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x71,
+	0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f, 0x73, 0x69, 0x7a,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x53, 0x69, 0x7a,
+	0x65, 0x22, 0x4b, 0x0a, 0x11, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f,
+	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x5f, 0x73, 0x69,
+	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x71, 0x53, 0x69, 0x7a,
+	0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x14,
+	0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x0d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x47, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75,
+	0x66, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x79,
+	0x74, 0x65, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00,
+	0x52, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12,
+	0x46, 0x0a, 0x0d, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x49, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x6c,
+	0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43,
+	0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d,
+	0x73, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x61, 0x72, 0x61,
+	0x6d, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x2f, 0x5a,
+	0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f,
+	0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72,
+	0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_benchmark_grpc_testing_payloads_proto_rawDescOnce sync.Once
+	file_benchmark_grpc_testing_payloads_proto_rawDescData = file_benchmark_grpc_testing_payloads_proto_rawDesc
+)
 
-func (m *PayloadConfig) GetBytebufParams() *ByteBufferParams {
-	if x, ok := m.GetPayload().(*PayloadConfig_BytebufParams); ok {
-		return x.BytebufParams
+func file_benchmark_grpc_testing_payloads_proto_rawDescGZIP() []byte {
+	file_benchmark_grpc_testing_payloads_proto_rawDescOnce.Do(func() {
+		file_benchmark_grpc_testing_payloads_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_payloads_proto_rawDescData)
+	})
+	return file_benchmark_grpc_testing_payloads_proto_rawDescData
+}
+
+var file_benchmark_grpc_testing_payloads_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_benchmark_grpc_testing_payloads_proto_goTypes = []interface{}{
+	(*ByteBufferParams)(nil),   // 0: grpc.testing.ByteBufferParams
+	(*SimpleProtoParams)(nil),  // 1: grpc.testing.SimpleProtoParams
+	(*ComplexProtoParams)(nil), // 2: grpc.testing.ComplexProtoParams
+	(*PayloadConfig)(nil),      // 3: grpc.testing.PayloadConfig
+}
+var file_benchmark_grpc_testing_payloads_proto_depIdxs = []int32{
+	0, // 0: grpc.testing.PayloadConfig.bytebuf_params:type_name -> grpc.testing.ByteBufferParams
+	1, // 1: grpc.testing.PayloadConfig.simple_params:type_name -> grpc.testing.SimpleProtoParams
+	2, // 2: grpc.testing.PayloadConfig.complex_params:type_name -> grpc.testing.ComplexProtoParams
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_benchmark_grpc_testing_payloads_proto_init() }
+func file_benchmark_grpc_testing_payloads_proto_init() {
+	if File_benchmark_grpc_testing_payloads_proto != nil {
+		return
 	}
-	return nil
-}
-
-func (m *PayloadConfig) GetSimpleParams() *SimpleProtoParams {
-	if x, ok := m.GetPayload().(*PayloadConfig_SimpleParams); ok {
-		return x.SimpleParams
+	if !protoimpl.UnsafeEnabled {
+		file_benchmark_grpc_testing_payloads_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ByteBufferParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_payloads_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleProtoParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_payloads_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ComplexProtoParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_payloads_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PayloadConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
-	return nil
-}
-
-func (m *PayloadConfig) GetComplexParams() *ComplexProtoParams {
-	if x, ok := m.GetPayload().(*PayloadConfig_ComplexParams); ok {
-		return x.ComplexParams
-	}
-	return nil
-}
-
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*PayloadConfig) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
+	file_benchmark_grpc_testing_payloads_proto_msgTypes[3].OneofWrappers = []interface{}{
 		(*PayloadConfig_BytebufParams)(nil),
 		(*PayloadConfig_SimpleParams)(nil),
 		(*PayloadConfig_ComplexParams)(nil),
 	}
-}
-
-func init() {
-	proto.RegisterType((*ByteBufferParams)(nil), "grpc.testing.ByteBufferParams")
-	proto.RegisterType((*SimpleProtoParams)(nil), "grpc.testing.SimpleProtoParams")
-	proto.RegisterType((*ComplexProtoParams)(nil), "grpc.testing.ComplexProtoParams")
-	proto.RegisterType((*PayloadConfig)(nil), "grpc.testing.PayloadConfig")
-}
-
-func init() {
-	proto.RegisterFile("benchmark/grpc_testing/payloads.proto", fileDescriptor_69438956f5d73a41)
-}
-
-var fileDescriptor_69438956f5d73a41 = []byte{
-	// 289 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x91, 0xc1, 0x4b, 0xc3, 0x30,
-	0x18, 0xc5, 0xd7, 0x89, 0x6e, 0xfb, 0x5c, 0x87, 0x16, 0x0f, 0x8a, 0xa0, 0xa3, 0x20, 0x78, 0x31,
-	0x05, 0xfd, 0x0f, 0x3a, 0xd0, 0xa9, 0x97, 0xd2, 0xdd, 0xbc, 0x94, 0xb4, 0x7e, 0x8d, 0xc5, 0xb6,
-	0xc9, 0x92, 0x0e, 0xec, 0xfe, 0x70, 0xcf, 0x92, 0x34, 0x83, 0xcd, 0x79, 0xf3, 0x9a, 0x97, 0xf7,
-	0xfb, 0xde, 0xe3, 0xc1, 0x4d, 0x8a, 0x75, 0xf6, 0x51, 0x51, 0xf9, 0x19, 0x30, 0x29, 0xb2, 0xa4,
-	0x41, 0xd5, 0x14, 0x35, 0x0b, 0x04, 0x6d, 0x4b, 0x4e, 0xdf, 0x15, 0x11, 0x92, 0x37, 0xdc, 0x1b,
-	0x6b, 0x91, 0x58, 0xd1, 0x7f, 0x81, 0x93, 0xb0, 0x6d, 0x30, 0x5c, 0xe5, 0x39, 0xca, 0x88, 0x4a,
-	0x5a, 0x29, 0xef, 0x02, 0x86, 0x12, 0x97, 0x89, 0x2a, 0xd6, 0x78, 0xee, 0x4c, 0x9d, 0xdb, 0xc3,
-	0x78, 0x20, 0x71, 0xb9, 0x28, 0xd6, 0xe8, 0x5d, 0xc2, 0x48, 0xa2, 0x12, 0x9d, 0xd6, 0x37, 0xda,
-	0x50, 0x3f, 0x68, 0xd1, 0x7f, 0x85, 0xd3, 0x45, 0x51, 0x89, 0x12, 0x23, 0x7d, 0xe8, 0x9f, 0xb0,
-	0x33, 0xf0, 0x66, 0x5c, 0xc3, 0xbe, 0xb6, 0x68, 0xfe, 0xb7, 0x03, 0x6e, 0xd4, 0xf5, 0x99, 0xf1,
-	0x3a, 0x2f, 0x98, 0xf7, 0x04, 0x93, 0xb4, 0x6d, 0x30, 0x5d, 0xe5, 0x89, 0x30, 0x7f, 0xcc, 0x95,
-	0xe3, 0xfb, 0x2b, 0xb2, 0xdd, 0x93, 0xfc, 0x2e, 0x39, 0xef, 0xc5, 0xae, 0xf5, 0xd9, 0xa0, 0x8f,
-	0xe0, 0x2a, 0x93, 0x7e, 0xc3, 0xe9, 0x1b, 0xce, 0xf5, 0x2e, 0x67, 0xaf, 0xe0, 0xbc, 0x17, 0x8f,
-	0x3b, 0x9f, 0xe5, 0x3c, 0xc3, 0x24, 0xeb, 0x82, 0x6f, 0x40, 0x07, 0x06, 0x34, 0xdd, 0x05, 0xed,
-	0x97, 0xd3, 0x91, 0xac, 0xb3, 0x7b, 0x08, 0x47, 0x30, 0xb0, 0xe3, 0x85, 0xc1, 0xdb, 0x1d, 0xe3,
-	0x9c, 0x95, 0x48, 0x18, 0x2f, 0x69, 0xcd, 0x08, 0x97, 0xcc, 0xcc, 0x1c, 0xfc, 0xbd, 0x7a, 0x7a,
-	0x64, 0xd6, 0x7e, 0xf8, 0x09, 0x00, 0x00, 0xff, 0xff, 0x15, 0x79, 0xad, 0xbf, 0x16, 0x02, 0x00,
-	0x00,
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_benchmark_grpc_testing_payloads_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_benchmark_grpc_testing_payloads_proto_goTypes,
+		DependencyIndexes: file_benchmark_grpc_testing_payloads_proto_depIdxs,
+		MessageInfos:      file_benchmark_grpc_testing_payloads_proto_msgTypes,
+	}.Build()
+	File_benchmark_grpc_testing_payloads_proto = out.File
+	file_benchmark_grpc_testing_payloads_proto_rawDesc = nil
+	file_benchmark_grpc_testing_payloads_proto_goTypes = nil
+	file_benchmark_grpc_testing_payloads_proto_depIdxs = nil
 }
diff --git a/benchmark/grpc_testing/services.pb.go b/benchmark/grpc_testing/services.pb.go
index 4d0acd35d4fb..70f66ffca46f 100644
--- a/benchmark/grpc_testing/services.pb.go
+++ b/benchmark/grpc_testing/services.pb.go
@@ -1,49 +1,152 @@
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: benchmark/grpc_testing/services.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+var File_benchmark_grpc_testing_services_proto protoreflect.FileDescriptor
 
-func init() {
-	proto.RegisterFile("benchmark/grpc_testing/services.proto", fileDescriptor_e86b6b5d31c265e4)
+var file_benchmark_grpc_testing_services_proto_rawDesc = []byte{
+	0x0a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b,
+	0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, 0x65,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x24, 0x62, 0x65,
+	0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x32, 0x87, 0x02, 0x0a, 0x10, 0x42, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b,
+	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79,
+	0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x4e, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x6c, 0x6c,
+	0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12,
+	0x5b, 0x0a, 0x1a, 0x55, 0x6e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x64,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x32, 0x97, 0x02, 0x0a,
+	0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45,
+	0x0a, 0x09, 0x52, 0x75, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x45, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x09,
+	0x43, 0x6f, 0x72, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x34, 0x0a, 0x0a, 0x51, 0x75, 0x69, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x12,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x56, 0x6f,
+	0x69, 0x64, 0x1a, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var file_benchmark_grpc_testing_services_proto_goTypes = []interface{}{
+	(*SimpleRequest)(nil),  // 0: grpc.testing.SimpleRequest
+	(*ServerArgs)(nil),     // 1: grpc.testing.ServerArgs
+	(*ClientArgs)(nil),     // 2: grpc.testing.ClientArgs
+	(*CoreRequest)(nil),    // 3: grpc.testing.CoreRequest
+	(*Void)(nil),           // 4: grpc.testing.Void
+	(*SimpleResponse)(nil), // 5: grpc.testing.SimpleResponse
+	(*ServerStatus)(nil),   // 6: grpc.testing.ServerStatus
+	(*ClientStatus)(nil),   // 7: grpc.testing.ClientStatus
+	(*CoreResponse)(nil),   // 8: grpc.testing.CoreResponse
+}
+var file_benchmark_grpc_testing_services_proto_depIdxs = []int32{
+	0, // 0: grpc.testing.BenchmarkService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	0, // 1: grpc.testing.BenchmarkService.StreamingCall:input_type -> grpc.testing.SimpleRequest
+	0, // 2: grpc.testing.BenchmarkService.UnconstrainedStreamingCall:input_type -> grpc.testing.SimpleRequest
+	1, // 3: grpc.testing.WorkerService.RunServer:input_type -> grpc.testing.ServerArgs
+	2, // 4: grpc.testing.WorkerService.RunClient:input_type -> grpc.testing.ClientArgs
+	3, // 5: grpc.testing.WorkerService.CoreCount:input_type -> grpc.testing.CoreRequest
+	4, // 6: grpc.testing.WorkerService.QuitWorker:input_type -> grpc.testing.Void
+	5, // 7: grpc.testing.BenchmarkService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	5, // 8: grpc.testing.BenchmarkService.StreamingCall:output_type -> grpc.testing.SimpleResponse
+	5, // 9: grpc.testing.BenchmarkService.UnconstrainedStreamingCall:output_type -> grpc.testing.SimpleResponse
+	6, // 10: grpc.testing.WorkerService.RunServer:output_type -> grpc.testing.ServerStatus
+	7, // 11: grpc.testing.WorkerService.RunClient:output_type -> grpc.testing.ClientStatus
+	8, // 12: grpc.testing.WorkerService.CoreCount:output_type -> grpc.testing.CoreResponse
+	4, // 13: grpc.testing.WorkerService.QuitWorker:output_type -> grpc.testing.Void
+	7, // [7:14] is the sub-list for method output_type
+	0, // [0:7] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-var fileDescriptor_e86b6b5d31c265e4 = []byte{
-	// 309 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x92, 0xc1, 0x4a, 0x3b, 0x31,
-	0x10, 0xc6, 0x69, 0x0f, 0x7f, 0x68, 0xf8, 0x17, 0x24, 0x27, 0x8d, 0x1e, 0x15, 0xbc, 0xb8, 0x2b,
-	0xd5, 0x17, 0xb0, 0x8b, 0x1e, 0x05, 0xbb, 0x54, 0x41, 0x0f, 0x92, 0x6e, 0x87, 0x18, 0x9a, 0x9d,
-	0x59, 0x27, 0xb3, 0x82, 0x4f, 0xe0, 0x23, 0xf8, 0xba, 0xd2, 0xdd, 0x56, 0x6a, 0xd9, 0x9e, 0xf4,
-	0x98, 0xf9, 0xbe, 0xf9, 0x4d, 0x26, 0xf9, 0xd4, 0xc9, 0x0c, 0xb0, 0x78, 0x29, 0x2d, 0x2f, 0x52,
-	0xc7, 0x55, 0xf1, 0x2c, 0x10, 0xc5, 0xa3, 0x4b, 0x23, 0xf0, 0x9b, 0x2f, 0x20, 0x26, 0x15, 0x93,
-	0x90, 0xfe, 0xbf, 0x14, 0x93, 0x95, 0x68, 0x76, 0x35, 0x95, 0x10, 0xa3, 0x75, 0xeb, 0x26, 0x73,
-	0xbc, 0xc3, 0x56, 0x10, 0x0a, 0x53, 0x68, 0x5d, 0xa3, 0x8f, 0xbe, 0xda, 0x1b, 0xaf, 0x8d, 0x79,
-	0x3b, 0x56, 0xdf, 0xa8, 0xc1, 0x14, 0x2d, 0xbf, 0x67, 0x36, 0x04, 0x7d, 0x98, 0x6c, 0x4e, 0x4f,
-	0x72, 0x5f, 0x56, 0x01, 0x26, 0xf0, 0x5a, 0x43, 0x14, 0x73, 0xd4, 0x2d, 0xc6, 0x8a, 0x30, 0x82,
-	0xbe, 0x55, 0xc3, 0x5c, 0x18, 0x6c, 0xe9, 0xd1, 0xfd, 0x92, 0x75, 0xda, 0x3b, 0xef, 0xe9, 0x27,
-	0x65, 0xa6, 0x58, 0x10, 0x46, 0x61, 0xeb, 0x11, 0xe6, 0x7f, 0x09, 0x1f, 0x7d, 0xf6, 0xd5, 0xf0,
-	0x81, 0x78, 0x01, 0xbc, 0x7e, 0x86, 0x6b, 0x35, 0x98, 0xd4, 0xb8, 0x3c, 0x01, 0xeb, 0xfd, 0x2d,
-	0x40, 0x53, 0xbd, 0x62, 0x17, 0x8d, 0xe9, 0x52, 0x72, 0xb1, 0x52, 0xc7, 0xe6, 0xd6, 0x2d, 0x26,
-	0x0b, 0x1e, 0x50, 0xb6, 0x31, 0x6d, 0xb5, 0x0b, 0xd3, 0x2a, 0x1b, 0x98, 0xb1, 0x1a, 0x64, 0xc4,
-	0x90, 0x51, 0x8d, 0xa2, 0x0f, 0xb6, 0xcc, 0xc4, 0xdf, 0x9b, 0x9a, 0x2e, 0x69, 0xf5, 0x21, 0x97,
-	0x4a, 0xdd, 0xd5, 0x5e, 0xda, 0x35, 0xb5, 0xfe, 0xe9, 0xbc, 0x27, 0x3f, 0x37, 0x1d, 0xb5, 0x71,
-	0xfa, 0x78, 0xe6, 0x88, 0x5c, 0x80, 0xc4, 0x51, 0xb0, 0xe8, 0x12, 0x62, 0xd7, 0x64, 0x2a, 0xed,
-	0x8e, 0xd8, 0xec, 0x5f, 0x93, 0xad, 0x8b, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x77, 0x96, 0x9e,
-	0xdb, 0xdf, 0x02, 0x00, 0x00,
+func init() { file_benchmark_grpc_testing_services_proto_init() }
+func file_benchmark_grpc_testing_services_proto_init() {
+	if File_benchmark_grpc_testing_services_proto != nil {
+		return
+	}
+	file_benchmark_grpc_testing_messages_proto_init()
+	file_benchmark_grpc_testing_control_proto_init()
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_benchmark_grpc_testing_services_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   0,
+			NumExtensions: 0,
+			NumServices:   2,
+		},
+		GoTypes:           file_benchmark_grpc_testing_services_proto_goTypes,
+		DependencyIndexes: file_benchmark_grpc_testing_services_proto_depIdxs,
+	}.Build()
+	File_benchmark_grpc_testing_services_proto = out.File
+	file_benchmark_grpc_testing_services_proto_rawDesc = nil
+	file_benchmark_grpc_testing_services_proto_goTypes = nil
+	file_benchmark_grpc_testing_services_proto_depIdxs = nil
 }
diff --git a/benchmark/grpc_testing/stats.pb.go b/benchmark/grpc_testing/stats.pb.go
index ebe310c41dcc..9dcf37ac3abd 100644
--- a/benchmark/grpc_testing/stats.pb.go
+++ b/benchmark/grpc_testing/stats.pb.go
@@ -1,312 +1,473 @@
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: benchmark/grpc_testing/stats.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type ServerStats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// wall clock time change in seconds since last reset
 	TimeElapsed float64 `protobuf:"fixed64,1,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
 	// change in user time (in seconds) used by the server since last reset
 	TimeUser float64 `protobuf:"fixed64,2,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
 	// change in server time (in seconds) used by the server process and all
 	// threads since last reset
-	TimeSystem           float64  `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	TimeSystem float64 `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
 }
 
-func (m *ServerStats) Reset()         { *m = ServerStats{} }
-func (m *ServerStats) String() string { return proto.CompactTextString(m) }
-func (*ServerStats) ProtoMessage()    {}
-func (*ServerStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0658d5f374c86d6, []int{0}
+func (x *ServerStats) Reset() {
+	*x = ServerStats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerStats) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerStats.Unmarshal(m, b)
-}
-func (m *ServerStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerStats.Marshal(b, m, deterministic)
-}
-func (m *ServerStats) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerStats.Merge(m, src)
+func (x *ServerStats) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerStats) XXX_Size() int {
-	return xxx_messageInfo_ServerStats.Size(m)
-}
-func (m *ServerStats) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerStats.DiscardUnknown(m)
+
+func (*ServerStats) ProtoMessage() {}
+
+func (x *ServerStats) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerStats proto.InternalMessageInfo
+// Deprecated: Use ServerStats.ProtoReflect.Descriptor instead.
+func (*ServerStats) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *ServerStats) GetTimeElapsed() float64 {
-	if m != nil {
-		return m.TimeElapsed
+func (x *ServerStats) GetTimeElapsed() float64 {
+	if x != nil {
+		return x.TimeElapsed
 	}
 	return 0
 }
 
-func (m *ServerStats) GetTimeUser() float64 {
-	if m != nil {
-		return m.TimeUser
+func (x *ServerStats) GetTimeUser() float64 {
+	if x != nil {
+		return x.TimeUser
 	}
 	return 0
 }
 
-func (m *ServerStats) GetTimeSystem() float64 {
-	if m != nil {
-		return m.TimeSystem
+func (x *ServerStats) GetTimeSystem() float64 {
+	if x != nil {
+		return x.TimeSystem
 	}
 	return 0
 }
 
 // Histogram params based on grpc/support/histogram.c
 type HistogramParams struct {
-	Resolution           float64  `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"`
-	MaxPossible          float64  `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *HistogramParams) Reset()         { *m = HistogramParams{} }
-func (m *HistogramParams) String() string { return proto.CompactTextString(m) }
-func (*HistogramParams) ProtoMessage()    {}
-func (*HistogramParams) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0658d5f374c86d6, []int{1}
+	Resolution  float64 `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"`                      // first bucket is [0, 1 + resolution)
+	MaxPossible float64 `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"` // use enough buckets to allow this value
 }
 
-func (m *HistogramParams) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HistogramParams.Unmarshal(m, b)
-}
-func (m *HistogramParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HistogramParams.Marshal(b, m, deterministic)
-}
-func (m *HistogramParams) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HistogramParams.Merge(m, src)
+func (x *HistogramParams) Reset() {
+	*x = HistogramParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *HistogramParams) XXX_Size() int {
-	return xxx_messageInfo_HistogramParams.Size(m)
+
+func (x *HistogramParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HistogramParams) XXX_DiscardUnknown() {
-	xxx_messageInfo_HistogramParams.DiscardUnknown(m)
+
+func (*HistogramParams) ProtoMessage() {}
+
+func (x *HistogramParams) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HistogramParams proto.InternalMessageInfo
+// Deprecated: Use HistogramParams.ProtoReflect.Descriptor instead.
+func (*HistogramParams) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *HistogramParams) GetResolution() float64 {
-	if m != nil {
-		return m.Resolution
+func (x *HistogramParams) GetResolution() float64 {
+	if x != nil {
+		return x.Resolution
 	}
 	return 0
 }
 
-func (m *HistogramParams) GetMaxPossible() float64 {
-	if m != nil {
-		return m.MaxPossible
+func (x *HistogramParams) GetMaxPossible() float64 {
+	if x != nil {
+		return x.MaxPossible
 	}
 	return 0
 }
 
 // Histogram data based on grpc/support/histogram.c
 type HistogramData struct {
-	Bucket               []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"`
-	MinSeen              float64  `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"`
-	MaxSeen              float64  `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"`
-	Sum                  float64  `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"`
-	SumOfSquares         float64  `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"`
-	Count                float64  `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *HistogramData) Reset()         { *m = HistogramData{} }
-func (m *HistogramData) String() string { return proto.CompactTextString(m) }
-func (*HistogramData) ProtoMessage()    {}
-func (*HistogramData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0658d5f374c86d6, []int{2}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Bucket       []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"`
+	MinSeen      float64  `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"`
+	MaxSeen      float64  `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"`
+	Sum          float64  `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"`
+	SumOfSquares float64  `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"`
+	Count        float64  `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"`
+}
+
+func (x *HistogramData) Reset() {
+	*x = HistogramData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HistogramData) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HistogramData.Unmarshal(m, b)
-}
-func (m *HistogramData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HistogramData.Marshal(b, m, deterministic)
-}
-func (m *HistogramData) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HistogramData.Merge(m, src)
+func (x *HistogramData) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HistogramData) XXX_Size() int {
-	return xxx_messageInfo_HistogramData.Size(m)
-}
-func (m *HistogramData) XXX_DiscardUnknown() {
-	xxx_messageInfo_HistogramData.DiscardUnknown(m)
+
+func (*HistogramData) ProtoMessage() {}
+
+func (x *HistogramData) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HistogramData proto.InternalMessageInfo
+// Deprecated: Use HistogramData.ProtoReflect.Descriptor instead.
+func (*HistogramData) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *HistogramData) GetBucket() []uint32 {
-	if m != nil {
-		return m.Bucket
+func (x *HistogramData) GetBucket() []uint32 {
+	if x != nil {
+		return x.Bucket
 	}
 	return nil
 }
 
-func (m *HistogramData) GetMinSeen() float64 {
-	if m != nil {
-		return m.MinSeen
+func (x *HistogramData) GetMinSeen() float64 {
+	if x != nil {
+		return x.MinSeen
 	}
 	return 0
 }
 
-func (m *HistogramData) GetMaxSeen() float64 {
-	if m != nil {
-		return m.MaxSeen
+func (x *HistogramData) GetMaxSeen() float64 {
+	if x != nil {
+		return x.MaxSeen
 	}
 	return 0
 }
 
-func (m *HistogramData) GetSum() float64 {
-	if m != nil {
-		return m.Sum
+func (x *HistogramData) GetSum() float64 {
+	if x != nil {
+		return x.Sum
 	}
 	return 0
 }
 
-func (m *HistogramData) GetSumOfSquares() float64 {
-	if m != nil {
-		return m.SumOfSquares
+func (x *HistogramData) GetSumOfSquares() float64 {
+	if x != nil {
+		return x.SumOfSquares
 	}
 	return 0
 }
 
-func (m *HistogramData) GetCount() float64 {
-	if m != nil {
-		return m.Count
+func (x *HistogramData) GetCount() float64 {
+	if x != nil {
+		return x.Count
 	}
 	return 0
 }
 
 type ClientStats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Latency histogram. Data points are in nanoseconds.
 	Latencies *HistogramData `protobuf:"bytes,1,opt,name=latencies,proto3" json:"latencies,omitempty"`
 	// See ServerStats for details.
-	TimeElapsed          float64  `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
-	TimeUser             float64  `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
-	TimeSystem           float64  `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	TimeElapsed float64 `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
+	TimeUser    float64 `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
+	TimeSystem  float64 `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
 }
 
-func (m *ClientStats) Reset()         { *m = ClientStats{} }
-func (m *ClientStats) String() string { return proto.CompactTextString(m) }
-func (*ClientStats) ProtoMessage()    {}
-func (*ClientStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a0658d5f374c86d6, []int{3}
+func (x *ClientStats) Reset() {
+	*x = ClientStats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClientStats) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientStats.Unmarshal(m, b)
+func (x *ClientStats) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientStats.Marshal(b, m, deterministic)
-}
-func (m *ClientStats) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientStats.Merge(m, src)
-}
-func (m *ClientStats) XXX_Size() int {
-	return xxx_messageInfo_ClientStats.Size(m)
-}
-func (m *ClientStats) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientStats.DiscardUnknown(m)
+
+func (*ClientStats) ProtoMessage() {}
+
+func (x *ClientStats) ProtoReflect() protoreflect.Message {
+	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClientStats proto.InternalMessageInfo
+// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead.
+func (*ClientStats) Descriptor() ([]byte, []int) {
+	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *ClientStats) GetLatencies() *HistogramData {
-	if m != nil {
-		return m.Latencies
+func (x *ClientStats) GetLatencies() *HistogramData {
+	if x != nil {
+		return x.Latencies
 	}
 	return nil
 }
 
-func (m *ClientStats) GetTimeElapsed() float64 {
-	if m != nil {
-		return m.TimeElapsed
+func (x *ClientStats) GetTimeElapsed() float64 {
+	if x != nil {
+		return x.TimeElapsed
 	}
 	return 0
 }
 
-func (m *ClientStats) GetTimeUser() float64 {
-	if m != nil {
-		return m.TimeUser
+func (x *ClientStats) GetTimeUser() float64 {
+	if x != nil {
+		return x.TimeUser
 	}
 	return 0
 }
 
-func (m *ClientStats) GetTimeSystem() float64 {
-	if m != nil {
-		return m.TimeSystem
+func (x *ClientStats) GetTimeSystem() float64 {
+	if x != nil {
+		return x.TimeSystem
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterType((*ServerStats)(nil), "grpc.testing.ServerStats")
-	proto.RegisterType((*HistogramParams)(nil), "grpc.testing.HistogramParams")
-	proto.RegisterType((*HistogramData)(nil), "grpc.testing.HistogramData")
-	proto.RegisterType((*ClientStats)(nil), "grpc.testing.ClientStats")
-}
-
-func init() {
-	proto.RegisterFile("benchmark/grpc_testing/stats.proto", fileDescriptor_a0658d5f374c86d6)
-}
-
-var fileDescriptor_a0658d5f374c86d6 = []byte{
-	// 376 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xcf, 0x0b, 0xd3, 0x30,
-	0x14, 0xc7, 0xe9, 0xba, 0xcd, 0xed, 0x75, 0x53, 0x09, 0x22, 0x95, 0x81, 0xce, 0xe2, 0x61, 0x17,
-	0x5b, 0xd0, 0x93, 0x57, 0x7f, 0x80, 0x37, 0xc7, 0xaa, 0x17, 0x2f, 0x25, 0xad, 0x6f, 0x31, 0xac,
-	0x49, 0x6a, 0x5e, 0x22, 0xf5, 0x4f, 0x12, 0xff, 0x49, 0x69, 0x5a, 0x74, 0xfe, 0x40, 0x6f, 0xcd,
-	0xe7, 0xf3, 0x25, 0x69, 0xf2, 0x7d, 0x90, 0xd5, 0xa8, 0x9b, 0x8f, 0x8a, 0xdb, 0x4b, 0x21, 0x6c,
-	0xd7, 0x54, 0x0e, 0xc9, 0x49, 0x2d, 0x0a, 0x72, 0xdc, 0x51, 0xde, 0x59, 0xe3, 0x0c, 0xdb, 0x0c,
-	0x26, 0x9f, 0x4c, 0xa6, 0x21, 0x29, 0xd1, 0x7e, 0x46, 0x5b, 0x0e, 0x11, 0xf6, 0x10, 0x36, 0x4e,
-	0x2a, 0xac, 0xb0, 0xe5, 0x1d, 0xe1, 0x87, 0x34, 0xda, 0x47, 0x87, 0xe8, 0x94, 0x0c, 0xec, 0xd5,
-	0x88, 0xd8, 0x0e, 0xd6, 0x21, 0xe2, 0x09, 0x6d, 0x3a, 0x0b, 0x7e, 0x35, 0x80, 0x77, 0x84, 0x96,
-	0x3d, 0x80, 0x90, 0xad, 0xe8, 0x0b, 0x39, 0x54, 0x69, 0x1c, 0x34, 0x0c, 0xa8, 0x0c, 0x24, 0x7b,
-	0x0b, 0xb7, 0x5e, 0x4b, 0x72, 0x46, 0x58, 0xae, 0x8e, 0xdc, 0x72, 0x45, 0xec, 0x3e, 0x80, 0x45,
-	0x32, 0xad, 0x77, 0xd2, 0xe8, 0xe9, 0xc4, 0x2b, 0x32, 0xfc, 0x93, 0xe2, 0x7d, 0xd5, 0x19, 0x22,
-	0x59, 0xb7, 0x38, 0x9d, 0x99, 0x28, 0xde, 0x1f, 0x27, 0x94, 0x7d, 0x8b, 0x60, 0xfb, 0x63, 0xdb,
-	0x97, 0xdc, 0x71, 0x76, 0x17, 0x96, 0xb5, 0x6f, 0x2e, 0xe8, 0xd2, 0x68, 0x1f, 0x1f, 0xb6, 0xa7,
-	0x69, 0xc5, 0xee, 0xc1, 0x4a, 0x49, 0x5d, 0x11, 0xa2, 0x9e, 0x36, 0xba, 0xa1, 0xa4, 0x2e, 0x11,
-	0x75, 0x50, 0xbc, 0x1f, 0x55, 0x3c, 0x29, 0xde, 0x07, 0x75, 0x1b, 0x62, 0xf2, 0x2a, 0x9d, 0x07,
-	0x3a, 0x7c, 0xb2, 0x47, 0x70, 0x93, 0xbc, 0xaa, 0xcc, 0xb9, 0xa2, 0x4f, 0x9e, 0x5b, 0xa4, 0x74,
-	0x11, 0xe4, 0x86, 0xbc, 0x7a, 0x73, 0x2e, 0x47, 0xc6, 0xee, 0xc0, 0xa2, 0x31, 0x5e, 0xbb, 0x74,
-	0x19, 0xe4, 0xb8, 0xc8, 0xbe, 0x46, 0x90, 0xbc, 0x68, 0x25, 0x6a, 0x37, 0x3e, 0xfa, 0x33, 0x58,
-	0xb7, 0xdc, 0xa1, 0x6e, 0x24, 0x52, 0xb8, 0x7f, 0xf2, 0x64, 0x97, 0x5f, 0xb7, 0x94, 0xff, 0x72,
-	0xb7, 0xd3, 0xcf, 0xf4, 0x1f, 0x7d, 0xcd, 0xfe, 0xd3, 0x57, 0xfc, 0xef, 0xbe, 0xe6, 0xbf, 0xf7,
-	0xf5, 0xbc, 0x78, 0xff, 0x58, 0x18, 0x23, 0x5a, 0xcc, 0x85, 0x69, 0xb9, 0x16, 0xb9, 0xb1, 0x22,
-	0xcc, 0x56, 0xf1, 0xf7, 0x51, 0xab, 0x97, 0x61, 0xca, 0x9e, 0x7e, 0x0f, 0x00, 0x00, 0xff, 0xff,
-	0xcc, 0xda, 0x99, 0x65, 0x8b, 0x02, 0x00, 0x00,
+var File_benchmark_grpc_testing_stats_proto protoreflect.FileDescriptor
+
+var file_benchmark_grpc_testing_stats_proto_rawDesc = []byte{
+	0x0a, 0x22, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x22, 0x6e, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65,
+	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61,
+	0x70, 0x73, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65,
+	0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65,
+	0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74,
+	0x65, 0x6d, 0x22, 0x54, 0x0a, 0x0f, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x50,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74,
+	0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c,
+	0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x73,
+	0x73, 0x69, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6d, 0x61, 0x78,
+	0x50, 0x6f, 0x73, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x22, 0xab, 0x01, 0x0a, 0x0d, 0x48, 0x69, 0x73,
+	0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75,
+	0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b,
+	0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x6d, 0x69, 0x6e, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x19, 0x0a,
+	0x08, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52,
+	0x07, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x75,
+	0x6d, 0x5f, 0x6f, 0x66, 0x5f, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x01, 0x52, 0x0c, 0x73, 0x75, 0x6d, 0x4f, 0x66, 0x53, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73,
+	0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52,
+	0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xa9, 0x01, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63,
+	0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72,
+	0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65,
+	0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65,
+	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61,
+	0x70, 0x73, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65,
+	0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65,
+	0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74,
+	0x65, 0x6d, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c,
+	0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e,
+	0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_benchmark_grpc_testing_stats_proto_rawDescOnce sync.Once
+	file_benchmark_grpc_testing_stats_proto_rawDescData = file_benchmark_grpc_testing_stats_proto_rawDesc
+)
+
+func file_benchmark_grpc_testing_stats_proto_rawDescGZIP() []byte {
+	file_benchmark_grpc_testing_stats_proto_rawDescOnce.Do(func() {
+		file_benchmark_grpc_testing_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_stats_proto_rawDescData)
+	})
+	return file_benchmark_grpc_testing_stats_proto_rawDescData
+}
+
+var file_benchmark_grpc_testing_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_benchmark_grpc_testing_stats_proto_goTypes = []interface{}{
+	(*ServerStats)(nil),     // 0: grpc.testing.ServerStats
+	(*HistogramParams)(nil), // 1: grpc.testing.HistogramParams
+	(*HistogramData)(nil),   // 2: grpc.testing.HistogramData
+	(*ClientStats)(nil),     // 3: grpc.testing.ClientStats
+}
+var file_benchmark_grpc_testing_stats_proto_depIdxs = []int32{
+	2, // 0: grpc.testing.ClientStats.latencies:type_name -> grpc.testing.HistogramData
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_benchmark_grpc_testing_stats_proto_init() }
+func file_benchmark_grpc_testing_stats_proto_init() {
+	if File_benchmark_grpc_testing_stats_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_benchmark_grpc_testing_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerStats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HistogramParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HistogramData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_benchmark_grpc_testing_stats_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientStats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_benchmark_grpc_testing_stats_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_benchmark_grpc_testing_stats_proto_goTypes,
+		DependencyIndexes: file_benchmark_grpc_testing_stats_proto_depIdxs,
+		MessageInfos:      file_benchmark_grpc_testing_stats_proto_msgTypes,
+	}.Build()
+	File_benchmark_grpc_testing_stats_proto = out.File
+	file_benchmark_grpc_testing_stats_proto_rawDesc = nil
+	file_benchmark_grpc_testing_stats_proto_goTypes = nil
+	file_benchmark_grpc_testing_stats_proto_depIdxs = nil
 }
diff --git a/binarylog/grpc_binarylog_v1/binarylog.pb.go b/binarylog/grpc_binarylog_v1/binarylog.pb.go
index f826ec76984d..da0472967615 100644
--- a/binarylog/grpc_binarylog_v1/binarylog.pb.go
+++ b/binarylog/grpc_binarylog_v1/binarylog.pb.go
@@ -1,26 +1,49 @@
+// Copyright 2018 The gRPC Authors
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/binlog/v1/binarylog.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/binlog/v1/binarylog.proto
 
 package grpc_binarylog_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Enumerates the type of event
 // Note the terminology is different from the RPC semantics
@@ -56,34 +79,55 @@ const (
 	GrpcLogEntry_EVENT_TYPE_CANCEL GrpcLogEntry_EventType = 7
 )
 
-var GrpcLogEntry_EventType_name = map[int32]string{
-	0: "EVENT_TYPE_UNKNOWN",
-	1: "EVENT_TYPE_CLIENT_HEADER",
-	2: "EVENT_TYPE_SERVER_HEADER",
-	3: "EVENT_TYPE_CLIENT_MESSAGE",
-	4: "EVENT_TYPE_SERVER_MESSAGE",
-	5: "EVENT_TYPE_CLIENT_HALF_CLOSE",
-	6: "EVENT_TYPE_SERVER_TRAILER",
-	7: "EVENT_TYPE_CANCEL",
-}
+// Enum value maps for GrpcLogEntry_EventType.
+var (
+	GrpcLogEntry_EventType_name = map[int32]string{
+		0: "EVENT_TYPE_UNKNOWN",
+		1: "EVENT_TYPE_CLIENT_HEADER",
+		2: "EVENT_TYPE_SERVER_HEADER",
+		3: "EVENT_TYPE_CLIENT_MESSAGE",
+		4: "EVENT_TYPE_SERVER_MESSAGE",
+		5: "EVENT_TYPE_CLIENT_HALF_CLOSE",
+		6: "EVENT_TYPE_SERVER_TRAILER",
+		7: "EVENT_TYPE_CANCEL",
+	}
+	GrpcLogEntry_EventType_value = map[string]int32{
+		"EVENT_TYPE_UNKNOWN":           0,
+		"EVENT_TYPE_CLIENT_HEADER":     1,
+		"EVENT_TYPE_SERVER_HEADER":     2,
+		"EVENT_TYPE_CLIENT_MESSAGE":    3,
+		"EVENT_TYPE_SERVER_MESSAGE":    4,
+		"EVENT_TYPE_CLIENT_HALF_CLOSE": 5,
+		"EVENT_TYPE_SERVER_TRAILER":    6,
+		"EVENT_TYPE_CANCEL":            7,
+	}
+)
 
-var GrpcLogEntry_EventType_value = map[string]int32{
-	"EVENT_TYPE_UNKNOWN":           0,
-	"EVENT_TYPE_CLIENT_HEADER":     1,
-	"EVENT_TYPE_SERVER_HEADER":     2,
-	"EVENT_TYPE_CLIENT_MESSAGE":    3,
-	"EVENT_TYPE_SERVER_MESSAGE":    4,
-	"EVENT_TYPE_CLIENT_HALF_CLOSE": 5,
-	"EVENT_TYPE_SERVER_TRAILER":    6,
-	"EVENT_TYPE_CANCEL":            7,
+func (x GrpcLogEntry_EventType) Enum() *GrpcLogEntry_EventType {
+	p := new(GrpcLogEntry_EventType)
+	*p = x
+	return p
 }
 
 func (x GrpcLogEntry_EventType) String() string {
-	return proto.EnumName(GrpcLogEntry_EventType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (GrpcLogEntry_EventType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_binlog_v1_binarylog_proto_enumTypes[0].Descriptor()
+}
+
+func (GrpcLogEntry_EventType) Type() protoreflect.EnumType {
+	return &file_grpc_binlog_v1_binarylog_proto_enumTypes[0]
+}
+
+func (x GrpcLogEntry_EventType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
+// Deprecated: Use GrpcLogEntry_EventType.Descriptor instead.
 func (GrpcLogEntry_EventType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{0, 0}
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0, 0}
 }
 
 // Enumerates the entity that generates the log entry
@@ -95,24 +139,45 @@ const (
 	GrpcLogEntry_LOGGER_SERVER  GrpcLogEntry_Logger = 2
 )
 
-var GrpcLogEntry_Logger_name = map[int32]string{
-	0: "LOGGER_UNKNOWN",
-	1: "LOGGER_CLIENT",
-	2: "LOGGER_SERVER",
-}
+// Enum value maps for GrpcLogEntry_Logger.
+var (
+	GrpcLogEntry_Logger_name = map[int32]string{
+		0: "LOGGER_UNKNOWN",
+		1: "LOGGER_CLIENT",
+		2: "LOGGER_SERVER",
+	}
+	GrpcLogEntry_Logger_value = map[string]int32{
+		"LOGGER_UNKNOWN": 0,
+		"LOGGER_CLIENT":  1,
+		"LOGGER_SERVER":  2,
+	}
+)
 
-var GrpcLogEntry_Logger_value = map[string]int32{
-	"LOGGER_UNKNOWN": 0,
-	"LOGGER_CLIENT":  1,
-	"LOGGER_SERVER":  2,
+func (x GrpcLogEntry_Logger) Enum() *GrpcLogEntry_Logger {
+	p := new(GrpcLogEntry_Logger)
+	*p = x
+	return p
 }
 
 func (x GrpcLogEntry_Logger) String() string {
-	return proto.EnumName(GrpcLogEntry_Logger_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
+func (GrpcLogEntry_Logger) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_binlog_v1_binarylog_proto_enumTypes[1].Descriptor()
+}
+
+func (GrpcLogEntry_Logger) Type() protoreflect.EnumType {
+	return &file_grpc_binlog_v1_binarylog_proto_enumTypes[1]
+}
+
+func (x GrpcLogEntry_Logger) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use GrpcLogEntry_Logger.Descriptor instead.
 func (GrpcLogEntry_Logger) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{0, 1}
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0, 1}
 }
 
 type Address_Type int32
@@ -128,30 +193,55 @@ const (
 	Address_TYPE_UNIX Address_Type = 3
 )
 
-var Address_Type_name = map[int32]string{
-	0: "TYPE_UNKNOWN",
-	1: "TYPE_IPV4",
-	2: "TYPE_IPV6",
-	3: "TYPE_UNIX",
-}
+// Enum value maps for Address_Type.
+var (
+	Address_Type_name = map[int32]string{
+		0: "TYPE_UNKNOWN",
+		1: "TYPE_IPV4",
+		2: "TYPE_IPV6",
+		3: "TYPE_UNIX",
+	}
+	Address_Type_value = map[string]int32{
+		"TYPE_UNKNOWN": 0,
+		"TYPE_IPV4":    1,
+		"TYPE_IPV6":    2,
+		"TYPE_UNIX":    3,
+	}
+)
 
-var Address_Type_value = map[string]int32{
-	"TYPE_UNKNOWN": 0,
-	"TYPE_IPV4":    1,
-	"TYPE_IPV6":    2,
-	"TYPE_UNIX":    3,
+func (x Address_Type) Enum() *Address_Type {
+	p := new(Address_Type)
+	*p = x
+	return p
 }
 
 func (x Address_Type) String() string {
-	return proto.EnumName(Address_Type_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (Address_Type) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_binlog_v1_binarylog_proto_enumTypes[2].Descriptor()
+}
+
+func (Address_Type) Type() protoreflect.EnumType {
+	return &file_grpc_binlog_v1_binarylog_proto_enumTypes[2]
 }
 
+func (x Address_Type) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use Address_Type.Descriptor instead.
 func (Address_Type) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{7, 0}
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{7, 0}
 }
 
 // Log entry we store in binary logs
 type GrpcLogEntry struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The timestamp of the binary log message
 	Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	// Uniquely identifies a call. The value must not be 0 in order to disambiguate
@@ -166,11 +256,11 @@ type GrpcLogEntry struct {
 	// durability or ordering is not guaranteed.
 	SequenceIdWithinCall uint64                 `protobuf:"varint,3,opt,name=sequence_id_within_call,json=sequenceIdWithinCall,proto3" json:"sequence_id_within_call,omitempty"`
 	Type                 GrpcLogEntry_EventType `protobuf:"varint,4,opt,name=type,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_EventType" json:"type,omitempty"`
-	Logger               GrpcLogEntry_Logger    `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"`
+	Logger               GrpcLogEntry_Logger    `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"` // One of the above Logger enum
 	// The logger uses one of the following fields to record the payload,
 	// according to the type of the log entry.
 	//
-	// Types that are valid to be assigned to Payload:
+	// Types that are assignable to Payload:
 	//	*GrpcLogEntry_ClientHeader
 	//	*GrpcLogEntry_ServerHeader
 	//	*GrpcLogEntry_Message
@@ -183,100 +273,76 @@ type GrpcLogEntry struct {
 	// EVENT_TYPE_SERVER_HEADER normally or EVENT_TYPE_SERVER_TRAILER in
 	// the case of trailers-only. On server side, peer is always
 	// logged on EVENT_TYPE_CLIENT_HEADER.
-	Peer                 *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Peer *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"`
 }
 
-func (m *GrpcLogEntry) Reset()         { *m = GrpcLogEntry{} }
-func (m *GrpcLogEntry) String() string { return proto.CompactTextString(m) }
-func (*GrpcLogEntry) ProtoMessage()    {}
-func (*GrpcLogEntry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{0}
+func (x *GrpcLogEntry) Reset() {
+	*x = GrpcLogEntry{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *GrpcLogEntry) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GrpcLogEntry.Unmarshal(m, b)
-}
-func (m *GrpcLogEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GrpcLogEntry.Marshal(b, m, deterministic)
-}
-func (m *GrpcLogEntry) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GrpcLogEntry.Merge(m, src)
+func (x *GrpcLogEntry) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GrpcLogEntry) XXX_Size() int {
-	return xxx_messageInfo_GrpcLogEntry.Size(m)
-}
-func (m *GrpcLogEntry) XXX_DiscardUnknown() {
-	xxx_messageInfo_GrpcLogEntry.DiscardUnknown(m)
+
+func (*GrpcLogEntry) ProtoMessage() {}
+
+func (x *GrpcLogEntry) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_GrpcLogEntry proto.InternalMessageInfo
+// Deprecated: Use GrpcLogEntry.ProtoReflect.Descriptor instead.
+func (*GrpcLogEntry) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *GrpcLogEntry) GetTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.Timestamp
+func (x *GrpcLogEntry) GetTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.Timestamp
 	}
 	return nil
 }
 
-func (m *GrpcLogEntry) GetCallId() uint64 {
-	if m != nil {
-		return m.CallId
+func (x *GrpcLogEntry) GetCallId() uint64 {
+	if x != nil {
+		return x.CallId
 	}
 	return 0
 }
 
-func (m *GrpcLogEntry) GetSequenceIdWithinCall() uint64 {
-	if m != nil {
-		return m.SequenceIdWithinCall
+func (x *GrpcLogEntry) GetSequenceIdWithinCall() uint64 {
+	if x != nil {
+		return x.SequenceIdWithinCall
 	}
 	return 0
 }
 
-func (m *GrpcLogEntry) GetType() GrpcLogEntry_EventType {
-	if m != nil {
-		return m.Type
+func (x *GrpcLogEntry) GetType() GrpcLogEntry_EventType {
+	if x != nil {
+		return x.Type
 	}
 	return GrpcLogEntry_EVENT_TYPE_UNKNOWN
 }
 
-func (m *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger {
-	if m != nil {
-		return m.Logger
+func (x *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger {
+	if x != nil {
+		return x.Logger
 	}
 	return GrpcLogEntry_LOGGER_UNKNOWN
 }
 
-type isGrpcLogEntry_Payload interface {
-	isGrpcLogEntry_Payload()
-}
-
-type GrpcLogEntry_ClientHeader struct {
-	ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"`
-}
-
-type GrpcLogEntry_ServerHeader struct {
-	ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"`
-}
-
-type GrpcLogEntry_Message struct {
-	Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"`
-}
-
-type GrpcLogEntry_Trailer struct {
-	Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"`
-}
-
-func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {}
-
-func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {}
-
-func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {}
-
-func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {}
-
 func (m *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload {
 	if m != nil {
 		return m.Payload
@@ -284,59 +350,82 @@ func (m *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload {
 	return nil
 }
 
-func (m *GrpcLogEntry) GetClientHeader() *ClientHeader {
-	if x, ok := m.GetPayload().(*GrpcLogEntry_ClientHeader); ok {
+func (x *GrpcLogEntry) GetClientHeader() *ClientHeader {
+	if x, ok := x.GetPayload().(*GrpcLogEntry_ClientHeader); ok {
 		return x.ClientHeader
 	}
 	return nil
 }
 
-func (m *GrpcLogEntry) GetServerHeader() *ServerHeader {
-	if x, ok := m.GetPayload().(*GrpcLogEntry_ServerHeader); ok {
+func (x *GrpcLogEntry) GetServerHeader() *ServerHeader {
+	if x, ok := x.GetPayload().(*GrpcLogEntry_ServerHeader); ok {
 		return x.ServerHeader
 	}
 	return nil
 }
 
-func (m *GrpcLogEntry) GetMessage() *Message {
-	if x, ok := m.GetPayload().(*GrpcLogEntry_Message); ok {
+func (x *GrpcLogEntry) GetMessage() *Message {
+	if x, ok := x.GetPayload().(*GrpcLogEntry_Message); ok {
 		return x.Message
 	}
 	return nil
 }
 
-func (m *GrpcLogEntry) GetTrailer() *Trailer {
-	if x, ok := m.GetPayload().(*GrpcLogEntry_Trailer); ok {
+func (x *GrpcLogEntry) GetTrailer() *Trailer {
+	if x, ok := x.GetPayload().(*GrpcLogEntry_Trailer); ok {
 		return x.Trailer
 	}
 	return nil
 }
 
-func (m *GrpcLogEntry) GetPayloadTruncated() bool {
-	if m != nil {
-		return m.PayloadTruncated
+func (x *GrpcLogEntry) GetPayloadTruncated() bool {
+	if x != nil {
+		return x.PayloadTruncated
 	}
 	return false
 }
 
-func (m *GrpcLogEntry) GetPeer() *Address {
-	if m != nil {
-		return m.Peer
+func (x *GrpcLogEntry) GetPeer() *Address {
+	if x != nil {
+		return x.Peer
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*GrpcLogEntry) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*GrpcLogEntry_ClientHeader)(nil),
-		(*GrpcLogEntry_ServerHeader)(nil),
-		(*GrpcLogEntry_Message)(nil),
-		(*GrpcLogEntry_Trailer)(nil),
-	}
+type isGrpcLogEntry_Payload interface {
+	isGrpcLogEntry_Payload()
 }
 
+type GrpcLogEntry_ClientHeader struct {
+	ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"`
+}
+
+type GrpcLogEntry_ServerHeader struct {
+	ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"`
+}
+
+type GrpcLogEntry_Message struct {
+	// Used by EVENT_TYPE_CLIENT_MESSAGE, EVENT_TYPE_SERVER_MESSAGE
+	Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"`
+}
+
+type GrpcLogEntry_Trailer struct {
+	Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"`
+}
+
+func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {}
+
+func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {}
+
+func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {}
+
+func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {}
+
 type ClientHeader struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// This contains only the metadata from the application.
 	Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
 	// The name of the RPC method, which looks something like:
@@ -350,106 +439,122 @@ type ClientHeader struct {
 	// <host> or <host>:<port> .
 	Authority string `protobuf:"bytes,3,opt,name=authority,proto3" json:"authority,omitempty"`
 	// the RPC timeout
-	Timeout              *duration.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
-	XXX_unrecognized     []byte             `json:"-"`
-	XXX_sizecache        int32              `json:"-"`
+	Timeout *duration.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"`
 }
 
-func (m *ClientHeader) Reset()         { *m = ClientHeader{} }
-func (m *ClientHeader) String() string { return proto.CompactTextString(m) }
-func (*ClientHeader) ProtoMessage()    {}
-func (*ClientHeader) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{1}
+func (x *ClientHeader) Reset() {
+	*x = ClientHeader{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ClientHeader) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ClientHeader.Unmarshal(m, b)
-}
-func (m *ClientHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ClientHeader.Marshal(b, m, deterministic)
-}
-func (m *ClientHeader) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ClientHeader.Merge(m, src)
-}
-func (m *ClientHeader) XXX_Size() int {
-	return xxx_messageInfo_ClientHeader.Size(m)
+func (x *ClientHeader) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ClientHeader) XXX_DiscardUnknown() {
-	xxx_messageInfo_ClientHeader.DiscardUnknown(m)
+
+func (*ClientHeader) ProtoMessage() {}
+
+func (x *ClientHeader) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ClientHeader proto.InternalMessageInfo
+// Deprecated: Use ClientHeader.ProtoReflect.Descriptor instead.
+func (*ClientHeader) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *ClientHeader) GetMetadata() *Metadata {
-	if m != nil {
-		return m.Metadata
+func (x *ClientHeader) GetMetadata() *Metadata {
+	if x != nil {
+		return x.Metadata
 	}
 	return nil
 }
 
-func (m *ClientHeader) GetMethodName() string {
-	if m != nil {
-		return m.MethodName
+func (x *ClientHeader) GetMethodName() string {
+	if x != nil {
+		return x.MethodName
 	}
 	return ""
 }
 
-func (m *ClientHeader) GetAuthority() string {
-	if m != nil {
-		return m.Authority
+func (x *ClientHeader) GetAuthority() string {
+	if x != nil {
+		return x.Authority
 	}
 	return ""
 }
 
-func (m *ClientHeader) GetTimeout() *duration.Duration {
-	if m != nil {
-		return m.Timeout
+func (x *ClientHeader) GetTimeout() *duration.Duration {
+	if x != nil {
+		return x.Timeout
 	}
 	return nil
 }
 
 type ServerHeader struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// This contains only the metadata from the application.
-	Metadata             *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}  `json:"-"`
-	XXX_unrecognized     []byte    `json:"-"`
-	XXX_sizecache        int32     `json:"-"`
+	Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
 }
 
-func (m *ServerHeader) Reset()         { *m = ServerHeader{} }
-func (m *ServerHeader) String() string { return proto.CompactTextString(m) }
-func (*ServerHeader) ProtoMessage()    {}
-func (*ServerHeader) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{2}
+func (x *ServerHeader) Reset() {
+	*x = ServerHeader{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerHeader) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerHeader.Unmarshal(m, b)
-}
-func (m *ServerHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerHeader.Marshal(b, m, deterministic)
+func (x *ServerHeader) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerHeader) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerHeader.Merge(m, src)
-}
-func (m *ServerHeader) XXX_Size() int {
-	return xxx_messageInfo_ServerHeader.Size(m)
-}
-func (m *ServerHeader) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerHeader.DiscardUnknown(m)
+
+func (*ServerHeader) ProtoMessage() {}
+
+func (x *ServerHeader) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerHeader proto.InternalMessageInfo
+// Deprecated: Use ServerHeader.ProtoReflect.Descriptor instead.
+func (*ServerHeader) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *ServerHeader) GetMetadata() *Metadata {
-	if m != nil {
-		return m.Metadata
+func (x *ServerHeader) GetMetadata() *Metadata {
+	if x != nil {
+		return x.Metadata
 	}
 	return nil
 }
 
 type Trailer struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// This contains only the metadata from the application.
 	Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
 	// The gRPC status code.
@@ -459,112 +564,124 @@ type Trailer struct {
 	StatusMessage string `protobuf:"bytes,3,opt,name=status_message,json=statusMessage,proto3" json:"status_message,omitempty"`
 	// The value of the 'grpc-status-details-bin' metadata key. If
 	// present, this is always an encoded 'google.rpc.Status' message.
-	StatusDetails        []byte   `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	StatusDetails []byte `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"`
 }
 
-func (m *Trailer) Reset()         { *m = Trailer{} }
-func (m *Trailer) String() string { return proto.CompactTextString(m) }
-func (*Trailer) ProtoMessage()    {}
-func (*Trailer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{3}
+func (x *Trailer) Reset() {
+	*x = Trailer{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Trailer) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Trailer.Unmarshal(m, b)
-}
-func (m *Trailer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Trailer.Marshal(b, m, deterministic)
-}
-func (m *Trailer) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Trailer.Merge(m, src)
-}
-func (m *Trailer) XXX_Size() int {
-	return xxx_messageInfo_Trailer.Size(m)
+func (x *Trailer) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Trailer) XXX_DiscardUnknown() {
-	xxx_messageInfo_Trailer.DiscardUnknown(m)
+
+func (*Trailer) ProtoMessage() {}
+
+func (x *Trailer) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Trailer proto.InternalMessageInfo
+// Deprecated: Use Trailer.ProtoReflect.Descriptor instead.
+func (*Trailer) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *Trailer) GetMetadata() *Metadata {
-	if m != nil {
-		return m.Metadata
+func (x *Trailer) GetMetadata() *Metadata {
+	if x != nil {
+		return x.Metadata
 	}
 	return nil
 }
 
-func (m *Trailer) GetStatusCode() uint32 {
-	if m != nil {
-		return m.StatusCode
+func (x *Trailer) GetStatusCode() uint32 {
+	if x != nil {
+		return x.StatusCode
 	}
 	return 0
 }
 
-func (m *Trailer) GetStatusMessage() string {
-	if m != nil {
-		return m.StatusMessage
+func (x *Trailer) GetStatusMessage() string {
+	if x != nil {
+		return x.StatusMessage
 	}
 	return ""
 }
 
-func (m *Trailer) GetStatusDetails() []byte {
-	if m != nil {
-		return m.StatusDetails
+func (x *Trailer) GetStatusDetails() []byte {
+	if x != nil {
+		return x.StatusDetails
 	}
 	return nil
 }
 
 // Message payload, used by CLIENT_MESSAGE and SERVER_MESSAGE
 type Message struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Length of the message. It may not be the same as the length of the
 	// data field, as the logging payload can be truncated or omitted.
 	Length uint32 `protobuf:"varint,1,opt,name=length,proto3" json:"length,omitempty"`
 	// May be truncated or omitted.
-	Data                 []byte   `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
 }
 
-func (m *Message) Reset()         { *m = Message{} }
-func (m *Message) String() string { return proto.CompactTextString(m) }
-func (*Message) ProtoMessage()    {}
-func (*Message) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{4}
+func (x *Message) Reset() {
+	*x = Message{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Message) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Message.Unmarshal(m, b)
-}
-func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Message.Marshal(b, m, deterministic)
+func (x *Message) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Message) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Message.Merge(m, src)
-}
-func (m *Message) XXX_Size() int {
-	return xxx_messageInfo_Message.Size(m)
-}
-func (m *Message) XXX_DiscardUnknown() {
-	xxx_messageInfo_Message.DiscardUnknown(m)
+
+func (*Message) ProtoMessage() {}
+
+func (x *Message) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Message proto.InternalMessageInfo
+// Deprecated: Use Message.ProtoReflect.Descriptor instead.
+func (*Message) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *Message) GetLength() uint32 {
-	if m != nil {
-		return m.Length
+func (x *Message) GetLength() uint32 {
+	if x != nil {
+		return x.Length
 	}
 	return 0
 }
 
-func (m *Message) GetData() []byte {
-	if m != nil {
-		return m.Data
+func (x *Message) GetData() []byte {
+	if x != nil {
+		return x.Data
 	}
 	return nil
 }
@@ -591,222 +708,480 @@ func (m *Message) GetData() []byte {
 // header is just a normal metadata key.
 // The pair will not count towards the size limit.
 type Metadata struct {
-	Entry                []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}         `json:"-"`
-	XXX_unrecognized     []byte           `json:"-"`
-	XXX_sizecache        int32            `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *Metadata) Reset()         { *m = Metadata{} }
-func (m *Metadata) String() string { return proto.CompactTextString(m) }
-func (*Metadata) ProtoMessage()    {}
-func (*Metadata) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{5}
+	Entry []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"`
 }
 
-func (m *Metadata) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Metadata.Unmarshal(m, b)
-}
-func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Metadata.Marshal(b, m, deterministic)
-}
-func (m *Metadata) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Metadata.Merge(m, src)
+func (x *Metadata) Reset() {
+	*x = Metadata{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Metadata) XXX_Size() int {
-	return xxx_messageInfo_Metadata.Size(m)
+
+func (x *Metadata) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Metadata) XXX_DiscardUnknown() {
-	xxx_messageInfo_Metadata.DiscardUnknown(m)
+
+func (*Metadata) ProtoMessage() {}
+
+func (x *Metadata) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Metadata proto.InternalMessageInfo
+// Deprecated: Use Metadata.ProtoReflect.Descriptor instead.
+func (*Metadata) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *Metadata) GetEntry() []*MetadataEntry {
-	if m != nil {
-		return m.Entry
+func (x *Metadata) GetEntry() []*MetadataEntry {
+	if x != nil {
+		return x.Entry
 	}
 	return nil
 }
 
 // A metadata key value pair
 type MetadataEntry struct {
-	Key                  string   `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Value                []byte   `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *MetadataEntry) Reset()         { *m = MetadataEntry{} }
-func (m *MetadataEntry) String() string { return proto.CompactTextString(m) }
-func (*MetadataEntry) ProtoMessage()    {}
-func (*MetadataEntry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{6}
+	Key   string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
-func (m *MetadataEntry) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MetadataEntry.Unmarshal(m, b)
-}
-func (m *MetadataEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MetadataEntry.Marshal(b, m, deterministic)
-}
-func (m *MetadataEntry) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MetadataEntry.Merge(m, src)
+func (x *MetadataEntry) Reset() {
+	*x = MetadataEntry{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *MetadataEntry) XXX_Size() int {
-	return xxx_messageInfo_MetadataEntry.Size(m)
+
+func (x *MetadataEntry) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *MetadataEntry) XXX_DiscardUnknown() {
-	xxx_messageInfo_MetadataEntry.DiscardUnknown(m)
+
+func (*MetadataEntry) ProtoMessage() {}
+
+func (x *MetadataEntry) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_MetadataEntry proto.InternalMessageInfo
+// Deprecated: Use MetadataEntry.ProtoReflect.Descriptor instead.
+func (*MetadataEntry) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *MetadataEntry) GetKey() string {
-	if m != nil {
-		return m.Key
+func (x *MetadataEntry) GetKey() string {
+	if x != nil {
+		return x.Key
 	}
 	return ""
 }
 
-func (m *MetadataEntry) GetValue() []byte {
-	if m != nil {
-		return m.Value
+func (x *MetadataEntry) GetValue() []byte {
+	if x != nil {
+		return x.Value
 	}
 	return nil
 }
 
 // Address information
 type Address struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	Type    Address_Type `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.binarylog.v1.Address_Type" json:"type,omitempty"`
 	Address string       `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"`
 	// only for TYPE_IPV4 and TYPE_IPV6
-	IpPort               uint32   `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	IpPort uint32 `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"`
 }
 
-func (m *Address) Reset()         { *m = Address{} }
-func (m *Address) String() string { return proto.CompactTextString(m) }
-func (*Address) ProtoMessage()    {}
-func (*Address) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b7972e58de45083a, []int{7}
+func (x *Address) Reset() {
+	*x = Address{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Address) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Address.Unmarshal(m, b)
+func (x *Address) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Address.Marshal(b, m, deterministic)
-}
-func (m *Address) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Address.Merge(m, src)
-}
-func (m *Address) XXX_Size() int {
-	return xxx_messageInfo_Address.Size(m)
-}
-func (m *Address) XXX_DiscardUnknown() {
-	xxx_messageInfo_Address.DiscardUnknown(m)
+
+func (*Address) ProtoMessage() {}
+
+func (x *Address) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Address proto.InternalMessageInfo
+// Deprecated: Use Address.ProtoReflect.Descriptor instead.
+func (*Address) Descriptor() ([]byte, []int) {
+	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *Address) GetType() Address_Type {
-	if m != nil {
-		return m.Type
+func (x *Address) GetType() Address_Type {
+	if x != nil {
+		return x.Type
 	}
 	return Address_TYPE_UNKNOWN
 }
 
-func (m *Address) GetAddress() string {
-	if m != nil {
-		return m.Address
+func (x *Address) GetAddress() string {
+	if x != nil {
+		return x.Address
 	}
 	return ""
 }
 
-func (m *Address) GetIpPort() uint32 {
-	if m != nil {
-		return m.IpPort
+func (x *Address) GetIpPort() uint32 {
+	if x != nil {
+		return x.IpPort
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_EventType", GrpcLogEntry_EventType_name, GrpcLogEntry_EventType_value)
-	proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_Logger", GrpcLogEntry_Logger_name, GrpcLogEntry_Logger_value)
-	proto.RegisterEnum("grpc.binarylog.v1.Address_Type", Address_Type_name, Address_Type_value)
-	proto.RegisterType((*GrpcLogEntry)(nil), "grpc.binarylog.v1.GrpcLogEntry")
-	proto.RegisterType((*ClientHeader)(nil), "grpc.binarylog.v1.ClientHeader")
-	proto.RegisterType((*ServerHeader)(nil), "grpc.binarylog.v1.ServerHeader")
-	proto.RegisterType((*Trailer)(nil), "grpc.binarylog.v1.Trailer")
-	proto.RegisterType((*Message)(nil), "grpc.binarylog.v1.Message")
-	proto.RegisterType((*Metadata)(nil), "grpc.binarylog.v1.Metadata")
-	proto.RegisterType((*MetadataEntry)(nil), "grpc.binarylog.v1.MetadataEntry")
-	proto.RegisterType((*Address)(nil), "grpc.binarylog.v1.Address")
-}
-
-func init() { proto.RegisterFile("grpc/binlog/v1/binarylog.proto", fileDescriptor_b7972e58de45083a) }
-
-var fileDescriptor_b7972e58de45083a = []byte{
-	// 904 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x51, 0x6f, 0xe3, 0x44,
-	0x10, 0xae, 0xdb, 0x34, 0x6e, 0x26, 0x49, 0xe5, 0xae, 0xca, 0x9d, 0xaf, 0x94, 0x6b, 0x64, 0x09,
-	0x14, 0x84, 0xe4, 0xa8, 0x29, 0xd7, 0xe3, 0x05, 0xa4, 0x24, 0xf5, 0xa5, 0x11, 0xb9, 0x34, 0xda,
-	0xe4, 0x7a, 0x80, 0x90, 0xac, 0x6d, 0xbc, 0x38, 0x16, 0x8e, 0xd7, 0xac, 0x37, 0x41, 0xf9, 0x59,
-	0xbc, 0x21, 0xdd, 0xef, 0xe2, 0x1d, 0x79, 0xd7, 0x4e, 0x4d, 0xd3, 0x82, 0xc4, 0xbd, 0xed, 0x7c,
-	0xf3, 0xcd, 0x37, 0xbb, 0xe3, 0x99, 0x31, 0xbc, 0xf4, 0x79, 0x3c, 0x6b, 0xdd, 0x05, 0x51, 0xc8,
-	0xfc, 0xd6, 0xea, 0x3c, 0x3d, 0x11, 0xbe, 0x0e, 0x99, 0x6f, 0xc7, 0x9c, 0x09, 0x86, 0x8e, 0x52,
-	0xbf, 0x7d, 0x8f, 0xae, 0xce, 0x4f, 0x5e, 0xfa, 0x8c, 0xf9, 0x21, 0x6d, 0x49, 0xc2, 0xdd, 0xf2,
-	0x97, 0x96, 0xb7, 0xe4, 0x44, 0x04, 0x2c, 0x52, 0x21, 0x27, 0x67, 0x0f, 0xfd, 0x22, 0x58, 0xd0,
-	0x44, 0x90, 0x45, 0xac, 0x08, 0xd6, 0x07, 0x1d, 0x6a, 0x7d, 0x1e, 0xcf, 0x86, 0xcc, 0x77, 0x22,
-	0xc1, 0xd7, 0xe8, 0x1b, 0xa8, 0x6c, 0x38, 0xa6, 0xd6, 0xd0, 0x9a, 0xd5, 0xf6, 0x89, 0xad, 0x54,
-	0xec, 0x5c, 0xc5, 0x9e, 0xe6, 0x0c, 0x7c, 0x4f, 0x46, 0xcf, 0x41, 0x9f, 0x91, 0x30, 0x74, 0x03,
-	0xcf, 0xdc, 0x6d, 0x68, 0xcd, 0x12, 0x2e, 0xa7, 0xe6, 0xc0, 0x43, 0xaf, 0xe0, 0x79, 0x42, 0x7f,
-	0x5b, 0xd2, 0x68, 0x46, 0xdd, 0xc0, 0x73, 0x7f, 0x0f, 0xc4, 0x3c, 0x88, 0xdc, 0xd4, 0x69, 0xee,
-	0x49, 0xe2, 0x71, 0xee, 0x1e, 0x78, 0xef, 0xa5, 0xb3, 0x47, 0xc2, 0x10, 0x7d, 0x0b, 0x25, 0xb1,
-	0x8e, 0xa9, 0x59, 0x6a, 0x68, 0xcd, 0xc3, 0xf6, 0x97, 0xf6, 0xd6, 0xeb, 0xed, 0xe2, 0xc5, 0x6d,
-	0x67, 0x45, 0x23, 0x31, 0x5d, 0xc7, 0x14, 0xcb, 0x30, 0xf4, 0x1d, 0x94, 0x43, 0xe6, 0xfb, 0x94,
-	0x9b, 0xfb, 0x52, 0xe0, 0x8b, 0xff, 0x12, 0x18, 0x4a, 0x36, 0xce, 0xa2, 0xd0, 0x1b, 0xa8, 0xcf,
-	0xc2, 0x80, 0x46, 0xc2, 0x9d, 0x53, 0xe2, 0x51, 0x6e, 0x96, 0x65, 0x31, 0xce, 0x1e, 0x91, 0xe9,
-	0x49, 0xde, 0xb5, 0xa4, 0x5d, 0xef, 0xe0, 0xda, 0xac, 0x60, 0xa7, 0x3a, 0x09, 0xe5, 0x2b, 0xca,
-	0x73, 0x1d, 0xfd, 0x49, 0x9d, 0x89, 0xe4, 0xdd, 0xeb, 0x24, 0x05, 0x1b, 0x5d, 0x82, 0xbe, 0xa0,
-	0x49, 0x42, 0x7c, 0x6a, 0x1e, 0xe4, 0x9f, 0x65, 0x4b, 0xe1, 0xad, 0x62, 0x5c, 0xef, 0xe0, 0x9c,
-	0x9c, 0xc6, 0x09, 0x4e, 0x82, 0x90, 0x72, 0xb3, 0xf2, 0x64, 0xdc, 0x54, 0x31, 0xd2, 0xb8, 0x8c,
-	0x8c, 0xbe, 0x82, 0xa3, 0x98, 0xac, 0x43, 0x46, 0x3c, 0x57, 0xf0, 0x65, 0x34, 0x23, 0x82, 0x7a,
-	0x26, 0x34, 0xb4, 0xe6, 0x01, 0x36, 0x32, 0xc7, 0x34, 0xc7, 0x91, 0x0d, 0xa5, 0x98, 0x52, 0x6e,
-	0x56, 0x9f, 0xcc, 0xd0, 0xf1, 0x3c, 0x4e, 0x93, 0x04, 0x4b, 0x9e, 0xf5, 0x97, 0x06, 0x95, 0xcd,
-	0x07, 0x43, 0xcf, 0x00, 0x39, 0xb7, 0xce, 0x68, 0xea, 0x4e, 0x7f, 0x1c, 0x3b, 0xee, 0xbb, 0xd1,
-	0xf7, 0xa3, 0x9b, 0xf7, 0x23, 0x63, 0x07, 0x9d, 0x82, 0x59, 0xc0, 0x7b, 0xc3, 0x41, 0x7a, 0xbe,
-	0x76, 0x3a, 0x57, 0x0e, 0x36, 0xb4, 0x07, 0xde, 0x89, 0x83, 0x6f, 0x1d, 0x9c, 0x7b, 0x77, 0xd1,
-	0x67, 0xf0, 0x62, 0x3b, 0xf6, 0xad, 0x33, 0x99, 0x74, 0xfa, 0x8e, 0xb1, 0xf7, 0xc0, 0x9d, 0x05,
-	0xe7, 0xee, 0x12, 0x6a, 0xc0, 0xe9, 0x23, 0x99, 0x3b, 0xc3, 0x37, 0x6e, 0x6f, 0x78, 0x33, 0x71,
-	0x8c, 0xfd, 0xc7, 0x05, 0xa6, 0xb8, 0x33, 0x18, 0x3a, 0xd8, 0x28, 0xa3, 0x4f, 0xe0, 0xa8, 0x28,
-	0xd0, 0x19, 0xf5, 0x9c, 0xa1, 0xa1, 0x5b, 0x5d, 0x28, 0xab, 0x36, 0x43, 0x08, 0x0e, 0x87, 0x37,
-	0xfd, 0xbe, 0x83, 0x0b, 0xef, 0x3d, 0x82, 0x7a, 0x86, 0xa9, 0x8c, 0x86, 0x56, 0x80, 0x54, 0x0a,
-	0x63, 0xb7, 0x5b, 0x01, 0x3d, 0xab, 0xbf, 0xf5, 0x41, 0x83, 0x5a, 0xb1, 0xf9, 0xd0, 0x6b, 0x38,
-	0x58, 0x50, 0x41, 0x3c, 0x22, 0x48, 0x36, 0xbc, 0x9f, 0x3e, 0xda, 0x25, 0x8a, 0x82, 0x37, 0x64,
-	0x74, 0x06, 0xd5, 0x05, 0x15, 0x73, 0xe6, 0xb9, 0x11, 0x59, 0x50, 0x39, 0xc0, 0x15, 0x0c, 0x0a,
-	0x1a, 0x91, 0x05, 0x45, 0xa7, 0x50, 0x21, 0x4b, 0x31, 0x67, 0x3c, 0x10, 0x6b, 0x39, 0xb6, 0x15,
-	0x7c, 0x0f, 0xa0, 0x0b, 0xd0, 0xd3, 0x45, 0xc0, 0x96, 0x42, 0x8e, 0x6b, 0xb5, 0xfd, 0x62, 0x6b,
-	0x67, 0x5c, 0x65, 0x9b, 0x09, 0xe7, 0x4c, 0xab, 0x0f, 0xb5, 0x62, 0xc7, 0xff, 0xef, 0xcb, 0x5b,
-	0x7f, 0x68, 0xa0, 0x67, 0x1d, 0xfc, 0x51, 0x15, 0x48, 0x04, 0x11, 0xcb, 0xc4, 0x9d, 0x31, 0x4f,
-	0x55, 0xa0, 0x8e, 0x41, 0x41, 0x3d, 0xe6, 0x51, 0xf4, 0x39, 0x1c, 0x66, 0x84, 0x7c, 0x0e, 0x55,
-	0x19, 0xea, 0x0a, 0xcd, 0x46, 0xaf, 0x40, 0xf3, 0xa8, 0x20, 0x41, 0x98, 0xc8, 0x8a, 0xd4, 0x72,
-	0xda, 0x95, 0x02, 0xad, 0x57, 0xa0, 0xe7, 0x11, 0xcf, 0xa0, 0x1c, 0xd2, 0xc8, 0x17, 0x73, 0x79,
-	0xe1, 0x3a, 0xce, 0x2c, 0x84, 0xa0, 0x24, 0x9f, 0xb1, 0x2b, 0xe3, 0xe5, 0xd9, 0xea, 0xc2, 0x41,
-	0x7e, 0x77, 0x74, 0x09, 0xfb, 0x34, 0xdd, 0x5c, 0xa6, 0xd6, 0xd8, 0x6b, 0x56, 0xdb, 0x8d, 0x7f,
-	0x79, 0xa7, 0xdc, 0x70, 0x58, 0xd1, 0xad, 0xd7, 0x50, 0xff, 0x07, 0x8e, 0x0c, 0xd8, 0xfb, 0x95,
-	0xae, 0x65, 0xf6, 0x0a, 0x4e, 0x8f, 0xe8, 0x18, 0xf6, 0x57, 0x24, 0x5c, 0xd2, 0x2c, 0xb7, 0x32,
-	0xac, 0x3f, 0x35, 0xd0, 0xb3, 0x39, 0x46, 0x17, 0xd9, 0x76, 0xd6, 0xe4, 0x72, 0x3d, 0x7b, 0x7a,
-	0xe2, 0xed, 0xc2, 0x4e, 0x36, 0x41, 0x27, 0x0a, 0xcd, 0x3a, 0x2c, 0x37, 0xd3, 0x9f, 0x47, 0x10,
-	0xbb, 0x31, 0xe3, 0x42, 0x56, 0xb5, 0x8e, 0xcb, 0x41, 0x3c, 0x66, 0x5c, 0x58, 0x0e, 0x94, 0xe4,
-	0x8e, 0x30, 0xa0, 0xf6, 0x60, 0x3b, 0xd4, 0xa1, 0x22, 0x91, 0xc1, 0xf8, 0xf6, 0x6b, 0x43, 0x2b,
-	0x9a, 0x97, 0xc6, 0xee, 0xc6, 0x7c, 0x37, 0x1a, 0xfc, 0x60, 0xec, 0x75, 0x7f, 0x86, 0xe3, 0x80,
-	0x6d, 0x5f, 0xb2, 0x7b, 0xd8, 0x95, 0xd6, 0x90, 0xf9, 0xe3, 0xb4, 0x51, 0xc7, 0xda, 0x4f, 0xed,
-	0xac, 0x71, 0x7d, 0x16, 0x92, 0xc8, 0xb7, 0x19, 0xf7, 0x5b, 0xf9, 0x7f, 0x59, 0x85, 0x49, 0xd3,
-	0xdd, 0x98, 0xee, 0xea, 0xfc, 0xae, 0x2c, 0xbb, 0xfc, 0xe2, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff,
-	0x10, 0x93, 0x68, 0x41, 0xc2, 0x07, 0x00, 0x00,
+var File_grpc_binlog_v1_binarylog_proto protoreflect.FileDescriptor
+
+var file_grpc_binlog_v1_binarylog_proto_rawDesc = []byte{
+	0x0a, 0x1e, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x2f, 0x76, 0x31,
+	0x2f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x11, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67,
+	0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbb, 0x07, 0x0a, 0x0c, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x6f, 0x67,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61,
+	0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
+	0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12,
+	0x17, 0x0a, 0x07, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04,
+	0x52, 0x06, 0x63, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x35, 0x0a, 0x17, 0x73, 0x65, 0x71, 0x75,
+	0x65, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x5f, 0x63,
+	0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x14, 0x73, 0x65, 0x71, 0x75, 0x65,
+	0x6e, 0x63, 0x65, 0x49, 0x64, 0x57, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x43, 0x61, 0x6c, 0x6c, 0x12,
+	0x3d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x29, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76,
+	0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45,
+	0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e,
+	0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e,
+	0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e,
+	0x4c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x12, 0x46,
+	0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18,
+	0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e,
+	0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x46, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76,
+	0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00,
+	0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x36,
+	0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67,
+	0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, 0x07, 0x6d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x65,
+	0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62,
+	0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x69,
+	0x6c, 0x65, 0x72, 0x48, 0x00, 0x52, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x2b,
+	0x0a, 0x11, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61,
+	0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x70, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x04, 0x70,
+	0x65, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x64,
+	0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x04, 0x70, 0x65, 0x65, 0x72, 0x22, 0xf5, 0x01, 0x0a, 0x09,
+	0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x56, 0x45,
+	0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10,
+	0x00, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12,
+	0x1c, 0x0a, 0x18, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45,
+	0x52, 0x56, 0x45, 0x52, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x02, 0x12, 0x1d, 0x0a,
+	0x19, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x49, 0x45,
+	0x4e, 0x54, 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x10, 0x03, 0x12, 0x1d, 0x0a, 0x19,
+	0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45,
+	0x52, 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x10, 0x04, 0x12, 0x20, 0x0a, 0x1c, 0x45,
+	0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54,
+	0x5f, 0x48, 0x41, 0x4c, 0x46, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x05, 0x12, 0x1d, 0x0a,
+	0x19, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x52, 0x56,
+	0x45, 0x52, 0x5f, 0x54, 0x52, 0x41, 0x49, 0x4c, 0x45, 0x52, 0x10, 0x06, 0x12, 0x15, 0x0a, 0x11,
+	0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45,
+	0x4c, 0x10, 0x07, 0x22, 0x42, 0x0a, 0x06, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x12, 0x12, 0x0a,
+	0x0e, 0x4c, 0x4f, 0x47, 0x47, 0x45, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10,
+	0x00, 0x12, 0x11, 0x0a, 0x0d, 0x4c, 0x4f, 0x47, 0x47, 0x45, 0x52, 0x5f, 0x43, 0x4c, 0x49, 0x45,
+	0x4e, 0x54, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x4c, 0x4f, 0x47, 0x47, 0x45, 0x52, 0x5f, 0x53,
+	0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x22, 0xbb, 0x01, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x48, 0x65, 0x61,
+	0x64, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e,
+	0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61,
+	0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1f, 0x0a, 0x0b,
+	0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a,
+	0x09, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x33, 0x0a, 0x07, 0x74,
+	0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44,
+	0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74,
+	0x22, 0x47, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72,
+	0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79,
+	0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52,
+	0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xb1, 0x01, 0x0a, 0x07, 0x54, 0x72,
+	0x61, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
+	0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62,
+	0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61,
+	0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1f,
+	0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x12,
+	0x25, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73,
+	0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d,
+	0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x35, 0x0a,
+	0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67,
+	0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68,
+	0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04,
+	0x64, 0x61, 0x74, 0x61, 0x22, 0x42, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x12, 0x36, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67,
+	0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x37, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61,
+	0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x22, 0xb8, 0x01, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x33, 0x0a,
+	0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e,
+	0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
+	0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x17, 0x0a, 0x07,
+	0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69,
+	0x70, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x45, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a,
+	0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12,
+	0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x56, 0x34, 0x10, 0x01, 0x12, 0x0d,
+	0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x56, 0x36, 0x10, 0x02, 0x12, 0x0d, 0x0a,
+	0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x58, 0x10, 0x03, 0x42, 0x5c, 0x0a, 0x14,
+	0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f,
+	0x67, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x4c, 0x6f, 0x67, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x32, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67,
+	0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62,
+	0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x69,
+	0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x33,
+}
+
+var (
+	file_grpc_binlog_v1_binarylog_proto_rawDescOnce sync.Once
+	file_grpc_binlog_v1_binarylog_proto_rawDescData = file_grpc_binlog_v1_binarylog_proto_rawDesc
+)
+
+func file_grpc_binlog_v1_binarylog_proto_rawDescGZIP() []byte {
+	file_grpc_binlog_v1_binarylog_proto_rawDescOnce.Do(func() {
+		file_grpc_binlog_v1_binarylog_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_binlog_v1_binarylog_proto_rawDescData)
+	})
+	return file_grpc_binlog_v1_binarylog_proto_rawDescData
+}
+
+var file_grpc_binlog_v1_binarylog_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_grpc_binlog_v1_binarylog_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
+var file_grpc_binlog_v1_binarylog_proto_goTypes = []interface{}{
+	(GrpcLogEntry_EventType)(0), // 0: grpc.binarylog.v1.GrpcLogEntry.EventType
+	(GrpcLogEntry_Logger)(0),    // 1: grpc.binarylog.v1.GrpcLogEntry.Logger
+	(Address_Type)(0),           // 2: grpc.binarylog.v1.Address.Type
+	(*GrpcLogEntry)(nil),        // 3: grpc.binarylog.v1.GrpcLogEntry
+	(*ClientHeader)(nil),        // 4: grpc.binarylog.v1.ClientHeader
+	(*ServerHeader)(nil),        // 5: grpc.binarylog.v1.ServerHeader
+	(*Trailer)(nil),             // 6: grpc.binarylog.v1.Trailer
+	(*Message)(nil),             // 7: grpc.binarylog.v1.Message
+	(*Metadata)(nil),            // 8: grpc.binarylog.v1.Metadata
+	(*MetadataEntry)(nil),       // 9: grpc.binarylog.v1.MetadataEntry
+	(*Address)(nil),             // 10: grpc.binarylog.v1.Address
+	(*timestamp.Timestamp)(nil), // 11: google.protobuf.Timestamp
+	(*duration.Duration)(nil),   // 12: google.protobuf.Duration
+}
+var file_grpc_binlog_v1_binarylog_proto_depIdxs = []int32{
+	11, // 0: grpc.binarylog.v1.GrpcLogEntry.timestamp:type_name -> google.protobuf.Timestamp
+	0,  // 1: grpc.binarylog.v1.GrpcLogEntry.type:type_name -> grpc.binarylog.v1.GrpcLogEntry.EventType
+	1,  // 2: grpc.binarylog.v1.GrpcLogEntry.logger:type_name -> grpc.binarylog.v1.GrpcLogEntry.Logger
+	4,  // 3: grpc.binarylog.v1.GrpcLogEntry.client_header:type_name -> grpc.binarylog.v1.ClientHeader
+	5,  // 4: grpc.binarylog.v1.GrpcLogEntry.server_header:type_name -> grpc.binarylog.v1.ServerHeader
+	7,  // 5: grpc.binarylog.v1.GrpcLogEntry.message:type_name -> grpc.binarylog.v1.Message
+	6,  // 6: grpc.binarylog.v1.GrpcLogEntry.trailer:type_name -> grpc.binarylog.v1.Trailer
+	10, // 7: grpc.binarylog.v1.GrpcLogEntry.peer:type_name -> grpc.binarylog.v1.Address
+	8,  // 8: grpc.binarylog.v1.ClientHeader.metadata:type_name -> grpc.binarylog.v1.Metadata
+	12, // 9: grpc.binarylog.v1.ClientHeader.timeout:type_name -> google.protobuf.Duration
+	8,  // 10: grpc.binarylog.v1.ServerHeader.metadata:type_name -> grpc.binarylog.v1.Metadata
+	8,  // 11: grpc.binarylog.v1.Trailer.metadata:type_name -> grpc.binarylog.v1.Metadata
+	9,  // 12: grpc.binarylog.v1.Metadata.entry:type_name -> grpc.binarylog.v1.MetadataEntry
+	2,  // 13: grpc.binarylog.v1.Address.type:type_name -> grpc.binarylog.v1.Address.Type
+	14, // [14:14] is the sub-list for method output_type
+	14, // [14:14] is the sub-list for method input_type
+	14, // [14:14] is the sub-list for extension type_name
+	14, // [14:14] is the sub-list for extension extendee
+	0,  // [0:14] is the sub-list for field type_name
+}
+
+func init() { file_grpc_binlog_v1_binarylog_proto_init() }
+func file_grpc_binlog_v1_binarylog_proto_init() {
+	if File_grpc_binlog_v1_binarylog_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GrpcLogEntry); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientHeader); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerHeader); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Trailer); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Message); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Metadata); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MetadataEntry); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_binlog_v1_binarylog_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Address); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_binlog_v1_binarylog_proto_msgTypes[0].OneofWrappers = []interface{}{
+		(*GrpcLogEntry_ClientHeader)(nil),
+		(*GrpcLogEntry_ServerHeader)(nil),
+		(*GrpcLogEntry_Message)(nil),
+		(*GrpcLogEntry_Trailer)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_binlog_v1_binarylog_proto_rawDesc,
+			NumEnums:      3,
+			NumMessages:   8,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_binlog_v1_binarylog_proto_goTypes,
+		DependencyIndexes: file_grpc_binlog_v1_binarylog_proto_depIdxs,
+		EnumInfos:         file_grpc_binlog_v1_binarylog_proto_enumTypes,
+		MessageInfos:      file_grpc_binlog_v1_binarylog_proto_msgTypes,
+	}.Build()
+	File_grpc_binlog_v1_binarylog_proto = out.File
+	file_grpc_binlog_v1_binarylog_proto_rawDesc = nil
+	file_grpc_binlog_v1_binarylog_proto_goTypes = nil
+	file_grpc_binlog_v1_binarylog_proto_depIdxs = nil
 }
diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go
index 9c364e1ebd9c..ca98e688b0b8 100644
--- a/channelz/grpc_channelz_v1/channelz.pb.go
+++ b/channelz/grpc_channelz_v1/channelz.pb.go
@@ -1,28 +1,54 @@
+// Copyright 2018 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file defines an interface for exporting monitoring information
+// out of gRPC servers.  See the full design at
+// https://github.com/grpc/proposal/blob/master/A14-channelz.md
+//
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/channelz/v1/channelz.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/channelz/v1/channelz.proto
 
 package grpc_channelz_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	any "github.com/golang/protobuf/ptypes/any"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	wrappers "github.com/golang/protobuf/ptypes/wrappers"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type ChannelConnectivityState_State int32
 
@@ -35,30 +61,51 @@ const (
 	ChannelConnectivityState_SHUTDOWN          ChannelConnectivityState_State = 5
 )
 
-var ChannelConnectivityState_State_name = map[int32]string{
-	0: "UNKNOWN",
-	1: "IDLE",
-	2: "CONNECTING",
-	3: "READY",
-	4: "TRANSIENT_FAILURE",
-	5: "SHUTDOWN",
-}
+// Enum value maps for ChannelConnectivityState_State.
+var (
+	ChannelConnectivityState_State_name = map[int32]string{
+		0: "UNKNOWN",
+		1: "IDLE",
+		2: "CONNECTING",
+		3: "READY",
+		4: "TRANSIENT_FAILURE",
+		5: "SHUTDOWN",
+	}
+	ChannelConnectivityState_State_value = map[string]int32{
+		"UNKNOWN":           0,
+		"IDLE":              1,
+		"CONNECTING":        2,
+		"READY":             3,
+		"TRANSIENT_FAILURE": 4,
+		"SHUTDOWN":          5,
+	}
+)
 
-var ChannelConnectivityState_State_value = map[string]int32{
-	"UNKNOWN":           0,
-	"IDLE":              1,
-	"CONNECTING":        2,
-	"READY":             3,
-	"TRANSIENT_FAILURE": 4,
-	"SHUTDOWN":          5,
+func (x ChannelConnectivityState_State) Enum() *ChannelConnectivityState_State {
+	p := new(ChannelConnectivityState_State)
+	*p = x
+	return p
 }
 
 func (x ChannelConnectivityState_State) String() string {
-	return proto.EnumName(ChannelConnectivityState_State_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ChannelConnectivityState_State) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_channelz_v1_channelz_proto_enumTypes[0].Descriptor()
 }
 
+func (ChannelConnectivityState_State) Type() protoreflect.EnumType {
+	return &file_grpc_channelz_v1_channelz_proto_enumTypes[0]
+}
+
+func (x ChannelConnectivityState_State) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ChannelConnectivityState_State.Descriptor instead.
 func (ChannelConnectivityState_State) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{2, 0}
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{2, 0}
 }
 
 // The supported severity levels of trace events.
@@ -71,34 +118,59 @@ const (
 	ChannelTraceEvent_CT_ERROR   ChannelTraceEvent_Severity = 3
 )
 
-var ChannelTraceEvent_Severity_name = map[int32]string{
-	0: "CT_UNKNOWN",
-	1: "CT_INFO",
-	2: "CT_WARNING",
-	3: "CT_ERROR",
-}
+// Enum value maps for ChannelTraceEvent_Severity.
+var (
+	ChannelTraceEvent_Severity_name = map[int32]string{
+		0: "CT_UNKNOWN",
+		1: "CT_INFO",
+		2: "CT_WARNING",
+		3: "CT_ERROR",
+	}
+	ChannelTraceEvent_Severity_value = map[string]int32{
+		"CT_UNKNOWN": 0,
+		"CT_INFO":    1,
+		"CT_WARNING": 2,
+		"CT_ERROR":   3,
+	}
+)
 
-var ChannelTraceEvent_Severity_value = map[string]int32{
-	"CT_UNKNOWN": 0,
-	"CT_INFO":    1,
-	"CT_WARNING": 2,
-	"CT_ERROR":   3,
+func (x ChannelTraceEvent_Severity) Enum() *ChannelTraceEvent_Severity {
+	p := new(ChannelTraceEvent_Severity)
+	*p = x
+	return p
 }
 
 func (x ChannelTraceEvent_Severity) String() string {
-	return proto.EnumName(ChannelTraceEvent_Severity_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
+func (ChannelTraceEvent_Severity) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_channelz_v1_channelz_proto_enumTypes[1].Descriptor()
+}
+
+func (ChannelTraceEvent_Severity) Type() protoreflect.EnumType {
+	return &file_grpc_channelz_v1_channelz_proto_enumTypes[1]
+}
+
+func (x ChannelTraceEvent_Severity) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ChannelTraceEvent_Severity.Descriptor instead.
 func (ChannelTraceEvent_Severity) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{4, 0}
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{4, 0}
 }
 
 // Channel is a logical grouping of channels, subchannels, and sockets.
 type Channel struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The identifier for this channel. This should bet set.
 	Ref *ChannelRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"`
 	// Data specific to this channel.
-	Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+	Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // At most one of 'channel_ref+subchannel_ref' and 'socket' is set.
 	// There are no ordering guarantees on the order of channel refs.
 	// There may not be cycles in the ref graph.
 	// A channel ref may be present in more than one channel or subchannel.
@@ -109,68 +181,72 @@ type Channel struct {
 	// A sub channel ref may be present in more than one channel or subchannel.
 	SubchannelRef []*SubchannelRef `protobuf:"bytes,4,rep,name=subchannel_ref,json=subchannelRef,proto3" json:"subchannel_ref,omitempty"`
 	// There are no ordering guarantees on the order of sockets.
-	SocketRef            []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
-	XXX_unrecognized     []byte       `json:"-"`
-	XXX_sizecache        int32        `json:"-"`
+	SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"`
 }
 
-func (m *Channel) Reset()         { *m = Channel{} }
-func (m *Channel) String() string { return proto.CompactTextString(m) }
-func (*Channel) ProtoMessage()    {}
-func (*Channel) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{0}
+func (x *Channel) Reset() {
+	*x = Channel{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Channel) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Channel.Unmarshal(m, b)
-}
-func (m *Channel) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Channel.Marshal(b, m, deterministic)
-}
-func (m *Channel) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Channel.Merge(m, src)
-}
-func (m *Channel) XXX_Size() int {
-	return xxx_messageInfo_Channel.Size(m)
+func (x *Channel) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Channel) XXX_DiscardUnknown() {
-	xxx_messageInfo_Channel.DiscardUnknown(m)
+
+func (*Channel) ProtoMessage() {}
+
+func (x *Channel) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Channel proto.InternalMessageInfo
+// Deprecated: Use Channel.ProtoReflect.Descriptor instead.
+func (*Channel) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *Channel) GetRef() *ChannelRef {
-	if m != nil {
-		return m.Ref
+func (x *Channel) GetRef() *ChannelRef {
+	if x != nil {
+		return x.Ref
 	}
 	return nil
 }
 
-func (m *Channel) GetData() *ChannelData {
-	if m != nil {
-		return m.Data
+func (x *Channel) GetData() *ChannelData {
+	if x != nil {
+		return x.Data
 	}
 	return nil
 }
 
-func (m *Channel) GetChannelRef() []*ChannelRef {
-	if m != nil {
-		return m.ChannelRef
+func (x *Channel) GetChannelRef() []*ChannelRef {
+	if x != nil {
+		return x.ChannelRef
 	}
 	return nil
 }
 
-func (m *Channel) GetSubchannelRef() []*SubchannelRef {
-	if m != nil {
-		return m.SubchannelRef
+func (x *Channel) GetSubchannelRef() []*SubchannelRef {
+	if x != nil {
+		return x.SubchannelRef
 	}
 	return nil
 }
 
-func (m *Channel) GetSocketRef() []*SocketRef {
-	if m != nil {
-		return m.SocketRef
+func (x *Channel) GetSocketRef() []*SocketRef {
+	if x != nil {
+		return x.SocketRef
 	}
 	return nil
 }
@@ -178,10 +254,14 @@ func (m *Channel) GetSocketRef() []*SocketRef {
 // Subchannel is a logical grouping of channels, subchannels, and sockets.
 // A subchannel is load balanced over by it's ancestor
 type Subchannel struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The identifier for this channel.
 	Ref *SubchannelRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"`
 	// Data specific to this channel.
-	Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+	Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // At most one of 'channel_ref+subchannel_ref' and 'socket' is set.
 	// There are no ordering guarantees on the order of channel refs.
 	// There may not be cycles in the ref graph.
 	// A channel ref may be present in more than one channel or subchannel.
@@ -192,68 +272,72 @@ type Subchannel struct {
 	// A sub channel ref may be present in more than one channel or subchannel.
 	SubchannelRef []*SubchannelRef `protobuf:"bytes,4,rep,name=subchannel_ref,json=subchannelRef,proto3" json:"subchannel_ref,omitempty"`
 	// There are no ordering guarantees on the order of sockets.
-	SocketRef            []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
-	XXX_unrecognized     []byte       `json:"-"`
-	XXX_sizecache        int32        `json:"-"`
+	SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"`
 }
 
-func (m *Subchannel) Reset()         { *m = Subchannel{} }
-func (m *Subchannel) String() string { return proto.CompactTextString(m) }
-func (*Subchannel) ProtoMessage()    {}
-func (*Subchannel) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{1}
+func (x *Subchannel) Reset() {
+	*x = Subchannel{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Subchannel) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Subchannel.Unmarshal(m, b)
-}
-func (m *Subchannel) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Subchannel.Marshal(b, m, deterministic)
+func (x *Subchannel) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Subchannel) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Subchannel.Merge(m, src)
-}
-func (m *Subchannel) XXX_Size() int {
-	return xxx_messageInfo_Subchannel.Size(m)
-}
-func (m *Subchannel) XXX_DiscardUnknown() {
-	xxx_messageInfo_Subchannel.DiscardUnknown(m)
+
+func (*Subchannel) ProtoMessage() {}
+
+func (x *Subchannel) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Subchannel proto.InternalMessageInfo
+// Deprecated: Use Subchannel.ProtoReflect.Descriptor instead.
+func (*Subchannel) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *Subchannel) GetRef() *SubchannelRef {
-	if m != nil {
-		return m.Ref
+func (x *Subchannel) GetRef() *SubchannelRef {
+	if x != nil {
+		return x.Ref
 	}
 	return nil
 }
 
-func (m *Subchannel) GetData() *ChannelData {
-	if m != nil {
-		return m.Data
+func (x *Subchannel) GetData() *ChannelData {
+	if x != nil {
+		return x.Data
 	}
 	return nil
 }
 
-func (m *Subchannel) GetChannelRef() []*ChannelRef {
-	if m != nil {
-		return m.ChannelRef
+func (x *Subchannel) GetChannelRef() []*ChannelRef {
+	if x != nil {
+		return x.ChannelRef
 	}
 	return nil
 }
 
-func (m *Subchannel) GetSubchannelRef() []*SubchannelRef {
-	if m != nil {
-		return m.SubchannelRef
+func (x *Subchannel) GetSubchannelRef() []*SubchannelRef {
+	if x != nil {
+		return x.SubchannelRef
 	}
 	return nil
 }
 
-func (m *Subchannel) GetSocketRef() []*SocketRef {
-	if m != nil {
-		return m.SocketRef
+func (x *Subchannel) GetSocketRef() []*SocketRef {
+	if x != nil {
+		return x.SocketRef
 	}
 	return nil
 }
@@ -261,46 +345,58 @@ func (m *Subchannel) GetSocketRef() []*SocketRef {
 // These come from the specified states in this document:
 // https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md
 type ChannelConnectivityState struct {
-	State                ChannelConnectivityState_State `protobuf:"varint,1,opt,name=state,proto3,enum=grpc.channelz.v1.ChannelConnectivityState_State" json:"state,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                       `json:"-"`
-	XXX_unrecognized     []byte                         `json:"-"`
-	XXX_sizecache        int32                          `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ChannelConnectivityState) Reset()         { *m = ChannelConnectivityState{} }
-func (m *ChannelConnectivityState) String() string { return proto.CompactTextString(m) }
-func (*ChannelConnectivityState) ProtoMessage()    {}
-func (*ChannelConnectivityState) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{2}
+	State ChannelConnectivityState_State `protobuf:"varint,1,opt,name=state,proto3,enum=grpc.channelz.v1.ChannelConnectivityState_State" json:"state,omitempty"`
 }
 
-func (m *ChannelConnectivityState) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ChannelConnectivityState.Unmarshal(m, b)
-}
-func (m *ChannelConnectivityState) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ChannelConnectivityState.Marshal(b, m, deterministic)
-}
-func (m *ChannelConnectivityState) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ChannelConnectivityState.Merge(m, src)
+func (x *ChannelConnectivityState) Reset() {
+	*x = ChannelConnectivityState{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ChannelConnectivityState) XXX_Size() int {
-	return xxx_messageInfo_ChannelConnectivityState.Size(m)
+
+func (x *ChannelConnectivityState) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ChannelConnectivityState) XXX_DiscardUnknown() {
-	xxx_messageInfo_ChannelConnectivityState.DiscardUnknown(m)
+
+func (*ChannelConnectivityState) ProtoMessage() {}
+
+func (x *ChannelConnectivityState) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ChannelConnectivityState proto.InternalMessageInfo
+// Deprecated: Use ChannelConnectivityState.ProtoReflect.Descriptor instead.
+func (*ChannelConnectivityState) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *ChannelConnectivityState) GetState() ChannelConnectivityState_State {
-	if m != nil {
-		return m.State
+func (x *ChannelConnectivityState) GetState() ChannelConnectivityState_State {
+	if x != nil {
+		return x.State
 	}
 	return ChannelConnectivityState_UNKNOWN
 }
 
 // Channel data is data related to a specific Channel or Subchannel.
 type ChannelData struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The connectivity state of the channel or subchannel.  Implementations
 	// should always set this.
 	State *ChannelConnectivityState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"`
@@ -316,81 +412,85 @@ type ChannelData struct {
 	CallsFailed int64 `protobuf:"varint,6,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"`
 	// The last time a call was started on the channel.
 	LastCallStartedTimestamp *timestamp.Timestamp `protobuf:"bytes,7,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"`
-	XXX_NoUnkeyedLiteral     struct{}             `json:"-"`
-	XXX_unrecognized         []byte               `json:"-"`
-	XXX_sizecache            int32                `json:"-"`
 }
 
-func (m *ChannelData) Reset()         { *m = ChannelData{} }
-func (m *ChannelData) String() string { return proto.CompactTextString(m) }
-func (*ChannelData) ProtoMessage()    {}
-func (*ChannelData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{3}
+func (x *ChannelData) Reset() {
+	*x = ChannelData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ChannelData) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ChannelData.Unmarshal(m, b)
-}
-func (m *ChannelData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ChannelData.Marshal(b, m, deterministic)
-}
-func (m *ChannelData) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ChannelData.Merge(m, src)
-}
-func (m *ChannelData) XXX_Size() int {
-	return xxx_messageInfo_ChannelData.Size(m)
+func (x *ChannelData) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ChannelData) XXX_DiscardUnknown() {
-	xxx_messageInfo_ChannelData.DiscardUnknown(m)
+
+func (*ChannelData) ProtoMessage() {}
+
+func (x *ChannelData) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ChannelData proto.InternalMessageInfo
+// Deprecated: Use ChannelData.ProtoReflect.Descriptor instead.
+func (*ChannelData) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *ChannelData) GetState() *ChannelConnectivityState {
-	if m != nil {
-		return m.State
+func (x *ChannelData) GetState() *ChannelConnectivityState {
+	if x != nil {
+		return x.State
 	}
 	return nil
 }
 
-func (m *ChannelData) GetTarget() string {
-	if m != nil {
-		return m.Target
+func (x *ChannelData) GetTarget() string {
+	if x != nil {
+		return x.Target
 	}
 	return ""
 }
 
-func (m *ChannelData) GetTrace() *ChannelTrace {
-	if m != nil {
-		return m.Trace
+func (x *ChannelData) GetTrace() *ChannelTrace {
+	if x != nil {
+		return x.Trace
 	}
 	return nil
 }
 
-func (m *ChannelData) GetCallsStarted() int64 {
-	if m != nil {
-		return m.CallsStarted
+func (x *ChannelData) GetCallsStarted() int64 {
+	if x != nil {
+		return x.CallsStarted
 	}
 	return 0
 }
 
-func (m *ChannelData) GetCallsSucceeded() int64 {
-	if m != nil {
-		return m.CallsSucceeded
+func (x *ChannelData) GetCallsSucceeded() int64 {
+	if x != nil {
+		return x.CallsSucceeded
 	}
 	return 0
 }
 
-func (m *ChannelData) GetCallsFailed() int64 {
-	if m != nil {
-		return m.CallsFailed
+func (x *ChannelData) GetCallsFailed() int64 {
+	if x != nil {
+		return x.CallsFailed
 	}
 	return 0
 }
 
-func (m *ChannelData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.LastCallStartedTimestamp
+func (x *ChannelData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastCallStartedTimestamp
 	}
 	return nil
 }
@@ -398,6 +498,10 @@ func (m *ChannelData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
 // A trace event is an interesting thing that happened to a channel or
 // subchannel, such as creation, address resolution, subchannel creation, etc.
 type ChannelTraceEvent struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// High level description of the event.
 	Description string `protobuf:"bytes,1,opt,name=description,proto3" json:"description,omitempty"`
 	// the severity of the trace event
@@ -409,77 +513,65 @@ type ChannelTraceEvent struct {
 	// this field would be filled if this trace event was for a subchannel being
 	// created.
 	//
-	// Types that are valid to be assigned to ChildRef:
+	// Types that are assignable to ChildRef:
 	//	*ChannelTraceEvent_ChannelRef
 	//	*ChannelTraceEvent_SubchannelRef
-	ChildRef             isChannelTraceEvent_ChildRef `protobuf_oneof:"child_ref"`
-	XXX_NoUnkeyedLiteral struct{}                     `json:"-"`
-	XXX_unrecognized     []byte                       `json:"-"`
-	XXX_sizecache        int32                        `json:"-"`
+	ChildRef isChannelTraceEvent_ChildRef `protobuf_oneof:"child_ref"`
 }
 
-func (m *ChannelTraceEvent) Reset()         { *m = ChannelTraceEvent{} }
-func (m *ChannelTraceEvent) String() string { return proto.CompactTextString(m) }
-func (*ChannelTraceEvent) ProtoMessage()    {}
-func (*ChannelTraceEvent) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{4}
+func (x *ChannelTraceEvent) Reset() {
+	*x = ChannelTraceEvent{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ChannelTraceEvent) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ChannelTraceEvent.Unmarshal(m, b)
-}
-func (m *ChannelTraceEvent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ChannelTraceEvent.Marshal(b, m, deterministic)
+func (x *ChannelTraceEvent) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ChannelTraceEvent) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ChannelTraceEvent.Merge(m, src)
-}
-func (m *ChannelTraceEvent) XXX_Size() int {
-	return xxx_messageInfo_ChannelTraceEvent.Size(m)
-}
-func (m *ChannelTraceEvent) XXX_DiscardUnknown() {
-	xxx_messageInfo_ChannelTraceEvent.DiscardUnknown(m)
+
+func (*ChannelTraceEvent) ProtoMessage() {}
+
+func (x *ChannelTraceEvent) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ChannelTraceEvent proto.InternalMessageInfo
+// Deprecated: Use ChannelTraceEvent.ProtoReflect.Descriptor instead.
+func (*ChannelTraceEvent) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *ChannelTraceEvent) GetDescription() string {
-	if m != nil {
-		return m.Description
+func (x *ChannelTraceEvent) GetDescription() string {
+	if x != nil {
+		return x.Description
 	}
 	return ""
 }
 
-func (m *ChannelTraceEvent) GetSeverity() ChannelTraceEvent_Severity {
-	if m != nil {
-		return m.Severity
+func (x *ChannelTraceEvent) GetSeverity() ChannelTraceEvent_Severity {
+	if x != nil {
+		return x.Severity
 	}
 	return ChannelTraceEvent_CT_UNKNOWN
 }
 
-func (m *ChannelTraceEvent) GetTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.Timestamp
+func (x *ChannelTraceEvent) GetTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.Timestamp
 	}
 	return nil
 }
 
-type isChannelTraceEvent_ChildRef interface {
-	isChannelTraceEvent_ChildRef()
-}
-
-type ChannelTraceEvent_ChannelRef struct {
-	ChannelRef *ChannelRef `protobuf:"bytes,4,opt,name=channel_ref,json=channelRef,proto3,oneof"`
-}
-
-type ChannelTraceEvent_SubchannelRef struct {
-	SubchannelRef *SubchannelRef `protobuf:"bytes,5,opt,name=subchannel_ref,json=subchannelRef,proto3,oneof"`
-}
-
-func (*ChannelTraceEvent_ChannelRef) isChannelTraceEvent_ChildRef() {}
-
-func (*ChannelTraceEvent_SubchannelRef) isChannelTraceEvent_ChildRef() {}
-
 func (m *ChannelTraceEvent) GetChildRef() isChannelTraceEvent_ChildRef {
 	if m != nil {
 		return m.ChildRef
@@ -487,30 +579,42 @@ func (m *ChannelTraceEvent) GetChildRef() isChannelTraceEvent_ChildRef {
 	return nil
 }
 
-func (m *ChannelTraceEvent) GetChannelRef() *ChannelRef {
-	if x, ok := m.GetChildRef().(*ChannelTraceEvent_ChannelRef); ok {
+func (x *ChannelTraceEvent) GetChannelRef() *ChannelRef {
+	if x, ok := x.GetChildRef().(*ChannelTraceEvent_ChannelRef); ok {
 		return x.ChannelRef
 	}
 	return nil
 }
 
-func (m *ChannelTraceEvent) GetSubchannelRef() *SubchannelRef {
-	if x, ok := m.GetChildRef().(*ChannelTraceEvent_SubchannelRef); ok {
+func (x *ChannelTraceEvent) GetSubchannelRef() *SubchannelRef {
+	if x, ok := x.GetChildRef().(*ChannelTraceEvent_SubchannelRef); ok {
 		return x.SubchannelRef
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*ChannelTraceEvent) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*ChannelTraceEvent_ChannelRef)(nil),
-		(*ChannelTraceEvent_SubchannelRef)(nil),
-	}
+type isChannelTraceEvent_ChildRef interface {
+	isChannelTraceEvent_ChildRef()
+}
+
+type ChannelTraceEvent_ChannelRef struct {
+	ChannelRef *ChannelRef `protobuf:"bytes,4,opt,name=channel_ref,json=channelRef,proto3,oneof"`
+}
+
+type ChannelTraceEvent_SubchannelRef struct {
+	SubchannelRef *SubchannelRef `protobuf:"bytes,5,opt,name=subchannel_ref,json=subchannelRef,proto3,oneof"`
 }
 
+func (*ChannelTraceEvent_ChannelRef) isChannelTraceEvent_ChildRef() {}
+
+func (*ChannelTraceEvent_SubchannelRef) isChannelTraceEvent_ChildRef() {}
+
 // ChannelTrace represents the recent events that have occurred on the channel.
 type ChannelTrace struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Number of events ever logged in this tracing object. This can differ from
 	// events.size() because events can be overwritten or garbage collected by
 	// implementations.
@@ -518,254 +622,290 @@ type ChannelTrace struct {
 	// Time that this channel was created.
 	CreationTimestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=creation_timestamp,json=creationTimestamp,proto3" json:"creation_timestamp,omitempty"`
 	// List of events that have occurred on this channel.
-	Events               []*ChannelTraceEvent `protobuf:"bytes,3,rep,name=events,proto3" json:"events,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
-	XXX_unrecognized     []byte               `json:"-"`
-	XXX_sizecache        int32                `json:"-"`
+	Events []*ChannelTraceEvent `protobuf:"bytes,3,rep,name=events,proto3" json:"events,omitempty"`
 }
 
-func (m *ChannelTrace) Reset()         { *m = ChannelTrace{} }
-func (m *ChannelTrace) String() string { return proto.CompactTextString(m) }
-func (*ChannelTrace) ProtoMessage()    {}
-func (*ChannelTrace) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{5}
+func (x *ChannelTrace) Reset() {
+	*x = ChannelTrace{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ChannelTrace) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ChannelTrace.Unmarshal(m, b)
-}
-func (m *ChannelTrace) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ChannelTrace.Marshal(b, m, deterministic)
-}
-func (m *ChannelTrace) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ChannelTrace.Merge(m, src)
+func (x *ChannelTrace) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ChannelTrace) XXX_Size() int {
-	return xxx_messageInfo_ChannelTrace.Size(m)
-}
-func (m *ChannelTrace) XXX_DiscardUnknown() {
-	xxx_messageInfo_ChannelTrace.DiscardUnknown(m)
+
+func (*ChannelTrace) ProtoMessage() {}
+
+func (x *ChannelTrace) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ChannelTrace proto.InternalMessageInfo
+// Deprecated: Use ChannelTrace.ProtoReflect.Descriptor instead.
+func (*ChannelTrace) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *ChannelTrace) GetNumEventsLogged() int64 {
-	if m != nil {
-		return m.NumEventsLogged
+func (x *ChannelTrace) GetNumEventsLogged() int64 {
+	if x != nil {
+		return x.NumEventsLogged
 	}
 	return 0
 }
 
-func (m *ChannelTrace) GetCreationTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.CreationTimestamp
+func (x *ChannelTrace) GetCreationTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.CreationTimestamp
 	}
 	return nil
 }
 
-func (m *ChannelTrace) GetEvents() []*ChannelTraceEvent {
-	if m != nil {
-		return m.Events
+func (x *ChannelTrace) GetEvents() []*ChannelTraceEvent {
+	if x != nil {
+		return x.Events
 	}
 	return nil
 }
 
 // ChannelRef is a reference to a Channel.
 type ChannelRef struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The globally unique id for this channel.  Must be a positive number.
 	ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"`
 	// An optional name associated with the channel.
-	Name                 string   `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *ChannelRef) Reset()         { *m = ChannelRef{} }
-func (m *ChannelRef) String() string { return proto.CompactTextString(m) }
-func (*ChannelRef) ProtoMessage()    {}
-func (*ChannelRef) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{6}
+func (x *ChannelRef) Reset() {
+	*x = ChannelRef{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ChannelRef) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ChannelRef.Unmarshal(m, b)
+func (x *ChannelRef) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ChannelRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ChannelRef.Marshal(b, m, deterministic)
-}
-func (m *ChannelRef) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ChannelRef.Merge(m, src)
-}
-func (m *ChannelRef) XXX_Size() int {
-	return xxx_messageInfo_ChannelRef.Size(m)
-}
-func (m *ChannelRef) XXX_DiscardUnknown() {
-	xxx_messageInfo_ChannelRef.DiscardUnknown(m)
+
+func (*ChannelRef) ProtoMessage() {}
+
+func (x *ChannelRef) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ChannelRef proto.InternalMessageInfo
+// Deprecated: Use ChannelRef.ProtoReflect.Descriptor instead.
+func (*ChannelRef) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *ChannelRef) GetChannelId() int64 {
-	if m != nil {
-		return m.ChannelId
+func (x *ChannelRef) GetChannelId() int64 {
+	if x != nil {
+		return x.ChannelId
 	}
 	return 0
 }
 
-func (m *ChannelRef) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *ChannelRef) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 // SubchannelRef is a reference to a Subchannel.
 type SubchannelRef struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The globally unique id for this subchannel.  Must be a positive number.
 	SubchannelId int64 `protobuf:"varint,7,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"`
 	// An optional name associated with the subchannel.
-	Name                 string   `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Name string `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *SubchannelRef) Reset()         { *m = SubchannelRef{} }
-func (m *SubchannelRef) String() string { return proto.CompactTextString(m) }
-func (*SubchannelRef) ProtoMessage()    {}
-func (*SubchannelRef) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{7}
+func (x *SubchannelRef) Reset() {
+	*x = SubchannelRef{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SubchannelRef) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SubchannelRef.Unmarshal(m, b)
-}
-func (m *SubchannelRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SubchannelRef.Marshal(b, m, deterministic)
-}
-func (m *SubchannelRef) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SubchannelRef.Merge(m, src)
-}
-func (m *SubchannelRef) XXX_Size() int {
-	return xxx_messageInfo_SubchannelRef.Size(m)
+func (x *SubchannelRef) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SubchannelRef) XXX_DiscardUnknown() {
-	xxx_messageInfo_SubchannelRef.DiscardUnknown(m)
+
+func (*SubchannelRef) ProtoMessage() {}
+
+func (x *SubchannelRef) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SubchannelRef proto.InternalMessageInfo
+// Deprecated: Use SubchannelRef.ProtoReflect.Descriptor instead.
+func (*SubchannelRef) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *SubchannelRef) GetSubchannelId() int64 {
-	if m != nil {
-		return m.SubchannelId
+func (x *SubchannelRef) GetSubchannelId() int64 {
+	if x != nil {
+		return x.SubchannelId
 	}
 	return 0
 }
 
-func (m *SubchannelRef) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *SubchannelRef) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 // SocketRef is a reference to a Socket.
 type SocketRef struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The globally unique id for this socket.  Must be a positive number.
 	SocketId int64 `protobuf:"varint,3,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"`
 	// An optional name associated with the socket.
-	Name                 string   `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *SocketRef) Reset()         { *m = SocketRef{} }
-func (m *SocketRef) String() string { return proto.CompactTextString(m) }
-func (*SocketRef) ProtoMessage()    {}
-func (*SocketRef) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{8}
+func (x *SocketRef) Reset() {
+	*x = SocketRef{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SocketRef) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SocketRef.Unmarshal(m, b)
-}
-func (m *SocketRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SocketRef.Marshal(b, m, deterministic)
-}
-func (m *SocketRef) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SocketRef.Merge(m, src)
+func (x *SocketRef) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SocketRef) XXX_Size() int {
-	return xxx_messageInfo_SocketRef.Size(m)
-}
-func (m *SocketRef) XXX_DiscardUnknown() {
-	xxx_messageInfo_SocketRef.DiscardUnknown(m)
+
+func (*SocketRef) ProtoMessage() {}
+
+func (x *SocketRef) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SocketRef proto.InternalMessageInfo
+// Deprecated: Use SocketRef.ProtoReflect.Descriptor instead.
+func (*SocketRef) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *SocketRef) GetSocketId() int64 {
-	if m != nil {
-		return m.SocketId
+func (x *SocketRef) GetSocketId() int64 {
+	if x != nil {
+		return x.SocketId
 	}
 	return 0
 }
 
-func (m *SocketRef) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *SocketRef) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 // ServerRef is a reference to a Server.
 type ServerRef struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// A globally unique identifier for this server.  Must be a positive number.
 	ServerId int64 `protobuf:"varint,5,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
 	// An optional name associated with the server.
-	Name                 string   `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *ServerRef) Reset()         { *m = ServerRef{} }
-func (m *ServerRef) String() string { return proto.CompactTextString(m) }
-func (*ServerRef) ProtoMessage()    {}
-func (*ServerRef) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{9}
+func (x *ServerRef) Reset() {
+	*x = ServerRef{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerRef) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerRef.Unmarshal(m, b)
-}
-func (m *ServerRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerRef.Marshal(b, m, deterministic)
+func (x *ServerRef) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerRef) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerRef.Merge(m, src)
-}
-func (m *ServerRef) XXX_Size() int {
-	return xxx_messageInfo_ServerRef.Size(m)
-}
-func (m *ServerRef) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerRef.DiscardUnknown(m)
+
+func (*ServerRef) ProtoMessage() {}
+
+func (x *ServerRef) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerRef proto.InternalMessageInfo
+// Deprecated: Use ServerRef.ProtoReflect.Descriptor instead.
+func (*ServerRef) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{9}
+}
 
-func (m *ServerRef) GetServerId() int64 {
-	if m != nil {
-		return m.ServerId
+func (x *ServerRef) GetServerId() int64 {
+	if x != nil {
+		return x.ServerId
 	}
 	return 0
 }
 
-func (m *ServerRef) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *ServerRef) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
@@ -773,66 +913,78 @@ func (m *ServerRef) GetName() string {
 // Server represents a single server.  There may be multiple servers in a single
 // program.
 type Server struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The identifier for a Server.  This should be set.
 	Ref *ServerRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"`
 	// The associated data of the Server.
 	Data *ServerData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
 	// The sockets that the server is listening on.  There are no ordering
 	// guarantees.  This may be absent.
-	ListenSocket         []*SocketRef `protobuf:"bytes,3,rep,name=listen_socket,json=listenSocket,proto3" json:"listen_socket,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
-	XXX_unrecognized     []byte       `json:"-"`
-	XXX_sizecache        int32        `json:"-"`
+	ListenSocket []*SocketRef `protobuf:"bytes,3,rep,name=listen_socket,json=listenSocket,proto3" json:"listen_socket,omitempty"`
 }
 
-func (m *Server) Reset()         { *m = Server{} }
-func (m *Server) String() string { return proto.CompactTextString(m) }
-func (*Server) ProtoMessage()    {}
-func (*Server) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{10}
+func (x *Server) Reset() {
+	*x = Server{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Server) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Server.Unmarshal(m, b)
+func (x *Server) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Server) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Server.Marshal(b, m, deterministic)
-}
-func (m *Server) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Server.Merge(m, src)
-}
-func (m *Server) XXX_Size() int {
-	return xxx_messageInfo_Server.Size(m)
-}
-func (m *Server) XXX_DiscardUnknown() {
-	xxx_messageInfo_Server.DiscardUnknown(m)
+
+func (*Server) ProtoMessage() {}
+
+func (x *Server) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Server proto.InternalMessageInfo
+// Deprecated: Use Server.ProtoReflect.Descriptor instead.
+func (*Server) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{10}
+}
 
-func (m *Server) GetRef() *ServerRef {
-	if m != nil {
-		return m.Ref
+func (x *Server) GetRef() *ServerRef {
+	if x != nil {
+		return x.Ref
 	}
 	return nil
 }
 
-func (m *Server) GetData() *ServerData {
-	if m != nil {
-		return m.Data
+func (x *Server) GetData() *ServerData {
+	if x != nil {
+		return x.Data
 	}
 	return nil
 }
 
-func (m *Server) GetListenSocket() []*SocketRef {
-	if m != nil {
-		return m.ListenSocket
+func (x *Server) GetListenSocket() []*SocketRef {
+	if x != nil {
+		return x.ListenSocket
 	}
 	return nil
 }
 
 // ServerData is data for a specific Server.
 type ServerData struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// A trace of recent events on the server.  May be absent.
 	Trace *ChannelTrace `protobuf:"bytes,1,opt,name=trace,proto3" json:"trace,omitempty"`
 	// The number of incoming calls started on the server
@@ -843,73 +995,81 @@ type ServerData struct {
 	CallsFailed int64 `protobuf:"varint,4,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"`
 	// The last time a call was started on the server.
 	LastCallStartedTimestamp *timestamp.Timestamp `protobuf:"bytes,5,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"`
-	XXX_NoUnkeyedLiteral     struct{}             `json:"-"`
-	XXX_unrecognized         []byte               `json:"-"`
-	XXX_sizecache            int32                `json:"-"`
 }
 
-func (m *ServerData) Reset()         { *m = ServerData{} }
-func (m *ServerData) String() string { return proto.CompactTextString(m) }
-func (*ServerData) ProtoMessage()    {}
-func (*ServerData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{11}
+func (x *ServerData) Reset() {
+	*x = ServerData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerData) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerData.Unmarshal(m, b)
-}
-func (m *ServerData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerData.Marshal(b, m, deterministic)
-}
-func (m *ServerData) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerData.Merge(m, src)
+func (x *ServerData) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerData) XXX_Size() int {
-	return xxx_messageInfo_ServerData.Size(m)
-}
-func (m *ServerData) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerData.DiscardUnknown(m)
+
+func (*ServerData) ProtoMessage() {}
+
+func (x *ServerData) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerData proto.InternalMessageInfo
+// Deprecated: Use ServerData.ProtoReflect.Descriptor instead.
+func (*ServerData) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{11}
+}
 
-func (m *ServerData) GetTrace() *ChannelTrace {
-	if m != nil {
-		return m.Trace
+func (x *ServerData) GetTrace() *ChannelTrace {
+	if x != nil {
+		return x.Trace
 	}
 	return nil
 }
 
-func (m *ServerData) GetCallsStarted() int64 {
-	if m != nil {
-		return m.CallsStarted
+func (x *ServerData) GetCallsStarted() int64 {
+	if x != nil {
+		return x.CallsStarted
 	}
 	return 0
 }
 
-func (m *ServerData) GetCallsSucceeded() int64 {
-	if m != nil {
-		return m.CallsSucceeded
+func (x *ServerData) GetCallsSucceeded() int64 {
+	if x != nil {
+		return x.CallsSucceeded
 	}
 	return 0
 }
 
-func (m *ServerData) GetCallsFailed() int64 {
-	if m != nil {
-		return m.CallsFailed
+func (x *ServerData) GetCallsFailed() int64 {
+	if x != nil {
+		return x.CallsFailed
 	}
 	return 0
 }
 
-func (m *ServerData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.LastCallStartedTimestamp
+func (x *ServerData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastCallStartedTimestamp
 	}
 	return nil
 }
 
 // Information about an actual connection.  Pronounced "sock-ay".
 type Socket struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The identifier for the Socket.
 	Ref *SocketRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"`
 	// Data specific to this Socket.
@@ -923,75 +1083,79 @@ type Socket struct {
 	Security *Security `protobuf:"bytes,5,opt,name=security,proto3" json:"security,omitempty"`
 	// Optional, represents the name of the remote endpoint, if different than
 	// the original target name.
-	RemoteName           string   `protobuf:"bytes,6,opt,name=remote_name,json=remoteName,proto3" json:"remote_name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	RemoteName string `protobuf:"bytes,6,opt,name=remote_name,json=remoteName,proto3" json:"remote_name,omitempty"`
 }
 
-func (m *Socket) Reset()         { *m = Socket{} }
-func (m *Socket) String() string { return proto.CompactTextString(m) }
-func (*Socket) ProtoMessage()    {}
-func (*Socket) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{12}
+func (x *Socket) Reset() {
+	*x = Socket{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Socket) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Socket.Unmarshal(m, b)
-}
-func (m *Socket) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Socket.Marshal(b, m, deterministic)
-}
-func (m *Socket) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Socket.Merge(m, src)
+func (x *Socket) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Socket) XXX_Size() int {
-	return xxx_messageInfo_Socket.Size(m)
-}
-func (m *Socket) XXX_DiscardUnknown() {
-	xxx_messageInfo_Socket.DiscardUnknown(m)
+
+func (*Socket) ProtoMessage() {}
+
+func (x *Socket) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Socket proto.InternalMessageInfo
+// Deprecated: Use Socket.ProtoReflect.Descriptor instead.
+func (*Socket) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{12}
+}
 
-func (m *Socket) GetRef() *SocketRef {
-	if m != nil {
-		return m.Ref
+func (x *Socket) GetRef() *SocketRef {
+	if x != nil {
+		return x.Ref
 	}
 	return nil
 }
 
-func (m *Socket) GetData() *SocketData {
-	if m != nil {
-		return m.Data
+func (x *Socket) GetData() *SocketData {
+	if x != nil {
+		return x.Data
 	}
 	return nil
 }
 
-func (m *Socket) GetLocal() *Address {
-	if m != nil {
-		return m.Local
+func (x *Socket) GetLocal() *Address {
+	if x != nil {
+		return x.Local
 	}
 	return nil
 }
 
-func (m *Socket) GetRemote() *Address {
-	if m != nil {
-		return m.Remote
+func (x *Socket) GetRemote() *Address {
+	if x != nil {
+		return x.Remote
 	}
 	return nil
 }
 
-func (m *Socket) GetSecurity() *Security {
-	if m != nil {
-		return m.Security
+func (x *Socket) GetSecurity() *Security {
+	if x != nil {
+		return x.Security
 	}
 	return nil
 }
 
-func (m *Socket) GetRemoteName() string {
-	if m != nil {
-		return m.RemoteName
+func (x *Socket) GetRemoteName() string {
+	if x != nil {
+		return x.RemoteName
 	}
 	return ""
 }
@@ -1000,6 +1164,10 @@ func (m *Socket) GetRemoteName() string {
 // are specific to the implementation, so there may be minor differences in
 // the semantics.  (e.g. flow control windows)
 type SocketData struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The number of streams that have been started.
 	StreamsStarted int64 `protobuf:"varint,1,opt,name=streams_started,json=streamsStarted,proto3" json:"streams_started,omitempty"`
 	// The number of streams that have ended successfully:
@@ -1037,164 +1205,204 @@ type SocketData struct {
 	RemoteFlowControlWindow *wrappers.Int64Value `protobuf:"bytes,12,opt,name=remote_flow_control_window,json=remoteFlowControlWindow,proto3" json:"remote_flow_control_window,omitempty"`
 	// Socket options set on this socket.  May be absent if 'summary' is set
 	// on GetSocketRequest.
-	Option               []*SocketOption `protobuf:"bytes,13,rep,name=option,proto3" json:"option,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}        `json:"-"`
-	XXX_unrecognized     []byte          `json:"-"`
-	XXX_sizecache        int32           `json:"-"`
+	Option []*SocketOption `protobuf:"bytes,13,rep,name=option,proto3" json:"option,omitempty"`
 }
 
-func (m *SocketData) Reset()         { *m = SocketData{} }
-func (m *SocketData) String() string { return proto.CompactTextString(m) }
-func (*SocketData) ProtoMessage()    {}
-func (*SocketData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{13}
+func (x *SocketData) Reset() {
+	*x = SocketData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SocketData) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SocketData.Unmarshal(m, b)
-}
-func (m *SocketData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SocketData.Marshal(b, m, deterministic)
-}
-func (m *SocketData) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SocketData.Merge(m, src)
-}
-func (m *SocketData) XXX_Size() int {
-	return xxx_messageInfo_SocketData.Size(m)
-}
-func (m *SocketData) XXX_DiscardUnknown() {
-	xxx_messageInfo_SocketData.DiscardUnknown(m)
+func (x *SocketData) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_SocketData proto.InternalMessageInfo
+func (*SocketData) ProtoMessage() {}
 
-func (m *SocketData) GetStreamsStarted() int64 {
-	if m != nil {
-		return m.StreamsStarted
+func (x *SocketData) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SocketData.ProtoReflect.Descriptor instead.
+func (*SocketData) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{13}
+}
+
+func (x *SocketData) GetStreamsStarted() int64 {
+	if x != nil {
+		return x.StreamsStarted
 	}
 	return 0
 }
 
-func (m *SocketData) GetStreamsSucceeded() int64 {
-	if m != nil {
-		return m.StreamsSucceeded
+func (x *SocketData) GetStreamsSucceeded() int64 {
+	if x != nil {
+		return x.StreamsSucceeded
 	}
 	return 0
 }
 
-func (m *SocketData) GetStreamsFailed() int64 {
-	if m != nil {
-		return m.StreamsFailed
+func (x *SocketData) GetStreamsFailed() int64 {
+	if x != nil {
+		return x.StreamsFailed
 	}
 	return 0
 }
 
-func (m *SocketData) GetMessagesSent() int64 {
-	if m != nil {
-		return m.MessagesSent
+func (x *SocketData) GetMessagesSent() int64 {
+	if x != nil {
+		return x.MessagesSent
 	}
 	return 0
 }
 
-func (m *SocketData) GetMessagesReceived() int64 {
-	if m != nil {
-		return m.MessagesReceived
+func (x *SocketData) GetMessagesReceived() int64 {
+	if x != nil {
+		return x.MessagesReceived
 	}
 	return 0
 }
 
-func (m *SocketData) GetKeepAlivesSent() int64 {
-	if m != nil {
-		return m.KeepAlivesSent
+func (x *SocketData) GetKeepAlivesSent() int64 {
+	if x != nil {
+		return x.KeepAlivesSent
 	}
 	return 0
 }
 
-func (m *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.LastLocalStreamCreatedTimestamp
+func (x *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastLocalStreamCreatedTimestamp
 	}
 	return nil
 }
 
-func (m *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.LastRemoteStreamCreatedTimestamp
+func (x *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastRemoteStreamCreatedTimestamp
 	}
 	return nil
 }
 
-func (m *SocketData) GetLastMessageSentTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.LastMessageSentTimestamp
+func (x *SocketData) GetLastMessageSentTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastMessageSentTimestamp
 	}
 	return nil
 }
 
-func (m *SocketData) GetLastMessageReceivedTimestamp() *timestamp.Timestamp {
-	if m != nil {
-		return m.LastMessageReceivedTimestamp
+func (x *SocketData) GetLastMessageReceivedTimestamp() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastMessageReceivedTimestamp
 	}
 	return nil
 }
 
-func (m *SocketData) GetLocalFlowControlWindow() *wrappers.Int64Value {
-	if m != nil {
-		return m.LocalFlowControlWindow
+func (x *SocketData) GetLocalFlowControlWindow() *wrappers.Int64Value {
+	if x != nil {
+		return x.LocalFlowControlWindow
 	}
 	return nil
 }
 
-func (m *SocketData) GetRemoteFlowControlWindow() *wrappers.Int64Value {
-	if m != nil {
-		return m.RemoteFlowControlWindow
+func (x *SocketData) GetRemoteFlowControlWindow() *wrappers.Int64Value {
+	if x != nil {
+		return x.RemoteFlowControlWindow
 	}
 	return nil
 }
 
-func (m *SocketData) GetOption() []*SocketOption {
-	if m != nil {
-		return m.Option
+func (x *SocketData) GetOption() []*SocketOption {
+	if x != nil {
+		return x.Option
 	}
 	return nil
 }
 
 // Address represents the address used to create the socket.
 type Address struct {
-	// Types that are valid to be assigned to Address:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Address:
 	//	*Address_TcpipAddress
 	//	*Address_UdsAddress_
 	//	*Address_OtherAddress_
-	Address              isAddress_Address `protobuf_oneof:"address"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	Address isAddress_Address `protobuf_oneof:"address"`
 }
 
-func (m *Address) Reset()         { *m = Address{} }
-func (m *Address) String() string { return proto.CompactTextString(m) }
-func (*Address) ProtoMessage()    {}
-func (*Address) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{14}
+func (x *Address) Reset() {
+	*x = Address{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Address) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *Address) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Address.Unmarshal(m, b)
+func (*Address) ProtoMessage() {}
+
+func (x *Address) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Address.Marshal(b, m, deterministic)
+
+// Deprecated: Use Address.ProtoReflect.Descriptor instead.
+func (*Address) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14}
 }
-func (m *Address) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Address.Merge(m, src)
+
+func (m *Address) GetAddress() isAddress_Address {
+	if m != nil {
+		return m.Address
+	}
+	return nil
 }
-func (m *Address) XXX_Size() int {
-	return xxx_messageInfo_Address.Size(m)
+
+func (x *Address) GetTcpipAddress() *Address_TcpIpAddress {
+	if x, ok := x.GetAddress().(*Address_TcpipAddress); ok {
+		return x.TcpipAddress
+	}
+	return nil
 }
-func (m *Address) XXX_DiscardUnknown() {
-	xxx_messageInfo_Address.DiscardUnknown(m)
+
+func (x *Address) GetUdsAddress() *Address_UdsAddress {
+	if x, ok := x.GetAddress().(*Address_UdsAddress_); ok {
+		return x.UdsAddress
+	}
+	return nil
 }
 
-var xxx_messageInfo_Address proto.InternalMessageInfo
+func (x *Address) GetOtherAddress() *Address_OtherAddress {
+	if x, ok := x.GetAddress().(*Address_OtherAddress_); ok {
+		return x.OtherAddress
+	}
+	return nil
+}
 
 type isAddress_Address interface {
 	isAddress_Address()
@@ -1218,1706 +1426,2802 @@ func (*Address_UdsAddress_) isAddress_Address() {}
 
 func (*Address_OtherAddress_) isAddress_Address() {}
 
-func (m *Address) GetAddress() isAddress_Address {
-	if m != nil {
-		return m.Address
+// Security represents details about how secure the socket is.
+type Security struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Model:
+	//	*Security_Tls_
+	//	*Security_Other
+	Model isSecurity_Model `protobuf_oneof:"model"`
+}
+
+func (x *Security) Reset() {
+	*x = Security{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *Address) GetTcpipAddress() *Address_TcpIpAddress {
-	if x, ok := m.GetAddress().(*Address_TcpipAddress); ok {
-		return x.TcpipAddress
+func (x *Security) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Security) ProtoMessage() {}
+
+func (x *Security) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-func (m *Address) GetUdsAddress() *Address_UdsAddress {
-	if x, ok := m.GetAddress().(*Address_UdsAddress_); ok {
-		return x.UdsAddress
+// Deprecated: Use Security.ProtoReflect.Descriptor instead.
+func (*Security) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15}
+}
+
+func (m *Security) GetModel() isSecurity_Model {
+	if m != nil {
+		return m.Model
 	}
 	return nil
 }
 
-func (m *Address) GetOtherAddress() *Address_OtherAddress {
-	if x, ok := m.GetAddress().(*Address_OtherAddress_); ok {
-		return x.OtherAddress
+func (x *Security) GetTls() *Security_Tls {
+	if x, ok := x.GetModel().(*Security_Tls_); ok {
+		return x.Tls
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*Address) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*Address_TcpipAddress)(nil),
-		(*Address_UdsAddress_)(nil),
-		(*Address_OtherAddress_)(nil),
+func (x *Security) GetOther() *Security_OtherSecurity {
+	if x, ok := x.GetModel().(*Security_Other); ok {
+		return x.Other
 	}
+	return nil
 }
 
-type Address_TcpIpAddress struct {
-	// Either the IPv4 or IPv6 address in bytes.  Will be either 4 bytes or 16
-	// bytes in length.
-	IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"`
-	// 0-64k, or -1 if not appropriate.
-	Port                 int32    `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+type isSecurity_Model interface {
+	isSecurity_Model()
 }
 
-func (m *Address_TcpIpAddress) Reset()         { *m = Address_TcpIpAddress{} }
-func (m *Address_TcpIpAddress) String() string { return proto.CompactTextString(m) }
-func (*Address_TcpIpAddress) ProtoMessage()    {}
-func (*Address_TcpIpAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{14, 0}
+type Security_Tls_ struct {
+	Tls *Security_Tls `protobuf:"bytes,1,opt,name=tls,proto3,oneof"`
 }
 
-func (m *Address_TcpIpAddress) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Address_TcpIpAddress.Unmarshal(m, b)
-}
-func (m *Address_TcpIpAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Address_TcpIpAddress.Marshal(b, m, deterministic)
-}
-func (m *Address_TcpIpAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Address_TcpIpAddress.Merge(m, src)
-}
-func (m *Address_TcpIpAddress) XXX_Size() int {
-	return xxx_messageInfo_Address_TcpIpAddress.Size(m)
-}
-func (m *Address_TcpIpAddress) XXX_DiscardUnknown() {
-	xxx_messageInfo_Address_TcpIpAddress.DiscardUnknown(m)
+type Security_Other struct {
+	Other *Security_OtherSecurity `protobuf:"bytes,2,opt,name=other,proto3,oneof"`
 }
 
-var xxx_messageInfo_Address_TcpIpAddress proto.InternalMessageInfo
+func (*Security_Tls_) isSecurity_Model() {}
 
-func (m *Address_TcpIpAddress) GetIpAddress() []byte {
-	if m != nil {
-		return m.IpAddress
-	}
-	return nil
+func (*Security_Other) isSecurity_Model() {}
+
+// SocketOption represents socket options for a socket.  Specifically, these
+// are the options returned by getsockopt().
+type SocketOption struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The full name of the socket option.  Typically this will be the upper case
+	// name, such as "SO_REUSEPORT".
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// The human readable value of this socket option.  At least one of value or
+	// additional will be set.
+	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+	// Additional data associated with the socket option.  At least one of value
+	// or additional will be set.
+	Additional *any.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"`
 }
 
-func (m *Address_TcpIpAddress) GetPort() int32 {
-	if m != nil {
-		return m.Port
+func (x *SocketOption) Reset() {
+	*x = SocketOption{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[16]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return 0
 }
 
-// A Unix Domain Socket address.
-type Address_UdsAddress struct {
-	Filename             string   `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *SocketOption) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *Address_UdsAddress) Reset()         { *m = Address_UdsAddress{} }
-func (m *Address_UdsAddress) String() string { return proto.CompactTextString(m) }
-func (*Address_UdsAddress) ProtoMessage()    {}
-func (*Address_UdsAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{14, 1}
-}
+func (*SocketOption) ProtoMessage() {}
 
-func (m *Address_UdsAddress) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Address_UdsAddress.Unmarshal(m, b)
-}
-func (m *Address_UdsAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Address_UdsAddress.Marshal(b, m, deterministic)
-}
-func (m *Address_UdsAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Address_UdsAddress.Merge(m, src)
-}
-func (m *Address_UdsAddress) XXX_Size() int {
-	return xxx_messageInfo_Address_UdsAddress.Size(m)
-}
-func (m *Address_UdsAddress) XXX_DiscardUnknown() {
-	xxx_messageInfo_Address_UdsAddress.DiscardUnknown(m)
+func (x *SocketOption) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[16]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Address_UdsAddress proto.InternalMessageInfo
+// Deprecated: Use SocketOption.ProtoReflect.Descriptor instead.
+func (*SocketOption) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{16}
+}
 
-func (m *Address_UdsAddress) GetFilename() string {
-	if m != nil {
-		return m.Filename
+func (x *SocketOption) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
-// An address type not included above.
-type Address_OtherAddress struct {
-	// The human readable version of the value.  This value should be set.
-	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	// The actual address message.
-	Value                *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *SocketOption) GetValue() string {
+	if x != nil {
+		return x.Value
+	}
+	return ""
 }
 
-func (m *Address_OtherAddress) Reset()         { *m = Address_OtherAddress{} }
-func (m *Address_OtherAddress) String() string { return proto.CompactTextString(m) }
-func (*Address_OtherAddress) ProtoMessage()    {}
-func (*Address_OtherAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{14, 2}
+func (x *SocketOption) GetAdditional() *any.Any {
+	if x != nil {
+		return x.Additional
+	}
+	return nil
 }
 
-func (m *Address_OtherAddress) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Address_OtherAddress.Unmarshal(m, b)
-}
-func (m *Address_OtherAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Address_OtherAddress.Marshal(b, m, deterministic)
-}
-func (m *Address_OtherAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Address_OtherAddress.Merge(m, src)
+// For use with SocketOption's additional field.  This is primarily used for
+// SO_RCVTIMEO and SO_SNDTIMEO
+type SocketOptionTimeout struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Duration *duration.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"`
 }
-func (m *Address_OtherAddress) XXX_Size() int {
-	return xxx_messageInfo_Address_OtherAddress.Size(m)
+
+func (x *SocketOptionTimeout) Reset() {
+	*x = SocketOptionTimeout{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[17]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Address_OtherAddress) XXX_DiscardUnknown() {
-	xxx_messageInfo_Address_OtherAddress.DiscardUnknown(m)
+
+func (x *SocketOptionTimeout) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_Address_OtherAddress proto.InternalMessageInfo
+func (*SocketOptionTimeout) ProtoMessage() {}
 
-func (m *Address_OtherAddress) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *SocketOptionTimeout) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[17]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return ""
+	return mi.MessageOf(x)
 }
 
-func (m *Address_OtherAddress) GetValue() *any.Any {
-	if m != nil {
-		return m.Value
+// Deprecated: Use SocketOptionTimeout.ProtoReflect.Descriptor instead.
+func (*SocketOptionTimeout) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{17}
+}
+
+func (x *SocketOptionTimeout) GetDuration() *duration.Duration {
+	if x != nil {
+		return x.Duration
 	}
 	return nil
 }
 
-// Security represents details about how secure the socket is.
-type Security struct {
-	// Types that are valid to be assigned to Model:
-	//	*Security_Tls_
-	//	*Security_Other
-	Model                isSecurity_Model `protobuf_oneof:"model"`
-	XXX_NoUnkeyedLiteral struct{}         `json:"-"`
-	XXX_unrecognized     []byte           `json:"-"`
-	XXX_sizecache        int32            `json:"-"`
-}
+// For use with SocketOption's additional field.  This is primarily used for
+// SO_LINGER.
+type SocketOptionLinger struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *Security) Reset()         { *m = Security{} }
-func (m *Security) String() string { return proto.CompactTextString(m) }
-func (*Security) ProtoMessage()    {}
-func (*Security) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{15}
+	// active maps to `struct linger.l_onoff`
+	Active bool `protobuf:"varint,1,opt,name=active,proto3" json:"active,omitempty"`
+	// duration maps to `struct linger.l_linger`
+	Duration *duration.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"`
 }
 
-func (m *Security) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Security.Unmarshal(m, b)
-}
-func (m *Security) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Security.Marshal(b, m, deterministic)
-}
-func (m *Security) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Security.Merge(m, src)
-}
-func (m *Security) XXX_Size() int {
-	return xxx_messageInfo_Security.Size(m)
+func (x *SocketOptionLinger) Reset() {
+	*x = SocketOptionLinger{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[18]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Security) XXX_DiscardUnknown() {
-	xxx_messageInfo_Security.DiscardUnknown(m)
+
+func (x *SocketOptionLinger) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_Security proto.InternalMessageInfo
+func (*SocketOptionLinger) ProtoMessage() {}
 
-type isSecurity_Model interface {
-	isSecurity_Model()
+func (x *SocketOptionLinger) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[18]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-type Security_Tls_ struct {
-	Tls *Security_Tls `protobuf:"bytes,1,opt,name=tls,proto3,oneof"`
+// Deprecated: Use SocketOptionLinger.ProtoReflect.Descriptor instead.
+func (*SocketOptionLinger) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{18}
 }
 
-type Security_Other struct {
-	Other *Security_OtherSecurity `protobuf:"bytes,2,opt,name=other,proto3,oneof"`
+func (x *SocketOptionLinger) GetActive() bool {
+	if x != nil {
+		return x.Active
+	}
+	return false
 }
 
-func (*Security_Tls_) isSecurity_Model() {}
-
-func (*Security_Other) isSecurity_Model() {}
-
-func (m *Security) GetModel() isSecurity_Model {
-	if m != nil {
-		return m.Model
+func (x *SocketOptionLinger) GetDuration() *duration.Duration {
+	if x != nil {
+		return x.Duration
 	}
 	return nil
 }
 
-func (m *Security) GetTls() *Security_Tls {
-	if x, ok := m.GetModel().(*Security_Tls_); ok {
-		return x.Tls
-	}
-	return nil
+// For use with SocketOption's additional field.  Tcp info for
+// SOL_TCP and TCP_INFO.
+type SocketOptionTcpInfo struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	TcpiState        uint32 `protobuf:"varint,1,opt,name=tcpi_state,json=tcpiState,proto3" json:"tcpi_state,omitempty"`
+	TcpiCaState      uint32 `protobuf:"varint,2,opt,name=tcpi_ca_state,json=tcpiCaState,proto3" json:"tcpi_ca_state,omitempty"`
+	TcpiRetransmits  uint32 `protobuf:"varint,3,opt,name=tcpi_retransmits,json=tcpiRetransmits,proto3" json:"tcpi_retransmits,omitempty"`
+	TcpiProbes       uint32 `protobuf:"varint,4,opt,name=tcpi_probes,json=tcpiProbes,proto3" json:"tcpi_probes,omitempty"`
+	TcpiBackoff      uint32 `protobuf:"varint,5,opt,name=tcpi_backoff,json=tcpiBackoff,proto3" json:"tcpi_backoff,omitempty"`
+	TcpiOptions      uint32 `protobuf:"varint,6,opt,name=tcpi_options,json=tcpiOptions,proto3" json:"tcpi_options,omitempty"`
+	TcpiSndWscale    uint32 `protobuf:"varint,7,opt,name=tcpi_snd_wscale,json=tcpiSndWscale,proto3" json:"tcpi_snd_wscale,omitempty"`
+	TcpiRcvWscale    uint32 `protobuf:"varint,8,opt,name=tcpi_rcv_wscale,json=tcpiRcvWscale,proto3" json:"tcpi_rcv_wscale,omitempty"`
+	TcpiRto          uint32 `protobuf:"varint,9,opt,name=tcpi_rto,json=tcpiRto,proto3" json:"tcpi_rto,omitempty"`
+	TcpiAto          uint32 `protobuf:"varint,10,opt,name=tcpi_ato,json=tcpiAto,proto3" json:"tcpi_ato,omitempty"`
+	TcpiSndMss       uint32 `protobuf:"varint,11,opt,name=tcpi_snd_mss,json=tcpiSndMss,proto3" json:"tcpi_snd_mss,omitempty"`
+	TcpiRcvMss       uint32 `protobuf:"varint,12,opt,name=tcpi_rcv_mss,json=tcpiRcvMss,proto3" json:"tcpi_rcv_mss,omitempty"`
+	TcpiUnacked      uint32 `protobuf:"varint,13,opt,name=tcpi_unacked,json=tcpiUnacked,proto3" json:"tcpi_unacked,omitempty"`
+	TcpiSacked       uint32 `protobuf:"varint,14,opt,name=tcpi_sacked,json=tcpiSacked,proto3" json:"tcpi_sacked,omitempty"`
+	TcpiLost         uint32 `protobuf:"varint,15,opt,name=tcpi_lost,json=tcpiLost,proto3" json:"tcpi_lost,omitempty"`
+	TcpiRetrans      uint32 `protobuf:"varint,16,opt,name=tcpi_retrans,json=tcpiRetrans,proto3" json:"tcpi_retrans,omitempty"`
+	TcpiFackets      uint32 `protobuf:"varint,17,opt,name=tcpi_fackets,json=tcpiFackets,proto3" json:"tcpi_fackets,omitempty"`
+	TcpiLastDataSent uint32 `protobuf:"varint,18,opt,name=tcpi_last_data_sent,json=tcpiLastDataSent,proto3" json:"tcpi_last_data_sent,omitempty"`
+	TcpiLastAckSent  uint32 `protobuf:"varint,19,opt,name=tcpi_last_ack_sent,json=tcpiLastAckSent,proto3" json:"tcpi_last_ack_sent,omitempty"`
+	TcpiLastDataRecv uint32 `protobuf:"varint,20,opt,name=tcpi_last_data_recv,json=tcpiLastDataRecv,proto3" json:"tcpi_last_data_recv,omitempty"`
+	TcpiLastAckRecv  uint32 `protobuf:"varint,21,opt,name=tcpi_last_ack_recv,json=tcpiLastAckRecv,proto3" json:"tcpi_last_ack_recv,omitempty"`
+	TcpiPmtu         uint32 `protobuf:"varint,22,opt,name=tcpi_pmtu,json=tcpiPmtu,proto3" json:"tcpi_pmtu,omitempty"`
+	TcpiRcvSsthresh  uint32 `protobuf:"varint,23,opt,name=tcpi_rcv_ssthresh,json=tcpiRcvSsthresh,proto3" json:"tcpi_rcv_ssthresh,omitempty"`
+	TcpiRtt          uint32 `protobuf:"varint,24,opt,name=tcpi_rtt,json=tcpiRtt,proto3" json:"tcpi_rtt,omitempty"`
+	TcpiRttvar       uint32 `protobuf:"varint,25,opt,name=tcpi_rttvar,json=tcpiRttvar,proto3" json:"tcpi_rttvar,omitempty"`
+	TcpiSndSsthresh  uint32 `protobuf:"varint,26,opt,name=tcpi_snd_ssthresh,json=tcpiSndSsthresh,proto3" json:"tcpi_snd_ssthresh,omitempty"`
+	TcpiSndCwnd      uint32 `protobuf:"varint,27,opt,name=tcpi_snd_cwnd,json=tcpiSndCwnd,proto3" json:"tcpi_snd_cwnd,omitempty"`
+	TcpiAdvmss       uint32 `protobuf:"varint,28,opt,name=tcpi_advmss,json=tcpiAdvmss,proto3" json:"tcpi_advmss,omitempty"`
+	TcpiReordering   uint32 `protobuf:"varint,29,opt,name=tcpi_reordering,json=tcpiReordering,proto3" json:"tcpi_reordering,omitempty"`
+}
+
+func (x *SocketOptionTcpInfo) Reset() {
+	*x = SocketOptionTcpInfo{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[19]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SocketOptionTcpInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SocketOptionTcpInfo) ProtoMessage() {}
+
+func (x *SocketOptionTcpInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[19]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SocketOptionTcpInfo.ProtoReflect.Descriptor instead.
+func (*SocketOptionTcpInfo) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{19}
 }
 
-func (m *Security) GetOther() *Security_OtherSecurity {
-	if x, ok := m.GetModel().(*Security_Other); ok {
-		return x.Other
+func (x *SocketOptionTcpInfo) GetTcpiState() uint32 {
+	if x != nil {
+		return x.TcpiState
 	}
-	return nil
+	return 0
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*Security) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*Security_Tls_)(nil),
-		(*Security_Other)(nil),
+func (x *SocketOptionTcpInfo) GetTcpiCaState() uint32 {
+	if x != nil {
+		return x.TcpiCaState
 	}
+	return 0
 }
 
-type Security_Tls struct {
-	// Types that are valid to be assigned to CipherSuite:
-	//	*Security_Tls_StandardName
-	//	*Security_Tls_OtherName
-	CipherSuite isSecurity_Tls_CipherSuite `protobuf_oneof:"cipher_suite"`
-	// the certificate used by this endpoint.
-	LocalCertificate []byte `protobuf:"bytes,3,opt,name=local_certificate,json=localCertificate,proto3" json:"local_certificate,omitempty"`
-	// the certificate used by the remote endpoint.
-	RemoteCertificate    []byte   `protobuf:"bytes,4,opt,name=remote_certificate,json=remoteCertificate,proto3" json:"remote_certificate,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *SocketOptionTcpInfo) GetTcpiRetransmits() uint32 {
+	if x != nil {
+		return x.TcpiRetransmits
+	}
+	return 0
 }
 
-func (m *Security_Tls) Reset()         { *m = Security_Tls{} }
-func (m *Security_Tls) String() string { return proto.CompactTextString(m) }
-func (*Security_Tls) ProtoMessage()    {}
-func (*Security_Tls) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{15, 0}
+func (x *SocketOptionTcpInfo) GetTcpiProbes() uint32 {
+	if x != nil {
+		return x.TcpiProbes
+	}
+	return 0
 }
 
-func (m *Security_Tls) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Security_Tls.Unmarshal(m, b)
-}
-func (m *Security_Tls) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Security_Tls.Marshal(b, m, deterministic)
-}
-func (m *Security_Tls) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Security_Tls.Merge(m, src)
-}
-func (m *Security_Tls) XXX_Size() int {
-	return xxx_messageInfo_Security_Tls.Size(m)
-}
-func (m *Security_Tls) XXX_DiscardUnknown() {
-	xxx_messageInfo_Security_Tls.DiscardUnknown(m)
+func (x *SocketOptionTcpInfo) GetTcpiBackoff() uint32 {
+	if x != nil {
+		return x.TcpiBackoff
+	}
+	return 0
 }
 
-var xxx_messageInfo_Security_Tls proto.InternalMessageInfo
+func (x *SocketOptionTcpInfo) GetTcpiOptions() uint32 {
+	if x != nil {
+		return x.TcpiOptions
+	}
+	return 0
+}
 
-type isSecurity_Tls_CipherSuite interface {
-	isSecurity_Tls_CipherSuite()
+func (x *SocketOptionTcpInfo) GetTcpiSndWscale() uint32 {
+	if x != nil {
+		return x.TcpiSndWscale
+	}
+	return 0
 }
 
-type Security_Tls_StandardName struct {
-	StandardName string `protobuf:"bytes,1,opt,name=standard_name,json=standardName,proto3,oneof"`
+func (x *SocketOptionTcpInfo) GetTcpiRcvWscale() uint32 {
+	if x != nil {
+		return x.TcpiRcvWscale
+	}
+	return 0
 }
 
-type Security_Tls_OtherName struct {
-	OtherName string `protobuf:"bytes,2,opt,name=other_name,json=otherName,proto3,oneof"`
+func (x *SocketOptionTcpInfo) GetTcpiRto() uint32 {
+	if x != nil {
+		return x.TcpiRto
+	}
+	return 0
 }
 
-func (*Security_Tls_StandardName) isSecurity_Tls_CipherSuite() {}
+func (x *SocketOptionTcpInfo) GetTcpiAto() uint32 {
+	if x != nil {
+		return x.TcpiAto
+	}
+	return 0
+}
 
-func (*Security_Tls_OtherName) isSecurity_Tls_CipherSuite() {}
+func (x *SocketOptionTcpInfo) GetTcpiSndMss() uint32 {
+	if x != nil {
+		return x.TcpiSndMss
+	}
+	return 0
+}
 
-func (m *Security_Tls) GetCipherSuite() isSecurity_Tls_CipherSuite {
-	if m != nil {
-		return m.CipherSuite
+func (x *SocketOptionTcpInfo) GetTcpiRcvMss() uint32 {
+	if x != nil {
+		return x.TcpiRcvMss
 	}
-	return nil
+	return 0
 }
 
-func (m *Security_Tls) GetStandardName() string {
-	if x, ok := m.GetCipherSuite().(*Security_Tls_StandardName); ok {
-		return x.StandardName
+func (x *SocketOptionTcpInfo) GetTcpiUnacked() uint32 {
+	if x != nil {
+		return x.TcpiUnacked
 	}
-	return ""
+	return 0
 }
 
-func (m *Security_Tls) GetOtherName() string {
-	if x, ok := m.GetCipherSuite().(*Security_Tls_OtherName); ok {
-		return x.OtherName
+func (x *SocketOptionTcpInfo) GetTcpiSacked() uint32 {
+	if x != nil {
+		return x.TcpiSacked
 	}
-	return ""
+	return 0
 }
 
-func (m *Security_Tls) GetLocalCertificate() []byte {
-	if m != nil {
-		return m.LocalCertificate
+func (x *SocketOptionTcpInfo) GetTcpiLost() uint32 {
+	if x != nil {
+		return x.TcpiLost
 	}
-	return nil
+	return 0
 }
 
-func (m *Security_Tls) GetRemoteCertificate() []byte {
-	if m != nil {
-		return m.RemoteCertificate
+func (x *SocketOptionTcpInfo) GetTcpiRetrans() uint32 {
+	if x != nil {
+		return x.TcpiRetrans
 	}
-	return nil
+	return 0
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*Security_Tls) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*Security_Tls_StandardName)(nil),
-		(*Security_Tls_OtherName)(nil),
+func (x *SocketOptionTcpInfo) GetTcpiFackets() uint32 {
+	if x != nil {
+		return x.TcpiFackets
 	}
+	return 0
 }
 
-type Security_OtherSecurity struct {
-	// The human readable version of the value.
-	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	// The actual security details message.
-	Value                *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *SocketOptionTcpInfo) GetTcpiLastDataSent() uint32 {
+	if x != nil {
+		return x.TcpiLastDataSent
+	}
+	return 0
 }
 
-func (m *Security_OtherSecurity) Reset()         { *m = Security_OtherSecurity{} }
-func (m *Security_OtherSecurity) String() string { return proto.CompactTextString(m) }
-func (*Security_OtherSecurity) ProtoMessage()    {}
-func (*Security_OtherSecurity) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{15, 1}
+func (x *SocketOptionTcpInfo) GetTcpiLastAckSent() uint32 {
+	if x != nil {
+		return x.TcpiLastAckSent
+	}
+	return 0
 }
 
-func (m *Security_OtherSecurity) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Security_OtherSecurity.Unmarshal(m, b)
-}
-func (m *Security_OtherSecurity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Security_OtherSecurity.Marshal(b, m, deterministic)
-}
-func (m *Security_OtherSecurity) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Security_OtherSecurity.Merge(m, src)
-}
-func (m *Security_OtherSecurity) XXX_Size() int {
-	return xxx_messageInfo_Security_OtherSecurity.Size(m)
-}
-func (m *Security_OtherSecurity) XXX_DiscardUnknown() {
-	xxx_messageInfo_Security_OtherSecurity.DiscardUnknown(m)
+func (x *SocketOptionTcpInfo) GetTcpiLastDataRecv() uint32 {
+	if x != nil {
+		return x.TcpiLastDataRecv
+	}
+	return 0
 }
 
-var xxx_messageInfo_Security_OtherSecurity proto.InternalMessageInfo
-
-func (m *Security_OtherSecurity) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *SocketOptionTcpInfo) GetTcpiLastAckRecv() uint32 {
+	if x != nil {
+		return x.TcpiLastAckRecv
 	}
-	return ""
+	return 0
 }
 
-func (m *Security_OtherSecurity) GetValue() *any.Any {
-	if m != nil {
-		return m.Value
+func (x *SocketOptionTcpInfo) GetTcpiPmtu() uint32 {
+	if x != nil {
+		return x.TcpiPmtu
 	}
-	return nil
+	return 0
 }
 
-// SocketOption represents socket options for a socket.  Specifically, these
-// are the options returned by getsockopt().
-type SocketOption struct {
-	// The full name of the socket option.  Typically this will be the upper case
-	// name, such as "SO_REUSEPORT".
-	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	// The human readable value of this socket option.  At least one of value or
-	// additional will be set.
-	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
-	// Additional data associated with the socket option.  At least one of value
-	// or additional will be set.
-	Additional           *any.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *SocketOptionTcpInfo) GetTcpiRcvSsthresh() uint32 {
+	if x != nil {
+		return x.TcpiRcvSsthresh
+	}
+	return 0
 }
 
-func (m *SocketOption) Reset()         { *m = SocketOption{} }
-func (m *SocketOption) String() string { return proto.CompactTextString(m) }
-func (*SocketOption) ProtoMessage()    {}
-func (*SocketOption) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{16}
+func (x *SocketOptionTcpInfo) GetTcpiRtt() uint32 {
+	if x != nil {
+		return x.TcpiRtt
+	}
+	return 0
 }
 
-func (m *SocketOption) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SocketOption.Unmarshal(m, b)
-}
-func (m *SocketOption) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SocketOption.Marshal(b, m, deterministic)
-}
-func (m *SocketOption) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SocketOption.Merge(m, src)
-}
-func (m *SocketOption) XXX_Size() int {
-	return xxx_messageInfo_SocketOption.Size(m)
-}
-func (m *SocketOption) XXX_DiscardUnknown() {
-	xxx_messageInfo_SocketOption.DiscardUnknown(m)
+func (x *SocketOptionTcpInfo) GetTcpiRttvar() uint32 {
+	if x != nil {
+		return x.TcpiRttvar
+	}
+	return 0
 }
 
-var xxx_messageInfo_SocketOption proto.InternalMessageInfo
-
-func (m *SocketOption) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *SocketOptionTcpInfo) GetTcpiSndSsthresh() uint32 {
+	if x != nil {
+		return x.TcpiSndSsthresh
 	}
-	return ""
+	return 0
 }
 
-func (m *SocketOption) GetValue() string {
-	if m != nil {
-		return m.Value
+func (x *SocketOptionTcpInfo) GetTcpiSndCwnd() uint32 {
+	if x != nil {
+		return x.TcpiSndCwnd
 	}
-	return ""
+	return 0
 }
 
-func (m *SocketOption) GetAdditional() *any.Any {
-	if m != nil {
-		return m.Additional
+func (x *SocketOptionTcpInfo) GetTcpiAdvmss() uint32 {
+	if x != nil {
+		return x.TcpiAdvmss
 	}
-	return nil
+	return 0
 }
 
-// For use with SocketOption's additional field.  This is primarily used for
-// SO_RCVTIMEO and SO_SNDTIMEO
-type SocketOptionTimeout struct {
-	Duration             *duration.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
-	XXX_unrecognized     []byte             `json:"-"`
-	XXX_sizecache        int32              `json:"-"`
+func (x *SocketOptionTcpInfo) GetTcpiReordering() uint32 {
+	if x != nil {
+		return x.TcpiReordering
+	}
+	return 0
 }
 
-func (m *SocketOptionTimeout) Reset()         { *m = SocketOptionTimeout{} }
-func (m *SocketOptionTimeout) String() string { return proto.CompactTextString(m) }
-func (*SocketOptionTimeout) ProtoMessage()    {}
-func (*SocketOptionTimeout) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{17}
-}
+type GetTopChannelsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *SocketOptionTimeout) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SocketOptionTimeout.Unmarshal(m, b)
-}
-func (m *SocketOptionTimeout) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SocketOptionTimeout.Marshal(b, m, deterministic)
-}
-func (m *SocketOptionTimeout) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SocketOptionTimeout.Merge(m, src)
-}
-func (m *SocketOptionTimeout) XXX_Size() int {
-	return xxx_messageInfo_SocketOptionTimeout.Size(m)
-}
-func (m *SocketOptionTimeout) XXX_DiscardUnknown() {
-	xxx_messageInfo_SocketOptionTimeout.DiscardUnknown(m)
+	// start_channel_id indicates that only channels at or above this id should be
+	// included in the results.
+	// To request the first page, this should be set to 0. To request
+	// subsequent pages, the client generates this value by adding 1 to
+	// the highest seen result ID.
+	StartChannelId int64 `protobuf:"varint,1,opt,name=start_channel_id,json=startChannelId,proto3" json:"start_channel_id,omitempty"`
+	// If non-zero, the server will return a page of results containing
+	// at most this many items. If zero, the server will choose a
+	// reasonable page size.  Must never be negative.
+	MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
 }
 
-var xxx_messageInfo_SocketOptionTimeout proto.InternalMessageInfo
-
-func (m *SocketOptionTimeout) GetDuration() *duration.Duration {
-	if m != nil {
-		return m.Duration
+func (x *GetTopChannelsRequest) Reset() {
+	*x = GetTopChannelsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[20]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-// For use with SocketOption's additional field.  This is primarily used for
-// SO_LINGER.
-type SocketOptionLinger struct {
-	// active maps to `struct linger.l_onoff`
-	Active bool `protobuf:"varint,1,opt,name=active,proto3" json:"active,omitempty"`
-	// duration maps to `struct linger.l_linger`
-	Duration             *duration.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
-	XXX_unrecognized     []byte             `json:"-"`
-	XXX_sizecache        int32              `json:"-"`
+func (x *GetTopChannelsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *SocketOptionLinger) Reset()         { *m = SocketOptionLinger{} }
-func (m *SocketOptionLinger) String() string { return proto.CompactTextString(m) }
-func (*SocketOptionLinger) ProtoMessage()    {}
-func (*SocketOptionLinger) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{18}
-}
+func (*GetTopChannelsRequest) ProtoMessage() {}
 
-func (m *SocketOptionLinger) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SocketOptionLinger.Unmarshal(m, b)
-}
-func (m *SocketOptionLinger) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SocketOptionLinger.Marshal(b, m, deterministic)
-}
-func (m *SocketOptionLinger) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SocketOptionLinger.Merge(m, src)
-}
-func (m *SocketOptionLinger) XXX_Size() int {
-	return xxx_messageInfo_SocketOptionLinger.Size(m)
-}
-func (m *SocketOptionLinger) XXX_DiscardUnknown() {
-	xxx_messageInfo_SocketOptionLinger.DiscardUnknown(m)
+func (x *GetTopChannelsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[20]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SocketOptionLinger proto.InternalMessageInfo
+// Deprecated: Use GetTopChannelsRequest.ProtoReflect.Descriptor instead.
+func (*GetTopChannelsRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{20}
+}
 
-func (m *SocketOptionLinger) GetActive() bool {
-	if m != nil {
-		return m.Active
+func (x *GetTopChannelsRequest) GetStartChannelId() int64 {
+	if x != nil {
+		return x.StartChannelId
 	}
-	return false
+	return 0
 }
 
-func (m *SocketOptionLinger) GetDuration() *duration.Duration {
-	if m != nil {
-		return m.Duration
+func (x *GetTopChannelsRequest) GetMaxResults() int64 {
+	if x != nil {
+		return x.MaxResults
 	}
-	return nil
+	return 0
 }
 
-// For use with SocketOption's additional field.  Tcp info for
-// SOL_TCP and TCP_INFO.
-type SocketOptionTcpInfo struct {
-	TcpiState            uint32   `protobuf:"varint,1,opt,name=tcpi_state,json=tcpiState,proto3" json:"tcpi_state,omitempty"`
-	TcpiCaState          uint32   `protobuf:"varint,2,opt,name=tcpi_ca_state,json=tcpiCaState,proto3" json:"tcpi_ca_state,omitempty"`
-	TcpiRetransmits      uint32   `protobuf:"varint,3,opt,name=tcpi_retransmits,json=tcpiRetransmits,proto3" json:"tcpi_retransmits,omitempty"`
-	TcpiProbes           uint32   `protobuf:"varint,4,opt,name=tcpi_probes,json=tcpiProbes,proto3" json:"tcpi_probes,omitempty"`
-	TcpiBackoff          uint32   `protobuf:"varint,5,opt,name=tcpi_backoff,json=tcpiBackoff,proto3" json:"tcpi_backoff,omitempty"`
-	TcpiOptions          uint32   `protobuf:"varint,6,opt,name=tcpi_options,json=tcpiOptions,proto3" json:"tcpi_options,omitempty"`
-	TcpiSndWscale        uint32   `protobuf:"varint,7,opt,name=tcpi_snd_wscale,json=tcpiSndWscale,proto3" json:"tcpi_snd_wscale,omitempty"`
-	TcpiRcvWscale        uint32   `protobuf:"varint,8,opt,name=tcpi_rcv_wscale,json=tcpiRcvWscale,proto3" json:"tcpi_rcv_wscale,omitempty"`
-	TcpiRto              uint32   `protobuf:"varint,9,opt,name=tcpi_rto,json=tcpiRto,proto3" json:"tcpi_rto,omitempty"`
-	TcpiAto              uint32   `protobuf:"varint,10,opt,name=tcpi_ato,json=tcpiAto,proto3" json:"tcpi_ato,omitempty"`
-	TcpiSndMss           uint32   `protobuf:"varint,11,opt,name=tcpi_snd_mss,json=tcpiSndMss,proto3" json:"tcpi_snd_mss,omitempty"`
-	TcpiRcvMss           uint32   `protobuf:"varint,12,opt,name=tcpi_rcv_mss,json=tcpiRcvMss,proto3" json:"tcpi_rcv_mss,omitempty"`
-	TcpiUnacked          uint32   `protobuf:"varint,13,opt,name=tcpi_unacked,json=tcpiUnacked,proto3" json:"tcpi_unacked,omitempty"`
-	TcpiSacked           uint32   `protobuf:"varint,14,opt,name=tcpi_sacked,json=tcpiSacked,proto3" json:"tcpi_sacked,omitempty"`
-	TcpiLost             uint32   `protobuf:"varint,15,opt,name=tcpi_lost,json=tcpiLost,proto3" json:"tcpi_lost,omitempty"`
-	TcpiRetrans          uint32   `protobuf:"varint,16,opt,name=tcpi_retrans,json=tcpiRetrans,proto3" json:"tcpi_retrans,omitempty"`
-	TcpiFackets          uint32   `protobuf:"varint,17,opt,name=tcpi_fackets,json=tcpiFackets,proto3" json:"tcpi_fackets,omitempty"`
-	TcpiLastDataSent     uint32   `protobuf:"varint,18,opt,name=tcpi_last_data_sent,json=tcpiLastDataSent,proto3" json:"tcpi_last_data_sent,omitempty"`
-	TcpiLastAckSent      uint32   `protobuf:"varint,19,opt,name=tcpi_last_ack_sent,json=tcpiLastAckSent,proto3" json:"tcpi_last_ack_sent,omitempty"`
-	TcpiLastDataRecv     uint32   `protobuf:"varint,20,opt,name=tcpi_last_data_recv,json=tcpiLastDataRecv,proto3" json:"tcpi_last_data_recv,omitempty"`
-	TcpiLastAckRecv      uint32   `protobuf:"varint,21,opt,name=tcpi_last_ack_recv,json=tcpiLastAckRecv,proto3" json:"tcpi_last_ack_recv,omitempty"`
-	TcpiPmtu             uint32   `protobuf:"varint,22,opt,name=tcpi_pmtu,json=tcpiPmtu,proto3" json:"tcpi_pmtu,omitempty"`
-	TcpiRcvSsthresh      uint32   `protobuf:"varint,23,opt,name=tcpi_rcv_ssthresh,json=tcpiRcvSsthresh,proto3" json:"tcpi_rcv_ssthresh,omitempty"`
-	TcpiRtt              uint32   `protobuf:"varint,24,opt,name=tcpi_rtt,json=tcpiRtt,proto3" json:"tcpi_rtt,omitempty"`
-	TcpiRttvar           uint32   `protobuf:"varint,25,opt,name=tcpi_rttvar,json=tcpiRttvar,proto3" json:"tcpi_rttvar,omitempty"`
-	TcpiSndSsthresh      uint32   `protobuf:"varint,26,opt,name=tcpi_snd_ssthresh,json=tcpiSndSsthresh,proto3" json:"tcpi_snd_ssthresh,omitempty"`
-	TcpiSndCwnd          uint32   `protobuf:"varint,27,opt,name=tcpi_snd_cwnd,json=tcpiSndCwnd,proto3" json:"tcpi_snd_cwnd,omitempty"`
-	TcpiAdvmss           uint32   `protobuf:"varint,28,opt,name=tcpi_advmss,json=tcpiAdvmss,proto3" json:"tcpi_advmss,omitempty"`
-	TcpiReordering       uint32   `protobuf:"varint,29,opt,name=tcpi_reordering,json=tcpiReordering,proto3" json:"tcpi_reordering,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *SocketOptionTcpInfo) Reset()         { *m = SocketOptionTcpInfo{} }
-func (m *SocketOptionTcpInfo) String() string { return proto.CompactTextString(m) }
-func (*SocketOptionTcpInfo) ProtoMessage()    {}
-func (*SocketOptionTcpInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{19}
-}
+type GetTopChannelsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *SocketOptionTcpInfo) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SocketOptionTcpInfo.Unmarshal(m, b)
-}
-func (m *SocketOptionTcpInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SocketOptionTcpInfo.Marshal(b, m, deterministic)
-}
-func (m *SocketOptionTcpInfo) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SocketOptionTcpInfo.Merge(m, src)
+	// list of channels that the connection detail service knows about.  Sorted in
+	// ascending channel_id order.
+	// Must contain at least 1 result, otherwise 'end' must be true.
+	Channel []*Channel `protobuf:"bytes,1,rep,name=channel,proto3" json:"channel,omitempty"`
+	// If set, indicates that the list of channels is the final list.  Requesting
+	// more channels can only return more if they are created after this RPC
+	// completes.
+	End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
 }
-func (m *SocketOptionTcpInfo) XXX_Size() int {
-	return xxx_messageInfo_SocketOptionTcpInfo.Size(m)
+
+func (x *GetTopChannelsResponse) Reset() {
+	*x = GetTopChannelsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[21]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *SocketOptionTcpInfo) XXX_DiscardUnknown() {
-	xxx_messageInfo_SocketOptionTcpInfo.DiscardUnknown(m)
+
+func (x *GetTopChannelsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_SocketOptionTcpInfo proto.InternalMessageInfo
+func (*GetTopChannelsResponse) ProtoMessage() {}
 
-func (m *SocketOptionTcpInfo) GetTcpiState() uint32 {
-	if m != nil {
-		return m.TcpiState
+func (x *GetTopChannelsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[21]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiCaState() uint32 {
-	if m != nil {
-		return m.TcpiCaState
-	}
-	return 0
+// Deprecated: Use GetTopChannelsResponse.ProtoReflect.Descriptor instead.
+func (*GetTopChannelsResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{21}
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRetransmits() uint32 {
-	if m != nil {
-		return m.TcpiRetransmits
+func (x *GetTopChannelsResponse) GetChannel() []*Channel {
+	if x != nil {
+		return x.Channel
 	}
-	return 0
+	return nil
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiProbes() uint32 {
-	if m != nil {
-		return m.TcpiProbes
+func (x *GetTopChannelsResponse) GetEnd() bool {
+	if x != nil {
+		return x.End
 	}
-	return 0
+	return false
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiBackoff() uint32 {
-	if m != nil {
-		return m.TcpiBackoff
-	}
-	return 0
+type GetServersRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// start_server_id indicates that only servers at or above this id should be
+	// included in the results.
+	// To request the first page, this must be set to 0. To request
+	// subsequent pages, the client generates this value by adding 1 to
+	// the highest seen result ID.
+	StartServerId int64 `protobuf:"varint,1,opt,name=start_server_id,json=startServerId,proto3" json:"start_server_id,omitempty"`
+	// If non-zero, the server will return a page of results containing
+	// at most this many items. If zero, the server will choose a
+	// reasonable page size.  Must never be negative.
+	MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiOptions() uint32 {
-	if m != nil {
-		return m.TcpiOptions
+func (x *GetServersRequest) Reset() {
+	*x = GetServersRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[22]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiSndWscale() uint32 {
-	if m != nil {
-		return m.TcpiSndWscale
-	}
-	return 0
+func (x *GetServersRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRcvWscale() uint32 {
-	if m != nil {
-		return m.TcpiRcvWscale
+func (*GetServersRequest) ProtoMessage() {}
+
+func (x *GetServersRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[22]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRto() uint32 {
-	if m != nil {
-		return m.TcpiRto
-	}
-	return 0
+// Deprecated: Use GetServersRequest.ProtoReflect.Descriptor instead.
+func (*GetServersRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{22}
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiAto() uint32 {
-	if m != nil {
-		return m.TcpiAto
+func (x *GetServersRequest) GetStartServerId() int64 {
+	if x != nil {
+		return x.StartServerId
 	}
 	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiSndMss() uint32 {
-	if m != nil {
-		return m.TcpiSndMss
+func (x *GetServersRequest) GetMaxResults() int64 {
+	if x != nil {
+		return x.MaxResults
 	}
 	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRcvMss() uint32 {
-	if m != nil {
-		return m.TcpiRcvMss
-	}
-	return 0
+type GetServersResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// list of servers that the connection detail service knows about.  Sorted in
+	// ascending server_id order.
+	// Must contain at least 1 result, otherwise 'end' must be true.
+	Server []*Server `protobuf:"bytes,1,rep,name=server,proto3" json:"server,omitempty"`
+	// If set, indicates that the list of servers is the final list.  Requesting
+	// more servers will only return more if they are created after this RPC
+	// completes.
+	End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiUnacked() uint32 {
-	if m != nil {
-		return m.TcpiUnacked
+func (x *GetServersResponse) Reset() {
+	*x = GetServersResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[23]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiSacked() uint32 {
-	if m != nil {
-		return m.TcpiSacked
-	}
-	return 0
+func (x *GetServersResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiLost() uint32 {
-	if m != nil {
-		return m.TcpiLost
+func (*GetServersResponse) ProtoMessage() {}
+
+func (x *GetServersResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[23]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRetrans() uint32 {
-	if m != nil {
-		return m.TcpiRetrans
-	}
-	return 0
+// Deprecated: Use GetServersResponse.ProtoReflect.Descriptor instead.
+func (*GetServersResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{23}
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiFackets() uint32 {
-	if m != nil {
-		return m.TcpiFackets
+func (x *GetServersResponse) GetServer() []*Server {
+	if x != nil {
+		return x.Server
 	}
-	return 0
+	return nil
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiLastDataSent() uint32 {
-	if m != nil {
-		return m.TcpiLastDataSent
+func (x *GetServersResponse) GetEnd() bool {
+	if x != nil {
+		return x.End
 	}
-	return 0
+	return false
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiLastAckSent() uint32 {
-	if m != nil {
-		return m.TcpiLastAckSent
-	}
-	return 0
+type GetServerRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// server_id is the identifier of the specific server to get.
+	ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiLastDataRecv() uint32 {
-	if m != nil {
-		return m.TcpiLastDataRecv
+func (x *GetServerRequest) Reset() {
+	*x = GetServerRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[24]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiLastAckRecv() uint32 {
-	if m != nil {
-		return m.TcpiLastAckRecv
-	}
-	return 0
+func (x *GetServerRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiPmtu() uint32 {
-	if m != nil {
-		return m.TcpiPmtu
+func (*GetServerRequest) ProtoMessage() {}
+
+func (x *GetServerRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[24]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRcvSsthresh() uint32 {
-	if m != nil {
-		return m.TcpiRcvSsthresh
-	}
-	return 0
+// Deprecated: Use GetServerRequest.ProtoReflect.Descriptor instead.
+func (*GetServerRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{24}
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRtt() uint32 {
-	if m != nil {
-		return m.TcpiRtt
+func (x *GetServerRequest) GetServerId() int64 {
+	if x != nil {
+		return x.ServerId
 	}
 	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiRttvar() uint32 {
-	if m != nil {
-		return m.TcpiRttvar
-	}
-	return 0
+type GetServerResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The Server that corresponds to the requested server_id.  This field
+	// should be set.
+	Server *Server `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"`
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiSndSsthresh() uint32 {
-	if m != nil {
-		return m.TcpiSndSsthresh
+func (x *GetServerResponse) Reset() {
+	*x = GetServerResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[25]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return 0
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiSndCwnd() uint32 {
-	if m != nil {
-		return m.TcpiSndCwnd
-	}
-	return 0
+func (x *GetServerResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiAdvmss() uint32 {
-	if m != nil {
-		return m.TcpiAdvmss
+func (*GetServerResponse) ProtoMessage() {}
+
+func (x *GetServerResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[25]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *SocketOptionTcpInfo) GetTcpiReordering() uint32 {
-	if m != nil {
-		return m.TcpiReordering
+// Deprecated: Use GetServerResponse.ProtoReflect.Descriptor instead.
+func (*GetServerResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{25}
+}
+
+func (x *GetServerResponse) GetServer() *Server {
+	if x != nil {
+		return x.Server
 	}
-	return 0
+	return nil
 }
 
-type GetTopChannelsRequest struct {
-	// start_channel_id indicates that only channels at or above this id should be
+type GetServerSocketsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
+	// start_socket_id indicates that only sockets at or above this id should be
 	// included in the results.
-	// To request the first page, this should be set to 0. To request
+	// To request the first page, this must be set to 0. To request
 	// subsequent pages, the client generates this value by adding 1 to
 	// the highest seen result ID.
-	StartChannelId int64 `protobuf:"varint,1,opt,name=start_channel_id,json=startChannelId,proto3" json:"start_channel_id,omitempty"`
+	StartSocketId int64 `protobuf:"varint,2,opt,name=start_socket_id,json=startSocketId,proto3" json:"start_socket_id,omitempty"`
 	// If non-zero, the server will return a page of results containing
 	// at most this many items. If zero, the server will choose a
 	// reasonable page size.  Must never be negative.
-	MaxResults           int64    `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	MaxResults int64 `protobuf:"varint,3,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
 }
 
-func (m *GetTopChannelsRequest) Reset()         { *m = GetTopChannelsRequest{} }
-func (m *GetTopChannelsRequest) String() string { return proto.CompactTextString(m) }
-func (*GetTopChannelsRequest) ProtoMessage()    {}
-func (*GetTopChannelsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{20}
+func (x *GetServerSocketsRequest) Reset() {
+	*x = GetServerSocketsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[26]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *GetTopChannelsRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetTopChannelsRequest.Unmarshal(m, b)
+func (x *GetServerSocketsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GetTopChannelsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetTopChannelsRequest.Marshal(b, m, deterministic)
-}
-func (m *GetTopChannelsRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetTopChannelsRequest.Merge(m, src)
-}
-func (m *GetTopChannelsRequest) XXX_Size() int {
-	return xxx_messageInfo_GetTopChannelsRequest.Size(m)
+
+func (*GetServerSocketsRequest) ProtoMessage() {}
+
+func (x *GetServerSocketsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[26]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *GetTopChannelsRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetTopChannelsRequest.DiscardUnknown(m)
+
+// Deprecated: Use GetServerSocketsRequest.ProtoReflect.Descriptor instead.
+func (*GetServerSocketsRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{26}
 }
 
-var xxx_messageInfo_GetTopChannelsRequest proto.InternalMessageInfo
+func (x *GetServerSocketsRequest) GetServerId() int64 {
+	if x != nil {
+		return x.ServerId
+	}
+	return 0
+}
 
-func (m *GetTopChannelsRequest) GetStartChannelId() int64 {
-	if m != nil {
-		return m.StartChannelId
+func (x *GetServerSocketsRequest) GetStartSocketId() int64 {
+	if x != nil {
+		return x.StartSocketId
 	}
 	return 0
 }
 
-func (m *GetTopChannelsRequest) GetMaxResults() int64 {
-	if m != nil {
-		return m.MaxResults
+func (x *GetServerSocketsRequest) GetMaxResults() int64 {
+	if x != nil {
+		return x.MaxResults
 	}
 	return 0
 }
 
-type GetTopChannelsResponse struct {
-	// list of channels that the connection detail service knows about.  Sorted in
-	// ascending channel_id order.
-	// Must contain at least 1 result, otherwise 'end' must be true.
-	Channel []*Channel `protobuf:"bytes,1,rep,name=channel,proto3" json:"channel,omitempty"`
-	// If set, indicates that the list of channels is the final list.  Requesting
-	// more channels can only return more if they are created after this RPC
+type GetServerSocketsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// list of socket refs that the connection detail service knows about.  Sorted in
+	// ascending socket_id order.
+	// Must contain at least 1 result, otherwise 'end' must be true.
+	SocketRef []*SocketRef `protobuf:"bytes,1,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"`
+	// If set, indicates that the list of sockets is the final list.  Requesting
+	// more sockets will only return more if they are created after this RPC
 	// completes.
-	End                  bool     `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
 }
 
-func (m *GetTopChannelsResponse) Reset()         { *m = GetTopChannelsResponse{} }
-func (m *GetTopChannelsResponse) String() string { return proto.CompactTextString(m) }
-func (*GetTopChannelsResponse) ProtoMessage()    {}
-func (*GetTopChannelsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{21}
+func (x *GetServerSocketsResponse) Reset() {
+	*x = GetServerSocketsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[27]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *GetTopChannelsResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetTopChannelsResponse.Unmarshal(m, b)
-}
-func (m *GetTopChannelsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetTopChannelsResponse.Marshal(b, m, deterministic)
-}
-func (m *GetTopChannelsResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetTopChannelsResponse.Merge(m, src)
+func (x *GetServerSocketsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GetTopChannelsResponse) XXX_Size() int {
-	return xxx_messageInfo_GetTopChannelsResponse.Size(m)
-}
-func (m *GetTopChannelsResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetTopChannelsResponse.DiscardUnknown(m)
+
+func (*GetServerSocketsResponse) ProtoMessage() {}
+
+func (x *GetServerSocketsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[27]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_GetTopChannelsResponse proto.InternalMessageInfo
+// Deprecated: Use GetServerSocketsResponse.ProtoReflect.Descriptor instead.
+func (*GetServerSocketsResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{27}
+}
 
-func (m *GetTopChannelsResponse) GetChannel() []*Channel {
-	if m != nil {
-		return m.Channel
+func (x *GetServerSocketsResponse) GetSocketRef() []*SocketRef {
+	if x != nil {
+		return x.SocketRef
 	}
 	return nil
 }
 
-func (m *GetTopChannelsResponse) GetEnd() bool {
-	if m != nil {
-		return m.End
+func (x *GetServerSocketsResponse) GetEnd() bool {
+	if x != nil {
+		return x.End
 	}
 	return false
 }
 
-type GetServersRequest struct {
-	// start_server_id indicates that only servers at or above this id should be
-	// included in the results.
-	// To request the first page, this must be set to 0. To request
-	// subsequent pages, the client generates this value by adding 1 to
-	// the highest seen result ID.
-	StartServerId int64 `protobuf:"varint,1,opt,name=start_server_id,json=startServerId,proto3" json:"start_server_id,omitempty"`
-	// If non-zero, the server will return a page of results containing
-	// at most this many items. If zero, the server will choose a
-	// reasonable page size.  Must never be negative.
-	MaxResults           int64    `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+type GetChannelRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GetServersRequest) Reset()         { *m = GetServersRequest{} }
-func (m *GetServersRequest) String() string { return proto.CompactTextString(m) }
-func (*GetServersRequest) ProtoMessage()    {}
-func (*GetServersRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{22}
+	// channel_id is the identifier of the specific channel to get.
+	ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"`
 }
 
-func (m *GetServersRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetServersRequest.Unmarshal(m, b)
-}
-func (m *GetServersRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetServersRequest.Marshal(b, m, deterministic)
-}
-func (m *GetServersRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetServersRequest.Merge(m, src)
-}
-func (m *GetServersRequest) XXX_Size() int {
-	return xxx_messageInfo_GetServersRequest.Size(m)
+func (x *GetChannelRequest) Reset() {
+	*x = GetChannelRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[28]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetServersRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetServersRequest.DiscardUnknown(m)
+
+func (x *GetChannelRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetServersRequest proto.InternalMessageInfo
+func (*GetChannelRequest) ProtoMessage() {}
 
-func (m *GetServersRequest) GetStartServerId() int64 {
-	if m != nil {
-		return m.StartServerId
+func (x *GetChannelRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[28]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *GetServersRequest) GetMaxResults() int64 {
-	if m != nil {
-		return m.MaxResults
+// Deprecated: Use GetChannelRequest.ProtoReflect.Descriptor instead.
+func (*GetChannelRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{28}
+}
+
+func (x *GetChannelRequest) GetChannelId() int64 {
+	if x != nil {
+		return x.ChannelId
 	}
 	return 0
 }
 
-type GetServersResponse struct {
-	// list of servers that the connection detail service knows about.  Sorted in
-	// ascending server_id order.
-	// Must contain at least 1 result, otherwise 'end' must be true.
-	Server []*Server `protobuf:"bytes,1,rep,name=server,proto3" json:"server,omitempty"`
-	// If set, indicates that the list of servers is the final list.  Requesting
-	// more servers will only return more if they are created after this RPC
-	// completes.
-	End                  bool     `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+type GetChannelResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GetServersResponse) Reset()         { *m = GetServersResponse{} }
-func (m *GetServersResponse) String() string { return proto.CompactTextString(m) }
-func (*GetServersResponse) ProtoMessage()    {}
-func (*GetServersResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{23}
+	// The Channel that corresponds to the requested channel_id.  This field
+	// should be set.
+	Channel *Channel `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"`
 }
 
-func (m *GetServersResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetServersResponse.Unmarshal(m, b)
-}
-func (m *GetServersResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetServersResponse.Marshal(b, m, deterministic)
-}
-func (m *GetServersResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetServersResponse.Merge(m, src)
-}
-func (m *GetServersResponse) XXX_Size() int {
-	return xxx_messageInfo_GetServersResponse.Size(m)
+func (x *GetChannelResponse) Reset() {
+	*x = GetChannelResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[29]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetServersResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetServersResponse.DiscardUnknown(m)
+
+func (x *GetChannelResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetServersResponse proto.InternalMessageInfo
+func (*GetChannelResponse) ProtoMessage() {}
 
-func (m *GetServersResponse) GetServer() []*Server {
-	if m != nil {
-		return m.Server
+func (x *GetChannelResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[29]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-func (m *GetServersResponse) GetEnd() bool {
-	if m != nil {
-		return m.End
-	}
-	return false
+// Deprecated: Use GetChannelResponse.ProtoReflect.Descriptor instead.
+func (*GetChannelResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{29}
 }
 
-type GetServerRequest struct {
-	// server_id is the identifier of the specific server to get.
-	ServerId             int64    `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *GetChannelResponse) GetChannel() *Channel {
+	if x != nil {
+		return x.Channel
+	}
+	return nil
 }
 
-func (m *GetServerRequest) Reset()         { *m = GetServerRequest{} }
-func (m *GetServerRequest) String() string { return proto.CompactTextString(m) }
-func (*GetServerRequest) ProtoMessage()    {}
-func (*GetServerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{24}
-}
+type GetSubchannelRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GetServerRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetServerRequest.Unmarshal(m, b)
-}
-func (m *GetServerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetServerRequest.Marshal(b, m, deterministic)
-}
-func (m *GetServerRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetServerRequest.Merge(m, src)
+	// subchannel_id is the identifier of the specific subchannel to get.
+	SubchannelId int64 `protobuf:"varint,1,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"`
 }
-func (m *GetServerRequest) XXX_Size() int {
-	return xxx_messageInfo_GetServerRequest.Size(m)
+
+func (x *GetSubchannelRequest) Reset() {
+	*x = GetSubchannelRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[30]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetServerRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetServerRequest.DiscardUnknown(m)
+
+func (x *GetSubchannelRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetServerRequest proto.InternalMessageInfo
+func (*GetSubchannelRequest) ProtoMessage() {}
 
-func (m *GetServerRequest) GetServerId() int64 {
-	if m != nil {
-		return m.ServerId
+func (x *GetSubchannelRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[30]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-type GetServerResponse struct {
-	// The Server that corresponds to the requested server_id.  This field
-	// should be set.
-	Server               *Server  `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+// Deprecated: Use GetSubchannelRequest.ProtoReflect.Descriptor instead.
+func (*GetSubchannelRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{30}
 }
 
-func (m *GetServerResponse) Reset()         { *m = GetServerResponse{} }
-func (m *GetServerResponse) String() string { return proto.CompactTextString(m) }
-func (*GetServerResponse) ProtoMessage()    {}
-func (*GetServerResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{25}
+func (x *GetSubchannelRequest) GetSubchannelId() int64 {
+	if x != nil {
+		return x.SubchannelId
+	}
+	return 0
 }
 
-func (m *GetServerResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetServerResponse.Unmarshal(m, b)
-}
-func (m *GetServerResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetServerResponse.Marshal(b, m, deterministic)
-}
-func (m *GetServerResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetServerResponse.Merge(m, src)
+type GetSubchannelResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The Subchannel that corresponds to the requested subchannel_id.  This
+	// field should be set.
+	Subchannel *Subchannel `protobuf:"bytes,1,opt,name=subchannel,proto3" json:"subchannel,omitempty"`
 }
-func (m *GetServerResponse) XXX_Size() int {
-	return xxx_messageInfo_GetServerResponse.Size(m)
+
+func (x *GetSubchannelResponse) Reset() {
+	*x = GetSubchannelResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[31]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetServerResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetServerResponse.DiscardUnknown(m)
+
+func (x *GetSubchannelResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetServerResponse proto.InternalMessageInfo
+func (*GetSubchannelResponse) ProtoMessage() {}
 
-func (m *GetServerResponse) GetServer() *Server {
-	if m != nil {
-		return m.Server
+func (x *GetSubchannelResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[31]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-type GetServerSocketsRequest struct {
-	ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
-	// start_socket_id indicates that only sockets at or above this id should be
-	// included in the results.
-	// To request the first page, this must be set to 0. To request
-	// subsequent pages, the client generates this value by adding 1 to
-	// the highest seen result ID.
-	StartSocketId int64 `protobuf:"varint,2,opt,name=start_socket_id,json=startSocketId,proto3" json:"start_socket_id,omitempty"`
-	// If non-zero, the server will return a page of results containing
-	// at most this many items. If zero, the server will choose a
-	// reasonable page size.  Must never be negative.
-	MaxResults           int64    `protobuf:"varint,3,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+// Deprecated: Use GetSubchannelResponse.ProtoReflect.Descriptor instead.
+func (*GetSubchannelResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{31}
 }
 
-func (m *GetServerSocketsRequest) Reset()         { *m = GetServerSocketsRequest{} }
-func (m *GetServerSocketsRequest) String() string { return proto.CompactTextString(m) }
-func (*GetServerSocketsRequest) ProtoMessage()    {}
-func (*GetServerSocketsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{26}
+func (x *GetSubchannelResponse) GetSubchannel() *Subchannel {
+	if x != nil {
+		return x.Subchannel
+	}
+	return nil
 }
 
-func (m *GetServerSocketsRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetServerSocketsRequest.Unmarshal(m, b)
-}
-func (m *GetServerSocketsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetServerSocketsRequest.Marshal(b, m, deterministic)
-}
-func (m *GetServerSocketsRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetServerSocketsRequest.Merge(m, src)
+type GetSocketRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// socket_id is the identifier of the specific socket to get.
+	SocketId int64 `protobuf:"varint,1,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"`
+	// If true, the response will contain only high level information
+	// that is inexpensive to obtain. Fields thay may be omitted are
+	// documented.
+	Summary bool `protobuf:"varint,2,opt,name=summary,proto3" json:"summary,omitempty"`
 }
-func (m *GetServerSocketsRequest) XXX_Size() int {
-	return xxx_messageInfo_GetServerSocketsRequest.Size(m)
+
+func (x *GetSocketRequest) Reset() {
+	*x = GetSocketRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[32]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetServerSocketsRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetServerSocketsRequest.DiscardUnknown(m)
+
+func (x *GetSocketRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetServerSocketsRequest proto.InternalMessageInfo
+func (*GetSocketRequest) ProtoMessage() {}
 
-func (m *GetServerSocketsRequest) GetServerId() int64 {
-	if m != nil {
-		return m.ServerId
+func (x *GetSocketRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[32]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *GetServerSocketsRequest) GetStartSocketId() int64 {
-	if m != nil {
-		return m.StartSocketId
-	}
-	return 0
+// Deprecated: Use GetSocketRequest.ProtoReflect.Descriptor instead.
+func (*GetSocketRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{32}
 }
 
-func (m *GetServerSocketsRequest) GetMaxResults() int64 {
-	if m != nil {
-		return m.MaxResults
+func (x *GetSocketRequest) GetSocketId() int64 {
+	if x != nil {
+		return x.SocketId
 	}
 	return 0
 }
 
-type GetServerSocketsResponse struct {
-	// list of socket refs that the connection detail service knows about.  Sorted in
-	// ascending socket_id order.
-	// Must contain at least 1 result, otherwise 'end' must be true.
-	SocketRef []*SocketRef `protobuf:"bytes,1,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"`
-	// If set, indicates that the list of sockets is the final list.  Requesting
-	// more sockets will only return more if they are created after this RPC
-	// completes.
-	End                  bool     `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *GetSocketRequest) GetSummary() bool {
+	if x != nil {
+		return x.Summary
+	}
+	return false
 }
 
-func (m *GetServerSocketsResponse) Reset()         { *m = GetServerSocketsResponse{} }
-func (m *GetServerSocketsResponse) String() string { return proto.CompactTextString(m) }
-func (*GetServerSocketsResponse) ProtoMessage()    {}
-func (*GetServerSocketsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{27}
-}
+type GetSocketResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GetServerSocketsResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetServerSocketsResponse.Unmarshal(m, b)
-}
-func (m *GetServerSocketsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetServerSocketsResponse.Marshal(b, m, deterministic)
-}
-func (m *GetServerSocketsResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetServerSocketsResponse.Merge(m, src)
+	// The Socket that corresponds to the requested socket_id.  This field
+	// should be set.
+	Socket *Socket `protobuf:"bytes,1,opt,name=socket,proto3" json:"socket,omitempty"`
 }
-func (m *GetServerSocketsResponse) XXX_Size() int {
-	return xxx_messageInfo_GetServerSocketsResponse.Size(m)
+
+func (x *GetSocketResponse) Reset() {
+	*x = GetSocketResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[33]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetServerSocketsResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetServerSocketsResponse.DiscardUnknown(m)
+
+func (x *GetSocketResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetServerSocketsResponse proto.InternalMessageInfo
+func (*GetSocketResponse) ProtoMessage() {}
 
-func (m *GetServerSocketsResponse) GetSocketRef() []*SocketRef {
-	if m != nil {
-		return m.SocketRef
+func (x *GetSocketResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[33]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-func (m *GetServerSocketsResponse) GetEnd() bool {
-	if m != nil {
-		return m.End
-	}
-	return false
+// Deprecated: Use GetSocketResponse.ProtoReflect.Descriptor instead.
+func (*GetSocketResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{33}
 }
 
-type GetChannelRequest struct {
-	// channel_id is the identifier of the specific channel to get.
-	ChannelId            int64    `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *GetSocketResponse) GetSocket() *Socket {
+	if x != nil {
+		return x.Socket
+	}
+	return nil
 }
 
-func (m *GetChannelRequest) Reset()         { *m = GetChannelRequest{} }
-func (m *GetChannelRequest) String() string { return proto.CompactTextString(m) }
-func (*GetChannelRequest) ProtoMessage()    {}
-func (*GetChannelRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{28}
-}
+type Address_TcpIpAddress struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GetChannelRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetChannelRequest.Unmarshal(m, b)
-}
-func (m *GetChannelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetChannelRequest.Marshal(b, m, deterministic)
-}
-func (m *GetChannelRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetChannelRequest.Merge(m, src)
+	// Either the IPv4 or IPv6 address in bytes.  Will be either 4 bytes or 16
+	// bytes in length.
+	IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"`
+	// 0-64k, or -1 if not appropriate.
+	Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
 }
-func (m *GetChannelRequest) XXX_Size() int {
-	return xxx_messageInfo_GetChannelRequest.Size(m)
+
+func (x *Address_TcpIpAddress) Reset() {
+	*x = Address_TcpIpAddress{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[34]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetChannelRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetChannelRequest.DiscardUnknown(m)
+
+func (x *Address_TcpIpAddress) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetChannelRequest proto.InternalMessageInfo
+func (*Address_TcpIpAddress) ProtoMessage() {}
 
-func (m *GetChannelRequest) GetChannelId() int64 {
-	if m != nil {
-		return m.ChannelId
+func (x *Address_TcpIpAddress) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[34]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-type GetChannelResponse struct {
-	// The Channel that corresponds to the requested channel_id.  This field
-	// should be set.
-	Channel              *Channel `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+// Deprecated: Use Address_TcpIpAddress.ProtoReflect.Descriptor instead.
+func (*Address_TcpIpAddress) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 0}
 }
 
-func (m *GetChannelResponse) Reset()         { *m = GetChannelResponse{} }
-func (m *GetChannelResponse) String() string { return proto.CompactTextString(m) }
-func (*GetChannelResponse) ProtoMessage()    {}
-func (*GetChannelResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{29}
+func (x *Address_TcpIpAddress) GetIpAddress() []byte {
+	if x != nil {
+		return x.IpAddress
+	}
+	return nil
 }
 
-func (m *GetChannelResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetChannelResponse.Unmarshal(m, b)
-}
-func (m *GetChannelResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetChannelResponse.Marshal(b, m, deterministic)
+func (x *Address_TcpIpAddress) GetPort() int32 {
+	if x != nil {
+		return x.Port
+	}
+	return 0
 }
-func (m *GetChannelResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetChannelResponse.Merge(m, src)
+
+// A Unix Domain Socket address.
+type Address_UdsAddress struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Filename string `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"`
 }
-func (m *GetChannelResponse) XXX_Size() int {
-	return xxx_messageInfo_GetChannelResponse.Size(m)
+
+func (x *Address_UdsAddress) Reset() {
+	*x = Address_UdsAddress{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[35]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetChannelResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetChannelResponse.DiscardUnknown(m)
+
+func (x *Address_UdsAddress) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetChannelResponse proto.InternalMessageInfo
+func (*Address_UdsAddress) ProtoMessage() {}
 
-func (m *GetChannelResponse) GetChannel() *Channel {
-	if m != nil {
-		return m.Channel
+func (x *Address_UdsAddress) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[35]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-type GetSubchannelRequest struct {
-	// subchannel_id is the identifier of the specific subchannel to get.
-	SubchannelId         int64    `protobuf:"varint,1,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+// Deprecated: Use Address_UdsAddress.ProtoReflect.Descriptor instead.
+func (*Address_UdsAddress) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 1}
 }
 
-func (m *GetSubchannelRequest) Reset()         { *m = GetSubchannelRequest{} }
-func (m *GetSubchannelRequest) String() string { return proto.CompactTextString(m) }
-func (*GetSubchannelRequest) ProtoMessage()    {}
-func (*GetSubchannelRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{30}
+func (x *Address_UdsAddress) GetFilename() string {
+	if x != nil {
+		return x.Filename
+	}
+	return ""
 }
 
-func (m *GetSubchannelRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetSubchannelRequest.Unmarshal(m, b)
-}
-func (m *GetSubchannelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetSubchannelRequest.Marshal(b, m, deterministic)
-}
-func (m *GetSubchannelRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetSubchannelRequest.Merge(m, src)
+// An address type not included above.
+type Address_OtherAddress struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The human readable version of the value.  This value should be set.
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// The actual address message.
+	Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
-func (m *GetSubchannelRequest) XXX_Size() int {
-	return xxx_messageInfo_GetSubchannelRequest.Size(m)
+
+func (x *Address_OtherAddress) Reset() {
+	*x = Address_OtherAddress{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[36]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetSubchannelRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetSubchannelRequest.DiscardUnknown(m)
+
+func (x *Address_OtherAddress) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_GetSubchannelRequest proto.InternalMessageInfo
+func (*Address_OtherAddress) ProtoMessage() {}
 
-func (m *GetSubchannelRequest) GetSubchannelId() int64 {
-	if m != nil {
-		return m.SubchannelId
+func (x *Address_OtherAddress) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[36]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-type GetSubchannelResponse struct {
-	// The Subchannel that corresponds to the requested subchannel_id.  This
-	// field should be set.
-	Subchannel           *Subchannel `protobuf:"bytes,1,opt,name=subchannel,proto3" json:"subchannel,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
+// Deprecated: Use Address_OtherAddress.ProtoReflect.Descriptor instead.
+func (*Address_OtherAddress) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 2}
 }
 
-func (m *GetSubchannelResponse) Reset()         { *m = GetSubchannelResponse{} }
-func (m *GetSubchannelResponse) String() string { return proto.CompactTextString(m) }
-func (*GetSubchannelResponse) ProtoMessage()    {}
-func (*GetSubchannelResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{31}
+func (x *Address_OtherAddress) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
 }
 
-func (m *GetSubchannelResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetSubchannelResponse.Unmarshal(m, b)
+func (x *Address_OtherAddress) GetValue() *any.Any {
+	if x != nil {
+		return x.Value
+	}
+	return nil
 }
-func (m *GetSubchannelResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetSubchannelResponse.Marshal(b, m, deterministic)
+
+type Security_Tls struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to CipherSuite:
+	//	*Security_Tls_StandardName
+	//	*Security_Tls_OtherName
+	CipherSuite isSecurity_Tls_CipherSuite `protobuf_oneof:"cipher_suite"`
+	// the certificate used by this endpoint.
+	LocalCertificate []byte `protobuf:"bytes,3,opt,name=local_certificate,json=localCertificate,proto3" json:"local_certificate,omitempty"`
+	// the certificate used by the remote endpoint.
+	RemoteCertificate []byte `protobuf:"bytes,4,opt,name=remote_certificate,json=remoteCertificate,proto3" json:"remote_certificate,omitempty"`
 }
-func (m *GetSubchannelResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetSubchannelResponse.Merge(m, src)
+
+func (x *Security_Tls) Reset() {
+	*x = Security_Tls{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[37]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetSubchannelResponse) XXX_Size() int {
-	return xxx_messageInfo_GetSubchannelResponse.Size(m)
+
+func (x *Security_Tls) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GetSubchannelResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetSubchannelResponse.DiscardUnknown(m)
+
+func (*Security_Tls) ProtoMessage() {}
+
+func (x *Security_Tls) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[37]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_GetSubchannelResponse proto.InternalMessageInfo
+// Deprecated: Use Security_Tls.ProtoReflect.Descriptor instead.
+func (*Security_Tls) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15, 0}
+}
 
-func (m *GetSubchannelResponse) GetSubchannel() *Subchannel {
+func (m *Security_Tls) GetCipherSuite() isSecurity_Tls_CipherSuite {
 	if m != nil {
-		return m.Subchannel
+		return m.CipherSuite
 	}
 	return nil
 }
 
-type GetSocketRequest struct {
-	// socket_id is the identifier of the specific socket to get.
-	SocketId int64 `protobuf:"varint,1,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"`
-	// If true, the response will contain only high level information
-	// that is inexpensive to obtain. Fields thay may be omitted are
-	// documented.
-	Summary              bool     `protobuf:"varint,2,opt,name=summary,proto3" json:"summary,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *Security_Tls) GetStandardName() string {
+	if x, ok := x.GetCipherSuite().(*Security_Tls_StandardName); ok {
+		return x.StandardName
+	}
+	return ""
 }
 
-func (m *GetSocketRequest) Reset()         { *m = GetSocketRequest{} }
-func (m *GetSocketRequest) String() string { return proto.CompactTextString(m) }
-func (*GetSocketRequest) ProtoMessage()    {}
-func (*GetSocketRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{32}
+func (x *Security_Tls) GetOtherName() string {
+	if x, ok := x.GetCipherSuite().(*Security_Tls_OtherName); ok {
+		return x.OtherName
+	}
+	return ""
 }
 
-func (m *GetSocketRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetSocketRequest.Unmarshal(m, b)
+func (x *Security_Tls) GetLocalCertificate() []byte {
+	if x != nil {
+		return x.LocalCertificate
+	}
+	return nil
 }
-func (m *GetSocketRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetSocketRequest.Marshal(b, m, deterministic)
+
+func (x *Security_Tls) GetRemoteCertificate() []byte {
+	if x != nil {
+		return x.RemoteCertificate
+	}
+	return nil
 }
-func (m *GetSocketRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetSocketRequest.Merge(m, src)
+
+type isSecurity_Tls_CipherSuite interface {
+	isSecurity_Tls_CipherSuite()
 }
-func (m *GetSocketRequest) XXX_Size() int {
-	return xxx_messageInfo_GetSocketRequest.Size(m)
+
+type Security_Tls_StandardName struct {
+	// The cipher suite name in the RFC 4346 format:
+	// https://tools.ietf.org/html/rfc4346#appendix-C
+	StandardName string `protobuf:"bytes,1,opt,name=standard_name,json=standardName,proto3,oneof"`
 }
-func (m *GetSocketRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetSocketRequest.DiscardUnknown(m)
+
+type Security_Tls_OtherName struct {
+	// Some other way to describe the cipher suite if
+	// the RFC 4346 name is not available.
+	OtherName string `protobuf:"bytes,2,opt,name=other_name,json=otherName,proto3,oneof"`
 }
 
-var xxx_messageInfo_GetSocketRequest proto.InternalMessageInfo
+func (*Security_Tls_StandardName) isSecurity_Tls_CipherSuite() {}
 
-func (m *GetSocketRequest) GetSocketId() int64 {
-	if m != nil {
-		return m.SocketId
-	}
-	return 0
+func (*Security_Tls_OtherName) isSecurity_Tls_CipherSuite() {}
+
+type Security_OtherSecurity struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The human readable version of the value.
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// The actual security details message.
+	Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
-func (m *GetSocketRequest) GetSummary() bool {
-	if m != nil {
-		return m.Summary
+func (x *Security_OtherSecurity) Reset() {
+	*x = Security_OtherSecurity{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[38]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return false
 }
 
-type GetSocketResponse struct {
-	// The Socket that corresponds to the requested socket_id.  This field
-	// should be set.
-	Socket               *Socket  `protobuf:"bytes,1,opt,name=socket,proto3" json:"socket,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *Security_OtherSecurity) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *GetSocketResponse) Reset()         { *m = GetSocketResponse{} }
-func (m *GetSocketResponse) String() string { return proto.CompactTextString(m) }
-func (*GetSocketResponse) ProtoMessage()    {}
-func (*GetSocketResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6ee37dfd35a8ab00, []int{33}
-}
+func (*Security_OtherSecurity) ProtoMessage() {}
 
-func (m *GetSocketResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetSocketResponse.Unmarshal(m, b)
-}
-func (m *GetSocketResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetSocketResponse.Marshal(b, m, deterministic)
-}
-func (m *GetSocketResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetSocketResponse.Merge(m, src)
-}
-func (m *GetSocketResponse) XXX_Size() int {
-	return xxx_messageInfo_GetSocketResponse.Size(m)
+func (x *Security_OtherSecurity) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[38]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *GetSocketResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetSocketResponse.DiscardUnknown(m)
+
+// Deprecated: Use Security_OtherSecurity.ProtoReflect.Descriptor instead.
+func (*Security_OtherSecurity) Descriptor() ([]byte, []int) {
+	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15, 1}
 }
 
-var xxx_messageInfo_GetSocketResponse proto.InternalMessageInfo
+func (x *Security_OtherSecurity) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
 
-func (m *GetSocketResponse) GetSocket() *Socket {
-	if m != nil {
-		return m.Socket
+func (x *Security_OtherSecurity) GetValue() *any.Any {
+	if x != nil {
+		return x.Value
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.channelz.v1.ChannelConnectivityState_State", ChannelConnectivityState_State_name, ChannelConnectivityState_State_value)
-	proto.RegisterEnum("grpc.channelz.v1.ChannelTraceEvent_Severity", ChannelTraceEvent_Severity_name, ChannelTraceEvent_Severity_value)
-	proto.RegisterType((*Channel)(nil), "grpc.channelz.v1.Channel")
-	proto.RegisterType((*Subchannel)(nil), "grpc.channelz.v1.Subchannel")
-	proto.RegisterType((*ChannelConnectivityState)(nil), "grpc.channelz.v1.ChannelConnectivityState")
-	proto.RegisterType((*ChannelData)(nil), "grpc.channelz.v1.ChannelData")
-	proto.RegisterType((*ChannelTraceEvent)(nil), "grpc.channelz.v1.ChannelTraceEvent")
-	proto.RegisterType((*ChannelTrace)(nil), "grpc.channelz.v1.ChannelTrace")
-	proto.RegisterType((*ChannelRef)(nil), "grpc.channelz.v1.ChannelRef")
-	proto.RegisterType((*SubchannelRef)(nil), "grpc.channelz.v1.SubchannelRef")
-	proto.RegisterType((*SocketRef)(nil), "grpc.channelz.v1.SocketRef")
-	proto.RegisterType((*ServerRef)(nil), "grpc.channelz.v1.ServerRef")
-	proto.RegisterType((*Server)(nil), "grpc.channelz.v1.Server")
-	proto.RegisterType((*ServerData)(nil), "grpc.channelz.v1.ServerData")
-	proto.RegisterType((*Socket)(nil), "grpc.channelz.v1.Socket")
-	proto.RegisterType((*SocketData)(nil), "grpc.channelz.v1.SocketData")
-	proto.RegisterType((*Address)(nil), "grpc.channelz.v1.Address")
-	proto.RegisterType((*Address_TcpIpAddress)(nil), "grpc.channelz.v1.Address.TcpIpAddress")
-	proto.RegisterType((*Address_UdsAddress)(nil), "grpc.channelz.v1.Address.UdsAddress")
-	proto.RegisterType((*Address_OtherAddress)(nil), "grpc.channelz.v1.Address.OtherAddress")
-	proto.RegisterType((*Security)(nil), "grpc.channelz.v1.Security")
-	proto.RegisterType((*Security_Tls)(nil), "grpc.channelz.v1.Security.Tls")
-	proto.RegisterType((*Security_OtherSecurity)(nil), "grpc.channelz.v1.Security.OtherSecurity")
-	proto.RegisterType((*SocketOption)(nil), "grpc.channelz.v1.SocketOption")
-	proto.RegisterType((*SocketOptionTimeout)(nil), "grpc.channelz.v1.SocketOptionTimeout")
-	proto.RegisterType((*SocketOptionLinger)(nil), "grpc.channelz.v1.SocketOptionLinger")
-	proto.RegisterType((*SocketOptionTcpInfo)(nil), "grpc.channelz.v1.SocketOptionTcpInfo")
-	proto.RegisterType((*GetTopChannelsRequest)(nil), "grpc.channelz.v1.GetTopChannelsRequest")
-	proto.RegisterType((*GetTopChannelsResponse)(nil), "grpc.channelz.v1.GetTopChannelsResponse")
-	proto.RegisterType((*GetServersRequest)(nil), "grpc.channelz.v1.GetServersRequest")
-	proto.RegisterType((*GetServersResponse)(nil), "grpc.channelz.v1.GetServersResponse")
-	proto.RegisterType((*GetServerRequest)(nil), "grpc.channelz.v1.GetServerRequest")
-	proto.RegisterType((*GetServerResponse)(nil), "grpc.channelz.v1.GetServerResponse")
-	proto.RegisterType((*GetServerSocketsRequest)(nil), "grpc.channelz.v1.GetServerSocketsRequest")
-	proto.RegisterType((*GetServerSocketsResponse)(nil), "grpc.channelz.v1.GetServerSocketsResponse")
-	proto.RegisterType((*GetChannelRequest)(nil), "grpc.channelz.v1.GetChannelRequest")
-	proto.RegisterType((*GetChannelResponse)(nil), "grpc.channelz.v1.GetChannelResponse")
-	proto.RegisterType((*GetSubchannelRequest)(nil), "grpc.channelz.v1.GetSubchannelRequest")
-	proto.RegisterType((*GetSubchannelResponse)(nil), "grpc.channelz.v1.GetSubchannelResponse")
-	proto.RegisterType((*GetSocketRequest)(nil), "grpc.channelz.v1.GetSocketRequest")
-	proto.RegisterType((*GetSocketResponse)(nil), "grpc.channelz.v1.GetSocketResponse")
-}
-
-func init() { proto.RegisterFile("grpc/channelz/v1/channelz.proto", fileDescriptor_6ee37dfd35a8ab00) }
-
-var fileDescriptor_6ee37dfd35a8ab00 = []byte{
-	// 2584 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x59, 0x4b, 0x6f, 0xdb, 0xd8,
-	0xf5, 0xb7, 0xde, 0xd4, 0xd1, 0x23, 0xf2, 0x4d, 0x26, 0x43, 0x2b, 0x99, 0xb1, 0xff, 0xf4, 0x4c,
-	0xc6, 0x93, 0xfc, 0x23, 0xc7, 0x9e, 0x34, 0x28, 0x3a, 0x2d, 0x3a, 0xb6, 0x62, 0xc7, 0x72, 0x1d,
-	0x39, 0xa0, 0xe4, 0x49, 0xa6, 0x28, 0xca, 0xa1, 0xc9, 0x6b, 0x99, 0x35, 0x45, 0xaa, 0xbc, 0x57,
-	0xf2, 0x24, 0x9b, 0x2e, 0xba, 0xef, 0xb2, 0x28, 0xfa, 0x01, 0xba, 0xe9, 0xa2, 0x40, 0x81, 0x02,
-	0xed, 0xb6, 0xdf, 0xa6, 0xdf, 0xa2, 0xb8, 0x0f, 0x3e, 0xf4, 0xb2, 0x14, 0x64, 0xd9, 0x8d, 0x21,
-	0x1e, 0xfe, 0xce, 0xef, 0x9c, 0x7b, 0x5e, 0xf7, 0xf2, 0x1a, 0xd6, 0x7b, 0xc1, 0xc0, 0xda, 0xb6,
-	0x2e, 0x4d, 0xcf, 0xc3, 0xee, 0xbb, 0xed, 0xd1, 0x4e, 0xf4, 0xbb, 0x31, 0x08, 0x7c, 0xea, 0xa3,
-	0x1a, 0x03, 0x34, 0x22, 0xe1, 0x68, 0xa7, 0xbe, 0xd6, 0xf3, 0xfd, 0x9e, 0x8b, 0xb7, 0xf9, 0xfb,
-	0xf3, 0xe1, 0xc5, 0xb6, 0xe9, 0xbd, 0x15, 0xe0, 0xfa, 0xa7, 0x93, 0xaf, 0xec, 0x61, 0x60, 0x52,
-	0xc7, 0xf7, 0xe4, 0xfb, 0xf5, 0xc9, 0xf7, 0xd4, 0xe9, 0x63, 0x42, 0xcd, 0xfe, 0x60, 0x1e, 0xc1,
-	0x75, 0x60, 0x0e, 0x06, 0x38, 0x20, 0xe2, 0xbd, 0xf6, 0xb7, 0x34, 0x14, 0x9a, 0xc2, 0x17, 0xd4,
-	0x80, 0x4c, 0x80, 0x2f, 0xd4, 0xd4, 0x46, 0x6a, 0xab, 0xb4, 0x7b, 0xbf, 0x31, 0xe9, 0x67, 0x43,
-	0xe2, 0x74, 0x7c, 0xa1, 0x33, 0x20, 0xda, 0x81, 0xac, 0x6d, 0x52, 0x53, 0x4d, 0x73, 0x85, 0x4f,
-	0xe6, 0x2a, 0x3c, 0x37, 0xa9, 0xa9, 0x73, 0x28, 0xfa, 0x19, 0x94, 0x24, 0xc0, 0x60, 0xa6, 0x32,
-	0x1b, 0x99, 0x85, 0xa6, 0xc0, 0x8a, 0x7e, 0xa3, 0x43, 0xa8, 0x92, 0xe1, 0x79, 0x92, 0x21, 0xcb,
-	0x19, 0xd6, 0xa7, 0x19, 0x3a, 0x11, 0x8e, 0x91, 0x54, 0x48, 0xf2, 0x11, 0xfd, 0x04, 0x80, 0xf8,
-	0xd6, 0x15, 0xa6, 0x9c, 0x23, 0xc7, 0x39, 0xee, 0xcd, 0xe0, 0xe0, 0x18, 0xa6, 0x5f, 0x24, 0xe1,
-	0x4f, 0xed, 0x1f, 0x69, 0x80, 0x98, 0x1c, 0xed, 0x24, 0x83, 0xb6, 0xd0, 0x8f, 0xff, 0xe1, 0xb8,
-	0xfd, 0x3b, 0x05, 0xaa, 0x74, 0xaf, 0xe9, 0x7b, 0x1e, 0xb6, 0xa8, 0x33, 0x72, 0xe8, 0xdb, 0x0e,
-	0x35, 0x29, 0x46, 0x87, 0x90, 0x23, 0xec, 0x07, 0x8f, 0x63, 0x75, 0xf7, 0xc9, 0xdc, 0x95, 0x4d,
-	0xa9, 0x36, 0xf8, 0x5f, 0x5d, 0xa8, 0x6b, 0xbf, 0x86, 0x9c, 0x20, 0x2c, 0x41, 0xe1, 0xac, 0xfd,
-	0x8b, 0xf6, 0xe9, 0xeb, 0x76, 0x6d, 0x05, 0x29, 0x90, 0x6d, 0x3d, 0x3f, 0x39, 0xa8, 0xa5, 0x50,
-	0x15, 0xa0, 0x79, 0xda, 0x6e, 0x1f, 0x34, 0xbb, 0xad, 0xf6, 0x8b, 0x5a, 0x1a, 0x15, 0x21, 0xa7,
-	0x1f, 0xec, 0x3d, 0xff, 0xae, 0x96, 0x41, 0x1f, 0xc1, 0x6a, 0x57, 0xdf, 0x6b, 0x77, 0x5a, 0x07,
-	0xed, 0xae, 0x71, 0xb8, 0xd7, 0x3a, 0x39, 0xd3, 0x0f, 0x6a, 0x59, 0x54, 0x06, 0xa5, 0x73, 0x74,
-	0xd6, 0x7d, 0xce, 0x98, 0x72, 0xda, 0x7f, 0xd2, 0x50, 0x4a, 0x64, 0x07, 0x7d, 0x93, 0xf4, 0xbb,
-	0xb4, 0xfb, 0x70, 0x79, 0xbf, 0xa5, 0xc7, 0xe8, 0x2e, 0xe4, 0xa9, 0x19, 0xf4, 0x30, 0xe5, 0xe5,
-	0x50, 0xd4, 0xe5, 0x13, 0x7a, 0x0a, 0x39, 0x1a, 0x98, 0x16, 0x56, 0x33, 0x9c, 0xf9, 0xd3, 0xb9,
-	0xcc, 0x5d, 0x86, 0xd2, 0x05, 0x18, 0x6d, 0x42, 0xc5, 0x32, 0x5d, 0x97, 0x18, 0x84, 0x9a, 0x01,
-	0xc5, 0xb6, 0x9a, 0xdd, 0x48, 0x6d, 0x65, 0xf4, 0x32, 0x17, 0x76, 0x84, 0x0c, 0x7d, 0x01, 0xb7,
-	0x24, 0x68, 0x68, 0x59, 0x18, 0xdb, 0xd8, 0x56, 0x73, 0x1c, 0x56, 0x15, 0xb0, 0x50, 0x8a, 0xfe,
-	0x0f, 0x84, 0xa2, 0x71, 0x61, 0x3a, 0x2e, 0xb6, 0xd5, 0x3c, 0x47, 0x95, 0xb8, 0xec, 0x90, 0x8b,
-	0xd0, 0x77, 0x70, 0xcf, 0x35, 0x09, 0x35, 0x98, 0x2c, 0x34, 0x6a, 0x44, 0x43, 0x48, 0x2d, 0x70,
-	0xe7, 0xeb, 0x0d, 0x31, 0x85, 0x1a, 0xe1, 0x14, 0x6a, 0x74, 0x43, 0x84, 0xae, 0x32, 0xf5, 0xa6,
-	0xe9, 0xba, 0xd2, 0xbb, 0xe8, 0x8d, 0xf6, 0xa7, 0x0c, 0xac, 0x26, 0xd7, 0x78, 0x30, 0xc2, 0x1e,
-	0x45, 0x1b, 0x50, 0xb2, 0x31, 0xb1, 0x02, 0x67, 0xc0, 0xc6, 0x20, 0x8f, 0x7b, 0x51, 0x4f, 0x8a,
-	0xd0, 0x11, 0x28, 0x04, 0x8f, 0x70, 0xe0, 0xd0, 0xb7, 0x3c, 0xa6, 0xd5, 0xdd, 0xff, 0xbf, 0x39,
-	0x78, 0x9c, 0xb8, 0xd1, 0x91, 0x3a, 0x7a, 0xa4, 0x8d, 0x7e, 0x0c, 0xc5, 0x78, 0x29, 0x99, 0x85,
-	0x4b, 0x89, 0xc1, 0xe8, 0xe7, 0xe3, 0xfd, 0x9a, 0x5d, 0x3c, 0x52, 0x8f, 0x56, 0xc6, 0x3a, 0xf6,
-	0x68, 0xaa, 0x63, 0x73, 0x4b, 0x4d, 0x98, 0xa3, 0x95, 0x89, 0x9e, 0xd5, 0x0e, 0x40, 0x09, 0x97,
-	0xc6, 0xcb, 0xbf, 0x6b, 0xc4, 0x8d, 0x51, 0x82, 0x42, 0xb3, 0x6b, 0xb4, 0xda, 0x87, 0xa7, 0xb2,
-	0x37, 0xba, 0xc6, 0xeb, 0x3d, 0xbd, 0x2d, 0x7a, 0xa3, 0x0c, 0x4a, 0xb3, 0x6b, 0x1c, 0xe8, 0xfa,
-	0xa9, 0x5e, 0xcb, 0xec, 0x97, 0xa0, 0x68, 0x5d, 0x3a, 0xae, 0xcd, 0x7c, 0x61, 0xbd, 0x5c, 0x4e,
-	0x46, 0x10, 0x3d, 0x84, 0x55, 0x6f, 0xd8, 0x37, 0x30, 0x8b, 0x24, 0x31, 0x5c, 0xbf, 0xd7, 0xc3,
-	0x36, 0xcf, 0x4d, 0x46, 0xbf, 0xe5, 0x0d, 0xfb, 0x3c, 0xc2, 0xe4, 0x84, 0x8b, 0x51, 0x0b, 0x90,
-	0x15, 0x60, 0xbe, 0x8b, 0x25, 0x2a, 0x25, 0xbd, 0x30, 0xbc, 0xab, 0xa1, 0x56, 0x24, 0x42, 0x5f,
-	0x43, 0x5e, 0x98, 0x94, 0x13, 0x71, 0x73, 0x89, 0x44, 0xeb, 0x52, 0x45, 0xb3, 0x00, 0xe2, 0xf0,
-	0xa3, 0x4f, 0x20, 0x0c, 0xbf, 0xe1, 0x84, 0xae, 0x17, 0xa5, 0xa4, 0x65, 0x23, 0x04, 0x59, 0xcf,
-	0xec, 0x63, 0xd9, 0xa4, 0xfc, 0xf7, 0x71, 0x56, 0xc9, 0xd4, 0xb2, 0xc7, 0x59, 0x25, 0x5b, 0xcb,
-	0x1d, 0x67, 0x95, 0x5c, 0x2d, 0x7f, 0x9c, 0x55, 0xf2, 0xb5, 0xc2, 0x71, 0x56, 0x29, 0xd4, 0x94,
-	0xe3, 0xac, 0xa2, 0xd4, 0x8a, 0x9a, 0x0b, 0x95, 0xb1, 0xfc, 0xb0, 0x0e, 0x4d, 0x24, 0xd6, 0xb1,
-	0x79, 0x8b, 0x64, 0xf4, 0x72, 0x2c, 0x4c, 0x58, 0x53, 0xc6, 0xac, 0xa5, 0x6a, 0xe9, 0xe3, 0xac,
-	0x92, 0xae, 0x65, 0xe6, 0x59, 0xd6, 0xbe, 0x87, 0x62, 0x34, 0x7b, 0xd1, 0x3d, 0x90, 0xd3, 0x97,
-	0x59, 0xc9, 0x70, 0x2b, 0x8a, 0x10, 0x24, 0x2c, 0x64, 0xe7, 0x5a, 0x98, 0xbd, 0x1e, 0x66, 0x01,
-	0x07, 0x23, 0x1c, 0x84, 0x16, 0xf8, 0x03, 0xb3, 0x90, 0x93, 0x16, 0xb8, 0x20, 0x61, 0x21, 0xbf,
-	0xd4, 0x1a, 0x62, 0x0b, 0x7f, 0x4d, 0x41, 0x5e, 0x98, 0x40, 0x8f, 0x93, 0x7b, 0xeb, 0xac, 0x7d,
-	0x26, 0xf4, 0x44, 0xec, 0xab, 0x4f, 0xc6, 0xf6, 0xd5, 0xfb, 0xf3, 0xf0, 0x89, 0x6d, 0xf5, 0x1b,
-	0xa8, 0xb8, 0x0e, 0xa1, 0xd8, 0x33, 0x44, 0x60, 0x64, 0x19, 0xdd, 0xb8, 0xa5, 0x95, 0x85, 0x86,
-	0x10, 0x68, 0x7f, 0x60, 0xa7, 0x81, 0x88, 0x36, 0x9e, 0xda, 0xa9, 0x0f, 0x9a, 0xda, 0xe9, 0xe5,
-	0xa6, 0x76, 0x66, 0xa9, 0xa9, 0x9d, 0x7d, 0xef, 0xa9, 0x9d, 0xfb, 0x80, 0xa9, 0xfd, 0x97, 0x34,
-	0xe4, 0x45, 0x6c, 0x16, 0xa7, 0x2f, 0x8a, 0xe9, 0x92, 0xe9, 0xe3, 0xf8, 0x44, 0xfa, 0xb6, 0x21,
-	0xe7, 0xfa, 0x96, 0xe9, 0xca, 0xd9, 0xbc, 0x36, 0xad, 0xb2, 0x67, 0xdb, 0x01, 0x26, 0x44, 0x17,
-	0x38, 0xb4, 0x03, 0xf9, 0x00, 0xf7, 0x7d, 0x8a, 0xe5, 0x44, 0xbe, 0x41, 0x43, 0x02, 0xd1, 0x33,
-	0xb6, 0x9b, 0x58, 0x43, 0xbe, 0x9b, 0x44, 0x71, 0x99, 0x2e, 0x2c, 0x81, 0xd0, 0x23, 0x2c, 0x5a,
-	0x87, 0x92, 0x60, 0x30, 0x12, 0x5d, 0x00, 0x42, 0xd4, 0x36, 0xfb, 0x58, 0xfb, 0x7d, 0x01, 0x20,
-	0x5e, 0x11, 0x4b, 0x2f, 0xa1, 0x01, 0x36, 0xfb, 0x71, 0x15, 0x88, 0x21, 0x54, 0x95, 0xe2, 0xb0,
-	0x0e, 0x1e, 0xc1, 0x6a, 0x04, 0x8c, 0x2a, 0x41, 0x14, 0x4c, 0x2d, 0x84, 0x46, 0xb5, 0xf0, 0x39,
-	0x84, 0xea, 0x61, 0x35, 0x88, 0x9a, 0xa9, 0x48, 0xa9, 0xac, 0x87, 0x4d, 0xa8, 0xf4, 0x31, 0x21,
-	0x66, 0x0f, 0x13, 0x83, 0x60, 0x8f, 0x86, 0xc7, 0x86, 0x50, 0xd8, 0x61, 0x3b, 0xef, 0x23, 0x58,
-	0x8d, 0x40, 0x01, 0xb6, 0xb0, 0x33, 0x8a, 0x0e, 0x0e, 0xb5, 0xf0, 0x85, 0x2e, 0xe5, 0x68, 0x0b,
-	0x6a, 0x57, 0x18, 0x0f, 0x0c, 0xd3, 0x75, 0x46, 0x21, 0xa9, 0x38, 0x3e, 0x54, 0x99, 0x7c, 0x8f,
-	0x8b, 0x39, 0xed, 0x25, 0x6c, 0xf2, 0x5a, 0xe4, 0x19, 0x32, 0x84, 0x5f, 0x06, 0x1f, 0xf5, 0xef,
-	0x79, 0x92, 0x58, 0x67, 0x34, 0x27, 0x8c, 0xa5, 0xc3, 0x49, 0x9a, 0x82, 0x23, 0xde, 0x2d, 0x7e,
-	0x03, 0x9f, 0x71, 0x4b, 0x32, 0x2f, 0x73, 0x4d, 0x29, 0x0b, 0x4d, 0x6d, 0x30, 0x1e, 0x9d, 0xd3,
-	0xcc, 0xb1, 0x15, 0x76, 0x98, 0x0c, 0x0c, 0x0f, 0x40, 0xc2, 0x44, 0x71, 0xb9, 0x0e, 0x7b, 0x29,
-	0xb4, 0x59, 0x9c, 0x62, 0x6a, 0x13, 0xd6, 0xc7, 0xa8, 0xc3, 0x5c, 0x24, 0xe8, 0x61, 0x21, 0xfd,
-	0xfd, 0x04, 0x7d, 0x98, 0xb4, 0xd8, 0xc4, 0xb7, 0xb0, 0x26, 0xd2, 0x71, 0xe1, 0xfa, 0xd7, 0x86,
-	0xe5, 0x7b, 0x34, 0xf0, 0x5d, 0xe3, 0xda, 0xf1, 0x6c, 0xff, 0x5a, 0x2d, 0x85, 0xfd, 0x3c, 0x41,
-	0xde, 0xf2, 0xe8, 0xb3, 0xa7, 0xdf, 0x9a, 0xee, 0x10, 0xeb, 0x77, 0xb9, 0xf6, 0xa1, 0xeb, 0x5f,
-	0x37, 0x85, 0xee, 0x6b, 0xae, 0x8a, 0xde, 0x40, 0x5d, 0x06, 0x7f, 0x16, 0x71, 0x79, 0x31, 0xf1,
-	0xc7, 0x42, 0x7d, 0x9a, 0xf9, 0x19, 0xe4, 0x7d, 0x71, 0x22, 0xac, 0xf0, 0x11, 0xfe, 0xe9, 0xbc,
-	0xf1, 0x71, 0xca, 0x51, 0xba, 0x44, 0x6b, 0xff, 0xcc, 0x40, 0x41, 0xb6, 0x3c, 0x7a, 0x09, 0x15,
-	0x6a, 0x0d, 0x9c, 0x81, 0x61, 0x0a, 0x81, 0x9c, 0x5c, 0x0f, 0xe6, 0x0e, 0x89, 0x46, 0xd7, 0x1a,
-	0xb4, 0x06, 0xf2, 0xe1, 0x68, 0x45, 0x2f, 0x73, 0xf5, 0x90, 0xee, 0x05, 0x94, 0x86, 0x36, 0x89,
-	0xc8, 0xc4, 0x58, 0xfb, 0x6c, 0x3e, 0xd9, 0x99, 0x4d, 0x62, 0x2a, 0x18, 0x46, 0x4f, 0xcc, 0x2f,
-	0x9f, 0x5e, 0xe2, 0x20, 0xa2, 0xca, 0x2c, 0xf2, 0xeb, 0x94, 0xc1, 0x13, 0x7e, 0xf9, 0x89, 0xe7,
-	0xfa, 0x1e, 0x94, 0x93, 0x7e, 0xb3, 0x93, 0xcf, 0xc4, 0x9a, 0xcb, 0x7a, 0x31, 0x5e, 0x06, 0x82,
-	0xec, 0xc0, 0x0f, 0xc4, 0xe7, 0x49, 0x4e, 0xe7, 0xbf, 0xeb, 0x5b, 0x00, 0xb1, 0xb7, 0xa8, 0x0e,
-	0xca, 0x85, 0xe3, 0x62, 0x3e, 0xe7, 0xc4, 0x79, 0x3c, 0x7a, 0xae, 0xb7, 0xa1, 0x9c, 0x74, 0x26,
-	0x3a, 0x15, 0xa4, 0xe2, 0x53, 0x01, 0x7a, 0x08, 0xb9, 0x11, 0xcb, 0xae, 0x0c, 0xd1, 0x9d, 0xa9,
-	0x02, 0xd8, 0xf3, 0xde, 0xea, 0x02, 0xb2, 0x5f, 0x84, 0x82, 0xf4, 0x54, 0xfb, 0x63, 0x86, 0x9d,
-	0x6c, 0xe5, 0xb8, 0xdd, 0x85, 0x0c, 0x75, 0xc9, 0xfc, 0x6d, 0x37, 0x04, 0x36, 0xba, 0x2e, 0x8b,
-	0x08, 0x03, 0xb3, 0x8f, 0x37, 0x1e, 0x18, 0x69, 0x77, 0xeb, 0x06, 0x2d, 0xbe, 0x86, 0xf0, 0xe9,
-	0x68, 0x45, 0x17, 0x8a, 0xf5, 0x7f, 0xa5, 0x20, 0xd3, 0x75, 0x09, 0xfa, 0x1c, 0x2a, 0x84, 0x9a,
-	0x9e, 0x6d, 0x06, 0xb6, 0x11, 0x2f, 0x8f, 0x45, 0x3e, 0x14, 0xb3, 0x91, 0x8f, 0xd6, 0x01, 0x44,
-	0x22, 0xe3, 0xa3, 0xe4, 0xd1, 0x8a, 0x5e, 0xe4, 0x32, 0x0e, 0x78, 0x04, 0xab, 0xa2, 0xef, 0x2c,
-	0x1c, 0x50, 0xe7, 0xc2, 0xb1, 0xd8, 0xa7, 0x65, 0x86, 0x67, 0xa4, 0xc6, 0x5f, 0x34, 0x63, 0x39,
-	0x7a, 0x0c, 0x48, 0x36, 0x53, 0x12, 0x9d, 0xe5, 0xe8, 0x55, 0xf1, 0x26, 0x01, 0xdf, 0xaf, 0x42,
-	0xd9, 0x72, 0x06, 0xcc, 0x3a, 0x19, 0x3a, 0x14, 0xd7, 0x4f, 0xa1, 0x32, 0xb6, 0xaa, 0x0f, 0x4e,
-	0x4d, 0x01, 0x72, 0x7d, 0xdf, 0xc6, 0xae, 0xe6, 0x41, 0x39, 0xd9, 0x6b, 0x33, 0x89, 0xef, 0x24,
-	0x89, 0x8b, 0x92, 0x02, 0x3d, 0x05, 0x30, 0x6d, 0xdb, 0x61, 0x5a, 0xd1, 0xae, 0x3e, 0xdb, 0x66,
-	0x02, 0xa7, 0x9d, 0xc0, 0xed, 0xa4, 0x3d, 0x36, 0xc6, 0xfc, 0x21, 0x45, 0x3f, 0x02, 0x25, 0xbc,
-	0x2d, 0x93, 0x75, 0xb1, 0x36, 0x45, 0xf5, 0x5c, 0x02, 0xf4, 0x08, 0xaa, 0x59, 0x80, 0x92, 0x6c,
-	0x27, 0x8e, 0xd7, 0xc3, 0x01, 0xfb, 0x4c, 0x37, 0xd9, 0xe7, 0xbb, 0x58, 0x85, 0xa2, 0xcb, 0xa7,
-	0x31, 0x23, 0xe9, 0xe5, 0x8d, 0xfc, 0x5d, 0x99, 0xf0, 0xd9, 0x1a, 0xb4, 0xbc, 0x0b, 0x9f, 0xf5,
-	0x22, 0x9b, 0x21, 0x46, 0x7c, 0xa9, 0x50, 0xd1, 0x8b, 0x4c, 0x22, 0x6e, 0x35, 0x34, 0x31, 0xa1,
-	0x0c, 0xcb, 0x94, 0x88, 0x34, 0x47, 0x94, 0x98, 0xb0, 0x69, 0x0a, 0xcc, 0x97, 0x50, 0xe3, 0x98,
-	0x00, 0xd3, 0xc0, 0xf4, 0x48, 0xdf, 0xa1, 0x62, 0x60, 0x54, 0xf4, 0x5b, 0x4c, 0xae, 0xc7, 0x62,
-	0x76, 0x46, 0xe1, 0xd0, 0x41, 0xe0, 0x9f, 0x63, 0xc2, 0x4b, 0xa7, 0xa2, 0x73, 0x07, 0x5e, 0x71,
-	0x09, 0x3b, 0x4a, 0x72, 0xc0, 0xb9, 0x69, 0x5d, 0xf9, 0x17, 0xe2, 0x1b, 0x54, 0x9a, 0xdb, 0x17,
-	0xa2, 0x08, 0x22, 0xe6, 0x29, 0xe1, 0x9b, 0xbc, 0x84, 0x88, 0xa5, 0x11, 0xf4, 0x00, 0x6e, 0x89,
-	0x45, 0x79, 0xb6, 0x71, 0x4d, 0x2c, 0xd3, 0xc5, 0x7c, 0x37, 0xaf, 0xe8, 0x7c, 0x31, 0x1d, 0xcf,
-	0x7e, 0xcd, 0x85, 0x11, 0x2e, 0xb0, 0x46, 0x21, 0x4e, 0x89, 0x71, 0xba, 0x35, 0x92, 0xb8, 0x35,
-	0x50, 0x04, 0x8e, 0xfa, 0x7c, 0x23, 0xad, 0xe8, 0x05, 0x0e, 0xa0, 0x7e, 0xf4, 0xca, 0xa4, 0x3e,
-	0xdf, 0x04, 0xe5, 0xab, 0x3d, 0xea, 0xa3, 0x0d, 0xe9, 0x28, 0xf3, 0xa2, 0x4f, 0x08, 0xdf, 0xc6,
-	0xe4, 0x6a, 0x3b, 0x9e, 0xfd, 0x92, 0x90, 0x08, 0xc1, 0xec, 0x33, 0x44, 0x39, 0x46, 0xe8, 0xd6,
-	0x88, 0x21, 0xc2, 0xc5, 0x0e, 0x3d, 0xd3, 0xba, 0xc2, 0xb6, 0x5a, 0x89, 0x17, 0x7b, 0x26, 0x44,
-	0x51, 0x4c, 0x89, 0x40, 0x54, 0x13, 0x56, 0x04, 0xe0, 0x1e, 0xf0, 0x84, 0x1a, 0xae, 0x4f, 0xa8,
-	0x7a, 0x8b, 0xbf, 0xe6, 0x3e, 0x9f, 0xf8, 0x84, 0x46, 0x06, 0x64, 0xf2, 0xd4, 0x5a, 0x6c, 0x40,
-	0x26, 0x2e, 0x82, 0x5c, 0x30, 0x3a, 0x4a, 0xd4, 0xd5, 0x18, 0x72, 0x28, 0x44, 0xe8, 0x31, 0xdc,
-	0x16, 0x26, 0xd8, 0x31, 0x81, 0x9d, 0x94, 0xc5, 0xf9, 0x0b, 0x71, 0x24, 0xaf, 0x8e, 0x13, 0x93,
-	0xf0, 0x63, 0xa7, 0x3c, 0xd8, 0xa1, 0x18, 0x6e, 0x5a, 0x57, 0x02, 0x7d, 0x3b, 0xae, 0x19, 0x86,
-	0xde, 0xb3, 0xae, 0x38, 0x78, 0x9a, 0x3b, 0xc0, 0xd6, 0x48, 0xbd, 0x33, 0xcd, 0xad, 0x63, 0x6b,
-	0x34, 0xcd, 0xcd, 0xd1, 0x1f, 0x4d, 0x71, 0x73, 0x70, 0x18, 0x9a, 0x41, 0x9f, 0x0e, 0xd5, 0xbb,
-	0x71, 0x68, 0x5e, 0xf5, 0xe9, 0x10, 0x3d, 0x84, 0xd5, 0x28, 0x3b, 0x84, 0xd0, 0xcb, 0x00, 0x93,
-	0x4b, 0xf5, 0xe3, 0x44, 0x61, 0x5b, 0xa3, 0x8e, 0x14, 0x27, 0x2a, 0x84, 0xaa, 0x6a, 0xb2, 0x42,
-	0x68, 0x94, 0x9f, 0x80, 0xd2, 0x91, 0x19, 0xa8, 0x6b, 0x89, 0x1c, 0x73, 0x49, 0x64, 0x87, 0xd5,
-	0x49, 0x64, 0xa7, 0x1e, 0xdb, 0xe9, 0x78, 0x76, 0x64, 0x27, 0xec, 0x47, 0x86, 0xb5, 0xae, 0x3d,
-	0x5b, 0xbd, 0x17, 0x27, 0xa3, 0xe3, 0xd9, 0xcd, 0x6b, 0x2f, 0x2e, 0x08, 0xd3, 0x1e, 0xb1, 0xa2,
-	0xba, 0x1f, 0x1b, 0xdc, 0xe3, 0x12, 0x76, 0xf2, 0x97, 0x39, 0xf7, 0x03, 0x1b, 0x07, 0x8e, 0xd7,
-	0x53, 0x3f, 0xe1, 0xa0, 0xaa, 0x48, 0x7b, 0x28, 0xd5, 0xce, 0xe1, 0xa3, 0x17, 0x98, 0x76, 0xfd,
-	0x81, 0xfc, 0x86, 0x24, 0x3a, 0xfe, 0xed, 0x10, 0x13, 0xca, 0x0e, 0xdb, 0xfc, 0x9b, 0xc1, 0x98,
-	0xba, 0xc1, 0xa8, 0x72, 0x79, 0x33, 0xba, 0x58, 0x58, 0x87, 0x52, 0xdf, 0xfc, 0xc1, 0x08, 0x30,
-	0x19, 0xba, 0x94, 0xc8, 0xcf, 0x06, 0xe8, 0x9b, 0x3f, 0xe8, 0x42, 0xa2, 0x19, 0x70, 0x77, 0xd2,
-	0x06, 0x19, 0xf8, 0x1e, 0xc1, 0xe8, 0x2b, 0x28, 0x48, 0x7a, 0x35, 0xc5, 0x8f, 0x58, 0x6b, 0xf3,
-	0xaf, 0xb3, 0x42, 0x24, 0xaa, 0x41, 0x06, 0x7b, 0xe2, 0xf3, 0x44, 0xd1, 0xd9, 0x4f, 0xed, 0x57,
-	0xb0, 0xfa, 0x02, 0x53, 0xf1, 0xc9, 0x1c, 0x2d, 0xe0, 0x01, 0xfb, 0xf8, 0x61, 0x0b, 0x88, 0xaf,
-	0x13, 0x52, 0xe1, 0x77, 0x8a, 0x19, 0x48, 0xf4, 0x32, 0xee, 0xbf, 0x01, 0x94, 0x64, 0x97, 0xae,
-	0x3f, 0x81, 0xbc, 0x20, 0x96, 0x9e, 0xab, 0x73, 0xaf, 0x12, 0x24, 0x6e, 0x86, 0xdf, 0xdb, 0x50,
-	0x8b, 0x98, 0x43, 0xb7, 0xc7, 0xee, 0x3f, 0x52, 0xe3, 0xf7, 0x1f, 0xda, 0x41, 0x62, 0xa1, 0x33,
-	0x3d, 0x49, 0x2d, 0xe3, 0x89, 0xf6, 0x3b, 0xf8, 0x38, 0xa2, 0x11, 0x3b, 0x06, 0x59, 0xc6, 0x7c,
-	0x22, 0xa4, 0xd1, 0x1d, 0x50, 0x3a, 0x19, 0xd2, 0xf0, 0x22, 0x68, 0x22, 0xa4, 0x99, 0xa9, 0x90,
-	0x5e, 0x82, 0x3a, 0xed, 0x80, 0x5c, 0xce, 0xf8, 0xff, 0x03, 0x52, 0xef, 0xf3, 0xff, 0x80, 0x19,
-	0x21, 0xde, 0xe5, 0x11, 0x8b, 0xee, 0xe4, 0xc4, 0x22, 0x6f, 0xbe, 0x97, 0xd3, 0x5a, 0x3c, 0xe1,
-	0x91, 0xce, 0xac, 0x5a, 0x4d, 0x2d, 0x57, 0xab, 0xda, 0xd7, 0x70, 0x87, 0x2d, 0x34, 0x71, 0x5b,
-	0x27, 0x3c, 0x98, 0xba, 0xb1, 0x4b, 0x4d, 0xdf, 0xd8, 0x69, 0x67, 0xbc, 0x37, 0x93, 0xca, 0xd2,
-	0x95, 0x9f, 0x02, 0xc4, 0xc0, 0xf9, 0xff, 0x5b, 0x4b, 0x68, 0x26, 0xf0, 0x5a, 0x4b, 0x54, 0x9d,
-	0x0c, 0x5a, 0x9c, 0xf6, 0x28, 0xa7, 0xa9, 0x89, 0x7b, 0x3d, 0x15, 0x0a, 0x64, 0xd8, 0xef, 0x9b,
-	0xc1, 0x5b, 0x19, 0xd9, 0xf0, 0x31, 0xac, 0x47, 0x49, 0x95, 0xa8, 0x47, 0x71, 0xf3, 0x35, 0xbf,
-	0x1e, 0x85, 0x86, 0xc4, 0xed, 0xfe, 0x39, 0x07, 0x8a, 0x0c, 0xdd, 0x3b, 0x64, 0x41, 0x75, 0x7c,
-	0x5a, 0xa0, 0x2f, 0xa6, 0x09, 0x66, 0xce, 0xac, 0xfa, 0xd6, 0x62, 0xa0, 0xf4, 0xf1, 0x35, 0x40,
-	0xdc, 0xd3, 0x68, 0x73, 0xa6, 0xde, 0xf8, 0x3c, 0xa9, 0x7f, 0x76, 0x33, 0x48, 0x12, 0x77, 0xa1,
-	0x18, 0x49, 0x91, 0x76, 0x83, 0x4a, 0x48, 0xbb, 0x79, 0x23, 0x46, 0xb2, 0x3a, 0x89, 0x41, 0x21,
-	0xfb, 0x05, 0x7d, 0x79, 0x83, 0xe2, 0x78, 0x53, 0xd7, 0x1f, 0x2e, 0x03, 0x1d, 0x8b, 0x4c, 0xf8,
-	0xef, 0xdb, 0xd9, 0xde, 0x8d, 0xb7, 0xd3, 0x9c, 0xc8, 0x4c, 0xf6, 0xcf, 0xf7, 0x50, 0x19, 0xab,
-	0x66, 0xf4, 0x60, 0xb6, 0x57, 0x93, 0xbd, 0x52, 0xff, 0x62, 0x21, 0x6e, 0x3c, 0xf6, 0xe2, 0xa2,
-	0x70, 0x4e, 0xec, 0x93, 0x55, 0x3f, 0x2f, 0xf6, 0x63, 0xe5, 0xbc, 0xff, 0x06, 0x6e, 0x3b, 0xfe,
-	0x14, 0x70, 0xbf, 0x12, 0x16, 0xec, 0x2b, 0x76, 0x24, 0x7f, 0x95, 0xfa, 0xe5, 0x13, 0x79, 0x44,
-	0xef, 0xf9, 0xae, 0xe9, 0xf5, 0x1a, 0x7e, 0xd0, 0xdb, 0x1e, 0xff, 0xb7, 0x3d, 0x7b, 0x0a, 0x77,
-	0xd3, 0x77, 0xc6, 0x68, 0xe7, 0x3c, 0xcf, 0x4f, 0xf3, 0x5f, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff,
-	0x54, 0xae, 0x0b, 0x93, 0xdf, 0x1f, 0x00, 0x00,
+var File_grpc_channelz_v1_channelz_proto protoreflect.FileDescriptor
+
+var file_grpc_channelz_v1_channelz_proto_rawDesc = []byte{
+	0x0a, 0x1f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2f,
+	0x76, 0x31, 0x2f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x10, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a,
+	0x2e, 0x76, 0x31, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
+	0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
+	0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a,
+	0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
+	0xaf, 0x02, 0x0a, 0x07, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x2e, 0x0a, 0x03, 0x72,
+	0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x31, 0x0a, 0x04, 0x64,
+	0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3d,
+	0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x03, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65,
+	0x66, 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x46, 0x0a,
+	0x0e, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18,
+	0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x3a, 0x0a, 0x0a, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f,
+	0x72, 0x65, 0x66, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63,
+	0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x09, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65,
+	0x66, 0x22, 0xb5, 0x02, 0x0a, 0x0a, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x12, 0x31, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31,
+	0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x03,
+	0x72, 0x65, 0x66, 0x12, 0x31, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x44, 0x61, 0x74, 0x61,
+	0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3d, 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x46, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31,
+	0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x0d,
+	0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x3a, 0x0a,
+	0x0a, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x05, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x09,
+	0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x22, 0xc2, 0x01, 0x0a, 0x18, 0x43, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74,
+	0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x46, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74,
+	0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x5e,
+	0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f,
+	0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0e,
+	0x0a, 0x0a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x09,
+	0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x03, 0x12, 0x15, 0x0a, 0x11, 0x54, 0x52, 0x41,
+	0x4e, 0x53, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x04,
+	0x12, 0x0c, 0x0a, 0x08, 0x53, 0x48, 0x55, 0x54, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x22, 0xe9,
+	0x02, 0x0a, 0x0b, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40,
+	0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31,
+	0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69,
+	0x76, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65,
+	0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x34, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63,
+	0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x23,
+	0x0a, 0x0d, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x53, 0x74, 0x61, 0x72,
+	0x74, 0x65, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x75, 0x63,
+	0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x63, 0x61,
+	0x6c, 0x6c, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c,
+	0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01,
+	0x28, 0x03, 0x52, 0x0b, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12,
+	0x59, 0x0a, 0x1b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x73, 0x74, 0x61,
+	0x72, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65,
+	0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x98, 0x03, 0x0a, 0x11, 0x43,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74,
+	0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54,
+	0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69,
+	0x74, 0x79, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x38, 0x0a, 0x09,
+	0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d,
+	0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3f, 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x48, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e,
+	0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66,
+	0x48, 0x00, 0x52, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65,
+	0x66, 0x22, 0x45, 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x0e, 0x0a,
+	0x0a, 0x43, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a,
+	0x07, 0x43, 0x54, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x54,
+	0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x54,
+	0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x42, 0x0b, 0x0a, 0x09, 0x63, 0x68, 0x69, 0x6c,
+	0x64, 0x5f, 0x72, 0x65, 0x66, 0x22, 0xc2, 0x01, 0x0a, 0x0c, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x5f, 0x65, 0x76,
+	0x65, 0x6e, 0x74, 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x67, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x4c, 0x6f, 0x67, 0x67,
+	0x65, 0x64, 0x12, 0x49, 0x0a, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74,
+	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
+	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x11, 0x63, 0x72, 0x65, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3b, 0x0a,
+	0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31,
+	0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65,
+	0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x63, 0x0a, 0x0a, 0x43, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x03, 0x10,
+	0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08,
+	0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22,
+	0x6c, 0x0a, 0x0d, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66,
+	0x12, 0x23, 0x0a, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69,
+	0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a,
+	0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10,
+	0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0x60, 0x0a,
+	0x09, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f,
+	0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x73,
+	0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10,
+	0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08,
+	0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22,
+	0x60, 0x0a, 0x09, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08,
+	0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a,
+	0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10,
+	0x09, 0x22, 0xab, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x2d, 0x0a, 0x03,
+	0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x30, 0x0a, 0x04, 0x64,
+	0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a,
+	0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x03,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65,
+	0x66, 0x52, 0x0c, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x22,
+	0x8e, 0x02, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x34,
+	0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31,
+	0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x05, 0x74,
+	0x72, 0x61, 0x63, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x74,
+	0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x61, 0x6c,
+	0x6c, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x61, 0x6c,
+	0x6c, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x03, 0x52, 0x0e, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
+	0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c,
+	0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x46,
+	0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x59, 0x0a, 0x1b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x61,
+	0x6c, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73,
+	0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
+	0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c,
+	0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x22, 0xa6, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x72,
+	0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b,
+	0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x61,
+	0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b,
+	0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2f, 0x0a, 0x05,
+	0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x41,
+	0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x12, 0x31, 0x0a,
+	0x06, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31,
+	0x2e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x06, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
+	0x12, 0x36, 0x0a, 0x08, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x52, 0x08,
+	0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f,
+	0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72,
+	0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x83, 0x07, 0x0a, 0x0a, 0x53, 0x6f,
+	0x63, 0x6b, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x0e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65,
+	0x64, 0x12, 0x2b, 0x0a, 0x11, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x5f, 0x73, 0x75, 0x63,
+	0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x73, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x12, 0x25,
+	0x0a, 0x0e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x46,
+	0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x73, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x65,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x53, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x6d, 0x65,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x18,
+	0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x52,
+	0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x6b, 0x65, 0x65, 0x70, 0x5f,
+	0x61, 0x6c, 0x69, 0x76, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x0e, 0x6b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x73, 0x53, 0x65, 0x6e,
+	0x74, 0x12, 0x68, 0x0a, 0x23, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f,
+	0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74,
+	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
+	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x1f, 0x6c, 0x61, 0x73, 0x74,
+	0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x72, 0x65, 0x61, 0x74,
+	0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x6a, 0x0a, 0x24, 0x6c,
+	0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
+	0x61, 0x6d, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
+	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x74,
+	0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69,
+	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x59, 0x0a, 0x1b, 0x6c, 0x61, 0x73, 0x74, 0x5f,
+	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d,
+	0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54,
+	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x65,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61,
+	0x6d, 0x70, 0x12, 0x61, 0x0a, 0x1f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61,
+	0x67, 0x65, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65,
+	0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
+	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x65, 0x73,
+	0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65,
+	0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x56, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x66,
+	0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64,
+	0x6f, 0x77, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34,
+	0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x16, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x46, 0x6c, 0x6f, 0x77,
+	0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x58, 0x0a,
+	0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e,
+	0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x0c, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x17,
+	0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
+	0x6c, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x36, 0x0a, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f,
+	0x6e, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65,
+	0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22,
+	0xb8, 0x03, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x4d, 0x0a, 0x0d, 0x74,
+	0x63, 0x70, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x54, 0x63,
+	0x70, 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x74, 0x63,
+	0x70, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x47, 0x0a, 0x0b, 0x75, 0x64,
+	0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e,
+	0x76, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x55, 0x64, 0x73, 0x41, 0x64,
+	0x64, 0x72, 0x65, 0x73, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x75, 0x64, 0x73, 0x41, 0x64, 0x64, 0x72,
+	0x65, 0x73, 0x73, 0x12, 0x4d, 0x0a, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64,
+	0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x64,
+	0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65,
+	0x73, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65,
+	0x73, 0x73, 0x1a, 0x41, 0x0a, 0x0c, 0x54, 0x63, 0x70, 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65,
+	0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73,
+	0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x28, 0x0a, 0x0a, 0x55, 0x64, 0x73, 0x41, 0x64, 0x64, 0x72,
+	0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x1a,
+	0x4e, 0x0a, 0x0c, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12,
+	0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42,
+	0x09, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x96, 0x03, 0x0a, 0x08, 0x53,
+	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x12, 0x32, 0x0a, 0x03, 0x74, 0x6c, 0x73, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
+	0x2e, 0x54, 0x6c, 0x73, 0x48, 0x00, 0x52, 0x03, 0x74, 0x6c, 0x73, 0x12, 0x40, 0x0a, 0x05, 0x6f,
+	0x74, 0x68, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x53, 0x65, 0x63, 0x75,
+	0x72, 0x69, 0x74, 0x79, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x1a, 0xb9, 0x01,
+	0x0a, 0x03, 0x54, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72,
+	0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c,
+	0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0a,
+	0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x48, 0x00, 0x52, 0x09, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a,
+	0x11, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61,
+	0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x43,
+	0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65,
+	0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x65,
+	0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x42, 0x0e, 0x0a, 0x0c, 0x63, 0x69, 0x70,
+	0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, 0x74, 0x65, 0x1a, 0x4f, 0x0a, 0x0d, 0x4f, 0x74, 0x68,
+	0x65, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2a,
+	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
+	0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x6d, 0x6f,
+	0x64, 0x65, 0x6c, 0x22, 0x6e, 0x0a, 0x0c, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x70, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x34, 0x0a,
+	0x0a, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x0a, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f,
+	0x6e, 0x61, 0x6c, 0x22, 0x4c, 0x0a, 0x13, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x70, 0x74,
+	0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44,
+	0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x22, 0x63, 0x0a, 0x12, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f,
+	0x6e, 0x4c, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12,
+	0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb2, 0x08, 0x0a, 0x13, 0x53, 0x6f, 0x63, 0x6b, 0x65,
+	0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x63, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d,
+	0x0a, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x09, 0x74, 0x63, 0x70, 0x69, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x22, 0x0a,
+	0x0d, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x63, 0x61, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x43, 0x61, 0x53, 0x74, 0x61, 0x74,
+	0x65, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x61, 0x6e,
+	0x73, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, 0x70,
+	0x69, 0x52, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b,
+	0x74, 0x63, 0x70, 0x69, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x12, 0x21, 0x0a,
+	0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x18, 0x05, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66,
+	0x12, 0x21, 0x0a, 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x4f, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f,
+	0x77, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, 0x63,
+	0x70, 0x69, 0x53, 0x6e, 0x64, 0x57, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x74,
+	0x63, 0x70, 0x69, 0x5f, 0x72, 0x63, 0x76, 0x5f, 0x77, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x18, 0x08,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, 0x63, 0x70, 0x69, 0x52, 0x63, 0x76, 0x57, 0x73, 0x63,
+	0x61, 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x74, 0x6f, 0x18,
+	0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x63, 0x70, 0x69, 0x52, 0x74, 0x6f, 0x12, 0x19,
+	0x0a, 0x08, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x61, 0x74, 0x6f, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d,
+	0x52, 0x07, 0x74, 0x63, 0x70, 0x69, 0x41, 0x74, 0x6f, 0x12, 0x20, 0x0a, 0x0c, 0x74, 0x63, 0x70,
+	0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, 0x6d, 0x73, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x0a, 0x74, 0x63, 0x70, 0x69, 0x53, 0x6e, 0x64, 0x4d, 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x74,
+	0x63, 0x70, 0x69, 0x5f, 0x72, 0x63, 0x76, 0x5f, 0x6d, 0x73, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x52, 0x63, 0x76, 0x4d, 0x73, 0x73, 0x12, 0x21, 0x0a,
+	0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x75, 0x6e, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x18, 0x0d, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x55, 0x6e, 0x61, 0x63, 0x6b, 0x65, 0x64,
+	0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x18,
+	0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x53, 0x61, 0x63, 0x6b, 0x65,
+	0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x6f, 0x73, 0x74, 0x18, 0x0f,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x74, 0x63, 0x70, 0x69, 0x4c, 0x6f, 0x73, 0x74, 0x12, 0x21,
+	0x0a, 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x18, 0x10,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x52, 0x65, 0x74, 0x72, 0x61, 0x6e,
+	0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x66, 0x61, 0x63, 0x6b, 0x65, 0x74,
+	0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x46, 0x61, 0x63,
+	0x6b, 0x65, 0x74, 0x73, 0x12, 0x2d, 0x0a, 0x13, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73,
+	0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x10, 0x74, 0x63, 0x70, 0x69, 0x4c, 0x61, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x53,
+	0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x12, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, 0x74,
+	0x5f, 0x61, 0x63, 0x6b, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x0f, 0x74, 0x63, 0x70, 0x69, 0x4c, 0x61, 0x73, 0x74, 0x41, 0x63, 0x6b, 0x53, 0x65, 0x6e, 0x74,
+	0x12, 0x2d, 0x0a, 0x13, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x64, 0x61,
+	0x74, 0x61, 0x5f, 0x72, 0x65, 0x63, 0x76, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x74,
+	0x63, 0x70, 0x69, 0x4c, 0x61, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x63, 0x76, 0x12,
+	0x2b, 0x0a, 0x12, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x61, 0x63, 0x6b,
+	0x5f, 0x72, 0x65, 0x63, 0x76, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, 0x70,
+	0x69, 0x4c, 0x61, 0x73, 0x74, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x63, 0x76, 0x12, 0x1b, 0x0a, 0x09,
+	0x74, 0x63, 0x70, 0x69, 0x5f, 0x70, 0x6d, 0x74, 0x75, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x08, 0x74, 0x63, 0x70, 0x69, 0x50, 0x6d, 0x74, 0x75, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x63, 0x70,
+	0x69, 0x5f, 0x72, 0x63, 0x76, 0x5f, 0x73, 0x73, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x18, 0x17,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, 0x70, 0x69, 0x52, 0x63, 0x76, 0x53, 0x73, 0x74,
+	0x68, 0x72, 0x65, 0x73, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x74,
+	0x74, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x63, 0x70, 0x69, 0x52, 0x74, 0x74,
+	0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x74, 0x74, 0x76, 0x61, 0x72, 0x18,
+	0x19, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x52, 0x74, 0x74, 0x76, 0x61,
+	0x72, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, 0x73, 0x73,
+	0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63,
+	0x70, 0x69, 0x53, 0x6e, 0x64, 0x53, 0x73, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x12, 0x22, 0x0a,
+	0x0d, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, 0x63, 0x77, 0x6e, 0x64, 0x18, 0x1b,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x53, 0x6e, 0x64, 0x43, 0x77, 0x6e,
+	0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x76, 0x6d, 0x73, 0x73,
+	0x18, 0x1c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x41, 0x64, 0x76, 0x6d,
+	0x73, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x65, 0x6f, 0x72, 0x64,
+	0x65, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x74, 0x63, 0x70,
+	0x69, 0x52, 0x65, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x62, 0x0a, 0x15, 0x47,
+	0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e,
+	0x73, 0x74, 0x61, 0x72, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x1f,
+	0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22,
+	0x5f, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x10,
+	0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6e, 0x64,
+	0x22, 0x5c, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x73,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d,
+	0x73, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a,
+	0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x58,
+	0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x06,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x2f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x22, 0x45, 0x0a, 0x11, 0x47, 0x65, 0x74,
+	0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30,
+	0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76,
+	0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x22, 0x7f, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f, 0x63,
+	0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72,
+	0x74, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64,
+	0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74,
+	0x73, 0x22, 0x68, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f,
+	0x63, 0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a,
+	0x0a, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x09,
+	0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x32, 0x0a, 0x11, 0x47,
+	0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x22,
+	0x49, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x22, 0x3b, 0x0a, 0x14, 0x47, 0x65,
+	0x74, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x75,
+	0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x3c, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x22, 0x49,
+	0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64, 0x12,
+	0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08,
+	0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x45, 0x0a, 0x11, 0x47, 0x65, 0x74,
+	0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30,
+	0x0a, 0x06, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76,
+	0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74,
+	0x32, 0x9a, 0x05, 0x0a, 0x08, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x12, 0x63, 0x0a,
+	0x0e, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12,
+	0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e,
+	0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x54,
+	0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x12, 0x57, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
+	0x12, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a,
+	0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x47,
+	0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e,
+	0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x69, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f,
+	0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x2a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a,
+	0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f, 0x63,
+	0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0a,
+	0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65,
+	0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e,
+	0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x63,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76,
+	0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x6f,
+	0x63, 0x6b, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65,
+	0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53,
+	0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x58, 0x0a,
+	0x13, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x7a, 0x2e, 0x76, 0x31, 0x42, 0x0d, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x50, 0x72,
+	0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f,
+	0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x68,
+	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x7a, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_channelz_v1_channelz_proto_rawDescOnce sync.Once
+	file_grpc_channelz_v1_channelz_proto_rawDescData = file_grpc_channelz_v1_channelz_proto_rawDesc
+)
+
+func file_grpc_channelz_v1_channelz_proto_rawDescGZIP() []byte {
+	file_grpc_channelz_v1_channelz_proto_rawDescOnce.Do(func() {
+		file_grpc_channelz_v1_channelz_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_channelz_v1_channelz_proto_rawDescData)
+	})
+	return file_grpc_channelz_v1_channelz_proto_rawDescData
+}
+
+var file_grpc_channelz_v1_channelz_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_grpc_channelz_v1_channelz_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
+var file_grpc_channelz_v1_channelz_proto_goTypes = []interface{}{
+	(ChannelConnectivityState_State)(0), // 0: grpc.channelz.v1.ChannelConnectivityState.State
+	(ChannelTraceEvent_Severity)(0),     // 1: grpc.channelz.v1.ChannelTraceEvent.Severity
+	(*Channel)(nil),                     // 2: grpc.channelz.v1.Channel
+	(*Subchannel)(nil),                  // 3: grpc.channelz.v1.Subchannel
+	(*ChannelConnectivityState)(nil),    // 4: grpc.channelz.v1.ChannelConnectivityState
+	(*ChannelData)(nil),                 // 5: grpc.channelz.v1.ChannelData
+	(*ChannelTraceEvent)(nil),           // 6: grpc.channelz.v1.ChannelTraceEvent
+	(*ChannelTrace)(nil),                // 7: grpc.channelz.v1.ChannelTrace
+	(*ChannelRef)(nil),                  // 8: grpc.channelz.v1.ChannelRef
+	(*SubchannelRef)(nil),               // 9: grpc.channelz.v1.SubchannelRef
+	(*SocketRef)(nil),                   // 10: grpc.channelz.v1.SocketRef
+	(*ServerRef)(nil),                   // 11: grpc.channelz.v1.ServerRef
+	(*Server)(nil),                      // 12: grpc.channelz.v1.Server
+	(*ServerData)(nil),                  // 13: grpc.channelz.v1.ServerData
+	(*Socket)(nil),                      // 14: grpc.channelz.v1.Socket
+	(*SocketData)(nil),                  // 15: grpc.channelz.v1.SocketData
+	(*Address)(nil),                     // 16: grpc.channelz.v1.Address
+	(*Security)(nil),                    // 17: grpc.channelz.v1.Security
+	(*SocketOption)(nil),                // 18: grpc.channelz.v1.SocketOption
+	(*SocketOptionTimeout)(nil),         // 19: grpc.channelz.v1.SocketOptionTimeout
+	(*SocketOptionLinger)(nil),          // 20: grpc.channelz.v1.SocketOptionLinger
+	(*SocketOptionTcpInfo)(nil),         // 21: grpc.channelz.v1.SocketOptionTcpInfo
+	(*GetTopChannelsRequest)(nil),       // 22: grpc.channelz.v1.GetTopChannelsRequest
+	(*GetTopChannelsResponse)(nil),      // 23: grpc.channelz.v1.GetTopChannelsResponse
+	(*GetServersRequest)(nil),           // 24: grpc.channelz.v1.GetServersRequest
+	(*GetServersResponse)(nil),          // 25: grpc.channelz.v1.GetServersResponse
+	(*GetServerRequest)(nil),            // 26: grpc.channelz.v1.GetServerRequest
+	(*GetServerResponse)(nil),           // 27: grpc.channelz.v1.GetServerResponse
+	(*GetServerSocketsRequest)(nil),     // 28: grpc.channelz.v1.GetServerSocketsRequest
+	(*GetServerSocketsResponse)(nil),    // 29: grpc.channelz.v1.GetServerSocketsResponse
+	(*GetChannelRequest)(nil),           // 30: grpc.channelz.v1.GetChannelRequest
+	(*GetChannelResponse)(nil),          // 31: grpc.channelz.v1.GetChannelResponse
+	(*GetSubchannelRequest)(nil),        // 32: grpc.channelz.v1.GetSubchannelRequest
+	(*GetSubchannelResponse)(nil),       // 33: grpc.channelz.v1.GetSubchannelResponse
+	(*GetSocketRequest)(nil),            // 34: grpc.channelz.v1.GetSocketRequest
+	(*GetSocketResponse)(nil),           // 35: grpc.channelz.v1.GetSocketResponse
+	(*Address_TcpIpAddress)(nil),        // 36: grpc.channelz.v1.Address.TcpIpAddress
+	(*Address_UdsAddress)(nil),          // 37: grpc.channelz.v1.Address.UdsAddress
+	(*Address_OtherAddress)(nil),        // 38: grpc.channelz.v1.Address.OtherAddress
+	(*Security_Tls)(nil),                // 39: grpc.channelz.v1.Security.Tls
+	(*Security_OtherSecurity)(nil),      // 40: grpc.channelz.v1.Security.OtherSecurity
+	(*timestamp.Timestamp)(nil),         // 41: google.protobuf.Timestamp
+	(*wrappers.Int64Value)(nil),         // 42: google.protobuf.Int64Value
+	(*any.Any)(nil),                     // 43: google.protobuf.Any
+	(*duration.Duration)(nil),           // 44: google.protobuf.Duration
+}
+var file_grpc_channelz_v1_channelz_proto_depIdxs = []int32{
+	8,  // 0: grpc.channelz.v1.Channel.ref:type_name -> grpc.channelz.v1.ChannelRef
+	5,  // 1: grpc.channelz.v1.Channel.data:type_name -> grpc.channelz.v1.ChannelData
+	8,  // 2: grpc.channelz.v1.Channel.channel_ref:type_name -> grpc.channelz.v1.ChannelRef
+	9,  // 3: grpc.channelz.v1.Channel.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef
+	10, // 4: grpc.channelz.v1.Channel.socket_ref:type_name -> grpc.channelz.v1.SocketRef
+	9,  // 5: grpc.channelz.v1.Subchannel.ref:type_name -> grpc.channelz.v1.SubchannelRef
+	5,  // 6: grpc.channelz.v1.Subchannel.data:type_name -> grpc.channelz.v1.ChannelData
+	8,  // 7: grpc.channelz.v1.Subchannel.channel_ref:type_name -> grpc.channelz.v1.ChannelRef
+	9,  // 8: grpc.channelz.v1.Subchannel.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef
+	10, // 9: grpc.channelz.v1.Subchannel.socket_ref:type_name -> grpc.channelz.v1.SocketRef
+	0,  // 10: grpc.channelz.v1.ChannelConnectivityState.state:type_name -> grpc.channelz.v1.ChannelConnectivityState.State
+	4,  // 11: grpc.channelz.v1.ChannelData.state:type_name -> grpc.channelz.v1.ChannelConnectivityState
+	7,  // 12: grpc.channelz.v1.ChannelData.trace:type_name -> grpc.channelz.v1.ChannelTrace
+	41, // 13: grpc.channelz.v1.ChannelData.last_call_started_timestamp:type_name -> google.protobuf.Timestamp
+	1,  // 14: grpc.channelz.v1.ChannelTraceEvent.severity:type_name -> grpc.channelz.v1.ChannelTraceEvent.Severity
+	41, // 15: grpc.channelz.v1.ChannelTraceEvent.timestamp:type_name -> google.protobuf.Timestamp
+	8,  // 16: grpc.channelz.v1.ChannelTraceEvent.channel_ref:type_name -> grpc.channelz.v1.ChannelRef
+	9,  // 17: grpc.channelz.v1.ChannelTraceEvent.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef
+	41, // 18: grpc.channelz.v1.ChannelTrace.creation_timestamp:type_name -> google.protobuf.Timestamp
+	6,  // 19: grpc.channelz.v1.ChannelTrace.events:type_name -> grpc.channelz.v1.ChannelTraceEvent
+	11, // 20: grpc.channelz.v1.Server.ref:type_name -> grpc.channelz.v1.ServerRef
+	13, // 21: grpc.channelz.v1.Server.data:type_name -> grpc.channelz.v1.ServerData
+	10, // 22: grpc.channelz.v1.Server.listen_socket:type_name -> grpc.channelz.v1.SocketRef
+	7,  // 23: grpc.channelz.v1.ServerData.trace:type_name -> grpc.channelz.v1.ChannelTrace
+	41, // 24: grpc.channelz.v1.ServerData.last_call_started_timestamp:type_name -> google.protobuf.Timestamp
+	10, // 25: grpc.channelz.v1.Socket.ref:type_name -> grpc.channelz.v1.SocketRef
+	15, // 26: grpc.channelz.v1.Socket.data:type_name -> grpc.channelz.v1.SocketData
+	16, // 27: grpc.channelz.v1.Socket.local:type_name -> grpc.channelz.v1.Address
+	16, // 28: grpc.channelz.v1.Socket.remote:type_name -> grpc.channelz.v1.Address
+	17, // 29: grpc.channelz.v1.Socket.security:type_name -> grpc.channelz.v1.Security
+	41, // 30: grpc.channelz.v1.SocketData.last_local_stream_created_timestamp:type_name -> google.protobuf.Timestamp
+	41, // 31: grpc.channelz.v1.SocketData.last_remote_stream_created_timestamp:type_name -> google.protobuf.Timestamp
+	41, // 32: grpc.channelz.v1.SocketData.last_message_sent_timestamp:type_name -> google.protobuf.Timestamp
+	41, // 33: grpc.channelz.v1.SocketData.last_message_received_timestamp:type_name -> google.protobuf.Timestamp
+	42, // 34: grpc.channelz.v1.SocketData.local_flow_control_window:type_name -> google.protobuf.Int64Value
+	42, // 35: grpc.channelz.v1.SocketData.remote_flow_control_window:type_name -> google.protobuf.Int64Value
+	18, // 36: grpc.channelz.v1.SocketData.option:type_name -> grpc.channelz.v1.SocketOption
+	36, // 37: grpc.channelz.v1.Address.tcpip_address:type_name -> grpc.channelz.v1.Address.TcpIpAddress
+	37, // 38: grpc.channelz.v1.Address.uds_address:type_name -> grpc.channelz.v1.Address.UdsAddress
+	38, // 39: grpc.channelz.v1.Address.other_address:type_name -> grpc.channelz.v1.Address.OtherAddress
+	39, // 40: grpc.channelz.v1.Security.tls:type_name -> grpc.channelz.v1.Security.Tls
+	40, // 41: grpc.channelz.v1.Security.other:type_name -> grpc.channelz.v1.Security.OtherSecurity
+	43, // 42: grpc.channelz.v1.SocketOption.additional:type_name -> google.protobuf.Any
+	44, // 43: grpc.channelz.v1.SocketOptionTimeout.duration:type_name -> google.protobuf.Duration
+	44, // 44: grpc.channelz.v1.SocketOptionLinger.duration:type_name -> google.protobuf.Duration
+	2,  // 45: grpc.channelz.v1.GetTopChannelsResponse.channel:type_name -> grpc.channelz.v1.Channel
+	12, // 46: grpc.channelz.v1.GetServersResponse.server:type_name -> grpc.channelz.v1.Server
+	12, // 47: grpc.channelz.v1.GetServerResponse.server:type_name -> grpc.channelz.v1.Server
+	10, // 48: grpc.channelz.v1.GetServerSocketsResponse.socket_ref:type_name -> grpc.channelz.v1.SocketRef
+	2,  // 49: grpc.channelz.v1.GetChannelResponse.channel:type_name -> grpc.channelz.v1.Channel
+	3,  // 50: grpc.channelz.v1.GetSubchannelResponse.subchannel:type_name -> grpc.channelz.v1.Subchannel
+	14, // 51: grpc.channelz.v1.GetSocketResponse.socket:type_name -> grpc.channelz.v1.Socket
+	43, // 52: grpc.channelz.v1.Address.OtherAddress.value:type_name -> google.protobuf.Any
+	43, // 53: grpc.channelz.v1.Security.OtherSecurity.value:type_name -> google.protobuf.Any
+	22, // 54: grpc.channelz.v1.Channelz.GetTopChannels:input_type -> grpc.channelz.v1.GetTopChannelsRequest
+	24, // 55: grpc.channelz.v1.Channelz.GetServers:input_type -> grpc.channelz.v1.GetServersRequest
+	26, // 56: grpc.channelz.v1.Channelz.GetServer:input_type -> grpc.channelz.v1.GetServerRequest
+	28, // 57: grpc.channelz.v1.Channelz.GetServerSockets:input_type -> grpc.channelz.v1.GetServerSocketsRequest
+	30, // 58: grpc.channelz.v1.Channelz.GetChannel:input_type -> grpc.channelz.v1.GetChannelRequest
+	32, // 59: grpc.channelz.v1.Channelz.GetSubchannel:input_type -> grpc.channelz.v1.GetSubchannelRequest
+	34, // 60: grpc.channelz.v1.Channelz.GetSocket:input_type -> grpc.channelz.v1.GetSocketRequest
+	23, // 61: grpc.channelz.v1.Channelz.GetTopChannels:output_type -> grpc.channelz.v1.GetTopChannelsResponse
+	25, // 62: grpc.channelz.v1.Channelz.GetServers:output_type -> grpc.channelz.v1.GetServersResponse
+	27, // 63: grpc.channelz.v1.Channelz.GetServer:output_type -> grpc.channelz.v1.GetServerResponse
+	29, // 64: grpc.channelz.v1.Channelz.GetServerSockets:output_type -> grpc.channelz.v1.GetServerSocketsResponse
+	31, // 65: grpc.channelz.v1.Channelz.GetChannel:output_type -> grpc.channelz.v1.GetChannelResponse
+	33, // 66: grpc.channelz.v1.Channelz.GetSubchannel:output_type -> grpc.channelz.v1.GetSubchannelResponse
+	35, // 67: grpc.channelz.v1.Channelz.GetSocket:output_type -> grpc.channelz.v1.GetSocketResponse
+	61, // [61:68] is the sub-list for method output_type
+	54, // [54:61] is the sub-list for method input_type
+	54, // [54:54] is the sub-list for extension type_name
+	54, // [54:54] is the sub-list for extension extendee
+	0,  // [0:54] is the sub-list for field type_name
+}
+
+func init() { file_grpc_channelz_v1_channelz_proto_init() }
+func file_grpc_channelz_v1_channelz_proto_init() {
+	if File_grpc_channelz_v1_channelz_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_channelz_v1_channelz_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Channel); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Subchannel); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChannelConnectivityState); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChannelData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChannelTraceEvent); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChannelTrace); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChannelRef); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SubchannelRef); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SocketRef); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerRef); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Server); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Socket); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SocketData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Address); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Security); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SocketOption); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SocketOptionTimeout); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SocketOptionLinger); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SocketOptionTcpInfo); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetTopChannelsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetTopChannelsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetServersRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetServersResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetServerRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetServerResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetServerSocketsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetServerSocketsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetChannelRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetChannelResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetSubchannelRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetSubchannelResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetSocketRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetSocketResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Address_TcpIpAddress); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Address_UdsAddress); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Address_OtherAddress); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Security_Tls); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_channelz_v1_channelz_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Security_OtherSecurity); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_channelz_v1_channelz_proto_msgTypes[4].OneofWrappers = []interface{}{
+		(*ChannelTraceEvent_ChannelRef)(nil),
+		(*ChannelTraceEvent_SubchannelRef)(nil),
+	}
+	file_grpc_channelz_v1_channelz_proto_msgTypes[14].OneofWrappers = []interface{}{
+		(*Address_TcpipAddress)(nil),
+		(*Address_UdsAddress_)(nil),
+		(*Address_OtherAddress_)(nil),
+	}
+	file_grpc_channelz_v1_channelz_proto_msgTypes[15].OneofWrappers = []interface{}{
+		(*Security_Tls_)(nil),
+		(*Security_Other)(nil),
+	}
+	file_grpc_channelz_v1_channelz_proto_msgTypes[37].OneofWrappers = []interface{}{
+		(*Security_Tls_StandardName)(nil),
+		(*Security_Tls_OtherName)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_channelz_v1_channelz_proto_rawDesc,
+			NumEnums:      2,
+			NumMessages:   39,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_channelz_v1_channelz_proto_goTypes,
+		DependencyIndexes: file_grpc_channelz_v1_channelz_proto_depIdxs,
+		EnumInfos:         file_grpc_channelz_v1_channelz_proto_enumTypes,
+		MessageInfos:      file_grpc_channelz_v1_channelz_proto_msgTypes,
+	}.Build()
+	File_grpc_channelz_v1_channelz_proto = out.File
+	file_grpc_channelz_v1_channelz_proto_rawDesc = nil
+	file_grpc_channelz_v1_channelz_proto_goTypes = nil
+	file_grpc_channelz_v1_channelz_proto_depIdxs = nil
 }
diff --git a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go
index 38c4832dfd4b..e0744eef355e 100644
--- a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go
@@ -1,26 +1,52 @@
+// Copyright 2018 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/altscontext.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/gcp/altscontext.proto
 
 package grpc_gcp
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type AltsContext struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The application protocol negotiated for this connection.
 	ApplicationProtocol string `protobuf:"bytes,1,opt,name=application_protocol,json=applicationProtocol,proto3" json:"application_protocol,omitempty"`
 	// The record protocol negotiated for this connection.
@@ -34,119 +60,205 @@ type AltsContext struct {
 	// The RPC protocol versions supported by the peer.
 	PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"`
 	// Additional attributes of the peer.
-	PeerAttributes       map[string]string `protobuf:"bytes,7,rep,name=peer_attributes,json=peerAttributes,proto3" json:"peer_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	PeerAttributes map[string]string `protobuf:"bytes,7,rep,name=peer_attributes,json=peerAttributes,proto3" json:"peer_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
-func (m *AltsContext) Reset()         { *m = AltsContext{} }
-func (m *AltsContext) String() string { return proto.CompactTextString(m) }
-func (*AltsContext) ProtoMessage()    {}
-func (*AltsContext) Descriptor() ([]byte, []int) {
-	return fileDescriptor_6647a41e53a575a3, []int{0}
+func (x *AltsContext) Reset() {
+	*x = AltsContext{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_altscontext_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *AltsContext) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_AltsContext.Unmarshal(m, b)
-}
-func (m *AltsContext) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_AltsContext.Marshal(b, m, deterministic)
+func (x *AltsContext) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *AltsContext) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_AltsContext.Merge(m, src)
-}
-func (m *AltsContext) XXX_Size() int {
-	return xxx_messageInfo_AltsContext.Size(m)
-}
-func (m *AltsContext) XXX_DiscardUnknown() {
-	xxx_messageInfo_AltsContext.DiscardUnknown(m)
+
+func (*AltsContext) ProtoMessage() {}
+
+func (x *AltsContext) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_altscontext_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_AltsContext proto.InternalMessageInfo
+// Deprecated: Use AltsContext.ProtoReflect.Descriptor instead.
+func (*AltsContext) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_altscontext_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *AltsContext) GetApplicationProtocol() string {
-	if m != nil {
-		return m.ApplicationProtocol
+func (x *AltsContext) GetApplicationProtocol() string {
+	if x != nil {
+		return x.ApplicationProtocol
 	}
 	return ""
 }
 
-func (m *AltsContext) GetRecordProtocol() string {
-	if m != nil {
-		return m.RecordProtocol
+func (x *AltsContext) GetRecordProtocol() string {
+	if x != nil {
+		return x.RecordProtocol
 	}
 	return ""
 }
 
-func (m *AltsContext) GetSecurityLevel() SecurityLevel {
-	if m != nil {
-		return m.SecurityLevel
+func (x *AltsContext) GetSecurityLevel() SecurityLevel {
+	if x != nil {
+		return x.SecurityLevel
 	}
 	return SecurityLevel_SECURITY_NONE
 }
 
-func (m *AltsContext) GetPeerServiceAccount() string {
-	if m != nil {
-		return m.PeerServiceAccount
+func (x *AltsContext) GetPeerServiceAccount() string {
+	if x != nil {
+		return x.PeerServiceAccount
 	}
 	return ""
 }
 
-func (m *AltsContext) GetLocalServiceAccount() string {
-	if m != nil {
-		return m.LocalServiceAccount
+func (x *AltsContext) GetLocalServiceAccount() string {
+	if x != nil {
+		return x.LocalServiceAccount
 	}
 	return ""
 }
 
-func (m *AltsContext) GetPeerRpcVersions() *RpcProtocolVersions {
-	if m != nil {
-		return m.PeerRpcVersions
+func (x *AltsContext) GetPeerRpcVersions() *RpcProtocolVersions {
+	if x != nil {
+		return x.PeerRpcVersions
 	}
 	return nil
 }
 
-func (m *AltsContext) GetPeerAttributes() map[string]string {
-	if m != nil {
-		return m.PeerAttributes
+func (x *AltsContext) GetPeerAttributes() map[string]string {
+	if x != nil {
+		return x.PeerAttributes
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterType((*AltsContext)(nil), "grpc.gcp.AltsContext")
-	proto.RegisterMapType((map[string]string)(nil), "grpc.gcp.AltsContext.PeerAttributesEntry")
-}
-
-func init() { proto.RegisterFile("grpc/gcp/altscontext.proto", fileDescriptor_6647a41e53a575a3) }
-
-var fileDescriptor_6647a41e53a575a3 = []byte{
-	// 411 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0x4d, 0x6f, 0x13, 0x31,
-	0x10, 0x86, 0xb5, 0x0d, 0x2d, 0xe0, 0x88, 0xb4, 0xb8, 0xa9, 0x58, 0x45, 0x42, 0x8a, 0xb8, 0xb0,
-	0x5c, 0x76, 0x21, 0x5c, 0x10, 0x07, 0x50, 0x8a, 0x38, 0x20, 0x71, 0x88, 0xb6, 0x12, 0x07, 0x2e,
-	0x2b, 0x77, 0x3a, 0xb2, 0x2c, 0x5c, 0x8f, 0x35, 0x76, 0x22, 0xf2, 0xb3, 0xf9, 0x07, 0x68, 0xed,
-	0xcd, 0x07, 0x1f, 0xb7, 0x9d, 0x79, 0x9f, 0x19, 0xbf, 0xb3, 0x33, 0x62, 0xa6, 0xd9, 0x43, 0xa3,
-	0xc1, 0x37, 0xca, 0xc6, 0x00, 0xe4, 0x22, 0xfe, 0x8c, 0xb5, 0x67, 0x8a, 0x24, 0x1f, 0xf5, 0x5a,
-	0xad, 0xc1, 0xcf, 0xaa, 0x3d, 0x15, 0x59, 0xb9, 0xe0, 0x89, 0x63, 0x17, 0x10, 0xd6, 0x6c, 0xe2,
-	0xb6, 0x03, 0xba, 0xbf, 0x27, 0x97, 0x6b, 0x5e, 0xfc, 0x1a, 0x89, 0xf1, 0xd2, 0xc6, 0xf0, 0x29,
-	0x77, 0x92, 0x6f, 0xc4, 0x54, 0x79, 0x6f, 0x0d, 0xa8, 0x68, 0xc8, 0x75, 0x09, 0x02, 0xb2, 0x65,
-	0x31, 0x2f, 0xaa, 0xc7, 0xed, 0xe5, 0x91, 0xb6, 0x1a, 0x24, 0xf9, 0x52, 0x9c, 0x33, 0x02, 0xf1,
-	0xdd, 0x81, 0x3e, 0x49, 0xf4, 0x24, 0xa7, 0xf7, 0xe0, 0x07, 0x31, 0xd9, 0x9b, 0xb0, 0xb8, 0x41,
-	0x5b, 0x8e, 0xe6, 0x45, 0x35, 0x59, 0x3c, 0xab, 0x77, 0xc6, 0xeb, 0x9b, 0x41, 0xff, 0xda, 0xcb,
-	0xed, 0x93, 0x70, 0x1c, 0xca, 0xd7, 0x62, 0xea, 0x11, 0xb9, 0x0b, 0xc8, 0x1b, 0x03, 0xd8, 0x29,
-	0x00, 0x5a, 0xbb, 0x58, 0x3e, 0x48, 0xaf, 0xc9, 0x5e, 0xbb, 0xc9, 0xd2, 0x32, 0x2b, 0x72, 0x21,
-	0xae, 0x2c, 0x81, 0xb2, 0xff, 0x94, 0x9c, 0xe6, 0x71, 0x92, 0xf8, 0x57, 0xcd, 0x17, 0xf1, 0x34,
-	0xbd, 0xc2, 0x1e, 0xba, 0x0d, 0x72, 0x30, 0xe4, 0x42, 0x79, 0x36, 0x2f, 0xaa, 0xf1, 0xe2, 0xf9,
-	0xc1, 0x68, 0xeb, 0x61, 0x37, 0xd7, 0xb7, 0x01, 0x6a, 0xcf, 0xfb, 0xba, 0xd6, 0xc3, 0x2e, 0x21,
-	0x5b, 0x91, 0x52, 0x9d, 0x8a, 0x91, 0xcd, 0xed, 0x3a, 0x62, 0x28, 0x1f, 0xce, 0x47, 0xd5, 0x78,
-	0xf1, 0xea, 0xd0, 0xe8, 0xe8, 0xe7, 0xd7, 0x2b, 0x44, 0x5e, 0xee, 0xd9, 0xcf, 0x2e, 0xf2, 0xb6,
-	0x9d, 0xf8, 0x3f, 0x92, 0xb3, 0xa5, 0xb8, 0xfc, 0x0f, 0x26, 0x2f, 0xc4, 0xe8, 0x07, 0x6e, 0x87,
-	0x35, 0xf5, 0x9f, 0x72, 0x2a, 0x4e, 0x37, 0xca, 0xae, 0x71, 0x58, 0x46, 0x0e, 0xde, 0x9f, 0xbc,
-	0x2b, 0xae, 0xad, 0xb8, 0x32, 0x94, 0x1d, 0xf4, 0x47, 0x54, 0x1b, 0x17, 0x91, 0x9d, 0xb2, 0xd7,
-	0x17, 0x47, 0x66, 0xd2, 0x74, 0xab, 0xe2, 0xfb, 0x47, 0x4d, 0xa4, 0x2d, 0xd6, 0x9a, 0xac, 0x72,
-	0xba, 0x26, 0xd6, 0x4d, 0x3a, 0x2e, 0x60, 0xbc, 0x43, 0x17, 0x8d, 0xb2, 0x21, 0x9d, 0x62, 0xb3,
-	0xeb, 0xd2, 0xa4, 0x2b, 0x48, 0x50, 0xa7, 0xc1, 0xdf, 0x9e, 0xa5, 0xf8, 0xed, 0xef, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0x9b, 0x8c, 0xe4, 0x6a, 0xba, 0x02, 0x00, 0x00,
+var File_grpc_gcp_altscontext_proto protoreflect.FileDescriptor
+
+var file_grpc_gcp_altscontext_proto_rawDesc = []byte{
+	0x0a, 0x1a, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x63,
+	0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x1a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70,
+	0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72,
+	0x69, 0x74, 0x79, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x22, 0xf1, 0x03, 0x0a, 0x0b, 0x41, 0x6c, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74,
+	0x12, 0x31, 0x0a, 0x14, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13,
+	0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f,
+	0x63, 0x6f, 0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65,
+	0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x3e, 0x0a, 0x0e,
+	0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03,
+	0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e,
+	0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x0d, 0x73,
+	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x30, 0x0a, 0x14,
+	0x70, 0x65, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63,
+	0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x65, 0x65, 0x72,
+	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x32,
+	0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f,
+	0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x6c,
+	0x6f, 0x63, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75,
+	0x6e, 0x74, 0x12, 0x49, 0x0a, 0x11, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76,
+	0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0f, 0x70, 0x65,
+	0x65, 0x72, 0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x52, 0x0a,
+	0x0f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73,
+	0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63,
+	0x70, 0x2e, 0x41, 0x6c, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x50, 0x65,
+	0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65,
+	0x73, 0x1a, 0x41, 0x0a, 0x13, 0x50, 0x65, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75,
+	0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x3a, 0x02, 0x38, 0x01, 0x42, 0x6c, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x61, 0x6c, 0x74, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x10, 0x41,
+	0x6c, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50,
+	0x01, 0x5a, 0x3f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67,
+	0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x73, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72,
+	0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x67,
+	0x63, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_gcp_altscontext_proto_rawDescOnce sync.Once
+	file_grpc_gcp_altscontext_proto_rawDescData = file_grpc_gcp_altscontext_proto_rawDesc
+)
+
+func file_grpc_gcp_altscontext_proto_rawDescGZIP() []byte {
+	file_grpc_gcp_altscontext_proto_rawDescOnce.Do(func() {
+		file_grpc_gcp_altscontext_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_gcp_altscontext_proto_rawDescData)
+	})
+	return file_grpc_gcp_altscontext_proto_rawDescData
+}
+
+var file_grpc_gcp_altscontext_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_grpc_gcp_altscontext_proto_goTypes = []interface{}{
+	(*AltsContext)(nil),         // 0: grpc.gcp.AltsContext
+	nil,                         // 1: grpc.gcp.AltsContext.PeerAttributesEntry
+	(SecurityLevel)(0),          // 2: grpc.gcp.SecurityLevel
+	(*RpcProtocolVersions)(nil), // 3: grpc.gcp.RpcProtocolVersions
+}
+var file_grpc_gcp_altscontext_proto_depIdxs = []int32{
+	2, // 0: grpc.gcp.AltsContext.security_level:type_name -> grpc.gcp.SecurityLevel
+	3, // 1: grpc.gcp.AltsContext.peer_rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions
+	1, // 2: grpc.gcp.AltsContext.peer_attributes:type_name -> grpc.gcp.AltsContext.PeerAttributesEntry
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_grpc_gcp_altscontext_proto_init() }
+func file_grpc_gcp_altscontext_proto_init() {
+	if File_grpc_gcp_altscontext_proto != nil {
+		return
+	}
+	file_grpc_gcp_transport_security_common_proto_init()
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_gcp_altscontext_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*AltsContext); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_gcp_altscontext_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_gcp_altscontext_proto_goTypes,
+		DependencyIndexes: file_grpc_gcp_altscontext_proto_depIdxs,
+		MessageInfos:      file_grpc_gcp_altscontext_proto_msgTypes,
+	}.Build()
+	File_grpc_gcp_altscontext_proto = out.File
+	file_grpc_gcp_altscontext_proto_rawDesc = nil
+	file_grpc_gcp_altscontext_proto_goTypes = nil
+	file_grpc_gcp_altscontext_proto_depIdxs = nil
 }
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
index 6d9c304e7981..ed94ab265bbb 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
@@ -1,24 +1,46 @@
+// Copyright 2018 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/handshaker.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/gcp/handshaker.proto
 
 package grpc_gcp
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type HandshakeProtocol int32
 
@@ -31,24 +53,45 @@ const (
 	HandshakeProtocol_ALTS HandshakeProtocol = 2
 )
 
-var HandshakeProtocol_name = map[int32]string{
-	0: "HANDSHAKE_PROTOCOL_UNSPECIFIED",
-	1: "TLS",
-	2: "ALTS",
-}
+// Enum value maps for HandshakeProtocol.
+var (
+	HandshakeProtocol_name = map[int32]string{
+		0: "HANDSHAKE_PROTOCOL_UNSPECIFIED",
+		1: "TLS",
+		2: "ALTS",
+	}
+	HandshakeProtocol_value = map[string]int32{
+		"HANDSHAKE_PROTOCOL_UNSPECIFIED": 0,
+		"TLS":                            1,
+		"ALTS":                           2,
+	}
+)
 
-var HandshakeProtocol_value = map[string]int32{
-	"HANDSHAKE_PROTOCOL_UNSPECIFIED": 0,
-	"TLS":                            1,
-	"ALTS":                           2,
+func (x HandshakeProtocol) Enum() *HandshakeProtocol {
+	p := new(HandshakeProtocol)
+	*p = x
+	return p
 }
 
 func (x HandshakeProtocol) String() string {
-	return proto.EnumName(HandshakeProtocol_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (HandshakeProtocol) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_gcp_handshaker_proto_enumTypes[0].Descriptor()
+}
+
+func (HandshakeProtocol) Type() protoreflect.EnumType {
+	return &file_grpc_gcp_handshaker_proto_enumTypes[0]
+}
+
+func (x HandshakeProtocol) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
+// Deprecated: Use HandshakeProtocol.Descriptor instead.
 func (HandshakeProtocol) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{0}
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{0}
 }
 
 type NetworkProtocol int32
@@ -59,138 +102,159 @@ const (
 	NetworkProtocol_UDP                          NetworkProtocol = 2
 )
 
-var NetworkProtocol_name = map[int32]string{
-	0: "NETWORK_PROTOCOL_UNSPECIFIED",
-	1: "TCP",
-	2: "UDP",
-}
+// Enum value maps for NetworkProtocol.
+var (
+	NetworkProtocol_name = map[int32]string{
+		0: "NETWORK_PROTOCOL_UNSPECIFIED",
+		1: "TCP",
+		2: "UDP",
+	}
+	NetworkProtocol_value = map[string]int32{
+		"NETWORK_PROTOCOL_UNSPECIFIED": 0,
+		"TCP":                          1,
+		"UDP":                          2,
+	}
+)
 
-var NetworkProtocol_value = map[string]int32{
-	"NETWORK_PROTOCOL_UNSPECIFIED": 0,
-	"TCP":                          1,
-	"UDP":                          2,
+func (x NetworkProtocol) Enum() *NetworkProtocol {
+	p := new(NetworkProtocol)
+	*p = x
+	return p
 }
 
 func (x NetworkProtocol) String() string {
-	return proto.EnumName(NetworkProtocol_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (NetworkProtocol) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_gcp_handshaker_proto_enumTypes[1].Descriptor()
+}
+
+func (NetworkProtocol) Type() protoreflect.EnumType {
+	return &file_grpc_gcp_handshaker_proto_enumTypes[1]
+}
+
+func (x NetworkProtocol) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
+// Deprecated: Use NetworkProtocol.Descriptor instead.
 func (NetworkProtocol) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{1}
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{1}
 }
 
 type Endpoint struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// IP address. It should contain an IPv4 or IPv6 string literal, e.g.
 	// "192.168.0.1" or "2001:db8::1".
 	IpAddress string `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"`
 	// Port number.
 	Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
 	// Network protocol (e.g., TCP, UDP) associated with this endpoint.
-	Protocol             NetworkProtocol `protobuf:"varint,3,opt,name=protocol,proto3,enum=grpc.gcp.NetworkProtocol" json:"protocol,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}        `json:"-"`
-	XXX_unrecognized     []byte          `json:"-"`
-	XXX_sizecache        int32           `json:"-"`
+	Protocol NetworkProtocol `protobuf:"varint,3,opt,name=protocol,proto3,enum=grpc.gcp.NetworkProtocol" json:"protocol,omitempty"`
 }
 
-func (m *Endpoint) Reset()         { *m = Endpoint{} }
-func (m *Endpoint) String() string { return proto.CompactTextString(m) }
-func (*Endpoint) ProtoMessage()    {}
-func (*Endpoint) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{0}
+func (x *Endpoint) Reset() {
+	*x = Endpoint{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Endpoint) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Endpoint.Unmarshal(m, b)
-}
-func (m *Endpoint) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Endpoint.Marshal(b, m, deterministic)
-}
-func (m *Endpoint) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Endpoint.Merge(m, src)
+func (x *Endpoint) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Endpoint) XXX_Size() int {
-	return xxx_messageInfo_Endpoint.Size(m)
-}
-func (m *Endpoint) XXX_DiscardUnknown() {
-	xxx_messageInfo_Endpoint.DiscardUnknown(m)
+
+func (*Endpoint) ProtoMessage() {}
+
+func (x *Endpoint) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Endpoint proto.InternalMessageInfo
+// Deprecated: Use Endpoint.ProtoReflect.Descriptor instead.
+func (*Endpoint) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *Endpoint) GetIpAddress() string {
-	if m != nil {
-		return m.IpAddress
+func (x *Endpoint) GetIpAddress() string {
+	if x != nil {
+		return x.IpAddress
 	}
 	return ""
 }
 
-func (m *Endpoint) GetPort() int32 {
-	if m != nil {
-		return m.Port
+func (x *Endpoint) GetPort() int32 {
+	if x != nil {
+		return x.Port
 	}
 	return 0
 }
 
-func (m *Endpoint) GetProtocol() NetworkProtocol {
-	if m != nil {
-		return m.Protocol
+func (x *Endpoint) GetProtocol() NetworkProtocol {
+	if x != nil {
+		return x.Protocol
 	}
 	return NetworkProtocol_NETWORK_PROTOCOL_UNSPECIFIED
 }
 
 type Identity struct {
-	// Types that are valid to be assigned to IdentityOneof:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to IdentityOneof:
 	//	*Identity_ServiceAccount
 	//	*Identity_Hostname
 	IdentityOneof isIdentity_IdentityOneof `protobuf_oneof:"identity_oneof"`
 	// Additional attributes of the identity.
-	Attributes           map[string]string `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	Attributes map[string]string `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
-func (m *Identity) Reset()         { *m = Identity{} }
-func (m *Identity) String() string { return proto.CompactTextString(m) }
-func (*Identity) ProtoMessage()    {}
-func (*Identity) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{1}
+func (x *Identity) Reset() {
+	*x = Identity{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Identity) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Identity.Unmarshal(m, b)
-}
-func (m *Identity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Identity.Marshal(b, m, deterministic)
-}
-func (m *Identity) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Identity.Merge(m, src)
-}
-func (m *Identity) XXX_Size() int {
-	return xxx_messageInfo_Identity.Size(m)
-}
-func (m *Identity) XXX_DiscardUnknown() {
-	xxx_messageInfo_Identity.DiscardUnknown(m)
+func (x *Identity) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_Identity proto.InternalMessageInfo
-
-type isIdentity_IdentityOneof interface {
-	isIdentity_IdentityOneof()
-}
+func (*Identity) ProtoMessage() {}
 
-type Identity_ServiceAccount struct {
-	ServiceAccount string `protobuf:"bytes,1,opt,name=service_account,json=serviceAccount,proto3,oneof"`
+func (x *Identity) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-type Identity_Hostname struct {
-	Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3,oneof"`
+// Deprecated: Use Identity.ProtoReflect.Descriptor instead.
+func (*Identity) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{1}
 }
 
-func (*Identity_ServiceAccount) isIdentity_IdentityOneof() {}
-
-func (*Identity_Hostname) isIdentity_IdentityOneof() {}
-
 func (m *Identity) GetIdentityOneof() isIdentity_IdentityOneof {
 	if m != nil {
 		return m.IdentityOneof
@@ -198,36 +262,50 @@ func (m *Identity) GetIdentityOneof() isIdentity_IdentityOneof {
 	return nil
 }
 
-func (m *Identity) GetServiceAccount() string {
-	if x, ok := m.GetIdentityOneof().(*Identity_ServiceAccount); ok {
+func (x *Identity) GetServiceAccount() string {
+	if x, ok := x.GetIdentityOneof().(*Identity_ServiceAccount); ok {
 		return x.ServiceAccount
 	}
 	return ""
 }
 
-func (m *Identity) GetHostname() string {
-	if x, ok := m.GetIdentityOneof().(*Identity_Hostname); ok {
+func (x *Identity) GetHostname() string {
+	if x, ok := x.GetIdentityOneof().(*Identity_Hostname); ok {
 		return x.Hostname
 	}
 	return ""
 }
 
-func (m *Identity) GetAttributes() map[string]string {
-	if m != nil {
-		return m.Attributes
+func (x *Identity) GetAttributes() map[string]string {
+	if x != nil {
+		return x.Attributes
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*Identity) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*Identity_ServiceAccount)(nil),
-		(*Identity_Hostname)(nil),
-	}
+type isIdentity_IdentityOneof interface {
+	isIdentity_IdentityOneof()
+}
+
+type Identity_ServiceAccount struct {
+	// Service account of a connection endpoint.
+	ServiceAccount string `protobuf:"bytes,1,opt,name=service_account,json=serviceAccount,proto3,oneof"`
+}
+
+type Identity_Hostname struct {
+	// Hostname of a connection endpoint.
+	Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3,oneof"`
 }
 
+func (*Identity_ServiceAccount) isIdentity_IdentityOneof() {}
+
+func (*Identity_Hostname) isIdentity_IdentityOneof() {}
+
 type StartClientHandshakeReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Handshake security protocol requested by the client.
 	HandshakeSecurityProtocol HandshakeProtocol `protobuf:"varint,1,opt,name=handshake_security_protocol,json=handshakeSecurityProtocol,proto3,enum=grpc.gcp.HandshakeProtocol" json:"handshake_security_protocol,omitempty"`
 	// The application protocols supported by the client, e.g., "h2" (for http2),
@@ -256,159 +334,175 @@ type StartClientHandshakeReq struct {
 	// (Optional) RPC protocol versions supported by the client.
 	RpcVersions *RpcProtocolVersions `protobuf:"bytes,9,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"`
 	// (Optional) Maximum frame size supported by the client.
-	MaxFrameSize         uint32   `protobuf:"varint,10,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	MaxFrameSize uint32 `protobuf:"varint,10,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"`
 }
 
-func (m *StartClientHandshakeReq) Reset()         { *m = StartClientHandshakeReq{} }
-func (m *StartClientHandshakeReq) String() string { return proto.CompactTextString(m) }
-func (*StartClientHandshakeReq) ProtoMessage()    {}
-func (*StartClientHandshakeReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{2}
+func (x *StartClientHandshakeReq) Reset() {
+	*x = StartClientHandshakeReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StartClientHandshakeReq) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StartClientHandshakeReq.Unmarshal(m, b)
+func (x *StartClientHandshakeReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StartClientHandshakeReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StartClientHandshakeReq.Marshal(b, m, deterministic)
-}
-func (m *StartClientHandshakeReq) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StartClientHandshakeReq.Merge(m, src)
-}
-func (m *StartClientHandshakeReq) XXX_Size() int {
-	return xxx_messageInfo_StartClientHandshakeReq.Size(m)
-}
-func (m *StartClientHandshakeReq) XXX_DiscardUnknown() {
-	xxx_messageInfo_StartClientHandshakeReq.DiscardUnknown(m)
+
+func (*StartClientHandshakeReq) ProtoMessage() {}
+
+func (x *StartClientHandshakeReq) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StartClientHandshakeReq proto.InternalMessageInfo
+// Deprecated: Use StartClientHandshakeReq.ProtoReflect.Descriptor instead.
+func (*StartClientHandshakeReq) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *StartClientHandshakeReq) GetHandshakeSecurityProtocol() HandshakeProtocol {
-	if m != nil {
-		return m.HandshakeSecurityProtocol
+func (x *StartClientHandshakeReq) GetHandshakeSecurityProtocol() HandshakeProtocol {
+	if x != nil {
+		return x.HandshakeSecurityProtocol
 	}
 	return HandshakeProtocol_HANDSHAKE_PROTOCOL_UNSPECIFIED
 }
 
-func (m *StartClientHandshakeReq) GetApplicationProtocols() []string {
-	if m != nil {
-		return m.ApplicationProtocols
+func (x *StartClientHandshakeReq) GetApplicationProtocols() []string {
+	if x != nil {
+		return x.ApplicationProtocols
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetRecordProtocols() []string {
-	if m != nil {
-		return m.RecordProtocols
+func (x *StartClientHandshakeReq) GetRecordProtocols() []string {
+	if x != nil {
+		return x.RecordProtocols
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetTargetIdentities() []*Identity {
-	if m != nil {
-		return m.TargetIdentities
+func (x *StartClientHandshakeReq) GetTargetIdentities() []*Identity {
+	if x != nil {
+		return x.TargetIdentities
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetLocalIdentity() *Identity {
-	if m != nil {
-		return m.LocalIdentity
+func (x *StartClientHandshakeReq) GetLocalIdentity() *Identity {
+	if x != nil {
+		return x.LocalIdentity
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetLocalEndpoint() *Endpoint {
-	if m != nil {
-		return m.LocalEndpoint
+func (x *StartClientHandshakeReq) GetLocalEndpoint() *Endpoint {
+	if x != nil {
+		return x.LocalEndpoint
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetRemoteEndpoint() *Endpoint {
-	if m != nil {
-		return m.RemoteEndpoint
+func (x *StartClientHandshakeReq) GetRemoteEndpoint() *Endpoint {
+	if x != nil {
+		return x.RemoteEndpoint
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetTargetName() string {
-	if m != nil {
-		return m.TargetName
+func (x *StartClientHandshakeReq) GetTargetName() string {
+	if x != nil {
+		return x.TargetName
 	}
 	return ""
 }
 
-func (m *StartClientHandshakeReq) GetRpcVersions() *RpcProtocolVersions {
-	if m != nil {
-		return m.RpcVersions
+func (x *StartClientHandshakeReq) GetRpcVersions() *RpcProtocolVersions {
+	if x != nil {
+		return x.RpcVersions
 	}
 	return nil
 }
 
-func (m *StartClientHandshakeReq) GetMaxFrameSize() uint32 {
-	if m != nil {
-		return m.MaxFrameSize
+func (x *StartClientHandshakeReq) GetMaxFrameSize() uint32 {
+	if x != nil {
+		return x.MaxFrameSize
 	}
 	return 0
 }
 
 type ServerHandshakeParameters struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The record protocols supported by the server, e.g.,
 	// "ALTSRP_GCM_AES128".
 	RecordProtocols []string `protobuf:"bytes,1,rep,name=record_protocols,json=recordProtocols,proto3" json:"record_protocols,omitempty"`
 	// (Optional) A list of local identities supported by the server, if
 	// specified. Otherwise, the handshaker chooses a default local identity.
-	LocalIdentities      []*Identity `protobuf:"bytes,2,rep,name=local_identities,json=localIdentities,proto3" json:"local_identities,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
+	LocalIdentities []*Identity `protobuf:"bytes,2,rep,name=local_identities,json=localIdentities,proto3" json:"local_identities,omitempty"`
 }
 
-func (m *ServerHandshakeParameters) Reset()         { *m = ServerHandshakeParameters{} }
-func (m *ServerHandshakeParameters) String() string { return proto.CompactTextString(m) }
-func (*ServerHandshakeParameters) ProtoMessage()    {}
-func (*ServerHandshakeParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{3}
+func (x *ServerHandshakeParameters) Reset() {
+	*x = ServerHandshakeParameters{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerHandshakeParameters) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerHandshakeParameters.Unmarshal(m, b)
-}
-func (m *ServerHandshakeParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerHandshakeParameters.Marshal(b, m, deterministic)
-}
-func (m *ServerHandshakeParameters) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerHandshakeParameters.Merge(m, src)
+func (x *ServerHandshakeParameters) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerHandshakeParameters) XXX_Size() int {
-	return xxx_messageInfo_ServerHandshakeParameters.Size(m)
-}
-func (m *ServerHandshakeParameters) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerHandshakeParameters.DiscardUnknown(m)
+
+func (*ServerHandshakeParameters) ProtoMessage() {}
+
+func (x *ServerHandshakeParameters) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServerHandshakeParameters proto.InternalMessageInfo
+// Deprecated: Use ServerHandshakeParameters.ProtoReflect.Descriptor instead.
+func (*ServerHandshakeParameters) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *ServerHandshakeParameters) GetRecordProtocols() []string {
-	if m != nil {
-		return m.RecordProtocols
+func (x *ServerHandshakeParameters) GetRecordProtocols() []string {
+	if x != nil {
+		return x.RecordProtocols
 	}
 	return nil
 }
 
-func (m *ServerHandshakeParameters) GetLocalIdentities() []*Identity {
-	if m != nil {
-		return m.LocalIdentities
+func (x *ServerHandshakeParameters) GetLocalIdentities() []*Identity {
+	if x != nil {
+		return x.LocalIdentities
 	}
 	return nil
 }
 
 type StartServerHandshakeReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The application protocols supported by the server, e.g., "h2" (for http2),
 	// "grpc".
 	ApplicationProtocols []string `protobuf:"bytes,1,rep,name=application_protocols,json=applicationProtocols,proto3" json:"application_protocols,omitempty"`
@@ -430,186 +524,184 @@ type StartServerHandshakeReq struct {
 	// (Optional) RPC protocol versions supported by the server.
 	RpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"`
 	// (Optional) Maximum frame size supported by the server.
-	MaxFrameSize         uint32   `protobuf:"varint,7,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	MaxFrameSize uint32 `protobuf:"varint,7,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"`
 }
 
-func (m *StartServerHandshakeReq) Reset()         { *m = StartServerHandshakeReq{} }
-func (m *StartServerHandshakeReq) String() string { return proto.CompactTextString(m) }
-func (*StartServerHandshakeReq) ProtoMessage()    {}
-func (*StartServerHandshakeReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{4}
+func (x *StartServerHandshakeReq) Reset() {
+	*x = StartServerHandshakeReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StartServerHandshakeReq) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StartServerHandshakeReq.Unmarshal(m, b)
-}
-func (m *StartServerHandshakeReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StartServerHandshakeReq.Marshal(b, m, deterministic)
+func (x *StartServerHandshakeReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StartServerHandshakeReq) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StartServerHandshakeReq.Merge(m, src)
-}
-func (m *StartServerHandshakeReq) XXX_Size() int {
-	return xxx_messageInfo_StartServerHandshakeReq.Size(m)
-}
-func (m *StartServerHandshakeReq) XXX_DiscardUnknown() {
-	xxx_messageInfo_StartServerHandshakeReq.DiscardUnknown(m)
+
+func (*StartServerHandshakeReq) ProtoMessage() {}
+
+func (x *StartServerHandshakeReq) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StartServerHandshakeReq proto.InternalMessageInfo
+// Deprecated: Use StartServerHandshakeReq.ProtoReflect.Descriptor instead.
+func (*StartServerHandshakeReq) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *StartServerHandshakeReq) GetApplicationProtocols() []string {
-	if m != nil {
-		return m.ApplicationProtocols
+func (x *StartServerHandshakeReq) GetApplicationProtocols() []string {
+	if x != nil {
+		return x.ApplicationProtocols
 	}
 	return nil
 }
 
-func (m *StartServerHandshakeReq) GetHandshakeParameters() map[int32]*ServerHandshakeParameters {
-	if m != nil {
-		return m.HandshakeParameters
+func (x *StartServerHandshakeReq) GetHandshakeParameters() map[int32]*ServerHandshakeParameters {
+	if x != nil {
+		return x.HandshakeParameters
 	}
 	return nil
 }
 
-func (m *StartServerHandshakeReq) GetInBytes() []byte {
-	if m != nil {
-		return m.InBytes
+func (x *StartServerHandshakeReq) GetInBytes() []byte {
+	if x != nil {
+		return x.InBytes
 	}
 	return nil
 }
 
-func (m *StartServerHandshakeReq) GetLocalEndpoint() *Endpoint {
-	if m != nil {
-		return m.LocalEndpoint
+func (x *StartServerHandshakeReq) GetLocalEndpoint() *Endpoint {
+	if x != nil {
+		return x.LocalEndpoint
 	}
 	return nil
 }
 
-func (m *StartServerHandshakeReq) GetRemoteEndpoint() *Endpoint {
-	if m != nil {
-		return m.RemoteEndpoint
+func (x *StartServerHandshakeReq) GetRemoteEndpoint() *Endpoint {
+	if x != nil {
+		return x.RemoteEndpoint
 	}
 	return nil
 }
 
-func (m *StartServerHandshakeReq) GetRpcVersions() *RpcProtocolVersions {
-	if m != nil {
-		return m.RpcVersions
+func (x *StartServerHandshakeReq) GetRpcVersions() *RpcProtocolVersions {
+	if x != nil {
+		return x.RpcVersions
 	}
 	return nil
 }
 
-func (m *StartServerHandshakeReq) GetMaxFrameSize() uint32 {
-	if m != nil {
-		return m.MaxFrameSize
+func (x *StartServerHandshakeReq) GetMaxFrameSize() uint32 {
+	if x != nil {
+		return x.MaxFrameSize
 	}
 	return 0
 }
 
 type NextHandshakeMessageReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Bytes in out_frames returned from the peer's HandshakerResp. It is possible
 	// that the peer's out_frames are split into multiple NextHandshakerMessageReq
 	// messages.
-	InBytes              []byte   `protobuf:"bytes,1,opt,name=in_bytes,json=inBytes,proto3" json:"in_bytes,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	InBytes []byte `protobuf:"bytes,1,opt,name=in_bytes,json=inBytes,proto3" json:"in_bytes,omitempty"`
 }
 
-func (m *NextHandshakeMessageReq) Reset()         { *m = NextHandshakeMessageReq{} }
-func (m *NextHandshakeMessageReq) String() string { return proto.CompactTextString(m) }
-func (*NextHandshakeMessageReq) ProtoMessage()    {}
-func (*NextHandshakeMessageReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{5}
+func (x *NextHandshakeMessageReq) Reset() {
+	*x = NextHandshakeMessageReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *NextHandshakeMessageReq) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_NextHandshakeMessageReq.Unmarshal(m, b)
-}
-func (m *NextHandshakeMessageReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_NextHandshakeMessageReq.Marshal(b, m, deterministic)
-}
-func (m *NextHandshakeMessageReq) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_NextHandshakeMessageReq.Merge(m, src)
+func (x *NextHandshakeMessageReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *NextHandshakeMessageReq) XXX_Size() int {
-	return xxx_messageInfo_NextHandshakeMessageReq.Size(m)
-}
-func (m *NextHandshakeMessageReq) XXX_DiscardUnknown() {
-	xxx_messageInfo_NextHandshakeMessageReq.DiscardUnknown(m)
+
+func (*NextHandshakeMessageReq) ProtoMessage() {}
+
+func (x *NextHandshakeMessageReq) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_NextHandshakeMessageReq proto.InternalMessageInfo
+// Deprecated: Use NextHandshakeMessageReq.ProtoReflect.Descriptor instead.
+func (*NextHandshakeMessageReq) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *NextHandshakeMessageReq) GetInBytes() []byte {
-	if m != nil {
-		return m.InBytes
+func (x *NextHandshakeMessageReq) GetInBytes() []byte {
+	if x != nil {
+		return x.InBytes
 	}
 	return nil
 }
 
 type HandshakerReq struct {
-	// Types that are valid to be assigned to ReqOneof:
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to ReqOneof:
 	//	*HandshakerReq_ClientStart
 	//	*HandshakerReq_ServerStart
 	//	*HandshakerReq_Next
-	ReqOneof             isHandshakerReq_ReqOneof `protobuf_oneof:"req_oneof"`
-	XXX_NoUnkeyedLiteral struct{}                 `json:"-"`
-	XXX_unrecognized     []byte                   `json:"-"`
-	XXX_sizecache        int32                    `json:"-"`
+	ReqOneof isHandshakerReq_ReqOneof `protobuf_oneof:"req_oneof"`
 }
 
-func (m *HandshakerReq) Reset()         { *m = HandshakerReq{} }
-func (m *HandshakerReq) String() string { return proto.CompactTextString(m) }
-func (*HandshakerReq) ProtoMessage()    {}
-func (*HandshakerReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{6}
+func (x *HandshakerReq) Reset() {
+	*x = HandshakerReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HandshakerReq) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HandshakerReq.Unmarshal(m, b)
-}
-func (m *HandshakerReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HandshakerReq.Marshal(b, m, deterministic)
-}
-func (m *HandshakerReq) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HandshakerReq.Merge(m, src)
-}
-func (m *HandshakerReq) XXX_Size() int {
-	return xxx_messageInfo_HandshakerReq.Size(m)
-}
-func (m *HandshakerReq) XXX_DiscardUnknown() {
-	xxx_messageInfo_HandshakerReq.DiscardUnknown(m)
+func (x *HandshakerReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_HandshakerReq proto.InternalMessageInfo
+func (*HandshakerReq) ProtoMessage() {}
 
-type isHandshakerReq_ReqOneof interface {
-	isHandshakerReq_ReqOneof()
-}
-
-type HandshakerReq_ClientStart struct {
-	ClientStart *StartClientHandshakeReq `protobuf:"bytes,1,opt,name=client_start,json=clientStart,proto3,oneof"`
-}
-
-type HandshakerReq_ServerStart struct {
-	ServerStart *StartServerHandshakeReq `protobuf:"bytes,2,opt,name=server_start,json=serverStart,proto3,oneof"`
+func (x *HandshakerReq) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-type HandshakerReq_Next struct {
-	Next *NextHandshakeMessageReq `protobuf:"bytes,3,opt,name=next,proto3,oneof"`
+// Deprecated: Use HandshakerReq.ProtoReflect.Descriptor instead.
+func (*HandshakerReq) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{6}
 }
 
-func (*HandshakerReq_ClientStart) isHandshakerReq_ReqOneof() {}
-
-func (*HandshakerReq_ServerStart) isHandshakerReq_ReqOneof() {}
-
-func (*HandshakerReq_Next) isHandshakerReq_ReqOneof() {}
-
 func (m *HandshakerReq) GetReqOneof() isHandshakerReq_ReqOneof {
 	if m != nil {
 		return m.ReqOneof
@@ -617,37 +709,57 @@ func (m *HandshakerReq) GetReqOneof() isHandshakerReq_ReqOneof {
 	return nil
 }
 
-func (m *HandshakerReq) GetClientStart() *StartClientHandshakeReq {
-	if x, ok := m.GetReqOneof().(*HandshakerReq_ClientStart); ok {
+func (x *HandshakerReq) GetClientStart() *StartClientHandshakeReq {
+	if x, ok := x.GetReqOneof().(*HandshakerReq_ClientStart); ok {
 		return x.ClientStart
 	}
 	return nil
 }
 
-func (m *HandshakerReq) GetServerStart() *StartServerHandshakeReq {
-	if x, ok := m.GetReqOneof().(*HandshakerReq_ServerStart); ok {
+func (x *HandshakerReq) GetServerStart() *StartServerHandshakeReq {
+	if x, ok := x.GetReqOneof().(*HandshakerReq_ServerStart); ok {
 		return x.ServerStart
 	}
 	return nil
 }
 
-func (m *HandshakerReq) GetNext() *NextHandshakeMessageReq {
-	if x, ok := m.GetReqOneof().(*HandshakerReq_Next); ok {
+func (x *HandshakerReq) GetNext() *NextHandshakeMessageReq {
+	if x, ok := x.GetReqOneof().(*HandshakerReq_Next); ok {
 		return x.Next
 	}
 	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*HandshakerReq) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*HandshakerReq_ClientStart)(nil),
-		(*HandshakerReq_ServerStart)(nil),
-		(*HandshakerReq_Next)(nil),
-	}
+type isHandshakerReq_ReqOneof interface {
+	isHandshakerReq_ReqOneof()
+}
+
+type HandshakerReq_ClientStart struct {
+	// The start client handshake request message.
+	ClientStart *StartClientHandshakeReq `protobuf:"bytes,1,opt,name=client_start,json=clientStart,proto3,oneof"`
 }
 
+type HandshakerReq_ServerStart struct {
+	// The start server handshake request message.
+	ServerStart *StartServerHandshakeReq `protobuf:"bytes,2,opt,name=server_start,json=serverStart,proto3,oneof"`
+}
+
+type HandshakerReq_Next struct {
+	// The next handshake request message.
+	Next *NextHandshakeMessageReq `protobuf:"bytes,3,opt,name=next,proto3,oneof"`
+}
+
+func (*HandshakerReq_ClientStart) isHandshakerReq_ReqOneof() {}
+
+func (*HandshakerReq_ServerStart) isHandshakerReq_ReqOneof() {}
+
+func (*HandshakerReq_Next) isHandshakerReq_ReqOneof() {}
+
 type HandshakerResult struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The application protocol negotiated for this connection.
 	ApplicationProtocol string `protobuf:"bytes,1,opt,name=application_protocol,json=applicationProtocol,proto3" json:"application_protocol,omitempty"`
 	// The record protocol negotiated for this connection.
@@ -667,143 +779,159 @@ type HandshakerResult struct {
 	// The RPC protocol versions supported by the peer.
 	PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,7,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"`
 	// The maximum frame size of the peer.
-	MaxFrameSize         uint32   `protobuf:"varint,8,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	MaxFrameSize uint32 `protobuf:"varint,8,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"`
 }
 
-func (m *HandshakerResult) Reset()         { *m = HandshakerResult{} }
-func (m *HandshakerResult) String() string { return proto.CompactTextString(m) }
-func (*HandshakerResult) ProtoMessage()    {}
-func (*HandshakerResult) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{7}
+func (x *HandshakerResult) Reset() {
+	*x = HandshakerResult{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HandshakerResult) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HandshakerResult.Unmarshal(m, b)
-}
-func (m *HandshakerResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HandshakerResult.Marshal(b, m, deterministic)
+func (x *HandshakerResult) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HandshakerResult) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HandshakerResult.Merge(m, src)
-}
-func (m *HandshakerResult) XXX_Size() int {
-	return xxx_messageInfo_HandshakerResult.Size(m)
-}
-func (m *HandshakerResult) XXX_DiscardUnknown() {
-	xxx_messageInfo_HandshakerResult.DiscardUnknown(m)
+
+func (*HandshakerResult) ProtoMessage() {}
+
+func (x *HandshakerResult) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HandshakerResult proto.InternalMessageInfo
+// Deprecated: Use HandshakerResult.ProtoReflect.Descriptor instead.
+func (*HandshakerResult) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *HandshakerResult) GetApplicationProtocol() string {
-	if m != nil {
-		return m.ApplicationProtocol
+func (x *HandshakerResult) GetApplicationProtocol() string {
+	if x != nil {
+		return x.ApplicationProtocol
 	}
 	return ""
 }
 
-func (m *HandshakerResult) GetRecordProtocol() string {
-	if m != nil {
-		return m.RecordProtocol
+func (x *HandshakerResult) GetRecordProtocol() string {
+	if x != nil {
+		return x.RecordProtocol
 	}
 	return ""
 }
 
-func (m *HandshakerResult) GetKeyData() []byte {
-	if m != nil {
-		return m.KeyData
+func (x *HandshakerResult) GetKeyData() []byte {
+	if x != nil {
+		return x.KeyData
 	}
 	return nil
 }
 
-func (m *HandshakerResult) GetPeerIdentity() *Identity {
-	if m != nil {
-		return m.PeerIdentity
+func (x *HandshakerResult) GetPeerIdentity() *Identity {
+	if x != nil {
+		return x.PeerIdentity
 	}
 	return nil
 }
 
-func (m *HandshakerResult) GetLocalIdentity() *Identity {
-	if m != nil {
-		return m.LocalIdentity
+func (x *HandshakerResult) GetLocalIdentity() *Identity {
+	if x != nil {
+		return x.LocalIdentity
 	}
 	return nil
 }
 
-func (m *HandshakerResult) GetKeepChannelOpen() bool {
-	if m != nil {
-		return m.KeepChannelOpen
+func (x *HandshakerResult) GetKeepChannelOpen() bool {
+	if x != nil {
+		return x.KeepChannelOpen
 	}
 	return false
 }
 
-func (m *HandshakerResult) GetPeerRpcVersions() *RpcProtocolVersions {
-	if m != nil {
-		return m.PeerRpcVersions
+func (x *HandshakerResult) GetPeerRpcVersions() *RpcProtocolVersions {
+	if x != nil {
+		return x.PeerRpcVersions
 	}
 	return nil
 }
 
-func (m *HandshakerResult) GetMaxFrameSize() uint32 {
-	if m != nil {
-		return m.MaxFrameSize
+func (x *HandshakerResult) GetMaxFrameSize() uint32 {
+	if x != nil {
+		return x.MaxFrameSize
 	}
 	return 0
 }
 
 type HandshakerStatus struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The status code. This could be the gRPC status code.
 	Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
 	// The status details.
-	Details              string   `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Details string `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"`
 }
 
-func (m *HandshakerStatus) Reset()         { *m = HandshakerStatus{} }
-func (m *HandshakerStatus) String() string { return proto.CompactTextString(m) }
-func (*HandshakerStatus) ProtoMessage()    {}
-func (*HandshakerStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{8}
+func (x *HandshakerStatus) Reset() {
+	*x = HandshakerStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HandshakerStatus) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HandshakerStatus.Unmarshal(m, b)
-}
-func (m *HandshakerStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HandshakerStatus.Marshal(b, m, deterministic)
-}
-func (m *HandshakerStatus) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HandshakerStatus.Merge(m, src)
+func (x *HandshakerStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HandshakerStatus) XXX_Size() int {
-	return xxx_messageInfo_HandshakerStatus.Size(m)
-}
-func (m *HandshakerStatus) XXX_DiscardUnknown() {
-	xxx_messageInfo_HandshakerStatus.DiscardUnknown(m)
+
+func (*HandshakerStatus) ProtoMessage() {}
+
+func (x *HandshakerStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HandshakerStatus proto.InternalMessageInfo
+// Deprecated: Use HandshakerStatus.ProtoReflect.Descriptor instead.
+func (*HandshakerStatus) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *HandshakerStatus) GetCode() uint32 {
-	if m != nil {
-		return m.Code
+func (x *HandshakerStatus) GetCode() uint32 {
+	if x != nil {
+		return x.Code
 	}
 	return 0
 }
 
-func (m *HandshakerStatus) GetDetails() string {
-	if m != nil {
-		return m.Details
+func (x *HandshakerStatus) GetDetails() string {
+	if x != nil {
+		return x.Details
 	}
 	return ""
 }
 
 type HandshakerResp struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Frames to be given to the peer for the NextHandshakeMessageReq. May be
 	// empty if no out_frames have to be sent to the peer or if in_bytes in the
 	// HandshakerReq are incomplete. All the non-empty out frames must be sent to
@@ -818,160 +946,481 @@ type HandshakerResp struct {
 	// to frames that needs to be forwarded to the peer.
 	Result *HandshakerResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"`
 	// Status of the handshaker.
-	Status               *HandshakerStatus `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	Status *HandshakerStatus `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"`
 }
 
-func (m *HandshakerResp) Reset()         { *m = HandshakerResp{} }
-func (m *HandshakerResp) String() string { return proto.CompactTextString(m) }
-func (*HandshakerResp) ProtoMessage()    {}
-func (*HandshakerResp) Descriptor() ([]byte, []int) {
-	return fileDescriptor_54c074f40c7c7e99, []int{9}
+func (x *HandshakerResp) Reset() {
+	*x = HandshakerResp{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_handshaker_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HandshakerResp) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HandshakerResp.Unmarshal(m, b)
+func (x *HandshakerResp) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HandshakerResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HandshakerResp.Marshal(b, m, deterministic)
-}
-func (m *HandshakerResp) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HandshakerResp.Merge(m, src)
-}
-func (m *HandshakerResp) XXX_Size() int {
-	return xxx_messageInfo_HandshakerResp.Size(m)
-}
-func (m *HandshakerResp) XXX_DiscardUnknown() {
-	xxx_messageInfo_HandshakerResp.DiscardUnknown(m)
+
+func (*HandshakerResp) ProtoMessage() {}
+
+func (x *HandshakerResp) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_handshaker_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HandshakerResp proto.InternalMessageInfo
+// Deprecated: Use HandshakerResp.ProtoReflect.Descriptor instead.
+func (*HandshakerResp) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{9}
+}
 
-func (m *HandshakerResp) GetOutFrames() []byte {
-	if m != nil {
-		return m.OutFrames
+func (x *HandshakerResp) GetOutFrames() []byte {
+	if x != nil {
+		return x.OutFrames
 	}
 	return nil
 }
 
-func (m *HandshakerResp) GetBytesConsumed() uint32 {
-	if m != nil {
-		return m.BytesConsumed
+func (x *HandshakerResp) GetBytesConsumed() uint32 {
+	if x != nil {
+		return x.BytesConsumed
 	}
 	return 0
 }
 
-func (m *HandshakerResp) GetResult() *HandshakerResult {
-	if m != nil {
-		return m.Result
+func (x *HandshakerResp) GetResult() *HandshakerResult {
+	if x != nil {
+		return x.Result
 	}
 	return nil
 }
 
-func (m *HandshakerResp) GetStatus() *HandshakerStatus {
-	if m != nil {
-		return m.Status
+func (x *HandshakerResp) GetStatus() *HandshakerStatus {
+	if x != nil {
+		return x.Status
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.gcp.HandshakeProtocol", HandshakeProtocol_name, HandshakeProtocol_value)
-	proto.RegisterEnum("grpc.gcp.NetworkProtocol", NetworkProtocol_name, NetworkProtocol_value)
-	proto.RegisterType((*Endpoint)(nil), "grpc.gcp.Endpoint")
-	proto.RegisterType((*Identity)(nil), "grpc.gcp.Identity")
-	proto.RegisterMapType((map[string]string)(nil), "grpc.gcp.Identity.AttributesEntry")
-	proto.RegisterType((*StartClientHandshakeReq)(nil), "grpc.gcp.StartClientHandshakeReq")
-	proto.RegisterType((*ServerHandshakeParameters)(nil), "grpc.gcp.ServerHandshakeParameters")
-	proto.RegisterType((*StartServerHandshakeReq)(nil), "grpc.gcp.StartServerHandshakeReq")
-	proto.RegisterMapType((map[int32]*ServerHandshakeParameters)(nil), "grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry")
-	proto.RegisterType((*NextHandshakeMessageReq)(nil), "grpc.gcp.NextHandshakeMessageReq")
-	proto.RegisterType((*HandshakerReq)(nil), "grpc.gcp.HandshakerReq")
-	proto.RegisterType((*HandshakerResult)(nil), "grpc.gcp.HandshakerResult")
-	proto.RegisterType((*HandshakerStatus)(nil), "grpc.gcp.HandshakerStatus")
-	proto.RegisterType((*HandshakerResp)(nil), "grpc.gcp.HandshakerResp")
-}
-
-func init() { proto.RegisterFile("grpc/gcp/handshaker.proto", fileDescriptor_54c074f40c7c7e99) }
-
-var fileDescriptor_54c074f40c7c7e99 = []byte{
-	// 1203 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45,
-	0x14, 0xce, 0xda, 0x4e, 0xe2, 0x1c, 0xc7, 0x3f, 0x99, 0xa6, 0xea, 0x26, 0x6d, 0xc1, 0x18, 0x10,
-	0x6e, 0x2f, 0x6c, 0x70, 0x41, 0xa5, 0x45, 0x55, 0x6b, 0x3b, 0x8e, 0x1c, 0x5a, 0x1c, 0x6b, 0x9d,
-	0x82, 0x44, 0x2f, 0x56, 0xd3, 0xf5, 0xd4, 0x59, 0x79, 0x3d, 0xb3, 0x9d, 0x19, 0x87, 0xb8, 0xf7,
-	0xbc, 0x04, 0xf7, 0xbc, 0x06, 0x2f, 0xc1, 0x33, 0x20, 0xf1, 0x18, 0x68, 0x67, 0x7f, 0x6d, 0xaf,
-	0xab, 0x22, 0xb8, 0xdb, 0x39, 0xf3, 0x7d, 0x67, 0xce, 0x9c, 0xf3, 0x9d, 0xb3, 0x03, 0x47, 0x13,
-	0xee, 0x5a, 0xcd, 0x89, 0xe5, 0x36, 0x2f, 0x31, 0x1d, 0x8b, 0x4b, 0x3c, 0x25, 0xbc, 0xe1, 0x72,
-	0x26, 0x19, 0xca, 0x7b, 0x5b, 0x8d, 0x89, 0xe5, 0x1e, 0xd7, 0x23, 0x90, 0xe4, 0x98, 0x0a, 0x97,
-	0x71, 0x69, 0x0a, 0x62, 0xcd, 0xb9, 0x2d, 0x17, 0xa6, 0xc5, 0x66, 0x33, 0x46, 0x7d, 0x4e, 0x4d,
-	0x42, 0xbe, 0x47, 0xc7, 0x2e, 0xb3, 0xa9, 0x44, 0x77, 0x01, 0x6c, 0xd7, 0xc4, 0xe3, 0x31, 0x27,
-	0x42, 0xe8, 0x5a, 0x55, 0xab, 0xef, 0x19, 0x7b, 0xb6, 0xdb, 0xf6, 0x0d, 0x08, 0x41, 0xce, 0x73,
-	0xa4, 0x67, 0xaa, 0x5a, 0x7d, 0xdb, 0x50, 0xdf, 0xe8, 0x1b, 0xc8, 0x2b, 0x3f, 0x16, 0x73, 0xf4,
-	0x6c, 0x55, 0xab, 0x97, 0x5a, 0x47, 0x8d, 0x30, 0x8a, 0xc6, 0x80, 0xc8, 0x5f, 0x18, 0x9f, 0x0e,
-	0x03, 0x80, 0x11, 0x41, 0x6b, 0x7f, 0x6b, 0x90, 0x3f, 0x1b, 0x13, 0x2a, 0x6d, 0xb9, 0x40, 0xf7,
-	0xa0, 0x2c, 0x08, 0xbf, 0xb2, 0x2d, 0x62, 0x62, 0xcb, 0x62, 0x73, 0x2a, 0xfd, 0xb3, 0xfb, 0x5b,
-	0x46, 0x29, 0xd8, 0x68, 0xfb, 0x76, 0x74, 0x07, 0xf2, 0x97, 0x4c, 0x48, 0x8a, 0x67, 0x44, 0x85,
-	0xe1, 0x61, 0x22, 0x0b, 0xea, 0x00, 0x60, 0x29, 0xb9, 0xfd, 0x7a, 0x2e, 0x89, 0xd0, 0xb3, 0xd5,
-	0x6c, 0xbd, 0xd0, 0xaa, 0xc5, 0xe1, 0x84, 0x07, 0x36, 0xda, 0x11, 0xa8, 0x47, 0x25, 0x5f, 0x18,
-	0x09, 0xd6, 0xf1, 0x13, 0x28, 0xaf, 0x6c, 0xa3, 0x0a, 0x64, 0xa7, 0x64, 0x11, 0xe4, 0xc3, 0xfb,
-	0x44, 0x87, 0xb0, 0x7d, 0x85, 0x9d, 0x79, 0x10, 0x83, 0xe1, 0x2f, 0x1e, 0x67, 0xbe, 0xd5, 0x3a,
-	0x15, 0x28, 0xd9, 0xc1, 0x31, 0x26, 0xa3, 0x84, 0xbd, 0xa9, 0xfd, 0x99, 0x83, 0x5b, 0x23, 0x89,
-	0xb9, 0xec, 0x3a, 0x36, 0xa1, 0xb2, 0x1f, 0x16, 0xcd, 0x20, 0x6f, 0xd1, 0x2b, 0xb8, 0x1d, 0x15,
-	0x31, 0xae, 0x4f, 0x94, 0x50, 0x4d, 0x25, 0xf4, 0x76, 0x7c, 0x83, 0x88, 0x1c, 0xa5, 0xf4, 0x28,
-	0xe2, 0x8f, 0x02, 0x7a, 0xb8, 0x85, 0x1e, 0xc0, 0x4d, 0xec, 0xba, 0x8e, 0x6d, 0x61, 0x69, 0x33,
-	0x1a, 0x79, 0x15, 0x7a, 0xa6, 0x9a, 0xad, 0xef, 0x19, 0x87, 0x89, 0xcd, 0x90, 0x23, 0xd0, 0x3d,
-	0xa8, 0x70, 0x62, 0x31, 0x3e, 0x4e, 0xe0, 0xb3, 0x0a, 0x5f, 0xf6, 0xed, 0x31, 0xf4, 0x29, 0x1c,
-	0x48, 0xcc, 0x27, 0x44, 0x9a, 0xc1, 0x8d, 0x6d, 0x22, 0xf4, 0x9c, 0x4a, 0x3a, 0x5a, 0x4f, 0xba,
-	0x51, 0xf1, 0xc1, 0x67, 0x11, 0x16, 0x3d, 0x82, 0x92, 0xc3, 0x2c, 0xec, 0x84, 0xfc, 0x85, 0xbe,
-	0x5d, 0xd5, 0x36, 0xb0, 0x8b, 0x0a, 0x19, 0x49, 0x26, 0xa2, 0x92, 0x40, 0xbb, 0xfa, 0xce, 0x2a,
-	0x35, 0x54, 0x75, 0x40, 0x8d, 0x44, 0xfe, 0x1d, 0x94, 0x39, 0x99, 0x31, 0x49, 0x62, 0xee, 0xee,
-	0x46, 0x6e, 0xc9, 0x87, 0x46, 0xe4, 0x8f, 0xa1, 0x10, 0xdc, 0x59, 0x49, 0x30, 0xaf, 0xca, 0x0f,
-	0xbe, 0x69, 0xe0, 0x49, 0xf0, 0x19, 0xec, 0x73, 0xd7, 0x32, 0xaf, 0x08, 0x17, 0x36, 0xa3, 0x42,
-	0xdf, 0x53, 0xae, 0xef, 0xc6, 0xae, 0x0d, 0xd7, 0x0a, 0x53, 0xf8, 0x63, 0x00, 0x32, 0x0a, 0xdc,
-	0xb5, 0xc2, 0x05, 0xfa, 0x0c, 0x4a, 0x33, 0x7c, 0x6d, 0xbe, 0xe1, 0x78, 0x46, 0x4c, 0x61, 0xbf,
-	0x23, 0x3a, 0x54, 0xb5, 0x7a, 0xd1, 0xd8, 0x9f, 0xe1, 0xeb, 0x53, 0xcf, 0x38, 0xb2, 0xdf, 0x91,
-	0xda, 0xaf, 0x1a, 0x1c, 0x8d, 0x08, 0xbf, 0x22, 0x3c, 0xd6, 0x04, 0xf6, 0x76, 0x25, 0xe1, 0xe9,
-	0x55, 0xd4, 0xd2, 0xab, 0xf8, 0x04, 0x2a, 0x4b, 0x45, 0xf0, 0x8a, 0x98, 0xd9, 0x58, 0xc4, 0x72,
-	0xb2, 0x0c, 0x36, 0x11, 0xb5, 0xdf, 0x43, 0x75, 0xaf, 0x04, 0xe3, 0xa9, 0x7b, 0xa3, 0x00, 0xb5,
-	0xf7, 0x08, 0x70, 0x06, 0x87, 0x71, 0x4b, 0xb8, 0xd1, 0x95, 0x82, 0x98, 0x1e, 0xc7, 0x31, 0x6d,
-	0x38, 0xb5, 0x91, 0x92, 0x0f, 0xbf, 0xcb, 0x6f, 0x5c, 0xa6, 0x64, 0xea, 0x08, 0xf2, 0x36, 0x35,
-	0x5f, 0x2f, 0xfc, 0x81, 0xa1, 0xd5, 0xf7, 0x8d, 0x5d, 0x9b, 0x76, 0xbc, 0x65, 0x8a, 0xc6, 0x72,
-	0xff, 0x41, 0x63, 0xdb, 0x1f, 0xac, 0xb1, 0x55, 0x09, 0xed, 0xfc, 0x0f, 0x12, 0xda, 0x5d, 0x97,
-	0xd0, 0xf1, 0x14, 0xf4, 0x4d, 0xb9, 0x4a, 0x8e, 0xbc, 0x6d, 0x7f, 0xe4, 0x3d, 0x4a, 0x8e, 0xbc,
-	0x42, 0xeb, 0xd3, 0x44, 0x21, 0x36, 0xc9, 0x30, 0x31, 0x17, 0x6b, 0x5f, 0xc3, 0xad, 0x01, 0xb9,
-	0x8e, 0xa7, 0xdf, 0x0f, 0x44, 0x08, 0x3c, 0x51, 0x32, 0x49, 0x96, 0x40, 0x5b, 0x2a, 0x41, 0xed,
-	0x2f, 0x0d, 0x8a, 0x11, 0x85, 0x7b, 0xe0, 0x53, 0xd8, 0xb7, 0xd4, 0x1c, 0x35, 0x85, 0x57, 0x7f,
-	0x45, 0x28, 0xb4, 0x3e, 0x59, 0x91, 0xc5, 0xfa, 0xa8, 0xed, 0x6f, 0x19, 0x05, 0x9f, 0xa8, 0x00,
-	0x9e, 0x1f, 0xa1, 0xe2, 0x0e, 0xfc, 0x64, 0x52, 0xfd, 0xac, 0xcb, 0xcb, 0xf3, 0xe3, 0x13, 0x7d,
-	0x3f, 0x0f, 0x21, 0x47, 0xc9, 0xb5, 0x54, 0xda, 0x59, 0xe2, 0x6f, 0xb8, 0x6d, 0x7f, 0xcb, 0x50,
-	0x84, 0x4e, 0x01, 0xf6, 0x38, 0x79, 0x1b, 0xfc, 0x23, 0x7e, 0xcb, 0x42, 0x25, 0x79, 0x4f, 0x31,
-	0x77, 0x24, 0xfa, 0x0a, 0x0e, 0xd3, 0xda, 0x27, 0xf8, 0x0f, 0xdd, 0x48, 0xe9, 0x1e, 0xf4, 0x05,
-	0x94, 0x57, 0xfa, 0x3e, 0xf8, 0x43, 0x95, 0x96, 0xdb, 0xde, 0xcb, 0xf9, 0x94, 0x2c, 0xcc, 0x31,
-	0x96, 0x38, 0x94, 0xfd, 0x94, 0x2c, 0x4e, 0xb0, 0xc4, 0xe8, 0x21, 0x14, 0x5d, 0x42, 0x78, 0x3c,
-	0x94, 0x73, 0x1b, 0x87, 0xf2, 0xbe, 0x07, 0x5c, 0x9f, 0xc9, 0xff, 0x7e, 0x9c, 0xdf, 0x87, 0x83,
-	0x29, 0x21, 0xae, 0x69, 0x5d, 0x62, 0x4a, 0x89, 0x63, 0x32, 0x97, 0x50, 0xa5, 0xfb, 0xbc, 0x51,
-	0xf6, 0x36, 0xba, 0xbe, 0xfd, 0xdc, 0x25, 0x14, 0x9d, 0xc1, 0x81, 0x8a, 0x6f, 0xa9, 0x47, 0x76,
-	0x3f, 0xa4, 0x47, 0xca, 0x1e, 0xcf, 0x78, 0x6f, 0x9f, 0xe4, 0x53, 0x46, 0xed, 0xb3, 0x64, 0x6d,
-	0x46, 0x12, 0xcb, 0xb9, 0x7a, 0x0a, 0x59, 0x6c, 0x4c, 0x54, 0x2d, 0x8a, 0x86, 0xfa, 0x46, 0x3a,
-	0xec, 0x8e, 0x89, 0xc4, 0xb6, 0xfa, 0xc3, 0x7a, 0x49, 0x0f, 0x97, 0xb5, 0x3f, 0x34, 0x28, 0x2d,
-	0x95, 0xd7, 0xf5, 0x9e, 0x5a, 0x6c, 0x2e, 0xfd, 0xa3, 0x43, 0xd9, 0xef, 0xb1, 0xb9, 0x54, 0xc7,
-	0x0a, 0xf4, 0x39, 0x94, 0x54, 0x43, 0x98, 0x16, 0xa3, 0x62, 0x3e, 0x23, 0x63, 0xe5, 0xb2, 0x68,
-	0x14, 0x95, 0xb5, 0x1b, 0x18, 0x51, 0x0b, 0x76, 0xb8, 0x12, 0x4b, 0xa0, 0xbf, 0xe3, 0x94, 0xa7,
-	0x42, 0x20, 0x27, 0x23, 0x40, 0x7a, 0x1c, 0xa1, 0x2e, 0x11, 0x14, 0x36, 0x95, 0xe3, 0x5f, 0xd3,
-	0x08, 0x90, 0xf7, 0xbf, 0x87, 0x83, 0xb5, 0xa7, 0x07, 0xaa, 0xc1, 0x47, 0xfd, 0xf6, 0xe0, 0x64,
-	0xd4, 0x6f, 0x3f, 0xef, 0x99, 0x43, 0xe3, 0xfc, 0xe2, 0xbc, 0x7b, 0xfe, 0xc2, 0x7c, 0x39, 0x18,
-	0x0d, 0x7b, 0xdd, 0xb3, 0xd3, 0xb3, 0xde, 0x49, 0x65, 0x0b, 0xed, 0x42, 0xf6, 0xe2, 0xc5, 0xa8,
-	0xa2, 0xa1, 0x3c, 0xe4, 0xda, 0x2f, 0x2e, 0x46, 0x95, 0xcc, 0xfd, 0x1e, 0x94, 0x57, 0xde, 0x85,
-	0xa8, 0x0a, 0x77, 0x06, 0xbd, 0x8b, 0x9f, 0xce, 0x8d, 0xe7, 0xef, 0xf3, 0xd3, 0x1d, 0x56, 0x34,
-	0xef, 0xe3, 0xe5, 0xc9, 0xb0, 0x92, 0x69, 0xbd, 0x4a, 0x84, 0xc4, 0x47, 0xfe, 0x2b, 0x11, 0x9d,
-	0x42, 0xe1, 0x84, 0x45, 0x66, 0x74, 0x2b, 0x3d, 0x1d, 0x6f, 0x8f, 0xf5, 0x0d, 0x79, 0x72, 0x6b,
-	0x5b, 0x75, 0xed, 0x4b, 0xad, 0x33, 0x85, 0x9b, 0x36, 0xf3, 0x31, 0xd8, 0x91, 0xa2, 0x61, 0x53,
-	0x49, 0x38, 0xc5, 0x4e, 0xa7, 0x1c, 0xc3, 0x55, 0xf4, 0x43, 0xed, 0xe7, 0xa7, 0x13, 0xc6, 0x26,
-	0x0e, 0x69, 0x4c, 0x98, 0x83, 0xe9, 0xa4, 0xc1, 0xf8, 0xa4, 0xa9, 0x1e, 0xdf, 0x16, 0x27, 0x4a,
-	0xde, 0xd8, 0x11, 0x4d, 0xcf, 0x49, 0x33, 0x74, 0xd2, 0x54, 0xbd, 0xa9, 0x40, 0xe6, 0xc4, 0x72,
-	0x5f, 0xef, 0xa8, 0xf5, 0x83, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc1, 0xf9, 0x9d, 0xf2, 0xd9,
-	0x0b, 0x00, 0x00,
+var File_grpc_gcp_handshaker_proto protoreflect.FileDescriptor
+
+var file_grpc_gcp_handshaker_proto_rawDesc = []byte{
+	0x0a, 0x19, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x68, 0x61, 0x6e, 0x64, 0x73,
+	0x68, 0x61, 0x6b, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x67, 0x63, 0x70, 0x1a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f,
+	0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69,
+	0x74, 0x79, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
+	0x74, 0x0a, 0x08, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x69,
+	0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x09, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f,
+	0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x35,
+	0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x4e, 0x65, 0x74, 0x77,
+	0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0xe8, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69,
+	0x74, 0x79, 0x12, 0x29, 0x0a, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63,
+	0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x73,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a,
+	0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48,
+	0x00, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x42, 0x0a, 0x0a, 0x61,
+	0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74,
+	0x69, 0x74, 0x79, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a,
+	0x3d, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74,
+	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x10,
+	0x0a, 0x0e, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6f, 0x6e, 0x65, 0x6f, 0x66,
+	0x22, 0xd3, 0x04, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x12, 0x5b, 0x0a, 0x1b,
+	0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69,
+	0x74, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0e, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e,
+	0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x19,
+	0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
+	0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x33, 0x0a, 0x15, 0x61, 0x70, 0x70,
+	0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12, 0x29,
+	0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64,
+	0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12, 0x3f, 0x0a, 0x11, 0x74, 0x61, 0x72,
+	0x67, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e,
+	0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74,
+	0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f,
+	0x63, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64,
+	0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x64, 0x65,
+	0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x65,
+	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+	0x74, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x12, 0x3b, 0x0a, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f,
+	0x69, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0e, 0x72,
+	0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1f, 0x0a,
+	0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x40,
+	0x0a, 0x0c, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e,
+	0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x73, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73,
+	0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x69,
+	0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x72, 0x61,
+	0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x85, 0x01, 0x0a, 0x19, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65,
+	0x74, 0x65, 0x72, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f,
+	0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12,
+	0x3d, 0x0a, 0x10, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74,
+	0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0f, 0x6c,
+	0x6f, 0x63, 0x61, 0x6c, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0xa5,
+	0x04, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x61,
+	0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x12, 0x33, 0x0a, 0x15, 0x61, 0x70,
+	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63,
+	0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x61, 0x70, 0x70, 0x6c, 0x69,
+	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12,
+	0x6d, 0x0a, 0x14, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x5f, 0x70, 0x61, 0x72,
+	0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71,
+	0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65,
+	0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x68, 0x61, 0x6e, 0x64, 0x73,
+	0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x19,
+	0x0a, 0x08, 0x69, 0x6e, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x07, 0x69, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, 0x63,
+	0x61, 0x6c, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64,
+	0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70,
+	0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x65,
+	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+	0x74, 0x52, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+	0x74, 0x12, 0x40, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+	0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67,
+	0x63, 0x70, 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65,
+	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65,
+	0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x78,
+	0x46, 0x72, 0x61, 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x1a, 0x6b, 0x0a, 0x18, 0x48, 0x61, 0x6e,
+	0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63,
+	0x70, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b,
+	0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x34, 0x0a, 0x17, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x61,
+	0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65,
+	0x71, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x6e, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0c, 0x52, 0x07, 0x69, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a,
+	0x0d, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x46,
+	0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e,
+	0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x73,
+	0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x46, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x48,
+	0x00, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x37,
+	0x0a, 0x04, 0x6e, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x61, 0x6e, 0x64,
+	0x73, 0x68, 0x61, 0x6b, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x48,
+	0x00, 0x52, 0x04, 0x6e, 0x65, 0x78, 0x74, 0x42, 0x0b, 0x0a, 0x09, 0x72, 0x65, 0x71, 0x5f, 0x6f,
+	0x6e, 0x65, 0x6f, 0x66, 0x22, 0x9a, 0x03, 0x0a, 0x10, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61,
+	0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x31, 0x0a, 0x14, 0x61, 0x70, 0x70,
+	0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x27, 0x0a, 0x0f,
+	0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f,
+	0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x64, 0x61, 0x74,
+	0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x44, 0x61, 0x74, 0x61,
+	0x12, 0x37, 0x0a, 0x0d, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74,
+	0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67,
+	0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0c, 0x70, 0x65, 0x65,
+	0x72, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, 0x63,
+	0x61, 0x6c, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65,
+	0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x74, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x6f, 0x70, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52,
+	0x0f, 0x6b, 0x65, 0x65, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4f, 0x70, 0x65, 0x6e,
+	0x12, 0x49, 0x0a, 0x11, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72,
+	0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63,
+	0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0f, 0x70, 0x65, 0x65, 0x72,
+	0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d,
+	0x61, 0x78, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x08, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x53, 0x69, 0x7a,
+	0x65, 0x22, 0x40, 0x0a, 0x10, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x53,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74,
+	0x61, 0x69, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61,
+	0x69, 0x6c, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x0e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b,
+	0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x75, 0x74, 0x5f, 0x66, 0x72,
+	0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6f, 0x75, 0x74, 0x46,
+	0x72, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x63,
+	0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x62,
+	0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x06,
+	0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b,
+	0x65, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74,
+	0x12, 0x32, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64,
+	0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74,
+	0x61, 0x74, 0x75, 0x73, 0x2a, 0x4a, 0x0a, 0x11, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b,
+	0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x22, 0x0a, 0x1e, 0x48, 0x41, 0x4e,
+	0x44, 0x53, 0x48, 0x41, 0x4b, 0x45, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f,
+	0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a,
+	0x03, 0x54, 0x4c, 0x53, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4c, 0x54, 0x53, 0x10, 0x02,
+	0x2a, 0x45, 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x74, 0x6f,
+	0x63, 0x6f, 0x6c, 0x12, 0x20, 0x0a, 0x1c, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x50,
+	0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46,
+	0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x07,
+	0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x02, 0x32, 0x5b, 0x0a, 0x11, 0x48, 0x61, 0x6e, 0x64, 0x73,
+	0x68, 0x61, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x46, 0x0a, 0x0b,
+	0x44, 0x6f, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x17, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65,
+	0x72, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e,
+	0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00,
+	0x28, 0x01, 0x30, 0x01, 0x42, 0x6b, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x61, 0x6c, 0x74, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x0f, 0x48,
+	0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
+	0x5a, 0x3f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e,
+	0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
+	0x69, 0x61, 0x6c, 0x73, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x67, 0x63,
+	0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_gcp_handshaker_proto_rawDescOnce sync.Once
+	file_grpc_gcp_handshaker_proto_rawDescData = file_grpc_gcp_handshaker_proto_rawDesc
+)
+
+func file_grpc_gcp_handshaker_proto_rawDescGZIP() []byte {
+	file_grpc_gcp_handshaker_proto_rawDescOnce.Do(func() {
+		file_grpc_gcp_handshaker_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_gcp_handshaker_proto_rawDescData)
+	})
+	return file_grpc_gcp_handshaker_proto_rawDescData
+}
+
+var file_grpc_gcp_handshaker_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_grpc_gcp_handshaker_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
+var file_grpc_gcp_handshaker_proto_goTypes = []interface{}{
+	(HandshakeProtocol)(0),            // 0: grpc.gcp.HandshakeProtocol
+	(NetworkProtocol)(0),              // 1: grpc.gcp.NetworkProtocol
+	(*Endpoint)(nil),                  // 2: grpc.gcp.Endpoint
+	(*Identity)(nil),                  // 3: grpc.gcp.Identity
+	(*StartClientHandshakeReq)(nil),   // 4: grpc.gcp.StartClientHandshakeReq
+	(*ServerHandshakeParameters)(nil), // 5: grpc.gcp.ServerHandshakeParameters
+	(*StartServerHandshakeReq)(nil),   // 6: grpc.gcp.StartServerHandshakeReq
+	(*NextHandshakeMessageReq)(nil),   // 7: grpc.gcp.NextHandshakeMessageReq
+	(*HandshakerReq)(nil),             // 8: grpc.gcp.HandshakerReq
+	(*HandshakerResult)(nil),          // 9: grpc.gcp.HandshakerResult
+	(*HandshakerStatus)(nil),          // 10: grpc.gcp.HandshakerStatus
+	(*HandshakerResp)(nil),            // 11: grpc.gcp.HandshakerResp
+	nil,                               // 12: grpc.gcp.Identity.AttributesEntry
+	nil,                               // 13: grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry
+	(*RpcProtocolVersions)(nil),       // 14: grpc.gcp.RpcProtocolVersions
+}
+var file_grpc_gcp_handshaker_proto_depIdxs = []int32{
+	1,  // 0: grpc.gcp.Endpoint.protocol:type_name -> grpc.gcp.NetworkProtocol
+	12, // 1: grpc.gcp.Identity.attributes:type_name -> grpc.gcp.Identity.AttributesEntry
+	0,  // 2: grpc.gcp.StartClientHandshakeReq.handshake_security_protocol:type_name -> grpc.gcp.HandshakeProtocol
+	3,  // 3: grpc.gcp.StartClientHandshakeReq.target_identities:type_name -> grpc.gcp.Identity
+	3,  // 4: grpc.gcp.StartClientHandshakeReq.local_identity:type_name -> grpc.gcp.Identity
+	2,  // 5: grpc.gcp.StartClientHandshakeReq.local_endpoint:type_name -> grpc.gcp.Endpoint
+	2,  // 6: grpc.gcp.StartClientHandshakeReq.remote_endpoint:type_name -> grpc.gcp.Endpoint
+	14, // 7: grpc.gcp.StartClientHandshakeReq.rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions
+	3,  // 8: grpc.gcp.ServerHandshakeParameters.local_identities:type_name -> grpc.gcp.Identity
+	13, // 9: grpc.gcp.StartServerHandshakeReq.handshake_parameters:type_name -> grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry
+	2,  // 10: grpc.gcp.StartServerHandshakeReq.local_endpoint:type_name -> grpc.gcp.Endpoint
+	2,  // 11: grpc.gcp.StartServerHandshakeReq.remote_endpoint:type_name -> grpc.gcp.Endpoint
+	14, // 12: grpc.gcp.StartServerHandshakeReq.rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions
+	4,  // 13: grpc.gcp.HandshakerReq.client_start:type_name -> grpc.gcp.StartClientHandshakeReq
+	6,  // 14: grpc.gcp.HandshakerReq.server_start:type_name -> grpc.gcp.StartServerHandshakeReq
+	7,  // 15: grpc.gcp.HandshakerReq.next:type_name -> grpc.gcp.NextHandshakeMessageReq
+	3,  // 16: grpc.gcp.HandshakerResult.peer_identity:type_name -> grpc.gcp.Identity
+	3,  // 17: grpc.gcp.HandshakerResult.local_identity:type_name -> grpc.gcp.Identity
+	14, // 18: grpc.gcp.HandshakerResult.peer_rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions
+	9,  // 19: grpc.gcp.HandshakerResp.result:type_name -> grpc.gcp.HandshakerResult
+	10, // 20: grpc.gcp.HandshakerResp.status:type_name -> grpc.gcp.HandshakerStatus
+	5,  // 21: grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry.value:type_name -> grpc.gcp.ServerHandshakeParameters
+	8,  // 22: grpc.gcp.HandshakerService.DoHandshake:input_type -> grpc.gcp.HandshakerReq
+	11, // 23: grpc.gcp.HandshakerService.DoHandshake:output_type -> grpc.gcp.HandshakerResp
+	23, // [23:24] is the sub-list for method output_type
+	22, // [22:23] is the sub-list for method input_type
+	22, // [22:22] is the sub-list for extension type_name
+	22, // [22:22] is the sub-list for extension extendee
+	0,  // [0:22] is the sub-list for field type_name
+}
+
+func init() { file_grpc_gcp_handshaker_proto_init() }
+func file_grpc_gcp_handshaker_proto_init() {
+	if File_grpc_gcp_handshaker_proto != nil {
+		return
+	}
+	file_grpc_gcp_transport_security_common_proto_init()
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_gcp_handshaker_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Endpoint); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Identity); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StartClientHandshakeReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerHandshakeParameters); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StartServerHandshakeReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*NextHandshakeMessageReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HandshakerReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HandshakerResult); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HandshakerStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_handshaker_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HandshakerResp); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_gcp_handshaker_proto_msgTypes[1].OneofWrappers = []interface{}{
+		(*Identity_ServiceAccount)(nil),
+		(*Identity_Hostname)(nil),
+	}
+	file_grpc_gcp_handshaker_proto_msgTypes[6].OneofWrappers = []interface{}{
+		(*HandshakerReq_ClientStart)(nil),
+		(*HandshakerReq_ServerStart)(nil),
+		(*HandshakerReq_Next)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_gcp_handshaker_proto_rawDesc,
+			NumEnums:      2,
+			NumMessages:   12,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_gcp_handshaker_proto_goTypes,
+		DependencyIndexes: file_grpc_gcp_handshaker_proto_depIdxs,
+		EnumInfos:         file_grpc_gcp_handshaker_proto_enumTypes,
+		MessageInfos:      file_grpc_gcp_handshaker_proto_msgTypes,
+	}.Build()
+	File_grpc_gcp_handshaker_proto = out.File
+	file_grpc_gcp_handshaker_proto_rawDesc = nil
+	file_grpc_gcp_handshaker_proto_goTypes = nil
+	file_grpc_gcp_handshaker_proto_depIdxs = nil
 }
diff --git a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go
index 992805165db6..922b531a55c2 100644
--- a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go
@@ -1,24 +1,46 @@
+// Copyright 2018 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/transport_security_common.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/gcp/transport_security_common.proto
 
 package grpc_gcp
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // The security level of the created channel. The list is sorted in increasing
 // level of security. This order must always be maintained.
@@ -30,155 +52,275 @@ const (
 	SecurityLevel_INTEGRITY_AND_PRIVACY SecurityLevel = 2
 )
 
-var SecurityLevel_name = map[int32]string{
-	0: "SECURITY_NONE",
-	1: "INTEGRITY_ONLY",
-	2: "INTEGRITY_AND_PRIVACY",
-}
+// Enum value maps for SecurityLevel.
+var (
+	SecurityLevel_name = map[int32]string{
+		0: "SECURITY_NONE",
+		1: "INTEGRITY_ONLY",
+		2: "INTEGRITY_AND_PRIVACY",
+	}
+	SecurityLevel_value = map[string]int32{
+		"SECURITY_NONE":         0,
+		"INTEGRITY_ONLY":        1,
+		"INTEGRITY_AND_PRIVACY": 2,
+	}
+)
 
-var SecurityLevel_value = map[string]int32{
-	"SECURITY_NONE":         0,
-	"INTEGRITY_ONLY":        1,
-	"INTEGRITY_AND_PRIVACY": 2,
+func (x SecurityLevel) Enum() *SecurityLevel {
+	p := new(SecurityLevel)
+	*p = x
+	return p
 }
 
 func (x SecurityLevel) String() string {
-	return proto.EnumName(SecurityLevel_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (SecurityLevel) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_gcp_transport_security_common_proto_enumTypes[0].Descriptor()
+}
+
+func (SecurityLevel) Type() protoreflect.EnumType {
+	return &file_grpc_gcp_transport_security_common_proto_enumTypes[0]
 }
 
+func (x SecurityLevel) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use SecurityLevel.Descriptor instead.
 func (SecurityLevel) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_b97e31e3cc23582a, []int{0}
+	return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0}
 }
 
 // Max and min supported RPC protocol versions.
 type RpcProtocolVersions struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Maximum supported RPC version.
 	MaxRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,1,opt,name=max_rpc_version,json=maxRpcVersion,proto3" json:"max_rpc_version,omitempty"`
 	// Minimum supported RPC version.
-	MinRpcVersion        *RpcProtocolVersions_Version `protobuf:"bytes,2,opt,name=min_rpc_version,json=minRpcVersion,proto3" json:"min_rpc_version,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                     `json:"-"`
-	XXX_unrecognized     []byte                       `json:"-"`
-	XXX_sizecache        int32                        `json:"-"`
+	MinRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,2,opt,name=min_rpc_version,json=minRpcVersion,proto3" json:"min_rpc_version,omitempty"`
 }
 
-func (m *RpcProtocolVersions) Reset()         { *m = RpcProtocolVersions{} }
-func (m *RpcProtocolVersions) String() string { return proto.CompactTextString(m) }
-func (*RpcProtocolVersions) ProtoMessage()    {}
-func (*RpcProtocolVersions) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b97e31e3cc23582a, []int{0}
+func (x *RpcProtocolVersions) Reset() {
+	*x = RpcProtocolVersions{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RpcProtocolVersions) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RpcProtocolVersions.Unmarshal(m, b)
-}
-func (m *RpcProtocolVersions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RpcProtocolVersions.Marshal(b, m, deterministic)
+func (x *RpcProtocolVersions) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RpcProtocolVersions) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RpcProtocolVersions.Merge(m, src)
-}
-func (m *RpcProtocolVersions) XXX_Size() int {
-	return xxx_messageInfo_RpcProtocolVersions.Size(m)
-}
-func (m *RpcProtocolVersions) XXX_DiscardUnknown() {
-	xxx_messageInfo_RpcProtocolVersions.DiscardUnknown(m)
+
+func (*RpcProtocolVersions) ProtoMessage() {}
+
+func (x *RpcProtocolVersions) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RpcProtocolVersions proto.InternalMessageInfo
+// Deprecated: Use RpcProtocolVersions.ProtoReflect.Descriptor instead.
+func (*RpcProtocolVersions) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *RpcProtocolVersions) GetMaxRpcVersion() *RpcProtocolVersions_Version {
-	if m != nil {
-		return m.MaxRpcVersion
+func (x *RpcProtocolVersions) GetMaxRpcVersion() *RpcProtocolVersions_Version {
+	if x != nil {
+		return x.MaxRpcVersion
 	}
 	return nil
 }
 
-func (m *RpcProtocolVersions) GetMinRpcVersion() *RpcProtocolVersions_Version {
-	if m != nil {
-		return m.MinRpcVersion
+func (x *RpcProtocolVersions) GetMinRpcVersion() *RpcProtocolVersions_Version {
+	if x != nil {
+		return x.MinRpcVersion
 	}
 	return nil
 }
 
 // RPC version contains a major version and a minor version.
 type RpcProtocolVersions_Version struct {
-	Major                uint32   `protobuf:"varint,1,opt,name=major,proto3" json:"major,omitempty"`
-	Minor                uint32   `protobuf:"varint,2,opt,name=minor,proto3" json:"minor,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *RpcProtocolVersions_Version) Reset()         { *m = RpcProtocolVersions_Version{} }
-func (m *RpcProtocolVersions_Version) String() string { return proto.CompactTextString(m) }
-func (*RpcProtocolVersions_Version) ProtoMessage()    {}
-func (*RpcProtocolVersions_Version) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b97e31e3cc23582a, []int{0, 0}
+	Major uint32 `protobuf:"varint,1,opt,name=major,proto3" json:"major,omitempty"`
+	Minor uint32 `protobuf:"varint,2,opt,name=minor,proto3" json:"minor,omitempty"`
 }
 
-func (m *RpcProtocolVersions_Version) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RpcProtocolVersions_Version.Unmarshal(m, b)
-}
-func (m *RpcProtocolVersions_Version) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RpcProtocolVersions_Version.Marshal(b, m, deterministic)
-}
-func (m *RpcProtocolVersions_Version) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RpcProtocolVersions_Version.Merge(m, src)
+func (x *RpcProtocolVersions_Version) Reset() {
+	*x = RpcProtocolVersions_Version{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *RpcProtocolVersions_Version) XXX_Size() int {
-	return xxx_messageInfo_RpcProtocolVersions_Version.Size(m)
+
+func (x *RpcProtocolVersions_Version) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RpcProtocolVersions_Version) XXX_DiscardUnknown() {
-	xxx_messageInfo_RpcProtocolVersions_Version.DiscardUnknown(m)
+
+func (*RpcProtocolVersions_Version) ProtoMessage() {}
+
+func (x *RpcProtocolVersions_Version) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RpcProtocolVersions_Version proto.InternalMessageInfo
+// Deprecated: Use RpcProtocolVersions_Version.ProtoReflect.Descriptor instead.
+func (*RpcProtocolVersions_Version) Descriptor() ([]byte, []int) {
+	return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0, 0}
+}
 
-func (m *RpcProtocolVersions_Version) GetMajor() uint32 {
-	if m != nil {
-		return m.Major
+func (x *RpcProtocolVersions_Version) GetMajor() uint32 {
+	if x != nil {
+		return x.Major
 	}
 	return 0
 }
 
-func (m *RpcProtocolVersions_Version) GetMinor() uint32 {
-	if m != nil {
-		return m.Minor
+func (x *RpcProtocolVersions_Version) GetMinor() uint32 {
+	if x != nil {
+		return x.Minor
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterEnum("grpc.gcp.SecurityLevel", SecurityLevel_name, SecurityLevel_value)
-	proto.RegisterType((*RpcProtocolVersions)(nil), "grpc.gcp.RpcProtocolVersions")
-	proto.RegisterType((*RpcProtocolVersions_Version)(nil), "grpc.gcp.RpcProtocolVersions.Version")
-}
-
-func init() {
-	proto.RegisterFile("grpc/gcp/transport_security_common.proto", fileDescriptor_b97e31e3cc23582a)
-}
-
-var fileDescriptor_b97e31e3cc23582a = []byte{
-	// 323 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x91, 0x41, 0x4b, 0x3b, 0x31,
-	0x10, 0xc5, 0xff, 0x5b, 0xf8, 0xab, 0x44, 0x56, 0xeb, 0x6a, 0x41, 0xc5, 0x83, 0x08, 0x42, 0xf1,
-	0x90, 0x05, 0xc5, 0xb3, 0xb4, 0xb5, 0x48, 0xa1, 0x6e, 0xeb, 0xb6, 0x16, 0xea, 0x25, 0xc4, 0x18,
-	0x42, 0x24, 0x9b, 0x09, 0xb3, 0xb1, 0xd4, 0xaf, 0xec, 0xa7, 0x90, 0x4d, 0xbb, 0x14, 0xc1, 0x8b,
-	0xb7, 0xbc, 0xc7, 0xcc, 0x6f, 0x32, 0xf3, 0x48, 0x5b, 0xa1, 0x13, 0xa9, 0x12, 0x2e, 0xf5, 0xc8,
-	0x6d, 0xe9, 0x00, 0x3d, 0x2b, 0xa5, 0xf8, 0x40, 0xed, 0x3f, 0x99, 0x80, 0xa2, 0x00, 0x4b, 0x1d,
-	0x82, 0x87, 0x64, 0xa7, 0xaa, 0xa4, 0x4a, 0xb8, 0x8b, 0xaf, 0x88, 0x1c, 0xe6, 0x4e, 0x8c, 0x2b,
-	0x5b, 0x80, 0x99, 0x49, 0x2c, 0x35, 0xd8, 0x32, 0x79, 0x24, 0xfb, 0x05, 0x5f, 0x32, 0x74, 0x82,
-	0x2d, 0x56, 0xde, 0x71, 0x74, 0x1e, 0xb5, 0x77, 0xaf, 0x2f, 0x69, 0xdd, 0x4b, 0x7f, 0xe9, 0xa3,
-	0xeb, 0x47, 0x1e, 0x17, 0x7c, 0x99, 0x3b, 0xb1, 0x96, 0x01, 0xa7, 0xed, 0x0f, 0x5c, 0xe3, 0x6f,
-	0x38, 0x6d, 0x37, 0xb8, 0xd3, 0x5b, 0xb2, 0x5d, 0x93, 0x8f, 0xc8, 0xff, 0x82, 0xbf, 0x03, 0x86,
-	0xef, 0xc5, 0xf9, 0x4a, 0x04, 0x57, 0x5b, 0xc0, 0x30, 0xa5, 0x72, 0x2b, 0x71, 0xf5, 0x44, 0xe2,
-	0xc9, 0xfa, 0x1e, 0x43, 0xb9, 0x90, 0x26, 0x39, 0x20, 0xf1, 0xa4, 0xdf, 0x7b, 0xce, 0x07, 0xd3,
-	0x39, 0xcb, 0x46, 0x59, 0xbf, 0xf9, 0x2f, 0x49, 0xc8, 0xde, 0x20, 0x9b, 0xf6, 0x1f, 0x82, 0x37,
-	0xca, 0x86, 0xf3, 0x66, 0x94, 0x9c, 0x90, 0xd6, 0xc6, 0xeb, 0x64, 0xf7, 0x6c, 0x9c, 0x0f, 0x66,
-	0x9d, 0xde, 0xbc, 0xd9, 0xe8, 0x2e, 0x49, 0x4b, 0xc3, 0x6a, 0x07, 0x6e, 0x7c, 0x49, 0xb5, 0xf5,
-	0x12, 0x2d, 0x37, 0xdd, 0xb3, 0x69, 0x9d, 0x41, 0x3d, 0xb2, 0x17, 0x12, 0x08, 0x2b, 0x8e, 0xa3,
-	0x97, 0x3b, 0x05, 0xa0, 0x8c, 0xa4, 0x0a, 0x0c, 0xb7, 0x8a, 0x02, 0xaa, 0x34, 0xc4, 0x27, 0x50,
-	0xbe, 0x49, 0xeb, 0x35, 0x37, 0x65, 0x5a, 0x11, 0xd3, 0x9a, 0x98, 0x86, 0xe8, 0x42, 0x11, 0x53,
-	0xc2, 0xbd, 0x6e, 0x05, 0x7d, 0xf3, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x31, 0x14, 0xb4, 0x11, 0xf6,
-	0x01, 0x00, 0x00,
+var File_grpc_gcp_transport_security_common_proto protoreflect.FileDescriptor
+
+var file_grpc_gcp_transport_security_common_proto_rawDesc = []byte{
+	0x0a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73,
+	0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x63, 0x6f,
+	0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x67, 0x63, 0x70, 0x22, 0xea, 0x01, 0x0a, 0x13, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4d, 0x0a, 0x0f,
+	0x6d, 0x61, 0x78, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70,
+	0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73,
+	0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x6d, 0x61,
+	0x78, 0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4d, 0x0a, 0x0f, 0x6d,
+	0x69, 0x6e, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e,
+	0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x6d, 0x69, 0x6e,
+	0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x35, 0x0a, 0x07, 0x56, 0x65,
+	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d,
+	0x69, 0x6e, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f,
+	0x72, 0x2a, 0x51, 0x0a, 0x0d, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x65, 0x76,
+	0x65, 0x6c, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x45, 0x43, 0x55, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x4e,
+	0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x54, 0x45, 0x47, 0x52, 0x49,
+	0x54, 0x59, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x49, 0x4e, 0x54,
+	0x45, 0x47, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x50, 0x52, 0x49, 0x56, 0x41,
+	0x43, 0x59, 0x10, 0x02, 0x42, 0x78, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x61, 0x6c, 0x74, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x1c, 0x54,
+	0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
+	0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
+	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
+	0x73, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x67, 0x63, 0x70, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_gcp_transport_security_common_proto_rawDescOnce sync.Once
+	file_grpc_gcp_transport_security_common_proto_rawDescData = file_grpc_gcp_transport_security_common_proto_rawDesc
+)
+
+func file_grpc_gcp_transport_security_common_proto_rawDescGZIP() []byte {
+	file_grpc_gcp_transport_security_common_proto_rawDescOnce.Do(func() {
+		file_grpc_gcp_transport_security_common_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_gcp_transport_security_common_proto_rawDescData)
+	})
+	return file_grpc_gcp_transport_security_common_proto_rawDescData
+}
+
+var file_grpc_gcp_transport_security_common_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_grpc_gcp_transport_security_common_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_grpc_gcp_transport_security_common_proto_goTypes = []interface{}{
+	(SecurityLevel)(0),                  // 0: grpc.gcp.SecurityLevel
+	(*RpcProtocolVersions)(nil),         // 1: grpc.gcp.RpcProtocolVersions
+	(*RpcProtocolVersions_Version)(nil), // 2: grpc.gcp.RpcProtocolVersions.Version
+}
+var file_grpc_gcp_transport_security_common_proto_depIdxs = []int32{
+	2, // 0: grpc.gcp.RpcProtocolVersions.max_rpc_version:type_name -> grpc.gcp.RpcProtocolVersions.Version
+	2, // 1: grpc.gcp.RpcProtocolVersions.min_rpc_version:type_name -> grpc.gcp.RpcProtocolVersions.Version
+	2, // [2:2] is the sub-list for method output_type
+	2, // [2:2] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_grpc_gcp_transport_security_common_proto_init() }
+func file_grpc_gcp_transport_security_common_proto_init() {
+	if File_grpc_gcp_transport_security_common_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_gcp_transport_security_common_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RpcProtocolVersions); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_gcp_transport_security_common_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RpcProtocolVersions_Version); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_gcp_transport_security_common_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_gcp_transport_security_common_proto_goTypes,
+		DependencyIndexes: file_grpc_gcp_transport_security_common_proto_depIdxs,
+		EnumInfos:         file_grpc_gcp_transport_security_common_proto_enumTypes,
+		MessageInfos:      file_grpc_gcp_transport_security_common_proto_msgTypes,
+	}.Build()
+	File_grpc_gcp_transport_security_common_proto = out.File
+	file_grpc_gcp_transport_security_common_proto_rawDesc = nil
+	file_grpc_gcp_transport_security_common_proto_goTypes = nil
+	file_grpc_gcp_transport_security_common_proto_depIdxs = nil
 }
diff --git a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
index 936dfb6ed4bf..afabe3b7b12d 100644
--- a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
@@ -1,4 +1,21 @@
+// Copyright 2020 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/tls/provider/meshca/experimental/config.proto
 
 // NOTE: This proto will very likely move to a different namespace and a
@@ -7,23 +24,25 @@
 package meshca_experimental
 
 import (
-	fmt "fmt"
 	v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Type of key to be embedded in CSRs sent to the MeshCA.
 type GoogleMeshCaConfig_KeyType int32
@@ -33,27 +52,52 @@ const (
 	GoogleMeshCaConfig_KEY_TYPE_RSA     GoogleMeshCaConfig_KeyType = 1
 )
 
-var GoogleMeshCaConfig_KeyType_name = map[int32]string{
-	0: "KEY_TYPE_UNKNOWN",
-	1: "KEY_TYPE_RSA",
-}
+// Enum value maps for GoogleMeshCaConfig_KeyType.
+var (
+	GoogleMeshCaConfig_KeyType_name = map[int32]string{
+		0: "KEY_TYPE_UNKNOWN",
+		1: "KEY_TYPE_RSA",
+	}
+	GoogleMeshCaConfig_KeyType_value = map[string]int32{
+		"KEY_TYPE_UNKNOWN": 0,
+		"KEY_TYPE_RSA":     1,
+	}
+)
 
-var GoogleMeshCaConfig_KeyType_value = map[string]int32{
-	"KEY_TYPE_UNKNOWN": 0,
-	"KEY_TYPE_RSA":     1,
+func (x GoogleMeshCaConfig_KeyType) Enum() *GoogleMeshCaConfig_KeyType {
+	p := new(GoogleMeshCaConfig_KeyType)
+	*p = x
+	return p
 }
 
 func (x GoogleMeshCaConfig_KeyType) String() string {
-	return proto.EnumName(GoogleMeshCaConfig_KeyType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (GoogleMeshCaConfig_KeyType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes[0].Descriptor()
 }
 
+func (GoogleMeshCaConfig_KeyType) Type() protoreflect.EnumType {
+	return &file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes[0]
+}
+
+func (x GoogleMeshCaConfig_KeyType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use GoogleMeshCaConfig_KeyType.Descriptor instead.
 func (GoogleMeshCaConfig_KeyType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_2f29f6dba7c86653, []int{0, 0}
+	return file_grpc_tls_provider_meshca_experimental_config_proto_rawDescGZIP(), []int{0, 0}
 }
 
 // GoogleMeshCaConfig contains all configuration parameters required by the
 // MeshCA CertificateProvider plugin implementation.
 type GoogleMeshCaConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// GoogleMeshCA server endpoint to get CSRs signed via the *CreateCertificate*
 	// unary call. This must have :ref:`api_type
 	// <envoy_api_field_config.core.v3.ApiConfigSource.api_type>` :ref:`GRPC
@@ -83,116 +127,205 @@ type GoogleMeshCaConfig struct {
 	// GCE location (region/zone) where the workload is located.
 	//
 	// GCE/GKE Metadata Server will be contacted if left unspecified.
-	Location             string   `protobuf:"bytes,6,opt,name=location,proto3" json:"location,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Location string `protobuf:"bytes,6,opt,name=location,proto3" json:"location,omitempty"`
 }
 
-func (m *GoogleMeshCaConfig) Reset()         { *m = GoogleMeshCaConfig{} }
-func (m *GoogleMeshCaConfig) String() string { return proto.CompactTextString(m) }
-func (*GoogleMeshCaConfig) ProtoMessage()    {}
-func (*GoogleMeshCaConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_2f29f6dba7c86653, []int{0}
+func (x *GoogleMeshCaConfig) Reset() {
+	*x = GoogleMeshCaConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *GoogleMeshCaConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GoogleMeshCaConfig.Unmarshal(m, b)
-}
-func (m *GoogleMeshCaConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GoogleMeshCaConfig.Marshal(b, m, deterministic)
-}
-func (m *GoogleMeshCaConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GoogleMeshCaConfig.Merge(m, src)
+func (x *GoogleMeshCaConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GoogleMeshCaConfig) XXX_Size() int {
-	return xxx_messageInfo_GoogleMeshCaConfig.Size(m)
-}
-func (m *GoogleMeshCaConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_GoogleMeshCaConfig.DiscardUnknown(m)
+
+func (*GoogleMeshCaConfig) ProtoMessage() {}
+
+func (x *GoogleMeshCaConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_GoogleMeshCaConfig proto.InternalMessageInfo
+// Deprecated: Use GoogleMeshCaConfig.ProtoReflect.Descriptor instead.
+func (*GoogleMeshCaConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_tls_provider_meshca_experimental_config_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *GoogleMeshCaConfig) GetServer() *v3.ApiConfigSource {
-	if m != nil {
-		return m.Server
+func (x *GoogleMeshCaConfig) GetServer() *v3.ApiConfigSource {
+	if x != nil {
+		return x.Server
 	}
 	return nil
 }
 
-func (m *GoogleMeshCaConfig) GetCertificateLifetime() *duration.Duration {
-	if m != nil {
-		return m.CertificateLifetime
+func (x *GoogleMeshCaConfig) GetCertificateLifetime() *duration.Duration {
+	if x != nil {
+		return x.CertificateLifetime
 	}
 	return nil
 }
 
-func (m *GoogleMeshCaConfig) GetRenewalGracePeriod() *duration.Duration {
-	if m != nil {
-		return m.RenewalGracePeriod
+func (x *GoogleMeshCaConfig) GetRenewalGracePeriod() *duration.Duration {
+	if x != nil {
+		return x.RenewalGracePeriod
 	}
 	return nil
 }
 
-func (m *GoogleMeshCaConfig) GetKeyType() GoogleMeshCaConfig_KeyType {
-	if m != nil {
-		return m.KeyType
+func (x *GoogleMeshCaConfig) GetKeyType() GoogleMeshCaConfig_KeyType {
+	if x != nil {
+		return x.KeyType
 	}
 	return GoogleMeshCaConfig_KEY_TYPE_UNKNOWN
 }
 
-func (m *GoogleMeshCaConfig) GetKeySize() uint32 {
-	if m != nil {
-		return m.KeySize
+func (x *GoogleMeshCaConfig) GetKeySize() uint32 {
+	if x != nil {
+		return x.KeySize
 	}
 	return 0
 }
 
-func (m *GoogleMeshCaConfig) GetLocation() string {
-	if m != nil {
-		return m.Location
+func (x *GoogleMeshCaConfig) GetLocation() string {
+	if x != nil {
+		return x.Location
 	}
 	return ""
 }
 
-func init() {
-	proto.RegisterEnum("grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig_KeyType", GoogleMeshCaConfig_KeyType_name, GoogleMeshCaConfig_KeyType_value)
-	proto.RegisterType((*GoogleMeshCaConfig)(nil), "grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig")
-}
-
-func init() {
-	proto.RegisterFile("grpc/tls/provider/meshca/experimental/config.proto", fileDescriptor_2f29f6dba7c86653)
-}
-
-var fileDescriptor_2f29f6dba7c86653 = []byte{
-	// 439 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x4f, 0x6f, 0xd3, 0x30,
-	0x18, 0xc6, 0x09, 0x83, 0x6e, 0x98, 0x3f, 0x2a, 0xa6, 0x87, 0xac, 0x07, 0x54, 0x4d, 0x9a, 0x94,
-	0x93, 0x2d, 0xda, 0x33, 0x87, 0x6e, 0x4c, 0x3b, 0x74, 0x94, 0x2a, 0x1d, 0x9a, 0x86, 0x90, 0x22,
-	0xcf, 0x7d, 0x9b, 0x59, 0x73, 0xed, 0xe8, 0x8d, 0x1b, 0xc8, 0x3e, 0x09, 0x9f, 0x84, 0xcf, 0x87,
-	0xe2, 0x78, 0x53, 0x04, 0x07, 0x7a, 0xcb, 0x1b, 0xfb, 0xf7, 0x3c, 0x8f, 0x1e, 0xbf, 0x64, 0x9c,
-	0x63, 0x21, 0xb9, 0xd3, 0x25, 0x2f, 0xd0, 0x56, 0x6a, 0x05, 0xc8, 0x37, 0x50, 0xde, 0x4a, 0xc1,
-	0xe1, 0x67, 0x01, 0xa8, 0x36, 0x60, 0x9c, 0xd0, 0x5c, 0x5a, 0xb3, 0x56, 0x39, 0x2b, 0xd0, 0x3a,
-	0x4b, 0x8f, 0x1b, 0x86, 0x39, 0x5d, 0xb2, 0x07, 0x86, 0xb5, 0x0c, 0xeb, 0x32, 0xc3, 0x04, 0x4c,
-	0x65, 0xeb, 0x80, 0x72, 0x69, 0x11, 0x78, 0x35, 0x09, 0x63, 0x56, 0xda, 0x2d, 0x4a, 0x68, 0x05,
-	0x87, 0xef, 0x73, 0x6b, 0x73, 0x0d, 0xdc, 0x4f, 0x37, 0xdb, 0x35, 0x5f, 0x6d, 0x51, 0x38, 0x65,
-	0x4d, 0x7b, 0x7e, 0xf4, 0x7b, 0x8f, 0xd0, 0x73, 0x7f, 0xe5, 0x33, 0x94, 0xb7, 0xa7, 0xe2, 0xd4,
-	0x6b, 0xd0, 0x8f, 0xa4, 0x57, 0x02, 0x56, 0x80, 0x71, 0x34, 0x8a, 0x92, 0x97, 0xe3, 0x63, 0xe6,
-	0x1d, 0x59, 0x08, 0xdb, 0x38, 0xb2, 0x6a, 0xc2, 0xa6, 0x85, 0x6a, 0x81, 0xa5, 0xf7, 0x4c, 0x03,
-	0x44, 0x2f, 0xc8, 0x40, 0x02, 0x3a, 0xb5, 0x56, 0x52, 0x38, 0xc8, 0xb4, 0x5a, 0x83, 0x53, 0x1b,
-	0x88, 0x9f, 0x7a, 0xb1, 0x43, 0xd6, 0x86, 0x62, 0x0f, 0xa1, 0xd8, 0xa7, 0x10, 0x2a, 0x7d, 0xd7,
-	0xc1, 0x2e, 0x02, 0x45, 0x67, 0x64, 0x80, 0x60, 0xe0, 0x87, 0xd0, 0x59, 0x8e, 0x42, 0x42, 0xd6,
-	0x34, 0x61, 0x57, 0xf1, 0xde, 0xff, 0xd4, 0x68, 0xc0, 0xce, 0x1b, 0x6a, 0xe1, 0x21, 0xfa, 0x9d,
-	0x1c, 0xdc, 0x41, 0x9d, 0xb9, 0xba, 0x80, 0xf8, 0xd9, 0x28, 0x4a, 0xde, 0x8c, 0xa7, 0x6c, 0xa7,
-	0xd2, 0xd9, 0xbf, 0x35, 0xb1, 0x19, 0xd4, 0x97, 0x75, 0x01, 0xe9, 0xfe, 0x5d, 0xfb, 0x41, 0x0f,
-	0x5b, 0xf5, 0x52, 0xdd, 0x43, 0xfc, 0x7c, 0x14, 0x25, 0xaf, 0xfd, 0xd1, 0x52, 0xdd, 0x03, 0x1d,
-	0x92, 0x03, 0x6d, 0xa5, 0x0f, 0x16, 0xf7, 0x46, 0x51, 0xf2, 0x22, 0x7d, 0x9c, 0x8f, 0x3e, 0x90,
-	0xfd, 0x20, 0x45, 0x07, 0xa4, 0x3f, 0x3b, 0xbb, 0xce, 0x2e, 0xaf, 0x17, 0x67, 0xd9, 0xd7, 0xf9,
-	0x6c, 0xfe, 0xe5, 0x6a, 0xde, 0x7f, 0x42, 0xfb, 0xe4, 0xd5, 0xe3, 0xdf, 0x74, 0x39, 0xed, 0x47,
-	0x27, 0xbf, 0x22, 0x92, 0x28, 0xbb, 0x5b, 0xf4, 0x93, 0xb7, 0xdd, 0xd4, 0x8b, 0xa6, 0xa7, 0x45,
-	0xf4, 0xed, 0x2a, 0xf4, 0x96, 0x5b, 0x2d, 0x4c, 0xce, 0x2c, 0xe6, 0xdc, 0x6f, 0xac, 0x44, 0x58,
-	0x81, 0x71, 0x4a, 0xe8, 0xd2, 0x6f, 0x6f, 0xf3, 0x24, 0x7f, 0x6f, 0xb0, 0x32, 0x0e, 0xd0, 0x08,
-	0x1d, 0xe6, 0xac, 0xeb, 0x76, 0xd3, 0xf3, 0x2f, 0x31, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff, 0x72,
-	0x44, 0x68, 0xc5, 0x01, 0x03, 0x00, 0x00,
+var File_grpc_tls_provider_meshca_experimental_config_proto protoreflect.FileDescriptor
+
+var file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc = []byte{
+	0x0a, 0x32, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x6c, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69,
+	0x64, 0x65, 0x72, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2f, 0x65, 0x78, 0x70, 0x65, 0x72,
+	0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x25, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x6c, 0x73, 0x2e, 0x70,
+	0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x65,
+	0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x1a, 0x28, 0x65, 0x6e, 0x76,
+	0x6f, 0x79, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76,
+	0x33, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb6, 0x03, 0x0a, 0x12, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x4d, 0x65, 0x73, 0x68, 0x43, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3d, 0x0a, 0x06,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x65,
+	0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x72, 0x65,
+	0x2e, 0x76, 0x33, 0x2e, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x4c, 0x0a, 0x14, 0x63,
+	0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74,
+	0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74,
+	0x65, 0x4c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x14, 0x72, 0x65, 0x6e,
+	0x65, 0x77, 0x61, 0x6c, 0x5f, 0x67, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f,
+	0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x52, 0x12, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x61, 0x6c, 0x47, 0x72, 0x61, 0x63, 0x65,
+	0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x5c, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x74, 0x79,
+	0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x41, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x6d, 0x65, 0x73,
+	0x68, 0x63, 0x61, 0x2e, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c,
+	0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x61, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x2e, 0x4b, 0x65, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x6b, 0x65, 0x79,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x69, 0x7a, 0x65,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x53, 0x69, 0x7a, 0x65, 0x12,
+	0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x31, 0x0a, 0x07, 0x4b,
+	0x65, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4b, 0x45, 0x59, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c,
+	0x4b, 0x45, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x10, 0x01, 0x42, 0x98,
+	0x01, 0x0a, 0x28, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x6c, 0x73, 0x2e, 0x70,
+	0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x65,
+	0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x42, 0x11, 0x4d, 0x65, 0x73,
+	0x68, 0x43, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
+	0x5a, 0x57, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e,
+	0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
+	0x69, 0x61, 0x6c, 0x73, 0x2f, 0x74, 0x6c, 0x73, 0x2f, 0x63, 0x65, 0x72, 0x74, 0x70, 0x72, 0x6f,
+	0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2f, 0x69, 0x6e, 0x74,
+	0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x5f, 0x65, 0x78, 0x70,
+	0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
+}
+
+var (
+	file_grpc_tls_provider_meshca_experimental_config_proto_rawDescOnce sync.Once
+	file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData = file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc
+)
+
+func file_grpc_tls_provider_meshca_experimental_config_proto_rawDescGZIP() []byte {
+	file_grpc_tls_provider_meshca_experimental_config_proto_rawDescOnce.Do(func() {
+		file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData)
+	})
+	return file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData
+}
+
+var file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_grpc_tls_provider_meshca_experimental_config_proto_goTypes = []interface{}{
+	(GoogleMeshCaConfig_KeyType)(0), // 0: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.KeyType
+	(*GoogleMeshCaConfig)(nil),      // 1: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig
+	(*v3.ApiConfigSource)(nil),      // 2: envoy.config.core.v3.ApiConfigSource
+	(*duration.Duration)(nil),       // 3: google.protobuf.Duration
+}
+var file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs = []int32{
+	2, // 0: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.server:type_name -> envoy.config.core.v3.ApiConfigSource
+	3, // 1: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.certificate_lifetime:type_name -> google.protobuf.Duration
+	3, // 2: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.renewal_grace_period:type_name -> google.protobuf.Duration
+	0, // 3: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.key_type:type_name -> grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.KeyType
+	4, // [4:4] is the sub-list for method output_type
+	4, // [4:4] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
+}
+
+func init() { file_grpc_tls_provider_meshca_experimental_config_proto_init() }
+func file_grpc_tls_provider_meshca_experimental_config_proto_init() {
+	if File_grpc_tls_provider_meshca_experimental_config_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GoogleMeshCaConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_tls_provider_meshca_experimental_config_proto_goTypes,
+		DependencyIndexes: file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs,
+		EnumInfos:         file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes,
+		MessageInfos:      file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes,
+	}.Build()
+	File_grpc_tls_provider_meshca_experimental_config_proto = out.File
+	file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc = nil
+	file_grpc_tls_provider_meshca_experimental_config_proto_goTypes = nil
+	file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs = nil
 }
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
index 16122f768488..bc0fcd452b47 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
@@ -1,155 +1,276 @@
+// Copyright 2019 Istio Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: istio/google/security/meshca/v1/meshca.proto
 
 package google_security_meshca_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Certificate request message.
 type MeshCertificateRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The request ID must be a valid UUID with the exception that zero UUID is
 	// not supported (00000000-0000-0000-0000-000000000000).
 	RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
 	// PEM-encoded certificate request.
 	Csr string `protobuf:"bytes,2,opt,name=csr,proto3" json:"csr,omitempty"`
 	// Optional: requested certificate validity period.
-	Validity             *duration.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
-	XXX_unrecognized     []byte             `json:"-"`
-	XXX_sizecache        int32              `json:"-"`
+	Validity *duration.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"` // Reserved 4
 }
 
-func (m *MeshCertificateRequest) Reset()         { *m = MeshCertificateRequest{} }
-func (m *MeshCertificateRequest) String() string { return proto.CompactTextString(m) }
-func (*MeshCertificateRequest) ProtoMessage()    {}
-func (*MeshCertificateRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f72841047b94fe5e, []int{0}
+func (x *MeshCertificateRequest) Reset() {
+	*x = MeshCertificateRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *MeshCertificateRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MeshCertificateRequest.Unmarshal(m, b)
+func (x *MeshCertificateRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *MeshCertificateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MeshCertificateRequest.Marshal(b, m, deterministic)
-}
-func (m *MeshCertificateRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MeshCertificateRequest.Merge(m, src)
-}
-func (m *MeshCertificateRequest) XXX_Size() int {
-	return xxx_messageInfo_MeshCertificateRequest.Size(m)
-}
-func (m *MeshCertificateRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_MeshCertificateRequest.DiscardUnknown(m)
+
+func (*MeshCertificateRequest) ProtoMessage() {}
+
+func (x *MeshCertificateRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_MeshCertificateRequest proto.InternalMessageInfo
+// Deprecated: Use MeshCertificateRequest.ProtoReflect.Descriptor instead.
+func (*MeshCertificateRequest) Descriptor() ([]byte, []int) {
+	return file_istio_google_security_meshca_v1_meshca_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *MeshCertificateRequest) GetRequestId() string {
-	if m != nil {
-		return m.RequestId
+func (x *MeshCertificateRequest) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
 	}
 	return ""
 }
 
-func (m *MeshCertificateRequest) GetCsr() string {
-	if m != nil {
-		return m.Csr
+func (x *MeshCertificateRequest) GetCsr() string {
+	if x != nil {
+		return x.Csr
 	}
 	return ""
 }
 
-func (m *MeshCertificateRequest) GetValidity() *duration.Duration {
-	if m != nil {
-		return m.Validity
+func (x *MeshCertificateRequest) GetValidity() *duration.Duration {
+	if x != nil {
+		return x.Validity
 	}
 	return nil
 }
 
 // Certificate response message.
 type MeshCertificateResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// PEM-encoded certificate chain.
 	// Leaf cert is element '0'. Root cert is element 'n'.
-	CertChain            []string `protobuf:"bytes,1,rep,name=cert_chain,json=certChain,proto3" json:"cert_chain,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	CertChain []string `protobuf:"bytes,1,rep,name=cert_chain,json=certChain,proto3" json:"cert_chain,omitempty"`
 }
 
-func (m *MeshCertificateResponse) Reset()         { *m = MeshCertificateResponse{} }
-func (m *MeshCertificateResponse) String() string { return proto.CompactTextString(m) }
-func (*MeshCertificateResponse) ProtoMessage()    {}
-func (*MeshCertificateResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f72841047b94fe5e, []int{1}
+func (x *MeshCertificateResponse) Reset() {
+	*x = MeshCertificateResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *MeshCertificateResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MeshCertificateResponse.Unmarshal(m, b)
+func (x *MeshCertificateResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *MeshCertificateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MeshCertificateResponse.Marshal(b, m, deterministic)
+
+func (*MeshCertificateResponse) ProtoMessage() {}
+
+func (x *MeshCertificateResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *MeshCertificateResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MeshCertificateResponse.Merge(m, src)
+
+// Deprecated: Use MeshCertificateResponse.ProtoReflect.Descriptor instead.
+func (*MeshCertificateResponse) Descriptor() ([]byte, []int) {
+	return file_istio_google_security_meshca_v1_meshca_proto_rawDescGZIP(), []int{1}
 }
-func (m *MeshCertificateResponse) XXX_Size() int {
-	return xxx_messageInfo_MeshCertificateResponse.Size(m)
+
+func (x *MeshCertificateResponse) GetCertChain() []string {
+	if x != nil {
+		return x.CertChain
+	}
+	return nil
 }
-func (m *MeshCertificateResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_MeshCertificateResponse.DiscardUnknown(m)
+
+var File_istio_google_security_meshca_v1_meshca_proto protoreflect.FileDescriptor
+
+var file_istio_google_security_meshca_v1_meshca_proto_rawDesc = []byte{
+	0x0a, 0x2c, 0x69, 0x73, 0x74, 0x69, 0x6f, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x73,
+	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2f, 0x76,
+	0x31, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e,
+	0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x80, 0x01, 0x0a, 0x16, 0x4d, 0x65,
+	0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f,
+	0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x73, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x63, 0x73, 0x72, 0x12, 0x35, 0x0a, 0x08, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74,
+	0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x52, 0x08, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74, 0x79, 0x22, 0x38, 0x0a, 0x17,
+	0x4d, 0x65, 0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f,
+	0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x63, 0x65, 0x72,
+	0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x32, 0x96, 0x01, 0x0a, 0x16, 0x4d, 0x65, 0x73, 0x68, 0x43,
+	0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x12, 0x7c, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69,
+	0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x31, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+	0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e,
+	0x76, 0x31, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61,
+	0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x6d, 0x65, 0x73, 0x68,
+	0x63, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66,
+	0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42,
+	0x2e, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x73, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x76, 0x31,
+	0x42, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
-var xxx_messageInfo_MeshCertificateResponse proto.InternalMessageInfo
+var (
+	file_istio_google_security_meshca_v1_meshca_proto_rawDescOnce sync.Once
+	file_istio_google_security_meshca_v1_meshca_proto_rawDescData = file_istio_google_security_meshca_v1_meshca_proto_rawDesc
+)
 
-func (m *MeshCertificateResponse) GetCertChain() []string {
-	if m != nil {
-		return m.CertChain
-	}
-	return nil
+func file_istio_google_security_meshca_v1_meshca_proto_rawDescGZIP() []byte {
+	file_istio_google_security_meshca_v1_meshca_proto_rawDescOnce.Do(func() {
+		file_istio_google_security_meshca_v1_meshca_proto_rawDescData = protoimpl.X.CompressGZIP(file_istio_google_security_meshca_v1_meshca_proto_rawDescData)
+	})
+	return file_istio_google_security_meshca_v1_meshca_proto_rawDescData
+}
+
+var file_istio_google_security_meshca_v1_meshca_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_istio_google_security_meshca_v1_meshca_proto_goTypes = []interface{}{
+	(*MeshCertificateRequest)(nil),  // 0: google.security.meshca.v1.MeshCertificateRequest
+	(*MeshCertificateResponse)(nil), // 1: google.security.meshca.v1.MeshCertificateResponse
+	(*duration.Duration)(nil),       // 2: google.protobuf.Duration
+}
+var file_istio_google_security_meshca_v1_meshca_proto_depIdxs = []int32{
+	2, // 0: google.security.meshca.v1.MeshCertificateRequest.validity:type_name -> google.protobuf.Duration
+	0, // 1: google.security.meshca.v1.MeshCertificateService.CreateCertificate:input_type -> google.security.meshca.v1.MeshCertificateRequest
+	1, // 2: google.security.meshca.v1.MeshCertificateService.CreateCertificate:output_type -> google.security.meshca.v1.MeshCertificateResponse
+	2, // [2:3] is the sub-list for method output_type
+	1, // [1:2] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
 }
 
-func init() {
-	proto.RegisterType((*MeshCertificateRequest)(nil), "google.security.meshca.v1.MeshCertificateRequest")
-	proto.RegisterType((*MeshCertificateResponse)(nil), "google.security.meshca.v1.MeshCertificateResponse")
-}
-
-func init() {
-	proto.RegisterFile("istio/google/security/meshca/v1/meshca.proto", fileDescriptor_f72841047b94fe5e)
-}
-
-var fileDescriptor_f72841047b94fe5e = []byte{
-	// 284 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4b, 0xc3, 0x30,
-	0x1c, 0xc5, 0x8d, 0x05, 0x71, 0xd9, 0x45, 0x73, 0xd0, 0x6e, 0x30, 0x29, 0x3d, 0xed, 0x20, 0x29,
-	0xad, 0x08, 0x9e, 0x57, 0x2f, 0x1e, 0x84, 0x51, 0x3f, 0xc0, 0xc8, 0xd2, 0xff, 0xd6, 0x3f, 0x6c,
-	0xcd, 0x4c, 0xd2, 0xc2, 0xc0, 0x83, 0x9f, 0xc2, 0xcf, 0x2b, 0x69, 0x33, 0x11, 0xe6, 0x0e, 0xde,
-	0x1e, 0x2f, 0xef, 0xe5, 0xff, 0xe3, 0xd1, 0x7b, 0x34, 0x16, 0x55, 0xb2, 0x56, 0x6a, 0xbd, 0x81,
-	0xc4, 0x80, 0x6c, 0x34, 0xda, 0x7d, 0xb2, 0x05, 0x53, 0x49, 0x91, 0xb4, 0xa9, 0x57, 0x7c, 0xa7,
-	0x95, 0x55, 0x6c, 0xd4, 0xe7, 0xf8, 0x21, 0xc7, 0xfd, 0x6b, 0x9b, 0x8e, 0xef, 0xfc, 0x17, 0x5d,
-	0x70, 0xd9, 0xac, 0x92, 0xb2, 0xd1, 0xc2, 0xa2, 0xaa, 0xfb, 0x6a, 0xfc, 0x49, 0xe8, 0xcd, 0x2b,
-	0x98, 0x2a, 0x07, 0x6d, 0x71, 0x85, 0x52, 0x58, 0x28, 0xe0, 0xbd, 0x01, 0x63, 0xd9, 0x84, 0x52,
-	0xdd, 0xcb, 0x05, 0x96, 0x21, 0x89, 0xc8, 0x74, 0x50, 0x0c, 0xbc, 0xf3, 0x52, 0xb2, 0x2b, 0x1a,
-	0x48, 0xa3, 0xc3, 0xf3, 0xce, 0x77, 0x92, 0x3d, 0xd2, 0xcb, 0x56, 0x6c, 0xb0, 0x44, 0xbb, 0x0f,
-	0x83, 0x88, 0x4c, 0x87, 0xd9, 0x88, 0x7b, 0xb2, 0xc3, 0x79, 0xfe, 0xec, 0xcf, 0x17, 0x3f, 0xd1,
-	0xf8, 0x89, 0xde, 0x1e, 0x11, 0x98, 0x9d, 0xaa, 0x0d, 0x38, 0x04, 0x09, 0xda, 0x2e, 0x64, 0x25,
-	0xb0, 0x0e, 0x49, 0x14, 0x38, 0x04, 0xe7, 0xe4, 0xce, 0xc8, 0xbe, 0x8e, 0xe1, 0xdf, 0x40, 0xb7,
-	0x28, 0x81, 0x7d, 0xd0, 0xeb, 0x5c, 0x83, 0xb0, 0xf0, 0xeb, 0x8d, 0xa5, 0xfc, 0xe4, 0x50, 0xfc,
-	0xef, 0x11, 0xc6, 0xd9, 0x7f, 0x2a, 0x3d, 0x75, 0x7c, 0x36, 0xe3, 0x74, 0x22, 0xd5, 0xf6, 0x74,
-	0x75, 0x36, 0xec, 0xba, 0x62, 0xee, 0x66, 0x99, 0x93, 0xe5, 0x45, 0xb7, 0xcf, 0xc3, 0x77, 0x00,
-	0x00, 0x00, 0xff, 0xff, 0xb7, 0x0d, 0xfd, 0xff, 0xf7, 0x01, 0x00, 0x00,
+func init() { file_istio_google_security_meshca_v1_meshca_proto_init() }
+func file_istio_google_security_meshca_v1_meshca_proto_init() {
+	if File_istio_google_security_meshca_v1_meshca_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_istio_google_security_meshca_v1_meshca_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MeshCertificateRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_istio_google_security_meshca_v1_meshca_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MeshCertificateResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_istio_google_security_meshca_v1_meshca_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_istio_google_security_meshca_v1_meshca_proto_goTypes,
+		DependencyIndexes: file_istio_google_security_meshca_v1_meshca_proto_depIdxs,
+		MessageInfos:      file_istio_google_security_meshca_v1_meshca_proto_msgTypes,
+	}.Build()
+	File_istio_google_security_meshca_v1_meshca_proto = out.File
+	file_istio_google_security_meshca_v1_meshca_proto_rawDesc = nil
+	file_istio_google_security_meshca_v1_meshca_proto_goTypes = nil
+	file_istio_google_security_meshca_v1_meshca_proto_depIdxs = nil
 }
diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go
index e56015bfe63e..8421ea7a3896 100644
--- a/encoding/proto/proto_test.go
+++ b/encoding/proto/proto_test.go
@@ -106,11 +106,11 @@ func (s) TestStaggeredMarshalAndUnmarshalUsingSamePool(t *testing.T) {
 	var err error
 
 	if m1, err = codec1.Marshal(&proto1); err != nil {
-		t.Errorf("codec.Marshal(%v) failed", proto1)
+		t.Errorf("codec.Marshal(%s) failed", &proto1)
 	}
 
 	if m2, err = codec2.Marshal(&proto2); err != nil {
-		t.Errorf("codec.Marshal(%v) failed", proto2)
+		t.Errorf("codec.Marshal(%s) failed", &proto2)
 	}
 
 	if err = codec1.Unmarshal(m1, &proto1); err != nil {
diff --git a/examples/examples_test.sh b/examples/examples_test.sh
index 85e2958bff8e..9015272f33e0 100755
--- a/examples/examples_test.sh
+++ b/examples/examples_test.sh
@@ -70,14 +70,14 @@ declare -A EXPECTED_SERVER_OUTPUT=(
     ["features/errors"]=""
     ["features/interceptor"]="unary echoing message \"hello world\""
     ["features/load_balancing"]="serving on :50051"
-    ["features/metadata"]="message:\"this is examples/metadata\" , sending echo"
+    ["features/metadata"]="message:\"this is examples/metadata\", sending echo"
     ["features/multiplex"]=":50051"
     ["features/name_resolving"]="serving on localhost:50051"
 )
 
 declare -A EXPECTED_CLIENT_OUTPUT=(
     ["helloworld"]="Greeting: Hello world"
-    ["route_guide"]="location:<latitude:416851321 longitude:-742674555 >"
+    ["route_guide"]="Feature: name: \"\", point:(416851321, -742674555)"
     ["features/authentication"]="UnaryEcho:  hello world"
     ["features/compression"]="UnaryEcho call returned \"compress\", <nil>"
     ["features/deadline"]="wanted = DeadlineExceeded, got = DeadlineExceeded"
diff --git a/examples/features/proto/echo/echo.pb.go b/examples/features/proto/echo/echo.pb.go
index 4048b43c0c2c..b096703e62ee 100644
--- a/examples/features/proto/echo/echo.pb.go
+++ b/examples/features/proto/echo/echo.pb.go
@@ -1,129 +1,265 @@
+//
+//
+// Copyright 2018 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: examples/features/proto/echo/echo.proto
 
 package echo
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // EchoRequest is the request for echo.
 type EchoRequest struct {
-	Message              string   `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *EchoRequest) Reset()         { *m = EchoRequest{} }
-func (m *EchoRequest) String() string { return proto.CompactTextString(m) }
-func (*EchoRequest) ProtoMessage()    {}
-func (*EchoRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_2fd1d686b7b805dc, []int{0}
+	Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
 }
 
-func (m *EchoRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EchoRequest.Unmarshal(m, b)
-}
-func (m *EchoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EchoRequest.Marshal(b, m, deterministic)
-}
-func (m *EchoRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EchoRequest.Merge(m, src)
+func (x *EchoRequest) Reset() {
+	*x = EchoRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_features_proto_echo_echo_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *EchoRequest) XXX_Size() int {
-	return xxx_messageInfo_EchoRequest.Size(m)
+
+func (x *EchoRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EchoRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_EchoRequest.DiscardUnknown(m)
+
+func (*EchoRequest) ProtoMessage() {}
+
+func (x *EchoRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_features_proto_echo_echo_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_EchoRequest proto.InternalMessageInfo
+// Deprecated: Use EchoRequest.ProtoReflect.Descriptor instead.
+func (*EchoRequest) Descriptor() ([]byte, []int) {
+	return file_examples_features_proto_echo_echo_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *EchoRequest) GetMessage() string {
-	if m != nil {
-		return m.Message
+func (x *EchoRequest) GetMessage() string {
+	if x != nil {
+		return x.Message
 	}
 	return ""
 }
 
 // EchoResponse is the response for echo.
 type EchoResponse struct {
-	Message              string   `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
 }
 
-func (m *EchoResponse) Reset()         { *m = EchoResponse{} }
-func (m *EchoResponse) String() string { return proto.CompactTextString(m) }
-func (*EchoResponse) ProtoMessage()    {}
-func (*EchoResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_2fd1d686b7b805dc, []int{1}
+func (x *EchoResponse) Reset() {
+	*x = EchoResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_features_proto_echo_echo_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *EchoResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EchoResponse.Unmarshal(m, b)
+func (x *EchoResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EchoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EchoResponse.Marshal(b, m, deterministic)
+
+func (*EchoResponse) ProtoMessage() {}
+
+func (x *EchoResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_features_proto_echo_echo_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *EchoResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EchoResponse.Merge(m, src)
+
+// Deprecated: Use EchoResponse.ProtoReflect.Descriptor instead.
+func (*EchoResponse) Descriptor() ([]byte, []int) {
+	return file_examples_features_proto_echo_echo_proto_rawDescGZIP(), []int{1}
 }
-func (m *EchoResponse) XXX_Size() int {
-	return xxx_messageInfo_EchoResponse.Size(m)
+
+func (x *EchoResponse) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
 }
-func (m *EchoResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_EchoResponse.DiscardUnknown(m)
+
+var File_examples_features_proto_echo_echo_proto protoreflect.FileDescriptor
+
+var file_examples_features_proto_echo_echo_proto_rawDesc = []byte{
+	0x0a, 0x27, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x74, 0x75,
+	0x72, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x65, 0x63, 0x68, 0x6f, 0x2f, 0x65,
+	0x63, 0x68, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x22, 0x27, 0x0a,
+	0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07,
+	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x28, 0x0a, 0x0c, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x32, 0xfb, 0x02, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x50, 0x0a, 0x09, 0x55, 0x6e, 0x61,
+	0x72, 0x79, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78,
+	0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65,
+	0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68,
+	0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x13, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x45, 0x63,
+	0x68, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c,
+	0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70,
+	0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x45, 0x63, 0x68, 0x6f,
+	0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73,
+	0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65,
+	0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x65, 0x0a, 0x1a, 0x42, 0x69, 0x64, 0x69, 0x72,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e,
+	0x67, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61,
+	0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78,
+	0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x35,
+	0x5a, 0x33, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e,
+	0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65,
+	0x73, 0x2f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2f, 0x65, 0x63, 0x68, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
-var xxx_messageInfo_EchoResponse proto.InternalMessageInfo
+var (
+	file_examples_features_proto_echo_echo_proto_rawDescOnce sync.Once
+	file_examples_features_proto_echo_echo_proto_rawDescData = file_examples_features_proto_echo_echo_proto_rawDesc
+)
 
-func (m *EchoResponse) GetMessage() string {
-	if m != nil {
-		return m.Message
-	}
-	return ""
+func file_examples_features_proto_echo_echo_proto_rawDescGZIP() []byte {
+	file_examples_features_proto_echo_echo_proto_rawDescOnce.Do(func() {
+		file_examples_features_proto_echo_echo_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_features_proto_echo_echo_proto_rawDescData)
+	})
+	return file_examples_features_proto_echo_echo_proto_rawDescData
+}
+
+var file_examples_features_proto_echo_echo_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_examples_features_proto_echo_echo_proto_goTypes = []interface{}{
+	(*EchoRequest)(nil),  // 0: grpc.examples.echo.EchoRequest
+	(*EchoResponse)(nil), // 1: grpc.examples.echo.EchoResponse
+}
+var file_examples_features_proto_echo_echo_proto_depIdxs = []int32{
+	0, // 0: grpc.examples.echo.Echo.UnaryEcho:input_type -> grpc.examples.echo.EchoRequest
+	0, // 1: grpc.examples.echo.Echo.ServerStreamingEcho:input_type -> grpc.examples.echo.EchoRequest
+	0, // 2: grpc.examples.echo.Echo.ClientStreamingEcho:input_type -> grpc.examples.echo.EchoRequest
+	0, // 3: grpc.examples.echo.Echo.BidirectionalStreamingEcho:input_type -> grpc.examples.echo.EchoRequest
+	1, // 4: grpc.examples.echo.Echo.UnaryEcho:output_type -> grpc.examples.echo.EchoResponse
+	1, // 5: grpc.examples.echo.Echo.ServerStreamingEcho:output_type -> grpc.examples.echo.EchoResponse
+	1, // 6: grpc.examples.echo.Echo.ClientStreamingEcho:output_type -> grpc.examples.echo.EchoResponse
+	1, // 7: grpc.examples.echo.Echo.BidirectionalStreamingEcho:output_type -> grpc.examples.echo.EchoResponse
+	4, // [4:8] is the sub-list for method output_type
+	0, // [0:4] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-func init() {
-	proto.RegisterType((*EchoRequest)(nil), "grpc.examples.echo.EchoRequest")
-	proto.RegisterType((*EchoResponse)(nil), "grpc.examples.echo.EchoResponse")
-}
-
-func init() {
-	proto.RegisterFile("examples/features/proto/echo/echo.proto", fileDescriptor_2fd1d686b7b805dc)
-}
-
-var fileDescriptor_2fd1d686b7b805dc = []byte{
-	// 236 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xb1, 0x4b, 0x03, 0x31,
-	0x14, 0xc6, 0x8d, 0x88, 0xd2, 0xe8, 0x14, 0x97, 0xd2, 0xc5, 0x72, 0x4b, 0x6f, 0x4a, 0x8a, 0xc5,
-	0x7f, 0xa0, 0xe2, 0x2e, 0x2d, 0x2e, 0xe2, 0x12, 0xcf, 0xcf, 0x34, 0x90, 0xcb, 0x3b, 0x5f, 0x52,
-	0xd1, 0xbf, 0xdd, 0x45, 0x72, 0x47, 0x41, 0x90, 0x3a, 0xd5, 0x25, 0xe4, 0xe3, 0xfd, 0xde, 0xf7,
-	0x5b, 0x9e, 0x9c, 0xe1, 0xc3, 0xb6, 0x5d, 0x40, 0x32, 0xaf, 0xb0, 0x79, 0xcb, 0x48, 0xa6, 0x63,
-	0xca, 0x64, 0xd0, 0x6c, 0x86, 0x47, 0xf7, 0x59, 0x29, 0xc7, 0x5d, 0xa3, 0x77, 0xb4, 0x2e, 0x93,
-	0x6a, 0x26, 0xcf, 0xef, 0x9a, 0x0d, 0xad, 0xf0, 0xb6, 0x45, 0xca, 0x6a, 0x2c, 0xcf, 0x5a, 0xa4,
-	0x64, 0x1d, 0xc6, 0x62, 0x2a, 0xea, 0xd1, 0x6a, 0x17, 0xab, 0x5a, 0x5e, 0x0c, 0x60, 0xea, 0x28,
-	0x26, 0xec, 0x27, 0xaf, 0xbf, 0x8e, 0xe5, 0x49, 0x41, 0xd5, 0xbd, 0x1c, 0x3d, 0x44, 0xcb, 0x9f,
-	0x7d, 0xb8, 0xd2, 0xbf, 0xed, 0xfa, 0x87, 0x7a, 0x32, 0xdd, 0x0f, 0x0c, 0xca, 0xea, 0x48, 0x3d,
-	0xc9, 0xcb, 0x35, 0xf8, 0x1d, 0xbc, 0xce, 0x0c, 0xdb, 0xfa, 0xe8, 0x0e, 0xd6, 0x3d, 0x17, 0xa5,
-	0xfd, 0x36, 0x78, 0xc4, 0x7c, 0xf8, 0xf6, 0x5a, 0x28, 0xc8, 0xc9, 0xd2, 0xbf, 0x78, 0x46, 0x93,
-	0x3d, 0x45, 0x1b, 0xfe, 0x43, 0x32, 0x17, 0xcb, 0x9b, 0xc7, 0x85, 0x23, 0x72, 0x01, 0xda, 0x51,
-	0xb0, 0xd1, 0x69, 0x62, 0x67, 0xca, 0xaa, 0xf9, 0xeb, 0x4c, 0x9e, 0x4f, 0xfb, 0xff, 0xe2, 0x3b,
-	0x00, 0x00, 0xff, 0xff, 0xf7, 0x79, 0x87, 0xf0, 0x4d, 0x02, 0x00, 0x00,
+func init() { file_examples_features_proto_echo_echo_proto_init() }
+func file_examples_features_proto_echo_echo_proto_init() {
+	if File_examples_features_proto_echo_echo_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_examples_features_proto_echo_echo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_examples_features_proto_echo_echo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_examples_features_proto_echo_echo_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_examples_features_proto_echo_echo_proto_goTypes,
+		DependencyIndexes: file_examples_features_proto_echo_echo_proto_depIdxs,
+		MessageInfos:      file_examples_features_proto_echo_echo_proto_msgTypes,
+	}.Build()
+	File_examples_features_proto_echo_echo_proto = out.File
+	file_examples_features_proto_echo_echo_proto_rawDesc = nil
+	file_examples_features_proto_echo_echo_proto_goTypes = nil
+	file_examples_features_proto_echo_echo_proto_depIdxs = nil
 }
diff --git a/examples/go.mod b/examples/go.mod
index d4e2b7c34a0f..9f056e4a992c 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -8,6 +8,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98
 	google.golang.org/grpc v1.31.0
+	google.golang.org/protobuf v1.25.0
 )
 
 replace google.golang.org/grpc => ../
diff --git a/examples/helloworld/helloworld/helloworld.pb.go b/examples/helloworld/helloworld/helloworld.pb.go
index 531c792a1e7c..5f9912a1f5f3 100644
--- a/examples/helloworld/helloworld/helloworld.pb.go
+++ b/examples/helloworld/helloworld/helloworld.pb.go
@@ -1,127 +1,240 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: examples/helloworld/helloworld/helloworld.proto
 
 package helloworld
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // The request message containing the user's name.
 type HelloRequest struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *HelloRequest) Reset()         { *m = HelloRequest{} }
-func (m *HelloRequest) String() string { return proto.CompactTextString(m) }
-func (*HelloRequest) ProtoMessage()    {}
-func (*HelloRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b83ea99a5323a2c7, []int{0}
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *HelloRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HelloRequest.Unmarshal(m, b)
-}
-func (m *HelloRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HelloRequest.Marshal(b, m, deterministic)
-}
-func (m *HelloRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HelloRequest.Merge(m, src)
+func (x *HelloRequest) Reset() {
+	*x = HelloRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *HelloRequest) XXX_Size() int {
-	return xxx_messageInfo_HelloRequest.Size(m)
+
+func (x *HelloRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HelloRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_HelloRequest.DiscardUnknown(m)
+
+func (*HelloRequest) ProtoMessage() {}
+
+func (x *HelloRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HelloRequest proto.InternalMessageInfo
+// Deprecated: Use HelloRequest.ProtoReflect.Descriptor instead.
+func (*HelloRequest) Descriptor() ([]byte, []int) {
+	return file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *HelloRequest) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *HelloRequest) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 // The response message containing the greetings
 type HelloReply struct {
-	Message              string   `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *HelloReply) Reset()         { *m = HelloReply{} }
-func (m *HelloReply) String() string { return proto.CompactTextString(m) }
-func (*HelloReply) ProtoMessage()    {}
-func (*HelloReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b83ea99a5323a2c7, []int{1}
+	Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
 }
 
-func (m *HelloReply) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HelloReply.Unmarshal(m, b)
-}
-func (m *HelloReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HelloReply.Marshal(b, m, deterministic)
-}
-func (m *HelloReply) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HelloReply.Merge(m, src)
+func (x *HelloReply) Reset() {
+	*x = HelloReply{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *HelloReply) XXX_Size() int {
-	return xxx_messageInfo_HelloReply.Size(m)
+
+func (x *HelloReply) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HelloReply) XXX_DiscardUnknown() {
-	xxx_messageInfo_HelloReply.DiscardUnknown(m)
+
+func (*HelloReply) ProtoMessage() {}
+
+func (x *HelloReply) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HelloReply proto.InternalMessageInfo
+// Deprecated: Use HelloReply.ProtoReflect.Descriptor instead.
+func (*HelloReply) Descriptor() ([]byte, []int) {
+	return file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *HelloReply) GetMessage() string {
-	if m != nil {
-		return m.Message
+func (x *HelloReply) GetMessage() string {
+	if x != nil {
+		return x.Message
 	}
 	return ""
 }
 
-func init() {
-	proto.RegisterType((*HelloRequest)(nil), "helloworld.HelloRequest")
-	proto.RegisterType((*HelloReply)(nil), "helloworld.HelloReply")
+var File_examples_helloworld_helloworld_helloworld_proto protoreflect.FileDescriptor
+
+var file_examples_helloworld_helloworld_helloworld_proto_rawDesc = []byte{
+	0x0a, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f,
+	0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64,
+	0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x0a, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x22, 0x22, 0x0a,
+	0x0c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a,
+	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x22, 0x26, 0x0a, 0x0a, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12,
+	0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, 0x49, 0x0a, 0x07, 0x47, 0x72, 0x65,
+	0x65, 0x74, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x08, 0x53, 0x61, 0x79, 0x48, 0x65, 0x6c, 0x6c, 0x6f,
+	0x12, 0x18, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x48, 0x65,
+	0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x68, 0x65, 0x6c,
+	0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x70,
+	0x6c, 0x79, 0x22, 0x00, 0x42, 0x67, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f,
+	0x72, 0x6c, 0x64, 0x42, 0x0f, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x35, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67,
+	0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x65,
+	0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72,
+	0x6c, 0x64, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_examples_helloworld_helloworld_helloworld_proto_rawDescOnce sync.Once
+	file_examples_helloworld_helloworld_helloworld_proto_rawDescData = file_examples_helloworld_helloworld_helloworld_proto_rawDesc
+)
+
+func file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP() []byte {
+	file_examples_helloworld_helloworld_helloworld_proto_rawDescOnce.Do(func() {
+		file_examples_helloworld_helloworld_helloworld_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_helloworld_helloworld_helloworld_proto_rawDescData)
+	})
+	return file_examples_helloworld_helloworld_helloworld_proto_rawDescData
 }
 
-func init() {
-	proto.RegisterFile("examples/helloworld/helloworld/helloworld.proto", fileDescriptor_b83ea99a5323a2c7)
+var file_examples_helloworld_helloworld_helloworld_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_examples_helloworld_helloworld_helloworld_proto_goTypes = []interface{}{
+	(*HelloRequest)(nil), // 0: helloworld.HelloRequest
+	(*HelloReply)(nil),   // 1: helloworld.HelloReply
+}
+var file_examples_helloworld_helloworld_helloworld_proto_depIdxs = []int32{
+	0, // 0: helloworld.Greeter.SayHello:input_type -> helloworld.HelloRequest
+	1, // 1: helloworld.Greeter.SayHello:output_type -> helloworld.HelloReply
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-var fileDescriptor_b83ea99a5323a2c7 = []byte{
-	// 205 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x4f, 0xad, 0x48, 0xcc,
-	0x2d, 0xc8, 0x49, 0x2d, 0xd6, 0xcf, 0x48, 0xcd, 0xc9, 0xc9, 0x2f, 0xcf, 0x2f, 0xca, 0x49, 0xc1,
-	0xce, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x42, 0x88, 0x28, 0x29, 0x71, 0xf1, 0x78,
-	0x80, 0x78, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x42, 0x42, 0x5c, 0x2c, 0x79, 0x89, 0xb9,
-	0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x60, 0xb6, 0x92, 0x1a, 0x17, 0x17, 0x54, 0x4d,
-	0x41, 0x4e, 0xa5, 0x90, 0x04, 0x17, 0x7b, 0x6e, 0x6a, 0x71, 0x71, 0x62, 0x3a, 0x4c, 0x11, 0x8c,
-	0x6b, 0xe4, 0xc9, 0xc5, 0xee, 0x5e, 0x94, 0x9a, 0x5a, 0x92, 0x5a, 0x24, 0x64, 0xc7, 0xc5, 0x11,
-	0x9c, 0x58, 0x09, 0xd6, 0x25, 0x24, 0xa1, 0x87, 0xe4, 0x02, 0x64, 0xcb, 0xa4, 0xc4, 0xb0, 0xc8,
-	0x14, 0xe4, 0x54, 0x2a, 0x31, 0x38, 0xa5, 0x73, 0x49, 0x67, 0xe6, 0xeb, 0xa5, 0x17, 0x15, 0x24,
-	0xeb, 0xc1, 0x7c, 0x87, 0xa4, 0xd6, 0x89, 0x1f, 0xac, 0x38, 0x1c, 0xc4, 0x0e, 0x00, 0x79, 0x29,
-	0x80, 0x31, 0xca, 0x34, 0x3d, 0x3f, 0x3f, 0x3d, 0x27, 0x55, 0x2f, 0x3d, 0x3f, 0x27, 0x31, 0x2f,
-	0x5d, 0x2f, 0xbf, 0x28, 0x5d, 0x1f, 0xa4, 0x9d, 0x40, 0xe0, 0x24, 0xb1, 0x81, 0x83, 0xc4, 0x18,
-	0x10, 0x00, 0x00, 0xff, 0xff, 0x46, 0xfe, 0x45, 0x5c, 0x45, 0x01, 0x00, 0x00,
+func init() { file_examples_helloworld_helloworld_helloworld_proto_init() }
+func file_examples_helloworld_helloworld_helloworld_proto_init() {
+	if File_examples_helloworld_helloworld_helloworld_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HelloRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HelloReply); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_examples_helloworld_helloworld_helloworld_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_examples_helloworld_helloworld_helloworld_proto_goTypes,
+		DependencyIndexes: file_examples_helloworld_helloworld_helloworld_proto_depIdxs,
+		MessageInfos:      file_examples_helloworld_helloworld_helloworld_proto_msgTypes,
+	}.Build()
+	File_examples_helloworld_helloworld_helloworld_proto = out.File
+	file_examples_helloworld_helloworld_helloworld_proto_rawDesc = nil
+	file_examples_helloworld_helloworld_helloworld_proto_goTypes = nil
+	file_examples_helloworld_helloworld_helloworld_proto_depIdxs = nil
 }
diff --git a/examples/route_guide/client/client.go b/examples/route_guide/client/client.go
index 3e9d4e1183cc..172f10fb308b 100644
--- a/examples/route_guide/client/client.go
+++ b/examples/route_guide/client/client.go
@@ -72,7 +72,8 @@ func printFeatures(client pb.RouteGuideClient, rect *pb.Rectangle) {
 		if err != nil {
 			log.Fatalf("%v.ListFeatures(_) = _, %v", client, err)
 		}
-		log.Println(feature)
+		log.Printf("Feature: name: %q, point:(%v, %v)", feature.GetName(),
+			feature.GetLocation().GetLatitude(), feature.GetLocation().GetLongitude())
 	}
 }
 
diff --git a/examples/route_guide/routeguide/route_guide.pb.go b/examples/route_guide/routeguide/route_guide.pb.go
index 78c856645a7b..f99a1dacda74 100644
--- a/examples/route_guide/routeguide/route_guide.pb.go
+++ b/examples/route_guide/routeguide/route_guide.pb.go
@@ -1,72 +1,99 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: examples/route_guide/routeguide/route_guide.proto
 
 package routeguide
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Points are represented as latitude-longitude pairs in the E7 representation
 // (degrees multiplied by 10**7 and rounded to the nearest integer).
 // Latitudes should be in the range +/- 90 degrees and longitude should be in
 // the range +/- 180 degrees (inclusive).
 type Point struct {
-	Latitude             int32    `protobuf:"varint,1,opt,name=latitude,proto3" json:"latitude,omitempty"`
-	Longitude            int32    `protobuf:"varint,2,opt,name=longitude,proto3" json:"longitude,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *Point) Reset()         { *m = Point{} }
-func (m *Point) String() string { return proto.CompactTextString(m) }
-func (*Point) ProtoMessage()    {}
-func (*Point) Descriptor() ([]byte, []int) {
-	return fileDescriptor_af806a20656386f8, []int{0}
+	Latitude  int32 `protobuf:"varint,1,opt,name=latitude,proto3" json:"latitude,omitempty"`
+	Longitude int32 `protobuf:"varint,2,opt,name=longitude,proto3" json:"longitude,omitempty"`
 }
 
-func (m *Point) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Point.Unmarshal(m, b)
-}
-func (m *Point) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Point.Marshal(b, m, deterministic)
-}
-func (m *Point) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Point.Merge(m, src)
+func (x *Point) Reset() {
+	*x = Point{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Point) XXX_Size() int {
-	return xxx_messageInfo_Point.Size(m)
+
+func (x *Point) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Point) XXX_DiscardUnknown() {
-	xxx_messageInfo_Point.DiscardUnknown(m)
+
+func (*Point) ProtoMessage() {}
+
+func (x *Point) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Point proto.InternalMessageInfo
+// Deprecated: Use Point.ProtoReflect.Descriptor instead.
+func (*Point) Descriptor() ([]byte, []int) {
+	return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *Point) GetLatitude() int32 {
-	if m != nil {
-		return m.Latitude
+func (x *Point) GetLatitude() int32 {
+	if x != nil {
+		return x.Latitude
 	}
 	return 0
 }
 
-func (m *Point) GetLongitude() int32 {
-	if m != nil {
-		return m.Longitude
+func (x *Point) GetLongitude() int32 {
+	if x != nil {
+		return x.Longitude
 	}
 	return 0
 }
@@ -74,50 +101,58 @@ func (m *Point) GetLongitude() int32 {
 // A latitude-longitude rectangle, represented as two diagonally opposite
 // points "lo" and "hi".
 type Rectangle struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// One corner of the rectangle.
 	Lo *Point `protobuf:"bytes,1,opt,name=lo,proto3" json:"lo,omitempty"`
 	// The other corner of the rectangle.
-	Hi                   *Point   `protobuf:"bytes,2,opt,name=hi,proto3" json:"hi,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Hi *Point `protobuf:"bytes,2,opt,name=hi,proto3" json:"hi,omitempty"`
 }
 
-func (m *Rectangle) Reset()         { *m = Rectangle{} }
-func (m *Rectangle) String() string { return proto.CompactTextString(m) }
-func (*Rectangle) ProtoMessage()    {}
-func (*Rectangle) Descriptor() ([]byte, []int) {
-	return fileDescriptor_af806a20656386f8, []int{1}
+func (x *Rectangle) Reset() {
+	*x = Rectangle{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Rectangle) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Rectangle.Unmarshal(m, b)
-}
-func (m *Rectangle) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Rectangle.Marshal(b, m, deterministic)
+func (x *Rectangle) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Rectangle) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Rectangle.Merge(m, src)
-}
-func (m *Rectangle) XXX_Size() int {
-	return xxx_messageInfo_Rectangle.Size(m)
-}
-func (m *Rectangle) XXX_DiscardUnknown() {
-	xxx_messageInfo_Rectangle.DiscardUnknown(m)
+
+func (*Rectangle) ProtoMessage() {}
+
+func (x *Rectangle) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Rectangle proto.InternalMessageInfo
+// Deprecated: Use Rectangle.ProtoReflect.Descriptor instead.
+func (*Rectangle) Descriptor() ([]byte, []int) {
+	return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *Rectangle) GetLo() *Point {
-	if m != nil {
-		return m.Lo
+func (x *Rectangle) GetLo() *Point {
+	if x != nil {
+		return x.Lo
 	}
 	return nil
 }
 
-func (m *Rectangle) GetHi() *Point {
-	if m != nil {
-		return m.Hi
+func (x *Rectangle) GetHi() *Point {
+	if x != nil {
+		return x.Hi
 	}
 	return nil
 }
@@ -126,100 +161,116 @@ func (m *Rectangle) GetHi() *Point {
 //
 // If a feature could not be named, the name is empty.
 type Feature struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The name of the feature.
 	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	// The point where the feature is detected.
-	Location             *Point   `protobuf:"bytes,2,opt,name=location,proto3" json:"location,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Location *Point `protobuf:"bytes,2,opt,name=location,proto3" json:"location,omitempty"`
 }
 
-func (m *Feature) Reset()         { *m = Feature{} }
-func (m *Feature) String() string { return proto.CompactTextString(m) }
-func (*Feature) ProtoMessage()    {}
-func (*Feature) Descriptor() ([]byte, []int) {
-	return fileDescriptor_af806a20656386f8, []int{2}
+func (x *Feature) Reset() {
+	*x = Feature{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Feature) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Feature.Unmarshal(m, b)
-}
-func (m *Feature) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Feature.Marshal(b, m, deterministic)
-}
-func (m *Feature) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Feature.Merge(m, src)
+func (x *Feature) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Feature) XXX_Size() int {
-	return xxx_messageInfo_Feature.Size(m)
-}
-func (m *Feature) XXX_DiscardUnknown() {
-	xxx_messageInfo_Feature.DiscardUnknown(m)
+
+func (*Feature) ProtoMessage() {}
+
+func (x *Feature) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Feature proto.InternalMessageInfo
+// Deprecated: Use Feature.ProtoReflect.Descriptor instead.
+func (*Feature) Descriptor() ([]byte, []int) {
+	return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *Feature) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *Feature) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
-func (m *Feature) GetLocation() *Point {
-	if m != nil {
-		return m.Location
+func (x *Feature) GetLocation() *Point {
+	if x != nil {
+		return x.Location
 	}
 	return nil
 }
 
 // A RouteNote is a message sent while at a given point.
 type RouteNote struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The location from which the message is sent.
 	Location *Point `protobuf:"bytes,1,opt,name=location,proto3" json:"location,omitempty"`
 	// The message to be sent.
-	Message              string   `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
 }
 
-func (m *RouteNote) Reset()         { *m = RouteNote{} }
-func (m *RouteNote) String() string { return proto.CompactTextString(m) }
-func (*RouteNote) ProtoMessage()    {}
-func (*RouteNote) Descriptor() ([]byte, []int) {
-	return fileDescriptor_af806a20656386f8, []int{3}
+func (x *RouteNote) Reset() {
+	*x = RouteNote{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RouteNote) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RouteNote.Unmarshal(m, b)
-}
-func (m *RouteNote) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RouteNote.Marshal(b, m, deterministic)
-}
-func (m *RouteNote) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RouteNote.Merge(m, src)
-}
-func (m *RouteNote) XXX_Size() int {
-	return xxx_messageInfo_RouteNote.Size(m)
+func (x *RouteNote) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RouteNote) XXX_DiscardUnknown() {
-	xxx_messageInfo_RouteNote.DiscardUnknown(m)
+
+func (*RouteNote) ProtoMessage() {}
+
+func (x *RouteNote) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RouteNote proto.InternalMessageInfo
+// Deprecated: Use RouteNote.ProtoReflect.Descriptor instead.
+func (*RouteNote) Descriptor() ([]byte, []int) {
+	return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *RouteNote) GetLocation() *Point {
-	if m != nil {
-		return m.Location
+func (x *RouteNote) GetLocation() *Point {
+	if x != nil {
+		return x.Location
 	}
 	return nil
 }
 
-func (m *RouteNote) GetMessage() string {
-	if m != nil {
-		return m.Message
+func (x *RouteNote) GetMessage() string {
+	if x != nil {
+		return x.Message
 	}
 	return ""
 }
@@ -230,6 +281,10 @@ func (m *RouteNote) GetMessage() string {
 // detected features, and the total distance covered as the cumulative sum of
 // the distance between each point.
 type RouteSummary struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The number of points received.
 	PointCount int32 `protobuf:"varint,1,opt,name=point_count,json=pointCount,proto3" json:"point_count,omitempty"`
 	// The number of known features passed while traversing the route.
@@ -237,105 +292,254 @@ type RouteSummary struct {
 	// The distance covered in metres.
 	Distance int32 `protobuf:"varint,3,opt,name=distance,proto3" json:"distance,omitempty"`
 	// The duration of the traversal in seconds.
-	ElapsedTime          int32    `protobuf:"varint,4,opt,name=elapsed_time,json=elapsedTime,proto3" json:"elapsed_time,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	ElapsedTime int32 `protobuf:"varint,4,opt,name=elapsed_time,json=elapsedTime,proto3" json:"elapsed_time,omitempty"`
 }
 
-func (m *RouteSummary) Reset()         { *m = RouteSummary{} }
-func (m *RouteSummary) String() string { return proto.CompactTextString(m) }
-func (*RouteSummary) ProtoMessage()    {}
-func (*RouteSummary) Descriptor() ([]byte, []int) {
-	return fileDescriptor_af806a20656386f8, []int{4}
+func (x *RouteSummary) Reset() {
+	*x = RouteSummary{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RouteSummary) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RouteSummary.Unmarshal(m, b)
+func (x *RouteSummary) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RouteSummary) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RouteSummary.Marshal(b, m, deterministic)
-}
-func (m *RouteSummary) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RouteSummary.Merge(m, src)
-}
-func (m *RouteSummary) XXX_Size() int {
-	return xxx_messageInfo_RouteSummary.Size(m)
-}
-func (m *RouteSummary) XXX_DiscardUnknown() {
-	xxx_messageInfo_RouteSummary.DiscardUnknown(m)
+
+func (*RouteSummary) ProtoMessage() {}
+
+func (x *RouteSummary) ProtoReflect() protoreflect.Message {
+	mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_RouteSummary proto.InternalMessageInfo
+// Deprecated: Use RouteSummary.ProtoReflect.Descriptor instead.
+func (*RouteSummary) Descriptor() ([]byte, []int) {
+	return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *RouteSummary) GetPointCount() int32 {
-	if m != nil {
-		return m.PointCount
+func (x *RouteSummary) GetPointCount() int32 {
+	if x != nil {
+		return x.PointCount
 	}
 	return 0
 }
 
-func (m *RouteSummary) GetFeatureCount() int32 {
-	if m != nil {
-		return m.FeatureCount
+func (x *RouteSummary) GetFeatureCount() int32 {
+	if x != nil {
+		return x.FeatureCount
 	}
 	return 0
 }
 
-func (m *RouteSummary) GetDistance() int32 {
-	if m != nil {
-		return m.Distance
+func (x *RouteSummary) GetDistance() int32 {
+	if x != nil {
+		return x.Distance
 	}
 	return 0
 }
 
-func (m *RouteSummary) GetElapsedTime() int32 {
-	if m != nil {
-		return m.ElapsedTime
+func (x *RouteSummary) GetElapsedTime() int32 {
+	if x != nil {
+		return x.ElapsedTime
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterType((*Point)(nil), "routeguide.Point")
-	proto.RegisterType((*Rectangle)(nil), "routeguide.Rectangle")
-	proto.RegisterType((*Feature)(nil), "routeguide.Feature")
-	proto.RegisterType((*RouteNote)(nil), "routeguide.RouteNote")
-	proto.RegisterType((*RouteSummary)(nil), "routeguide.RouteSummary")
-}
-
-func init() {
-	proto.RegisterFile("examples/route_guide/routeguide/route_guide.proto", fileDescriptor_af806a20656386f8)
-}
-
-var fileDescriptor_af806a20656386f8 = []byte{
-	// 434 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x53, 0xcd, 0x6e, 0xd3, 0x40,
-	0x10, 0xae, 0x43, 0x4b, 0xe3, 0x49, 0x10, 0x62, 0x10, 0x92, 0x15, 0x90, 0xa0, 0xe6, 0xd2, 0x0b,
-	0x4e, 0x29, 0x52, 0x8f, 0x45, 0xb4, 0x12, 0xbd, 0x54, 0x28, 0x98, 0x9e, 0xb8, 0x44, 0x8b, 0x3d,
-	0x6c, 0x56, 0x5a, 0x7b, 0x2c, 0x7b, 0x2d, 0xc1, 0x03, 0xf0, 0x04, 0xbc, 0x30, 0xda, 0x5d, 0x3b,
-	0x76, 0x69, 0xaa, 0xde, 0x66, 0xbe, 0xf9, 0xbe, 0xf9, 0xd5, 0xc0, 0x7b, 0xfa, 0x25, 0x8a, 0x4a,
-	0x53, 0xb3, 0xac, 0xb9, 0x35, 0xb4, 0x96, 0xad, 0xca, 0xc9, 0xdb, 0x23, 0xd3, 0xc3, 0x49, 0x55,
-	0xb3, 0x61, 0x84, 0x21, 0x1a, 0x7f, 0x82, 0x83, 0x15, 0xab, 0xd2, 0xe0, 0x02, 0xa6, 0x5a, 0x18,
-	0x65, 0xda, 0x9c, 0xa2, 0xe0, 0x4d, 0x70, 0x7c, 0x90, 0x6e, 0x7d, 0x7c, 0x05, 0xa1, 0xe6, 0x52,
-	0xfa, 0xe0, 0xc4, 0x05, 0x07, 0x20, 0xfe, 0x0a, 0x61, 0x4a, 0x99, 0x11, 0xa5, 0xd4, 0x84, 0x47,
-	0x30, 0xd1, 0xec, 0x12, 0xcc, 0x4e, 0x9f, 0x25, 0x43, 0xa1, 0xc4, 0x55, 0x49, 0x27, 0x9a, 0x2d,
-	0x65, 0xa3, 0x5c, 0x9a, 0xdd, 0x94, 0x8d, 0x8a, 0xaf, 0xe1, 0xf0, 0x33, 0x09, 0xd3, 0xd6, 0x84,
-	0x08, 0xfb, 0xa5, 0x28, 0x7c, 0x4f, 0x61, 0xea, 0x6c, 0x7c, 0x07, 0x53, 0xcd, 0x99, 0x30, 0x8a,
-	0xcb, 0xfb, 0xf3, 0x6c, 0x29, 0xf1, 0x0d, 0x84, 0xa9, 0x8d, 0x7e, 0x61, 0x73, 0x5b, 0x1b, 0x3c,
-	0xa8, 0xc5, 0x08, 0x0e, 0x0b, 0x6a, 0x1a, 0x21, 0xfd, 0xe0, 0x61, 0xda, 0xbb, 0xf1, 0xdf, 0x00,
-	0xe6, 0x2e, 0xed, 0xb7, 0xb6, 0x28, 0x44, 0xfd, 0x1b, 0x5f, 0xc3, 0xac, 0xb2, 0xea, 0x75, 0xc6,
-	0x6d, 0x69, 0xba, 0x25, 0x82, 0x83, 0x2e, 0x2d, 0x82, 0x6f, 0xe1, 0xc9, 0x4f, 0x3f, 0x55, 0x47,
-	0xf1, 0xab, 0x9c, 0x77, 0xa0, 0x27, 0x2d, 0x60, 0x9a, 0xab, 0xc6, 0x88, 0x32, 0xa3, 0xe8, 0x91,
-	0xbf, 0x43, 0xef, 0xe3, 0x11, 0xcc, 0x49, 0x8b, 0xaa, 0xa1, 0x7c, 0x6d, 0x54, 0x41, 0xd1, 0xbe,
-	0x8b, 0xcf, 0x3a, 0xec, 0x46, 0x15, 0x74, 0xfa, 0x67, 0x02, 0xe0, 0xba, 0xba, 0xb2, 0xe3, 0xe0,
-	0x19, 0xc0, 0x15, 0x99, 0x7e, 0x97, 0x77, 0x27, 0x5d, 0x3c, 0x1f, 0x43, 0x1d, 0x2f, 0xde, 0xc3,
-	0x73, 0x98, 0x5f, 0xab, 0xa6, 0x17, 0x36, 0xf8, 0x62, 0x4c, 0xdb, 0x5e, 0xfb, 0x1e, 0xf5, 0x49,
-	0x80, 0xe7, 0x30, 0x4b, 0x29, 0xe3, 0x3a, 0x77, 0xbd, 0xec, 0x2a, 0x1c, 0xdd, 0xca, 0x38, 0xda,
-	0x63, 0xbc, 0x77, 0x1c, 0xe0, 0xc7, 0xee, 0x64, 0x97, 0x1b, 0x61, 0xfe, 0x2b, 0xde, 0x5f, 0x72,
-	0xb1, 0x1b, 0xb6, 0xf2, 0x93, 0xe0, 0x62, 0x03, 0x2f, 0x15, 0x27, 0xb2, 0xae, 0xb2, 0xa4, 0x7f,
-	0x90, 0x11, 0xfd, 0xe2, 0xe9, 0xb0, 0xa3, 0x95, 0xfd, 0x89, 0x55, 0xf0, 0xfd, 0x4c, 0x32, 0x4b,
-	0x4d, 0x89, 0x64, 0x2d, 0x4a, 0x99, 0x70, 0x2d, 0x97, 0x56, 0xbe, 0x7c, 0xe0, 0xbf, 0x7e, 0x3c,
-	0x76, 0x4f, 0xf5, 0xe1, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x28, 0xef, 0x54, 0xdd, 0x89, 0x03,
-	0x00, 0x00,
+var File_examples_route_guide_routeguide_route_guide_proto protoreflect.FileDescriptor
+
+var file_examples_route_guide_routeguide_route_guide_proto_rawDesc = []byte{
+	0x0a, 0x31, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65,
+	0x5f, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64,
+	0x65, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x22,
+	0x41, 0x0a, 0x05, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x74, 0x69,
+	0x74, 0x75, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x6c, 0x61, 0x74, 0x69,
+	0x74, 0x75, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75,
+	0x64, 0x65, 0x22, 0x51, 0x0a, 0x09, 0x52, 0x65, 0x63, 0x74, 0x61, 0x6e, 0x67, 0x6c, 0x65, 0x12,
+	0x21, 0x0a, 0x02, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x6f,
+	0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x02,
+	0x6c, 0x6f, 0x12, 0x21, 0x0a, 0x02, 0x68, 0x69, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11,
+	0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e,
+	0x74, 0x52, 0x02, 0x68, 0x69, 0x22, 0x4c, 0x0a, 0x07, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65,
+	0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
+	0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75,
+	0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x22, 0x54, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4e, 0x6f, 0x74, 0x65,
+	0x12, 0x2d, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e,
+	0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12,
+	0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x0c, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x6f,
+	0x69, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x0a, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x66,
+	0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74,
+	0x12, 0x1a, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c,
+	0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x0b, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x32,
+	0x85, 0x02, 0x0a, 0x0a, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x47, 0x75, 0x69, 0x64, 0x65, 0x12, 0x36,
+	0x0a, 0x0a, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x11, 0x2e, 0x72,
+	0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x1a,
+	0x13, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x46, 0x65, 0x61,
+	0x74, 0x75, 0x72, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65,
+	0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x15, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75,
+	0x69, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x74, 0x61, 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x13, 0x2e,
+	0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75,
+	0x72, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x0b, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
+	0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69,
+	0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65,
+	0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61,
+	0x72, 0x79, 0x22, 0x00, 0x28, 0x01, 0x12, 0x3f, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x43,
+	0x68, 0x61, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65,
+	0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4e, 0x6f, 0x74, 0x65, 0x1a, 0x15, 0x2e, 0x72, 0x6f, 0x75,
+	0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4e, 0x6f, 0x74,
+	0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x68, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x72, 0x6f, 0x75, 0x74,
+	0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x42, 0x0f, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x47, 0x75, 0x69,
+	0x64, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x36, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65,
+	0x5f, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64,
+	0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_examples_route_guide_routeguide_route_guide_proto_rawDescOnce sync.Once
+	file_examples_route_guide_routeguide_route_guide_proto_rawDescData = file_examples_route_guide_routeguide_route_guide_proto_rawDesc
+)
+
+func file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP() []byte {
+	file_examples_route_guide_routeguide_route_guide_proto_rawDescOnce.Do(func() {
+		file_examples_route_guide_routeguide_route_guide_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_route_guide_routeguide_route_guide_proto_rawDescData)
+	})
+	return file_examples_route_guide_routeguide_route_guide_proto_rawDescData
+}
+
+var file_examples_route_guide_routeguide_route_guide_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_examples_route_guide_routeguide_route_guide_proto_goTypes = []interface{}{
+	(*Point)(nil),        // 0: routeguide.Point
+	(*Rectangle)(nil),    // 1: routeguide.Rectangle
+	(*Feature)(nil),      // 2: routeguide.Feature
+	(*RouteNote)(nil),    // 3: routeguide.RouteNote
+	(*RouteSummary)(nil), // 4: routeguide.RouteSummary
+}
+var file_examples_route_guide_routeguide_route_guide_proto_depIdxs = []int32{
+	0, // 0: routeguide.Rectangle.lo:type_name -> routeguide.Point
+	0, // 1: routeguide.Rectangle.hi:type_name -> routeguide.Point
+	0, // 2: routeguide.Feature.location:type_name -> routeguide.Point
+	0, // 3: routeguide.RouteNote.location:type_name -> routeguide.Point
+	0, // 4: routeguide.RouteGuide.GetFeature:input_type -> routeguide.Point
+	1, // 5: routeguide.RouteGuide.ListFeatures:input_type -> routeguide.Rectangle
+	0, // 6: routeguide.RouteGuide.RecordRoute:input_type -> routeguide.Point
+	3, // 7: routeguide.RouteGuide.RouteChat:input_type -> routeguide.RouteNote
+	2, // 8: routeguide.RouteGuide.GetFeature:output_type -> routeguide.Feature
+	2, // 9: routeguide.RouteGuide.ListFeatures:output_type -> routeguide.Feature
+	4, // 10: routeguide.RouteGuide.RecordRoute:output_type -> routeguide.RouteSummary
+	3, // 11: routeguide.RouteGuide.RouteChat:output_type -> routeguide.RouteNote
+	8, // [8:12] is the sub-list for method output_type
+	4, // [4:8] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
+}
+
+func init() { file_examples_route_guide_routeguide_route_guide_proto_init() }
+func file_examples_route_guide_routeguide_route_guide_proto_init() {
+	if File_examples_route_guide_routeguide_route_guide_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Point); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Rectangle); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Feature); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouteNote); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouteSummary); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_examples_route_guide_routeguide_route_guide_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   5,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_examples_route_guide_routeguide_route_guide_proto_goTypes,
+		DependencyIndexes: file_examples_route_guide_routeguide_route_guide_proto_depIdxs,
+		MessageInfos:      file_examples_route_guide_routeguide_route_guide_proto_msgTypes,
+	}.Build()
+	File_examples_route_guide_routeguide_route_guide_proto = out.File
+	file_examples_route_guide_routeguide_route_guide_proto_rawDesc = nil
+	file_examples_route_guide_routeguide_route_guide_proto_goTypes = nil
+	file_examples_route_guide_routeguide_route_guide_proto_depIdxs = nil
 }
diff --git a/go.mod b/go.mod
index f4a728fa21b1..6feb697bbdab 100644
--- a/go.mod
+++ b/go.mod
@@ -7,10 +7,11 @@ require (
 	github.com/envoyproxy/go-control-plane v0.9.7
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.4.2
-	github.com/google/go-cmp v0.4.0
+	github.com/google/go-cmp v0.5.0
 	github.com/google/uuid v1.1.2
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
-	google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
+	google.golang.org/protobuf v1.25.0
 )
diff --git a/go.sum b/go.sum
index 215a4599a965..293f5490353f 100644
--- a/go.sum
+++ b/go.sum
@@ -17,24 +17,23 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrp
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -45,7 +44,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -65,7 +63,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -73,8 +70,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
@@ -84,10 +82,12 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go
index e9919c0073b6..2b6803ecc72a 100644
--- a/health/grpc_health_v1/health.pb.go
+++ b/health/grpc_health_v1/health.pb.go
@@ -1,24 +1,46 @@
+// Copyright 2015 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The canonical version of this proto can be found at
+// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/health/v1/health.proto
 
 package grpc_health_v1
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type HealthCheckResponse_ServingStatus int32
 
@@ -26,136 +48,266 @@ const (
 	HealthCheckResponse_UNKNOWN         HealthCheckResponse_ServingStatus = 0
 	HealthCheckResponse_SERVING         HealthCheckResponse_ServingStatus = 1
 	HealthCheckResponse_NOT_SERVING     HealthCheckResponse_ServingStatus = 2
-	HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3
+	HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3 // Used only by the Watch method.
 )
 
-var HealthCheckResponse_ServingStatus_name = map[int32]string{
-	0: "UNKNOWN",
-	1: "SERVING",
-	2: "NOT_SERVING",
-	3: "SERVICE_UNKNOWN",
-}
+// Enum value maps for HealthCheckResponse_ServingStatus.
+var (
+	HealthCheckResponse_ServingStatus_name = map[int32]string{
+		0: "UNKNOWN",
+		1: "SERVING",
+		2: "NOT_SERVING",
+		3: "SERVICE_UNKNOWN",
+	}
+	HealthCheckResponse_ServingStatus_value = map[string]int32{
+		"UNKNOWN":         0,
+		"SERVING":         1,
+		"NOT_SERVING":     2,
+		"SERVICE_UNKNOWN": 3,
+	}
+)
 
-var HealthCheckResponse_ServingStatus_value = map[string]int32{
-	"UNKNOWN":         0,
-	"SERVING":         1,
-	"NOT_SERVING":     2,
-	"SERVICE_UNKNOWN": 3,
+func (x HealthCheckResponse_ServingStatus) Enum() *HealthCheckResponse_ServingStatus {
+	p := new(HealthCheckResponse_ServingStatus)
+	*p = x
+	return p
 }
 
 func (x HealthCheckResponse_ServingStatus) String() string {
-	return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
-func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_e265fd9d4e077217, []int{1, 0}
+func (HealthCheckResponse_ServingStatus) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_health_v1_health_proto_enumTypes[0].Descriptor()
 }
 
-type HealthCheckRequest struct {
-	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (HealthCheckResponse_ServingStatus) Type() protoreflect.EnumType {
+	return &file_grpc_health_v1_health_proto_enumTypes[0]
 }
 
-func (m *HealthCheckRequest) Reset()         { *m = HealthCheckRequest{} }
-func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) }
-func (*HealthCheckRequest) ProtoMessage()    {}
-func (*HealthCheckRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e265fd9d4e077217, []int{0}
+func (x HealthCheckResponse_ServingStatus) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
-func (m *HealthCheckRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HealthCheckRequest.Unmarshal(m, b)
+// Deprecated: Use HealthCheckResponse_ServingStatus.Descriptor instead.
+func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{1, 0}
 }
-func (m *HealthCheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HealthCheckRequest.Marshal(b, m, deterministic)
+
+type HealthCheckRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
 }
-func (m *HealthCheckRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HealthCheckRequest.Merge(m, src)
+
+func (x *HealthCheckRequest) Reset() {
+	*x = HealthCheckRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_health_v1_health_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *HealthCheckRequest) XXX_Size() int {
-	return xxx_messageInfo_HealthCheckRequest.Size(m)
+
+func (x *HealthCheckRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HealthCheckRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_HealthCheckRequest.DiscardUnknown(m)
+
+func (*HealthCheckRequest) ProtoMessage() {}
+
+func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_health_v1_health_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_HealthCheckRequest proto.InternalMessageInfo
+// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead.
+func (*HealthCheckRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *HealthCheckRequest) GetService() string {
-	if m != nil {
-		return m.Service
+func (x *HealthCheckRequest) GetService() string {
+	if x != nil {
+		return x.Service
 	}
 	return ""
 }
 
 type HealthCheckResponse struct {
-	Status               HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                          `json:"-"`
-	XXX_unrecognized     []byte                            `json:"-"`
-	XXX_sizecache        int32                             `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"`
 }
 
-func (m *HealthCheckResponse) Reset()         { *m = HealthCheckResponse{} }
-func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) }
-func (*HealthCheckResponse) ProtoMessage()    {}
-func (*HealthCheckResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e265fd9d4e077217, []int{1}
+func (x *HealthCheckResponse) Reset() {
+	*x = HealthCheckResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_health_v1_health_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *HealthCheckResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HealthCheckResponse.Unmarshal(m, b)
+func (x *HealthCheckResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *HealthCheckResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HealthCheckResponse.Marshal(b, m, deterministic)
+
+func (*HealthCheckResponse) ProtoMessage() {}
+
+func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_health_v1_health_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *HealthCheckResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HealthCheckResponse.Merge(m, src)
+
+// Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead.
+func (*HealthCheckResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{1}
 }
-func (m *HealthCheckResponse) XXX_Size() int {
-	return xxx_messageInfo_HealthCheckResponse.Size(m)
+
+func (x *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus {
+	if x != nil {
+		return x.Status
+	}
+	return HealthCheckResponse_UNKNOWN
 }
-func (m *HealthCheckResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_HealthCheckResponse.DiscardUnknown(m)
+
+var File_grpc_health_v1_health_proto protoreflect.FileDescriptor
+
+var file_grpc_health_v1_health_proto_rawDesc = []byte{
+	0x0a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31,
+	0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x22, 0x2e, 0x0a,
+	0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0xb1, 0x01,
+	0x0a, 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61,
+	0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65,
+	0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69,
+	0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73,
+	0x22, 0x4f, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b,
+	0x0a, 0x07, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4e,
+	0x4f, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f,
+	0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10,
+	0x03, 0x32, 0xae, 0x01, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x50, 0x0a, 0x05,
+	0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61,
+	0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65,
+	0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74,
+	0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52,
+	0x0a, 0x05, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68,
+	0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43,
+	0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61,
+	0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x30, 0x01, 0x42, 0x61, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65,
+	0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67,
+	0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68,
+	0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74,
+	0x68, 0x5f, 0x76, 0x31, 0xaa, 0x02, 0x0e, 0x47, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x65, 0x61, 0x6c,
+	0x74, 0x68, 0x2e, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
-var xxx_messageInfo_HealthCheckResponse proto.InternalMessageInfo
+var (
+	file_grpc_health_v1_health_proto_rawDescOnce sync.Once
+	file_grpc_health_v1_health_proto_rawDescData = file_grpc_health_v1_health_proto_rawDesc
+)
 
-func (m *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus {
-	if m != nil {
-		return m.Status
-	}
-	return HealthCheckResponse_UNKNOWN
+func file_grpc_health_v1_health_proto_rawDescGZIP() []byte {
+	file_grpc_health_v1_health_proto_rawDescOnce.Do(func() {
+		file_grpc_health_v1_health_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_health_v1_health_proto_rawDescData)
+	})
+	return file_grpc_health_v1_health_proto_rawDescData
 }
 
-func init() {
-	proto.RegisterEnum("grpc.health.v1.HealthCheckResponse_ServingStatus", HealthCheckResponse_ServingStatus_name, HealthCheckResponse_ServingStatus_value)
-	proto.RegisterType((*HealthCheckRequest)(nil), "grpc.health.v1.HealthCheckRequest")
-	proto.RegisterType((*HealthCheckResponse)(nil), "grpc.health.v1.HealthCheckResponse")
-}
-
-func init() { proto.RegisterFile("grpc/health/v1/health.proto", fileDescriptor_e265fd9d4e077217) }
-
-var fileDescriptor_e265fd9d4e077217 = []byte{
-	// 297 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4e, 0x2f, 0x2a, 0x48,
-	0xd6, 0xcf, 0x48, 0x4d, 0xcc, 0x29, 0xc9, 0xd0, 0x2f, 0x33, 0x84, 0xb2, 0xf4, 0x0a, 0x8a, 0xf2,
-	0x4b, 0xf2, 0x85, 0xf8, 0x40, 0x92, 0x7a, 0x50, 0xa1, 0x32, 0x43, 0x25, 0x3d, 0x2e, 0x21, 0x0f,
-	0x30, 0xc7, 0x39, 0x23, 0x35, 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x82,
-	0x8b, 0xbd, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08,
-	0xc6, 0x55, 0xda, 0xc8, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8,
-	0x93, 0x8b, 0xad, 0xb8, 0x24, 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x50, 0x0f, 0xd5,
-	0x22, 0x3d, 0x2c, 0x9a, 0xf4, 0x82, 0x41, 0x86, 0xe6, 0xa5, 0x07, 0x83, 0x35, 0x06, 0x41, 0x0d,
-	0x50, 0xf2, 0xe7, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, 0x0f,
-	0xf7, 0x13, 0x60, 0x00, 0x71, 0x82, 0x5d, 0x83, 0xc2, 0x3c, 0xfd, 0xdc, 0x05, 0x18, 0x85, 0xf8,
-	0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x42, 0xc2, 0x5c, 0xfc, 0x60, 0x8e, 0xb3,
-	0x6b, 0x3c, 0x4c, 0x0b, 0xb3, 0xd1, 0x3a, 0x46, 0x2e, 0x36, 0x88, 0xf5, 0x42, 0x01, 0x5c, 0xac,
-	0x60, 0x27, 0x08, 0x29, 0xe1, 0x75, 0x1f, 0x38, 0x14, 0xa4, 0x94, 0x89, 0xf0, 0x83, 0x50, 0x10,
-	0x17, 0x6b, 0x78, 0x62, 0x49, 0x72, 0x06, 0xd5, 0x4c, 0x34, 0x60, 0x74, 0x4a, 0xe4, 0x12, 0xcc,
-	0xcc, 0x47, 0x53, 0xea, 0xc4, 0x0d, 0x51, 0x1b, 0x00, 0x8a, 0xc6, 0x00, 0xc6, 0x28, 0x9d, 0xf4,
-	0xfc, 0xfc, 0xf4, 0x9c, 0x54, 0xbd, 0xf4, 0xfc, 0x9c, 0xc4, 0xbc, 0x74, 0xbd, 0xfc, 0xa2, 0x74,
-	0x7d, 0xe4, 0x78, 0x07, 0xb1, 0xe3, 0x21, 0xec, 0xf8, 0x32, 0xc3, 0x55, 0x4c, 0x7c, 0xee, 0x20,
-	0xd3, 0x20, 0x46, 0xe8, 0x85, 0x19, 0x26, 0xb1, 0x81, 0x93, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff,
-	0xff, 0x12, 0x7d, 0x96, 0xcb, 0x2d, 0x02, 0x00, 0x00,
+var file_grpc_health_v1_health_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_grpc_health_v1_health_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_grpc_health_v1_health_proto_goTypes = []interface{}{
+	(HealthCheckResponse_ServingStatus)(0), // 0: grpc.health.v1.HealthCheckResponse.ServingStatus
+	(*HealthCheckRequest)(nil),             // 1: grpc.health.v1.HealthCheckRequest
+	(*HealthCheckResponse)(nil),            // 2: grpc.health.v1.HealthCheckResponse
+}
+var file_grpc_health_v1_health_proto_depIdxs = []int32{
+	0, // 0: grpc.health.v1.HealthCheckResponse.status:type_name -> grpc.health.v1.HealthCheckResponse.ServingStatus
+	1, // 1: grpc.health.v1.Health.Check:input_type -> grpc.health.v1.HealthCheckRequest
+	1, // 2: grpc.health.v1.Health.Watch:input_type -> grpc.health.v1.HealthCheckRequest
+	2, // 3: grpc.health.v1.Health.Check:output_type -> grpc.health.v1.HealthCheckResponse
+	2, // 4: grpc.health.v1.Health.Watch:output_type -> grpc.health.v1.HealthCheckResponse
+	3, // [3:5] is the sub-list for method output_type
+	1, // [1:3] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_grpc_health_v1_health_proto_init() }
+func file_grpc_health_v1_health_proto_init() {
+	if File_grpc_health_v1_health_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_health_v1_health_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HealthCheckRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_health_v1_health_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HealthCheckResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_health_v1_health_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_health_v1_health_proto_goTypes,
+		DependencyIndexes: file_grpc_health_v1_health_proto_depIdxs,
+		EnumInfos:         file_grpc_health_v1_health_proto_enumTypes,
+		MessageInfos:      file_grpc_health_v1_health_proto_msgTypes,
+	}.Build()
+	File_grpc_health_v1_health_proto = out.File
+	file_grpc_health_v1_health_proto_rawDesc = nil
+	file_grpc_health_v1_health_proto_goTypes = nil
+	file_grpc_health_v1_health_proto_depIdxs = nil
 }
diff --git a/internal/proto/grpc_service_config/service_config.pb.go b/internal/proto/grpc_service_config/service_config.pb.go
index c0436b62a64b..8130a39bcc83 100644
--- a/internal/proto/grpc_service_config/service_config.pb.go
+++ b/internal/proto/grpc_service_config/service_config.pb.go
@@ -1,27 +1,56 @@
+// Copyright 2016 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A ServiceConfig is supplied when a service is deployed. It mostly contains
+// parameters for how clients that connect to the service should behave (for
+// example, the load balancing policy to use to pick between service replicas).
+//
+// The configuration options provided here act as overrides to automatically
+// chosen option values. Service owners should be conservative in specifying
+// options as the system is likely to choose better values for these options in
+// the vast majority of cases. In other words, please specify a configuration
+// option only if you really have to, and avoid copy-paste inclusion of configs.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: grpc/service_config/service_config.proto
 
 package grpc_service_config
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
 	duration "github.com/golang/protobuf/ptypes/duration"
 	wrappers "github.com/golang/protobuf/ptypes/wrappers"
 	code "google.golang.org/genproto/googleapis/rpc/code"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Load balancing policy.
 //
@@ -48,26 +77,51 @@ const (
 	ServiceConfig_ROUND_ROBIN ServiceConfig_LoadBalancingPolicy = 1
 )
 
-var ServiceConfig_LoadBalancingPolicy_name = map[int32]string{
-	0: "UNSPECIFIED",
-	1: "ROUND_ROBIN",
-}
+// Enum value maps for ServiceConfig_LoadBalancingPolicy.
+var (
+	ServiceConfig_LoadBalancingPolicy_name = map[int32]string{
+		0: "UNSPECIFIED",
+		1: "ROUND_ROBIN",
+	}
+	ServiceConfig_LoadBalancingPolicy_value = map[string]int32{
+		"UNSPECIFIED": 0,
+		"ROUND_ROBIN": 1,
+	}
+)
 
-var ServiceConfig_LoadBalancingPolicy_value = map[string]int32{
-	"UNSPECIFIED": 0,
-	"ROUND_ROBIN": 1,
+func (x ServiceConfig_LoadBalancingPolicy) Enum() *ServiceConfig_LoadBalancingPolicy {
+	p := new(ServiceConfig_LoadBalancingPolicy)
+	*p = x
+	return p
 }
 
 func (x ServiceConfig_LoadBalancingPolicy) String() string {
-	return proto.EnumName(ServiceConfig_LoadBalancingPolicy_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ServiceConfig_LoadBalancingPolicy) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_service_config_service_config_proto_enumTypes[0].Descriptor()
+}
+
+func (ServiceConfig_LoadBalancingPolicy) Type() protoreflect.EnumType {
+	return &file_grpc_service_config_service_config_proto_enumTypes[0]
+}
+
+func (x ServiceConfig_LoadBalancingPolicy) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
+// Deprecated: Use ServiceConfig_LoadBalancingPolicy.Descriptor instead.
 func (ServiceConfig_LoadBalancingPolicy) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{11, 0}
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{11, 0}
 }
 
 // Configuration for a method.
 type MethodConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	Name []*MethodConfig_Name `protobuf:"bytes,1,rep,name=name,proto3" json:"name,omitempty"`
 	// Whether RPCs sent to this method should wait until the connection is
 	// ready by default. If false, the RPC will abort immediately if there is
@@ -125,71 +179,96 @@ type MethodConfig struct {
 	// Only one of retry_policy or hedging_policy may be set. If neither is set,
 	// RPCs will not be retried or hedged.
 	//
-	// Types that are valid to be assigned to RetryOrHedgingPolicy:
+	// Types that are assignable to RetryOrHedgingPolicy:
 	//	*MethodConfig_RetryPolicy_
 	//	*MethodConfig_HedgingPolicy_
 	RetryOrHedgingPolicy isMethodConfig_RetryOrHedgingPolicy `protobuf_oneof:"retry_or_hedging_policy"`
-	XXX_NoUnkeyedLiteral struct{}                            `json:"-"`
-	XXX_unrecognized     []byte                              `json:"-"`
-	XXX_sizecache        int32                               `json:"-"`
 }
 
-func (m *MethodConfig) Reset()         { *m = MethodConfig{} }
-func (m *MethodConfig) String() string { return proto.CompactTextString(m) }
-func (*MethodConfig) ProtoMessage()    {}
-func (*MethodConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{0}
+func (x *MethodConfig) Reset() {
+	*x = MethodConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *MethodConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MethodConfig.Unmarshal(m, b)
+func (x *MethodConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *MethodConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MethodConfig.Marshal(b, m, deterministic)
+
+func (*MethodConfig) ProtoMessage() {}
+
+func (x *MethodConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *MethodConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MethodConfig.Merge(m, src)
+
+// Deprecated: Use MethodConfig.ProtoReflect.Descriptor instead.
+func (*MethodConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{0}
 }
-func (m *MethodConfig) XXX_Size() int {
-	return xxx_messageInfo_MethodConfig.Size(m)
+
+func (x *MethodConfig) GetName() []*MethodConfig_Name {
+	if x != nil {
+		return x.Name
+	}
+	return nil
 }
-func (m *MethodConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_MethodConfig.DiscardUnknown(m)
+
+func (x *MethodConfig) GetWaitForReady() *wrappers.BoolValue {
+	if x != nil {
+		return x.WaitForReady
+	}
+	return nil
 }
 
-var xxx_messageInfo_MethodConfig proto.InternalMessageInfo
+func (x *MethodConfig) GetTimeout() *duration.Duration {
+	if x != nil {
+		return x.Timeout
+	}
+	return nil
+}
 
-func (m *MethodConfig) GetName() []*MethodConfig_Name {
-	if m != nil {
-		return m.Name
+func (x *MethodConfig) GetMaxRequestMessageBytes() *wrappers.UInt32Value {
+	if x != nil {
+		return x.MaxRequestMessageBytes
 	}
 	return nil
 }
 
-func (m *MethodConfig) GetWaitForReady() *wrappers.BoolValue {
-	if m != nil {
-		return m.WaitForReady
+func (x *MethodConfig) GetMaxResponseMessageBytes() *wrappers.UInt32Value {
+	if x != nil {
+		return x.MaxResponseMessageBytes
 	}
 	return nil
 }
 
-func (m *MethodConfig) GetTimeout() *duration.Duration {
+func (m *MethodConfig) GetRetryOrHedgingPolicy() isMethodConfig_RetryOrHedgingPolicy {
 	if m != nil {
-		return m.Timeout
+		return m.RetryOrHedgingPolicy
 	}
 	return nil
 }
 
-func (m *MethodConfig) GetMaxRequestMessageBytes() *wrappers.UInt32Value {
-	if m != nil {
-		return m.MaxRequestMessageBytes
+func (x *MethodConfig) GetRetryPolicy() *MethodConfig_RetryPolicy {
+	if x, ok := x.GetRetryOrHedgingPolicy().(*MethodConfig_RetryPolicy_); ok {
+		return x.RetryPolicy
 	}
 	return nil
 }
 
-func (m *MethodConfig) GetMaxResponseMessageBytes() *wrappers.UInt32Value {
-	if m != nil {
-		return m.MaxResponseMessageBytes
+func (x *MethodConfig) GetHedgingPolicy() *MethodConfig_HedgingPolicy {
+	if x, ok := x.GetRetryOrHedgingPolicy().(*MethodConfig_HedgingPolicy_); ok {
+		return x.HedgingPolicy
 	}
 	return nil
 }
@@ -210,619 +289,312 @@ func (*MethodConfig_RetryPolicy_) isMethodConfig_RetryOrHedgingPolicy() {}
 
 func (*MethodConfig_HedgingPolicy_) isMethodConfig_RetryOrHedgingPolicy() {}
 
-func (m *MethodConfig) GetRetryOrHedgingPolicy() isMethodConfig_RetryOrHedgingPolicy {
-	if m != nil {
-		return m.RetryOrHedgingPolicy
-	}
-	return nil
+// Configuration for pick_first LB policy.
+type PickFirstConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *MethodConfig) GetRetryPolicy() *MethodConfig_RetryPolicy {
-	if x, ok := m.GetRetryOrHedgingPolicy().(*MethodConfig_RetryPolicy_); ok {
-		return x.RetryPolicy
+func (x *PickFirstConfig) Reset() {
+	*x = PickFirstConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *MethodConfig) GetHedgingPolicy() *MethodConfig_HedgingPolicy {
-	if x, ok := m.GetRetryOrHedgingPolicy().(*MethodConfig_HedgingPolicy_); ok {
-		return x.HedgingPolicy
-	}
-	return nil
+func (x *PickFirstConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*MethodConfig) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*MethodConfig_RetryPolicy_)(nil),
-		(*MethodConfig_HedgingPolicy_)(nil),
+func (*PickFirstConfig) ProtoMessage() {}
+
+func (x *PickFirstConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
+	return mi.MessageOf(x)
 }
 
-// The names of the methods to which this configuration applies.
-// - MethodConfig without names (empty list) will be skipped.
-// - Each name entry must be unique across the entire ServiceConfig.
-// - If the 'method' field is empty, this MethodConfig specifies the defaults
-//   for all methods for the specified service.
-// - If the 'service' field is empty, the 'method' field must be empty, and
-//   this MethodConfig specifies the default for all methods (it's the default
-//   config).
-//
-// When determining which MethodConfig to use for a given RPC, the most
-// specific match wins. For example, let's say that the service config
-// contains the following MethodConfig entries:
-//
-// method_config { name { } ... }
-// method_config { name { service: "MyService" } ... }
-// method_config { name { service: "MyService" method: "Foo" } ... }
-//
-// MyService/Foo will use the third entry, because it exactly matches the
-// service and method name. MyService/Bar will use the second entry, because
-// it provides the default for all methods of MyService. AnotherService/Baz
-// will use the first entry, because it doesn't match the other two.
-//
-// In JSON representation, value "", value `null`, and not present are the
-// same. The following are the same Name:
-// - { "service": "s" }
-// - { "service": "s", "method": null }
-// - { "service": "s", "method": "" }
-type MethodConfig_Name struct {
-	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
-	Method               string   `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+// Deprecated: Use PickFirstConfig.ProtoReflect.Descriptor instead.
+func (*PickFirstConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{1}
 }
 
-func (m *MethodConfig_Name) Reset()         { *m = MethodConfig_Name{} }
-func (m *MethodConfig_Name) String() string { return proto.CompactTextString(m) }
-func (*MethodConfig_Name) ProtoMessage()    {}
-func (*MethodConfig_Name) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{0, 0}
+// Configuration for round_robin LB policy.
+type RoundRobinConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *MethodConfig_Name) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MethodConfig_Name.Unmarshal(m, b)
-}
-func (m *MethodConfig_Name) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MethodConfig_Name.Marshal(b, m, deterministic)
-}
-func (m *MethodConfig_Name) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MethodConfig_Name.Merge(m, src)
-}
-func (m *MethodConfig_Name) XXX_Size() int {
-	return xxx_messageInfo_MethodConfig_Name.Size(m)
+func (x *RoundRobinConfig) Reset() {
+	*x = RoundRobinConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *MethodConfig_Name) XXX_DiscardUnknown() {
-	xxx_messageInfo_MethodConfig_Name.DiscardUnknown(m)
+
+func (x *RoundRobinConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_MethodConfig_Name proto.InternalMessageInfo
+func (*RoundRobinConfig) ProtoMessage() {}
 
-func (m *MethodConfig_Name) GetService() string {
-	if m != nil {
-		return m.Service
+func (x *RoundRobinConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return ""
+	return mi.MessageOf(x)
 }
 
-func (m *MethodConfig_Name) GetMethod() string {
-	if m != nil {
-		return m.Method
-	}
-	return ""
+// Deprecated: Use RoundRobinConfig.ProtoReflect.Descriptor instead.
+func (*RoundRobinConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{2}
 }
 
-// The retry policy for outgoing RPCs.
-type MethodConfig_RetryPolicy struct {
-	// The maximum number of RPC attempts, including the original attempt.
-	//
-	// This field is required and must be greater than 1.
-	// Any value greater than 5 will be treated as if it were 5.
-	MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"`
-	// Exponential backoff parameters. The initial retry attempt will occur at
-	// random(0, initial_backoff). In general, the nth attempt will occur at
-	// random(0,
-	//   min(initial_backoff*backoff_multiplier**(n-1), max_backoff)).
-	// Required. Must be greater than zero.
-	InitialBackoff *duration.Duration `protobuf:"bytes,2,opt,name=initial_backoff,json=initialBackoff,proto3" json:"initial_backoff,omitempty"`
-	// Required. Must be greater than zero.
-	MaxBackoff        *duration.Duration `protobuf:"bytes,3,opt,name=max_backoff,json=maxBackoff,proto3" json:"max_backoff,omitempty"`
-	BackoffMultiplier float32            `protobuf:"fixed32,4,opt,name=backoff_multiplier,json=backoffMultiplier,proto3" json:"backoff_multiplier,omitempty"`
-	// The set of status codes which may be retried.
-	//
-	// This field is required and must be non-empty.
-	RetryableStatusCodes []code.Code `protobuf:"varint,5,rep,packed,name=retryable_status_codes,json=retryableStatusCodes,proto3,enum=google.rpc.Code" json:"retryable_status_codes,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
-}
+// Configuration for priority LB policy.
+type PriorityLoadBalancingPolicyConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *MethodConfig_RetryPolicy) Reset()         { *m = MethodConfig_RetryPolicy{} }
-func (m *MethodConfig_RetryPolicy) String() string { return proto.CompactTextString(m) }
-func (*MethodConfig_RetryPolicy) ProtoMessage()    {}
-func (*MethodConfig_RetryPolicy) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{0, 1}
+	Children map[string]*PriorityLoadBalancingPolicyConfig_Child `protobuf:"bytes,1,rep,name=children,proto3" json:"children,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// A list of child names in decreasing priority order
+	// (i.e., first element is the highest priority).
+	Priorities []string `protobuf:"bytes,2,rep,name=priorities,proto3" json:"priorities,omitempty"`
 }
 
-func (m *MethodConfig_RetryPolicy) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MethodConfig_RetryPolicy.Unmarshal(m, b)
-}
-func (m *MethodConfig_RetryPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MethodConfig_RetryPolicy.Marshal(b, m, deterministic)
-}
-func (m *MethodConfig_RetryPolicy) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MethodConfig_RetryPolicy.Merge(m, src)
-}
-func (m *MethodConfig_RetryPolicy) XXX_Size() int {
-	return xxx_messageInfo_MethodConfig_RetryPolicy.Size(m)
+func (x *PriorityLoadBalancingPolicyConfig) Reset() {
+	*x = PriorityLoadBalancingPolicyConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *MethodConfig_RetryPolicy) XXX_DiscardUnknown() {
-	xxx_messageInfo_MethodConfig_RetryPolicy.DiscardUnknown(m)
+
+func (x *PriorityLoadBalancingPolicyConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_MethodConfig_RetryPolicy proto.InternalMessageInfo
+func (*PriorityLoadBalancingPolicyConfig) ProtoMessage() {}
 
-func (m *MethodConfig_RetryPolicy) GetMaxAttempts() uint32 {
-	if m != nil {
-		return m.MaxAttempts
+func (x *PriorityLoadBalancingPolicyConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *MethodConfig_RetryPolicy) GetInitialBackoff() *duration.Duration {
-	if m != nil {
-		return m.InitialBackoff
-	}
-	return nil
+// Deprecated: Use PriorityLoadBalancingPolicyConfig.ProtoReflect.Descriptor instead.
+func (*PriorityLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{3}
 }
 
-func (m *MethodConfig_RetryPolicy) GetMaxBackoff() *duration.Duration {
-	if m != nil {
-		return m.MaxBackoff
+func (x *PriorityLoadBalancingPolicyConfig) GetChildren() map[string]*PriorityLoadBalancingPolicyConfig_Child {
+	if x != nil {
+		return x.Children
 	}
 	return nil
 }
 
-func (m *MethodConfig_RetryPolicy) GetBackoffMultiplier() float32 {
-	if m != nil {
-		return m.BackoffMultiplier
-	}
-	return 0
-}
-
-func (m *MethodConfig_RetryPolicy) GetRetryableStatusCodes() []code.Code {
-	if m != nil {
-		return m.RetryableStatusCodes
+func (x *PriorityLoadBalancingPolicyConfig) GetPriorities() []string {
+	if x != nil {
+		return x.Priorities
 	}
 	return nil
 }
 
-// The hedging policy for outgoing RPCs. Hedged RPCs may execute more than
-// once on the server, so only idempotent methods should specify a hedging
-// policy.
-type MethodConfig_HedgingPolicy struct {
-	// The hedging policy will send up to max_requests RPCs.
-	// This number represents the total number of all attempts, including
-	// the original attempt.
-	//
-	// This field is required and must be greater than 1.
-	// Any value greater than 5 will be treated as if it were 5.
-	MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"`
-	// The first RPC will be sent immediately, but the max_requests-1 subsequent
-	// hedged RPCs will be sent at intervals of every hedging_delay. Set this
-	// to 0 to immediately send all max_requests RPCs.
-	HedgingDelay *duration.Duration `protobuf:"bytes,2,opt,name=hedging_delay,json=hedgingDelay,proto3" json:"hedging_delay,omitempty"`
-	// The set of status codes which indicate other hedged RPCs may still
-	// succeed. If a non-fatal status code is returned by the server, hedged
-	// RPCs will continue. Otherwise, outstanding requests will be canceled and
-	// the error returned to the client application layer.
-	//
-	// This field is optional.
-	NonFatalStatusCodes  []code.Code `protobuf:"varint,3,rep,packed,name=non_fatal_status_codes,json=nonFatalStatusCodes,proto3,enum=google.rpc.Code" json:"non_fatal_status_codes,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
-}
+// Configuration for weighted_target LB policy.
+type WeightedTargetLoadBalancingPolicyConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *MethodConfig_HedgingPolicy) Reset()         { *m = MethodConfig_HedgingPolicy{} }
-func (m *MethodConfig_HedgingPolicy) String() string { return proto.CompactTextString(m) }
-func (*MethodConfig_HedgingPolicy) ProtoMessage()    {}
-func (*MethodConfig_HedgingPolicy) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{0, 2}
+	Targets map[string]*WeightedTargetLoadBalancingPolicyConfig_Target `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
-func (m *MethodConfig_HedgingPolicy) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MethodConfig_HedgingPolicy.Unmarshal(m, b)
-}
-func (m *MethodConfig_HedgingPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MethodConfig_HedgingPolicy.Marshal(b, m, deterministic)
-}
-func (m *MethodConfig_HedgingPolicy) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MethodConfig_HedgingPolicy.Merge(m, src)
-}
-func (m *MethodConfig_HedgingPolicy) XXX_Size() int {
-	return xxx_messageInfo_MethodConfig_HedgingPolicy.Size(m)
+func (x *WeightedTargetLoadBalancingPolicyConfig) Reset() {
+	*x = WeightedTargetLoadBalancingPolicyConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *MethodConfig_HedgingPolicy) XXX_DiscardUnknown() {
-	xxx_messageInfo_MethodConfig_HedgingPolicy.DiscardUnknown(m)
+
+func (x *WeightedTargetLoadBalancingPolicyConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_MethodConfig_HedgingPolicy proto.InternalMessageInfo
+func (*WeightedTargetLoadBalancingPolicyConfig) ProtoMessage() {}
 
-func (m *MethodConfig_HedgingPolicy) GetMaxAttempts() uint32 {
-	if m != nil {
-		return m.MaxAttempts
+func (x *WeightedTargetLoadBalancingPolicyConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return 0
+	return mi.MessageOf(x)
 }
 
-func (m *MethodConfig_HedgingPolicy) GetHedgingDelay() *duration.Duration {
-	if m != nil {
-		return m.HedgingDelay
-	}
-	return nil
+// Deprecated: Use WeightedTargetLoadBalancingPolicyConfig.ProtoReflect.Descriptor instead.
+func (*WeightedTargetLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{4}
 }
 
-func (m *MethodConfig_HedgingPolicy) GetNonFatalStatusCodes() []code.Code {
-	if m != nil {
-		return m.NonFatalStatusCodes
+func (x *WeightedTargetLoadBalancingPolicyConfig) GetTargets() map[string]*WeightedTargetLoadBalancingPolicyConfig_Target {
+	if x != nil {
+		return x.Targets
 	}
 	return nil
 }
 
-// Configuration for pick_first LB policy.
-type PickFirstConfig struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *PickFirstConfig) Reset()         { *m = PickFirstConfig{} }
-func (m *PickFirstConfig) String() string { return proto.CompactTextString(m) }
-func (*PickFirstConfig) ProtoMessage()    {}
-func (*PickFirstConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{1}
-}
-
-func (m *PickFirstConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PickFirstConfig.Unmarshal(m, b)
-}
-func (m *PickFirstConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PickFirstConfig.Marshal(b, m, deterministic)
-}
-func (m *PickFirstConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PickFirstConfig.Merge(m, src)
-}
-func (m *PickFirstConfig) XXX_Size() int {
-	return xxx_messageInfo_PickFirstConfig.Size(m)
-}
-func (m *PickFirstConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_PickFirstConfig.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_PickFirstConfig proto.InternalMessageInfo
+// Configuration for grpclb LB policy.
+type GrpcLbConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-// Configuration for round_robin LB policy.
-type RoundRobinConfig struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	// Optional.  What LB policy to use for routing between the backend
+	// addresses.  If unset, defaults to round_robin.
+	// Currently, the only supported values are round_robin and pick_first.
+	// Note that this will be used both in balancer mode and in fallback mode.
+	// Multiple LB policies can be specified; clients will iterate through
+	// the list in order and stop at the first policy that they support.
+	ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
+	// Optional.  If specified, overrides the name of the service to be sent to
+	// the balancer.
+	ServiceName string `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
 }
 
-func (m *RoundRobinConfig) Reset()         { *m = RoundRobinConfig{} }
-func (m *RoundRobinConfig) String() string { return proto.CompactTextString(m) }
-func (*RoundRobinConfig) ProtoMessage()    {}
-func (*RoundRobinConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{2}
+func (x *GrpcLbConfig) Reset() {
+	*x = GrpcLbConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *RoundRobinConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_RoundRobinConfig.Unmarshal(m, b)
-}
-func (m *RoundRobinConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_RoundRobinConfig.Marshal(b, m, deterministic)
-}
-func (m *RoundRobinConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_RoundRobinConfig.Merge(m, src)
-}
-func (m *RoundRobinConfig) XXX_Size() int {
-	return xxx_messageInfo_RoundRobinConfig.Size(m)
+func (x *GrpcLbConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *RoundRobinConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_RoundRobinConfig.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_RoundRobinConfig proto.InternalMessageInfo
 
-// Configuration for priority LB policy.
-type PriorityLoadBalancingPolicyConfig struct {
-	Children map[string]*PriorityLoadBalancingPolicyConfig_Child `protobuf:"bytes,1,rep,name=children,proto3" json:"children,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	// A list of child names in decreasing priority order
-	// (i.e., first element is the highest priority).
-	Priorities           []string `protobuf:"bytes,2,rep,name=priorities,proto3" json:"priorities,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+func (*GrpcLbConfig) ProtoMessage() {}
 
-func (m *PriorityLoadBalancingPolicyConfig) Reset()         { *m = PriorityLoadBalancingPolicyConfig{} }
-func (m *PriorityLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
-func (*PriorityLoadBalancingPolicyConfig) ProtoMessage()    {}
-func (*PriorityLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{3}
+func (x *GrpcLbConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-func (m *PriorityLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Unmarshal(m, b)
-}
-func (m *PriorityLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
-}
-func (m *PriorityLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Merge(m, src)
-}
-func (m *PriorityLoadBalancingPolicyConfig) XXX_Size() int {
-	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Size(m)
-}
-func (m *PriorityLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_PriorityLoadBalancingPolicyConfig.DiscardUnknown(m)
+// Deprecated: Use GrpcLbConfig.ProtoReflect.Descriptor instead.
+func (*GrpcLbConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{5}
 }
 
-var xxx_messageInfo_PriorityLoadBalancingPolicyConfig proto.InternalMessageInfo
-
-func (m *PriorityLoadBalancingPolicyConfig) GetChildren() map[string]*PriorityLoadBalancingPolicyConfig_Child {
-	if m != nil {
-		return m.Children
+func (x *GrpcLbConfig) GetChildPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.ChildPolicy
 	}
 	return nil
 }
 
-func (m *PriorityLoadBalancingPolicyConfig) GetPriorities() []string {
-	if m != nil {
-		return m.Priorities
+func (x *GrpcLbConfig) GetServiceName() string {
+	if x != nil {
+		return x.ServiceName
 	}
-	return nil
+	return ""
 }
 
-// A map of name to child policy configuration.
-// The names are used to allow the priority policy to update
-// existing child policies instead of creating new ones every
-// time it receives a config update.
-type PriorityLoadBalancingPolicyConfig_Child struct {
-	Config               []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=config,proto3" json:"config,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
-	XXX_unrecognized     []byte                 `json:"-"`
-	XXX_sizecache        int32                  `json:"-"`
-}
+// Configuration for the cds LB policy.
+type CdsConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *PriorityLoadBalancingPolicyConfig_Child) Reset() {
-	*m = PriorityLoadBalancingPolicyConfig_Child{}
-}
-func (m *PriorityLoadBalancingPolicyConfig_Child) String() string { return proto.CompactTextString(m) }
-func (*PriorityLoadBalancingPolicyConfig_Child) ProtoMessage()    {}
-func (*PriorityLoadBalancingPolicyConfig_Child) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{3, 0}
+	Cluster string `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"` // Required.
 }
 
-func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Unmarshal(m, b)
-}
-func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Marshal(b, m, deterministic)
-}
-func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Merge(m, src)
-}
-func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Size() int {
-	return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Size(m)
+func (x *CdsConfig) Reset() {
+	*x = CdsConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_DiscardUnknown() {
-	xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.DiscardUnknown(m)
+
+func (x *CdsConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child proto.InternalMessageInfo
+func (*CdsConfig) ProtoMessage() {}
 
-func (m *PriorityLoadBalancingPolicyConfig_Child) GetConfig() []*LoadBalancingConfig {
-	if m != nil {
-		return m.Config
+func (x *CdsConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-// Configuration for weighted_target LB policy.
-type WeightedTargetLoadBalancingPolicyConfig struct {
-	Targets              map[string]*WeightedTargetLoadBalancingPolicyConfig_Target `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}                                                   `json:"-"`
-	XXX_unrecognized     []byte                                                     `json:"-"`
-	XXX_sizecache        int32                                                      `json:"-"`
+// Deprecated: Use CdsConfig.ProtoReflect.Descriptor instead.
+func (*CdsConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{6}
 }
 
-func (m *WeightedTargetLoadBalancingPolicyConfig) Reset() {
-	*m = WeightedTargetLoadBalancingPolicyConfig{}
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
-func (*WeightedTargetLoadBalancingPolicyConfig) ProtoMessage()    {}
-func (*WeightedTargetLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{4}
-}
-
-func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Unmarshal(m, b)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Merge(m, src)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Size() int {
-	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Size(m)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig proto.InternalMessageInfo
-
-func (m *WeightedTargetLoadBalancingPolicyConfig) GetTargets() map[string]*WeightedTargetLoadBalancingPolicyConfig_Target {
-	if m != nil {
-		return m.Targets
-	}
-	return nil
-}
-
-type WeightedTargetLoadBalancingPolicyConfig_Target struct {
-	Weight               uint32                 `protobuf:"varint,1,opt,name=weight,proto3" json:"weight,omitempty"`
-	ChildPolicy          []*LoadBalancingConfig `protobuf:"bytes,2,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
-	XXX_unrecognized     []byte                 `json:"-"`
-	XXX_sizecache        int32                  `json:"-"`
-}
-
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) Reset() {
-	*m = WeightedTargetLoadBalancingPolicyConfig_Target{}
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) String() string {
-	return proto.CompactTextString(m)
-}
-func (*WeightedTargetLoadBalancingPolicyConfig_Target) ProtoMessage() {}
-func (*WeightedTargetLoadBalancingPolicyConfig_Target) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{4, 0}
-}
-
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Unmarshal(m, b)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Marshal(b, m, deterministic)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Merge(m, src)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Size() int {
-	return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Size(m)
-}
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_DiscardUnknown() {
-	xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target proto.InternalMessageInfo
-
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) GetWeight() uint32 {
-	if m != nil {
-		return m.Weight
-	}
-	return 0
-}
-
-func (m *WeightedTargetLoadBalancingPolicyConfig_Target) GetChildPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.ChildPolicy
-	}
-	return nil
-}
-
-// Configuration for grpclb LB policy.
-type GrpcLbConfig struct {
-	// Optional.  What LB policy to use for routing between the backend
-	// addresses.  If unset, defaults to round_robin.
-	// Currently, the only supported values are round_robin and pick_first.
-	// Note that this will be used both in balancer mode and in fallback mode.
-	// Multiple LB policies can be specified; clients will iterate through
-	// the list in order and stop at the first policy that they support.
-	ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
-	// Optional.  If specified, overrides the name of the service to be sent to
-	// the balancer.
-	ServiceName          string   `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *GrpcLbConfig) Reset()         { *m = GrpcLbConfig{} }
-func (m *GrpcLbConfig) String() string { return proto.CompactTextString(m) }
-func (*GrpcLbConfig) ProtoMessage()    {}
-func (*GrpcLbConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{5}
-}
-
-func (m *GrpcLbConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GrpcLbConfig.Unmarshal(m, b)
-}
-func (m *GrpcLbConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GrpcLbConfig.Marshal(b, m, deterministic)
-}
-func (m *GrpcLbConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GrpcLbConfig.Merge(m, src)
-}
-func (m *GrpcLbConfig) XXX_Size() int {
-	return xxx_messageInfo_GrpcLbConfig.Size(m)
-}
-func (m *GrpcLbConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_GrpcLbConfig.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GrpcLbConfig proto.InternalMessageInfo
-
-func (m *GrpcLbConfig) GetChildPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.ChildPolicy
-	}
-	return nil
-}
-
-func (m *GrpcLbConfig) GetServiceName() string {
-	if m != nil {
-		return m.ServiceName
-	}
-	return ""
-}
-
-// Configuration for the cds LB policy.
-type CdsConfig struct {
-	Cluster              string   `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *CdsConfig) Reset()         { *m = CdsConfig{} }
-func (m *CdsConfig) String() string { return proto.CompactTextString(m) }
-func (*CdsConfig) ProtoMessage()    {}
-func (*CdsConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{6}
-}
-
-func (m *CdsConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_CdsConfig.Unmarshal(m, b)
-}
-func (m *CdsConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_CdsConfig.Marshal(b, m, deterministic)
-}
-func (m *CdsConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_CdsConfig.Merge(m, src)
-}
-func (m *CdsConfig) XXX_Size() int {
-	return xxx_messageInfo_CdsConfig.Size(m)
-}
-func (m *CdsConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_CdsConfig.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_CdsConfig proto.InternalMessageInfo
-
-func (m *CdsConfig) GetCluster() string {
-	if m != nil {
-		return m.Cluster
-	}
-	return ""
+func (x *CdsConfig) GetCluster() string {
+	if x != nil {
+		return x.Cluster
+	}
+	return ""
 }
 
 // Configuration for xds LB policy.
 type XdsConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Name of balancer to connect to.
-	BalancerName string `protobuf:"bytes,1,opt,name=balancer_name,json=balancerName,proto3" json:"balancer_name,omitempty"` // Deprecated: Do not use.
+	//
+	// Deprecated: Do not use.
+	BalancerName string `protobuf:"bytes,1,opt,name=balancer_name,json=balancerName,proto3" json:"balancer_name,omitempty"`
 	// Optional.  What LB policy to use for intra-locality routing.
 	// If unset, will use whatever algorithm is specified by the balancer.
 	// Multiple LB policies can be specified; clients will iterate through
@@ -841,74 +613,82 @@ type XdsConfig struct {
 	// If set to the empty string, load reporting will be sent to the same
 	// server that we obtained CDS data from.
 	LrsLoadReportingServerName *wrappers.StringValue `protobuf:"bytes,5,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
-	XXX_NoUnkeyedLiteral       struct{}              `json:"-"`
-	XXX_unrecognized           []byte                `json:"-"`
-	XXX_sizecache              int32                 `json:"-"`
 }
 
-func (m *XdsConfig) Reset()         { *m = XdsConfig{} }
-func (m *XdsConfig) String() string { return proto.CompactTextString(m) }
-func (*XdsConfig) ProtoMessage()    {}
-func (*XdsConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{7}
+func (x *XdsConfig) Reset() {
+	*x = XdsConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *XdsConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_XdsConfig.Unmarshal(m, b)
-}
-func (m *XdsConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_XdsConfig.Marshal(b, m, deterministic)
-}
-func (m *XdsConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_XdsConfig.Merge(m, src)
+func (x *XdsConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *XdsConfig) XXX_Size() int {
-	return xxx_messageInfo_XdsConfig.Size(m)
-}
-func (m *XdsConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_XdsConfig.DiscardUnknown(m)
+
+func (*XdsConfig) ProtoMessage() {}
+
+func (x *XdsConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_XdsConfig proto.InternalMessageInfo
+// Deprecated: Use XdsConfig.ProtoReflect.Descriptor instead.
+func (*XdsConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{7}
+}
 
 // Deprecated: Do not use.
-func (m *XdsConfig) GetBalancerName() string {
-	if m != nil {
-		return m.BalancerName
+func (x *XdsConfig) GetBalancerName() string {
+	if x != nil {
+		return x.BalancerName
 	}
 	return ""
 }
 
-func (m *XdsConfig) GetChildPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.ChildPolicy
+func (x *XdsConfig) GetChildPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.ChildPolicy
 	}
 	return nil
 }
 
-func (m *XdsConfig) GetFallbackPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.FallbackPolicy
+func (x *XdsConfig) GetFallbackPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.FallbackPolicy
 	}
 	return nil
 }
 
-func (m *XdsConfig) GetEdsServiceName() string {
-	if m != nil {
-		return m.EdsServiceName
+func (x *XdsConfig) GetEdsServiceName() string {
+	if x != nil {
+		return x.EdsServiceName
 	}
 	return ""
 }
 
-func (m *XdsConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
-	if m != nil {
-		return m.LrsLoadReportingServerName
+func (x *XdsConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
+	if x != nil {
+		return x.LrsLoadReportingServerName
 	}
 	return nil
 }
 
 // Configuration for eds LB policy.
 type EdsLoadBalancingPolicyConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Cluster name.  Required.
 	Cluster string `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"`
 	// EDS service name, as returned in CDS.
@@ -931,73 +711,81 @@ type EdsLoadBalancingPolicyConfig struct {
 	// locality-policy's config.
 	// Optional; defaults to "round_robin".
 	EndpointPickingPolicy []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=endpoint_picking_policy,json=endpointPickingPolicy,proto3" json:"endpoint_picking_policy,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{}               `json:"-"`
-	XXX_unrecognized      []byte                 `json:"-"`
-	XXX_sizecache         int32                  `json:"-"`
 }
 
-func (m *EdsLoadBalancingPolicyConfig) Reset()         { *m = EdsLoadBalancingPolicyConfig{} }
-func (m *EdsLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
-func (*EdsLoadBalancingPolicyConfig) ProtoMessage()    {}
-func (*EdsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{8}
+func (x *EdsLoadBalancingPolicyConfig) Reset() {
+	*x = EdsLoadBalancingPolicyConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *EdsLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Unmarshal(m, b)
-}
-func (m *EdsLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
-}
-func (m *EdsLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EdsLoadBalancingPolicyConfig.Merge(m, src)
+func (x *EdsLoadBalancingPolicyConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EdsLoadBalancingPolicyConfig) XXX_Size() int {
-	return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Size(m)
-}
-func (m *EdsLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_EdsLoadBalancingPolicyConfig.DiscardUnknown(m)
+
+func (*EdsLoadBalancingPolicyConfig) ProtoMessage() {}
+
+func (x *EdsLoadBalancingPolicyConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_EdsLoadBalancingPolicyConfig proto.InternalMessageInfo
+// Deprecated: Use EdsLoadBalancingPolicyConfig.ProtoReflect.Descriptor instead.
+func (*EdsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *EdsLoadBalancingPolicyConfig) GetCluster() string {
-	if m != nil {
-		return m.Cluster
+func (x *EdsLoadBalancingPolicyConfig) GetCluster() string {
+	if x != nil {
+		return x.Cluster
 	}
 	return ""
 }
 
-func (m *EdsLoadBalancingPolicyConfig) GetEdsServiceName() string {
-	if m != nil {
-		return m.EdsServiceName
+func (x *EdsLoadBalancingPolicyConfig) GetEdsServiceName() string {
+	if x != nil {
+		return x.EdsServiceName
 	}
 	return ""
 }
 
-func (m *EdsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
-	if m != nil {
-		return m.LrsLoadReportingServerName
+func (x *EdsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
+	if x != nil {
+		return x.LrsLoadReportingServerName
 	}
 	return nil
 }
 
-func (m *EdsLoadBalancingPolicyConfig) GetLocalityPickingPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.LocalityPickingPolicy
+func (x *EdsLoadBalancingPolicyConfig) GetLocalityPickingPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.LocalityPickingPolicy
 	}
 	return nil
 }
 
-func (m *EdsLoadBalancingPolicyConfig) GetEndpointPickingPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.EndpointPickingPolicy
+func (x *EdsLoadBalancingPolicyConfig) GetEndpointPickingPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.EndpointPickingPolicy
 	}
 	return nil
 }
 
 // Configuration for lrs LB policy.
 type LrsLoadBalancingPolicyConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Cluster name.  Required.
 	ClusterName string `protobuf:"bytes,1,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"`
 	// EDS service name, as returned in CDS.
@@ -1009,126 +797,74 @@ type LrsLoadBalancingPolicyConfig struct {
 	LrsLoadReportingServerName string                                 `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
 	Locality                   *LrsLoadBalancingPolicyConfig_Locality `protobuf:"bytes,4,opt,name=locality,proto3" json:"locality,omitempty"`
 	// Endpoint-picking policy.
-	ChildPolicy          []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
-	XXX_unrecognized     []byte                 `json:"-"`
-	XXX_sizecache        int32                  `json:"-"`
+	ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
 }
 
-func (m *LrsLoadBalancingPolicyConfig) Reset()         { *m = LrsLoadBalancingPolicyConfig{} }
-func (m *LrsLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) }
-func (*LrsLoadBalancingPolicyConfig) ProtoMessage()    {}
-func (*LrsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{9}
+func (x *LrsLoadBalancingPolicyConfig) Reset() {
+	*x = LrsLoadBalancingPolicyConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *LrsLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Unmarshal(m, b)
-}
-func (m *LrsLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Marshal(b, m, deterministic)
-}
-func (m *LrsLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LrsLoadBalancingPolicyConfig.Merge(m, src)
-}
-func (m *LrsLoadBalancingPolicyConfig) XXX_Size() int {
-	return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Size(m)
-}
-func (m *LrsLoadBalancingPolicyConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_LrsLoadBalancingPolicyConfig.DiscardUnknown(m)
+func (x *LrsLoadBalancingPolicyConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_LrsLoadBalancingPolicyConfig proto.InternalMessageInfo
+func (*LrsLoadBalancingPolicyConfig) ProtoMessage() {}
 
-func (m *LrsLoadBalancingPolicyConfig) GetClusterName() string {
-	if m != nil {
-		return m.ClusterName
+func (x *LrsLoadBalancingPolicyConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return ""
+	return mi.MessageOf(x)
 }
 
-func (m *LrsLoadBalancingPolicyConfig) GetEdsServiceName() string {
-	if m != nil {
-		return m.EdsServiceName
-	}
-	return ""
+// Deprecated: Use LrsLoadBalancingPolicyConfig.ProtoReflect.Descriptor instead.
+func (*LrsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{9}
 }
 
-func (m *LrsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() string {
-	if m != nil {
-		return m.LrsLoadReportingServerName
+func (x *LrsLoadBalancingPolicyConfig) GetClusterName() string {
+	if x != nil {
+		return x.ClusterName
 	}
 	return ""
 }
 
-func (m *LrsLoadBalancingPolicyConfig) GetLocality() *LrsLoadBalancingPolicyConfig_Locality {
-	if m != nil {
-		return m.Locality
-	}
-	return nil
-}
-
-func (m *LrsLoadBalancingPolicyConfig) GetChildPolicy() []*LoadBalancingConfig {
-	if m != nil {
-		return m.ChildPolicy
+func (x *LrsLoadBalancingPolicyConfig) GetEdsServiceName() string {
+	if x != nil {
+		return x.EdsServiceName
 	}
-	return nil
-}
-
-// The locality for which this policy will report load.  Required.
-type LrsLoadBalancingPolicyConfig_Locality struct {
-	Region               string   `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"`
-	Zone                 string   `protobuf:"bytes,2,opt,name=zone,proto3" json:"zone,omitempty"`
-	Subzone              string   `protobuf:"bytes,3,opt,name=subzone,proto3" json:"subzone,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *LrsLoadBalancingPolicyConfig_Locality) Reset()         { *m = LrsLoadBalancingPolicyConfig_Locality{} }
-func (m *LrsLoadBalancingPolicyConfig_Locality) String() string { return proto.CompactTextString(m) }
-func (*LrsLoadBalancingPolicyConfig_Locality) ProtoMessage()    {}
-func (*LrsLoadBalancingPolicyConfig_Locality) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{9, 0}
-}
-
-func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Unmarshal(m, b)
-}
-func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Marshal(b, m, deterministic)
-}
-func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Merge(m, src)
-}
-func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Size() int {
-	return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Size(m)
-}
-func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_DiscardUnknown() {
-	xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.DiscardUnknown(m)
+	return ""
 }
 
-var xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality proto.InternalMessageInfo
-
-func (m *LrsLoadBalancingPolicyConfig_Locality) GetRegion() string {
-	if m != nil {
-		return m.Region
+func (x *LrsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() string {
+	if x != nil {
+		return x.LrsLoadReportingServerName
 	}
 	return ""
 }
 
-func (m *LrsLoadBalancingPolicyConfig_Locality) GetZone() string {
-	if m != nil {
-		return m.Zone
+func (x *LrsLoadBalancingPolicyConfig) GetLocality() *LrsLoadBalancingPolicyConfig_Locality {
+	if x != nil {
+		return x.Locality
 	}
-	return ""
+	return nil
 }
 
-func (m *LrsLoadBalancingPolicyConfig_Locality) GetSubzone() string {
-	if m != nil {
-		return m.Subzone
+func (x *LrsLoadBalancingPolicyConfig) GetChildPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.ChildPolicy
 	}
-	return ""
+	return nil
 }
 
 // Selects LB policy and provides corresponding configuration.
@@ -1143,9 +879,13 @@ func (m *LrsLoadBalancingPolicyConfig_Locality) GetSubzone() string {
 // - If the list doesn't contain any supported policy, the whole service config
 //   is invalid.
 type LoadBalancingConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Exactly one LB policy may be configured.
 	//
-	// Types that are valid to be assigned to Policy:
+	// Types that are assignable to Policy:
 	//	*LoadBalancingConfig_PickFirst
 	//	*LoadBalancingConfig_RoundRobin
 	//	*LoadBalancingConfig_Grpclb
@@ -1156,36 +896,119 @@ type LoadBalancingConfig struct {
 	//	*LoadBalancingConfig_Lrs
 	//	*LoadBalancingConfig_Xds
 	//	*LoadBalancingConfig_XdsExperimental
-	Policy               isLoadBalancingConfig_Policy `protobuf_oneof:"policy"`
-	XXX_NoUnkeyedLiteral struct{}                     `json:"-"`
-	XXX_unrecognized     []byte                       `json:"-"`
-	XXX_sizecache        int32                        `json:"-"`
+	Policy isLoadBalancingConfig_Policy `protobuf_oneof:"policy"`
+}
+
+func (x *LoadBalancingConfig) Reset() {
+	*x = LoadBalancingConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancingConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancingConfig) ProtoMessage() {}
+
+func (x *LoadBalancingConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-func (m *LoadBalancingConfig) Reset()         { *m = LoadBalancingConfig{} }
-func (m *LoadBalancingConfig) String() string { return proto.CompactTextString(m) }
-func (*LoadBalancingConfig) ProtoMessage()    {}
+// Deprecated: Use LoadBalancingConfig.ProtoReflect.Descriptor instead.
 func (*LoadBalancingConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{10}
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{10}
+}
+
+func (m *LoadBalancingConfig) GetPolicy() isLoadBalancingConfig_Policy {
+	if m != nil {
+		return m.Policy
+	}
+	return nil
+}
+
+func (x *LoadBalancingConfig) GetPickFirst() *PickFirstConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_PickFirst); ok {
+		return x.PickFirst
+	}
+	return nil
+}
+
+func (x *LoadBalancingConfig) GetRoundRobin() *RoundRobinConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_RoundRobin); ok {
+		return x.RoundRobin
+	}
+	return nil
+}
+
+func (x *LoadBalancingConfig) GetGrpclb() *GrpcLbConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_Grpclb); ok {
+		return x.Grpclb
+	}
+	return nil
+}
+
+func (x *LoadBalancingConfig) GetPriority() *PriorityLoadBalancingPolicyConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_Priority); ok {
+		return x.Priority
+	}
+	return nil
 }
 
-func (m *LoadBalancingConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadBalancingConfig.Unmarshal(m, b)
+func (x *LoadBalancingConfig) GetWeightedTarget() *WeightedTargetLoadBalancingPolicyConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_WeightedTarget); ok {
+		return x.WeightedTarget
+	}
+	return nil
 }
-func (m *LoadBalancingConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadBalancingConfig.Marshal(b, m, deterministic)
+
+func (x *LoadBalancingConfig) GetCds() *CdsConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_Cds); ok {
+		return x.Cds
+	}
+	return nil
 }
-func (m *LoadBalancingConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadBalancingConfig.Merge(m, src)
+
+func (x *LoadBalancingConfig) GetEds() *EdsLoadBalancingPolicyConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_Eds); ok {
+		return x.Eds
+	}
+	return nil
 }
-func (m *LoadBalancingConfig) XXX_Size() int {
-	return xxx_messageInfo_LoadBalancingConfig.Size(m)
+
+func (x *LoadBalancingConfig) GetLrs() *LrsLoadBalancingPolicyConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_Lrs); ok {
+		return x.Lrs
+	}
+	return nil
 }
-func (m *LoadBalancingConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadBalancingConfig.DiscardUnknown(m)
+
+// Deprecated: Do not use.
+func (x *LoadBalancingConfig) GetXds() *XdsConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_Xds); ok {
+		return x.Xds
+	}
+	return nil
 }
 
-var xxx_messageInfo_LoadBalancingConfig proto.InternalMessageInfo
+// Deprecated: Do not use.
+func (x *LoadBalancingConfig) GetXdsExperimental() *XdsConfig {
+	if x, ok := x.GetPolicy().(*LoadBalancingConfig_XdsExperimental); ok {
+		return x.XdsExperimental
+	}
+	return nil
+}
 
 type isLoadBalancingConfig_Policy interface {
 	isLoadBalancingConfig_Policy()
@@ -1200,6 +1023,9 @@ type LoadBalancingConfig_RoundRobin struct {
 }
 
 type LoadBalancingConfig_Grpclb struct {
+	// gRPC lookaside load balancing.
+	// This will eventually be deprecated by the new xDS-based local
+	// balancing policy.
 	Grpclb *GrpcLbConfig `protobuf:"bytes,3,opt,name=grpclb,proto3,oneof"`
 }
 
@@ -1212,6 +1038,10 @@ type LoadBalancingConfig_WeightedTarget struct {
 }
 
 type LoadBalancingConfig_Cds struct {
+	// EXPERIMENTAL -- DO NOT USE
+	// xDS-based load balancing.
+	// The policy is known as xds_experimental while it is under development.
+	// It will be renamed to xds once it is ready for public use.
 	Cds *CdsConfig `protobuf:"bytes,6,opt,name=cds,proto3,oneof"`
 }
 
@@ -1224,10 +1054,15 @@ type LoadBalancingConfig_Lrs struct {
 }
 
 type LoadBalancingConfig_Xds struct {
+	// Deprecated: Do not use.
 	Xds *XdsConfig `protobuf:"bytes,2,opt,name=xds,proto3,oneof"`
 }
 
 type LoadBalancingConfig_XdsExperimental struct {
+	// TODO(rekarthik): Deprecate this field after the xds policy
+	// is ready for public use.
+	//
+	// Deprecated: Do not use.
 	XdsExperimental *XdsConfig `protobuf:"bytes,5,opt,name=xds_experimental,proto3,oneof"`
 }
 
@@ -1251,179 +1086,519 @@ func (*LoadBalancingConfig_Xds) isLoadBalancingConfig_Policy() {}
 
 func (*LoadBalancingConfig_XdsExperimental) isLoadBalancingConfig_Policy() {}
 
-func (m *LoadBalancingConfig) GetPolicy() isLoadBalancingConfig_Policy {
-	if m != nil {
-		return m.Policy
-	}
-	return nil
+// A ServiceConfig represents information about a service but is not specific to
+// any name resolver.
+type ServiceConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Deprecated: Do not use.
+	LoadBalancingPolicy ServiceConfig_LoadBalancingPolicy `protobuf:"varint,1,opt,name=load_balancing_policy,json=loadBalancingPolicy,proto3,enum=grpc.service_config.ServiceConfig_LoadBalancingPolicy" json:"load_balancing_policy,omitempty"`
+	// Multiple LB policies can be specified; clients will iterate through
+	// the list in order and stop at the first policy that they support. If none
+	// are supported, the service config is considered invalid.
+	LoadBalancingConfig []*LoadBalancingConfig `protobuf:"bytes,4,rep,name=load_balancing_config,json=loadBalancingConfig,proto3" json:"load_balancing_config,omitempty"`
+	// Per-method configuration.
+	MethodConfig      []*MethodConfig                      `protobuf:"bytes,2,rep,name=method_config,json=methodConfig,proto3" json:"method_config,omitempty"`
+	RetryThrottling   *ServiceConfig_RetryThrottlingPolicy `protobuf:"bytes,3,opt,name=retry_throttling,json=retryThrottling,proto3" json:"retry_throttling,omitempty"`
+	HealthCheckConfig *ServiceConfig_HealthCheckConfig     `protobuf:"bytes,5,opt,name=health_check_config,json=healthCheckConfig,proto3" json:"health_check_config,omitempty"`
 }
 
-func (m *LoadBalancingConfig) GetPickFirst() *PickFirstConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_PickFirst); ok {
-		return x.PickFirst
+func (x *ServiceConfig) Reset() {
+	*x = ServiceConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *LoadBalancingConfig) GetRoundRobin() *RoundRobinConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_RoundRobin); ok {
-		return x.RoundRobin
-	}
-	return nil
+func (x *ServiceConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *LoadBalancingConfig) GetGrpclb() *GrpcLbConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Grpclb); ok {
-		return x.Grpclb
+func (*ServiceConfig) ProtoMessage() {}
+
+func (x *ServiceConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
-	return nil
+	return mi.MessageOf(x)
 }
 
-func (m *LoadBalancingConfig) GetPriority() *PriorityLoadBalancingPolicyConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Priority); ok {
-		return x.Priority
-	}
-	return nil
+// Deprecated: Use ServiceConfig.ProtoReflect.Descriptor instead.
+func (*ServiceConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{11}
 }
 
-func (m *LoadBalancingConfig) GetWeightedTarget() *WeightedTargetLoadBalancingPolicyConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_WeightedTarget); ok {
-		return x.WeightedTarget
+// Deprecated: Do not use.
+func (x *ServiceConfig) GetLoadBalancingPolicy() ServiceConfig_LoadBalancingPolicy {
+	if x != nil {
+		return x.LoadBalancingPolicy
 	}
-	return nil
+	return ServiceConfig_UNSPECIFIED
 }
 
-func (m *LoadBalancingConfig) GetCds() *CdsConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Cds); ok {
-		return x.Cds
+func (x *ServiceConfig) GetLoadBalancingConfig() []*LoadBalancingConfig {
+	if x != nil {
+		return x.LoadBalancingConfig
 	}
 	return nil
 }
 
-func (m *LoadBalancingConfig) GetEds() *EdsLoadBalancingPolicyConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Eds); ok {
-		return x.Eds
+func (x *ServiceConfig) GetMethodConfig() []*MethodConfig {
+	if x != nil {
+		return x.MethodConfig
 	}
 	return nil
 }
 
-func (m *LoadBalancingConfig) GetLrs() *LrsLoadBalancingPolicyConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Lrs); ok {
-		return x.Lrs
+func (x *ServiceConfig) GetRetryThrottling() *ServiceConfig_RetryThrottlingPolicy {
+	if x != nil {
+		return x.RetryThrottling
 	}
 	return nil
 }
 
-// Deprecated: Do not use.
-func (m *LoadBalancingConfig) GetXds() *XdsConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_Xds); ok {
-		return x.Xds
+func (x *ServiceConfig) GetHealthCheckConfig() *ServiceConfig_HealthCheckConfig {
+	if x != nil {
+		return x.HealthCheckConfig
 	}
 	return nil
 }
 
-// Deprecated: Do not use.
-func (m *LoadBalancingConfig) GetXdsExperimental() *XdsConfig {
-	if x, ok := m.GetPolicy().(*LoadBalancingConfig_XdsExperimental); ok {
-		return x.XdsExperimental
+// The names of the methods to which this configuration applies.
+// - MethodConfig without names (empty list) will be skipped.
+// - Each name entry must be unique across the entire ServiceConfig.
+// - If the 'method' field is empty, this MethodConfig specifies the defaults
+//   for all methods for the specified service.
+// - If the 'service' field is empty, the 'method' field must be empty, and
+//   this MethodConfig specifies the default for all methods (it's the default
+//   config).
+//
+// When determining which MethodConfig to use for a given RPC, the most
+// specific match wins. For example, let's say that the service config
+// contains the following MethodConfig entries:
+//
+// method_config { name { } ... }
+// method_config { name { service: "MyService" } ... }
+// method_config { name { service: "MyService" method: "Foo" } ... }
+//
+// MyService/Foo will use the third entry, because it exactly matches the
+// service and method name. MyService/Bar will use the second entry, because
+// it provides the default for all methods of MyService. AnotherService/Baz
+// will use the first entry, because it doesn't match the other two.
+//
+// In JSON representation, value "", value `null`, and not present are the
+// same. The following are the same Name:
+// - { "service": "s" }
+// - { "service": "s", "method": null }
+// - { "service": "s", "method": "" }
+type MethodConfig_Name struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` // Required. Includes proto package name.
+	Method  string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
+}
+
+func (x *MethodConfig_Name) Reset() {
+	*x = MethodConfig_Name{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*LoadBalancingConfig) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*LoadBalancingConfig_PickFirst)(nil),
-		(*LoadBalancingConfig_RoundRobin)(nil),
-		(*LoadBalancingConfig_Grpclb)(nil),
-		(*LoadBalancingConfig_Priority)(nil),
-		(*LoadBalancingConfig_WeightedTarget)(nil),
-		(*LoadBalancingConfig_Cds)(nil),
-		(*LoadBalancingConfig_Eds)(nil),
-		(*LoadBalancingConfig_Lrs)(nil),
-		(*LoadBalancingConfig_Xds)(nil),
-		(*LoadBalancingConfig_XdsExperimental)(nil),
+func (x *MethodConfig_Name) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MethodConfig_Name) ProtoMessage() {}
+
+func (x *MethodConfig_Name) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
+	return mi.MessageOf(x)
 }
 
-// A ServiceConfig represents information about a service but is not specific to
-// any name resolver.
-type ServiceConfig struct {
-	LoadBalancingPolicy ServiceConfig_LoadBalancingPolicy `protobuf:"varint,1,opt,name=load_balancing_policy,json=loadBalancingPolicy,proto3,enum=grpc.service_config.ServiceConfig_LoadBalancingPolicy" json:"load_balancing_policy,omitempty"` // Deprecated: Do not use.
-	// Multiple LB policies can be specified; clients will iterate through
-	// the list in order and stop at the first policy that they support. If none
-	// are supported, the service config is considered invalid.
-	LoadBalancingConfig []*LoadBalancingConfig `protobuf:"bytes,4,rep,name=load_balancing_config,json=loadBalancingConfig,proto3" json:"load_balancing_config,omitempty"`
-	// Per-method configuration.
-	MethodConfig         []*MethodConfig                      `protobuf:"bytes,2,rep,name=method_config,json=methodConfig,proto3" json:"method_config,omitempty"`
-	RetryThrottling      *ServiceConfig_RetryThrottlingPolicy `protobuf:"bytes,3,opt,name=retry_throttling,json=retryThrottling,proto3" json:"retry_throttling,omitempty"`
-	HealthCheckConfig    *ServiceConfig_HealthCheckConfig     `protobuf:"bytes,5,opt,name=health_check_config,json=healthCheckConfig,proto3" json:"health_check_config,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                             `json:"-"`
-	XXX_unrecognized     []byte                               `json:"-"`
-	XXX_sizecache        int32                                `json:"-"`
+// Deprecated: Use MethodConfig_Name.ProtoReflect.Descriptor instead.
+func (*MethodConfig_Name) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{0, 0}
 }
 
-func (m *ServiceConfig) Reset()         { *m = ServiceConfig{} }
-func (m *ServiceConfig) String() string { return proto.CompactTextString(m) }
-func (*ServiceConfig) ProtoMessage()    {}
-func (*ServiceConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{11}
+func (x *MethodConfig_Name) GetService() string {
+	if x != nil {
+		return x.Service
+	}
+	return ""
 }
 
-func (m *ServiceConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServiceConfig.Unmarshal(m, b)
+func (x *MethodConfig_Name) GetMethod() string {
+	if x != nil {
+		return x.Method
+	}
+	return ""
 }
-func (m *ServiceConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServiceConfig.Marshal(b, m, deterministic)
+
+// The retry policy for outgoing RPCs.
+type MethodConfig_RetryPolicy struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The maximum number of RPC attempts, including the original attempt.
+	//
+	// This field is required and must be greater than 1.
+	// Any value greater than 5 will be treated as if it were 5.
+	MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"`
+	// Exponential backoff parameters. The initial retry attempt will occur at
+	// random(0, initial_backoff). In general, the nth attempt will occur at
+	// random(0,
+	//   min(initial_backoff*backoff_multiplier**(n-1), max_backoff)).
+	// Required. Must be greater than zero.
+	InitialBackoff *duration.Duration `protobuf:"bytes,2,opt,name=initial_backoff,json=initialBackoff,proto3" json:"initial_backoff,omitempty"`
+	// Required. Must be greater than zero.
+	MaxBackoff        *duration.Duration `protobuf:"bytes,3,opt,name=max_backoff,json=maxBackoff,proto3" json:"max_backoff,omitempty"`
+	BackoffMultiplier float32            `protobuf:"fixed32,4,opt,name=backoff_multiplier,json=backoffMultiplier,proto3" json:"backoff_multiplier,omitempty"` // Required. Must be greater than zero.
+	// The set of status codes which may be retried.
+	//
+	// This field is required and must be non-empty.
+	RetryableStatusCodes []code.Code `protobuf:"varint,5,rep,packed,name=retryable_status_codes,json=retryableStatusCodes,proto3,enum=google.rpc.Code" json:"retryable_status_codes,omitempty"`
 }
-func (m *ServiceConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServiceConfig.Merge(m, src)
+
+func (x *MethodConfig_RetryPolicy) Reset() {
+	*x = MethodConfig_RetryPolicy{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *ServiceConfig) XXX_Size() int {
-	return xxx_messageInfo_ServiceConfig.Size(m)
+
+func (x *MethodConfig_RetryPolicy) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServiceConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServiceConfig.DiscardUnknown(m)
+
+func (*MethodConfig_RetryPolicy) ProtoMessage() {}
+
+func (x *MethodConfig_RetryPolicy) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServiceConfig proto.InternalMessageInfo
+// Deprecated: Use MethodConfig_RetryPolicy.ProtoReflect.Descriptor instead.
+func (*MethodConfig_RetryPolicy) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{0, 1}
+}
 
-// Deprecated: Do not use.
-func (m *ServiceConfig) GetLoadBalancingPolicy() ServiceConfig_LoadBalancingPolicy {
-	if m != nil {
-		return m.LoadBalancingPolicy
+func (x *MethodConfig_RetryPolicy) GetMaxAttempts() uint32 {
+	if x != nil {
+		return x.MaxAttempts
 	}
-	return ServiceConfig_UNSPECIFIED
+	return 0
 }
 
-func (m *ServiceConfig) GetLoadBalancingConfig() []*LoadBalancingConfig {
-	if m != nil {
-		return m.LoadBalancingConfig
+func (x *MethodConfig_RetryPolicy) GetInitialBackoff() *duration.Duration {
+	if x != nil {
+		return x.InitialBackoff
 	}
 	return nil
 }
 
-func (m *ServiceConfig) GetMethodConfig() []*MethodConfig {
-	if m != nil {
-		return m.MethodConfig
+func (x *MethodConfig_RetryPolicy) GetMaxBackoff() *duration.Duration {
+	if x != nil {
+		return x.MaxBackoff
 	}
 	return nil
 }
 
-func (m *ServiceConfig) GetRetryThrottling() *ServiceConfig_RetryThrottlingPolicy {
-	if m != nil {
-		return m.RetryThrottling
+func (x *MethodConfig_RetryPolicy) GetBackoffMultiplier() float32 {
+	if x != nil {
+		return x.BackoffMultiplier
+	}
+	return 0
+}
+
+func (x *MethodConfig_RetryPolicy) GetRetryableStatusCodes() []code.Code {
+	if x != nil {
+		return x.RetryableStatusCodes
 	}
 	return nil
 }
 
-func (m *ServiceConfig) GetHealthCheckConfig() *ServiceConfig_HealthCheckConfig {
-	if m != nil {
-		return m.HealthCheckConfig
+// The hedging policy for outgoing RPCs. Hedged RPCs may execute more than
+// once on the server, so only idempotent methods should specify a hedging
+// policy.
+type MethodConfig_HedgingPolicy struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The hedging policy will send up to max_requests RPCs.
+	// This number represents the total number of all attempts, including
+	// the original attempt.
+	//
+	// This field is required and must be greater than 1.
+	// Any value greater than 5 will be treated as if it were 5.
+	MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"`
+	// The first RPC will be sent immediately, but the max_requests-1 subsequent
+	// hedged RPCs will be sent at intervals of every hedging_delay. Set this
+	// to 0 to immediately send all max_requests RPCs.
+	HedgingDelay *duration.Duration `protobuf:"bytes,2,opt,name=hedging_delay,json=hedgingDelay,proto3" json:"hedging_delay,omitempty"`
+	// The set of status codes which indicate other hedged RPCs may still
+	// succeed. If a non-fatal status code is returned by the server, hedged
+	// RPCs will continue. Otherwise, outstanding requests will be canceled and
+	// the error returned to the client application layer.
+	//
+	// This field is optional.
+	NonFatalStatusCodes []code.Code `protobuf:"varint,3,rep,packed,name=non_fatal_status_codes,json=nonFatalStatusCodes,proto3,enum=google.rpc.Code" json:"non_fatal_status_codes,omitempty"`
+}
+
+func (x *MethodConfig_HedgingPolicy) Reset() {
+	*x = MethodConfig_HedgingPolicy{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *MethodConfig_HedgingPolicy) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MethodConfig_HedgingPolicy) ProtoMessage() {}
+
+func (x *MethodConfig_HedgingPolicy) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MethodConfig_HedgingPolicy.ProtoReflect.Descriptor instead.
+func (*MethodConfig_HedgingPolicy) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{0, 2}
+}
+
+func (x *MethodConfig_HedgingPolicy) GetMaxAttempts() uint32 {
+	if x != nil {
+		return x.MaxAttempts
+	}
+	return 0
+}
+
+func (x *MethodConfig_HedgingPolicy) GetHedgingDelay() *duration.Duration {
+	if x != nil {
+		return x.HedgingDelay
 	}
 	return nil
 }
 
+func (x *MethodConfig_HedgingPolicy) GetNonFatalStatusCodes() []code.Code {
+	if x != nil {
+		return x.NonFatalStatusCodes
+	}
+	return nil
+}
+
+// A map of name to child policy configuration.
+// The names are used to allow the priority policy to update
+// existing child policies instead of creating new ones every
+// time it receives a config update.
+type PriorityLoadBalancingPolicyConfig_Child struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Config []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=config,proto3" json:"config,omitempty"`
+}
+
+func (x *PriorityLoadBalancingPolicyConfig_Child) Reset() {
+	*x = PriorityLoadBalancingPolicyConfig_Child{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PriorityLoadBalancingPolicyConfig_Child) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PriorityLoadBalancingPolicyConfig_Child) ProtoMessage() {}
+
+func (x *PriorityLoadBalancingPolicyConfig_Child) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PriorityLoadBalancingPolicyConfig_Child.ProtoReflect.Descriptor instead.
+func (*PriorityLoadBalancingPolicyConfig_Child) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{3, 0}
+}
+
+func (x *PriorityLoadBalancingPolicyConfig_Child) GetConfig() []*LoadBalancingConfig {
+	if x != nil {
+		return x.Config
+	}
+	return nil
+}
+
+type WeightedTargetLoadBalancingPolicyConfig_Target struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Weight      uint32                 `protobuf:"varint,1,opt,name=weight,proto3" json:"weight,omitempty"`
+	ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,2,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"`
+}
+
+func (x *WeightedTargetLoadBalancingPolicyConfig_Target) Reset() {
+	*x = WeightedTargetLoadBalancingPolicyConfig_Target{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[17]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *WeightedTargetLoadBalancingPolicyConfig_Target) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WeightedTargetLoadBalancingPolicyConfig_Target) ProtoMessage() {}
+
+func (x *WeightedTargetLoadBalancingPolicyConfig_Target) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[17]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WeightedTargetLoadBalancingPolicyConfig_Target.ProtoReflect.Descriptor instead.
+func (*WeightedTargetLoadBalancingPolicyConfig_Target) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{4, 0}
+}
+
+func (x *WeightedTargetLoadBalancingPolicyConfig_Target) GetWeight() uint32 {
+	if x != nil {
+		return x.Weight
+	}
+	return 0
+}
+
+func (x *WeightedTargetLoadBalancingPolicyConfig_Target) GetChildPolicy() []*LoadBalancingConfig {
+	if x != nil {
+		return x.ChildPolicy
+	}
+	return nil
+}
+
+// The locality for which this policy will report load.  Required.
+type LrsLoadBalancingPolicyConfig_Locality struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Region  string `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"`
+	Zone    string `protobuf:"bytes,2,opt,name=zone,proto3" json:"zone,omitempty"`
+	Subzone string `protobuf:"bytes,3,opt,name=subzone,proto3" json:"subzone,omitempty"`
+}
+
+func (x *LrsLoadBalancingPolicyConfig_Locality) Reset() {
+	*x = LrsLoadBalancingPolicyConfig_Locality{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[19]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LrsLoadBalancingPolicyConfig_Locality) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LrsLoadBalancingPolicyConfig_Locality) ProtoMessage() {}
+
+func (x *LrsLoadBalancingPolicyConfig_Locality) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[19]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LrsLoadBalancingPolicyConfig_Locality.ProtoReflect.Descriptor instead.
+func (*LrsLoadBalancingPolicyConfig_Locality) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{9, 0}
+}
+
+func (x *LrsLoadBalancingPolicyConfig_Locality) GetRegion() string {
+	if x != nil {
+		return x.Region
+	}
+	return ""
+}
+
+func (x *LrsLoadBalancingPolicyConfig_Locality) GetZone() string {
+	if x != nil {
+		return x.Zone
+	}
+	return ""
+}
+
+func (x *LrsLoadBalancingPolicyConfig_Locality) GetSubzone() string {
+	if x != nil {
+		return x.Subzone
+	}
+	return ""
+}
+
 // If a RetryThrottlingPolicy is provided, gRPC will automatically throttle
 // retry attempts and hedged RPCs when the client's ratio of failures to
 // successes exceeds a threshold.
@@ -1438,6 +1613,10 @@ func (m *ServiceConfig) GetHealthCheckConfig() *ServiceConfig_HealthCheckConfig
 // If token_count is less than or equal to max_tokens / 2, then RPCs will not
 // be retried and hedged RPCs will not be sent.
 type ServiceConfig_RetryThrottlingPolicy struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The number of tokens starts at max_tokens. The token_count will always be
 	// between 0 and max_tokens.
 	//
@@ -1448,221 +1627,793 @@ type ServiceConfig_RetryThrottlingPolicy struct {
 	//
 	// This field is required and must be greater than zero. Up to 3 decimal
 	// places are supported.
-	TokenRatio           float32  `protobuf:"fixed32,2,opt,name=token_ratio,json=tokenRatio,proto3" json:"token_ratio,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	TokenRatio float32 `protobuf:"fixed32,2,opt,name=token_ratio,json=tokenRatio,proto3" json:"token_ratio,omitempty"`
 }
 
-func (m *ServiceConfig_RetryThrottlingPolicy) Reset()         { *m = ServiceConfig_RetryThrottlingPolicy{} }
-func (m *ServiceConfig_RetryThrottlingPolicy) String() string { return proto.CompactTextString(m) }
-func (*ServiceConfig_RetryThrottlingPolicy) ProtoMessage()    {}
-func (*ServiceConfig_RetryThrottlingPolicy) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{11, 0}
+func (x *ServiceConfig_RetryThrottlingPolicy) Reset() {
+	*x = ServiceConfig_RetryThrottlingPolicy{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[20]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Unmarshal(m, b)
+func (x *ServiceConfig_RetryThrottlingPolicy) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Marshal(b, m, deterministic)
-}
-func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Merge(m, src)
-}
-func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Size() int {
-	return xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Size(m)
-}
-func (m *ServiceConfig_RetryThrottlingPolicy) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.DiscardUnknown(m)
+
+func (*ServiceConfig_RetryThrottlingPolicy) ProtoMessage() {}
+
+func (x *ServiceConfig_RetryThrottlingPolicy) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[20]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy proto.InternalMessageInfo
+// Deprecated: Use ServiceConfig_RetryThrottlingPolicy.ProtoReflect.Descriptor instead.
+func (*ServiceConfig_RetryThrottlingPolicy) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{11, 0}
+}
 
-func (m *ServiceConfig_RetryThrottlingPolicy) GetMaxTokens() uint32 {
-	if m != nil {
-		return m.MaxTokens
+func (x *ServiceConfig_RetryThrottlingPolicy) GetMaxTokens() uint32 {
+	if x != nil {
+		return x.MaxTokens
 	}
 	return 0
 }
 
-func (m *ServiceConfig_RetryThrottlingPolicy) GetTokenRatio() float32 {
-	if m != nil {
-		return m.TokenRatio
+func (x *ServiceConfig_RetryThrottlingPolicy) GetTokenRatio() float32 {
+	if x != nil {
+		return x.TokenRatio
 	}
 	return 0
 }
 
 type ServiceConfig_HealthCheckConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Service name to use in the health-checking request.
-	ServiceName          *wrappers.StringValue `protobuf:"bytes,1,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}              `json:"-"`
-	XXX_unrecognized     []byte                `json:"-"`
-	XXX_sizecache        int32                 `json:"-"`
+	ServiceName *wrappers.StringValue `protobuf:"bytes,1,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
 }
 
-func (m *ServiceConfig_HealthCheckConfig) Reset()         { *m = ServiceConfig_HealthCheckConfig{} }
-func (m *ServiceConfig_HealthCheckConfig) String() string { return proto.CompactTextString(m) }
-func (*ServiceConfig_HealthCheckConfig) ProtoMessage()    {}
-func (*ServiceConfig_HealthCheckConfig) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e32d3cb2c41c77ce, []int{11, 1}
+func (x *ServiceConfig_HealthCheckConfig) Reset() {
+	*x = ServiceConfig_HealthCheckConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_service_config_service_config_proto_msgTypes[21]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServiceConfig_HealthCheckConfig) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServiceConfig_HealthCheckConfig.Unmarshal(m, b)
+func (x *ServiceConfig_HealthCheckConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServiceConfig_HealthCheckConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServiceConfig_HealthCheckConfig.Marshal(b, m, deterministic)
-}
-func (m *ServiceConfig_HealthCheckConfig) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServiceConfig_HealthCheckConfig.Merge(m, src)
-}
-func (m *ServiceConfig_HealthCheckConfig) XXX_Size() int {
-	return xxx_messageInfo_ServiceConfig_HealthCheckConfig.Size(m)
-}
-func (m *ServiceConfig_HealthCheckConfig) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServiceConfig_HealthCheckConfig.DiscardUnknown(m)
+
+func (*ServiceConfig_HealthCheckConfig) ProtoMessage() {}
+
+func (x *ServiceConfig_HealthCheckConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_service_config_service_config_proto_msgTypes[21]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServiceConfig_HealthCheckConfig proto.InternalMessageInfo
+// Deprecated: Use ServiceConfig_HealthCheckConfig.ProtoReflect.Descriptor instead.
+func (*ServiceConfig_HealthCheckConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{11, 1}
+}
 
-func (m *ServiceConfig_HealthCheckConfig) GetServiceName() *wrappers.StringValue {
-	if m != nil {
-		return m.ServiceName
+func (x *ServiceConfig_HealthCheckConfig) GetServiceName() *wrappers.StringValue {
+	if x != nil {
+		return x.ServiceName
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.service_config.ServiceConfig_LoadBalancingPolicy", ServiceConfig_LoadBalancingPolicy_name, ServiceConfig_LoadBalancingPolicy_value)
-	proto.RegisterType((*MethodConfig)(nil), "grpc.service_config.MethodConfig")
-	proto.RegisterType((*MethodConfig_Name)(nil), "grpc.service_config.MethodConfig.Name")
-	proto.RegisterType((*MethodConfig_RetryPolicy)(nil), "grpc.service_config.MethodConfig.RetryPolicy")
-	proto.RegisterType((*MethodConfig_HedgingPolicy)(nil), "grpc.service_config.MethodConfig.HedgingPolicy")
-	proto.RegisterType((*PickFirstConfig)(nil), "grpc.service_config.PickFirstConfig")
-	proto.RegisterType((*RoundRobinConfig)(nil), "grpc.service_config.RoundRobinConfig")
-	proto.RegisterType((*PriorityLoadBalancingPolicyConfig)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig")
-	proto.RegisterMapType((map[string]*PriorityLoadBalancingPolicyConfig_Child)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig.ChildrenEntry")
-	proto.RegisterType((*PriorityLoadBalancingPolicyConfig_Child)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig.Child")
-	proto.RegisterType((*WeightedTargetLoadBalancingPolicyConfig)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig")
-	proto.RegisterMapType((map[string]*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.TargetsEntry")
-	proto.RegisterType((*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.Target")
-	proto.RegisterType((*GrpcLbConfig)(nil), "grpc.service_config.GrpcLbConfig")
-	proto.RegisterType((*CdsConfig)(nil), "grpc.service_config.CdsConfig")
-	proto.RegisterType((*XdsConfig)(nil), "grpc.service_config.XdsConfig")
-	proto.RegisterType((*EdsLoadBalancingPolicyConfig)(nil), "grpc.service_config.EdsLoadBalancingPolicyConfig")
-	proto.RegisterType((*LrsLoadBalancingPolicyConfig)(nil), "grpc.service_config.LrsLoadBalancingPolicyConfig")
-	proto.RegisterType((*LrsLoadBalancingPolicyConfig_Locality)(nil), "grpc.service_config.LrsLoadBalancingPolicyConfig.Locality")
-	proto.RegisterType((*LoadBalancingConfig)(nil), "grpc.service_config.LoadBalancingConfig")
-	proto.RegisterType((*ServiceConfig)(nil), "grpc.service_config.ServiceConfig")
-	proto.RegisterType((*ServiceConfig_RetryThrottlingPolicy)(nil), "grpc.service_config.ServiceConfig.RetryThrottlingPolicy")
-	proto.RegisterType((*ServiceConfig_HealthCheckConfig)(nil), "grpc.service_config.ServiceConfig.HealthCheckConfig")
-}
-
-func init() {
-	proto.RegisterFile("grpc/service_config/service_config.proto", fileDescriptor_e32d3cb2c41c77ce)
-}
-
-var fileDescriptor_e32d3cb2c41c77ce = []byte{
-	// 1589 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0x1a, 0x47,
-	0x16, 0x16, 0xa0, 0x1b, 0x07, 0x84, 0x50, 0x6b, 0x25, 0x61, 0xca, 0xab, 0x95, 0xa8, 0xf5, 0x9a,
-	0x17, 0xa3, 0x5a, 0x79, 0xcb, 0xeb, 0xd2, 0x6e, 0xed, 0x3a, 0xe8, 0x12, 0x54, 0x91, 0x65, 0xd2,
-	0x92, 0x6f, 0x55, 0xa9, 0x1a, 0x0d, 0x33, 0x0d, 0x4c, 0x69, 0x66, 0x9a, 0xf4, 0x34, 0x16, 0xe4,
-	0x21, 0x79, 0xcd, 0x2f, 0x49, 0x7e, 0x43, 0xfe, 0x49, 0x9e, 0x52, 0x79, 0xcf, 0x3f, 0xc8, 0x43,
-	0x2a, 0xd5, 0x97, 0x41, 0x0c, 0x8c, 0x10, 0xb2, 0xfc, 0xc6, 0x9c, 0xee, 0xef, 0x3b, 0xd7, 0x3e,
-	0x7d, 0x1a, 0x28, 0xb7, 0x58, 0xc7, 0xda, 0x09, 0x08, 0xfb, 0xe0, 0x58, 0xc4, 0xb0, 0xa8, 0xdf,
-	0x74, 0x5a, 0x23, 0x9f, 0x95, 0x0e, 0xa3, 0x9c, 0xa2, 0x55, 0xb1, 0xb3, 0x12, 0x5d, 0x2a, 0x6e,
-	0xb6, 0x28, 0x6d, 0xb9, 0x64, 0x47, 0x6e, 0x69, 0x74, 0x9b, 0x3b, 0x76, 0x97, 0x99, 0xdc, 0xa1,
-	0xbe, 0x02, 0x8d, 0xaf, 0x5f, 0x31, 0xb3, 0xd3, 0x21, 0x2c, 0xd0, 0xeb, 0x6b, 0x7a, 0x5d, 0x18,
-	0x61, 0x51, 0x9b, 0x28, 0x71, 0xe9, 0x97, 0x45, 0xc8, 0xbe, 0x24, 0xbc, 0x4d, 0xed, 0x7d, 0xa9,
-	0x07, 0xed, 0xc1, 0xac, 0x6f, 0x7a, 0xa4, 0x90, 0xd8, 0x4a, 0x95, 0x33, 0xbb, 0xff, 0xa8, 0xc4,
-	0xd8, 0x52, 0x19, 0x06, 0x54, 0x4e, 0x4d, 0x8f, 0x60, 0x89, 0x41, 0x2f, 0x20, 0x77, 0x65, 0x3a,
-	0xdc, 0x68, 0x52, 0x66, 0x30, 0x62, 0xda, 0xfd, 0x42, 0x72, 0x2b, 0x51, 0xce, 0xec, 0x16, 0x2b,
-	0x4a, 0x79, 0x25, 0x34, 0xae, 0x52, 0xa5, 0xd4, 0x7d, 0x63, 0xba, 0x5d, 0x82, 0xb3, 0x02, 0x71,
-	0x44, 0x19, 0x16, 0xfb, 0xd1, 0x53, 0x58, 0xe0, 0x8e, 0x47, 0x68, 0x97, 0x17, 0x52, 0x12, 0xfa,
-	0x60, 0x0c, 0x7a, 0xa0, 0xfd, 0xc6, 0xe1, 0x4e, 0xf4, 0x16, 0x1e, 0x78, 0x66, 0xcf, 0x60, 0xe4,
-	0xeb, 0x2e, 0x09, 0xb8, 0xe1, 0x91, 0x20, 0x30, 0x5b, 0xc4, 0x68, 0xf4, 0x39, 0x09, 0x0a, 0xb3,
-	0x92, 0xe6, 0xe1, 0x18, 0xcd, 0xeb, 0x63, 0x9f, 0x3f, 0xdd, 0x55, 0x36, 0xac, 0x7b, 0x66, 0x0f,
-	0x2b, 0xf4, 0x4b, 0x05, 0xae, 0x0a, 0x2c, 0x7a, 0x0f, 0x45, 0x45, 0x1c, 0x74, 0xa8, 0x1f, 0x90,
-	0x11, 0xe6, 0xb9, 0x29, 0x98, 0x37, 0x24, 0xb3, 0x82, 0x47, 0xa8, 0x31, 0x64, 0x19, 0xe1, 0xac,
-	0x6f, 0x74, 0xa8, 0xeb, 0x58, 0xfd, 0xc2, 0xbc, 0x24, 0x7b, 0x72, 0x7b, 0xb8, 0xb1, 0x40, 0xd5,
-	0x25, 0xa8, 0x36, 0x83, 0x33, 0xec, 0xfa, 0x13, 0xbd, 0x83, 0x5c, 0x9b, 0xd8, 0x2d, 0xc7, 0x6f,
-	0x85, 0xac, 0x0b, 0x92, 0x75, 0xe7, 0x76, 0xd6, 0x9a, 0xc2, 0x0d, 0x78, 0x97, 0xda, 0xc3, 0x82,
-	0xe2, 0x73, 0x98, 0x15, 0x69, 0x46, 0x05, 0x58, 0xd0, 0x2c, 0x85, 0xc4, 0x56, 0xa2, 0x9c, 0xc6,
-	0xe1, 0x27, 0x5a, 0x87, 0x79, 0x4f, 0x12, 0xca, 0x94, 0xa7, 0xb1, 0xfe, 0x2a, 0xfe, 0x98, 0x84,
-	0xcc, 0x90, 0xc9, 0x68, 0x1b, 0xb2, 0x22, 0xa4, 0x26, 0xe7, 0xc4, 0xeb, 0xf0, 0x40, 0xd2, 0x2c,
-	0xe1, 0x8c, 0x67, 0xf6, 0x3e, 0xd3, 0x22, 0x54, 0x85, 0x65, 0xc7, 0x77, 0xb8, 0x63, 0xba, 0x46,
-	0xc3, 0xb4, 0x2e, 0x69, 0xb3, 0xa9, 0xcb, 0x68, 0x42, 0x2d, 0xe4, 0x34, 0xa2, 0xaa, 0x00, 0x68,
-	0x0f, 0x04, 0xe5, 0x00, 0x7f, 0x6b, 0x2d, 0x81, 0x67, 0xf6, 0x42, 0xec, 0x13, 0x40, 0x1a, 0x67,
-	0x78, 0x5d, 0x97, 0x3b, 0x1d, 0xd7, 0x21, 0x4c, 0xd6, 0x51, 0x12, 0xaf, 0xe8, 0x95, 0x97, 0x83,
-	0x05, 0x74, 0x04, 0xeb, 0x32, 0x09, 0x66, 0xc3, 0x25, 0x46, 0xc0, 0x4d, 0xde, 0x0d, 0x0c, 0x71,
-	0xc0, 0x44, 0x81, 0xa4, 0xca, 0xb9, 0xdd, 0x7c, 0xa8, 0x55, 0xe4, 0x60, 0x9f, 0xda, 0x04, 0xff,
-	0x65, 0xb0, 0xff, 0x4c, 0x6e, 0x17, 0xc2, 0xa0, 0xf8, 0x53, 0x02, 0x96, 0x22, 0x69, 0x98, 0x26,
-	0x56, 0xff, 0x83, 0x30, 0x53, 0x86, 0x4d, 0x5c, 0xb3, 0x7f, 0x7b, 0xa4, 0xb2, 0x7a, 0xff, 0x81,
-	0xd8, 0x8e, 0x0e, 0x61, 0xdd, 0xa7, 0xbe, 0xd1, 0x34, 0xb9, 0xe9, 0x46, 0x8d, 0x4f, 0xdd, 0x60,
-	0xfc, 0xaa, 0x4f, 0xfd, 0x23, 0xb1, 0x7d, 0xc8, 0xf6, 0xea, 0x03, 0xd8, 0x50, 0xd5, 0x4c, 0x99,
-	0x11, 0x2d, 0xc1, 0xd2, 0x0a, 0x2c, 0xd7, 0x1d, 0xeb, 0xf2, 0xc8, 0x61, 0x01, 0x57, 0xc5, 0x56,
-	0x42, 0x90, 0xc7, 0xb4, 0xeb, 0xdb, 0x98, 0x36, 0x1c, 0x5f, 0xcb, 0x7e, 0x4b, 0xc2, 0x76, 0x9d,
-	0x39, 0x94, 0x39, 0xbc, 0x7f, 0x42, 0x4d, 0xbb, 0x6a, 0xba, 0xa6, 0x6f, 0x0d, 0x62, 0xa1, 0x9b,
-	0xd3, 0x05, 0x2c, 0x5a, 0x6d, 0xc7, 0xb5, 0x19, 0xf1, 0x75, 0x83, 0x3a, 0x88, 0xad, 0xed, 0x5b,
-	0x99, 0x2a, 0xfb, 0x9a, 0xe6, 0xd0, 0xe7, 0xac, 0x8f, 0x07, 0xac, 0x68, 0x13, 0xa0, 0xa3, 0xc0,
-	0x0e, 0x09, 0x0a, 0xc9, 0xad, 0x54, 0x39, 0x8d, 0x87, 0x24, 0xc5, 0x63, 0x98, 0x93, 0x50, 0xf4,
-	0x02, 0xe6, 0x95, 0x32, 0x6d, 0x48, 0x39, 0xd6, 0x90, 0x88, 0x01, 0x4a, 0x35, 0xd6, 0xb8, 0x62,
-	0x1f, 0x96, 0x22, 0x56, 0xa0, 0x3c, 0xa4, 0x2e, 0x49, 0x5f, 0x9f, 0x2c, 0xf1, 0x13, 0x61, 0x98,
-	0xfb, 0x20, 0xfa, 0x88, 0x4e, 0xeb, 0x7f, 0xef, 0xe3, 0x2c, 0x56, 0x54, 0x7b, 0xc9, 0xe7, 0x89,
-	0xd2, 0x1f, 0x49, 0x78, 0xfc, 0x96, 0x38, 0xad, 0x36, 0x27, 0xf6, 0xb9, 0xc9, 0x5a, 0x84, 0xdf,
-	0x1c, 0x73, 0x0b, 0x16, 0xb8, 0xdc, 0x12, 0x68, 0x4f, 0x8f, 0x63, 0xad, 0x98, 0x92, 0xae, 0xa2,
-	0xd6, 0x03, 0x15, 0xf7, 0x90, 0xb9, 0xe8, 0xc1, 0xbc, 0x5a, 0x10, 0x8d, 0xe4, 0x4a, 0x52, 0xe9,
-	0x72, 0xd7, 0x5f, 0xe8, 0x0b, 0xc8, 0xca, 0x24, 0x85, 0xad, 0x2d, 0x79, 0xc7, 0xa8, 0x67, 0x24,
-	0x5a, 0xf7, 0xb3, 0xef, 0x20, 0x3b, 0x6c, 0x47, 0x4c, 0xe4, 0xdf, 0x47, 0x23, 0xbf, 0xff, 0x09,
-	0x7c, 0x1e, 0x4e, 0xc0, 0xb7, 0x90, 0xfd, 0x9c, 0x75, 0xac, 0x93, 0x86, 0x0e, 0xf2, 0xa8, 0x77,
-	0x89, 0x7b, 0x78, 0x27, 0xfa, 0x46, 0x08, 0x91, 0x57, 0xb9, 0xea, 0xc8, 0x19, 0x2d, 0x13, 0x8d,
-	0xbc, 0xf4, 0x08, 0xd2, 0xfb, 0x76, 0xa0, 0x95, 0x17, 0x60, 0xc1, 0x72, 0xbb, 0x01, 0x27, 0x2c,
-	0xec, 0xea, 0xfa, 0xb3, 0xf4, 0x6b, 0x12, 0xd2, 0xef, 0x06, 0xfb, 0x1e, 0xc3, 0x52, 0x43, 0xea,
-	0x25, 0xcc, 0xd0, 0x33, 0x42, 0xa2, 0x9c, 0xae, 0x26, 0x0b, 0x09, 0x9c, 0x0d, 0x17, 0xe4, 0x35,
-	0xf1, 0x29, 0x73, 0x85, 0xbe, 0x84, 0xe5, 0xa6, 0xe9, 0xba, 0xa2, 0xf1, 0x86, 0x7c, 0xa9, 0x3b,
-	0xf2, 0xe5, 0x42, 0x02, 0x4d, 0x59, 0x86, 0x3c, 0xb1, 0x03, 0x23, 0x12, 0xa4, 0x59, 0xe9, 0x79,
-	0x8e, 0xd8, 0xc1, 0xd9, 0x75, 0x9c, 0xd0, 0x05, 0x6c, 0xba, 0x2c, 0x30, 0x5c, 0x6a, 0xda, 0x06,
-	0x23, 0x1d, 0xca, 0xb8, 0x68, 0x6d, 0x02, 0x18, 0xc6, 0xe0, 0xa6, 0x29, 0xe0, 0x8c, 0x33, 0xc7,
-	0x6f, 0xa9, 0x29, 0xa0, 0xe8, 0xb2, 0x40, 0xd8, 0x85, 0x43, 0x86, 0x33, 0x49, 0x20, 0x33, 0xf1,
-	0x7d, 0x0a, 0x1e, 0x1e, 0xda, 0xc1, 0xcd, 0xe7, 0xef, 0xc6, 0xec, 0xc4, 0xba, 0x91, 0xfc, 0x48,
-	0x37, 0x52, 0xf7, 0x73, 0x03, 0x5d, 0xc0, 0x86, 0x4b, 0x2d, 0xd3, 0x75, 0x78, 0xdf, 0xe8, 0x38,
-	0xd6, 0xe5, 0xd0, 0x10, 0x32, 0x7b, 0xc7, 0x6c, 0xad, 0x85, 0x44, 0x75, 0xc5, 0xa3, 0x93, 0x76,
-	0x01, 0x1b, 0xc4, 0xb7, 0x3b, 0xd4, 0xf1, 0xf9, 0xa8, 0x86, 0xb9, 0xbb, 0x6a, 0x08, 0x89, 0x22,
-	0x1a, 0x4a, 0x3f, 0xa4, 0xe0, 0xe1, 0x09, 0x9b, 0x90, 0x8a, 0x6d, 0xc8, 0xea, 0xd8, 0x0f, 0xd5,
-	0x3f, 0xce, 0x68, 0x99, 0x8c, 0xc3, 0xf4, 0x39, 0xa9, 0x4e, 0x95, 0x93, 0xf4, 0xc4, 0xa8, 0xbf,
-	0x81, 0xc5, 0x30, 0x58, 0x7a, 0xd0, 0xdd, 0x8b, 0x0f, 0xc2, 0x04, 0xaf, 0x2a, 0x27, 0x9a, 0x01,
-	0x0f, 0xb8, 0xc6, 0x0e, 0xf0, 0xdc, 0x7d, 0x9a, 0x6d, 0x1d, 0x16, 0x43, 0x15, 0xa2, 0xbb, 0x33,
-	0xd2, 0x72, 0xa8, 0xaf, 0x63, 0xa7, 0xbf, 0x10, 0x82, 0xd9, 0x6f, 0xa8, 0x1f, 0x86, 0x4a, 0xfe,
-	0x96, 0xc3, 0x66, 0xb7, 0x21, 0xc5, 0x29, 0x3d, 0x6c, 0xaa, 0xcf, 0xd2, 0xef, 0x73, 0xb0, 0x1a,
-	0xa3, 0x16, 0x1d, 0x01, 0x88, 0xca, 0x30, 0x9a, 0x62, 0xd8, 0xd0, 0x01, 0xf9, 0x7b, 0xfc, 0x9d,
-	0x19, 0x1d, 0x49, 0x6a, 0x33, 0x78, 0x08, 0x89, 0x8e, 0x21, 0xc3, 0xc4, 0x80, 0x62, 0x30, 0x31,
-	0xa1, 0x48, 0x53, 0x33, 0xbb, 0x8f, 0x62, 0x89, 0x46, 0x07, 0x19, 0x39, 0x93, 0x5f, 0x63, 0xd1,
-	0x7f, 0x60, 0x5e, 0xc0, 0xdc, 0x86, 0x3e, 0x61, 0xdb, 0xb1, 0x2c, 0xc3, 0x77, 0x41, 0x6d, 0x06,
-	0x6b, 0x08, 0x3a, 0x87, 0x45, 0x3d, 0x7a, 0xf4, 0x0b, 0x69, 0x09, 0x7f, 0xf6, 0x71, 0x13, 0x40,
-	0x6d, 0x06, 0x0f, 0x98, 0x50, 0x0b, 0x96, 0xaf, 0xf4, 0xc5, 0x65, 0xa8, 0xfb, 0xb7, 0x00, 0x13,
-	0xc6, 0x8b, 0x29, 0x2f, 0xb9, 0xda, 0x0c, 0xce, 0x5d, 0x45, 0xb6, 0xa2, 0x5d, 0x48, 0x59, 0x76,
-	0xa0, 0x9f, 0x36, 0x9b, 0xb1, 0xe4, 0x83, 0x4b, 0xa8, 0x36, 0x83, 0xc5, 0x66, 0x74, 0x08, 0x29,
-	0x62, 0x07, 0xfa, 0xe1, 0xf2, 0xcf, 0x58, 0xcc, 0xa4, 0x6e, 0x29, 0x68, 0x88, 0xa2, 0x71, 0x59,
-	0x50, 0x58, 0x9c, 0x40, 0x33, 0xe9, 0x4c, 0x08, 0x1a, 0x97, 0x05, 0xe8, 0x19, 0xa4, 0x7a, 0x76,
-	0xa0, 0x67, 0x80, 0x78, 0x0f, 0x06, 0xd7, 0xa3, 0xb8, 0x07, 0x05, 0xae, 0x67, 0x07, 0xa8, 0x0e,
-	0xf9, 0x9e, 0x1d, 0x18, 0xa4, 0xd7, 0x21, 0xcc, 0xf1, 0x88, 0xcf, 0x4d, 0x57, 0x5f, 0x14, 0xd3,
-	0x91, 0x8c, 0xa1, 0xab, 0x8b, 0x30, 0xaf, 0x07, 0xea, 0x9f, 0xe7, 0x60, 0x49, 0xf7, 0x11, 0x5d,
-	0xf6, 0x3e, 0xac, 0xc9, 0x2e, 0xd2, 0x08, 0x5d, 0xb9, 0x9e, 0x22, 0x12, 0xe5, 0xdc, 0x0d, 0x35,
-	0x13, 0xa1, 0xa8, 0xc4, 0x44, 0x42, 0xde, 0xeb, 0xab, 0xee, 0xf8, 0x02, 0xfa, 0x6a, 0x4c, 0x9f,
-	0x9e, 0x84, 0xef, 0xda, 0xe9, 0xa3, 0xec, 0x83, 0x43, 0xbc, 0xa4, 0xde, 0x8e, 0x21, 0xab, 0x9a,
-	0x1e, 0xb6, 0x6f, 0x7d, 0xc4, 0xe2, 0xac, 0x37, 0xfc, 0x47, 0x86, 0x05, 0x79, 0xf5, 0x26, 0xe1,
-	0x6d, 0x46, 0x39, 0x77, 0x1d, 0xbf, 0xa5, 0xcf, 0xe0, 0xf3, 0x29, 0x02, 0x22, 0xdf, 0xac, 0xe7,
-	0x03, 0xa4, 0xf2, 0x1c, 0x2f, 0xb3, 0xa8, 0x18, 0xd9, 0xb0, 0xda, 0x26, 0xa6, 0xcb, 0xdb, 0x86,
-	0xd5, 0x26, 0xd6, 0x65, 0x68, 0xb2, 0xca, 0xf5, 0xbf, 0xa6, 0xd0, 0x53, 0x93, 0xe8, 0x7d, 0x01,
-	0xd6, 0x5e, 0xac, 0xb4, 0x47, 0x45, 0xc5, 0xb7, 0xb0, 0x16, 0x6b, 0x0f, 0xfa, 0x2b, 0x88, 0x87,
-	0xab, 0xc1, 0xe9, 0x25, 0xf1, 0xc3, 0xf7, 0x61, 0xda, 0x33, 0x7b, 0xe7, 0x52, 0x80, 0xfe, 0x06,
-	0x19, 0xb9, 0x64, 0xc8, 0xb7, 0x9f, 0x2c, 0xe3, 0x24, 0x06, 0x29, 0xc2, 0x42, 0x52, 0x3c, 0x87,
-	0x95, 0x31, 0x03, 0xd0, 0xff, 0x47, 0xc6, 0xc7, 0xc4, 0x14, 0xa3, 0x41, 0x64, 0xb8, 0xfc, 0xf7,
-	0x48, 0x77, 0xd6, 0xc6, 0x2e, 0x43, 0xe6, 0xf5, 0xe9, 0x59, 0xfd, 0x70, 0xff, 0xf8, 0xe8, 0xf8,
-	0xf0, 0x20, 0x3f, 0x23, 0x04, 0xf8, 0xd5, 0xeb, 0xd3, 0x03, 0x03, 0xbf, 0xaa, 0x1e, 0x9f, 0xe6,
-	0x13, 0xd5, 0x27, 0xb0, 0xe6, 0xd0, 0x48, 0xd0, 0x54, 0xcc, 0xaa, 0x28, 0x12, 0xb4, 0xba, 0xb0,
-	0xa0, 0x9e, 0x68, 0xcc, 0x4b, 0x53, 0x9e, 0xfe, 0x19, 0x00, 0x00, 0xff, 0xff, 0xae, 0xd3, 0x19,
-	0x42, 0x5a, 0x13, 0x00, 0x00,
+var File_grpc_service_config_service_config_proto protoreflect.FileDescriptor
+
+var file_grpc_service_config_service_config_proto_rawDesc = []byte{
+	0x0a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f,
+	0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a,
+	0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a,
+	0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a,
+	0x15, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6f, 0x64, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xde, 0x08, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3a, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+	0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4d, 0x65, 0x74, 0x68,
+	0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x5f,
+	0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f,
+	0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0c, 0x77, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72,
+	0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x33, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x57, 0x0a, 0x19, 0x6d, 0x61,
+	0x78, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
+	0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x16, 0x6d, 0x61, 0x78,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x79,
+	0x74, 0x65, 0x73, 0x12, 0x59, 0x0a, 0x1a, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65,
+	0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32,
+	0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x17, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x52,
+	0x0a, 0x0c, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x06,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x74, 0x72, 0x79, 0x50, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x48, 0x00, 0x52, 0x0b, 0x72, 0x65, 0x74, 0x72, 0x79, 0x50, 0x6f, 0x6c, 0x69,
+	0x63, 0x79, 0x12, 0x58, 0x0a, 0x0e, 0x68, 0x65, 0x64, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x6f,
+	0x6c, 0x69, 0x63, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x48, 0x65,
+	0x64, 0x67, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x48, 0x00, 0x52, 0x0d, 0x68,
+	0x65, 0x64, 0x67, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x1a, 0x38, 0x0a, 0x04,
+	0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16,
+	0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
+	0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0xa7, 0x02, 0x0a, 0x0b, 0x52, 0x65, 0x74, 0x72, 0x79,
+	0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x74,
+	0x74, 0x65, 0x6d, 0x70, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x6d, 0x61,
+	0x78, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73, 0x12, 0x42, 0x0a, 0x0f, 0x69, 0x6e, 0x69,
+	0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x69,
+	0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x12, 0x3a, 0x0a,
+	0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d,
+	0x61, 0x78, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x12, 0x2d, 0x0a, 0x12, 0x62, 0x61, 0x63,
+	0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x69, 0x65, 0x72, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x75,
+	0x6c, 0x74, 0x69, 0x70, 0x6c, 0x69, 0x65, 0x72, 0x12, 0x46, 0x0a, 0x16, 0x72, 0x65, 0x74, 0x72,
+	0x79, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64,
+	0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x14, 0x72, 0x65, 0x74, 0x72,
+	0x79, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x73,
+	0x1a, 0xb9, 0x01, 0x0a, 0x0d, 0x48, 0x65, 0x64, 0x67, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69,
+	0x63, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x74, 0x74, 0x65, 0x6d, 0x70,
+	0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x41, 0x74, 0x74,
+	0x65, 0x6d, 0x70, 0x74, 0x73, 0x12, 0x3e, 0x0a, 0x0d, 0x68, 0x65, 0x64, 0x67, 0x69, 0x6e, 0x67,
+	0x5f, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44,
+	0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x68, 0x65, 0x64, 0x67, 0x69, 0x6e, 0x67,
+	0x44, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x45, 0x0a, 0x16, 0x6e, 0x6f, 0x6e, 0x5f, 0x66, 0x61, 0x74,
+	0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x73, 0x18,
+	0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72,
+	0x70, 0x63, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x13, 0x6e, 0x6f, 0x6e, 0x46, 0x61, 0x74, 0x61,
+	0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x73, 0x42, 0x19, 0x0a, 0x17,
+	0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x6f, 0x72, 0x5f, 0x68, 0x65, 0x64, 0x67, 0x69, 0x6e, 0x67,
+	0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x50, 0x69, 0x63, 0x6b, 0x46,
+	0x69, 0x72, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x12, 0x0a, 0x10, 0x52, 0x6f,
+	0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xeb,
+	0x02, 0x0a, 0x21, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x6f, 0x61, 0x64, 0x42,
+	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f,
+	0x6e, 0x66, 0x69, 0x67, 0x12, 0x60, 0x0a, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x69,
+	0x6f, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69,
+	0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43,
+	0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x63, 0x68,
+	0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69,
+	0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x69, 0x6f,
+	0x72, 0x69, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x49, 0x0a, 0x05, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x12,
+	0x40, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x1a, 0x79, 0x0a, 0x0d, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x45, 0x6e, 0x74,
+	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x03, 0x6b, 0x65, 0x79, 0x12, 0x52, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69,
+	0x74, 0x79, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50,
+	0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x68, 0x69, 0x6c,
+	0x64, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfe, 0x02, 0x0a,
+	0x27, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4c,
+	0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69,
+	0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x63, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e,
+	0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63,
+	0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x1a, 0x6d, 0x0a,
+	0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68,
+	0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12,
+	0x4b, 0x0a, 0x0c, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18,
+	0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64,
+	0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
+	0x0b, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x1a, 0x7f, 0x0a, 0x0c,
+	0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
+	0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x59,
+	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x43, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x2e, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x54, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50,
+	0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7e, 0x0a,
+	0x0c, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x62, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4b, 0x0a,
+	0x0c, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x01, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
+	0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x63,
+	0x68, 0x69, 0x6c, 0x64, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x25, 0x0a,
+	0x09, 0x43, 0x64, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6c,
+	0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x75,
+	0x73, 0x74, 0x65, 0x72, 0x22, 0xe0, 0x02, 0x0a, 0x09, 0x58, 0x64, 0x73, 0x43, 0x6f, 0x6e, 0x66,
+	0x69, 0x67, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x5f, 0x6e,
+	0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, 0x62,
+	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x63,
+	0x68, 0x69, 0x6c, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61,
+	0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x63, 0x68, 0x69,
+	0x6c, 0x64, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x51, 0x0a, 0x0f, 0x66, 0x61, 0x6c, 0x6c,
+	0x62, 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x03, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61,
+	0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x66, 0x61, 0x6c,
+	0x6c, 0x62, 0x61, 0x63, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x65,
+	0x64, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x64, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x60, 0x0a, 0x1e, 0x6c, 0x72, 0x73, 0x5f, 0x6c, 0x6f, 0x61,
+	0x64, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
+	0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x1a, 0x6c, 0x72, 0x73,
+	0x4c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x88, 0x03, 0x0a, 0x1c, 0x45, 0x64, 0x73, 0x4c,
+	0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69,
+	0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6c, 0x75, 0x73,
+	0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x75, 0x73, 0x74,
+	0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x64, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x64,
+	0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x60, 0x0a, 0x1e,
+	0x6c, 0x72, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69,
+	0x6e, 0x67, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c,
+	0x75, 0x65, 0x52, 0x1a, 0x6c, 0x72, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x70, 0x6f, 0x72,
+	0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x60,
+	0x0a, 0x17, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x69, 0x63, 0x6b, 0x69,
+	0x6e, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c,
+	0x69, 0x74, 0x79, 0x50, 0x69, 0x63, 0x6b, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79,
+	0x12, 0x60, 0x0a, 0x17, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x70, 0x69, 0x63,
+	0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x05, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61,
+	0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x15, 0x65, 0x6e, 0x64,
+	0x70, 0x6f, 0x69, 0x6e, 0x74, 0x50, 0x69, 0x63, 0x6b, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69,
+	0x63, 0x79, 0x22, 0xa6, 0x03, 0x0a, 0x1c, 0x4c, 0x72, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
+	0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e,
+	0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74,
+	0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x64, 0x73, 0x5f, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0e, 0x65, 0x64, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65,
+	0x12, 0x42, 0x0a, 0x1e, 0x6c, 0x72, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, 0x65, 0x70,
+	0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x6c, 0x72, 0x73, 0x4c, 0x6f, 0x61,
+	0x64, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x4e, 0x61, 0x6d, 0x65, 0x12, 0x56, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x69, 0x74, 0x79,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x72, 0x73,
+	0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x69,
+	0x74, 0x79, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0c,
+	0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x05, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c,
+	0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x63, 0x68,
+	0x69, 0x6c, 0x64, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x1a, 0x50, 0x0a, 0x08, 0x4c, 0x6f, 0x63,
+	0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a,
+	0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e,
+	0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x7a, 0x6f, 0x6e, 0x65, 0x22, 0xfa, 0x05, 0x0a, 0x13,
+	0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x12, 0x46, 0x0a, 0x0a, 0x70, 0x69, 0x63, 0x6b, 0x5f, 0x66, 0x69, 0x72, 0x73,
+	0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x69,
+	0x63, 0x6b, 0x46, 0x69, 0x72, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52,
+	0x0a, 0x70, 0x69, 0x63, 0x6b, 0x5f, 0x66, 0x69, 0x72, 0x73, 0x74, 0x12, 0x49, 0x0a, 0x0b, 0x72,
+	0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x72, 0x6f, 0x62, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f,
+	0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69,
+	0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0b, 0x72, 0x6f, 0x75, 0x6e, 0x64,
+	0x5f, 0x72, 0x6f, 0x62, 0x69, 0x6e, 0x12, 0x3b, 0x0a, 0x06, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x72, 0x70,
+	0x63, 0x4c, 0x62, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x06, 0x67, 0x72, 0x70,
+	0x63, 0x6c, 0x62, 0x12, 0x54, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18,
+	0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x69, 0x6f,
+	0x72, 0x69, 0x74, 0x79, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e,
+	0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52,
+	0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x67, 0x0a, 0x0f, 0x77, 0x65, 0x69,
+	0x67, 0x68, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x0a, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65,
+	0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e,
+	0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x48, 0x00, 0x52, 0x0e, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x54, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x12, 0x32, 0x0a, 0x03, 0x63, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x64, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48,
+	0x00, 0x52, 0x03, 0x63, 0x64, 0x73, 0x12, 0x45, 0x0a, 0x03, 0x65, 0x64, 0x73, 0x18, 0x07, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x64, 0x73, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x03, 0x65, 0x64, 0x73, 0x12, 0x45, 0x0a,
+	0x03, 0x6c, 0x72, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x2e, 0x4c, 0x72, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e,
+	0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52,
+	0x03, 0x6c, 0x72, 0x73, 0x12, 0x36, 0x0a, 0x03, 0x78, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x58, 0x64, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x42, 0x02, 0x18, 0x01, 0x48, 0x00, 0x52, 0x03, 0x78, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x10,
+	0x78, 0x64, 0x73, 0x5f, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x58, 0x64, 0x73,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x02, 0x18, 0x01, 0x48, 0x00, 0x52, 0x10, 0x78, 0x64,
+	0x73, 0x5f, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x42, 0x08,
+	0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0xd8, 0x05, 0x0a, 0x0d, 0x53, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x6e, 0x0a, 0x15, 0x6c, 0x6f,
+	0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e,
+	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63,
+	0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x13, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e,
+	0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x5c, 0x0a, 0x15, 0x6c, 0x6f,
+	0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e,
+	0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x52, 0x13, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69,
+	0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x46, 0x0a, 0x0d, 0x6d, 0x65, 0x74, 0x68,
+	0x6f, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x21, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66,
+	0x69, 0x67, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x12, 0x63, 0x0a, 0x10, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74,
+	0x6c, 0x69, 0x6e, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52,
+	0x65, 0x74, 0x72, 0x79, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x69, 0x6e, 0x67, 0x50, 0x6f,
+	0x6c, 0x69, 0x63, 0x79, 0x52, 0x0f, 0x72, 0x65, 0x74, 0x72, 0x79, 0x54, 0x68, 0x72, 0x6f, 0x74,
+	0x74, 0x6c, 0x69, 0x6e, 0x67, 0x12, 0x64, 0x0a, 0x13, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f,
+	0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x34, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65,
+	0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68,
+	0x43, 0x68, 0x65, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x57, 0x0a, 0x15, 0x52,
+	0x65, 0x74, 0x72, 0x79, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x69, 0x6e, 0x67, 0x50, 0x6f,
+	0x6c, 0x69, 0x63, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x78, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
+	0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6d, 0x61, 0x78, 0x54, 0x6f, 0x6b,
+	0x65, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x72, 0x61, 0x74,
+	0x69, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0a, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x52,
+	0x61, 0x74, 0x69, 0x6f, 0x1a, 0x54, 0x0a, 0x11, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68,
+	0x65, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3f, 0x0a, 0x0c, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x73,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x37, 0x0a, 0x13, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x69, 0x63,
+	0x79, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44,
+	0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x4f, 0x55, 0x4e, 0x44, 0x5f, 0x52, 0x4f, 0x42, 0x49,
+	0x4e, 0x10, 0x01, 0x42, 0x2d, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x73,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x12, 0x53, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x6f,
+	0x50, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_service_config_service_config_proto_rawDescOnce sync.Once
+	file_grpc_service_config_service_config_proto_rawDescData = file_grpc_service_config_service_config_proto_rawDesc
+)
+
+func file_grpc_service_config_service_config_proto_rawDescGZIP() []byte {
+	file_grpc_service_config_service_config_proto_rawDescOnce.Do(func() {
+		file_grpc_service_config_service_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_service_config_service_config_proto_rawDescData)
+	})
+	return file_grpc_service_config_service_config_proto_rawDescData
+}
+
+var file_grpc_service_config_service_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_grpc_service_config_service_config_proto_msgTypes = make([]protoimpl.MessageInfo, 22)
+var file_grpc_service_config_service_config_proto_goTypes = []interface{}{
+	(ServiceConfig_LoadBalancingPolicy)(0),          // 0: grpc.service_config.ServiceConfig.LoadBalancingPolicy
+	(*MethodConfig)(nil),                            // 1: grpc.service_config.MethodConfig
+	(*PickFirstConfig)(nil),                         // 2: grpc.service_config.PickFirstConfig
+	(*RoundRobinConfig)(nil),                        // 3: grpc.service_config.RoundRobinConfig
+	(*PriorityLoadBalancingPolicyConfig)(nil),       // 4: grpc.service_config.PriorityLoadBalancingPolicyConfig
+	(*WeightedTargetLoadBalancingPolicyConfig)(nil), // 5: grpc.service_config.WeightedTargetLoadBalancingPolicyConfig
+	(*GrpcLbConfig)(nil),                            // 6: grpc.service_config.GrpcLbConfig
+	(*CdsConfig)(nil),                               // 7: grpc.service_config.CdsConfig
+	(*XdsConfig)(nil),                               // 8: grpc.service_config.XdsConfig
+	(*EdsLoadBalancingPolicyConfig)(nil),            // 9: grpc.service_config.EdsLoadBalancingPolicyConfig
+	(*LrsLoadBalancingPolicyConfig)(nil),            // 10: grpc.service_config.LrsLoadBalancingPolicyConfig
+	(*LoadBalancingConfig)(nil),                     // 11: grpc.service_config.LoadBalancingConfig
+	(*ServiceConfig)(nil),                           // 12: grpc.service_config.ServiceConfig
+	(*MethodConfig_Name)(nil),                       // 13: grpc.service_config.MethodConfig.Name
+	(*MethodConfig_RetryPolicy)(nil),                // 14: grpc.service_config.MethodConfig.RetryPolicy
+	(*MethodConfig_HedgingPolicy)(nil),              // 15: grpc.service_config.MethodConfig.HedgingPolicy
+	(*PriorityLoadBalancingPolicyConfig_Child)(nil), // 16: grpc.service_config.PriorityLoadBalancingPolicyConfig.Child
+	nil, // 17: grpc.service_config.PriorityLoadBalancingPolicyConfig.ChildrenEntry
+	(*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), // 18: grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.Target
+	nil, // 19: grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.TargetsEntry
+	(*LrsLoadBalancingPolicyConfig_Locality)(nil), // 20: grpc.service_config.LrsLoadBalancingPolicyConfig.Locality
+	(*ServiceConfig_RetryThrottlingPolicy)(nil),   // 21: grpc.service_config.ServiceConfig.RetryThrottlingPolicy
+	(*ServiceConfig_HealthCheckConfig)(nil),       // 22: grpc.service_config.ServiceConfig.HealthCheckConfig
+	(*wrappers.BoolValue)(nil),                    // 23: google.protobuf.BoolValue
+	(*duration.Duration)(nil),                     // 24: google.protobuf.Duration
+	(*wrappers.UInt32Value)(nil),                  // 25: google.protobuf.UInt32Value
+	(*wrappers.StringValue)(nil),                  // 26: google.protobuf.StringValue
+	(code.Code)(0),                                // 27: google.rpc.Code
+}
+var file_grpc_service_config_service_config_proto_depIdxs = []int32{
+	13, // 0: grpc.service_config.MethodConfig.name:type_name -> grpc.service_config.MethodConfig.Name
+	23, // 1: grpc.service_config.MethodConfig.wait_for_ready:type_name -> google.protobuf.BoolValue
+	24, // 2: grpc.service_config.MethodConfig.timeout:type_name -> google.protobuf.Duration
+	25, // 3: grpc.service_config.MethodConfig.max_request_message_bytes:type_name -> google.protobuf.UInt32Value
+	25, // 4: grpc.service_config.MethodConfig.max_response_message_bytes:type_name -> google.protobuf.UInt32Value
+	14, // 5: grpc.service_config.MethodConfig.retry_policy:type_name -> grpc.service_config.MethodConfig.RetryPolicy
+	15, // 6: grpc.service_config.MethodConfig.hedging_policy:type_name -> grpc.service_config.MethodConfig.HedgingPolicy
+	17, // 7: grpc.service_config.PriorityLoadBalancingPolicyConfig.children:type_name -> grpc.service_config.PriorityLoadBalancingPolicyConfig.ChildrenEntry
+	19, // 8: grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.targets:type_name -> grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.TargetsEntry
+	11, // 9: grpc.service_config.GrpcLbConfig.child_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	11, // 10: grpc.service_config.XdsConfig.child_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	11, // 11: grpc.service_config.XdsConfig.fallback_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	26, // 12: grpc.service_config.XdsConfig.lrs_load_reporting_server_name:type_name -> google.protobuf.StringValue
+	26, // 13: grpc.service_config.EdsLoadBalancingPolicyConfig.lrs_load_reporting_server_name:type_name -> google.protobuf.StringValue
+	11, // 14: grpc.service_config.EdsLoadBalancingPolicyConfig.locality_picking_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	11, // 15: grpc.service_config.EdsLoadBalancingPolicyConfig.endpoint_picking_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	20, // 16: grpc.service_config.LrsLoadBalancingPolicyConfig.locality:type_name -> grpc.service_config.LrsLoadBalancingPolicyConfig.Locality
+	11, // 17: grpc.service_config.LrsLoadBalancingPolicyConfig.child_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	2,  // 18: grpc.service_config.LoadBalancingConfig.pick_first:type_name -> grpc.service_config.PickFirstConfig
+	3,  // 19: grpc.service_config.LoadBalancingConfig.round_robin:type_name -> grpc.service_config.RoundRobinConfig
+	6,  // 20: grpc.service_config.LoadBalancingConfig.grpclb:type_name -> grpc.service_config.GrpcLbConfig
+	4,  // 21: grpc.service_config.LoadBalancingConfig.priority:type_name -> grpc.service_config.PriorityLoadBalancingPolicyConfig
+	5,  // 22: grpc.service_config.LoadBalancingConfig.weighted_target:type_name -> grpc.service_config.WeightedTargetLoadBalancingPolicyConfig
+	7,  // 23: grpc.service_config.LoadBalancingConfig.cds:type_name -> grpc.service_config.CdsConfig
+	9,  // 24: grpc.service_config.LoadBalancingConfig.eds:type_name -> grpc.service_config.EdsLoadBalancingPolicyConfig
+	10, // 25: grpc.service_config.LoadBalancingConfig.lrs:type_name -> grpc.service_config.LrsLoadBalancingPolicyConfig
+	8,  // 26: grpc.service_config.LoadBalancingConfig.xds:type_name -> grpc.service_config.XdsConfig
+	8,  // 27: grpc.service_config.LoadBalancingConfig.xds_experimental:type_name -> grpc.service_config.XdsConfig
+	0,  // 28: grpc.service_config.ServiceConfig.load_balancing_policy:type_name -> grpc.service_config.ServiceConfig.LoadBalancingPolicy
+	11, // 29: grpc.service_config.ServiceConfig.load_balancing_config:type_name -> grpc.service_config.LoadBalancingConfig
+	1,  // 30: grpc.service_config.ServiceConfig.method_config:type_name -> grpc.service_config.MethodConfig
+	21, // 31: grpc.service_config.ServiceConfig.retry_throttling:type_name -> grpc.service_config.ServiceConfig.RetryThrottlingPolicy
+	22, // 32: grpc.service_config.ServiceConfig.health_check_config:type_name -> grpc.service_config.ServiceConfig.HealthCheckConfig
+	24, // 33: grpc.service_config.MethodConfig.RetryPolicy.initial_backoff:type_name -> google.protobuf.Duration
+	24, // 34: grpc.service_config.MethodConfig.RetryPolicy.max_backoff:type_name -> google.protobuf.Duration
+	27, // 35: grpc.service_config.MethodConfig.RetryPolicy.retryable_status_codes:type_name -> google.rpc.Code
+	24, // 36: grpc.service_config.MethodConfig.HedgingPolicy.hedging_delay:type_name -> google.protobuf.Duration
+	27, // 37: grpc.service_config.MethodConfig.HedgingPolicy.non_fatal_status_codes:type_name -> google.rpc.Code
+	11, // 38: grpc.service_config.PriorityLoadBalancingPolicyConfig.Child.config:type_name -> grpc.service_config.LoadBalancingConfig
+	16, // 39: grpc.service_config.PriorityLoadBalancingPolicyConfig.ChildrenEntry.value:type_name -> grpc.service_config.PriorityLoadBalancingPolicyConfig.Child
+	11, // 40: grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.Target.child_policy:type_name -> grpc.service_config.LoadBalancingConfig
+	18, // 41: grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.TargetsEntry.value:type_name -> grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.Target
+	26, // 42: grpc.service_config.ServiceConfig.HealthCheckConfig.service_name:type_name -> google.protobuf.StringValue
+	43, // [43:43] is the sub-list for method output_type
+	43, // [43:43] is the sub-list for method input_type
+	43, // [43:43] is the sub-list for extension type_name
+	43, // [43:43] is the sub-list for extension extendee
+	0,  // [0:43] is the sub-list for field type_name
+}
+
+func init() { file_grpc_service_config_service_config_proto_init() }
+func file_grpc_service_config_service_config_proto_init() {
+	if File_grpc_service_config_service_config_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_service_config_service_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MethodConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PickFirstConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RoundRobinConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PriorityLoadBalancingPolicyConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*WeightedTargetLoadBalancingPolicyConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GrpcLbConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CdsConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*XdsConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EdsLoadBalancingPolicyConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LrsLoadBalancingPolicyConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancingConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServiceConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MethodConfig_Name); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MethodConfig_RetryPolicy); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MethodConfig_HedgingPolicy); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PriorityLoadBalancingPolicyConfig_Child); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*WeightedTargetLoadBalancingPolicyConfig_Target); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LrsLoadBalancingPolicyConfig_Locality); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServiceConfig_RetryThrottlingPolicy); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_service_config_service_config_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServiceConfig_HealthCheckConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_service_config_service_config_proto_msgTypes[0].OneofWrappers = []interface{}{
+		(*MethodConfig_RetryPolicy_)(nil),
+		(*MethodConfig_HedgingPolicy_)(nil),
+	}
+	file_grpc_service_config_service_config_proto_msgTypes[10].OneofWrappers = []interface{}{
+		(*LoadBalancingConfig_PickFirst)(nil),
+		(*LoadBalancingConfig_RoundRobin)(nil),
+		(*LoadBalancingConfig_Grpclb)(nil),
+		(*LoadBalancingConfig_Priority)(nil),
+		(*LoadBalancingConfig_WeightedTarget)(nil),
+		(*LoadBalancingConfig_Cds)(nil),
+		(*LoadBalancingConfig_Eds)(nil),
+		(*LoadBalancingConfig_Lrs)(nil),
+		(*LoadBalancingConfig_Xds)(nil),
+		(*LoadBalancingConfig_XdsExperimental)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_service_config_service_config_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   22,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_service_config_service_config_proto_goTypes,
+		DependencyIndexes: file_grpc_service_config_service_config_proto_depIdxs,
+		EnumInfos:         file_grpc_service_config_service_config_proto_enumTypes,
+		MessageInfos:      file_grpc_service_config_service_config_proto_msgTypes,
+	}.Build()
+	File_grpc_service_config_service_config_proto = out.File
+	file_grpc_service_config_service_config_proto_rawDesc = nil
+	file_grpc_service_config_service_config_proto_goTypes = nil
+	file_grpc_service_config_service_config_proto_depIdxs = nil
 }
diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 9b44623c25ae..1f3bb0b9f692 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -1,24 +1,46 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: interop/grpc_testing/test.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // The type of payload that should be returned.
 type PayloadType int32
@@ -32,24 +54,45 @@ const (
 	PayloadType_RANDOM PayloadType = 2
 )
 
-var PayloadType_name = map[int32]string{
-	0: "COMPRESSABLE",
-	1: "UNCOMPRESSABLE",
-	2: "RANDOM",
-}
+// Enum value maps for PayloadType.
+var (
+	PayloadType_name = map[int32]string{
+		0: "COMPRESSABLE",
+		1: "UNCOMPRESSABLE",
+		2: "RANDOM",
+	}
+	PayloadType_value = map[string]int32{
+		"COMPRESSABLE":   0,
+		"UNCOMPRESSABLE": 1,
+		"RANDOM":         2,
+	}
+)
 
-var PayloadType_value = map[string]int32{
-	"COMPRESSABLE":   0,
-	"UNCOMPRESSABLE": 1,
-	"RANDOM":         2,
+func (x PayloadType) Enum() *PayloadType {
+	p := new(PayloadType)
+	*p = x
+	return p
 }
 
 func (x PayloadType) String() string {
-	return proto.EnumName(PayloadType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (PayloadType) Descriptor() protoreflect.EnumDescriptor {
+	return file_interop_grpc_testing_test_proto_enumTypes[0].Descriptor()
+}
+
+func (PayloadType) Type() protoreflect.EnumType {
+	return &file_interop_grpc_testing_test_proto_enumTypes[0]
 }
 
+func (x PayloadType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use PayloadType.Descriptor instead.
 func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{0}
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{0}
 }
 
 // The type of route that a client took to reach a server w.r.t. gRPCLB.
@@ -69,103 +112,139 @@ const (
 	GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND GrpclbRouteType = 2
 )
 
-var GrpclbRouteType_name = map[int32]string{
-	0: "GRPCLB_ROUTE_TYPE_UNKNOWN",
-	1: "GRPCLB_ROUTE_TYPE_FALLBACK",
-	2: "GRPCLB_ROUTE_TYPE_BACKEND",
-}
+// Enum value maps for GrpclbRouteType.
+var (
+	GrpclbRouteType_name = map[int32]string{
+		0: "GRPCLB_ROUTE_TYPE_UNKNOWN",
+		1: "GRPCLB_ROUTE_TYPE_FALLBACK",
+		2: "GRPCLB_ROUTE_TYPE_BACKEND",
+	}
+	GrpclbRouteType_value = map[string]int32{
+		"GRPCLB_ROUTE_TYPE_UNKNOWN":  0,
+		"GRPCLB_ROUTE_TYPE_FALLBACK": 1,
+		"GRPCLB_ROUTE_TYPE_BACKEND":  2,
+	}
+)
 
-var GrpclbRouteType_value = map[string]int32{
-	"GRPCLB_ROUTE_TYPE_UNKNOWN":  0,
-	"GRPCLB_ROUTE_TYPE_FALLBACK": 1,
-	"GRPCLB_ROUTE_TYPE_BACKEND":  2,
+func (x GrpclbRouteType) Enum() *GrpclbRouteType {
+	p := new(GrpclbRouteType)
+	*p = x
+	return p
 }
 
 func (x GrpclbRouteType) String() string {
-	return proto.EnumName(GrpclbRouteType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
-func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{1}
+func (GrpclbRouteType) Descriptor() protoreflect.EnumDescriptor {
+	return file_interop_grpc_testing_test_proto_enumTypes[1].Descriptor()
 }
 
-type Empty struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (GrpclbRouteType) Type() protoreflect.EnumType {
+	return &file_interop_grpc_testing_test_proto_enumTypes[1]
 }
 
-func (m *Empty) Reset()         { *m = Empty{} }
-func (m *Empty) String() string { return proto.CompactTextString(m) }
-func (*Empty) ProtoMessage()    {}
-func (*Empty) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{0}
+func (x GrpclbRouteType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
-func (m *Empty) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Empty.Unmarshal(m, b)
+// Deprecated: Use GrpclbRouteType.Descriptor instead.
+func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{1}
 }
-func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Empty.Marshal(b, m, deterministic)
+
+type Empty struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
-func (m *Empty) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Empty.Merge(m, src)
+
+func (x *Empty) Reset() {
+	*x = Empty{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Empty) XXX_Size() int {
-	return xxx_messageInfo_Empty.Size(m)
+
+func (x *Empty) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Empty) XXX_DiscardUnknown() {
-	xxx_messageInfo_Empty.DiscardUnknown(m)
+
+func (*Empty) ProtoMessage() {}
+
+func (x *Empty) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Empty proto.InternalMessageInfo
+// Deprecated: Use Empty.ProtoReflect.Descriptor instead.
+func (*Empty) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{0}
+}
 
 // A block of data, to simply increase gRPC message size.
 type Payload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The type of data in body.
 	Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"`
 	// Primary contents of payload.
-	Body                 []byte   `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
 }
 
-func (m *Payload) Reset()         { *m = Payload{} }
-func (m *Payload) String() string { return proto.CompactTextString(m) }
-func (*Payload) ProtoMessage()    {}
-func (*Payload) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{1}
+func (x *Payload) Reset() {
+	*x = Payload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Payload) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Payload.Unmarshal(m, b)
-}
-func (m *Payload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Payload.Marshal(b, m, deterministic)
-}
-func (m *Payload) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Payload.Merge(m, src)
+func (x *Payload) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Payload) XXX_Size() int {
-	return xxx_messageInfo_Payload.Size(m)
-}
-func (m *Payload) XXX_DiscardUnknown() {
-	xxx_messageInfo_Payload.DiscardUnknown(m)
+
+func (*Payload) ProtoMessage() {}
+
+func (x *Payload) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Payload proto.InternalMessageInfo
+// Deprecated: Use Payload.ProtoReflect.Descriptor instead.
+func (*Payload) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *Payload) GetType() PayloadType {
-	if m != nil {
-		return m.Type
+func (x *Payload) GetType() PayloadType {
+	if x != nil {
+		return x.Type
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *Payload) GetBody() []byte {
-	if m != nil {
-		return m.Body
+func (x *Payload) GetBody() []byte {
+	if x != nil {
+		return x.Body
 	}
 	return nil
 }
@@ -173,54 +252,66 @@ func (m *Payload) GetBody() []byte {
 // A protobuf representation for grpc status. This is used by test
 // clients to specify a status that the server should attempt to return.
 type EchoStatus struct {
-	Code                 int32    `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
-	Message              string   `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *EchoStatus) Reset()         { *m = EchoStatus{} }
-func (m *EchoStatus) String() string { return proto.CompactTextString(m) }
-func (*EchoStatus) ProtoMessage()    {}
-func (*EchoStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{2}
+	Code    int32  `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
+	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
 }
 
-func (m *EchoStatus) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EchoStatus.Unmarshal(m, b)
-}
-func (m *EchoStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EchoStatus.Marshal(b, m, deterministic)
-}
-func (m *EchoStatus) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EchoStatus.Merge(m, src)
+func (x *EchoStatus) Reset() {
+	*x = EchoStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *EchoStatus) XXX_Size() int {
-	return xxx_messageInfo_EchoStatus.Size(m)
+
+func (x *EchoStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EchoStatus) XXX_DiscardUnknown() {
-	xxx_messageInfo_EchoStatus.DiscardUnknown(m)
+
+func (*EchoStatus) ProtoMessage() {}
+
+func (x *EchoStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_EchoStatus proto.InternalMessageInfo
+// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead.
+func (*EchoStatus) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *EchoStatus) GetCode() int32 {
-	if m != nil {
-		return m.Code
+func (x *EchoStatus) GetCode() int32 {
+	if x != nil {
+		return x.Code
 	}
 	return 0
 }
 
-func (m *EchoStatus) GetMessage() string {
-	if m != nil {
-		return m.Message
+func (x *EchoStatus) GetMessage() string {
+	if x != nil {
+		return x.Message
 	}
 	return ""
 }
 
 // Unary request.
 type SimpleRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload type in the response from the server.
 	// If response_type is RANDOM, server randomly chooses one from other formats.
 	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
@@ -238,95 +329,103 @@ type SimpleRequest struct {
 	// Whether SimpleResponse should include server_id.
 	FillServerId bool `protobuf:"varint,9,opt,name=fill_server_id,json=fillServerId,proto3" json:"fill_server_id,omitempty"`
 	// Whether SimpleResponse should include grpclb_route_type.
-	FillGrpclbRouteType  bool     `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	FillGrpclbRouteType bool `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"`
 }
 
-func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
-func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
-func (*SimpleRequest) ProtoMessage()    {}
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{3}
+func (x *SimpleRequest) Reset() {
+	*x = SimpleRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleRequest.Unmarshal(m, b)
-}
-func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic)
+func (x *SimpleRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleRequest.Merge(m, src)
-}
-func (m *SimpleRequest) XXX_Size() int {
-	return xxx_messageInfo_SimpleRequest.Size(m)
-}
-func (m *SimpleRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleRequest.DiscardUnknown(m)
+
+func (*SimpleRequest) ProtoMessage() {}
+
+func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo
+// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
+func (*SimpleRequest) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *SimpleRequest) GetResponseType() PayloadType {
-	if m != nil {
-		return m.ResponseType
+func (x *SimpleRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *SimpleRequest) GetResponseSize() int32 {
-	if m != nil {
-		return m.ResponseSize
+func (x *SimpleRequest) GetResponseSize() int32 {
+	if x != nil {
+		return x.ResponseSize
 	}
 	return 0
 }
 
-func (m *SimpleRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *SimpleRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *SimpleRequest) GetFillUsername() bool {
-	if m != nil {
-		return m.FillUsername
+func (x *SimpleRequest) GetFillUsername() bool {
+	if x != nil {
+		return x.FillUsername
 	}
 	return false
 }
 
-func (m *SimpleRequest) GetFillOauthScope() bool {
-	if m != nil {
-		return m.FillOauthScope
+func (x *SimpleRequest) GetFillOauthScope() bool {
+	if x != nil {
+		return x.FillOauthScope
 	}
 	return false
 }
 
-func (m *SimpleRequest) GetResponseStatus() *EchoStatus {
-	if m != nil {
-		return m.ResponseStatus
+func (x *SimpleRequest) GetResponseStatus() *EchoStatus {
+	if x != nil {
+		return x.ResponseStatus
 	}
 	return nil
 }
 
-func (m *SimpleRequest) GetFillServerId() bool {
-	if m != nil {
-		return m.FillServerId
+func (x *SimpleRequest) GetFillServerId() bool {
+	if x != nil {
+		return x.FillServerId
 	}
 	return false
 }
 
-func (m *SimpleRequest) GetFillGrpclbRouteType() bool {
-	if m != nil {
-		return m.FillGrpclbRouteType
+func (x *SimpleRequest) GetFillGrpclbRouteType() bool {
+	if x != nil {
+		return x.FillGrpclbRouteType
 	}
 	return false
 }
 
 // Unary response, as configured by the request.
 type SimpleResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Payload to increase message size.
 	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 	// The user the request came from, for verifying authentication was
@@ -340,215 +439,247 @@ type SimpleResponse struct {
 	// gRPCLB Path.
 	GrpclbRouteType GrpclbRouteType `protobuf:"varint,5,opt,name=grpclb_route_type,json=grpclbRouteType,proto3,enum=grpc.testing.GrpclbRouteType" json:"grpclb_route_type,omitempty"`
 	// Server hostname.
-	Hostname             string   `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"`
 }
 
-func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
-func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
-func (*SimpleResponse) ProtoMessage()    {}
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{4}
+func (x *SimpleResponse) Reset() {
+	*x = SimpleResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleResponse.Unmarshal(m, b)
-}
-func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic)
+func (x *SimpleResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleResponse.Merge(m, src)
-}
-func (m *SimpleResponse) XXX_Size() int {
-	return xxx_messageInfo_SimpleResponse.Size(m)
-}
-func (m *SimpleResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleResponse.DiscardUnknown(m)
+
+func (*SimpleResponse) ProtoMessage() {}
+
+func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo
+// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
+func (*SimpleResponse) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *SimpleResponse) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *SimpleResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *SimpleResponse) GetUsername() string {
-	if m != nil {
-		return m.Username
+func (x *SimpleResponse) GetUsername() string {
+	if x != nil {
+		return x.Username
 	}
 	return ""
 }
 
-func (m *SimpleResponse) GetOauthScope() string {
-	if m != nil {
-		return m.OauthScope
+func (x *SimpleResponse) GetOauthScope() string {
+	if x != nil {
+		return x.OauthScope
 	}
 	return ""
 }
 
-func (m *SimpleResponse) GetServerId() string {
-	if m != nil {
-		return m.ServerId
+func (x *SimpleResponse) GetServerId() string {
+	if x != nil {
+		return x.ServerId
 	}
 	return ""
 }
 
-func (m *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType {
-	if m != nil {
-		return m.GrpclbRouteType
+func (x *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType {
+	if x != nil {
+		return x.GrpclbRouteType
 	}
 	return GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN
 }
 
-func (m *SimpleResponse) GetHostname() string {
-	if m != nil {
-		return m.Hostname
+func (x *SimpleResponse) GetHostname() string {
+	if x != nil {
+		return x.Hostname
 	}
 	return ""
 }
 
 // Client-streaming request.
 type StreamingInputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Optional input payload sent along with the request.
-	Payload              *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingInputCallRequest) Reset()         { *m = StreamingInputCallRequest{} }
-func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) }
-func (*StreamingInputCallRequest) ProtoMessage()    {}
-func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{5}
+func (x *StreamingInputCallRequest) Reset() {
+	*x = StreamingInputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingInputCallRequest.Unmarshal(m, b)
-}
-func (m *StreamingInputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingInputCallRequest.Marshal(b, m, deterministic)
-}
-func (m *StreamingInputCallRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingInputCallRequest.Merge(m, src)
+func (x *StreamingInputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingInputCallRequest) XXX_Size() int {
-	return xxx_messageInfo_StreamingInputCallRequest.Size(m)
-}
-func (m *StreamingInputCallRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingInputCallRequest.DiscardUnknown(m)
+
+func (*StreamingInputCallRequest) ProtoMessage() {}
+
+func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingInputCallRequest proto.InternalMessageInfo
+// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *StreamingInputCallRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingInputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
 // Client-streaming response.
 type StreamingInputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Aggregated size of payloads received from the client.
-	AggregatedPayloadSize int32    `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{} `json:"-"`
-	XXX_unrecognized      []byte   `json:"-"`
-	XXX_sizecache         int32    `json:"-"`
+	AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
 }
 
-func (m *StreamingInputCallResponse) Reset()         { *m = StreamingInputCallResponse{} }
-func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) }
-func (*StreamingInputCallResponse) ProtoMessage()    {}
-func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{6}
+func (x *StreamingInputCallResponse) Reset() {
+	*x = StreamingInputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingInputCallResponse.Unmarshal(m, b)
-}
-func (m *StreamingInputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingInputCallResponse.Marshal(b, m, deterministic)
-}
-func (m *StreamingInputCallResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingInputCallResponse.Merge(m, src)
+func (x *StreamingInputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingInputCallResponse) XXX_Size() int {
-	return xxx_messageInfo_StreamingInputCallResponse.Size(m)
-}
-func (m *StreamingInputCallResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingInputCallResponse.DiscardUnknown(m)
+
+func (*StreamingInputCallResponse) ProtoMessage() {}
+
+func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingInputCallResponse proto.InternalMessageInfo
+// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
-	if m != nil {
-		return m.AggregatedPayloadSize
+func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
+	if x != nil {
+		return x.AggregatedPayloadSize
 	}
 	return 0
 }
 
 // Configuration for a particular response.
 type ResponseParameters struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload sizes in responses from the server.
 	// If response_type is COMPRESSABLE, this denotes the size before compression.
 	Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"`
 	// Desired interval between consecutive responses in the response stream in
 	// microseconds.
-	IntervalUs           int32    `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
 }
 
-func (m *ResponseParameters) Reset()         { *m = ResponseParameters{} }
-func (m *ResponseParameters) String() string { return proto.CompactTextString(m) }
-func (*ResponseParameters) ProtoMessage()    {}
-func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{7}
+func (x *ResponseParameters) Reset() {
+	*x = ResponseParameters{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ResponseParameters) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ResponseParameters.Unmarshal(m, b)
-}
-func (m *ResponseParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ResponseParameters.Marshal(b, m, deterministic)
-}
-func (m *ResponseParameters) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ResponseParameters.Merge(m, src)
+func (x *ResponseParameters) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ResponseParameters) XXX_Size() int {
-	return xxx_messageInfo_ResponseParameters.Size(m)
-}
-func (m *ResponseParameters) XXX_DiscardUnknown() {
-	xxx_messageInfo_ResponseParameters.DiscardUnknown(m)
+
+func (*ResponseParameters) ProtoMessage() {}
+
+func (x *ResponseParameters) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ResponseParameters proto.InternalMessageInfo
+// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead.
+func (*ResponseParameters) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *ResponseParameters) GetSize() int32 {
-	if m != nil {
-		return m.Size
+func (x *ResponseParameters) GetSize() int32 {
+	if x != nil {
+		return x.Size
 	}
 	return 0
 }
 
-func (m *ResponseParameters) GetIntervalUs() int32 {
-	if m != nil {
-		return m.IntervalUs
+func (x *ResponseParameters) GetIntervalUs() int32 {
+	if x != nil {
+		return x.IntervalUs
 	}
 	return 0
 }
 
 // Server-streaming request.
 type StreamingOutputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload type in the response from the server.
 	// If response_type is RANDOM, the payload from each response in the stream
 	// might be of different types. This is to simulate a mixed type of payload
@@ -559,344 +690,748 @@ type StreamingOutputCallRequest struct {
 	// Optional input payload sent along with the request.
 	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
 	// Whether server should return a given status
-	ResponseStatus       *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
+	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
 }
 
-func (m *StreamingOutputCallRequest) Reset()         { *m = StreamingOutputCallRequest{} }
-func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) }
-func (*StreamingOutputCallRequest) ProtoMessage()    {}
-func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{8}
+func (x *StreamingOutputCallRequest) Reset() {
+	*x = StreamingOutputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingOutputCallRequest.Unmarshal(m, b)
-}
-func (m *StreamingOutputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingOutputCallRequest.Marshal(b, m, deterministic)
+func (x *StreamingOutputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingOutputCallRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingOutputCallRequest.Merge(m, src)
-}
-func (m *StreamingOutputCallRequest) XXX_Size() int {
-	return xxx_messageInfo_StreamingOutputCallRequest.Size(m)
-}
-func (m *StreamingOutputCallRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingOutputCallRequest.DiscardUnknown(m)
+
+func (*StreamingOutputCallRequest) ProtoMessage() {}
+
+func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingOutputCallRequest proto.InternalMessageInfo
+// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *StreamingOutputCallRequest) GetResponseType() PayloadType {
-	if m != nil {
-		return m.ResponseType
+func (x *StreamingOutputCallRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
-	if m != nil {
-		return m.ResponseParameters
+func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
+	if x != nil {
+		return x.ResponseParameters
 	}
 	return nil
 }
 
-func (m *StreamingOutputCallRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingOutputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
-	if m != nil {
-		return m.ResponseStatus
+func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
+	if x != nil {
+		return x.ResponseStatus
 	}
 	return nil
 }
 
 // Server-streaming response, as configured by the request and parameters.
 type StreamingOutputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Payload to increase response size.
-	Payload              *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingOutputCallResponse) Reset()         { *m = StreamingOutputCallResponse{} }
-func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) }
-func (*StreamingOutputCallResponse) ProtoMessage()    {}
-func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{9}
+func (x *StreamingOutputCallResponse) Reset() {
+	*x = StreamingOutputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingOutputCallResponse.Unmarshal(m, b)
-}
-func (m *StreamingOutputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingOutputCallResponse.Marshal(b, m, deterministic)
-}
-func (m *StreamingOutputCallResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingOutputCallResponse.Merge(m, src)
+func (x *StreamingOutputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingOutputCallResponse) XXX_Size() int {
-	return xxx_messageInfo_StreamingOutputCallResponse.Size(m)
-}
-func (m *StreamingOutputCallResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingOutputCallResponse.DiscardUnknown(m)
+
+func (*StreamingOutputCallResponse) ProtoMessage() {}
+
+func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingOutputCallResponse proto.InternalMessageInfo
+// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{9}
+}
 
-func (m *StreamingOutputCallResponse) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingOutputCallResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
 type LoadBalancerStatsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Request stats for the next num_rpcs sent by client.
 	NumRpcs int32 `protobuf:"varint,1,opt,name=num_rpcs,json=numRpcs,proto3" json:"num_rpcs,omitempty"`
 	// If num_rpcs have not completed within timeout_sec, return partial results.
-	TimeoutSec           int32    `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	TimeoutSec int32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
 }
 
-func (m *LoadBalancerStatsRequest) Reset()         { *m = LoadBalancerStatsRequest{} }
-func (m *LoadBalancerStatsRequest) String() string { return proto.CompactTextString(m) }
-func (*LoadBalancerStatsRequest) ProtoMessage()    {}
-func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{10}
+func (x *LoadBalancerStatsRequest) Reset() {
+	*x = LoadBalancerStatsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *LoadBalancerStatsRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadBalancerStatsRequest.Unmarshal(m, b)
-}
-func (m *LoadBalancerStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadBalancerStatsRequest.Marshal(b, m, deterministic)
-}
-func (m *LoadBalancerStatsRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadBalancerStatsRequest.Merge(m, src)
+func (x *LoadBalancerStatsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *LoadBalancerStatsRequest) XXX_Size() int {
-	return xxx_messageInfo_LoadBalancerStatsRequest.Size(m)
-}
-func (m *LoadBalancerStatsRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadBalancerStatsRequest.DiscardUnknown(m)
+
+func (*LoadBalancerStatsRequest) ProtoMessage() {}
+
+func (x *LoadBalancerStatsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_LoadBalancerStatsRequest proto.InternalMessageInfo
+// Deprecated: Use LoadBalancerStatsRequest.ProtoReflect.Descriptor instead.
+func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{10}
+}
 
-func (m *LoadBalancerStatsRequest) GetNumRpcs() int32 {
-	if m != nil {
-		return m.NumRpcs
+func (x *LoadBalancerStatsRequest) GetNumRpcs() int32 {
+	if x != nil {
+		return x.NumRpcs
 	}
 	return 0
 }
 
-func (m *LoadBalancerStatsRequest) GetTimeoutSec() int32 {
-	if m != nil {
-		return m.TimeoutSec
+func (x *LoadBalancerStatsRequest) GetTimeoutSec() int32 {
+	if x != nil {
+		return x.TimeoutSec
 	}
 	return 0
 }
 
 type LoadBalancerStatsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The number of completed RPCs for each peer.
 	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
 	// The number of RPCs that failed to record a remote peer.
 	NumFailures int32 `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"`
 	// The number of completed RPCs for each method (UnaryCall or EmptyCall).
-	RpcsByMethod         map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}                                         `json:"-"`
-	XXX_unrecognized     []byte                                           `json:"-"`
-	XXX_sizecache        int32                                            `json:"-"`
+	RpcsByMethod map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
-func (m *LoadBalancerStatsResponse) Reset()         { *m = LoadBalancerStatsResponse{} }
-func (m *LoadBalancerStatsResponse) String() string { return proto.CompactTextString(m) }
-func (*LoadBalancerStatsResponse) ProtoMessage()    {}
-func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{11}
+func (x *LoadBalancerStatsResponse) Reset() {
+	*x = LoadBalancerStatsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *LoadBalancerStatsResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadBalancerStatsResponse.Unmarshal(m, b)
-}
-func (m *LoadBalancerStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadBalancerStatsResponse.Marshal(b, m, deterministic)
-}
-func (m *LoadBalancerStatsResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadBalancerStatsResponse.Merge(m, src)
-}
-func (m *LoadBalancerStatsResponse) XXX_Size() int {
-	return xxx_messageInfo_LoadBalancerStatsResponse.Size(m)
+func (x *LoadBalancerStatsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *LoadBalancerStatsResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadBalancerStatsResponse.DiscardUnknown(m)
+
+func (*LoadBalancerStatsResponse) ProtoMessage() {}
+
+func (x *LoadBalancerStatsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_LoadBalancerStatsResponse proto.InternalMessageInfo
+// Deprecated: Use LoadBalancerStatsResponse.ProtoReflect.Descriptor instead.
+func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{11}
+}
 
-func (m *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 {
-	if m != nil {
-		return m.RpcsByPeer
+func (x *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 {
+	if x != nil {
+		return x.RpcsByPeer
 	}
 	return nil
 }
 
-func (m *LoadBalancerStatsResponse) GetNumFailures() int32 {
-	if m != nil {
-		return m.NumFailures
+func (x *LoadBalancerStatsResponse) GetNumFailures() int32 {
+	if x != nil {
+		return x.NumFailures
 	}
 	return 0
 }
 
-func (m *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer {
-	if m != nil {
-		return m.RpcsByMethod
+func (x *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer {
+	if x != nil {
+		return x.RpcsByMethod
 	}
 	return nil
 }
 
 type LoadBalancerStatsResponse_RpcsByPeer struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The number of completed RPCs for each peer.
-	RpcsByPeer           map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}         `json:"-"`
-	XXX_unrecognized     []byte           `json:"-"`
-	XXX_sizecache        int32            `json:"-"`
+	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
 }
 
-func (m *LoadBalancerStatsResponse_RpcsByPeer) Reset()         { *m = LoadBalancerStatsResponse_RpcsByPeer{} }
-func (m *LoadBalancerStatsResponse_RpcsByPeer) String() string { return proto.CompactTextString(m) }
-func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage()    {}
-func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_534063719f48d90d, []int{11, 0}
+func (x *LoadBalancerStatsResponse_RpcsByPeer) Reset() {
+	*x = LoadBalancerStatsResponse_RpcsByPeer{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Unmarshal(m, b)
-}
-func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Marshal(b, m, deterministic)
-}
-func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Merge(m, src)
+func (x *LoadBalancerStatsResponse_RpcsByPeer) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Size() int {
-	return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Size(m)
-}
-func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_DiscardUnknown() {
-	xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.DiscardUnknown(m)
+
+func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {}
+
+func (x *LoadBalancerStatsResponse_RpcsByPeer) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer proto.InternalMessageInfo
+// Deprecated: Use LoadBalancerStatsResponse_RpcsByPeer.ProtoReflect.Descriptor instead.
+func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{11, 0}
+}
 
-func (m *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 {
-	if m != nil {
-		return m.RpcsByPeer
+func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 {
+	if x != nil {
+		return x.RpcsByPeer
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value)
-	proto.RegisterEnum("grpc.testing.GrpclbRouteType", GrpclbRouteType_name, GrpclbRouteType_value)
-	proto.RegisterType((*Empty)(nil), "grpc.testing.Empty")
-	proto.RegisterType((*Payload)(nil), "grpc.testing.Payload")
-	proto.RegisterType((*EchoStatus)(nil), "grpc.testing.EchoStatus")
-	proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest")
-	proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse")
-	proto.RegisterType((*StreamingInputCallRequest)(nil), "grpc.testing.StreamingInputCallRequest")
-	proto.RegisterType((*StreamingInputCallResponse)(nil), "grpc.testing.StreamingInputCallResponse")
-	proto.RegisterType((*ResponseParameters)(nil), "grpc.testing.ResponseParameters")
-	proto.RegisterType((*StreamingOutputCallRequest)(nil), "grpc.testing.StreamingOutputCallRequest")
-	proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse")
-	proto.RegisterType((*LoadBalancerStatsRequest)(nil), "grpc.testing.LoadBalancerStatsRequest")
-	proto.RegisterType((*LoadBalancerStatsResponse)(nil), "grpc.testing.LoadBalancerStatsResponse")
-	proto.RegisterMapType((map[string]*LoadBalancerStatsResponse_RpcsByPeer)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry")
-	proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry")
-	proto.RegisterType((*LoadBalancerStatsResponse_RpcsByPeer)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer")
-	proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry")
-}
-
-func init() { proto.RegisterFile("interop/grpc_testing/test.proto", fileDescriptor_534063719f48d90d) }
-
-var fileDescriptor_534063719f48d90d = []byte{
-	// 1083 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xdd, 0x72, 0xdb, 0xc4,
-	0x17, 0x8f, 0x1c, 0x3b, 0x8e, 0x8f, 0x5d, 0xc7, 0xd9, 0xb4, 0xff, 0xbf, 0xe2, 0x50, 0x6a, 0x04,
-	0x43, 0x4d, 0x99, 0x3a, 0x8c, 0x3b, 0x7c, 0x75, 0xa6, 0x30, 0x71, 0xe2, 0xa4, 0x99, 0x3a, 0xb6,
-	0x91, 0x63, 0x98, 0x72, 0xa3, 0xd9, 0xc8, 0x1b, 0x45, 0x83, 0xa4, 0x15, 0xab, 0x55, 0x06, 0xf7,
-	0x86, 0x19, 0x1e, 0x81, 0x57, 0xe0, 0x09, 0xb8, 0xe6, 0x6d, 0x78, 0x12, 0x66, 0x57, 0x92, 0x3f,
-	0x15, 0x1a, 0x93, 0x81, 0x2b, 0xef, 0x9e, 0xcf, 0xdf, 0xf9, 0x9d, 0x3d, 0xbb, 0x16, 0x3c, 0xb2,
-	0x3d, 0x4e, 0x18, 0xf5, 0xf7, 0x2d, 0xe6, 0x9b, 0x06, 0x27, 0x01, 0xb7, 0x3d, 0x6b, 0x5f, 0xfc,
-	0x36, 0x7c, 0x46, 0x39, 0x45, 0x25, 0xa1, 0x68, 0xc4, 0x0a, 0x2d, 0x0f, 0xb9, 0xb6, 0xeb, 0xf3,
-	0xb1, 0xd6, 0x81, 0x7c, 0x1f, 0x8f, 0x1d, 0x8a, 0x47, 0xe8, 0x29, 0x64, 0xf9, 0xd8, 0x27, 0xaa,
-	0x52, 0x53, 0xea, 0xe5, 0xe6, 0x6e, 0x63, 0xd6, 0xa1, 0x11, 0x1b, 0x9d, 0x8f, 0x7d, 0xa2, 0x4b,
-	0x33, 0x84, 0x20, 0x7b, 0x41, 0x47, 0x63, 0x35, 0x53, 0x53, 0xea, 0x25, 0x5d, 0xae, 0xb5, 0xe7,
-	0x00, 0x6d, 0xf3, 0x8a, 0x0e, 0x38, 0xe6, 0x61, 0x20, 0x2c, 0x4c, 0x3a, 0x8a, 0x02, 0xe6, 0x74,
-	0xb9, 0x46, 0x2a, 0xe4, 0x5d, 0x12, 0x04, 0xd8, 0x22, 0xd2, 0xb1, 0xa0, 0x27, 0x5b, 0xed, 0xd7,
-	0x75, 0xb8, 0x37, 0xb0, 0x5d, 0xdf, 0x21, 0x3a, 0xf9, 0x31, 0x24, 0x01, 0x47, 0x5f, 0xc1, 0x3d,
-	0x46, 0x02, 0x9f, 0x7a, 0x01, 0x31, 0x6e, 0x87, 0xac, 0x94, 0xd8, 0x8b, 0x1d, 0x7a, 0x7f, 0xc6,
-	0x3f, 0xb0, 0xdf, 0x44, 0x19, 0x73, 0x53, 0xa3, 0x81, 0xfd, 0x86, 0xa0, 0x7d, 0xc8, 0xfb, 0x51,
-	0x04, 0x75, 0xbd, 0xa6, 0xd4, 0x8b, 0xcd, 0x07, 0xa9, 0xe1, 0xf5, 0xc4, 0x4a, 0x44, 0xbd, 0xb4,
-	0x1d, 0xc7, 0x08, 0x03, 0xc2, 0x3c, 0xec, 0x12, 0x35, 0x5b, 0x53, 0xea, 0x9b, 0x7a, 0x49, 0x08,
-	0x87, 0xb1, 0x0c, 0xd5, 0xa1, 0x22, 0x8d, 0x28, 0x0e, 0xf9, 0x95, 0x11, 0x98, 0xd4, 0x27, 0x6a,
-	0x4e, 0xda, 0x95, 0x85, 0xbc, 0x27, 0xc4, 0x03, 0x21, 0x45, 0x07, 0xb0, 0x35, 0x05, 0x29, 0x79,
-	0x53, 0xf3, 0x12, 0x87, 0x3a, 0x8f, 0x63, 0xca, 0xab, 0x5e, 0x9e, 0x14, 0x10, 0xf1, 0xfc, 0x01,
-	0xc8, 0xa0, 0x46, 0x40, 0xd8, 0x35, 0x61, 0x86, 0x3d, 0x52, 0x0b, 0x53, 0x48, 0x03, 0x29, 0x3c,
-	0x1d, 0xa1, 0x67, 0xf0, 0x3f, 0x69, 0x25, 0xa2, 0x3a, 0x17, 0x06, 0xa3, 0x21, 0x8f, 0x69, 0x05,
-	0x69, 0xbd, 0x23, 0xb4, 0x27, 0x52, 0xa9, 0x0b, 0x9d, 0xa0, 0x50, 0xfb, 0x25, 0x03, 0xe5, 0xa4,
-	0x29, 0x51, 0xce, 0x59, 0xc2, 0x94, 0x5b, 0x11, 0x56, 0x85, 0xcd, 0x09, 0x57, 0x51, 0xcf, 0x27,
-	0x7b, 0xf4, 0x08, 0x8a, 0xb3, 0x14, 0xad, 0x4b, 0x35, 0xd0, 0x29, 0x3d, 0x7b, 0x50, 0x98, 0x96,
-	0x95, 0x8d, 0xbc, 0x83, 0xa4, 0xa4, 0x53, 0xd8, 0x5e, 0xae, 0x26, 0x27, 0x0f, 0xc9, 0xc3, 0x79,
-	0x50, 0x0b, 0x75, 0xe9, 0x5b, 0xd6, 0xbc, 0x40, 0x80, 0xbc, 0xa2, 0x01, 0x97, 0x20, 0x37, 0xa2,
-	0x34, 0xc9, 0x5e, 0xeb, 0xc0, 0xee, 0x80, 0x33, 0x82, 0x5d, 0xdb, 0xb3, 0x4e, 0x3d, 0x3f, 0xe4,
-	0x87, 0xd8, 0x71, 0x92, 0x43, 0xba, 0x2a, 0x1d, 0xda, 0x39, 0x54, 0xd3, 0xa2, 0xc5, 0xec, 0x7e,
-	0x06, 0xff, 0xc7, 0x96, 0xc5, 0x88, 0x85, 0x39, 0x19, 0x19, 0xb1, 0x4f, 0x74, 0x7a, 0xa3, 0x31,
-	0x7a, 0x30, 0x55, 0xc7, 0xa1, 0xc5, 0x31, 0xd6, 0x4e, 0x01, 0x25, 0x31, 0xfa, 0x98, 0x61, 0x97,
-	0x70, 0xc2, 0xe4, 0x04, 0xce, 0xb8, 0xca, 0xb5, 0xa0, 0x5c, 0xde, 0x15, 0xd7, 0x58, 0x9c, 0xe1,
-	0x78, 0x26, 0x20, 0x11, 0x0d, 0x03, 0xed, 0xb7, 0xcc, 0x0c, 0xc2, 0x5e, 0xc8, 0x17, 0x0a, 0xbe,
-	0xeb, 0x54, 0x7e, 0x03, 0x3b, 0x13, 0x7f, 0x7f, 0x02, 0x55, 0xcd, 0xd4, 0xd6, 0xeb, 0xc5, 0x66,
-	0x6d, 0x3e, 0xca, 0x72, 0x49, 0x3a, 0x62, 0xcb, 0x65, 0xae, 0x3c, 0xc3, 0x77, 0x1f, 0x3a, 0xad,
-	0x0b, 0x7b, 0xa9, 0x24, 0xfd, 0xc3, 0x29, 0xd1, 0xbe, 0x05, 0xb5, 0x43, 0xf1, 0xa8, 0x85, 0x1d,
-	0xec, 0x99, 0x84, 0x89, 0x2c, 0x41, 0x42, 0xf9, 0x2e, 0x6c, 0x7a, 0xa1, 0x6b, 0x30, 0xdf, 0x0c,
-	0xe2, 0x56, 0xe6, 0xbd, 0xd0, 0xd5, 0x7d, 0x33, 0x10, 0xdd, 0xe4, 0xb6, 0x4b, 0x68, 0xc8, 0x8d,
-	0x80, 0x98, 0x49, 0x37, 0x63, 0xd1, 0x80, 0x98, 0xda, 0x9f, 0x59, 0xd8, 0x4d, 0x09, 0x1c, 0xc3,
-	0x7c, 0x0d, 0x25, 0x11, 0xd5, 0xb8, 0x18, 0x1b, 0x3e, 0x21, 0x4c, 0x55, 0x64, 0x17, 0x3e, 0x9f,
-	0xc7, 0x7a, 0xa3, 0x7b, 0x43, 0x40, 0x68, 0x8d, 0xfb, 0x84, 0xb0, 0xb6, 0xc7, 0xd9, 0x58, 0x07,
-	0x36, 0x11, 0xa0, 0xf7, 0xa0, 0x24, 0x40, 0x5f, 0x62, 0xdb, 0x09, 0x19, 0x49, 0x0e, 0x5a, 0xd1,
-	0x0b, 0xdd, 0xe3, 0x58, 0x84, 0x0c, 0x28, 0x27, 0xd9, 0x5d, 0xc2, 0xaf, 0xa8, 0x68, 0x9f, 0xc8,
-	0xff, 0xe5, 0x6a, 0xf9, 0xcf, 0xa4, 0x6f, 0x84, 0xa0, 0xc4, 0x66, 0x44, 0xd5, 0xdf, 0x15, 0x80,
-	0x29, 0x46, 0x34, 0x4a, 0xad, 0xb6, 0xb5, 0x7a, 0xb5, 0x7f, 0x57, 0x78, 0xf5, 0x05, 0x6c, 0x2d,
-	0xa8, 0x51, 0x05, 0xd6, 0x7f, 0x20, 0x63, 0xd9, 0xbb, 0x82, 0x2e, 0x96, 0xe8, 0x3e, 0xe4, 0xae,
-	0xb1, 0x13, 0x26, 0x6f, 0x52, 0xb4, 0x79, 0x9e, 0xf9, 0x42, 0xb9, 0xab, 0x7b, 0x00, 0xdb, 0x4b,
-	0xac, 0xa4, 0x04, 0x78, 0x39, 0x1b, 0xa0, 0xd8, 0x6c, 0xae, 0xce, 0xc1, 0x4c, 0xd2, 0x27, 0x5f,
-	0x43, 0x71, 0x66, 0xe0, 0x51, 0x05, 0x4a, 0x87, 0xbd, 0xb3, 0xbe, 0xde, 0x1e, 0x0c, 0x0e, 0x5a,
-	0x9d, 0x76, 0x65, 0x0d, 0x21, 0x28, 0x0f, 0xbb, 0x73, 0x32, 0x05, 0x01, 0x6c, 0xe8, 0x07, 0xdd,
-	0xa3, 0xde, 0x59, 0x25, 0xf3, 0x84, 0xc2, 0xd6, 0xc2, 0x15, 0x8d, 0x1e, 0xc2, 0xee, 0x89, 0xde,
-	0x3f, 0xec, 0xb4, 0x0c, 0xbd, 0x37, 0x3c, 0x6f, 0x1b, 0xe7, 0xaf, 0xfb, 0x6d, 0x63, 0xd8, 0x7d,
-	0xd5, 0xed, 0x7d, 0xd7, 0xad, 0xac, 0xa1, 0x77, 0xa1, 0xba, 0xac, 0x3e, 0x3e, 0xe8, 0x74, 0x5a,
-	0x07, 0x87, 0xaf, 0x2a, 0x4a, 0xba, 0xbb, 0xd0, 0xb5, 0xbb, 0x47, 0x95, 0x4c, 0xf3, 0x8f, 0x2c,
-	0x14, 0xcf, 0x49, 0xc0, 0xc5, 0xf3, 0x68, 0x9b, 0x04, 0x7d, 0x0a, 0x05, 0xf9, 0x87, 0x48, 0x0c,
-	0x31, 0xda, 0x59, 0xb8, 0x05, 0x84, 0xa2, 0x9a, 0x26, 0x44, 0xc7, 0x50, 0x18, 0x7a, 0x98, 0x45,
-	0x6e, 0x7b, 0xf3, 0x16, 0x73, 0x7f, 0x66, 0xaa, 0xef, 0xa4, 0x2b, 0xe3, 0x39, 0x74, 0x60, 0x27,
-	0xe5, 0x36, 0x41, 0xf5, 0x05, 0xa7, 0x1b, 0x6f, 0xe5, 0xea, 0x47, 0xb7, 0xb0, 0x8c, 0x72, 0x7d,
-	0xa2, 0x20, 0x1b, 0xd0, 0xf2, 0x13, 0x84, 0x1e, 0xdf, 0x10, 0x62, 0xf1, 0xc9, 0xab, 0xd6, 0xdf,
-	0x6e, 0x18, 0xa5, 0xaa, 0x8b, 0x54, 0xe5, 0xe3, 0xd0, 0x71, 0x8e, 0x42, 0xdf, 0x21, 0x3f, 0xfd,
-	0x6b, 0x35, 0xd5, 0x15, 0x59, 0x55, 0xf9, 0x25, 0x76, 0x2e, 0xff, 0x83, 0x54, 0xcd, 0x21, 0xdc,
-	0x1f, 0x7a, 0xb2, 0x83, 0x2e, 0xf1, 0x38, 0x19, 0x25, 0xa7, 0xe8, 0x05, 0x6c, 0xcf, 0xc9, 0x57,
-	0x3b, 0x4d, 0xcd, 0x9f, 0x53, 0xde, 0x80, 0x24, 0xb4, 0x09, 0xe5, 0x13, 0xc2, 0x0f, 0x1d, 0x9b,
-	0x78, 0x5c, 0x2a, 0xd0, 0x87, 0x6f, 0x9d, 0xd9, 0xa8, 0xb6, 0xc7, 0xb7, 0x9c, 0x6d, 0x6d, 0xad,
-	0xf5, 0xf4, 0xfb, 0x8f, 0x2d, 0x4a, 0x2d, 0x87, 0x34, 0x2c, 0xea, 0x60, 0xcf, 0x6a, 0x50, 0x66,
-	0xc9, 0x2f, 0x8a, 0xfd, 0xb4, 0xcf, 0x8b, 0x8b, 0x0d, 0xf9, 0x69, 0xf1, 0xec, 0xaf, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0x8c, 0x0b, 0x8c, 0x16, 0x7d, 0x0c, 0x00, 0x00,
+var File_interop_grpc_testing_test_proto protoreflect.FileDescriptor
+
+var file_interop_grpc_testing_test_proto_rawDesc = []byte{
+	0x0a, 0x1f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6f, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22,
+	0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x4c, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
+	0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x3a, 0x0a, 0x0a, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74,
+	0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73,
+	0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61,
+	0x67, 0x65, 0x22, 0x92, 0x03, 0x0a, 0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69,
+	0x6c, 0x6c, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12,
+	0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63,
+	0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x4f,
+	0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x24, 0x0a, 0x0e,
+	0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x09,
+	0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x67, 0x72, 0x70, 0x63, 0x6c,
+	0x62, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01,
+	0x28, 0x08, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, 0x82, 0x02, 0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70,
+	0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x75,
+	0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,
+	0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x61, 0x75, 0x74, 0x68,
+	0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x61,
+	0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x49, 0x0a, 0x11, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x5f,
+	0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52,
+	0x0f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65,
+	0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x4c, 0x0a, 0x19,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61,
+	0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72,
+	0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73,
+	0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65,
+	0x67, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65,
+	0x22, 0x49, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61,
+	0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e,
+	0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x22, 0xa3, 0x02, 0x0a, 0x1a,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43,
+	0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72,
+	0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50,
+	0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x41,
+	0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x22, 0x4e, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
+	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x22, 0x56, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
+	0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a,
+	0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x07, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65,
+	0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74,
+	0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x22, 0xe2, 0x04, 0x0a, 0x19, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f,
+	0x62, 0x79, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65,
+	0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65,
+	0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x6e, 0x75, 0x6d, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72,
+	0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x46, 0x61, 0x69,
+	0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x5f, 0x0a, 0x0e, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79,
+	0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74,
+	0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79,
+	0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0xb1, 0x01, 0x0a, 0x0a, 0x52, 0x70, 0x63, 0x73, 0x42,
+	0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x64, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79,
+	0x5f, 0x70, 0x65, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42,
+	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x2e,
+	0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
+	0x0a, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x52,
+	0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
+	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
+	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3d, 0x0a, 0x0f, 0x52, 0x70,
+	0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x73, 0x0a, 0x11, 0x52, 0x70, 0x63,
+	0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
+	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
+	0x12, 0x48, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x32, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c,
+	0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50,
+	0x65, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x3f,
+	0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a,
+	0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x12,
+	0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c,
+	0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10, 0x02, 0x2a,
+	0x6f, 0x0a, 0x0f, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55,
+	0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10,
+	0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10,
+	0x01, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, 0x02,
+	0x32, 0xbb, 0x04, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x12, 0x35, 0x0a, 0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70,
+	0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79,
+	0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x6c, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70,
+	0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f,
+	0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43,
+	0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x69, 0x0a,
+	0x12, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43,
+	0x61, 0x6c, 0x6c, 0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75,
+	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c,
+	0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
+	0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74,
+	0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28,
+	0x01, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70, 0x6c, 0x65,
+	0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
+	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61,
+	0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x32, 0x55,
+	0x0a, 0x14, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x11, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
+	0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x7f, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c,
+	0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x65, 0x12, 0x63, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74,
+	0x61, 0x74, 0x73, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53,
+	0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42,
+	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x2d, 0x5a, 0x2b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6f, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_interop_grpc_testing_test_proto_rawDescOnce sync.Once
+	file_interop_grpc_testing_test_proto_rawDescData = file_interop_grpc_testing_test_proto_rawDesc
+)
+
+func file_interop_grpc_testing_test_proto_rawDescGZIP() []byte {
+	file_interop_grpc_testing_test_proto_rawDescOnce.Do(func() {
+		file_interop_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_interop_grpc_testing_test_proto_rawDescData)
+	})
+	return file_interop_grpc_testing_test_proto_rawDescData
+}
+
+var file_interop_grpc_testing_test_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_interop_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 16)
+var file_interop_grpc_testing_test_proto_goTypes = []interface{}{
+	(PayloadType)(0),                             // 0: grpc.testing.PayloadType
+	(GrpclbRouteType)(0),                         // 1: grpc.testing.GrpclbRouteType
+	(*Empty)(nil),                                // 2: grpc.testing.Empty
+	(*Payload)(nil),                              // 3: grpc.testing.Payload
+	(*EchoStatus)(nil),                           // 4: grpc.testing.EchoStatus
+	(*SimpleRequest)(nil),                        // 5: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil),                       // 6: grpc.testing.SimpleResponse
+	(*StreamingInputCallRequest)(nil),            // 7: grpc.testing.StreamingInputCallRequest
+	(*StreamingInputCallResponse)(nil),           // 8: grpc.testing.StreamingInputCallResponse
+	(*ResponseParameters)(nil),                   // 9: grpc.testing.ResponseParameters
+	(*StreamingOutputCallRequest)(nil),           // 10: grpc.testing.StreamingOutputCallRequest
+	(*StreamingOutputCallResponse)(nil),          // 11: grpc.testing.StreamingOutputCallResponse
+	(*LoadBalancerStatsRequest)(nil),             // 12: grpc.testing.LoadBalancerStatsRequest
+	(*LoadBalancerStatsResponse)(nil),            // 13: grpc.testing.LoadBalancerStatsResponse
+	(*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 14: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	nil, // 15: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	nil, // 16: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	nil, // 17: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+}
+var file_interop_grpc_testing_test_proto_depIdxs = []int32{
+	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
+	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
+	3,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
+	4,  // 3: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
+	3,  // 4: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
+	1,  // 5: grpc.testing.SimpleResponse.grpclb_route_type:type_name -> grpc.testing.GrpclbRouteType
+	3,  // 6: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
+	0,  // 7: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
+	9,  // 8: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
+	3,  // 9: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
+	4,  // 10: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
+	3,  // 11: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
+	15, // 12: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	16, // 13: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	17, // 14: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	14, // 15: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	2,  // 16: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty
+	5,  // 17: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	10, // 18: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	7,  // 19: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest
+	10, // 20: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	10, // 21: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	2,  // 22: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty
+	12, // 23: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest
+	2,  // 24: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty
+	6,  // 25: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	11, // 26: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	8,  // 27: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse
+	11, // 28: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	11, // 29: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	2,  // 30: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty
+	13, // 31: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse
+	24, // [24:32] is the sub-list for method output_type
+	16, // [16:24] is the sub-list for method input_type
+	16, // [16:16] is the sub-list for extension type_name
+	16, // [16:16] is the sub-list for extension extendee
+	0,  // [0:16] is the sub-list for field type_name
+}
+
+func init() { file_interop_grpc_testing_test_proto_init() }
+func file_interop_grpc_testing_test_proto_init() {
+	if File_interop_grpc_testing_test_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_interop_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Empty); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Payload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ResponseParameters); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerStatsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerStatsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerStatsResponse_RpcsByPeer); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_interop_grpc_testing_test_proto_rawDesc,
+			NumEnums:      2,
+			NumMessages:   16,
+			NumExtensions: 0,
+			NumServices:   3,
+		},
+		GoTypes:           file_interop_grpc_testing_test_proto_goTypes,
+		DependencyIndexes: file_interop_grpc_testing_test_proto_depIdxs,
+		EnumInfos:         file_interop_grpc_testing_test_proto_enumTypes,
+		MessageInfos:      file_interop_grpc_testing_test_proto_msgTypes,
+	}.Build()
+	File_interop_grpc_testing_test_proto = out.File
+	file_interop_grpc_testing_test_proto_rawDesc = nil
+	file_interop_grpc_testing_test_proto_goTypes = nil
+	file_interop_grpc_testing_test_proto_depIdxs = nil
 }
diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go
index 831a62721383..889c14eb9e89 100644
--- a/profiling/proto/service.pb.go
+++ b/profiling/proto/service.pb.go
@@ -1,170 +1,219 @@
+// Copyright 2019 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: profiling/proto/service.proto
 
 package proto
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // EnableRequest defines the fields in a /Profiling/Enable method request to
 // toggle profiling on and off within a gRPC program.
 type EnableRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Setting this to true will enable profiling. Setting this to false will
 	// disable profiling.
-	Enabled              bool     `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
 }
 
-func (m *EnableRequest) Reset()         { *m = EnableRequest{} }
-func (m *EnableRequest) String() string { return proto.CompactTextString(m) }
-func (*EnableRequest) ProtoMessage()    {}
-func (*EnableRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1ab2aa17b47c6fb, []int{0}
+func (x *EnableRequest) Reset() {
+	*x = EnableRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_profiling_proto_service_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *EnableRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EnableRequest.Unmarshal(m, b)
+func (x *EnableRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EnableRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EnableRequest.Marshal(b, m, deterministic)
-}
-func (m *EnableRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EnableRequest.Merge(m, src)
-}
-func (m *EnableRequest) XXX_Size() int {
-	return xxx_messageInfo_EnableRequest.Size(m)
-}
-func (m *EnableRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_EnableRequest.DiscardUnknown(m)
+
+func (*EnableRequest) ProtoMessage() {}
+
+func (x *EnableRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_profiling_proto_service_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_EnableRequest proto.InternalMessageInfo
+// Deprecated: Use EnableRequest.ProtoReflect.Descriptor instead.
+func (*EnableRequest) Descriptor() ([]byte, []int) {
+	return file_profiling_proto_service_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *EnableRequest) GetEnabled() bool {
-	if m != nil {
-		return m.Enabled
+func (x *EnableRequest) GetEnabled() bool {
+	if x != nil {
+		return x.Enabled
 	}
 	return false
 }
 
 // EnableResponse defines the fields in a /Profiling/Enable method response.
 type EnableResponse struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *EnableResponse) Reset()         { *m = EnableResponse{} }
-func (m *EnableResponse) String() string { return proto.CompactTextString(m) }
-func (*EnableResponse) ProtoMessage()    {}
-func (*EnableResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1ab2aa17b47c6fb, []int{1}
+func (x *EnableResponse) Reset() {
+	*x = EnableResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_profiling_proto_service_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *EnableResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EnableResponse.Unmarshal(m, b)
-}
-func (m *EnableResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EnableResponse.Marshal(b, m, deterministic)
-}
-func (m *EnableResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EnableResponse.Merge(m, src)
-}
-func (m *EnableResponse) XXX_Size() int {
-	return xxx_messageInfo_EnableResponse.Size(m)
+func (x *EnableResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *EnableResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_EnableResponse.DiscardUnknown(m)
+
+func (*EnableResponse) ProtoMessage() {}
+
+func (x *EnableResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_profiling_proto_service_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_EnableResponse proto.InternalMessageInfo
+// Deprecated: Use EnableResponse.ProtoReflect.Descriptor instead.
+func (*EnableResponse) Descriptor() ([]byte, []int) {
+	return file_profiling_proto_service_proto_rawDescGZIP(), []int{1}
+}
 
 // GetStreamStatsRequest defines the fields in a /Profiling/GetStreamStats
 // method request to retrieve stream-level stats in a gRPC client/server.
 type GetStreamStatsRequest struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *GetStreamStatsRequest) Reset()         { *m = GetStreamStatsRequest{} }
-func (m *GetStreamStatsRequest) String() string { return proto.CompactTextString(m) }
-func (*GetStreamStatsRequest) ProtoMessage()    {}
-func (*GetStreamStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1ab2aa17b47c6fb, []int{2}
+func (x *GetStreamStatsRequest) Reset() {
+	*x = GetStreamStatsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_profiling_proto_service_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *GetStreamStatsRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetStreamStatsRequest.Unmarshal(m, b)
-}
-func (m *GetStreamStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetStreamStatsRequest.Marshal(b, m, deterministic)
-}
-func (m *GetStreamStatsRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetStreamStatsRequest.Merge(m, src)
+func (x *GetStreamStatsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GetStreamStatsRequest) XXX_Size() int {
-	return xxx_messageInfo_GetStreamStatsRequest.Size(m)
-}
-func (m *GetStreamStatsRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetStreamStatsRequest.DiscardUnknown(m)
+
+func (*GetStreamStatsRequest) ProtoMessage() {}
+
+func (x *GetStreamStatsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_profiling_proto_service_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_GetStreamStatsRequest proto.InternalMessageInfo
+// Deprecated: Use GetStreamStatsRequest.ProtoReflect.Descriptor instead.
+func (*GetStreamStatsRequest) Descriptor() ([]byte, []int) {
+	return file_profiling_proto_service_proto_rawDescGZIP(), []int{2}
+}
 
 // GetStreamStatsResponse defines the fields in a /Profiling/GetStreamStats
 // method response.
 type GetStreamStatsResponse struct {
-	StreamStats          []*Stat  `protobuf:"bytes,1,rep,name=stream_stats,json=streamStats,proto3" json:"stream_stats,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GetStreamStatsResponse) Reset()         { *m = GetStreamStatsResponse{} }
-func (m *GetStreamStatsResponse) String() string { return proto.CompactTextString(m) }
-func (*GetStreamStatsResponse) ProtoMessage()    {}
-func (*GetStreamStatsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1ab2aa17b47c6fb, []int{3}
+	StreamStats []*Stat `protobuf:"bytes,1,rep,name=stream_stats,json=streamStats,proto3" json:"stream_stats,omitempty"`
 }
 
-func (m *GetStreamStatsResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetStreamStatsResponse.Unmarshal(m, b)
-}
-func (m *GetStreamStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetStreamStatsResponse.Marshal(b, m, deterministic)
-}
-func (m *GetStreamStatsResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetStreamStatsResponse.Merge(m, src)
+func (x *GetStreamStatsResponse) Reset() {
+	*x = GetStreamStatsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_profiling_proto_service_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *GetStreamStatsResponse) XXX_Size() int {
-	return xxx_messageInfo_GetStreamStatsResponse.Size(m)
+
+func (x *GetStreamStatsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GetStreamStatsResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetStreamStatsResponse.DiscardUnknown(m)
+
+func (*GetStreamStatsResponse) ProtoMessage() {}
+
+func (x *GetStreamStatsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_profiling_proto_service_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_GetStreamStatsResponse proto.InternalMessageInfo
+// Deprecated: Use GetStreamStatsResponse.ProtoReflect.Descriptor instead.
+func (*GetStreamStatsResponse) Descriptor() ([]byte, []int) {
+	return file_profiling_proto_service_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *GetStreamStatsResponse) GetStreamStats() []*Stat {
-	if m != nil {
-		return m.StreamStats
+func (x *GetStreamStatsResponse) GetStreamStats() []*Stat {
+	if x != nil {
+		return x.StreamStats
 	}
 	return nil
 }
@@ -173,6 +222,10 @@ func (m *GetStreamStatsResponse) GetStreamStats() []*Stat {
 // gRPC that's being profiled. It includes a tag and some additional metadata
 // to identify itself.
 type Timer struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// tags is a comma-separated list of strings used to tag a timer.
 	Tags string `protobuf:"bytes,1,opt,name=tags,proto3" json:"tags,omitempty"`
 	// begin_sec and begin_nsec are the start epoch second and nanosecond,
@@ -186,75 +239,79 @@ type Timer struct {
 	EndSec  int64 `protobuf:"varint,4,opt,name=end_sec,json=endSec,proto3" json:"end_sec,omitempty"`
 	EndNsec int32 `protobuf:"varint,5,opt,name=end_nsec,json=endNsec,proto3" json:"end_nsec,omitempty"`
 	// go_id is the goroutine ID of the component being profiled.
-	GoId                 int64    `protobuf:"varint,6,opt,name=go_id,json=goId,proto3" json:"go_id,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	GoId int64 `protobuf:"varint,6,opt,name=go_id,json=goId,proto3" json:"go_id,omitempty"`
 }
 
-func (m *Timer) Reset()         { *m = Timer{} }
-func (m *Timer) String() string { return proto.CompactTextString(m) }
-func (*Timer) ProtoMessage()    {}
-func (*Timer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1ab2aa17b47c6fb, []int{4}
+func (x *Timer) Reset() {
+	*x = Timer{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_profiling_proto_service_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Timer) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Timer.Unmarshal(m, b)
-}
-func (m *Timer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Timer.Marshal(b, m, deterministic)
-}
-func (m *Timer) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Timer.Merge(m, src)
+func (x *Timer) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Timer) XXX_Size() int {
-	return xxx_messageInfo_Timer.Size(m)
-}
-func (m *Timer) XXX_DiscardUnknown() {
-	xxx_messageInfo_Timer.DiscardUnknown(m)
+
+func (*Timer) ProtoMessage() {}
+
+func (x *Timer) ProtoReflect() protoreflect.Message {
+	mi := &file_profiling_proto_service_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Timer proto.InternalMessageInfo
+// Deprecated: Use Timer.ProtoReflect.Descriptor instead.
+func (*Timer) Descriptor() ([]byte, []int) {
+	return file_profiling_proto_service_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *Timer) GetTags() string {
-	if m != nil {
-		return m.Tags
+func (x *Timer) GetTags() string {
+	if x != nil {
+		return x.Tags
 	}
 	return ""
 }
 
-func (m *Timer) GetBeginSec() int64 {
-	if m != nil {
-		return m.BeginSec
+func (x *Timer) GetBeginSec() int64 {
+	if x != nil {
+		return x.BeginSec
 	}
 	return 0
 }
 
-func (m *Timer) GetBeginNsec() int32 {
-	if m != nil {
-		return m.BeginNsec
+func (x *Timer) GetBeginNsec() int32 {
+	if x != nil {
+		return x.BeginNsec
 	}
 	return 0
 }
 
-func (m *Timer) GetEndSec() int64 {
-	if m != nil {
-		return m.EndSec
+func (x *Timer) GetEndSec() int64 {
+	if x != nil {
+		return x.EndSec
 	}
 	return 0
 }
 
-func (m *Timer) GetEndNsec() int32 {
-	if m != nil {
-		return m.EndNsec
+func (x *Timer) GetEndNsec() int32 {
+	if x != nil {
+		return x.EndNsec
 	}
 	return 0
 }
 
-func (m *Timer) GetGoId() int64 {
-	if m != nil {
-		return m.GoId
+func (x *Timer) GetGoId() int64 {
+	if x != nil {
+		return x.GoId
 	}
 	return 0
 }
@@ -262,6 +319,10 @@ func (m *Timer) GetGoId() int64 {
 // A Stat is a collection of Timers along with some additional
 // metadata to tag and identify itself.
 type Stat struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// tags is a comma-separated list of strings used to categorize a stat.
 	Tags string `protobuf:"bytes,1,opt,name=tags,proto3" json:"tags,omitempty"`
 	// timers is an array of Timers, each representing a different
@@ -271,94 +332,247 @@ type Stat struct {
 	// undefined encoding format. For example, the Stats returned by the
 	// /Profiling/GetStreamStats service use the metadata field to encode the
 	// connection ID and the stream ID of each query.
-	Metadata             []byte   `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Metadata []byte `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"`
 }
 
-func (m *Stat) Reset()         { *m = Stat{} }
-func (m *Stat) String() string { return proto.CompactTextString(m) }
-func (*Stat) ProtoMessage()    {}
-func (*Stat) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e1ab2aa17b47c6fb, []int{5}
+func (x *Stat) Reset() {
+	*x = Stat{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_profiling_proto_service_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Stat) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Stat.Unmarshal(m, b)
+func (x *Stat) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Stat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Stat.Marshal(b, m, deterministic)
-}
-func (m *Stat) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Stat.Merge(m, src)
-}
-func (m *Stat) XXX_Size() int {
-	return xxx_messageInfo_Stat.Size(m)
-}
-func (m *Stat) XXX_DiscardUnknown() {
-	xxx_messageInfo_Stat.DiscardUnknown(m)
+
+func (*Stat) ProtoMessage() {}
+
+func (x *Stat) ProtoReflect() protoreflect.Message {
+	mi := &file_profiling_proto_service_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Stat proto.InternalMessageInfo
+// Deprecated: Use Stat.ProtoReflect.Descriptor instead.
+func (*Stat) Descriptor() ([]byte, []int) {
+	return file_profiling_proto_service_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *Stat) GetTags() string {
-	if m != nil {
-		return m.Tags
+func (x *Stat) GetTags() string {
+	if x != nil {
+		return x.Tags
 	}
 	return ""
 }
 
-func (m *Stat) GetTimers() []*Timer {
-	if m != nil {
-		return m.Timers
+func (x *Stat) GetTimers() []*Timer {
+	if x != nil {
+		return x.Timers
 	}
 	return nil
 }
 
-func (m *Stat) GetMetadata() []byte {
-	if m != nil {
-		return m.Metadata
+func (x *Stat) GetMetadata() []byte {
+	if x != nil {
+		return x.Metadata
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterType((*EnableRequest)(nil), "grpc.go.profiling.v1alpha.EnableRequest")
-	proto.RegisterType((*EnableResponse)(nil), "grpc.go.profiling.v1alpha.EnableResponse")
-	proto.RegisterType((*GetStreamStatsRequest)(nil), "grpc.go.profiling.v1alpha.GetStreamStatsRequest")
-	proto.RegisterType((*GetStreamStatsResponse)(nil), "grpc.go.profiling.v1alpha.GetStreamStatsResponse")
-	proto.RegisterType((*Timer)(nil), "grpc.go.profiling.v1alpha.Timer")
-	proto.RegisterType((*Stat)(nil), "grpc.go.profiling.v1alpha.Stat")
-}
-
-func init() { proto.RegisterFile("profiling/proto/service.proto", fileDescriptor_e1ab2aa17b47c6fb) }
-
-var fileDescriptor_e1ab2aa17b47c6fb = []byte{
-	// 392 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0x41, 0xab, 0xd3, 0x40,
-	0x10, 0xc7, 0xc9, 0x6b, 0x12, 0xd3, 0x79, 0xcf, 0x87, 0xac, 0xe8, 0xcb, 0xab, 0x14, 0x43, 0x0e,
-	0x92, 0x5e, 0x36, 0xb6, 0x5e, 0x3c, 0x17, 0x44, 0xbc, 0x88, 0x6c, 0x3d, 0x89, 0x52, 0xb6, 0xc9,
-	0xb8, 0x06, 0xd2, 0x6c, 0xcc, 0x6e, 0xfb, 0x79, 0xfc, 0x6a, 0x7e, 0x13, 0xd9, 0x49, 0x5b, 0x69,
-	0xa9, 0xc5, 0x77, 0x4a, 0x66, 0xe6, 0xff, 0x9b, 0xfd, 0x0f, 0x33, 0x30, 0x6e, 0x3b, 0xfd, 0xbd,
-	0xaa, 0xab, 0x46, 0xe5, 0x6d, 0xa7, 0xad, 0xce, 0x0d, 0x76, 0xdb, 0xaa, 0x40, 0x4e, 0x11, 0xbb,
-	0x57, 0x5d, 0x5b, 0x70, 0xa5, 0xf9, 0x41, 0xc6, 0xb7, 0x53, 0x59, 0xb7, 0x3f, 0x64, 0x3a, 0x81,
-	0xc7, 0xef, 0x1a, 0xb9, 0xaa, 0x51, 0xe0, 0xcf, 0x0d, 0x1a, 0xcb, 0x62, 0x78, 0x84, 0x94, 0x28,
-	0x63, 0x2f, 0xf1, 0xb2, 0x48, 0xec, 0xc3, 0xf4, 0x09, 0xdc, 0xee, 0xa5, 0xa6, 0xd5, 0x8d, 0xc1,
-	0xf4, 0x0e, 0x9e, 0xbd, 0x47, 0xbb, 0xb0, 0x1d, 0xca, 0xf5, 0xc2, 0x4a, 0x6b, 0x76, 0x4d, 0xd2,
-	0xaf, 0xf0, 0xfc, 0xb4, 0xd0, 0x23, 0x6c, 0x0e, 0x37, 0x86, 0xd2, 0x4b, 0xe3, 0xf2, 0xb1, 0x97,
-	0x0c, 0xb2, 0xeb, 0xd9, 0x4b, 0xfe, 0x4f, 0x87, 0xdc, 0xf1, 0xe2, 0xda, 0xfc, 0xed, 0x95, 0xfe,
-	0xf2, 0x20, 0xf8, 0x5c, 0xad, 0xb1, 0x63, 0x0c, 0x7c, 0x2b, 0x95, 0x21, 0xa7, 0x43, 0x41, 0xff,
-	0xec, 0x05, 0x0c, 0x57, 0xa8, 0xaa, 0x66, 0x69, 0xb0, 0x88, 0xaf, 0x12, 0x2f, 0x1b, 0x88, 0x88,
-	0x12, 0x0b, 0x2c, 0xd8, 0x18, 0xa0, 0x2f, 0x36, 0xae, 0x3a, 0x48, 0xbc, 0x2c, 0x10, 0xbd, 0xfc,
-	0xa3, 0xc1, 0x82, 0xdd, 0xb9, 0xe1, 0x4b, 0x22, 0x7d, 0x22, 0x43, 0x6c, 0x4a, 0xc7, 0xdd, 0x43,
-	0xe4, 0x0a, 0x44, 0x05, 0x44, 0x39, 0x21, 0x31, 0x4f, 0x21, 0x50, 0x7a, 0x59, 0x95, 0x71, 0x48,
-	0x84, 0xaf, 0xf4, 0x87, 0x32, 0x6d, 0xc1, 0x77, 0x5e, 0xcf, 0x1a, 0x7c, 0x0b, 0xa1, 0x75, 0xee,
-	0x4d, 0x7c, 0x45, 0xc3, 0x27, 0x17, 0x86, 0xa7, 0x31, 0xc5, 0x4e, 0xcf, 0x46, 0x10, 0xad, 0xd1,
-	0xca, 0x52, 0x5a, 0x49, 0xde, 0x6f, 0xc4, 0x21, 0x9e, 0xfd, 0xf6, 0x60, 0xf8, 0x69, 0xcf, 0xb3,
-	0x6f, 0x10, 0xf6, 0xbb, 0x62, 0xd9, 0x85, 0xee, 0x47, 0x9b, 0x1f, 0x4d, 0xfe, 0x43, 0xb9, 0xdb,
-	0xe2, 0x06, 0x6e, 0x8f, 0xf7, 0xcb, 0x5e, 0x5f, 0x80, 0xcf, 0xde, 0xc8, 0x68, 0xfa, 0x00, 0xa2,
-	0x7f, 0x76, 0x9e, 0x7d, 0x79, 0xa5, 0xb4, 0x56, 0x35, 0x72, 0xa5, 0x6b, 0xd9, 0x28, 0xae, 0x3b,
-	0x95, 0xbb, 0x2e, 0xf9, 0xc9, 0xfd, 0xaf, 0x42, 0xfa, 0xbc, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff,
-	0x5d, 0x47, 0x09, 0xa9, 0x19, 0x03, 0x00, 0x00,
+var File_profiling_proto_service_proto protoreflect.FileDescriptor
+
+var file_profiling_proto_service_proto_rawDesc = []byte{
+	0x0a, 0x1d, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+	0x19, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69,
+	0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x22, 0x29, 0x0a, 0x0d, 0x45, 0x6e,
+	0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65,
+	0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e,
+	0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x10, 0x0a, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x22, 0x5c, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61,
+	0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0c, 0x73, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b,
+	0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69,
+	0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x74, 0x61,
+	0x74, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x22, 0xa0,
+	0x01, 0x0a, 0x05, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09,
+	0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x08, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x53, 0x65, 0x63, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x65, 0x67,
+	0x69, 0x6e, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x62,
+	0x65, 0x67, 0x69, 0x6e, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x6e, 0x64, 0x5f,
+	0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x65, 0x6e, 0x64, 0x53, 0x65,
+	0x63, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x05, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x13, 0x0a, 0x05,
+	0x67, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x67, 0x6f, 0x49,
+	0x64, 0x22, 0x70, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67,
+	0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x38, 0x0a,
+	0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e,
+	0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x52,
+	0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
+	0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
+	0x61, 0x74, 0x61, 0x32, 0xe1, 0x01, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e,
+	0x67, 0x12, 0x5d, 0x0a, 0x06, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x28, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e,
+	0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e,
+	0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
+	0x61, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x75, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61,
+	0x74, 0x73, 0x12, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f,
+	0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x47,
+	0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70,
+	0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
+	0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x2f, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_profiling_proto_service_proto_rawDescOnce sync.Once
+	file_profiling_proto_service_proto_rawDescData = file_profiling_proto_service_proto_rawDesc
+)
+
+func file_profiling_proto_service_proto_rawDescGZIP() []byte {
+	file_profiling_proto_service_proto_rawDescOnce.Do(func() {
+		file_profiling_proto_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_profiling_proto_service_proto_rawDescData)
+	})
+	return file_profiling_proto_service_proto_rawDescData
+}
+
+var file_profiling_proto_service_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
+var file_profiling_proto_service_proto_goTypes = []interface{}{
+	(*EnableRequest)(nil),          // 0: grpc.go.profiling.v1alpha.EnableRequest
+	(*EnableResponse)(nil),         // 1: grpc.go.profiling.v1alpha.EnableResponse
+	(*GetStreamStatsRequest)(nil),  // 2: grpc.go.profiling.v1alpha.GetStreamStatsRequest
+	(*GetStreamStatsResponse)(nil), // 3: grpc.go.profiling.v1alpha.GetStreamStatsResponse
+	(*Timer)(nil),                  // 4: grpc.go.profiling.v1alpha.Timer
+	(*Stat)(nil),                   // 5: grpc.go.profiling.v1alpha.Stat
+}
+var file_profiling_proto_service_proto_depIdxs = []int32{
+	5, // 0: grpc.go.profiling.v1alpha.GetStreamStatsResponse.stream_stats:type_name -> grpc.go.profiling.v1alpha.Stat
+	4, // 1: grpc.go.profiling.v1alpha.Stat.timers:type_name -> grpc.go.profiling.v1alpha.Timer
+	0, // 2: grpc.go.profiling.v1alpha.Profiling.Enable:input_type -> grpc.go.profiling.v1alpha.EnableRequest
+	2, // 3: grpc.go.profiling.v1alpha.Profiling.GetStreamStats:input_type -> grpc.go.profiling.v1alpha.GetStreamStatsRequest
+	1, // 4: grpc.go.profiling.v1alpha.Profiling.Enable:output_type -> grpc.go.profiling.v1alpha.EnableResponse
+	3, // 5: grpc.go.profiling.v1alpha.Profiling.GetStreamStats:output_type -> grpc.go.profiling.v1alpha.GetStreamStatsResponse
+	4, // [4:6] is the sub-list for method output_type
+	2, // [2:4] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_profiling_proto_service_proto_init() }
+func file_profiling_proto_service_proto_init() {
+	if File_profiling_proto_service_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_profiling_proto_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EnableRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_profiling_proto_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EnableResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_profiling_proto_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetStreamStatsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_profiling_proto_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetStreamStatsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_profiling_proto_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Timer); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_profiling_proto_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Stat); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_profiling_proto_service_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   6,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_profiling_proto_service_proto_goTypes,
+		DependencyIndexes: file_profiling_proto_service_proto_depIdxs,
+		MessageInfos:      file_profiling_proto_service_proto_msgTypes,
+	}.Build()
+	File_profiling_proto_service_proto = out.File
+	file_profiling_proto_service_proto_rawDesc = nil
+	file_profiling_proto_service_proto_goTypes = nil
+	file_profiling_proto_service_proto_depIdxs = nil
 }
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 382612d520dd..6b8e0b77f067 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -1,72 +1,143 @@
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Service exported by server reflection
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: reflection/grpc_reflection_v1alpha/reflection.proto
 
 package grpc_reflection_v1alpha
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // The message sent by the client when calling ServerReflectionInfo method.
 type ServerReflectionRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"`
 	// To use reflection service, the client should set one of the following
 	// fields in message_request. The server distinguishes requests by their
 	// defined field and then handles them using corresponding methods.
 	//
-	// Types that are valid to be assigned to MessageRequest:
+	// Types that are assignable to MessageRequest:
 	//	*ServerReflectionRequest_FileByFilename
 	//	*ServerReflectionRequest_FileContainingSymbol
 	//	*ServerReflectionRequest_FileContainingExtension
 	//	*ServerReflectionRequest_AllExtensionNumbersOfType
 	//	*ServerReflectionRequest_ListServices
-	MessageRequest       isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"`
-	XXX_NoUnkeyedLiteral struct{}                                 `json:"-"`
-	XXX_unrecognized     []byte                                   `json:"-"`
-	XXX_sizecache        int32                                    `json:"-"`
+	MessageRequest isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"`
+}
+
+func (x *ServerReflectionRequest) Reset() {
+	*x = ServerReflectionRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ServerReflectionRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *ServerReflectionRequest) Reset()         { *m = ServerReflectionRequest{} }
-func (m *ServerReflectionRequest) String() string { return proto.CompactTextString(m) }
-func (*ServerReflectionRequest) ProtoMessage()    {}
+func (*ServerReflectionRequest) ProtoMessage() {}
+
+func (x *ServerReflectionRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ServerReflectionRequest.ProtoReflect.Descriptor instead.
 func (*ServerReflectionRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{0}
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{0}
 }
 
-func (m *ServerReflectionRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerReflectionRequest.Unmarshal(m, b)
+func (x *ServerReflectionRequest) GetHost() string {
+	if x != nil {
+		return x.Host
+	}
+	return ""
 }
-func (m *ServerReflectionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerReflectionRequest.Marshal(b, m, deterministic)
+
+func (m *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest {
+	if m != nil {
+		return m.MessageRequest
+	}
+	return nil
 }
-func (m *ServerReflectionRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerReflectionRequest.Merge(m, src)
+
+func (x *ServerReflectionRequest) GetFileByFilename() string {
+	if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileByFilename); ok {
+		return x.FileByFilename
+	}
+	return ""
 }
-func (m *ServerReflectionRequest) XXX_Size() int {
-	return xxx_messageInfo_ServerReflectionRequest.Size(m)
+
+func (x *ServerReflectionRequest) GetFileContainingSymbol() string {
+	if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileContainingSymbol); ok {
+		return x.FileContainingSymbol
+	}
+	return ""
 }
-func (m *ServerReflectionRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerReflectionRequest.DiscardUnknown(m)
+
+func (x *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest {
+	if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileContainingExtension); ok {
+		return x.FileContainingExtension
+	}
+	return nil
 }
 
-var xxx_messageInfo_ServerReflectionRequest proto.InternalMessageInfo
+func (x *ServerReflectionRequest) GetAllExtensionNumbersOfType() string {
+	if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_AllExtensionNumbersOfType); ok {
+		return x.AllExtensionNumbersOfType
+	}
+	return ""
+}
 
-func (m *ServerReflectionRequest) GetHost() string {
-	if m != nil {
-		return m.Host
+func (x *ServerReflectionRequest) GetListServices() string {
+	if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_ListServices); ok {
+		return x.ListServices
 	}
 	return ""
 }
@@ -76,22 +147,38 @@ type isServerReflectionRequest_MessageRequest interface {
 }
 
 type ServerReflectionRequest_FileByFilename struct {
+	// Find a proto file by the file name.
 	FileByFilename string `protobuf:"bytes,3,opt,name=file_by_filename,json=fileByFilename,proto3,oneof"`
 }
 
 type ServerReflectionRequest_FileContainingSymbol struct {
+	// Find the proto file that declares the given fully-qualified symbol name.
+	// This field should be a fully-qualified symbol name
+	// (e.g. <package>.<service>[.<method>] or <package>.<type>).
 	FileContainingSymbol string `protobuf:"bytes,4,opt,name=file_containing_symbol,json=fileContainingSymbol,proto3,oneof"`
 }
 
 type ServerReflectionRequest_FileContainingExtension struct {
+	// Find the proto file which defines an extension extending the given
+	// message type with the given field number.
 	FileContainingExtension *ExtensionRequest `protobuf:"bytes,5,opt,name=file_containing_extension,json=fileContainingExtension,proto3,oneof"`
 }
 
 type ServerReflectionRequest_AllExtensionNumbersOfType struct {
+	// Finds the tag numbers used by all known extensions of extendee_type, and
+	// appends them to ExtensionNumberResponse in an undefined order.
+	// Its corresponding method is best-effort: it's not guaranteed that the
+	// reflection service will implement this method, and it's not guaranteed
+	// that this method will provide all extensions. Returns
+	// StatusCode::UNIMPLEMENTED if it's not implemented.
+	// This field should be a fully-qualified type name. The format is
+	// <package>.<type>
 	AllExtensionNumbersOfType string `protobuf:"bytes,6,opt,name=all_extension_numbers_of_type,json=allExtensionNumbersOfType,proto3,oneof"`
 }
 
 type ServerReflectionRequest_ListServices struct {
+	// List the full names of registered services. The content will not be
+	// checked.
 	ListServices string `protobuf:"bytes,7,opt,name=list_services,json=listServices,proto3,oneof"`
 }
 
@@ -106,162 +193,160 @@ func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequ
 
 func (*ServerReflectionRequest_ListServices) isServerReflectionRequest_MessageRequest() {}
 
-func (m *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest {
-	if m != nil {
-		return m.MessageRequest
-	}
-	return nil
-}
-
-func (m *ServerReflectionRequest) GetFileByFilename() string {
-	if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_FileByFilename); ok {
-		return x.FileByFilename
-	}
-	return ""
-}
+// The type name and extension number sent by the client when requesting
+// file_containing_extension.
+type ExtensionRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *ServerReflectionRequest) GetFileContainingSymbol() string {
-	if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_FileContainingSymbol); ok {
-		return x.FileContainingSymbol
-	}
-	return ""
+	// Fully-qualified type name. The format should be <package>.<type>
+	ContainingType  string `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"`
+	ExtensionNumber int32  `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"`
 }
 
-func (m *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest {
-	if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_FileContainingExtension); ok {
-		return x.FileContainingExtension
+func (x *ExtensionRequest) Reset() {
+	*x = ExtensionRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-func (m *ServerReflectionRequest) GetAllExtensionNumbersOfType() string {
-	if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_AllExtensionNumbersOfType); ok {
-		return x.AllExtensionNumbersOfType
-	}
-	return ""
+func (x *ExtensionRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *ServerReflectionRequest) GetListServices() string {
-	if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_ListServices); ok {
-		return x.ListServices
-	}
-	return ""
-}
+func (*ExtensionRequest) ProtoMessage() {}
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*ServerReflectionRequest) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*ServerReflectionRequest_FileByFilename)(nil),
-		(*ServerReflectionRequest_FileContainingSymbol)(nil),
-		(*ServerReflectionRequest_FileContainingExtension)(nil),
-		(*ServerReflectionRequest_AllExtensionNumbersOfType)(nil),
-		(*ServerReflectionRequest_ListServices)(nil),
+func (x *ExtensionRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
 	}
+	return mi.MessageOf(x)
 }
 
-// The type name and extension number sent by the client when requesting
-// file_containing_extension.
-type ExtensionRequest struct {
-	// Fully-qualified type name. The format should be <package>.<type>
-	ContainingType       string   `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"`
-	ExtensionNumber      int32    `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *ExtensionRequest) Reset()         { *m = ExtensionRequest{} }
-func (m *ExtensionRequest) String() string { return proto.CompactTextString(m) }
-func (*ExtensionRequest) ProtoMessage()    {}
+// Deprecated: Use ExtensionRequest.ProtoReflect.Descriptor instead.
 func (*ExtensionRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{1}
-}
-
-func (m *ExtensionRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ExtensionRequest.Unmarshal(m, b)
-}
-func (m *ExtensionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ExtensionRequest.Marshal(b, m, deterministic)
-}
-func (m *ExtensionRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ExtensionRequest.Merge(m, src)
-}
-func (m *ExtensionRequest) XXX_Size() int {
-	return xxx_messageInfo_ExtensionRequest.Size(m)
-}
-func (m *ExtensionRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_ExtensionRequest.DiscardUnknown(m)
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{1}
 }
 
-var xxx_messageInfo_ExtensionRequest proto.InternalMessageInfo
-
-func (m *ExtensionRequest) GetContainingType() string {
-	if m != nil {
-		return m.ContainingType
+func (x *ExtensionRequest) GetContainingType() string {
+	if x != nil {
+		return x.ContainingType
 	}
 	return ""
 }
 
-func (m *ExtensionRequest) GetExtensionNumber() int32 {
-	if m != nil {
-		return m.ExtensionNumber
+func (x *ExtensionRequest) GetExtensionNumber() int32 {
+	if x != nil {
+		return x.ExtensionNumber
 	}
 	return 0
 }
 
 // The message sent by the server to answer ServerReflectionInfo method.
 type ServerReflectionResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	ValidHost       string                   `protobuf:"bytes,1,opt,name=valid_host,json=validHost,proto3" json:"valid_host,omitempty"`
 	OriginalRequest *ServerReflectionRequest `protobuf:"bytes,2,opt,name=original_request,json=originalRequest,proto3" json:"original_request,omitempty"`
 	// The server sets one of the following fields according to the
 	// message_request in the request.
 	//
-	// Types that are valid to be assigned to MessageResponse:
+	// Types that are assignable to MessageResponse:
 	//	*ServerReflectionResponse_FileDescriptorResponse
 	//	*ServerReflectionResponse_AllExtensionNumbersResponse
 	//	*ServerReflectionResponse_ListServicesResponse
 	//	*ServerReflectionResponse_ErrorResponse
-	MessageResponse      isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"`
-	XXX_NoUnkeyedLiteral struct{}                                   `json:"-"`
-	XXX_unrecognized     []byte                                     `json:"-"`
-	XXX_sizecache        int32                                      `json:"-"`
+	MessageResponse isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"`
 }
 
-func (m *ServerReflectionResponse) Reset()         { *m = ServerReflectionResponse{} }
-func (m *ServerReflectionResponse) String() string { return proto.CompactTextString(m) }
-func (*ServerReflectionResponse) ProtoMessage()    {}
-func (*ServerReflectionResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{2}
+func (x *ServerReflectionResponse) Reset() {
+	*x = ServerReflectionResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServerReflectionResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServerReflectionResponse.Unmarshal(m, b)
-}
-func (m *ServerReflectionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServerReflectionResponse.Marshal(b, m, deterministic)
+func (x *ServerReflectionResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServerReflectionResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServerReflectionResponse.Merge(m, src)
+
+func (*ServerReflectionResponse) ProtoMessage() {}
+
+func (x *ServerReflectionResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *ServerReflectionResponse) XXX_Size() int {
-	return xxx_messageInfo_ServerReflectionResponse.Size(m)
+
+// Deprecated: Use ServerReflectionResponse.ProtoReflect.Descriptor instead.
+func (*ServerReflectionResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{2}
 }
-func (m *ServerReflectionResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServerReflectionResponse.DiscardUnknown(m)
+
+func (x *ServerReflectionResponse) GetValidHost() string {
+	if x != nil {
+		return x.ValidHost
+	}
+	return ""
 }
 
-var xxx_messageInfo_ServerReflectionResponse proto.InternalMessageInfo
+func (x *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest {
+	if x != nil {
+		return x.OriginalRequest
+	}
+	return nil
+}
 
-func (m *ServerReflectionResponse) GetValidHost() string {
+func (m *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse {
 	if m != nil {
-		return m.ValidHost
+		return m.MessageResponse
 	}
-	return ""
+	return nil
 }
 
-func (m *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest {
-	if m != nil {
-		return m.OriginalRequest
+func (x *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse {
+	if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_FileDescriptorResponse); ok {
+		return x.FileDescriptorResponse
+	}
+	return nil
+}
+
+func (x *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse {
+	if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_AllExtensionNumbersResponse); ok {
+		return x.AllExtensionNumbersResponse
+	}
+	return nil
+}
+
+func (x *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse {
+	if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_ListServicesResponse); ok {
+		return x.ListServicesResponse
+	}
+	return nil
+}
+
+func (x *ServerReflectionResponse) GetErrorResponse() *ErrorResponse {
+	if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_ErrorResponse); ok {
+		return x.ErrorResponse
 	}
 	return nil
 }
@@ -271,18 +356,27 @@ type isServerReflectionResponse_MessageResponse interface {
 }
 
 type ServerReflectionResponse_FileDescriptorResponse struct {
+	// This message is used to answer file_by_filename, file_containing_symbol,
+	// file_containing_extension requests with transitive dependencies.
+	// As the repeated label is not allowed in oneof fields, we use a
+	// FileDescriptorResponse message to encapsulate the repeated fields.
+	// The reflection service is allowed to avoid sending FileDescriptorProtos
+	// that were previously sent in response to earlier requests in the stream.
 	FileDescriptorResponse *FileDescriptorResponse `protobuf:"bytes,4,opt,name=file_descriptor_response,json=fileDescriptorResponse,proto3,oneof"`
 }
 
 type ServerReflectionResponse_AllExtensionNumbersResponse struct {
+	// This message is used to answer all_extension_numbers_of_type requests.
 	AllExtensionNumbersResponse *ExtensionNumberResponse `protobuf:"bytes,5,opt,name=all_extension_numbers_response,json=allExtensionNumbersResponse,proto3,oneof"`
 }
 
 type ServerReflectionResponse_ListServicesResponse struct {
+	// This message is used to answer list_services requests.
 	ListServicesResponse *ListServiceResponse `protobuf:"bytes,6,opt,name=list_services_response,json=listServicesResponse,proto3,oneof"`
 }
 
 type ServerReflectionResponse_ErrorResponse struct {
+	// This message is used when an error occurs.
 	ErrorResponse *ErrorResponse `protobuf:"bytes,7,opt,name=error_response,json=errorResponse,proto3,oneof"`
 }
 
@@ -296,92 +390,55 @@ func (*ServerReflectionResponse_ListServicesResponse) isServerReflectionResponse
 
 func (*ServerReflectionResponse_ErrorResponse) isServerReflectionResponse_MessageResponse() {}
 
-func (m *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse {
-	if m != nil {
-		return m.MessageResponse
-	}
-	return nil
-}
-
-func (m *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse {
-	if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_FileDescriptorResponse); ok {
-		return x.FileDescriptorResponse
-	}
-	return nil
-}
-
-func (m *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse {
-	if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_AllExtensionNumbersResponse); ok {
-		return x.AllExtensionNumbersResponse
-	}
-	return nil
-}
-
-func (m *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse {
-	if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_ListServicesResponse); ok {
-		return x.ListServicesResponse
-	}
-	return nil
-}
-
-func (m *ServerReflectionResponse) GetErrorResponse() *ErrorResponse {
-	if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_ErrorResponse); ok {
-		return x.ErrorResponse
-	}
-	return nil
-}
-
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*ServerReflectionResponse) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*ServerReflectionResponse_FileDescriptorResponse)(nil),
-		(*ServerReflectionResponse_AllExtensionNumbersResponse)(nil),
-		(*ServerReflectionResponse_ListServicesResponse)(nil),
-		(*ServerReflectionResponse_ErrorResponse)(nil),
-	}
-}
-
 // Serialized FileDescriptorProto messages sent by the server answering
 // a file_by_filename, file_containing_symbol, or file_containing_extension
 // request.
 type FileDescriptorResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Serialized FileDescriptorProto messages. We avoid taking a dependency on
 	// descriptor.proto, which uses proto2 only features, by making them opaque
 	// bytes instead.
-	FileDescriptorProto  [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	FileDescriptorProto [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"`
 }
 
-func (m *FileDescriptorResponse) Reset()         { *m = FileDescriptorResponse{} }
-func (m *FileDescriptorResponse) String() string { return proto.CompactTextString(m) }
-func (*FileDescriptorResponse) ProtoMessage()    {}
-func (*FileDescriptorResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{3}
+func (x *FileDescriptorResponse) Reset() {
+	*x = FileDescriptorResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *FileDescriptorResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_FileDescriptorResponse.Unmarshal(m, b)
-}
-func (m *FileDescriptorResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_FileDescriptorResponse.Marshal(b, m, deterministic)
-}
-func (m *FileDescriptorResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_FileDescriptorResponse.Merge(m, src)
+func (x *FileDescriptorResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *FileDescriptorResponse) XXX_Size() int {
-	return xxx_messageInfo_FileDescriptorResponse.Size(m)
-}
-func (m *FileDescriptorResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_FileDescriptorResponse.DiscardUnknown(m)
+
+func (*FileDescriptorResponse) ProtoMessage() {}
+
+func (x *FileDescriptorResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_FileDescriptorResponse proto.InternalMessageInfo
+// Deprecated: Use FileDescriptorResponse.ProtoReflect.Descriptor instead.
+func (*FileDescriptorResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *FileDescriptorResponse) GetFileDescriptorProto() [][]byte {
-	if m != nil {
-		return m.FileDescriptorProto
+func (x *FileDescriptorResponse) GetFileDescriptorProto() [][]byte {
+	if x != nil {
+		return x.FileDescriptorProto
 	}
 	return nil
 }
@@ -389,92 +446,108 @@ func (m *FileDescriptorResponse) GetFileDescriptorProto() [][]byte {
 // A list of extension numbers sent by the server answering
 // all_extension_numbers_of_type request.
 type ExtensionNumberResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Full name of the base type, including the package name. The format
 	// is <package>.<type>
-	BaseTypeName         string   `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"`
-	ExtensionNumber      []int32  `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	BaseTypeName    string  `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"`
+	ExtensionNumber []int32 `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"`
 }
 
-func (m *ExtensionNumberResponse) Reset()         { *m = ExtensionNumberResponse{} }
-func (m *ExtensionNumberResponse) String() string { return proto.CompactTextString(m) }
-func (*ExtensionNumberResponse) ProtoMessage()    {}
-func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{4}
+func (x *ExtensionNumberResponse) Reset() {
+	*x = ExtensionNumberResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ExtensionNumberResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ExtensionNumberResponse.Unmarshal(m, b)
-}
-func (m *ExtensionNumberResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ExtensionNumberResponse.Marshal(b, m, deterministic)
+func (x *ExtensionNumberResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ExtensionNumberResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ExtensionNumberResponse.Merge(m, src)
-}
-func (m *ExtensionNumberResponse) XXX_Size() int {
-	return xxx_messageInfo_ExtensionNumberResponse.Size(m)
-}
-func (m *ExtensionNumberResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_ExtensionNumberResponse.DiscardUnknown(m)
+
+func (*ExtensionNumberResponse) ProtoMessage() {}
+
+func (x *ExtensionNumberResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ExtensionNumberResponse proto.InternalMessageInfo
+// Deprecated: Use ExtensionNumberResponse.ProtoReflect.Descriptor instead.
+func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *ExtensionNumberResponse) GetBaseTypeName() string {
-	if m != nil {
-		return m.BaseTypeName
+func (x *ExtensionNumberResponse) GetBaseTypeName() string {
+	if x != nil {
+		return x.BaseTypeName
 	}
 	return ""
 }
 
-func (m *ExtensionNumberResponse) GetExtensionNumber() []int32 {
-	if m != nil {
-		return m.ExtensionNumber
+func (x *ExtensionNumberResponse) GetExtensionNumber() []int32 {
+	if x != nil {
+		return x.ExtensionNumber
 	}
 	return nil
 }
 
 // A list of ServiceResponse sent by the server answering list_services request.
 type ListServiceResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The information of each service may be expanded in the future, so we use
 	// ServiceResponse message to encapsulate it.
-	Service              []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
-	XXX_unrecognized     []byte             `json:"-"`
-	XXX_sizecache        int32              `json:"-"`
+	Service []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"`
 }
 
-func (m *ListServiceResponse) Reset()         { *m = ListServiceResponse{} }
-func (m *ListServiceResponse) String() string { return proto.CompactTextString(m) }
-func (*ListServiceResponse) ProtoMessage()    {}
-func (*ListServiceResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{5}
+func (x *ListServiceResponse) Reset() {
+	*x = ListServiceResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ListServiceResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ListServiceResponse.Unmarshal(m, b)
-}
-func (m *ListServiceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ListServiceResponse.Marshal(b, m, deterministic)
-}
-func (m *ListServiceResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ListServiceResponse.Merge(m, src)
-}
-func (m *ListServiceResponse) XXX_Size() int {
-	return xxx_messageInfo_ListServiceResponse.Size(m)
+func (x *ListServiceResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ListServiceResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_ListServiceResponse.DiscardUnknown(m)
+
+func (*ListServiceResponse) ProtoMessage() {}
+
+func (x *ListServiceResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ListServiceResponse proto.InternalMessageInfo
+// Deprecated: Use ListServiceResponse.ProtoReflect.Descriptor instead.
+func (*ListServiceResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *ListServiceResponse) GetService() []*ServiceResponse {
-	if m != nil {
-		return m.Service
+func (x *ListServiceResponse) GetService() []*ServiceResponse {
+	if x != nil {
+		return x.Service
 	}
 	return nil
 }
@@ -482,153 +555,399 @@ func (m *ListServiceResponse) GetService() []*ServiceResponse {
 // The information of a single service used by ListServiceResponse to answer
 // list_services request.
 type ServiceResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Full name of a registered service, including its package name. The format
 	// is <package>.<service>
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *ServiceResponse) Reset()         { *m = ServiceResponse{} }
-func (m *ServiceResponse) String() string { return proto.CompactTextString(m) }
-func (*ServiceResponse) ProtoMessage()    {}
-func (*ServiceResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{6}
+func (x *ServiceResponse) Reset() {
+	*x = ServiceResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ServiceResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ServiceResponse.Unmarshal(m, b)
-}
-func (m *ServiceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ServiceResponse.Marshal(b, m, deterministic)
+func (x *ServiceResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ServiceResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ServiceResponse.Merge(m, src)
-}
-func (m *ServiceResponse) XXX_Size() int {
-	return xxx_messageInfo_ServiceResponse.Size(m)
-}
-func (m *ServiceResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_ServiceResponse.DiscardUnknown(m)
+
+func (*ServiceResponse) ProtoMessage() {}
+
+func (x *ServiceResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ServiceResponse proto.InternalMessageInfo
+// Deprecated: Use ServiceResponse.ProtoReflect.Descriptor instead.
+func (*ServiceResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *ServiceResponse) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *ServiceResponse) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 // The error code and error message sent by the server when an error occurs.
 type ErrorResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// This field uses the error codes defined in grpc::StatusCode.
-	ErrorCode            int32    `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"`
-	ErrorMessage         string   `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	ErrorCode    int32  `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"`
+	ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`
 }
 
-func (m *ErrorResponse) Reset()         { *m = ErrorResponse{} }
-func (m *ErrorResponse) String() string { return proto.CompactTextString(m) }
-func (*ErrorResponse) ProtoMessage()    {}
-func (*ErrorResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e8cf9f2921ad6c95, []int{7}
+func (x *ErrorResponse) Reset() {
+	*x = ErrorResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ErrorResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ErrorResponse.Unmarshal(m, b)
+func (x *ErrorResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ErrorResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ErrorResponse.Marshal(b, m, deterministic)
-}
-func (m *ErrorResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ErrorResponse.Merge(m, src)
-}
-func (m *ErrorResponse) XXX_Size() int {
-	return xxx_messageInfo_ErrorResponse.Size(m)
-}
-func (m *ErrorResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_ErrorResponse.DiscardUnknown(m)
+
+func (*ErrorResponse) ProtoMessage() {}
+
+func (x *ErrorResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ErrorResponse proto.InternalMessageInfo
+// Deprecated: Use ErrorResponse.ProtoReflect.Descriptor instead.
+func (*ErrorResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *ErrorResponse) GetErrorCode() int32 {
-	if m != nil {
-		return m.ErrorCode
+func (x *ErrorResponse) GetErrorCode() int32 {
+	if x != nil {
+		return x.ErrorCode
 	}
 	return 0
 }
 
-func (m *ErrorResponse) GetErrorMessage() string {
-	if m != nil {
-		return m.ErrorMessage
+func (x *ErrorResponse) GetErrorMessage() string {
+	if x != nil {
+		return x.ErrorMessage
 	}
 	return ""
 }
 
-func init() {
-	proto.RegisterType((*ServerReflectionRequest)(nil), "grpc.reflection.v1alpha.ServerReflectionRequest")
-	proto.RegisterType((*ExtensionRequest)(nil), "grpc.reflection.v1alpha.ExtensionRequest")
-	proto.RegisterType((*ServerReflectionResponse)(nil), "grpc.reflection.v1alpha.ServerReflectionResponse")
-	proto.RegisterType((*FileDescriptorResponse)(nil), "grpc.reflection.v1alpha.FileDescriptorResponse")
-	proto.RegisterType((*ExtensionNumberResponse)(nil), "grpc.reflection.v1alpha.ExtensionNumberResponse")
-	proto.RegisterType((*ListServiceResponse)(nil), "grpc.reflection.v1alpha.ListServiceResponse")
-	proto.RegisterType((*ServiceResponse)(nil), "grpc.reflection.v1alpha.ServiceResponse")
-	proto.RegisterType((*ErrorResponse)(nil), "grpc.reflection.v1alpha.ErrorResponse")
-}
-
-func init() {
-	proto.RegisterFile("reflection/grpc_reflection_v1alpha/reflection.proto", fileDescriptor_e8cf9f2921ad6c95)
-}
-
-var fileDescriptor_e8cf9f2921ad6c95 = []byte{
-	// 686 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x41, 0x6f, 0xd3, 0x4c,
-	0x10, 0xad, 0xdb, 0xa4, 0x55, 0x26, 0x69, 0x92, 0x6f, 0xdb, 0xaf, 0x71, 0x41, 0x45, 0x91, 0xa1,
-	0x90, 0x22, 0x94, 0xb4, 0xa9, 0x84, 0x84, 0xb8, 0xa5, 0x80, 0x82, 0x54, 0x5a, 0xe4, 0x70, 0x01,
-	0x0e, 0x2b, 0x27, 0x99, 0xb8, 0x06, 0xc7, 0x6b, 0x76, 0xdd, 0x40, 0x4e, 0xfc, 0x08, 0x7e, 0x14,
-	0x7f, 0x89, 0x23, 0xda, 0xb5, 0x63, 0x3b, 0x6e, 0x4c, 0xd5, 0x53, 0x9c, 0x37, 0x33, 0xfb, 0x66,
-	0xf6, 0xbd, 0xb1, 0xe1, 0x94, 0xe3, 0xc4, 0xc5, 0x51, 0xe0, 0x30, 0xaf, 0x63, 0x73, 0x7f, 0x44,
-	0x93, 0xff, 0x74, 0x76, 0x62, 0xb9, 0xfe, 0x95, 0xd5, 0x49, 0xa0, 0xb6, 0xcf, 0x59, 0xc0, 0x48,
-	0x43, 0x66, 0xb6, 0x53, 0x70, 0x94, 0x69, 0xfc, 0x59, 0x87, 0xc6, 0x00, 0xf9, 0x0c, 0xb9, 0x19,
-	0x07, 0x4d, 0xfc, 0x76, 0x8d, 0x22, 0x20, 0x04, 0x0a, 0x57, 0x4c, 0x04, 0xba, 0xd6, 0xd4, 0x5a,
-	0x25, 0x53, 0x3d, 0x93, 0xa7, 0x50, 0x9f, 0x38, 0x2e, 0xd2, 0xe1, 0x9c, 0xca, 0x5f, 0xcf, 0x9a,
-	0xa2, 0xbe, 0x21, 0xe3, 0xfd, 0x35, 0xb3, 0x2a, 0x91, 0xde, 0xfc, 0x4d, 0x84, 0x93, 0xe7, 0xb0,
-	0xa7, 0x72, 0x47, 0xcc, 0x0b, 0x2c, 0xc7, 0x73, 0x3c, 0x9b, 0x8a, 0xf9, 0x74, 0xc8, 0x5c, 0xbd,
-	0x10, 0x55, 0xec, 0xca, 0xf8, 0x59, 0x1c, 0x1e, 0xa8, 0x28, 0xb1, 0x61, 0x3f, 0x5b, 0x87, 0x3f,
-	0x02, 0xf4, 0x84, 0xc3, 0x3c, 0xbd, 0xd8, 0xd4, 0x5a, 0xe5, 0xee, 0x51, 0x3b, 0x67, 0xa0, 0xf6,
-	0xeb, 0x45, 0x66, 0x34, 0x45, 0x7f, 0xcd, 0x6c, 0x2c, 0xb3, 0xc4, 0x19, 0xa4, 0x07, 0x07, 0x96,
-	0xeb, 0x26, 0x87, 0x53, 0xef, 0x7a, 0x3a, 0x44, 0x2e, 0x28, 0x9b, 0xd0, 0x60, 0xee, 0xa3, 0xbe,
-	0x19, 0xf5, 0xb9, 0x6f, 0xb9, 0x6e, 0x5c, 0x76, 0x11, 0x26, 0x5d, 0x4e, 0x3e, 0xcc, 0x7d, 0x24,
-	0x87, 0xb0, 0xed, 0x3a, 0x22, 0xa0, 0x02, 0xf9, 0xcc, 0x19, 0xa1, 0xd0, 0xb7, 0xa2, 0x9a, 0x8a,
-	0x84, 0x07, 0x11, 0xda, 0xfb, 0x0f, 0x6a, 0x53, 0x14, 0xc2, 0xb2, 0x91, 0xf2, 0xb0, 0x31, 0x63,
-	0x02, 0xf5, 0x6c, 0xb3, 0xe4, 0x09, 0xd4, 0x52, 0x53, 0xab, 0x1e, 0xc2, 0xdb, 0xaf, 0x26, 0xb0,
-	0xa2, 0x3d, 0x82, 0x7a, 0xb6, 0x6d, 0x7d, 0xbd, 0xa9, 0xb5, 0x8a, 0x66, 0x0d, 0x97, 0x1b, 0x35,
-	0x7e, 0x17, 0x40, 0xbf, 0x29, 0xb1, 0xf0, 0x99, 0x27, 0x90, 0x1c, 0x00, 0xcc, 0x2c, 0xd7, 0x19,
-	0xd3, 0x94, 0xd2, 0x25, 0x85, 0xf4, 0xa5, 0xdc, 0x9f, 0xa1, 0xce, 0xb8, 0x63, 0x3b, 0x9e, 0xe5,
-	0x2e, 0xfa, 0x56, 0x34, 0xe5, 0xee, 0x71, 0xae, 0x02, 0x39, 0x76, 0x32, 0x6b, 0x8b, 0x93, 0x16,
-	0xc3, 0x7e, 0x05, 0x5d, 0xe9, 0x3c, 0x46, 0x31, 0xe2, 0x8e, 0x1f, 0x30, 0x4e, 0x79, 0xd4, 0x97,
-	0x72, 0x48, 0xb9, 0xdb, 0xc9, 0x25, 0x91, 0x26, 0x7b, 0x15, 0xd7, 0x2d, 0xc6, 0xe9, 0xaf, 0x99,
-	0xca, 0x72, 0x37, 0x23, 0xe4, 0x3b, 0x3c, 0x58, 0xad, 0x75, 0x4c, 0x59, 0xbc, 0x65, 0xae, 0x8c,
-	0x01, 0x52, 0x9c, 0xf7, 0x57, 0xd8, 0x23, 0x26, 0x1e, 0xc3, 0xde, 0x92, 0x41, 0x12, 0xc2, 0x4d,
-	0x45, 0xf8, 0x2c, 0x97, 0xf0, 0x3c, 0x31, 0x50, 0x8a, 0x6c, 0x37, 0xed, 0xab, 0x98, 0xe5, 0x12,
-	0xaa, 0xc8, 0x79, 0xfa, 0x06, 0xb7, 0xd4, 0xe9, 0x8f, 0xf3, 0xc7, 0x91, 0xe9, 0xa9, 0x73, 0xb7,
-	0x31, 0x0d, 0xf4, 0x08, 0xd4, 0x13, 0xc3, 0x86, 0x98, 0x71, 0x0e, 0x7b, 0xab, 0xef, 0x9d, 0x74,
-	0xe1, 0xff, 0xac, 0x94, 0xea, 0xc5, 0xa3, 0x6b, 0xcd, 0x8d, 0x56, 0xc5, 0xdc, 0x59, 0x16, 0xe5,
-	0xbd, 0x0c, 0x19, 0x5f, 0xa0, 0x91, 0x73, 0xa5, 0xe4, 0x11, 0x54, 0x87, 0x96, 0x40, 0xb5, 0x00,
-	0x54, 0xbd, 0x63, 0x42, 0x67, 0x56, 0x24, 0x2a, 0xfd, 0x7f, 0x21, 0xdf, 0x2f, 0xab, 0x77, 0x60,
-	0x63, 0xd5, 0x0e, 0x7c, 0x84, 0x9d, 0x15, 0xb7, 0x49, 0x7a, 0xb0, 0x15, 0xc9, 0xa2, 0x1a, 0x2d,
-	0x77, 0x5b, 0xff, 0x74, 0x75, 0xaa, 0xd4, 0x5c, 0x14, 0x1a, 0x87, 0x50, 0xcb, 0x1e, 0x4b, 0xa0,
-	0x90, 0x6a, 0x5a, 0x3d, 0x1b, 0x03, 0xd8, 0x5e, 0xba, 0x71, 0xb9, 0x79, 0xa1, 0x62, 0x23, 0x36,
-	0x0e, 0x53, 0x8b, 0x66, 0x49, 0x21, 0x67, 0x6c, 0x8c, 0xe4, 0x21, 0x84, 0x82, 0xd0, 0x48, 0x05,
-	0xb5, 0x76, 0x25, 0xb3, 0xa2, 0xc0, 0x77, 0x21, 0xd6, 0xfd, 0xa5, 0x41, 0x3d, 0xbb, 0x6e, 0xe4,
-	0x27, 0xec, 0x66, 0xb1, 0xb7, 0xde, 0x84, 0x91, 0x3b, 0x6f, 0xec, 0xbd, 0x93, 0x3b, 0x54, 0x84,
-	0x53, 0xb5, 0xb4, 0x63, 0xad, 0xf7, 0xf2, 0xd3, 0x0b, 0x9b, 0x31, 0xdb, 0xc5, 0xb6, 0xcd, 0x5c,
-	0xcb, 0xb3, 0xdb, 0x8c, 0xdb, 0xea, 0x53, 0xd5, 0xb9, 0xfd, 0xd3, 0x35, 0xdc, 0x54, 0xbe, 0x39,
-	0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x74, 0x3a, 0x67, 0xe7, 0x06, 0x00, 0x00,
+var File_reflection_grpc_reflection_v1alpha_reflection_proto protoreflect.FileDescriptor
+
+var file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDesc = []byte{
+	0x0a, 0x33, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x5f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x61,
+	0x6c, 0x70, 0x68, 0x61, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x17, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x22, 0xf8,
+	0x02, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f,
+	0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x2a,
+	0x0a, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x65,
+	0x42, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x66, 0x69,
+	0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x79,
+	0x6d, 0x62, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x14, 0x66, 0x69,
+	0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6d, 0x62,
+	0x6f, 0x6c, 0x12, 0x67, 0x0a, 0x19, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61,
+	0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18,
+	0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e,
+	0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x48, 0x00, 0x52, 0x17, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69,
+	0x6e, 0x67, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x42, 0x0a, 0x1d, 0x61,
+	0x6c, 0x6c, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d,
+	0x62, 0x65, 0x72, 0x73, 0x5f, 0x6f, 0x66, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01,
+	0x28, 0x09, 0x48, 0x00, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69,
+	0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x4f, 0x66, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x25, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
+	0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x6c, 0x69, 0x73, 0x74, 0x53, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x42, 0x11, 0x0a, 0x0f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x65, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x66, 0x0a, 0x10, 0x45, 0x78, 0x74,
+	0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a,
+	0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x79, 0x70, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69,
+	0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73,
+	0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65,
+	0x72, 0x22, 0xc7, 0x04, 0x0a, 0x18, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d,
+	0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x5b, 0x0a,
+	0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72,
+	0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
+	0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0f, 0x6f, 0x72, 0x69, 0x67, 0x69,
+	0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x6b, 0x0a, 0x18, 0x66, 0x69,
+	0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76,
+	0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52,
+	0x16, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, 0x0a, 0x1e, 0x61, 0x6c, 0x6c, 0x5f, 0x65,
+	0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73,
+	0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73,
+	0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x48, 0x00, 0x52, 0x1b, 0x61, 0x6c, 0x6c, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f,
+	0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x64, 0x0a, 0x16, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x73, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00,
+	0x52, 0x14, 0x6c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f,
+	0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x12, 0x0a, 0x10, 0x6d, 0x65, 0x73, 0x73, 0x61,
+	0x67, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4c, 0x0a, 0x16, 0x46,
+	0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x65,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x18, 0x01,
+	0x20, 0x03, 0x28, 0x0c, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69,
+	0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6a, 0x0a, 0x17, 0x45, 0x78, 0x74,
+	0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70,
+	0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x61,
+	0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78,
+	0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02,
+	0x20, 0x03, 0x28, 0x05, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e,
+	0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0x59, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x07,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e,
+	0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x22, 0x25, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x53, 0x0a, 0x0d, 0x45, 0x72, 0x72, 0x6f, 0x72,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f,
+	0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x65, 0x72,
+	0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72,
+	0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c,
+	0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, 0x93, 0x01, 0x0a,
+	0x10, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x12, 0x7f, 0x0a, 0x14, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c,
+	0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31,
+	0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01,
+	0x30, 0x01, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c,
+	0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x72, 0x65, 0x66,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescOnce sync.Once
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescData = file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDesc
+)
+
+func file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP() []byte {
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescOnce.Do(func() {
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescData)
+	})
+	return file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDescData
+}
+
+var file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
+var file_reflection_grpc_reflection_v1alpha_reflection_proto_goTypes = []interface{}{
+	(*ServerReflectionRequest)(nil),  // 0: grpc.reflection.v1alpha.ServerReflectionRequest
+	(*ExtensionRequest)(nil),         // 1: grpc.reflection.v1alpha.ExtensionRequest
+	(*ServerReflectionResponse)(nil), // 2: grpc.reflection.v1alpha.ServerReflectionResponse
+	(*FileDescriptorResponse)(nil),   // 3: grpc.reflection.v1alpha.FileDescriptorResponse
+	(*ExtensionNumberResponse)(nil),  // 4: grpc.reflection.v1alpha.ExtensionNumberResponse
+	(*ListServiceResponse)(nil),      // 5: grpc.reflection.v1alpha.ListServiceResponse
+	(*ServiceResponse)(nil),          // 6: grpc.reflection.v1alpha.ServiceResponse
+	(*ErrorResponse)(nil),            // 7: grpc.reflection.v1alpha.ErrorResponse
+}
+var file_reflection_grpc_reflection_v1alpha_reflection_proto_depIdxs = []int32{
+	1, // 0: grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension:type_name -> grpc.reflection.v1alpha.ExtensionRequest
+	0, // 1: grpc.reflection.v1alpha.ServerReflectionResponse.original_request:type_name -> grpc.reflection.v1alpha.ServerReflectionRequest
+	3, // 2: grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response:type_name -> grpc.reflection.v1alpha.FileDescriptorResponse
+	4, // 3: grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response:type_name -> grpc.reflection.v1alpha.ExtensionNumberResponse
+	5, // 4: grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response:type_name -> grpc.reflection.v1alpha.ListServiceResponse
+	7, // 5: grpc.reflection.v1alpha.ServerReflectionResponse.error_response:type_name -> grpc.reflection.v1alpha.ErrorResponse
+	6, // 6: grpc.reflection.v1alpha.ListServiceResponse.service:type_name -> grpc.reflection.v1alpha.ServiceResponse
+	0, // 7: grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo:input_type -> grpc.reflection.v1alpha.ServerReflectionRequest
+	2, // 8: grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo:output_type -> grpc.reflection.v1alpha.ServerReflectionResponse
+	8, // [8:9] is the sub-list for method output_type
+	7, // [7:8] is the sub-list for method input_type
+	7, // [7:7] is the sub-list for extension type_name
+	7, // [7:7] is the sub-list for extension extendee
+	0, // [0:7] is the sub-list for field type_name
+}
+
+func init() { file_reflection_grpc_reflection_v1alpha_reflection_proto_init() }
+func file_reflection_grpc_reflection_v1alpha_reflection_proto_init() {
+	if File_reflection_grpc_reflection_v1alpha_reflection_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerReflectionRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ExtensionRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerReflectionResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FileDescriptorResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ExtensionNumberResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ListServiceResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServiceResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ErrorResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[0].OneofWrappers = []interface{}{
+		(*ServerReflectionRequest_FileByFilename)(nil),
+		(*ServerReflectionRequest_FileContainingSymbol)(nil),
+		(*ServerReflectionRequest_FileContainingExtension)(nil),
+		(*ServerReflectionRequest_AllExtensionNumbersOfType)(nil),
+		(*ServerReflectionRequest_ListServices)(nil),
+	}
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes[2].OneofWrappers = []interface{}{
+		(*ServerReflectionResponse_FileDescriptorResponse)(nil),
+		(*ServerReflectionResponse_AllExtensionNumbersResponse)(nil),
+		(*ServerReflectionResponse_ListServicesResponse)(nil),
+		(*ServerReflectionResponse_ErrorResponse)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   8,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_reflection_grpc_reflection_v1alpha_reflection_proto_goTypes,
+		DependencyIndexes: file_reflection_grpc_reflection_v1alpha_reflection_proto_depIdxs,
+		MessageInfos:      file_reflection_grpc_reflection_v1alpha_reflection_proto_msgTypes,
+	}.Build()
+	File_reflection_grpc_reflection_v1alpha_reflection_proto = out.File
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_rawDesc = nil
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_goTypes = nil
+	file_reflection_grpc_reflection_v1alpha_reflection_proto_depIdxs = nil
 }
diff --git a/reflection/grpc_testing/proto2.pb.go b/reflection/grpc_testing/proto2.pb.go
index d34adb82a85e..aa5d35f4c67e 100644
--- a/reflection/grpc_testing/proto2.pb.go
+++ b/reflection/grpc_testing/proto2.pb.go
@@ -1,90 +1,177 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: reflection/grpc_testing/proto2.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoiface "google.golang.org/protobuf/runtime/protoiface"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type ToBeExtended struct {
-	Foo                          *int32   `protobuf:"varint,1,req,name=foo" json:"foo,omitempty"`
-	XXX_NoUnkeyedLiteral         struct{} `json:"-"`
-	proto.XXX_InternalExtensions `json:"-"`
-	XXX_unrecognized             []byte `json:"-"`
-	XXX_sizecache                int32  `json:"-"`
-}
+	state           protoimpl.MessageState
+	sizeCache       protoimpl.SizeCache
+	unknownFields   protoimpl.UnknownFields
+	extensionFields protoimpl.ExtensionFields
 
-func (m *ToBeExtended) Reset()         { *m = ToBeExtended{} }
-func (m *ToBeExtended) String() string { return proto.CompactTextString(m) }
-func (*ToBeExtended) ProtoMessage()    {}
-func (*ToBeExtended) Descriptor() ([]byte, []int) {
-	return fileDescriptor_dddbb2c1ebdcf2b8, []int{0}
+	Foo *int32 `protobuf:"varint,1,req,name=foo" json:"foo,omitempty"`
 }
 
-var extRange_ToBeExtended = []proto.ExtensionRange{
-	{Start: 10, End: 30},
+func (x *ToBeExtended) Reset() {
+	*x = ToBeExtended{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_testing_proto2_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (*ToBeExtended) ExtensionRangeArray() []proto.ExtensionRange {
-	return extRange_ToBeExtended
+func (x *ToBeExtended) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *ToBeExtended) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ToBeExtended.Unmarshal(m, b)
-}
-func (m *ToBeExtended) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ToBeExtended.Marshal(b, m, deterministic)
-}
-func (m *ToBeExtended) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ToBeExtended.Merge(m, src)
+func (*ToBeExtended) ProtoMessage() {}
+
+func (x *ToBeExtended) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_testing_proto2_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *ToBeExtended) XXX_Size() int {
-	return xxx_messageInfo_ToBeExtended.Size(m)
+
+// Deprecated: Use ToBeExtended.ProtoReflect.Descriptor instead.
+func (*ToBeExtended) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_testing_proto2_proto_rawDescGZIP(), []int{0}
 }
-func (m *ToBeExtended) XXX_DiscardUnknown() {
-	xxx_messageInfo_ToBeExtended.DiscardUnknown(m)
+
+var extRange_ToBeExtended = []protoiface.ExtensionRangeV1{
+	{Start: 10, End: 30},
 }
 
-var xxx_messageInfo_ToBeExtended proto.InternalMessageInfo
+// Deprecated: Use ToBeExtended.ProtoReflect.Descriptor.ExtensionRanges instead.
+func (*ToBeExtended) ExtensionRangeArray() []protoiface.ExtensionRangeV1 {
+	return extRange_ToBeExtended
+}
 
-func (m *ToBeExtended) GetFoo() int32 {
-	if m != nil && m.Foo != nil {
-		return *m.Foo
+func (x *ToBeExtended) GetFoo() int32 {
+	if x != nil && x.Foo != nil {
+		return *x.Foo
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterType((*ToBeExtended)(nil), "grpc.testing.ToBeExtended")
+var File_reflection_grpc_testing_proto2_proto protoreflect.FileDescriptor
+
+var file_reflection_grpc_testing_proto2_proto_rawDesc = []byte{
+	0x0a, 0x24, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x22, 0x26, 0x0a, 0x0c, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65,
+	0x6e, 0x64, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x66, 0x6f, 0x6f, 0x18, 0x01, 0x20, 0x02, 0x28,
+	0x05, 0x52, 0x03, 0x66, 0x6f, 0x6f, 0x2a, 0x04, 0x08, 0x0a, 0x10, 0x1f, 0x42, 0x30, 0x5a, 0x2e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72,
+	0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+}
+
+var (
+	file_reflection_grpc_testing_proto2_proto_rawDescOnce sync.Once
+	file_reflection_grpc_testing_proto2_proto_rawDescData = file_reflection_grpc_testing_proto2_proto_rawDesc
+)
+
+func file_reflection_grpc_testing_proto2_proto_rawDescGZIP() []byte {
+	file_reflection_grpc_testing_proto2_proto_rawDescOnce.Do(func() {
+		file_reflection_grpc_testing_proto2_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_proto2_proto_rawDescData)
+	})
+	return file_reflection_grpc_testing_proto2_proto_rawDescData
 }
 
-func init() {
-	proto.RegisterFile("reflection/grpc_testing/proto2.proto", fileDescriptor_dddbb2c1ebdcf2b8)
+var file_reflection_grpc_testing_proto2_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_reflection_grpc_testing_proto2_proto_goTypes = []interface{}{
+	(*ToBeExtended)(nil), // 0: grpc.testing.ToBeExtended
+}
+var file_reflection_grpc_testing_proto2_proto_depIdxs = []int32{
+	0, // [0:0] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-var fileDescriptor_dddbb2c1ebdcf2b8 = []byte{
-	// 130 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x29, 0x4a, 0x4d, 0xcb,
-	0x49, 0x4d, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x4f, 0x2f, 0x2a, 0x48, 0x8e, 0x2f, 0x49, 0x2d, 0x2e,
-	0xc9, 0xcc, 0x4b, 0xd7, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x37, 0xd2, 0x03, 0x53, 0x42, 0x3c, 0x20,
-	0x29, 0x3d, 0xa8, 0x94, 0x92, 0x1a, 0x17, 0x4f, 0x48, 0xbe, 0x53, 0xaa, 0x6b, 0x45, 0x49, 0x6a,
-	0x5e, 0x4a, 0x6a, 0x8a, 0x90, 0x00, 0x17, 0x73, 0x5a, 0x7e, 0xbe, 0x04, 0xa3, 0x02, 0x93, 0x06,
-	0x6b, 0x10, 0x88, 0xa9, 0xc5, 0xc2, 0xc1, 0x25, 0x20, 0xef, 0x64, 0x10, 0xa5, 0x97, 0x9e, 0x9f,
-	0x9f, 0x9e, 0x93, 0xaa, 0x97, 0x9e, 0x9f, 0x93, 0x98, 0x97, 0xae, 0x97, 0x5f, 0x94, 0x0e, 0xb6,
-	0x44, 0x1f, 0x87, 0xa5, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x05, 0x50, 0x64, 0x8e, 0x00,
-	0x00, 0x00,
+func init() { file_reflection_grpc_testing_proto2_proto_init() }
+func file_reflection_grpc_testing_proto2_proto_init() {
+	if File_reflection_grpc_testing_proto2_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_reflection_grpc_testing_proto2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ToBeExtended); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			case 3:
+				return &v.extensionFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_reflection_grpc_testing_proto2_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_reflection_grpc_testing_proto2_proto_goTypes,
+		DependencyIndexes: file_reflection_grpc_testing_proto2_proto_depIdxs,
+		MessageInfos:      file_reflection_grpc_testing_proto2_proto_msgTypes,
+	}.Build()
+	File_reflection_grpc_testing_proto2_proto = out.File
+	file_reflection_grpc_testing_proto2_proto_rawDesc = nil
+	file_reflection_grpc_testing_proto2_proto_goTypes = nil
+	file_reflection_grpc_testing_proto2_proto_depIdxs = nil
 }
diff --git a/reflection/grpc_testing/proto2_ext.pb.go b/reflection/grpc_testing/proto2_ext.pb.go
index 631b5146d56c..4d96fa358aea 100644
--- a/reflection/grpc_testing/proto2_ext.pb.go
+++ b/reflection/grpc_testing/proto2_ext.pb.go
@@ -1,116 +1,228 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: reflection/grpc_testing/proto2_ext.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type Extension struct {
-	Whatzit              *int32   `protobuf:"varint,1,opt,name=whatzit" json:"whatzit,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *Extension) Reset()         { *m = Extension{} }
-func (m *Extension) String() string { return proto.CompactTextString(m) }
-func (*Extension) ProtoMessage()    {}
-func (*Extension) Descriptor() ([]byte, []int) {
-	return fileDescriptor_071dc827b8673a0c, []int{0}
+	Whatzit *int32 `protobuf:"varint,1,opt,name=whatzit" json:"whatzit,omitempty"`
 }
 
-func (m *Extension) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Extension.Unmarshal(m, b)
-}
-func (m *Extension) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Extension.Marshal(b, m, deterministic)
-}
-func (m *Extension) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Extension.Merge(m, src)
+func (x *Extension) Reset() {
+	*x = Extension{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Extension) XXX_Size() int {
-	return xxx_messageInfo_Extension.Size(m)
+
+func (x *Extension) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Extension) XXX_DiscardUnknown() {
-	xxx_messageInfo_Extension.DiscardUnknown(m)
+
+func (*Extension) ProtoMessage() {}
+
+func (x *Extension) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Extension proto.InternalMessageInfo
+// Deprecated: Use Extension.ProtoReflect.Descriptor instead.
+func (*Extension) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_testing_proto2_ext_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *Extension) GetWhatzit() int32 {
-	if m != nil && m.Whatzit != nil {
-		return *m.Whatzit
+func (x *Extension) GetWhatzit() int32 {
+	if x != nil && x.Whatzit != nil {
+		return *x.Whatzit
 	}
 	return 0
 }
 
-var E_Foo = &proto.ExtensionDesc{
-	ExtendedType:  (*ToBeExtended)(nil),
-	ExtensionType: (*int32)(nil),
-	Field:         13,
-	Name:          "grpc.testing.foo",
-	Tag:           "varint,13,opt,name=foo",
-	Filename:      "reflection/grpc_testing/proto2_ext.proto",
+var file_reflection_grpc_testing_proto2_ext_proto_extTypes = []protoimpl.ExtensionInfo{
+	{
+		ExtendedType:  (*ToBeExtended)(nil),
+		ExtensionType: (*int32)(nil),
+		Field:         13,
+		Name:          "grpc.testing.foo",
+		Tag:           "varint,13,opt,name=foo",
+		Filename:      "reflection/grpc_testing/proto2_ext.proto",
+	},
+	{
+		ExtendedType:  (*ToBeExtended)(nil),
+		ExtensionType: (*Extension)(nil),
+		Field:         17,
+		Name:          "grpc.testing.bar",
+		Tag:           "bytes,17,opt,name=bar",
+		Filename:      "reflection/grpc_testing/proto2_ext.proto",
+	},
+	{
+		ExtendedType:  (*ToBeExtended)(nil),
+		ExtensionType: (*SearchRequest)(nil),
+		Field:         19,
+		Name:          "grpc.testing.baz",
+		Tag:           "bytes,19,opt,name=baz",
+		Filename:      "reflection/grpc_testing/proto2_ext.proto",
+	},
 }
 
-var E_Bar = &proto.ExtensionDesc{
-	ExtendedType:  (*ToBeExtended)(nil),
-	ExtensionType: (*Extension)(nil),
-	Field:         17,
-	Name:          "grpc.testing.bar",
-	Tag:           "bytes,17,opt,name=bar",
-	Filename:      "reflection/grpc_testing/proto2_ext.proto",
-}
+// Extension fields to ToBeExtended.
+var (
+	// optional int32 foo = 13;
+	E_Foo = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[0]
+	// optional grpc.testing.Extension bar = 17;
+	E_Bar = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[1]
+	// optional grpc.testing.SearchRequest baz = 19;
+	E_Baz = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[2]
+)
+
+var File_reflection_grpc_testing_proto2_ext_proto protoreflect.FileDescriptor
 
-var E_Baz = &proto.ExtensionDesc{
-	ExtendedType:  (*ToBeExtended)(nil),
-	ExtensionType: (*SearchRequest)(nil),
-	Field:         19,
-	Name:          "grpc.testing.baz",
-	Tag:           "bytes,19,opt,name=baz",
-	Filename:      "reflection/grpc_testing/proto2_ext.proto",
+var file_reflection_grpc_testing_proto2_ext_proto_rawDesc = []byte{
+	0x0a, 0x28, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32,
+	0x5f, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x24, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22,
+	0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x22, 0x25, 0x0a, 0x09, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12,
+	0x18, 0x0a, 0x07, 0x77, 0x68, 0x61, 0x74, 0x7a, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x07, 0x77, 0x68, 0x61, 0x74, 0x7a, 0x69, 0x74, 0x3a, 0x2c, 0x0a, 0x03, 0x66, 0x6f, 0x6f,
+	0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x0d, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x03, 0x66, 0x6f, 0x6f, 0x3a, 0x45, 0x0a, 0x03, 0x62, 0x61, 0x72, 0x12, 0x1a,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f,
+	0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x62, 0x61, 0x72, 0x3a, 0x49,
+	0x0a, 0x03, 0x62, 0x61, 0x7a, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65,
+	0x64, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x52, 0x03, 0x62, 0x61, 0x7a, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67,
+	0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67,
+	0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
 }
 
-func init() {
-	proto.RegisterType((*Extension)(nil), "grpc.testing.Extension")
-	proto.RegisterExtension(E_Foo)
-	proto.RegisterExtension(E_Bar)
-	proto.RegisterExtension(E_Baz)
+var (
+	file_reflection_grpc_testing_proto2_ext_proto_rawDescOnce sync.Once
+	file_reflection_grpc_testing_proto2_ext_proto_rawDescData = file_reflection_grpc_testing_proto2_ext_proto_rawDesc
+)
+
+func file_reflection_grpc_testing_proto2_ext_proto_rawDescGZIP() []byte {
+	file_reflection_grpc_testing_proto2_ext_proto_rawDescOnce.Do(func() {
+		file_reflection_grpc_testing_proto2_ext_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_proto2_ext_proto_rawDescData)
+	})
+	return file_reflection_grpc_testing_proto2_ext_proto_rawDescData
 }
 
-func init() {
-	proto.RegisterFile("reflection/grpc_testing/proto2_ext.proto", fileDescriptor_071dc827b8673a0c)
+var file_reflection_grpc_testing_proto2_ext_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_reflection_grpc_testing_proto2_ext_proto_goTypes = []interface{}{
+	(*Extension)(nil),     // 0: grpc.testing.Extension
+	(*ToBeExtended)(nil),  // 1: grpc.testing.ToBeExtended
+	(*SearchRequest)(nil), // 2: grpc.testing.SearchRequest
+}
+var file_reflection_grpc_testing_proto2_ext_proto_depIdxs = []int32{
+	1, // 0: grpc.testing.foo:extendee -> grpc.testing.ToBeExtended
+	1, // 1: grpc.testing.bar:extendee -> grpc.testing.ToBeExtended
+	1, // 2: grpc.testing.baz:extendee -> grpc.testing.ToBeExtended
+	0, // 3: grpc.testing.bar:type_name -> grpc.testing.Extension
+	2, // 4: grpc.testing.baz:type_name -> grpc.testing.SearchRequest
+	5, // [5:5] is the sub-list for method output_type
+	5, // [5:5] is the sub-list for method input_type
+	3, // [3:5] is the sub-list for extension type_name
+	0, // [0:3] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-var fileDescriptor_071dc827b8673a0c = []byte{
-	// 224 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0xc1, 0x4b, 0x85, 0x40,
-	0x10, 0xc6, 0x91, 0x47, 0x44, 0x5b, 0x1d, 0xb2, 0x43, 0x62, 0x97, 0x87, 0x14, 0x78, 0x88, 0x35,
-	0x3c, 0x7a, 0x14, 0x3c, 0x74, 0xb5, 0x4e, 0x5d, 0x64, 0xd3, 0x71, 0x5d, 0x90, 0x1d, 0x5b, 0x27,
-	0x12, 0xff, 0xfa, 0xd8, 0xb5, 0xe2, 0x79, 0xd0, 0xd3, 0xee, 0x30, 0xbf, 0xdf, 0xc7, 0xf0, 0xb1,
-	0xd8, 0x40, 0xdb, 0x43, 0x4d, 0x0a, 0x75, 0x22, 0xcd, 0x50, 0x57, 0x04, 0x23, 0x29, 0x2d, 0x93,
-	0xc1, 0x20, 0x61, 0x5a, 0xc1, 0x44, 0xdc, 0x7d, 0xfd, 0x2b, 0xbb, 0xe6, 0xbf, 0xeb, 0xf0, 0x61,
-	0xdf, 0x5b, 0x9c, 0x30, 0xda, 0xa2, 0xec, 0xbb, 0x30, 0xd1, 0x23, 0xbb, 0x28, 0x26, 0x02, 0x3d,
-	0x2a, 0xd4, 0x7e, 0xc0, 0xce, 0xbf, 0x3b, 0x41, 0xb3, 0xa2, 0xc0, 0x3b, 0x7a, 0xf1, 0x59, 0xf9,
-	0x37, 0x66, 0x4f, 0xec, 0xd0, 0x22, 0xfa, 0x21, 0x3f, 0x3d, 0x83, 0xbf, 0x61, 0x0e, 0xce, 0x6e,
-	0xa0, 0x09, 0xae, 0x9d, 0x61, 0xb1, 0xac, 0x60, 0x87, 0x0f, 0x61, 0x76, 0xe9, 0x9b, 0xa3, 0x17,
-	0x5f, 0xa6, 0x77, 0x6b, 0xe2, 0xff, 0x92, 0xd2, 0xfa, 0xd9, 0x8b, 0x8d, 0x99, 0x77, 0x63, 0x6e,
-	0x5d, 0xcc, 0xfd, 0x9a, 0x78, 0x05, 0x61, 0xea, 0xae, 0x84, 0xcf, 0x2f, 0x18, 0xc9, 0x46, 0xcd,
-	0xf9, 0xf3, 0x3b, 0x97, 0x88, 0xb2, 0x07, 0x2e, 0xb1, 0x17, 0x5a, 0x72, 0x34, 0xd2, 0x75, 0x92,
-	0x6c, 0x74, 0xf4, 0x13, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x14, 0xcf, 0xc9, 0x9b, 0x01, 0x00, 0x00,
+func init() { file_reflection_grpc_testing_proto2_ext_proto_init() }
+func file_reflection_grpc_testing_proto2_ext_proto_init() {
+	if File_reflection_grpc_testing_proto2_ext_proto != nil {
+		return
+	}
+	file_reflection_grpc_testing_proto2_proto_init()
+	file_reflection_grpc_testing_test_proto_init()
+	if !protoimpl.UnsafeEnabled {
+		file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Extension); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_reflection_grpc_testing_proto2_ext_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 3,
+			NumServices:   0,
+		},
+		GoTypes:           file_reflection_grpc_testing_proto2_ext_proto_goTypes,
+		DependencyIndexes: file_reflection_grpc_testing_proto2_ext_proto_depIdxs,
+		MessageInfos:      file_reflection_grpc_testing_proto2_ext_proto_msgTypes,
+		ExtensionInfos:    file_reflection_grpc_testing_proto2_ext_proto_extTypes,
+	}.Build()
+	File_reflection_grpc_testing_proto2_ext_proto = out.File
+	file_reflection_grpc_testing_proto2_ext_proto_rawDesc = nil
+	file_reflection_grpc_testing_proto2_ext_proto_goTypes = nil
+	file_reflection_grpc_testing_proto2_ext_proto_depIdxs = nil
 }
diff --git a/reflection/grpc_testing/proto2_ext2.pb.go b/reflection/grpc_testing/proto2_ext2.pb.go
index ad2fef3addd3..88b96565afdb 100644
--- a/reflection/grpc_testing/proto2_ext2.pb.go
+++ b/reflection/grpc_testing/proto2_ext2.pb.go
@@ -1,105 +1,209 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: reflection/grpc_testing/proto2_ext2.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type AnotherExtension struct {
-	Whatchamacallit      *int32   `protobuf:"varint,1,opt,name=whatchamacallit" json:"whatchamacallit,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *AnotherExtension) Reset()         { *m = AnotherExtension{} }
-func (m *AnotherExtension) String() string { return proto.CompactTextString(m) }
-func (*AnotherExtension) ProtoMessage()    {}
-func (*AnotherExtension) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ead6f7bd8a66fb18, []int{0}
+	Whatchamacallit *int32 `protobuf:"varint,1,opt,name=whatchamacallit" json:"whatchamacallit,omitempty"`
 }
 
-func (m *AnotherExtension) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_AnotherExtension.Unmarshal(m, b)
-}
-func (m *AnotherExtension) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_AnotherExtension.Marshal(b, m, deterministic)
-}
-func (m *AnotherExtension) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_AnotherExtension.Merge(m, src)
+func (x *AnotherExtension) Reset() {
+	*x = AnotherExtension{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *AnotherExtension) XXX_Size() int {
-	return xxx_messageInfo_AnotherExtension.Size(m)
+
+func (x *AnotherExtension) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *AnotherExtension) XXX_DiscardUnknown() {
-	xxx_messageInfo_AnotherExtension.DiscardUnknown(m)
+
+func (*AnotherExtension) ProtoMessage() {}
+
+func (x *AnotherExtension) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_AnotherExtension proto.InternalMessageInfo
+// Deprecated: Use AnotherExtension.ProtoReflect.Descriptor instead.
+func (*AnotherExtension) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_testing_proto2_ext2_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *AnotherExtension) GetWhatchamacallit() int32 {
-	if m != nil && m.Whatchamacallit != nil {
-		return *m.Whatchamacallit
+func (x *AnotherExtension) GetWhatchamacallit() int32 {
+	if x != nil && x.Whatchamacallit != nil {
+		return *x.Whatchamacallit
 	}
 	return 0
 }
 
-var E_Frob = &proto.ExtensionDesc{
-	ExtendedType:  (*ToBeExtended)(nil),
-	ExtensionType: (*string)(nil),
-	Field:         23,
-	Name:          "grpc.testing.frob",
-	Tag:           "bytes,23,opt,name=frob",
-	Filename:      "reflection/grpc_testing/proto2_ext2.proto",
+var file_reflection_grpc_testing_proto2_ext2_proto_extTypes = []protoimpl.ExtensionInfo{
+	{
+		ExtendedType:  (*ToBeExtended)(nil),
+		ExtensionType: (*string)(nil),
+		Field:         23,
+		Name:          "grpc.testing.frob",
+		Tag:           "bytes,23,opt,name=frob",
+		Filename:      "reflection/grpc_testing/proto2_ext2.proto",
+	},
+	{
+		ExtendedType:  (*ToBeExtended)(nil),
+		ExtensionType: (*AnotherExtension)(nil),
+		Field:         29,
+		Name:          "grpc.testing.nitz",
+		Tag:           "bytes,29,opt,name=nitz",
+		Filename:      "reflection/grpc_testing/proto2_ext2.proto",
+	},
 }
 
-var E_Nitz = &proto.ExtensionDesc{
-	ExtendedType:  (*ToBeExtended)(nil),
-	ExtensionType: (*AnotherExtension)(nil),
-	Field:         29,
-	Name:          "grpc.testing.nitz",
-	Tag:           "bytes,29,opt,name=nitz",
-	Filename:      "reflection/grpc_testing/proto2_ext2.proto",
+// Extension fields to ToBeExtended.
+var (
+	// optional string frob = 23;
+	E_Frob = &file_reflection_grpc_testing_proto2_ext2_proto_extTypes[0]
+	// optional grpc.testing.AnotherExtension nitz = 29;
+	E_Nitz = &file_reflection_grpc_testing_proto2_ext2_proto_extTypes[1]
+)
+
+var File_reflection_grpc_testing_proto2_ext2_proto protoreflect.FileDescriptor
+
+var file_reflection_grpc_testing_proto2_ext2_proto_rawDesc = []byte{
+	0x0a, 0x29, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32,
+	0x5f, 0x65, 0x78, 0x74, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x24, 0x72, 0x65, 0x66, 0x6c, 0x65,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
+	0x3c, 0x0a, 0x10, 0x41, 0x6e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73,
+	0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x0f, 0x77, 0x68, 0x61, 0x74, 0x63, 0x68, 0x61, 0x6d, 0x61,
+	0x63, 0x61, 0x6c, 0x6c, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x77, 0x68,
+	0x61, 0x74, 0x63, 0x68, 0x61, 0x6d, 0x61, 0x63, 0x61, 0x6c, 0x6c, 0x69, 0x74, 0x3a, 0x2e, 0x0a,
+	0x04, 0x66, 0x72, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65,
+	0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x62, 0x3a, 0x4e, 0x0a,
+	0x04, 0x6e, 0x69, 0x74, 0x7a, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65,
+	0x64, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x6e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x45, 0x78,
+	0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x6e, 0x69, 0x74, 0x7a, 0x42, 0x30, 0x5a,
+	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f,
+	0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
 }
 
-func init() {
-	proto.RegisterType((*AnotherExtension)(nil), "grpc.testing.AnotherExtension")
-	proto.RegisterExtension(E_Frob)
-	proto.RegisterExtension(E_Nitz)
+var (
+	file_reflection_grpc_testing_proto2_ext2_proto_rawDescOnce sync.Once
+	file_reflection_grpc_testing_proto2_ext2_proto_rawDescData = file_reflection_grpc_testing_proto2_ext2_proto_rawDesc
+)
+
+func file_reflection_grpc_testing_proto2_ext2_proto_rawDescGZIP() []byte {
+	file_reflection_grpc_testing_proto2_ext2_proto_rawDescOnce.Do(func() {
+		file_reflection_grpc_testing_proto2_ext2_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_proto2_ext2_proto_rawDescData)
+	})
+	return file_reflection_grpc_testing_proto2_ext2_proto_rawDescData
 }
 
-func init() {
-	proto.RegisterFile("reflection/grpc_testing/proto2_ext2.proto", fileDescriptor_ead6f7bd8a66fb18)
+var file_reflection_grpc_testing_proto2_ext2_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_reflection_grpc_testing_proto2_ext2_proto_goTypes = []interface{}{
+	(*AnotherExtension)(nil), // 0: grpc.testing.AnotherExtension
+	(*ToBeExtended)(nil),     // 1: grpc.testing.ToBeExtended
+}
+var file_reflection_grpc_testing_proto2_ext2_proto_depIdxs = []int32{
+	1, // 0: grpc.testing.frob:extendee -> grpc.testing.ToBeExtended
+	1, // 1: grpc.testing.nitz:extendee -> grpc.testing.ToBeExtended
+	0, // 2: grpc.testing.nitz:type_name -> grpc.testing.AnotherExtension
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	2, // [2:3] is the sub-list for extension type_name
+	0, // [0:2] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-var fileDescriptor_ead6f7bd8a66fb18 = []byte{
-	// 208 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x8f, 0x31, 0x4b, 0xc7, 0x30,
-	0x10, 0xc5, 0x09, 0xfc, 0x1d, 0x8c, 0x82, 0xd2, 0xc5, 0x52, 0x50, 0x8a, 0x38, 0xc4, 0xe5, 0x2a,
-	0x1d, 0x8b, 0x8b, 0x05, 0x57, 0x87, 0xe2, 0xe4, 0x52, 0x62, 0x7a, 0x4d, 0x03, 0x31, 0x57, 0xd2,
-	0x03, 0x8b, 0x9f, 0x5e, 0x8c, 0x0e, 0x5a, 0xb0, 0xdb, 0x71, 0xef, 0xf7, 0xde, 0xdd, 0x93, 0xb7,
-	0x11, 0x47, 0x8f, 0x86, 0x1d, 0x85, 0xca, 0xc6, 0xd9, 0xf4, 0x8c, 0x0b, 0xbb, 0x60, 0xab, 0x39,
-	0x12, 0x53, 0xdd, 0xe3, 0xca, 0x35, 0xa4, 0x39, 0x3b, 0xfd, 0xd2, 0xe1, 0x47, 0x2f, 0x6e, 0xf6,
-	0x8d, 0xdf, 0x9e, 0xeb, 0x7b, 0x79, 0xfe, 0x10, 0x88, 0x27, 0x8c, 0x8f, 0x2b, 0x63, 0x58, 0x1c,
-	0x85, 0x4c, 0xc9, 0xb3, 0xf7, 0x49, 0xb3, 0x99, 0xf4, 0x9b, 0x36, 0xda, 0x7b, 0xc7, 0xb9, 0x28,
-	0x85, 0x3a, 0xea, 0xb6, 0xeb, 0x06, 0xe4, 0x61, 0x8c, 0xf4, 0x9a, 0x15, 0xf0, 0xfb, 0x34, 0x3c,
-	0x53, 0x8b, 0x29, 0x6e, 0xc0, 0x21, 0xbf, 0x28, 0x85, 0x3a, 0xee, 0x12, 0xd7, 0x3c, 0xc9, 0x43,
-	0x70, 0xfc, 0xb1, 0xcb, 0x5f, 0x96, 0x42, 0x9d, 0xd4, 0x57, 0x7f, 0x89, 0xed, 0x8f, 0x5d, 0xca,
-	0x69, 0xef, 0x5e, 0xc0, 0x12, 0x59, 0x8f, 0x60, 0xc9, 0xeb, 0x60, 0x81, 0xa2, 0x4d, 0x65, 0xab,
-	0x7f, 0xca, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0x18, 0x74, 0xc1, 0x53, 0x4f, 0x01, 0x00, 0x00,
+func init() { file_reflection_grpc_testing_proto2_ext2_proto_init() }
+func file_reflection_grpc_testing_proto2_ext2_proto_init() {
+	if File_reflection_grpc_testing_proto2_ext2_proto != nil {
+		return
+	}
+	file_reflection_grpc_testing_proto2_proto_init()
+	if !protoimpl.UnsafeEnabled {
+		file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*AnotherExtension); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_reflection_grpc_testing_proto2_ext2_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 2,
+			NumServices:   0,
+		},
+		GoTypes:           file_reflection_grpc_testing_proto2_ext2_proto_goTypes,
+		DependencyIndexes: file_reflection_grpc_testing_proto2_ext2_proto_depIdxs,
+		MessageInfos:      file_reflection_grpc_testing_proto2_ext2_proto_msgTypes,
+		ExtensionInfos:    file_reflection_grpc_testing_proto2_ext2_proto_extTypes,
+	}.Build()
+	File_reflection_grpc_testing_proto2_ext2_proto = out.File
+	file_reflection_grpc_testing_proto2_ext2_proto_rawDesc = nil
+	file_reflection_grpc_testing_proto2_ext2_proto_goTypes = nil
+	file_reflection_grpc_testing_proto2_ext2_proto_depIdxs = nil
 }
diff --git a/reflection/grpc_testing/test.pb.go b/reflection/grpc_testing/test.pb.go
index 464bf39dd5c4..b5e589e3155d 100644
--- a/reflection/grpc_testing/test.pb.go
+++ b/reflection/grpc_testing/test.pb.go
@@ -1,185 +1,326 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: reflection/grpc_testing/test.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type SearchResponse struct {
-	Results              []*SearchResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                 `json:"-"`
-	XXX_unrecognized     []byte                   `json:"-"`
-	XXX_sizecache        int32                    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *SearchResponse) Reset()         { *m = SearchResponse{} }
-func (m *SearchResponse) String() string { return proto.CompactTextString(m) }
-func (*SearchResponse) ProtoMessage()    {}
-func (*SearchResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b179ea967ba71047, []int{0}
+	Results []*SearchResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"`
 }
 
-func (m *SearchResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SearchResponse.Unmarshal(m, b)
+func (x *SearchResponse) Reset() {
+	*x = SearchResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_testing_test_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *SearchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SearchResponse.Marshal(b, m, deterministic)
+
+func (x *SearchResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SearchResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SearchResponse.Merge(m, src)
+
+func (*SearchResponse) ProtoMessage() {}
+
+func (x *SearchResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_testing_test_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *SearchResponse) XXX_Size() int {
-	return xxx_messageInfo_SearchResponse.Size(m)
+
+// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead.
+func (*SearchResponse) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{0}
 }
-func (m *SearchResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_SearchResponse.DiscardUnknown(m)
+
+func (x *SearchResponse) GetResults() []*SearchResponse_Result {
+	if x != nil {
+		return x.Results
+	}
+	return nil
 }
 
-var xxx_messageInfo_SearchResponse proto.InternalMessageInfo
+type SearchRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
+}
 
-func (m *SearchResponse) GetResults() []*SearchResponse_Result {
-	if m != nil {
-		return m.Results
+func (x *SearchRequest) Reset() {
+	*x = SearchRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_testing_test_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return nil
 }
 
-type SearchResponse_Result struct {
-	Url                  string   `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
-	Title                string   `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"`
-	Snippets             []string `protobuf:"bytes,3,rep,name=snippets,proto3" json:"snippets,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *SearchRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-func (m *SearchResponse_Result) Reset()         { *m = SearchResponse_Result{} }
-func (m *SearchResponse_Result) String() string { return proto.CompactTextString(m) }
-func (*SearchResponse_Result) ProtoMessage()    {}
-func (*SearchResponse_Result) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b179ea967ba71047, []int{0, 0}
+func (*SearchRequest) ProtoMessage() {}
+
+func (x *SearchRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_testing_test_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-func (m *SearchResponse_Result) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SearchResponse_Result.Unmarshal(m, b)
+// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead.
+func (*SearchRequest) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{1}
 }
-func (m *SearchResponse_Result) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SearchResponse_Result.Marshal(b, m, deterministic)
+
+func (x *SearchRequest) GetQuery() string {
+	if x != nil {
+		return x.Query
+	}
+	return ""
 }
-func (m *SearchResponse_Result) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SearchResponse_Result.Merge(m, src)
+
+type SearchResponse_Result struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Url      string   `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
+	Title    string   `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"`
+	Snippets []string `protobuf:"bytes,3,rep,name=snippets,proto3" json:"snippets,omitempty"`
 }
-func (m *SearchResponse_Result) XXX_Size() int {
-	return xxx_messageInfo_SearchResponse_Result.Size(m)
+
+func (x *SearchResponse_Result) Reset() {
+	*x = SearchResponse_Result{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_reflection_grpc_testing_test_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *SearchResponse_Result) XXX_DiscardUnknown() {
-	xxx_messageInfo_SearchResponse_Result.DiscardUnknown(m)
+
+func (x *SearchResponse_Result) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_SearchResponse_Result proto.InternalMessageInfo
+func (*SearchResponse_Result) ProtoMessage() {}
 
-func (m *SearchResponse_Result) GetUrl() string {
-	if m != nil {
-		return m.Url
+func (x *SearchResponse_Result) ProtoReflect() protoreflect.Message {
+	mi := &file_reflection_grpc_testing_test_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SearchResponse_Result.ProtoReflect.Descriptor instead.
+func (*SearchResponse_Result) Descriptor() ([]byte, []int) {
+	return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{0, 0}
+}
+
+func (x *SearchResponse_Result) GetUrl() string {
+	if x != nil {
+		return x.Url
 	}
 	return ""
 }
 
-func (m *SearchResponse_Result) GetTitle() string {
-	if m != nil {
-		return m.Title
+func (x *SearchResponse_Result) GetTitle() string {
+	if x != nil {
+		return x.Title
 	}
 	return ""
 }
 
-func (m *SearchResponse_Result) GetSnippets() []string {
-	if m != nil {
-		return m.Snippets
+func (x *SearchResponse_Result) GetSnippets() []string {
+	if x != nil {
+		return x.Snippets
 	}
 	return nil
 }
 
-type SearchRequest struct {
-	Query                string   `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+var File_reflection_grpc_testing_test_proto protoreflect.FileDescriptor
 
-func (m *SearchRequest) Reset()         { *m = SearchRequest{} }
-func (m *SearchRequest) String() string { return proto.CompactTextString(m) }
-func (*SearchRequest) ProtoMessage()    {}
-func (*SearchRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b179ea967ba71047, []int{1}
+var file_reflection_grpc_testing_test_proto_rawDesc = []byte{
+	0x0a, 0x22, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70,
+	0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x22, 0x9d, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73,
+	0x75, 0x6c, 0x74, 0x73, 0x1a, 0x4c, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x10,
+	0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c,
+	0x12, 0x14, 0x0a, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65,
+	0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65,
+	0x74, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x32, 0xa6, 0x01, 0x0a, 0x0d, 0x53, 0x65,
+	0x61, 0x72, 0x63, 0x68, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x53,
+	0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x50, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x61,
+	0x72, 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01,
+	0x30, 0x01, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c,
+	0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
-func (m *SearchRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SearchRequest.Unmarshal(m, b)
-}
-func (m *SearchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SearchRequest.Marshal(b, m, deterministic)
-}
-func (m *SearchRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SearchRequest.Merge(m, src)
+var (
+	file_reflection_grpc_testing_test_proto_rawDescOnce sync.Once
+	file_reflection_grpc_testing_test_proto_rawDescData = file_reflection_grpc_testing_test_proto_rawDesc
+)
+
+func file_reflection_grpc_testing_test_proto_rawDescGZIP() []byte {
+	file_reflection_grpc_testing_test_proto_rawDescOnce.Do(func() {
+		file_reflection_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_test_proto_rawDescData)
+	})
+	return file_reflection_grpc_testing_test_proto_rawDescData
 }
-func (m *SearchRequest) XXX_Size() int {
-	return xxx_messageInfo_SearchRequest.Size(m)
+
+var file_reflection_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_reflection_grpc_testing_test_proto_goTypes = []interface{}{
+	(*SearchResponse)(nil),        // 0: grpc.testing.SearchResponse
+	(*SearchRequest)(nil),         // 1: grpc.testing.SearchRequest
+	(*SearchResponse_Result)(nil), // 2: grpc.testing.SearchResponse.Result
 }
-func (m *SearchRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_SearchRequest.DiscardUnknown(m)
+var file_reflection_grpc_testing_test_proto_depIdxs = []int32{
+	2, // 0: grpc.testing.SearchResponse.results:type_name -> grpc.testing.SearchResponse.Result
+	1, // 1: grpc.testing.SearchService.Search:input_type -> grpc.testing.SearchRequest
+	1, // 2: grpc.testing.SearchService.StreamingSearch:input_type -> grpc.testing.SearchRequest
+	0, // 3: grpc.testing.SearchService.Search:output_type -> grpc.testing.SearchResponse
+	0, // 4: grpc.testing.SearchService.StreamingSearch:output_type -> grpc.testing.SearchResponse
+	3, // [3:5] is the sub-list for method output_type
+	1, // [1:3] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
 }
 
-var xxx_messageInfo_SearchRequest proto.InternalMessageInfo
-
-func (m *SearchRequest) GetQuery() string {
-	if m != nil {
-		return m.Query
+func init() { file_reflection_grpc_testing_test_proto_init() }
+func file_reflection_grpc_testing_test_proto_init() {
+	if File_reflection_grpc_testing_test_proto != nil {
+		return
 	}
-	return ""
-}
-
-func init() {
-	proto.RegisterType((*SearchResponse)(nil), "grpc.testing.SearchResponse")
-	proto.RegisterType((*SearchResponse_Result)(nil), "grpc.testing.SearchResponse.Result")
-	proto.RegisterType((*SearchRequest)(nil), "grpc.testing.SearchRequest")
-}
-
-func init() {
-	proto.RegisterFile("reflection/grpc_testing/test.proto", fileDescriptor_b179ea967ba71047)
-}
-
-var fileDescriptor_b179ea967ba71047 = []byte{
-	// 267 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x51, 0x3f, 0x4b, 0xfd, 0x30,
-	0x14, 0x25, 0xbf, 0xf2, 0xab, 0xbe, 0xeb, 0x5f, 0x82, 0x43, 0xa9, 0x0e, 0xa5, 0x22, 0x74, 0x4a,
-	0x1f, 0xcf, 0xd9, 0x45, 0x57, 0x07, 0x69, 0x37, 0x17, 0xa9, 0xe5, 0x1a, 0x03, 0x31, 0xe9, 0xbb,
-	0xb9, 0x15, 0xfc, 0x30, 0xae, 0x7e, 0x4e, 0x69, 0xfb, 0x2a, 0x0a, 0xea, 0xe2, 0x94, 0x9c, 0x73,
-	0xcf, 0x39, 0xc9, 0xe5, 0x40, 0x4e, 0xf8, 0x60, 0xb1, 0x65, 0xe3, 0x5d, 0xa9, 0xa9, 0x6b, 0xef,
-	0x18, 0x03, 0x1b, 0xa7, 0xcb, 0xe1, 0x54, 0x1d, 0x79, 0xf6, 0x72, 0x77, 0x18, 0xa8, 0xcd, 0x20,
-	0x7f, 0x15, 0xb0, 0x5f, 0x63, 0x43, 0xed, 0x63, 0x85, 0xa1, 0xf3, 0x2e, 0xa0, 0xbc, 0x80, 0x2d,
-	0xc2, 0xd0, 0x5b, 0x0e, 0x89, 0xc8, 0xa2, 0x62, 0x67, 0x75, 0xaa, 0x3e, 0x5b, 0xd4, 0x57, 0xb9,
-	0xaa, 0x46, 0x6d, 0x35, 0x7b, 0xd2, 0x6b, 0x88, 0x27, 0x4a, 0x1e, 0x42, 0xd4, 0x93, 0x4d, 0x44,
-	0x26, 0x8a, 0x45, 0x35, 0x5c, 0xe5, 0x11, 0xfc, 0x67, 0xc3, 0x16, 0x93, 0x7f, 0x23, 0x37, 0x01,
-	0x99, 0xc2, 0x76, 0x70, 0xa6, 0xeb, 0x90, 0x43, 0x12, 0x65, 0x51, 0xb1, 0xa8, 0x3e, 0x70, 0x7e,
-	0x06, 0x7b, 0xf3, 0x7b, 0xeb, 0x1e, 0x03, 0x0f, 0x11, 0xeb, 0x1e, 0xe9, 0x65, 0x13, 0x3b, 0x81,
-	0xd5, 0x9b, 0x98, 0x75, 0x35, 0xd2, 0xb3, 0x69, 0x51, 0x5e, 0x41, 0x3c, 0x11, 0xf2, 0xf8, 0xfb,
-	0xef, 0x8f, 0x71, 0xe9, 0xc9, 0x6f, 0xbb, 0xc9, 0x1b, 0x38, 0xa8, 0x99, 0xb0, 0x79, 0x32, 0x4e,
-	0xff, 0x39, 0xad, 0x10, 0x4b, 0x71, 0xb9, 0xbc, 0x55, 0xda, 0x7b, 0x6d, 0x51, 0x69, 0x6f, 0x1b,
-	0xa7, 0x95, 0x27, 0x3d, 0x56, 0x55, 0xfe, 0x50, 0xdd, 0x7d, 0x3c, 0xd6, 0x76, 0xfe, 0x1e, 0x00,
-	0x00, 0xff, 0xff, 0x38, 0x42, 0x3b, 0xd2, 0xdc, 0x01, 0x00, 0x00,
+	if !protoimpl.UnsafeEnabled {
+		file_reflection_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SearchResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SearchRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_reflection_grpc_testing_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SearchResponse_Result); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_reflection_grpc_testing_test_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   3,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_reflection_grpc_testing_test_proto_goTypes,
+		DependencyIndexes: file_reflection_grpc_testing_test_proto_depIdxs,
+		MessageInfos:      file_reflection_grpc_testing_test_proto_msgTypes,
+	}.Build()
+	File_reflection_grpc_testing_test_proto = out.File
+	file_reflection_grpc_testing_test_proto_rawDesc = nil
+	file_reflection_grpc_testing_test_proto_goTypes = nil
+	file_reflection_grpc_testing_test_proto_depIdxs = nil
 }
diff --git a/regenerate.sh b/regenerate.sh
index 647a55afbbfc..3443ad975913 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -31,8 +31,8 @@ echo "remove existing generated files"
 # intentionally generated by an older version of protoc-gen-go.
 rm -f $(find . -name '*.pb.go' | grep -v 'grpc_testingv3/testv3.pb.go')
 
-echo "go install github.com/golang/protobuf/protoc-gen-go"
-(cd test/tools && go install github.com/golang/protobuf/protoc-gen-go)
+echo "go install google.golang.org/protobuf/cmd/protoc-gen-go"
+(cd test/tools && go install google.golang.org/protobuf/cmd/protoc-gen-go)
 
 echo "go install cmd/protoc-gen-go-grpc"
 (cd cmd/protoc-gen-go-grpc && go install .)
diff --git a/stats/grpc_testing/test.pb.go b/stats/grpc_testing/test.pb.go
index df3796ebbd30..59866b746384 100644
--- a/stats/grpc_testing/test.pb.go
+++ b/stats/grpc_testing/test.pb.go
@@ -1,125 +1,254 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: stats/grpc_testing/test.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type SimpleRequest struct {
-	Id                   int32    `protobuf:"varint,2,opt,name=id,proto3" json:"id,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
-func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
-func (*SimpleRequest) ProtoMessage()    {}
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_d7a50f7a8e9e8e09, []int{0}
+	Id int32 `protobuf:"varint,2,opt,name=id,proto3" json:"id,omitempty"`
 }
 
-func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleRequest.Unmarshal(m, b)
-}
-func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic)
-}
-func (m *SimpleRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleRequest.Merge(m, src)
+func (x *SimpleRequest) Reset() {
+	*x = SimpleRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_stats_grpc_testing_test_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *SimpleRequest) XXX_Size() int {
-	return xxx_messageInfo_SimpleRequest.Size(m)
+
+func (x *SimpleRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleRequest.DiscardUnknown(m)
+
+func (*SimpleRequest) ProtoMessage() {}
+
+func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_stats_grpc_testing_test_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo
+// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
+func (*SimpleRequest) Descriptor() ([]byte, []int) {
+	return file_stats_grpc_testing_test_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *SimpleRequest) GetId() int32 {
-	if m != nil {
-		return m.Id
+func (x *SimpleRequest) GetId() int32 {
+	if x != nil {
+		return x.Id
 	}
 	return 0
 }
 
 type SimpleResponse struct {
-	Id                   int32    `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
-func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
-func (*SimpleResponse) ProtoMessage()    {}
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_d7a50f7a8e9e8e09, []int{1}
+	Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"`
 }
 
-func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleResponse.Unmarshal(m, b)
-}
-func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic)
-}
-func (m *SimpleResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleResponse.Merge(m, src)
+func (x *SimpleResponse) Reset() {
+	*x = SimpleResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_stats_grpc_testing_test_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *SimpleResponse) XXX_Size() int {
-	return xxx_messageInfo_SimpleResponse.Size(m)
+
+func (x *SimpleResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleResponse.DiscardUnknown(m)
+
+func (*SimpleResponse) ProtoMessage() {}
+
+func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_stats_grpc_testing_test_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo
+// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
+func (*SimpleResponse) Descriptor() ([]byte, []int) {
+	return file_stats_grpc_testing_test_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *SimpleResponse) GetId() int32 {
-	if m != nil {
-		return m.Id
+func (x *SimpleResponse) GetId() int32 {
+	if x != nil {
+		return x.Id
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest")
-	proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse")
-}
-
-func init() { proto.RegisterFile("stats/grpc_testing/test.proto", fileDescriptor_d7a50f7a8e9e8e09) }
-
-var fileDescriptor_d7a50f7a8e9e8e09 = []byte{
-	// 233 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x91, 0xcf, 0x4a, 0x03, 0x31,
-	0x10, 0xc6, 0x49, 0x44, 0xc1, 0x51, 0x17, 0xc9, 0x49, 0xfc, 0x83, 0xa5, 0xa7, 0x8a, 0x90, 0x2d,
-	0xfa, 0x06, 0x56, 0x7a, 0x2d, 0x74, 0xf5, 0xe2, 0x45, 0x62, 0x3b, 0x84, 0xc0, 0x34, 0x89, 0xc9,
-	0xac, 0xe8, 0x1b, 0xfa, 0x58, 0x92, 0xb2, 0x0b, 0x8a, 0xde, 0xd6, 0xd3, 0x1c, 0xbe, 0x8f, 0xdf,
-	0xc7, 0xf0, 0x83, 0x8b, 0xcc, 0x86, 0x73, 0x6d, 0x53, 0x5c, 0x3d, 0x33, 0x66, 0x76, 0xde, 0xd6,
-	0xe5, 0xea, 0x98, 0x02, 0x07, 0x75, 0x58, 0x02, 0xdd, 0x05, 0xe3, 0x4b, 0x38, 0x6a, 0xdc, 0x26,
-	0x12, 0x2e, 0xf1, 0xb5, 0xc5, 0xcc, 0xaa, 0x02, 0xe9, 0xd6, 0x27, 0x72, 0x24, 0x26, 0xbb, 0x4b,
-	0xe9, 0xd6, 0xe3, 0x11, 0x54, 0x7d, 0x21, 0xc7, 0xe0, 0x33, 0x76, 0x8d, 0x9d, 0xbe, 0x71, 0xf3,
-	0x29, 0xe1, 0xe0, 0x01, 0x33, 0x37, 0x98, 0xde, 0xdc, 0x0a, 0xd5, 0x1c, 0xf6, 0x1f, 0xbd, 0x49,
-	0x1f, 0x33, 0x43, 0xa4, 0xce, 0xf4, 0xf7, 0x39, 0xfd, 0x63, 0xeb, 0xf4, 0xfc, 0xef, 0xb0, 0xdb,
-	0x59, 0x40, 0x35, 0x6f, 0x89, 0xee, 0xdb, 0x48, 0xf8, 0x3e, 0x10, 0x36, 0x11, 0x53, 0xa1, 0x16,
-	0x70, 0x3c, 0x23, 0x87, 0x9e, 0x1b, 0x4e, 0x68, 0x36, 0x83, 0x91, 0x05, 0x58, 0x9e, 0xc6, 0xf4,
-	0x2f, 0xc0, 0xa9, 0xb8, 0xbb, 0x7e, 0xba, 0xb2, 0x21, 0x58, 0x42, 0x6d, 0x03, 0x19, 0x6f, 0x75,
-	0x48, 0x76, 0x2b, 0xb2, 0xfe, 0xed, 0xf4, 0x65, 0x6f, 0xeb, 0xf3, 0xf6, 0x2b, 0x00, 0x00, 0xff,
-	0xff, 0x7c, 0x26, 0xce, 0x3c, 0xf0, 0x01, 0x00, 0x00,
+var File_stats_grpc_testing_test_proto protoreflect.FileDescriptor
+
+var file_stats_grpc_testing_test_proto_rawDesc = []byte{
+	0x0a, 0x1d, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+	0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x1f, 0x0a,
+	0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e,
+	0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x22, 0x20,
+	0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64,
+	0x32, 0xc8, 0x02, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c,
+	0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x10, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x4f, 0x0a, 0x10, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69,
+	0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x2b, 0x5a, 0x29, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
+	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_stats_grpc_testing_test_proto_rawDescOnce sync.Once
+	file_stats_grpc_testing_test_proto_rawDescData = file_stats_grpc_testing_test_proto_rawDesc
+)
+
+func file_stats_grpc_testing_test_proto_rawDescGZIP() []byte {
+	file_stats_grpc_testing_test_proto_rawDescOnce.Do(func() {
+		file_stats_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_stats_grpc_testing_test_proto_rawDescData)
+	})
+	return file_stats_grpc_testing_test_proto_rawDescData
+}
+
+var file_stats_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_stats_grpc_testing_test_proto_goTypes = []interface{}{
+	(*SimpleRequest)(nil),  // 0: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil), // 1: grpc.testing.SimpleResponse
+}
+var file_stats_grpc_testing_test_proto_depIdxs = []int32{
+	0, // 0: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	0, // 1: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.SimpleRequest
+	0, // 2: grpc.testing.TestService.ClientStreamCall:input_type -> grpc.testing.SimpleRequest
+	0, // 3: grpc.testing.TestService.ServerStreamCall:input_type -> grpc.testing.SimpleRequest
+	1, // 4: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	1, // 5: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.SimpleResponse
+	1, // 6: grpc.testing.TestService.ClientStreamCall:output_type -> grpc.testing.SimpleResponse
+	1, // 7: grpc.testing.TestService.ServerStreamCall:output_type -> grpc.testing.SimpleResponse
+	4, // [4:8] is the sub-list for method output_type
+	0, // [0:4] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_stats_grpc_testing_test_proto_init() }
+func file_stats_grpc_testing_test_proto_init() {
+	if File_stats_grpc_testing_test_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_stats_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_stats_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_stats_grpc_testing_test_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_stats_grpc_testing_test_proto_goTypes,
+		DependencyIndexes: file_stats_grpc_testing_test_proto_depIdxs,
+		MessageInfos:      file_stats_grpc_testing_test_proto_msgTypes,
+	}.Build()
+	File_stats_grpc_testing_test_proto = out.File
+	file_stats_grpc_testing_test_proto_rawDesc = nil
+	file_stats_grpc_testing_test_proto_goTypes = nil
+	file_stats_grpc_testing_test_proto_depIdxs = nil
 }
diff --git a/stress/grpc_testing/metrics.pb.go b/stress/grpc_testing/metrics.pb.go
index 40a0123c44bd..45d34c662ec2 100644
--- a/stress/grpc_testing/metrics.pb.go
+++ b/stress/grpc_testing/metrics.pb.go
@@ -1,66 +1,128 @@
+// Copyright 2015-2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Contains the definitions for a metrics service and the type of metrics
+// exposed by the service.
+//
+// Currently, 'Gauge' (i.e a metric that represents the measured value of
+// something at an instant of time) is the only metric type supported by the
+// service.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: stress/grpc_testing/metrics.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Response message containing the gauge name and value
 type GaugeResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	// Types that are valid to be assigned to Value:
+	// Types that are assignable to Value:
 	//	*GaugeResponse_LongValue
 	//	*GaugeResponse_DoubleValue
 	//	*GaugeResponse_StringValue
-	Value                isGaugeResponse_Value `protobuf_oneof:"value"`
-	XXX_NoUnkeyedLiteral struct{}              `json:"-"`
-	XXX_unrecognized     []byte                `json:"-"`
-	XXX_sizecache        int32                 `json:"-"`
+	Value isGaugeResponse_Value `protobuf_oneof:"value"`
 }
 
-func (m *GaugeResponse) Reset()         { *m = GaugeResponse{} }
-func (m *GaugeResponse) String() string { return proto.CompactTextString(m) }
-func (*GaugeResponse) ProtoMessage()    {}
-func (*GaugeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_028251bc41da09ab, []int{0}
+func (x *GaugeResponse) Reset() {
+	*x = GaugeResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_stress_grpc_testing_metrics_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *GaugeResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GaugeResponse.Unmarshal(m, b)
+func (x *GaugeResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *GaugeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GaugeResponse.Marshal(b, m, deterministic)
+
+func (*GaugeResponse) ProtoMessage() {}
+
+func (x *GaugeResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_stress_grpc_testing_metrics_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *GaugeResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GaugeResponse.Merge(m, src)
+
+// Deprecated: Use GaugeResponse.ProtoReflect.Descriptor instead.
+func (*GaugeResponse) Descriptor() ([]byte, []int) {
+	return file_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{0}
 }
-func (m *GaugeResponse) XXX_Size() int {
-	return xxx_messageInfo_GaugeResponse.Size(m)
+
+func (x *GaugeResponse) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
 }
-func (m *GaugeResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GaugeResponse.DiscardUnknown(m)
+
+func (m *GaugeResponse) GetValue() isGaugeResponse_Value {
+	if m != nil {
+		return m.Value
+	}
+	return nil
 }
 
-var xxx_messageInfo_GaugeResponse proto.InternalMessageInfo
+func (x *GaugeResponse) GetLongValue() int64 {
+	if x, ok := x.GetValue().(*GaugeResponse_LongValue); ok {
+		return x.LongValue
+	}
+	return 0
+}
 
-func (m *GaugeResponse) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *GaugeResponse) GetDoubleValue() float64 {
+	if x, ok := x.GetValue().(*GaugeResponse_DoubleValue); ok {
+		return x.DoubleValue
+	}
+	return 0
+}
+
+func (x *GaugeResponse) GetStringValue() string {
+	if x, ok := x.GetValue().(*GaugeResponse_StringValue); ok {
+		return x.StringValue
 	}
 	return ""
 }
@@ -87,140 +149,221 @@ func (*GaugeResponse_DoubleValue) isGaugeResponse_Value() {}
 
 func (*GaugeResponse_StringValue) isGaugeResponse_Value() {}
 
-func (m *GaugeResponse) GetValue() isGaugeResponse_Value {
-	if m != nil {
-		return m.Value
-	}
-	return nil
-}
+// Request message containing the gauge name
+type GaugeRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 
-func (m *GaugeResponse) GetLongValue() int64 {
-	if x, ok := m.GetValue().(*GaugeResponse_LongValue); ok {
-		return x.LongValue
-	}
-	return 0
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 }
 
-func (m *GaugeResponse) GetDoubleValue() float64 {
-	if x, ok := m.GetValue().(*GaugeResponse_DoubleValue); ok {
-		return x.DoubleValue
+func (x *GaugeRequest) Reset() {
+	*x = GaugeRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_stress_grpc_testing_metrics_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
 	}
-	return 0
 }
 
-func (m *GaugeResponse) GetStringValue() string {
-	if x, ok := m.GetValue().(*GaugeResponse_StringValue); ok {
-		return x.StringValue
-	}
-	return ""
+func (x *GaugeRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-// XXX_OneofWrappers is for the internal use of the proto package.
-func (*GaugeResponse) XXX_OneofWrappers() []interface{} {
-	return []interface{}{
-		(*GaugeResponse_LongValue)(nil),
-		(*GaugeResponse_DoubleValue)(nil),
-		(*GaugeResponse_StringValue)(nil),
-	}
-}
+func (*GaugeRequest) ProtoMessage() {}
 
-// Request message containing the gauge name
-type GaugeRequest struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (x *GaugeRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_stress_grpc_testing_metrics_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-func (m *GaugeRequest) Reset()         { *m = GaugeRequest{} }
-func (m *GaugeRequest) String() string { return proto.CompactTextString(m) }
-func (*GaugeRequest) ProtoMessage()    {}
+// Deprecated: Use GaugeRequest.ProtoReflect.Descriptor instead.
 func (*GaugeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_028251bc41da09ab, []int{1}
+	return file_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{1}
 }
 
-func (m *GaugeRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GaugeRequest.Unmarshal(m, b)
-}
-func (m *GaugeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GaugeRequest.Marshal(b, m, deterministic)
-}
-func (m *GaugeRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GaugeRequest.Merge(m, src)
-}
-func (m *GaugeRequest) XXX_Size() int {
-	return xxx_messageInfo_GaugeRequest.Size(m)
-}
-func (m *GaugeRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GaugeRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GaugeRequest proto.InternalMessageInfo
-
-func (m *GaugeRequest) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *GaugeRequest) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
 type EmptyMessage struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
 
-func (m *EmptyMessage) Reset()         { *m = EmptyMessage{} }
-func (m *EmptyMessage) String() string { return proto.CompactTextString(m) }
-func (*EmptyMessage) ProtoMessage()    {}
-func (*EmptyMessage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_028251bc41da09ab, []int{2}
+func (x *EmptyMessage) Reset() {
+	*x = EmptyMessage{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_stress_grpc_testing_metrics_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *EmptyMessage) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EmptyMessage.Unmarshal(m, b)
-}
-func (m *EmptyMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EmptyMessage.Marshal(b, m, deterministic)
-}
-func (m *EmptyMessage) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EmptyMessage.Merge(m, src)
-}
-func (m *EmptyMessage) XXX_Size() int {
-	return xxx_messageInfo_EmptyMessage.Size(m)
-}
-func (m *EmptyMessage) XXX_DiscardUnknown() {
-	xxx_messageInfo_EmptyMessage.DiscardUnknown(m)
+func (x *EmptyMessage) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
 
-var xxx_messageInfo_EmptyMessage proto.InternalMessageInfo
+func (*EmptyMessage) ProtoMessage() {}
 
-func init() {
-	proto.RegisterType((*GaugeResponse)(nil), "grpc.testing.GaugeResponse")
-	proto.RegisterType((*GaugeRequest)(nil), "grpc.testing.GaugeRequest")
-	proto.RegisterType((*EmptyMessage)(nil), "grpc.testing.EmptyMessage")
+func (x *EmptyMessage) ProtoReflect() protoreflect.Message {
+	mi := &file_stress_grpc_testing_metrics_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-func init() { proto.RegisterFile("stress/grpc_testing/metrics.proto", fileDescriptor_028251bc41da09ab) }
+// Deprecated: Use EmptyMessage.ProtoReflect.Descriptor instead.
+func (*EmptyMessage) Descriptor() ([]byte, []int) {
+	return file_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{2}
+}
+
+var File_stress_grpc_testing_metrics_proto protoreflect.FileDescriptor
+
+var file_stress_grpc_testing_metrics_proto_rawDesc = []byte{
+	0x0a, 0x21, 0x73, 0x74, 0x72, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x22, 0x97, 0x01, 0x0a, 0x0d, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x6c, 0x6f, 0x6e, 0x67, 0x5f,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x09, 0x6c,
+	0x6f, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62,
+	0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00,
+	0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a,
+	0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20,
+	0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c,
+	0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x22, 0x0a, 0x0c, 0x47,
+	0x61, 0x75, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22,
+	0x0e, 0x0a, 0x0c, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32,
+	0xa0, 0x01, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x47, 0x61, 0x75, 0x67,
+	0x65, 0x73, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1b,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61,
+	0x75, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x43, 0x0a,
+	0x08, 0x47, 0x65, 0x74, 0x47, 0x61, 0x75, 0x67, 0x65, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c,
+	0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x72,
+	0x65, 0x73, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_stress_grpc_testing_metrics_proto_rawDescOnce sync.Once
+	file_stress_grpc_testing_metrics_proto_rawDescData = file_stress_grpc_testing_metrics_proto_rawDesc
+)
 
-var fileDescriptor_028251bc41da09ab = []byte{
-	// 288 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x3f, 0x4f, 0xc3, 0x30,
-	0x10, 0xc5, 0x6b, 0x5a, 0xfe, 0xf4, 0x08, 0x1d, 0x3c, 0x55, 0x65, 0x20, 0x84, 0x25, 0x42, 0xc8,
-	0x41, 0xf0, 0x09, 0x28, 0x42, 0x29, 0x43, 0x97, 0x20, 0x31, 0xb0, 0x54, 0x69, 0x38, 0x59, 0x91,
-	0x9c, 0x38, 0xf8, 0x9c, 0x4a, 0x7c, 0x12, 0x56, 0x3e, 0x2a, 0x8a, 0x13, 0x55, 0x29, 0xaa, 0xba,
-	0x59, 0xbf, 0xf7, 0xfc, 0x7c, 0xe7, 0x07, 0xd7, 0x64, 0x0d, 0x12, 0x45, 0xd2, 0x54, 0xd9, 0xca,
-	0x22, 0xd9, 0xbc, 0x94, 0x51, 0x81, 0xd6, 0xe4, 0x19, 0x89, 0xca, 0x68, 0xab, 0xb9, 0xd7, 0x68,
-	0xa2, 0xd3, 0x82, 0x1f, 0x06, 0x17, 0x71, 0x5a, 0x4b, 0x4c, 0x90, 0x2a, 0x5d, 0x12, 0x72, 0x0e,
-	0xa3, 0x32, 0x2d, 0x70, 0xca, 0x7c, 0x16, 0x8e, 0x13, 0x77, 0xe6, 0x57, 0x00, 0x4a, 0x97, 0x72,
-	0xb5, 0x49, 0x55, 0x8d, 0xd3, 0x23, 0x9f, 0x85, 0xc3, 0xc5, 0x20, 0x19, 0x37, 0xec, 0xbd, 0x41,
-	0xfc, 0x06, 0xbc, 0x4f, 0x5d, 0xaf, 0x15, 0x76, 0x96, 0xa1, 0xcf, 0x42, 0xb6, 0x18, 0x24, 0xe7,
-	0x2d, 0xdd, 0x9a, 0xc8, 0x9a, 0x7c, 0x9b, 0x33, 0x6a, 0x5e, 0x68, 0x4c, 0x2d, 0x75, 0xa6, 0xf9,
-	0x29, 0x1c, 0x3b, 0x35, 0x08, 0xc0, 0xeb, 0x06, 0xfb, 0xaa, 0x91, 0xec, 0xbe, 0xb9, 0x82, 0x09,
-	0x78, 0x2f, 0x45, 0x65, 0xbf, 0x97, 0x48, 0x94, 0x4a, 0x7c, 0xf8, 0x65, 0x30, 0x59, 0xb6, 0xdb,
-	0xbe, 0xa1, 0xd9, 0xe4, 0x19, 0xf2, 0x57, 0xf0, 0x62, 0xb4, 0x4f, 0x4a, 0xb9, 0x30, 0xe2, 0x33,
-	0xd1, 0xdf, 0x5f, 0xf4, 0xaf, 0xcf, 0x2e, 0x77, 0xb5, 0x9d, 0x7f, 0xb9, 0x67, 0xfc, 0x19, 0xce,
-	0x62, 0xb4, 0x8e, 0xfe, 0x8f, 0xe9, 0x4f, 0x7a, 0x30, 0x66, 0x7e, 0xf7, 0x71, 0x2b, 0xb5, 0x96,
-	0x0a, 0x85, 0xd4, 0x2a, 0x2d, 0xa5, 0xd0, 0x46, 0xba, 0xba, 0xa2, 0x3d, 0xd5, 0xad, 0x4f, 0x5c,
-	0x67, 0x8f, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x8c, 0x62, 0x73, 0xd8, 0x01, 0x00, 0x00,
+func file_stress_grpc_testing_metrics_proto_rawDescGZIP() []byte {
+	file_stress_grpc_testing_metrics_proto_rawDescOnce.Do(func() {
+		file_stress_grpc_testing_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_stress_grpc_testing_metrics_proto_rawDescData)
+	})
+	return file_stress_grpc_testing_metrics_proto_rawDescData
+}
+
+var file_stress_grpc_testing_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_stress_grpc_testing_metrics_proto_goTypes = []interface{}{
+	(*GaugeResponse)(nil), // 0: grpc.testing.GaugeResponse
+	(*GaugeRequest)(nil),  // 1: grpc.testing.GaugeRequest
+	(*EmptyMessage)(nil),  // 2: grpc.testing.EmptyMessage
+}
+var file_stress_grpc_testing_metrics_proto_depIdxs = []int32{
+	2, // 0: grpc.testing.MetricsService.GetAllGauges:input_type -> grpc.testing.EmptyMessage
+	1, // 1: grpc.testing.MetricsService.GetGauge:input_type -> grpc.testing.GaugeRequest
+	0, // 2: grpc.testing.MetricsService.GetAllGauges:output_type -> grpc.testing.GaugeResponse
+	0, // 3: grpc.testing.MetricsService.GetGauge:output_type -> grpc.testing.GaugeResponse
+	2, // [2:4] is the sub-list for method output_type
+	0, // [0:2] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_stress_grpc_testing_metrics_proto_init() }
+func file_stress_grpc_testing_metrics_proto_init() {
+	if File_stress_grpc_testing_metrics_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_stress_grpc_testing_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GaugeResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_stress_grpc_testing_metrics_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GaugeRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_stress_grpc_testing_metrics_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EmptyMessage); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_stress_grpc_testing_metrics_proto_msgTypes[0].OneofWrappers = []interface{}{
+		(*GaugeResponse_LongValue)(nil),
+		(*GaugeResponse_DoubleValue)(nil),
+		(*GaugeResponse_StringValue)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_stress_grpc_testing_metrics_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   3,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_stress_grpc_testing_metrics_proto_goTypes,
+		DependencyIndexes: file_stress_grpc_testing_metrics_proto_depIdxs,
+		MessageInfos:      file_stress_grpc_testing_metrics_proto_msgTypes,
+	}.Build()
+	File_stress_grpc_testing_metrics_proto = out.File
+	file_stress_grpc_testing_metrics_proto_rawDesc = nil
+	file_stress_grpc_testing_metrics_proto_goTypes = nil
+	file_stress_grpc_testing_metrics_proto_depIdxs = nil
 }
diff --git a/test/codec_perf/perf.pb.go b/test/codec_perf/perf.pb.go
index b98764ade4ec..d9c81ee855a2 100644
--- a/test/codec_perf/perf.pb.go
+++ b/test/codec_perf/perf.pb.go
@@ -1,80 +1,168 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Messages used for performance tests that may not reference grpc directly for
+// reasons of import cycles.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: test/codec_perf/perf.proto
 
 package codec_perf
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // Buffer is a message that contains a body of bytes that is used to exercise
 // encoding and decoding overheads.
 type Buffer struct {
-	Body                 []byte   `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Body []byte `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"`
 }
 
-func (m *Buffer) Reset()         { *m = Buffer{} }
-func (m *Buffer) String() string { return proto.CompactTextString(m) }
-func (*Buffer) ProtoMessage()    {}
-func (*Buffer) Descriptor() ([]byte, []int) {
-	return fileDescriptor_a913550de912e506, []int{0}
+func (x *Buffer) Reset() {
+	*x = Buffer{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_codec_perf_perf_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Buffer) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Buffer.Unmarshal(m, b)
+func (x *Buffer) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Buffer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Buffer.Marshal(b, m, deterministic)
+
+func (*Buffer) ProtoMessage() {}
+
+func (x *Buffer) ProtoReflect() protoreflect.Message {
+	mi := &file_test_codec_perf_perf_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
-func (m *Buffer) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Buffer.Merge(m, src)
+
+// Deprecated: Use Buffer.ProtoReflect.Descriptor instead.
+func (*Buffer) Descriptor() ([]byte, []int) {
+	return file_test_codec_perf_perf_proto_rawDescGZIP(), []int{0}
 }
-func (m *Buffer) XXX_Size() int {
-	return xxx_messageInfo_Buffer.Size(m)
+
+func (x *Buffer) GetBody() []byte {
+	if x != nil {
+		return x.Body
+	}
+	return nil
 }
-func (m *Buffer) XXX_DiscardUnknown() {
-	xxx_messageInfo_Buffer.DiscardUnknown(m)
+
+var File_test_codec_perf_perf_proto protoreflect.FileDescriptor
+
+var file_test_codec_perf_perf_proto_rawDesc = []byte{
+	0x0a, 0x1a, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x70, 0x65, 0x72,
+	0x66, 0x2f, 0x70, 0x65, 0x72, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f,
+	0x64, 0x65, 0x63, 0x2e, 0x70, 0x65, 0x72, 0x66, 0x22, 0x1c, 0x0a, 0x06, 0x42, 0x75, 0x66, 0x66,
+	0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63,
+	0x2f, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x70, 0x65, 0x72, 0x66,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
-var xxx_messageInfo_Buffer proto.InternalMessageInfo
+var (
+	file_test_codec_perf_perf_proto_rawDescOnce sync.Once
+	file_test_codec_perf_perf_proto_rawDescData = file_test_codec_perf_perf_proto_rawDesc
+)
 
-func (m *Buffer) GetBody() []byte {
-	if m != nil {
-		return m.Body
-	}
-	return nil
+func file_test_codec_perf_perf_proto_rawDescGZIP() []byte {
+	file_test_codec_perf_perf_proto_rawDescOnce.Do(func() {
+		file_test_codec_perf_perf_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_codec_perf_perf_proto_rawDescData)
+	})
+	return file_test_codec_perf_perf_proto_rawDescData
 }
 
-func init() {
-	proto.RegisterType((*Buffer)(nil), "codec.perf.Buffer")
+var file_test_codec_perf_perf_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_test_codec_perf_perf_proto_goTypes = []interface{}{
+	(*Buffer)(nil), // 0: codec.perf.Buffer
+}
+var file_test_codec_perf_perf_proto_depIdxs = []int32{
+	0, // [0:0] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
 }
 
-func init() { proto.RegisterFile("test/codec_perf/perf.proto", fileDescriptor_a913550de912e506) }
-
-var fileDescriptor_a913550de912e506 = []byte{
-	// 118 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2a, 0x49, 0x2d, 0x2e,
-	0xd1, 0x4f, 0xce, 0x4f, 0x49, 0x4d, 0x8e, 0x2f, 0x48, 0x2d, 0x4a, 0xd3, 0x07, 0x11, 0x7a, 0x05,
-	0x45, 0xf9, 0x25, 0xf9, 0x42, 0x5c, 0x60, 0x61, 0x3d, 0x90, 0x88, 0x92, 0x0c, 0x17, 0x9b, 0x53,
-	0x69, 0x5a, 0x5a, 0x6a, 0x91, 0x90, 0x10, 0x17, 0x4b, 0x52, 0x7e, 0x4a, 0xa5, 0x04, 0xa3, 0x02,
-	0xa3, 0x06, 0x4f, 0x10, 0x98, 0xed, 0xa4, 0x11, 0xa5, 0x96, 0x9e, 0x9f, 0x9f, 0x9e, 0x93, 0xaa,
-	0x97, 0x9e, 0x9f, 0x93, 0x98, 0x97, 0xae, 0x97, 0x5f, 0x94, 0xae, 0x9f, 0x5e, 0x54, 0x90, 0xac,
-	0x8f, 0x66, 0x7c, 0x12, 0x1b, 0xd8, 0x68, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xdb,
-	0x49, 0x8b, 0x78, 0x00, 0x00, 0x00,
+func init() { file_test_codec_perf_perf_proto_init() }
+func file_test_codec_perf_perf_proto_init() {
+	if File_test_codec_perf_perf_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_test_codec_perf_perf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Buffer); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_test_codec_perf_perf_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_test_codec_perf_perf_proto_goTypes,
+		DependencyIndexes: file_test_codec_perf_perf_proto_depIdxs,
+		MessageInfos:      file_test_codec_perf_perf_proto_msgTypes,
+	}.Build()
+	File_test_codec_perf_perf_proto = out.File
+	file_test_codec_perf_perf_proto_rawDesc = nil
+	file_test_codec_perf_perf_proto_goTypes = nil
+	file_test_codec_perf_perf_proto_depIdxs = nil
 }
diff --git a/test/grpc_testing/test.pb.go b/test/grpc_testing/test.pb.go
index 70e3b89228bc..a632338e9a6b 100644
--- a/test/grpc_testing/test.pb.go
+++ b/test/grpc_testing/test.pb.go
@@ -1,24 +1,46 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.3.0
 // source: test/grpc_testing/test.proto
 
 package grpc_testing
 
 import (
-	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	math "math"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 // The type of payload that should be returned.
 type PayloadType int32
@@ -32,109 +54,149 @@ const (
 	PayloadType_RANDOM PayloadType = 2
 )
 
-var PayloadType_name = map[int32]string{
-	0: "COMPRESSABLE",
-	1: "UNCOMPRESSABLE",
-	2: "RANDOM",
-}
+// Enum value maps for PayloadType.
+var (
+	PayloadType_name = map[int32]string{
+		0: "COMPRESSABLE",
+		1: "UNCOMPRESSABLE",
+		2: "RANDOM",
+	}
+	PayloadType_value = map[string]int32{
+		"COMPRESSABLE":   0,
+		"UNCOMPRESSABLE": 1,
+		"RANDOM":         2,
+	}
+)
 
-var PayloadType_value = map[string]int32{
-	"COMPRESSABLE":   0,
-	"UNCOMPRESSABLE": 1,
-	"RANDOM":         2,
+func (x PayloadType) Enum() *PayloadType {
+	p := new(PayloadType)
+	*p = x
+	return p
 }
 
 func (x PayloadType) String() string {
-	return proto.EnumName(PayloadType_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
-func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{0}
+func (PayloadType) Descriptor() protoreflect.EnumDescriptor {
+	return file_test_grpc_testing_test_proto_enumTypes[0].Descriptor()
 }
 
-type Empty struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (PayloadType) Type() protoreflect.EnumType {
+	return &file_test_grpc_testing_test_proto_enumTypes[0]
 }
 
-func (m *Empty) Reset()         { *m = Empty{} }
-func (m *Empty) String() string { return proto.CompactTextString(m) }
-func (*Empty) ProtoMessage()    {}
-func (*Empty) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{0}
+func (x PayloadType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
 
-func (m *Empty) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Empty.Unmarshal(m, b)
+// Deprecated: Use PayloadType.Descriptor instead.
+func (PayloadType) EnumDescriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{0}
 }
-func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Empty.Marshal(b, m, deterministic)
+
+type Empty struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
 }
-func (m *Empty) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Empty.Merge(m, src)
+
+func (x *Empty) Reset() {
+	*x = Empty{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *Empty) XXX_Size() int {
-	return xxx_messageInfo_Empty.Size(m)
+
+func (x *Empty) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Empty) XXX_DiscardUnknown() {
-	xxx_messageInfo_Empty.DiscardUnknown(m)
+
+func (*Empty) ProtoMessage() {}
+
+func (x *Empty) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Empty proto.InternalMessageInfo
+// Deprecated: Use Empty.ProtoReflect.Descriptor instead.
+func (*Empty) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{0}
+}
 
 // A block of data, to simply increase gRPC message size.
 type Payload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// The type of data in body.
 	Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"`
 	// Primary contents of payload.
-	Body                 []byte   `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
 }
 
-func (m *Payload) Reset()         { *m = Payload{} }
-func (m *Payload) String() string { return proto.CompactTextString(m) }
-func (*Payload) ProtoMessage()    {}
-func (*Payload) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{1}
+func (x *Payload) Reset() {
+	*x = Payload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *Payload) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Payload.Unmarshal(m, b)
-}
-func (m *Payload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Payload.Marshal(b, m, deterministic)
-}
-func (m *Payload) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Payload.Merge(m, src)
+func (x *Payload) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *Payload) XXX_Size() int {
-	return xxx_messageInfo_Payload.Size(m)
-}
-func (m *Payload) XXX_DiscardUnknown() {
-	xxx_messageInfo_Payload.DiscardUnknown(m)
+
+func (*Payload) ProtoMessage() {}
+
+func (x *Payload) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_Payload proto.InternalMessageInfo
+// Deprecated: Use Payload.ProtoReflect.Descriptor instead.
+func (*Payload) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{1}
+}
 
-func (m *Payload) GetType() PayloadType {
-	if m != nil {
-		return m.Type
+func (x *Payload) GetType() PayloadType {
+	if x != nil {
+		return x.Type
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *Payload) GetBody() []byte {
-	if m != nil {
-		return m.Body
+func (x *Payload) GetBody() []byte {
+	if x != nil {
+		return x.Body
 	}
 	return nil
 }
 
 // Unary request.
 type SimpleRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload type in the response from the server.
 	// If response_type is RANDOM, server randomly chooses one from other formats.
 	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
@@ -146,268 +208,308 @@ type SimpleRequest struct {
 	// Whether SimpleResponse should include username.
 	FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"`
 	// Whether SimpleResponse should include OAuth scope.
-	FillOauthScope       bool     `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"`
 }
 
-func (m *SimpleRequest) Reset()         { *m = SimpleRequest{} }
-func (m *SimpleRequest) String() string { return proto.CompactTextString(m) }
-func (*SimpleRequest) ProtoMessage()    {}
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{2}
+func (x *SimpleRequest) Reset() {
+	*x = SimpleRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SimpleRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleRequest.Unmarshal(m, b)
-}
-func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic)
-}
-func (m *SimpleRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleRequest.Merge(m, src)
+func (x *SimpleRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleRequest) XXX_Size() int {
-	return xxx_messageInfo_SimpleRequest.Size(m)
-}
-func (m *SimpleRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleRequest.DiscardUnknown(m)
+
+func (*SimpleRequest) ProtoMessage() {}
+
+func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo
+// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
+func (*SimpleRequest) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{2}
+}
 
-func (m *SimpleRequest) GetResponseType() PayloadType {
-	if m != nil {
-		return m.ResponseType
+func (x *SimpleRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *SimpleRequest) GetResponseSize() int32 {
-	if m != nil {
-		return m.ResponseSize
+func (x *SimpleRequest) GetResponseSize() int32 {
+	if x != nil {
+		return x.ResponseSize
 	}
 	return 0
 }
 
-func (m *SimpleRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *SimpleRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *SimpleRequest) GetFillUsername() bool {
-	if m != nil {
-		return m.FillUsername
+func (x *SimpleRequest) GetFillUsername() bool {
+	if x != nil {
+		return x.FillUsername
 	}
 	return false
 }
 
-func (m *SimpleRequest) GetFillOauthScope() bool {
-	if m != nil {
-		return m.FillOauthScope
+func (x *SimpleRequest) GetFillOauthScope() bool {
+	if x != nil {
+		return x.FillOauthScope
 	}
 	return false
 }
 
 // Unary response, as configured by the request.
 type SimpleResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Payload to increase message size.
 	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 	// The user the request came from, for verifying authentication was
 	// successful when the client expected it.
 	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
 	// OAuth scope.
-	OauthScope           string   `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
 }
 
-func (m *SimpleResponse) Reset()         { *m = SimpleResponse{} }
-func (m *SimpleResponse) String() string { return proto.CompactTextString(m) }
-func (*SimpleResponse) ProtoMessage()    {}
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{3}
+func (x *SimpleResponse) Reset() {
+	*x = SimpleResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *SimpleResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SimpleResponse.Unmarshal(m, b)
-}
-func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic)
-}
-func (m *SimpleResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SimpleResponse.Merge(m, src)
+func (x *SimpleResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *SimpleResponse) XXX_Size() int {
-	return xxx_messageInfo_SimpleResponse.Size(m)
-}
-func (m *SimpleResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_SimpleResponse.DiscardUnknown(m)
+
+func (*SimpleResponse) ProtoMessage() {}
+
+func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo
+// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
+func (*SimpleResponse) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{3}
+}
 
-func (m *SimpleResponse) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *SimpleResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func (m *SimpleResponse) GetUsername() string {
-	if m != nil {
-		return m.Username
+func (x *SimpleResponse) GetUsername() string {
+	if x != nil {
+		return x.Username
 	}
 	return ""
 }
 
-func (m *SimpleResponse) GetOauthScope() string {
-	if m != nil {
-		return m.OauthScope
+func (x *SimpleResponse) GetOauthScope() string {
+	if x != nil {
+		return x.OauthScope
 	}
 	return ""
 }
 
 // Client-streaming request.
 type StreamingInputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Optional input payload sent along with the request.
-	Payload              *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingInputCallRequest) Reset()         { *m = StreamingInputCallRequest{} }
-func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) }
-func (*StreamingInputCallRequest) ProtoMessage()    {}
-func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{4}
+func (x *StreamingInputCallRequest) Reset() {
+	*x = StreamingInputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingInputCallRequest.Unmarshal(m, b)
+func (x *StreamingInputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingInputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingInputCallRequest.Marshal(b, m, deterministic)
-}
-func (m *StreamingInputCallRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingInputCallRequest.Merge(m, src)
-}
-func (m *StreamingInputCallRequest) XXX_Size() int {
-	return xxx_messageInfo_StreamingInputCallRequest.Size(m)
-}
-func (m *StreamingInputCallRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingInputCallRequest.DiscardUnknown(m)
+
+func (*StreamingInputCallRequest) ProtoMessage() {}
+
+func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingInputCallRequest proto.InternalMessageInfo
+// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{4}
+}
 
-func (m *StreamingInputCallRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingInputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
 // Client-streaming response.
 type StreamingInputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Aggregated size of payloads received from the client.
-	AggregatedPayloadSize int32    `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
-	XXX_NoUnkeyedLiteral  struct{} `json:"-"`
-	XXX_unrecognized      []byte   `json:"-"`
-	XXX_sizecache         int32    `json:"-"`
+	AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
 }
 
-func (m *StreamingInputCallResponse) Reset()         { *m = StreamingInputCallResponse{} }
-func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) }
-func (*StreamingInputCallResponse) ProtoMessage()    {}
-func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{5}
+func (x *StreamingInputCallResponse) Reset() {
+	*x = StreamingInputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingInputCallResponse.Unmarshal(m, b)
-}
-func (m *StreamingInputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingInputCallResponse.Marshal(b, m, deterministic)
+func (x *StreamingInputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingInputCallResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingInputCallResponse.Merge(m, src)
-}
-func (m *StreamingInputCallResponse) XXX_Size() int {
-	return xxx_messageInfo_StreamingInputCallResponse.Size(m)
-}
-func (m *StreamingInputCallResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingInputCallResponse.DiscardUnknown(m)
+
+func (*StreamingInputCallResponse) ProtoMessage() {}
+
+func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingInputCallResponse proto.InternalMessageInfo
+// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{5}
+}
 
-func (m *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
-	if m != nil {
-		return m.AggregatedPayloadSize
+func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
+	if x != nil {
+		return x.AggregatedPayloadSize
 	}
 	return 0
 }
 
 // Configuration for a particular response.
 type ResponseParameters struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload sizes in responses from the server.
 	// If response_type is COMPRESSABLE, this denotes the size before compression.
 	Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"`
 	// Desired interval between consecutive responses in the response stream in
 	// microseconds.
-	IntervalUs           int32    `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
 }
 
-func (m *ResponseParameters) Reset()         { *m = ResponseParameters{} }
-func (m *ResponseParameters) String() string { return proto.CompactTextString(m) }
-func (*ResponseParameters) ProtoMessage()    {}
-func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{6}
+func (x *ResponseParameters) Reset() {
+	*x = ResponseParameters{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *ResponseParameters) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ResponseParameters.Unmarshal(m, b)
+func (x *ResponseParameters) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *ResponseParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ResponseParameters.Marshal(b, m, deterministic)
-}
-func (m *ResponseParameters) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ResponseParameters.Merge(m, src)
-}
-func (m *ResponseParameters) XXX_Size() int {
-	return xxx_messageInfo_ResponseParameters.Size(m)
-}
-func (m *ResponseParameters) XXX_DiscardUnknown() {
-	xxx_messageInfo_ResponseParameters.DiscardUnknown(m)
+
+func (*ResponseParameters) ProtoMessage() {}
+
+func (x *ResponseParameters) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_ResponseParameters proto.InternalMessageInfo
+// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead.
+func (*ResponseParameters) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{6}
+}
 
-func (m *ResponseParameters) GetSize() int32 {
-	if m != nil {
-		return m.Size
+func (x *ResponseParameters) GetSize() int32 {
+	if x != nil {
+		return x.Size
 	}
 	return 0
 }
 
-func (m *ResponseParameters) GetIntervalUs() int32 {
-	if m != nil {
-		return m.IntervalUs
+func (x *ResponseParameters) GetIntervalUs() int32 {
+	if x != nil {
+		return x.IntervalUs
 	}
 	return 0
 }
 
 // Server-streaming request.
 type StreamingOutputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Desired payload type in the response from the server.
 	// If response_type is RANDOM, the payload from each response in the stream
 	// might be of different types. This is to simulate a mixed type of payload
@@ -416,153 +518,412 @@ type StreamingOutputCallRequest struct {
 	// Configuration for each expected response message.
 	ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"`
 	// Optional input payload sent along with the request.
-	Payload              *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingOutputCallRequest) Reset()         { *m = StreamingOutputCallRequest{} }
-func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) }
-func (*StreamingOutputCallRequest) ProtoMessage()    {}
-func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{7}
+func (x *StreamingOutputCallRequest) Reset() {
+	*x = StreamingOutputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingOutputCallRequest.Unmarshal(m, b)
-}
-func (m *StreamingOutputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingOutputCallRequest.Marshal(b, m, deterministic)
-}
-func (m *StreamingOutputCallRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingOutputCallRequest.Merge(m, src)
-}
-func (m *StreamingOutputCallRequest) XXX_Size() int {
-	return xxx_messageInfo_StreamingOutputCallRequest.Size(m)
+func (x *StreamingOutputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingOutputCallRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingOutputCallRequest.DiscardUnknown(m)
+
+func (*StreamingOutputCallRequest) ProtoMessage() {}
+
+func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingOutputCallRequest proto.InternalMessageInfo
+// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{7}
+}
 
-func (m *StreamingOutputCallRequest) GetResponseType() PayloadType {
-	if m != nil {
-		return m.ResponseType
+func (x *StreamingOutputCallRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
 	}
 	return PayloadType_COMPRESSABLE
 }
 
-func (m *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
-	if m != nil {
-		return m.ResponseParameters
+func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
+	if x != nil {
+		return x.ResponseParameters
 	}
 	return nil
 }
 
-func (m *StreamingOutputCallRequest) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingOutputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
 // Server-streaming response, as configured by the request and parameters.
 type StreamingOutputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
 	// Payload to increase response size.
-	Payload              *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
 }
 
-func (m *StreamingOutputCallResponse) Reset()         { *m = StreamingOutputCallResponse{} }
-func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) }
-func (*StreamingOutputCallResponse) ProtoMessage()    {}
-func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_139516ae706ad4b7, []int{8}
+func (x *StreamingOutputCallResponse) Reset() {
+	*x = StreamingOutputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_grpc_testing_test_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
 
-func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StreamingOutputCallResponse.Unmarshal(m, b)
-}
-func (m *StreamingOutputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StreamingOutputCallResponse.Marshal(b, m, deterministic)
-}
-func (m *StreamingOutputCallResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamingOutputCallResponse.Merge(m, src)
+func (x *StreamingOutputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *StreamingOutputCallResponse) XXX_Size() int {
-	return xxx_messageInfo_StreamingOutputCallResponse.Size(m)
-}
-func (m *StreamingOutputCallResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamingOutputCallResponse.DiscardUnknown(m)
+
+func (*StreamingOutputCallResponse) ProtoMessage() {}
+
+func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_test_grpc_testing_test_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_StreamingOutputCallResponse proto.InternalMessageInfo
+// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
+	return file_test_grpc_testing_test_proto_rawDescGZIP(), []int{8}
+}
 
-func (m *StreamingOutputCallResponse) GetPayload() *Payload {
-	if m != nil {
-		return m.Payload
+func (x *StreamingOutputCallResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
 	}
 	return nil
 }
 
-func init() {
-	proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value)
-	proto.RegisterType((*Empty)(nil), "grpc.testing.Empty")
-	proto.RegisterType((*Payload)(nil), "grpc.testing.Payload")
-	proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest")
-	proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse")
-	proto.RegisterType((*StreamingInputCallRequest)(nil), "grpc.testing.StreamingInputCallRequest")
-	proto.RegisterType((*StreamingInputCallResponse)(nil), "grpc.testing.StreamingInputCallResponse")
-	proto.RegisterType((*ResponseParameters)(nil), "grpc.testing.ResponseParameters")
-	proto.RegisterType((*StreamingOutputCallRequest)(nil), "grpc.testing.StreamingOutputCallRequest")
-	proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse")
-}
-
-func init() { proto.RegisterFile("test/grpc_testing/test.proto", fileDescriptor_139516ae706ad4b7) }
-
-var fileDescriptor_139516ae706ad4b7 = []byte{
-	// 615 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xdd, 0x6e, 0xd3, 0x4c,
-	0x10, 0xfd, 0xb6, 0x4d, 0xff, 0x26, 0x69, 0x14, 0x6d, 0x55, 0x7d, 0x69, 0x5a, 0x89, 0xc8, 0x5c,
-	0x60, 0x2a, 0x91, 0xa0, 0x20, 0xb8, 0x04, 0xf5, 0x57, 0x54, 0x6a, 0x9b, 0x62, 0x37, 0x37, 0xdc,
-	0x44, 0xdb, 0x76, 0x6a, 0x2c, 0x6d, 0xbc, 0xcb, 0x7a, 0x5d, 0x91, 0x5e, 0xf0, 0x62, 0xbc, 0x0c,
-	0x0f, 0xc1, 0x03, 0xa0, 0xdd, 0xd8, 0xa9, 0xd3, 0xb8, 0x22, 0x05, 0xc1, 0x55, 0x36, 0x33, 0x67,
-	0xce, 0x9c, 0xe3, 0x19, 0x7b, 0x61, 0x4b, 0x63, 0xac, 0xdb, 0x81, 0x92, 0x97, 0x7d, 0x73, 0x0a,
-	0xa3, 0xa0, 0x6d, 0x7e, 0x5b, 0x52, 0x09, 0x2d, 0x68, 0xc5, 0x24, 0x5a, 0x69, 0xc2, 0x59, 0x82,
-	0x85, 0x83, 0x81, 0xd4, 0x43, 0xe7, 0x18, 0x96, 0xce, 0xd8, 0x90, 0x0b, 0x76, 0x45, 0x5f, 0x40,
-	0x49, 0x0f, 0x25, 0xd6, 0x49, 0x93, 0xb8, 0xd5, 0xce, 0x46, 0x2b, 0x5f, 0xd0, 0x4a, 0x41, 0xe7,
-	0x43, 0x89, 0x9e, 0x85, 0x51, 0x0a, 0xa5, 0x0b, 0x71, 0x35, 0xac, 0xcf, 0x35, 0x89, 0x5b, 0xf1,
-	0xec, 0xd9, 0xf9, 0x41, 0x60, 0xd5, 0x0f, 0x07, 0x92, 0xa3, 0x87, 0x9f, 0x13, 0x8c, 0x35, 0x7d,
-	0x0b, 0xab, 0x0a, 0x63, 0x29, 0xa2, 0x18, 0xfb, 0xb3, 0xb1, 0x57, 0x32, 0xbc, 0xf9, 0x47, 0x9f,
-	0xe6, 0xea, 0xe3, 0xf0, 0x16, 0x6d, 0xbb, 0x85, 0x3b, 0x90, 0x1f, 0xde, 0x22, 0x6d, 0xc3, 0x92,
-	0x1c, 0x31, 0xd4, 0xe7, 0x9b, 0xc4, 0x2d, 0x77, 0xd6, 0x0b, 0xe9, 0xbd, 0x0c, 0x65, 0x58, 0xaf,
-	0x43, 0xce, 0xfb, 0x49, 0x8c, 0x2a, 0x62, 0x03, 0xac, 0x97, 0x9a, 0xc4, 0x5d, 0xf6, 0x2a, 0x26,
-	0xd8, 0x4b, 0x63, 0xd4, 0x85, 0x9a, 0x05, 0x09, 0x96, 0xe8, 0x4f, 0xfd, 0xf8, 0x52, 0x48, 0xac,
-	0x2f, 0x58, 0x5c, 0xd5, 0xc4, 0xbb, 0x26, 0xec, 0x9b, 0xa8, 0xf3, 0x15, 0xaa, 0x99, 0xeb, 0x91,
-	0xaa, 0xbc, 0x22, 0x32, 0x93, 0xa2, 0x06, 0x2c, 0x8f, 0xc5, 0x18, 0x8b, 0x2b, 0xde, 0xf8, 0x3f,
-	0x7d, 0x02, 0xe5, 0xbc, 0x86, 0x79, 0x9b, 0x06, 0x71, 0xd7, 0xff, 0x18, 0x36, 0x7c, 0xad, 0x90,
-	0x0d, 0xc2, 0x28, 0x38, 0x8a, 0x64, 0xa2, 0xf7, 0x18, 0xe7, 0xd9, 0x04, 0x1e, 0x2b, 0xc5, 0x39,
-	0x87, 0x46, 0x11, 0x5b, 0xea, 0xec, 0x0d, 0xfc, 0xcf, 0x82, 0x40, 0x61, 0xc0, 0x34, 0x5e, 0xf5,
-	0xd3, 0x9a, 0xd1, 0x68, 0x88, 0x1d, 0xcd, 0xfa, 0x5d, 0x3a, 0xa5, 0x36, 0x33, 0x72, 0x8e, 0x80,
-	0x66, 0x1c, 0x67, 0x4c, 0xb1, 0x01, 0x6a, 0x54, 0xb1, 0x59, 0xa2, 0x5c, 0xa9, 0x3d, 0x1b, 0xbb,
-	0x61, 0xa4, 0x51, 0xdd, 0x30, 0x33, 0xa0, 0x74, 0xe0, 0x90, 0x85, 0x7a, 0xb1, 0xf3, 0x9d, 0xe4,
-	0x14, 0x76, 0x13, 0x7d, 0xcf, 0xf0, 0x9f, 0xae, 0xdc, 0x07, 0x58, 0x1b, 0xd7, 0xcb, 0xb1, 0xd4,
-	0xfa, 0x5c, 0x73, 0xde, 0x2d, 0x77, 0x9a, 0x93, 0x2c, 0xd3, 0x96, 0x3c, 0xaa, 0xa6, 0x6d, 0x3e,
-	0x76, 0x41, 0x9d, 0x53, 0xd8, 0x2c, 0x74, 0xf8, 0x9b, 0xeb, 0xb5, 0xfd, 0x0e, 0xca, 0x39, 0xc3,
-	0xb4, 0x06, 0x95, 0xbd, 0xee, 0xc9, 0x99, 0x77, 0xe0, 0xfb, 0x3b, 0xbb, 0xc7, 0x07, 0xb5, 0xff,
-	0x28, 0x85, 0x6a, 0xef, 0x74, 0x22, 0x46, 0x28, 0xc0, 0xa2, 0xb7, 0x73, 0xba, 0xdf, 0x3d, 0xa9,
-	0xcd, 0x75, 0xbe, 0x95, 0xa0, 0x7c, 0x8e, 0xb1, 0xf6, 0x51, 0xdd, 0x84, 0x97, 0x48, 0x5f, 0xc3,
-	0x8a, 0xfd, 0x80, 0x18, 0x59, 0x74, 0x6d, 0xb2, 0xbb, 0x4d, 0x34, 0x8a, 0x82, 0xf4, 0x10, 0x56,
-	0x7a, 0x11, 0x53, 0xa3, 0xb2, 0xcd, 0x49, 0xc4, 0xc4, 0x87, 0xa3, 0xb1, 0x55, 0x9c, 0x4c, 0x1f,
-	0x00, 0x87, 0xb5, 0x82, 0xe7, 0x43, 0xdd, 0x7b, 0x45, 0x0f, 0x2e, 0x49, 0xe3, 0xf9, 0x0c, 0xc8,
-	0x51, 0xaf, 0x97, 0x84, 0x86, 0x40, 0xa7, 0xdf, 0x08, 0xfa, 0xec, 0x01, 0x8a, 0xfb, 0x6f, 0x60,
-	0xc3, 0xfd, 0x35, 0x70, 0xd4, 0xca, 0x35, 0xad, 0xaa, 0x87, 0x09, 0xe7, 0xfb, 0x89, 0xe4, 0xf8,
-	0xe5, 0xaf, 0x79, 0x72, 0x89, 0x75, 0x55, 0x7d, 0xcf, 0xf8, 0xf5, 0x3f, 0x68, 0xb5, 0xbb, 0xfd,
-	0xd1, 0x0d, 0x84, 0x08, 0x38, 0xb6, 0x02, 0xc1, 0x59, 0x14, 0xb4, 0x84, 0x0a, 0xec, 0x4d, 0xd5,
-	0x9e, 0xba, 0xb3, 0x2e, 0x16, 0xed, 0x7d, 0xf5, 0xea, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e,
-	0x50, 0x51, 0x5b, 0xcf, 0x06, 0x00, 0x00,
+var File_test_grpc_testing_test_proto protoreflect.FileDescriptor
+
+var file_test_grpc_testing_test_proto_rawDesc = []byte{
+	0x0a, 0x1c, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x07, 0x0a, 0x05,
+	0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x4c, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
+	0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62,
+	0x6f, 0x64, 0x79, 0x22, 0xf4, 0x01, 0x0a, 0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66,
+	0x69, 0x6c, 0x6c, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65,
+	0x12, 0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73,
+	0x63, 0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x6c,
+	0x4f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x7e, 0x0a, 0x0e, 0x53, 0x69,
+	0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07,
+	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a,
+	0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x61, 0x75,
+	0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
+	0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x4c, 0x0a, 0x19, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67,
+	0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x69, 0x7a,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61,
+	0x74, 0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x49,
+	0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65,
+	0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x65,
+	0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x69,
+	0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x22, 0xe0, 0x01, 0x0a, 0x1a, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c,
+	0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32,
+	0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18,
+	0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72,
+	0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x07, 0x70,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x4e, 0x0a, 0x1b,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43,
+	0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2a, 0x3f, 0x0a, 0x0b,
+	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x0c, 0x43,
+	0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x12, 0x0a,
+	0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10,
+	0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10, 0x02, 0x32, 0xbb, 0x04,
+	0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x35, 0x0a,
+	0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a,
+	0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45,
+	0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, 0x61, 0x6c,
+	0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69,
+	0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x13,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43,
+	0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70,
+	0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72,
+	0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x12, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
+	0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61,
+	0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69,
+	0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c, 0x44, 0x75, 0x70,
+	0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67,
+	0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74,
+	0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01,
+	0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61,
+	0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75,
+	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x2a, 0x5a, 0x28, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
+	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_test_grpc_testing_test_proto_rawDescOnce sync.Once
+	file_test_grpc_testing_test_proto_rawDescData = file_test_grpc_testing_test_proto_rawDesc
+)
+
+func file_test_grpc_testing_test_proto_rawDescGZIP() []byte {
+	file_test_grpc_testing_test_proto_rawDescOnce.Do(func() {
+		file_test_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_grpc_testing_test_proto_rawDescData)
+	})
+	return file_test_grpc_testing_test_proto_rawDescData
+}
+
+var file_test_grpc_testing_test_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_test_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_test_grpc_testing_test_proto_goTypes = []interface{}{
+	(PayloadType)(0),                    // 0: grpc.testing.PayloadType
+	(*Empty)(nil),                       // 1: grpc.testing.Empty
+	(*Payload)(nil),                     // 2: grpc.testing.Payload
+	(*SimpleRequest)(nil),               // 3: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil),              // 4: grpc.testing.SimpleResponse
+	(*StreamingInputCallRequest)(nil),   // 5: grpc.testing.StreamingInputCallRequest
+	(*StreamingInputCallResponse)(nil),  // 6: grpc.testing.StreamingInputCallResponse
+	(*ResponseParameters)(nil),          // 7: grpc.testing.ResponseParameters
+	(*StreamingOutputCallRequest)(nil),  // 8: grpc.testing.StreamingOutputCallRequest
+	(*StreamingOutputCallResponse)(nil), // 9: grpc.testing.StreamingOutputCallResponse
+}
+var file_test_grpc_testing_test_proto_depIdxs = []int32{
+	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
+	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
+	2,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
+	2,  // 3: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
+	2,  // 4: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
+	0,  // 5: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
+	7,  // 6: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
+	2,  // 7: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
+	2,  // 8: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
+	1,  // 9: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty
+	3,  // 10: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	8,  // 11: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	5,  // 12: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest
+	8,  // 13: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	8,  // 14: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	1,  // 15: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty
+	4,  // 16: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	9,  // 17: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	6,  // 18: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse
+	9,  // 19: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	9,  // 20: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	15, // [15:21] is the sub-list for method output_type
+	9,  // [9:15] is the sub-list for method input_type
+	9,  // [9:9] is the sub-list for extension type_name
+	9,  // [9:9] is the sub-list for extension extendee
+	0,  // [0:9] is the sub-list for field type_name
+}
+
+func init() { file_test_grpc_testing_test_proto_init() }
+func file_test_grpc_testing_test_proto_init() {
+	if File_test_grpc_testing_test_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_test_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Empty); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Payload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ResponseParameters); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_grpc_testing_test_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_test_grpc_testing_test_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   9,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_test_grpc_testing_test_proto_goTypes,
+		DependencyIndexes: file_test_grpc_testing_test_proto_depIdxs,
+		EnumInfos:         file_test_grpc_testing_test_proto_enumTypes,
+		MessageInfos:      file_test_grpc_testing_test_proto_msgTypes,
+	}.Build()
+	File_test_grpc_testing_test_proto = out.File
+	file_test_grpc_testing_test_proto_rawDesc = nil
+	file_test_grpc_testing_test_proto_goTypes = nil
+	file_test_grpc_testing_test_proto_depIdxs = nil
 }
diff --git a/test/tools/go.mod b/test/tools/go.mod
index e683f01362c9..874268d34fce 100644
--- a/test/tools/go.mod
+++ b/test/tools/go.mod
@@ -3,10 +3,10 @@ module google.golang.org/grpc/test/tools
 go 1.11
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/client9/misspell v0.3.4
-	github.com/golang/protobuf v1.3.3
+	github.com/golang/protobuf v1.4.1
 	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3
 	golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135
+	google.golang.org/protobuf v1.25.0 // indirect
 	honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc
 )
diff --git a/test/tools/go.sum b/test/tools/go.sum
index 8dd58fbbf069..09acda10d25c 100644
--- a/test/tools/go.sum
+++ b/test/tools/go.sum
@@ -1,18 +1,71 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From 0e8f1cda01321329950add4362b0fb0180f58b28 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 22 Oct 2020 13:14:27 -0700
Subject: [PATCH 241/481] manual resolver: delete
 GenerateAndRegisterManualResolver (#3960)

Tests are using WithResolver dial option.
---
 resolver/manual/manual.go | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/resolver/manual/manual.go b/resolver/manual/manual.go
index e141f96a62df..3679d702ab96 100644
--- a/resolver/manual/manual.go
+++ b/resolver/manual/manual.go
@@ -21,9 +21,6 @@
 package manual
 
 import (
-	"strconv"
-	"time"
-
 	"google.golang.org/grpc/resolver"
 )
 
@@ -81,13 +78,3 @@ func (*Resolver) Close() {}
 func (r *Resolver) UpdateState(s resolver.State) {
 	r.CC.UpdateState(s)
 }
-
-// GenerateAndRegisterManualResolver generates a random scheme and a Resolver
-// with it. It also registers this Resolver.
-// It returns the Resolver and a cleanup function to unregister it.
-func GenerateAndRegisterManualResolver() (*Resolver, func()) {
-	scheme := strconv.FormatInt(time.Now().UnixNano(), 36)
-	r := NewBuilderWithScheme(scheme)
-	resolver.Register(r)
-	return r, func() { resolver.UnregisterForTesting(scheme) }
-}

From 37b72f944a8f18ceb4dc5772ebfe95d0aea66574 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 22 Oct 2020 13:20:03 -0700
Subject: [PATCH 242/481] xdsclient: populate error details for NACK (#3975)

---
 xds/internal/client/client_xds.go         | 10 +++---
 xds/internal/client/transport_helper.go   | 17 ++++++----
 xds/internal/client/v2/client.go          | 10 ++++--
 xds/internal/client/v2/client_ack_test.go | 40 +++++++++++++++--------
 xds/internal/client/v3/client.go          | 10 ++++--
 5 files changed, 57 insertions(+), 30 deletions(-)

diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index e2a7b13f9521..0acfd69cf89b 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -47,7 +47,7 @@ func UnmarshalListener(resources []*anypb.Any, logger *grpclog.PrefixLogger) (ma
 	update := make(map[string]ListenerUpdate)
 	for _, r := range resources {
 		if !IsListenerResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in LDS response", r.GetTypeUrl())
+			return nil, fmt.Errorf("xds: unexpected resource type: %q in LDS response", r.GetTypeUrl())
 		}
 		lis := &v3listenerpb.Listener{}
 		if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
@@ -71,7 +71,7 @@ func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.
 	}
 	apiLisAny := lis.GetApiListener().GetApiListener()
 	if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
-		return "", fmt.Errorf("xds: unexpected resource type: %s in LDS response", apiLisAny.GetTypeUrl())
+		return "", fmt.Errorf("xds: unexpected resource type: %q in LDS response", apiLisAny.GetTypeUrl())
 	}
 	apiLis := &v3httppb.HttpConnectionManager{}
 	if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
@@ -108,7 +108,7 @@ func UnmarshalRouteConfig(resources []*anypb.Any, logger *grpclog.PrefixLogger)
 	update := make(map[string]RouteConfigUpdate)
 	for _, r := range resources {
 		if !IsRouteConfigResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in RDS response", r.GetTypeUrl())
+			return nil, fmt.Errorf("xds: unexpected resource type: %q in RDS response", r.GetTypeUrl())
 		}
 		rc := &v3routepb.RouteConfiguration{}
 		if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
@@ -361,7 +361,7 @@ func UnmarshalCluster(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map
 	update := make(map[string]ClusterUpdate)
 	for _, r := range resources {
 		if !IsClusterResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in CDS response", r.GetTypeUrl())
+			return nil, fmt.Errorf("xds: unexpected resource type: %q in CDS response", r.GetTypeUrl())
 		}
 
 		cluster := &v3clusterpb.Cluster{}
@@ -477,7 +477,7 @@ func UnmarshalEndpoints(resources []*anypb.Any, logger *grpclog.PrefixLogger) (m
 	update := make(map[string]EndpointsUpdate)
 	for _, r := range resources {
 		if !IsEndpointsResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %s in EDS response", r.GetTypeUrl())
+			return nil, fmt.Errorf("xds: unexpected resource type: %q in EDS response", r.GetTypeUrl())
 		}
 
 		cla := &v3endpointpb.ClusterLoadAssignment{}
diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
index 3ce1f8721b3b..607f26fd5e17 100644
--- a/xds/internal/client/transport_helper.go
+++ b/xds/internal/client/transport_helper.go
@@ -51,7 +51,7 @@ type VersionedClient interface {
 
 	// SendRequest constructs and sends out a DiscoveryRequest message specific
 	// to the underlying transport protocol version.
-	SendRequest(s grpc.ClientStream, resourceNames []string, rType ResourceType, version string, nonce string) error
+	SendRequest(s grpc.ClientStream, resourceNames []string, rType ResourceType, version, nonce, errMsg string) error
 
 	// RecvResponse uses the provided stream to receive a response specific to
 	// the underlying transport protocol version.
@@ -246,10 +246,10 @@ func (t *TransportHelper) send(ctx context.Context) {
 			t.sendCh.Load()
 
 			var (
-				target         []string
-				rType          ResourceType
-				version, nonce string
-				send           bool
+				target                 []string
+				rType                  ResourceType
+				version, nonce, errMsg string
+				send                   bool
 			)
 			switch update := u.(type) {
 			case *watchAction:
@@ -259,6 +259,7 @@ func (t *TransportHelper) send(ctx context.Context) {
 				if !send {
 					continue
 				}
+				errMsg = update.errMsg
 			}
 			if stream == nil {
 				// There's no stream yet. Skip the request. This request
@@ -267,7 +268,7 @@ func (t *TransportHelper) send(ctx context.Context) {
 				// sending response back).
 				continue
 			}
-			if err := t.vClient.SendRequest(stream, target, rType, version, nonce); err != nil {
+			if err := t.vClient.SendRequest(stream, target, rType, version, nonce, errMsg); err != nil {
 				t.logger.Warningf("ADS request for {target: %q, type: %v, version: %q, nonce: %q} failed: %v", target, rType, version, nonce, err)
 				// send failed, clear the current stream.
 				stream = nil
@@ -292,7 +293,7 @@ func (t *TransportHelper) sendExisting(stream grpc.ClientStream) bool {
 	t.nonceMap = make(map[ResourceType]string)
 
 	for rType, s := range t.watchMap {
-		if err := t.vClient.SendRequest(stream, mapToSlice(s), rType, "", ""); err != nil {
+		if err := t.vClient.SendRequest(stream, mapToSlice(s), rType, "", "", ""); err != nil {
 			t.logger.Errorf("ADS request failed: %v", err)
 			return false
 		}
@@ -321,6 +322,7 @@ func (t *TransportHelper) recv(stream grpc.ClientStream) bool {
 				rType:   rType,
 				version: "",
 				nonce:   nonce,
+				errMsg:  err.Error(),
 				stream:  stream,
 			})
 			t.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", rType, version, nonce, err)
@@ -387,6 +389,7 @@ type ackAction struct {
 	rType   ResourceType
 	version string // NACK if version is an empty string.
 	nonce   string
+	errMsg  string // Empty unless it's a NACK.
 	// ACK/NACK are tagged with the stream it's for. When the stream is down,
 	// all the ACK/NACK for this stream will be dropped, and the version/nonce
 	// won't be updated.
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index 674bba4798f9..7b063ad4f559 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
@@ -33,6 +34,7 @@ import (
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
+	statuspb "google.golang.org/genproto/googleapis/rpc/status"
 )
 
 func init() {
@@ -106,7 +108,7 @@ func (v2c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
 // - If this is an ack, version will be the version from the response.
 // - If this is a nack, version will be the previous acked version (from
 //   versionMap). If there was no ack before, it will be empty.
-func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce string) error {
+func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce, errMsg string) error {
 	stream, ok := s.(adsStream)
 	if !ok {
 		return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s)
@@ -117,7 +119,11 @@ func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rTyp
 		ResourceNames: resourceNames,
 		VersionInfo:   version,
 		ResponseNonce: nonce,
-		// TODO: populate ErrorDetails for nack.
+	}
+	if errMsg != "" {
+		req.ErrorDetail = &statuspb.Status{
+			Code: int32(codes.InvalidArgument), Message: errMsg,
+		}
 	}
 	if err := stream.Send(req); err != nil {
 		return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err)
diff --git a/xds/internal/client/v2/client_ack_test.go b/xds/internal/client/v2/client_ack_test.go
index 2b38d3aa3709..87437aa218c7 100644
--- a/xds/internal/client/v2/client_ack_test.go
+++ b/xds/internal/client/v2/client_ack_test.go
@@ -29,6 +29,7 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
@@ -73,7 +74,7 @@ func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cb
 }
 
 // compareXDSRequest reads requests from channel, compare it with want.
-func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string) error {
+func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string, wantErr bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	val, err := ch.Receive(ctx)
@@ -84,11 +85,22 @@ func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver,
 	if req.Err != nil {
 		return fmt.Errorf("unexpected error from request: %v", req.Err)
 	}
+
+	xdsReq := req.Req.(*xdspb.DiscoveryRequest)
+	if (xdsReq.ErrorDetail != nil) != wantErr {
+		return fmt.Errorf("received request with error details: %v, wantErr: %v", xdsReq.ErrorDetail, wantErr)
+	}
+	// All NACK request.ErrorDetails have hardcoded status code InvalidArguments.
+	if xdsReq.ErrorDetail != nil && xdsReq.ErrorDetail.Code != int32(codes.InvalidArgument) {
+		return fmt.Errorf("received request with error details: %v, want status with code: %v", xdsReq.ErrorDetail, codes.InvalidArgument)
+	}
+
+	xdsReq.ErrorDetail = nil // Clear the error details field before comparing.
 	wantClone := proto.Clone(want).(*xdspb.DiscoveryRequest)
 	wantClone.VersionInfo = ver
 	wantClone.ResponseNonce = nonce
-	if !cmp.Equal(req.Req, wantClone, cmp.Comparer(proto.Equal)) {
-		return fmt.Errorf("received request different from want, diff: %s", cmp.Diff(req.Req, wantClone))
+	if !cmp.Equal(xdsReq, wantClone, cmp.Comparer(proto.Equal)) {
+		return fmt.Errorf("received request different from want, diff: %s", cmp.Diff(req.Req, wantClone, cmp.Comparer(proto.Equal)))
 	}
 	return nil
 }
@@ -118,7 +130,7 @@ func startXDS(t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *
 	}
 	v2c.AddWatch(rType, nameToWatch)
 
-	if err := compareXDSRequest(reqChan, req, preVersion, preNonce); err != nil {
+	if err := compareXDSRequest(reqChan, req, preVersion, preNonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", rType, err)
 	}
 	t.Logf("FakeServer received %v request...", rType)
@@ -133,7 +145,7 @@ func sendGoodResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakese
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, ver)
 	t.Logf("Good %v response pushed to fakeServer...", rType)
 
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce, false); err != nil {
 		return "", fmt.Errorf("failed to receive %v request: %v", rType, err)
 	}
 	t.Logf("Good %v response acked", rType)
@@ -168,7 +180,7 @@ func sendBadResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeser
 		TypeUrl:   typeURL,
 	}, ver)
 	t.Logf("Bad %v response pushed to fakeServer...", rType)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce, true); err != nil {
 		return fmt.Errorf("failed to receive %v request: %v", rType, err)
 	}
 	t.Logf("Bad %v response nacked", rType)
@@ -274,7 +286,7 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 
 	// The expected version string is an empty string, because this is the first
 	// response, and it's nacked (so there's no previous ack version).
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, "", nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, "", nonce, true); err != nil {
 		t.Errorf("Failed to receive request: %v", err)
 	}
 	t.Logf("Bad response nacked")
@@ -314,7 +326,7 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	t.Logf("Bad response pushed to fakeServer...")
 
 	// The expected version string is the previous acked version.
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS-1), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS-1), nonce, true); err != nil {
 		t.Errorf("Failed to receive request: %v", err)
 	}
 	t.Logf("Bad response nacked")
@@ -339,7 +351,7 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 
 	// Start a CDS watch.
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", "", false); err != nil {
 		t.Fatal(err)
 	}
 	t.Logf("FakeServer received %v request...", xdsclient.ClusterResource)
@@ -356,12 +368,12 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
 	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
 	// Wait for a request with correct resource names and version.
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	versionCDS++
@@ -394,7 +406,7 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 
 	// Start a CDS watch.
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", "", false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	t.Logf("FakeServer received %v request...", xdsclient.ClusterResource)
@@ -410,7 +422,7 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
 	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	versionCDS++
@@ -440,7 +452,7 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	// Start a new watch. The new watch should have the nonce from the response
 	// above, and version from the first good response.
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce); err != nil {
+	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 328cd8b9cbea..5d8d7198ce2d 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -24,7 +24,9 @@ import (
 	"fmt"
 
 	"github.com/golang/protobuf/proto"
+	statuspb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
@@ -106,7 +108,7 @@ func (v3c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) {
 // - If this is an ack, version will be the version from the response.
 // - If this is a nack, version will be the previous acked version (from
 //   versionMap). If there was no ack before, it will be empty.
-func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce string) error {
+func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce, errMsg string) error {
 	stream, ok := s.(adsStream)
 	if !ok {
 		return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s)
@@ -117,7 +119,11 @@ func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rTyp
 		ResourceNames: resourceNames,
 		VersionInfo:   version,
 		ResponseNonce: nonce,
-		// TODO: populate ErrorDetails for nack.
+	}
+	if errMsg != "" {
+		req.ErrorDetail = &statuspb.Status{
+			Code: int32(codes.InvalidArgument), Message: errMsg,
+		}
 	}
 	if err := stream.Send(req); err != nil {
 		return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err)

From eb7fc22e4562764adc369346aef9ce81180ebdb7 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 22 Oct 2020 13:37:57 -0700
Subject: [PATCH 243/481] credentials/xds: Handle no acceptedSANs correctly.
 (#3965)

---
 credentials/xds/xds.go      |  5 +++++
 credentials/xds/xds_test.go | 36 ++++++++++++++++++++++--------------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go
index cecc27d14559..e3f73d0757e8 100644
--- a/credentials/xds/xds.go
+++ b/credentials/xds/xds.go
@@ -174,6 +174,11 @@ func (hi *HandshakeInfo) makeTLSConfig(ctx context.Context) (*tls.Config, error)
 }
 
 func (hi *HandshakeInfo) matchingSANExists(cert *x509.Certificate) bool {
+	if len(hi.acceptedSANs) == 0 {
+		// An empty list of acceptedSANs means "accept everything".
+		return true
+	}
+
 	var sans []string
 	// SANs can be specified in any of these four fields on the parsed cert.
 	sans = append(sans, cert.DNSNames...)
diff --git a/credentials/xds/xds_test.go b/credentials/xds/xds_test.go
index a2adcf4558ea..18aa86c9cdda 100644
--- a/credentials/xds/xds_test.go
+++ b/credentials/xds/xds_test.go
@@ -358,26 +358,37 @@ func (s) TestClientCredsSuccess(t *testing.T) {
 	tests := []struct {
 		desc             string
 		handshakeFunc    testHandshakeFunc
-		rootProvider     certprovider.Provider
-		identityProvider certprovider.Provider
+		handshakeInfoCtx func(ctx context.Context) context.Context
 	}{
 		{
-			// Since we don't specify rootProvider and identityProvider here,
-			// the test does not add a HandshakeInfo context value, and thereby
-			// the ClientHandshake() method will delegate to the fallback.
 			desc:          "fallback",
 			handshakeFunc: testServerTLSHandshake,
+			handshakeInfoCtx: func(ctx context.Context) context.Context {
+				// Since we don't add a HandshakeInfo to the context, the
+				// ClientHandshake() method will delegate to the fallback.
+				return ctx
+			},
 		},
 		{
 			desc:          "TLS",
 			handshakeFunc: testServerTLSHandshake,
-			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
+			handshakeInfoCtx: func(ctx context.Context) context.Context {
+				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
+			},
 		},
 		{
-			desc:             "mTLS",
-			handshakeFunc:    testServerMutualTLSHandshake,
-			rootProvider:     makeRootProvider(t, "x509/server_ca_cert.pem"),
-			identityProvider: makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"),
+			desc:          "mTLS",
+			handshakeFunc: testServerMutualTLSHandshake,
+			handshakeInfoCtx: func(ctx context.Context) context.Context {
+				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), defaultTestCertSAN)
+			},
+		},
+		{
+			desc:          "mTLS with no acceptedSANs specified",
+			handshakeFunc: testServerMutualTLSHandshake,
+			handshakeInfoCtx: func(ctx context.Context) context.Context {
+				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
+			},
 		},
 	}
 
@@ -400,10 +411,7 @@ func (s) TestClientCredsSuccess(t *testing.T) {
 
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
-			if test.rootProvider != nil || test.identityProvider != nil {
-				ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider, defaultTestCertSAN)
-			}
-			_, ai, err := creds.ClientHandshake(ctx, authority, conn)
+			_, ai, err := creds.ClientHandshake(test.handshakeInfoCtx(ctx), authority, conn)
 			if err != nil {
 				t.Fatalf("ClientHandshake() returned failed: %q", err)
 			}

From 3e67a200ebbf773f7f0715852864c542e408d596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=A1rk=20S=C3=A1gi-Kaz=C3=A1r?=
 <sagikazarmark@users.noreply.github.com>
Date: Fri, 23 Oct 2020 22:09:56 +0200
Subject: [PATCH 244/481] Add GitHub release action (#3962)

---
 .github/workflows/release.yml | 61 +++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 000000000000..6c0748048cc9
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,61 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release:
+    name: Release cmd/protoc-gen-go-grpc
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.release.tag_name, 'cmd/protoc-gen-go-grpc/')
+    strategy:
+      matrix:
+        goos: [linux, darwin, windows]
+        goarch: [386, amd64]
+        exclude:
+          - goos: darwin
+            goarch: 386
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+
+      - name: Download dependencies
+        run: |
+          cd cmd/protoc-gen-go-grpc
+          go mod download
+
+      - name: Prepare build directory
+        run: |
+          mkdir -p build/
+          cp README.md build/
+          cp LICENSE build/
+
+      - name: Build
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+        run: |
+          cd cmd/protoc-gen-go-grpc
+          go build -trimpath -o $GITHUB_WORKSPACE/build
+
+      - name: Create package
+        id: package
+        run: |
+          PACKAGE_NAME=protoc-gen-go-grpc.${GITHUB_REF#refs/tags/cmd/protoc-gen-go-grpc/}.${{ matrix.goos }}.${{ matrix.goarch }}.tar.gz
+          tar -czvf $PACKAGE_NAME -C build .
+          echo ::set-output name=name::${PACKAGE_NAME}
+
+      - name: Upload asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ./${{ steps.package.outputs.name }}
+          asset_name: ${{ steps.package.outputs.name }}
+          asset_content_type: application/gzip

From a80e65018272ac6bd58ee44d0d3e37374b090c1f Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 23 Oct 2020 13:18:32 -0700
Subject: [PATCH 245/481] cmd/protoc-gen-go-grpc: bump -version to 1.0.1 for
 release (#3984)

---
 cmd/protoc-gen-go-grpc/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index 9ef889fa470e..7e22561745e9 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -38,7 +38,7 @@ import (
 	"google.golang.org/protobuf/types/pluginpb"
 )
 
-const version = "1.0"
+const version = "1.0.1"
 
 var requireUnimplemented *bool
 

From e8e22874d164e14311334b4c118d11fd311f915e Mon Sep 17 00:00:00 2001
From: Igor Zibarev <hypnoglow@gmail.com>
Date: Mon, 26 Oct 2020 17:33:18 +0300
Subject: [PATCH 246/481] Remove experimental comment from client interceptors
 (#3948)

---
 interceptor.go | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/interceptor.go b/interceptor.go
index d1a609e4893d..668e0adcf0a9 100644
--- a/interceptor.go
+++ b/interceptor.go
@@ -25,25 +25,41 @@ import (
 // UnaryInvoker is called by UnaryClientInterceptor to complete RPCs.
 type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error
 
-// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. invoker is the handler to complete the RPC
-// and it is the responsibility of the interceptor to call it.
+// UnaryClientInterceptor intercepts the execution of a unary RPC on the client.
+// Unary interceptors can be specified as a DialOption, using
+// WithUnaryInterceptor() or WithChainUnaryInterceptor(), when creating a
+// ClientConn. When a unary interceptor(s) is set on a ClientConn, gRPC
+// delegates all unary RPC invocations to the interceptor, and it is the
+// responsibility of the interceptor to call invoker to complete the processing
+// of the RPC.
 //
-// Experimental
+// method is the RPC name. req and reply are the corresponding request and
+// response messages. cc is the ClientConn on which the RPC was invoked. invoker
+// is the handler to complete the RPC and it is the responsibility of the
+// interceptor to call it. opts contain all applicable call options, including
+// defaults from the ClientConn as well as per-call options.
 //
-// Notice: This type is EXPERIMENTAL and may be changed or removed in a
-// later release.
+// The returned error must be compatible with the status package.
 type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error
 
 // Streamer is called by StreamClientInterceptor to create a ClientStream.
 type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error)
 
-// StreamClientInterceptor intercepts the creation of ClientStream. It may return a custom ClientStream to intercept all I/O
-// operations. streamer is the handler to create a ClientStream and it is the responsibility of the interceptor to call it.
+// StreamClientInterceptor intercepts the creation of a ClientStream. Stream
+// interceptors can be specified as a DialOption, using WithStreamInterceptor()
+// or WithChainStreamInterceptor(), when creating a ClientConn. When a stream
+// interceptor(s) is set on the ClientConn, gRPC delegates all stream creations
+// to the interceptor, and it is the responsibility of the interceptor to call
+// streamer.
 //
-// Experimental
+// desc contains a description of the stream. cc is the ClientConn on which the
+// RPC was invoked. streamer is the handler to create a ClientStream and it is
+// the responsibility of the interceptor to call it. opts contain all applicable
+// call options, including defaults from the ClientConn as well as per-call
+// options.
 //
-// Notice: This type is EXPERIMENTAL and may be changed or removed in a
-// later release.
+// StreamClientInterceptor may return a custom ClientStream to intercept all I/O
+// operations. The returned error must be compatible with the status package.
 type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error)
 
 // UnaryServerInfo consists of various information about a unary RPC on

From a223251d8bced825764bf1e450b9840fd9227195 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 26 Oct 2020 15:47:47 -0700
Subject: [PATCH 247/481] lrs: handle multiple clusters in LRS stream (#3935)

---
 xds/internal/balancer/edsbalancer/eds.go      |   4 +-
 .../balancer/edsbalancer/eds_impl_test.go     |   5 +-
 .../edsbalancer/xds_client_wrapper.go         |  82 ++++++----
 .../balancer/edsbalancer/xds_lrs_test.go      |   4 +-
 xds/internal/balancer/lrs/balancer.go         | 102 +++++++-----
 xds/internal/balancer/lrs/balancer_test.go    |   4 +-
 xds/internal/client/client.go                 |  37 +++--
 xds/internal/client/client_loadreport.go      | 139 +++++++++++-----
 xds/internal/client/client_loadreport_test.go | 148 ++++++++++++++++++
 xds/internal/client/client_test.go            |   2 +-
 xds/internal/client/transport_helper.go       |  27 ++--
 xds/internal/client/v2/client.go              |   3 -
 xds/internal/client/v2/loadreport.go          |  57 +++----
 xds/internal/client/v3/client.go              |   3 -
 xds/internal/client/v3/loadreport.go          |  57 +++----
 xds/internal/testutils/fakeclient/client.go   |   8 +-
 xds/internal/testutils/fakeserver/server.go   |   4 +-
 17 files changed, 451 insertions(+), 235 deletions(-)
 create mode 100644 xds/internal/client/client_loadreport_test.go

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 6b124c27071d..a148db0e42e6 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -177,7 +177,9 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			return
 		}
 
-		x.client.handleUpdate(cfg, u.ResolverState.Attributes)
+		if err := x.client.handleUpdate(cfg, u.ResolverState.Attributes); err != nil {
+			x.logger.Warningf("failed to update xds clients: %v", err)
+		}
 
 		if x.config == nil {
 			x.config = cfg
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index a56acb7e293d..292ea4f80a70 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -688,9 +688,10 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// be used.
 	loadStore := load.NewStore()
 	lsWrapper := &loadStoreWrapper{}
-	lsWrapper.update(loadStore, testClusterNames[0])
+	lsWrapper.updateServiceName(testClusterNames[0])
+	lsWrapper.updateLoadStore(loadStore)
 	cw := &xdsClientWrapper{
-		load: lsWrapper,
+		loadWrapper: lsWrapper,
 	}
 
 	cc := testutils.NewTestClientConn(t)
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index f22c1624a3b6..fe4e996a7b58 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -19,6 +19,7 @@
 package edsbalancer
 
 import (
+	"fmt"
 	"sync"
 
 	"google.golang.org/grpc"
@@ -35,8 +36,7 @@ import (
 // balancer. It's defined so we can override xdsclientNew function in tests.
 type xdsClientInterface interface {
 	WatchEndpoints(clusterName string, edsCb func(xdsclient.EndpointsUpdate, error)) (cancel func())
-	LoadStore() *load.Store
-	ReportLoad(server string, clusterName string) (cancel func())
+	ReportLoad(server string) (loadStore *load.Store, cancel func())
 	Close()
 }
 
@@ -48,23 +48,37 @@ var (
 )
 
 type loadStoreWrapper struct {
-	mu         sync.RWMutex
+	mu      sync.RWMutex
+	service string
+	// Both store and perCluster will be nil if load reporting is disabled (EDS
+	// response doesn't have LRS server name). Note that methods on Store and
+	// perCluster all handle nil, so there's no need to check nil before calling
+	// them.
 	store      *load.Store
-	service    string
 	perCluster load.PerClusterReporter
 }
 
-func (lsw *loadStoreWrapper) update(store *load.Store, service string) {
+func (lsw *loadStoreWrapper) updateServiceName(service string) {
 	lsw.mu.Lock()
 	defer lsw.mu.Unlock()
-	if store == lsw.store && service == lsw.service {
+	if lsw.service == service {
 		return
 	}
-	lsw.store = store
 	lsw.service = service
 	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
 }
 
+func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	lsw.perCluster = nil
+	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
+}
+
 func (lsw *loadStoreWrapper) CallStarted(locality string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
@@ -102,7 +116,9 @@ type xdsClientWrapper struct {
 	// xdsClient could come from attributes, or created with balancerName.
 	xdsClient xdsClientInterface
 
-	load *loadStoreWrapper
+	// loadWrapper is a wrapper with loadOriginal, with clusterName and
+	// edsServiceName. It's used children to report loads.
+	loadWrapper *loadStoreWrapper
 	// edsServiceName is the edsServiceName currently being watched, not
 	// necessary the edsServiceName from service config.
 	//
@@ -127,7 +143,7 @@ func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bb
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
 		bbo:          bbo,
-		load:         &loadStoreWrapper{},
+		loadWrapper:  &loadStoreWrapper{},
 	}
 }
 
@@ -168,12 +184,12 @@ func (c *xdsClientWrapper) replaceXDSClient(newClient xdsClientInterface, newBal
 // the balancerName (from bootstrap file or from service config) changed.
 // - if balancer names are the same, do nothing, and return false
 // - if balancer names are different, create new one, and return true
-func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.Attributes) bool {
+func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.Attributes) (bool, error) {
 	if attr != nil {
 		if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil {
 			// This will also clear balancerName, to indicate that client is
 			// from attributes.
-			return c.replaceXDSClient(clientFromAttr, "")
+			return c.replaceXDSClient(clientFromAttr, ""), nil
 		}
 	}
 
@@ -184,7 +200,7 @@ func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 	}
 
 	if c.balancerName == clientConfig.BalancerName {
-		return false
+		return false, nil
 	}
 
 	var dopts []grpc.DialOption
@@ -192,16 +208,20 @@ func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 		dopts = []grpc.DialOption{grpc.WithContextDialer(dialer)}
 	}
 
+	// TODO: there's no longer a need to read bootstrap file and create a new
+	// xds client. The EDS balancer should always get the xds client from
+	// attributes. Otherwise, this function should just fail. Also, xdsclient
+	// will be shared by multiple clients, so trying to make an xds client is
+	// just the wrong move.
 	newClient, err := xdsclientNew(xdsclient.Options{Config: *clientConfig, DialOpts: dopts})
 	if err != nil {
 		// This should never fail. xdsclientnew does a non-blocking dial, and
 		// all the config passed in should be validated.
 		//
 		// This could leave c.xdsClient as nil if this is the first update.
-		c.logger.Warningf("eds: failed to create xdsClient, error: %v", err)
-		return false
+		return false, fmt.Errorf("eds: failed to create xdsClient, error: %v", err)
 	}
-	return c.replaceXDSClient(newClient, clientConfig.BalancerName)
+	return c.replaceXDSClient(newClient, clientConfig.BalancerName), nil
 }
 
 // startEndpointsWatch starts the EDS watch. Caller can call this when the
@@ -214,10 +234,6 @@ func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.A
 // This usually means load report needs to be restarted, but this function does
 // NOT do that. Caller needs to call startLoadReport separately.
 func (c *xdsClientWrapper) startEndpointsWatch() {
-	if c.xdsClient == nil {
-		return
-	}
-
 	if c.cancelEndpointsWatch != nil {
 		c.cancelEndpointsWatch()
 	}
@@ -238,31 +254,32 @@ func (c *xdsClientWrapper) startEndpointsWatch() {
 // Caller can cal this when the loadReportServer name changes, but
 // edsServiceName doesn't (so we only need to restart load reporting, not EDS
 // watch).
-func (c *xdsClientWrapper) startLoadReport(loadReportServer *string) {
-	if c.xdsClient == nil {
-		c.logger.Warningf("xds: xdsClient is nil when trying to start load reporting. This means xdsClient wasn't passed in from the resolver, and xdsClient.New failed")
-		return
-	}
+func (c *xdsClientWrapper) startLoadReport(loadReportServer *string) *load.Store {
 	if c.cancelLoadReport != nil {
 		c.cancelLoadReport()
 	}
 	c.loadReportServer = loadReportServer
+	var loadStore *load.Store
 	if c.loadReportServer != nil {
-		c.cancelLoadReport = c.xdsClient.ReportLoad(*c.loadReportServer, c.edsServiceName)
+		loadStore, c.cancelLoadReport = c.xdsClient.ReportLoad(*c.loadReportServer)
 	}
+	return loadStore
 }
 
 func (c *xdsClientWrapper) loadStore() load.PerClusterReporter {
-	if c == nil || c.load.store == nil {
+	if c == nil {
 		return nil
 	}
-	return c.load
+	return c.loadWrapper
 }
 
 // handleUpdate applies the service config and attributes updates to the client,
 // including updating the xds_client to use, and updating the EDS name to watch.
-func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) {
-	clientChanged := c.updateXDSClient(config, attr)
+func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) error {
+	clientChanged, err := c.updateXDSClient(config, attr)
+	if err != nil {
+		return err
+	}
 
 	// Need to restart EDS watch when one of the following happens:
 	// - the xds_client is updated
@@ -277,14 +294,17 @@ func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attr
 		//
 		// This is OK for now, because we don't actually expect edsServiceName
 		// to change. Fix this (a bigger change) will happen later.
-		c.load.update(c.xdsClient.LoadStore(), c.edsServiceName)
+		c.loadWrapper.updateServiceName(c.edsServiceName)
 	}
 
 	// Only need to restart load reporting when:
 	// - the loadReportServer name changed
 	if !equalStringPointers(c.loadReportServer, config.LrsLoadReportingServerName) {
-		c.startLoadReport(config.LrsLoadReportingServerName)
+		loadStore := c.startLoadReport(config.LrsLoadReportingServerName)
+		c.loadWrapper.updateLoadStore(loadStore)
 	}
+
+	return nil
 }
 
 func (c *xdsClientWrapper) cancelWatch() {
diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
index 8d888ec6f3b0..955f54401c86 100644
--- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
@@ -66,7 +66,7 @@ func (s) TestXDSLoadReporting(t *testing.T) {
 	if err != nil {
 		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
 	}
-	if got.Server != "" || got.Cluster != testEDSClusterName {
-		t.Fatalf("xdsClient.ReportLoad called with {%v, %v}: want {\"\", %v}", got.Server, got.Cluster, testEDSClusterName)
+	if got.Server != "" {
+		t.Fatalf("xdsClient.ReportLoad called with {%v}: want {\"\"}", got.Server)
 	}
 }
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index 1361fb15728f..f8e7673f7d8e 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -80,7 +80,9 @@ func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	// Update load reporting config or xds client. This needs to be done before
 	// updating the child policy because we need the loadStore from the updated
 	// client to be passed to the ccWrapper.
-	b.client.update(newConfig, s.ResolverState.Attributes)
+	if err := b.client.update(newConfig, s.ResolverState.Attributes); err != nil {
+		return err
+	}
 
 	// If child policy is a different type, recreate the sub-balancer.
 	if b.config == nil || b.config.ChildPolicy.Name != newConfig.ChildPolicy.Name {
@@ -144,31 +146,44 @@ func (ccw *ccWrapper) UpdateState(s balancer.State) {
 // xdsClientInterface contains only the xds_client methods needed by LRS
 // balancer. It's defined so we can override xdsclient in tests.
 type xdsClientInterface interface {
-	LoadStore() *load.Store
-	ReportLoad(server string, clusterName string) func()
+	ReportLoad(server string) (*load.Store, func())
 	Close()
 }
 
 type loadStoreWrapper struct {
 	mu         sync.RWMutex
-	store      *load.Store
 	cluster    string
 	edsService string
+	// Both store and perCluster will be nil if load reporting is disabled (EDS
+	// response doesn't have LRS server name). Note that methods on Store and
+	// perCluster all handle nil, so there's no need to check nil before calling
+	// them.
+	store      *load.Store
 	perCluster load.PerClusterReporter
 }
 
-func (lsw *loadStoreWrapper) update(store *load.Store, cluster, edsService string) {
+func (lsw *loadStoreWrapper) updateClusterAndService(cluster, edsService string) {
 	lsw.mu.Lock()
 	defer lsw.mu.Unlock()
-	if store == lsw.store && cluster == lsw.cluster && edsService == lsw.edsService {
+	if cluster == lsw.cluster && edsService == lsw.edsService {
 		return
 	}
-	lsw.store = store
 	lsw.cluster = cluster
 	lsw.edsService = edsService
 	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
 }
 
+func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	lsw.perCluster = nil
+	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
+}
+
 func (lsw *loadStoreWrapper) CallStarted(locality string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
@@ -199,44 +214,62 @@ type xdsClientWrapper struct {
 	clusterName      string
 	edsServiceName   string
 	lrsServerName    string
-	load             *loadStoreWrapper
+	// loadWrapper is a wrapper with loadOriginal, with clusterName and
+	// edsServiceName. It's used children to report loads.
+	loadWrapper *loadStoreWrapper
 }
 
 func newXDSClientWrapper() *xdsClientWrapper {
 	return &xdsClientWrapper{
-		load: &loadStoreWrapper{},
+		loadWrapper: &loadStoreWrapper{},
 	}
 }
 
 // update checks the config and xdsclient, and decides whether it needs to
 // restart the load reporting stream.
-func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attributes) {
+func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attributes) error {
 	var (
-		restartLoadReport bool
-		updateLoadStore   bool
+		restartLoadReport           bool
+		updateLoadClusterAndService bool
 	)
-	if attr != nil {
-		if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil {
-			if w.c != clientFromAttr {
-				// xds client is different, restart.
-				restartLoadReport = true
-				updateLoadStore = true
-				w.c = clientFromAttr
-			}
-		}
+
+	if attr == nil {
+		return fmt.Errorf("lrs: failed to get xdsClient from attributes: attributes is nil")
+	}
+	clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface)
+	if clientFromAttr == nil {
+		return fmt.Errorf("lrs: failed to get xdsClient from attributes: xdsClient not found in attributes")
+	}
+
+	if w.c != clientFromAttr {
+		// xds client is different, restart.
+		restartLoadReport = true
+		w.c = clientFromAttr
 	}
 
 	// ClusterName is different, restart. ClusterName is from ClusterName and
 	// EdsServiceName.
 	if w.clusterName != newConfig.ClusterName {
-		updateLoadStore = true
+		updateLoadClusterAndService = true
 		w.clusterName = newConfig.ClusterName
 	}
 	if w.edsServiceName != newConfig.EdsServiceName {
-		updateLoadStore = true
+		updateLoadClusterAndService = true
 		w.edsServiceName = newConfig.EdsServiceName
 	}
 
+	if updateLoadClusterAndService {
+		// This updates the clusterName and serviceName that will reported for the
+		// loads. The update here is too early, the perfect timing is when the
+		// picker is updated with the new connection. But from this balancer's point
+		// of view, it's impossible to tell.
+		//
+		// On the other hand, this will almost never happen. Each LRS policy
+		// shouldn't get updated config. The parent should do a graceful switch when
+		// the clusterName or serviceName is changed.
+		w.loadWrapper.updateClusterAndService(w.clusterName, w.edsServiceName)
+	}
+
 	if w.lrsServerName != newConfig.LrsLoadReportingServerName {
 		// LrsLoadReportingServerName is different, load should be report to a
 		// different server, restart.
@@ -244,34 +277,23 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attribut
 		w.lrsServerName = newConfig.LrsLoadReportingServerName
 	}
 
-	// This updates the clusterName and serviceName that will reported for the
-	// loads. The update here is too early, the perfect timing is when the
-	// picker is updated with the new connection. But from this balancer's point
-	// of view, it's impossible to tell.
-	//
-	// On the other hand, this will almost never happen. Each LRS policy
-	// shouldn't get updated config. The parent should do a graceful switch when
-	// the clusterName or serviceName is changed.
-	if updateLoadStore {
-		w.load.update(w.c.LoadStore(), w.clusterName, w.edsServiceName)
-	}
-
 	if restartLoadReport {
 		if w.cancelLoadReport != nil {
 			w.cancelLoadReport()
 			w.cancelLoadReport = nil
 		}
+		var loadStore *load.Store
 		if w.c != nil {
-			w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName, w.clusterName)
+			loadStore, w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName)
 		}
+		w.loadWrapper.updateLoadStore(loadStore)
 	}
+
+	return nil
 }
 
 func (w *xdsClientWrapper) loadStore() load.PerClusterReporter {
-	if w.load.store == nil {
-		return nil
-	}
-	return w.load
+	return w.loadWrapper
 }
 
 func (w *xdsClientWrapper) close() {
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index 789cfea0c00a..38dd573ef14b 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -84,8 +84,8 @@ func TestLoadReporting(t *testing.T) {
 	if err != nil {
 		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
 	}
-	if got.Server != testLRSServerName || got.Cluster != testClusterName {
-		t.Fatalf("xdsClient.ReportLoad called with {%q, %q}: want {%q, %q}", got.Server, got.Cluster, testLRSServerName, testClusterName)
+	if got.Server != testLRSServerName {
+		t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerName)
 	}
 
 	sc1 := <-cc.NewSubConnCh
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 0d61963e1b79..347856927976 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -30,6 +30,7 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc/xds/internal/client/load"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/backoff"
@@ -39,7 +40,6 @@ import (
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -78,9 +78,6 @@ type BuildOptions struct {
 	// Backoff returns the amount of time to backoff before retrying broken
 	// streams.
 	Backoff func(int) time.Duration
-	// LoadStore contains load reports which need to be pushed to the management
-	// server.
-	LoadStore *load.Store
 	// Logger provides enhanced logging capabilities.
 	Logger *grpclog.PrefixLogger
 }
@@ -99,6 +96,12 @@ type APIClientBuilder interface {
 
 // APIClient represents the functionality provided by transport protocol
 // version specific implementations of the xDS client.
+//
+// TODO: unexport this interface and all the methods after the PR to make
+// xdsClient sharable by clients. AddWatch and RemoveWatch are exported for
+// v2/v3 to override because they need to keep track of LDS name for RDS to use.
+// After the share xdsClient change, that's no longer necessary. After that, we
+// will still keep this interface for testing purposes.
 type APIClient interface {
 	// AddWatch adds a watch for an xDS resource given its type and name.
 	AddWatch(ResourceType, string)
@@ -107,21 +110,18 @@ type APIClient interface {
 	// given its type and name.
 	RemoveWatch(ResourceType, string)
 
-	// ReportLoad starts an LRS stream to periodically report load using the
+	// reportLoad starts an LRS stream to periodically report load using the
 	// provided ClientConn, which represent a connection to the management
 	// server.
-	ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions)
+	reportLoad(ctx context.Context, cc *grpc.ClientConn, opts loadReportingOptions)
 
 	// Close cleans up resources allocated by the API client.
 	Close()
 }
 
-// LoadReportingOptions contains configuration knobs for reporting load data.
-type LoadReportingOptions struct {
-	// ClusterName is the cluster name for which load is being reported.
-	ClusterName string
-	// TargetName is the target of the parent ClientConn.
-	TargetName string
+// loadReportingOptions contains configuration knobs for reporting load data.
+type loadReportingOptions struct {
+	loadStore *load.Store
 }
 
 // UpdateHandler receives and processes (by taking appropriate actions) xDS
@@ -328,7 +328,6 @@ type Client struct {
 	opts      Options
 	cc        *grpc.ClientConn // Connection to the xDS server
 	apiClient APIClient
-	loadStore *load.Store
 
 	logger *grpclog.PrefixLogger
 
@@ -342,6 +341,11 @@ type Client struct {
 	cdsCache    map[string]ClusterUpdate
 	edsWatchers map[string]map[*watchInfo]bool
 	edsCache    map[string]EndpointsUpdate
+
+	// Changes to map lrsClients and the lrsClient inside the map need to be
+	// protected by lrsMu.
+	lrsMu      sync.Mutex
+	lrsClients map[string]*lrsClient
 }
 
 // New returns a new xdsClient configured with opts.
@@ -382,9 +386,8 @@ func New(opts Options) (*Client, error) {
 	}
 
 	c := &Client{
-		done:      grpcsync.NewEvent(),
-		opts:      opts,
-		loadStore: load.NewStore(),
+		done: grpcsync.NewEvent(),
+		opts: opts,
 
 		updateCh:    buffer.NewUnbounded(),
 		ldsWatchers: make(map[string]map[*watchInfo]bool),
@@ -395,6 +398,7 @@ func New(opts Options) (*Client, error) {
 		cdsCache:    make(map[string]ClusterUpdate),
 		edsWatchers: make(map[string]map[*watchInfo]bool),
 		edsCache:    make(map[string]EndpointsUpdate),
+		lrsClients:  make(map[string]*lrsClient),
 	}
 
 	cc, err := grpc.Dial(opts.Config.BalancerName, dopts...)
@@ -410,7 +414,6 @@ func New(opts Options) (*Client, error) {
 		Parent:    c,
 		NodeProto: opts.Config.NodeProto,
 		Backoff:   backoff.DefaultExponential.Backoff,
-		LoadStore: c.loadStore,
 		Logger:    c.logger,
 	})
 	if err != nil {
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go
index e52c3b93f90e..e91316b9fe48 100644
--- a/xds/internal/client/client_loadreport.go
+++ b/xds/internal/client/client_loadreport.go
@@ -24,53 +24,116 @@ import (
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
-// NodeMetadataHostnameKey is the metadata key for specifying the target name in
-// the node proto of an LRS request.
-const NodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME"
+// ReportLoad starts an load reporting stream to the given server. If the server
+// is not an empty string, and is different from the xds server, a new
+// ClientConn will be created.
+//
+// The same options used for creating the Client will be used (including
+// NodeProto, and dial options if necessary).
+//
+// It returns a Store for the user to report loads, a function to cancel the
+// load reporting stream.
+func (c *Client) ReportLoad(server string) (*load.Store, func()) {
+	c.lrsMu.Lock()
+	defer c.lrsMu.Unlock()
+
+	// If there's already a client to this server, use it. Otherwise, create
+	// one.
+	lrsC, ok := c.lrsClients[server]
+	if !ok {
+		lrsC = newLRSClient(c, server)
+		c.lrsClients[server] = lrsC
+	}
+
+	store := lrsC.ref()
+	return store, func() {
+		// This is a callback, need to hold lrsMu.
+		c.lrsMu.Lock()
+		defer c.lrsMu.Unlock()
+		if lrsC.unRef() {
+			// Delete the lrsClient from map if this is the last reference.
+			delete(c.lrsClients, server)
+		}
+	}
+}
+
+// lrsClient maps to one lrsServer. It contains:
+// - a ClientConn to this server (only if it's different from the xds server)
+// - a load.Store that contains loads only for this server
+type lrsClient struct {
+	parent *Client
+	server string
 
-// LoadStore returns the underlying load data store used by the xDS client.
-func (c *Client) LoadStore() *load.Store {
-	return c.loadStore
+	cc           *grpc.ClientConn // nil if the server is same as the xds server
+	refCount     int
+	cancelStream func()
+	loadStore    *load.Store
+}
+
+// newLRSClient creates a new LRS stream to the server.
+func newLRSClient(parent *Client, server string) *lrsClient {
+	return &lrsClient{
+		parent:   parent,
+		server:   server,
+		refCount: 0,
+	}
 }
 
-// ReportLoad sends the load of the given clusterName to the given server. If
-// the server is not an empty string, and is different from the xds server, a
-// new ClientConn will be created.
+// ref increments the refCount. If this is the first ref, it starts the LRS stream.
 //
-// The same options used for creating the Client will be used (including
-// NodeProto, and dial options if necessary).
+// Not thread-safe, caller needs to synchronize.
+func (lrsC *lrsClient) ref() *load.Store {
+	lrsC.refCount++
+	if lrsC.refCount == 1 {
+		lrsC.startStream()
+	}
+	return lrsC.loadStore
+}
+
+// unRef decrements the refCount, and closes the stream if refCount reaches 0
+// (and close the cc if cc is not xDS cc). It returns whether refCount reached 0
+// after this call.
 //
-// It returns a function to cancel the load reporting stream. If server is
-// different from xds server, the ClientConn will also be closed.
-func (c *Client) ReportLoad(server string, clusterName string) func() {
-	var (
-		cc      *grpc.ClientConn
-		closeCC bool
-	)
-	c.logger.Infof("Starting load report to server: %s", server)
-	if server == "" || server == c.opts.Config.BalancerName {
-		cc = c.cc
+// Not thread-safe, caller needs to synchronize.
+func (lrsC *lrsClient) unRef() (closed bool) {
+	lrsC.refCount--
+	if lrsC.refCount != 0 {
+		return false
+	}
+	lrsC.parent.logger.Infof("Stopping load report to server: %s", lrsC.server)
+	lrsC.cancelStream()
+	if lrsC.cc != nil {
+		lrsC.cc.Close()
+	}
+	return true
+}
+
+// startStream starts the LRS stream to the server. If server is not the same
+// xDS server from the parent, it also creates a ClientConn.
+func (lrsC *lrsClient) startStream() {
+	var cc *grpc.ClientConn
+
+	lrsC.parent.logger.Infof("Starting load report to server: %s", lrsC.server)
+	if lrsC.server == "" || lrsC.server == lrsC.parent.opts.Config.BalancerName {
+		// Reuse the xDS client if server is the same.
+		cc = lrsC.parent.cc
 	} else {
-		c.logger.Infof("LRS server is different from xDS server, starting a new ClientConn")
-		dopts := append([]grpc.DialOption{c.opts.Config.Creds}, c.opts.DialOpts...)
-		ccNew, err := grpc.Dial(server, dopts...)
+		lrsC.parent.logger.Infof("LRS server is different from xDS server, starting a new ClientConn")
+		dopts := append([]grpc.DialOption{lrsC.parent.opts.Config.Creds}, lrsC.parent.opts.DialOpts...)
+		ccNew, err := grpc.Dial(lrsC.server, dopts...)
 		if err != nil {
 			// An error from a non-blocking dial indicates something serious.
-			c.logger.Infof("xds: failed to dial load report server {%s}: %v", server, err)
-			return func() {}
+			lrsC.parent.logger.Infof("xds: failed to dial load report server {%s}: %v", lrsC.server, err)
+			return
 		}
 		cc = ccNew
-		closeCC = true
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	go c.apiClient.ReportLoad(ctx, c.cc, LoadReportingOptions{
-		ClusterName: clusterName,
-		TargetName:  c.opts.TargetName,
-	})
-	return func() {
-		cancel()
-		if closeCC {
-			cc.Close()
-		}
+		lrsC.cc = ccNew
 	}
+
+	var ctx context.Context
+	ctx, lrsC.cancelStream = context.WithCancel(context.Background())
+
+	// Create the store and stream.
+	lrsC.loadStore = load.NewStore()
+	go lrsC.parent.apiClient.reportLoad(ctx, cc, loadReportingOptions{loadStore: lrsC.loadStore})
 }
diff --git a/xds/internal/client/client_loadreport_test.go b/xds/internal/client/client_loadreport_test.go
new file mode 100644
index 000000000000..d426247c10d9
--- /dev/null
+++ b/xds/internal/client/client_loadreport_test.go
@@ -0,0 +1,148 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
+	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
+	durationpb "github.com/golang/protobuf/ptypes/duration"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	_ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 xDS API client.
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestLRSClient(t *testing.T) {
+	fs, sCleanup, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("failed to start fake xDS server: %v", err)
+	}
+	defer sCleanup()
+
+	xdsC, err := client.New(client.Options{
+		Config: bootstrap.Config{
+			BalancerName: fs.Address,
+			Creds:        grpc.WithInsecure(),
+			NodeProto:    &v2corepb.Node{},
+			TransportAPI: version.TransportV2,
+		},
+	})
+	if err != nil {
+		t.Fatalf("failed to create xds client: %v", err)
+	}
+	defer xdsC.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if u, err := fs.NewConnChan.Receive(ctx); err != nil {
+		t.Errorf("unexpected timeout: %v, %v, want NewConn", u, err)
+	}
+
+	// Report to the same address should not create new ClientConn.
+	store1, lrsCancel1 := xdsC.ReportLoad(fs.Address)
+	defer lrsCancel1()
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := fs.NewConnChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Errorf("unexpected NewConn: %v, %v, want channel recv timeout", u, err)
+	}
+
+	fs2, sCleanup2, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("failed to start fake xDS server: %v", err)
+	}
+	defer sCleanup2()
+
+	// Report to a different address should create new ClientConn.
+	store2, lrsCancel2 := xdsC.ReportLoad(fs2.Address)
+	defer lrsCancel2()
+	if u, err := fs2.NewConnChan.Receive(ctx); err != nil {
+		t.Errorf("unexpected timeout: %v, %v, want NewConn", u, err)
+	}
+
+	if store1 == store2 {
+		t.Fatalf("got same store for different servers, want different")
+	}
+
+	if u, err := fs2.LRSRequestChan.Receive(ctx); err != nil {
+		t.Errorf("unexpected timeout: %v, %v, want NewConn", u, err)
+	}
+	store2.PerCluster("cluster", "eds").CallDropped("test")
+
+	// Send one resp to the client.
+	fs2.LRSResponseChan <- &fakeserver.Response{
+		Resp: &lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000},
+		},
+	}
+
+	// Server should receive a req with the loads.
+	u, err := fs2.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("unexpected LRS request: %v, %v, want error canceled", u, err)
+	}
+	receivedLoad := u.(*fakeserver.Request).Req.(*lrspb.LoadStatsRequest).ClusterStats
+	if len(receivedLoad) <= 0 {
+		t.Fatalf("unexpected load received, want load for cluster, eds, dropped for test")
+	}
+	receivedLoad[0].LoadReportInterval = nil
+	want := (&endpointpb.ClusterStats{
+		ClusterName:          "cluster",
+		ClusterServiceName:   "eds",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+	})
+	if d := cmp.Diff(want, receivedLoad[0], protocmp.Transform()); d != "" {
+		t.Fatalf("unexpected load received, want load for cluster, eds, dropped for test, diff (-want +got):\n%s", d)
+	}
+
+	// Cancel this load reporting stream, server should see error canceled.
+	lrsCancel2()
+
+	// Server should receive a stream canceled error.
+	if u, err := fs2.LRSRequestChan.Receive(ctx); err != nil || status.Code(u.(*fakeserver.Request).Err) != codes.Canceled {
+		t.Errorf("unexpected LRS request: %v, %v, want error canceled", u, err)
+	}
+}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 4a531384fff0..a2a92a24f7fb 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -115,7 +115,7 @@ func (c *testAPIClient) RemoveWatch(resourceType ResourceType, resourceName stri
 	c.removeWatches[resourceType].Send(resourceName)
 }
 
-func (c *testAPIClient) ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions) {
+func (c *testAPIClient) reportLoad(context.Context, *grpc.ClientConn, loadReportingOptions) {
 }
 
 func (c *testAPIClient) Close() {}
diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
index 607f26fd5e17..b286a61d638b 100644
--- a/xds/internal/client/transport_helper.go
+++ b/xds/internal/client/transport_helper.go
@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc/xds/internal/client/load"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/buffer"
@@ -71,19 +72,21 @@ type VersionedClient interface {
 	NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error)
 
 	// SendFirstLoadStatsRequest constructs and sends the first request on the
-	// LRS stream. This contains the node proto with appropriate metadata
-	// fields.
-	SendFirstLoadStatsRequest(s grpc.ClientStream, targetName string) error
+	// LRS stream.
+	SendFirstLoadStatsRequest(s grpc.ClientStream) error
 
 	// HandleLoadStatsResponse receives the first response from the server which
 	// contains the load reporting interval and the clusters for which the
 	// server asks the client to report load for.
-	HandleLoadStatsResponse(s grpc.ClientStream, clusterName string) (time.Duration, error)
+	//
+	// If the response sets SendAllClusters to true, the returned clusters is
+	// nil.
+	HandleLoadStatsResponse(s grpc.ClientStream) (clusters []string, _ time.Duration, _ error)
 
 	// SendLoadStatsRequest will be invoked at regular intervals to send load
 	// report with load data reported since the last time this method was
 	// invoked.
-	SendLoadStatsRequest(s grpc.ClientStream, clusterName string) error
+	SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error
 }
 
 // TransportHelper contains all xDS transport protocol related functionality
@@ -443,9 +446,9 @@ func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStrea
 	return target, rType, version, nonce, send
 }
 
-// ReportLoad starts an LRS stream to report load data to the management server.
+// reportLoad starts an LRS stream to report load data to the management server.
 // It blocks until the context is cancelled.
-func (t *TransportHelper) ReportLoad(ctx context.Context, cc *grpc.ClientConn, opts LoadReportingOptions) {
+func (t *TransportHelper) reportLoad(ctx context.Context, cc *grpc.ClientConn, opts loadReportingOptions) {
 	retries := 0
 	for {
 		if ctx.Err() != nil {
@@ -472,23 +475,23 @@ func (t *TransportHelper) ReportLoad(ctx context.Context, cc *grpc.ClientConn, o
 		}
 		logger.Infof("lrs: created LRS stream")
 
-		if err := t.vClient.SendFirstLoadStatsRequest(stream, opts.TargetName); err != nil {
+		if err := t.vClient.SendFirstLoadStatsRequest(stream); err != nil {
 			logger.Warningf("lrs: failed to send first request: %v", err)
 			continue
 		}
 
-		interval, err := t.vClient.HandleLoadStatsResponse(stream, opts.ClusterName)
+		clusters, interval, err := t.vClient.HandleLoadStatsResponse(stream)
 		if err != nil {
 			logger.Warning(err)
 			continue
 		}
 
 		retries = 0
-		t.sendLoads(ctx, stream, opts.ClusterName, interval)
+		t.sendLoads(ctx, stream, opts.loadStore, clusters, interval)
 	}
 }
 
-func (t *TransportHelper) sendLoads(ctx context.Context, stream grpc.ClientStream, clusterName string, interval time.Duration) {
+func (t *TransportHelper) sendLoads(ctx context.Context, stream grpc.ClientStream, store *load.Store, clusterNames []string, interval time.Duration) {
 	tick := time.NewTicker(interval)
 	defer tick.Stop()
 	for {
@@ -497,7 +500,7 @@ func (t *TransportHelper) sendLoads(ctx context.Context, stream grpc.ClientStrea
 		case <-ctx.Done():
 			return
 		}
-		if err := t.vClient.SendLoadStatsRequest(stream, clusterName); err != nil {
+		if err := t.vClient.SendLoadStatsRequest(stream, store.Stats(clusterNames)); err != nil {
 			logger.Warning(err)
 			return
 		}
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index 7b063ad4f559..433e6907d9fe 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -28,7 +28,6 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/version"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
@@ -69,7 +68,6 @@ func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIC
 		cc:        cc,
 		parent:    opts.Parent,
 		nodeProto: nodeProto,
-		loadStore: opts.LoadStore,
 		logger:    opts.Logger,
 	}
 	v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background())
@@ -88,7 +86,6 @@ type client struct {
 	ctx       context.Context
 	cancelCtx context.CancelFunc
 	parent    xdsclient.UpdateHandler
-	loadStore *load.Store
 	logger    *grpclog.PrefixLogger
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
diff --git a/xds/internal/client/v2/loadreport.go b/xds/internal/client/v2/loadreport.go
index a06dcb8e9f0d..69405fcd9ad3 100644
--- a/xds/internal/client/v2/loadreport.go
+++ b/xds/internal/client/v2/loadreport.go
@@ -26,17 +26,18 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/xds/internal/client/load"
 
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v2endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
 	lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
 	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2"
-	structpb "github.com/golang/protobuf/ptypes/struct"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/xds/internal"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
+const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters"
+
 type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient
 
 func (v2c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) {
@@ -44,7 +45,7 @@ func (v2c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn)
 	return c.StreamLoadStats(ctx)
 }
 
-func (v2c *client) SendFirstLoadStatsRequest(s grpc.ClientStream, targetName string) error {
+func (v2c *client) SendFirstLoadStatsRequest(s grpc.ClientStream) error {
 	stream, ok := s.(lrsStream)
 	if !ok {
 		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
@@ -53,72 +54,52 @@ func (v2c *client) SendFirstLoadStatsRequest(s grpc.ClientStream, targetName str
 	if node == nil {
 		node = &v2corepb.Node{}
 	}
-	if node.Metadata == nil {
-		node.Metadata = &structpb.Struct{}
-	}
-	if node.Metadata.Fields == nil {
-		node.Metadata.Fields = make(map[string]*structpb.Value)
-	}
-	node.Metadata.Fields[xdsclient.NodeMetadataHostnameKey] = &structpb.Value{
-		Kind: &structpb.Value_StringValue{StringValue: targetName},
-	}
+	node.ClientFeatures = append(node.ClientFeatures, clientFeatureLRSSendAllClusters)
 
 	req := &lrspb.LoadStatsRequest{Node: node}
 	v2c.logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
 	return stream.Send(req)
 }
 
-func (v2c *client) HandleLoadStatsResponse(s grpc.ClientStream, clusterName string) (time.Duration, error) {
+func (v2c *client) HandleLoadStatsResponse(s grpc.ClientStream) ([]string, time.Duration, error) {
 	stream, ok := s.(lrsStream)
 	if !ok {
-		return 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s)
+		return nil, 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s)
 	}
 
 	resp, err := stream.Recv()
 	if err != nil {
-		return 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
+		return nil, 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
 	}
 	v2c.logger.Infof("lrs: received first LoadStatsResponse: %+v", resp)
 
 	interval, err := ptypes.Duration(resp.GetLoadReportingInterval())
 	if err != nil {
-		return 0, fmt.Errorf("lrs: failed to convert report interval: %v", err)
+		return nil, 0, fmt.Errorf("lrs: failed to convert report interval: %v", err)
 	}
 
-	// The LRS client should join the clusters it knows with the cluster
-	// list from response, and send loads for them.
-	//
-	// But the LRS client now only supports one cluster. TODO: extend it to
-	// support multiple clusters.
-	var clusterFoundInResponse bool
-	for _, c := range resp.Clusters {
-		if c == clusterName {
-			clusterFoundInResponse = true
-		}
-	}
-	if !clusterFoundInResponse {
-		return 0, fmt.Errorf("lrs: received clusters %v does not contain expected {%v}", resp.Clusters, clusterName)
-	}
 	if resp.ReportEndpointGranularity {
 		// TODO: fixme to support per endpoint loads.
-		return 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation")
+		return nil, 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation")
 	}
 
-	return interval, nil
+	clusters := resp.Clusters
+	if resp.SendAllClusters {
+		// Return nil to send stats for all clusters.
+		clusters = nil
+	}
+
+	return clusters, interval, nil
 }
 
-func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string) error {
+func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error {
 	stream, ok := s.(lrsStream)
 	if !ok {
 		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
 	}
-	if v2c.loadStore == nil {
-		return errors.New("lrs: LoadStore is not initialized")
-	}
 
 	var clusterStats []*v2endpointpb.ClusterStats
-	sds := v2c.loadStore.Stats([]string{clusterName})
-	for _, sd := range sds {
+	for _, sd := range loads {
 		var (
 			droppedReqs   []*v2endpointpb.ClusterStats_DroppedRequests
 			localityStats []*v2endpointpb.UpstreamLocalityStats
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 5d8d7198ce2d..00e74e5974e2 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -29,7 +29,6 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/version"
 
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -69,7 +68,6 @@ func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIC
 		cc:        cc,
 		parent:    opts.Parent,
 		nodeProto: nodeProto,
-		loadStore: opts.LoadStore,
 		logger:    opts.Logger,
 	}
 	v3c.ctx, v3c.cancelCtx = context.WithCancel(context.Background())
@@ -88,7 +86,6 @@ type client struct {
 	ctx       context.Context
 	cancelCtx context.CancelFunc
 	parent    xdsclient.UpdateHandler
-	loadStore *load.Store
 	logger    *grpclog.PrefixLogger
 
 	// ClientConn to the xDS gRPC server. Owned by the parent xdsClient.
diff --git a/xds/internal/client/v3/loadreport.go b/xds/internal/client/v3/loadreport.go
index beca34c49bfd..74e18632aa07 100644
--- a/xds/internal/client/v3/loadreport.go
+++ b/xds/internal/client/v3/loadreport.go
@@ -26,17 +26,18 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/xds/internal/client/load"
 
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
 	lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
-	structpb "github.com/golang/protobuf/ptypes/struct"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/xds/internal"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
+const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters"
+
 type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient
 
 func (v3c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) {
@@ -44,7 +45,7 @@ func (v3c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn)
 	return c.StreamLoadStats(ctx)
 }
 
-func (v3c *client) SendFirstLoadStatsRequest(s grpc.ClientStream, targetName string) error {
+func (v3c *client) SendFirstLoadStatsRequest(s grpc.ClientStream) error {
 	stream, ok := s.(lrsStream)
 	if !ok {
 		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
@@ -53,72 +54,52 @@ func (v3c *client) SendFirstLoadStatsRequest(s grpc.ClientStream, targetName str
 	if node == nil {
 		node = &v3corepb.Node{}
 	}
-	if node.Metadata == nil {
-		node.Metadata = &structpb.Struct{}
-	}
-	if node.Metadata.Fields == nil {
-		node.Metadata.Fields = make(map[string]*structpb.Value)
-	}
-	node.Metadata.Fields[xdsclient.NodeMetadataHostnameKey] = &structpb.Value{
-		Kind: &structpb.Value_StringValue{StringValue: targetName},
-	}
+	node.ClientFeatures = append(node.ClientFeatures, clientFeatureLRSSendAllClusters)
 
 	req := &lrspb.LoadStatsRequest{Node: node}
 	v3c.logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
 	return stream.Send(req)
 }
 
-func (v3c *client) HandleLoadStatsResponse(s grpc.ClientStream, clusterName string) (time.Duration, error) {
+func (v3c *client) HandleLoadStatsResponse(s grpc.ClientStream) ([]string, time.Duration, error) {
 	stream, ok := s.(lrsStream)
 	if !ok {
-		return 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s)
+		return nil, 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s)
 	}
 
 	resp, err := stream.Recv()
 	if err != nil {
-		return 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
+		return nil, 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
 	}
 	v3c.logger.Infof("lrs: received first LoadStatsResponse: %+v", resp)
 
 	interval, err := ptypes.Duration(resp.GetLoadReportingInterval())
 	if err != nil {
-		return 0, fmt.Errorf("lrs: failed to convert report interval: %v", err)
+		return nil, 0, fmt.Errorf("lrs: failed to convert report interval: %v", err)
 	}
 
-	// The LRS client should join the clusters it knows with the cluster
-	// list from response, and send loads for them.
-	//
-	// But the LRS client now only supports one cluster. TODO: extend it to
-	// support multiple clusters.
-	var clusterFoundInResponse bool
-	for _, c := range resp.Clusters {
-		if c == clusterName {
-			clusterFoundInResponse = true
-		}
-	}
-	if !clusterFoundInResponse {
-		return 0, fmt.Errorf("lrs: received clusters %v does not contain expected {%v}", resp.Clusters, clusterName)
-	}
 	if resp.ReportEndpointGranularity {
 		// TODO: fixme to support per endpoint loads.
-		return 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation")
+		return nil, 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation")
 	}
 
-	return interval, nil
+	clusters := resp.Clusters
+	if resp.SendAllClusters {
+		// Return nil to send stats for all clusters.
+		clusters = nil
+	}
+
+	return clusters, interval, nil
 }
 
-func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, clusterName string) error {
+func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error {
 	stream, ok := s.(lrsStream)
 	if !ok {
 		return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s)
 	}
-	if v3c.loadStore == nil {
-		return errors.New("lrs: LoadStore is not initialized")
-	}
 
 	var clusterStats []*v3endpointpb.ClusterStats
-	sds := v3c.loadStore.Stats([]string{clusterName})
-	for _, sd := range sds {
+	for _, sd := range loads {
 		var (
 			droppedReqs   []*v3endpointpb.ClusterStats_DroppedRequests
 			localityStats []*v3endpointpb.UpstreamLocalityStats
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index cd2710e612aa..408817c17846 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -156,14 +156,12 @@ func (xdsC *Client) WaitForCancelEDSWatch(ctx context.Context) error {
 type ReportLoadArgs struct {
 	// Server is the name of the server to which the load is reported.
 	Server string
-	// Cluster is the name of the cluster for which load is reported.
-	Cluster string
 }
 
 // ReportLoad starts reporting load about clusterName to server.
-func (xdsC *Client) ReportLoad(server string, clusterName string) (cancel func()) {
-	xdsC.loadReportCh.Send(ReportLoadArgs{Server: server, Cluster: clusterName})
-	return func() {}
+func (xdsC *Client) ReportLoad(server string) (loadStore *load.Store, cancel func()) {
+	xdsC.loadReportCh.Send(ReportLoadArgs{Server: server})
+	return xdsC.loadStore, func() {}
 }
 
 // LoadStore returns the underlying load data store.
diff --git a/xds/internal/testutils/fakeserver/server.go b/xds/internal/testutils/fakeserver/server.go
index 4cff72087cec..994f5308f3af 100644
--- a/xds/internal/testutils/fakeserver/server.go
+++ b/xds/internal/testutils/fakeserver/server.go
@@ -203,10 +203,10 @@ type lrsServer struct {
 
 func (lrsS *lrsServer) StreamLoadStats(s lrsgrpc.LoadReportingService_StreamLoadStatsServer) error {
 	req, err := s.Recv()
+	lrsS.reqChan.Send(&Request{req, err})
 	if err != nil {
 		return err
 	}
-	lrsS.reqChan.Send(&Request{req, err})
 
 	select {
 	case r := <-lrsS.respChan:
@@ -222,12 +222,12 @@ func (lrsS *lrsServer) StreamLoadStats(s lrsgrpc.LoadReportingService_StreamLoad
 
 	for {
 		req, err := s.Recv()
+		lrsS.reqChan.Send(&Request{req, err})
 		if err != nil {
 			if err == io.EOF {
 				return nil
 			}
 			return err
 		}
-		lrsS.reqChan.Send(&Request{req, err})
 	}
 }

From 829af019ba6029b1352853b6e2c531092cdc5cd6 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Tue, 27 Oct 2020 09:58:31 -0700
Subject: [PATCH 248/481] advancedtls: add examples for reloading from file
 system (#3976)

* add examples for reloading from file system
---
 .travis.yml                                   |   2 +-
 .../credential_reloading_from_files/README.md |  26 ++
 .../client/main.go                            |  99 +++++
 .../server/main.go                            | 105 +++++
 .../advancedtls/examples/examples_test.sh     | 126 ++++++
 security/advancedtls/examples/go.mod          |   9 +
 security/advancedtls/examples/go.sum          | 371 ++++++++++++++++++
 security/advancedtls/go.mod                   |   2 +-
 8 files changed, 738 insertions(+), 2 deletions(-)
 create mode 100644 security/advancedtls/examples/credential_reloading_from_files/README.md
 create mode 100644 security/advancedtls/examples/credential_reloading_from_files/client/main.go
 create mode 100644 security/advancedtls/examples/credential_reloading_from_files/server/main.go
 create mode 100755 security/advancedtls/examples/examples_test.sh
 create mode 100644 security/advancedtls/examples/go.mod
 create mode 100644 security/advancedtls/examples/go.sum

diff --git a/.travis.yml b/.travis.yml
index 3e495fa23b9d..5847d94e5512 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -35,7 +35,7 @@ install:
 
 script:
   - set -e
-  - if [[ -n "${TESTEXTRAS}" ]]; then examples/examples_test.sh; interop/interop_test.sh; make testsubmodule; exit 0; fi
+  - if [[ -n "${TESTEXTRAS}" ]]; then examples/examples_test.sh; security/advancedtls/examples/examples_test.sh; interop/interop_test.sh; make testsubmodule; exit 0; fi
   - if [[ -n "${VET}" ]]; then ./vet.sh; fi
   - if [[ -n "${GAE}" ]]; then make testappengine; exit 0; fi
   - if [[ -n "${RACE}" ]]; then make testrace; exit 0; fi
diff --git a/security/advancedtls/examples/credential_reloading_from_files/README.md b/security/advancedtls/examples/credential_reloading_from_files/README.md
new file mode 100644
index 000000000000..afb13d44cfb6
--- /dev/null
+++ b/security/advancedtls/examples/credential_reloading_from_files/README.md
@@ -0,0 +1,26 @@
+# Credential Reloading From Files
+
+Credential reloading is a feature supported in the advancedtls library. 
+A very common way to achieve this is to reload from files.
+
+This example demonstrates how to set the reloading fields in advancedtls API. 
+Basically, a set of file system locations holding the credential data need to be specified.
+Once the credential data needs to be updated, users just change the credential data in the file system, and gRPC will pick up the changes automatically.
+
+This example only shows how to set the API, without demonstrating the way to reload credentials on file system. 
+To learn more about how to do that in Go, please see `advancedtls_integration_test.go`.
+
+A couple of things to note:
+ 1. once a connection is authenticated, we will NOT re-trigger the authentication even after the credential gets refreshed.
+ 2. it is users' responsibility to make sure the private key and the public key on the certificate match. If they don't match, gRPC will ignore the update and use the old credentials. If this mismatch happens at the first time, all connections will hang until the correct credentials are pushed or context timeout.  
+
+## Try it
+In directory `security/advancedtls/examples`:
+
+```
+go run server/main.go
+```
+
+```
+go run client/main.go
+```
\ No newline at end of file
diff --git a/security/advancedtls/examples/credential_reloading_from_files/client/main.go b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
new file mode 100644
index 000000000000..c300ba4cf941
--- /dev/null
+++ b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
@@ -0,0 +1,99 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// The client demonstrates how to use the credential reloading feature in
+// advancedtls to make a mTLS connection to the server.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"google.golang.org/grpc"
+	pb "google.golang.org/grpc/examples/helloworld/helloworld"
+	"google.golang.org/grpc/security/advancedtls"
+	"google.golang.org/grpc/security/advancedtls/testdata"
+)
+
+var address = "localhost:50051"
+
+const (
+	// Default timeout for normal connections.
+	defaultConnTimeout = 10 * time.Second
+	// Intervals that set to monitor the credential updates.
+	credRefreshingInterval = 1 * time.Minute
+)
+
+func main() {
+	flag.Parse()
+
+	// TODO(ZhenLian): change function signatures to reflect the changes in
+	// https://github.com/grpc/grpc-go/pull/3981.
+	identityOptions := advancedtls.PEMFileProviderOptions{
+		CertFile:         testdata.Path("client_cert_1.pem"),
+		KeyFile:          testdata.Path("client_key_1.pem"),
+		IdentityInterval: credRefreshingInterval,
+	}
+	identityProvider, err := advancedtls.NewPEMFileProvider(identityOptions)
+	if err != nil {
+		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", identityOptions, err)
+	}
+	rootOptions := advancedtls.PEMFileProviderOptions{
+		TrustFile:    testdata.Path("client_trust_cert_1.pem"),
+		RootInterval: credRefreshingInterval,
+	}
+	rootProvider, err := advancedtls.NewPEMFileProvider(rootOptions)
+	if err != nil {
+		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", rootOptions, err)
+	}
+
+	options := &advancedtls.ClientOptions{
+		IdentityOptions: advancedtls.IdentityCertificateOptions{
+			IdentityProvider: identityProvider,
+		},
+		VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
+			return &advancedtls.VerificationResults{}, nil
+		},
+		RootOptions: advancedtls.RootCertificateOptions{
+			RootProvider: rootProvider,
+		},
+		VType: advancedtls.CertVerification,
+	}
+	clientTLSCreds, err := advancedtls.NewClientCreds(options)
+	if err != nil {
+		log.Fatalf("advancedtls.NewClientCreds(%v) failed: %v", options, err)
+	}
+
+	// At initialization, the connection should be good.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultConnTimeout)
+	defer cancel()
+	conn, err := grpc.DialContext(ctx, address, grpc.WithTransportCredentials(clientTLSCreds))
+	if err != nil {
+		log.Fatalf("grpc.DialContext to %s failed: %v", address, err)
+	}
+	greetClient := pb.NewGreeterClient(conn)
+	reply, err := greetClient.SayHello(ctx, &pb.HelloRequest{Name: "gRPC"}, grpc.WaitForReady(true))
+	if err != nil {
+		log.Fatalf("greetClient.SayHello failed: %v", err)
+	}
+	defer conn.Close()
+	fmt.Printf("Getting message from server: %s...\n", reply.Message)
+}
diff --git a/security/advancedtls/examples/credential_reloading_from_files/server/main.go b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
new file mode 100644
index 000000000000..b60762fa625a
--- /dev/null
+++ b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
@@ -0,0 +1,105 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// The server demonstrates how to use the credential reloading feature in
+// advancedtls to serve mTLS connections from the client.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/security/advancedtls"
+	"google.golang.org/grpc/security/advancedtls/testdata"
+
+	pb "google.golang.org/grpc/examples/helloworld/helloworld"
+)
+
+var port = ":50051"
+
+// Intervals that set to monitor the credential updates.
+const credRefreshingInterval = 1 * time.Minute
+
+type greeterServer struct {
+	pb.UnimplementedGreeterServer
+}
+
+// sayHello is a simple implementation of the pb.GreeterServer SayHello method.
+func (greeterServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+	return &pb.HelloReply{Message: "Hello " + in.Name}, nil
+}
+
+func main() {
+	flag.Parse()
+	fmt.Printf("server starting on port %s...\n", port)
+
+	// TODO(ZhenLian): change function signatures to reflect the changes in
+	// https://github.com/grpc/grpc-go/pull/3981.
+	identityOptions := advancedtls.PEMFileProviderOptions{
+		CertFile:         testdata.Path("server_cert_1.pem"),
+		KeyFile:          testdata.Path("server_key_1.pem"),
+		IdentityInterval: credRefreshingInterval,
+	}
+	identityProvider, err := advancedtls.NewPEMFileProvider(identityOptions)
+	if err != nil {
+		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", identityOptions, err)
+	}
+	defer identityProvider.Close()
+	rootOptions := advancedtls.PEMFileProviderOptions{
+		TrustFile:    testdata.Path("server_trust_cert_1.pem"),
+		RootInterval: credRefreshingInterval,
+	}
+	rootProvider, err := advancedtls.NewPEMFileProvider(rootOptions)
+	if err != nil {
+		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", rootOptions, err)
+	}
+	defer rootProvider.Close()
+
+	// Start a server and create a client using advancedtls API with Provider.
+	options := &advancedtls.ServerOptions{
+		IdentityOptions: advancedtls.IdentityCertificateOptions{
+			IdentityProvider: identityProvider,
+		},
+		RootOptions: advancedtls.RootCertificateOptions{
+			RootProvider: rootProvider,
+		},
+		RequireClientCert: true,
+		VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
+			return &advancedtls.VerificationResults{}, nil
+		},
+		VType: advancedtls.CertVerification,
+	}
+	serverTLSCreds, err := advancedtls.NewServerCreds(options)
+	if err != nil {
+		log.Fatalf("advancedtls.NewServerCreds(%v) failed: %v", options, err)
+	}
+	s := grpc.NewServer(grpc.Creds(serverTLSCreds))
+	lis, err := net.Listen("tcp", port)
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+	pb.RegisterGreeterServer(s, greeterServer{})
+	if err := s.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
+}
diff --git a/security/advancedtls/examples/examples_test.sh b/security/advancedtls/examples/examples_test.sh
new file mode 100755
index 000000000000..44dbadd5ab17
--- /dev/null
+++ b/security/advancedtls/examples/examples_test.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+#
+#  Copyright 2020 gRPC authors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+set +e
+
+export TMPDIR=$(mktemp -d)
+trap "rm -rf ${TMPDIR}" EXIT
+
+clean () {
+  for i in {1..10}; do
+    jobs -p | xargs -n1 pkill -P
+    # A simple "wait" just hangs sometimes.  Running `jobs` seems to help.
+    sleep 1
+    if jobs | read; then
+      return
+    fi
+  done
+  echo "$(tput setaf 1) clean failed to kill tests $(tput sgr 0)"
+  jobs
+  pstree
+  exit 1
+}
+
+fail () {
+    echo "$(tput setaf 1) $1 $(tput sgr 0)"
+    clean
+    exit 1
+}
+
+pass () {
+    echo "$(tput setaf 2) $1 $(tput sgr 0)"
+}
+
+EXAMPLES=(
+    "credential_reloading_from_files"
+)
+
+declare -A EXPECTED_SERVER_OUTPUT=(
+    ["credential_reloading_from_files"]=""
+)
+
+declare -A EXPECTED_CLIENT_OUTPUT=(
+    ["credential_reloading_from_files"]="Getting message from server: Hello gRPC..."
+)
+
+cd ./security/advancedtls/examples
+
+for example in ${EXAMPLES[@]}; do
+    echo "$(tput setaf 4) testing: ${example} $(tput sgr 0)"
+
+    # Build server
+    if ! go build -o /dev/null ./${example}/*server/*.go; then
+        fail "failed to build server"
+    else
+        pass "successfully built server"
+    fi
+
+    # Build client
+    if ! go build -o /dev/null ./${example}/*client/*.go; then
+        fail "failed to build client"
+    else
+        pass "successfully built client"
+    fi
+
+    # Start server
+    SERVER_LOG="$(mktemp)"
+    go run ./$example/*server/*.go &> $SERVER_LOG  &
+
+    CLIENT_LOG="$(mktemp)"
+    if ! timeout 20 go run ${example}/*client/*.go &> $CLIENT_LOG; then
+        fail "client failed to communicate with server
+        got server log:
+        $(cat $SERVER_LOG)
+        got client log:
+        $(cat $CLIENT_LOG)
+        "
+    else
+        pass "client successfully communitcated with server"
+    fi
+
+    # Check server log for expected output if expecting an
+    # output
+    if [ -n "${EXPECTED_SERVER_OUTPUT[$example]}" ]; then
+        if ! grep -q "${EXPECTED_SERVER_OUTPUT[$example]}" $SERVER_LOG; then
+            fail "server log missing output: ${EXPECTED_SERVER_OUTPUT[$example]}
+            got server log:
+            $(cat $SERVER_LOG)
+            got client log:
+            $(cat $CLIENT_LOG)
+            "
+        else
+            pass "server log contains expected output: ${EXPECTED_SERVER_OUTPUT[$example]}"
+        fi
+    fi
+
+    # Check client log for expected output if expecting an
+    # output
+    if [ -n "${EXPECTED_CLIENT_OUTPUT[$example]}" ]; then
+        if ! grep -q "${EXPECTED_CLIENT_OUTPUT[$example]}" $CLIENT_LOG; then
+            fail "client log missing output: ${EXPECTED_CLIENT_OUTPUT[$example]}
+            got server log:
+            $(cat $SERVER_LOG)
+            got client log:
+            $(cat $CLIENT_LOG)
+            "
+        else
+            pass "client log contains expected output: ${EXPECTED_CLIENT_OUTPUT[$example]}"
+        fi
+    fi
+    clean
+    echo ""
+done
diff --git a/security/advancedtls/examples/go.mod b/security/advancedtls/examples/go.mod
new file mode 100644
index 000000000000..ed473445ef28
--- /dev/null
+++ b/security/advancedtls/examples/go.mod
@@ -0,0 +1,9 @@
+module google.golang.org/grpc/security/advancedtls/examples
+
+go 1.15
+
+require (
+	google.golang.org/grpc v1.33.1
+	google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d
+	google.golang.org/grpc/security/advancedtls v0.0.0-20201020200225-9519efffeb5d
+)
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
new file mode 100644
index 000000000000..9289089aa26c
--- /dev/null
+++ b/security/advancedtls/examples/go.sum
@@ -0,0 +1,371 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f/go.mod h1:TGiSRL2BBv2WqzfsFNWYp/pkWdtf5kbZS/DQ9Ee3mWk=
+google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d h1:AW0XWR8UrnkpSYMW9c9obDY/xXI0phgJOn+7oAp2DpA=
+google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d/go.mod h1:Lh55/1hxmVHEkOvSIQ2uj0P12QyOCUNyRwnUlSS13hw=
+google.golang.org/grpc/security/advancedtls v0.0.0-20201020200225-9519efffeb5d h1:BQWmmlFW/jQsHms4c99jRsiJAVv52QoUmCHTSzJXEAM=
+google.golang.org/grpc/security/advancedtls v0.0.0-20201020200225-9519efffeb5d/go.mod h1:39RL6g/iGSMf3cQevYAO8tFbj0cakVVRjYayPce2p+k=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index 442735f6fc03..7df0f3fc56c1 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/google/go-cmp v0.5.1
 	google.golang.org/grpc v1.31.0
-	google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f
+	google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d
 )
 
 replace google.golang.org/grpc => ../../

From 4e6166f62b1566078120f53adc1ac8bd651be587 Mon Sep 17 00:00:00 2001
From: Aaron Jheng <wentworth@outlook.com>
Date: Wed, 28 Oct 2020 01:54:58 +0800
Subject: [PATCH 249/481] Documentation: fix outgoing metadata example code
 (#3979)

---
 Documentation/grpc-metadata.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Documentation/grpc-metadata.md b/Documentation/grpc-metadata.md
index c278a98ef8c7..ff4de6e71de3 100644
--- a/Documentation/grpc-metadata.md
+++ b/Documentation/grpc-metadata.md
@@ -110,9 +110,9 @@ md := metadata.Pairs("k1", "v1", "k1", "v2", "k2", "v3")
 ctx := metadata.NewOutgoingContext(context.Background(), md)
 
 // later, add some more metadata to the context (e.g. in an interceptor)
-md, _ := metadata.FromOutgoingContext(ctx)
+send, _ := metadata.FromOutgoingContext(ctx)
 newMD := metadata.Pairs("k3", "v3")
-ctx = metadata.NewContext(ctx, metadata.Join(metadata.New(send), newMD))
+ctx = metadata.NewOutgoingContext(ctx, metadata.Join(send, newMD))
 
 // make unary RPC
 response, err := client.SomeRPC(ctx, someRequest)

From 71ef09e69af4a8faf36c26a572f065176aa03259 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 27 Oct 2020 15:56:59 -0700
Subject: [PATCH 250/481] xdsResolver: call WatchListener and WatchRDS, instead
 of WatchService (#3972)

---
 xds/internal/client/client.go                 |   7 -
 xds/internal/client/client_rds_test.go        |  96 ---
 xds/internal/client/client_test.go            |  12 -
 xds/internal/client/client_watchers.go        | 134 +---
 ...st.go => client_watchers_listener_test.go} |  16 +-
 ..._test.go => client_watchers_route_test.go} |  12 +-
 .../client/client_watchers_service_test.go    | 572 ------------------
 xds/internal/client/client_xds.go             |  95 ---
 xds/internal/resolver/serviceconfig.go        |   2 +-
 xds/internal/resolver/serviceconfig_test.go   |   4 +-
 xds/internal/resolver/watch_service.go        | 236 ++++++++
 xds/internal/resolver/watch_service_test.go   | 291 +++++++++
 xds/internal/resolver/xds_resolver.go         |   9 +-
 xds/internal/resolver/xds_resolver_test.go    | 112 +++-
 xds/internal/testutils/fakeclient/client.go   | 109 ++--
 15 files changed, 705 insertions(+), 1002 deletions(-)
 rename xds/internal/client/{client_watchers_lds_test.go => client_watchers_listener_test.go} (94%)
 rename xds/internal/client/{client_watchers_rds_test.go => client_watchers_route_test.go} (94%)
 delete mode 100644 xds/internal/client/client_watchers_service_test.go
 create mode 100644 xds/internal/resolver/watch_service.go
 create mode 100644 xds/internal/resolver/watch_service_test.go

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 347856927976..f1c35d7be520 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -190,13 +190,6 @@ type Int64Range struct {
 	End   int64 `json:"end"`
 }
 
-// ServiceUpdate contains information received from LDS and RDS responses,
-// which is of interest to the registered service watcher.
-type ServiceUpdate struct {
-	// Routes contain matchers+actions to route RPCs.
-	Routes []*Route
-}
-
 // SecurityConfig contains the security configuration received as part of the
 // Cluster resource.
 type SecurityConfig struct {
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index 286472af8c73..ee092117cac5 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -633,102 +633,6 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 	}
 }
 
-func (s) TestMatchTypeForDomain(t *testing.T) {
-	tests := []struct {
-		d    string
-		want domainMatchType
-	}{
-		{d: "", want: domainMatchTypeInvalid},
-		{d: "*", want: domainMatchTypeUniversal},
-		{d: "bar.*", want: domainMatchTypePrefix},
-		{d: "*.abc.com", want: domainMatchTypeSuffix},
-		{d: "foo.bar.com", want: domainMatchTypeExact},
-		{d: "foo.*.com", want: domainMatchTypeInvalid},
-	}
-	for _, tt := range tests {
-		if got := matchTypeForDomain(tt.d); got != tt.want {
-			t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want)
-		}
-	}
-}
-
-func (s) TestMatch(t *testing.T) {
-	tests := []struct {
-		name        string
-		domain      string
-		host        string
-		wantTyp     domainMatchType
-		wantMatched bool
-	}{
-		{name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
-		{name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
-		{name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true},
-		{name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true},
-		{name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false},
-		{name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true},
-		{name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false},
-		{name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true},
-		{name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched {
-				t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched)
-			}
-		})
-	}
-}
-
-func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
-	var (
-		oneExactMatch = &VirtualHost{
-			Domains: []string{"foo.bar.com"},
-		}
-		oneSuffixMatch = &VirtualHost{
-			Domains: []string{"*.bar.com"},
-		}
-		onePrefixMatch = &VirtualHost{
-			Domains: []string{"foo.bar.*"},
-		}
-		oneUniversalMatch = &VirtualHost{
-			Domains: []string{"*"},
-		}
-		longExactMatch = &VirtualHost{
-			Domains: []string{"v2.foo.bar.com"},
-		}
-		multipleMatch = &VirtualHost{
-			Domains: []string{"pi.foo.bar.com", "314.*", "*.159"},
-		}
-		vhs = []*VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
-	)
-
-	tests := []struct {
-		name   string
-		host   string
-		vHosts []*VirtualHost
-		want   *VirtualHost
-	}{
-		{name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch},
-		{name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch},
-		{name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch},
-		{name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch},
-		{name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch},
-		// Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact.
-		{name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch},
-		// Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix.
-		{name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch},
-		// Matches suffix "*.bar.com" and prefix "314.*". Takes suffix.
-		{name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := findBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) {
-				t.Errorf("findBestMatchingVirtualHost() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func (s) TestRoutesProtoToSlice(t *testing.T) {
 	tests := []struct {
 		name       string
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index a2a92a24f7fb..932975ce9151 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -193,18 +193,6 @@ func verifyRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel, w
 	return nil
 }
 
-func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate ServiceUpdate) error {
-	u, err := updateCh.Receive(ctx)
-	if err != nil {
-		return fmt.Errorf("timeout when waiting for service update: %v", err)
-	}
-	gotUpdate := u.(serviceUpdateErr)
-	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()) {
-		return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil)", gotUpdate.u, gotUpdate.err, wantUpdate)
-	}
-	return nil
-}
-
 func verifyClusterUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate ClusterUpdate) error {
 	u, err := updateCh.Receive(ctx)
 	if err != nil {
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index 33ac789d3794..a3524fd9ae38 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -205,22 +205,10 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 
 // WatchListener uses LDS to discover information about the provided listener.
 //
-// WatchListener is expected to called only from the server side implementation
-// of xDS. Clients will use WatchService instead.
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchListener(listener string, cb func(ListenerUpdate, error)) (cancel func()) {
-	return c.watchLDS(listener, cb)
-}
-
-// watchLDS starts a listener watcher for the service..
-//
 // Note that during race (e.g. an xDS response is received while the user is
 // calling cancel()), there's a small window where the callback can be called
 // after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) {
+func (c *Client) WatchListener(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
 		rType:       ListenerResource,
@@ -234,12 +222,12 @@ func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (c
 	return c.watch(wi)
 }
 
-// watchRDS starts a listener watcher for the service..
+// WatchRouteConfig starts a listener watcher for the service..
 //
 // Note that during race (e.g. an xDS response is received while the user is
 // calling cancel()), there's a small window where the callback can be called
 // after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) watchRDS(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) {
+func (c *Client) WatchRouteConfig(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
 		rType:       RouteConfigResource,
@@ -253,122 +241,6 @@ func (c *Client) watchRDS(routeName string, cb func(RouteConfigUpdate, error)) (
 	return c.watch(wi)
 }
 
-// WatchService uses LDS and RDS to discover information about the provided
-// serviceName.
-//
-// WatchService can only be called once. The second call will not start a
-// watcher and the callback will get an error. It's this case because an xDS
-// client is expected to be used only by one ClientConn.
-//
-// Note that during race (e.g. an xDS response is received while the user is
-// calling cancel()), there's a small window where the callback can be called
-// after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error)) (cancel func()) {
-	c.mu.Lock()
-	if len(c.ldsWatchers) != 0 {
-		go cb(ServiceUpdate{}, fmt.Errorf("unexpected WatchService when there's another service being watched"))
-		c.mu.Unlock()
-		return func() {}
-	}
-	c.mu.Unlock()
-
-	w := &serviceUpdateWatcher{c: c, serviceName: serviceName, serviceCb: cb}
-	w.ldsCancel = c.watchLDS(serviceName, w.handleLDSResp)
-
-	return w.close
-}
-
-// serviceUpdateWatcher handles LDS and RDS response, and calls the service
-// callback at the right time.
-//
-// TODO: move serviceUpdateWatcher and all its functions into xds resolver. The
-// resolver should be responsible for making WatchListener() and WatchRoute()
-// calls, and finding the best matching virtual host.
-type serviceUpdateWatcher struct {
-	c           *Client
-	serviceName string
-	ldsCancel   func()
-	serviceCb   func(ServiceUpdate, error)
-
-	mu        sync.Mutex
-	closed    bool
-	rdsName   string
-	rdsCancel func()
-}
-
-func (w *serviceUpdateWatcher) handleLDSResp(update ListenerUpdate, err error) {
-	w.c.logger.Infof("xds: client received LDS update: %+v, err: %v", update, err)
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	if w.closed {
-		return
-	}
-	if err != nil {
-		// We check the error type and do different things. For now, the only
-		// type we check is ResourceNotFound, which indicates the LDS resource
-		// was removed, and besides sending the error to callback, we also
-		// cancel the RDS watch.
-		if ErrType(err) == ErrorTypeResourceNotFound && w.rdsCancel != nil {
-			w.rdsCancel()
-			w.rdsName = ""
-			w.rdsCancel = nil
-		}
-		// The other error cases still return early without canceling the
-		// existing RDS watch.
-		w.serviceCb(ServiceUpdate{}, err)
-		return
-	}
-
-	if w.rdsName == update.RouteConfigName {
-		// If the new RouteConfigName is same as the previous, don't cancel and
-		// restart the RDS watch.
-		return
-	}
-	w.rdsName = update.RouteConfigName
-	if w.rdsCancel != nil {
-		w.rdsCancel()
-	}
-	w.rdsCancel = w.c.watchRDS(update.RouteConfigName, w.handleRDSResp)
-}
-
-func (w *serviceUpdateWatcher) handleRDSResp(update RouteConfigUpdate, err error) {
-	w.c.logger.Infof("xds: client received RDS update: %+v, err: %v", update, err)
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	if w.closed {
-		return
-	}
-	if w.rdsCancel == nil {
-		// This mean only the RDS watch is canceled, can happen if the LDS
-		// resource is removed.
-		return
-	}
-	if err != nil {
-		w.serviceCb(ServiceUpdate{}, err)
-		return
-	}
-
-	matchVh := findBestMatchingVirtualHost(w.serviceName, update.VirtualHosts)
-	if matchVh == nil {
-		// No matching virtual host found.
-		w.serviceCb(ServiceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName))
-		return
-	}
-
-	w.serviceCb(ServiceUpdate{Routes: matchVh.Routes}, nil)
-}
-
-func (w *serviceUpdateWatcher) close() {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	w.closed = true
-	w.ldsCancel()
-	if w.rdsCancel != nil {
-		w.rdsCancel()
-		w.rdsCancel = nil
-	}
-}
-
 // WatchCluster uses CDS to discover information about the provided
 // clusterName.
 //
diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_listener_test.go
similarity index 94%
rename from xds/internal/client/client_watchers_lds_test.go
rename to xds/internal/client/client_watchers_listener_test.go
index e922654d45b8..62a0fe493ef3 100644
--- a/xds/internal/client/client_watchers_lds_test.go
+++ b/xds/internal/client/client_watchers_listener_test.go
@@ -53,7 +53,7 @@ func (s) TestLDSWatch(t *testing.T) {
 	apiClient := c.(*testAPIClient)
 
 	ldsUpdateCh := testutils.NewChannel()
-	cancelWatch := client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+	cancelWatch := client.WatchListener(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
@@ -114,7 +114,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
-		cancelLastWatch = client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+		cancelLastWatch = client.WatchListener(testLDSName, func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
 
@@ -178,7 +178,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		ldsUpdateCh := testutils.NewChannel()
 		ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh)
-		client.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
+		client.WatchListener(testLDSName+"1", func(update ListenerUpdate, err error) {
 			ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 		})
 
@@ -193,7 +193,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 
 	// Third watch for a different name.
 	ldsUpdateCh2 := testutils.NewChannel()
-	client.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
+	client.WatchListener(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
@@ -238,7 +238,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	apiClient := c.(*testAPIClient)
 
 	ldsUpdateCh := testutils.NewChannel()
-	client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+	client.WatchListener(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
@@ -253,7 +253,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 
 	// Another watch for the resource in cache.
 	ldsUpdateCh2 := testutils.NewChannel()
-	client.watchLDS(testLDSName, func(update ListenerUpdate, err error) {
+	client.WatchListener(testLDSName, func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
@@ -300,7 +300,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	apiClient := c.(*testAPIClient)
 
 	ldsUpdateCh1 := testutils.NewChannel()
-	client.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) {
+	client.WatchListener(testLDSName+"1", func(update ListenerUpdate, err error) {
 		ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
@@ -308,7 +308,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	}
 	// Another watch for a different name.
 	ldsUpdateCh2 := testutils.NewChannel()
-	client.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) {
+	client.WatchListener(testLDSName+"2", func(update ListenerUpdate, err error) {
 		ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_route_test.go
similarity index 94%
rename from xds/internal/client/client_watchers_rds_test.go
rename to xds/internal/client/client_watchers_route_test.go
index 63b028d9c1ab..02425d6c8fe2 100644
--- a/xds/internal/client/client_watchers_rds_test.go
+++ b/xds/internal/client/client_watchers_route_test.go
@@ -55,7 +55,7 @@ func (s) TestRDSWatch(t *testing.T) {
 	apiClient := c.(*testAPIClient)
 
 	rdsUpdateCh := testutils.NewChannel()
-	cancelWatch := client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+	cancelWatch := client.WatchRouteConfig(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
@@ -121,7 +121,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
-		cancelLastWatch = client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+		cancelLastWatch = client.WatchRouteConfig(testRDSName, func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
 
@@ -191,7 +191,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	for i := 0; i < count; i++ {
 		rdsUpdateCh := testutils.NewChannel()
 		rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh)
-		client.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) {
+		client.WatchRouteConfig(testRDSName+"1", func(update RouteConfigUpdate, err error) {
 			rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 		})
 
@@ -206,7 +206,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 
 	// Third watch for a different name.
 	rdsUpdateCh2 := testutils.NewChannel()
-	client.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) {
+	client.WatchRouteConfig(testRDSName+"2", func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
@@ -265,7 +265,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	apiClient := c.(*testAPIClient)
 
 	rdsUpdateCh := testutils.NewChannel()
-	client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+	client.WatchRouteConfig(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err})
 	})
 	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
@@ -287,7 +287,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 
 	// Another watch for the resource in cache.
 	rdsUpdateCh2 := testutils.NewChannel()
-	client.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) {
+	client.WatchRouteConfig(testRDSName, func(update RouteConfigUpdate, err error) {
 		rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err})
 	})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go
deleted file mode 100644
index 1c935c60f31b..000000000000
--- a/xds/internal/client/client_watchers_service_test.go
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package client
-
-import (
-	"context"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-
-	"google.golang.org/grpc/internal/testutils"
-)
-
-type serviceUpdateErr struct {
-	u   ServiceUpdate
-	err error
-}
-
-// TestServiceWatch covers the cases:
-// - an update is received after a watch()
-// - an update with routes received
-func (s) TestServiceWatch(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, false))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-
-	wantUpdate2 := ServiceUpdate{
-		Routes: []*Route{{
-			Prefix: newStringP(""),
-			Action: map[string]uint32{testCDSName: 1},
-		}},
-	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS
-// response, the second LDS response trigger an new RDS watch, and an update of
-// the old RDS watch doesn't trigger update to service callback.
-func (s) TestServiceWatchLDSUpdate(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, false))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-
-	// Another LDS update with a different RDS_name.
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName + "2"}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-
-	// Another update for the old name.
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
-	defer sCancel()
-	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
-	}
-
-	// RDS update for the new name.
-	wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName + "2": {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}},
-				},
-			},
-		},
-	})
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// TestServiceWatchSecond covers the case where a second WatchService() gets an
-// error (because only one is allowed). But the first watch still receives
-// updates.
-func (s) TestServiceWatchSecond(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, false))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-
-	// Call WatchService() again, with the same or different name.
-	serviceUpdateCh2 := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh2.Send(serviceUpdateErr{u: update, err: err})
-	})
-	u, err := serviceUpdateCh2.Receive(ctx)
-	if err != nil {
-		t.Fatalf("failed to get serviceUpdate: %v", err)
-	}
-	uu := u.(serviceUpdateErr)
-	if !cmp.Equal(uu.u, ServiceUpdate{}) {
-		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
-	}
-	if uu.err == nil {
-		t.Errorf("unexpected serviceError: <nil>, want error watcher timeout")
-	}
-
-	// Send update again, first callback should be called, second should
-	// timeout.
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-
-	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
-	defer sCancel()
-	if u, err := serviceUpdateCh2.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
-	}
-}
-
-// TestServiceWatchWithNoResponseFromServer tests the case where the xDS server
-// does not respond to the requests being sent out as part of registering a
-// service update watcher. The callback will get an error.
-func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, true))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	u, err := serviceUpdateCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("failed to get serviceUpdate: %v", err)
-	}
-	uu := u.(serviceUpdateErr)
-	if !cmp.Equal(uu.u, ServiceUpdate{}) {
-		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
-	}
-	if uu.err == nil {
-		t.Errorf("unexpected serviceError: <nil>, want error watcher timeout")
-	}
-}
-
-// TestServiceWatchEmptyRDS tests the case where the underlying apiClient
-// receives an empty RDS response. The callback will get an error.
-func (s) TestServiceWatchEmptyRDS(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, true))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{})
-	u, err := serviceUpdateCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("failed to get serviceUpdate: %v", err)
-	}
-	uu := u.(serviceUpdateErr)
-	if !cmp.Equal(uu.u, ServiceUpdate{}) {
-		t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{})
-	}
-	if uu.err == nil {
-		t.Errorf("unexpected serviceError: <nil>, want error watcher timeout")
-	}
-}
-
-// TestServiceWatchWithClientClose tests the case where xDS responses are
-// received after the client is closed, and we make sure that the registered
-// watcher callback is not invoked.
-func (s) TestServiceWatchWithClientClose(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, true))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-
-	// Client is closed before it receives the RDS response.
-	client.Close()
-	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
-	defer sCancel()
-	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err)
-	}
-}
-
-// TestServiceNotCancelRDSOnSameLDSUpdate covers the case that if the second LDS
-// update contains the same RDS name as the previous, the RDS watch isn't
-// canceled and restarted.
-func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, false))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-
-	// Another LDS update with a the same RDS_name.
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
-	defer sCancel()
-	if _, err := apiClient.removeWatches[RouteConfigResource].Receive(sCtx); err != context.DeadlineExceeded {
-		t.Fatalf("unexpected rds watch cancel")
-	}
-}
-
-// TestServiceResourceRemoved covers the cases:
-// - an update is received after a watch()
-// - another update is received, with one resource removed
-//   - this should trigger callback with resource removed error
-// - one more update without the removed resource
-//   - the callback (above) shouldn't receive any update
-func (s) TestServiceResourceRemoved(t *testing.T) {
-	apiClientCh, cleanup := overrideNewAPIClient()
-	defer cleanup()
-
-	client, err := New(clientOpts(testXDSServer, false))
-	if err != nil {
-		t.Fatalf("failed to create client: %v", err)
-	}
-	defer client.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := apiClientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("timeout when waiting for API client to be created: %v", err)
-	}
-	apiClient := c.(*testAPIClient)
-
-	serviceUpdateCh := testutils.NewChannel()
-	client.WatchService(testLDSName, func(update ServiceUpdate, err error) {
-		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
-	})
-	if _, err := apiClient.addWatches[ListenerResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
-				},
-			},
-		},
-	})
-	wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-
-	// Remove LDS resource, should cancel the RDS watch, and trigger resource
-	// removed error.
-	client.NewListeners(map[string]ListenerUpdate{})
-	if _, err := apiClient.removeWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want watch to be canceled, got error %v", err)
-	}
-	if u, err := serviceUpdateCh.Receive(ctx); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound {
-		t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err)
-	}
-
-	// Send RDS update for the removed LDS resource, expect no updates to
-	// callback, because RDS should be canceled.
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}},
-				},
-			},
-		},
-	})
-	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
-	defer sCancel()
-	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
-	}
-
-	// Add LDS resource, but not RDS resource, should
-	//  - start a new RDS watch
-	//  - timeout on service channel, because RDS cache was cleared
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: {RouteConfigName: testRDSName}})
-	if _, err := apiClient.addWatches[RouteConfigResource].Receive(ctx); err != nil {
-		t.Fatalf("want new watch to start, got error %v", err)
-	}
-	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
-	defer sCancel()
-	if u, err := serviceUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u)
-	}
-
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{
-		testRDSName: {
-			VirtualHosts: []*VirtualHost{
-				{
-					Domains: []string{testLDSName},
-					Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}},
-				},
-			},
-		},
-	})
-	wantUpdate = ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}
-	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
-		t.Fatal(err)
-	}
-}
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 0acfd69cf89b..6f6245374342 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"net"
 	"strconv"
-	"strings"
 
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -260,100 +259,6 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 	return routesRet, nil
 }
 
-type domainMatchType int
-
-const (
-	domainMatchTypeInvalid domainMatchType = iota
-	domainMatchTypeUniversal
-	domainMatchTypePrefix
-	domainMatchTypeSuffix
-	domainMatchTypeExact
-)
-
-// Exact > Suffix > Prefix > Universal > Invalid.
-func (t domainMatchType) betterThan(b domainMatchType) bool {
-	return t > b
-}
-
-func matchTypeForDomain(d string) domainMatchType {
-	if d == "" {
-		return domainMatchTypeInvalid
-	}
-	if d == "*" {
-		return domainMatchTypeUniversal
-	}
-	if strings.HasPrefix(d, "*") {
-		return domainMatchTypeSuffix
-	}
-	if strings.HasSuffix(d, "*") {
-		return domainMatchTypePrefix
-	}
-	if strings.Contains(d, "*") {
-		return domainMatchTypeInvalid
-	}
-	return domainMatchTypeExact
-}
-
-func match(domain, host string) (domainMatchType, bool) {
-	switch typ := matchTypeForDomain(domain); typ {
-	case domainMatchTypeInvalid:
-		return typ, false
-	case domainMatchTypeUniversal:
-		return typ, true
-	case domainMatchTypePrefix:
-		// abc.*
-		return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*"))
-	case domainMatchTypeSuffix:
-		// *.123
-		return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*"))
-	case domainMatchTypeExact:
-		return typ, domain == host
-	default:
-		return domainMatchTypeInvalid, false
-	}
-}
-
-// findBestMatchingVirtualHost returns the virtual host whose domains field best
-// matches host
-//
-// The domains field support 4 different matching pattern types:
-//  - Exact match
-//  - Suffix match (e.g. “*ABC”)
-//  - Prefix match (e.g. “ABC*)
-//  - Universal match (e.g. “*”)
-//
-// The best match is defined as:
-//  - A match is better if it’s matching pattern type is better
-//    - Exact match > suffix match > prefix match > universal match
-//  - If two matches are of the same pattern type, the longer match is better
-//    - This is to compare the length of the matching pattern, e.g. “*ABCDE” >
-//    “*ABC”
-func findBestMatchingVirtualHost(host string, vHosts []*VirtualHost) *VirtualHost {
-	var (
-		matchVh   *VirtualHost
-		matchType = domainMatchTypeInvalid
-		matchLen  int
-	)
-	for _, vh := range vHosts {
-		for _, domain := range vh.Domains {
-			typ, matched := match(domain, host)
-			if typ == domainMatchTypeInvalid {
-				// The rds response is invalid.
-				return nil
-			}
-			if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched {
-				// The previous match has better type, or the previous match has
-				// better length, or this domain isn't a match.
-				continue
-			}
-			matchVh = vh
-			matchType = typ
-			matchLen = len(domain)
-		}
-	}
-	return matchVh
-}
-
 // UnmarshalCluster processes resources received in an CDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 805d8d41104c..965817c6c8ba 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -137,6 +137,6 @@ func weightedClusterToBalancerConfig(wc map[string]uint32) balancerConfig {
 	return bc
 }
 
-func (r *xdsResolver) serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) {
+func (r *xdsResolver) serviceUpdateToJSON(su serviceUpdate) (string, error) {
 	return r.routesToJSON(su.Routes)
 }
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index 1fc03d1fac16..5969bd83caed 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -311,13 +311,13 @@ func TestRoutesToJSON(t *testing.T) {
 func TestServiceUpdateToJSON(t *testing.T) {
 	tests := []struct {
 		name     string
-		su       xdsclient.ServiceUpdate
+		su       serviceUpdate
 		wantJSON string
 		wantErr  bool
 	}{
 		{
 			name: "routing",
-			su: xdsclient.ServiceUpdate{
+			su: serviceUpdate{
 				Routes: []*xdsclient.Route{{
 					Path:   newStringP("/service_1/method_1"),
 					Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25},
diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go
new file mode 100644
index 000000000000..01aad899e8be
--- /dev/null
+++ b/xds/internal/resolver/watch_service.go
@@ -0,0 +1,236 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"google.golang.org/grpc/internal/grpclog"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+// serviceUpdate contains information received from the RDS responses which is
+// of interested to the xds resolver. The RDS request is built by first making a
+// LDS to get the RouteConfig name.
+type serviceUpdate struct {
+	// Routes contain matchers+actions to route RPCs.
+	Routes []*xdsclient.Route
+}
+
+// watchService uses LDS and RDS to discover information about the provided
+// serviceName.
+//
+// Note that during race (e.g. an xDS response is received while the user is
+// calling cancel()), there's a small window where the callback can be called
+// after the watcher is canceled. The caller needs to handle this case.
+func watchService(c xdsClientInterface, serviceName string, cb func(serviceUpdate, error), logger *grpclog.PrefixLogger) (cancel func()) {
+	w := &serviceUpdateWatcher{
+		logger:      logger,
+		c:           c,
+		serviceName: serviceName,
+		serviceCb:   cb,
+	}
+	w.ldsCancel = c.WatchListener(serviceName, w.handleLDSResp)
+
+	return w.close
+}
+
+// serviceUpdateWatcher handles LDS and RDS response, and calls the service
+// callback at the right time.
+type serviceUpdateWatcher struct {
+	logger      *grpclog.PrefixLogger
+	c           xdsClientInterface
+	serviceName string
+	ldsCancel   func()
+	serviceCb   func(serviceUpdate, error)
+
+	mu        sync.Mutex
+	closed    bool
+	rdsName   string
+	rdsCancel func()
+}
+
+func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, err error) {
+	w.logger.Infof("received LDS update: %+v, err: %v", update, err)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return
+	}
+	if err != nil {
+		// We check the error type and do different things. For now, the only
+		// type we check is ResourceNotFound, which indicates the LDS resource
+		// was removed, and besides sending the error to callback, we also
+		// cancel the RDS watch.
+		if xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound && w.rdsCancel != nil {
+			w.rdsCancel()
+			w.rdsName = ""
+			w.rdsCancel = nil
+		}
+		// The other error cases still return early without canceling the
+		// existing RDS watch.
+		w.serviceCb(serviceUpdate{}, err)
+		return
+	}
+
+	if w.rdsName == update.RouteConfigName {
+		// If the new RouteConfigName is same as the previous, don't cancel and
+		// restart the RDS watch.
+		return
+	}
+	w.rdsName = update.RouteConfigName
+	if w.rdsCancel != nil {
+		w.rdsCancel()
+	}
+	w.rdsCancel = w.c.WatchRouteConfig(update.RouteConfigName, w.handleRDSResp)
+}
+
+func (w *serviceUpdateWatcher) handleRDSResp(update xdsclient.RouteConfigUpdate, err error) {
+	w.logger.Infof("received RDS update: %+v, err: %v", update, err)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closed {
+		return
+	}
+	if w.rdsCancel == nil {
+		// This mean only the RDS watch is canceled, can happen if the LDS
+		// resource is removed.
+		return
+	}
+	if err != nil {
+		w.serviceCb(serviceUpdate{}, err)
+		return
+	}
+
+	matchVh := findBestMatchingVirtualHost(w.serviceName, update.VirtualHosts)
+	if matchVh == nil {
+		// No matching virtual host found.
+		w.serviceCb(serviceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName))
+		return
+	}
+
+	w.serviceCb(serviceUpdate{Routes: matchVh.Routes}, nil)
+}
+
+func (w *serviceUpdateWatcher) close() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.closed = true
+	w.ldsCancel()
+	if w.rdsCancel != nil {
+		w.rdsCancel()
+		w.rdsCancel = nil
+	}
+}
+
+type domainMatchType int
+
+const (
+	domainMatchTypeInvalid domainMatchType = iota
+	domainMatchTypeUniversal
+	domainMatchTypePrefix
+	domainMatchTypeSuffix
+	domainMatchTypeExact
+)
+
+// Exact > Suffix > Prefix > Universal > Invalid.
+func (t domainMatchType) betterThan(b domainMatchType) bool {
+	return t > b
+}
+
+func matchTypeForDomain(d string) domainMatchType {
+	if d == "" {
+		return domainMatchTypeInvalid
+	}
+	if d == "*" {
+		return domainMatchTypeUniversal
+	}
+	if strings.HasPrefix(d, "*") {
+		return domainMatchTypeSuffix
+	}
+	if strings.HasSuffix(d, "*") {
+		return domainMatchTypePrefix
+	}
+	if strings.Contains(d, "*") {
+		return domainMatchTypeInvalid
+	}
+	return domainMatchTypeExact
+}
+
+func match(domain, host string) (domainMatchType, bool) {
+	switch typ := matchTypeForDomain(domain); typ {
+	case domainMatchTypeInvalid:
+		return typ, false
+	case domainMatchTypeUniversal:
+		return typ, true
+	case domainMatchTypePrefix:
+		// abc.*
+		return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*"))
+	case domainMatchTypeSuffix:
+		// *.123
+		return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*"))
+	case domainMatchTypeExact:
+		return typ, domain == host
+	default:
+		return domainMatchTypeInvalid, false
+	}
+}
+
+// findBestMatchingVirtualHost returns the virtual host whose domains field best
+// matches host
+//
+// The domains field support 4 different matching pattern types:
+//  - Exact match
+//  - Suffix match (e.g. “*ABC”)
+//  - Prefix match (e.g. “ABC*)
+//  - Universal match (e.g. “*”)
+//
+// The best match is defined as:
+//  - A match is better if it’s matching pattern type is better
+//    - Exact match > suffix match > prefix match > universal match
+//  - If two matches are of the same pattern type, the longer match is better
+//    - This is to compare the length of the matching pattern, e.g. “*ABCDE” >
+//    “*ABC”
+func findBestMatchingVirtualHost(host string, vHosts []*xdsclient.VirtualHost) *xdsclient.VirtualHost {
+	var (
+		matchVh   *xdsclient.VirtualHost
+		matchType = domainMatchTypeInvalid
+		matchLen  int
+	)
+	for _, vh := range vHosts {
+		for _, domain := range vh.Domains {
+			typ, matched := match(domain, host)
+			if typ == domainMatchTypeInvalid {
+				// The rds response is invalid.
+				return nil
+			}
+			if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched {
+				// The previous match has better type, or the previous match has
+				// better length, or this domain isn't a match.
+				continue
+			}
+			matchVh = vh
+			matchType = typ
+			matchLen = len(domain)
+		}
+	}
+	return matchVh
+}
diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go
new file mode 100644
index 000000000000..6fd2103bcd58
--- /dev/null
+++ b/xds/internal/resolver/watch_service_test.go
@@ -0,0 +1,291 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/testutils"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+	"google.golang.org/protobuf/proto"
+)
+
+func (s) TestMatchTypeForDomain(t *testing.T) {
+	tests := []struct {
+		d    string
+		want domainMatchType
+	}{
+		{d: "", want: domainMatchTypeInvalid},
+		{d: "*", want: domainMatchTypeUniversal},
+		{d: "bar.*", want: domainMatchTypePrefix},
+		{d: "*.abc.com", want: domainMatchTypeSuffix},
+		{d: "foo.bar.com", want: domainMatchTypeExact},
+		{d: "foo.*.com", want: domainMatchTypeInvalid},
+	}
+	for _, tt := range tests {
+		if got := matchTypeForDomain(tt.d); got != tt.want {
+			t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want)
+		}
+	}
+}
+
+func (s) TestMatch(t *testing.T) {
+	tests := []struct {
+		name        string
+		domain      string
+		host        string
+		wantTyp     domainMatchType
+		wantMatched bool
+	}{
+		{name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
+		{name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
+		{name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true},
+		{name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true},
+		{name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false},
+		{name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true},
+		{name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false},
+		{name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true},
+		{name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched {
+				t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched)
+			}
+		})
+	}
+}
+
+func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
+	var (
+		oneExactMatch = &xdsclient.VirtualHost{
+			Domains: []string{"foo.bar.com"},
+		}
+		oneSuffixMatch = &xdsclient.VirtualHost{
+			Domains: []string{"*.bar.com"},
+		}
+		onePrefixMatch = &xdsclient.VirtualHost{
+			Domains: []string{"foo.bar.*"},
+		}
+		oneUniversalMatch = &xdsclient.VirtualHost{
+			Domains: []string{"*"},
+		}
+		longExactMatch = &xdsclient.VirtualHost{
+			Domains: []string{"v2.foo.bar.com"},
+		}
+		multipleMatch = &xdsclient.VirtualHost{
+			Domains: []string{"pi.foo.bar.com", "314.*", "*.159"},
+		}
+		vhs = []*xdsclient.VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
+	)
+
+	tests := []struct {
+		name   string
+		host   string
+		vHosts []*xdsclient.VirtualHost
+		want   *xdsclient.VirtualHost
+	}{
+		{name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch},
+		{name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch},
+		{name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch},
+		{name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch},
+		{name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch},
+		// Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact.
+		{name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch},
+		// Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix.
+		{name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch},
+		// Matches suffix "*.bar.com" and prefix "314.*". Takes suffix.
+		{name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := findBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) {
+				t.Errorf("findBestMatchingxdsclient.VirtualHost() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type serviceUpdateErr struct {
+	u   serviceUpdate
+	err error
+}
+
+func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate serviceUpdate) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for service update: %v", err)
+	}
+	gotUpdate := u.(serviceUpdateErr)
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()) {
+		return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil),  diff (-want +got):\n%s", gotUpdate.u, gotUpdate.err, wantUpdate, cmp.Diff(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()))
+	}
+	return nil
+}
+
+// TestServiceWatch covers the cases:
+// - an update is received after a watch()
+// - an update with routes received
+func (s) TestServiceWatch(t *testing.T) {
+	serviceUpdateCh := testutils.NewChannel()
+	xdsC := fakeclient.NewClient()
+	cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	}, nil)
+	defer cancelWatch()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	wantUpdate := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	wantUpdate2 := serviceUpdate{
+		Routes: []*xdsclient.Route{{
+			Path:   newStringP(""),
+			Action: map[string]uint32{cluster: 1},
+		}},
+	}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Path: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+			{
+				// Another virtual host, with different domains.
+				Domains: []string{"random"},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS
+// response, the second LDS response trigger an new RDS watch, and an update of
+// the old RDS watch doesn't trigger update to service callback.
+func (s) TestServiceWatchLDSUpdate(t *testing.T) {
+	serviceUpdateCh := testutils.NewChannel()
+	xdsC := fakeclient.NewClient()
+	cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	}, nil)
+	defer cancelWatch()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	wantUpdate := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Another LDS update with a different RDS_name.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr + "2"}, nil)
+	if err := xdsC.WaitForCancelRouteConfigWatch(ctx); err != nil {
+		t.Fatalf("wait for cancel route watch failed: %v, want nil", err)
+	}
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr+"2")
+
+	// RDS update for the new name.
+	wantUpdate2 := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestServiceNotCancelRDSOnSameLDSUpdate covers the case that if the second LDS
+// update contains the same RDS name as the previous, the RDS watch isn't
+// canceled and restarted.
+func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
+	serviceUpdateCh := testutils.NewChannel()
+	xdsC := fakeclient.NewClient()
+	cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	}, nil)
+	defer cancelWatch()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	wantUpdate := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
+
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Another LDS update with a the same RDS_name.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelRouteConfigWatch(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("wait for cancel route watch failed: %v, want nil", err)
+	}
+}
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 359ee129fb2f..c3e83231fd22 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -80,7 +80,7 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 	r.client = client
 
 	// Register a watch on the xdsClient for the user's dial target.
-	cancelWatch := r.client.WatchService(r.target.Endpoint, r.handleServiceUpdate)
+	cancelWatch := watchService(r.client, r.target.Endpoint, r.handleServiceUpdate, r.logger)
 	r.logger.Infof("Watch started on resource name %v with xds-client %p", r.target.Endpoint, r.client)
 	r.cancelWatch = func() {
 		cancelWatch()
@@ -99,14 +99,15 @@ func (*xdsResolverBuilder) Scheme() string {
 // xdsClientInterface contains methods from xdsClient.Client which are used by
 // the resolver. This will be faked out in unittests.
 type xdsClientInterface interface {
-	WatchService(string, func(xdsclient.ServiceUpdate, error)) func()
+	WatchListener(serviceName string, cb func(xdsclient.ListenerUpdate, error)) func()
+	WatchRouteConfig(routeName string, cb func(xdsclient.RouteConfigUpdate, error)) func()
 	Close()
 }
 
 // suWithError wraps the ServiceUpdate and error received through a watch API
 // callback, so that it can pushed onto the update channel as a single entity.
 type suWithError struct {
-	su  xdsclient.ServiceUpdate
+	su  serviceUpdate
 	err error
 }
 
@@ -184,7 +185,7 @@ func (r *xdsResolver) run() {
 // handleServiceUpdate is the callback which handles service updates. It writes
 // the received update to the update channel, which is picked by the run
 // goroutine.
-func (r *xdsResolver) handleServiceUpdate(su xdsclient.ServiceUpdate, err error) {
+func (r *xdsResolver) handleServiceUpdate(su serviceUpdate, err error) {
 	if r.closed.HasFired() {
 		// Do not pass updates to the ClientConn once the resolver is closed.
 		return
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 7eb0629a560a..776ac24fc714 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -44,10 +44,12 @@ import (
 )
 
 const (
-	targetStr          = "target"
-	cluster            = "cluster"
-	balancerName       = "dummyBalancer"
-	defaultTestTimeout = 1 * time.Second
+	targetStr               = "target"
+	routeStr                = "route"
+	cluster                 = "cluster"
+	balancerName            = "dummyBalancer"
+	defaultTestTimeout      = 1 * time.Second
+	defaultTestShortTimeout = 100 * time.Microsecond
 )
 
 var (
@@ -254,13 +256,28 @@ func testSetup(t *testing.T, opts setupOpts) (*xdsResolver, *testClientConn, fun
 	return r.(*xdsResolver), tcc, cancel
 }
 
-// waitForWatchService waits for the WatchService method to be called on the
+// waitForWatchListener waits for the WatchListener method to be called on the
 // xdsClient within a reasonable amount of time, and also verifies that the
 // watch is called with the expected target.
-func waitForWatchService(t *testing.T, xdsC *fakeclient.Client, wantTarget string) {
+func waitForWatchListener(ctx context.Context, t *testing.T, xdsC *fakeclient.Client, wantTarget string) {
 	t.Helper()
 
-	gotTarget, err := xdsC.WaitForWatchService(context.Background())
+	gotTarget, err := xdsC.WaitForWatchListener(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchService failed with error: %v", err)
+	}
+	if gotTarget != wantTarget {
+		t.Fatalf("xdsClient.WatchService() called with target: %v, want %v", gotTarget, wantTarget)
+	}
+}
+
+// waitForWatchRouteConfig waits for the WatchRoute method to be called on the
+// xdsClient within a reasonable amount of time, and also verifies that the
+// watch is called with the expected target.
+func waitForWatchRouteConfig(ctx context.Context, t *testing.T, xdsC *fakeclient.Client, wantTarget string) {
+	t.Helper()
+
+	gotTarget, err := xdsC.WaitForWatchRouteConfig(ctx)
 	if err != nil {
 		t.Fatalf("xdsClient.WatchService failed with error: %v", err)
 	}
@@ -279,15 +296,24 @@ func (s) TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	})
 	defer cancel()
 
-	waitForWatchService(t, xdsC, targetStr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Call the watchAPI callback after closing the resolver, and make sure no
 	// update is triggerred on the ClientConn.
 	xdsR.Close()
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if gotVal, gotErr := tcc.stateCh.Receive(ctx); gotErr != context.DeadlineExceeded {
 		t.Fatalf("ClientConn.UpdateState called after xdsResolver is closed: %v", gotVal)
 	}
@@ -306,15 +332,17 @@ func (s) TestXDSResolverBadServiceUpdate(t *testing.T) {
 		xdsR.Close()
 	}()
 
-	waitForWatchService(t, xdsC, targetStr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr := errors.New("bad serviceupdate")
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != nil || gotErrVal != suErr {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr)
 	}
@@ -333,28 +361,39 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 		xdsR.Close()
 	}()
 
-	waitForWatchService(t, xdsC, targetStr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 	defer replaceRandNumGenerator(0)()
 
 	for _, tt := range []struct {
-		su       xdsclient.ServiceUpdate
+		routes   []*xdsclient.Route
 		wantJSON string
 	}{
 		{
-			su:       xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}}},
+			routes:   []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}},
 			wantJSON: testOneClusterOnlyJSON,
 		},
 		{
-			su: client.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
+			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
 				"cluster_1": 75,
 				"cluster_2": 25,
-			}}}},
+			}}},
 			wantJSON: testWeightedCDSJSON,
 		},
 	} {
 		// Invoke the watchAPI callback with a good service update and wait for the
 		// UpdateState method to be called on the ClientConn.
-		xdsC.InvokeWatchServiceCallback(tt.su, nil)
+		xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+			VirtualHosts: []*xdsclient.VirtualHost{
+				{
+					Domains: []string{targetStr},
+					Routes:  tt.routes,
+				},
+			},
+		}, nil)
 
 		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 		defer cancel()
@@ -392,22 +431,31 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 		xdsR.Close()
 	}()
 
-	waitForWatchService(t, xdsC, targetStr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr := errors.New("bad serviceupdate")
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != nil || gotErrVal != suErr {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr)
 	}
 
 	// Invoke the watchAPI callback with a good service update and wait for the
 	// UpdateState method to be called on the ClientConn.
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
@@ -423,7 +471,7 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr2 := errors.New("bad serviceupdate 2")
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr2)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr2)
 	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != nil || gotErrVal != suErr2 {
 		t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr2)
 	}
@@ -443,15 +491,17 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 		xdsR.Close()
 	}()
 
-	waitForWatchService(t, xdsC, targetStr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a bad service update and wait for the
 	// ReportError method to be called on the ClientConn.
 	suErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "resource removed error")
-	xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if gotErrVal, gotErr := tcc.errorCh.Receive(ctx); gotErr != context.DeadlineExceeded {
 		t.Fatalf("ClientConn.ReportError() received %v, %v, want channel recv timeout", gotErrVal, gotErr)
 	}
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 408817c17846..c1fd0510c844 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -21,7 +21,6 @@ package fakeclient
 
 import (
 	"context"
-	"sync"
 
 	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -32,57 +31,94 @@ import (
 // channels to signal the occurrence of various events.
 type Client struct {
 	name         string
-	suWatchCh    *testutils.Channel
+	ldsWatchCh   *testutils.Channel
+	rdsWatchCh   *testutils.Channel
 	cdsWatchCh   *testutils.Channel
 	edsWatchCh   *testutils.Channel
-	suCancelCh   *testutils.Channel
+	ldsCancelCh  *testutils.Channel
+	rdsCancelCh  *testutils.Channel
 	cdsCancelCh  *testutils.Channel
 	edsCancelCh  *testutils.Channel
 	loadReportCh *testutils.Channel
 	closeCh      *testutils.Channel
 	loadStore    *load.Store
 
-	mu        sync.Mutex
-	serviceCb func(xdsclient.ServiceUpdate, error)
-	cdsCb     func(xdsclient.ClusterUpdate, error)
-	edsCb     func(xdsclient.EndpointsUpdate, error)
+	ldsCb func(xdsclient.ListenerUpdate, error)
+	rdsCb func(xdsclient.RouteConfigUpdate, error)
+	cdsCb func(xdsclient.ClusterUpdate, error)
+	edsCb func(xdsclient.EndpointsUpdate, error)
 }
 
-// WatchService registers a LDS/RDS watch.
-func (xdsC *Client) WatchService(target string, callback func(xdsclient.ServiceUpdate, error)) func() {
-	xdsC.mu.Lock()
-	defer xdsC.mu.Unlock()
-
-	xdsC.serviceCb = callback
-	xdsC.suWatchCh.Send(target)
+// WatchListener registers a LDS watch.
+func (xdsC *Client) WatchListener(serviceName string, callback func(xdsclient.ListenerUpdate, error)) func() {
+	xdsC.ldsCb = callback
+	xdsC.ldsWatchCh.Send(serviceName)
 	return func() {
-		xdsC.suCancelCh.Send(nil)
+		xdsC.ldsCancelCh.Send(nil)
 	}
 }
 
-// WaitForWatchService waits for WatchService to be invoked on this client and
+// WaitForWatchListener waits for WatchCluster to be invoked on this client and
 // returns the serviceName being watched.
-func (xdsC *Client) WaitForWatchService(ctx context.Context) (string, error) {
-	val, err := xdsC.suWatchCh.Receive(ctx)
+func (xdsC *Client) WaitForWatchListener(ctx context.Context) (string, error) {
+	val, err := xdsC.ldsWatchCh.Receive(ctx)
+	if err != nil {
+		return "", err
+	}
+	return val.(string), err
+}
+
+// InvokeWatchListenerCallback invokes the registered ldsWatch callback.
+//
+// Not thread safe with WatchListener. Only call this after
+// WaitForWatchListener.
+func (xdsC *Client) InvokeWatchListenerCallback(update xdsclient.ListenerUpdate, err error) {
+	xdsC.ldsCb(update, err)
+}
+
+// WaitForCancelListenerWatch waits for a LDS watch to be cancelled  and returns
+// context.DeadlineExceeded otherwise.
+func (xdsC *Client) WaitForCancelListenerWatch(ctx context.Context) error {
+	_, err := xdsC.ldsCancelCh.Receive(ctx)
+	return err
+}
+
+// WatchRouteConfig registers a RDS watch.
+func (xdsC *Client) WatchRouteConfig(routeName string, callback func(xdsclient.RouteConfigUpdate, error)) func() {
+	xdsC.rdsCb = callback
+	xdsC.rdsWatchCh.Send(routeName)
+	return func() {
+		xdsC.rdsCancelCh.Send(nil)
+	}
+}
+
+// WaitForWatchRouteConfig waits for WatchCluster to be invoked on this client and
+// returns the routeName being watched.
+func (xdsC *Client) WaitForWatchRouteConfig(ctx context.Context) (string, error) {
+	val, err := xdsC.rdsWatchCh.Receive(ctx)
 	if err != nil {
 		return "", err
 	}
 	return val.(string), err
 }
 
-// InvokeWatchServiceCallback invokes the registered service watch callback.
-func (xdsC *Client) InvokeWatchServiceCallback(u xdsclient.ServiceUpdate, err error) {
-	xdsC.mu.Lock()
-	defer xdsC.mu.Unlock()
+// InvokeWatchRouteConfigCallback invokes the registered rdsWatch callback.
+//
+// Not thread safe with WatchRouteConfig. Only call this after
+// WaitForWatchRouteConfig.
+func (xdsC *Client) InvokeWatchRouteConfigCallback(update xdsclient.RouteConfigUpdate, err error) {
+	xdsC.rdsCb(update, err)
+}
 
-	xdsC.serviceCb(u, err)
+// WaitForCancelRouteConfigWatch waits for a RDS watch to be cancelled  and returns
+// context.DeadlineExceeded otherwise.
+func (xdsC *Client) WaitForCancelRouteConfigWatch(ctx context.Context) error {
+	_, err := xdsC.rdsCancelCh.Receive(ctx)
+	return err
 }
 
 // WatchCluster registers a CDS watch.
 func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsclient.ClusterUpdate, error)) func() {
-	xdsC.mu.Lock()
-	defer xdsC.mu.Unlock()
-
 	xdsC.cdsCb = callback
 	xdsC.cdsWatchCh.Send(clusterName)
 	return func() {
@@ -101,10 +137,10 @@ func (xdsC *Client) WaitForWatchCluster(ctx context.Context) (string, error) {
 }
 
 // InvokeWatchClusterCallback invokes the registered cdsWatch callback.
+//
+// Not thread safe with WatchCluster. Only call this after
+// WaitForWatchCluster.
 func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.ClusterUpdate, err error) {
-	xdsC.mu.Lock()
-	defer xdsC.mu.Unlock()
-
 	xdsC.cdsCb(update, err)
 }
 
@@ -117,9 +153,6 @@ func (xdsC *Client) WaitForCancelClusterWatch(ctx context.Context) error {
 
 // WatchEndpoints registers an EDS watch for provided clusterName.
 func (xdsC *Client) WatchEndpoints(clusterName string, callback func(xdsclient.EndpointsUpdate, error)) (cancel func()) {
-	xdsC.mu.Lock()
-	defer xdsC.mu.Unlock()
-
 	xdsC.edsCb = callback
 	xdsC.edsWatchCh.Send(clusterName)
 	return func() {
@@ -138,10 +171,10 @@ func (xdsC *Client) WaitForWatchEDS(ctx context.Context) (string, error) {
 }
 
 // InvokeWatchEDSCallback invokes the registered edsWatch callback.
+//
+// Not thread safe with WatchEndpoints. Only call this after
+// WaitForWatchEDS.
 func (xdsC *Client) InvokeWatchEDSCallback(update xdsclient.EndpointsUpdate, err error) {
-	xdsC.mu.Lock()
-	defer xdsC.mu.Unlock()
-
 	xdsC.edsCb(update, err)
 }
 
@@ -204,10 +237,12 @@ func NewClient() *Client {
 func NewClientWithName(name string) *Client {
 	return &Client{
 		name:         name,
-		suWatchCh:    testutils.NewChannel(),
+		ldsWatchCh:   testutils.NewChannel(),
+		rdsWatchCh:   testutils.NewChannel(),
 		cdsWatchCh:   testutils.NewChannel(),
 		edsWatchCh:   testutils.NewChannel(),
-		suCancelCh:   testutils.NewChannel(),
+		ldsCancelCh:  testutils.NewChannel(),
+		rdsCancelCh:  testutils.NewChannel(),
 		cdsCancelCh:  testutils.NewChannel(),
 		edsCancelCh:  testutils.NewChannel(),
 		loadReportCh: testutils.NewChannel(),

From 15a78f19307d5faf10cfdd9d4e664c65a387cbd1 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 27 Oct 2020 17:29:21 -0700
Subject: [PATCH 251/481] xds: Minor cleanup in client/v2 tests. (#3989)

---
 xds/internal/client/v2/client_ack_test.go | 122 ++++++++++++----------
 xds/internal/client/v2/client_rds_test.go |   9 +-
 xds/internal/client/v2/client_test.go     |  18 ++--
 3 files changed, 84 insertions(+), 65 deletions(-)

diff --git a/xds/internal/client/v2/client_ack_test.go b/xds/internal/client/v2/client_ack_test.go
index 87437aa218c7..57a6d1c200dc 100644
--- a/xds/internal/client/v2/client_ack_test.go
+++ b/xds/internal/client/v2/client_ack_test.go
@@ -36,7 +36,10 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-const defaultTestTimeout = 1 * time.Second
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
 
 func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cbRDS, cbCDS, cbEDS *testutils.Channel, cleanup func()) {
 	cbLDS = testutils.NewChannel()
@@ -74,9 +77,7 @@ func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cb
 }
 
 // compareXDSRequest reads requests from channel, compare it with want.
-func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string, wantErr bool) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+func compareXDSRequest(ctx context.Context, ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string, wantErr bool) error {
 	val, err := ch.Receive(ctx)
 	if err != nil {
 		return err
@@ -116,7 +117,7 @@ func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *
 
 // startXDS calls watch to send the first request. It then sends a good response
 // and checks for ack.
-func startXDS(t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
+func startXDS(ctx context.Context, t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) {
 	nameToWatch := ""
 	switch rType {
 	case xdsclient.ListenerResource:
@@ -130,7 +131,7 @@ func startXDS(t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *
 	}
 	v2c.AddWatch(rType, nameToWatch)
 
-	if err := compareXDSRequest(reqChan, req, preVersion, preNonce, false); err != nil {
+	if err := compareXDSRequest(ctx, reqChan, req, preVersion, preNonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", rType, err)
 	}
 	t.Logf("FakeServer received %v request...", rType)
@@ -141,17 +142,15 @@ func startXDS(t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *
 //
 // It also waits and checks that the ack request contains the given version, and
 // the generated nonce.
-func sendGoodResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
+func sendGoodResp(ctx context.Context, t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) {
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, ver)
 	t.Logf("Good %v response pushed to fakeServer...", rType)
 
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce, false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce, false); err != nil {
 		return "", fmt.Errorf("failed to receive %v request: %v", rType, err)
 	}
 	t.Logf("Good %v response acked", rType)
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if _, err := callbackCh.Receive(ctx); err != nil {
 		return "", fmt.Errorf("timeout when expecting %v update", rType)
 	}
@@ -163,7 +162,7 @@ func sendGoodResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakese
 // be nacked, so we expect a request with the previous version (version-1).
 //
 // But the nonce in request should be the new nonce.
-func sendBadResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, wantReq *xdspb.DiscoveryRequest) error {
+func sendBadResp(ctx context.Context, t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, wantReq *xdspb.DiscoveryRequest) error {
 	var typeURL string
 	switch rType {
 	case xdsclient.ListenerResource:
@@ -180,7 +179,7 @@ func sendBadResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeser
 		TypeUrl:   typeURL,
 	}, ver)
 	t.Logf("Bad %v response pushed to fakeServer...", rType)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce, true); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce, true); err != nil {
 		return fmt.Errorf("failed to receive %v request: %v", rType, err)
 	}
 	t.Logf("Bad %v response nacked", rType)
@@ -205,60 +204,63 @@ func (s) TestV2ClientAck(t *testing.T) {
 	v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2cCleanup := startXDSV2Client(t, cc)
 	defer v2cCleanup()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// Start the watch, send a good response, and check for ack.
-	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
-	if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+	startXDS(ctx, t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	if _, err := sendGoodResp(ctx, t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
-	startXDS(t, xdsclient.RouteConfigResource, v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
-	if _, err := sendGoodResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
+	startXDS(ctx, t, xdsclient.RouteConfigResource, v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "")
+	if _, err := sendGoodResp(ctx, t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
 		t.Fatal(err)
 	}
 	versionRDS++
-	startXDS(t, xdsclient.ClusterResource, v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
-	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	startXDS(ctx, t, xdsclient.ClusterResource, v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "")
+	if _, err := sendGoodResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
-	startXDS(t, xdsclient.EndpointsResource, v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
-	if _, err := sendGoodResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
+	startXDS(ctx, t, xdsclient.EndpointsResource, v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "")
+	if _, err := sendGoodResp(ctx, t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
 		t.Fatal(err)
 	}
 	versionEDS++
 
 	// Send a bad response, and check for nack.
-	if err := sendBadResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSRequest); err != nil {
+	if err := sendBadResp(ctx, t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
-	if err := sendBadResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSRequest); err != nil {
+	if err := sendBadResp(ctx, t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionRDS++
-	if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
+	if err := sendBadResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
-	if err := sendBadResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSRequest); err != nil {
+	if err := sendBadResp(ctx, t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionEDS++
 
 	// send another good response, and check for ack, with the new version.
-	if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
-	if _, err := sendGoodResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil {
 		t.Fatal(err)
 	}
 	versionRDS++
-	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
-	if _, err := sendGoodResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil {
 		t.Fatal(err)
 	}
 	versionEDS++
@@ -275,8 +277,11 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 	v2c, cbLDS, _, _, _, v2cCleanup := startXDSV2Client(t, cc)
 	defer v2cCleanup()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// Start the watch, send a good response, and check for ack.
-	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	startXDS(ctx, t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
 
 	nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{{}},
@@ -286,13 +291,13 @@ func (s) TestV2ClientAckFirstIsNack(t *testing.T) {
 
 	// The expected version string is an empty string, because this is the first
 	// response, and it's nacked (so there's no previous ack version).
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, "", nonce, true); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, goodLDSRequest, "", nonce, true); err != nil {
 		t.Errorf("Failed to receive request: %v", err)
 	}
 	t.Logf("Bad response nacked")
 	versionLDS++
 
-	sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	sendGoodResp(ctx, t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 	versionLDS++
 }
 
@@ -307,15 +312,18 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	v2c, cbLDS, _, _, _, v2cCleanup := startXDSV2Client(t, cc)
 	defer v2cCleanup()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// Start the watch, send a good response, and check for ack.
-	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
-	nonce, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
+	startXDS(ctx, t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "")
+	nonce, err := sendGoodResp(ctx, t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Start a new watch. The version in the new request should be the version
 	// from the previous response, thus versionLDS before ++.
-	startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
+	startXDS(ctx, t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce)
 	versionLDS++
 
 	// This is an invalid response after the new watch.
@@ -326,13 +334,13 @@ func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) {
 	t.Logf("Bad response pushed to fakeServer...")
 
 	// The expected version string is the previous acked version.
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS-1), nonce, true); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS-1), nonce, true); err != nil {
 		t.Errorf("Failed to receive request: %v", err)
 	}
 	t.Logf("Bad response nacked")
 	versionLDS++
 
-	if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil {
 		t.Fatal(err)
 	}
 	versionLDS++
@@ -349,16 +357,19 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	v2c, _, _, cbCDS, _, v2cCleanup := startXDSV2Client(t, cc)
 	defer v2cCleanup()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// Start a CDS watch.
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", "", false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, goodCDSRequest, "", "", false); err != nil {
 		t.Fatal(err)
 	}
 	t.Logf("FakeServer received %v request...", xdsclient.ClusterResource)
 
 	// Send a good CDS response, this function waits for the ACK with the right
 	// version.
-	nonce, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	nonce, err := sendGoodResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -368,24 +379,24 @@ func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) {
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
 	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce, false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
 	// Wait for a request with correct resource names and version.
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce, false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	versionCDS++
 
 	// Send a bad response with the next version.
-	if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
+	if err := sendBadResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
@@ -404,15 +415,18 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	v2c, _, _, cbCDS, _, v2cCleanup := startXDSV2Client(t, cc)
 	defer v2cCleanup()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// Start a CDS watch.
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", "", false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, goodCDSRequest, "", "", false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	t.Logf("FakeServer received %v request...", xdsclient.ClusterResource)
 
 	// send a good response, and check for ack, with the new version.
-	nonce, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
+	nonce, err := sendGoodResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -422,14 +436,14 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	// Wait for a request with no resource names, because the only watch was
 	// removed.
 	emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL}
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce, false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 	versionCDS++
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if req, err := fakeServer.XDSRequestChan.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if req, err := fakeServer.XDSRequestChan.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
 	}
 
@@ -438,13 +452,13 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	t.Logf("Good %v response pushed to fakeServer...", xdsclient.ClusterResource)
 
 	// Expect no ACK because watch was canceled.
-	if req, err := fakeServer.XDSRequestChan.Receive(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if req, err := fakeServer.XDSRequestChan.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatalf("Got unexpected xds request after watch is canceled: %v", req)
 	}
 
 	// Still expected an callback update, because response was good.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if _, err := cbCDS.Receive(ctx); err != nil {
 		t.Fatalf("Timeout when expecting %v update", xdsclient.ClusterResource)
 	}
@@ -452,18 +466,18 @@ func (s) TestV2ClientAckCancelResponseRace(t *testing.T) {
 	// Start a new watch. The new watch should have the nonce from the response
 	// above, and version from the first good response.
 	v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1)
-	if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce, false); err != nil {
+	if err := compareXDSRequest(ctx, fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce, false); err != nil {
 		t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err)
 	}
 
 	// Send a bad response with the next version.
-	if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
+	if err := sendBadResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
 
 	// send another good response, and check for ack, with the new version.
-	if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
+	if _, err := sendGoodResp(ctx, t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil {
 		t.Fatal(err)
 	}
 	versionCDS++
diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/client_rds_test.go
index 1d5538ec5492..c685d4696985 100644
--- a/xds/internal/client/v2/client_rds_test.go
+++ b/xds/internal/client/v2/client_rds_test.go
@@ -34,10 +34,8 @@ import (
 // This is called by RDS tests to start LDS first, because LDS is a
 // pre-requirement for RDS, and RDS handle would fail without an existing LDS
 // watch.
-func doLDS(t *testing.T, v2c xdsclient.APIClient, fakeServer *fakeserver.Server) {
+func doLDS(ctx context.Context, t *testing.T, v2c xdsclient.APIClient, fakeServer *fakeserver.Server) {
 	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout waiting for LDS request: %v", err)
 	}
@@ -137,7 +135,10 @@ func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer v2c.Close()
-	doLDS(t, v2c, fakeServer)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	doLDS(ctx, t, v2c, fakeServer)
 
 	if v2c.handleRDSResponse(badResourceTypeInRDSResponse) == nil {
 		t.Fatal("v2c.handleRDSResponse() succeeded, should have failed")
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index 97634b203069..7b8d6786b03e 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -452,7 +452,9 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	// Cannot directly compare test.wantUpdate with nil (typed vs non-typed nil:
 	// https://golang.org/doc/faq#nil_error).
 	if c := test.wantUpdate; c == nil || (reflect.ValueOf(c).Kind() == reflect.Ptr && reflect.ValueOf(c).IsNil()) {
-		update, err := gotUpdateCh.Receive(ctx)
+		sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		defer sCancel()
+		update, err := gotUpdateCh.Receive(sCtx)
 		if err == context.DeadlineExceeded {
 			return
 		}
@@ -543,7 +545,7 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")}
 	t.Log("Bad LDS response pushed to fakeServer...")
 
-	timer := time.NewTimer(1 * time.Second)
+	timer := time.NewTimer(defaultTestShortTimeout)
 	select {
 	case <-timer.C:
 		t.Fatal("Timeout when expecting LDS update")
@@ -607,7 +609,9 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")}
 	t.Log("Bad LDS response pushed to fakeServer...")
 
-	val, err := fakeServer.XDSRequestChan.Receive(ctx)
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	val, err := fakeServer.XDSRequestChan.Receive(sCtx)
 	if err == context.DeadlineExceeded {
 		t.Fatalf("Timeout expired when expecting LDS update")
 	}
@@ -661,9 +665,9 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1)
 
 	// The watcher should receive an update, with a timeout error in it.
-	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
-	defer cancel()
-	if v, err := callbackCh.Receive(ctx); err == nil {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if v, err := callbackCh.Receive(sCtx); err == nil {
 		t.Fatalf("Expect an timeout error from watcher, got %v", v)
 	}
 
@@ -673,7 +677,7 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 		Addresses: []resolver.Address{{Addr: fakeServer.Address}},
 	})
 
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	if _, err := fakeServer.XDSRequestChan.Receive(ctx); err != nil {
 		t.Fatalf("Timeout expired when expecting an LDS request")

From fe98b4c668eb0fb9eb4b368a7fede14f6cb47384 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 29 Oct 2020 10:15:45 -0700
Subject: [PATCH 252/481] doc: update keepalive.md to add why (#3993)

---
 Documentation/keepalive.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Documentation/keepalive.md b/Documentation/keepalive.md
index 43cfda82daf7..6bd95360e02d 100644
--- a/Documentation/keepalive.md
+++ b/Documentation/keepalive.md
@@ -8,6 +8,17 @@ activity on the connection.
 For how to configure keepalive, see
 https://godoc.org/google.golang.org/grpc/keepalive for the options.
 
+## Why do I need this?
+
+Keepalive can be useful to detect TCP level connection failures. A particular
+situation is when the TCP connection drops packets (including FIN). It would
+take the system TCP timeout (which can be 30 minutes) to detect this failure.
+Keepalive would allow gRPC to detect this failure much sooner.
+
+Another usage is (as the name suggests) to keep the connection alive. For
+example in cases where the L4 proxies are configured to kill "idle" connections.
+Sending pings would make the connections not "idle".
+
 ## What should I set?
 
 It should be sufficient for most users to set [client

From b045bc88c648f1788bd00b49491cc1438bbdd95a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 29 Oct 2020 10:16:05 -0700
Subject: [PATCH 253/481] priority: add policy config parsing (#3936)

---
 xds/internal/balancer/priority/config.go      |  64 +++++++++++
 xds/internal/balancer/priority/config_test.go | 107 ++++++++++++++++++
 xds/internal/balancer/priority/doc.go         |  24 ++++
 3 files changed, 195 insertions(+)
 create mode 100644 xds/internal/balancer/priority/config.go
 create mode 100644 xds/internal/balancer/priority/config_test.go
 create mode 100644 xds/internal/balancer/priority/doc.go

diff --git a/xds/internal/balancer/priority/config.go b/xds/internal/balancer/priority/config.go
new file mode 100644
index 000000000000..da085908c71d
--- /dev/null
+++ b/xds/internal/balancer/priority/config.go
@@ -0,0 +1,64 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"encoding/json"
+	"fmt"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type child struct {
+	Config *internalserviceconfig.BalancerConfig
+}
+
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+
+	// Children is a map from the child balancer names to their configs. Child
+	// names can be found in field Priorities.
+	Children map[string]*child
+	// Priorities is a list of child balancer names. They are sorted from
+	// highest priority to low. The type/config for each child can be found in
+	// field Children, with the balancer name as the key.
+	Priorities []string
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	var cfg lbConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+
+	prioritiesSet := make(map[string]bool)
+	for _, name := range cfg.Priorities {
+		if _, ok := cfg.Children[name]; !ok {
+			return nil, fmt.Errorf("LB policy name %q found in Priorities field (%v) is not found in Children field (%+v)", name, cfg.Priorities, cfg.Children)
+		}
+		prioritiesSet[name] = true
+	}
+	for name := range cfg.Children {
+		if _, ok := prioritiesSet[name]; !ok {
+			return nil, fmt.Errorf("LB policy name %q found in Children field (%v) is not found in Priorities field (%+v)", name, cfg.Children, cfg.Priorities)
+		}
+	}
+	return &cfg, nil
+}
diff --git a/xds/internal/balancer/priority/config_test.go b/xds/internal/balancer/priority/config_test.go
new file mode 100644
index 000000000000..15c4069dd1e7
--- /dev/null
+++ b/xds/internal/balancer/priority/config_test.go
@@ -0,0 +1,107 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer/roundrobin"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name: "child not found",
+			js: `{
+  "priorities": ["child-1", "child-2", "child-3"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "child not used",
+			js: `{
+  "priorities": ["child-1", "child-2"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-2": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "good",
+			js: `{
+  "priorities": ["child-1", "child-2", "child-3"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-2": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			want: &lbConfig{
+				Children: map[string]*child{
+					"child-1": {
+						&internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+					"child-2": {
+						&internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+					"child-3": {
+						&internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+				},
+				Priorities: []string{"child-1", "child-2", "child-3"},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(got, tt.want); diff != "" {
+				t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff)
+			}
+		})
+	}
+}
diff --git a/xds/internal/balancer/priority/doc.go b/xds/internal/balancer/priority/doc.go
new file mode 100644
index 000000000000..53bd270cb10e
--- /dev/null
+++ b/xds/internal/balancer/priority/doc.go
@@ -0,0 +1,24 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package priority implements the priority balancer.
+//
+// This balancer will be kept in internal until we use it in the xds balancers,
+// and are confident its functionalities are stable. It will then be exported
+// for more users.
+package priority

From 89faf1c3e8283dd3c863b877bcf1631d1fe6f50c Mon Sep 17 00:00:00 2001
From: Nick Jones <dntj@users.noreply.github.com>
Date: Fri, 30 Oct 2020 07:52:55 +1100
Subject: [PATCH 254/481] internal/transport: fix a bug causing -bin metadata
 to be incorrectly encoded. (#3985)

Most interfaces for adding metadata to a context downcase keys, except metadata.AppendToOutgoingContext (presumably for performance reasons).  Documentation for metadata.AppendToOutgoingContext, referring to that of metadata.Pairs, states that metadata keys are downcased.

The bug is manifest if you use metadata.AppendToOutgoingContext to add metadata with a non-downcased `-Bin` suffix.  http2Client will not encode such metadata, as encodeMetadataHeader just performs a lower suffix check.
---
 internal/transport/http2_client.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 6a4776bb1531..0364df53f868 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -489,14 +489,14 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 		for _, vv := range added {
 			for i, v := range vv {
 				if i%2 == 0 {
-					k = v
+					k = strings.ToLower(v)
 					continue
 				}
 				// HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set.
 				if isReservedHeader(k) {
 					continue
 				}
-				headerFields = append(headerFields, hpack.HeaderField{Name: strings.ToLower(k), Value: encodeMetadataHeader(k, v)})
+				headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
 			}
 		}
 	}

From c8ef9bc957120e98c96f6c44984ef310b092a5d8 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 30 Oct 2020 09:34:18 -0700
Subject: [PATCH 255/481] xds: Check name of transport socket received in
 Cluster response. (#3988)

---
 xds/internal/client/client_cds_test.go | 27 ++++++++++++++++++++++++++
 xds/internal/client/client_xds.go      |  7 +++++++
 2 files changed, 34 insertions(+)

diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go
index 9d36d70be244..6cba7ef12a08 100644
--- a/xds/internal/client/client_cds_test.go
+++ b/xds/internal/client/client_cds_test.go
@@ -201,6 +201,30 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		wantUpdate ClusterUpdate
 		wantErr    bool
 	}{
+		{
+			name: "transport-socket-unsupported-name",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "unsupported-foo",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "transport-socket-unsupported-typeURL",
 			cluster: &v3clusterpb.Cluster{
@@ -298,6 +322,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
 						TypedConfig: &anypb.Any{
 							TypeUrl: version.V3UpstreamTLSContextURL,
@@ -342,6 +367,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
 						TypedConfig: &anypb.Any{
 							TypeUrl: version.V3UpstreamTLSContextURL,
@@ -392,6 +418,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
 						TypedConfig: &anypb.Any{
 							TypeUrl: version.V3UpstreamTLSContextURL,
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 6f6245374342..b8598c0247d9 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -259,6 +259,10 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 	return routesRet, nil
 }
 
+// TransportSocket proto message has a `name` field which is expected to be set
+// to this value by the management server.
+const transportSocketName = "envoy.transport_sockets.tls"
+
 // UnmarshalCluster processes resources received in an CDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
@@ -322,6 +326,9 @@ func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, e
 	if ts == nil {
 		return nil, nil
 	}
+	if name := ts.GetName(); name != transportSocketName {
+		return nil, fmt.Errorf("xds: transport_socket field has unexpected name: %s", name)
+	}
 	any := ts.GetTypedConfig()
 	if any == nil || any.TypeUrl != version.V3UpstreamTLSContextURL {
 		return nil, fmt.Errorf("xds: transport_socket field has unexpected typeURL: %s", any.TypeUrl)

From f4d9cca4f761e105b63c387755d0153ebf15afe7 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 30 Oct 2020 14:37:13 -0700
Subject: [PATCH 256/481] github: add Github Actions workflow for tests;
 support in vet.sh (#4005)

---
 .github/workflows/testing.yml | 97 +++++++++++++++++++++++++++++++++++
 vet.sh                        |  8 +++
 2 files changed, 105 insertions(+)
 create mode 100644 .github/workflows/testing.yml

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
new file mode 100644
index 000000000000..dbfc9db328da
--- /dev/null
+++ b/.github/workflows/testing.yml
@@ -0,0 +1,97 @@
+name: Testing
+
+# Trigger on pushes, PRs (excluding documentation changes), and nightly.
+on:
+  push:
+  pull_request:
+    paths-ignore:
+      - 'Documentation/**'
+      - 'version.go'
+  schedule:
+    - cron: 0 0 * * * # daily at 00:00
+
+# Always force the use of Go modules
+env:
+  GO111MODULE: on
+
+jobs:
+  # Check generated protos match their source repos (optional for PRs).
+  vet-proto:
+    runs-on: ubuntu-latest
+    steps:
+      # Setup the environment.
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      # Run the vet checks.
+      - name: vet
+        run: ./vet.sh -install && ./vet.sh
+
+  # Run the main gRPC-Go tests.
+  tests:
+    # Proto checks are run in the above job.
+    env:
+      VET_SKIP_PROTO: 1
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - type: vet
+            goversion: 1.14
+          - type: race
+            goversion: 1.14
+          - type: 386
+            goversion: 1.14
+          - type: retry
+            goversion: 1.14
+          - type: extras
+            goversion: 1.14
+          - type: tests
+            goversion: 1.13
+          - type: tests
+            goversion: 1.12
+          - type: tests
+            goversion: 1.11  # Keep until interop tests no longer require Go1.11
+
+    steps:
+      # Setup the environment.
+      - name: Setup GOARCH=386
+        if: ${{ matrix.type == '386' }}
+        run: echo "GOARCH=386" >> $GITHUB_ENV
+      - name: Setup RETRY
+        if: ${{ matrix.type == 'retry' }}
+        run: echo "GRPC_GO_RETRY=on" >> $GITHUB_ENV
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.goversion }}
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      # Only run vet for 'vet' runs.
+      - name: Run vet.sh
+        if: ${{ matrix.type == 'vet' }}
+        run: ./vet.sh -install && ./vet.sh
+
+      # Main tests run for everything except when testing "extras" and the race detector.
+      - name: Run tests
+        if: ${{ matrix.type != 'extras' && matrix.type != 'race' }}
+        run: make test
+
+      # Race detector tests
+      - name: Run test race
+        if: ${{ matrix.TYPE == 'race' }}
+        run: make testrace
+
+      # Non-core gRPC tests (examples, interop, etc)
+      - name: Run extras tests
+        if: ${{ matrix.TYPE == 'extras' }}
+        run: |
+          examples/examples_test.sh
+          security/advancedtls/examples/examples_test.sh
+          interop/interop_test.sh
+          make testsubmodule
diff --git a/vet.sh b/vet.sh
index 7e14befe7e3d..48652f6040f2 100755
--- a/vet.sh
+++ b/vet.sh
@@ -60,6 +60,14 @@ if [[ "$1" = "-install" ]]; then
       unzip ${PROTOC_FILENAME}
       bin/protoc --version
       popd
+    elif [[ "${GITHUB_ACTIONS}" = "true" ]]; then
+      PROTOBUF_VERSION=3.3.0
+      PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
+      pushd /home/runner/go
+      wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}
+      unzip ${PROTOC_FILENAME}
+      bin/protoc --version
+      popd
     elif not which protoc > /dev/null; then
       die "Please install protoc into your path"
     fi

From fe9c99ff4c285512a20b6fcbdc3371deb9ddd4a3 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 30 Oct 2020 15:52:14 -0700
Subject: [PATCH 257/481] xds: CDS balancer security integration. (#3955)

---
 credentials/tls/certprovider/store.go         |  16 +-
 credentials/xds/xds.go                        |  26 +-
 credentials/xds/xds_test.go                   |   7 +-
 internal/internal.go                          |   4 +
 .../balancer/cdsbalancer/cdsbalancer.go       | 313 ++++++--
 .../cdsbalancer/cdsbalancer_security_test.go  | 690 ++++++++++++++++++
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 125 ++--
 xds/internal/balancer/edsbalancer/eds.go      |   4 +-
 xds/internal/client/client.go                 |   8 +
 xds/internal/testutils/balancer.go            |  17 +
 xds/internal/testutils/fakeclient/client.go   |  12 +
 11 files changed, 1074 insertions(+), 148 deletions(-)
 create mode 100644 xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go

diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go
index 990e6ab06c34..7f41fddb3bbe 100644
--- a/credentials/tls/certprovider/store.go
+++ b/credentials/tls/certprovider/store.go
@@ -80,9 +80,19 @@ func GetProvider(name string, config interface{}, opts Options) (Provider, error
 	if builder == nil {
 		return nil, fmt.Errorf("no registered builder for provider name: %s", name)
 	}
-	stableConfig, err := builder.ParseConfig(config)
-	if err != nil {
-		return nil, err
+
+	var (
+		stableConfig StableConfig
+		err          error
+	)
+	if c, ok := config.(StableConfig); ok {
+		// The config passed to the store has already been parsed.
+		stableConfig = c
+	} else {
+		stableConfig, err = builder.ParseConfig(config)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	sk := storeKey{
diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go
index e3f73d0757e8..2cc52ce6b798 100644
--- a/credentials/xds/xds.go
+++ b/credentials/xds/xds.go
@@ -37,10 +37,15 @@ import (
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
 	credinternal "google.golang.org/grpc/internal/credentials"
 	"google.golang.org/grpc/resolver"
 )
 
+func init() {
+	internal.GetXDSHandshakeInfoForTesting = getHandshakeInfo
+}
+
 // ClientOptions contains parameters to configure a new client-side xDS
 // credentials implementation.
 type ClientOptions struct {
@@ -124,6 +129,18 @@ func (hi *HandshakeInfo) SetAcceptedSANs(sans []string) {
 	hi.mu.Unlock()
 }
 
+// UseFallbackCreds returns true when fallback credentials are to be used based
+// on the contents of the HandshakeInfo.
+func (hi *HandshakeInfo) UseFallbackCreds() bool {
+	if hi == nil {
+		return true
+	}
+
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+	return hi.identityProvider == nil && hi.rootProvider == nil
+}
+
 func (hi *HandshakeInfo) validate(isClient bool) error {
 	hi.mu.Lock()
 	defer hi.mu.Unlock()
@@ -245,10 +262,9 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 		return c.fallback.ClientHandshake(ctx, authority, rawConn)
 	}
 	hi := getHandshakeInfo(chi.Attributes)
-	if hi == nil {
+	if hi.UseFallbackCreds() {
 		return c.fallback.ClientHandshake(ctx, authority, rawConn)
 	}
-
 	if err := hi.validate(c.isClient); err != nil {
 		return nil, nil, err
 	}
@@ -355,3 +371,9 @@ func (c *credsImpl) Clone() credentials.TransportCredentials {
 func (c *credsImpl) OverrideServerName(_ string) error {
 	return errors.New("serverName for peer validation must be configured as a list of acceptable SANs")
 }
+
+// UsesXDS returns true if c uses xDS to fetch security configuration
+// used at handshake time, and false otherwise.
+func (c *credsImpl) UsesXDS() bool {
+	return true
+}
diff --git a/credentials/xds/xds_test.go b/credentials/xds/xds_test.go
index 18aa86c9cdda..07bc48b3f64d 100644
--- a/credentials/xds/xds_test.go
+++ b/credentials/xds/xds_test.go
@@ -303,12 +303,7 @@ func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) {
 
 	pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	ctx := newTestContextWithHandshakeInfo(pCtx, nil, nil)
-	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
-		t.Fatal("ClientHandshake succeeded without certificate providers in HandshakeInfo")
-	}
-
-	ctx = newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{})
+	ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{})
 	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
 		t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo")
 	}
diff --git a/internal/internal.go b/internal/internal.go
index 716d92800b9a..f83d6625988d 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -57,6 +57,10 @@ var (
 	// bootstrap code while parsing certificate provider configs in the
 	// bootstrap file.
 	GetCertificateProviderBuilder interface{} // func(string) certprovider.Builder
+	// GetXDSHandshakeInfoForTesting returns a pointer to the xds.HandshakeInfo
+	// stored in the passed in attributes. This is set by
+	// credentials/xds/xds.go.
+	GetXDSHandshakeInfoForTesting interface{} // func (attr *attributes.Attributes) *xds.HandshakeInfo
 )
 
 // HealthChecker defines the signature of the client-side LB channel health checking function.
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index c7df3fb10fdc..55d6e8c9f0aa 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -26,12 +26,16 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -57,6 +61,8 @@ var (
 		// not deal with subConns.
 		return builder.Build(cc, opts), nil
 	}
+
+	getProvider = certprovider.GetProvider
 )
 
 func init() {
@@ -72,14 +78,31 @@ type cdsBB struct{}
 // Build creates a new CDS balancer with the ClientConn.
 func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
 	b := &cdsBalancer{
-		cc:          cc,
 		bOpts:       opts,
 		updateCh:    buffer.NewUnbounded(),
 		closed:      grpcsync.NewEvent(),
 		cancelWatch: func() {}, // No-op at this point.
+		xdsHI:       xds.NewHandshakeInfo(nil, nil),
 	}
 	b.logger = prefixLogger((b))
 	b.logger.Infof("Created")
+
+	var creds credentials.TransportCredentials
+	switch {
+	case opts.DialCreds != nil:
+		creds = opts.DialCreds
+	case opts.CredsBundle != nil:
+		creds = opts.CredsBundle.TransportCredentials()
+	}
+	if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() {
+		b.xdsCredsInUse = true
+	}
+	b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse)
+
+	b.ccw = &ccWrapper{
+		ClientConn: cc,
+		xdsHI:      b.xdsHI,
+	}
 	go b.run()
 	return b
 }
@@ -110,6 +133,7 @@ func (cdsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 // the cdsBalancer. This will be faked out in unittests.
 type xdsClientInterface interface {
 	WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func()
+	CertProviderConfigs() map[string]bootstrap.CertProviderConfig
 	Close()
 }
 
@@ -145,15 +169,194 @@ type watchUpdate struct {
 // is exposed to gRPC and implements the balancer.ClientConn interface which is
 // exposed to the edsBalancer.
 type cdsBalancer struct {
-	cc             balancer.ClientConn
-	bOpts          balancer.BuildOptions
-	updateCh       *buffer.Unbounded
-	client         xdsClientInterface
-	cancelWatch    func()
-	edsLB          balancer.Balancer
+	ccw            *ccWrapper            // ClientConn interface passed to child LB.
+	bOpts          balancer.BuildOptions // BuildOptions passed to child LB.
+	updateCh       *buffer.Unbounded     // Channel for gRPC and xdsClient updates.
+	xdsClient      xdsClientInterface    // xDS client to watch Cluster resource.
+	cancelWatch    func()                // Cluster watch cancel func.
+	edsLB          balancer.Balancer     // EDS child policy.
 	clusterToWatch string
 	logger         *grpclog.PrefixLogger
 	closed         *grpcsync.Event
+
+	// The certificate providers are cached here to that they can be closed when
+	// a new provider is to be created.
+	cachedRoot     certprovider.Provider
+	cachedIdentity certprovider.Provider
+	xdsHI          *xds.HandshakeInfo
+	xdsCredsInUse  bool
+}
+
+// handleClientConnUpdate handles a ClientConnUpdate received from gRPC. Good
+// updates lead to registration of a CDS watch. Updates with error lead to
+// cancellation of existing watch and propagation of the same error to the
+// edsBalancer.
+func (b *cdsBalancer) handleClientConnUpdate(update *ccUpdate) {
+	// We first handle errors, if any, and then proceed with handling the
+	// update, only if the status quo has changed.
+	if err := update.err; err != nil {
+		b.handleErrorFromUpdate(err, true)
+	}
+	if b.xdsClient == update.client && b.clusterToWatch == update.clusterName {
+		return
+	}
+	if update.client != nil {
+		// Since the cdsBalancer doesn't own the xdsClient object, we don't have
+		// to bother about closing the old client here, but we still need to
+		// cancel the watch on the old client.
+		b.cancelWatch()
+		b.xdsClient = update.client
+	}
+	if update.clusterName != "" {
+		cancelWatch := b.xdsClient.WatchCluster(update.clusterName, b.handleClusterUpdate)
+		b.logger.Infof("Watch started on resource name %v with xds-client %p", update.clusterName, b.xdsClient)
+		b.cancelWatch = func() {
+			cancelWatch()
+			b.logger.Infof("Watch cancelled on resource name %v with xds-client %p", update.clusterName, b.xdsClient)
+		}
+		b.clusterToWatch = update.clusterName
+	}
+}
+
+// handleSecurityConfig processes the security configuration received from the
+// management server, creates appropriate certificate provider plugins, and
+// updates the HandhakeInfo which is added as an address attribute in
+// NewSubConn() calls.
+func (b *cdsBalancer) handleSecurityConfig(config *xdsclient.SecurityConfig) error {
+	// If xdsCredentials are not in use, i.e, the user did not want to get
+	// security configuration from an xDS server, we should not be acting on the
+	// received security config here. Doing so poses a security threat.
+	if !b.xdsCredsInUse {
+		return nil
+	}
+
+	// Security config being nil is a valid case where the management server has
+	// not sent any security configuration. The xdsCredentials implementation
+	// handles this by delegating to its fallback credentials.
+	if config == nil {
+		// We need to explicitly set the fields to nil here since this might be
+		// a case of switching from a good security configuration to an empty
+		// one where fallback credentials are to be used.
+		b.xdsHI.SetRootCertProvider(nil)
+		b.xdsHI.SetIdentityCertProvider(nil)
+		b.xdsHI.SetAcceptedSANs(nil)
+		return nil
+	}
+
+	cpc := b.xdsClient.CertProviderConfigs()
+	if cpc == nil {
+		// Bootstrap did not find any certificate provider configs, but the user
+		// has specified xdsCredentials and the management server has sent down
+		// security configuration.
+		return errors.New("xds: certificate_providers config missing in bootstrap file")
+	}
+
+	// A root provider is required whether we are using TLS or mTLS.
+	rootCfg, ok := cpc[config.RootInstanceName]
+	if !ok {
+		return fmt.Errorf("certificate provider instance %q not found in bootstrap file", config.RootInstanceName)
+	}
+	rootProvider, err := getProvider(rootCfg.Name, rootCfg.Config, certprovider.Options{
+		CertName: config.RootCertName,
+		WantRoot: true,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", rootCfg, err)
+	}
+	if b.cachedRoot != nil {
+		b.cachedRoot.Close()
+	}
+
+	// The identity provider is only present when using mTLS.
+	var identityProvider certprovider.Provider
+	if name := config.IdentityInstanceName; name != "" {
+		identityCfg := cpc[name]
+		if !ok {
+			return fmt.Errorf("certificate provider instance %q not found in bootstrap file", config.IdentityInstanceName)
+		}
+		identityProvider, err = getProvider(identityCfg.Name, identityCfg.Config, certprovider.Options{
+			CertName:     config.IdentityCertName,
+			WantIdentity: true,
+		})
+		if err != nil {
+			// This error is not expected since the bootstrap process parses the
+			// config and makes sure that it is acceptable to the plugin. Still,
+			// it is possible that the plugin parses the config successfully,
+			// but its Build() method errors out.
+			return fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", identityCfg, err)
+		}
+	}
+	if b.cachedIdentity != nil {
+		b.cachedIdentity.Close()
+	}
+
+	b.cachedRoot = rootProvider
+	b.cachedIdentity = identityProvider
+
+	// We set all fields here, even if some of them are nil, since they
+	// could have been non-nil earlier.
+	b.xdsHI.SetRootCertProvider(rootProvider)
+	b.xdsHI.SetIdentityCertProvider(identityProvider)
+	b.xdsHI.SetAcceptedSANs(config.AcceptedSANs)
+	return nil
+}
+
+// handleWatchUpdate handles a watch update from the xDS Client. Good updates
+// lead to clientConn updates being invoked on the underlying edsBalancer.
+func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
+	if err := update.err; err != nil {
+		b.logger.Warningf("Watch error from xds-client %p: %v", b.xdsClient, err)
+		b.handleErrorFromUpdate(err, false)
+		return
+	}
+
+	b.logger.Infof("Watch update from xds-client %p, content: %+v", b.xdsClient, update.cds)
+
+	// Process the security config from the received update before building the
+	// child policy or forwarding the update to it. We do this because the child
+	// policy may try to create a new subConn inline. Processing the security
+	// configuration here and setting up the handshakeInfo will make sure that
+	// such attempts are handled properly.
+	if err := b.handleSecurityConfig(update.cds.SecurityCfg); err != nil {
+		// If the security config is invalid, for example, if the provider
+		// instance is not found in the bootstrap config, we need to put the
+		// channel in transient failure.
+		b.logger.Warningf("Invalid security config update from xds-client %p: %v", b.xdsClient, err)
+		b.handleErrorFromUpdate(err, false)
+		return
+	}
+
+	// The first good update from the watch API leads to the instantiation of an
+	// edsBalancer. Further updates/errors are propagated to the existing
+	// edsBalancer.
+	if b.edsLB == nil {
+		edsLB, err := newEDSBalancer(b.ccw, b.bOpts)
+		if err != nil {
+			b.logger.Errorf("Failed to create child policy of type %s, %v", edsName, err)
+			return
+		}
+		b.edsLB = edsLB
+		b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
+	}
+	lbCfg := &edsbalancer.EDSConfig{EDSServiceName: update.cds.ServiceName}
+	if update.cds.EnableLRS {
+		// An empty string here indicates that the edsBalancer should use the
+		// same xDS server for load reporting as it does for EDS
+		// requests/responses.
+		lbCfg.LrsLoadReportingServerName = new(string)
+
+	}
+	ccState := balancer.ClientConnState{
+		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, b.xdsClient)},
+		BalancerConfig: lbCfg,
+	}
+	if err := b.edsLB.UpdateClientConnState(ccState); err != nil {
+		b.logger.Errorf("xds: edsBalancer.UpdateClientConnState(%+v) returned error: %v", ccState, err)
+	}
 }
 
 // run is a long-running goroutine which handles all updates from gRPC. All
@@ -165,81 +368,18 @@ func (b *cdsBalancer) run() {
 		case u := <-b.updateCh.Get():
 			b.updateCh.Load()
 			switch update := u.(type) {
-			// Good clientConn updates lead to registration of a CDS watch.
-			// Updates with error lead to cancellation of existing watch and
-			// propagation of the same error to the edsBalancer.
 			case *ccUpdate:
-				// We first handle errors, if any, and then proceed with handling
-				// the update, only if the status quo has changed.
-				if err := update.err; err != nil {
-					b.handleErrorFromUpdate(err, true)
-				}
-				if b.client == update.client && b.clusterToWatch == update.clusterName {
-					break
-				}
-				if update.client != nil {
-					// Since the cdsBalancer doesn't own the xdsClient object, we
-					// don't have to bother about closing the old client here, but
-					// we still need to cancel the watch on the old client.
-					b.cancelWatch()
-					b.client = update.client
-				}
-				if update.clusterName != "" {
-					cancelWatch := b.client.WatchCluster(update.clusterName, b.handleClusterUpdate)
-					b.logger.Infof("Watch started on resource name %v with xds-client %p", update.clusterName, b.client)
-					b.cancelWatch = func() {
-						cancelWatch()
-						b.logger.Infof("Watch cancelled on resource name %v with xds-client %p", update.clusterName, b.client)
-					}
-					b.clusterToWatch = update.clusterName
-				}
-
-			// SubConn updates are passthrough and are simply handed over to the
-			// underlying edsBalancer.
+				b.handleClientConnUpdate(update)
 			case *scUpdate:
+				// SubConn updates are passthrough and are simply handed over to
+				// the underlying edsBalancer.
 				if b.edsLB == nil {
 					b.logger.Errorf("xds: received scUpdate {%+v} with no edsBalancer", update)
 					break
 				}
 				b.edsLB.UpdateSubConnState(update.subConn, update.state)
-
-			// Watch API updates lead to clientConn updates being invoked on the
-			// underlying edsBalancer.
 			case *watchUpdate:
-				if err := update.err; err != nil {
-					b.logger.Warningf("Watch error from xds-client %p: %v", b.client, err)
-					b.handleErrorFromUpdate(err, false)
-					break
-				}
-
-				b.logger.Infof("Watch update from xds-client %p, content: %+v", b.client, update.cds)
-				// The first good update from the watch API leads to the
-				// instantiation of an edsBalancer. Further updates/errors are
-				// propagated to the existing edsBalancer.
-				if b.edsLB == nil {
-					var err error
-					b.edsLB, err = newEDSBalancer(b.cc, b.bOpts)
-					if b.edsLB == nil {
-						b.logger.Errorf("Failed to create child policy of type %s, %v", edsName, err)
-						break
-					}
-					b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
-				}
-				lbCfg := &edsbalancer.EDSConfig{EDSServiceName: update.cds.ServiceName}
-				if update.cds.EnableLRS {
-					// An empty string here indicates that the edsBalancer
-					// should use the same xDS server for load reporting as
-					// it does for EDS requests/responses.
-					lbCfg.LrsLoadReportingServerName = new(string)
-
-				}
-				ccState := balancer.ClientConnState{
-					ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, b.client)},
-					BalancerConfig: lbCfg,
-				}
-				if err := b.edsLB.UpdateClientConnState(ccState); err != nil {
-					b.logger.Errorf("xds: edsBalancer.UpdateClientConnState(%+v) returned error: %v", ccState, err)
-				}
+				b.handleWatchUpdate(update)
 			}
 
 		// Close results in cancellation of the CDS watch and closing of the
@@ -290,7 +430,7 @@ func (b *cdsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
 	} else {
 		// If eds balancer was never created, fail the RPCs with
 		// errors.
-		b.cc.UpdateState(balancer.State{
+		b.ccw.UpdateState(balancer.State{
 			ConnectivityState: connectivity.TransientFailure,
 			Picker:            base.NewErrPicker(err),
 		})
@@ -365,3 +505,26 @@ func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 func (b *cdsBalancer) Close() {
 	b.closed.Fire()
 }
+
+// ccWrapper wraps the balancer.ClientConn that was passed in to the CDS
+// balancer during creation and intercepts the NewSubConn() call from the child
+// policy. Other methods of the balancer.ClientConn interface are not overridden
+// and hence get the original implementation.
+type ccWrapper struct {
+	balancer.ClientConn
+
+	// The certificate providers in this HandshakeInfo are updated based on the
+	// received security configuration in the Cluster resource.
+	xdsHI *xds.HandshakeInfo
+}
+
+// NewSubConn intercepts NewSubConn() calls from the child policy and adds an
+// address attribute which provides all information required by the xdsCreds
+// handshaker to perform the TLS handshake.
+func (ccw *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xds.SetHandshakeInfo(addr, ccw.xdsHI)
+	}
+	return ccw.ClientConn.NewSubConn(newAddrs, opts)
+}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
new file mode 100644
index 000000000000..15c097d3f096
--- /dev/null
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -0,0 +1,690 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/credentials/local"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+)
+
+const (
+	fakeProvider1Name = "fake-certificate-provider-1"
+	fakeProvider2Name = "fake-certificate-provider-2"
+	fakeConfig        = "my fake config"
+)
+
+var (
+	fpb1, fpb2                   *fakeProviderBuilder
+	bootstrapCertProviderConfigs = map[string]bootstrap.CertProviderConfig{
+		"default1": {
+			Name:   fakeProvider1Name,
+			Config: &fakeStableConfig{config: fakeConfig + "1111"},
+		},
+		"default2": {
+			Name:   fakeProvider2Name,
+			Config: &fakeStableConfig{config: fakeConfig + "2222"},
+		},
+	}
+	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
+		ServiceName: serviceName,
+		SecurityCfg: &xdsclient.SecurityConfig{
+			RootInstanceName:     "default1",
+			IdentityInstanceName: "default2",
+		},
+	}
+	cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{
+		ServiceName: serviceName,
+		SecurityCfg: &xdsclient.SecurityConfig{
+			RootInstanceName: "not-default",
+		},
+	}
+)
+
+func init() {
+	fpb1 = &fakeProviderBuilder{name: fakeProvider1Name}
+	fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
+	certprovider.Register(fpb1)
+	certprovider.Register(fpb2)
+}
+
+// fakeProviderBuilder builds new instances of fakeProvider and interprets the
+// config provided to it as a string.
+type fakeProviderBuilder struct {
+	name string
+}
+
+func (b *fakeProviderBuilder) Build(certprovider.StableConfig, certprovider.Options) certprovider.Provider {
+	p := &fakeProvider{}
+	return p
+}
+
+func (b *fakeProviderBuilder) ParseConfig(config interface{}) (certprovider.StableConfig, error) {
+	s, ok := config.(string)
+	if !ok {
+		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
+	}
+	return &fakeStableConfig{config: s}, nil
+}
+
+func (b *fakeProviderBuilder) Name() string {
+	return b.name
+}
+
+type fakeStableConfig struct {
+	config string
+}
+
+func (c *fakeStableConfig) Canonical() []byte {
+	return []byte(c.config)
+}
+
+// fakeProvider is an implementation of the Provider interface which provides a
+// method for tests to invoke to push new key materials.
+type fakeProvider struct {
+	certprovider.Provider
+}
+
+// setupWithXDSCreds performs all the setup steps required for tests which use
+// xDSCredentials.
+func setupWithXDSCreds(t *testing.T, storeErr bool) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, *testutils.Channel, func()) {
+	t.Helper()
+
+	builder := balancer.Get(cdsName)
+	if builder == nil {
+		t.Fatalf("balancer.Get(%q) returned nil", cdsName)
+	}
+	// Create and pass xdsCredentials while building the CDS balancer.
+	creds, err := xds.NewClientCredentials(xds.ClientOptions{
+		FallbackCreds: local.NewCredentials(), // Placeholder fallback credentials.
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client creds: %v", err)
+	}
+	// Create a new CDS balancer and pass it a fake balancer.ClientConn which we
+	// can use to inspect the different calls made by the balancer.
+	tcc := xdstestutils.NewTestClientConn(t)
+	cdsB := builder.Build(tcc, balancer.BuildOptions{DialCreds: creds})
+
+	// Override the creation of the EDS balancer to return a fake EDS balancer
+	// implementation.
+	edsB := newTestEDSBalancer()
+	oldEDSBalancerBuilder := newEDSBalancer
+	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
+		edsB.parentCC = cc
+		return edsB, nil
+	}
+
+	// Create a fake xDS client and push a ClientConnState update to the CDS
+	// balancer with a cluster name and the fake xDS client in the attributes.
+	xdsC := fakeclient.NewClient()
+	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
+		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
+	}
+
+	// Make sure the CDS balancer registers a Cluster watch with the xDS client
+	// passed via attributes in the above update.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	gotCluster, err := xdsC.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != clusterName {
+		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName)
+	}
+
+	// Override the certificate provider creation function to get notified about
+	// provider creation. Create a channel with size 2, so we have buffer to
+	// push notifications for both providers.
+	providerCh := testutils.NewChannelWithSize(2)
+	origGetProviderFunc := getProvider
+	if storeErr {
+		getProvider = func(string, interface{}, certprovider.Options) (certprovider.Provider, error) {
+			return nil, errors.New("certprovider.Store failed to created provider")
+		}
+	} else {
+		getProvider = func(name string, cfg interface{}, opts certprovider.Options) (certprovider.Provider, error) {
+			providerCh.Send(nil)
+			return origGetProviderFunc(name, cfg, opts)
+		}
+	}
+
+	return xdsC, cdsB.(*cdsBalancer), edsB, tcc, providerCh, func() {
+		newEDSBalancer = oldEDSBalancerBuilder
+		getProvider = origGetProviderFunc
+	}
+}
+
+// makeNewSubConn invokes the NewSubConn() call on the balancer.ClientConn
+// passed to the EDS balancer, and verifies that the CDS balancer forwards the
+// call appropriately to its parent balancer.ClientConn with or without
+// attributes bases on the value of wantAttributes.
+func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xdstestutils.TestClientConn, wantFallback bool) error {
+	dummyAddr := "foo-address"
+	addrs := []resolver.Address{{Addr: dummyAddr}}
+	if _, err := edsCC.NewSubConn(addrs, balancer.NewSubConnOptions{}); err != nil {
+		return fmt.Errorf("NewSubConn(%+v) on parent ClientConn failed: %v", addrs, err)
+	}
+
+	select {
+	case <-ctx.Done():
+		return errors.New("timeout when waiting for new SubConn")
+	case gotAddrs := <-parentCC.NewSubConnAddrsCh:
+		if len(gotAddrs) != 1 {
+			return fmt.Errorf("NewSubConn expected 1 address, got %d", len(gotAddrs))
+		}
+		if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
+			return fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want)
+		}
+		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xds.HandshakeInfo)
+		hi := getHI(gotAddrs[0].Attributes)
+		if hi == nil {
+			return errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes")
+		}
+		if gotFallback := hi.UseFallbackCreds(); gotFallback != wantFallback {
+			return fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback)
+		}
+	}
+	return nil
+}
+
+// TestSecurityConfigWithoutXDSCreds tests the case where xdsCredentials are not
+// in use, but the CDS balancer receives a Cluster update with security
+// configuration. Verifies that no certificate providers are created, and that
+// the address attributes added as part of the intercepted NewSubConn() method
+// indicate the use of fallback credentials.
+func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
+	// Override the certificate provider creation function to get notified about
+	// provider creation.
+	providerCh := testutils.NewChannel()
+	origGetProviderFunc := getProvider
+	getProvider = func(name string, cfg interface{}, opts certprovider.Options) (certprovider.Provider, error) {
+		providerCh.Send(nil)
+		return origGetProviderFunc(name, cfg, opts)
+	}
+	defer func() { getProvider = origGetProviderFunc }()
+
+	// This creates a CDS balancer, pushes a ClientConnState update with a fake
+	// xdsClient, and makes sure that the CDS balancer registers a watch on the
+	// provided xdsClient.
+	xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a NewSubConn and verify that attributes are not added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
+		t.Fatal(err)
+	}
+
+	// Again, since xdsCredentials are not in use, no certificate providers
+	// should have been initialized by the CDS balancer.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("cds balancer created certificate providers when not using xds credentials")
+	}
+}
+
+// TestNoSecurityConfigWithXDSCreds tests the case where xdsCredentials are in
+// use, but the CDS balancer receives a Cluster update without security
+// configuration. Verifies that no certificate providers are created, and that
+// the address attributes added as part of the intercepted NewSubConn() method
+// indicate the use of fallback credentials.
+func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup. No security config is
+	// passed to the CDS balancer as part of this update.
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a NewSubConn and verify that attributes are not added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
+		t.Fatal(err)
+	}
+
+	// Again, since no security configuration was received, no certificate
+	// providers should have been initialized by the CDS balancer.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("cds balancer created certificate providers when not using xds credentials")
+	}
+}
+
+// TestSecurityConfigNotFoundInBootstrap tests the case where the security
+// config returned by the xDS server cannot be resolved based on the contents of
+// the bootstrap config. Verifies that the balancer puts the channel in a failed
+// state, and returns an error picker.
+func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
+	// We test two cases here:
+	// 0: Bootstrap contains security config. But received plugin instance name
+	//    is not found in the bootstrap config.
+	// 1: Bootstrap contains no security config.
+	for i := 0; i < 2; i++ {
+		// This creates a CDS balancer which uses xdsCredentials, pushes a
+		// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+		// balancer registers a watch on the provided xdsClient.
+		xdsC, cdsB, edsB, tcc, _, cancel := setupWithXDSCreds(t, false)
+		defer func() {
+			cancel()
+			cdsB.Close()
+		}()
+
+		if i == 0 {
+			// Set the bootstrap config used by the fake client.
+			xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+		}
+
+		// Here we invoke the watch callback registered on the fake xdsClient. A bad
+		// security config is passed here. So, we expect the CDS balancer to not
+		// create an EDS balancer and instead reject this update and put the channel
+		// in a bad state.
+		xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil)
+
+		// The CDS balancer has not yet created an EDS balancer. So, this bad
+		// watcher update should not be forwarded forwarded to our fake EDS balancer
+		// as an error.
+		sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		defer sCancel()
+		if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded {
+			t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
+		}
+
+		// Make sure the CDS balancer reports an error picker.
+		ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer ctxCancel()
+		if err := tcc.WaitForErrPicker(ctx); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// TestCertproviderStoreError tests the case where the certprovider.Store
+// returns an error when the CDS balancer attempts to create a provider.
+func (s) TestCertproviderStoreError(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, _, cancel := setupWithXDSCreds(t, true)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Set the bootstrap config used by the fake client.
+	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+
+	// Here we invoke the watch callback registered on the fake xdsClient. Even
+	// though the received update is good, the certprovider.Store is configured
+	// to return an error. So, CDS balancer should reject this config and report
+	// an error.
+	xdsC.InvokeWatchClusterCallback(cdsUpdateWithGoodSecurityCfg, nil)
+
+	// The CDS balancer has not yet created an EDS balancer. So, this bad
+	// watcher update should not be forwarded forwarded to our fake EDS balancer
+	// as an error.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded {
+		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
+	}
+
+	// Make sure the CDS balancer reports an error picker.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := tcc.WaitForErrPicker(ctx); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Set the bootstrap config used by the fake client.
+	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+
+	// Here we invoke the watch callback registered on the fake xdsClient. A bad
+	// security config is passed here. So, we expect the CDS balancer to not
+	// create an EDS balancer and instead reject this update and put the channel
+	// in a bad state.
+	xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil)
+
+	// The CDS balancer has not yet created an EDS balancer. So, this bad
+	// watcher update should not be forwarded forwarded to our fake EDS balancer
+	// as an error.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded {
+		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
+	}
+
+	// Make sure the CDS balancer reports an error picker.
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := tcc.WaitForErrPicker(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+	// Make sure two certificate providers are created.
+	for i := 0; i < 2; i++ {
+		if _, err := providerCh.Receive(ctx); err != nil {
+			t.Fatalf("Failed to create certificate provider upon receipt of security config")
+		}
+	}
+
+	// Make a NewSubConn and verify that attributes are added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestGoodSecurityConfig tests the case where the CDS balancer receives
+// security configuration as part of the Cluster resource which can be
+// successfully resolved using the bootstrap file contents. Verifies that
+// certificate providers are created, and that the NewSubConn() call adds
+// appropriate address attributes.
+func (s) TestGoodSecurityConfig(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Set the bootstrap config used by the fake client.
+	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+	// Make sure two certificate providers are created.
+	for i := 0; i < 2; i++ {
+		if _, err := providerCh.Receive(ctx); err != nil {
+			t.Fatalf("Failed to create certificate provider upon receipt of security config")
+		}
+	}
+
+	// Make a NewSubConn and verify that attributes are added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Set the bootstrap config used by the fake client.
+	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+	// Make sure two certificate providers are created.
+	for i := 0; i < 2; i++ {
+		if _, err := providerCh.Receive(ctx); err != nil {
+			t.Fatalf("Failed to create certificate provider upon receipt of security config")
+		}
+	}
+
+	// Make a NewSubConn and verify that attributes are added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Here we invoke the watch callback registered on the fake xdsClient with
+	// an update which contains bad security config. So, we expect the CDS
+	// balancer to forward this error to the EDS balancer and eventually the
+	// channel needs to be put in a bad state.
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a NewSubConn and verify that fallback creds are used.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestSecurityConfigUpdate_GoodToBad tests the case where the first security
+// config returned by the xDS server is successful, but the second update cannot
+// be resolved based on the contents of the bootstrap config. Verifies that the
+// error is forwarded to the EDS balancer (which was created as part of the
+// first successful update).
+func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Set the bootstrap config used by the fake client.
+	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+	// Make sure two certificate providers are created.
+	for i := 0; i < 2; i++ {
+		if _, err := providerCh.Receive(ctx); err != nil {
+			t.Fatalf("Failed to create certificate provider upon receipt of security config")
+		}
+	}
+
+	// Make a NewSubConn and verify that attributes are added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Here we invoke the watch callback registered on the fake xdsClient with
+	// an update which contains bad security config. So, we expect the CDS
+	// balancer to forward this error to the EDS balancer and eventually the
+	// channel needs to be put in a bad state.
+	xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil)
+
+	// We manually check that an error is forwarded to the EDS balancer instead
+	// of using one of the helper methods on the testEDSBalancer, because all we
+	// care here is whether an error is sent to it or not. We don't care about
+	// the exact error.
+	gotErr, err := edsB.resolverErrCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("timeout waiting for CDS balancer to forward error to EDS balancer upon receipt of bad security config")
+	}
+	if gotErr == nil {
+		t.Fatal("CDS balancer did not forward error to EDS balancer upon receipt of bad security config")
+	}
+
+	// Since the error being pushed here is not a resource-not-found-error, the
+	// registered watch should not be cancelled.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
+	}
+}
+
+// TestSecurityConfigUpdate_GoodToGood tests the case where the CDS balancer
+// receives two different but successful updates with security configuration.
+// Verifies that appropriate providers are created, and that address attributes
+// are added.
+func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
+	// This creates a CDS balancer which uses xdsCredentials, pushes a
+	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
+	// balancer registers a watch on the provided xdsClient.
+	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Set the bootstrap config used by the fake client.
+	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will attempt to
+	// create a new EDS balancer. The fake EDS balancer created above will be
+	// returned to the CDS balancer, because we have overridden the
+	// newEDSBalancer function as part of test setup.
+	cdsUpdate := xdsclient.ClusterUpdate{
+		ServiceName: serviceName,
+		SecurityCfg: &xdsclient.SecurityConfig{
+			RootInstanceName: "default1",
+		},
+	}
+	wantCCS := edsCCS(serviceName, false, xdsC)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// We specified only the root provider. So, expect only one provider here.
+	if _, err := providerCh.Receive(ctx); err != nil {
+		t.Fatalf("Failed to create certificate provider upon receipt of security config")
+	}
+
+	// Make a NewSubConn and verify that attributes are added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push another update with a new security configuration.
+	cdsUpdate = xdsclient.ClusterUpdate{
+		ServiceName: serviceName,
+		SecurityCfg: &xdsclient.SecurityConfig{
+			RootInstanceName: "default2",
+		},
+	}
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// We specified only the root provider. So, expect only one provider here.
+	if _, err := providerCh.Receive(ctx); err != nil {
+		t.Fatalf("Failed to create certificate provider upon receipt of security config")
+	}
+
+	// Make a NewSubConn and verify that attributes are added.
+	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// The HandshakeInfo type does not expose its internals. So, we cannot
+	// verify that the HandshakeInfo carried by the attributes have actually
+	// been changed. This will be covered in e2e/interop tests.
+	// TODO(easwars): Remove this TODO once appropriate e2e/intertop tests have
+	// been added.
+}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 1c83845363bb..2257929e40cd 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -42,11 +42,10 @@ import (
 )
 
 const (
-	clusterName        = "cluster1"
-	serviceName        = "service1"
-	defaultTestTimeout = 1 * time.Second
-	// Used when waiting for something that is expected to *not* happen.
-	defaultTestShortTimeout = 10 * time.Millisecond
+	clusterName             = "cluster1"
+	serviceName             = "service1"
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
 )
 
 type s struct {
@@ -65,10 +64,7 @@ type cdsWatchInfo struct {
 
 // invokeWatchCb invokes the CDS watch callback registered by the cdsBalancer
 // and waits for appropriate state to be pushed to the provided edsBalancer.
-func invokeWatchCbAndWait(xdsC *fakeclient.Client, cdsW cdsWatchInfo, wantCCS balancer.ClientConnState, edsB *testEDSBalancer) error {
-	// TODO(easwars): Change this func to accept a context.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+func invokeWatchCbAndWait(ctx context.Context, xdsC *fakeclient.Client, cdsW cdsWatchInfo, wantCCS balancer.ClientConnState, edsB *testEDSBalancer) error {
 	xdsC.InvokeWatchClusterCallback(cdsW.update, cdsW.err)
 	if cdsW.err != nil {
 		return edsB.waitForResolverError(ctx, cdsW.err)
@@ -88,6 +84,9 @@ type testEDSBalancer struct {
 	resolverErrCh *testutils.Channel
 	// closeCh is a channel used to signal the closing of this balancer.
 	closeCh *testutils.Channel
+	// parentCC is the balancer.ClientConn passed to this test balancer as part
+	// of the Build() call.
+	parentCC balancer.ClientConn
 }
 
 type subConnWithState struct {
@@ -209,6 +208,8 @@ func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.Clie
 // setup creates a cdsBalancer and an edsBalancer (and overrides the
 // newEDSBalancer function to return it), and also returns a cleanup function.
 func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
+	t.Helper()
+
 	builder := balancer.Get(cdsName)
 	if builder == nil {
 		t.Fatalf("balancer.Get(%q) returned nil", cdsName)
@@ -219,6 +220,7 @@ func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClie
 	edsB := newTestEDSBalancer()
 	oldEDSBalancerBuilder := newEDSBalancer
 	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
+		edsB.parentCC = cc
 		return edsB, nil
 	}
 
@@ -379,7 +381,9 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{test.cdsUpdate, test.updateErr}, test.wantCCS, edsB); err != nil {
+			ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer ctxCancel()
+			if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{test.cdsUpdate, test.updateErr}, test.wantCCS, edsB); err != nil {
 				t.Fatal(err)
 			}
 		})
@@ -408,22 +412,24 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 
 	// Since the error being pushed here is not a resource-not-found-error, the
 	// registered watch should not be cancelled.
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// The CDS balancer has not yet created an EDS balancer. So, this resolver
 	// error should not be forwarded to our fake EDS balancer.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := edsB.waitForResolverError(ctx, watcherErr); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForResolverError(sCtx, watcherErr); err != context.DeadlineExceeded {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
+
 	// Make sure the CDS balancer reports an error picker.
-	timer := time.NewTimer(defaultTestTimeout)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
 	select {
-	case <-timer.C:
+	case <-ctx.Done():
 		t.Fatalf("timeout when waiting for an error picker")
 	case picker := <-tcc.NewPickerCh:
 		if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
@@ -438,21 +444,19 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
-	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
 	// Again push a non-resource-not-found-error through the watcher callback.
 	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, watcherErr)
 	// Make sure the registered watch is not cancelled.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// Make sure the error is forwarded to the EDS balancer.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := edsB.waitForResolverError(ctx, watcherErr); err != nil {
 		t.Fatalf("Watch callback error is not forwarded to EDS balancer")
 	}
@@ -462,14 +466,12 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, resourceErr)
 	// Make sure that the watch is not cancelled. This error indicates that the
 	// request cluster resource is not found. We should continue to watch it.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a resource-not-found-error")
 	}
 	// Make sure the error is forwarded to the EDS balancer.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := edsB.waitForResolverError(ctx, resourceErr); err != nil {
 		t.Fatalf("Watch callback error is not forwarded to EDS balancer")
 	}
@@ -496,22 +498,23 @@ func (s) TestResolverError(t *testing.T) {
 
 	// Since the error being pushed here is not a resource-not-found-error, the
 	// registered watch should not be cancelled.
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// The CDS balancer has not yet created an EDS balancer. So, this resolver
 	// error should not be forwarded to our fake EDS balancer.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := edsB.waitForResolverError(ctx, resolverErr); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForResolverError(sCtx, resolverErr); err != context.DeadlineExceeded {
 		t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
 	}
 	// Make sure the CDS balancer reports an error picker.
-	timer := time.NewTimer(defaultTestTimeout)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
 	select {
-	case <-timer.C:
+	case <-ctx.Done():
 		t.Fatalf("timeout when waiting for an error picker")
 	case picker := <-tcc.NewPickerCh:
 		if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
@@ -526,21 +529,19 @@ func (s) TestResolverError(t *testing.T) {
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
-	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
 	// Again push a non-resource-not-found-error.
 	cdsB.ResolverError(resolverErr)
 	// Make sure the registered watch is not cancelled.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// Make sure the error is forwarded to the EDS balancer.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := edsB.waitForResolverError(ctx, resolverErr); err != nil {
 		t.Fatalf("ResolverError() not forwarded to EDS balancer")
 	}
@@ -549,14 +550,10 @@ func (s) TestResolverError(t *testing.T) {
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
 	cdsB.ResolverError(resourceErr)
 	// Make sure the registered watch is cancelled.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, watchForCancel failed: %v", err)
 	}
 	// Make sure the error is forwarded to the EDS balancer.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := edsB.waitForResolverError(ctx, resourceErr); err != nil {
 		t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err)
 	}
@@ -581,7 +578,9 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
-	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
@@ -591,8 +590,6 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	cdsB.UpdateSubConnState(sc, state)
 
 	// Make sure that the update is forwarded to the EDS balancer.
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := edsB.waitForSubConnUpdate(ctx, subConnWithState{sc: sc, state: state}); err != nil {
 		t.Fatal(err)
 	}
@@ -613,7 +610,9 @@ func (s) TestClose(t *testing.T) {
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, false, xdsC)
-	if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
 
@@ -622,15 +621,19 @@ func (s) TestClose(t *testing.T) {
 
 	// Make sure that the cluster watch registered by the CDS balancer is
 	// cancelled.
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatal(err)
 	}
 
+	// Make sure that a cluster update is not acted upon.
+	xdsC.InvokeWatchClusterCallback(cdsUpdate, nil)
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForClientConnUpdate(sCtx, balancer.ClientConnState{}); err != context.DeadlineExceeded {
+		t.Fatalf("ClusterUpdate after close forwaded to EDS balancer")
+	}
+
 	// Make sure that the underlying EDS balancer is closed.
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := edsB.waitForClose(ctx); err != nil {
 		t.Fatal(err)
 	}
@@ -644,9 +647,9 @@ func (s) TestClose(t *testing.T) {
 	// Make sure that the UpdateSubConnState() method on the CDS balancer does
 	// not forward the update to the EDS balancer.
 	cdsB.UpdateSubConnState(&xdstestutils.TestSubConn{}, balancer.SubConnState{})
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := edsB.waitForSubConnUpdate(ctx, subConnWithState{}); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForSubConnUpdate(sCtx, subConnWithState{}); err != context.DeadlineExceeded {
 		t.Fatal("UpdateSubConnState() forwarded to EDS balancer after Close()")
 	}
 
@@ -654,9 +657,9 @@ func (s) TestClose(t *testing.T) {
 	// forward the update to the EDS balancer.
 	rErr := errors.New("cdsBalancer resolver error")
 	cdsB.ResolverError(rErr)
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer ctxCancel()
-	if err := edsB.waitForResolverError(ctx, rErr); err != context.DeadlineExceeded {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsB.waitForResolverError(sCtx, rErr); err != context.DeadlineExceeded {
 		t.Fatal("ResolverError() forwarded to EDS balancer after Close()")
 	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index a148db0e42e6..2aa98f9d314b 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
@@ -153,6 +154,7 @@ func (x *edsBalancer) run() {
 // In both cases, the sub-balancers will be closed, and the future picks will
 // fail.
 func (x *edsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
+	x.logger.Warningf("Received error: %v", err)
 	if xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound {
 		if fromParent {
 			// This is an error from the parent ClientConn (can be the parent
@@ -201,7 +203,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 		x.handleErrorFromUpdate(u, true)
 	default:
 		// unreachable path
-		panic("wrong update type")
+		x.logger.Errorf("wrong update type: %T", update)
 	}
 }
 
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index f1c35d7be520..c2bbeafbdd8d 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -418,6 +418,14 @@ func New(opts Options) (*Client, error) {
 	return c, nil
 }
 
+// CertProviderConfigs returns the certificate provider configuration from the
+// "certificate_providers" field of the bootstrap file. The returned value is a
+// map from plugin_instance_name to {plugin_name, plugin_config}. Callers must
+// not modify the returned map.
+func (c *Client) CertProviderConfigs() map[string]bootstrap.CertProviderConfig {
+	return c.opts.Config.CertProviderConfigs
+}
+
 // run is a goroutine for all the callbacks.
 //
 // Callback can be called in watch(), if an item is found in cache. Without this
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index e655ffec8c51..cd81b8fa4166 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -20,6 +20,8 @@
 package testutils
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"testing"
 
@@ -153,6 +155,21 @@ func (tcc *TestClientConn) Target() string {
 	panic("not implemented")
 }
 
+// WaitForErrPicker waits until an error picker is pushed to this ClientConn.
+// Returns error if the provided context expires or a non-error picker is pushed
+// to the ClientConn.
+func (tcc *TestClientConn) WaitForErrPicker(ctx context.Context) error {
+	select {
+	case <-ctx.Done():
+		return errors.New("timeout when waiting for an error picker")
+	case picker := <-tcc.NewPickerCh:
+		if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil {
+			return fmt.Errorf("balancer returned a picker which is not an error picker")
+		}
+	}
+	return nil
+}
+
 // IsRoundRobin checks whether f's return value is roundrobin of elements from
 // want. But it doesn't check for the order. Note that want can contain
 // duplicate items, which makes it weight-round-robin.
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index c1fd0510c844..6496d1326d11 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -24,6 +24,7 @@ import (
 
 	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
@@ -42,6 +43,7 @@ type Client struct {
 	loadReportCh *testutils.Channel
 	closeCh      *testutils.Channel
 	loadStore    *load.Store
+	certConfigs  map[string]bootstrap.CertProviderConfig
 
 	ldsCb func(xdsclient.ListenerUpdate, error)
 	rdsCb func(xdsclient.RouteConfigUpdate, error)
@@ -221,6 +223,16 @@ func (xdsC *Client) WaitForClose(ctx context.Context) error {
 	return err
 }
 
+// CertProviderConfigs returns the configured certificate provider configs.
+func (xdsC *Client) CertProviderConfigs() map[string]bootstrap.CertProviderConfig {
+	return xdsC.certConfigs
+}
+
+// SetCertProviderConfigs updates the certificate provider configs.
+func (xdsC *Client) SetCertProviderConfigs(configs map[string]bootstrap.CertProviderConfig) {
+	xdsC.certConfigs = configs
+}
+
 // Name returns the name of the xds client.
 func (xdsC *Client) Name() string {
 	return xdsC.name

From 4e179b8d3ec42e48c4802f1938f9d593a5e91408 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 30 Oct 2020 15:52:55 -0700
Subject: [PATCH 258/481] pemfile: Move file watcher plugin from advancedtls to
 gRPC (#3981)

---
 .../tls/certprovider/pemfile/watcher.go       | 252 +++++++++++
 .../tls/certprovider/pemfile/watcher_test.go  | 426 ++++++++++++++++++
 .../advancedtls_integration_test.go           |  40 +-
 security/advancedtls/pemfile_provider.go      | 197 --------
 security/advancedtls/pemfile_provider_test.go | 220 ---------
 5 files changed, 699 insertions(+), 436 deletions(-)
 create mode 100644 credentials/tls/certprovider/pemfile/watcher.go
 create mode 100644 credentials/tls/certprovider/pemfile/watcher_test.go
 delete mode 100644 security/advancedtls/pemfile_provider.go
 delete mode 100644 security/advancedtls/pemfile_provider_test.go

diff --git a/credentials/tls/certprovider/pemfile/watcher.go b/credentials/tls/certprovider/pemfile/watcher.go
new file mode 100644
index 000000000000..29ea8b2b0654
--- /dev/null
+++ b/credentials/tls/certprovider/pemfile/watcher.go
@@ -0,0 +1,252 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package pemfile provides a file watching certificate provider plugin
+// implementation which works for files with PEM contents.
+//
+// Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
+package pemfile
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/grpclog"
+)
+
+const (
+	defaultCertRefreshDuration = 1 * time.Hour
+	defaultRootRefreshDuration = 2 * time.Hour
+)
+
+var (
+	// For overriding from unit tests.
+	newDistributor = func() distributor { return certprovider.NewDistributor() }
+
+	logger = grpclog.Component("pemfile")
+)
+
+// Options configures a certificate provider plugin that watches a specified set
+// of files that contain certificates and keys in PEM format.
+type Options struct {
+	// CertFile is the file that holds the identity certificate.
+	// Optional. If this is set, KeyFile must also be set.
+	CertFile string
+	// KeyFile is the file that holds identity private key.
+	// Optional. If this is set, CertFile must also be set.
+	KeyFile string
+	// RootFile is the file that holds trusted root certificate(s).
+	// Optional.
+	RootFile string
+	// CertRefreshDuration is the amount of time the plugin waits before
+	// checking for updates in the specified identity certificate and key file.
+	// Optional. If not set, a default value (1 hour) will be used.
+	CertRefreshDuration time.Duration
+	// RootRefreshDuration is the amount of time the plugin waits before
+	// checking for updates in the specified root file.
+	// Optional. If not set, a default value (2 hour) will be used.
+	RootRefreshDuration time.Duration
+}
+
+// NewProvider returns a new certificate provider plugin that is configured to
+// watch the PEM files specified in the passed in options.
+func NewProvider(o Options) (certprovider.Provider, error) {
+	if o.CertFile == "" && o.KeyFile == "" && o.RootFile == "" {
+		return nil, fmt.Errorf("pemfile: at least one credential file needs to be specified")
+	}
+	if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified {
+		return nil, fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified")
+	}
+	if o.CertRefreshDuration == 0 {
+		o.CertRefreshDuration = defaultCertRefreshDuration
+	}
+	if o.RootRefreshDuration == 0 {
+		o.RootRefreshDuration = defaultRootRefreshDuration
+	}
+
+	provider := &watcher{opts: o}
+	if o.CertFile != "" && o.KeyFile != "" {
+		provider.identityDistributor = newDistributor()
+	}
+	if o.RootFile != "" {
+		provider.rootDistributor = newDistributor()
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	provider.cancel = cancel
+	go provider.run(ctx)
+
+	return provider, nil
+}
+
+// watcher is a certificate provider plugin that implements the
+// certprovider.Provider interface. It watches a set of certificate and key
+// files and provides the most up-to-date key material for consumption by
+// credentials implementation.
+type watcher struct {
+	identityDistributor distributor
+	rootDistributor     distributor
+	opts                Options
+	certFileContents    []byte
+	keyFileContents     []byte
+	rootFileContents    []byte
+	cancel              context.CancelFunc
+}
+
+// distributor wraps the methods on certprovider.Distributor which are used by
+// the plugin. This is very useful in tests which need to know exactly when the
+// plugin updates its key material.
+type distributor interface {
+	KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error)
+	Set(km *certprovider.KeyMaterial, err error)
+	Stop()
+}
+
+// updateIdentityDistributor checks if the cert/key files that the plugin is
+// watching have changed, and if so, reads the new contents and updates the
+// identityDistributor with the new key material.
+//
+// Skips updates when file reading or parsing fails.
+// TODO(easwars): Retry with limit (on the number of retries or the amount of
+// time) upon failures.
+func (w *watcher) updateIdentityDistributor() {
+	if w.identityDistributor == nil {
+		return
+	}
+
+	certFileContents, err := ioutil.ReadFile(w.opts.CertFile)
+	if err != nil {
+		logger.Warningf("certFile (%s) read failed: %v", w.opts.CertFile, err)
+		return
+	}
+	keyFileContents, err := ioutil.ReadFile(w.opts.KeyFile)
+	if err != nil {
+		logger.Warningf("keyFile (%s) read failed: %v", w.opts.KeyFile, err)
+		return
+	}
+	// If the file contents have not changed, skip updating the distributor.
+	if bytes.Equal(w.certFileContents, certFileContents) && bytes.Equal(w.keyFileContents, keyFileContents) {
+		return
+	}
+
+	cert, err := tls.X509KeyPair(certFileContents, keyFileContents)
+	if err != nil {
+		logger.Warningf("tls.X509KeyPair(%q, %q) failed: %v", certFileContents, keyFileContents, err)
+		return
+	}
+	w.certFileContents = certFileContents
+	w.keyFileContents = keyFileContents
+	w.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}, nil)
+}
+
+// updateRootDistributor checks if the root cert file that the plugin is
+// watching hs changed, and if so, updates the rootDistributor with the new key
+// material.
+//
+// Skips updates when root cert reading or parsing fails.
+// TODO(easwars): Retry with limit (on the number of retries or the amount of
+// time) upon failures.
+func (w *watcher) updateRootDistributor() {
+	if w.rootDistributor == nil {
+		return
+	}
+
+	rootFileContents, err := ioutil.ReadFile(w.opts.RootFile)
+	if err != nil {
+		logger.Warningf("rootFile (%s) read failed: %v", w.opts.RootFile, err)
+		return
+	}
+	trustPool := x509.NewCertPool()
+	if !trustPool.AppendCertsFromPEM(rootFileContents) {
+		logger.Warning("failed to parse root certificate")
+		return
+	}
+	// If the file contents have not changed, skip updating the distributor.
+	if bytes.Equal(w.rootFileContents, rootFileContents) {
+		return
+	}
+
+	w.rootFileContents = rootFileContents
+	w.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil)
+}
+
+// run is a long running goroutine which watches the configured files for
+// changes, and pushes new key material into the appropriate distributors which
+// is returned from calls to KeyMaterial().
+func (w *watcher) run(ctx context.Context) {
+	// Update both root and identity certs at the beginning. Subsequently,
+	// update only the appropriate file whose ticker has fired.
+	w.updateIdentityDistributor()
+	w.updateRootDistributor()
+
+	identityTicker := time.NewTicker(w.opts.CertRefreshDuration)
+	rootTicker := time.NewTicker(w.opts.RootRefreshDuration)
+	for {
+		select {
+		case <-ctx.Done():
+			identityTicker.Stop()
+			rootTicker.Stop()
+			if w.identityDistributor != nil {
+				w.identityDistributor.Stop()
+			}
+			if w.rootDistributor != nil {
+				w.rootDistributor.Stop()
+			}
+			return
+		case <-identityTicker.C:
+			w.updateIdentityDistributor()
+		case <-rootTicker.C:
+			w.updateRootDistributor()
+		}
+	}
+}
+
+// KeyMaterial returns the key material sourced by the watcher.
+// Callers are expected to use the returned value as read-only.
+func (w *watcher) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
+	km := &certprovider.KeyMaterial{}
+	if w.identityDistributor != nil {
+		identityKM, err := w.identityDistributor.KeyMaterial(ctx)
+		if err != nil {
+			return nil, err
+		}
+		km.Certs = identityKM.Certs
+	}
+	if w.rootDistributor != nil {
+		rootKM, err := w.rootDistributor.KeyMaterial(ctx)
+		if err != nil {
+			return nil, err
+		}
+		km.Roots = rootKM.Roots
+	}
+	return km, nil
+}
+
+// Close cleans up resources allocated by the watcher.
+func (w *watcher) Close() {
+	w.cancel()
+}
diff --git a/credentials/tls/certprovider/pemfile/watcher_test.go b/credentials/tls/certprovider/pemfile/watcher_test.go
new file mode 100644
index 000000000000..092bd30ece63
--- /dev/null
+++ b/credentials/tls/certprovider/pemfile/watcher_test.go
@@ -0,0 +1,426 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pemfile
+
+import (
+	"context"
+	"crypto/x509"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/testdata"
+)
+
+const (
+	// These are the names of files inside temporary directories, which the
+	// plugin is asked to watch.
+	certFile = "cert.pem"
+	keyFile  = "key.pem"
+	rootFile = "ca.pem"
+
+	defaultTestRefreshDuration = 100 * time.Millisecond
+	defaultTestTimeout         = 5 * time.Second
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// TestNewProvider tests the NewProvider() function with different inputs.
+func (s) TestNewProvider(t *testing.T) {
+	tests := []struct {
+		desc      string
+		options   Options
+		wantError bool
+	}{
+		{
+			desc:      "No credential files specified",
+			options:   Options{},
+			wantError: true,
+		},
+		{
+			desc: "Only identity cert is specified",
+			options: Options{
+				CertFile: testdata.Path("x509/client1_cert.pem"),
+			},
+			wantError: true,
+		},
+		{
+			desc: "Only identity key is specified",
+			options: Options{
+				KeyFile: testdata.Path("x509/client1_key.pem"),
+			},
+			wantError: true,
+		},
+		{
+			desc: "Identity cert/key pair is specified",
+			options: Options{
+				KeyFile:  testdata.Path("x509/client1_key.pem"),
+				CertFile: testdata.Path("x509/client1_cert.pem"),
+			},
+		},
+		{
+			desc: "Only root certs are specified",
+			options: Options{
+				RootFile: testdata.Path("x509/client_ca_cert.pem"),
+			},
+		},
+		{
+			desc: "Everything is specified",
+			options: Options{
+				KeyFile:  testdata.Path("x509/client1_key.pem"),
+				CertFile: testdata.Path("x509/client1_cert.pem"),
+				RootFile: testdata.Path("x509/client_ca_cert.pem"),
+			},
+			wantError: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			provider, err := NewProvider(test.options)
+			if (err != nil) != test.wantError {
+				t.Fatalf("NewProvider(%v) = %v, want %v", test.options, err, test.wantError)
+			}
+			if err != nil {
+				return
+			}
+			provider.Close()
+		})
+	}
+}
+
+// wrappedDistributor wraps a distributor and pushes on a channel whenever new
+// key material is pushed to the distributor.
+type wrappedDistributor struct {
+	*certprovider.Distributor
+	distCh *testutils.Channel
+}
+
+func newWrappedDistributor(distCh *testutils.Channel) *wrappedDistributor {
+	return &wrappedDistributor{
+		distCh:      distCh,
+		Distributor: certprovider.NewDistributor(),
+	}
+}
+
+func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) {
+	wd.Distributor.Set(km, err)
+	wd.distCh.Send(nil)
+}
+
+func createTmpFile(t *testing.T, src, dst string) {
+	t.Helper()
+
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		t.Fatalf("ioutil.ReadFile(%q) failed: %v", src, err)
+	}
+	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
+		t.Fatalf("ioutil.WriteFile(%q) failed: %v", dst, err)
+	}
+	t.Logf("Wrote file at: %s", dst)
+	t.Logf("%s", string(data))
+}
+
+// createTempDirWithFiles creates a temporary directory under the system default
+// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
+// rootSrc files are creates appropriate files under the newly create tempDir.
+// Returns the name of the created tempDir.
+func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc string) string {
+	t.Helper()
+
+	// Create a temp directory. Passing an empty string for the first argument
+	// uses the system temp directory.
+	dir, err := ioutil.TempDir("", dirSuffix)
+	if err != nil {
+		t.Fatalf("ioutil.TempDir() failed: %v", err)
+	}
+	t.Logf("Using tmpdir: %s", dir)
+
+	createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile))
+	createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile))
+	createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile))
+	return dir
+}
+
+// initializeProvider performs setup steps common to all tests (except the one
+// which uses symlinks).
+func initializeProvider(t *testing.T, testName string) (string, certprovider.Provider, *testutils.Channel, func()) {
+	t.Helper()
+
+	// Override the newDistributor to one which pushes on a channel that we
+	// can block on.
+	origDistributorFunc := newDistributor
+	distCh := testutils.NewChannel()
+	d := newWrappedDistributor(distCh)
+	newDistributor = func() distributor { return d }
+
+	// Create a new provider to watch the files in tmpdir.
+	dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
+	opts := Options{
+		CertFile:            path.Join(dir, certFile),
+		KeyFile:             path.Join(dir, keyFile),
+		RootFile:            path.Join(dir, rootFile),
+		CertRefreshDuration: defaultTestRefreshDuration,
+		RootRefreshDuration: defaultTestRefreshDuration,
+	}
+	prov, err := NewProvider(opts)
+	if err != nil {
+		t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
+	}
+
+	// Make sure the provider picks up the files and pushes the key material on
+	// to the distributors.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for i := 0; i < 2; i++ {
+		// Since we have root and identity certs, we need to make sure the
+		// update is pushed on both of them.
+		if _, err := distCh.Receive(ctx); err != nil {
+			t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
+		}
+	}
+
+	return dir, prov, distCh, func() {
+		newDistributor = origDistributorFunc
+		prov.Close()
+	}
+}
+
+// TestProvider_NoUpdate tests the case where a file watcher plugin is created
+// successfully, and the underlying files do not change. Verifies that the
+// plugin does not push new updates to the distributor in this case.
+func (s) TestProvider_NoUpdate(t *testing.T) {
+	_, prov, distCh, cancel := initializeProvider(t, "no_update")
+	defer cancel()
+
+	// Make sure the provider is healthy and returns key material.
+	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cc()
+	if _, err := prov.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Files haven't change. Make sure no updates are pushed by the provider.
+	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+	defer sc()
+	if _, err := distCh.Receive(sCtx); err == nil {
+		t.Fatal("new key material pushed to distributor when underlying files did not change")
+	}
+}
+
+// TestProvider_UpdateSuccess tests the case where a file watcher plugin is
+// created successfully and the underlying files change. Verifies that the
+// changes are picked up by the provider.
+func (s) TestProvider_UpdateSuccess(t *testing.T) {
+	dir, prov, distCh, cancel := initializeProvider(t, "update_success")
+	defer cancel()
+
+	// Make sure the provider is healthy and returns key material.
+	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cc()
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Change only the root file.
+	createTmpFile(t, testdata.Path("x509/server_ca_cert.pem"), path.Join(dir, rootFile))
+	if _, err := distCh.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for new key material to be pushed to the distributor")
+	}
+
+	// Make sure update is picked up.
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if cmp.Equal(km1, km2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+		t.Fatal("expected provider to return new key material after update to underlying file")
+	}
+
+	// Change only cert/key files.
+	createTmpFile(t, testdata.Path("x509/client2_cert.pem"), path.Join(dir, certFile))
+	createTmpFile(t, testdata.Path("x509/client2_key.pem"), path.Join(dir, keyFile))
+	if _, err := distCh.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for new key material to be pushed to the distributor")
+	}
+
+	// Make sure update is picked up.
+	km3, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if cmp.Equal(km2, km3, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+		t.Fatal("expected provider to return new key material after update to underlying file")
+	}
+}
+
+// TestProvider_UpdateSuccessWithSymlink tests the case where a file watcher
+// plugin is created successfully to watch files through a symlink and the
+// symlink is updates to point to new files. Verifies that the changes are
+// picked up by the provider.
+func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) {
+	// Override the newDistributor to one which pushes on a channel that we
+	// can block on.
+	origDistributorFunc := newDistributor
+	distCh := testutils.NewChannel()
+	d := newWrappedDistributor(distCh)
+	newDistributor = func() distributor { return d }
+	defer func() { newDistributor = origDistributorFunc }()
+
+	// Create two tempDirs with different files.
+	dir1 := createTmpDirWithFiles(t, "update_with_symlink1_*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
+	dir2 := createTmpDirWithFiles(t, "update_with_symlink2_*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/server_ca_cert.pem")
+
+	// Create a symlink under a new tempdir, and make it point to dir1.
+	tmpdir, err := ioutil.TempDir("", "test_symlink_*")
+	if err != nil {
+		t.Fatalf("ioutil.TempDir() failed: %v", err)
+	}
+	symLinkName := path.Join(tmpdir, "test_symlink")
+	if err := os.Symlink(dir1, symLinkName); err != nil {
+		t.Fatalf("failed to create symlink to %q: %v", dir1, err)
+	}
+
+	// Create a provider which watches the files pointed to by the symlink.
+	opts := Options{
+		CertFile:            path.Join(symLinkName, certFile),
+		KeyFile:             path.Join(symLinkName, keyFile),
+		RootFile:            path.Join(symLinkName, rootFile),
+		CertRefreshDuration: defaultTestRefreshDuration,
+		RootRefreshDuration: defaultTestRefreshDuration,
+	}
+	prov, err := NewProvider(opts)
+	if err != nil {
+		t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
+	}
+	defer prov.Close()
+
+	// Make sure the provider picks up the files and pushes the key material on
+	// to the distributors.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for i := 0; i < 2; i++ {
+		// Since we have root and identity certs, we need to make sure the
+		// update is pushed on both of them.
+		if _, err := distCh.Receive(ctx); err != nil {
+			t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
+		}
+	}
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Update the symlink to point to dir2.
+	symLinkTmpName := path.Join(tmpdir, "test_symlink.tmp")
+	if err := os.Symlink(dir2, symLinkTmpName); err != nil {
+		t.Fatalf("failed to create symlink to %q: %v", dir2, err)
+	}
+	if err := os.Rename(symLinkTmpName, symLinkName); err != nil {
+		t.Fatalf("failed to update symlink: %v", err)
+	}
+
+	// Make sure the provider picks up the new files and pushes the key material
+	// on to the distributors.
+	for i := 0; i < 2; i++ {
+		// Since we have root and identity certs, we need to make sure the
+		// update is pushed on both of them.
+		if _, err := distCh.Receive(ctx); err != nil {
+			t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
+		}
+	}
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	if cmp.Equal(km1, km2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+		t.Fatal("expected provider to return new key material after symlink update")
+	}
+}
+
+// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
+// files fail. Verifies that the failed update does not push anything on the
+// distributor. Then the update succeeds, and the test verifies that the key
+// material is updated.
+func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) {
+	dir, prov, distCh, cancel := initializeProvider(t, "update_failure")
+	defer cancel()
+
+	// Make sure the provider is healthy and returns key material.
+	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cc()
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Update only the cert file. The key file is left unchanged. This should
+	// lead to these two files being not compatible with each other. This
+	// simulates the case where the watching goroutine might catch the files in
+	// the midst of an update.
+	createTmpFile(t, testdata.Path("x509/server1_cert.pem"), path.Join(dir, certFile))
+
+	// Since the last update left the files in an incompatible state, the update
+	// should not be picked up by our provider.
+	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+	defer sc()
+	if _, err := distCh.Receive(sCtx); err == nil {
+		t.Fatal("new key material pushed to distributor when underlying files did not change")
+	}
+
+	// The provider should return key material corresponding to the old state.
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if !cmp.Equal(km1, km2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+		t.Fatal("expected provider to not update key material")
+	}
+
+	// Update the key file to match the cert file.
+	createTmpFile(t, testdata.Path("x509/server1_key.pem"), path.Join(dir, keyFile))
+
+	// Make sure update is picked up.
+	if _, err := distCh.Receive(ctx); err != nil {
+		t.Fatal("timeout waiting for new key material to be pushed to the distributor")
+	}
+	km3, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if cmp.Equal(km2, km3, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+		t.Fatal("expected provider to return new key material after update to underlying file")
+	}
+}
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 20a9a5857968..d554468557a3 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -32,6 +32,8 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 	"google.golang.org/grpc/security/advancedtls/internal/testutils"
 	"google.golang.org/grpc/security/advancedtls/testdata"
@@ -511,38 +513,38 @@ func copyFileContents(sourceFile, destinationFile string) error {
 
 // Create PEMFileProvider(s) watching the content changes of temporary
 // files.
-func createProviders(tmpFiles *tmpCredsFiles) (*PEMFileProvider, *PEMFileProvider, *PEMFileProvider, *PEMFileProvider, error) {
-	clientIdentityOptions := PEMFileProviderOptions{
-		CertFile:         tmpFiles.clientCertTmp.Name(),
-		KeyFile:          tmpFiles.clientKeyTmp.Name(),
-		IdentityInterval: credRefreshingInterval,
+func createProviders(tmpFiles *tmpCredsFiles) (certprovider.Provider, certprovider.Provider, certprovider.Provider, certprovider.Provider, error) {
+	clientIdentityOptions := pemfile.Options{
+		CertFile:            tmpFiles.clientCertTmp.Name(),
+		KeyFile:             tmpFiles.clientKeyTmp.Name(),
+		CertRefreshDuration: credRefreshingInterval,
 	}
-	clientIdentityProvider, err := NewPEMFileProvider(clientIdentityOptions)
+	clientIdentityProvider, err := pemfile.NewProvider(clientIdentityOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
-	clientRootOptions := PEMFileProviderOptions{
-		TrustFile:    tmpFiles.clientTrustTmp.Name(),
-		RootInterval: credRefreshingInterval,
+	clientRootOptions := pemfile.Options{
+		RootFile:            tmpFiles.clientTrustTmp.Name(),
+		RootRefreshDuration: credRefreshingInterval,
 	}
-	clientRootProvider, err := NewPEMFileProvider(clientRootOptions)
+	clientRootProvider, err := pemfile.NewProvider(clientRootOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
-	serverIdentityOptions := PEMFileProviderOptions{
-		CertFile:         tmpFiles.serverCertTmp.Name(),
-		KeyFile:          tmpFiles.serverKeyTmp.Name(),
-		IdentityInterval: credRefreshingInterval,
+	serverIdentityOptions := pemfile.Options{
+		CertFile:            tmpFiles.serverCertTmp.Name(),
+		KeyFile:             tmpFiles.serverKeyTmp.Name(),
+		CertRefreshDuration: credRefreshingInterval,
 	}
-	serverIdentityProvider, err := NewPEMFileProvider(serverIdentityOptions)
+	serverIdentityProvider, err := pemfile.NewProvider(serverIdentityOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
-	serverRootOptions := PEMFileProviderOptions{
-		TrustFile:    tmpFiles.serverTrustTmp.Name(),
-		RootInterval: credRefreshingInterval,
+	serverRootOptions := pemfile.Options{
+		RootFile:            tmpFiles.serverTrustTmp.Name(),
+		RootRefreshDuration: credRefreshingInterval,
 	}
-	serverRootProvider, err := NewPEMFileProvider(serverRootOptions)
+	serverRootProvider, err := pemfile.NewProvider(serverRootOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
diff --git a/security/advancedtls/pemfile_provider.go b/security/advancedtls/pemfile_provider.go
deleted file mode 100644
index 96b3587776e8..000000000000
--- a/security/advancedtls/pemfile_provider.go
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package advancedtls
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"fmt"
-	"io/ioutil"
-	"time"
-
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/grpclog"
-)
-
-const defaultIdentityInterval = 1 * time.Hour
-const defaultRootInterval = 2 * time.Hour
-
-// readKeyCertPairFunc will be overridden from unit tests.
-var readKeyCertPairFunc = tls.LoadX509KeyPair
-
-// readTrustCertFunc will be overridden from unit tests.
-var readTrustCertFunc = func(trustFile string) (*x509.CertPool, error) {
-	trustData, err := ioutil.ReadFile(trustFile)
-	if err != nil {
-		return nil, err
-	}
-	trustPool := x509.NewCertPool()
-	if !trustPool.AppendCertsFromPEM(trustData) {
-		return nil, fmt.Errorf("AppendCertsFromPEM failed to parse certificates")
-	}
-	return trustPool, nil
-}
-
-var logger = grpclog.Component("advancedtls")
-
-// PEMFileProviderOptions contains options to configure a PEMFileProvider.
-// Note that these fields will only take effect during construction. Once the
-// PEMFileProvider starts, changing fields in PEMFileProviderOptions will have
-// no effect.
-type PEMFileProviderOptions struct {
-	// CertFile is the file path that holds identity certificate whose updates
-	// will be captured by a watching goroutine.
-	// Optional. If this is set, KeyFile must also be set.
-	CertFile string
-	// KeyFile is the file path that holds identity private key whose updates
-	// will be captured by a watching goroutine.
-	// Optional. If this is set, CertFile must also be set.
-	KeyFile string
-	// TrustFile is the file path that holds trust certificate whose updates will
-	// be captured by a watching goroutine.
-	// Optional.
-	TrustFile string
-	// IdentityInterval is the time duration between two credential update checks
-	// for identity certs.
-	// Optional. If not set, we will use the default interval(1 hour).
-	IdentityInterval time.Duration
-	// RootInterval is the time duration between two credential update checks
-	// for root certs.
-	// Optional. If not set, we will use the default interval(2 hours).
-	RootInterval time.Duration
-}
-
-// PEMFileProvider implements certprovider.Provider.
-// It provides the most up-to-date identity private key-cert pairs and/or
-// root certificates.
-type PEMFileProvider struct {
-	identityDistributor *certprovider.Distributor
-	rootDistributor     *certprovider.Distributor
-	cancel              context.CancelFunc
-}
-
-func updateIdentityDistributor(distributor *certprovider.Distributor, certFile, keyFile string) {
-	if distributor == nil {
-		return
-	}
-	// Read identity certs from PEM files.
-	identityCert, err := readKeyCertPairFunc(certFile, keyFile)
-	if err != nil {
-		// If the reading produces an error, we will skip the update for this
-		// round and log the error.
-		logger.Warningf("tls.LoadX509KeyPair reads %s and %s failed: %v", certFile, keyFile, err)
-		return
-	}
-	distributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{identityCert}}, nil)
-}
-
-func updateRootDistributor(distributor *certprovider.Distributor, trustFile string) {
-	if distributor == nil {
-		return
-	}
-	// Read root certs from PEM files.
-	trustPool, err := readTrustCertFunc(trustFile)
-	if err != nil {
-		// If the reading produces an error, we will skip the update for this
-		// round and log the error.
-		logger.Warningf("readTrustCertFunc reads %v failed: %v", trustFile, err)
-		return
-	}
-	distributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil)
-}
-
-// NewPEMFileProvider returns a new PEMFileProvider constructed using the
-// provided options.
-func NewPEMFileProvider(o PEMFileProviderOptions) (*PEMFileProvider, error) {
-	if o.CertFile == "" && o.KeyFile == "" && o.TrustFile == "" {
-		return nil, fmt.Errorf("at least one credential file needs to be specified")
-	}
-	if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified {
-		return nil, fmt.Errorf("private key file and identity cert file should be both specified or not specified")
-	}
-	if o.IdentityInterval == 0 {
-		o.IdentityInterval = defaultIdentityInterval
-	}
-	if o.RootInterval == 0 {
-		o.RootInterval = defaultRootInterval
-	}
-	provider := &PEMFileProvider{}
-	if o.CertFile != "" && o.KeyFile != "" {
-		provider.identityDistributor = certprovider.NewDistributor()
-	}
-	if o.TrustFile != "" {
-		provider.rootDistributor = certprovider.NewDistributor()
-	}
-	// A goroutine to pull file changes.
-	identityTicker := time.NewTicker(o.IdentityInterval)
-	rootTicker := time.NewTicker(o.RootInterval)
-	ctx, cancel := context.WithCancel(context.Background())
-
-	go func() {
-		for {
-			updateIdentityDistributor(provider.identityDistributor, o.CertFile, o.KeyFile)
-			updateRootDistributor(provider.rootDistributor, o.TrustFile)
-			select {
-			case <-ctx.Done():
-				identityTicker.Stop()
-				rootTicker.Stop()
-				return
-			case <-identityTicker.C:
-				break
-			case <-rootTicker.C:
-				break
-			}
-		}
-	}()
-	provider.cancel = cancel
-	return provider, nil
-}
-
-// KeyMaterial returns the key material sourced by the PEMFileProvider.
-// Callers are expected to use the returned value as read-only.
-func (p *PEMFileProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
-	km := &certprovider.KeyMaterial{}
-	if p.identityDistributor != nil {
-		identityKM, err := p.identityDistributor.KeyMaterial(ctx)
-		if err != nil {
-			return nil, err
-		}
-		km.Certs = identityKM.Certs
-	}
-	if p.rootDistributor != nil {
-		rootKM, err := p.rootDistributor.KeyMaterial(ctx)
-		if err != nil {
-			return nil, err
-		}
-		km.Roots = rootKM.Roots
-	}
-	return km, nil
-}
-
-// Close cleans up resources allocated by the PEMFileProvider.
-func (p *PEMFileProvider) Close() {
-	p.cancel()
-	if p.identityDistributor != nil {
-		p.identityDistributor.Stop()
-	}
-	if p.rootDistributor != nil {
-		p.rootDistributor.Stop()
-	}
-}
diff --git a/security/advancedtls/pemfile_provider_test.go b/security/advancedtls/pemfile_provider_test.go
deleted file mode 100644
index 48e0bd2f1c3b..000000000000
--- a/security/advancedtls/pemfile_provider_test.go
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package advancedtls
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"fmt"
-	"math/big"
-	"testing"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/security/advancedtls/internal/testutils"
-	"google.golang.org/grpc/security/advancedtls/testdata"
-)
-
-func (s) TestNewPEMFileProvider(t *testing.T) {
-	tests := []struct {
-		desc      string
-		options   PEMFileProviderOptions
-		certFile  string
-		keyFile   string
-		trustFile string
-		wantError bool
-	}{
-		{
-			desc:      "Expect error if no credential files specified",
-			options:   PEMFileProviderOptions{},
-			wantError: true,
-		},
-		{
-			desc: "Expect error if only certFile is specified",
-			options: PEMFileProviderOptions{
-				CertFile: testdata.Path("client_cert_1.pem"),
-			},
-			wantError: true,
-		},
-		{
-			desc: "Should be good if only identity key cert pairs are specified",
-			options: PEMFileProviderOptions{
-				KeyFile:  testdata.Path("client_key_1.pem"),
-				CertFile: testdata.Path("client_cert_1.pem"),
-			},
-			wantError: false,
-		},
-		{
-			desc: "Should be good if only root certs are specified",
-			options: PEMFileProviderOptions{
-				TrustFile: testdata.Path("client_trust_cert_1.pem"),
-			},
-			wantError: false,
-		},
-		{
-			desc: "Should be good if both identity pairs and root certs are specified",
-			options: PEMFileProviderOptions{
-				KeyFile:   testdata.Path("client_key_1.pem"),
-				CertFile:  testdata.Path("client_cert_1.pem"),
-				TrustFile: testdata.Path("client_trust_cert_1.pem"),
-			},
-			wantError: false,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.desc, func(t *testing.T) {
-			provider, err := NewPEMFileProvider(test.options)
-			if (err != nil) != test.wantError {
-				t.Fatalf("NewPEMFileProvider(%v) = %v, want %v", test.options, err, test.wantError)
-			}
-			if err != nil {
-				return
-			}
-			provider.Close()
-		})
-	}
-
-}
-
-// This test overwrites the credential reading function used by the watching
-// goroutine. It is tested under different stages:
-// At stage 0, we force reading function to load ClientCert1 and ServerTrust1,
-// and see if the credentials are picked up by the watching go routine.
-// At stage 1, we force reading function to cause an error. The watching go
-// routine should log the error while leaving the credentials unchanged.
-// At stage 2, we force reading function to load ClientCert2 and ServerTrust2,
-// and see if the new credentials are picked up.
-func (s) TestWatchingRoutineUpdates(t *testing.T) {
-	// Load certificates.
-	cs := &testutils.CertStore{}
-	if err := cs.LoadCerts(); err != nil {
-		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
-	}
-	tests := []struct {
-		desc         string
-		options      PEMFileProviderOptions
-		wantKmStage0 certprovider.KeyMaterial
-		wantKmStage1 certprovider.KeyMaterial
-		wantKmStage2 certprovider.KeyMaterial
-	}{
-		{
-			desc: "use identity certs and root certs",
-			options: PEMFileProviderOptions{
-				CertFile:  "not_empty_cert_file",
-				KeyFile:   "not_empty_key_file",
-				TrustFile: "not_empty_trust_file",
-			},
-			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}, Roots: cs.ServerTrust1},
-			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}, Roots: cs.ServerTrust1},
-			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert2}, Roots: cs.ServerTrust2},
-		},
-		{
-			desc: "use identity certs only",
-			options: PEMFileProviderOptions{
-				CertFile: "not_empty_cert_file",
-				KeyFile:  "not_empty_key_file",
-			},
-			wantKmStage0: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}},
-			wantKmStage1: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}},
-			wantKmStage2: certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert2}},
-		},
-		{
-			desc: "use trust certs only",
-			options: PEMFileProviderOptions{
-				TrustFile: "not_empty_trust_file",
-			},
-			wantKmStage0: certprovider.KeyMaterial{Roots: cs.ServerTrust1},
-			wantKmStage1: certprovider.KeyMaterial{Roots: cs.ServerTrust1},
-			wantKmStage2: certprovider.KeyMaterial{Roots: cs.ServerTrust2},
-		},
-	}
-	for _, test := range tests {
-		testInterval := 200 * time.Millisecond
-		test.options.IdentityInterval = testInterval
-		test.options.RootInterval = testInterval
-		t.Run(test.desc, func(t *testing.T) {
-			stage := &stageInfo{}
-			oldReadKeyCertPairFunc := readKeyCertPairFunc
-			readKeyCertPairFunc = func(certFile, keyFile string) (tls.Certificate, error) {
-				switch stage.read() {
-				case 0:
-					return cs.ClientCert1, nil
-				case 1:
-					return tls.Certificate{}, fmt.Errorf("error occurred while reloading")
-				case 2:
-					return cs.ClientCert2, nil
-				default:
-					return tls.Certificate{}, fmt.Errorf("test stage not supported")
-				}
-			}
-			defer func() {
-				readKeyCertPairFunc = oldReadKeyCertPairFunc
-			}()
-			oldReadTrustCertFunc := readTrustCertFunc
-			readTrustCertFunc = func(trustFile string) (*x509.CertPool, error) {
-				switch stage.read() {
-				case 0:
-					return cs.ServerTrust1, nil
-				case 1:
-					return nil, fmt.Errorf("error occurred while reloading")
-				case 2:
-					return cs.ServerTrust2, nil
-				default:
-					return nil, fmt.Errorf("test stage not supported")
-				}
-			}
-			defer func() {
-				readTrustCertFunc = oldReadTrustCertFunc
-			}()
-			provider, err := NewPEMFileProvider(test.options)
-			if err != nil {
-				t.Fatalf("NewPEMFileProvider failed: %v", err)
-			}
-			defer provider.Close()
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-			//// ------------------------Stage 0------------------------------------
-			// Wait for the refreshing go-routine to pick up the changes.
-			time.Sleep(1 * time.Second)
-			gotKM, err := provider.KeyMaterial(ctx)
-			if !cmp.Equal(*gotKM, test.wantKmStage0, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
-				t.Fatalf("provider.KeyMaterial() = %+v, want %+v", *gotKM, test.wantKmStage0)
-			}
-			// ------------------------Stage 1------------------------------------
-			stage.increase()
-			// Wait for the refreshing go-routine to pick up the changes.
-			time.Sleep(1 * time.Second)
-			gotKM, err = provider.KeyMaterial(ctx)
-			if !cmp.Equal(*gotKM, test.wantKmStage1, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
-				t.Fatalf("provider.KeyMaterial() = %+v, want %+v", *gotKM, test.wantKmStage1)
-			}
-			//// ------------------------Stage 2------------------------------------
-			// Wait for the refreshing go-routine to pick up the changes.
-			stage.increase()
-			time.Sleep(1 * time.Second)
-			gotKM, err = provider.KeyMaterial(ctx)
-			if !cmp.Equal(*gotKM, test.wantKmStage2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
-				t.Fatalf("provider.KeyMaterial() = %+v, want %+v", *gotKM, test.wantKmStage2)
-			}
-			stage.reset()
-		})
-	}
-}

From 15ae9fc2b247ca1a5dd8cca129641955d4a22af2 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 2 Nov 2020 13:53:44 -0800
Subject: [PATCH 259/481] internal: increase timeout non timeout test in
 xds/client/v2 (#4006)

---
 xds/internal/client/v2/client_test.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index 7b8d6786b03e..5c9fbd56d6e4 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -609,10 +609,8 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 	fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")}
 	t.Log("Bad LDS response pushed to fakeServer...")
 
-	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer sCancel()
-	val, err := fakeServer.XDSRequestChan.Receive(sCtx)
-	if err == context.DeadlineExceeded {
+	val, err := fakeServer.XDSRequestChan.Receive(ctx)
+	if err != nil {
 		t.Fatalf("Timeout expired when expecting LDS update")
 	}
 	gotRequest := val.(*fakeserver.Request)

From 9a3c6618eeeed92cbfceb7d7a0cec7289aaad75b Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 3 Nov 2020 15:34:12 -0800
Subject: [PATCH 260/481] xds: remove xdsclient.New from EDS balancer (#4001)

---
 xds/internal/balancer/edsbalancer/config.go   |   4 -
 xds/internal/balancer/edsbalancer/eds.go      |  12 +-
 xds/internal/balancer/edsbalancer/eds_test.go | 169 +++++-------------
 .../edsbalancer/xds_client_wrapper.go         |  92 ++--------
 .../edsbalancer/xds_client_wrapper_test.go    |  85 +++------
 5 files changed, 84 insertions(+), 278 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/config.go b/xds/internal/balancer/edsbalancer/config.go
index 95fa6d5b6aa4..9b59cfa01ab3 100644
--- a/xds/internal/balancer/edsbalancer/config.go
+++ b/xds/internal/balancer/edsbalancer/config.go
@@ -29,8 +29,6 @@ import (
 // for EDS balancers.
 type EDSConfig struct {
 	serviceconfig.LoadBalancingConfig
-	// BalancerName represents the load balancer to use.
-	BalancerName string
 	// ChildPolicy represents the load balancing config for the child
 	// policy.
 	ChildPolicy *loadBalancingConfig
@@ -50,7 +48,6 @@ type EDSConfig struct {
 // and Fallbackspolicy are post-processed, and for each, the first installed
 // policy is kept.
 type edsConfigJSON struct {
-	BalancerName               string
 	ChildPolicy                []*loadBalancingConfig
 	FallbackPolicy             []*loadBalancingConfig
 	EDSServiceName             string
@@ -66,7 +63,6 @@ func (l *EDSConfig) UnmarshalJSON(data []byte) error {
 		return err
 	}
 
-	l.BalancerName = configJSON.BalancerName
 	l.EDSServiceName = configJSON.EDSServiceName
 	l.LrsLoadReportingServerName = configJSON.LRSLoadReportingServerName
 
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 2aa98f9d314b..9312b0532cbf 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -52,17 +52,16 @@ func init() {
 type edsBalancerBuilder struct{}
 
 // Build helps implement the balancer.Builder interface.
-func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
 	x := &edsBalancer{
 		cc:                cc,
-		buildOpts:         opts,
 		closed:            grpcsync.NewEvent(),
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
 	}
 	x.logger = prefixLogger((x))
-	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.buildOpts, x.logger)
+	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.logger)
 	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.client, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
@@ -103,10 +102,9 @@ type edsBalancerImplInterface interface {
 //
 // It currently has only an edsBalancer. Later, we may add fallback.
 type edsBalancer struct {
-	cc        balancer.ClientConn // *xdsClientConn
-	buildOpts balancer.BuildOptions
-	closed    *grpcsync.Event
-	logger    *grpclog.PrefixLogger
+	cc     balancer.ClientConn // *xdsClientConn
+	closed *grpcsync.Event
+	logger *grpclog.PrefixLogger
 
 	// edsBalancer continuously monitor the channels below, and will handle events from them in sync.
 	grpcUpdate        chan interface{}
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 62075aed29bb..085310d44b39 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -30,8 +30,9 @@ import (
 	"github.com/golang/protobuf/jsonpb"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/attributes"
+	xdsinternal "google.golang.org/grpc/xds/internal"
 
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpclog"
@@ -41,8 +42,6 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 
 	_ "google.golang.org/grpc/xds/internal/client/v2" // V2 client registration.
@@ -52,14 +51,6 @@ const defaultTestTimeout = 1 * time.Second
 
 func init() {
 	balancer.Register(&edsBalancerBuilder{})
-
-	bootstrapConfigNew = func() (*bootstrap.Config, error) {
-		return &bootstrap.Config{
-			BalancerName: testBalancerNameFooBar,
-			Creds:        grpc.WithInsecure(),
-			NodeProto:    xdstestutils.EmptyNodeProtoV2,
-		}, nil
-	}
 }
 
 func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
@@ -182,32 +173,6 @@ type fakeSubConn struct{}
 func (*fakeSubConn) UpdateAddresses([]resolver.Address) { panic("implement me") }
 func (*fakeSubConn) Connect()                           { panic("implement me") }
 
-// waitForNewXDSClientWithEDSWatch makes sure that a new xdsClient is created
-// with the provided name. It also make sure that the newly created client
-// registers an eds watcher.
-func waitForNewXDSClientWithEDSWatch(t *testing.T, ch *testutils.Channel, wantName string) *fakeclient.Client {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	val, err := ch.Receive(ctx)
-	if err != nil {
-		t.Fatalf("error when waiting for a new xds client: %v", err)
-		return nil
-	}
-	xdsC := val.(*fakeclient.Client)
-	if xdsC.Name() != wantName {
-		t.Fatalf("xdsClient created to balancer: %v, want %v", xdsC.Name(), wantName)
-		return nil
-	}
-	_, err = xdsC.WaitForWatchEDS(ctx)
-	if err != nil {
-		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
-		return nil
-	}
-	return xdsC
-}
-
 // waitForNewEDSLB makes sure that a new edsLB is created by the top-level
 // edsBalancer.
 func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
@@ -227,70 +192,15 @@ func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
 // edsLB, creates fake version of them and makes them available on the provided
 // channels. The returned cancel function should be called by the test for
 // cleanup.
-func setup(edsLBCh *testutils.Channel, xdsClientCh *testutils.Channel) func() {
+func setup(edsLBCh *testutils.Channel) func() {
 	origNewEDSBalancer := newEDSBalancer
 	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
 		edsLB := newFakeEDSBalancer(cc)
 		defer func() { edsLBCh.Send(edsLB) }()
 		return edsLB
 	}
-
-	origXdsClientNew := xdsclientNew
-	xdsclientNew = func(opts xdsclient.Options) (xdsClientInterface, error) {
-		xdsC := fakeclient.NewClientWithName(opts.Config.BalancerName)
-		defer func() { xdsClientCh.Send(xdsC) }()
-		return xdsC, nil
-	}
 	return func() {
 		newEDSBalancer = origNewEDSBalancer
-		xdsclientNew = origXdsClientNew
-	}
-}
-
-// TestXDSConfigBalancerNameUpdate verifies different scenarios where the
-// balancer name in the lbConfig is updated.
-//
-// The test does the following:
-// * Builds a new xds balancer.
-// * Repeatedly pushes new ClientConnState which specifies different
-//   balancerName in the lbConfig. We expect xdsClient objects to created
-//   whenever the balancerName changes.
-func (s) TestXDSConfigBalancerNameUpdate(t *testing.T) {
-	oldBootstrapConfigNew := bootstrapConfigNew
-	bootstrapConfigNew = func() (*bootstrap.Config, error) {
-		// Return an error from bootstrap, so the eds balancer will use
-		// BalancerName from the config.
-		//
-		// TODO: remove this when deleting BalancerName from config.
-		return nil, fmt.Errorf("no bootstrap available")
-	}
-	defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
-	edsLBCh := testutils.NewChannel()
-	xdsClientCh := testutils.NewChannel()
-	cancel := setup(edsLBCh, xdsClientCh)
-	defer cancel()
-
-	builder := balancer.Get(edsName)
-	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
-	if !ok {
-		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
-	}
-	defer edsB.Close()
-
-	addrs := []resolver.Address{{Addr: "1.1.1.1:10001"}, {Addr: "2.2.2.2:10002"}, {Addr: "3.3.3.3:10003"}}
-	for i := 0; i < 2; i++ {
-		balancerName := fmt.Sprintf("balancer-%d", i)
-		edsB.UpdateClientConnState(balancer.ClientConnState{
-			ResolverState: resolver.State{Addresses: addrs},
-			BalancerConfig: &EDSConfig{
-				BalancerName:   balancerName,
-				EDSServiceName: testEDSClusterName,
-			},
-		})
-
-		xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, balancerName)
-		xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	}
 }
 
@@ -351,9 +261,9 @@ func (b *fakeBalancer) Close() {}
 //   This time around, we expect no new xdsClient or edsLB to be created.
 //   Instead, we expect the existing edsLB to receive the new child policy.
 func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
+	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	xdsClientCh := testutils.NewChannel()
-	cancel := setup(edsLBCh, xdsClientCh)
+	cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -365,8 +275,8 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 	defer edsB.Close()
 
 	edsB.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{
-			BalancerName: testBalancerNameFooBar,
 			ChildPolicy: &loadBalancingConfig{
 				Name:   fakeBalancerA,
 				Config: json.RawMessage("{}"),
@@ -374,7 +284,12 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 			EDSServiceName: testEDSClusterName,
 		},
 	})
-	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
+		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
+	}
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	edsLB := waitForNewEDSLB(t, edsLBCh)
 	edsLB.waitForChildPolicy(&loadBalancingConfig{
@@ -383,8 +298,8 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 	})
 
 	edsB.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{
-			BalancerName: testBalancerNameFooBar,
 			ChildPolicy: &loadBalancingConfig{
 				Name:   fakeBalancerB,
 				Config: json.RawMessage("{}"),
@@ -401,9 +316,9 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 // TestXDSSubConnStateChange verifies if the top-level edsBalancer passes on
 // the subConnStateChange to appropriate child balancers.
 func (s) TestXDSSubConnStateChange(t *testing.T) {
+	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	xdsClientCh := testutils.NewChannel()
-	cancel := setup(edsLBCh, xdsClientCh)
+	cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -414,16 +329,16 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 	}
 	defer edsB.Close()
 
-	addrs := []resolver.Address{{Addr: "1.1.1.1:10001"}, {Addr: "2.2.2.2:10002"}, {Addr: "3.3.3.3:10003"}}
 	edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState: resolver.State{Addresses: addrs},
-		BalancerConfig: &EDSConfig{
-			BalancerName:   testBalancerNameFooBar,
-			EDSServiceName: testEDSClusterName,
-		},
+		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
+		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
 	})
 
-	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
+		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
+	}
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	edsLB := waitForNewEDSLB(t, edsLBCh)
 
@@ -442,9 +357,9 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 // If it's connection error, nothing will happen. This will need to change to
 // handle fallback.
 func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
+	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	xdsClientCh := testutils.NewChannel()
-	cancel := setup(edsLBCh, xdsClientCh)
+	cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -456,15 +371,17 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	defer edsB.Close()
 
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{
-			BalancerName:   testBalancerNameFooBar,
-			EDSServiceName: testEDSClusterName,
-		},
+		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
+		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
 	}); err != nil {
 		t.Fatal(err)
 	}
 
-	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
+		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
+	}
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	edsLB := waitForNewEDSLB(t, edsLBCh)
 	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
@@ -474,8 +391,6 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, connectionErr)
 
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
@@ -506,9 +421,9 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 // If it's connection error, nothing will happen. This will need to change to
 // handle fallback.
 func (s) TestErrorFromResolver(t *testing.T) {
+	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	xdsClientCh := testutils.NewChannel()
-	cancel := setup(edsLBCh, xdsClientCh)
+	cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -520,15 +435,17 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	defer edsB.Close()
 
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{
-			BalancerName:   testBalancerNameFooBar,
-			EDSServiceName: testEDSClusterName,
-		},
+		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
+		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
 	}); err != nil {
 		t.Fatal(err)
 	}
 
-	xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
+		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
+	}
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
 	edsLB := waitForNewEDSLB(t, edsLBCh)
 	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
@@ -538,8 +455,6 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
 	edsB.ResolverError(connectionErr)
 
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
@@ -610,7 +525,6 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 			name: "manually-generated",
 			js: json.RawMessage(`
 {
-  "balancerName": "fake.foo.bar",
   "childPolicy": [
     {"fake_balancer_C": {}},
     {"fake_balancer_A": {}},
@@ -625,7 +539,6 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
   "lrsLoadReportingServerName": "lrs.server"
 }`),
 			want: &EDSConfig{
-				BalancerName: "fake.foo.bar",
 				ChildPolicy: &loadBalancingConfig{
 					Name:   "fake_balancer_A",
 					Config: json.RawMessage("{}"),
@@ -645,11 +558,9 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 			name: "no-lrs-server-name",
 			js: json.RawMessage(`
 {
-  "balancerName": "fake.foo.bar",
   "edsServiceName": "eds.service"
 }`),
 			want: &EDSConfig{
-				BalancerName:               "fake.foo.bar",
 				EDSServiceName:             testEDSName,
 				LrsLoadReportingServerName: nil,
 			},
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index fe4e996a7b58..16466016bbcd 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -22,13 +22,10 @@ import (
 	"fmt"
 	"sync"
 
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
-	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
@@ -40,13 +37,6 @@ type xdsClientInterface interface {
 	Close()
 }
 
-var (
-	xdsclientNew = func(opts xdsclient.Options) (xdsClientInterface, error) {
-		return xdsclient.New(opts)
-	}
-	bootstrapConfigNew = bootstrap.NewConfig
-)
-
 type loadStoreWrapper struct {
 	mu      sync.RWMutex
 	service string
@@ -110,9 +100,7 @@ type xdsClientWrapper struct {
 	logger *grpclog.PrefixLogger
 
 	newEDSUpdate func(xdsclient.EndpointsUpdate, error)
-	bbo          balancer.BuildOptions
 
-	balancerName string
 	// xdsClient could come from attributes, or created with balancerName.
 	xdsClient xdsClientInterface
 
@@ -138,38 +126,14 @@ type xdsClientWrapper struct {
 //
 // The given callbacks won't be called until the underlying xds_client is
 // working and sends updates.
-func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, logger *grpclog.PrefixLogger) *xdsClientWrapper {
+func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), logger *grpclog.PrefixLogger) *xdsClientWrapper {
 	return &xdsClientWrapper{
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
-		bbo:          bbo,
 		loadWrapper:  &loadStoreWrapper{},
 	}
 }
 
-// replaceXDSClient replaces xdsClient fields to the newClient if they are
-// different. If xdsClient is replaced, the balancerName field will also be
-// updated to newBalancerName.
-//
-// If the old xdsClient is replaced, and was created locally (not from
-// attributes), it will be closed.
-//
-// It returns whether xdsClient is replaced.
-func (c *xdsClientWrapper) replaceXDSClient(newClient xdsClientInterface, newBalancerName string) bool {
-	if c.xdsClient == newClient {
-		return false
-	}
-	oldClient := c.xdsClient
-	oldBalancerName := c.balancerName
-	c.xdsClient = newClient
-	c.balancerName = newBalancerName
-	if oldBalancerName != "" {
-		// OldBalancerName!="" means if the old client was not from attributes.
-		oldClient.Close()
-	}
-	return true
-}
-
 // updateXDSClient sets xdsClient in wrapper to the correct one based on the
 // attributes and service config.
 //
@@ -184,44 +148,26 @@ func (c *xdsClientWrapper) replaceXDSClient(newClient xdsClientInterface, newBal
 // the balancerName (from bootstrap file or from service config) changed.
 // - if balancer names are the same, do nothing, and return false
 // - if balancer names are different, create new one, and return true
-func (c *xdsClientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.Attributes) (bool, error) {
-	if attr != nil {
-		if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil {
-			// This will also clear balancerName, to indicate that client is
-			// from attributes.
-			return c.replaceXDSClient(clientFromAttr, ""), nil
-		}
+func (c *xdsClientWrapper) updateXDSClient(attr *attributes.Attributes) (bool, error) {
+	if attr == nil {
+		return false, fmt.Errorf("unexported nil attributes, want attributes with xdsClient")
 	}
-
-	clientConfig, err := bootstrapConfigNew()
-	if err != nil {
-		// TODO: propagate this error to ClientConn, and fail RPCs if necessary.
-		clientConfig = &bootstrap.Config{BalancerName: config.BalancerName}
+	// TODO: change the way xdsClient is retrieved from attributes. One option
+	// is to add helper functions.
+	//
+	// Or, since xdsClient will become a singleton, this can just call
+	// xdsclient.New() instead. And if we decide to do this, do it in Build
+	// instead of when handling updates.
+	clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface)
+	if clientFromAttr == nil {
+		return false, fmt.Errorf("no xdsClient found in attributes")
 	}
 
-	if c.balancerName == clientConfig.BalancerName {
+	if c.xdsClient == clientFromAttr {
 		return false, nil
 	}
-
-	var dopts []grpc.DialOption
-	if dialer := c.bbo.Dialer; dialer != nil {
-		dopts = []grpc.DialOption{grpc.WithContextDialer(dialer)}
-	}
-
-	// TODO: there's no longer a need to read bootstrap file and create a new
-	// xds client. The EDS balancer should always get the xds client from
-	// attributes. Otherwise, this function should just fail. Also, xdsclient
-	// will be shared by multiple clients, so trying to make an xds client is
-	// just the wrong move.
-	newClient, err := xdsclientNew(xdsclient.Options{Config: *clientConfig, DialOpts: dopts})
-	if err != nil {
-		// This should never fail. xdsclientnew does a non-blocking dial, and
-		// all the config passed in should be validated.
-		//
-		// This could leave c.xdsClient as nil if this is the first update.
-		return false, fmt.Errorf("eds: failed to create xdsClient, error: %v", err)
-	}
-	return c.replaceXDSClient(newClient, clientConfig.BalancerName), nil
+	c.xdsClient = clientFromAttr
+	return true, nil
 }
 
 // startEndpointsWatch starts the EDS watch. Caller can call this when the
@@ -276,7 +222,7 @@ func (c *xdsClientWrapper) loadStore() load.PerClusterReporter {
 // handleUpdate applies the service config and attributes updates to the client,
 // including updating the xds_client to use, and updating the EDS name to watch.
 func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) error {
-	clientChanged, err := c.updateXDSClient(config, attr)
+	clientChanged, err := c.updateXDSClient(attr)
 	if err != nil {
 		return err
 	}
@@ -320,10 +266,6 @@ func (c *xdsClientWrapper) cancelWatch() {
 
 func (c *xdsClientWrapper) close() {
 	c.cancelWatch()
-	if c.xdsClient != nil && c.balancerName != "" {
-		// Only close xdsClient if it's not from attributes.
-		c.xdsClient.Close()
-	}
 }
 
 // equalStringPointers returns true if
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index c8dac9c9925e..38162d109ffa 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -24,22 +24,12 @@ import (
 	"fmt"
 	"testing"
 
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
-
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
-	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/testutils"
-	"google.golang.org/grpc/resolver"
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
-	"google.golang.org/grpc/xds/internal/version"
 )
 
 var (
@@ -49,28 +39,25 @@ var (
 
 // Given a list of resource names, verifies that EDS requests for the same are
 // received at the fake server.
-func verifyExpectedRequests(fs *fakeserver.Server, resourceNames ...string) error {
-	wantReq := &xdspb.DiscoveryRequest{
-		TypeUrl: version.V2EndpointsURL,
-		Node:    xdstestutils.EmptyNodeProtoV2,
-	}
+func verifyExpectedRequests(fc *fakeclient.Client, resourceNames ...string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	for _, name := range resourceNames {
-		if name != "" {
-			wantReq.ResourceNames = []string{name}
+		if name == "" {
+			// ResourceName empty string indicates a cancel.
+			if err := fc.WaitForCancelEDSWatch(ctx); err != nil {
+				return fmt.Errorf("timed out when expecting resource %q", name)
+			}
+			return nil
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		req, err := fs.XDSRequestChan.Receive(ctx)
+		resName, err := fc.WaitForWatchEDS(ctx)
 		if err != nil {
-			return fmt.Errorf("timed out when expecting request {%+v} at fake server", wantReq)
-		}
-		edsReq := req.(*fakeserver.Request)
-		if edsReq.Err != nil {
-			return fmt.Errorf("eds RPC failed with err: %v", edsReq.Err)
+			return fmt.Errorf("timed out when expecting resource %q, %p", name, fc)
 		}
-		if !proto.Equal(edsReq.Req, wantReq) {
-			return fmt.Errorf("got EDS request %v, expected: %v, diff: %s", edsReq.Req, wantReq, cmp.Diff(edsReq.Req, wantReq, cmp.Comparer(proto.Equal)))
+		if resName != name {
+			return fmt.Errorf("got EDS request for resource %q, expected: %q", resName, name)
 		}
 	}
 	return nil
@@ -86,34 +73,16 @@ func verifyExpectedRequests(fs *fakeserver.Server, resourceNames ...string) erro
 // * Sends updates with different edsServiceNames and expects new watches to be
 //   registered.
 func (s) TestClientWrapperWatchEDS(t *testing.T) {
-	fakeServer, cleanup, err := fakeserver.StartServer()
-	if err != nil {
-		t.Fatalf("Failed to start fake xDS server: %v", err)
-	}
-	defer cleanup()
-	t.Logf("Started fake xDS server at %s...", fakeServer.Address)
+	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 
-	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil)
+	cw := newXDSClientWrapper(nil, nil)
 	defer cw.close()
 	t.Logf("Started xDS client wrapper for endpoint %s...", testServiceName)
 
-	oldBootstrapConfigNew := bootstrapConfigNew
-	bootstrapConfigNew = func() (*bootstrap.Config, error) {
-		return &bootstrap.Config{
-			BalancerName: fakeServer.Address,
-			Creds:        grpc.WithInsecure(),
-			NodeProto:    xdstestutils.EmptyNodeProtoV2,
-		}, nil
-	}
-	defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
-
 	// Update with an non-empty edsServiceName should trigger an EDS watch for
 	// the same.
-	cw.handleUpdate(&EDSConfig{
-		BalancerName:   fakeServer.Address,
-		EDSServiceName: "foobar-1",
-	}, nil)
-	if err := verifyExpectedRequests(fakeServer, "foobar-1"); err != nil {
+	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-1"}, attributes.New(xdsinternal.XDSClientID, xdsC))
+	if err := verifyExpectedRequests(xdsC, "foobar-1"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -121,11 +90,8 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 	// name to another, and make sure a new watch is registered. The previously
 	// registered watch will be cancelled, which will result in an EDS request
 	// with no resource names being sent to the server.
-	cw.handleUpdate(&EDSConfig{
-		BalancerName:   fakeServer.Address,
-		EDSServiceName: "foobar-2",
-	}, nil)
-	if err := verifyExpectedRequests(fakeServer, "", "foobar-2"); err != nil {
+	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-2"}, attributes.New(xdsinternal.XDSClientID, xdsC))
+	if err := verifyExpectedRequests(xdsC, "", "foobar-2"); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -146,7 +112,7 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 		edsRespChan.Send(&edsUpdate{resp: update, err: err})
 	}
 
-	cw := newXDSClientWrapper(newEDS, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil)
+	cw := newXDSClientWrapper(newEDS, nil)
 	defer cw.close()
 
 	xdsC := fakeclient.NewClient()
@@ -184,14 +150,7 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 // clientWrapper receives the xdsClient to use in the attributes section of the
 // update.
 func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
-	oldxdsclientNew := xdsclientNew
-	xdsclientNew = func(_ xdsclient.Options) (xdsClientInterface, error) {
-		t.Fatalf("unexpected call to xdsclientNew when xds_client is set in attributes")
-		return nil, nil
-	}
-	defer func() { xdsclientNew = oldxdsclientNew }()
-
-	cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil)
+	cw := newXDSClientWrapper(nil, nil)
 	defer cw.close()
 
 	xdsC1 := fakeclient.NewClient()

From ea89aca6c66e00e3b4ffedcdf2f72df1f805ba4e Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 4 Nov 2020 07:37:33 -0800
Subject: [PATCH 261/481] xds: Add a package to hold environment variables.
 (#4002)

---
 xds/internal/client/bootstrap/bootstrap.go    | 24 ++-----
 .../client/bootstrap/bootstrap_test.go        | 67 ++++++++++---------
 xds/internal/env/env.go                       | 42 ++++++++++++
 3 files changed, 84 insertions(+), 49 deletions(-)
 create mode 100644 xds/internal/env/env.go

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 51ab98dee6f5..0ef3809496f9 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -25,7 +25,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"os"
 
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -35,14 +34,11 @@ import (
 	"google.golang.org/grpc/credentials/google"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
 const (
-	// Environment variable which holds the name of the xDS bootstrap file.
-	bootstrapFileEnv = "GRPC_XDS_BOOTSTRAP"
-	// Environment variable which controls the use of xDS v3 API.
-	v3SupportEnv = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
 	// The "server_features" field in the bootstrap file contains a list of
 	// features supported by the server. A value of "xds_v3" indicates that the
 	// server supports the v3 version of the xDS transport protocol.
@@ -142,11 +138,11 @@ type xdsServer struct {
 func NewConfig() (*Config, error) {
 	config := &Config{}
 
-	fName, ok := os.LookupEnv(bootstrapFileEnv)
-	if !ok {
-		return nil, fmt.Errorf("xds: Environment variable %v not defined", bootstrapFileEnv)
+	fName := env.BootstrapFileName
+	if fName == "" {
+		return nil, fmt.Errorf("xds: Environment variable %q not defined", "GRPC_XDS_BOOTSTRAP")
 	}
-	logger.Infof("Got bootstrap file location from %v environment variable: %v", bootstrapFileEnv, fName)
+	logger.Infof("Got bootstrap file location %q", fName)
 
 	data, err := bootstrapFileReadFunc(fName)
 	if err != nil {
@@ -259,14 +255,8 @@ func NewConfig() (*Config, error) {
 	// 2. Environment variable "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" is set to
 	//    true.
 	// The default value of the enum type "version.TransportAPI" is v2.
-	//
-	// TODO: there are multiple env variables, GRPC_XDS_BOOTSTRAP and
-	// GRPC_XDS_EXPERIMENTAL_V3_SUPPORT. Move all env variables into a separate
-	// package.
-	if v3Env := os.Getenv(v3SupportEnv); v3Env == "true" {
-		if serverSupportsV3 {
-			config.TransportAPI = version.TransportV3
-		}
+	if env.V3Support && serverSupportsV3 {
+		config.TransportAPI = version.TransportV3
 	}
 
 	if err := config.updateNodeProto(); err != nil {
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 22665060b43e..6ee68c346c45 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/credentials/google"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -267,10 +268,7 @@ func setupBootstrapOverride(bootstrapFileMap map[string]string) func() {
 		}
 		return nil, os.ErrNotExist
 	}
-	return func() {
-		bootstrapFileReadFunc = oldFileReadFunc
-		os.Unsetenv(bootstrapFileEnv)
-	}
+	return func() { bootstrapFileReadFunc = oldFileReadFunc }
 }
 
 // TODO: enable leak check for this package when
@@ -336,9 +334,10 @@ func TestNewConfigV2ProtoFailure(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
-				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
-			}
+			origBootstrapFileName := env.BootstrapFileName
+			env.BootstrapFileName = test.name
+			defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
 			if _, err := NewConfig(); err == nil {
 				t.Fatalf("NewConfig() returned nil error, expected to fail")
 			}
@@ -379,9 +378,10 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
-				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
-			}
+			origBootstrapFileName := env.BootstrapFileName
+			env.BootstrapFileName = test.name
+			defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
 			c, err := NewConfig()
 			if err != nil {
 				t.Fatalf("NewConfig() failed: %v", err)
@@ -398,10 +398,9 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 // on the client. In this case, whether the server supports v3 or not, the
 // client will end up using v2.
 func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) {
-	if err := os.Setenv(v3SupportEnv, "false"); err != nil {
-		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
-	}
-	defer os.Unsetenv(v3SupportEnv)
+	origV3Support := env.V3Support
+	env.V3Support = false
+	defer func() { env.V3Support = origV3Support }()
 
 	cancel := setupBootstrapOverride(v3BootstrapFileMap)
 	defer cancel()
@@ -416,9 +415,10 @@ func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
-				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
-			}
+			origBootstrapFileName := env.BootstrapFileName
+			env.BootstrapFileName = test.name
+			defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
 			c, err := NewConfig()
 			if err != nil {
 				t.Fatalf("NewConfig() failed: %v", err)
@@ -435,10 +435,9 @@ func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) {
 // client. Here the client ends up using v2 or v3 based on what the server
 // supports.
 func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
-	if err := os.Setenv(v3SupportEnv, "true"); err != nil {
-		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
-	}
-	defer os.Unsetenv(v3SupportEnv)
+	origV3Support := env.V3Support
+	env.V3Support = true
+	defer func() { env.V3Support = origV3Support }()
 
 	cancel := setupBootstrapOverride(v3BootstrapFileMap)
 	defer cancel()
@@ -453,9 +452,10 @@ func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
-				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
-			}
+			origBootstrapFileName := env.BootstrapFileName
+			env.BootstrapFileName = test.name
+			defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
 			c, err := NewConfig()
 			if err != nil {
 				t.Fatalf("NewConfig() failed: %v", err)
@@ -470,7 +470,10 @@ func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
 // TestNewConfigBootstrapFileEnvNotSet tests the case where the bootstrap file
 // environment variable is not set.
 func TestNewConfigBootstrapFileEnvNotSet(t *testing.T) {
-	os.Unsetenv(bootstrapFileEnv)
+	origBootstrapFileName := env.BootstrapFileName
+	env.BootstrapFileName = ""
+	defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
 	if _, err := NewConfig(); err == nil {
 		t.Errorf("NewConfig() returned nil error, expected to fail")
 	}
@@ -637,10 +640,9 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 		t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err)
 	}
 
-	if err := os.Setenv(v3SupportEnv, "true"); err != nil {
-		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
-	}
-	defer os.Unsetenv(v3SupportEnv)
+	origV3Support := env.V3Support
+	env.V3Support = true
+	defer func() { env.V3Support = origV3Support }()
 
 	cancel := setupBootstrapOverride(bootstrapFileMap)
 	defer cancel()
@@ -684,9 +686,10 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
-				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
-			}
+			origBootstrapFileName := env.BootstrapFileName
+			env.BootstrapFileName = test.name
+			defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
 			c, err := NewConfig()
 			if (err != nil) != test.wantErr {
 				t.Fatalf("NewConfig() returned: (%+v, %v), wantErr: %v", c.CertProviderConfigs, err, test.wantErr)
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
new file mode 100644
index 000000000000..3b338eae05fd
--- /dev/null
+++ b/xds/internal/env/env.go
@@ -0,0 +1,42 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package env acts a single source of definition for all environment variables
+// related to the xDS implementation in gRPC.
+package env
+
+import (
+	"os"
+	"strings"
+)
+
+const (
+	bootstrapFileNameEnv = "GRPC_XDS_BOOTSTRAP"
+	xdsV3SupportEnv      = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
+)
+
+var (
+	// BootstrapFileName holds the name of the file which contains xDS bootstrap
+	// configuration. Users can specify the location of the bootstrap file by
+	// setting the environment variable "GRPC_XDS_BOOSTRAP".
+	BootstrapFileName = os.Getenv(bootstrapFileNameEnv)
+	// V3Support indicates whether xDS v3 API support is enabled, which can be
+	// done by setting the environment variable
+	// "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" to "true".
+	V3Support = strings.EqualFold(os.Getenv(xdsV3SupportEnv), "true")
+)

From c6fa12175f1c5d2bd78977262c537847146b6dce Mon Sep 17 00:00:00 2001
From: Chongyang Shi <8771937+chongyangshi@users.noreply.github.com>
Date: Wed, 4 Nov 2020 17:28:19 +0000
Subject: [PATCH 262/481] Fix a typo in interceptor example's README.md (#4014)

---
 examples/features/interceptor/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/features/interceptor/README.md b/examples/features/interceptor/README.md
index 1e74b2881cc7..8498fbe11a1c 100644
--- a/examples/features/interceptor/README.md
+++ b/examples/features/interceptor/README.md
@@ -76,7 +76,7 @@ user to operate on.
 
 In the example, we define a new struct `wrappedStream`, which is embedded with a
 `ClientStream`. Then, we implement (overload) the `SendMsg` and `RecvMsg`
-methods on `wrappedStream` to intercepts these two operations on the embedded
+methods on `wrappedStream` to intercept these two operations on the embedded
 `ClientStream`. In the example, we log the message type info and time info for
 interception purpose.
 

From d7a7a304ffab898ad56dac8aff05aa16e3557f01 Mon Sep 17 00:00:00 2001
From: Gaurav Gahlot <gauravgahlot0107@gmail.com>
Date: Thu, 5 Nov 2020 22:55:17 +0530
Subject: [PATCH 263/481] testing: Avoid using context.Background (#3949)

---
 balancer/grpclb/grpclb_test.go                |  82 ++++---
 balancer_switching_test.go                    |  13 +-
 benchmark/primitives/context_test.go          |   6 +-
 call_test.go                                  |  26 ++-
 channelz/service/service_sktopt_test.go       |   4 +-
 channelz/service/service_test.go              |  38 ++-
 .../internal/handshaker/handshaker_test.go    |  19 +-
 credentials/alts/utils_test.go                |   9 +-
 credentials/credentials_test.go               |  15 +-
 credentials/local/local_test.go               |   7 +-
 credentials/sts/sts_test.go                   |  32 ++-
 health/client_test.go                         |   6 +-
 internal/transport/keepalive_test.go          |  68 ++++--
 internal/transport/proxy_test.go              |   5 +-
 internal/transport/transport_test.go          |  66 ++++--
 metadata/metadata_test.go                     |  25 +-
 reflection/serverreflection_test.go           |   7 +-
 stats/stats_test.go                           |  32 ++-
 test/channelz_test.go                         |  12 +-
 test/creds_test.go                            |  30 ++-
 test/end2end_test.go                          | 218 ++++++++++++------
 test/healthcheck_test.go                      |  33 ++-
 test/server_test.go                           |  16 +-
 test/stream_cleanup_test.go                   |   6 +-
 24 files changed, 539 insertions(+), 236 deletions(-)

diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index 48082e2069fa..5413613375a2 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -495,7 +495,7 @@ func (s) TestGRPCLBWeighted(t *testing.T) {
 		tss.ls.sls <- &lbpb.ServerList{Servers: bes}
 
 		for i := 0; i < 1000; i++ {
-			if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+			if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 				t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 			}
 			result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port])
@@ -605,18 +605,18 @@ func (s) TestDropRequest(t *testing.T) {
 			// 1st RPCs pick the first item in server list. They should succeed
 			// since they choose the non-drop-request backend according to the
 			// round robin policy.
-			if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil {
+			if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil {
 				t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 			}
 			// 2nd RPCs pick the second item in server list. They should succeed
 			// since they choose the non-drop-request backend according to the
 			// round robin policy.
-			if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil {
+			if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil {
 				t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 			}
 			// 3rd RPCs should fail, because they pick last item in server list,
 			// with Drop set to true.
-			if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(!failfast)); status.Code(err) != codes.Unavailable {
+			if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); status.Code(err) != codes.Unavailable {
 				t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, %s", testC, err, codes.Unavailable)
 			}
 		}
@@ -625,7 +625,7 @@ func (s) TestDropRequest(t *testing.T) {
 	// Make one more RPC to move the picker index one step further, so it's not
 	// 0. The following RPCs will test that drop index is not reset. If picker
 	// index is at 0, we cannot tell whether it's reset or not.
-	if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 	}
 
@@ -636,18 +636,18 @@ func (s) TestDropRequest(t *testing.T) {
 	time.Sleep(time.Second)
 	for i := 0; i < 3; i++ {
 		var p peer.Peer
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if want := tss.bePorts[1]; p.Addr.(*net.TCPAddr).Port != want {
 			t.Errorf("got peer: %v, want peer port: %v", p.Addr, want)
 		}
 
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.Unavailable {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.Unavailable {
 			t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, %s", testC, err, codes.Unavailable)
 		}
 
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if want := tss.bePorts[1]; p.Addr.(*net.TCPAddr).Port != want {
@@ -709,7 +709,7 @@ func (s) TestBalancerDisconnects(t *testing.T) {
 	}}})
 
 	var p peer.Peer
-	if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+	if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 	}
 	if p.Addr.(*net.TCPAddr).Port != tests[0].bePorts[0] {
@@ -720,7 +720,7 @@ func (s) TestBalancerDisconnects(t *testing.T) {
 	// Stop balancer[0], balancer[1] should be used by grpclb.
 	// Check peer address to see if that happened.
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if p.Addr.(*net.TCPAddr).Port == tests[1].bePorts[0] {
@@ -784,7 +784,7 @@ func (s) TestFallback(t *testing.T) {
 	}}})
 
 	var p peer.Peer
-	if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+	if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 		t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 	if p.Addr.String() != beLis.Addr().String() {
@@ -802,7 +802,7 @@ func (s) TestFallback(t *testing.T) {
 
 	var backendUsed bool
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] {
@@ -821,7 +821,7 @@ func (s) TestFallback(t *testing.T) {
 
 	var fallbackUsed bool
 	for i := 0; i < 2000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			// Because we are hard-closing the connection, above, it's possible
 			// for the first RPC attempt to be sent on the old connection,
 			// which will lead to an Unavailable error when it is closed.
@@ -847,7 +847,7 @@ func (s) TestFallback(t *testing.T) {
 
 	var backendUsed2 bool
 	for i := 0; i < 2000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] {
@@ -913,7 +913,7 @@ func (s) TestExplicitFallback(t *testing.T) {
 	var p peer.Peer
 	var backendUsed bool
 	for i := 0; i < 2000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] {
@@ -931,7 +931,7 @@ func (s) TestExplicitFallback(t *testing.T) {
 
 	var fallbackUsed bool
 	for i := 0; i < 2000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if p.Addr.String() == beLis.Addr().String() {
@@ -949,7 +949,7 @@ func (s) TestExplicitFallback(t *testing.T) {
 
 	backendUsed = false
 	for i := 0; i < 2000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] {
@@ -1067,7 +1067,7 @@ func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 
 		var backendUsed bool
 		for i := 0; i < 1000; i++ {
-			if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+			if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 				t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 			}
 			if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] {
@@ -1144,7 +1144,7 @@ func (s) TestGRPCLBPickFirst(t *testing.T) {
 
 	result = ""
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port])
@@ -1156,7 +1156,7 @@ func (s) TestGRPCLBPickFirst(t *testing.T) {
 	tss.ls.sls <- &lbpb.ServerList{Servers: beServers[2:]}
 	result = ""
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port])
@@ -1168,7 +1168,7 @@ func (s) TestGRPCLBPickFirst(t *testing.T) {
 	tss.ls.sls <- &lbpb.ServerList{Servers: beServers[1:]}
 	result = ""
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port])
@@ -1194,7 +1194,7 @@ func (s) TestGRPCLBPickFirst(t *testing.T) {
 
 	result = ""
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port])
@@ -1206,7 +1206,7 @@ func (s) TestGRPCLBPickFirst(t *testing.T) {
 	tss.ls.sls <- &lbpb.ServerList{Servers: beServers[0:3]}
 	result = ""
 	for i := 0; i < 1000; i++ {
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port])
@@ -1295,12 +1295,14 @@ const (
 func (s) TestGRPCLBStatsUnarySuccess(t *testing.T) {
 	if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) {
 		testC := testpb.NewTestServiceClient(cc)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout)
+		defer cancel()
 		// The first non-failfast RPC succeeds, all connections are up.
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		for i := 0; i < countRPC-1; i++ {
-			testC.EmptyCall(context.Background(), &testpb.Empty{})
+			testC.EmptyCall(ctx, &testpb.Empty{})
 		}
 	}, &rpcStats{
 		numCallsStarted:               int64(countRPC),
@@ -1314,12 +1316,14 @@ func (s) TestGRPCLBStatsUnarySuccess(t *testing.T) {
 func (s) TestGRPCLBStatsUnaryDrop(t *testing.T) {
 	if err := runAndCheckStats(t, true, nil, func(cc *grpc.ClientConn) {
 		testC := testpb.NewTestServiceClient(cc)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout)
+		defer cancel()
 		// The first non-failfast RPC succeeds, all connections are up.
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		for i := 0; i < countRPC-1; i++ {
-			testC.EmptyCall(context.Background(), &testpb.Empty{})
+			testC.EmptyCall(ctx, &testpb.Empty{})
 		}
 	}, &rpcStats{
 		numCallsStarted:               int64(countRPC),
@@ -1334,12 +1338,14 @@ func (s) TestGRPCLBStatsUnaryDrop(t *testing.T) {
 func (s) TestGRPCLBStatsUnaryFailedToSend(t *testing.T) {
 	if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) {
 		testC := testpb.NewTestServiceClient(cc)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout)
+		defer cancel()
 		// The first non-failfast RPC succeeds, all connections are up.
-		if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 			t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
 		for i := 0; i < countRPC-1; i++ {
-			cc.Invoke(context.Background(), failtosendURI, &testpb.Empty{}, nil)
+			cc.Invoke(ctx, failtosendURI, &testpb.Empty{}, nil)
 		}
 	}, &rpcStats{
 		numCallsStarted:                        int64(countRPC),
@@ -1354,8 +1360,10 @@ func (s) TestGRPCLBStatsUnaryFailedToSend(t *testing.T) {
 func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) {
 	if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) {
 		testC := testpb.NewTestServiceClient(cc)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout)
+		defer cancel()
 		// The first non-failfast RPC succeeds, all connections are up.
-		stream, err := testC.FullDuplexCall(context.Background(), grpc.WaitForReady(true))
+		stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true))
 		if err != nil {
 			t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
@@ -1365,7 +1373,7 @@ func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) {
 			}
 		}
 		for i := 0; i < countRPC-1; i++ {
-			stream, err = testC.FullDuplexCall(context.Background())
+			stream, err = testC.FullDuplexCall(ctx)
 			if err == nil {
 				// Wait for stream to end if err is nil.
 				for {
@@ -1387,8 +1395,10 @@ func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) {
 func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) {
 	if err := runAndCheckStats(t, true, nil, func(cc *grpc.ClientConn) {
 		testC := testpb.NewTestServiceClient(cc)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout)
+		defer cancel()
 		// The first non-failfast RPC succeeds, all connections are up.
-		stream, err := testC.FullDuplexCall(context.Background(), grpc.WaitForReady(true))
+		stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true))
 		if err != nil {
 			t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
@@ -1398,7 +1408,7 @@ func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) {
 			}
 		}
 		for i := 0; i < countRPC-1; i++ {
-			stream, err = testC.FullDuplexCall(context.Background())
+			stream, err = testC.FullDuplexCall(ctx)
 			if err == nil {
 				// Wait for stream to end if err is nil.
 				for {
@@ -1421,8 +1431,10 @@ func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) {
 func (s) TestGRPCLBStatsStreamingFailedToSend(t *testing.T) {
 	if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) {
 		testC := testpb.NewTestServiceClient(cc)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout)
+		defer cancel()
 		// The first non-failfast RPC succeeds, all connections are up.
-		stream, err := testC.FullDuplexCall(context.Background(), grpc.WaitForReady(true))
+		stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true))
 		if err != nil {
 			t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, <nil>", testC, err)
 		}
@@ -1432,7 +1444,7 @@ func (s) TestGRPCLBStatsStreamingFailedToSend(t *testing.T) {
 			}
 		}
 		for i := 0; i < countRPC-1; i++ {
-			cc.NewStream(context.Background(), &grpc.StreamDesc{}, failtosendURI)
+			cc.NewStream(ctx, &grpc.StreamDesc{}, failtosendURI)
 		}
 	}, &rpcStats{
 		numCallsStarted:                        int64(countRPC),
diff --git a/balancer_switching_test.go b/balancer_switching_test.go
index ed132121280c..2c6ed576620f 100644
--- a/balancer_switching_test.go
+++ b/balancer_switching_test.go
@@ -83,8 +83,10 @@ func checkPickFirst(cc *ClientConn, servers []*server) error {
 		err   error
 	)
 	connected := false
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i := 0; i < 5000; i++ {
-		if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); errorDesc(err) == servers[0].port {
+		if err = cc.Invoke(ctx, "/foo/bar", &req, &reply); errorDesc(err) == servers[0].port {
 			if connected {
 				// connected is set to false if peer is not server[0]. So if
 				// connected is true here, this is the second time we saw
@@ -100,9 +102,10 @@ func checkPickFirst(cc *ClientConn, servers []*server) error {
 	if !connected {
 		return fmt.Errorf("pickfirst is not in effect after 5 second, EmptyCall() = _, %v, want _, %v", err, servers[0].port)
 	}
+
 	// The following RPCs should all succeed with the first server.
 	for i := 0; i < 3; i++ {
-		err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply)
+		err = cc.Invoke(ctx, "/foo/bar", &req, &reply)
 		if errorDesc(err) != servers[0].port {
 			return fmt.Errorf("index %d: want peer %v, got peer %v", i, servers[0].port, err)
 		}
@@ -117,6 +120,8 @@ func checkRoundRobin(cc *ClientConn, servers []*server) error {
 		err   error
 	)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure connections to all servers are up.
 	for i := 0; i < 2; i++ {
 		// Do this check twice, otherwise the first RPC's transport may still be
@@ -124,7 +129,7 @@ func checkRoundRobin(cc *ClientConn, servers []*server) error {
 		for _, s := range servers {
 			var up bool
 			for i := 0; i < 5000; i++ {
-				if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); errorDesc(err) == s.port {
+				if err = cc.Invoke(ctx, "/foo/bar", &req, &reply); errorDesc(err) == s.port {
 					up = true
 					break
 				}
@@ -138,7 +143,7 @@ func checkRoundRobin(cc *ClientConn, servers []*server) error {
 
 	serverCount := len(servers)
 	for i := 0; i < 3*serverCount; i++ {
-		err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply)
+		err = cc.Invoke(ctx, "/foo/bar", &req, &reply)
 		if errorDesc(err) != servers[i%serverCount].port {
 			return fmt.Errorf("index %d: want peer %v, got peer %v", i, servers[i%serverCount].port, err)
 		}
diff --git a/benchmark/primitives/context_test.go b/benchmark/primitives/context_test.go
index c94acd74597c..faae50703e7d 100644
--- a/benchmark/primitives/context_test.go
+++ b/benchmark/primitives/context_test.go
@@ -24,6 +24,8 @@ import (
 	"time"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 func BenchmarkCancelContextErrNoErr(b *testing.B) {
 	ctx, cancel := context.WithCancel(context.Background())
 	for i := 0; i < b.N; i++ {
@@ -72,7 +74,7 @@ func BenchmarkCancelContextChannelGotErr(b *testing.B) {
 }
 
 func BenchmarkTimerContextErrNoErr(b *testing.B) {
-	ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	for i := 0; i < b.N; i++ {
 		if err := ctx.Err(); err != nil {
 			b.Fatal("error")
@@ -92,7 +94,7 @@ func BenchmarkTimerContextErrGotErr(b *testing.B) {
 }
 
 func BenchmarkTimerContextChannelNoErr(b *testing.B) {
-	ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	for i := 0; i < b.N; i++ {
 		select {
 		case <-ctx.Done():
diff --git a/call_test.go b/call_test.go
index 78760ba5297a..abc4537ddb7d 100644
--- a/call_test.go
+++ b/call_test.go
@@ -43,6 +43,8 @@ var (
 	canceled         = 0
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 type testCodec struct {
 }
 
@@ -237,7 +239,8 @@ func (s) TestUnaryClientInterceptor(t *testing.T) {
 	}()
 
 	var reply string
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0)
 	if err := cc.Invoke(parentCtx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse {
 		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want <nil>", err)
@@ -305,7 +308,8 @@ func (s) TestChainUnaryClientInterceptor(t *testing.T) {
 	}()
 
 	var reply string
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0)
 	if err := cc.Invoke(parentCtx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse+"321" {
 		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want <nil>", err)
@@ -346,7 +350,8 @@ func (s) TestChainOnBaseUnaryClientInterceptor(t *testing.T) {
 	}()
 
 	var reply string
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0)
 	if err := cc.Invoke(parentCtx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse {
 		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want <nil>", err)
@@ -407,7 +412,8 @@ func (s) TestChainStreamClientInterceptor(t *testing.T) {
 		server.stop()
 	}()
 
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0)
 	_, err := cc.NewStream(parentCtx, &StreamDesc{}, "/foo/bar")
 	if err != nil {
@@ -418,7 +424,9 @@ func (s) TestChainStreamClientInterceptor(t *testing.T) {
 func (s) TestInvoke(t *testing.T) {
 	server, cc := setUp(t, 0, math.MaxUint32)
 	var reply string
-	if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := cc.Invoke(ctx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse {
 		t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want <nil>", err)
 	}
 	cc.Close()
@@ -429,7 +437,9 @@ func (s) TestInvokeLargeErr(t *testing.T) {
 	server, cc := setUp(t, 0, math.MaxUint32)
 	var reply string
 	req := "hello"
-	err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	err := cc.Invoke(ctx, "/foo/bar", &req, &reply)
 	if _, ok := status.FromError(err); !ok {
 		t.Fatalf("grpc.Invoke(_, _, _, _, _) receives non rpc error.")
 	}
@@ -445,7 +455,9 @@ func (s) TestInvokeErrorSpecialChars(t *testing.T) {
 	server, cc := setUp(t, 0, math.MaxUint32)
 	var reply string
 	req := "weird error"
-	err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	err := cc.Invoke(ctx, "/foo/bar", &req, &reply)
 	if _, ok := status.FromError(err); !ok {
 		t.Fatalf("grpc.Invoke(_, _, _, _, _) receives non rpc error.")
 	}
diff --git a/channelz/service/service_sktopt_test.go b/channelz/service/service_sktopt_test.go
index e2d024f83652..ecd4a2ad05f9 100644
--- a/channelz/service/service_sktopt_test.go
+++ b/channelz/service/service_sktopt_test.go
@@ -145,8 +145,10 @@ func (s) TestGetSocketOptions(t *testing.T) {
 		ids[i] = channelz.RegisterNormalSocket(s, svrID, strconv.Itoa(i))
 		defer channelz.RemoveEntry(ids[i])
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i, s := range ss {
-		resp, _ := svr.GetSocket(context.Background(), &channelzpb.GetSocketRequest{SocketId: ids[i]})
+		resp, _ := svr.GetSocket(ctx, &channelzpb.GetSocketRequest{SocketId: ids[i]})
 		metrics := resp.GetSocket()
 		if !reflect.DeepEqual(metrics.GetRef(), &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}) || !reflect.DeepEqual(socketProtoToStruct(metrics), s) {
 			t.Fatalf("resp.GetSocket() want: metrics.GetRef() = %#v and %#v, got: metrics.GetRef() = %#v and %#v", &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}, s, metrics.GetRef(), socketProtoToStruct(metrics))
diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go
index 467ceb4f431c..03d2b29c27b4 100644
--- a/channelz/service/service_test.go
+++ b/channelz/service/service_test.go
@@ -69,6 +69,8 @@ var protoToSocketOpt protoToSocketOptFunc
 // TODO: Go1.7 is no longer supported - does this need a change?
 var emptyTime time.Time
 
+const defaultTestTimeout = 10 * time.Second
+
 type dummyChannel struct {
 	state                    connectivity.State
 	target                   string
@@ -327,7 +329,9 @@ func (s) TestGetTopChannels(t *testing.T) {
 		defer channelz.RemoveEntry(id)
 	}
 	s := newCZServer()
-	resp, _ := s.GetTopChannels(context.Background(), &channelzpb.GetTopChannelsRequest{StartChannelId: 0})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, _ := s.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{StartChannelId: 0})
 	if !resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want true, got %v", resp.GetEnd())
 	}
@@ -340,7 +344,7 @@ func (s) TestGetTopChannels(t *testing.T) {
 		id := channelz.RegisterChannel(tcs[0], 0, "")
 		defer channelz.RemoveEntry(id)
 	}
-	resp, _ = s.GetTopChannels(context.Background(), &channelzpb.GetTopChannelsRequest{StartChannelId: 0})
+	resp, _ = s.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{StartChannelId: 0})
 	if resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd())
 	}
@@ -374,7 +378,9 @@ func (s) TestGetServers(t *testing.T) {
 		defer channelz.RemoveEntry(id)
 	}
 	svr := newCZServer()
-	resp, _ := svr.GetServers(context.Background(), &channelzpb.GetServersRequest{StartServerId: 0})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, _ := svr.GetServers(ctx, &channelzpb.GetServersRequest{StartServerId: 0})
 	if !resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want true, got %v", resp.GetEnd())
 	}
@@ -387,7 +393,7 @@ func (s) TestGetServers(t *testing.T) {
 		id := channelz.RegisterServer(ss[0], "")
 		defer channelz.RemoveEntry(id)
 	}
-	resp, _ = svr.GetServers(context.Background(), &channelzpb.GetServersRequest{StartServerId: 0})
+	resp, _ = svr.GetServers(ctx, &channelzpb.GetServersRequest{StartServerId: 0})
 	if resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd())
 	}
@@ -407,7 +413,9 @@ func (s) TestGetServerSockets(t *testing.T) {
 		defer channelz.RemoveEntry(id)
 	}
 	svr := newCZServer()
-	resp, _ := svr.GetServerSockets(context.Background(), &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: 0})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, _ := svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: 0})
 	if !resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want: true, got: %v", resp.GetEnd())
 	}
@@ -424,7 +432,7 @@ func (s) TestGetServerSockets(t *testing.T) {
 		id := channelz.RegisterNormalSocket(&dummySocket{}, svrID, "")
 		defer channelz.RemoveEntry(id)
 	}
-	resp, _ = svr.GetServerSockets(context.Background(), &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: 0})
+	resp, _ = svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: 0})
 	if resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd())
 	}
@@ -446,9 +454,11 @@ func (s) TestGetServerSocketsNonZeroStartID(t *testing.T) {
 		defer channelz.RemoveEntry(id)
 	}
 	svr := newCZServer()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make GetServerSockets with startID = ids[1]+1, so socket-1 won't be
 	// included in the response.
-	resp, _ := svr.GetServerSockets(context.Background(), &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: ids[1] + 1})
+	resp, _ := svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: ids[1] + 1})
 	if !resp.GetEnd() {
 		t.Fatalf("resp.GetEnd() want: true, got: %v", resp.GetEnd())
 	}
@@ -512,7 +522,9 @@ func (s) TestGetChannel(t *testing.T) {
 		defer channelz.RemoveEntry(id)
 	}
 	svr := newCZServer()
-	resp, _ := svr.GetChannel(context.Background(), &channelzpb.GetChannelRequest{ChannelId: ids[0]})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, _ := svr.GetChannel(ctx, &channelzpb.GetChannelRequest{ChannelId: ids[0]})
 	metrics := resp.GetChannel()
 	subChans := metrics.GetSubchannelRef()
 	if len(subChans) != 1 || subChans[0].GetName() != refNames[2] || subChans[0].GetSubchannelId() != ids[2] {
@@ -552,7 +564,7 @@ func (s) TestGetChannel(t *testing.T) {
 			}
 		}
 	}
-	resp, _ = svr.GetChannel(context.Background(), &channelzpb.GetChannelRequest{ChannelId: ids[1]})
+	resp, _ = svr.GetChannel(ctx, &channelzpb.GetChannelRequest{ChannelId: ids[1]})
 	metrics = resp.GetChannel()
 	nestedChans = metrics.GetChannelRef()
 	if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[3] || nestedChans[0].GetChannelId() != ids[3] {
@@ -598,7 +610,9 @@ func (s) TestGetSubChannel(t *testing.T) {
 		defer channelz.RemoveEntry(id)
 	}
 	svr := newCZServer()
-	resp, _ := svr.GetSubchannel(context.Background(), &channelzpb.GetSubchannelRequest{SubchannelId: ids[1]})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, _ := svr.GetSubchannel(ctx, &channelzpb.GetSubchannelRequest{SubchannelId: ids[1]})
 	metrics := resp.GetSubchannel()
 	want := map[int64]string{
 		ids[2]: refNames[2],
@@ -719,8 +733,10 @@ func (s) TestGetSocket(t *testing.T) {
 		ids[i] = channelz.RegisterNormalSocket(s, svrID, strconv.Itoa(i))
 		defer channelz.RemoveEntry(ids[i])
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for i, s := range ss {
-		resp, _ := svr.GetSocket(context.Background(), &channelzpb.GetSocketRequest{SocketId: ids[i]})
+		resp, _ := svr.GetSocket(ctx, &channelzpb.GetSocketRequest{SocketId: ids[i]})
 		metrics := resp.GetSocket()
 		if !reflect.DeepEqual(metrics.GetRef(), &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}) || !reflect.DeepEqual(socketProtoToStruct(metrics), s) {
 			t.Fatalf("resp.GetSocket() want: metrics.GetRef() = %#v and %#v, got: metrics.GetRef() = %#v and %#v", &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}, s, metrics.GetRef(), socketProtoToStruct(metrics))
diff --git a/credentials/alts/internal/handshaker/handshaker_test.go b/credentials/alts/internal/handshaker/handshaker_test.go
index 9214f647c853..bf516dc53c87 100644
--- a/credentials/alts/internal/handshaker/handshaker_test.go
+++ b/credentials/alts/internal/handshaker/handshaker_test.go
@@ -56,6 +56,8 @@ var (
 	}
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 // testRPCStream mimics a altspb.HandshakerService_DoHandshakeClient object.
 type testRPCStream struct {
 	grpc.ClientStream
@@ -133,6 +135,10 @@ func (s) TestClientHandshake(t *testing.T) {
 	} {
 		errc := make(chan error)
 		stat.Reset()
+
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+
 		for i := 0; i < testCase.numberOfHandshakes; i++ {
 			stream := &testRPCStream{
 				t:        t,
@@ -155,7 +161,7 @@ func (s) TestClientHandshake(t *testing.T) {
 				side: core.ClientSide,
 			}
 			go func() {
-				_, context, err := chs.ClientHandshake(context.Background())
+				_, context, err := chs.ClientHandshake(ctx)
 				if err == nil && context == nil {
 					panic("expected non-nil ALTS context")
 				}
@@ -188,6 +194,10 @@ func (s) TestServerHandshake(t *testing.T) {
 	} {
 		errc := make(chan error)
 		stat.Reset()
+
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+
 		for i := 0; i < testCase.numberOfHandshakes; i++ {
 			stream := &testRPCStream{
 				t:        t,
@@ -207,7 +217,7 @@ func (s) TestServerHandshake(t *testing.T) {
 				side:       core.ServerSide,
 			}
 			go func() {
-				_, context, err := shs.ServerHandshake(context.Background())
+				_, context, err := shs.ServerHandshake(ctx)
 				if err == nil && context == nil {
 					panic("expected non-nil ALTS context")
 				}
@@ -258,7 +268,10 @@ func (s) TestPeerNotResponding(t *testing.T) {
 		},
 		side: core.ClientSide,
 	}
-	_, context, err := chs.ClientHandshake(context.Background())
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, context, err := chs.ClientHandshake(ctx)
 	chs.Close()
 	if context != nil {
 		t.Error("expected non-nil ALTS context")
diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go
index b9e752ebbac9..5b54b1d5f77c 100644
--- a/credentials/alts/utils_test.go
+++ b/credentials/alts/utils_test.go
@@ -26,6 +26,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"google.golang.org/grpc/codes"
 	altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp"
@@ -37,6 +38,8 @@ const (
 	testServiceAccount1 = "service_account1"
 	testServiceAccount2 = "service_account2"
 	testServiceAccount3 = "service_account3"
+
+	defaultTestTimeout = 10 * time.Second
 )
 
 func setupManufacturerReader(testOS string, reader func() (io.Reader, error)) func() {
@@ -101,7 +104,8 @@ func (s) TestIsRunningOnGCPNoProductNameFile(t *testing.T) {
 }
 
 func (s) TestAuthInfoFromContext(t *testing.T) {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	altsAuthInfo := &fakeALTSAuthInfo{}
 	p := &peer.Peer{
 		AuthInfo: altsAuthInfo,
@@ -158,7 +162,8 @@ func (s) TestAuthInfoFromPeer(t *testing.T) {
 }
 
 func (s) TestClientAuthorizationCheck(t *testing.T) {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	altsAuthInfo := &fakeALTSAuthInfo{testServiceAccount1}
 	p := &peer.Peer{
 		AuthInfo: altsAuthInfo,
diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go
index ea0cf5819ff8..dee0f2ca8304 100644
--- a/credentials/credentials_test.go
+++ b/credentials/credentials_test.go
@@ -24,12 +24,15 @@ import (
 	"net"
 	"strings"
 	"testing"
+	"time"
 
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/testdata"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 type s struct {
 	grpctest.Tester
 }
@@ -60,7 +63,9 @@ func createTestContext(s SecurityLevel) context.Context {
 		Method:   "testInfo",
 		AuthInfo: auth,
 	}
-	return internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(context.Background(), ri)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	return internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(ctx, ri)
 }
 
 func (s) TestCheckSecurityLevel(t *testing.T) {
@@ -112,7 +117,9 @@ func (s) TestCheckSecurityLevelNoGetCommonAuthInfoMethod(t *testing.T) {
 		Method:   "testInfo",
 		AuthInfo: auth,
 	}
-	ctxWithRequestInfo := internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(context.Background(), ri)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctxWithRequestInfo := internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(ctx, ri)
 	if err := CheckSecurityLevel(ctxWithRequestInfo, PrivacyAndIntegrity); err != nil {
 		t.Fatalf("CheckSeurityLevel() returned failure but want success")
 	}
@@ -296,7 +303,9 @@ func gRPCServerHandshake(conn net.Conn) (AuthInfo, error) {
 // Client handshake implementation in gRPC.
 func gRPCClientHandshake(conn net.Conn, lisAddr string) (AuthInfo, error) {
 	clientTLS := NewTLS(&tls.Config{InsecureSkipVerify: true})
-	_, authInfo, err := clientTLS.ClientHandshake(context.Background(), lisAddr, conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, authInfo, err := clientTLS.ClientHandshake(ctx, lisAddr, conn)
 	if err != nil {
 		return nil, err
 	}
diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go
index 3c65010e8b2a..64e2ec3e7fc6 100644
--- a/credentials/local/local_test.go
+++ b/credentials/local/local_test.go
@@ -31,6 +31,8 @@ import (
 	"google.golang.org/grpc/internal/grpctest"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 type s struct {
 	grpctest.Tester
 }
@@ -89,7 +91,10 @@ func serverLocalHandshake(conn net.Conn) (credentials.AuthInfo, error) {
 // Client local handshake implementation.
 func clientLocalHandshake(conn net.Conn, lisAddr string) (credentials.AuthInfo, error) {
 	cred := NewCredentials()
-	_, authInfo, err := cred.ClientHandshake(context.Background(), lisAddr, conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	_, authInfo, err := cred.ClientHandshake(ctx, lisAddr, conn)
 	if err != nil {
 		return nil, err
 	}
diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go
index 9cfa12097d60..ac680e001112 100644
--- a/credentials/sts/sts_test.go
+++ b/credentials/sts/sts_test.go
@@ -255,7 +255,10 @@ func (s) TestGetRequestMetadataSuccess(t *testing.T) {
 	errCh := make(chan error, 1)
 	go receiveAndCompareRequest(fc.ReqChan, errCh)
 
-	gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "")
 	if err != nil {
 		t.Fatalf("creds.GetRequestMetadata() = %v", err)
 	}
@@ -270,7 +273,7 @@ func (s) TestGetRequestMetadataSuccess(t *testing.T) {
 	// from the cache. This will fail if the credentials tries to send a fresh
 	// request here since we have not configured our fakeClient to return any
 	// response on retries.
-	gotMetadata, err = creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
+	gotMetadata, err = creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "")
 	if err != nil {
 		t.Fatalf("creds.GetRequestMetadata() = %v", err)
 	}
@@ -290,7 +293,9 @@ func (s) TestGetRequestMetadataBadSecurityLevel(t *testing.T) {
 		t.Fatalf("NewCredentials(%v) = %v", goodOptions, err)
 	}
 
-	gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.IntegrityOnly), "")
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.IntegrityOnly), "")
 	if err == nil {
 		t.Fatalf("creds.GetRequestMetadata() succeeded with metadata %v, expected to fail", gotMetadata)
 	}
@@ -335,7 +340,9 @@ func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) {
 		}
 		fc.RespChan.Send(resp)
 
-		gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "")
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "")
 		if err != nil {
 			t.Fatalf("creds.GetRequestMetadata() = %v", err)
 		}
@@ -374,6 +381,8 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) {
 		},
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			defer overrideSubjectTokenGood()()
@@ -393,7 +402,7 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) {
 			go receiveAndCompareRequest(fc.ReqChan, errCh)
 
 			fc.RespChan.Send(test.response)
-			if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil {
+			if _, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), ""); err == nil {
 				t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail")
 			}
 			if err := <-errCh; err != nil {
@@ -426,7 +435,9 @@ func (s) TestGetRequestMetadataBadSubjectTokenRead(t *testing.T) {
 		errCh <- nil
 	}()
 
-	if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), ""); err == nil {
 		t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail")
 	}
 	if err := <-errCh; err != nil {
@@ -604,6 +615,9 @@ func (s) TestConstructRequest(t *testing.T) {
 			},
 		},
 	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			if test.subjectTokenReadErr {
@@ -618,7 +632,7 @@ func (s) TestConstructRequest(t *testing.T) {
 				defer overrideActorTokenGood()()
 			}
 
-			gotRequest, err := constructRequest(context.Background(), test.opts)
+			gotRequest, err := constructRequest(ctx, test.opts)
 			if (err != nil) != test.wantErr {
 				t.Fatalf("constructRequest(%v) = %v, wantErr: %v", test.opts, err, test.wantErr)
 			}
@@ -634,7 +648,9 @@ func (s) TestConstructRequest(t *testing.T) {
 
 func (s) TestSendRequest(t *testing.T) {
 	defer overrideSubjectTokenGood()()
-	req, err := constructRequest(context.Background(), goodOptions)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	req, err := constructRequest(ctx, goodOptions)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/health/client_test.go b/health/client_test.go
index fa218afada72..ba933f95b84f 100644
--- a/health/client_test.go
+++ b/health/client_test.go
@@ -28,6 +28,8 @@ import (
 	"google.golang.org/grpc/connectivity"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 func (s) TestClientHealthCheckBackoff(t *testing.T) {
 	const maxRetries = 5
 
@@ -51,7 +53,9 @@ func (s) TestClientHealthCheckBackoff(t *testing.T) {
 	}
 	defer func() { backoffFunc = oldBackoffFunc }()
 
-	clientHealthCheck(context.Background(), newStream, func(connectivity.State, error) {}, "test")
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	clientHealthCheck(ctx, newStream, func(connectivity.State, error) {}, "test")
 
 	if !reflect.DeepEqual(got, want) {
 		t.Fatalf("Backoff durations for %v retries are %v. (expected: %v)", maxRetries, got, want)
diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go
index 37b77bb539c4..c8f177fecf1b 100644
--- a/internal/transport/keepalive_test.go
+++ b/internal/transport/keepalive_test.go
@@ -34,6 +34,8 @@ import (
 	"google.golang.org/grpc/keepalive"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 // TestMaxConnectionIdle tests that a server will send GoAway to an idle
 // client. An idle client is one who doesn't make any RPC calls for a duration
 // of MaxConnectionIdle time.
@@ -50,7 +52,9 @@ func (s) TestMaxConnectionIdle(t *testing.T) {
 		cancel()
 	}()
 
-	stream, err := client.NewStream(context.Background(), &CallHdr{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
@@ -87,7 +91,9 @@ func (s) TestMaxConnectionIdleBusyClient(t *testing.T) {
 		cancel()
 	}()
 
-	_, err := client.NewStream(context.Background(), &CallHdr{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
@@ -121,7 +127,9 @@ func (s) TestMaxConnectionAge(t *testing.T) {
 		cancel()
 	}()
 
-	_, err := client.NewStream(context.Background(), &CallHdr{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
@@ -228,8 +236,10 @@ func (s) TestKeepaliveServerWithResponsiveClient(t *testing.T) {
 	// Give keepalive logic some time by sleeping.
 	time.Sleep(4 * time.Second)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 }
@@ -258,8 +268,10 @@ func (s) TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) {
 	// Sleep for keepalive to close the connection.
 	time.Sleep(4 * time.Second)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is not healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err == nil {
 		t.Fatal("client.NewStream() should have failed, but succeeded")
 	}
 }
@@ -287,8 +299,10 @@ func (s) TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) {
 	// Give keepalive some time.
 	time.Sleep(4 * time.Second)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 }
@@ -311,8 +325,10 @@ func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) {
 	}
 	defer conn.Close()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Create a stream, but send no data on it.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 
@@ -320,7 +336,7 @@ func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) {
 	time.Sleep(4 * time.Second)
 
 	// Make sure the client transport is not healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err == nil {
 		t.Fatal("client.NewStream() should have failed, but succeeded")
 	}
 }
@@ -344,8 +360,10 @@ func (s) TestKeepaliveClientStaysHealthyWithResponsiveServer(t *testing.T) {
 	// Give keepalive some time.
 	time.Sleep(4 * time.Second)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 }
@@ -391,8 +409,10 @@ func (s) TestKeepaliveClientFrequency(t *testing.T) {
 		t.Fatalf("client transport still healthy; expected GoAway from the server.")
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is not healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err == nil {
 		t.Fatal("client.NewStream() should have failed, but succeeded")
 	}
 }
@@ -434,8 +454,10 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) {
 		t.Fatalf("client transport still healthy; expected GoAway from the server.")
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is not healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err == nil {
 		t.Fatal("client.NewStream() should have failed, but succeeded")
 	}
 }
@@ -463,7 +485,9 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) {
 		cancel()
 	}()
 
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 
@@ -481,7 +505,7 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) {
 	}
 
 	// Make sure the client transport is not healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err == nil {
 		t.Fatal("client.NewStream() should have failed, but succeeded")
 	}
 }
@@ -514,8 +538,10 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientNoRPC(t *testing.T) {
 	// Give keepalive enough time.
 	time.Sleep(3 * time.Second)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 }
@@ -543,7 +569,9 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) {
 		cancel()
 	}()
 
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 
@@ -551,7 +579,7 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) {
 	time.Sleep(3 * time.Second)
 
 	// Make sure the client transport is healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 }
@@ -584,8 +612,10 @@ func (s) TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient(t *testing.T
 	// No active streams on the client. Give keepalive enough time.
 	time.Sleep(5 * time.Second)
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Make sure the client transport is healthy.
-	if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil {
+	if _, err := client.NewStream(ctx, &CallHdr{}); err != nil {
 		t.Fatalf("client.NewStream() failed: %v", err)
 	}
 }
@@ -633,7 +663,9 @@ func (s) TestTCPUserTimeout(t *testing.T) {
 			cancel()
 		}()
 
-		stream, err := client.NewStream(context.Background(), &CallHdr{})
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		stream, err := client.NewStream(ctx, &CallHdr{})
 		if err != nil {
 			t.Fatalf("client.NewStream() failed: %v", err)
 		}
diff --git a/internal/transport/proxy_test.go b/internal/transport/proxy_test.go
index 628b1fddc494..a2f1aa438546 100644
--- a/internal/transport/proxy_test.go
+++ b/internal/transport/proxy_test.go
@@ -210,8 +210,11 @@ func (s) TestMapAddressEnv(t *testing.T) {
 	}
 	defer overwrite(hpfe)()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// envTestAddr should be handled by ProxyFromEnvironment.
-	got, err := mapAddress(context.Background(), envTestAddr)
+	got, err := mapAddress(ctx, envTestAddr)
 	if err != nil {
 		t.Error(err)
 	}
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 391ad9925c1a..0058df0d806f 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -483,7 +483,9 @@ func (s) TestInflightStreamClosing(t *testing.T) {
 	defer server.stop()
 	defer client.Close()
 
-	stream, err := client.NewStream(context.Background(), &CallHdr{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Client failed to create RPC request: %v", err)
 	}
@@ -519,14 +521,16 @@ func (s) TestClientSendAndReceive(t *testing.T) {
 		Host:   "localhost",
 		Method: "foo.Small",
 	}
-	s1, err1 := ct.NewStream(context.Background(), callHdr)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	s1, err1 := ct.NewStream(ctx, callHdr)
 	if err1 != nil {
 		t.Fatalf("failed to open stream: %v", err1)
 	}
 	if s1.id != 1 {
 		t.Fatalf("wrong stream id: %d", s1.id)
 	}
-	s2, err2 := ct.NewStream(context.Background(), callHdr)
+	s2, err2 := ct.NewStream(ctx, callHdr)
 	if err2 != nil {
 		t.Fatalf("failed to open stream: %v", err2)
 	}
@@ -564,7 +568,9 @@ func performOneRPC(ct ClientTransport) {
 		Host:   "localhost",
 		Method: "foo.Small",
 	}
-	s, err := ct.NewStream(context.Background(), callHdr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := ct.NewStream(ctx, callHdr)
 	if err != nil {
 		return
 	}
@@ -606,12 +612,14 @@ func (s) TestLargeMessage(t *testing.T) {
 		Host:   "localhost",
 		Method: "foo.Large",
 	}
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
 	var wg sync.WaitGroup
 	for i := 0; i < 2; i++ {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			s, err := ct.NewStream(context.Background(), callHdr)
+			s, err := ct.NewStream(ctx, callHdr)
 			if err != nil {
 				t.Errorf("%v.NewStream(_, _) = _, %v, want _, <nil>", ct, err)
 			}
@@ -771,7 +779,7 @@ func (s) TestGracefulClose(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			str, err := ct.NewStream(context.Background(), &CallHdr{})
+			str, err := ct.NewStream(ctx, &CallHdr{})
 			if err == ErrConnClosing {
 				return
 			} else if err != nil {
@@ -839,7 +847,9 @@ func (s) TestMaxStreams(t *testing.T) {
 		Host:   "localhost",
 		Method: "foo.Large",
 	}
-	s, err := ct.NewStream(context.Background(), callHdr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := ct.NewStream(ctx, callHdr)
 	if err != nil {
 		t.Fatalf("Failed to open stream: %v", err)
 	}
@@ -924,7 +934,9 @@ func (s) TestServerContextCanceledOnClosedConnection(t *testing.T) {
 		server.mu.Unlock()
 		break
 	}
-	s, err := ct.NewStream(context.Background(), callHdr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := ct.NewStream(ctx, callHdr)
 	if err != nil {
 		t.Fatalf("Failed to open stream: %v", err)
 	}
@@ -988,7 +1000,9 @@ func (s) TestClientConnDecoupledFromApplicationRead(t *testing.T) {
 	notifyChan := make(chan struct{})
 	server.h.notify = notifyChan
 	server.mu.Unlock()
-	cstream1, err := client.NewStream(context.Background(), &CallHdr{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cstream1, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Client failed to create first stream. Err: %v", err)
 	}
@@ -1015,7 +1029,7 @@ func (s) TestClientConnDecoupledFromApplicationRead(t *testing.T) {
 	server.h.notify = notifyChan
 	server.mu.Unlock()
 	// Create another stream on client.
-	cstream2, err := client.NewStream(context.Background(), &CallHdr{})
+	cstream2, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Client failed to create second stream. Err: %v", err)
 	}
@@ -1070,8 +1084,10 @@ func (s) TestServerConnDecoupledFromApplicationRead(t *testing.T) {
 	for k := range server.conns {
 		st = k.(*http2Server)
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	server.mu.Unlock()
-	cstream1, err := client.NewStream(context.Background(), &CallHdr{})
+	cstream1, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Failed to create 1st stream. Err: %v", err)
 	}
@@ -1080,7 +1096,7 @@ func (s) TestServerConnDecoupledFromApplicationRead(t *testing.T) {
 		t.Fatalf("Client failed to write data. Err: %v", err)
 	}
 	//Client should be able to create another stream and send data on it.
-	cstream2, err := client.NewStream(context.Background(), &CallHdr{})
+	cstream2, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Failed to create 2nd stream. Err: %v", err)
 	}
@@ -1287,7 +1303,7 @@ func (s) TestClientWithMisbehavedServer(t *testing.T) {
 		t.Fatalf("Error while creating client transport: %v", err)
 	}
 	defer ct.Close()
-	str, err := ct.NewStream(context.Background(), &CallHdr{})
+	str, err := ct.NewStream(connectCtx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Error while creating stream: %v", err)
 	}
@@ -1312,7 +1328,9 @@ func (s) TestEncodingRequiredStatus(t *testing.T) {
 		Host:   "localhost",
 		Method: "foo",
 	}
-	s, err := ct.NewStream(context.Background(), callHdr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := ct.NewStream(ctx, callHdr)
 	if err != nil {
 		return
 	}
@@ -1338,7 +1356,9 @@ func (s) TestInvalidHeaderField(t *testing.T) {
 		Host:   "localhost",
 		Method: "foo",
 	}
-	s, err := ct.NewStream(context.Background(), callHdr)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := ct.NewStream(ctx, callHdr)
 	if err != nil {
 		return
 	}
@@ -1356,7 +1376,9 @@ func (s) TestHeaderChanClosedAfterReceivingAnInvalidHeader(t *testing.T) {
 	defer cancel()
 	defer server.stop()
 	defer ct.Close()
-	s, err := ct.NewStream(context.Background(), &CallHdr{Host: "localhost", Method: "foo"})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := ct.NewStream(ctx, &CallHdr{Host: "localhost", Method: "foo"})
 	if err != nil {
 		t.Fatalf("failed to create the stream")
 	}
@@ -1473,12 +1495,14 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig)
 	for k := range server.conns {
 		st = k.(*http2Server)
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	server.mu.Unlock()
 	const numStreams = 10
 	clientStreams := make([]*Stream, numStreams)
 	for i := 0; i < numStreams; i++ {
 		var err error
-		clientStreams[i], err = client.NewStream(context.Background(), &CallHdr{})
+		clientStreams[i], err = client.NewStream(ctx, &CallHdr{})
 		if err != nil {
 			t.Fatalf("Failed to create stream. Err: %v", err)
 		}
@@ -1669,7 +1693,9 @@ func runPingPongTest(t *testing.T, msgSize int) {
 		}
 		return false, nil
 	})
-	stream, err := client.NewStream(context.Background(), &CallHdr{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := client.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Failed to create stream. Err: %v", err)
 	}
@@ -1748,7 +1774,9 @@ func (s) TestHeaderTblSize(t *testing.T) {
 	defer cancel()
 	defer ct.Close()
 	defer server.stop()
-	_, err := ct.NewStream(context.Background(), &CallHdr{})
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	_, err := ct.NewStream(ctx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("failed to open stream: %v", err)
 	}
diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go
index 84845d5b1278..f1fb5f6d324e 100644
--- a/metadata/metadata_test.go
+++ b/metadata/metadata_test.go
@@ -23,10 +23,13 @@ import (
 	"reflect"
 	"strconv"
 	"testing"
+	"time"
 
 	"google.golang.org/grpc/internal/grpctest"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 type s struct {
 	grpctest.Tester
 }
@@ -168,7 +171,9 @@ func (s) TestAppend(t *testing.T) {
 
 func (s) TestAppendToOutgoingContext(t *testing.T) {
 	// Pre-existing metadata
-	ctx := NewOutgoingContext(context.Background(), Pairs("k1", "v1", "k2", "v2"))
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx := NewOutgoingContext(tCtx, Pairs("k1", "v1", "k2", "v2"))
 	ctx = AppendToOutgoingContext(ctx, "k1", "v3")
 	ctx = AppendToOutgoingContext(ctx, "k1", "v4")
 	md, ok := FromOutgoingContext(ctx)
@@ -181,7 +186,7 @@ func (s) TestAppendToOutgoingContext(t *testing.T) {
 	}
 
 	// No existing metadata
-	ctx = AppendToOutgoingContext(context.Background(), "k1", "v1")
+	ctx = AppendToOutgoingContext(tCtx, "k1", "v1")
 	md, ok = FromOutgoingContext(ctx)
 	if !ok {
 		t.Errorf("Expected MD to exist in ctx, but got none")
@@ -193,7 +198,8 @@ func (s) TestAppendToOutgoingContext(t *testing.T) {
 }
 
 func (s) TestAppendToOutgoingContext_Repeated(t *testing.T) {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 
 	for i := 0; i < 100; i = i + 2 {
 		ctx1 := AppendToOutgoingContext(ctx, "k", strconv.Itoa(i))
@@ -213,7 +219,9 @@ func (s) TestAppendToOutgoingContext_Repeated(t *testing.T) {
 func (s) TestAppendToOutgoingContext_FromKVSlice(t *testing.T) {
 	const k, v = "a", "b"
 	kv := []string{k, v}
-	ctx := AppendToOutgoingContext(context.Background(), kv...)
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx := AppendToOutgoingContext(tCtx, kv...)
 	md, _ := FromOutgoingContext(ctx)
 	if md[k][0] != v {
 		t.Fatalf("md[%q] = %q; want %q", k, md[k], v)
@@ -230,7 +238,8 @@ func Benchmark_AddingMetadata_ContextManipulationApproach(b *testing.B) {
 	// TODO: Add in N=1-100 tests once Go1.6 support is removed.
 	const num = 10
 	for n := 0; n < b.N; n++ {
-		ctx := context.Background()
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
 		for i := 0; i < num; i++ {
 			md, _ := FromOutgoingContext(ctx)
 			NewOutgoingContext(ctx, Join(Pairs("k1", "v1", "k2", "v2"), md))
@@ -241,8 +250,9 @@ func Benchmark_AddingMetadata_ContextManipulationApproach(b *testing.B) {
 // Newer/faster approach to adding metadata to context
 func BenchmarkAppendToOutgoingContext(b *testing.B) {
 	const num = 10
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	for n := 0; n < b.N; n++ {
-		ctx := context.Background()
 		for i := 0; i < num; i++ {
 			ctx = AppendToOutgoingContext(ctx, "k1", "v1", "k2", "v2")
 		}
@@ -250,7 +260,8 @@ func BenchmarkAppendToOutgoingContext(b *testing.B) {
 }
 
 func BenchmarkFromOutgoingContext(b *testing.B) {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	ctx = NewOutgoingContext(ctx, MD{"k3": {"v3", "v4"}})
 	ctx = AppendToOutgoingContext(ctx, "k1", "v1", "k2", "v2")
 
diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go
index 55d1840fdc3a..24070141c2f2 100644
--- a/reflection/serverreflection_test.go
+++ b/reflection/serverreflection_test.go
@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"sort"
 	"testing"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
@@ -51,6 +52,8 @@ var (
 	fdProto2Ext2Byte []byte
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 type x struct {
 	grpctest.Tester
 }
@@ -209,7 +212,9 @@ func (x) TestReflectionEnd2end(t *testing.T) {
 	defer conn.Close()
 
 	c := rpb.NewServerReflectionClient(conn)
-	stream, err := c.ServerReflectionInfo(context.Background(), grpc.WaitForReady(true))
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := c.ServerReflectionInfo(ctx, grpc.WaitForReady(true))
 	if err != nil {
 		t.Fatalf("cannot get ServerReflectionInfo: %v", err)
 	}
diff --git a/stats/stats_test.go b/stats/stats_test.go
index d047d48bc5e2..875a57eeddfc 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -37,6 +37,8 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+const defaultTestTimeout = 10 * time.Second
+
 type s struct {
 	grpctest.Tester
 }
@@ -281,8 +283,10 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 	} else {
 		req = &testpb.SimpleRequest{Id: errorID}
 	}
-	ctx := metadata.NewOutgoingContext(context.Background(), testMetadata)
-	resp, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(!c.failfast))
+
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, err = tc.UnaryCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast))
 	return req, resp, err
 }
 
@@ -293,7 +297,9 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest
 		err   error
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
-	stream, err := tc.FullDuplexCall(metadata.NewOutgoingContext(context.Background(), testMetadata), grpc.WaitForReady(!c.failfast))
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.FullDuplexCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast))
 	if err != nil {
 		return reqs, resps, err
 	}
@@ -332,7 +338,9 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test
 		err  error
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
-	stream, err := tc.ClientStreamCall(metadata.NewOutgoingContext(context.Background(), testMetadata), grpc.WaitForReady(!c.failfast))
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.ClientStreamCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast))
 	if err != nil {
 		return reqs, resp, err
 	}
@@ -367,7 +375,9 @@ func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*test
 		startID = errorID
 	}
 	req = &testpb.SimpleRequest{Id: startID}
-	stream, err := tc.ServerStreamCall(metadata.NewOutgoingContext(context.Background(), testMetadata), req, grpc.WaitForReady(!c.failfast))
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.ServerStreamCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast))
 	if err != nil {
 		return req, resps, err
 	}
@@ -1286,7 +1296,9 @@ func (s) TestClientStatsFullDuplexRPCError(t *testing.T) {
 
 func (s) TestTags(t *testing.T) {
 	b := []byte{5, 2, 4, 3, 1}
-	ctx := stats.SetTags(context.Background(), b)
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx := stats.SetTags(tCtx, b)
 	if tg := stats.OutgoingTags(ctx); !reflect.DeepEqual(tg, b) {
 		t.Errorf("OutgoingTags(%v) = %v; want %v", ctx, tg, b)
 	}
@@ -1294,7 +1306,7 @@ func (s) TestTags(t *testing.T) {
 		t.Errorf("Tags(%v) = %v; want nil", ctx, tg)
 	}
 
-	ctx = stats.SetIncomingTags(context.Background(), b)
+	ctx = stats.SetIncomingTags(tCtx, b)
 	if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, b) {
 		t.Errorf("Tags(%v) = %v; want %v", ctx, tg, b)
 	}
@@ -1305,7 +1317,9 @@ func (s) TestTags(t *testing.T) {
 
 func (s) TestTrace(t *testing.T) {
 	b := []byte{5, 2, 4, 3, 1}
-	ctx := stats.SetTrace(context.Background(), b)
+	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx := stats.SetTrace(tCtx, b)
 	if tr := stats.OutgoingTrace(ctx); !reflect.DeepEqual(tr, b) {
 		t.Errorf("OutgoingTrace(%v) = %v; want %v", ctx, tr, b)
 	}
@@ -1313,7 +1327,7 @@ func (s) TestTrace(t *testing.T) {
 		t.Errorf("Trace(%v) = %v; want nil", ctx, tr)
 	}
 
-	ctx = stats.SetIncomingTrace(context.Background(), b)
+	ctx = stats.SetIncomingTrace(tCtx, b)
 	if tr := stats.Trace(ctx); !reflect.DeepEqual(tr, b) {
 		t.Errorf("Trace(%v) = %v; want %v", ctx, tr, b)
 	}
diff --git a/test/channelz_test.go b/test/channelz_test.go
index 9f8af01e7c74..7c074961d771 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -695,13 +695,17 @@ func (t *testServiceClientWrapper) HalfDuplexCall(ctx context.Context, opts ...g
 }
 
 func doSuccessfulUnaryCall(tc testpb.TestServiceClient, t *testing.T) {
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 }
 
 func doStreamingInputCallWithLargePayload(tc testpb.TestServiceClient, t *testing.T) {
-	s, err := tc.StreamingInputCall(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s, err := tc.StreamingInputCall(ctx)
 	if err != nil {
 		t.Fatalf("TestService/StreamingInputCall(_) = _, %v, want <nil>", err)
 	}
@@ -725,7 +729,9 @@ func doServerSideFailedUnaryCall(tc testpb.TestServiceClient, t *testing.T) {
 		ResponseSize: int32(smallSize),
 		Payload:      largePayload,
 	}
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 }
diff --git a/test/creds_test.go b/test/creds_test.go
index 46bdd30dc85e..6b3fc2a46076 100644
--- a/test/creds_test.go
+++ b/test/creds_test.go
@@ -92,7 +92,9 @@ func (s) TestCredsBundleBoth(t *testing.T) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
@@ -114,7 +116,9 @@ func (s) TestCredsBundleTransportCredentials(t *testing.T) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
@@ -130,7 +134,9 @@ func (s) TestCredsBundlePerRPCCredentials(t *testing.T) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
@@ -164,8 +170,10 @@ func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
 
 	cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{}))
 	tc := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// This unary call should succeed, because ClientHandshake will succeed for the second time.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want <nil>", err)
 	}
 }
@@ -236,7 +244,7 @@ func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
 		// with Unavailable because the connection hasn't started. When the
 		// first connection failed with creds error, the next RPC should also
 		// fail with the expected error.
-		if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
+		if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
 			return
 		}
 		time.Sleep(time.Millisecond)
@@ -317,7 +325,9 @@ func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
@@ -336,7 +346,9 @@ func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
@@ -376,7 +388,9 @@ func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil {
 		t.Fatalf("Test failed. Reason: %v", err)
 	}
 }
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 0842dccaad01..07efccd0ac76 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -917,11 +917,13 @@ func (s) TestContextDeadlineNotIgnored(t *testing.T) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
+	cancel()
 	atomic.StoreInt32(&(lc.beLazy), 1)
-	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond)
 	defer cancel()
 	t1 := time.Now()
 	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
@@ -951,13 +953,15 @@ func testTimeoutOnDeadServer(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 	te.srv.Stop()
+	cancel()
 
 	// Wait for the client to notice the connection is gone.
-	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	ctx, cancel = context.WithTimeout(context.Background(), 500*time.Millisecond)
 	state := cc.GetState()
 	for ; state == connectivity.Ready && cc.WaitForStateChange(ctx, state); state = cc.GetState() {
 	}
@@ -1071,7 +1075,7 @@ func testServerGoAwayPendingRPC(t *testing.T, e env) {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
 	// Finish an RPC to make sure the connection is good.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
 	}
 	ch := make(chan struct{})
@@ -1145,7 +1149,7 @@ func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
 	// Finish an RPC to make sure the connection is good.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
 	}
 	ch1 := make(chan struct{})
@@ -1226,7 +1230,9 @@ func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
 	}
 	ch := make(chan struct{})
@@ -1263,14 +1269,18 @@ func testConcurrentServerStopAndGoAway(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	stream, err := tc.FullDuplexCall(context.Background(), grpc.WaitForReady(true))
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true))
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
+
 	// Finish an RPC to make sure the connection is good.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
 	}
+
 	ch := make(chan struct{})
 	go func() {
 		te.srv.GracefulStop()
@@ -1396,10 +1406,10 @@ func testFailFast(t *testing.T, e env) {
 		time.Sleep(10 * time.Millisecond)
 	}
 	// The client keeps reconnecting and ongoing fail-fast RPCs should fail with code.Unavailable.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.Unavailable {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
 		t.Fatalf("TestService/EmptyCall(_, _, _) = _, %v, want _, error code: %s", err, codes.Unavailable)
 	}
-	if _, err := tc.StreamingInputCall(context.Background()); status.Code(err) != codes.Unavailable {
+	if _, err := tc.StreamingInputCall(ctx); status.Code(err) != codes.Unavailable {
 		t.Fatalf("TestService/StreamingInputCall(_) = _, %v, want _, error code: %s", err, codes.Unavailable)
 	}
 
@@ -1475,9 +1485,11 @@ func (s) TestGetMethodConfig(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// The following RPCs are expected to become non-fail-fast ones with 1ms deadline.
 	var err error
-	if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
+	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded)
 	}
 
@@ -1512,7 +1524,7 @@ func (s) TestGetMethodConfig(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 	// The following RPCs are expected to become fail-fast.
-	if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.Unavailable {
+	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.Unavailable)
 	}
 }
@@ -1556,13 +1568,14 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) {
 		}
 		time.Sleep(time.Millisecond)
 	}
-
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// The following RPCs are expected to become non-fail-fast ones with 1ms deadline.
 	var err error
-	if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
+	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded)
 	}
-	if _, err := tc.FullDuplexCall(context.Background(), grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded)
 	}
 
@@ -1597,10 +1610,10 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 	// The following RPCs are expected to become non-fail-fast ones with 1ms deadline.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded)
 	}
-	if _, err := tc.FullDuplexCall(context.Background()); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.FullDuplexCall(ctx); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded)
 	}
 }
@@ -1768,16 +1781,17 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 		}
 		time.Sleep(time.Millisecond)
 	}
-
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Test for unary RPC recv.
-	if _, err = tc.UnaryCall(context.Background(), req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = extraLargePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -1838,14 +1852,14 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	req.Payload = smallPayload
 	req.ResponseSize = int32(largeSize)
 
-	if _, err = tc.UnaryCall(context.Background(), req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = largePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -1899,24 +1913,24 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) {
 	req.Payload = smallPayload
 	req.ResponseSize = int32(largeSize)
 
-	if _, err = tc.UnaryCall(context.Background(), req, grpc.WaitForReady(true)); err != nil {
+	if _, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want <nil>", err)
 	}
 
 	req.ResponseSize = int32(extraLargeSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = largePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err != nil {
+	if _, err := tc.UnaryCall(ctx, req); err != nil {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want <nil>", err)
 	}
 
 	req.Payload = extraLargePayload
-	if _, err = tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err = tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -2137,8 +2151,11 @@ func testMaxMsgSizeClientDefault(t *testing.T, e env) {
 		ResponseSize: int32(largeSize),
 		Payload:      smallPayload,
 	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Test for unary RPC recv.
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -2206,15 +2223,18 @@ func testMaxMsgSizeClientAPI(t *testing.T, e env) {
 		ResponseSize: int32(largeSize),
 		Payload:      smallPayload,
 	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Test for unary RPC recv.
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = largePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -2291,15 +2311,18 @@ func testMaxMsgSizeServerAPI(t *testing.T, e env) {
 		ResponseSize: int32(largeSize),
 		Payload:      smallPayload,
 	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Test for unary RPC send.
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC recv.
 	req.Payload = largePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -2380,7 +2403,9 @@ func testTap(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 	if ttap.cnt != 1 {
@@ -2397,7 +2422,7 @@ func testTap(t *testing.T, e env) {
 		ResponseSize: 45,
 		Payload:      payload,
 	}
-	if _, err := tc.UnaryCall(context.Background(), req); status.Code(err) != codes.Unavailable {
+	if _, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.Unavailable {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, %s", err, codes.Unavailable)
 	}
 }
@@ -2727,7 +2752,9 @@ func testEmptyUnaryWithUserAgent(t *testing.T, e env) {
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
 	var header metadata.MD
-	reply, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Header(&header))
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	reply, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.Header(&header))
 	if err != nil || !proto.Equal(&testpb.Empty{}, reply) {
 		t.Fatalf("TestService/EmptyCall(_, _) = %v, %v, want %v, <nil>", reply, err, &testpb.Empty{})
 	}
@@ -2788,7 +2815,10 @@ func testLargeUnary(t *testing.T, e env) {
 		ResponseSize: respSize,
 		Payload:      payload,
 	}
-	reply, err := tc.UnaryCall(context.Background(), req)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	reply, err := tc.UnaryCall(ctx, req)
 	if err != nil {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, <nil>", err)
 	}
@@ -2832,13 +2862,16 @@ func testExceedMsgLimit(t *testing.T, e env) {
 		ResponseSize: smallSize,
 		Payload:      largePayload,
 	}
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 	// Make sure the client cannot receive a unary RPC of largeSize.
 	req.ResponseSize = largeSize
 	req.Payload = smallPayload
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -2893,7 +2926,9 @@ func testPeerClientSide(t *testing.T, e env) {
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	peer := new(peer.Peer)
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 	pa := peer.Addr.String()
@@ -2952,8 +2987,10 @@ func testPeerFailedRPC(t *testing.T, e env) {
 	defer te.tearDown()
 	tc := testpb.NewTestServiceClient(te.clientConn())
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// first make a successful request to the server
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 
@@ -2969,7 +3006,7 @@ func testPeerFailedRPC(t *testing.T, e env) {
 	}
 
 	peer := new(peer.Peer)
-	if _, err := tc.UnaryCall(context.Background(), req, grpc.Peer(peer)); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req, grpc.Peer(peer)); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	} else {
 		pa := peer.Addr.String()
@@ -3822,7 +3859,10 @@ func testServerStreaming(t *testing.T, e env) {
 		ResponseType:       testpb.PayloadType_COMPRESSABLE,
 		ResponseParameters: respParam,
 	}
-	stream, err := tc.StreamingOutputCall(context.Background(), req)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.StreamingOutputCall(ctx, req)
 	if err != nil {
 		t.Fatalf("%v.StreamingOutputCall(_) = _, %v, want <nil>", tc, err)
 	}
@@ -3930,7 +3970,9 @@ func testServerStreamingConcurrent(t *testing.T, e env) {
 
 	doStreamingCall := func() {
 		req := &testpb.StreamingOutputCallRequest{}
-		stream, err := tc.StreamingOutputCall(context.Background(), req)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		stream, err := tc.StreamingOutputCall(ctx, req)
 		if err != nil {
 			t.Errorf("%v.StreamingOutputCall(_) = _, %v, want <nil>", tc, err)
 			return
@@ -4219,11 +4261,14 @@ func testCompressServerHasNoSupport(t *testing.T, e env) {
 		ResponseSize: respSize,
 		Payload:      payload,
 	}
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.Unimplemented {
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.Unimplemented {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code %s", err, codes.Unimplemented)
 	}
 	// Streaming RPC
-	stream, err := tc.FullDuplexCall(context.Background())
+	stream, err := tc.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
@@ -4371,7 +4416,9 @@ func testUnaryClientInterceptor(t *testing.T, e env) {
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.NotFound {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.NotFound {
 		t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, error code %s", tc, err, codes.NotFound)
 	}
 }
@@ -4411,7 +4458,9 @@ func testStreamClientInterceptor(t *testing.T, e env) {
 		ResponseParameters: respParam,
 		Payload:            payload,
 	}
-	if _, err := tc.StreamingOutputCall(context.Background(), req); status.Code(err) != codes.NotFound {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.StreamingOutputCall(ctx, req); status.Code(err) != codes.NotFound {
 		t.Fatalf("%v.StreamingOutputCall(_) = _, %v, want _, error code %s", tc, err, codes.NotFound)
 	}
 }
@@ -4433,7 +4482,9 @@ func testUnaryServerInterceptor(t *testing.T, e env) {
 	defer te.tearDown()
 
 	tc := testpb.NewTestServiceClient(te.clientConn())
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.PermissionDenied {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.PermissionDenied {
 		t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, error code %s", tc, err, codes.PermissionDenied)
 	}
 }
@@ -4477,14 +4528,16 @@ func testStreamServerInterceptor(t *testing.T, e env) {
 		ResponseParameters: respParam,
 		Payload:            payload,
 	}
-	s1, err := tc.StreamingOutputCall(context.Background(), req)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	s1, err := tc.StreamingOutputCall(ctx, req)
 	if err != nil {
 		t.Fatalf("%v.StreamingOutputCall(_) = _, %v, want _, <nil>", tc, err)
 	}
 	if _, err := s1.Recv(); status.Code(err) != codes.PermissionDenied {
 		t.Fatalf("%v.StreamingInputCall(_) = _, %v, want _, error code %s", tc, err, codes.PermissionDenied)
 	}
-	s2, err := tc.FullDuplexCall(context.Background())
+	s2, err := tc.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
@@ -4788,7 +4841,10 @@ func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) {
 	te.maxClientReceiveMsgSize = newInt(10)
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	stream, err := tc.FullDuplexCall(context.Background())
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
@@ -4881,7 +4937,6 @@ func (s) TestFlowControlLogicalRace(t *testing.T) {
 	go s.Serve(lis)
 
 	ctx := context.Background()
-
 	cc, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure(), grpc.WithBlock())
 	if err != nil {
 		t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err)
@@ -5413,7 +5468,7 @@ func (s) TestTapTimeout(t *testing.T) {
 		res, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
 		cancel()
 		if s, ok := status.FromError(err); !ok || s.Code() != codes.Canceled {
-			t.Fatalf("ss.client.EmptyCall(context.Background(), _) = %v, %v; want nil, <status with Code()=Canceled>", res, err)
+			t.Fatalf("ss.client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=Canceled>", res, err)
 		}
 	}
 
@@ -5497,7 +5552,9 @@ func testConfigurableWindowSize(t *testing.T, e env, wc windowSizeConfig) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
-	stream, err := tc.FullDuplexCall(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
 	}
@@ -5597,11 +5654,13 @@ func testEncodeDoesntPanic(t *testing.T, e env) {
 	defer te.tearDown()
 	te.customCodec = nil
 	tc := testpb.NewTestServiceClient(te.clientConn())
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Failure case, should not panic.
-	tc.EmptyCall(context.Background(), &testpb.Empty{})
+	tc.EmptyCall(ctx, &testpb.Empty{})
 	erc.noError = true
 	// Passing case.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("EmptyCall(_, _) = _, %v, want _, <nil>", err)
 	}
 }
@@ -5709,8 +5768,10 @@ func testGetMethodConfigTD(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// The following RPCs are expected to become non-fail-fast ones with 1ms deadline.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded)
 	}
 
@@ -5723,13 +5784,13 @@ func testGetMethodConfigTD(t *testing.T, e env) {
 	ch <- sc
 	// Wait for the new service config to propagate.
 	for {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) == codes.DeadlineExceeded {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.DeadlineExceeded {
 			continue
 		}
 		break
 	}
 	// The following RPCs are expected to become fail-fast.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.Unavailable {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.Unavailable)
 	}
 }
@@ -5759,11 +5820,13 @@ func testServiceConfigWaitForReadyTD(t *testing.T, e env) {
 
 	cc := te.clientConn()
 	tc := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// The following RPCs are expected to become non-fail-fast ones with 1ms deadline.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded)
 	}
-	if _, err := tc.FullDuplexCall(context.Background(), grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded)
 	}
 
@@ -5789,10 +5852,10 @@ func testServiceConfigWaitForReadyTD(t *testing.T, e env) {
 		break
 	}
 	// The following RPCs are expected to become non-fail-fast ones with 1ms deadline.
-	if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded)
 	}
-	if _, err := tc.FullDuplexCall(context.Background()); status.Code(err) != codes.DeadlineExceeded {
+	if _, err := tc.FullDuplexCall(ctx); status.Code(err) != codes.DeadlineExceeded {
 		t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded)
 	}
 }
@@ -5917,15 +5980,18 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 		ResponseSize: int32(extraLargeSize),
 		Payload:      smallPayload,
 	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// Test for unary RPC recv.
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = extraLargePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -5975,14 +6041,14 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	req.Payload = smallPayload
 	req.ResponseSize = int32(largeSize)
 
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = largePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -6024,24 +6090,24 @@ func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) {
 	req.Payload = smallPayload
 	req.ResponseSize = int32(largeSize)
 
-	if _, err := tc.UnaryCall(context.Background(), req); err != nil {
+	if _, err := tc.UnaryCall(ctx, req); err != nil {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want <nil>", err)
 	}
 
 	req.ResponseSize = int32(extraLargeSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
 	// Test for unary RPC send.
 	req.Payload = largePayload
 	req.ResponseSize = int32(smallSize)
-	if _, err := tc.UnaryCall(context.Background(), req); err != nil {
+	if _, err := tc.UnaryCall(ctx, req); err != nil {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want <nil>", err)
 	}
 
 	req.Payload = extraLargePayload
-	if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted {
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted)
 	}
 
@@ -6098,7 +6164,9 @@ func (s) TestMethodFromServerStream(t *testing.T) {
 
 	te.startServer(nil)
 	defer te.tearDown()
-	_ = te.clientConn().Invoke(context.Background(), testMethod, nil, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_ = te.clientConn().Invoke(ctx, testMethod, nil, nil)
 	if !ok || method != testMethod {
 		t.Fatalf("Invoke with method %q, got %q, %v, want %q, true", testMethod, method, ok, testMethod)
 	}
@@ -6165,7 +6233,9 @@ func (s) TestInterceptorCanAccessCallOptions(t *testing.T) {
 	var headers metadata.MD
 	var trailers metadata.MD
 	var pr peer.Peer
-	tc.UnaryCall(context.Background(), &testpb.SimpleRequest{},
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	tc.UnaryCall(ctx, &testpb.SimpleRequest{},
 		grpc.MaxCallRecvMsgSize(100),
 		grpc.MaxCallSendMsgSize(200),
 		grpc.PerRPCCredentials(testPerRPCCredentials{}),
@@ -6188,7 +6258,7 @@ func (s) TestInterceptorCanAccessCallOptions(t *testing.T) {
 
 	observedOpts = observedOptions{} // reset
 
-	tc.StreamingInputCall(context.Background(),
+	tc.StreamingInputCall(ctx,
 		grpc.WaitForReady(false),
 		grpc.MaxCallSendMsgSize(2020),
 		grpc.UseCompressor("comp-type"),
diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go
index 0a60f8c927ac..99f7d8951ebd 100644
--- a/test/healthcheck_test.go
+++ b/test/healthcheck_test.go
@@ -315,9 +315,12 @@ func (s) TestHealthCheckWithGoAway(t *testing.T) {
 	}
 }`)})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
 	// make some rpcs to make sure connection is working.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
@@ -326,8 +329,6 @@ func (s) TestHealthCheckWithGoAway(t *testing.T) {
 	}
 
 	// the stream rpc will persist through goaway event.
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
 	stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true))
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
@@ -407,9 +408,11 @@ func (s) TestHealthCheckWithConnClose(t *testing.T) {
 	}
 }`)})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// make some rpcs to make sure connection is working.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
@@ -470,9 +473,11 @@ func (s) TestHealthCheckWithAddrConnDrain(t *testing.T) {
 		ServiceConfig: sc,
 	})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// make some rpcs to make sure connection is working.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
@@ -481,8 +486,6 @@ func (s) TestHealthCheckWithAddrConnDrain(t *testing.T) {
 	}
 
 	// the stream rpc will persist through goaway event.
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
 	stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true))
 	if err != nil {
 		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
@@ -561,9 +564,11 @@ func (s) TestHealthCheckWithClientConnClose(t *testing.T) {
 	}
 }`)})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// make some rpcs to make sure connection is working.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
@@ -765,9 +770,11 @@ func testHealthCheckDisableWithDialOption(t *testing.T, addr string) {
 	}
 }`)})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// send some rpcs to make sure transport has been created and is ready for use.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
@@ -804,9 +811,11 @@ func testHealthCheckDisableWithBalancer(t *testing.T, addr string) {
 	}
 }`)})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// send some rpcs to make sure transport has been created and is ready for use.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
@@ -837,9 +846,11 @@ func testHealthCheckDisableWithServiceConfig(t *testing.T, addr string) {
 
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: addr}}})
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	// send some rpcs to make sure transport has been created and is ready for use.
 	if err := verifyResultWithDelay(func() (bool, error) {
-		if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 			return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
 		}
 		return true, nil
diff --git a/test/server_test.go b/test/server_test.go
index c6a5fe74bd55..41466157a19f 100644
--- a/test/server_test.go
+++ b/test/server_test.go
@@ -135,9 +135,11 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) {
 	}
 	defer ss.Stop()
 
-	resp, err := ss.client.UnaryCall(context.Background(), &testpb.SimpleRequest{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, err := ss.client.UnaryCall(ctx, &testpb.SimpleRequest{})
 	if s, ok := status.FromError(err); !ok || s.Code() != codes.OK {
-		t.Fatalf("ss.client.UnaryCall(context.Background(), _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
+		t.Fatalf("ss.client.UnaryCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
 	}
 
 	respBytes := resp.Payload.GetBody()
@@ -181,9 +183,11 @@ func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) {
 	}
 	defer ss.Stop()
 
-	resp, err := ss.client.EmptyCall(context.Background(), &testpb.Empty{})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
 	if s, ok := status.FromError(err); !ok || s.Code() != codes.OK {
-		t.Fatalf("ss.client.EmptyCall(context.Background(), _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
+		t.Fatalf("ss.client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
 	}
 }
 
@@ -268,7 +272,9 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) {
 	}
 	defer ss.Stop()
 
-	stream, err := ss.client.FullDuplexCall(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.client.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("failed to FullDuplexCall: %v", err)
 	}
diff --git a/test/stream_cleanup_test.go b/test/stream_cleanup_test.go
index cb31b4eb2876..77d9477cf17e 100644
--- a/test/stream_cleanup_test.go
+++ b/test/stream_cleanup_test.go
@@ -50,10 +50,12 @@ func (s) TestStreamCleanup(t *testing.T) {
 	}
 	defer ss.Stop()
 
-	if _, err := ss.client.UnaryCall(context.Background(), &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("should fail with ResourceExhausted, message's body size: %v, maximum message size the client can receive: %v", bodySize, callRecvMsgSize)
 	}
-	if _, err := ss.client.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("should succeed, err: %v", err)
 	}
 }

From 25ddfddb4f030e94c55bc2725cbc2ef5ef179901 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 5 Nov 2020 15:04:08 -0800
Subject: [PATCH 264/481] rds: allow case_insensitive path matching (#3997)

- in xds_client, accept (not NACK) RDS resp with case_insensitive=true
- pass case_insensitive to xds resolver and routing balancer
  - Note that after the config selector change, the routing balancer will be removed, and
    this will be handled in the resolver config selector
---
 .../balancer/xdsrouting/matcher_path.go       | 40 +++++++++++++++----
 .../balancer/xdsrouting/matcher_path_test.go  | 28 ++++++++-----
 .../balancer/xdsrouting/matcher_test.go       | 28 ++++++++-----
 xds/internal/balancer/xdsrouting/routing.go   |  4 +-
 .../balancer/xdsrouting/routing_config.go     |  5 +++
 .../xdsrouting/routing_picker_test.go         | 20 +++++-----
 xds/internal/client/client.go                 |  9 +++--
 xds/internal/client/client_rds_test.go        | 27 +++++++++++--
 xds/internal/client/client_xds.go             |  8 ++--
 xds/internal/resolver/serviceconfig.go        | 22 +++++-----
 xds/internal/resolver/serviceconfig_test.go   |  6 ++-
 11 files changed, 136 insertions(+), 61 deletions(-)

diff --git a/xds/internal/balancer/xdsrouting/matcher_path.go b/xds/internal/balancer/xdsrouting/matcher_path.go
index 9c783acbb577..f6f4b7ddd9e1 100644
--- a/xds/internal/balancer/xdsrouting/matcher_path.go
+++ b/xds/internal/balancer/xdsrouting/matcher_path.go
@@ -30,14 +30,26 @@ type pathMatcherInterface interface {
 }
 
 type pathExactMatcher struct {
-	fullPath string
+	// fullPath is all upper case if caseInsensitive is true.
+	fullPath        string
+	caseInsensitive bool
 }
 
-func newPathExactMatcher(p string) *pathExactMatcher {
-	return &pathExactMatcher{fullPath: p}
+func newPathExactMatcher(p string, caseInsensitive bool) *pathExactMatcher {
+	ret := &pathExactMatcher{
+		fullPath:        p,
+		caseInsensitive: caseInsensitive,
+	}
+	if caseInsensitive {
+		ret.fullPath = strings.ToUpper(p)
+	}
+	return ret
 }
 
 func (pem *pathExactMatcher) match(path string) bool {
+	if pem.caseInsensitive {
+		return pem.fullPath == strings.ToUpper(path)
+	}
 	return pem.fullPath == path
 }
 
@@ -46,7 +58,7 @@ func (pem *pathExactMatcher) equal(m pathMatcherInterface) bool {
 	if !ok {
 		return false
 	}
-	return pem.fullPath == mm.fullPath
+	return pem.fullPath == mm.fullPath && pem.caseInsensitive == mm.caseInsensitive
 }
 
 func (pem *pathExactMatcher) String() string {
@@ -54,14 +66,26 @@ func (pem *pathExactMatcher) String() string {
 }
 
 type pathPrefixMatcher struct {
-	prefix string
+	// prefix is all upper case if caseInsensitive is true.
+	prefix          string
+	caseInsensitive bool
 }
 
-func newPathPrefixMatcher(p string) *pathPrefixMatcher {
-	return &pathPrefixMatcher{prefix: p}
+func newPathPrefixMatcher(p string, caseInsensitive bool) *pathPrefixMatcher {
+	ret := &pathPrefixMatcher{
+		prefix:          p,
+		caseInsensitive: caseInsensitive,
+	}
+	if caseInsensitive {
+		ret.prefix = strings.ToUpper(p)
+	}
+	return ret
 }
 
 func (ppm *pathPrefixMatcher) match(path string) bool {
+	if ppm.caseInsensitive {
+		return strings.HasPrefix(strings.ToUpper(path), ppm.prefix)
+	}
 	return strings.HasPrefix(path, ppm.prefix)
 }
 
@@ -70,7 +94,7 @@ func (ppm *pathPrefixMatcher) equal(m pathMatcherInterface) bool {
 	if !ok {
 		return false
 	}
-	return ppm.prefix == mm.prefix
+	return ppm.prefix == mm.prefix && ppm.caseInsensitive == mm.caseInsensitive
 }
 
 func (ppm *pathPrefixMatcher) String() string {
diff --git a/xds/internal/balancer/xdsrouting/matcher_path_test.go b/xds/internal/balancer/xdsrouting/matcher_path_test.go
index ad7db7481060..b50d5d84d8bc 100644
--- a/xds/internal/balancer/xdsrouting/matcher_path_test.go
+++ b/xds/internal/balancer/xdsrouting/matcher_path_test.go
@@ -25,17 +25,21 @@ import (
 
 func TestPathFullMatcherMatch(t *testing.T) {
 	tests := []struct {
-		name     string
-		fullPath string
-		path     string
-		want     bool
+		name            string
+		fullPath        string
+		caseInsensitive bool
+		path            string
+		want            bool
 	}{
 		{name: "match", fullPath: "/s/m", path: "/s/m", want: true},
+		{name: "case insensitive match", fullPath: "/s/m", caseInsensitive: true, path: "/S/m", want: true},
+		{name: "case insensitive match 2", fullPath: "/s/M", caseInsensitive: true, path: "/S/m", want: true},
 		{name: "not match", fullPath: "/s/m", path: "/a/b", want: false},
+		{name: "case insensitive not match", fullPath: "/s/m", caseInsensitive: true, path: "/a/b", want: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			fpm := newPathExactMatcher(tt.fullPath)
+			fpm := newPathExactMatcher(tt.fullPath, tt.caseInsensitive)
 			if got := fpm.match(tt.path); got != tt.want {
 				t.Errorf("{%q}.match(%q) = %v, want %v", tt.fullPath, tt.path, got, tt.want)
 			}
@@ -45,17 +49,21 @@ func TestPathFullMatcherMatch(t *testing.T) {
 
 func TestPathPrefixMatcherMatch(t *testing.T) {
 	tests := []struct {
-		name   string
-		prefix string
-		path   string
-		want   bool
+		name            string
+		prefix          string
+		caseInsensitive bool
+		path            string
+		want            bool
 	}{
 		{name: "match", prefix: "/s/", path: "/s/m", want: true},
+		{name: "case insensitive match", prefix: "/s/", caseInsensitive: true, path: "/S/m", want: true},
+		{name: "case insensitive match 2", prefix: "/S/", caseInsensitive: true, path: "/s/m", want: true},
 		{name: "not match", prefix: "/s/", path: "/a/b", want: false},
+		{name: "case insensitive not match", prefix: "/s/", caseInsensitive: true, path: "/a/b", want: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			fpm := newPathPrefixMatcher(tt.prefix)
+			fpm := newPathPrefixMatcher(tt.prefix, tt.caseInsensitive)
 			if got := fpm.match(tt.path); got != tt.want {
 				t.Errorf("{%q}.match(%q) = %v, want %v", tt.prefix, tt.path, got, tt.want)
 			}
diff --git a/xds/internal/balancer/xdsrouting/matcher_test.go b/xds/internal/balancer/xdsrouting/matcher_test.go
index e7d76e27469f..74c570a1a5b7 100644
--- a/xds/internal/balancer/xdsrouting/matcher_test.go
+++ b/xds/internal/balancer/xdsrouting/matcher_test.go
@@ -38,7 +38,17 @@ func TestAndMatcherMatch(t *testing.T) {
 	}{
 		{
 			name: "both match",
-			pm:   newPathExactMatcher("/a/b"),
+			pm:   newPathExactMatcher("/a/b", false),
+			hm:   newHeaderExactMatcher("th", "tv"),
+			info: balancer.PickInfo{
+				FullMethodName: "/a/b",
+				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			},
+			want: true,
+		},
+		{
+			name: "both match with path case insensitive",
+			pm:   newPathExactMatcher("/A/B", true),
 			hm:   newHeaderExactMatcher("th", "tv"),
 			info: balancer.PickInfo{
 				FullMethodName: "/a/b",
@@ -48,7 +58,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		},
 		{
 			name: "only one match",
-			pm:   newPathExactMatcher("/a/b"),
+			pm:   newPathExactMatcher("/a/b", false),
 			hm:   newHeaderExactMatcher("th", "tv"),
 			info: balancer.PickInfo{
 				FullMethodName: "/z/y",
@@ -58,7 +68,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		},
 		{
 			name: "both not match",
-			pm:   newPathExactMatcher("/z/y"),
+			pm:   newPathExactMatcher("/z/y", false),
 			hm:   newHeaderExactMatcher("th", "abc"),
 			info: balancer.PickInfo{
 				FullMethodName: "/a/b",
@@ -68,7 +78,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		},
 		{
 			name: "fake header",
-			pm:   newPathPrefixMatcher("/"),
+			pm:   newPathPrefixMatcher("/", false),
 			hm:   newHeaderExactMatcher("content-type", "fake"),
 			info: balancer.PickInfo{
 				FullMethodName: "/a/b",
@@ -80,7 +90,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		},
 		{
 			name: "binary header",
-			pm:   newPathPrefixMatcher("/"),
+			pm:   newPathPrefixMatcher("/", false),
 			hm:   newHeaderPresentMatcher("t-bin", true),
 			info: balancer.PickInfo{
 				FullMethodName: "/a/b",
@@ -146,8 +156,8 @@ func TestCompositeMatcherEqual(t *testing.T) {
 	}{
 		{
 			name: "equal",
-			pm:   newPathExactMatcher("/a/b"),
-			mm:   newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil),
+			pm:   newPathExactMatcher("/a/b", false),
+			mm:   newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil),
 			want: true,
 		},
 		{
@@ -158,9 +168,9 @@ func TestCompositeMatcherEqual(t *testing.T) {
 		},
 		{
 			name: "not equal",
-			pm:   newPathExactMatcher("/a/b"),
+			pm:   newPathExactMatcher("/a/b", false),
 			fm:   newFractionMatcher(123),
-			mm:   newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil),
+			mm:   newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil),
 			want: false,
 		},
 	}
diff --git a/xds/internal/balancer/xdsrouting/routing.go b/xds/internal/balancer/xdsrouting/routing.go
index ad5373a2f773..fd92fd91b8a8 100644
--- a/xds/internal/balancer/xdsrouting/routing.go
+++ b/xds/internal/balancer/xdsrouting/routing.go
@@ -136,9 +136,9 @@ func routeToMatcher(r routeConfig) (*compositeMatcher, error) {
 		}
 		pathMatcher = newPathRegexMatcher(re)
 	case r.path != "":
-		pathMatcher = newPathExactMatcher(r.path)
+		pathMatcher = newPathExactMatcher(r.path, r.caseInsensitive)
 	default:
-		pathMatcher = newPathPrefixMatcher(r.prefix)
+		pathMatcher = newPathPrefixMatcher(r.prefix, r.caseInsensitive)
 	}
 
 	var headerMatchers []headerMatcherInterface
diff --git a/xds/internal/balancer/xdsrouting/routing_config.go b/xds/internal/balancer/xdsrouting/routing_config.go
index 78716a098136..526fe27692f0 100644
--- a/xds/internal/balancer/xdsrouting/routing_config.go
+++ b/xds/internal/balancer/xdsrouting/routing_config.go
@@ -52,6 +52,9 @@ type routeConfig struct {
 	// Path, Prefix and Regex can have at most one set. This is guaranteed by
 	// config parsing.
 	path, prefix, regex string
+	// Indicates if prefix/path matching should be case insensitive. The default
+	// is false (case sensitive).
+	caseInsensitive bool
 
 	headers  []headerMatcher
 	fraction *uint32
@@ -74,6 +77,7 @@ type lbConfig struct {
 type routeJSON struct {
 	// Path, Prefix and Regex can have at most one non-nil.
 	Path, Prefix, Regex *string
+	CaseInsensitive     bool
 	// Zero or more header matchers.
 	Headers       []*xdsclient.HeaderMatcher
 	MatchFraction *wrapperspb.UInt32Value
@@ -99,6 +103,7 @@ func (jc lbConfigJSON) toLBConfig() *lbConfig {
 		case r.Regex != nil:
 			tempR.regex = *r.Regex
 		}
+		tempR.caseInsensitive = r.CaseInsensitive
 		for _, h := range r.Headers {
 			var tempHeader headerMatcher
 			switch {
diff --git a/xds/internal/balancer/xdsrouting/routing_picker_test.go b/xds/internal/balancer/xdsrouting/routing_picker_test.go
index 83e494a4bf8d..fd03c3b975cb 100644
--- a/xds/internal/balancer/xdsrouting/routing_picker_test.go
+++ b/xds/internal/balancer/xdsrouting/routing_picker_test.go
@@ -52,7 +52,7 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		{
 			name: "one route no match",
 			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
 			},
 			pickers: map[string]*subBalancerState{
 				"action-0": {state: balancer.State{
@@ -66,7 +66,7 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		{
 			name: "one route one match",
 			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
 			},
 			pickers: map[string]*subBalancerState{
 				"action-0": {state: balancer.State{
@@ -80,8 +80,8 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		{
 			name: "two routes first match",
 			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/z/", false), nil, nil), action: "action-1"},
 			},
 			pickers: map[string]*subBalancerState{
 				"action-0": {state: balancer.State{
@@ -99,8 +99,8 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		{
 			name: "two routes second match",
 			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/z/", false), nil, nil), action: "action-1"},
 			},
 			pickers: map[string]*subBalancerState{
 				"action-0": {state: balancer.State{
@@ -118,8 +118,8 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		{
 			name: "two routes both match former more specific",
 			routes: []route{
-				{m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-1"},
+				{m: newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-1"},
 			},
 			pickers: map[string]*subBalancerState{
 				"action-0": {state: balancer.State{
@@ -138,8 +138,8 @@ func (s) TestRoutingPickerGroupPick(t *testing.T) {
 		{
 			name: "tow routes both match latter more specific",
 			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-1"},
+				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
+				{m: newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil), action: "action-1"},
 			},
 			pickers: map[string]*subBalancerState{
 				"action-0": {state: balancer.State{
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index c2bbeafbdd8d..583f09baf1fd 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -167,9 +167,12 @@ type VirtualHost struct {
 // indication of the action to take upon match.
 type Route struct {
 	Path, Prefix, Regex *string
-	Headers             []*HeaderMatcher
-	Fraction            *uint32
-	Action              map[string]uint32 // action is weighted clusters.
+	// Indicates if prefix/path matching should be case insensitive. The default
+	// is false (case sensitive).
+	CaseInsensitive bool
+	Headers         []*HeaderMatcher
+	Fraction        *uint32
+	Action          map[string]uint32 // action is weighted clusters.
 }
 
 // HeaderMatcher represents header matchers.
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index ee092117cac5..7b39158b8e36 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -280,7 +280,14 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 							Route: &v3routepb.RouteAction{
 								ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
 							}}}}}}},
-			wantError: true,
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP("/"), CaseInsensitive: true, Action: map[string]uint32{clusterName: 1}}},
+					},
+				},
+			},
 		},
 		{
 			name: "good-route-config-with-empty-string-route",
@@ -434,7 +441,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, nil)
 			if (gotError != nil) != test.wantError || !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, ldsTarget, gotUpdate, test.wantUpdate)
+				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) returned unexpected, diff (-want +got):\\n%s", test.rc, ldsTarget, cmp.Diff(test.wantUpdate, gotUpdate, cmpopts.EquateEmpty()))
 			}
 		})
 	}
@@ -654,8 +661,22 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 					PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
 					CaseSensitive: &wrapperspb.BoolValue{Value: false},
 				},
+				Action: &v3routepb.Route_Route{
+					Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+							WeightedClusters: &v3routepb.WeightedCluster{
+								Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+									{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
+									{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+								},
+								TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+							}}}},
+			}},
+			wantRoutes: []*Route{{
+				Prefix:          newStringP("/"),
+				CaseInsensitive: true,
+				Action:          map[string]uint32{"A": 40, "B": 60},
 			}},
-			wantErr: true,
 		},
 		{
 			name: "good",
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index b8598c0247d9..af25ce226bd4 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -171,10 +171,6 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			continue
 		}
 
-		if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value {
-			return nil, fmt.Errorf("route %+v has case-sensitive false", r)
-		}
-
 		pathSp := match.GetPathSpecifier()
 		if pathSp == nil {
 			return nil, fmt.Errorf("route %+v doesn't have a path specifier", r)
@@ -193,6 +189,10 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			continue
 		}
 
+		if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil {
+			route.CaseInsensitive = !caseSensitive.Value
+		}
+
 		for _, h := range match.GetHeaders() {
 			var header HeaderMatcher
 			switch ht := h.GetHeaderMatchSpecifier().(type) {
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 965817c6c8ba..f585e4379678 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -56,12 +56,13 @@ type cdsBalancerConfig struct {
 }
 
 type route struct {
-	Path     *string                    `json:"path,omitempty"`
-	Prefix   *string                    `json:"prefix,omitempty"`
-	Regex    *string                    `json:"regex,omitempty"`
-	Headers  []*xdsclient.HeaderMatcher `json:"headers,omitempty"`
-	Fraction *wrapperspb.UInt32Value    `json:"matchFraction,omitempty"`
-	Action   string                     `json:"action"`
+	Path            *string                    `json:"path,omitempty"`
+	Prefix          *string                    `json:"prefix,omitempty"`
+	Regex           *string                    `json:"regex,omitempty"`
+	CaseInsensitive bool                       `json:"caseInsensitive"`
+	Headers         []*xdsclient.HeaderMatcher `json:"headers,omitempty"`
+	Fraction        *wrapperspb.UInt32Value    `json:"matchFraction,omitempty"`
+	Action          string                     `json:"action"`
 }
 
 type xdsActionConfig struct {
@@ -80,10 +81,11 @@ func (r *xdsResolver) routesToJSON(routes []*xdsclient.Route) (string, error) {
 	var rts []*route
 	for _, rt := range routes {
 		t := &route{
-			Path:    rt.Path,
-			Prefix:  rt.Prefix,
-			Regex:   rt.Regex,
-			Headers: rt.Headers,
+			Path:            rt.Path,
+			Prefix:          rt.Prefix,
+			Regex:           rt.Regex,
+			Headers:         rt.Headers,
+			CaseInsensitive: rt.CaseInsensitive,
 		}
 
 		if f := rt.Fraction; f != nil {
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index 5969bd83caed..cce30c183910 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -144,6 +144,7 @@ const (
       },
       {
         "prefix":"/service_2/method_1",
+        "caseInsensitive":true,
         "action":"cluster_1_0"
       },
       {
@@ -214,8 +215,9 @@ func TestRoutesToJSON(t *testing.T) {
 					Action: map[string]uint32{"cluster_1": 1},
 				},
 				{
-					Prefix: newStringP("/service_2/method_1"),
-					Action: map[string]uint32{"cluster_1": 1},
+					Prefix:          newStringP("/service_2/method_1"),
+					CaseInsensitive: true,
+					Action:          map[string]uint32{"cluster_1": 1},
 				},
 				{
 					Regex:  newStringP("^/service_2/method_3$"),

From 5d7f8c9ae4223acc93e34ebc3d79476d5a55dbae Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Thu, 5 Nov 2020 23:51:15 -0800
Subject: [PATCH 265/481] advancedtls: make example to use new pemfile API
 (#4012)

* advancedtls: make example to use new pemfile API
---
 .../client/main.go                            | 25 ++++++-------
 .../server/main.go                            | 25 ++++++-------
 security/advancedtls/examples/go.mod          |  8 +++-
 security/advancedtls/examples/go.sum          | 37 ++-----------------
 4 files changed, 34 insertions(+), 61 deletions(-)

diff --git a/security/advancedtls/examples/credential_reloading_from_files/client/main.go b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
index c300ba4cf941..2aa4dace587e 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/client/main.go
+++ b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 	"google.golang.org/grpc/security/advancedtls"
 	"google.golang.org/grpc/security/advancedtls/testdata"
@@ -45,24 +46,22 @@ const (
 func main() {
 	flag.Parse()
 
-	// TODO(ZhenLian): change function signatures to reflect the changes in
-	// https://github.com/grpc/grpc-go/pull/3981.
-	identityOptions := advancedtls.PEMFileProviderOptions{
-		CertFile:         testdata.Path("client_cert_1.pem"),
-		KeyFile:          testdata.Path("client_key_1.pem"),
-		IdentityInterval: credRefreshingInterval,
+	identityOptions := pemfile.Options{
+		CertFile:            testdata.Path("client_cert_1.pem"),
+		KeyFile:             testdata.Path("client_key_1.pem"),
+		CertRefreshDuration: credRefreshingInterval,
 	}
-	identityProvider, err := advancedtls.NewPEMFileProvider(identityOptions)
+	identityProvider, err := pemfile.NewProvider(identityOptions)
 	if err != nil {
-		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", identityOptions, err)
+		log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err)
 	}
-	rootOptions := advancedtls.PEMFileProviderOptions{
-		TrustFile:    testdata.Path("client_trust_cert_1.pem"),
-		RootInterval: credRefreshingInterval,
+	rootOptions := pemfile.Options{
+		RootFile:            testdata.Path("client_trust_cert_1.pem"),
+		RootRefreshDuration: credRefreshingInterval,
 	}
-	rootProvider, err := advancedtls.NewPEMFileProvider(rootOptions)
+	rootProvider, err := pemfile.NewProvider(rootOptions)
 	if err != nil {
-		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", rootOptions, err)
+		log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err)
 	}
 
 	options := &advancedtls.ClientOptions{
diff --git a/security/advancedtls/examples/credential_reloading_from_files/server/main.go b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
index b60762fa625a..c1431866f7b2 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/server/main.go
+++ b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	"google.golang.org/grpc/security/advancedtls"
 	"google.golang.org/grpc/security/advancedtls/testdata"
 
@@ -53,25 +54,23 @@ func main() {
 	flag.Parse()
 	fmt.Printf("server starting on port %s...\n", port)
 
-	// TODO(ZhenLian): change function signatures to reflect the changes in
-	// https://github.com/grpc/grpc-go/pull/3981.
-	identityOptions := advancedtls.PEMFileProviderOptions{
-		CertFile:         testdata.Path("server_cert_1.pem"),
-		KeyFile:          testdata.Path("server_key_1.pem"),
-		IdentityInterval: credRefreshingInterval,
+	identityOptions := pemfile.Options{
+		CertFile:            testdata.Path("server_cert_1.pem"),
+		KeyFile:             testdata.Path("server_key_1.pem"),
+		CertRefreshDuration: credRefreshingInterval,
 	}
-	identityProvider, err := advancedtls.NewPEMFileProvider(identityOptions)
+	identityProvider, err := pemfile.NewProvider(identityOptions)
 	if err != nil {
-		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", identityOptions, err)
+		log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err)
 	}
 	defer identityProvider.Close()
-	rootOptions := advancedtls.PEMFileProviderOptions{
-		TrustFile:    testdata.Path("server_trust_cert_1.pem"),
-		RootInterval: credRefreshingInterval,
+	rootOptions := pemfile.Options{
+		RootFile:            testdata.Path("server_trust_cert_1.pem"),
+		RootRefreshDuration: credRefreshingInterval,
 	}
-	rootProvider, err := advancedtls.NewPEMFileProvider(rootOptions)
+	rootProvider, err := pemfile.NewProvider(rootOptions)
 	if err != nil {
-		log.Fatalf("advancedtls.NewPEMFileProvider(%v) failed: %v", rootOptions, err)
+		log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err)
 	}
 	defer rootProvider.Close()
 
diff --git a/security/advancedtls/examples/go.mod b/security/advancedtls/examples/go.mod
index ed473445ef28..05cf3fc75c80 100644
--- a/security/advancedtls/examples/go.mod
+++ b/security/advancedtls/examples/go.mod
@@ -5,5 +5,11 @@ go 1.15
 require (
 	google.golang.org/grpc v1.33.1
 	google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d
-	google.golang.org/grpc/security/advancedtls v0.0.0-20201020200225-9519efffeb5d
+	google.golang.org/grpc/security/advancedtls v0.0.0-20201102215344-15ae9fc2b247
 )
+
+replace google.golang.org/grpc => ../../..
+
+replace google.golang.org/grpc/examples => ../../../examples
+
+replace google.golang.org/grpc/security/advancedtls => ../
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
index 9289089aa26c..e99f515c1397 100644
--- a/security/advancedtls/examples/go.sum
+++ b/security/advancedtls/examples/go.sum
@@ -1,4 +1,3 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
@@ -37,12 +36,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -51,7 +47,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
@@ -109,6 +104,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -134,7 +130,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -153,7 +148,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -183,7 +177,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,7 +184,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -227,7 +219,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -287,14 +278,12 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -320,30 +309,10 @@ google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f/go.mod h1:TGiSRL2BBv2WqzfsFNWYp/pkWdtf5kbZS/DQ9Ee3mWk=
-google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d h1:AW0XWR8UrnkpSYMW9c9obDY/xXI0phgJOn+7oAp2DpA=
-google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d/go.mod h1:Lh55/1hxmVHEkOvSIQ2uj0P12QyOCUNyRwnUlSS13hw=
-google.golang.org/grpc/security/advancedtls v0.0.0-20201020200225-9519efffeb5d h1:BQWmmlFW/jQsHms4c99jRsiJAVv52QoUmCHTSzJXEAM=
-google.golang.org/grpc/security/advancedtls v0.0.0-20201020200225-9519efffeb5d/go.mod h1:39RL6g/iGSMf3cQevYAO8tFbj0cakVVRjYayPce2p+k=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

From bc01f3fa878091a871245822426e598380a727a9 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 6 Nov 2020 11:24:57 -0800
Subject: [PATCH 266/481] certprovider: API tweaks. (#3987)

---
 .../tls/certprovider/meshca/builder.go        |  44 +++--
 .../tls/certprovider/meshca/builder_test.go   |  16 +-
 credentials/tls/certprovider/meshca/config.go |   2 +-
 .../tls/certprovider/meshca/config_test.go    |  31 ++--
 credentials/tls/certprovider/meshca/plugin.go |  14 +-
 .../tls/certprovider/meshca/plugin_test.go    |  27 +--
 credentials/tls/certprovider/provider.go      |  24 +--
 credentials/tls/certprovider/store.go         | 116 +++++++-----
 credentials/tls/certprovider/store_test.go    | 127 +++++++-------
 .../balancer/cdsbalancer/cdsbalancer.go       |  68 ++++---
 .../cdsbalancer/cdsbalancer_security_test.go  | 166 ++++++++----------
 xds/internal/client/bootstrap/bootstrap.go    |  25 +--
 .../client/bootstrap/bootstrap_test.go        |  26 ++-
 xds/internal/client/client.go                 |   8 +-
 xds/internal/testutils/fakeclient/client.go   |   8 +-
 15 files changed, 332 insertions(+), 370 deletions(-)

diff --git a/credentials/tls/certprovider/meshca/builder.go b/credentials/tls/certprovider/meshca/builder.go
index 3544a1647f67..4b8af7c9b3c5 100644
--- a/credentials/tls/certprovider/meshca/builder.go
+++ b/credentials/tls/certprovider/meshca/builder.go
@@ -65,7 +65,7 @@ type refCountedCC struct {
 }
 
 // pluginBuilder is an implementation of the certprovider.Builder interface,
-// which build certificate provider instances which get certificates signed from
+// which builds certificate provider instances to get certificates signed from
 // the MeshCA.
 type pluginBuilder struct {
 	// A collection of ClientConns to the MeshCA server along with a reference
@@ -75,22 +75,30 @@ type pluginBuilder struct {
 	clients map[ccMapKey]*refCountedCC
 }
 
-// Build returns a MeshCA certificate provider for the passed in configuration
-// and options.
+// ParseConfig parses the configuration to be passed to the MeshCA plugin
+// implementation. Expects the config to be a json.RawMessage which contains a
+// serialized JSON representation of the meshca_experimental.GoogleMeshCaConfig
+// proto message.
 //
-// This builder takes care of sharing the ClientConn to the MeshCA server among
+// Takes care of sharing the ClientConn to the MeshCA server among
 // different plugin instantiations.
-func (b *pluginBuilder) Build(c certprovider.StableConfig, opts certprovider.Options) certprovider.Provider {
-	cfg, ok := c.(*pluginConfig)
+func (b *pluginBuilder) ParseConfig(c interface{}) (*certprovider.BuildableConfig, error) {
+	data, ok := c.(json.RawMessage)
 	if !ok {
-		// This is not expected when passing config returned by ParseConfig().
-		// This could indicate a bug in the certprovider.Store implementation or
-		// in cases where the user is directly using these APIs, could be a user
-		// error.
-		logger.Errorf("unsupported config type: %T", c)
-		return nil
+		return nil, fmt.Errorf("meshca: unsupported config type: %T", c)
+	}
+	cfg, err := pluginConfigFromJSON(data)
+	if err != nil {
+		return nil, err
 	}
+	return certprovider.NewBuildableConfig(pluginName, cfg.canonical(), func(opts certprovider.BuildOptions) certprovider.Provider {
+		return b.buildFromConfig(cfg, opts)
+	}), nil
+}
 
+// buildFromConfig builds a certificate provider instance for the given config
+// and options. Provider instances are shared wherever possible.
+func (b *pluginBuilder) buildFromConfig(cfg *pluginConfig, opts certprovider.BuildOptions) certprovider.Provider {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
@@ -151,18 +159,6 @@ func (b *pluginBuilder) Build(c certprovider.StableConfig, opts certprovider.Opt
 	return p
 }
 
-// ParseConfig parses the configuration to be passed to the MeshCA plugin
-// implementation. Expects the config to be a json.RawMessage which contains a
-// serialized JSON representation of the meshca_experimental.GoogleMeshCaConfig
-// proto message.
-func (b *pluginBuilder) ParseConfig(c interface{}) (certprovider.StableConfig, error) {
-	data, ok := c.(json.RawMessage)
-	if !ok {
-		return nil, fmt.Errorf("meshca: unsupported config type: %T", c)
-	}
-	return pluginConfigFromJSON(data)
-}
-
 // Name returns the MeshCA plugin name.
 func (b *pluginBuilder) Name() string {
 	return pluginName
diff --git a/credentials/tls/certprovider/meshca/builder_test.go b/credentials/tls/certprovider/meshca/builder_test.go
index b395f4f4b91d..d21307f4a95b 100644
--- a/credentials/tls/certprovider/meshca/builder_test.go
+++ b/credentials/tls/certprovider/meshca/builder_test.go
@@ -71,7 +71,7 @@ func (s) TestBuildSameConfig(t *testing.T) {
 	// invocations of Build().
 	inputConfig := makeJSONConfig(t, goodConfigFullySpecified)
 	builder := newPluginBuilder()
-	stableConfig, err := builder.ParseConfig(inputConfig)
+	buildableConfig, err := builder.ParseConfig(inputConfig)
 	if err != nil {
 		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
 	}
@@ -80,9 +80,9 @@ func (s) TestBuildSameConfig(t *testing.T) {
 	// end up sharing the same ClientConn.
 	providers := []certprovider.Provider{}
 	for i := 0; i < cnt; i++ {
-		p := builder.Build(stableConfig, certprovider.Options{})
-		if p == nil {
-			t.Fatalf("builder.Build(%s) failed: %v", string(stableConfig.Canonical()), err)
+		p, err := buildableConfig.Build(certprovider.BuildOptions{})
+		if err != nil {
+			t.Fatalf("Build(%+v) failed: %v", buildableConfig, err)
 		}
 		providers = append(providers, p)
 	}
@@ -146,14 +146,14 @@ func (s) TestBuildDifferentConfig(t *testing.T) {
 		cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
 		cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = fmt.Sprintf("test-mesh-ca:%d", i)
 		inputConfig := makeJSONConfig(t, cfg)
-		stableConfig, err := builder.ParseConfig(inputConfig)
+		buildableConfig, err := builder.ParseConfig(inputConfig)
 		if err != nil {
 			t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
 		}
 
-		p := builder.Build(stableConfig, certprovider.Options{})
-		if p == nil {
-			t.Fatalf("builder.Build(%s) failed: %v", string(stableConfig.Canonical()), err)
+		p, err := buildableConfig.Build(certprovider.BuildOptions{})
+		if err != nil {
+			t.Fatalf("Build(%+v) failed: %v", buildableConfig, err)
 		}
 		providers = append(providers, p)
 	}
diff --git a/credentials/tls/certprovider/meshca/config.go b/credentials/tls/certprovider/meshca/config.go
index 38186fa84477..2800becc3db6 100644
--- a/credentials/tls/certprovider/meshca/config.go
+++ b/credentials/tls/certprovider/meshca/config.go
@@ -161,7 +161,7 @@ func pluginConfigFromJSON(data json.RawMessage) (*pluginConfig, error) {
 	return pc, nil
 }
 
-func (pc *pluginConfig) Canonical() []byte {
+func (pc *pluginConfig) canonical() []byte {
 	return []byte(fmt.Sprintf("%s:%s:%s:%s:%s:%s:%d:%s", pc.serverURI, pc.stsOpts, pc.callTimeout, pc.certLifetime, pc.certGraceTime, pc.keyType, pc.keySize, pc.location))
 }
 
diff --git a/credentials/tls/certprovider/meshca/config_test.go b/credentials/tls/certprovider/meshca/config_test.go
index 34dd9f75f558..f5e9b41747aa 100644
--- a/credentials/tls/certprovider/meshca/config_test.go
+++ b/credentials/tls/certprovider/meshca/config_test.go
@@ -160,13 +160,13 @@ func (s) TestParseConfigSuccessFullySpecified(t *testing.T) {
 	inputConfig := makeJSONConfig(t, goodConfigFullySpecified)
 	wantConfig := "test-meshca:http://test-sts:test-resource:test-audience:test-scope:test-requested-token-type:test-subject-token-path:test-subject-token-type:test-actor-token-path:test-actor-token-type:10s:24h0m0s:12h0m0s:RSA:2048:us-west1-b"
 
-	builder := newPluginBuilder()
-	gotConfig, err := builder.ParseConfig(inputConfig)
+	cfg, err := pluginConfigFromJSON(inputConfig)
 	if err != nil {
-		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
+		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", inputConfig, err)
 	}
-	if diff := cmp.Diff(wantConfig, string(gotConfig.Canonical())); diff != "" {
-		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
+	gotConfig := cfg.canonical()
+	if diff := cmp.Diff(wantConfig, string(gotConfig)); diff != "" {
+		t.Errorf("pluginConfigFromJSON(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
 	}
 }
 
@@ -248,13 +248,12 @@ func (s) TestParseConfigSuccessWithDefaults(t *testing.T) {
 		errCh <- nil
 	}()
 
-	builder := newPluginBuilder()
-	gotConfig, err := builder.ParseConfig(inputConfig)
+	cfg, err := pluginConfigFromJSON(inputConfig)
 	if err != nil {
-		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
-
+		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", inputConfig, err)
 	}
-	if diff := cmp.Diff(wantConfig, string(gotConfig.Canonical())); diff != "" {
+	gotConfig := cfg.canonical()
+	if diff := cmp.Diff(wantConfig, string(gotConfig)); diff != "" {
 		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
 	}
 
@@ -268,14 +267,9 @@ func (s) TestParseConfigSuccessWithDefaults(t *testing.T) {
 func (s) TestParseConfigFailureCases(t *testing.T) {
 	tests := []struct {
 		desc        string
-		inputConfig interface{}
+		inputConfig json.RawMessage
 		wantErr     string
 	}{
-		{
-			desc:        "bad config type",
-			inputConfig: struct{ foo string }{foo: "bar"},
-			wantErr:     "unsupported config type",
-		},
 		{
 			desc:        "invalid JSON",
 			inputConfig: json.RawMessage(`bad bad json`),
@@ -396,10 +390,9 @@ func (s) TestParseConfigFailureCases(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			builder := newPluginBuilder()
-			sc, err := builder.ParseConfig(test.inputConfig)
+			cfg, err := pluginConfigFromJSON(test.inputConfig)
 			if err == nil {
-				t.Fatalf("builder.ParseConfig(%q) = %v, expected to return error (%v)", test.inputConfig, string(sc.Canonical()), test.wantErr)
+				t.Fatalf("pluginConfigFromJSON(%q) = %v, expected to return error (%v)", test.inputConfig, string(cfg.canonical()), test.wantErr)
 
 			}
 			if !strings.Contains(err.Error(), test.wantErr) {
diff --git a/credentials/tls/certprovider/meshca/plugin.go b/credentials/tls/certprovider/meshca/plugin.go
index 5ff3e9cf2152..ab1958ac1fd0 100644
--- a/credentials/tls/certprovider/meshca/plugin.go
+++ b/credentials/tls/certprovider/meshca/plugin.go
@@ -65,12 +65,12 @@ type distributor interface {
 type providerPlugin struct {
 	distributor // Holds the key material.
 	cancel      context.CancelFunc
-	cc          *grpc.ClientConn        // Connection to MeshCA server.
-	cfg         *pluginConfig           // Plugin configuration.
-	opts        certprovider.Options    // Key material options.
-	logger      *grpclog.PrefixLogger   // Plugin instance specific prefix.
-	backoff     func(int) time.Duration // Exponential backoff.
-	doneFunc    func()                  // Notify the builder when done.
+	cc          *grpc.ClientConn          // Connection to MeshCA server.
+	cfg         *pluginConfig             // Plugin configuration.
+	opts        certprovider.BuildOptions // Key material options.
+	logger      *grpclog.PrefixLogger     // Plugin instance specific prefix.
+	backoff     func(int) time.Duration   // Exponential backoff.
+	doneFunc    func()                    // Notify the builder when done.
 }
 
 // providerParams wraps params passed to the provider plugin at creation time.
@@ -78,7 +78,7 @@ type providerParams struct {
 	// This ClientConn to the MeshCA server is owned by the builder.
 	cc       *grpc.ClientConn
 	cfg      *pluginConfig
-	opts     certprovider.Options
+	opts     certprovider.BuildOptions
 	backoff  func(int) time.Duration
 	doneFunc func()
 }
diff --git a/credentials/tls/certprovider/meshca/plugin_test.go b/credentials/tls/certprovider/meshca/plugin_test.go
index 48740c140c15..5b3f068d61e5 100644
--- a/credentials/tls/certprovider/meshca/plugin_test.go
+++ b/credentials/tls/certprovider/meshca/plugin_test.go
@@ -297,14 +297,15 @@ func (s) TestCreateCertificate(t *testing.T) {
 	e, addr, cancel := setup(t, opts{})
 	defer cancel()
 
-	// Set the MeshCA targetURI in the plugin configuration to point to our fake
-	// MeshCA.
+	// Set the MeshCA targetURI to point to our fake MeshCA.
 	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
 	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
 	inputConfig := makeJSONConfig(t, cfg)
-	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.Options{})
+
+	// Lookup MeshCA plugin builder, parse config and start the plugin.
+	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
 	if err != nil {
-		t.Fatalf("certprovider.GetProvider(%s, %s) failed: %v", pluginName, cfg, err)
+		t.Fatalf("GetProvider(%s, %s) failed: %v", pluginName, string(inputConfig), err)
 	}
 	defer prov.Close()
 
@@ -339,14 +340,15 @@ func (s) TestCreateCertificateWithBackoff(t *testing.T) {
 	e, addr, cancel := setup(t, opts{withbackoff: true})
 	defer cancel()
 
-	// Set the MeshCA targetURI in the plugin configuration to point to our fake
-	// MeshCA.
+	// Set the MeshCA targetURI to point to our fake MeshCA.
 	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
 	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
 	inputConfig := makeJSONConfig(t, cfg)
-	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.Options{})
+
+	// Lookup MeshCA plugin builder, parse config and start the plugin.
+	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
 	if err != nil {
-		t.Fatalf("certprovider.GetProvider(%s, %s) failed: %v", pluginName, cfg, err)
+		t.Fatalf("GetProvider(%s, %s) failed: %v", pluginName, string(inputConfig), err)
 	}
 	defer prov.Close()
 
@@ -394,14 +396,15 @@ func (s) TestCreateCertificateWithRefresh(t *testing.T) {
 	e, addr, cancel := setup(t, opts{withShortLife: true})
 	defer cancel()
 
-	// Set the MeshCA targetURI in the plugin configuration to point to our fake
-	// MeshCA.
+	// Set the MeshCA targetURI to point to our fake MeshCA.
 	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
 	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
 	inputConfig := makeJSONConfig(t, cfg)
-	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.Options{})
+
+	// Lookup MeshCA plugin builder, parse config and start the plugin.
+	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
 	if err != nil {
-		t.Fatalf("certprovider.GetProvider(%s, %s) failed: %v", pluginName, cfg, err)
+		t.Fatalf("GetProvider(%s, %s) failed: %v", pluginName, string(inputConfig), err)
 	}
 	defer prov.Close()
 
diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go
index 8d8ae80a053b..275c176afdf8 100644
--- a/credentials/tls/certprovider/provider.go
+++ b/credentials/tls/certprovider/provider.go
@@ -64,28 +64,14 @@ func getBuilder(name string) Builder {
 
 // Builder creates a Provider.
 type Builder interface {
-	// Build creates a new Provider and initializes it with the given config and
-	// options combination.
-	Build(StableConfig, Options) Provider
-
-	// ParseConfig converts config input in a format specific to individual
-	// implementations and returns an implementation of the StableConfig
-	// interface.
-	// Equivalent configurations must return StableConfig types whose
-	// Canonical() method returns the same output.
-	ParseConfig(interface{}) (StableConfig, error)
+	// ParseConfig parses the given config, which is in a format specific to individual
+	// implementations, and returns a BuildableConfig on success.
+	ParseConfig(interface{}) (*BuildableConfig, error)
 
 	// Name returns the name of providers built by this builder.
 	Name() string
 }
 
-// StableConfig wraps the method to return a stable Provider configuration.
-type StableConfig interface {
-	// Canonical returns Provider config as an arbitrary byte slice.
-	// Equivalent configurations must return the same output.
-	Canonical() []byte
-}
-
 // Provider makes it possible to keep channel credential implementations up to
 // date with secrets that they rely on to secure communications on the
 // underlying channel.
@@ -110,8 +96,8 @@ type KeyMaterial struct {
 	Roots *x509.CertPool
 }
 
-// Options contains configuration knobs passed to a Provider at creation time.
-type Options struct {
+// BuildOptions contains parameters passed to a Provider at build time.
+type BuildOptions struct {
 	// CertName holds the certificate name, whose key material is of interest to
 	// the caller.
 	CertName string
diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go
index 7f41fddb3bbe..90f98b3c9e71 100644
--- a/credentials/tls/certprovider/store.go
+++ b/credentials/tls/certprovider/store.go
@@ -39,7 +39,7 @@ type storeKey struct {
 	// configuration of the certificate provider in string form.
 	config string
 	// opts contains the certificate name and other keyMaterial options.
-	opts Options
+	opts BuildOptions
 }
 
 // wrappedProvider wraps a provider instance with a reference count.
@@ -59,45 +59,55 @@ type store struct {
 	providers map[storeKey]*wrappedProvider
 }
 
-// GetProvider returns a provider instance from which keyMaterial can be read.
-//
-// name is the registered name of the provider, config is the provider-specific
-// configuration, opts contains extra information that controls the keyMaterial
-// returned by the provider.
-//
-// Implementations of the Builder interface should clearly document the type of
-// configuration accepted by them.
-//
-// If a provider exists for passed arguments, its reference count is incremented
-// before returning. If no provider exists for the passed arguments, a new one
-// is created using the registered builder. If no registered builder is found,
-// or the provider configuration is rejected by it, a non-nil error is returned.
-func GetProvider(name string, config interface{}, opts Options) (Provider, error) {
-	provStore.mu.Lock()
-	defer provStore.mu.Unlock()
+// Close overrides the Close method of the embedded provider. It releases the
+// reference held by the caller on the underlying provider and if the
+// provider's reference count reaches zero, it is removed from the store, and
+// its Close method is also invoked.
+func (wp *wrappedProvider) Close() {
+	ps := wp.store
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
 
-	builder := getBuilder(name)
-	if builder == nil {
-		return nil, fmt.Errorf("no registered builder for provider name: %s", name)
+	wp.refCount--
+	if wp.refCount == 0 {
+		wp.Provider.Close()
+		delete(ps.providers, wp.storeKey)
 	}
+}
 
-	var (
-		stableConfig StableConfig
-		err          error
-	)
-	if c, ok := config.(StableConfig); ok {
-		// The config passed to the store has already been parsed.
-		stableConfig = c
-	} else {
-		stableConfig, err = builder.ParseConfig(config)
-		if err != nil {
-			return nil, err
-		}
+// BuildableConfig wraps parsed provider configuration and functionality to
+// instantiate provider instances.
+type BuildableConfig struct {
+	name    string
+	config  []byte
+	starter func(BuildOptions) Provider
+	pStore  *store
+}
+
+// NewBuildableConfig creates a new BuildableConfig with the given arguments.
+// Provider implementations are expected to invoke this function after parsing
+// the given configuration as part of their ParseConfig() method.
+// Equivalent configurations are expected to invoke this function with the same
+// config argument.
+func NewBuildableConfig(name string, config []byte, starter func(BuildOptions) Provider) *BuildableConfig {
+	return &BuildableConfig{
+		name:    name,
+		config:  config,
+		starter: starter,
+		pStore:  provStore,
 	}
+}
+
+// Build kicks off a provider instance with the wrapped configuration. Multiple
+// invocations of this method with the same opts will result in provider
+// instances being reused.
+func (bc *BuildableConfig) Build(opts BuildOptions) (Provider, error) {
+	provStore.mu.Lock()
+	defer provStore.mu.Unlock()
 
 	sk := storeKey{
-		name:   name,
-		config: string(stableConfig.Canonical()),
+		name:   bc.name,
+		config: string(bc.config),
 		opts:   opts,
 	}
 	if wp, ok := provStore.providers[sk]; ok {
@@ -105,9 +115,9 @@ func GetProvider(name string, config interface{}, opts Options) (Provider, error
 		return wp, nil
 	}
 
-	provider := builder.Build(stableConfig, opts)
+	provider := bc.starter(opts)
 	if provider == nil {
-		return nil, fmt.Errorf("certprovider.Build(%v) failed", sk)
+		return nil, fmt.Errorf("provider(%q, %q).Build(%v) failed", sk.name, sk.config, opts)
 	}
 	wp := &wrappedProvider{
 		Provider: provider,
@@ -119,18 +129,28 @@ func GetProvider(name string, config interface{}, opts Options) (Provider, error
 	return wp, nil
 }
 
-// Close overrides the Close method of the embedded provider. It releases the
-// reference held by the caller on the underlying provider and if the
-// provider's reference count reaches zero, it is removed from the store, and
-// its Close method is also invoked.
-func (wp *wrappedProvider) Close() {
-	ps := wp.store
-	ps.mu.Lock()
-	defer ps.mu.Unlock()
+// String returns the provider name and config as a colon separated string.
+func (bc *BuildableConfig) String() string {
+	return fmt.Sprintf("%s:%s", bc.name, string(bc.config))
+}
 
-	wp.refCount--
-	if wp.refCount == 0 {
-		wp.Provider.Close()
-		delete(ps.providers, wp.storeKey)
+// ParseConfig is a convenience function to create a BuildableConfig given a
+// provider name and configuration. Returns an error if there is no registered
+// builder for the given name or if the config parsing fails.
+func ParseConfig(name string, config interface{}) (*BuildableConfig, error) {
+	parser := getBuilder(name)
+	if parser == nil {
+		return nil, fmt.Errorf("no certificate provider builder found for %q", name)
+	}
+	return parser.ParseConfig(config)
+}
+
+// GetProvider is a convenience function to create a provider given the name,
+// config and build options.
+func GetProvider(name string, config interface{}, opts BuildOptions) (Provider, error) {
+	bc, err := ParseConfig(name, config)
+	if err != nil {
+		return nil, err
 	}
+	return bc.Build(opts)
 }
diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go
index 618c2d7d9b00..00d33a2be872 100644
--- a/credentials/tls/certprovider/store_test.go
+++ b/credentials/tls/certprovider/store_test.go
@@ -37,10 +37,11 @@ import (
 )
 
 const (
-	fakeProvider1Name  = "fake-certificate-provider-1"
-	fakeProvider2Name  = "fake-certificate-provider-2"
-	fakeConfig         = "my fake config"
-	defaultTestTimeout = 1 * time.Second
+	fakeProvider1Name       = "fake-certificate-provider-1"
+	fakeProvider2Name       = "fake-certificate-provider-2"
+	fakeConfig              = "my fake config"
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
 )
 
 var fpb1, fpb2 *fakeProviderBuilder
@@ -73,36 +74,36 @@ type fakeProviderBuilder struct {
 	providerChan *testutils.Channel
 }
 
-func (b *fakeProviderBuilder) Build(StableConfig, Options) Provider {
-	p := &fakeProvider{Distributor: NewDistributor()}
-	b.providerChan.Send(p)
-	return p
-}
-
-func (b *fakeProviderBuilder) ParseConfig(config interface{}) (StableConfig, error) {
+func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*BuildableConfig, error) {
 	s, ok := config.(string)
 	if !ok {
 		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
 	}
-	return &fakeStableConfig{config: s}, nil
+	return NewBuildableConfig(b.name, []byte(s), func(BuildOptions) Provider {
+		fp := &fakeProvider{
+			Distributor: NewDistributor(),
+			config:      s,
+		}
+		b.providerChan.Send(fp)
+		return fp
+	}), nil
 }
 
 func (b *fakeProviderBuilder) Name() string {
 	return b.name
 }
 
-type fakeStableConfig struct {
-	config string
-}
-
-func (c *fakeStableConfig) Canonical() []byte {
-	return []byte(c.config)
-}
-
 // fakeProvider is an implementation of the Provider interface which provides a
 // method for tests to invoke to push new key materials.
 type fakeProvider struct {
 	*Distributor
+	config string
+}
+
+func (p *fakeProvider) Start(BuildOptions) Provider {
+	// This is practically a no-op since this provider doesn't do any work which
+	// needs to be started at this point.
+	return p
 }
 
 // newKeyMaterial allows tests to push new key material to the fake provider
@@ -166,15 +167,19 @@ func compareKeyMaterial(got, want *KeyMaterial) error {
 	return nil
 }
 
+func createProvider(t *testing.T, name, config string, opts BuildOptions) Provider {
+	t.Helper()
+	prov, err := GetProvider(name, config, opts)
+	if err != nil {
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", name, config, opts, err)
+	}
+	return prov
+}
+
 // TestStoreSingleProvider creates a single provider through the store and calls
 // methods on them.
 func (s) TestStoreSingleProvider(t *testing.T) {
-	// Create a Provider through the store.
-	kmOpts := Options{CertName: "default"}
-	prov, err := GetProvider(fakeProvider1Name, fakeConfig, kmOpts)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, kmOpts, err)
-	}
+	prov := createProvider(t, fakeProvider1Name, fakeConfig, BuildOptions{CertName: "default"})
 	defer prov.Close()
 
 	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
@@ -190,7 +195,9 @@ func (s) TestStoreSingleProvider(t *testing.T) {
 	// Attempt to read from key material from the Provider returned by the
 	// store. This will fail because we have not pushed any key material into
 	// our fake provider.
-	if err := readAndVerifyKeyMaterial(ctx, prov, nil); !errors.Is(err, context.DeadlineExceeded) {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := readAndVerifyKeyMaterial(sCtx, prov, nil); !errors.Is(err, context.DeadlineExceeded) {
 		t.Fatal(err)
 	}
 
@@ -198,8 +205,6 @@ func (s) TestStoreSingleProvider(t *testing.T) {
 	// and attempt to read from the Provider returned by the store.
 	testKM1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
 	fakeProv.newKeyMaterial(testKM1, nil)
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, prov, testKM1); err != nil {
 		t.Fatal(err)
 	}
@@ -220,18 +225,14 @@ func (s) TestStoreSingleProvider(t *testing.T) {
 func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	// Create three readers on the same fake provider. Two of these readers use
 	// certName `foo`, while the third one uses certName `bar`.
-	optsFoo := Options{CertName: "foo"}
-	optsBar := Options{CertName: "bar"}
-	provFoo1, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err)
-	}
-	defer provFoo1.Close()
-	provFoo2, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err)
-	}
-	defer provFoo2.Close()
+	optsFoo := BuildOptions{CertName: "foo"}
+	provFoo1 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo)
+	provFoo2 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo)
+	defer func() {
+		provFoo1.Close()
+		provFoo2.Close()
+	}()
+
 	// Our fakeProviderBuilder pushes newly created providers on a channel.
 	// Grab the fake provider for optsFoo.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
@@ -242,11 +243,18 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	}
 	fakeProvFoo := p.(*fakeProvider)
 
-	provBar1, err := GetProvider(fakeProvider1Name, fakeConfig, optsBar)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsBar, err)
+	// Make sure only provider was created by the builder so far. The store
+	// should be able to share the providers.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fpb1.providerChan.Receive(sCtx); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("A second provider created when expected to be shared by the store")
 	}
+
+	optsBar := BuildOptions{CertName: "bar"}
+	provBar1 := createProvider(t, fakeProvider1Name, fakeConfig, optsBar)
 	defer provBar1.Close()
+
 	// Grab the fake provider for optsBar.
 	p, err = fpb1.providerChan.Receive(ctx)
 	if err != nil {
@@ -264,7 +272,9 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	if err := readAndVerifyKeyMaterial(ctx, provFoo2, fooKM); err != nil {
 		t.Fatal(err)
 	}
-	if err := readAndVerifyKeyMaterial(ctx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := readAndVerifyKeyMaterial(sCtx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) {
 		t.Fatal(err)
 	}
 
@@ -272,8 +282,6 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 	// appropriate key material.
 	barKM := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
 	fakeProvBar.newKeyMaterial(barKM, nil)
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := readAndVerifyKeyMaterial(ctx, provBar1, barKM); err != nil {
 		t.Fatal(err)
 	}
@@ -290,13 +298,11 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
 // would take place.
 func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 	// Create two providers of the same type, but with different configs.
-	opts := Options{CertName: "foo"}
+	opts := BuildOptions{CertName: "foo"}
 	cfg1 := fakeConfig + "1111"
 	cfg2 := fakeConfig + "2222"
-	prov1, err := GetProvider(fakeProvider1Name, cfg1, opts)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg1, opts, err)
-	}
+
+	prov1 := createProvider(t, fakeProvider1Name, cfg1, opts)
 	defer prov1.Close()
 	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
 	// the fake provider from that channel.
@@ -308,10 +314,7 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 	}
 	fakeProv1 := p1.(*fakeProvider)
 
-	prov2, err := GetProvider(fakeProvider1Name, cfg2, opts)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg2, opts, err)
-	}
+	prov2 := createProvider(t, fakeProvider1Name, cfg2, opts)
 	defer prov2.Close()
 	// Grab the second provider from the channel.
 	p2, err := fpb1.providerChan.Receive(ctx)
@@ -354,11 +357,8 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
 // TestStoreMultipleProviders creates providers of different types and makes
 // sure closing of one does not affect the other.
 func (s) TestStoreMultipleProviders(t *testing.T) {
-	opts := Options{CertName: "foo"}
-	prov1, err := GetProvider(fakeProvider1Name, fakeConfig, opts)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err)
-	}
+	opts := BuildOptions{CertName: "foo"}
+	prov1 := createProvider(t, fakeProvider1Name, fakeConfig, opts)
 	defer prov1.Close()
 	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
 	// the fake provider from that channel.
@@ -370,10 +370,7 @@ func (s) TestStoreMultipleProviders(t *testing.T) {
 	}
 	fakeProv1 := p1.(*fakeProvider)
 
-	prov2, err := GetProvider(fakeProvider2Name, fakeConfig, opts)
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err)
-	}
+	prov2 := createProvider(t, fakeProvider2Name, fakeConfig, opts)
 	defer prov2.Close()
 	// Grab the second provider from the channel.
 	p2, err := fpb2.providerChan.Receive(ctx)
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 55d6e8c9f0aa..ab4b78ea07d6 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -35,7 +35,6 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -61,8 +60,7 @@ var (
 		// not deal with subConns.
 		return builder.Build(cc, opts), nil
 	}
-
-	getProvider = certprovider.GetProvider
+	buildProvider = buildProviderFunc
 )
 
 func init() {
@@ -133,7 +131,7 @@ func (cdsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 // the cdsBalancer. This will be faked out in unittests.
 type xdsClientInterface interface {
 	WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func()
-	CertProviderConfigs() map[string]bootstrap.CertProviderConfig
+	CertProviderConfigs() map[string]*certprovider.BuildableConfig
 	Close()
 }
 
@@ -252,48 +250,28 @@ func (b *cdsBalancer) handleSecurityConfig(config *xdsclient.SecurityConfig) err
 	}
 
 	// A root provider is required whether we are using TLS or mTLS.
-	rootCfg, ok := cpc[config.RootInstanceName]
-	if !ok {
-		return fmt.Errorf("certificate provider instance %q not found in bootstrap file", config.RootInstanceName)
-	}
-	rootProvider, err := getProvider(rootCfg.Name, rootCfg.Config, certprovider.Options{
-		CertName: config.RootCertName,
-		WantRoot: true,
-	})
+	rootProvider, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
 	if err != nil {
-		// This error is not expected since the bootstrap process parses the
-		// config and makes sure that it is acceptable to the plugin. Still, it
-		// is possible that the plugin parses the config successfully, but its
-		// Build() method errors out.
-		return fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", rootCfg, err)
-	}
-	if b.cachedRoot != nil {
-		b.cachedRoot.Close()
+		return err
 	}
 
 	// The identity provider is only present when using mTLS.
 	var identityProvider certprovider.Provider
-	if name := config.IdentityInstanceName; name != "" {
-		identityCfg := cpc[name]
-		if !ok {
-			return fmt.Errorf("certificate provider instance %q not found in bootstrap file", config.IdentityInstanceName)
-		}
-		identityProvider, err = getProvider(identityCfg.Name, identityCfg.Config, certprovider.Options{
-			CertName:     config.IdentityCertName,
-			WantIdentity: true,
-		})
+	if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" {
+		var err error
+		identityProvider, err = buildProvider(cpc, name, cert, true, false)
 		if err != nil {
-			// This error is not expected since the bootstrap process parses the
-			// config and makes sure that it is acceptable to the plugin. Still,
-			// it is possible that the plugin parses the config successfully,
-			// but its Build() method errors out.
-			return fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", identityCfg, err)
+			return err
 		}
 	}
+
+	// Close the old providers and cache the new ones.
+	if b.cachedRoot != nil {
+		b.cachedRoot.Close()
+	}
 	if b.cachedIdentity != nil {
 		b.cachedIdentity.Close()
 	}
-
 	b.cachedRoot = rootProvider
 	b.cachedIdentity = identityProvider
 
@@ -305,6 +283,26 @@ func (b *cdsBalancer) handleSecurityConfig(config *xdsclient.SecurityConfig) err
 	return nil
 }
 
+func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
+	cfg, ok := configs[instanceName]
+	if !ok {
+		return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName)
+	}
+	provider, err := cfg.Build(certprovider.BuildOptions{
+		CertName:     certName,
+		WantIdentity: wantIdentity,
+		WantRoot:     wantRoot,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err)
+	}
+	return provider, nil
+}
+
 // handleWatchUpdate handles a watch update from the xDS Client. Good updates
 // lead to clientConn updates being invoked on the underlying edsBalancer.
 func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 15c097d3f096..a3f923dfa684 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -31,7 +31,6 @@ import (
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
@@ -44,16 +43,7 @@ const (
 
 var (
 	fpb1, fpb2                   *fakeProviderBuilder
-	bootstrapCertProviderConfigs = map[string]bootstrap.CertProviderConfig{
-		"default1": {
-			Name:   fakeProvider1Name,
-			Config: &fakeStableConfig{config: fakeConfig + "1111"},
-		},
-		"default2": {
-			Name:   fakeProvider2Name,
-			Config: &fakeStableConfig{config: fakeConfig + "2222"},
-		},
-	}
+	bootstrapCertProviderConfigs map[string]*certprovider.BuildableConfig
 	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
 		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
@@ -72,6 +62,12 @@ var (
 func init() {
 	fpb1 = &fakeProviderBuilder{name: fakeProvider1Name}
 	fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
+	cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111")
+	cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222")
+	bootstrapCertProviderConfigs = map[string]*certprovider.BuildableConfig{
+		"default1": cfg1,
+		"default2": cfg2,
+	}
 	certprovider.Register(fpb1)
 	certprovider.Register(fpb2)
 }
@@ -82,40 +78,38 @@ type fakeProviderBuilder struct {
 	name string
 }
 
-func (b *fakeProviderBuilder) Build(certprovider.StableConfig, certprovider.Options) certprovider.Provider {
-	p := &fakeProvider{}
-	return p
-}
-
-func (b *fakeProviderBuilder) ParseConfig(config interface{}) (certprovider.StableConfig, error) {
+func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.BuildableConfig, error) {
 	s, ok := config.(string)
 	if !ok {
 		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
 	}
-	return &fakeStableConfig{config: s}, nil
+	return certprovider.NewBuildableConfig(b.name, []byte(s), func(certprovider.BuildOptions) certprovider.Provider {
+		return &fakeProvider{
+			Distributor: certprovider.NewDistributor(),
+			config:      s,
+		}
+	}), nil
 }
 
 func (b *fakeProviderBuilder) Name() string {
 	return b.name
 }
 
-type fakeStableConfig struct {
-	config string
-}
-
-func (c *fakeStableConfig) Canonical() []byte {
-	return []byte(c.config)
-}
-
 // fakeProvider is an implementation of the Provider interface which provides a
 // method for tests to invoke to push new key materials.
 type fakeProvider struct {
-	certprovider.Provider
+	*certprovider.Distributor
+	config string
+}
+
+// Close helps implement the Provider interface.
+func (p *fakeProvider) Close() {
+	p.Distributor.Stop()
 }
 
 // setupWithXDSCreds performs all the setup steps required for tests which use
 // xDSCredentials.
-func setupWithXDSCreds(t *testing.T, storeErr bool) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, *testutils.Channel, func()) {
+func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
 	t.Helper()
 
 	builder := balancer.Get(cdsName)
@@ -162,25 +156,8 @@ func setupWithXDSCreds(t *testing.T, storeErr bool) (*fakeclient.Client, *cdsBal
 		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName)
 	}
 
-	// Override the certificate provider creation function to get notified about
-	// provider creation. Create a channel with size 2, so we have buffer to
-	// push notifications for both providers.
-	providerCh := testutils.NewChannelWithSize(2)
-	origGetProviderFunc := getProvider
-	if storeErr {
-		getProvider = func(string, interface{}, certprovider.Options) (certprovider.Provider, error) {
-			return nil, errors.New("certprovider.Store failed to created provider")
-		}
-	} else {
-		getProvider = func(name string, cfg interface{}, opts certprovider.Options) (certprovider.Provider, error) {
-			providerCh.Send(nil)
-			return origGetProviderFunc(name, cfg, opts)
-		}
-	}
-
-	return xdsC, cdsB.(*cdsBalancer), edsB, tcc, providerCh, func() {
+	return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() {
 		newEDSBalancer = oldEDSBalancerBuilder
-		getProvider = origGetProviderFunc
 	}
 }
 
@@ -223,16 +200,6 @@ func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xd
 // the address attributes added as part of the intercepted NewSubConn() method
 // indicate the use of fallback credentials.
 func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
-	// Override the certificate provider creation function to get notified about
-	// provider creation.
-	providerCh := testutils.NewChannel()
-	origGetProviderFunc := getProvider
-	getProvider = func(name string, cfg interface{}, opts certprovider.Options) (certprovider.Provider, error) {
-		providerCh.Send(nil)
-		return origGetProviderFunc(name, cfg, opts)
-	}
-	defer func() { getProvider = origGetProviderFunc }()
-
 	// This creates a CDS balancer, pushes a ClientConnState update with a fake
 	// xdsClient, and makes sure that the CDS balancer registers a watch on the
 	// provided xdsClient.
@@ -242,6 +209,17 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 		cdsB.Close()
 	}()
 
+	// Override the provider builder function to push on a channel. We do not
+	// expect this function to be called as part of this test.
+	providerCh := testutils.NewChannel()
+	origBuildProvider := buildProvider
+	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
+		p, err := origBuildProvider(c, id, cert, wi, wr)
+		providerCh.Send(nil)
+		return p, err
+	}
+	defer func() { buildProvider = origBuildProvider }()
+
 	// Here we invoke the watch callback registered on the fake xdsClient. This
 	// will trigger the watch handler on the CDS balancer, which will attempt to
 	// create a new EDS balancer. The fake EDS balancer created above will be
@@ -255,7 +233,9 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Make a NewSubConn and verify that attributes are not added.
+	// Make a NewSubConn and verify that the HandshakeInfo does not contain any
+	// certificate providers, forcing the credentials implementation to use
+	// fallback creds.
 	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
 		t.Fatal(err)
 	}
@@ -278,12 +258,23 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
+	// Override the provider builder function to push on a channel. We do not
+	// expect this function to be called as part of this test.
+	providerCh := testutils.NewChannel()
+	origBuildProvider := buildProvider
+	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
+		p, err := origBuildProvider(c, id, cert, wi, wr)
+		providerCh.Send(nil)
+		return p, err
+	}
+	defer func() { buildProvider = origBuildProvider }()
+
 	// Here we invoke the watch callback registered on the fake xdsClient. This
 	// will trigger the watch handler on the CDS balancer, which will attempt to
 	// create a new EDS balancer. The fake EDS balancer created above will be
@@ -298,7 +289,9 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Make a NewSubConn and verify that attributes are not added.
+	// Make a NewSubConn and verify that the HandshakeInfo does not contain any
+	// certificate providers, forcing the credentials implementation to use
+	// fallback creds.
 	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
 		t.Fatal(err)
 	}
@@ -325,7 +318,7 @@ func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
 		// This creates a CDS balancer which uses xdsCredentials, pushes a
 		// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 		// balancer registers a watch on the provided xdsClient.
-		xdsC, cdsB, edsB, tcc, _, cancel := setupWithXDSCreds(t, false)
+		xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 		defer func() {
 			cancel()
 			cdsB.Close()
@@ -366,12 +359,19 @@ func (s) TestCertproviderStoreError(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, _, cancel := setupWithXDSCreds(t, true)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
+	// Override the provider builder function to return an error.
+	origBuildProvider := buildProvider
+	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
+		return nil, errors.New("certprovider store error")
+	}
+	defer func() { buildProvider = origBuildProvider }()
+
 	// Set the bootstrap config used by the fake client.
 	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
 
@@ -402,7 +402,7 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -442,12 +442,6 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
-	// Make sure two certificate providers are created.
-	for i := 0; i < 2; i++ {
-		if _, err := providerCh.Receive(ctx); err != nil {
-			t.Fatalf("Failed to create certificate provider upon receipt of security config")
-		}
-	}
 
 	// Make a NewSubConn and verify that attributes are added.
 	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
@@ -464,7 +458,7 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -484,12 +478,6 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
-	// Make sure two certificate providers are created.
-	for i := 0; i < 2; i++ {
-		if _, err := providerCh.Receive(ctx); err != nil {
-			t.Fatalf("Failed to create certificate provider upon receipt of security config")
-		}
-	}
 
 	// Make a NewSubConn and verify that attributes are added.
 	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
@@ -501,7 +489,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -521,12 +509,6 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
-	// Make sure two certificate providers are created.
-	for i := 0; i < 2; i++ {
-		if _, err := providerCh.Receive(ctx); err != nil {
-			t.Fatalf("Failed to create certificate provider upon receipt of security config")
-		}
-	}
 
 	// Make a NewSubConn and verify that attributes are added.
 	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
@@ -557,7 +539,7 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
@@ -577,12 +559,6 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
-	// Make sure two certificate providers are created.
-	for i := 0; i < 2; i++ {
-		if _, err := providerCh.Receive(ctx); err != nil {
-			t.Fatalf("Failed to create certificate provider upon receipt of security config")
-		}
-	}
 
 	// Make a NewSubConn and verify that attributes are added.
 	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
@@ -624,12 +600,22 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	// This creates a CDS balancer which uses xdsCredentials, pushes a
 	// ClientConnState update with a fake xdsClient, and makes sure that the CDS
 	// balancer registers a watch on the provided xdsClient.
-	xdsC, cdsB, edsB, tcc, providerCh, cancel := setupWithXDSCreds(t, false)
+	xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
 	defer func() {
 		cancel()
 		cdsB.Close()
 	}()
 
+	// Override the provider builder function to push on a channel.
+	providerCh := testutils.NewChannel()
+	origBuildProvider := buildProvider
+	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
+		p, err := origBuildProvider(c, id, cert, wi, wr)
+		providerCh.Send(nil)
+		return p, err
+	}
+	defer func() { buildProvider = origBuildProvider }()
+
 	// Set the bootstrap config used by the fake client.
 	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
 
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 0ef3809496f9..789d1d0e45ad 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -75,18 +75,9 @@ type Config struct {
 	// NodeProto contains the Node proto to be used in xDS requests. The actual
 	// type depends on the transport protocol version used.
 	NodeProto proto.Message
-	// CertProviderConfigs contain parsed configs for supported certificate
-	// provider plugins found in the bootstrap file.
-	CertProviderConfigs map[string]CertProviderConfig
-}
-
-// CertProviderConfig wraps the certificate provider plugin name and config
-// (corresponding to one plugin instance) found in the bootstrap file.
-type CertProviderConfig struct {
-	// Name is the registered name of the certificate provider.
-	Name string
-	// Config is the parsed config to be passed to the certificate provider.
-	Config certprovider.StableConfig
+	// CertProviderConfigs contains a mapping from certificate provider plugin
+	// instance names to parsed buildable configs.
+	CertProviderConfigs map[string]*certprovider.BuildableConfig
 }
 
 type channelCreds struct {
@@ -207,7 +198,7 @@ func NewConfig() (*Config, error) {
 			if err := json.Unmarshal(v, &providerInstances); err != nil {
 				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
 			}
-			configs := make(map[string]CertProviderConfig)
+			configs := make(map[string]*certprovider.BuildableConfig)
 			getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
 			for instance, data := range providerInstances {
 				var nameAndConfig struct {
@@ -224,15 +215,11 @@ func NewConfig() (*Config, error) {
 					// We ignore plugins that we do not know about.
 					continue
 				}
-				cfg := nameAndConfig.Config
-				c, err := parser.ParseConfig(cfg)
+				bc, err := parser.ParseConfig(nameAndConfig.Config)
 				if err != nil {
 					return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
 				}
-				configs[instance] = CertProviderConfig{
-					Name:   name,
-					Config: c,
-				}
+				configs[instance] = bc
 			}
 			config.CertProviderConfigs = configs
 		}
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 6ee68c346c45..1b9decac185a 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -251,10 +251,10 @@ func (c *Config) compare(want *Config) error {
 	for instance, gotCfg := range gotCfgs {
 		wantCfg, ok := wantCfgs[instance]
 		if !ok {
-			return fmt.Errorf("config.CertProviderConfigs has unexpected plugin instance %q with config %q", instance, string(gotCfg.Config.Canonical()))
+			return fmt.Errorf("config.CertProviderConfigs has unexpected plugin instance %q with config %q", instance, gotCfg.String())
 		}
-		if gotCfg.Name != wantCfg.Name || !cmp.Equal(gotCfg.Config.Canonical(), wantCfg.Config.Canonical()) {
-			return fmt.Errorf("config.CertProviderConfigs for plugin instance %q has config {%s, %s, want {%s, %s}", instance, gotCfg.Name, string(gotCfg.Config.Canonical()), wantCfg.Name, string(wantCfg.Config.Canonical()))
+		if got, want := gotCfg.String(), wantCfg.String(); got != want {
+			return fmt.Errorf("config.CertProviderConfigs for plugin instance %q has config %q, want %q", instance, got, want)
 		}
 	}
 	return nil
@@ -489,13 +489,9 @@ const fakeCertProviderName = "fake-certificate-provider"
 // interprets the config provided to it as JSON with a single key and value.
 type fakeCertProviderBuilder struct{}
 
-func (b *fakeCertProviderBuilder) Build(certprovider.StableConfig, certprovider.Options) certprovider.Provider {
-	return &fakeCertProvider{}
-}
-
 // ParseConfig expects input in JSON format containing a map from string to
 // string, with a single entry and mapKey being "configKey".
-func (b *fakeCertProviderBuilder) ParseConfig(cfg interface{}) (certprovider.StableConfig, error) {
+func (b *fakeCertProviderBuilder) ParseConfig(cfg interface{}) (*certprovider.BuildableConfig, error) {
 	config, ok := cfg.(json.RawMessage)
 	if !ok {
 		return nil, fmt.Errorf("fakeCertProviderBuilder received config of type %T, want []byte", config)
@@ -507,7 +503,10 @@ func (b *fakeCertProviderBuilder) ParseConfig(cfg interface{}) (certprovider.Sta
 	if len(cfgData) != 1 || cfgData["configKey"] == "" {
 		return nil, errors.New("fakeCertProviderBuilder received invalid config")
 	}
-	return &fakeStableConfig{config: cfgData}, nil
+	fc := &fakeStableConfig{config: cfgData}
+	return certprovider.NewBuildableConfig(fakeCertProviderName, fc.canonical(), func(certprovider.BuildOptions) certprovider.Provider {
+		return &fakeCertProvider{}
+	}), nil
 }
 
 func (b *fakeCertProviderBuilder) Name() string {
@@ -518,7 +517,7 @@ type fakeStableConfig struct {
 	config map[string]string
 }
 
-func (c *fakeStableConfig) Canonical() []byte {
+func (c *fakeStableConfig) canonical() []byte {
 	var cfg string
 	for k, v := range c.config {
 		cfg = fmt.Sprintf("%s:%s", k, v)
@@ -652,11 +651,8 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 		Creds:        grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
 		TransportAPI: version.TransportV3,
 		NodeProto:    v3NodeProto,
-		CertProviderConfigs: map[string]CertProviderConfig{
-			"fakeProviderInstance": {
-				Name:   fakeCertProviderName,
-				Config: wantCfg,
-			},
+		CertProviderConfigs: map[string]*certprovider.BuildableConfig{
+			"fakeProviderInstance": wantCfg,
 		},
 	}
 	tests := []struct {
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 583f09baf1fd..5497fbd97941 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -30,6 +30,7 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/xds/internal/client/load"
 
 	"google.golang.org/grpc"
@@ -422,10 +423,9 @@ func New(opts Options) (*Client, error) {
 }
 
 // CertProviderConfigs returns the certificate provider configuration from the
-// "certificate_providers" field of the bootstrap file. The returned value is a
-// map from plugin_instance_name to {plugin_name, plugin_config}. Callers must
-// not modify the returned map.
-func (c *Client) CertProviderConfigs() map[string]bootstrap.CertProviderConfig {
+// "certificate_providers" field of the bootstrap file. The key in the returned
+// map is the plugin_instance_name. Callers must not modify the returned map.
+func (c *Client) CertProviderConfigs() map[string]*certprovider.BuildableConfig {
 	return c.opts.Config.CertProviderConfigs
 }
 
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 6496d1326d11..c540b57392aa 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -22,9 +22,9 @@ package fakeclient
 import (
 	"context"
 
+	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
@@ -43,7 +43,7 @@ type Client struct {
 	loadReportCh *testutils.Channel
 	closeCh      *testutils.Channel
 	loadStore    *load.Store
-	certConfigs  map[string]bootstrap.CertProviderConfig
+	certConfigs  map[string]*certprovider.BuildableConfig
 
 	ldsCb func(xdsclient.ListenerUpdate, error)
 	rdsCb func(xdsclient.RouteConfigUpdate, error)
@@ -224,12 +224,12 @@ func (xdsC *Client) WaitForClose(ctx context.Context) error {
 }
 
 // CertProviderConfigs returns the configured certificate provider configs.
-func (xdsC *Client) CertProviderConfigs() map[string]bootstrap.CertProviderConfig {
+func (xdsC *Client) CertProviderConfigs() map[string]*certprovider.BuildableConfig {
 	return xdsC.certConfigs
 }
 
 // SetCertProviderConfigs updates the certificate provider configs.
-func (xdsC *Client) SetCertProviderConfigs(configs map[string]bootstrap.CertProviderConfig) {
+func (xdsC *Client) SetCertProviderConfigs(configs map[string]*certprovider.BuildableConfig) {
 	xdsC.certConfigs = configs
 }
 

From 9c2f82d9a79cdc362a2361d4e64495e1a8df6113 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 6 Nov 2020 11:25:19 -0800
Subject: [PATCH 267/481] xds: Server implementation (#3999)

* xds: Server implementation.

* Remove security related code.

* Add a blocking newListenerWrapper() method.

* Fix some comments.

* Use non-blocking dial.

* Use WaitForReady().

* Use localhost instead of 127.0.0.1

* Another attempt to make the tests happy on GA.

* Make vet happy.

* Add a missing return.
---
 .../test/xds_server_integration_test.go       | 193 +++++++++
 xds/server.go                                 | 346 ++++++++++++++++
 xds/server_test.go                            | 391 ++++++++++++++++++
 xds/xds.go                                    |   8 +-
 4 files changed, 936 insertions(+), 2 deletions(-)
 create mode 100644 xds/internal/test/xds_server_integration_test.go
 create mode 100644 xds/server.go
 create mode 100644 xds/server_test.go

diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
new file mode 100644
index 000000000000..5534d311eb81
--- /dev/null
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -0,0 +1,193 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds_test contains e2e tests for xDS use on the server.
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v2discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpctest"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+	"google.golang.org/grpc/xds"
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+const (
+	defaultTestTimeout = 10 * time.Second
+	localAddress       = "localhost:9999"
+	listenerName       = "grpc/server?udpa.resource.listening_address=localhost:9999"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func setupListenerResponse(respCh chan *fakeserver.Response, name string) {
+	respCh <- &fakeserver.Response{
+		Resp: &v2discoverypb.DiscoveryResponse{
+			Resources: []*anypb.Any{
+				{
+					TypeUrl: version.V2ListenerURL,
+					Value: func() []byte {
+						l := &v3listenerpb.Listener{
+							// This needs to match the name we are querying for.
+							Name: listenerName,
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V2HTTPConnManagerURL,
+									Value: func() []byte {
+										cm := &v3httppb.HttpConnectionManager{
+											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+												Rds: &v3httppb.Rds{
+													ConfigSource: &v3corepb.ConfigSource{
+														ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+													},
+													RouteConfigName: "route-config",
+												},
+											},
+										}
+										mcm, _ := proto.Marshal(cm)
+										return mcm
+									}(),
+								},
+							},
+						}
+						ml, _ := proto.Marshal(l)
+						return ml
+					}(),
+				},
+			},
+			TypeUrl: version.V2ListenerURL,
+		},
+	}
+}
+
+func setupBootstrapFile(t *testing.T, serverURI string) func() {
+	// Create a bootstrap file in a temporary directory.
+	tmpdir, err := ioutil.TempDir("", "xds-server-test*")
+	if err != nil {
+		t.Fatalf("failed to create tempdir: %v", err)
+	}
+	bootstrapContents := fmt.Sprintf(`
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "%s",
+				"channel_creds": [
+					{ "type": "insecure" }
+				]
+			}]
+		}`, serverURI)
+	bootstrapFileName := path.Join(tmpdir, "bootstrap")
+	if err := ioutil.WriteFile(bootstrapFileName, []byte(bootstrapContents), os.ModePerm); err != nil {
+		t.Fatalf("failed to write bootstrap file: %v", err)
+	}
+
+	origBootstrapFileName := env.BootstrapFileName
+	env.BootstrapFileName = bootstrapFileName
+	t.Logf("Create bootstrap file at %s with contents\n%s", bootstrapFileName, bootstrapContents)
+	return func() { env.BootstrapFileName = origBootstrapFileName }
+}
+
+// TestServerSideXDS is an e2e tests for xDS use on the server. This does not
+// use any xDS features because we have not implemented any on the server side.
+func (s) TestServerSideXDS(t *testing.T) {
+	// Spin up a fake xDS management server on a local port.
+	// TODO(easwars): Switch to using the server from envoy-go-control-plane.
+	fs, cleanup, err := fakeserver.StartServer()
+	if err != nil {
+		t.Fatalf("failed to start fake xDS server: %v", err)
+	}
+	defer cleanup()
+	t.Logf("Started xDS management server at %s", fs.Address)
+
+	// Setup the fakeserver to respond with a Listener resource.
+	setupListenerResponse(fs.XDSResponseChan, listenerName)
+	// Create a bootstrap file in a temporary directory.
+	defer setupBootstrapFile(t, fs.Address)()
+
+	// Initialize a gRPC server which uses xDS, and register stubServer on it.
+	server := xds.NewGRPCServer()
+	testpb.RegisterTestServiceServer(server, &testService{})
+
+	errCh := make(chan error, 1)
+	go func() {
+		defer server.Stop()
+
+		// Create a clientconn and make a successful RPC
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		cc, err := grpc.DialContext(ctx, localAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+		if err != nil {
+			errCh <- fmt.Errorf("failed to dial local test server: %v", err)
+			return
+		}
+		defer cc.Close()
+
+		client := testpb.NewTestServiceClient(cc)
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+			errCh <- fmt.Errorf("rpc EmptyCall() failed: %v", err)
+			return
+		}
+		errCh <- nil
+	}()
+
+	opts := xds.ServeOptions{Network: "tcp", Address: localAddress}
+	if err := server.Serve(opts); err != nil {
+		t.Fatalf("Serve(%+v) failed: %v", opts, err)
+	}
+
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+}
+
+type testService struct {
+	testpb.TestServiceServer
+}
+
+func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	return &testpb.Empty{}, nil
+}
diff --git a/xds/server.go b/xds/server.go
new file mode 100644
index 000000000000..0b85527ce96f
--- /dev/null
+++ b/xds/server.go
@@ -0,0 +1,346 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+)
+
+const (
+	serverPrefix = "[xds-server %p] "
+
+	// The resource_name in the LDS request sent by the xDS-enabled gRPC server
+	// is of this format where the formatting directive at the end is replaced
+	// with the IP:Port specified by the user application.
+	listenerResourceNameFormat = "grpc/server?udpa.resource.listening_address=%s"
+)
+
+var (
+	// These new functions will be overridden in unit tests.
+	newXDSClient = func(opts xdsclient.Options) (xdsClientInterface, error) {
+		return xdsclient.New(opts)
+	}
+	newXDSConfig  = bootstrap.NewConfig
+	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface {
+		return grpc.NewServer(opts...)
+	}
+
+	logger = grpclog.Component("xds")
+)
+
+func prefixLogger(p *GRPCServer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(serverPrefix, p))
+}
+
+// xdsClientInterface contains methods from xdsClient.Client which are used by
+// the server. This is useful for overriding in unit tests.
+type xdsClientInterface interface {
+	WatchListener(string, func(xdsclient.ListenerUpdate, error)) func()
+	Close()
+}
+
+// grpcServerInterface contains methods from grpc.Server which are used by the
+// GRPCServer type here. This is useful for overriding in unit tests.
+type grpcServerInterface interface {
+	RegisterService(*grpc.ServiceDesc, interface{})
+	Serve(net.Listener) error
+	Stop()
+	GracefulStop()
+}
+
+// ServeOptions contains parameters to configure the Serve() method.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
+type ServeOptions struct {
+	// Network identifies the local network to listen on. The network must be
+	// "tcp", "tcp4", "tcp6".
+	Network string
+	// Address contains the local address to listen on. This should be of the
+	// form "host:port", where the host must be a literal IP address, and port
+	// must be a literal port number. If the host is a literal IPv6 address it
+	// must be enclosed in square brackets, as in "[2001:db8::1]:80. The host
+	// portion can be left unspecified.
+	Address string
+}
+
+func (so *ServeOptions) validate() error {
+	if net := so.Network; net != "tcp" && net != "tcp4" && net != "tcp6" {
+		return fmt.Errorf("xds: unsupported network type %q for server listener", net)
+	}
+	if _, _, err := net.SplitHostPort(so.Address); err != nil {
+		return fmt.Errorf("xds: unsupported address %q for server listener", so.Address)
+	}
+	return nil
+}
+
+// GRPCServer wraps a gRPC server and provides server-side xDS functionality, by
+// communication with a management server using xDS APIs. It implements the
+// grpc.ServiceRegistrar interface and can be passed to service registration
+// functions in IDL generated code.
+//
+// Experimental
+//
+// Notice: This type is EXPERIMENTAL and may be changed or removed in a
+// later release.
+type GRPCServer struct {
+	gs     grpcServerInterface
+	quit   *grpcsync.Event
+	logger *internalgrpclog.PrefixLogger
+
+	// clientMu is used only in initXDSClient(), which is called at the
+	// beginning of Serve(), where we have to decide if we have to create a
+	// client or use an existing one.
+	clientMu sync.Mutex
+	xdsC     xdsClientInterface
+}
+
+// NewGRPCServer creates an xDS-enabled gRPC server using the passed in opts.
+// The underlying gRPC server has no service registered and has not started to
+// accept requests yet.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a later
+// release.
+func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer {
+	newOpts := []grpc.ServerOption{
+		grpc.ChainUnaryInterceptor(xdsUnaryInterceptor),
+		grpc.ChainStreamInterceptor(xdsStreamInterceptor),
+	}
+	newOpts = append(newOpts, opts...)
+	s := &GRPCServer{
+		gs:   newGRPCServer(newOpts...),
+		quit: grpcsync.NewEvent(),
+	}
+	s.logger = prefixLogger(s)
+	s.logger.Infof("Created xds.GRPCServer")
+	return s
+}
+
+// RegisterService registers a service and its implementation to the underlying
+// gRPC server. It is called from the IDL generated code. This must be called
+// before invoking Serve.
+func (s *GRPCServer) RegisterService(sd *grpc.ServiceDesc, ss interface{}) {
+	s.gs.RegisterService(sd, ss)
+}
+
+// initXDSClient creates a new xdsClient if there is no existing one available.
+func (s *GRPCServer) initXDSClient() error {
+	s.clientMu.Lock()
+	defer s.clientMu.Unlock()
+
+	if s.xdsC != nil {
+		return nil
+	}
+
+	// Read the bootstrap file as part of initializing the xdsClient.
+	config, err := newXDSConfig()
+	if err != nil {
+		return fmt.Errorf("xds: failed to read bootstrap file: %v", err)
+	}
+	client, err := newXDSClient(xdsclient.Options{Config: *config})
+	if err != nil {
+		return fmt.Errorf("xds: failed to create xds-client: %v", err)
+	}
+	s.xdsC = client
+	s.logger.Infof("Created an xdsClient")
+	return nil
+}
+
+// Serve gets the underlying gRPC server to accept incoming connections on the
+// listening address in opts. A connection to the management server, to receive
+// xDS configuration, is initiated here.
+//
+// Serve will return a non-nil error unless Stop or GracefulStop is called.
+func (s *GRPCServer) Serve(opts ServeOptions) error {
+	s.logger.Infof("Serve() called with options: %+v", opts)
+
+	// Validate the listening address in opts.
+	if err := opts.validate(); err != nil {
+		return err
+	}
+
+	// If this is the first time Serve() is being called, we need to initialize
+	// our xdsClient. If not, we can use the existing one.
+	if err := s.initXDSClient(); err != nil {
+		return err
+	}
+	lw, err := s.newListenerWrapper(opts)
+	if lw == nil {
+		// Error returned can be nil (when Stop/GracefulStop() is called). So,
+		// we need to check the returned listenerWrapper instead.
+		return err
+	}
+	return s.gs.Serve(lw)
+}
+
+// newListenerWrapper starts a net.Listener on the address specified in opts. It
+// then registers a watch for a Listener resource and blocks until a good
+// response is received or the server is stopped by a call to
+// Stop/GracefulStop().
+//
+// Returns a listenerWrapper, which implements the net.Listener interface, that
+// can be passed to grpcServer.Serve().
+func (s *GRPCServer) newListenerWrapper(opts ServeOptions) (*listenerWrapper, error) {
+	lis, err := net.Listen(opts.Network, opts.Address)
+	if err != nil {
+		return nil, fmt.Errorf("xds: failed to listen on %+v: %v", opts, err)
+	}
+	lw := &listenerWrapper{Listener: lis}
+	s.logger.Infof("Started a net.Listener on %s", lis.Addr().String())
+
+	// This is used to notify that a good update has been received and that
+	// Serve() can be invoked on the underlying gRPC server. Using a
+	// grpcsync.Event instead of a vanilla channel simplifies the update handler
+	// as it need not keep track of whether the received update is the first one
+	// or not.
+	goodUpdate := grpcsync.NewEvent()
+
+	// Register an LDS watch using our xdsClient, and specify the listening
+	// address as the resource name.
+	// TODO(easwars): Check if literal IPv6 addresses need an enclosing [].
+	name := fmt.Sprintf(listenerResourceNameFormat, opts.Address)
+	cancelWatch := s.xdsC.WatchListener(name, func(update xdsclient.ListenerUpdate, err error) {
+		if err != nil {
+			// We simply log an error here and hope we get a successful update
+			// in the future. The error could be because of a timeout or an
+			// actual error, like the requested resource not found. In any case,
+			// it is fine for the server to hang indefinitely until Stop() is
+			// called.
+			s.logger.Warningf("Received error for resource %q: %+v", name, err)
+			return
+		}
+
+		s.logger.Infof("Received update for resource %q: %+v", name, update)
+
+		// TODO(easwars): Handle security configuration, create appropriate
+		// certificate providers and update the listenerWrapper before firing
+		// the event. Errors encountered during any of these steps should result
+		// in an early exit, and the update event should not fire.
+		goodUpdate.Fire()
+	})
+
+	s.logger.Infof("Watch started on resource name %v", name)
+	lw.cancelWatch = func() {
+		cancelWatch()
+		s.logger.Infof("Watch cancelled on resource name %v", name)
+	}
+
+	// Block until a good LDS response is received or the server is stopped.
+	select {
+	case <-s.quit.Done():
+		// Since the listener has not yet been handed over to gs.Serve(), we
+		// need to explicitly close the listener. Cancellation of the xDS watch
+		// is handled by the listenerWrapper.
+		lw.Close()
+		return nil, nil
+	case <-goodUpdate.Done():
+	}
+	return lw, nil
+}
+
+// Stop stops the underlying gRPC server. It immediately closes all open
+// connections. It cancels all active RPCs on the server side and the
+// corresponding pending RPCs on the client side will get notified by connection
+// errors.
+func (s *GRPCServer) Stop() {
+	s.quit.Fire()
+	s.gs.Stop()
+	if s.xdsC != nil {
+		s.xdsC.Close()
+	}
+}
+
+// GracefulStop stops the underlying gRPC server gracefully. It stops the server
+// from accepting new connections and RPCs and blocks until all the pending RPCs
+// are finished.
+func (s *GRPCServer) GracefulStop() {
+	s.quit.Fire()
+	s.gs.GracefulStop()
+	if s.xdsC != nil {
+		s.xdsC.Close()
+	}
+}
+
+// xdsUnaryInterceptor is the unary interceptor added to the gRPC server to
+// perform any xDS specific functionality on unary RPCs.
+//
+// This is a no-op at this point.
+func xdsUnaryInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+	return handler(ctx, req)
+}
+
+// xdsStreamInterceptor is the stream interceptor added to the gRPC server to
+// perform any xDS specific functionality on streaming RPCs.
+//
+// This is a no-op at this point.
+func xdsStreamInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	return handler(srv, ss)
+}
+
+// listenerWrapper wraps the net.Listener associated with the listening address
+// passed to Serve(). It also contains all other state associated with this
+// particular invocation of Serve().
+type listenerWrapper struct {
+	net.Listener
+	cancelWatch func()
+
+	// TODO(easwars): Add fields for certificate providers.
+}
+
+// Accept blocks on an Accept() on the underlying listener, and wraps the
+// returned net.Conn with the configured certificate providers.
+func (l *listenerWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	return &conn{Conn: c}, nil
+}
+
+// Close closes the underlying listener. It also cancels the xDS watch
+// registered in Serve() and closes any certificate provider instances created
+// based on security configuration received in the LDS response.
+func (l *listenerWrapper) Close() error {
+	l.Listener.Close()
+	if l.cancelWatch != nil {
+		l.cancelWatch()
+	}
+	return nil
+}
+
+// conn is a thin wrapper around a net.Conn returned by Accept().
+type conn struct {
+	net.Conn
+
+	// TODO(easwars): Add fields for certificate providers.
+}
diff --git a/xds/server_test.go b/xds/server_test.go
new file mode 100644
index 000000000000..e95316236979
--- /dev/null
+++ b/xds/server_test.go
@@ -0,0 +1,391 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"context"
+	"errors"
+	"net"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestServeOptions_Validate(t *testing.T) {
+	tests := []struct {
+		desc    string
+		opts    ServeOptions
+		wantErr bool
+	}{
+		{
+			desc:    "empty options",
+			opts:    ServeOptions{},
+			wantErr: true,
+		},
+		{
+			desc:    "unsupported network",
+			opts:    ServeOptions{Network: "foo"},
+			wantErr: true,
+		},
+		{
+			desc:    "bad address",
+			opts:    ServeOptions{Network: "tcp", Address: "I'm a bad IP address"},
+			wantErr: true,
+		},
+		{
+			desc:    "no port",
+			opts:    ServeOptions{Network: "tcp", Address: "1.2.3.4"},
+			wantErr: true,
+		},
+		{
+			desc: "empty hostname",
+			opts: ServeOptions{Network: "tcp", Address: ":1234"},
+		},
+		{
+			desc: "ipv4",
+			opts: ServeOptions{Network: "tcp", Address: "1.2.3.4:1234"},
+		},
+		{
+			desc: "ipv6",
+			opts: ServeOptions{Network: "tcp", Address: "[1:2::3:4]:1234"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			err := test.opts.validate()
+			if (err != nil) != test.wantErr {
+				t.Errorf("ServeOptions.validate(%+v) returned err %v, wantErr: %v", test.opts, err, test.wantErr)
+			}
+		})
+	}
+}
+
+type fakeGRPCServer struct {
+	done              chan struct{}
+	registerServiceCh *testutils.Channel
+	serveCh           *testutils.Channel
+	stopCh            *testutils.Channel
+	gracefulStopCh    *testutils.Channel
+}
+
+func (f *fakeGRPCServer) RegisterService(*grpc.ServiceDesc, interface{}) {
+	f.registerServiceCh.Send(nil)
+}
+
+func (f *fakeGRPCServer) Serve(net.Listener) error {
+	f.serveCh.Send(nil)
+	<-f.done
+	return nil
+}
+
+func (f *fakeGRPCServer) Stop() {
+	close(f.done)
+	f.stopCh.Send(nil)
+}
+func (f *fakeGRPCServer) GracefulStop() {
+	close(f.done)
+	f.gracefulStopCh.Send(nil)
+}
+
+func newFakeGRPCServer() *fakeGRPCServer {
+	return &fakeGRPCServer{
+		done:              make(chan struct{}),
+		registerServiceCh: testutils.NewChannel(),
+		serveCh:           testutils.NewChannel(),
+		stopCh:            testutils.NewChannel(),
+		gracefulStopCh:    testutils.NewChannel(),
+	}
+}
+
+func (s) TestNewServer(t *testing.T) {
+	// The xds package adds a couple of server options (unary and stream
+	// interceptors) to the server options passed in by the user.
+	serverOpts := []grpc.ServerOption{grpc.Creds(insecure.NewCredentials())}
+	wantServerOpts := len(serverOpts) + 2
+
+	origNewGRPCServer := newGRPCServer
+	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface {
+		if got := len(opts); got != wantServerOpts {
+			t.Fatalf("%d ServerOptions passed to grpc.Server, want %d", got, wantServerOpts)
+		}
+		// Verify that the user passed ServerOptions are forwarded as is.
+		if !reflect.DeepEqual(opts[2:], serverOpts) {
+			t.Fatalf("got ServerOptions %v, want %v", opts[2:], serverOpts)
+		}
+		return newFakeGRPCServer()
+	}
+	defer func() {
+		newGRPCServer = origNewGRPCServer
+	}()
+
+	s := NewGRPCServer(serverOpts...)
+	defer s.Stop()
+}
+
+func (s) TestRegisterService(t *testing.T) {
+	fs := newFakeGRPCServer()
+
+	origNewGRPCServer := newGRPCServer
+	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface { return fs }
+	defer func() { newGRPCServer = origNewGRPCServer }()
+
+	s := NewGRPCServer()
+	defer s.Stop()
+
+	s.RegisterService(&grpc.ServiceDesc{}, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := fs.registerServiceCh.Receive(ctx); err != nil {
+		t.Fatalf("timeout when expecting RegisterService() to called on grpc.Server: %v", err)
+	}
+}
+
+// setupOverrides sets up overrides for bootstrap config, new xdsClient creation
+// and new gRPC.Server creation.
+func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func()) {
+	t.Helper()
+
+	origNewXDSConfig := newXDSConfig
+	newXDSConfig = func() (*bootstrap.Config, error) {
+		return &bootstrap.Config{
+			BalancerName: "dummyBalancer",
+			Creds:        grpc.WithInsecure(),
+			NodeProto:    xdstestutils.EmptyNodeProtoV3,
+		}, nil
+	}
+
+	clientCh := testutils.NewChannel()
+	origNewXDSClient := newXDSClient
+	newXDSClient = func(xdsclient.Options) (xdsClientInterface, error) {
+		c := fakeclient.NewClient()
+		clientCh.Send(c)
+		return c, nil
+	}
+
+	fs := newFakeGRPCServer()
+	origNewGRPCServer := newGRPCServer
+	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface { return fs }
+
+	return fs, clientCh, func() {
+		newXDSConfig = origNewXDSConfig
+		newXDSClient = origNewXDSClient
+		newGRPCServer = origNewGRPCServer
+	}
+}
+
+// TestServeSuccess tests the successful case of calling Serve().
+// The following sequence of events happen:
+// 1. Create a new GRPCServer and call Serve() in a goroutine.
+// 2. Make sure an xdsClient is created, and an LDS watch is registered.
+// 3. Push an error response from the xdsClient, and make sure that Serve() does
+//    not exit.
+// 4. Push a good response from the xdsClient, and make sure that Serve() on the
+// 	  underlying grpc.Server is called.
+func (s) TestServeSuccess(t *testing.T) {
+	fs, clientCh, cleanup := setupOverrides(t)
+	defer cleanup()
+
+	server := NewGRPCServer()
+	defer server.Stop()
+
+	// Call Serve() in a goroutine, and push on a channel when Serve returns.
+	serveDone := testutils.NewChannel()
+	go func() {
+		server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		serveDone.Send(nil)
+	}()
+
+	// Wait for an xdsClient to be created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := clientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for new xdsClient to be created: %v", err)
+	}
+	client := c.(*fakeclient.Client)
+
+	// Wait for a listener watch to be registered on the xdsClient.
+	name, err := client.WaitForWatchListener(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
+	}
+	wantPrefix := "grpc/server?udpa.resource.listening_address=localhost:"
+	if !strings.HasPrefix(name, wantPrefix) {
+		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantPrefix)
+	}
+
+	// Push an error to the registered listener watch callback and make sure
+	// that Serve does not return.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, errors.New("LDS error"))
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Serve() returned after a bad LDS response")
+	}
+
+	// Push a good LDS response, and wait for Serve() to be invoked on the
+	// underlying grpc.Server.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: "routeconfig"}, nil)
+	if _, err := fs.serveCh.Receive(ctx); err != nil {
+		t.Fatalf("error when waiting for Serve() to be invoked on the grpc.Server")
+	}
+}
+
+// TestServeWithStop tests the case where Stop() is called before an LDS update
+// is received. This should cause Serve() to exit before calling Serve() on the
+// underlying grpc.Server.
+func (s) TestServeWithStop(t *testing.T) {
+	fs, clientCh, cleanup := setupOverrides(t)
+	defer cleanup()
+
+	// Note that we are not deferring the Stop() here since we explicitly call
+	// it after the LDS watch has been registered.
+	server := NewGRPCServer()
+
+	// Call Serve() in a goroutine, and push on a channel when Serve returns.
+	serveDone := testutils.NewChannel()
+	go func() {
+		server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		serveDone.Send(nil)
+	}()
+
+	// Wait for an xdsClient to be created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := clientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for new xdsClient to be created: %v", err)
+	}
+	client := c.(*fakeclient.Client)
+
+	// Wait for a listener watch to be registered on the xdsClient.
+	name, err := client.WaitForWatchListener(ctx)
+	if err != nil {
+		server.Stop()
+		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
+	}
+	wantPrefix := "grpc/server?udpa.resource.listening_address=localhost:"
+	if !strings.HasPrefix(name, wantPrefix) {
+		server.Stop()
+		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantPrefix)
+	}
+
+	// Call Stop() on the server before a listener update is received, and
+	// expect Serve() to exit.
+	server.Stop()
+	if _, err := serveDone.Receive(ctx); err != nil {
+		t.Fatalf("error when waiting for Serve() to exit")
+	}
+
+	// Make sure that Serve() on the underlying grpc.Server is not called.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fs.serveCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Serve() called on underlying grpc.Server")
+	}
+}
+
+// TestServeBootstrapFailure tests the case where xDS bootstrap fails and
+// verifies that Serve() exits with a non-nil error.
+func (s) TestServeBootstrapFailure(t *testing.T) {
+	// Since we have not setup fakes for anything, this will attempt to do real
+	// xDS bootstrap and that will fail because the bootstrap environment
+	// variable is not set.
+	server := NewGRPCServer()
+	defer server.Stop()
+
+	serveDone := testutils.NewChannel()
+	go func() {
+		err := server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		serveDone.Send(err)
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	v, err := serveDone.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for Serve() to exit: %v", err)
+	}
+	if err, ok := v.(error); !ok || err == nil {
+		t.Fatal("Serve() did not exit with error")
+	}
+}
+
+// TestServeNewClientFailure tests the case where xds client creation fails and
+// verifies that Server() exits with a non-nil error.
+func (s) TestServeNewClientFailure(t *testing.T) {
+	origNewXDSConfig := newXDSConfig
+	newXDSConfig = func() (*bootstrap.Config, error) {
+		return &bootstrap.Config{
+			BalancerName: "dummyBalancer",
+			Creds:        grpc.WithInsecure(),
+			NodeProto:    xdstestutils.EmptyNodeProtoV3,
+		}, nil
+	}
+	defer func() { newXDSConfig = origNewXDSConfig }()
+
+	origNewXDSClient := newXDSClient
+	newXDSClient = func(xdsclient.Options) (xdsClientInterface, error) {
+		return nil, errors.New("xdsClient creation failed")
+	}
+	defer func() { newXDSClient = origNewXDSClient }()
+
+	server := NewGRPCServer()
+	defer server.Stop()
+
+	serveDone := testutils.NewChannel()
+	go func() {
+		err := server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		serveDone.Send(err)
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	v, err := serveDone.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for Serve() to exit: %v", err)
+	}
+	if err, ok := v.(error); !ok || err == nil {
+		t.Fatal("Serve() did not exit with error")
+	}
+}
diff --git a/xds/xds.go b/xds/xds.go
index fa0d699e8734..3cd0fc20b871 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -16,8 +16,12 @@
  *
  */
 
-// Package xds contains xds implementation. Users need to import this package to
-// get all xds functionality.
+// Package xds contains an implementation of the xDS suite of protocols, to be
+// used by gRPC client and server applications.
+//
+// On the client-side, users simply need to import this package to get all xDS
+// functionality. On the server-side, users need to use the GRPCServer type
+// exported by this package instead of the regular grpc.Server.
 //
 // See https://github.com/grpc/grpc-go/tree/master/examples/features/xds for
 // example.

From d6f8e6fbaf32b34e467137ee8c5676da5bb41844 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 9 Nov 2020 14:08:58 -0800
Subject: [PATCH 268/481] grpclb: send custom user-agent (#4011)

---
 balancer/balancer.go                      |  4 ++
 balancer/grpclb/grpclb_remote_balancer.go |  3 ++
 balancer/grpclb/grpclb_test.go            | 52 +++++++++++++++--------
 clientconn.go                             |  1 +
 4 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/balancer/balancer.go b/balancer/balancer.go
index 8bf359dbfda3..788759bde4b5 100644
--- a/balancer/balancer.go
+++ b/balancer/balancer.go
@@ -174,6 +174,10 @@ type BuildOptions struct {
 	Dialer func(context.Context, string) (net.Conn, error)
 	// ChannelzParentID is the entity parent's channelz unique identification number.
 	ChannelzParentID int64
+	// CustomUserAgent is the custom user agent set on the parent ClientConn.
+	// The balancer should set the same custom user agent if it creates a
+	// ClientConn.
+	CustomUserAgent string
 	// Target contains the parsed address info of the dial target. It is the same resolver.Target as
 	// passed to the resolver.
 	// See the documentation for the resolver.Target type for details about what it contains.
diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index 8eb45be28e32..8fdda09034f5 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -224,6 +224,9 @@ func (lb *lbBalancer) newRemoteBalancerCCWrapper() {
 	if lb.opt.Dialer != nil {
 		dopts = append(dopts, grpc.WithContextDialer(lb.opt.Dialer))
 	}
+	if lb.opt.CustomUserAgent != "" {
+		dopts = append(dopts, grpc.WithUserAgent(lb.opt.CustomUserAgent))
+	}
 	// Explicitly set pickfirst as the balancer.
 	dopts = append(dopts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"pick_first"}`))
 	dopts = append(dopts, grpc.WithResolvers(lb.manualResolver))
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index 5413613375a2..dcc5235703a7 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -194,15 +194,18 @@ type remoteBalancer struct {
 	stats     *rpcStats
 	statsChan chan *lbpb.ClientStats
 	fbChan    chan struct{}
+
+	customUserAgent string
 }
 
-func newRemoteBalancer(intervals []time.Duration, statsChan chan *lbpb.ClientStats) *remoteBalancer {
+func newRemoteBalancer(customUserAgent string, statsChan chan *lbpb.ClientStats) *remoteBalancer {
 	return &remoteBalancer{
-		sls:       make(chan *lbpb.ServerList, 1),
-		done:      make(chan struct{}),
-		stats:     newRPCStats(),
-		statsChan: statsChan,
-		fbChan:    make(chan struct{}),
+		sls:             make(chan *lbpb.ServerList, 1),
+		done:            make(chan struct{}),
+		stats:           newRPCStats(),
+		statsChan:       statsChan,
+		fbChan:          make(chan struct{}),
+		customUserAgent: customUserAgent,
 	}
 }
 
@@ -216,6 +219,17 @@ func (b *remoteBalancer) fallbackNow() {
 }
 
 func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServer) error {
+	md, ok := metadata.FromIncomingContext(stream.Context())
+	if !ok {
+		return status.Error(codes.Internal, "failed to receive metadata")
+	}
+	if b.customUserAgent != "" {
+		ua := md["user-agent"]
+		if len(ua) == 0 || !strings.HasPrefix(ua[0], b.customUserAgent) {
+			return status.Errorf(codes.InvalidArgument, "received unexpected user-agent: %v, want prefix %q", ua, b.customUserAgent)
+		}
+	}
+
 	req, err := stream.Recv()
 	if err != nil {
 		return err
@@ -333,7 +347,7 @@ type testServers struct {
 	beListeners []net.Listener
 }
 
-func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (tss *testServers, cleanup func(), err error) {
+func newLoadBalancer(numberOfBackends int, customUserAgent string, statsChan chan *lbpb.ClientStats) (tss *testServers, cleanup func(), err error) {
 	var (
 		beListeners []net.Listener
 		ls          *remoteBalancer
@@ -366,7 +380,7 @@ func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (ts
 		sn: lbServerName,
 	}
 	lb = grpc.NewServer(grpc.Creds(lbCreds))
-	ls = newRemoteBalancer(nil, statsChan)
+	ls = newRemoteBalancer(customUserAgent, statsChan)
 	lbgrpc.RegisterLoadBalancerServer(lb, ls)
 	go func() {
 		lb.Serve(lbLis)
@@ -398,7 +412,8 @@ var grpclbConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}`
 func (s) TestGRPCLB(t *testing.T) {
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(1, nil)
+	const testUserAgent = "test-user-agent"
+	tss, cleanup, err := newLoadBalancer(1, testUserAgent, nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -419,7 +434,8 @@ func (s) TestGRPCLB(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r),
-		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer))
+		grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer),
+		grpc.WithUserAgent(testUserAgent))
 	if err != nil {
 		t.Fatalf("Failed to dial to the backend %v", err)
 	}
@@ -445,7 +461,7 @@ func (s) TestGRPCLB(t *testing.T) {
 func (s) TestGRPCLBWeighted(t *testing.T) {
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(2, nil)
+	tss, cleanup, err := newLoadBalancer(2, "", nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -510,7 +526,7 @@ func (s) TestGRPCLBWeighted(t *testing.T) {
 func (s) TestDropRequest(t *testing.T) {
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(2, nil)
+	tss, cleanup, err := newLoadBalancer(2, "", nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -665,7 +681,7 @@ func (s) TestBalancerDisconnects(t *testing.T) {
 		lbs   []*grpc.Server
 	)
 	for i := 0; i < 2; i++ {
-		tss, cleanup, err := newLoadBalancer(1, nil)
+		tss, cleanup, err := newLoadBalancer(1, "", nil)
 		if err != nil {
 			t.Fatalf("failed to create new load balancer: %v", err)
 		}
@@ -737,7 +753,7 @@ func (s) TestFallback(t *testing.T) {
 
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(1, nil)
+	tss, cleanup, err := newLoadBalancer(1, "", nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -864,7 +880,7 @@ func (s) TestFallback(t *testing.T) {
 func (s) TestExplicitFallback(t *testing.T) {
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(1, nil)
+	tss, cleanup, err := newLoadBalancer(1, "", nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -974,7 +990,7 @@ func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 		resolveNowCh <- struct{}{}
 	}
 
-	tss, cleanup, err := newLoadBalancer(1, nil)
+	tss, cleanup, err := newLoadBalancer(1, "", nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -1085,7 +1101,7 @@ func (s) TestFallBackWithNoServerAddress(t *testing.T) {
 func (s) TestGRPCLBPickFirst(t *testing.T) {
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(3, nil)
+	tss, cleanup, err := newLoadBalancer(3, "", nil)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
@@ -1239,7 +1255,7 @@ func checkStats(stats, expected *rpcStats) error {
 func runAndCheckStats(t *testing.T, drop bool, statsChan chan *lbpb.ClientStats, runRPCs func(*grpc.ClientConn), statsWant *rpcStats) error {
 	r := manual.NewBuilderWithScheme("whatever")
 
-	tss, cleanup, err := newLoadBalancer(1, statsChan)
+	tss, cleanup, err := newLoadBalancer(1, "", statsChan)
 	if err != nil {
 		t.Fatalf("failed to create new load balancer: %v", err)
 	}
diff --git a/clientconn.go b/clientconn.go
index 11ed4af607a6..1e551a7041a0 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -288,6 +288,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		DialCreds:        credsClone,
 		CredsBundle:      cc.dopts.copts.CredsBundle,
 		Dialer:           cc.dopts.copts.Dialer,
+		CustomUserAgent:  cc.dopts.copts.UserAgent,
 		ChannelzParentID: cc.channelzID,
 		Target:           cc.parsedTarget,
 	}

From aeb04798c5560596fc5548e7b0641fe3f43535a0 Mon Sep 17 00:00:00 2001
From: yihuaz <yihuaz@google.com>
Date: Mon, 9 Nov 2020 15:33:53 -0800
Subject: [PATCH 269/481] credentials: fix PerRPCCredentials
 w/RequireTransportSecurity and security levels (#3995)

---
 credentials/credentials.go         |  19 +++---
 credentials/credentials_test.go    |  28 ++------
 credentials/insecure/insecure.go   |  12 ++--
 credentials/local/local.go         |  14 ++--
 credentials/local/local_test.go    |  32 ++++++---
 credentials/oauth/oauth.go         |  12 ++--
 credentials/sts/sts.go             |   3 +-
 internal/transport/http2_client.go |  19 +++++-
 test/insecure_creds_test.go        | 101 +++++++++++++++++++++++++++--
 test/local_creds_test.go           |  11 +++-
 10 files changed, 182 insertions(+), 69 deletions(-)

diff --git a/credentials/credentials.go b/credentials/credentials.go
index 02766443ae74..e69562e78786 100644
--- a/credentials/credentials.go
+++ b/credentials/credentials.go
@@ -58,9 +58,9 @@ type PerRPCCredentials interface {
 type SecurityLevel int
 
 const (
-	// Invalid indicates an invalid security level.
+	// InvalidSecurityLevel indicates an invalid security level.
 	// The zero SecurityLevel value is invalid for backward compatibility.
-	Invalid SecurityLevel = iota
+	InvalidSecurityLevel SecurityLevel = iota
 	// NoSecurity indicates a connection is insecure.
 	NoSecurity
 	// IntegrityOnly indicates a connection only provides integrity protection.
@@ -92,7 +92,7 @@ type CommonAuthInfo struct {
 }
 
 // GetCommonAuthInfo returns the pointer to CommonAuthInfo struct.
-func (c *CommonAuthInfo) GetCommonAuthInfo() *CommonAuthInfo {
+func (c CommonAuthInfo) GetCommonAuthInfo() CommonAuthInfo {
 	return c
 }
 
@@ -229,17 +229,16 @@ func ClientHandshakeInfoFromContext(ctx context.Context) ClientHandshakeInfo {
 // or 3) CommonAuthInfo.SecurityLevel has an invalid zero value. For 2) and 3), it is for the purpose of backward-compatibility.
 //
 // This API is experimental.
-func CheckSecurityLevel(ctx context.Context, level SecurityLevel) error {
+func CheckSecurityLevel(ai AuthInfo, level SecurityLevel) error {
 	type internalInfo interface {
-		GetCommonAuthInfo() *CommonAuthInfo
+		GetCommonAuthInfo() CommonAuthInfo
 	}
-	ri, _ := RequestInfoFromContext(ctx)
-	if ri.AuthInfo == nil {
-		return errors.New("unable to obtain SecurityLevel from context")
+	if ai == nil {
+		return errors.New("AuthInfo is nil")
 	}
-	if ci, ok := ri.AuthInfo.(internalInfo); ok {
+	if ci, ok := ai.(internalInfo); ok {
 		// CommonAuthInfo.SecurityLevel has an invalid value.
-		if ci.GetCommonAuthInfo().SecurityLevel == Invalid {
+		if ci.GetCommonAuthInfo().SecurityLevel == InvalidSecurityLevel {
 			return nil
 		}
 		if ci.GetCommonAuthInfo().SecurityLevel < level {
diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go
index dee0f2ca8304..08eb9c430ffc 100644
--- a/credentials/credentials_test.go
+++ b/credentials/credentials_test.go
@@ -26,7 +26,6 @@ import (
 	"testing"
 	"time"
 
-	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/testdata"
 )
@@ -57,17 +56,6 @@ func (ta testAuthInfo) AuthType() string {
 	return "testAuthInfo"
 }
 
-func createTestContext(s SecurityLevel) context.Context {
-	auth := &testAuthInfo{CommonAuthInfo: CommonAuthInfo{SecurityLevel: s}}
-	ri := RequestInfo{
-		Method:   "testInfo",
-		AuthInfo: auth,
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	return internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(ctx, ri)
-}
-
 func (s) TestCheckSecurityLevel(t *testing.T) {
 	testCases := []struct {
 		authLevel SecurityLevel
@@ -90,18 +78,18 @@ func (s) TestCheckSecurityLevel(t *testing.T) {
 			want:      true,
 		},
 		{
-			authLevel: Invalid,
+			authLevel: InvalidSecurityLevel,
 			testLevel: IntegrityOnly,
 			want:      true,
 		},
 		{
-			authLevel: Invalid,
+			authLevel: InvalidSecurityLevel,
 			testLevel: PrivacyAndIntegrity,
 			want:      true,
 		},
 	}
 	for _, tc := range testCases {
-		err := CheckSecurityLevel(createTestContext(tc.authLevel), tc.testLevel)
+		err := CheckSecurityLevel(testAuthInfo{CommonAuthInfo: CommonAuthInfo{SecurityLevel: tc.authLevel}}, tc.testLevel)
 		if tc.want && (err != nil) {
 			t.Fatalf("CheckSeurityLevel(%s, %s) returned failure but want success", tc.authLevel.String(), tc.testLevel.String())
 		} else if !tc.want && (err == nil) {
@@ -112,15 +100,7 @@ func (s) TestCheckSecurityLevel(t *testing.T) {
 }
 
 func (s) TestCheckSecurityLevelNoGetCommonAuthInfoMethod(t *testing.T) {
-	auth := &testAuthInfoNoGetCommonAuthInfoMethod{}
-	ri := RequestInfo{
-		Method:   "testInfo",
-		AuthInfo: auth,
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	ctxWithRequestInfo := internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(ctx, ri)
-	if err := CheckSecurityLevel(ctxWithRequestInfo, PrivacyAndIntegrity); err != nil {
+	if err := CheckSecurityLevel(testAuthInfoNoGetCommonAuthInfoMethod{}, PrivacyAndIntegrity); err != nil {
 		t.Fatalf("CheckSeurityLevel() returned failure but want success")
 	}
 }
diff --git a/credentials/insecure/insecure.go b/credentials/insecure/insecure.go
index 7fc11717f765..c4fa27c920da 100644
--- a/credentials/insecure/insecure.go
+++ b/credentials/insecure/insecure.go
@@ -43,11 +43,11 @@ func NewCredentials() credentials.TransportCredentials {
 type insecureTC struct{}
 
 func (insecureTC) ClientHandshake(ctx context.Context, _ string, conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil
+	return conn, info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil
 }
 
 func (insecureTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
-	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil
+	return conn, info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil
 }
 
 func (insecureTC) Info() credentials.ProtocolInfo {
@@ -62,13 +62,13 @@ func (insecureTC) OverrideServerName(string) error {
 	return nil
 }
 
-// Info contains the auth information for an insecure connection.
+// info contains the auth information for an insecure connection.
 // It implements the AuthInfo interface.
-type Info struct {
+type info struct {
 	credentials.CommonAuthInfo
 }
 
-// AuthType returns the type of Info as a string.
-func (Info) AuthType() string {
+// AuthType returns the type of info as a string.
+func (info) AuthType() string {
 	return "insecure"
 }
diff --git a/credentials/local/local.go b/credentials/local/local.go
index a9d446ecaa92..f772bc1307b2 100644
--- a/credentials/local/local.go
+++ b/credentials/local/local.go
@@ -38,14 +38,14 @@ import (
 	"google.golang.org/grpc/credentials"
 )
 
-// Info contains the auth information for a local connection.
+// info contains the auth information for a local connection.
 // It implements the AuthInfo interface.
-type Info struct {
+type info struct {
 	credentials.CommonAuthInfo
 }
 
-// AuthType returns the type of Info as a string.
-func (Info) AuthType() string {
+// AuthType returns the type of info as a string.
+func (info) AuthType() string {
 	return "local"
 }
 
@@ -70,7 +70,7 @@ func getSecurityLevel(network, addr string) (credentials.SecurityLevel, error) {
 		return credentials.PrivacyAndIntegrity, nil
 	// Not a local connection and should fail
 	default:
-		return credentials.Invalid, fmt.Errorf("local credentials rejected connection to non-local address %q", addr)
+		return credentials.InvalidSecurityLevel, fmt.Errorf("local credentials rejected connection to non-local address %q", addr)
 	}
 }
 
@@ -79,7 +79,7 @@ func (*localTC) ClientHandshake(ctx context.Context, authority string, conn net.
 	if err != nil {
 		return nil, nil, err
 	}
-	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil
+	return conn, info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil
 }
 
 func (*localTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
@@ -87,7 +87,7 @@ func (*localTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo,
 	if err != nil {
 		return nil, nil, err
 	}
-	return conn, Info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil
+	return conn, info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil
 }
 
 // NewCredentials returns a local credential implementing credentials.TransportCredentials.
diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go
index 64e2ec3e7fc6..00ae39f07e56 100644
--- a/credentials/local/local_test.go
+++ b/credentials/local/local_test.go
@@ -65,7 +65,7 @@ func (s) TestGetSecurityLevel(t *testing.T) {
 		{
 			testNetwork: "tcp",
 			testAddr:    "192.168.0.1:10000",
-			want:        credentials.Invalid,
+			want:        credentials.InvalidSecurityLevel,
 		},
 	}
 	for _, tc := range testCases {
@@ -78,6 +78,15 @@ func (s) TestGetSecurityLevel(t *testing.T) {
 
 type serverHandshake func(net.Conn) (credentials.AuthInfo, error)
 
+func getSecurityLevelFromAuthInfo(ai credentials.AuthInfo) credentials.SecurityLevel {
+	if c, ok := ai.(interface {
+		GetCommonAuthInfo() credentials.CommonAuthInfo
+	}); ok {
+		return c.GetCommonAuthInfo().SecurityLevel
+	}
+	return credentials.InvalidSecurityLevel
+}
+
 // Server local handshake implementation.
 func serverLocalHandshake(conn net.Conn) (credentials.AuthInfo, error) {
 	cred := NewCredentials()
@@ -140,21 +149,26 @@ func serverAndClientHandshake(lis net.Listener) (credentials.SecurityLevel, erro
 	defer lis.Close()
 	clientAuthInfo, err := clientHandle(clientLocalHandshake, lis.Addr().Network(), lis.Addr().String())
 	if err != nil {
-		return credentials.Invalid, fmt.Errorf("Error at client-side: %v", err)
+		return credentials.InvalidSecurityLevel, fmt.Errorf("Error at client-side: %v", err)
 	}
 	select {
 	case <-timer.C:
-		return credentials.Invalid, fmt.Errorf("Test didn't finish in time")
+		return credentials.InvalidSecurityLevel, fmt.Errorf("Test didn't finish in time")
 	case serverHandleResult := <-done:
 		if serverHandleResult.err != nil {
-			return credentials.Invalid, fmt.Errorf("Error at server-side: %v", serverHandleResult.err)
+			return credentials.InvalidSecurityLevel, fmt.Errorf("Error at server-side: %v", serverHandleResult.err)
+		}
+		clientSecLevel := getSecurityLevelFromAuthInfo(clientAuthInfo)
+		serverSecLevel := getSecurityLevelFromAuthInfo(serverHandleResult.authInfo)
+
+		if clientSecLevel == credentials.InvalidSecurityLevel {
+			return credentials.InvalidSecurityLevel, fmt.Errorf("Error at client-side: client's AuthInfo does not implement GetCommonAuthInfo()")
+		}
+		if serverSecLevel == credentials.InvalidSecurityLevel {
+			return credentials.InvalidSecurityLevel, fmt.Errorf("Error at server-side: server's AuthInfo does not implement GetCommonAuthInfo()")
 		}
-		clientLocal, _ := clientAuthInfo.(Info)
-		serverLocal, _ := serverHandleResult.authInfo.(Info)
-		clientSecLevel := clientLocal.CommonAuthInfo.SecurityLevel
-		serverSecLevel := serverLocal.CommonAuthInfo.SecurityLevel
 		if clientSecLevel != serverSecLevel {
-			return credentials.Invalid, fmt.Errorf("client's AuthInfo contains %s but server's AuthInfo contains %s", clientSecLevel.String(), serverSecLevel.String())
+			return credentials.InvalidSecurityLevel, fmt.Errorf("client's AuthInfo contains %s but server's AuthInfo contains %s", clientSecLevel.String(), serverSecLevel.String())
 		}
 		return clientSecLevel, nil
 	}
diff --git a/credentials/oauth/oauth.go b/credentials/oauth/oauth.go
index 6657055d6609..852ae375cfc7 100644
--- a/credentials/oauth/oauth.go
+++ b/credentials/oauth/oauth.go
@@ -42,7 +42,8 @@ func (ts TokenSource) GetRequestMetadata(ctx context.Context, uri ...string) (ma
 	if err != nil {
 		return nil, err
 	}
-	if err = credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	if err = credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil {
 		return nil, fmt.Errorf("unable to transfer TokenSource PerRPCCredentials: %v", err)
 	}
 	return map[string]string{
@@ -84,7 +85,8 @@ func (j jwtAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[s
 	if err != nil {
 		return nil, err
 	}
-	if err = credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	if err = credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil {
 		return nil, fmt.Errorf("unable to transfer jwtAccess PerRPCCredentials: %v", err)
 	}
 	return map[string]string{
@@ -107,7 +109,8 @@ func NewOauthAccess(token *oauth2.Token) credentials.PerRPCCredentials {
 }
 
 func (oa oauthAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
-	if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil {
 		return nil, fmt.Errorf("unable to transfer oauthAccess PerRPCCredentials: %v", err)
 	}
 	return map[string]string{
@@ -144,7 +147,8 @@ func (s *serviceAccount) GetRequestMetadata(ctx context.Context, uri ...string)
 			return nil, err
 		}
 	}
-	if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil {
 		return nil, fmt.Errorf("unable to transfer serviceAccount PerRPCCredentials: %v", err)
 	}
 	return map[string]string{
diff --git a/credentials/sts/sts.go b/credentials/sts/sts.go
index f4d58011be86..9285192a8eba 100644
--- a/credentials/sts/sts.go
+++ b/credentials/sts/sts.go
@@ -151,7 +151,8 @@ type callCreds struct {
 // GetRequestMetadata returns the cached accessToken, if available and valid, or
 // fetches a new one by performing an STS token exchange.
 func (c *callCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) {
-	if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil {
 		return nil, fmt.Errorf("unable to transfer STS PerRPCCredentials: %v", err)
 	}
 
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 0364df53f868..fef365c0d281 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -234,6 +234,18 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		if err != nil {
 			return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err)
 		}
+		for _, cd := range perRPCCreds {
+			if cd.RequireTransportSecurity() {
+				if ci, ok := authInfo.(interface {
+					GetCommonAuthInfo() credentials.CommonAuthInfo
+				}); ok {
+					secLevel := ci.GetCommonAuthInfo().SecurityLevel
+					if secLevel != credentials.InvalidSecurityLevel && secLevel < credentials.PrivacyAndIntegrity {
+						return nil, connectionErrorf(true, nil, "transport: cannot send secure credentials on an insecure connection")
+					}
+				}
+			}
+		}
 		isSecure = true
 		if transportCreds.Info().SecurityProtocol == "tls" {
 			scheme = "https"
@@ -557,8 +569,11 @@ func (t *http2Client) getCallAuthData(ctx context.Context, audience string, call
 	// Note: if these credentials are provided both via dial options and call
 	// options, then both sets of credentials will be applied.
 	if callCreds := callHdr.Creds; callCreds != nil {
-		if !t.isSecure && callCreds.RequireTransportSecurity() {
-			return nil, status.Error(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection")
+		if callCreds.RequireTransportSecurity() {
+			ri, _ := credentials.RequestInfoFromContext(ctx)
+			if !t.isSecure || credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity) != nil {
+				return nil, status.Error(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection")
+			}
 		}
 		data, err := callCreds.GetRequestMetadata(ctx, audience)
 		if err != nil {
diff --git a/test/insecure_creds_test.go b/test/insecure_creds_test.go
index 81d5a5ba5d0b..dd56a8c46ee5 100644
--- a/test/insecure_creds_test.go
+++ b/test/insecure_creds_test.go
@@ -21,6 +21,7 @@ package test
 import (
 	"context"
 	"net"
+	"strings"
 	"testing"
 	"time"
 
@@ -30,11 +31,32 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
+
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
 const defaultTestTimeout = 5 * time.Second
 
+// testLegacyPerRPCCredentials is a PerRPCCredentials that has yet incorporated security level.
+type testLegacyPerRPCCredentials struct{}
+
+func (cr testLegacyPerRPCCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
+	return nil, nil
+}
+
+func (cr testLegacyPerRPCCredentials) RequireTransportSecurity() bool {
+	return true
+}
+
+func getSecurityLevel(ai credentials.AuthInfo) credentials.SecurityLevel {
+	if c, ok := ai.(interface {
+		GetCommonAuthInfo() credentials.CommonAuthInfo
+	}); ok {
+		return c.GetCommonAuthInfo().SecurityLevel
+	}
+	return credentials.InvalidSecurityLevel
+}
+
 // TestInsecureCreds tests the use of insecure creds on the server and client
 // side, and verifies that expect security level and auth info are returned.
 // Also verifies that this credential can interop with existing `WithInsecure`
@@ -73,11 +95,11 @@ func (s) TestInsecureCreds(t *testing.T) {
 						return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
 					}
 					// Check security level.
-					info := pr.AuthInfo.(insecure.Info)
-					if at := info.AuthType(); at != "insecure" {
-						return nil, status.Errorf(codes.Unauthenticated, "Wrong AuthType: got %q, want insecure", at)
+					secLevel := getSecurityLevel(pr.AuthInfo)
+					if secLevel == credentials.InvalidSecurityLevel {
+						return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()")
 					}
-					if secLevel := info.CommonAuthInfo.SecurityLevel; secLevel != credentials.NoSecurity {
+					if secLevel != credentials.NoSecurity {
 						return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity)
 					}
 					return &testpb.Empty{}, nil
@@ -122,3 +144,74 @@ func (s) TestInsecureCreds(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestInsecureCredsWithPerRPCCredentials(t *testing.T) {
+	tests := []struct {
+		desc                      string
+		perRPCCredsViaDialOptions bool
+		perRPCCredsViaCallOptions bool
+		wantErr                   string
+	}{
+		{
+			desc:                      "send PerRPCCredentials via DialOptions",
+			perRPCCredsViaDialOptions: true,
+			perRPCCredsViaCallOptions: false,
+			wantErr:                   "context deadline exceeded",
+		},
+		{
+			desc:                      "send PerRPCCredentials via CallOptions",
+			perRPCCredsViaDialOptions: false,
+			perRPCCredsViaCallOptions: true,
+			wantErr:                   "transport: cannot send secure credentials on an insecure connection",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ss := &stubServer{
+				emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+					return &testpb.Empty{}, nil
+				},
+			}
+
+			sOpts := []grpc.ServerOption{}
+			sOpts = append(sOpts, grpc.Creds(insecure.NewCredentials()))
+			s := grpc.NewServer(sOpts...)
+			defer s.Stop()
+
+			testpb.RegisterTestServiceServer(s, ss)
+
+			lis, err := net.Listen("tcp", "localhost:0")
+			if err != nil {
+				t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+			}
+
+			go s.Serve(lis)
+
+			addr := lis.Addr().String()
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			cOpts := []grpc.DialOption{grpc.WithBlock()}
+			cOpts = append(cOpts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+
+			if test.perRPCCredsViaDialOptions {
+				cOpts = append(cOpts, grpc.WithPerRPCCredentials(testLegacyPerRPCCredentials{}))
+				if _, err := grpc.DialContext(ctx, addr, cOpts...); !strings.Contains(err.Error(), test.wantErr) {
+					t.Fatalf("InsecureCredsWithPerRPCCredentials/send_PerRPCCredentials_via_DialOptions  = %v; want %s", err, test.wantErr)
+				}
+			}
+
+			if test.perRPCCredsViaCallOptions {
+				cc, err := grpc.DialContext(ctx, addr, cOpts...)
+				if err != nil {
+					t.Fatalf("grpc.Dial(%q) failed: %v", addr, err)
+				}
+				defer cc.Close()
+
+				c := testpb.NewTestServiceClient(cc)
+				if _, err = c.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testLegacyPerRPCCredentials{})); !strings.Contains(err.Error(), test.wantErr) {
+					t.Fatalf("InsecureCredsWithPerRPCCredentials/send_PerRPCCredentials_via_CallOptions  = %v; want %s", err, test.wantErr)
+				}
+			}
+		})
+	}
+}
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index b55b73bdcbce..b9115f0d5ac8 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -43,9 +43,16 @@ func testLocalCredsE2ESucceed(network, address string) error {
 			if !ok {
 				return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
 			}
+			type internalInfo interface {
+				GetCommonAuthInfo() credentials.CommonAuthInfo
+			}
+			var secLevel credentials.SecurityLevel
+			if info, ok := (pr.AuthInfo).(internalInfo); ok {
+				secLevel = info.GetCommonAuthInfo().SecurityLevel
+			} else {
+				return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()")
+			}
 			// Check security level
-			info := pr.AuthInfo.(local.Info)
-			secLevel := info.CommonAuthInfo.SecurityLevel
 			switch network {
 			case "unix":
 				if secLevel != credentials.PrivacyAndIntegrity {

From b5d479d642af427866ee9c431bb1a12b89265754 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 10 Nov 2020 13:56:15 -0800
Subject: [PATCH 270/481] xds: Deflake xds_server_integration_test.go (#4025)

---
 .../test/xds_server_integration_test.go       | 44 +++++++++----------
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 5534d311eb81..7e1e7f63c4a0 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -143,44 +143,42 @@ func (s) TestServerSideXDS(t *testing.T) {
 	defer cleanup()
 	t.Logf("Started xDS management server at %s", fs.Address)
 
-	// Setup the fakeserver to respond with a Listener resource.
-	setupListenerResponse(fs.XDSResponseChan, listenerName)
 	// Create a bootstrap file in a temporary directory.
 	defer setupBootstrapFile(t, fs.Address)()
 
 	// Initialize a gRPC server which uses xDS, and register stubServer on it.
 	server := xds.NewGRPCServer()
 	testpb.RegisterTestServiceServer(server, &testService{})
+	defer server.Stop()
 
-	errCh := make(chan error, 1)
 	go func() {
-		defer server.Stop()
-
-		// Create a clientconn and make a successful RPC
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		cc, err := grpc.DialContext(ctx, localAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
-		if err != nil {
-			errCh <- fmt.Errorf("failed to dial local test server: %v", err)
-			return
+		opts := xds.ServeOptions{Network: "tcp", Address: localAddress}
+		if err := server.Serve(opts); err != nil {
+			t.Errorf("Serve(%+v) failed: %v", opts, err)
 		}
-		defer cc.Close()
+	}()
+
+	// Create a ClientConn and make a successful RPC.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 
-		client := testpb.NewTestServiceClient(cc)
-		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
-			errCh <- fmt.Errorf("rpc EmptyCall() failed: %v", err)
-			return
+	// Setup the fake management server to respond with a Listener resource.
+	go func() {
+		if _, err := fs.XDSRequestChan.Receive(ctx); err != nil {
+			t.Errorf("timeout when waiting for listener request: %v", err)
 		}
-		errCh <- nil
+		setupListenerResponse(fs.XDSResponseChan, listenerName)
 	}()
 
-	opts := xds.ServeOptions{Network: "tcp", Address: localAddress}
-	if err := server.Serve(opts); err != nil {
-		t.Fatalf("Serve(%+v) failed: %v", opts, err)
+	cc, err := grpc.DialContext(ctx, localAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
 	}
+	defer cc.Close()
 
-	if err := <-errCh; err != nil {
-		t.Fatal(err)
+	client := testpb.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
 	}
 }
 

From 28c130fe3d7ec9dcfbde6f3a05e9f32bbfa9e6b4 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 11 Nov 2020 10:12:52 -0800
Subject: [PATCH 271/481] xds: Minor improvements to xDS server API and test.
 (#4026)

---
 .../test/xds_server_integration_test.go       | 16 +++--
 xds/internal/testutils/local_address.go       | 40 +++++++++++
 xds/server.go                                 | 21 +++---
 xds/server_test.go                            | 67 +++++++++++++------
 4 files changed, 107 insertions(+), 37 deletions(-)
 create mode 100644 xds/internal/testutils/local_address.go

diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 7e1e7f63c4a0..3f218eb458ca 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -41,14 +41,13 @@ import (
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/xds"
 	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
 const (
 	defaultTestTimeout = 10 * time.Second
-	localAddress       = "localhost:9999"
-	listenerName       = "grpc/server?udpa.resource.listening_address=localhost:9999"
 )
 
 type s struct {
@@ -68,7 +67,7 @@ func setupListenerResponse(respCh chan *fakeserver.Response, name string) {
 					Value: func() []byte {
 						l := &v3listenerpb.Listener{
 							// This needs to match the name we are querying for.
-							Name: listenerName,
+							Name: name,
 							ApiListener: &v3listenerpb.ApiListener{
 								ApiListener: &anypb.Any{
 									TypeUrl: version.V2HTTPConnManagerURL,
@@ -151,8 +150,13 @@ func (s) TestServerSideXDS(t *testing.T) {
 	testpb.RegisterTestServiceServer(server, &testService{})
 	defer server.Stop()
 
+	localAddr, err := testutils.AvailableHostPort()
+	if err != nil {
+		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+	}
+
 	go func() {
-		opts := xds.ServeOptions{Network: "tcp", Address: localAddress}
+		opts := xds.ServeOptions{Address: localAddr}
 		if err := server.Serve(opts); err != nil {
 			t.Errorf("Serve(%+v) failed: %v", opts, err)
 		}
@@ -167,10 +171,10 @@ func (s) TestServerSideXDS(t *testing.T) {
 		if _, err := fs.XDSRequestChan.Receive(ctx); err != nil {
 			t.Errorf("timeout when waiting for listener request: %v", err)
 		}
-		setupListenerResponse(fs.XDSResponseChan, listenerName)
+		setupListenerResponse(fs.XDSResponseChan, fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr))
 	}()
 
-	cc, err := grpc.DialContext(ctx, localAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	cc, err := grpc.DialContext(ctx, localAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
diff --git a/xds/internal/testutils/local_address.go b/xds/internal/testutils/local_address.go
new file mode 100644
index 000000000000..5ada7368271c
--- /dev/null
+++ b/xds/internal/testutils/local_address.go
@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import "net"
+
+// AvailableHostPort returns a local address to listen on. This will be of the
+// form "host:port", where the host will be a literal IP address, and port
+// must be a literal port number. If the host is a literal IPv6 address it
+// will be enclosed in square brackets, as in "[2001:db8::1]:80.
+//
+// This is useful for tests which need to call the Serve() method on
+// xds.GRPCServer which needs to be passed an IP:Port to listen on, where the IP
+// must be a literal IP and not localhost. This approach will work on support
+// one or both of IPv4 or IPv6.
+func AvailableHostPort() (string, error) {
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return "", err
+	}
+	addr := l.Addr().String()
+	l.Close()
+	return addr, nil
+}
diff --git a/xds/server.go b/xds/server.go
index 0b85527ce96f..449c27a958ce 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strconv"
 	"sync"
 
 	"google.golang.org/grpc"
@@ -81,24 +82,24 @@ type grpcServerInterface interface {
 // Notice: This type is EXPERIMENTAL and may be changed or removed in a
 // later release.
 type ServeOptions struct {
-	// Network identifies the local network to listen on. The network must be
-	// "tcp", "tcp4", "tcp6".
-	Network string
 	// Address contains the local address to listen on. This should be of the
 	// form "host:port", where the host must be a literal IP address, and port
 	// must be a literal port number. If the host is a literal IPv6 address it
-	// must be enclosed in square brackets, as in "[2001:db8::1]:80. The host
-	// portion can be left unspecified.
+	// must be enclosed in square brackets, as in "[2001:db8::1]:80.
 	Address string
 }
 
 func (so *ServeOptions) validate() error {
-	if net := so.Network; net != "tcp" && net != "tcp4" && net != "tcp6" {
-		return fmt.Errorf("xds: unsupported network type %q for server listener", net)
-	}
-	if _, _, err := net.SplitHostPort(so.Address); err != nil {
+	addr, port, err := net.SplitHostPort(so.Address)
+	if err != nil {
 		return fmt.Errorf("xds: unsupported address %q for server listener", so.Address)
 	}
+	if net.ParseIP(addr) == nil {
+		return fmt.Errorf("xds: failed to parse %q as a valid literal IP address", addr)
+	}
+	if _, err := strconv.Atoi(port); err != nil {
+		return fmt.Errorf("%q is not a valid listener port", port)
+	}
 	return nil
 }
 
@@ -211,7 +212,7 @@ func (s *GRPCServer) Serve(opts ServeOptions) error {
 // Returns a listenerWrapper, which implements the net.Listener interface, that
 // can be passed to grpcServer.Serve().
 func (s *GRPCServer) newListenerWrapper(opts ServeOptions) (*listenerWrapper, error) {
-	lis, err := net.Listen(opts.Network, opts.Address)
+	lis, err := net.Listen("tcp", opts.Address)
 	if err != nil {
 		return nil, fmt.Errorf("xds: failed to listen on %+v: %v", opts, err)
 	}
diff --git a/xds/server_test.go b/xds/server_test.go
index e95316236979..8769cf386002 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -21,9 +21,9 @@ package xds
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"reflect"
-	"strings"
 	"testing"
 	"time"
 
@@ -62,31 +62,32 @@ func (s) TestServeOptions_Validate(t *testing.T) {
 			wantErr: true,
 		},
 		{
-			desc:    "unsupported network",
-			opts:    ServeOptions{Network: "foo"},
+			desc:    "bad address",
+			opts:    ServeOptions{Address: "I'm a bad IP address"},
 			wantErr: true,
 		},
 		{
-			desc:    "bad address",
-			opts:    ServeOptions{Network: "tcp", Address: "I'm a bad IP address"},
+			desc:    "no port",
+			opts:    ServeOptions{Address: "1.2.3.4"},
 			wantErr: true,
 		},
 		{
-			desc:    "no port",
-			opts:    ServeOptions{Network: "tcp", Address: "1.2.3.4"},
+			desc:    "empty hostname",
+			opts:    ServeOptions{Address: ":1234"},
 			wantErr: true,
 		},
 		{
-			desc: "empty hostname",
-			opts: ServeOptions{Network: "tcp", Address: ":1234"},
+			desc:    "localhost",
+			opts:    ServeOptions{Address: "localhost:1234"},
+			wantErr: true,
 		},
 		{
 			desc: "ipv4",
-			opts: ServeOptions{Network: "tcp", Address: "1.2.3.4:1234"},
+			opts: ServeOptions{Address: "1.2.3.4:1234"},
 		},
 		{
 			desc: "ipv6",
-			opts: ServeOptions{Network: "tcp", Address: "[1:2::3:4]:1234"},
+			opts: ServeOptions{Address: "[1:2::3:4]:1234"},
 		},
 	}
 
@@ -228,10 +229,17 @@ func (s) TestServeSuccess(t *testing.T) {
 	server := NewGRPCServer()
 	defer server.Stop()
 
+	localAddr, err := xdstestutils.AvailableHostPort()
+	if err != nil {
+		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+	}
+
 	// Call Serve() in a goroutine, and push on a channel when Serve returns.
 	serveDone := testutils.NewChannel()
 	go func() {
-		server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		if err := server.Serve(ServeOptions{Address: localAddr}); err != nil {
+			t.Error(err)
+		}
 		serveDone.Send(nil)
 	}()
 
@@ -249,9 +257,9 @@ func (s) TestServeSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantPrefix := "grpc/server?udpa.resource.listening_address=localhost:"
-	if !strings.HasPrefix(name, wantPrefix) {
-		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantPrefix)
+	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr)
+	if name != wantName {
+		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
 
 	// Push an error to the registered listener watch callback and make sure
@@ -282,10 +290,17 @@ func (s) TestServeWithStop(t *testing.T) {
 	// it after the LDS watch has been registered.
 	server := NewGRPCServer()
 
+	localAddr, err := xdstestutils.AvailableHostPort()
+	if err != nil {
+		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+	}
+
 	// Call Serve() in a goroutine, and push on a channel when Serve returns.
 	serveDone := testutils.NewChannel()
 	go func() {
-		server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		if err := server.Serve(ServeOptions{Address: localAddr}); err != nil {
+			t.Error(err)
+		}
 		serveDone.Send(nil)
 	}()
 
@@ -304,10 +319,10 @@ func (s) TestServeWithStop(t *testing.T) {
 		server.Stop()
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantPrefix := "grpc/server?udpa.resource.listening_address=localhost:"
-	if !strings.HasPrefix(name, wantPrefix) {
+	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr)
+	if name != wantName {
 		server.Stop()
-		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantPrefix)
+		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantName)
 	}
 
 	// Call Stop() on the server before a listener update is received, and
@@ -334,9 +349,14 @@ func (s) TestServeBootstrapFailure(t *testing.T) {
 	server := NewGRPCServer()
 	defer server.Stop()
 
+	localAddr, err := xdstestutils.AvailableHostPort()
+	if err != nil {
+		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+	}
+
 	serveDone := testutils.NewChannel()
 	go func() {
-		err := server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		err := server.Serve(ServeOptions{Address: localAddr})
 		serveDone.Send(err)
 	}()
 
@@ -373,9 +393,14 @@ func (s) TestServeNewClientFailure(t *testing.T) {
 	server := NewGRPCServer()
 	defer server.Stop()
 
+	localAddr, err := xdstestutils.AvailableHostPort()
+	if err != nil {
+		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+	}
+
 	serveDone := testutils.NewChannel()
 	go func() {
-		err := server.Serve(ServeOptions{Network: "tcp", Address: "localhost:0"})
+		err := server.Serve(ServeOptions{Address: localAddr})
 		serveDone.Send(err)
 	}()
 

From 6caf9d8621b902e41e367d1d5e98e4249acd8c8e Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 12 Nov 2020 11:08:36 -0800
Subject: [PATCH 272/481] xds: make xds client a singleton (#4015)

- xdsclient.New() no longer takes any input, all configs are from bootstrap file
  - added a NewForTesting()
- The returned *Client is a wrapper of the underlying client implementation, for ref-couting
- xds-resolver and xds-server no longer calls bootstrap.NewConfig. It only calls xdsclient.New()
---
 xds/internal/client/bootstrap/bootstrap.go    |  10 +-
 xds/internal/client/client.go                 |  96 +++++--------
 xds/internal/client/client_callback.go        |  12 +-
 xds/internal/client/client_loadreport.go      |  22 +--
 xds/internal/client/client_loadreport_test.go |  16 +--
 xds/internal/client/client_logging.go         |   2 +-
 xds/internal/client/client_singleton.go       | 101 ++++++++++++++
 xds/internal/client/client_test.go            | 132 ++++++++++++++++--
 xds/internal/client/client_watchers.go        |  20 +--
 .../client/client_watchers_cluster_test.go    |  14 +-
 .../client/client_watchers_endpoints_test.go  |  10 +-
 .../client/client_watchers_listener_test.go   |  10 +-
 .../client/client_watchers_route_test.go      |   8 +-
 xds/internal/client/tests/client_test.go      |  65 ++++-----
 xds/internal/client/v2/client.go              |  16 ++-
 xds/internal/client/v3/client.go              |  16 ++-
 xds/internal/resolver/xds_resolver.go         |  21 +--
 xds/internal/resolver/xds_resolver_test.go    | 117 ++--------------
 xds/internal/testutils/fakeserver/server.go   |   2 +-
 xds/server.go                                 |  11 +-
 xds/server_test.go                            |   4 +-
 21 files changed, 381 insertions(+), 324 deletions(-)
 create mode 100644 xds/internal/client/client_singleton.go

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 789d1d0e45ad..ca7c96c9928c 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -57,10 +57,10 @@ var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version)
 var bootstrapFileReadFunc = ioutil.ReadFile
 
 // Config provides the xDS client with several key bits of information that it
-// requires in its interaction with an xDS server. The Config is initialized
-// from the bootstrap file.
+// requires in its interaction with the management server. The Config is
+// initialized from the bootstrap file.
 type Config struct {
-	// BalancerName is the name of the xDS server to connect to.
+	// BalancerName is the name of the management server to connect to.
 	//
 	// The bootstrap file contains a list of servers (with name+creds), but we
 	// pick the first one.
@@ -96,7 +96,7 @@ type xdsServer struct {
 // The format of the bootstrap file will be as follows:
 // {
 //    "xds_server": {
-//      "server_uri": <string containing URI of xds server>,
+//      "server_uri": <string containing URI of management server>,
 //      "channel_creds": [
 //        {
 //          "type": <string containing channel cred type>,
@@ -168,7 +168,7 @@ func NewConfig() (*Config, error) {
 				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
 			}
 			if len(servers) < 1 {
-				return nil, fmt.Errorf("xds: bootstrap file parsing failed during bootstrap: file doesn't contain any xds server to connect to")
+				return nil, fmt.Errorf("xds: bootstrap file parsing failed during bootstrap: file doesn't contain any management server to connect to")
 			}
 			xs := servers[0]
 			config.BalancerName = xs.ServerURI
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 5497fbd97941..aa715f52c636 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -277,28 +277,6 @@ type EndpointsUpdate struct {
 	Localities []Locality
 }
 
-// Options provides all parameters required for the creation of an xDS client.
-type Options struct {
-	// Config contains a fully populated bootstrap config. It is the
-	// responsibility of the caller to use some sane defaults here if the
-	// bootstrap process returned with certain fields left unspecified.
-	Config bootstrap.Config
-	// DialOpts contains dial options to be used when dialing the xDS server.
-	DialOpts []grpc.DialOption
-	// TargetName is the target of the parent ClientConn.
-	TargetName string
-	// WatchExpiryTimeout is the amount of time the client is willing to wait
-	// for the first response from the server for any resource being watched.
-	// Expiry will not cause cancellation of the watch. It will only trigger the
-	// invocation of the registered callback and it is left up to the caller to
-	// decide whether or not they want to cancel the watch.
-	//
-	// If this field is left unspecified, a default value of 15 seconds will be
-	// used. This is based on the default value of the initial_fetch_timeout
-	// field in corepb.ConfigSource proto.
-	WatchExpiryTimeout time.Duration
-}
-
 // Function to be overridden in tests.
 var newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, opts BuildOptions) (APIClient, error) {
 	cb := getAPIClientBuilder(apiVersion)
@@ -308,23 +286,19 @@ var newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, op
 	return cb.Build(cc, opts)
 }
 
-// Client is a full fledged gRPC client which queries a set of discovery APIs
-// (collectively termed as xDS) on a remote management server, to discover
-// various dynamic resources.
-//
-// A single client object will be shared by the xds resolver and balancer
-// implementations. But the same client can only be shared by the same parent
-// ClientConn.
+// clientImpl is the real implementation of the xds client. The exported Client
+// is a wrapper of this struct with a ref count.
 //
 // Implements UpdateHandler interface.
 // TODO(easwars): Make a wrapper struct which implements this interface in the
 // style of ccBalancerWrapper so that the Client type does not implement these
 // exported methods.
-type Client struct {
-	done      *grpcsync.Event
-	opts      Options
-	cc        *grpc.ClientConn // Connection to the xDS server
-	apiClient APIClient
+type clientImpl struct {
+	done               *grpcsync.Event
+	config             *bootstrap.Config
+	cc                 *grpc.ClientConn // Connection to the management server.
+	apiClient          APIClient
+	watchExpiryTimeout time.Duration
 
 	logger *grpclog.PrefixLogger
 
@@ -345,46 +319,40 @@ type Client struct {
 	lrsClients map[string]*lrsClient
 }
 
-// New returns a new xdsClient configured with opts.
-func New(opts Options) (*Client, error) {
+// newWithConfig returns a new xdsClient with the given config.
+func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration) (*clientImpl, error) {
 	switch {
-	case opts.Config.BalancerName == "":
+	case config.BalancerName == "":
 		return nil, errors.New("xds: no xds_server name provided in options")
-	case opts.Config.Creds == nil:
+	case config.Creds == nil:
 		return nil, errors.New("xds: no credentials provided in options")
-	case opts.Config.NodeProto == nil:
+	case config.NodeProto == nil:
 		return nil, errors.New("xds: no node_proto provided in options")
 	}
 
-	switch opts.Config.TransportAPI {
+	switch config.TransportAPI {
 	case version.TransportV2:
-		if _, ok := opts.Config.NodeProto.(*v2corepb.Node); !ok {
-			return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", opts.Config.NodeProto, opts.Config.TransportAPI)
+		if _, ok := config.NodeProto.(*v2corepb.Node); !ok {
+			return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", config.NodeProto, config.TransportAPI)
 		}
 	case version.TransportV3:
-		if _, ok := opts.Config.NodeProto.(*v3corepb.Node); !ok {
-			return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", opts.Config.NodeProto, opts.Config.TransportAPI)
+		if _, ok := config.NodeProto.(*v3corepb.Node); !ok {
+			return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", config.NodeProto, config.TransportAPI)
 		}
 	}
 
 	dopts := []grpc.DialOption{
-		opts.Config.Creds,
+		config.Creds,
 		grpc.WithKeepaliveParams(keepalive.ClientParameters{
 			Time:    5 * time.Minute,
 			Timeout: 20 * time.Second,
 		}),
 	}
-	dopts = append(dopts, opts.DialOpts...)
-
-	if opts.WatchExpiryTimeout == 0 {
-		// This is based on the default value of the initial_fetch_timeout field
-		// in corepb.ConfigSource proto.
-		opts.WatchExpiryTimeout = 15 * time.Second
-	}
 
-	c := &Client{
-		done: grpcsync.NewEvent(),
-		opts: opts,
+	c := &clientImpl{
+		done:               grpcsync.NewEvent(),
+		config:             config,
+		watchExpiryTimeout: watchExpiryTimeout,
 
 		updateCh:    buffer.NewUnbounded(),
 		ldsWatchers: make(map[string]map[*watchInfo]bool),
@@ -398,18 +366,18 @@ func New(opts Options) (*Client, error) {
 		lrsClients:  make(map[string]*lrsClient),
 	}
 
-	cc, err := grpc.Dial(opts.Config.BalancerName, dopts...)
+	cc, err := grpc.Dial(config.BalancerName, dopts...)
 	if err != nil {
 		// An error from a non-blocking dial indicates something serious.
-		return nil, fmt.Errorf("xds: failed to dial balancer {%s}: %v", opts.Config.BalancerName, err)
+		return nil, fmt.Errorf("xds: failed to dial balancer {%s}: %v", config.BalancerName, err)
 	}
 	c.cc = cc
 	c.logger = prefixLogger((c))
-	c.logger.Infof("Created ClientConn to xDS server: %s", opts.Config.BalancerName)
+	c.logger.Infof("Created ClientConn to xDS management server: %s", config.BalancerName)
 
-	apiClient, err := newAPIClient(opts.Config.TransportAPI, cc, BuildOptions{
+	apiClient, err := newAPIClient(config.TransportAPI, cc, BuildOptions{
 		Parent:    c,
-		NodeProto: opts.Config.NodeProto,
+		NodeProto: config.NodeProto,
 		Backoff:   backoff.DefaultExponential.Backoff,
 		Logger:    c.logger,
 	})
@@ -426,7 +394,7 @@ func New(opts Options) (*Client, error) {
 // "certificate_providers" field of the bootstrap file. The key in the returned
 // map is the plugin_instance_name. Callers must not modify the returned map.
 func (c *Client) CertProviderConfigs() map[string]*certprovider.BuildableConfig {
-	return c.opts.Config.CertProviderConfigs
+	return c.config.CertProviderConfigs
 }
 
 // run is a goroutine for all the callbacks.
@@ -435,7 +403,7 @@ func (c *Client) CertProviderConfigs() map[string]*certprovider.BuildableConfig
 // goroutine, the callback will be called inline, which might cause a deadlock
 // in user's code. Callbacks also cannot be simple `go callback()` because the
 // order matters.
-func (c *Client) run() {
+func (c *clientImpl) run() {
 	for {
 		select {
 		case t := <-c.updateCh.Get():
@@ -450,8 +418,8 @@ func (c *Client) run() {
 	}
 }
 
-// Close closes the gRPC connection to the xDS server.
-func (c *Client) Close() {
+// Close closes the gRPC connection to the management server.
+func (c *clientImpl) Close() {
 	if c.done.HasFired() {
 		return
 	}
diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go
index a135dae745b9..90ffcdec6219 100644
--- a/xds/internal/client/client_callback.go
+++ b/xds/internal/client/client_callback.go
@@ -26,7 +26,7 @@ type watcherInfoWithUpdate struct {
 
 // scheduleCallback should only be called by methods of watchInfo, which checks
 // for watcher states and maintain consistency.
-func (c *Client) scheduleCallback(wi *watchInfo, update interface{}, err error) {
+func (c *clientImpl) scheduleCallback(wi *watchInfo, update interface{}, err error) {
 	c.updateCh.Put(&watcherInfoWithUpdate{
 		wi:     wi,
 		update: update,
@@ -34,7 +34,7 @@ func (c *Client) scheduleCallback(wi *watchInfo, update interface{}, err error)
 	})
 }
 
-func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
+func (c *clientImpl) callCallback(wiu *watcherInfoWithUpdate) {
 	c.mu.Lock()
 	// Use a closure to capture the callback and type assertion, to save one
 	// more switch case.
@@ -74,7 +74,7 @@ func (c *Client) callCallback(wiu *watcherInfoWithUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) NewListeners(updates map[string]ListenerUpdate) {
+func (c *clientImpl) NewListeners(updates map[string]ListenerUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -109,7 +109,7 @@ func (c *Client) NewListeners(updates map[string]ListenerUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) NewRouteConfigs(updates map[string]RouteConfigUpdate) {
+func (c *clientImpl) NewRouteConfigs(updates map[string]RouteConfigUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -130,7 +130,7 @@ func (c *Client) NewRouteConfigs(updates map[string]RouteConfigUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) NewClusters(updates map[string]ClusterUpdate) {
+func (c *clientImpl) NewClusters(updates map[string]ClusterUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -165,7 +165,7 @@ func (c *Client) NewClusters(updates map[string]ClusterUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *Client) NewEndpoints(updates map[string]EndpointsUpdate) {
+func (c *clientImpl) NewEndpoints(updates map[string]EndpointsUpdate) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go
index e91316b9fe48..be42a6e0c383 100644
--- a/xds/internal/client/client_loadreport.go
+++ b/xds/internal/client/client_loadreport.go
@@ -25,7 +25,7 @@ import (
 )
 
 // ReportLoad starts an load reporting stream to the given server. If the server
-// is not an empty string, and is different from the xds server, a new
+// is not an empty string, and is different from the management server, a new
 // ClientConn will be created.
 //
 // The same options used for creating the Client will be used (including
@@ -33,7 +33,7 @@ import (
 //
 // It returns a Store for the user to report loads, a function to cancel the
 // load reporting stream.
-func (c *Client) ReportLoad(server string) (*load.Store, func()) {
+func (c *clientImpl) ReportLoad(server string) (*load.Store, func()) {
 	c.lrsMu.Lock()
 	defer c.lrsMu.Unlock()
 
@@ -58,20 +58,21 @@ func (c *Client) ReportLoad(server string) (*load.Store, func()) {
 }
 
 // lrsClient maps to one lrsServer. It contains:
-// - a ClientConn to this server (only if it's different from the xds server)
+// - a ClientConn to this server (only if it's different from the management
+// server)
 // - a load.Store that contains loads only for this server
 type lrsClient struct {
-	parent *Client
+	parent *clientImpl
 	server string
 
-	cc           *grpc.ClientConn // nil if the server is same as the xds server
+	cc           *grpc.ClientConn // nil if the server is same as the management server
 	refCount     int
 	cancelStream func()
 	loadStore    *load.Store
 }
 
 // newLRSClient creates a new LRS stream to the server.
-func newLRSClient(parent *Client, server string) *lrsClient {
+func newLRSClient(parent *clientImpl, server string) *lrsClient {
 	return &lrsClient{
 		parent:   parent,
 		server:   server,
@@ -109,18 +110,17 @@ func (lrsC *lrsClient) unRef() (closed bool) {
 }
 
 // startStream starts the LRS stream to the server. If server is not the same
-// xDS server from the parent, it also creates a ClientConn.
+// management server from the parent, it also creates a ClientConn.
 func (lrsC *lrsClient) startStream() {
 	var cc *grpc.ClientConn
 
 	lrsC.parent.logger.Infof("Starting load report to server: %s", lrsC.server)
-	if lrsC.server == "" || lrsC.server == lrsC.parent.opts.Config.BalancerName {
+	if lrsC.server == "" || lrsC.server == lrsC.parent.config.BalancerName {
 		// Reuse the xDS client if server is the same.
 		cc = lrsC.parent.cc
 	} else {
-		lrsC.parent.logger.Infof("LRS server is different from xDS server, starting a new ClientConn")
-		dopts := append([]grpc.DialOption{lrsC.parent.opts.Config.Creds}, lrsC.parent.opts.DialOpts...)
-		ccNew, err := grpc.Dial(lrsC.server, dopts...)
+		lrsC.parent.logger.Infof("LRS server is different from management server, starting a new ClientConn")
+		ccNew, err := grpc.Dial(lrsC.server, lrsC.parent.config.Creds)
 		if err != nil {
 			// An error from a non-blocking dial indicates something serious.
 			lrsC.parent.logger.Infof("xds: failed to dial load report server {%s}: %v", lrsC.server, err)
diff --git a/xds/internal/client/client_loadreport_test.go b/xds/internal/client/client_loadreport_test.go
index d426247c10d9..c9a2709962eb 100644
--- a/xds/internal/client/client_loadreport_test.go
+++ b/xds/internal/client/client_loadreport_test.go
@@ -44,6 +44,8 @@ import (
 const (
 	defaultTestTimeout      = 5 * time.Second
 	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+
+	defaultClientWatchExpiryTimeout = 15 * time.Second
 )
 
 type s struct {
@@ -61,14 +63,12 @@ func (s) TestLRSClient(t *testing.T) {
 	}
 	defer sCleanup()
 
-	xdsC, err := client.New(client.Options{
-		Config: bootstrap.Config{
-			BalancerName: fs.Address,
-			Creds:        grpc.WithInsecure(),
-			NodeProto:    &v2corepb.Node{},
-			TransportAPI: version.TransportV2,
-		},
-	})
+	xdsC, err := client.NewWithConfigForTesting(&bootstrap.Config{
+		BalancerName: fs.Address,
+		Creds:        grpc.WithInsecure(),
+		NodeProto:    &v2corepb.Node{},
+		TransportAPI: version.TransportV2,
+	}, defaultClientWatchExpiryTimeout)
 	if err != nil {
 		t.Fatalf("failed to create xds client: %v", err)
 	}
diff --git a/xds/internal/client/client_logging.go b/xds/internal/client/client_logging.go
index a47e5247fe22..bff3fb1d3df3 100644
--- a/xds/internal/client/client_logging.go
+++ b/xds/internal/client/client_logging.go
@@ -29,6 +29,6 @@ const prefix = "[xds-client %p] "
 
 var logger = grpclog.Component("xds")
 
-func prefixLogger(p *Client) *internalgrpclog.PrefixLogger {
+func prefixLogger(p *clientImpl) *internalgrpclog.PrefixLogger {
 	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
 }
diff --git a/xds/internal/client/client_singleton.go b/xds/internal/client/client_singleton.go
new file mode 100644
index 000000000000..5d92b4146bb3
--- /dev/null
+++ b/xds/internal/client/client_singleton.go
@@ -0,0 +1,101 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+)
+
+const defaultWatchExpiryTimeout = 15 * time.Second
+
+// This is the Client returned by New(). It contains one client implementation,
+// and maintains the refcount.
+var singletonClient = &Client{}
+
+// To override in tests.
+var bootstrapNewConfig = bootstrap.NewConfig
+
+// Client is a full fledged gRPC client which queries a set of discovery APIs
+// (collectively termed as xDS) on a remote management server, to discover
+// various dynamic resources.
+//
+// The xds client is a singleton. It will be shared by the xds resolver and
+// balancer implementations, across multiple ClientConns and Servers.
+type Client struct {
+	*clientImpl
+
+	// This mu protects all the fields, including the embedded clientImpl above.
+	mu       sync.Mutex
+	refCount int
+}
+
+// New returns a new xdsClient configured by the bootstrap file specified in env
+// variable GRPC_XDS_BOOTSTRAP.
+func New() (*Client, error) {
+	singletonClient.mu.Lock()
+	defer singletonClient.mu.Unlock()
+	// If the client implementation was created, increment ref count and return
+	// the client.
+	if singletonClient.clientImpl != nil {
+		singletonClient.refCount++
+		return singletonClient, nil
+	}
+
+	// Create the new client implementation.
+	config, err := bootstrapNewConfig()
+	if err != nil {
+		return nil, fmt.Errorf("xds: failed to read bootstrap file: %v", err)
+	}
+	c, err := newWithConfig(config, defaultWatchExpiryTimeout)
+	if err != nil {
+		return nil, err
+	}
+
+	singletonClient.clientImpl = c
+	singletonClient.refCount++
+	return singletonClient, nil
+}
+
+// Close closes the client. It does ref count of the xds client implementation,
+// and closes the gRPC connection to the management server when ref count
+// reaches 0.
+func (c *Client) Close() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.refCount--
+	if c.refCount == 0 {
+		c.clientImpl.Close()
+		// Set clientImpl back to nil. So if New() is called after this, a new
+		// implementation will be created.
+		c.clientImpl = nil
+	}
+}
+
+// NewWithConfigForTesting is exported for testing only.
+func NewWithConfigForTesting(config *bootstrap.Config, watchExpiryTimeout time.Duration) (*Client, error) {
+	cl, err := newWithConfig(config, watchExpiryTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{clientImpl: cl, refCount: 1}, nil
+}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 932975ce9151..bb17169dbb9b 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/grpcsync"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpctest"
@@ -44,8 +45,7 @@ func Test(t *testing.T) {
 }
 
 const (
-	testXDSServer   = "xds-server"
-	chanRecvTimeout = 100 * time.Millisecond
+	testXDSServer = "xds-server"
 
 	testLDSName = "test-lds"
 	testRDSName = "test-rds"
@@ -57,22 +57,20 @@ const (
 	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
 )
 
-func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options {
-	watchExpiryTimeout := time.Duration(0)
-	if overrideWatchExpiryTImeout {
+func clientOpts(balancerName string, overrideWatchExpiryTimeout bool) (*bootstrap.Config, time.Duration) {
+	watchExpiryTimeout := defaultWatchExpiryTimeout
+	if overrideWatchExpiryTimeout {
 		watchExpiryTimeout = defaultTestWatchExpiryTimeout
 	}
-	return Options{
-		Config: bootstrap.Config{
-			BalancerName: balancerName,
-			Creds:        grpc.WithInsecure(),
-			NodeProto:    xdstestutils.EmptyNodeProtoV2,
-		},
-		WatchExpiryTimeout: watchExpiryTimeout,
-	}
+	return &bootstrap.Config{
+		BalancerName: balancerName,
+		Creds:        grpc.WithInsecure(),
+		NodeProto:    xdstestutils.EmptyNodeProtoV2,
+	}, watchExpiryTimeout
 }
 
 type testAPIClient struct {
+	done          *grpcsync.Event
 	addWatches    map[ResourceType]*testutils.Channel
 	removeWatches map[ResourceType]*testutils.Channel
 }
@@ -102,6 +100,7 @@ func newTestAPIClient() *testAPIClient {
 		EndpointsResource:   testutils.NewChannel(),
 	}
 	return &testAPIClient{
+		done:          grpcsync.NewEvent(),
 		addWatches:    addWatches,
 		removeWatches: removeWatches,
 	}
@@ -118,7 +117,9 @@ func (c *testAPIClient) RemoveWatch(resourceType ResourceType, resourceName stri
 func (c *testAPIClient) reportLoad(context.Context, *grpc.ClientConn, loadReportingOptions) {
 }
 
-func (c *testAPIClient) Close() {}
+func (c *testAPIClient) Close() {
+	c.done.Fire()
+}
 
 // TestWatchCallAnotherWatch covers the case where watch() is called inline by a
 // callback. It makes sure it doesn't cause a deadlock.
@@ -126,7 +127,7 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -216,3 +217,104 @@ func verifyEndpointsUpdate(ctx context.Context, updateCh *testutils.Channel, wan
 	}
 	return nil
 }
+
+// Test that multiple New() returns the same Client. And only when the last
+// client is closed, the underlying client is closed.
+func (s) TestClientNewSingleton(t *testing.T) {
+	oldBootstrapNewConfig := bootstrapNewConfig
+	bootstrapNewConfig = func() (*bootstrap.Config, error) {
+		return &bootstrap.Config{
+			BalancerName: testXDSServer,
+			Creds:        grpc.WithInsecure(),
+			NodeProto:    xdstestutils.EmptyNodeProtoV2,
+		}, nil
+	}
+	defer func() { bootstrapNewConfig = oldBootstrapNewConfig }()
+
+	apiClientCh, cleanup := overrideNewAPIClient()
+	defer cleanup()
+
+	// The first New(). Should create a Client and a new APIClient.
+	client, err := New()
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	clientImpl := client.clientImpl
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient := c.(*testAPIClient)
+
+	// Call New() again. They should all return the same client implementation,
+	// and should not create new API client.
+	const count = 9
+	for i := 0; i < count; i++ {
+		tc, terr := New()
+		if terr != nil {
+			client.Close()
+			t.Fatalf("%d-th call to New() failed with error: %v", i, terr)
+		}
+		if tc.clientImpl != clientImpl {
+			client.Close()
+			tc.Close()
+			t.Fatalf("%d-th call to New() got a different client %p, want %p", i, tc.clientImpl, clientImpl)
+		}
+
+		sctx, scancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		defer scancel()
+		_, err := apiClientCh.Receive(sctx)
+		if err == nil {
+			client.Close()
+			t.Fatalf("%d-th call to New() created a new API client", i)
+		}
+	}
+
+	// Call Close(). Nothing should be actually closed until the last ref calls
+	// Close().
+	for i := 0; i < count; i++ {
+		client.Close()
+		if clientImpl.done.HasFired() {
+			t.Fatalf("%d-th call to Close(), unexpected done in the client implemenation", i)
+		}
+		if apiClient.done.HasFired() {
+			t.Fatalf("%d-th call to Close(), unexpected done in the API client", i)
+		}
+	}
+
+	// Call the last Close(). The underlying implementation and API Client
+	// should all be closed.
+	client.Close()
+	if !clientImpl.done.HasFired() {
+		t.Fatalf("want client implementation to be closed, got not done")
+	}
+	if !apiClient.done.HasFired() {
+		t.Fatalf("want API client to be closed, got not done")
+	}
+
+	// Call New() again after the previous Client is actually closed. Should
+	// create a Client and a new APIClient.
+	client2, err2 := New()
+	if err2 != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer client2.Close()
+	c2, err := apiClientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for API client to be created: %v", err)
+	}
+	apiClient2 := c2.(*testAPIClient)
+
+	// The client wrapper with ref count should be the same.
+	if client2 != client {
+		t.Fatalf("New() after Close() should return the same client wrapper, got different %p, %p", client2, client)
+	}
+	if client2.clientImpl == clientImpl {
+		t.Fatalf("New() after Close() should return different client implementation, got the same %p", client2.clientImpl)
+	}
+	if apiClient2 == apiClient {
+		t.Fatalf("New() after Close() should return different API client, got the same %p", apiClient2)
+	}
+}
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go
index a3524fd9ae38..31704afc0790 100644
--- a/xds/internal/client/client_watchers.go
+++ b/xds/internal/client/client_watchers.go
@@ -35,7 +35,7 @@ const (
 
 // watchInfo holds all the information from a watch() call.
 type watchInfo struct {
-	c      *Client
+	c      *clientImpl
 	rType  ResourceType
 	target string
 
@@ -113,7 +113,7 @@ func (wi *watchInfo) cancel() {
 	wi.state = watchInfoStateCanceled
 }
 
-func (c *Client) watch(wi *watchInfo) (cancel func()) {
+func (c *clientImpl) watch(wi *watchInfo) (cancel func()) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.logger.Debugf("new watch for type %v, resource name %v", wi.rType, wi.target)
@@ -208,7 +208,7 @@ func (c *Client) watch(wi *watchInfo) (cancel func()) {
 // Note that during race (e.g. an xDS response is received while the user is
 // calling cancel()), there's a small window where the callback can be called
 // after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchListener(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) {
+func (c *clientImpl) WatchListener(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
 		rType:       ListenerResource,
@@ -216,7 +216,7 @@ func (c *Client) WatchListener(serviceName string, cb func(ListenerUpdate, error
 		ldsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.watchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
@@ -227,7 +227,7 @@ func (c *Client) WatchListener(serviceName string, cb func(ListenerUpdate, error
 // Note that during race (e.g. an xDS response is received while the user is
 // calling cancel()), there's a small window where the callback can be called
 // after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchRouteConfig(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) {
+func (c *clientImpl) WatchRouteConfig(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
 		rType:       RouteConfigResource,
@@ -235,7 +235,7 @@ func (c *Client) WatchRouteConfig(routeName string, cb func(RouteConfigUpdate, e
 		rdsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.watchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
@@ -250,7 +250,7 @@ func (c *Client) WatchRouteConfig(routeName string, cb func(RouteConfigUpdate, e
 // Note that during race (e.g. an xDS response is received while the user is
 // calling cancel()), there's a small window where the callback can be called
 // after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
+func (c *clientImpl) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
 		rType:       ClusterResource,
@@ -258,7 +258,7 @@ func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error))
 		cdsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.watchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
@@ -272,7 +272,7 @@ func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error))
 // Note that during race (e.g. an xDS response is received while the user is
 // calling cancel()), there's a small window where the callback can be called
 // after the watcher is canceled. The caller needs to handle this case.
-func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
+func (c *clientImpl) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) {
 	wi := &watchInfo{
 		c:           c,
 		rType:       EndpointsResource,
@@ -280,7 +280,7 @@ func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, err
 		edsCallback: cb,
 	}
 
-	wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() {
+	wi.expiryTimer = time.AfterFunc(c.watchExpiryTimeout, func() {
 		wi.timeout()
 	})
 	return c.watch(wi)
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go
index c5126754a401..b6b8d61216df 100644
--- a/xds/internal/client/client_watchers_cluster_test.go
+++ b/xds/internal/client/client_watchers_cluster_test.go
@@ -40,7 +40,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -93,7 +93,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -156,7 +156,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -221,7 +221,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -282,7 +282,7 @@ func (s) TestClusterWatchExpiryTimer(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, true))
+	client, err := newWithConfig(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -321,7 +321,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, true))
+	client, err := newWithConfig(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -369,7 +369,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go
index 822ee596678e..2bf6abab234a 100644
--- a/xds/internal/client/client_watchers_endpoints_test.go
+++ b/xds/internal/client/client_watchers_endpoints_test.go
@@ -58,7 +58,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -110,7 +110,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -175,7 +175,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -240,7 +240,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -299,7 +299,7 @@ func (s) TestEndpointsWatchExpiryTimer(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, true))
+	client, err := newWithConfig(clientOpts(testXDSServer, true))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_listener_test.go b/xds/internal/client/client_watchers_listener_test.go
index 62a0fe493ef3..a233cd305417 100644
--- a/xds/internal/client/client_watchers_listener_test.go
+++ b/xds/internal/client/client_watchers_listener_test.go
@@ -38,7 +38,7 @@ func (s) TestLDSWatch(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -91,7 +91,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -157,7 +157,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -223,7 +223,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -285,7 +285,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/client_watchers_route_test.go b/xds/internal/client/client_watchers_route_test.go
index 02425d6c8fe2..ec04383c2df1 100644
--- a/xds/internal/client/client_watchers_route_test.go
+++ b/xds/internal/client/client_watchers_route_test.go
@@ -40,7 +40,7 @@ func (s) TestRDSWatch(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -99,7 +99,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -171,7 +171,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
@@ -250,7 +250,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 	apiClientCh, cleanup := overrideNewAPIClient()
 	defer cleanup()
 
-	client, err := New(clientOpts(testXDSServer, false))
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
diff --git a/xds/internal/client/tests/client_test.go b/xds/internal/client/tests/client_test.go
index 7887835cbadc..ae1209ae81a8 100644
--- a/xds/internal/client/tests/client_test.go
+++ b/xds/internal/client/tests/client_test.go
@@ -20,6 +20,7 @@ package tests_test
 
 import (
 	"testing"
+	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpctest"
@@ -42,77 +43,67 @@ const (
 	testXDSServer = "xds-server"
 )
 
-func clientOpts(balancerName string) xdsclient.Options {
-	return xdsclient.Options{
-		Config: bootstrap.Config{
-			BalancerName: balancerName,
-			Creds:        grpc.WithInsecure(),
-			NodeProto:    testutils.EmptyNodeProtoV2,
-		},
-	}
-}
-
 func (s) TestNew(t *testing.T) {
 	tests := []struct {
 		name    string
-		opts    xdsclient.Options
+		config  *bootstrap.Config
 		wantErr bool
 	}{
-		{name: "empty-opts", opts: xdsclient.Options{}, wantErr: true},
+		{
+			name:    "empty-opts",
+			config:  &bootstrap.Config{},
+			wantErr: true,
+		},
 		{
 			name: "empty-balancer-name",
-			opts: xdsclient.Options{
-				Config: bootstrap.Config{
-					Creds:     grpc.WithInsecure(),
-					NodeProto: testutils.EmptyNodeProtoV2,
-				},
+			config: &bootstrap.Config{
+				Creds:     grpc.WithInsecure(),
+				NodeProto: testutils.EmptyNodeProtoV2,
 			},
 			wantErr: true,
 		},
 		{
 			name: "empty-dial-creds",
-			opts: xdsclient.Options{
-				Config: bootstrap.Config{
-					BalancerName: testXDSServer,
-					NodeProto:    testutils.EmptyNodeProtoV2,
-				},
+			config: &bootstrap.Config{
+				BalancerName: testXDSServer,
+				NodeProto:    testutils.EmptyNodeProtoV2,
 			},
 			wantErr: true,
 		},
 		{
 			name: "empty-node-proto",
-			opts: xdsclient.Options{
-				Config: bootstrap.Config{
-					BalancerName: testXDSServer,
-					Creds:        grpc.WithInsecure(),
-				},
+			config: &bootstrap.Config{
+				BalancerName: testXDSServer,
+				Creds:        grpc.WithInsecure(),
 			},
 			wantErr: true,
 		},
 		{
 			name: "node-proto-version-mismatch",
-			opts: xdsclient.Options{
-				Config: bootstrap.Config{
-					BalancerName: testXDSServer,
-					Creds:        grpc.WithInsecure(),
-					NodeProto:    testutils.EmptyNodeProtoV3,
-					TransportAPI: version.TransportV2,
-				},
+			config: &bootstrap.Config{
+				BalancerName: testXDSServer,
+				Creds:        grpc.WithInsecure(),
+				NodeProto:    testutils.EmptyNodeProtoV3,
+				TransportAPI: version.TransportV2,
 			},
 			wantErr: true,
 		},
 		// TODO(easwars): Add cases for v3 API client.
 		{
 			name: "happy-case",
-			opts: clientOpts(testXDSServer),
+			config: &bootstrap.Config{
+				BalancerName: testXDSServer,
+				Creds:        grpc.WithInsecure(),
+				NodeProto:    testutils.EmptyNodeProtoV2,
+			},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			c, err := xdsclient.New(test.opts)
+			c, err := xdsclient.NewWithConfigForTesting(test.config, 15*time.Second)
 			if (err != nil) != test.wantErr {
-				t.Fatalf("New(%+v) = %v, wantErr: %v", test.opts, err, test.wantErr)
+				t.Fatalf("New(%+v) = %v, wantErr: %v", test.config, err, test.wantErr)
 			}
 			if c != nil {
 				c.Close()
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index 433e6907d9fe..faf9c6e082e3 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -181,8 +181,9 @@ func (v2c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, stri
 	return rType, resp.GetVersionInfo(), resp.GetNonce(), err
 }
 
-// handleLDSResponse processes an LDS response received from the xDS server. On
-// receipt of a good response, it also invokes the registered watcher callback.
+// handleLDSResponse processes an LDS response received from the management
+// server. On receipt of a good response, it also invokes the registered watcher
+// callback.
 func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, err := xdsclient.UnmarshalListener(resp.GetResources(), v2c.logger)
 	if err != nil {
@@ -192,9 +193,9 @@ func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	return nil
 }
 
-// handleRDSResponse processes an RDS response received from the xDS server. On
-// receipt of a good response, it caches validated resources and also invokes
-// the registered watcher callback.
+// handleRDSResponse processes an RDS response received from the management
+// server. On receipt of a good response, it caches validated resources and also
+// invokes the registered watcher callback.
 func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), v2c.logger)
 	if err != nil {
@@ -204,8 +205,9 @@ func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	return nil
 }
 
-// handleCDSResponse processes an CDS response received from the xDS server. On
-// receipt of a good response, it also invokes the registered watcher callback.
+// handleCDSResponse processes an CDS response received from the management
+// server. On receipt of a good response, it also invokes the registered watcher
+// callback.
 func (v2c *client) handleCDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v2c.logger)
 	if err != nil {
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 00e74e5974e2..de34da819639 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -181,8 +181,9 @@ func (v3c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, stri
 	return rType, resp.GetVersionInfo(), resp.GetNonce(), err
 }
 
-// handleLDSResponse processes an LDS response received from the xDS server. On
-// receipt of a good response, it also invokes the registered watcher callback.
+// handleLDSResponse processes an LDS response received from the management
+// server. On receipt of a good response, it also invokes the registered watcher
+// callback.
 func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, err := xdsclient.UnmarshalListener(resp.GetResources(), v3c.logger)
 	if err != nil {
@@ -192,9 +193,9 @@ func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 	return nil
 }
 
-// handleRDSResponse processes an RDS response received from the xDS server. On
-// receipt of a good response, it caches validated resources and also invokes
-// the registered watcher callback.
+// handleRDSResponse processes an RDS response received from the management
+// server. On receipt of a good response, it caches validated resources and also
+// invokes the registered watcher callback.
 func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), v3c.logger)
 	if err != nil {
@@ -204,8 +205,9 @@ func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 	return nil
 }
 
-// handleCDSResponse processes an CDS response received from the xDS server. On
-// receipt of a good response, it also invokes the registered watcher callback.
+// handleCDSResponse processes an CDS response received from the management
+// server. On receipt of a good response, it also invokes the registered watcher
+// callback.
 func (v3c *client) handleCDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v3c.logger)
 	if err != nil {
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index c3e83231fd22..667e7aea3cfe 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -22,7 +22,6 @@ package resolver
 import (
 	"fmt"
 
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
@@ -30,17 +29,15 @@ import (
 
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
 
 const xdsScheme = "xds"
 
 // For overriding in unittests.
 var (
-	newXDSClient = func(opts xdsclient.Options) (xdsClientInterface, error) {
-		return xdsclient.New(opts)
+	newXDSClient = func() (xdsClientInterface, error) {
+		return xdsclient.New()
 	}
-	newXDSConfig = bootstrap.NewConfig
 )
 
 func init() {
@@ -53,12 +50,7 @@ type xdsResolverBuilder struct{}
 //
 // The xds bootstrap process is performed (and a new xds client is built) every
 // time an xds resolver is built.
-func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rbo resolver.BuildOptions) (resolver.Resolver, error) {
-	config, err := newXDSConfig()
-	if err != nil {
-		return nil, fmt.Errorf("xds: failed to read bootstrap file: %v", err)
-	}
-
+func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
 	r := &xdsResolver{
 		target:   t,
 		cc:       cc,
@@ -68,12 +60,7 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rb
 	r.logger = prefixLogger((r))
 	r.logger.Infof("Creating resolver for target: %+v", t)
 
-	var dopts []grpc.DialOption
-	if rbo.Dialer != nil {
-		dopts = []grpc.DialOption{grpc.WithContextDialer(rbo.Dialer)}
-	}
-
-	client, err := newXDSClient(xdsclient.Options{Config: *config, DialOpts: dopts, TargetName: t.Endpoint})
+	client, err := newXDSClient()
 	if err != nil {
 		return nil, fmt.Errorf("xds: failed to create xds-client: %v", err)
 	}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 776ac24fc714..21714b4152da 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -21,13 +21,10 @@ package resolver
 import (
 	"context"
 	"errors"
-	"fmt"
-	"net"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpctest"
@@ -38,8 +35,6 @@ import (
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
@@ -53,11 +48,6 @@ const (
 )
 
 var (
-	validConfig = bootstrap.Config{
-		BalancerName: balancerName,
-		Creds:        grpc.WithInsecure(),
-		NodeProto:    xdstestutils.EmptyNodeProtoV2,
-	}
 	target = resolver.Target{Endpoint: targetStr}
 )
 
@@ -104,87 +94,24 @@ func newTestClientConn() *testClientConn {
 	}
 }
 
-func getXDSClientMakerFunc(wantOpts xdsclient.Options) func(xdsclient.Options) (xdsClientInterface, error) {
-	return func(gotOpts xdsclient.Options) (xdsClientInterface, error) {
-		if gotOpts.Config.BalancerName != wantOpts.Config.BalancerName {
-			return nil, fmt.Errorf("got balancerName: %s, want: %s", gotOpts.Config.BalancerName, wantOpts.Config.BalancerName)
-		}
-		// We cannot compare two DialOption objects to see if they are equal
-		// because each of these is a function pointer. So, the only thing we
-		// can do here is to check if the got option is nil or not based on
-		// what the want option is. We should be able to do extensive
-		// credential testing in e2e tests.
-		if (gotOpts.Config.Creds != nil) != (wantOpts.Config.Creds != nil) {
-			return nil, fmt.Errorf("got len(creds): %v, want: %v", gotOpts.Config.Creds, wantOpts.Config.Creds)
-		}
-		if len(gotOpts.DialOpts) != len(wantOpts.DialOpts) {
-			return nil, fmt.Errorf("got len(DialOpts): %v, want: %v", len(gotOpts.DialOpts), len(wantOpts.DialOpts))
-		}
-		return fakeclient.NewClient(), nil
-	}
-}
-
-func errorDialer(_ context.Context, _ string) (net.Conn, error) {
-	return nil, errors.New("dial error")
-}
-
 // TestResolverBuilder tests the xdsResolverBuilder's Build method with
 // different parameters.
 func (s) TestResolverBuilder(t *testing.T) {
 	tests := []struct {
 		name          string
-		rbo           resolver.BuildOptions
-		config        bootstrap.Config
-		xdsClientFunc func(xdsclient.Options) (xdsClientInterface, error)
+		xdsClientFunc func() (xdsClientInterface, error)
 		wantErr       bool
 	}{
 		{
-			name:    "empty-config",
-			rbo:     resolver.BuildOptions{},
-			config:  bootstrap.Config{},
-			wantErr: true,
-		},
-		{
-			name: "no-balancer-name-in-config",
-			rbo:  resolver.BuildOptions{},
-			config: bootstrap.Config{
-				Creds:     grpc.WithInsecure(),
-				NodeProto: xdstestutils.EmptyNodeProtoV2,
-			},
-			wantErr: true,
-		},
-		{
-			name: "no-creds-in-config",
-			rbo:  resolver.BuildOptions{},
-			config: bootstrap.Config{
-				BalancerName: balancerName,
-				NodeProto:    xdstestutils.EmptyNodeProtoV2,
+			name: "simple-good",
+			xdsClientFunc: func() (xdsClientInterface, error) {
+				return fakeclient.NewClient(), nil
 			},
-			xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}),
-			wantErr:       true,
-		},
-		{
-			name:   "error-dialer-in-rbo",
-			rbo:    resolver.BuildOptions{Dialer: errorDialer},
-			config: validConfig,
-			xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{
-				Config:   validConfig,
-				DialOpts: []grpc.DialOption{grpc.WithContextDialer(errorDialer)},
-			}),
 			wantErr: false,
 		},
 		{
-			name:          "simple-good",
-			rbo:           resolver.BuildOptions{},
-			config:        validConfig,
-			xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}),
-			wantErr:       false,
-		},
-		{
-			name:   "newXDSClient-throws-error",
-			rbo:    resolver.BuildOptions{},
-			config: validConfig,
-			xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) {
+			name: "newXDSClient-throws-error",
+			xdsClientFunc: func() (xdsClientInterface, error) {
 				return nil, errors.New("newXDSClient-throws-error")
 			},
 			wantErr: true,
@@ -192,19 +119,10 @@ func (s) TestResolverBuilder(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			// Fake out the bootstrap process by providing our own config.
-			oldConfigMaker := newXDSConfig
-			newXDSConfig = func() (*bootstrap.Config, error) {
-				if test.config.BalancerName == "" {
-					return nil, fmt.Errorf("no balancer name found in config")
-				}
-				return &test.config, nil
-			}
 			// Fake out the xdsClient creation process by providing a fake.
 			oldClientMaker := newXDSClient
 			newXDSClient = test.xdsClientFunc
 			defer func() {
-				newXDSConfig = oldConfigMaker
 				newXDSClient = oldClientMaker
 			}()
 
@@ -213,7 +131,7 @@ func (s) TestResolverBuilder(t *testing.T) {
 				t.Fatalf("resolver.Get(%v) returned nil", xdsScheme)
 			}
 
-			r, err := builder.Build(target, newTestClientConn(), test.rbo)
+			r, err := builder.Build(target, newTestClientConn(), resolver.BuildOptions{})
 			if (err != nil) != test.wantErr {
 				t.Fatalf("builder.Build(%v) returned err: %v, wantErr: %v", target, err, test.wantErr)
 			}
@@ -227,19 +145,15 @@ func (s) TestResolverBuilder(t *testing.T) {
 }
 
 type setupOpts struct {
-	config        *bootstrap.Config
-	xdsClientFunc func(xdsclient.Options) (xdsClientInterface, error)
+	xdsClientFunc func() (xdsClientInterface, error)
 }
 
 func testSetup(t *testing.T, opts setupOpts) (*xdsResolver, *testClientConn, func()) {
 	t.Helper()
 
-	oldConfigMaker := newXDSConfig
-	newXDSConfig = func() (*bootstrap.Config, error) { return opts.config, nil }
 	oldClientMaker := newXDSClient
 	newXDSClient = opts.xdsClientFunc
 	cancel := func() {
-		newXDSConfig = oldConfigMaker
 		newXDSClient = oldClientMaker
 	}
 
@@ -291,8 +205,7 @@ func waitForWatchRouteConfig(ctx context.Context, t *testing.T, xdsC *fakeclient
 func (s) TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
-		config:        &validConfig,
-		xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil },
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
 	defer cancel()
 
@@ -324,8 +237,7 @@ func (s) TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 func (s) TestXDSResolverBadServiceUpdate(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
-		config:        &validConfig,
-		xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil },
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
 	defer func() {
 		cancel()
@@ -353,8 +265,7 @@ func (s) TestXDSResolverBadServiceUpdate(t *testing.T) {
 func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
-		config:        &validConfig,
-		xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil },
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
 	defer func() {
 		cancel()
@@ -423,8 +334,7 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
-		config:        &validConfig,
-		xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil },
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
 	defer func() {
 		cancel()
@@ -483,8 +393,7 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
-		config:        &validConfig,
-		xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil },
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
 	defer func() {
 		cancel()
diff --git a/xds/internal/testutils/fakeserver/server.go b/xds/internal/testutils/fakeserver/server.go
index 994f5308f3af..6dd5436b1bd9 100644
--- a/xds/internal/testutils/fakeserver/server.go
+++ b/xds/internal/testutils/fakeserver/server.go
@@ -16,7 +16,7 @@
  *
  */
 
-// Package fakeserver provides a fake implementation of an xDS server.
+// Package fakeserver provides a fake implementation of the management server.
 package fakeserver
 
 import (
diff --git a/xds/server.go b/xds/server.go
index 449c27a958ce..51acf08a8694 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -44,8 +44,8 @@ const (
 
 var (
 	// These new functions will be overridden in unit tests.
-	newXDSClient = func(opts xdsclient.Options) (xdsClientInterface, error) {
-		return xdsclient.New(opts)
+	newXDSClient = func() (xdsClientInterface, error) {
+		return xdsclient.New()
 	}
 	newXDSConfig  = bootstrap.NewConfig
 	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface {
@@ -163,12 +163,7 @@ func (s *GRPCServer) initXDSClient() error {
 		return nil
 	}
 
-	// Read the bootstrap file as part of initializing the xdsClient.
-	config, err := newXDSConfig()
-	if err != nil {
-		return fmt.Errorf("xds: failed to read bootstrap file: %v", err)
-	}
-	client, err := newXDSClient(xdsclient.Options{Config: *config})
+	client, err := newXDSClient()
 	if err != nil {
 		return fmt.Errorf("xds: failed to create xds-client: %v", err)
 	}
diff --git a/xds/server_test.go b/xds/server_test.go
index 8769cf386002..c9b185d71d24 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -197,7 +197,7 @@ func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func())
 
 	clientCh := testutils.NewChannel()
 	origNewXDSClient := newXDSClient
-	newXDSClient = func(xdsclient.Options) (xdsClientInterface, error) {
+	newXDSClient = func() (xdsClientInterface, error) {
 		c := fakeclient.NewClient()
 		clientCh.Send(c)
 		return c, nil
@@ -385,7 +385,7 @@ func (s) TestServeNewClientFailure(t *testing.T) {
 	defer func() { newXDSConfig = origNewXDSConfig }()
 
 	origNewXDSClient := newXDSClient
-	newXDSClient = func(xdsclient.Options) (xdsClientInterface, error) {
+	newXDSClient = func() (xdsClientInterface, error) {
 		return nil, errors.New("xdsClient creation failed")
 	}
 	defer func() { newXDSClient = origNewXDSClient }()

From d2629bd403ebfea6d867ed5f149ab9c875d2dfcb Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 12 Nov 2020 11:13:04 -0800
Subject: [PATCH 273/481] xds/bootstrap: Add support for
 `grpc_server_resource_name_id`. (#4030)

---
 xds/internal/client/bootstrap/bootstrap.go    | 33 +++++---
 .../client/bootstrap/bootstrap_test.go        | 81 +++++++++++++++++++
 2 files changed, 102 insertions(+), 12 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index ca7c96c9928c..87abecd286b3 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -78,6 +78,10 @@ type Config struct {
 	// CertProviderConfigs contains a mapping from certificate provider plugin
 	// instance names to parsed buildable configs.
 	CertProviderConfigs map[string]*certprovider.BuildableConfig
+	// ServerResourceNameID contains the value to be used as the id in the
+	// resource name used to fetch the Listener resource on the xDS-enabled gRPC
+	// server.
+	ServerResourceNameID string
 }
 
 type channelCreds struct {
@@ -103,19 +107,20 @@ type xdsServer struct {
 //          "config": <JSON object containing config for the type>
 //        }
 //      ],
-//      "server_features": [ ... ]
-//		"certificate_providers" : {
-//			"default": {
-//				"plugin_name": "default-plugin-name",
-//				"config": { default plugin config in JSON }
-//			},
-//			"foo": {
-//				"plugin_name": "foo",
-//				"config": { foo plugin config in JSON }
-//			}
-//		}
 //    },
-//    "node": <JSON form of Node proto>
+//    "node": <JSON form of Node proto>,
+//    "server_features": [ ... ],
+//    "certificate_providers" : {
+//      "default": {
+//        "plugin_name": "default-plugin-name",
+//        "config": { default plugin config in JSON }
+//       },
+//      "foo": {
+//        "plugin_name": "foo",
+//        "config": { foo plugin config in JSON }
+//      }
+//    },
+//    "grpc_server_resource_name_id": "grpc/server"
 // }
 //
 // Currently, we support exactly one type of credential, which is
@@ -222,6 +227,10 @@ func NewConfig() (*Config, error) {
 				configs[instance] = bc
 			}
 			config.CertProviderConfigs = configs
+		case "grpc_server_resource_name_id":
+			if err := json.Unmarshal(v, &config.ServerResourceNameID); err != nil {
+				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
+			}
 		}
 		// Do not fail the xDS bootstrap when an unknown field is seen. This can
 		// happen when an older version client reads a newer version bootstrap
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 1b9decac185a..6691ff4b8907 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -239,6 +239,9 @@ func (c *Config) compare(want *Config) error {
 	if diff := cmp.Diff(want.NodeProto, c.NodeProto, cmp.Comparer(proto.Equal)); diff != "" {
 		return fmt.Errorf("config.NodeProto diff (-want, +got):\n%s", diff)
 	}
+	if c.ServerResourceNameID != want.ServerResourceNameID {
+		return fmt.Errorf("config.ServerResourceNameID is %q, want %q", c.ServerResourceNameID, want.ServerResourceNameID)
+	}
 
 	// A vanilla cmp.Equal or cmp.Diff will not produce useful error message
 	// here. So, we iterate through the list of configs and compare them one at
@@ -699,3 +702,81 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 		})
 	}
 }
+
+func TestNewConfigWithServerResourceNameID(t *testing.T) {
+	cancel := setupBootstrapOverride(map[string]string{
+		"badServerResourceNameID": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"grpc_server_resource_name_id": 123456789
+		}`,
+		"goodServerResourceNameID": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"grpc_server_resource_name_id": "grpc/server"
+		}`,
+	})
+	defer cancel()
+
+	tests := []struct {
+		name       string
+		wantConfig *Config
+		wantErr    bool
+	}{
+		{
+			name:    "badServerResourceNameID",
+			wantErr: true,
+		},
+		{
+			name: "goodServerResourceNameID",
+			wantConfig: &Config{
+				BalancerName:         "trafficdirector.googleapis.com:443",
+				Creds:                grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
+				TransportAPI:         version.TransportV2,
+				NodeProto:            v2NodeProto,
+				ServerResourceNameID: "grpc/server",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			origBootstrapFileName := env.BootstrapFileName
+			env.BootstrapFileName = test.name
+			defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
+			c, err := NewConfig()
+			if (err != nil) != test.wantErr {
+				t.Fatalf("NewConfig() returned (%+v, %v), wantErr: %v", c, err, test.wantErr)
+			}
+			if test.wantErr {
+				return
+			}
+			if err := c.compare(test.wantConfig); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

From 22dba5eb4ba042f1be9000cb188e94476e962da4 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 12 Nov 2020 12:13:25 -0800
Subject: [PATCH 274/481] xds: Remove usages of grpc.WithInsecure(). (#4028)

---
 xds/internal/client/bootstrap/bootstrap.go      |  3 ++-
 xds/internal/client/bootstrap/bootstrap_test.go |  5 +++--
 xds/internal/client/client_loadreport_test.go   |  7 ++++---
 xds/internal/client/client_test.go              |  3 ++-
 xds/internal/client/tests/client_test.go        | 11 +++++------
 xds/internal/client/v2/client_test.go           |  4 ++--
 xds/internal/resolver/xds_resolver_test.go      |  5 +----
 xds/internal/testutils/fakeserver/server.go     |  3 ++-
 xds/server_test.go                              |  4 ++--
 9 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 87abecd286b3..e9699717c956 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -32,6 +32,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/xds/internal/env"
@@ -183,7 +184,7 @@ func NewConfig() (*Config, error) {
 					config.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials())
 					break
 				} else if cc.Type == credsInsecure {
-					config.Creds = grpc.WithInsecure()
+					config.Creds = grpc.WithTransportCredentials(insecure.NewCredentials())
 					break
 				}
 			}
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 6691ff4b8907..b9d2b9ab63d9 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -33,6 +33,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/xds/internal/env"
@@ -207,7 +208,7 @@ var (
 	}
 	nilCredsConfigV2 = &Config{
 		BalancerName: "trafficdirector.googleapis.com:443",
-		Creds:        grpc.WithInsecure(),
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 		NodeProto:    v2NodeProto,
 	}
 	nonNilCredsConfigV2 = &Config{
@@ -362,7 +363,7 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 		{
 			"emptyNodeProto", &Config{
 				BalancerName: "trafficdirector.googleapis.com:443",
-				Creds:        grpc.WithInsecure(),
+				Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 				NodeProto: &v2corepb.Node{
 					BuildVersion:         gRPCVersion,
 					UserAgentName:        gRPCUserAgentName,
diff --git a/xds/internal/client/client_loadreport_test.go b/xds/internal/client/client_loadreport_test.go
index c9a2709962eb..e4979505f363 100644
--- a/xds/internal/client/client_loadreport_test.go
+++ b/xds/internal/client/client_loadreport_test.go
@@ -30,6 +30,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/client"
@@ -65,7 +66,7 @@ func (s) TestLRSClient(t *testing.T) {
 
 	xdsC, err := client.NewWithConfigForTesting(&bootstrap.Config{
 		BalancerName: fs.Address,
-		Creds:        grpc.WithInsecure(),
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 		NodeProto:    &v2corepb.Node{},
 		TransportAPI: version.TransportV2,
 	}, defaultClientWatchExpiryTimeout)
@@ -128,12 +129,12 @@ func (s) TestLRSClient(t *testing.T) {
 		t.Fatalf("unexpected load received, want load for cluster, eds, dropped for test")
 	}
 	receivedLoad[0].LoadReportInterval = nil
-	want := (&endpointpb.ClusterStats{
+	want := &endpointpb.ClusterStats{
 		ClusterName:          "cluster",
 		ClusterServiceName:   "eds",
 		TotalDroppedRequests: 1,
 		DroppedRequests:      []*endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
-	})
+	}
 	if d := cmp.Diff(want, receivedLoad[0], protocmp.Transform()); d != "" {
 		t.Fatalf("unexpected load received, want load for cluster, eds, dropped for test, diff (-want +got):\n%s", d)
 	}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index bb17169dbb9b..721492e5f430 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/internal/grpcsync"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
@@ -64,7 +65,7 @@ func clientOpts(balancerName string, overrideWatchExpiryTimeout bool) (*bootstra
 	}
 	return &bootstrap.Config{
 		BalancerName: balancerName,
-		Creds:        grpc.WithInsecure(),
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 		NodeProto:    xdstestutils.EmptyNodeProtoV2,
 	}, watchExpiryTimeout
 }
diff --git a/xds/internal/client/tests/client_test.go b/xds/internal/client/tests/client_test.go
index ae1209ae81a8..f5a57fbcd218 100644
--- a/xds/internal/client/tests/client_test.go
+++ b/xds/internal/client/tests/client_test.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpctest"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
@@ -39,9 +40,7 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-const (
-	testXDSServer = "xds-server"
-)
+const testXDSServer = "xds-server"
 
 func (s) TestNew(t *testing.T) {
 	tests := []struct {
@@ -57,7 +56,7 @@ func (s) TestNew(t *testing.T) {
 		{
 			name: "empty-balancer-name",
 			config: &bootstrap.Config{
-				Creds:     grpc.WithInsecure(),
+				Creds:     grpc.WithTransportCredentials(insecure.NewCredentials()),
 				NodeProto: testutils.EmptyNodeProtoV2,
 			},
 			wantErr: true,
@@ -74,7 +73,7 @@ func (s) TestNew(t *testing.T) {
 			name: "empty-node-proto",
 			config: &bootstrap.Config{
 				BalancerName: testXDSServer,
-				Creds:        grpc.WithInsecure(),
+				Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 			},
 			wantErr: true,
 		},
@@ -82,7 +81,7 @@ func (s) TestNew(t *testing.T) {
 			name: "node-proto-version-mismatch",
 			config: &bootstrap.Config{
 				BalancerName: testXDSServer,
-				Creds:        grpc.WithInsecure(),
+				Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 				NodeProto:    testutils.EmptyNodeProtoV3,
 				TransportAPI: version.TransportV2,
 			},
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index 5c9fbd56d6e4..db97788843ae 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
@@ -60,7 +61,6 @@ const (
 	goodRouteName1           = "GoodRouteConfig1"
 	goodRouteName2           = "GoodRouteConfig2"
 	goodEDSName              = "GoodClusterAssignment1"
-	uninterestingRouteName   = "UninterestingRouteName"
 	uninterestingDomain      = "uninteresting.domain"
 	goodClusterName1         = "GoodClusterName1"
 	goodClusterName2         = "GoodClusterName2"
@@ -635,7 +635,7 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 	rb := manual.NewBuilderWithScheme(scheme)
 	rb.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: "no.such.server"}}})
 
-	cc, err := grpc.Dial(scheme+":///whatever", grpc.WithInsecure(), grpc.WithResolvers(rb))
+	cc, err := grpc.Dial(scheme+":///whatever", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(rb))
 	if err != nil {
 		t.Fatalf("Failed to dial ClientConn: %v", err)
 	}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 21714b4152da..6e2715b64cd1 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -42,14 +42,11 @@ const (
 	targetStr               = "target"
 	routeStr                = "route"
 	cluster                 = "cluster"
-	balancerName            = "dummyBalancer"
 	defaultTestTimeout      = 1 * time.Second
 	defaultTestShortTimeout = 100 * time.Microsecond
 )
 
-var (
-	target = resolver.Target{Endpoint: targetStr}
-)
+var target = resolver.Target{Endpoint: targetStr}
 
 type s struct {
 	grpctest.Tester
diff --git a/xds/internal/testutils/fakeserver/server.go b/xds/internal/testutils/fakeserver/server.go
index 6dd5436b1bd9..d37c1c3ef0e2 100644
--- a/xds/internal/testutils/fakeserver/server.go
+++ b/xds/internal/testutils/fakeserver/server.go
@@ -29,6 +29,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/status"
 
@@ -138,7 +139,7 @@ func (xdsS *Server) XDSClientConn() (*grpc.ClientConn, func(), error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultDialTimeout)
 	defer cancel()
 
-	cc, err := grpc.DialContext(ctx, xdsS.Address, grpc.WithInsecure(), grpc.WithBlock())
+	cc, err := grpc.DialContext(ctx, xdsS.Address, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
 	if err != nil {
 		return nil, nil, fmt.Errorf("grpc.DialContext(%s) failed: %v", xdsS.Address, err)
 	}
diff --git a/xds/server_test.go b/xds/server_test.go
index c9b185d71d24..189edd880111 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -190,7 +190,7 @@ func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func())
 	newXDSConfig = func() (*bootstrap.Config, error) {
 		return &bootstrap.Config{
 			BalancerName: "dummyBalancer",
-			Creds:        grpc.WithInsecure(),
+			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 			NodeProto:    xdstestutils.EmptyNodeProtoV3,
 		}, nil
 	}
@@ -378,7 +378,7 @@ func (s) TestServeNewClientFailure(t *testing.T) {
 	newXDSConfig = func() (*bootstrap.Config, error) {
 		return &bootstrap.Config{
 			BalancerName: "dummyBalancer",
-			Creds:        grpc.WithInsecure(),
+			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
 			NodeProto:    xdstestutils.EmptyNodeProtoV3,
 		}, nil
 	}

From 90f1b3ee835b5acaf18e3ecd3028c7a5cebe5481 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <gargut@google.com>
Date: Thu, 12 Nov 2020 13:52:55 -0800
Subject: [PATCH 275/481] client: use "localhost:port" as authority if target
 is ":port" (#4017)

---
 clientconn.go          |  2 +
 test/authority_test.go | 85 +++++++++++++++++++++++++++++++-----------
 2 files changed, 65 insertions(+), 22 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index 1e551a7041a0..020cdae57e40 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -259,6 +259,8 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		cc.authority = cc.dopts.authority
 	} else if strings.HasPrefix(cc.target, "unix:") {
 		cc.authority = "localhost"
+	} else if strings.HasPrefix(cc.parsedTarget.Endpoint, ":") {
+		cc.authority = "localhost" + cc.parsedTarget.Endpoint
 	} else {
 		// Use endpoint from "scheme://authority/endpoint" as the default
 		// authority for ClientConn.
diff --git a/test/authority_test.go b/test/authority_test.go
index 11c3c4c1af8d..af289bc69ed0 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"sync"
 	"testing"
 	"time"
 
@@ -33,27 +34,31 @@ import (
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
+func authorityChecker(ctx context.Context, expectedAuthority string) (*testpb.Empty, error) {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return nil, status.Error(codes.InvalidArgument, "failed to parse metadata")
+	}
+	auths, ok := md[":authority"]
+	if !ok {
+		return nil, status.Error(codes.InvalidArgument, "no authority header")
+	}
+	if len(auths) != 1 {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("no authority header, auths = %v", auths))
+	}
+	if auths[0] != expectedAuthority {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority))
+	}
+	return &testpb.Empty{}, nil
+}
+
 func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer func(context.Context, string) (net.Conn, error)) {
 	if err := os.RemoveAll(address); err != nil {
 		t.Fatalf("Error removing socket file %v: %v\n", address, err)
 	}
-	us := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-			md, ok := metadata.FromIncomingContext(ctx)
-			if !ok {
-				return nil, status.Error(codes.InvalidArgument, "failed to parse metadata")
-			}
-			auths, ok := md[":authority"]
-			if !ok {
-				return nil, status.Error(codes.InvalidArgument, "no authority header")
-			}
-			if len(auths) < 1 {
-				return nil, status.Error(codes.InvalidArgument, "no authority header")
-			}
-			if auths[0] != expectedAuthority {
-				return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority))
-			}
-			return &testpb.Empty{}, nil
+	ss := &stubServer{
+		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return authorityChecker(ctx, expectedAuthority)
 		},
 		network: "unix",
 		address: address,
@@ -63,14 +68,13 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer
 	if dialer != nil {
 		opts = append(opts, grpc.WithContextDialer(dialer))
 	}
-	if err := us.Start(nil, opts...); err != nil {
+	if err := ss.Start(nil, opts...); err != nil {
 		t.Fatalf("Error starting endpoint server: %v", err)
-		return
 	}
-	defer us.Stop()
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer ss.Stop()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	_, err := us.client.EmptyCall(ctx, &testpb.Empty{})
+	_, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
 	if err != nil {
 		t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err)
 	}
@@ -147,3 +151,40 @@ func (s) TestUnixCustomDialer(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestColonPortAuthority(t *testing.T) {
+	expectedAuthority := ""
+	var authorityMu sync.Mutex
+	ss := &stubServer{
+		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			authorityMu.Lock()
+			defer authorityMu.Unlock()
+			return authorityChecker(ctx, expectedAuthority)
+		},
+		network: "tcp",
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+	_, port, err := net.SplitHostPort(ss.address)
+	if err != nil {
+		t.Fatalf("Failed splitting host from post: %v", err)
+	}
+	authorityMu.Lock()
+	expectedAuthority = "localhost:" + port
+	authorityMu.Unlock()
+	// ss.Start dials, but not the ":[port]" target that is being tested here.
+	// Dial again, with ":[port]" as the target.
+	cc, err := grpc.Dial(":"+port, grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("grpc.Dial(%q) = %v", ss.target, err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	_, err = testpb.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{})
+	if err != nil {
+		t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+}

From 1d01bf995d7474ba59a17a54514bebaaed868ff4 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 16 Nov 2020 11:57:00 -0800
Subject: [PATCH 276/481] grpclb: consider IDLE SubConns as connecting (#4031)

Otherwise, when the first response is received from the grpclb server, the
parent ClientConn enters TransientFailure, and the first several
non-wait-for-ready RPCs will fail.
---
 balancer/grpclb/grpclb.go      | 8 ++++++--
 balancer/grpclb/grpclb_test.go | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go
index a7424cf8d2d7..a43d8964119f 100644
--- a/balancer/grpclb/grpclb.go
+++ b/balancer/grpclb/grpclb.go
@@ -288,7 +288,11 @@ func (lb *lbBalancer) regeneratePicker(resetDrop bool) {
 //
 // The aggregated state is:
 //  - If at least one SubConn in Ready, the aggregated state is Ready;
-//  - Else if at least one SubConn in Connecting, the aggregated state is Connecting;
+//  - Else if at least one SubConn in Connecting or IDLE, the aggregated state is Connecting;
+//    - It's OK to consider IDLE as Connecting. SubConns never stay in IDLE,
+//    they start to connect immediately. But there's a race between the overall
+//    state is reported, and when the new SubConn state arrives. And SubConns
+//    never go back to IDLE.
 //  - Else the aggregated state is TransientFailure.
 func (lb *lbBalancer) aggregateSubConnStates() connectivity.State {
 	var numConnecting uint64
@@ -298,7 +302,7 @@ func (lb *lbBalancer) aggregateSubConnStates() connectivity.State {
 			switch state {
 			case connectivity.Ready:
 				return connectivity.Ready
-			case connectivity.Connecting:
+			case connectivity.Connecting, connectivity.Idle:
 				numConnecting++
 			}
 		}
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index dcc5235703a7..dc94ca877843 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -452,7 +452,7 @@ func (s) TestGRPCLB(t *testing.T) {
 
 	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
-	if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+	if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, <nil>", testC, err)
 	}
 }

From 707e298f55ef081e8a6001502989a55cb02c29e5 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 16 Nov 2020 14:34:08 -0800
Subject: [PATCH 277/481] internal: fix net.Dial fail without IP (#4037)

---
 test/authority_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/authority_test.go b/test/authority_test.go
index af289bc69ed0..e8307fee225d 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -176,7 +176,12 @@ func (s) TestColonPortAuthority(t *testing.T) {
 	authorityMu.Unlock()
 	// ss.Start dials, but not the ":[port]" target that is being tested here.
 	// Dial again, with ":[port]" as the target.
-	cc, err := grpc.Dial(":"+port, grpc.WithInsecure())
+	//
+	// Append "localhost" before calling net.Dial, in case net.Dial on certain
+	// platforms doesn't work well for address without the IP.
+	cc, err := grpc.Dial(":"+port, grpc.WithInsecure(), grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, "tcp", "localhost"+addr)
+	}))
 	if err != nil {
 		t.Fatalf("grpc.Dial(%q) = %v", ss.target, err)
 	}

From 20636e76a99aed65ef8706949d2400079ba003da Mon Sep 17 00:00:00 2001
From: Timothy Gu <timothygu99@gmail.com>
Date: Mon, 16 Nov 2020 19:59:46 -0500
Subject: [PATCH 278/481] protobuf: update protoc to latest version (#4038)

Presently, protoc 3.3.0 (released in 2017) is still being used. This
commit updates the vet.sh script as well as existing generated files to
use protoc 3.14.0 instead.

The only concrete change this brings is updated import paths of
Timestamp and Duration types. However, in practice this is a no-op,
since the types themselves are already aliased to the new import path:

- https://pkg.go.dev/github.com/golang/protobuf@v1.4.3/ptypes/duration#Duration
- https://pkg.go.dev/github.com/golang/protobuf@v1.4.3/ptypes/timestamp#Timestamp
---
 .../grpclb/grpc_lb_v1/load_balancer.pb.go     | 18 ++---
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  2 +-
 .../proto/grpc_lookup_v1/rls_config.pb.go     | 18 ++---
 benchmark/grpc_testing/control.pb.go          |  2 +-
 benchmark/grpc_testing/messages.pb.go         |  2 +-
 benchmark/grpc_testing/payloads.pb.go         |  2 +-
 benchmark/grpc_testing/services.pb.go         |  2 +-
 benchmark/grpc_testing/stats.pb.go            |  2 +-
 binarylog/grpc_binarylog_v1/binarylog.pb.go   | 40 +++++-----
 channelz/grpc_channelz_v1/channelz.pb.go      | 78 +++++++++----------
 .../internal/proto/grpc_gcp/altscontext.pb.go |  2 +-
 .../internal/proto/grpc_gcp/handshaker.pb.go  |  2 +-
 .../grpc_gcp/transport_security_common.pb.go  |  2 +-
 .../internal/meshca_experimental/config.pb.go | 14 ++--
 .../meshca/internal/v1/meshca.pb.go           | 10 +--
 examples/features/proto/echo/echo.pb.go       |  2 +-
 .../helloworld/helloworld/helloworld.pb.go    |  2 +-
 .../route_guide/routeguide/route_guide.pb.go  |  2 +-
 health/grpc_health_v1/health.pb.go            |  2 +-
 .../grpc_service_config/service_config.pb.go  | 56 ++++++-------
 interop/grpc_testing/test.pb.go               |  2 +-
 profiling/proto/service.pb.go                 |  2 +-
 .../grpc_reflection_v1alpha/reflection.pb.go  |  2 +-
 reflection/grpc_testing/proto2.pb.go          |  2 +-
 reflection/grpc_testing/proto2_ext.pb.go      |  2 +-
 reflection/grpc_testing/proto2_ext2.pb.go     |  2 +-
 reflection/grpc_testing/test.pb.go            |  2 +-
 stats/grpc_testing/test.pb.go                 |  2 +-
 stress/grpc_testing/metrics.pb.go             |  2 +-
 test/codec_perf/perf.pb.go                    |  2 +-
 test/grpc_testing/test.pb.go                  |  2 +-
 vet.sh                                        |  4 +-
 32 files changed, 143 insertions(+), 143 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
index 4725858e79d9..c393d7ffd3b2 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go
@@ -20,17 +20,17 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/lb/v1/load_balancer.proto
 
 package grpc_lb_v1
 
 import (
 	proto "github.com/golang/protobuf/proto"
-	duration "github.com/golang/protobuf/ptypes/duration"
-	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -246,7 +246,7 @@ type ClientStats struct {
 	unknownFields protoimpl.UnknownFields
 
 	// The timestamp of generating the report.
-	Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	// The total number of RPCs that started.
 	NumCallsStarted int64 `protobuf:"varint,2,opt,name=num_calls_started,json=numCallsStarted,proto3" json:"num_calls_started,omitempty"`
 	// The total number of RPCs that finished.
@@ -292,7 +292,7 @@ func (*ClientStats) Descriptor() ([]byte, []int) {
 	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{3}
 }
 
-func (x *ClientStats) GetTimestamp() *timestamp.Timestamp {
+func (x *ClientStats) GetTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.Timestamp
 	}
@@ -479,7 +479,7 @@ type InitialLoadBalanceResponse struct {
 	// This interval defines how often the client should send the client stats
 	// to the load balancer. Stats should only be reported when the duration is
 	// positive.
-	ClientStatsReportInterval *duration.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"`
+	ClientStatsReportInterval *durationpb.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"`
 }
 
 func (x *InitialLoadBalanceResponse) Reset() {
@@ -514,7 +514,7 @@ func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) {
 	return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{6}
 }
 
-func (x *InitialLoadBalanceResponse) GetClientStatsReportInterval() *duration.Duration {
+func (x *InitialLoadBalanceResponse) GetClientStatsReportInterval() *durationpb.Duration {
 	if x != nil {
 		return x.ClientStatsReportInterval
 	}
@@ -793,8 +793,8 @@ var file_grpc_lb_v1_load_balancer_proto_goTypes = []interface{}{
 	(*InitialLoadBalanceResponse)(nil), // 6: grpc.lb.v1.InitialLoadBalanceResponse
 	(*ServerList)(nil),                 // 7: grpc.lb.v1.ServerList
 	(*Server)(nil),                     // 8: grpc.lb.v1.Server
-	(*timestamp.Timestamp)(nil),        // 9: google.protobuf.Timestamp
-	(*duration.Duration)(nil),          // 10: google.protobuf.Duration
+	(*timestamppb.Timestamp)(nil),      // 9: google.protobuf.Timestamp
+	(*durationpb.Duration)(nil),        // 10: google.protobuf.Duration
 }
 var file_grpc_lb_v1_load_balancer_proto_depIdxs = []int32{
 	1,  // 0: grpc.lb.v1.LoadBalanceRequest.initial_request:type_name -> grpc.lb.v1.InitialLoadBalanceRequest
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index aad29f104b19..d48a1a6de84d 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/lookup/v1/rls.proto
 
 package grpc_lookup_v1
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
index e8a7a6dae544..6b0924b335fa 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
@@ -15,16 +15,16 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/lookup/v1/rls_config.proto
 
 package grpc_lookup_v1
 
 import (
 	proto "github.com/golang/protobuf/proto"
-	duration "github.com/golang/protobuf/ptypes/duration"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -328,17 +328,17 @@ type RouteLookupConfig struct {
 	LookupService string `protobuf:"bytes,3,opt,name=lookup_service,json=lookupService,proto3" json:"lookup_service,omitempty"`
 	// Configure a timeout value for lookup service requests.
 	// Defaults to 10 seconds if not specified.
-	LookupServiceTimeout *duration.Duration `protobuf:"bytes,4,opt,name=lookup_service_timeout,json=lookupServiceTimeout,proto3" json:"lookup_service_timeout,omitempty"`
+	LookupServiceTimeout *durationpb.Duration `protobuf:"bytes,4,opt,name=lookup_service_timeout,json=lookupServiceTimeout,proto3" json:"lookup_service_timeout,omitempty"`
 	// How long are responses valid for (like HTTP Cache-Control).
 	// If omitted or zero, the longest valid cache time is used.
 	// This value is clamped to 5 minutes to avoid unflushable bad responses.
-	MaxAge *duration.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"`
+	MaxAge *durationpb.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"`
 	// After a response has been in the client cache for this amount of time
 	// and is re-requested, start an asynchronous RPC to re-validate it.
 	// This value should be less than max_age by at least the length of a
 	// typical RTT to the Route Lookup Service to fully mask the RTT latency.
 	// If omitted, keys are only re-requested after they have expired.
-	StaleAge *duration.Duration `protobuf:"bytes,6,opt,name=stale_age,json=staleAge,proto3" json:"stale_age,omitempty"`
+	StaleAge *durationpb.Duration `protobuf:"bytes,6,opt,name=stale_age,json=staleAge,proto3" json:"stale_age,omitempty"`
 	// Rough indicator of amount of memory to use for the client cache.  Some of
 	// the data structure overhead is not accounted for, so actual memory consumed
 	// will be somewhat greater than this value.  If this field is omitted or set
@@ -409,21 +409,21 @@ func (x *RouteLookupConfig) GetLookupService() string {
 	return ""
 }
 
-func (x *RouteLookupConfig) GetLookupServiceTimeout() *duration.Duration {
+func (x *RouteLookupConfig) GetLookupServiceTimeout() *durationpb.Duration {
 	if x != nil {
 		return x.LookupServiceTimeout
 	}
 	return nil
 }
 
-func (x *RouteLookupConfig) GetMaxAge() *duration.Duration {
+func (x *RouteLookupConfig) GetMaxAge() *durationpb.Duration {
 	if x != nil {
 		return x.MaxAge
 	}
 	return nil
 }
 
-func (x *RouteLookupConfig) GetStaleAge() *duration.Duration {
+func (x *RouteLookupConfig) GetStaleAge() *durationpb.Duration {
 	if x != nil {
 		return x.StaleAge
 	}
@@ -611,7 +611,7 @@ var file_grpc_lookup_v1_rls_config_proto_goTypes = []interface{}{
 	(*HttpKeyBuilder)(nil),      // 2: grpc.lookup.v1.HttpKeyBuilder
 	(*RouteLookupConfig)(nil),   // 3: grpc.lookup.v1.RouteLookupConfig
 	(*GrpcKeyBuilder_Name)(nil), // 4: grpc.lookup.v1.GrpcKeyBuilder.Name
-	(*duration.Duration)(nil),   // 5: google.protobuf.Duration
+	(*durationpb.Duration)(nil), // 5: google.protobuf.Duration
 }
 var file_grpc_lookup_v1_rls_config_proto_depIdxs = []int32{
 	4, // 0: grpc.lookup.v1.GrpcKeyBuilder.names:type_name -> grpc.lookup.v1.GrpcKeyBuilder.Name
diff --git a/benchmark/grpc_testing/control.pb.go b/benchmark/grpc_testing/control.pb.go
index 9cb9a2fcde6e..af117985dbda 100644
--- a/benchmark/grpc_testing/control.pb.go
+++ b/benchmark/grpc_testing/control.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: benchmark/grpc_testing/control.proto
 
 package grpc_testing
diff --git a/benchmark/grpc_testing/messages.pb.go b/benchmark/grpc_testing/messages.pb.go
index 924e24e1ef9c..eb6e2975b326 100644
--- a/benchmark/grpc_testing/messages.pb.go
+++ b/benchmark/grpc_testing/messages.pb.go
@@ -17,7 +17,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: benchmark/grpc_testing/messages.proto
 
 package grpc_testing
diff --git a/benchmark/grpc_testing/payloads.pb.go b/benchmark/grpc_testing/payloads.pb.go
index 5e6313896009..f8a3edbd854f 100644
--- a/benchmark/grpc_testing/payloads.pb.go
+++ b/benchmark/grpc_testing/payloads.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: benchmark/grpc_testing/payloads.proto
 
 package grpc_testing
diff --git a/benchmark/grpc_testing/services.pb.go b/benchmark/grpc_testing/services.pb.go
index 70f66ffca46f..17d33e05678b 100644
--- a/benchmark/grpc_testing/services.pb.go
+++ b/benchmark/grpc_testing/services.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: benchmark/grpc_testing/services.proto
 
 package grpc_testing
diff --git a/benchmark/grpc_testing/stats.pb.go b/benchmark/grpc_testing/stats.pb.go
index 9dcf37ac3abd..87fc9bf2d3bc 100644
--- a/benchmark/grpc_testing/stats.pb.go
+++ b/benchmark/grpc_testing/stats.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: benchmark/grpc_testing/stats.proto
 
 package grpc_testing
diff --git a/binarylog/grpc_binarylog_v1/binarylog.pb.go b/binarylog/grpc_binarylog_v1/binarylog.pb.go
index da0472967615..ed75290cdf34 100644
--- a/binarylog/grpc_binarylog_v1/binarylog.pb.go
+++ b/binarylog/grpc_binarylog_v1/binarylog.pb.go
@@ -19,17 +19,17 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/binlog/v1/binarylog.proto
 
 package grpc_binarylog_v1
 
 import (
 	proto "github.com/golang/protobuf/proto"
-	duration "github.com/golang/protobuf/ptypes/duration"
-	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -243,7 +243,7 @@ type GrpcLogEntry struct {
 	unknownFields protoimpl.UnknownFields
 
 	// The timestamp of the binary log message
-	Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	// Uniquely identifies a call. The value must not be 0 in order to disambiguate
 	// from an unset value.
 	// Each call may have several log entries, they will all have the same call_id.
@@ -308,7 +308,7 @@ func (*GrpcLogEntry) Descriptor() ([]byte, []int) {
 	return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0}
 }
 
-func (x *GrpcLogEntry) GetTimestamp() *timestamp.Timestamp {
+func (x *GrpcLogEntry) GetTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.Timestamp
 	}
@@ -439,7 +439,7 @@ type ClientHeader struct {
 	// <host> or <host>:<port> .
 	Authority string `protobuf:"bytes,3,opt,name=authority,proto3" json:"authority,omitempty"`
 	// the RPC timeout
-	Timeout *duration.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"`
+	Timeout *durationpb.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"`
 }
 
 func (x *ClientHeader) Reset() {
@@ -495,7 +495,7 @@ func (x *ClientHeader) GetAuthority() string {
 	return ""
 }
 
-func (x *ClientHeader) GetTimeout() *duration.Duration {
+func (x *ClientHeader) GetTimeout() *durationpb.Duration {
 	if x != nil {
 		return x.Timeout
 	}
@@ -1020,19 +1020,19 @@ func file_grpc_binlog_v1_binarylog_proto_rawDescGZIP() []byte {
 var file_grpc_binlog_v1_binarylog_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
 var file_grpc_binlog_v1_binarylog_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
 var file_grpc_binlog_v1_binarylog_proto_goTypes = []interface{}{
-	(GrpcLogEntry_EventType)(0), // 0: grpc.binarylog.v1.GrpcLogEntry.EventType
-	(GrpcLogEntry_Logger)(0),    // 1: grpc.binarylog.v1.GrpcLogEntry.Logger
-	(Address_Type)(0),           // 2: grpc.binarylog.v1.Address.Type
-	(*GrpcLogEntry)(nil),        // 3: grpc.binarylog.v1.GrpcLogEntry
-	(*ClientHeader)(nil),        // 4: grpc.binarylog.v1.ClientHeader
-	(*ServerHeader)(nil),        // 5: grpc.binarylog.v1.ServerHeader
-	(*Trailer)(nil),             // 6: grpc.binarylog.v1.Trailer
-	(*Message)(nil),             // 7: grpc.binarylog.v1.Message
-	(*Metadata)(nil),            // 8: grpc.binarylog.v1.Metadata
-	(*MetadataEntry)(nil),       // 9: grpc.binarylog.v1.MetadataEntry
-	(*Address)(nil),             // 10: grpc.binarylog.v1.Address
-	(*timestamp.Timestamp)(nil), // 11: google.protobuf.Timestamp
-	(*duration.Duration)(nil),   // 12: google.protobuf.Duration
+	(GrpcLogEntry_EventType)(0),   // 0: grpc.binarylog.v1.GrpcLogEntry.EventType
+	(GrpcLogEntry_Logger)(0),      // 1: grpc.binarylog.v1.GrpcLogEntry.Logger
+	(Address_Type)(0),             // 2: grpc.binarylog.v1.Address.Type
+	(*GrpcLogEntry)(nil),          // 3: grpc.binarylog.v1.GrpcLogEntry
+	(*ClientHeader)(nil),          // 4: grpc.binarylog.v1.ClientHeader
+	(*ServerHeader)(nil),          // 5: grpc.binarylog.v1.ServerHeader
+	(*Trailer)(nil),               // 6: grpc.binarylog.v1.Trailer
+	(*Message)(nil),               // 7: grpc.binarylog.v1.Message
+	(*Metadata)(nil),              // 8: grpc.binarylog.v1.Metadata
+	(*MetadataEntry)(nil),         // 9: grpc.binarylog.v1.MetadataEntry
+	(*Address)(nil),               // 10: grpc.binarylog.v1.Address
+	(*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp
+	(*durationpb.Duration)(nil),   // 12: google.protobuf.Duration
 }
 var file_grpc_binlog_v1_binarylog_proto_depIdxs = []int32{
 	11, // 0: grpc.binarylog.v1.GrpcLogEntry.timestamp:type_name -> google.protobuf.Timestamp
diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go
index ca98e688b0b8..416b3528d5c6 100644
--- a/channelz/grpc_channelz_v1/channelz.pb.go
+++ b/channelz/grpc_channelz_v1/channelz.pb.go
@@ -22,19 +22,19 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/channelz/v1/channelz.proto
 
 package grpc_channelz_v1
 
 import (
 	proto "github.com/golang/protobuf/proto"
-	any "github.com/golang/protobuf/ptypes/any"
-	duration "github.com/golang/protobuf/ptypes/duration"
-	timestamp "github.com/golang/protobuf/ptypes/timestamp"
-	wrappers "github.com/golang/protobuf/ptypes/wrappers"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	anypb "google.golang.org/protobuf/types/known/anypb"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+	wrapperspb "google.golang.org/protobuf/types/known/wrapperspb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -411,7 +411,7 @@ type ChannelData struct {
 	// The number of calls that have completed with a non-OK status
 	CallsFailed int64 `protobuf:"varint,6,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"`
 	// The last time a call was started on the channel.
-	LastCallStartedTimestamp *timestamp.Timestamp `protobuf:"bytes,7,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"`
+	LastCallStartedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"`
 }
 
 func (x *ChannelData) Reset() {
@@ -488,7 +488,7 @@ func (x *ChannelData) GetCallsFailed() int64 {
 	return 0
 }
 
-func (x *ChannelData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
+func (x *ChannelData) GetLastCallStartedTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.LastCallStartedTimestamp
 	}
@@ -507,7 +507,7 @@ type ChannelTraceEvent struct {
 	// the severity of the trace event
 	Severity ChannelTraceEvent_Severity `protobuf:"varint,2,opt,name=severity,proto3,enum=grpc.channelz.v1.ChannelTraceEvent_Severity" json:"severity,omitempty"`
 	// When this event occurred.
-	Timestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	Timestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	// ref of referenced channel or subchannel.
 	// Optional, only present if this event refers to a child object. For example,
 	// this field would be filled if this trace event was for a subchannel being
@@ -565,7 +565,7 @@ func (x *ChannelTraceEvent) GetSeverity() ChannelTraceEvent_Severity {
 	return ChannelTraceEvent_CT_UNKNOWN
 }
 
-func (x *ChannelTraceEvent) GetTimestamp() *timestamp.Timestamp {
+func (x *ChannelTraceEvent) GetTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.Timestamp
 	}
@@ -620,7 +620,7 @@ type ChannelTrace struct {
 	// implementations.
 	NumEventsLogged int64 `protobuf:"varint,1,opt,name=num_events_logged,json=numEventsLogged,proto3" json:"num_events_logged,omitempty"`
 	// Time that this channel was created.
-	CreationTimestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=creation_timestamp,json=creationTimestamp,proto3" json:"creation_timestamp,omitempty"`
+	CreationTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=creation_timestamp,json=creationTimestamp,proto3" json:"creation_timestamp,omitempty"`
 	// List of events that have occurred on this channel.
 	Events []*ChannelTraceEvent `protobuf:"bytes,3,rep,name=events,proto3" json:"events,omitempty"`
 }
@@ -664,7 +664,7 @@ func (x *ChannelTrace) GetNumEventsLogged() int64 {
 	return 0
 }
 
-func (x *ChannelTrace) GetCreationTimestamp() *timestamp.Timestamp {
+func (x *ChannelTrace) GetCreationTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.CreationTimestamp
 	}
@@ -994,7 +994,7 @@ type ServerData struct {
 	// The number of incoming calls that have a completed with a non-OK status
 	CallsFailed int64 `protobuf:"varint,4,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"`
 	// The last time a call was started on the server.
-	LastCallStartedTimestamp *timestamp.Timestamp `protobuf:"bytes,5,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"`
+	LastCallStartedTimestamp *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"`
 }
 
 func (x *ServerData) Reset() {
@@ -1057,7 +1057,7 @@ func (x *ServerData) GetCallsFailed() int64 {
 	return 0
 }
 
-func (x *ServerData) GetLastCallStartedTimestamp() *timestamp.Timestamp {
+func (x *ServerData) GetLastCallStartedTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.LastCallStartedTimestamp
 	}
@@ -1187,22 +1187,22 @@ type SocketData struct {
 	KeepAlivesSent int64 `protobuf:"varint,6,opt,name=keep_alives_sent,json=keepAlivesSent,proto3" json:"keep_alives_sent,omitempty"`
 	// The last time a stream was created by this endpoint.  Usually unset for
 	// servers.
-	LastLocalStreamCreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,7,opt,name=last_local_stream_created_timestamp,json=lastLocalStreamCreatedTimestamp,proto3" json:"last_local_stream_created_timestamp,omitempty"`
+	LastLocalStreamCreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=last_local_stream_created_timestamp,json=lastLocalStreamCreatedTimestamp,proto3" json:"last_local_stream_created_timestamp,omitempty"`
 	// The last time a stream was created by the remote endpoint.  Usually unset
 	// for clients.
-	LastRemoteStreamCreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,8,opt,name=last_remote_stream_created_timestamp,json=lastRemoteStreamCreatedTimestamp,proto3" json:"last_remote_stream_created_timestamp,omitempty"`
+	LastRemoteStreamCreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=last_remote_stream_created_timestamp,json=lastRemoteStreamCreatedTimestamp,proto3" json:"last_remote_stream_created_timestamp,omitempty"`
 	// The last time a message was sent by this endpoint.
-	LastMessageSentTimestamp *timestamp.Timestamp `protobuf:"bytes,9,opt,name=last_message_sent_timestamp,json=lastMessageSentTimestamp,proto3" json:"last_message_sent_timestamp,omitempty"`
+	LastMessageSentTimestamp *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=last_message_sent_timestamp,json=lastMessageSentTimestamp,proto3" json:"last_message_sent_timestamp,omitempty"`
 	// The last time a message was received by this endpoint.
-	LastMessageReceivedTimestamp *timestamp.Timestamp `protobuf:"bytes,10,opt,name=last_message_received_timestamp,json=lastMessageReceivedTimestamp,proto3" json:"last_message_received_timestamp,omitempty"`
+	LastMessageReceivedTimestamp *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=last_message_received_timestamp,json=lastMessageReceivedTimestamp,proto3" json:"last_message_received_timestamp,omitempty"`
 	// The amount of window, granted to the local endpoint by the remote endpoint.
 	// This may be slightly out of date due to network latency.  This does NOT
 	// include stream level or TCP level flow control info.
-	LocalFlowControlWindow *wrappers.Int64Value `protobuf:"bytes,11,opt,name=local_flow_control_window,json=localFlowControlWindow,proto3" json:"local_flow_control_window,omitempty"`
+	LocalFlowControlWindow *wrapperspb.Int64Value `protobuf:"bytes,11,opt,name=local_flow_control_window,json=localFlowControlWindow,proto3" json:"local_flow_control_window,omitempty"`
 	// The amount of window, granted to the remote endpoint by the local endpoint.
 	// This may be slightly out of date due to network latency.  This does NOT
 	// include stream level or TCP level flow control info.
-	RemoteFlowControlWindow *wrappers.Int64Value `protobuf:"bytes,12,opt,name=remote_flow_control_window,json=remoteFlowControlWindow,proto3" json:"remote_flow_control_window,omitempty"`
+	RemoteFlowControlWindow *wrapperspb.Int64Value `protobuf:"bytes,12,opt,name=remote_flow_control_window,json=remoteFlowControlWindow,proto3" json:"remote_flow_control_window,omitempty"`
 	// Socket options set on this socket.  May be absent if 'summary' is set
 	// on GetSocketRequest.
 	Option []*SocketOption `protobuf:"bytes,13,rep,name=option,proto3" json:"option,omitempty"`
@@ -1282,42 +1282,42 @@ func (x *SocketData) GetKeepAlivesSent() int64 {
 	return 0
 }
 
-func (x *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamp.Timestamp {
+func (x *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.LastLocalStreamCreatedTimestamp
 	}
 	return nil
 }
 
-func (x *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamp.Timestamp {
+func (x *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.LastRemoteStreamCreatedTimestamp
 	}
 	return nil
 }
 
-func (x *SocketData) GetLastMessageSentTimestamp() *timestamp.Timestamp {
+func (x *SocketData) GetLastMessageSentTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.LastMessageSentTimestamp
 	}
 	return nil
 }
 
-func (x *SocketData) GetLastMessageReceivedTimestamp() *timestamp.Timestamp {
+func (x *SocketData) GetLastMessageReceivedTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.LastMessageReceivedTimestamp
 	}
 	return nil
 }
 
-func (x *SocketData) GetLocalFlowControlWindow() *wrappers.Int64Value {
+func (x *SocketData) GetLocalFlowControlWindow() *wrapperspb.Int64Value {
 	if x != nil {
 		return x.LocalFlowControlWindow
 	}
 	return nil
 }
 
-func (x *SocketData) GetRemoteFlowControlWindow() *wrappers.Int64Value {
+func (x *SocketData) GetRemoteFlowControlWindow() *wrapperspb.Int64Value {
 	if x != nil {
 		return x.RemoteFlowControlWindow
 	}
@@ -1522,7 +1522,7 @@ type SocketOption struct {
 	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 	// Additional data associated with the socket option.  At least one of value
 	// or additional will be set.
-	Additional *any.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"`
+	Additional *anypb.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"`
 }
 
 func (x *SocketOption) Reset() {
@@ -1571,7 +1571,7 @@ func (x *SocketOption) GetValue() string {
 	return ""
 }
 
-func (x *SocketOption) GetAdditional() *any.Any {
+func (x *SocketOption) GetAdditional() *anypb.Any {
 	if x != nil {
 		return x.Additional
 	}
@@ -1585,7 +1585,7 @@ type SocketOptionTimeout struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Duration *duration.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"`
+	Duration *durationpb.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"`
 }
 
 func (x *SocketOptionTimeout) Reset() {
@@ -1620,7 +1620,7 @@ func (*SocketOptionTimeout) Descriptor() ([]byte, []int) {
 	return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{17}
 }
 
-func (x *SocketOptionTimeout) GetDuration() *duration.Duration {
+func (x *SocketOptionTimeout) GetDuration() *durationpb.Duration {
 	if x != nil {
 		return x.Duration
 	}
@@ -1637,7 +1637,7 @@ type SocketOptionLinger struct {
 	// active maps to `struct linger.l_onoff`
 	Active bool `protobuf:"varint,1,opt,name=active,proto3" json:"active,omitempty"`
 	// duration maps to `struct linger.l_linger`
-	Duration *duration.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"`
+	Duration *durationpb.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"`
 }
 
 func (x *SocketOptionLinger) Reset() {
@@ -1679,7 +1679,7 @@ func (x *SocketOptionLinger) GetActive() bool {
 	return false
 }
 
-func (x *SocketOptionLinger) GetDuration() *duration.Duration {
+func (x *SocketOptionLinger) GetDuration() *durationpb.Duration {
 	if x != nil {
 		return x.Duration
 	}
@@ -2853,7 +2853,7 @@ type Address_OtherAddress struct {
 	// The human readable version of the value.  This value should be set.
 	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	// The actual address message.
-	Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+	Value *anypb.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
 func (x *Address_OtherAddress) Reset() {
@@ -2895,7 +2895,7 @@ func (x *Address_OtherAddress) GetName() string {
 	return ""
 }
 
-func (x *Address_OtherAddress) GetValue() *any.Any {
+func (x *Address_OtherAddress) GetValue() *anypb.Any {
 	if x != nil {
 		return x.Value
 	}
@@ -3012,7 +3012,7 @@ type Security_OtherSecurity struct {
 	// The human readable version of the value.
 	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	// The actual security details message.
-	Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+	Value *anypb.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
 func (x *Security_OtherSecurity) Reset() {
@@ -3054,7 +3054,7 @@ func (x *Security_OtherSecurity) GetName() string {
 	return ""
 }
 
-func (x *Security_OtherSecurity) GetValue() *any.Any {
+func (x *Security_OtherSecurity) GetValue() *anypb.Any {
 	if x != nil {
 		return x.Value
 	}
@@ -3632,10 +3632,10 @@ var file_grpc_channelz_v1_channelz_proto_goTypes = []interface{}{
 	(*Address_OtherAddress)(nil),        // 38: grpc.channelz.v1.Address.OtherAddress
 	(*Security_Tls)(nil),                // 39: grpc.channelz.v1.Security.Tls
 	(*Security_OtherSecurity)(nil),      // 40: grpc.channelz.v1.Security.OtherSecurity
-	(*timestamp.Timestamp)(nil),         // 41: google.protobuf.Timestamp
-	(*wrappers.Int64Value)(nil),         // 42: google.protobuf.Int64Value
-	(*any.Any)(nil),                     // 43: google.protobuf.Any
-	(*duration.Duration)(nil),           // 44: google.protobuf.Duration
+	(*timestamppb.Timestamp)(nil),       // 41: google.protobuf.Timestamp
+	(*wrapperspb.Int64Value)(nil),       // 42: google.protobuf.Int64Value
+	(*anypb.Any)(nil),                   // 43: google.protobuf.Any
+	(*durationpb.Duration)(nil),         // 44: google.protobuf.Duration
 }
 var file_grpc_channelz_v1_channelz_proto_depIdxs = []int32{
 	8,  // 0: grpc.channelz.v1.Channel.ref:type_name -> grpc.channelz.v1.ChannelRef
diff --git a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go
index e0744eef355e..703b48da753b 100644
--- a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/gcp/altscontext.proto
 
 package grpc_gcp
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
index ed94ab265bbb..40570e9bf2de 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/gcp/handshaker.proto
 
 package grpc_gcp
diff --git a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go
index 922b531a55c2..4fc3c79d6a39 100644
--- a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/gcp/transport_security_common.proto
 
 package grpc_gcp
diff --git a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
index afabe3b7b12d..846586782137 100644
--- a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/tls/provider/meshca/experimental/config.proto
 
 // NOTE: This proto will very likely move to a different namespace and a
@@ -26,9 +26,9 @@ package meshca_experimental
 import (
 	v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	proto "github.com/golang/protobuf/proto"
-	duration "github.com/golang/protobuf/ptypes/duration"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -111,11 +111,11 @@ type GoogleMeshCaConfig struct {
 	// Certificate lifetime to request in CSRs sent to the MeshCA.
 	//
 	// A default value of 24h will be used if left unspecified.
-	CertificateLifetime *duration.Duration `protobuf:"bytes,2,opt,name=certificate_lifetime,json=certificateLifetime,proto3" json:"certificate_lifetime,omitempty"`
+	CertificateLifetime *durationpb.Duration `protobuf:"bytes,2,opt,name=certificate_lifetime,json=certificateLifetime,proto3" json:"certificate_lifetime,omitempty"`
 	// How long before certificate expiration should the certificate be renewed.
 	//
 	// A default value of 12h will be used if left unspecified.
-	RenewalGracePeriod *duration.Duration `protobuf:"bytes,3,opt,name=renewal_grace_period,json=renewalGracePeriod,proto3" json:"renewal_grace_period,omitempty"`
+	RenewalGracePeriod *durationpb.Duration `protobuf:"bytes,3,opt,name=renewal_grace_period,json=renewalGracePeriod,proto3" json:"renewal_grace_period,omitempty"`
 	// Type of key.
 	//
 	// RSA keys will be used if left unspecified.
@@ -169,14 +169,14 @@ func (x *GoogleMeshCaConfig) GetServer() *v3.ApiConfigSource {
 	return nil
 }
 
-func (x *GoogleMeshCaConfig) GetCertificateLifetime() *duration.Duration {
+func (x *GoogleMeshCaConfig) GetCertificateLifetime() *durationpb.Duration {
 	if x != nil {
 		return x.CertificateLifetime
 	}
 	return nil
 }
 
-func (x *GoogleMeshCaConfig) GetRenewalGracePeriod() *duration.Duration {
+func (x *GoogleMeshCaConfig) GetRenewalGracePeriod() *durationpb.Duration {
 	if x != nil {
 		return x.RenewalGracePeriod
 	}
@@ -276,7 +276,7 @@ var file_grpc_tls_provider_meshca_experimental_config_proto_goTypes = []interfac
 	(GoogleMeshCaConfig_KeyType)(0), // 0: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.KeyType
 	(*GoogleMeshCaConfig)(nil),      // 1: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig
 	(*v3.ApiConfigSource)(nil),      // 2: envoy.config.core.v3.ApiConfigSource
-	(*duration.Duration)(nil),       // 3: google.protobuf.Duration
+	(*durationpb.Duration)(nil),     // 3: google.protobuf.Duration
 }
 var file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs = []int32{
 	2, // 0: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.server:type_name -> envoy.config.core.v3.ApiConfigSource
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
index bc0fcd452b47..387f8c55abc0 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
@@ -15,16 +15,16 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: istio/google/security/meshca/v1/meshca.proto
 
 package google_security_meshca_v1
 
 import (
 	proto "github.com/golang/protobuf/proto"
-	duration "github.com/golang/protobuf/ptypes/duration"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -52,7 +52,7 @@ type MeshCertificateRequest struct {
 	// PEM-encoded certificate request.
 	Csr string `protobuf:"bytes,2,opt,name=csr,proto3" json:"csr,omitempty"`
 	// Optional: requested certificate validity period.
-	Validity *duration.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"` // Reserved 4
+	Validity *durationpb.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"` // Reserved 4
 }
 
 func (x *MeshCertificateRequest) Reset() {
@@ -101,7 +101,7 @@ func (x *MeshCertificateRequest) GetCsr() string {
 	return ""
 }
 
-func (x *MeshCertificateRequest) GetValidity() *duration.Duration {
+func (x *MeshCertificateRequest) GetValidity() *durationpb.Duration {
 	if x != nil {
 		return x.Validity
 	}
@@ -211,7 +211,7 @@ var file_istio_google_security_meshca_v1_meshca_proto_msgTypes = make([]protoimp
 var file_istio_google_security_meshca_v1_meshca_proto_goTypes = []interface{}{
 	(*MeshCertificateRequest)(nil),  // 0: google.security.meshca.v1.MeshCertificateRequest
 	(*MeshCertificateResponse)(nil), // 1: google.security.meshca.v1.MeshCertificateResponse
-	(*duration.Duration)(nil),       // 2: google.protobuf.Duration
+	(*durationpb.Duration)(nil),     // 2: google.protobuf.Duration
 }
 var file_istio_google_security_meshca_v1_meshca_proto_depIdxs = []int32{
 	2, // 0: google.security.meshca.v1.MeshCertificateRequest.validity:type_name -> google.protobuf.Duration
diff --git a/examples/features/proto/echo/echo.pb.go b/examples/features/proto/echo/echo.pb.go
index b096703e62ee..5af638d5280e 100644
--- a/examples/features/proto/echo/echo.pb.go
+++ b/examples/features/proto/echo/echo.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: examples/features/proto/echo/echo.proto
 
 package echo
diff --git a/examples/helloworld/helloworld/helloworld.pb.go b/examples/helloworld/helloworld/helloworld.pb.go
index 5f9912a1f5f3..2d5cbf5d7805 100644
--- a/examples/helloworld/helloworld/helloworld.pb.go
+++ b/examples/helloworld/helloworld/helloworld.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: examples/helloworld/helloworld/helloworld.proto
 
 package helloworld
diff --git a/examples/route_guide/routeguide/route_guide.pb.go b/examples/route_guide/routeguide/route_guide.pb.go
index f99a1dacda74..85c3033c7cd2 100644
--- a/examples/route_guide/routeguide/route_guide.pb.go
+++ b/examples/route_guide/routeguide/route_guide.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: examples/route_guide/routeguide/route_guide.proto
 
 package routeguide
diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go
index 2b6803ecc72a..a66024d23e30 100644
--- a/health/grpc_health_v1/health.pb.go
+++ b/health/grpc_health_v1/health.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/health/v1/health.proto
 
 package grpc_health_v1
diff --git a/internal/proto/grpc_service_config/service_config.pb.go b/internal/proto/grpc_service_config/service_config.pb.go
index 8130a39bcc83..b50d59d79352 100644
--- a/internal/proto/grpc_service_config/service_config.pb.go
+++ b/internal/proto/grpc_service_config/service_config.pb.go
@@ -25,18 +25,18 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: grpc/service_config/service_config.proto
 
 package grpc_service_config
 
 import (
 	proto "github.com/golang/protobuf/proto"
-	duration "github.com/golang/protobuf/ptypes/duration"
-	wrappers "github.com/golang/protobuf/ptypes/wrappers"
 	code "google.golang.org/genproto/googleapis/rpc/code"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	wrapperspb "google.golang.org/protobuf/types/known/wrapperspb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -133,7 +133,7 @@ type MethodConfig struct {
 	// also affect transient errors encountered during name resolution, which
 	// cannot be caught by the value here, since the service config is
 	// obtained by the gRPC client via name resolution.
-	WaitForReady *wrappers.BoolValue `protobuf:"bytes,2,opt,name=wait_for_ready,json=waitForReady,proto3" json:"wait_for_ready,omitempty"`
+	WaitForReady *wrapperspb.BoolValue `protobuf:"bytes,2,opt,name=wait_for_ready,json=waitForReady,proto3" json:"wait_for_ready,omitempty"`
 	// The default timeout in seconds for RPCs sent to this method. This can be
 	// overridden in code. If no reply is received in the specified amount of
 	// time, the request is aborted and a DEADLINE_EXCEEDED error status
@@ -143,7 +143,7 @@ type MethodConfig struct {
 	// and the value set by the application via the gRPC client API.  If either
 	// one is not set, then the other will be used.  If neither is set, then the
 	// request has no deadline.
-	Timeout *duration.Duration `protobuf:"bytes,3,opt,name=timeout,proto3" json:"timeout,omitempty"`
+	Timeout *durationpb.Duration `protobuf:"bytes,3,opt,name=timeout,proto3" json:"timeout,omitempty"`
 	// The maximum allowed payload size for an individual request or object in a
 	// stream (client->server) in bytes. The size which is measured is the
 	// serialized payload after per-message compression (but before stream
@@ -159,7 +159,7 @@ type MethodConfig struct {
 	// be sent and the client will see a ClientError.
 	// Note that 0 is a valid value, meaning that the request message
 	// must be empty.
-	MaxRequestMessageBytes *wrappers.UInt32Value `protobuf:"bytes,4,opt,name=max_request_message_bytes,json=maxRequestMessageBytes,proto3" json:"max_request_message_bytes,omitempty"`
+	MaxRequestMessageBytes *wrapperspb.UInt32Value `protobuf:"bytes,4,opt,name=max_request_message_bytes,json=maxRequestMessageBytes,proto3" json:"max_request_message_bytes,omitempty"`
 	// The maximum allowed payload size for an individual response or object in a
 	// stream (server->client) in bytes. The size which is measured is the
 	// serialized payload after per-message compression (but before stream
@@ -175,7 +175,7 @@ type MethodConfig struct {
 	// be sent, and a ServerError will be sent to the client instead.
 	// Note that 0 is a valid value, meaning that the response message
 	// must be empty.
-	MaxResponseMessageBytes *wrappers.UInt32Value `protobuf:"bytes,5,opt,name=max_response_message_bytes,json=maxResponseMessageBytes,proto3" json:"max_response_message_bytes,omitempty"`
+	MaxResponseMessageBytes *wrapperspb.UInt32Value `protobuf:"bytes,5,opt,name=max_response_message_bytes,json=maxResponseMessageBytes,proto3" json:"max_response_message_bytes,omitempty"`
 	// Only one of retry_policy or hedging_policy may be set. If neither is set,
 	// RPCs will not be retried or hedged.
 	//
@@ -224,28 +224,28 @@ func (x *MethodConfig) GetName() []*MethodConfig_Name {
 	return nil
 }
 
-func (x *MethodConfig) GetWaitForReady() *wrappers.BoolValue {
+func (x *MethodConfig) GetWaitForReady() *wrapperspb.BoolValue {
 	if x != nil {
 		return x.WaitForReady
 	}
 	return nil
 }
 
-func (x *MethodConfig) GetTimeout() *duration.Duration {
+func (x *MethodConfig) GetTimeout() *durationpb.Duration {
 	if x != nil {
 		return x.Timeout
 	}
 	return nil
 }
 
-func (x *MethodConfig) GetMaxRequestMessageBytes() *wrappers.UInt32Value {
+func (x *MethodConfig) GetMaxRequestMessageBytes() *wrapperspb.UInt32Value {
 	if x != nil {
 		return x.MaxRequestMessageBytes
 	}
 	return nil
 }
 
-func (x *MethodConfig) GetMaxResponseMessageBytes() *wrappers.UInt32Value {
+func (x *MethodConfig) GetMaxResponseMessageBytes() *wrapperspb.UInt32Value {
 	if x != nil {
 		return x.MaxResponseMessageBytes
 	}
@@ -612,7 +612,7 @@ type XdsConfig struct {
 	// If not present, load reporting will be disabled.
 	// If set to the empty string, load reporting will be sent to the same
 	// server that we obtained CDS data from.
-	LrsLoadReportingServerName *wrappers.StringValue `protobuf:"bytes,5,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
+	LrsLoadReportingServerName *wrapperspb.StringValue `protobuf:"bytes,5,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
 }
 
 func (x *XdsConfig) Reset() {
@@ -676,7 +676,7 @@ func (x *XdsConfig) GetEdsServiceName() string {
 	return ""
 }
 
-func (x *XdsConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
+func (x *XdsConfig) GetLrsLoadReportingServerName() *wrapperspb.StringValue {
 	if x != nil {
 		return x.LrsLoadReportingServerName
 	}
@@ -698,7 +698,7 @@ type EdsLoadBalancingPolicyConfig struct {
 	// If unset, no load reporting is done.
 	// If set to empty string, load reporting will be sent to the same
 	// server as we are getting xds data from.
-	LrsLoadReportingServerName *wrappers.StringValue `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
+	LrsLoadReportingServerName *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"`
 	// Locality-picking policy.
 	// This policy's config is expected to be in the format used
 	// by the weighted_target policy.  Note that the config should include
@@ -759,7 +759,7 @@ func (x *EdsLoadBalancingPolicyConfig) GetEdsServiceName() string {
 	return ""
 }
 
-func (x *EdsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() *wrappers.StringValue {
+func (x *EdsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() *wrapperspb.StringValue {
 	if x != nil {
 		return x.LrsLoadReportingServerName
 	}
@@ -1271,10 +1271,10 @@ type MethodConfig_RetryPolicy struct {
 	// random(0,
 	//   min(initial_backoff*backoff_multiplier**(n-1), max_backoff)).
 	// Required. Must be greater than zero.
-	InitialBackoff *duration.Duration `protobuf:"bytes,2,opt,name=initial_backoff,json=initialBackoff,proto3" json:"initial_backoff,omitempty"`
+	InitialBackoff *durationpb.Duration `protobuf:"bytes,2,opt,name=initial_backoff,json=initialBackoff,proto3" json:"initial_backoff,omitempty"`
 	// Required. Must be greater than zero.
-	MaxBackoff        *duration.Duration `protobuf:"bytes,3,opt,name=max_backoff,json=maxBackoff,proto3" json:"max_backoff,omitempty"`
-	BackoffMultiplier float32            `protobuf:"fixed32,4,opt,name=backoff_multiplier,json=backoffMultiplier,proto3" json:"backoff_multiplier,omitempty"` // Required. Must be greater than zero.
+	MaxBackoff        *durationpb.Duration `protobuf:"bytes,3,opt,name=max_backoff,json=maxBackoff,proto3" json:"max_backoff,omitempty"`
+	BackoffMultiplier float32              `protobuf:"fixed32,4,opt,name=backoff_multiplier,json=backoffMultiplier,proto3" json:"backoff_multiplier,omitempty"` // Required. Must be greater than zero.
 	// The set of status codes which may be retried.
 	//
 	// This field is required and must be non-empty.
@@ -1320,14 +1320,14 @@ func (x *MethodConfig_RetryPolicy) GetMaxAttempts() uint32 {
 	return 0
 }
 
-func (x *MethodConfig_RetryPolicy) GetInitialBackoff() *duration.Duration {
+func (x *MethodConfig_RetryPolicy) GetInitialBackoff() *durationpb.Duration {
 	if x != nil {
 		return x.InitialBackoff
 	}
 	return nil
 }
 
-func (x *MethodConfig_RetryPolicy) GetMaxBackoff() *duration.Duration {
+func (x *MethodConfig_RetryPolicy) GetMaxBackoff() *durationpb.Duration {
 	if x != nil {
 		return x.MaxBackoff
 	}
@@ -1366,7 +1366,7 @@ type MethodConfig_HedgingPolicy struct {
 	// The first RPC will be sent immediately, but the max_requests-1 subsequent
 	// hedged RPCs will be sent at intervals of every hedging_delay. Set this
 	// to 0 to immediately send all max_requests RPCs.
-	HedgingDelay *duration.Duration `protobuf:"bytes,2,opt,name=hedging_delay,json=hedgingDelay,proto3" json:"hedging_delay,omitempty"`
+	HedgingDelay *durationpb.Duration `protobuf:"bytes,2,opt,name=hedging_delay,json=hedgingDelay,proto3" json:"hedging_delay,omitempty"`
 	// The set of status codes which indicate other hedged RPCs may still
 	// succeed. If a non-fatal status code is returned by the server, hedged
 	// RPCs will continue. Otherwise, outstanding requests will be canceled and
@@ -1415,7 +1415,7 @@ func (x *MethodConfig_HedgingPolicy) GetMaxAttempts() uint32 {
 	return 0
 }
 
-func (x *MethodConfig_HedgingPolicy) GetHedgingDelay() *duration.Duration {
+func (x *MethodConfig_HedgingPolicy) GetHedgingDelay() *durationpb.Duration {
 	if x != nil {
 		return x.HedgingDelay
 	}
@@ -1682,7 +1682,7 @@ type ServiceConfig_HealthCheckConfig struct {
 	unknownFields protoimpl.UnknownFields
 
 	// Service name to use in the health-checking request.
-	ServiceName *wrappers.StringValue `protobuf:"bytes,1,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
+	ServiceName *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
 }
 
 func (x *ServiceConfig_HealthCheckConfig) Reset() {
@@ -1717,7 +1717,7 @@ func (*ServiceConfig_HealthCheckConfig) Descriptor() ([]byte, []int) {
 	return file_grpc_service_config_service_config_proto_rawDescGZIP(), []int{11, 1}
 }
 
-func (x *ServiceConfig_HealthCheckConfig) GetServiceName() *wrappers.StringValue {
+func (x *ServiceConfig_HealthCheckConfig) GetServiceName() *wrapperspb.StringValue {
 	if x != nil {
 		return x.ServiceName
 	}
@@ -2077,10 +2077,10 @@ var file_grpc_service_config_service_config_proto_goTypes = []interface{}{
 	(*LrsLoadBalancingPolicyConfig_Locality)(nil), // 20: grpc.service_config.LrsLoadBalancingPolicyConfig.Locality
 	(*ServiceConfig_RetryThrottlingPolicy)(nil),   // 21: grpc.service_config.ServiceConfig.RetryThrottlingPolicy
 	(*ServiceConfig_HealthCheckConfig)(nil),       // 22: grpc.service_config.ServiceConfig.HealthCheckConfig
-	(*wrappers.BoolValue)(nil),                    // 23: google.protobuf.BoolValue
-	(*duration.Duration)(nil),                     // 24: google.protobuf.Duration
-	(*wrappers.UInt32Value)(nil),                  // 25: google.protobuf.UInt32Value
-	(*wrappers.StringValue)(nil),                  // 26: google.protobuf.StringValue
+	(*wrapperspb.BoolValue)(nil),                  // 23: google.protobuf.BoolValue
+	(*durationpb.Duration)(nil),                   // 24: google.protobuf.Duration
+	(*wrapperspb.UInt32Value)(nil),                // 25: google.protobuf.UInt32Value
+	(*wrapperspb.StringValue)(nil),                // 26: google.protobuf.StringValue
 	(code.Code)(0),                                // 27: google.rpc.Code
 }
 var file_grpc_service_config_service_config_proto_depIdxs = []int32{
diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 1f3bb0b9f692..8b8178f3ce53 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: interop/grpc_testing/test.proto
 
 package grpc_testing
diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go
index 889c14eb9e89..22bc4bc47f48 100644
--- a/profiling/proto/service.pb.go
+++ b/profiling/proto/service.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: profiling/proto/service.proto
 
 package proto
diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go
index 6b8e0b77f067..1f859f764881 100644
--- a/reflection/grpc_reflection_v1alpha/reflection.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go
@@ -17,7 +17,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: reflection/grpc_reflection_v1alpha/reflection.proto
 
 package grpc_reflection_v1alpha
diff --git a/reflection/grpc_testing/proto2.pb.go b/reflection/grpc_testing/proto2.pb.go
index aa5d35f4c67e..9a8f643adb17 100644
--- a/reflection/grpc_testing/proto2.pb.go
+++ b/reflection/grpc_testing/proto2.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: reflection/grpc_testing/proto2.proto
 
 package grpc_testing
diff --git a/reflection/grpc_testing/proto2_ext.pb.go b/reflection/grpc_testing/proto2_ext.pb.go
index 4d96fa358aea..4fe2b2a17d86 100644
--- a/reflection/grpc_testing/proto2_ext.pb.go
+++ b/reflection/grpc_testing/proto2_ext.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: reflection/grpc_testing/proto2_ext.proto
 
 package grpc_testing
diff --git a/reflection/grpc_testing/proto2_ext2.pb.go b/reflection/grpc_testing/proto2_ext2.pb.go
index 88b96565afdb..e84c44f22c96 100644
--- a/reflection/grpc_testing/proto2_ext2.pb.go
+++ b/reflection/grpc_testing/proto2_ext2.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: reflection/grpc_testing/proto2_ext2.proto
 
 package grpc_testing
diff --git a/reflection/grpc_testing/test.pb.go b/reflection/grpc_testing/test.pb.go
index b5e589e3155d..6740b629d767 100644
--- a/reflection/grpc_testing/test.pb.go
+++ b/reflection/grpc_testing/test.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: reflection/grpc_testing/test.proto
 
 package grpc_testing
diff --git a/stats/grpc_testing/test.pb.go b/stats/grpc_testing/test.pb.go
index 59866b746384..7acea50d128f 100644
--- a/stats/grpc_testing/test.pb.go
+++ b/stats/grpc_testing/test.pb.go
@@ -15,7 +15,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: stats/grpc_testing/test.proto
 
 package grpc_testing
diff --git a/stress/grpc_testing/metrics.pb.go b/stress/grpc_testing/metrics.pb.go
index 45d34c662ec2..f8e95188fec0 100644
--- a/stress/grpc_testing/metrics.pb.go
+++ b/stress/grpc_testing/metrics.pb.go
@@ -22,7 +22,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: stress/grpc_testing/metrics.proto
 
 package grpc_testing
diff --git a/test/codec_perf/perf.pb.go b/test/codec_perf/perf.pb.go
index d9c81ee855a2..d09ce726b63f 100644
--- a/test/codec_perf/perf.pb.go
+++ b/test/codec_perf/perf.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: test/codec_perf/perf.proto
 
 package codec_perf
diff --git a/test/grpc_testing/test.pb.go b/test/grpc_testing/test.pb.go
index a632338e9a6b..89ebb3420f24 100644
--- a/test/grpc_testing/test.pb.go
+++ b/test/grpc_testing/test.pb.go
@@ -18,7 +18,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.25.0
-// 	protoc        v3.3.0
+// 	protoc        v3.14.0
 // source: test/grpc_testing/test.proto
 
 package grpc_testing
diff --git a/vet.sh b/vet.sh
index 48652f6040f2..b41df6dc8607 100755
--- a/vet.sh
+++ b/vet.sh
@@ -53,7 +53,7 @@ if [[ "$1" = "-install" ]]; then
   fi
   if [[ -z "${VET_SKIP_PROTO}" ]]; then
     if [[ "${TRAVIS}" = "true" ]]; then
-      PROTOBUF_VERSION=3.3.0
+      PROTOBUF_VERSION=3.14.0
       PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
       pushd /home/travis
       wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}
@@ -61,7 +61,7 @@ if [[ "$1" = "-install" ]]; then
       bin/protoc --version
       popd
     elif [[ "${GITHUB_ACTIONS}" = "true" ]]; then
-      PROTOBUF_VERSION=3.3.0
+      PROTOBUF_VERSION=3.14.0
       PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
       pushd /home/runner/go
       wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}

From b88744b832669c12e6477a4da0d7c971a73a1a47 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 17 Nov 2020 13:22:28 -0800
Subject: [PATCH 279/481] xds: add ConfigSelector to support RouteAction
 timeouts (#3991)

---
 benchmark/primitives/primitives_test.go       |  26 +++
 .../primitives/safe_config_selector_test.go   | 114 ++++++++++
 clientconn.go                                 |  65 ++++--
 internal/resolver/config_selector.go          |  93 ++++++++
 internal/resolver/config_selector_test.go     | 153 +++++++++++++
 internal/serviceconfig/serviceconfig.go       |  56 +++++
 internal/testutils/channel.go                 |  47 +++-
 service_config.go                             |  74 ++-----
 stream.go                                     |  34 ++-
 test/end2end_test.go                          |   5 +-
 test/resolver_test.go                         | 203 ++++++++++++++++++
 11 files changed, 774 insertions(+), 96 deletions(-)
 create mode 100644 benchmark/primitives/safe_config_selector_test.go
 create mode 100644 internal/resolver/config_selector.go
 create mode 100644 internal/resolver/config_selector_test.go
 create mode 100644 test/resolver_test.go

diff --git a/benchmark/primitives/primitives_test.go b/benchmark/primitives/primitives_test.go
index 71fc26e29418..8272f2d4afe0 100644
--- a/benchmark/primitives/primitives_test.go
+++ b/benchmark/primitives/primitives_test.go
@@ -399,3 +399,29 @@ func runStructTypeAssertion(b *testing.B, fer interface{}) {
 		b.Fatal("error")
 	}
 }
+
+func BenchmarkWaitGroupAddDone(b *testing.B) {
+	wg := sync.WaitGroup{}
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for ; pb.Next(); i++ {
+			wg.Add(1)
+		}
+		for ; i > 0; i-- {
+			wg.Done()
+		}
+	})
+}
+
+func BenchmarkRLockUnlock(b *testing.B) {
+	mu := sync.RWMutex{}
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for ; pb.Next(); i++ {
+			mu.RLock()
+		}
+		for ; i > 0; i-- {
+			mu.RUnlock()
+		}
+	})
+}
diff --git a/benchmark/primitives/safe_config_selector_test.go b/benchmark/primitives/safe_config_selector_test.go
new file mode 100644
index 000000000000..3d368d8d7cf3
--- /dev/null
+++ b/benchmark/primitives/safe_config_selector_test.go
@@ -0,0 +1,114 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Benchmark options for safe config selector type.
+
+package primitives_test
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+type safeUpdaterAtomicAndCounter struct {
+	ptr unsafe.Pointer // *countingFunc
+}
+
+type countingFunc struct {
+	mu sync.RWMutex
+	f  func()
+}
+
+func (s *safeUpdaterAtomicAndCounter) call() {
+	cfPtr := atomic.LoadPointer(&s.ptr)
+	var cf *countingFunc
+	for {
+		cf = (*countingFunc)(cfPtr)
+		cf.mu.RLock()
+		cfPtr2 := atomic.LoadPointer(&s.ptr)
+		if cfPtr == cfPtr2 {
+			// Use cf with confidence!
+			break
+		}
+		// cf changed; try to use the new one instead, because the old one is
+		// no longer valid to use.
+		cf.mu.RUnlock()
+		cfPtr = cfPtr2
+	}
+	defer cf.mu.RUnlock()
+	cf.f()
+}
+
+func (s *safeUpdaterAtomicAndCounter) update(f func()) {
+	newCF := &countingFunc{f: f}
+	oldCFPtr := atomic.SwapPointer(&s.ptr, unsafe.Pointer(newCF))
+	if oldCFPtr == nil {
+		return
+	}
+	(*countingFunc)(oldCFPtr).mu.Lock()
+	(*countingFunc)(oldCFPtr).mu.Unlock() //lint:ignore SA2001 necessary to unlock after locking to unblock any RLocks
+}
+
+type safeUpdaterRWMutex struct {
+	mu sync.RWMutex
+	f  func()
+}
+
+func (s *safeUpdaterRWMutex) call() {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	s.f()
+}
+
+func (s *safeUpdaterRWMutex) update(f func()) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.f = f
+}
+
+type updater interface {
+	call()
+	update(f func())
+}
+
+func benchmarkSafeUpdater(b *testing.B, u updater) {
+	t := time.NewTicker(time.Second)
+	go func() {
+		for range t.C {
+			u.update(func() {})
+		}
+	}()
+	b.RunParallel(func(pb *testing.PB) {
+		u.update(func() {})
+		for pb.Next() {
+			u.call()
+		}
+	})
+	t.Stop()
+}
+
+func BenchmarkSafeUpdaterAtomicAndCounter(b *testing.B) {
+	benchmarkSafeUpdater(b, &safeUpdaterAtomicAndCounter{})
+}
+
+func BenchmarkSafeUpdaterRWMutex(b *testing.B) {
+	benchmarkSafeUpdater(b, &safeUpdaterRWMutex{})
+}
diff --git a/clientconn.go b/clientconn.go
index 020cdae57e40..b35ba94f292d 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -38,6 +38,7 @@ import (
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpcutil"
+	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/resolver"
@@ -104,6 +105,17 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) {
 	return DialContext(context.Background(), target, opts...)
 }
 
+type defaultConfigSelector struct {
+	sc *ServiceConfig
+}
+
+func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) *iresolver.RPCConfig {
+	return &iresolver.RPCConfig{
+		Context:      rpcInfo.Context,
+		MethodConfig: getMethodConfig(dcs.sc, rpcInfo.Method),
+	}
+}
+
 // DialContext creates a client connection to the given target. By default, it's
 // a non-blocking dial (the function won't wait for connections to be
 // established, and connecting happens in the background). To make it a blocking
@@ -224,6 +236,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		case sc, ok := <-cc.dopts.scChan:
 			if ok {
 				cc.sc = &sc
+				cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{&sc})
 				scSet = true
 			}
 		default:
@@ -273,6 +286,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		case sc, ok := <-cc.dopts.scChan:
 			if ok {
 				cc.sc = &sc
+				cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{&sc})
 			}
 		case <-ctx.Done():
 			return nil, ctx.Err()
@@ -479,6 +493,8 @@ type ClientConn struct {
 	balancerBuildOpts balancer.BuildOptions
 	blockingpicker    *pickerWrapper
 
+	safeConfigSelector iresolver.SafeConfigSelector
+
 	mu              sync.RWMutex
 	resolverWrapper *ccResolverWrapper
 	sc              *ServiceConfig
@@ -539,6 +555,7 @@ func (cc *ClientConn) scWatcher() {
 			// TODO: load balance policy runtime change is ignored.
 			// We may revisit this decision in the future.
 			cc.sc = &sc
+			cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{&sc})
 			cc.mu.Unlock()
 		case <-cc.ctx.Done():
 			return
@@ -577,13 +594,13 @@ func init() {
 
 func (cc *ClientConn) maybeApplyDefaultServiceConfig(addrs []resolver.Address) {
 	if cc.sc != nil {
-		cc.applyServiceConfigAndBalancer(cc.sc, addrs)
+		cc.applyServiceConfigAndBalancer(cc.sc, nil, addrs)
 		return
 	}
 	if cc.dopts.defaultServiceConfig != nil {
-		cc.applyServiceConfigAndBalancer(cc.dopts.defaultServiceConfig, addrs)
+		cc.applyServiceConfigAndBalancer(cc.dopts.defaultServiceConfig, &defaultConfigSelector{cc.dopts.defaultServiceConfig}, addrs)
 	} else {
-		cc.applyServiceConfigAndBalancer(emptyServiceConfig, addrs)
+		cc.applyServiceConfigAndBalancer(emptyServiceConfig, &defaultConfigSelector{emptyServiceConfig}, addrs)
 	}
 }
 
@@ -620,7 +637,15 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error {
 		// default, per the error handling design?
 	} else {
 		if sc, ok := s.ServiceConfig.Config.(*ServiceConfig); s.ServiceConfig.Err == nil && ok {
-			cc.applyServiceConfigAndBalancer(sc, s.Addresses)
+			configSelector := iresolver.GetConfigSelector(s)
+			if configSelector != nil {
+				if len(s.ServiceConfig.Config.(*ServiceConfig).Methods) != 0 {
+					channelz.Infof(logger, cc.channelzID, "method configs in service config will be ignored due to presence of config selector")
+				}
+			} else {
+				configSelector = &defaultConfigSelector{sc}
+			}
+			cc.applyServiceConfigAndBalancer(sc, configSelector, s.Addresses)
 		} else {
 			ret = balancer.ErrBadResolverState
 			if cc.balancerWrapper == nil {
@@ -630,6 +655,7 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error {
 				} else {
 					err = status.Errorf(codes.Unavailable, "illegal service config type: %T", s.ServiceConfig.Config)
 				}
+				cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{cc.sc})
 				cc.blockingpicker.updatePicker(base.NewErrPicker(err))
 				cc.csMgr.updateState(connectivity.TransientFailure)
 				cc.mu.Unlock()
@@ -864,6 +890,20 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool {
 	return curAddrFound
 }
 
+func getMethodConfig(sc *ServiceConfig, method string) MethodConfig {
+	if sc == nil {
+		return MethodConfig{}
+	}
+	if m, ok := sc.Methods[method]; ok {
+		return m
+	}
+	i := strings.LastIndex(method, "/")
+	if m, ok := sc.Methods[method[:i+1]]; ok {
+		return m
+	}
+	return sc.Methods[""]
+}
+
 // GetMethodConfig gets the method config of the input method.
 // If there's an exact match for input method (i.e. /service/method), we return
 // the corresponding MethodConfig.
@@ -876,17 +916,7 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig {
 	// TODO: Avoid the locking here.
 	cc.mu.RLock()
 	defer cc.mu.RUnlock()
-	if cc.sc == nil {
-		return MethodConfig{}
-	}
-	if m, ok := cc.sc.Methods[method]; ok {
-		return m
-	}
-	i := strings.LastIndex(method, "/")
-	if m, ok := cc.sc.Methods[method[:i+1]]; ok {
-		return m
-	}
-	return cc.sc.Methods[""]
+	return getMethodConfig(cc.sc, method)
 }
 
 func (cc *ClientConn) healthCheckConfig() *healthCheckConfig {
@@ -909,12 +939,15 @@ func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method st
 	return t, done, nil
 }
 
-func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, addrs []resolver.Address) {
+func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, configSelector iresolver.ConfigSelector, addrs []resolver.Address) {
 	if sc == nil {
 		// should never reach here.
 		return
 	}
 	cc.sc = sc
+	if configSelector != nil {
+		cc.safeConfigSelector.UpdateConfigSelector(configSelector)
+	}
 
 	if cc.sc.retryThrottling != nil {
 		newThrottler := &retryThrottler{
diff --git a/internal/resolver/config_selector.go b/internal/resolver/config_selector.go
new file mode 100644
index 000000000000..5ef9262e51dc
--- /dev/null
+++ b/internal/resolver/config_selector.go
@@ -0,0 +1,93 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package resolver provides internal resolver-related functionality.
+package resolver
+
+import (
+	"context"
+	"sync"
+
+	"google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+)
+
+// ConfigSelector controls what configuration to use for every RPC.
+type ConfigSelector interface {
+	// Selects the configuration for the RPC.
+	SelectConfig(RPCInfo) *RPCConfig
+}
+
+// RPCInfo contains RPC information needed by a ConfigSelector.
+type RPCInfo struct {
+	// Context is the user's context for the RPC and contains headers and
+	// application timeout.  It is passed for interception purposes and for
+	// efficiency reasons.  SelectConfig should not be blocking.
+	Context context.Context
+	Method  string // i.e. "/Service/Method"
+}
+
+// RPCConfig describes the configuration to use for each RPC.
+type RPCConfig struct {
+	// The context to use for the remainder of the RPC; can pass info to LB
+	// policy or affect timeout or metadata.
+	Context      context.Context
+	MethodConfig serviceconfig.MethodConfig // configuration to use for this RPC
+	OnCommitted  func()                     // Called when the RPC has been committed (retries no longer possible)
+}
+
+type csKeyType string
+
+const csKey = csKeyType("grpc.internal.resolver.configSelector")
+
+// SetConfigSelector sets the config selector in state and returns the new
+// state.
+func SetConfigSelector(state resolver.State, cs ConfigSelector) resolver.State {
+	state.Attributes = state.Attributes.WithValues(csKey, cs)
+	return state
+}
+
+// GetConfigSelector retrieves the config selector from state, if present, and
+// returns it or nil if absent.
+func GetConfigSelector(state resolver.State) ConfigSelector {
+	cs, _ := state.Attributes.Value(csKey).(ConfigSelector)
+	return cs
+}
+
+// SafeConfigSelector allows for safe switching of ConfigSelector
+// implementations such that previous values are guaranteed to not be in use
+// when UpdateConfigSelector returns.
+type SafeConfigSelector struct {
+	mu sync.RWMutex
+	cs ConfigSelector
+}
+
+// UpdateConfigSelector swaps to the provided ConfigSelector and blocks until
+// all uses of the previous ConfigSelector have completed.
+func (scs *SafeConfigSelector) UpdateConfigSelector(cs ConfigSelector) {
+	scs.mu.Lock()
+	defer scs.mu.Unlock()
+	scs.cs = cs
+}
+
+// SelectConfig defers to the current ConfigSelector in scs.
+func (scs *SafeConfigSelector) SelectConfig(r RPCInfo) *RPCConfig {
+	scs.mu.RLock()
+	defer scs.mu.RUnlock()
+	return scs.cs.SelectConfig(r)
+}
diff --git a/internal/resolver/config_selector_test.go b/internal/resolver/config_selector_test.go
new file mode 100644
index 000000000000..f41b8eb0ed08
--- /dev/null
+++ b/internal/resolver/config_selector_test.go
@@ -0,0 +1,153 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package resolver
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/serviceconfig"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type fakeConfigSelector struct {
+	selectConfig func(RPCInfo) *RPCConfig
+}
+
+func (f *fakeConfigSelector) SelectConfig(r RPCInfo) *RPCConfig {
+	return f.selectConfig(r)
+}
+
+func (s) TestSafeConfigSelector(t *testing.T) {
+	testRPCInfo := RPCInfo{Method: "test method"}
+
+	retChan1 := make(chan *RPCConfig)
+	retChan2 := make(chan *RPCConfig)
+
+	one := 1
+	two := 2
+
+	resp1 := &RPCConfig{MethodConfig: serviceconfig.MethodConfig{MaxReqSize: &one}}
+	resp2 := &RPCConfig{MethodConfig: serviceconfig.MethodConfig{MaxReqSize: &two}}
+
+	cs1Called := make(chan struct{})
+	cs2Called := make(chan struct{})
+
+	cs1 := &fakeConfigSelector{
+		selectConfig: func(r RPCInfo) *RPCConfig {
+			cs1Called <- struct{}{}
+			if diff := cmp.Diff(r, testRPCInfo); diff != "" {
+				t.Errorf("SelectConfig(%v) called; want %v\n  Diffs:\n%s", r, testRPCInfo, diff)
+			}
+			return <-retChan1
+		},
+	}
+	cs2 := &fakeConfigSelector{
+		selectConfig: func(r RPCInfo) *RPCConfig {
+			cs2Called <- struct{}{}
+			if diff := cmp.Diff(r, testRPCInfo); diff != "" {
+				t.Errorf("SelectConfig(%v) called; want %v\n  Diffs:\n%s", r, testRPCInfo, diff)
+			}
+			return <-retChan2
+		},
+	}
+
+	scs := &SafeConfigSelector{}
+	scs.UpdateConfigSelector(cs1)
+
+	cs1Returned := make(chan struct{})
+	go func() {
+		got := scs.SelectConfig(testRPCInfo) // blocks until send to retChan1
+		if got != resp1 {
+			t.Errorf("SelectConfig(%v) = %v; want %v", testRPCInfo, got, resp1)
+		}
+		close(cs1Returned)
+	}()
+
+	// cs1 is blocked but should be called
+	select {
+	case <-time.After(500 * time.Millisecond):
+		t.Fatalf("timed out waiting for cs1 to be called")
+	case <-cs1Called:
+	}
+
+	// swap in cs2 now that cs1 is called
+	csSwapped := make(chan struct{})
+	go func() {
+		// wait awhile first to ensure cs1 could be called below.
+		time.Sleep(50 * time.Millisecond)
+		scs.UpdateConfigSelector(cs2) // Blocks until cs1 done
+		close(csSwapped)
+	}()
+
+	// Allow cs1 to return and cs2 to eventually be swapped in.
+	retChan1 <- resp1
+
+	cs1Done := false // set when cs2 is first called
+	for dl := time.Now().Add(150 * time.Millisecond); !time.Now().After(dl); {
+		gotConfigChan := make(chan *RPCConfig)
+		go func() {
+			gotConfigChan <- scs.SelectConfig(testRPCInfo)
+		}()
+		select {
+		case <-time.After(500 * time.Millisecond):
+			t.Fatalf("timed out waiting for cs1 or cs2 to be called")
+		case <-cs1Called:
+			// Initially, before swapping to cs2, cs1 should be called
+			retChan1 <- resp1
+			go func() { <-gotConfigChan }()
+			if cs1Done {
+				t.Fatalf("cs1 called after cs2")
+			}
+		case <-cs2Called:
+			// Success! the new config selector is being called
+			if !cs1Done {
+				select {
+				case <-csSwapped:
+				case <-time.After(50 * time.Millisecond):
+					t.Fatalf("timed out waiting for UpdateConfigSelector to return")
+				}
+				select {
+				case <-cs1Returned:
+				case <-time.After(50 * time.Millisecond):
+					t.Fatalf("timed out waiting for cs1 to return")
+				}
+				cs1Done = true
+			}
+			retChan2 <- resp2
+			got := <-gotConfigChan
+			if diff := cmp.Diff(got, resp2); diff != "" {
+				t.Fatalf("SelectConfig(%v) = %v; want %v\n  Diffs:\n%s", testRPCInfo, got, resp2, diff)
+			}
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if !cs1Done {
+		t.Fatalf("timed out waiting for cs2 to be called")
+	}
+}
diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go
index af3e2b5f7e8b..bd4b8875f1a7 100644
--- a/internal/serviceconfig/serviceconfig.go
+++ b/internal/serviceconfig/serviceconfig.go
@@ -22,8 +22,10 @@ package serviceconfig
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
 	externalserviceconfig "google.golang.org/grpc/serviceconfig"
 )
@@ -104,3 +106,57 @@ func (bc *BalancerConfig) UnmarshalJSON(b []byte) error {
 	// case.
 	return fmt.Errorf("invalid loadBalancingConfig: no supported policies found")
 }
+
+// MethodConfig defines the configuration recommended by the service providers for a
+// particular method.
+type MethodConfig struct {
+	// WaitForReady indicates whether RPCs sent to this method should wait until
+	// the connection is ready by default (!failfast). The value specified via the
+	// gRPC client API will override the value set here.
+	WaitForReady *bool
+	// Timeout is the default timeout for RPCs sent to this method. The actual
+	// deadline used will be the minimum of the value specified here and the value
+	// set by the application via the gRPC client API.  If either one is not set,
+	// then the other will be used.  If neither is set, then the RPC has no deadline.
+	Timeout *time.Duration
+	// MaxReqSize is the maximum allowed payload size for an individual request in a
+	// stream (client->server) in bytes. The size which is measured is the serialized
+	// payload after per-message compression (but before stream compression) in bytes.
+	// The actual value used is the minimum of the value specified here and the value set
+	// by the application via the gRPC client API. If either one is not set, then the other
+	// will be used.  If neither is set, then the built-in default is used.
+	MaxReqSize *int
+	// MaxRespSize is the maximum allowed payload size for an individual response in a
+	// stream (server->client) in bytes.
+	MaxRespSize *int
+	// RetryPolicy configures retry options for the method.
+	RetryPolicy *RetryPolicy
+}
+
+// RetryPolicy defines the go-native version of the retry policy defined by the
+// service config here:
+// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config
+type RetryPolicy struct {
+	// MaxAttempts is the maximum number of attempts, including the original RPC.
+	//
+	// This field is required and must be two or greater.
+	MaxAttempts int
+
+	// Exponential backoff parameters. The initial retry attempt will occur at
+	// random(0, initialBackoff). In general, the nth attempt will occur at
+	// random(0,
+	//   min(initialBackoff*backoffMultiplier**(n-1), maxBackoff)).
+	//
+	// These fields are required and must be greater than zero.
+	InitialBackoff    time.Duration
+	MaxBackoff        time.Duration
+	BackoffMultiplier float64
+
+	// The set of status codes which may be retried.
+	//
+	// Status codes are specified as strings, e.g., "UNAVAILABLE".
+	//
+	// This field is required and must be non-empty.
+	// Note: a set is used to store this for easy lookup.
+	RetryableStatusCodes map[codes.Code]bool
+}
diff --git a/internal/testutils/channel.go b/internal/testutils/channel.go
index c92b1311c0ed..6a08a94a099f 100644
--- a/internal/testutils/channel.go
+++ b/internal/testutils/channel.go
@@ -30,17 +30,50 @@ type Channel struct {
 }
 
 // Send sends value on the underlying channel.
-func (cwt *Channel) Send(value interface{}) {
-	cwt.ch <- value
+func (c *Channel) Send(value interface{}) {
+	c.ch <- value
+}
+
+// SendContext sends value on the underlying channel, or returns an error if
+// the context expires.
+func (c *Channel) SendContext(ctx context.Context, value interface{}) error {
+	select {
+	case c.ch <- value:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// SendOrFail attempts to send value on the underlying channel.  Returns true
+// if successful or false if the channel was full.
+func (c *Channel) SendOrFail(value interface{}) bool {
+	select {
+	case c.ch <- value:
+		return true
+	default:
+		return false
+	}
+}
+
+// ReceiveOrFail returns the value on the underlying channel and true, or nil
+// and false if the channel was empty.
+func (c *Channel) ReceiveOrFail() (interface{}, bool) {
+	select {
+	case got := <-c.ch:
+		return got, true
+	default:
+		return nil, false
+	}
 }
 
 // Receive returns the value received on the underlying channel, or the error
 // returned by ctx if it is closed or cancelled.
-func (cwt *Channel) Receive(ctx context.Context) (interface{}, error) {
+func (c *Channel) Receive(ctx context.Context) (interface{}, error) {
 	select {
 	case <-ctx.Done():
 		return nil, ctx.Err()
-	case got := <-cwt.ch:
+	case got := <-c.ch:
 		return got, nil
 	}
 }
@@ -50,12 +83,12 @@ func (cwt *Channel) Receive(ctx context.Context) (interface{}, error) {
 // It's expected to be used with a size-1 channel, to only keep the most
 // up-to-date item. This method is inherently racy when invoked concurrently
 // from multiple goroutines.
-func (cwt *Channel) Replace(value interface{}) {
+func (c *Channel) Replace(value interface{}) {
 	for {
 		select {
-		case cwt.ch <- value:
+		case c.ch <- value:
 			return
-		case <-cwt.ch:
+		case <-c.ch:
 		}
 	}
 }
diff --git a/service_config.go b/service_config.go
index 5e434ca7f354..22c4240cf7e8 100644
--- a/service_config.go
+++ b/service_config.go
@@ -41,29 +41,7 @@ const maxInt = int(^uint(0) >> 1)
 // Deprecated: Users should not use this struct. Service config should be received
 // through name resolver, as specified here
 // https://github.com/grpc/grpc/blob/master/doc/service_config.md
-type MethodConfig struct {
-	// WaitForReady indicates whether RPCs sent to this method should wait until
-	// the connection is ready by default (!failfast). The value specified via the
-	// gRPC client API will override the value set here.
-	WaitForReady *bool
-	// Timeout is the default timeout for RPCs sent to this method. The actual
-	// deadline used will be the minimum of the value specified here and the value
-	// set by the application via the gRPC client API.  If either one is not set,
-	// then the other will be used.  If neither is set, then the RPC has no deadline.
-	Timeout *time.Duration
-	// MaxReqSize is the maximum allowed payload size for an individual request in a
-	// stream (client->server) in bytes. The size which is measured is the serialized
-	// payload after per-message compression (but before stream compression) in bytes.
-	// The actual value used is the minimum of the value specified here and the value set
-	// by the application via the gRPC client API. If either one is not set, then the other
-	// will be used.  If neither is set, then the built-in default is used.
-	MaxReqSize *int
-	// MaxRespSize is the maximum allowed payload size for an individual response in a
-	// stream (server->client) in bytes.
-	MaxRespSize *int
-	// RetryPolicy configures retry options for the method.
-	retryPolicy *retryPolicy
-}
+type MethodConfig = internalserviceconfig.MethodConfig
 
 type lbConfig struct {
 	name string
@@ -127,34 +105,6 @@ type healthCheckConfig struct {
 	ServiceName string
 }
 
-// retryPolicy defines the go-native version of the retry policy defined by the
-// service config here:
-// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config
-type retryPolicy struct {
-	// MaxAttempts is the maximum number of attempts, including the original RPC.
-	//
-	// This field is required and must be two or greater.
-	maxAttempts int
-
-	// Exponential backoff parameters. The initial retry attempt will occur at
-	// random(0, initialBackoff). In general, the nth attempt will occur at
-	// random(0,
-	//   min(initialBackoff*backoffMultiplier**(n-1), maxBackoff)).
-	//
-	// These fields are required and must be greater than zero.
-	initialBackoff    time.Duration
-	maxBackoff        time.Duration
-	backoffMultiplier float64
-
-	// The set of status codes which may be retried.
-	//
-	// Status codes are specified as strings, e.g., "UNAVAILABLE".
-	//
-	// This field is required and must be non-empty.
-	// Note: a set is used to store this for easy lookup.
-	retryableStatusCodes map[codes.Code]bool
-}
-
 type jsonRetryPolicy struct {
 	MaxAttempts          int
 	InitialBackoff       string
@@ -313,7 +263,7 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 			WaitForReady: m.WaitForReady,
 			Timeout:      d,
 		}
-		if mc.retryPolicy, err = convertRetryPolicy(m.RetryPolicy); err != nil {
+		if mc.RetryPolicy, err = convertRetryPolicy(m.RetryPolicy); err != nil {
 			logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err)
 			return &serviceconfig.ParseResult{Err: err}
 		}
@@ -359,7 +309,7 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult {
 	return &serviceconfig.ParseResult{Config: &sc}
 }
 
-func convertRetryPolicy(jrp *jsonRetryPolicy) (p *retryPolicy, err error) {
+func convertRetryPolicy(jrp *jsonRetryPolicy) (p *internalserviceconfig.RetryPolicy, err error) {
 	if jrp == nil {
 		return nil, nil
 	}
@@ -381,19 +331,19 @@ func convertRetryPolicy(jrp *jsonRetryPolicy) (p *retryPolicy, err error) {
 		return nil, nil
 	}
 
-	rp := &retryPolicy{
-		maxAttempts:          jrp.MaxAttempts,
-		initialBackoff:       *ib,
-		maxBackoff:           *mb,
-		backoffMultiplier:    jrp.BackoffMultiplier,
-		retryableStatusCodes: make(map[codes.Code]bool),
+	rp := &internalserviceconfig.RetryPolicy{
+		MaxAttempts:          jrp.MaxAttempts,
+		InitialBackoff:       *ib,
+		MaxBackoff:           *mb,
+		BackoffMultiplier:    jrp.BackoffMultiplier,
+		RetryableStatusCodes: make(map[codes.Code]bool),
 	}
-	if rp.maxAttempts > 5 {
+	if rp.MaxAttempts > 5 {
 		// TODO(retry): Make the max maxAttempts configurable.
-		rp.maxAttempts = 5
+		rp.MaxAttempts = 5
 	}
 	for _, code := range jrp.RetryableStatusCodes {
-		rp.retryableStatusCodes[code] = true
+		rp.RetryableStatusCodes[code] = true
 	}
 	return rp, nil
 }
diff --git a/stream.go b/stream.go
index 5fd856a38212..8d4694a33f90 100644
--- a/stream.go
+++ b/stream.go
@@ -36,6 +36,8 @@ import (
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpcutil"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
@@ -170,7 +172,18 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 	if err := cc.waitForResolvedAddrs(ctx); err != nil {
 		return nil, err
 	}
-	mc := cc.GetMethodConfig(method)
+
+	var mc serviceconfig.MethodConfig
+	var onCommit func()
+	rpcConfig := cc.safeConfigSelector.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: method})
+	if rpcConfig != nil {
+		if rpcConfig.Context != nil {
+			ctx = rpcConfig.Context
+		}
+		mc = rpcConfig.MethodConfig
+		onCommit = rpcConfig.OnCommitted
+	}
+
 	if mc.WaitForReady != nil {
 		c.failFast = !*mc.WaitForReady
 	}
@@ -272,6 +285,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 		cancel:       cancel,
 		beginTime:    beginTime,
 		firstAttempt: true,
+		onCommit:     onCommit,
 	}
 	if !cc.dopts.disableRetry {
 		cs.retryThrottler = cc.retryThrottler.Load().(*retryThrottler)
@@ -432,7 +446,8 @@ type clientStream struct {
 	// place where we need to check if the attempt is nil.
 	attempt *csAttempt
 	// TODO(hedging): hedging will have multiple attempts simultaneously.
-	committed  bool                       // active attempt committed for retry?
+	committed  bool // active attempt committed for retry?
+	onCommit   func()
 	buffer     []func(a *csAttempt) error // operations to replay on retry
 	bufferSize int                        // current size of buffer
 }
@@ -461,6 +476,9 @@ type csAttempt struct {
 }
 
 func (cs *clientStream) commitAttemptLocked() {
+	if !cs.committed && cs.onCommit != nil {
+		cs.onCommit()
+	}
 	cs.committed = true
 	cs.buffer = nil
 }
@@ -539,8 +557,8 @@ func (cs *clientStream) shouldRetry(err error) error {
 		code = status.Convert(err).Code()
 	}
 
-	rp := cs.methodConfig.retryPolicy
-	if rp == nil || !rp.retryableStatusCodes[code] {
+	rp := cs.methodConfig.RetryPolicy
+	if rp == nil || !rp.RetryableStatusCodes[code] {
 		return err
 	}
 
@@ -549,7 +567,7 @@ func (cs *clientStream) shouldRetry(err error) error {
 	if cs.retryThrottler.throttle() {
 		return err
 	}
-	if cs.numRetries+1 >= rp.maxAttempts {
+	if cs.numRetries+1 >= rp.MaxAttempts {
 		return err
 	}
 
@@ -558,9 +576,9 @@ func (cs *clientStream) shouldRetry(err error) error {
 		dur = time.Millisecond * time.Duration(pushback)
 		cs.numRetriesSincePushback = 0
 	} else {
-		fact := math.Pow(rp.backoffMultiplier, float64(cs.numRetriesSincePushback))
-		cur := float64(rp.initialBackoff) * fact
-		if max := float64(rp.maxBackoff); cur > max {
+		fact := math.Pow(rp.BackoffMultiplier, float64(cs.numRetriesSincePushback))
+		cur := float64(rp.InitialBackoff) * fact
+		if max := float64(rp.MaxBackoff); cur > max {
 			cur = max
 		}
 		dur = time.Duration(grpcrand.Int63n(int64(cur)))
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 07efccd0ac76..a3850d20874d 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -5784,10 +5784,9 @@ func testGetMethodConfigTD(t *testing.T, e env) {
 	ch <- sc
 	// Wait for the new service config to propagate.
 	for {
-		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.DeadlineExceeded {
-			continue
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
+			break
 		}
-		break
 	}
 	// The following RPCs are expected to become fail-fast.
 	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
diff --git a/test/resolver_test.go b/test/resolver_test.go
new file mode 100644
index 000000000000..6f50047edadc
--- /dev/null
+++ b/test/resolver_test.go
@@ -0,0 +1,203 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
+
+type funcConfigSelector struct {
+	f func(iresolver.RPCInfo) *iresolver.RPCConfig
+}
+
+func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) *iresolver.RPCConfig {
+	return f.f(i)
+}
+
+func (s) TestConfigSelector(t *testing.T) {
+	gotContextChan := testutils.NewChannelWithSize(1)
+
+	ss := &stubServer{
+		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			gotContextChan.SendContext(ctx, ctx)
+			return &testpb.Empty{}, nil
+		},
+	}
+	ss.r = manual.NewBuilderWithScheme("confSel")
+
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctxDeadline := time.Now().Add(10 * time.Second)
+	ctx, cancel := context.WithDeadline(context.Background(), ctxDeadline)
+	defer cancel()
+
+	longCtxDeadline := time.Now().Add(30 * time.Second)
+	longdeadlineCtx, cancel := context.WithDeadline(context.Background(), longCtxDeadline)
+	defer cancel()
+	shorterTimeout := 3 * time.Second
+
+	testMD := metadata.MD{"footest": []string{"bazbar"}}
+	mdOut := metadata.MD{"handler": []string{"value"}}
+
+	var onCommittedCalled bool
+
+	testCases := []struct {
+		name   string
+		md     metadata.MD
+		config *iresolver.RPCConfig
+
+		wantMD       metadata.MD
+		wantDeadline time.Time
+		wantTimeout  time.Duration
+	}{{
+		name:         "basic",
+		md:           testMD,
+		config:       &iresolver.RPCConfig{},
+		wantMD:       testMD,
+		wantDeadline: ctxDeadline,
+	}, {
+		name: "alter MD",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			Context: metadata.NewOutgoingContext(ctx, mdOut),
+		},
+		wantMD:       mdOut,
+		wantDeadline: ctxDeadline,
+	}, {
+		name: "alter timeout; remove MD",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			Context: longdeadlineCtx, // no metadata
+		},
+		wantMD:       nil,
+		wantDeadline: longCtxDeadline,
+	}, {
+		name:         "nil config",
+		md:           metadata.MD{},
+		config:       nil,
+		wantMD:       nil,
+		wantDeadline: ctxDeadline,
+	}, {
+		name: "alter timeout via method config; remove MD",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			MethodConfig: serviceconfig.MethodConfig{
+				Timeout: &shorterTimeout,
+			},
+		},
+		wantMD:      nil,
+		wantTimeout: shorterTimeout,
+	}, {
+		name: "onCommitted callback",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			OnCommitted: func() {
+				onCommittedCalled = true
+			},
+		},
+		wantMD:       testMD,
+		wantDeadline: ctxDeadline,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var gotInfo *iresolver.RPCInfo
+			state := iresolver.SetConfigSelector(resolver.State{
+				Addresses:     []resolver.Address{{Addr: ss.address}},
+				ServiceConfig: parseCfg(ss.r, "{}"),
+			}, funcConfigSelector{
+				f: func(i iresolver.RPCInfo) *iresolver.RPCConfig {
+					gotInfo = &i
+					cfg := tc.config
+					if cfg != nil && cfg.Context == nil {
+						cfg.Context = i.Context
+					}
+					return cfg
+				},
+			})
+			ss.r.UpdateState(state) // Blocks until config selector is applied
+
+			onCommittedCalled = false
+			ctx := metadata.NewOutgoingContext(ctx, tc.md)
+			startTime := time.Now()
+			if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, nil", err)
+			}
+
+			if gotInfo == nil {
+				t.Fatalf("no config selector data")
+			}
+
+			if want := "/grpc.testing.TestService/EmptyCall"; gotInfo.Method != want {
+				t.Errorf("gotInfo.Method = %q; want %q", gotInfo.Method, want)
+			}
+
+			gotContextI, ok := gotContextChan.ReceiveOrFail()
+			if !ok {
+				t.Fatalf("no context received")
+			}
+			gotContext := gotContextI.(context.Context)
+
+			gotMD, _ := metadata.FromOutgoingContext(gotInfo.Context)
+			if diff := cmp.Diff(tc.md, gotMD); diff != "" {
+				t.Errorf("gotInfo.Context contains MD %v; want %v\nDiffs: %v", gotMD, tc.md, diff)
+			}
+
+			gotMD, _ = metadata.FromIncomingContext(gotContext)
+			// Remove entries from gotMD not in tc.wantMD (e.g. authority header).
+			for k := range gotMD {
+				if _, ok := tc.wantMD[k]; !ok {
+					delete(gotMD, k)
+				}
+			}
+			if diff := cmp.Diff(tc.wantMD, gotMD, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("received md = %v; want %v\nDiffs: %v", gotMD, tc.wantMD, diff)
+			}
+
+			wantDeadline := tc.wantDeadline
+			if wantDeadline == (time.Time{}) {
+				wantDeadline = startTime.Add(tc.wantTimeout)
+			}
+			deadlineGot, _ := gotContext.Deadline()
+			if diff := deadlineGot.Sub(wantDeadline); diff > time.Second || diff < -time.Second {
+				t.Errorf("received deadline = %v; want ~%v", deadlineGot, wantDeadline)
+			}
+
+			if tc.config != nil && tc.config.OnCommitted != nil && !onCommittedCalled {
+				t.Errorf("OnCommitted callback not called")
+			}
+		})
+	}
+
+}

From f5c42ca71413d2fd9959b9e2f543c42f45fd3a12 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 17 Nov 2020 13:31:38 -0800
Subject: [PATCH 280/481] xds/client: Export a method to return bootstrap
 config. (#4033)

---
 .../balancer/cdsbalancer/cdsbalancer.go       |  8 +++---
 .../cdsbalancer/cdsbalancer_security_test.go  | 25 +++++++++++--------
 xds/internal/client/client.go                 | 11 ++++----
 xds/internal/testutils/fakeclient/client.go   | 16 ++++++------
 4 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index ab4b78ea07d6..b37cc733d966 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
 	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -131,7 +132,7 @@ func (cdsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig,
 // the cdsBalancer. This will be faked out in unittests.
 type xdsClientInterface interface {
 	WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func()
-	CertProviderConfigs() map[string]*certprovider.BuildableConfig
+	BootstrapConfig() *bootstrap.Config
 	Close()
 }
 
@@ -241,13 +242,14 @@ func (b *cdsBalancer) handleSecurityConfig(config *xdsclient.SecurityConfig) err
 		return nil
 	}
 
-	cpc := b.xdsClient.CertProviderConfigs()
-	if cpc == nil {
+	bc := b.xdsClient.BootstrapConfig()
+	if bc == nil || bc.CertProviderConfigs == nil {
 		// Bootstrap did not find any certificate provider configs, but the user
 		// has specified xdsCredentials and the management server has sent down
 		// security configuration.
 		return errors.New("xds: certificate_providers config missing in bootstrap file")
 	}
+	cpc := bc.CertProviderConfigs
 
 	// A root provider is required whether we are using TLS or mTLS.
 	rootProvider, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index a3f923dfa684..972d16b7fd30 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
@@ -43,7 +44,7 @@ const (
 
 var (
 	fpb1, fpb2                   *fakeProviderBuilder
-	bootstrapCertProviderConfigs map[string]*certprovider.BuildableConfig
+	bootstrapConfig              *bootstrap.Config
 	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
 		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
@@ -64,9 +65,11 @@ func init() {
 	fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
 	cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111")
 	cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222")
-	bootstrapCertProviderConfigs = map[string]*certprovider.BuildableConfig{
-		"default1": cfg1,
-		"default2": cfg2,
+	bootstrapConfig = &bootstrap.Config{
+		CertProviderConfigs: map[string]*certprovider.BuildableConfig{
+			"default1": cfg1,
+			"default2": cfg2,
+		},
 	}
 	certprovider.Register(fpb1)
 	certprovider.Register(fpb2)
@@ -326,7 +329,7 @@ func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
 
 		if i == 0 {
 			// Set the bootstrap config used by the fake client.
-			xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+			xdsC.SetBootstrapConfig(bootstrapConfig)
 		}
 
 		// Here we invoke the watch callback registered on the fake xdsClient. A bad
@@ -373,7 +376,7 @@ func (s) TestCertproviderStoreError(t *testing.T) {
 	defer func() { buildProvider = origBuildProvider }()
 
 	// Set the bootstrap config used by the fake client.
-	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+	xdsC.SetBootstrapConfig(bootstrapConfig)
 
 	// Here we invoke the watch callback registered on the fake xdsClient. Even
 	// though the received update is good, the certprovider.Store is configured
@@ -409,7 +412,7 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
 	}()
 
 	// Set the bootstrap config used by the fake client.
-	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+	xdsC.SetBootstrapConfig(bootstrapConfig)
 
 	// Here we invoke the watch callback registered on the fake xdsClient. A bad
 	// security config is passed here. So, we expect the CDS balancer to not
@@ -465,7 +468,7 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 	}()
 
 	// Set the bootstrap config used by the fake client.
-	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+	xdsC.SetBootstrapConfig(bootstrapConfig)
 
 	// Here we invoke the watch callback registered on the fake xdsClient. This
 	// will trigger the watch handler on the CDS balancer, which will attempt to
@@ -496,7 +499,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	}()
 
 	// Set the bootstrap config used by the fake client.
-	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+	xdsC.SetBootstrapConfig(bootstrapConfig)
 
 	// Here we invoke the watch callback registered on the fake xdsClient. This
 	// will trigger the watch handler on the CDS balancer, which will attempt to
@@ -546,7 +549,7 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	}()
 
 	// Set the bootstrap config used by the fake client.
-	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+	xdsC.SetBootstrapConfig(bootstrapConfig)
 
 	// Here we invoke the watch callback registered on the fake xdsClient. This
 	// will trigger the watch handler on the CDS balancer, which will attempt to
@@ -617,7 +620,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	defer func() { buildProvider = origBuildProvider }()
 
 	// Set the bootstrap config used by the fake client.
-	xdsC.SetCertProviderConfigs(bootstrapCertProviderConfigs)
+	xdsC.SetBootstrapConfig(bootstrapConfig)
 
 	// Here we invoke the watch callback registered on the fake xdsClient. This
 	// will trigger the watch handler on the CDS balancer, which will attempt to
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index aa715f52c636..09729ec3e281 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -30,7 +30,7 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/proto"
-	"google.golang.org/grpc/credentials/tls/certprovider"
+
 	"google.golang.org/grpc/xds/internal/client/load"
 
 	"google.golang.org/grpc"
@@ -390,11 +390,10 @@ func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration) (
 	return c, nil
 }
 
-// CertProviderConfigs returns the certificate provider configuration from the
-// "certificate_providers" field of the bootstrap file. The key in the returned
-// map is the plugin_instance_name. Callers must not modify the returned map.
-func (c *Client) CertProviderConfigs() map[string]*certprovider.BuildableConfig {
-	return c.config.CertProviderConfigs
+// BootstrapConfig returns the configuration read from the bootstrap file.
+// Callers must treat the return value as read-only.
+func (c *Client) BootstrapConfig() *bootstrap.Config {
+	return c.config
 }
 
 // run is a goroutine for all the callbacks.
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index c540b57392aa..0978125b8aeb 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -22,9 +22,9 @@ package fakeclient
 import (
 	"context"
 
-	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
@@ -43,7 +43,7 @@ type Client struct {
 	loadReportCh *testutils.Channel
 	closeCh      *testutils.Channel
 	loadStore    *load.Store
-	certConfigs  map[string]*certprovider.BuildableConfig
+	bootstrapCfg *bootstrap.Config
 
 	ldsCb func(xdsclient.ListenerUpdate, error)
 	rdsCb func(xdsclient.RouteConfigUpdate, error)
@@ -223,14 +223,14 @@ func (xdsC *Client) WaitForClose(ctx context.Context) error {
 	return err
 }
 
-// CertProviderConfigs returns the configured certificate provider configs.
-func (xdsC *Client) CertProviderConfigs() map[string]*certprovider.BuildableConfig {
-	return xdsC.certConfigs
+// BootstrapConfig returns the bootstrap config.
+func (xdsC *Client) BootstrapConfig() *bootstrap.Config {
+	return xdsC.bootstrapCfg
 }
 
-// SetCertProviderConfigs updates the certificate provider configs.
-func (xdsC *Client) SetCertProviderConfigs(configs map[string]*certprovider.BuildableConfig) {
-	xdsC.certConfigs = configs
+// SetBootstrapConfig updates the bootstrap config.
+func (xdsC *Client) SetBootstrapConfig(cfg *bootstrap.Config) {
+	xdsC.bootstrapCfg = cfg
 }
 
 // Name returns the name of the xds client.

From 3d14af97a5e943748f9133db4792e83aa6d77bb2 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 17 Nov 2020 15:23:49 -0800
Subject: [PATCH 281/481] Change version to 1.35.0-dev (#4044)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index 725093bb882a..1e75a5c7954f 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.34.0-dev"
+const Version = "1.35.0-dev"

From fa59d20167639a6f5f9b10d57c9cb08aaaee367c Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 17 Nov 2020 15:36:28 -0800
Subject: [PATCH 282/481] pemfile: Implement certprovider config parsing API
 (#4023)

---
 credentials/tls/certprovider/meshca/config.go |   1 -
 .../tls/certprovider/pemfile/builder.go       |  96 ++++++
 .../tls/certprovider/pemfile/builder_test.go  | 130 ++++++++
 .../tls/certprovider/pemfile/watcher.go       |  73 +++--
 .../tls/certprovider/pemfile/watcher_test.go  |  18 +-
 examples/go.mod                               |   1 -
 examples/go.sum                               | 279 +-----------------
 .../advancedtls_integration_test.go           |  20 +-
 .../client/main.go                            |  10 +-
 .../server/main.go                            |  10 +-
 security/advancedtls/examples/go.mod          |   4 +-
 security/advancedtls/examples/go.sum          | 277 +----------------
 security/advancedtls/go.mod                   |   2 +-
 security/advancedtls/go.sum                   | 276 +----------------
 14 files changed, 310 insertions(+), 887 deletions(-)
 create mode 100644 credentials/tls/certprovider/pemfile/builder.go
 create mode 100644 credentials/tls/certprovider/pemfile/builder_test.go

diff --git a/credentials/tls/certprovider/meshca/config.go b/credentials/tls/certprovider/meshca/config.go
index 2800becc3db6..029181a38fc1 100644
--- a/credentials/tls/certprovider/meshca/config.go
+++ b/credentials/tls/certprovider/meshca/config.go
@@ -69,7 +69,6 @@ var (
 	readAudienceFunc = readAudience
 )
 
-// Implements the certprovider.StableConfig interface.
 type pluginConfig struct {
 	serverURI     string
 	stsOpts       sts.Options
diff --git a/credentials/tls/certprovider/pemfile/builder.go b/credentials/tls/certprovider/pemfile/builder.go
new file mode 100644
index 000000000000..957523caad10
--- /dev/null
+++ b/credentials/tls/certprovider/pemfile/builder.go
@@ -0,0 +1,96 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pemfile
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+const (
+	pluginName             = "file_watcher"
+	defaultRefreshInterval = 10 * time.Minute
+)
+
+func init() {
+	certprovider.Register(&pluginBuilder{})
+}
+
+type pluginBuilder struct{}
+
+func (p *pluginBuilder) ParseConfig(c interface{}) (*certprovider.BuildableConfig, error) {
+	data, ok := c.(json.RawMessage)
+	if !ok {
+		return nil, fmt.Errorf("meshca: unsupported config type: %T", c)
+	}
+	opts, err := pluginConfigFromJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	return certprovider.NewBuildableConfig(pluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider {
+		return newProvider(opts)
+	}), nil
+}
+
+func (p *pluginBuilder) Name() string {
+	return pluginName
+}
+
+func pluginConfigFromJSON(jd json.RawMessage) (Options, error) {
+	// The only difference between this anonymous struct and the Options struct
+	// is that the refresh_interval is represented here as a duration proto,
+	// while in the latter a time.Duration is used.
+	cfg := &struct {
+		CertificateFile   string          `json:"certificate_file,omitempty"`
+		PrivateKeyFile    string          `json:"private_key_file,omitempty"`
+		CACertificateFile string          `json:"ca_certificate_file,omitempty"`
+		RefreshInterval   json.RawMessage `json:"refresh_interval,omitempty"`
+	}{}
+	if err := json.Unmarshal(jd, cfg); err != nil {
+		return Options{}, fmt.Errorf("pemfile: json.Unmarshal(%s) failed: %v", string(jd), err)
+	}
+
+	opts := Options{
+		CertFile: cfg.CertificateFile,
+		KeyFile:  cfg.PrivateKeyFile,
+		RootFile: cfg.CACertificateFile,
+		// Refresh interval is the only field in the configuration for which we
+		// support a default value. We cannot possibly have valid defaults for
+		// file paths to watch. Also, it is valid to specify an empty path for
+		// some of those fields if the user does not want to watch them.
+		RefreshDuration: defaultRefreshInterval,
+	}
+	if cfg.RefreshInterval != nil {
+		dur := &durationpb.Duration{}
+		if err := protojson.Unmarshal(cfg.RefreshInterval, dur); err != nil {
+			return Options{}, fmt.Errorf("pemfile: protojson.Unmarshal(%+v) failed: %v", cfg.RefreshInterval, err)
+		}
+		opts.RefreshDuration = dur.AsDuration()
+	}
+
+	if err := opts.validate(); err != nil {
+		return Options{}, err
+	}
+	return opts, nil
+}
diff --git a/credentials/tls/certprovider/pemfile/builder_test.go b/credentials/tls/certprovider/pemfile/builder_test.go
new file mode 100644
index 000000000000..bef00e10c19d
--- /dev/null
+++ b/credentials/tls/certprovider/pemfile/builder_test.go
@@ -0,0 +1,130 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pemfile
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		desc       string
+		input      interface{}
+		wantOutput string
+		wantErr    bool
+	}{
+		{
+			desc:    "non JSON input",
+			input:   new(int),
+			wantErr: true,
+		},
+		{
+			desc:    "invalid JSON",
+			input:   json.RawMessage(`bad bad json`),
+			wantErr: true,
+		},
+		{
+			desc:    "JSON input does not match expected",
+			input:   json.RawMessage(`["foo": "bar"]`),
+			wantErr: true,
+		},
+		{
+			desc:    "no credential files",
+			input:   json.RawMessage(`{}`),
+			wantErr: true,
+		},
+		{
+			desc: "only cert file",
+			input: json.RawMessage(`
+			{
+				"certificate_file": "/a/b/cert.pem"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "only key file",
+			input: json.RawMessage(`
+			{
+				"private_key_file": "/a/b/key.pem"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "cert and key in different directories",
+			input: json.RawMessage(`
+			{
+				"certificate_file": "/b/a/cert.pem",
+				"private_key_file": "/a/b/key.pem"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "bad refresh duration",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem",
+				"refresh_interval":   "duration"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "good config with default refresh interval",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem"
+			}`),
+			wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem:10m0s",
+		},
+		{
+			desc: "good config",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem",
+				"refresh_interval":   "200s"
+			}`),
+			wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem:3m20s",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			builder := &pluginBuilder{}
+
+			bc, err := builder.ParseConfig(test.input)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("ParseConfig(%+v) failed: %v", test.input, err)
+			}
+			if test.wantErr {
+				return
+			}
+
+			gotConfig := bc.String()
+			if gotConfig != test.wantOutput {
+				t.Fatalf("ParseConfig(%v) = %s, want %s", test.input, gotConfig, test.wantOutput)
+			}
+		})
+	}
+}
diff --git a/credentials/tls/certprovider/pemfile/watcher.go b/credentials/tls/certprovider/pemfile/watcher.go
index 29ea8b2b0654..a46b149dd03d 100644
--- a/credentials/tls/certprovider/pemfile/watcher.go
+++ b/credentials/tls/certprovider/pemfile/watcher.go
@@ -30,18 +30,17 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	"time"
 
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/grpclog"
 )
 
-const (
-	defaultCertRefreshDuration = 1 * time.Hour
-	defaultRootRefreshDuration = 2 * time.Hour
-)
+const defaultCertRefreshDuration = 1 * time.Hour
 
 var (
 	// For overriding from unit tests.
@@ -62,30 +61,48 @@ type Options struct {
 	// RootFile is the file that holds trusted root certificate(s).
 	// Optional.
 	RootFile string
-	// CertRefreshDuration is the amount of time the plugin waits before
-	// checking for updates in the specified identity certificate and key file.
+	// RefreshDuration is the amount of time the plugin waits before checking
+	// for updates in the specified files.
 	// Optional. If not set, a default value (1 hour) will be used.
-	CertRefreshDuration time.Duration
-	// RootRefreshDuration is the amount of time the plugin waits before
-	// checking for updates in the specified root file.
-	// Optional. If not set, a default value (2 hour) will be used.
-	RootRefreshDuration time.Duration
+	RefreshDuration time.Duration
 }
 
-// NewProvider returns a new certificate provider plugin that is configured to
-// watch the PEM files specified in the passed in options.
-func NewProvider(o Options) (certprovider.Provider, error) {
+func (o Options) canonical() []byte {
+	return []byte(fmt.Sprintf("%s:%s:%s:%s", o.CertFile, o.KeyFile, o.RootFile, o.RefreshDuration))
+}
+
+func (o Options) validate() error {
 	if o.CertFile == "" && o.KeyFile == "" && o.RootFile == "" {
-		return nil, fmt.Errorf("pemfile: at least one credential file needs to be specified")
+		return fmt.Errorf("pemfile: at least one credential file needs to be specified")
 	}
 	if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified {
-		return nil, fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified")
+		return fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified")
 	}
-	if o.CertRefreshDuration == 0 {
-		o.CertRefreshDuration = defaultCertRefreshDuration
+	// C-core has a limitation that they cannot verify that a certificate file
+	// matches a key file. So, the only way to get around this is to make sure
+	// that both files are in the same directory and that they do an atomic
+	// read. Even though Java/Go do not have this limitation, we want the
+	// overall plugin behavior to be consistent across languages.
+	if certDir, keyDir := filepath.Dir(o.CertFile), filepath.Dir(o.KeyFile); certDir != keyDir {
+		return errors.New("pemfile: certificate and key file must be in the same directory")
 	}
-	if o.RootRefreshDuration == 0 {
-		o.RootRefreshDuration = defaultRootRefreshDuration
+	return nil
+}
+
+// NewProvider returns a new certificate provider plugin that is configured to
+// watch the PEM files specified in the passed in options.
+func NewProvider(o Options) (certprovider.Provider, error) {
+	if err := o.validate(); err != nil {
+		return nil, err
+	}
+	return newProvider(o), nil
+}
+
+// newProvider is used to create a new certificate provider plugin after
+// validating the options, and hence does not return an error.
+func newProvider(o Options) certprovider.Provider {
+	if o.RefreshDuration == 0 {
+		o.RefreshDuration = defaultCertRefreshDuration
 	}
 
 	provider := &watcher{opts: o}
@@ -99,8 +116,7 @@ func NewProvider(o Options) (certprovider.Provider, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	provider.cancel = cancel
 	go provider.run(ctx)
-
-	return provider, nil
+	return provider
 }
 
 // watcher is a certificate provider plugin that implements the
@@ -203,13 +219,13 @@ func (w *watcher) run(ctx context.Context) {
 	w.updateIdentityDistributor()
 	w.updateRootDistributor()
 
-	identityTicker := time.NewTicker(w.opts.CertRefreshDuration)
-	rootTicker := time.NewTicker(w.opts.RootRefreshDuration)
+	ticker := time.NewTicker(w.opts.RefreshDuration)
 	for {
+		w.updateIdentityDistributor()
+		w.updateRootDistributor()
 		select {
 		case <-ctx.Done():
-			identityTicker.Stop()
-			rootTicker.Stop()
+			ticker.Stop()
 			if w.identityDistributor != nil {
 				w.identityDistributor.Stop()
 			}
@@ -217,10 +233,7 @@ func (w *watcher) run(ctx context.Context) {
 				w.rootDistributor.Stop()
 			}
 			return
-		case <-identityTicker.C:
-			w.updateIdentityDistributor()
-		case <-rootTicker.C:
-			w.updateRootDistributor()
+		case <-ticker.C:
 		}
 	}
 }
diff --git a/credentials/tls/certprovider/pemfile/watcher_test.go b/credentials/tls/certprovider/pemfile/watcher_test.go
index 092bd30ece63..d5ce5ab7e94d 100644
--- a/credentials/tls/certprovider/pemfile/watcher_test.go
+++ b/credentials/tls/certprovider/pemfile/watcher_test.go
@@ -187,11 +187,10 @@ func initializeProvider(t *testing.T, testName string) (string, certprovider.Pro
 	// Create a new provider to watch the files in tmpdir.
 	dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
 	opts := Options{
-		CertFile:            path.Join(dir, certFile),
-		KeyFile:             path.Join(dir, keyFile),
-		RootFile:            path.Join(dir, rootFile),
-		CertRefreshDuration: defaultTestRefreshDuration,
-		RootRefreshDuration: defaultTestRefreshDuration,
+		CertFile:        path.Join(dir, certFile),
+		KeyFile:         path.Join(dir, keyFile),
+		RootFile:        path.Join(dir, rootFile),
+		RefreshDuration: defaultTestRefreshDuration,
 	}
 	prov, err := NewProvider(opts)
 	if err != nil {
@@ -314,11 +313,10 @@ func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) {
 
 	// Create a provider which watches the files pointed to by the symlink.
 	opts := Options{
-		CertFile:            path.Join(symLinkName, certFile),
-		KeyFile:             path.Join(symLinkName, keyFile),
-		RootFile:            path.Join(symLinkName, rootFile),
-		CertRefreshDuration: defaultTestRefreshDuration,
-		RootRefreshDuration: defaultTestRefreshDuration,
+		CertFile:        path.Join(symLinkName, certFile),
+		KeyFile:         path.Join(symLinkName, keyFile),
+		RootFile:        path.Join(symLinkName, rootFile),
+		RefreshDuration: defaultTestRefreshDuration,
 	}
 	prov, err := NewProvider(opts)
 	if err != nil {
diff --git a/examples/go.mod b/examples/go.mod
index 9f056e4a992c..18c67afed969 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -3,7 +3,6 @@ module google.golang.org/grpc/examples
 go 1.11
 
 require (
-	cloud.google.com/go v0.63.0 // indirect
 	github.com/golang/protobuf v1.4.2
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98
diff --git a/examples/go.sum b/examples/go.sum
index 314ec0703382..b7082ec1e78d 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -1,43 +1,7 @@
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.63.0 h1:A+DfAZQ/eWca7gvu42CS6FNSDX4R8cghF+XfWLn4R6g=
-cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -45,25 +9,9 @@ github.com/envoyproxy/go-control-plane v0.9.7 h1:EARl0OvqMoxq/UMgMSCLnXzkaXbxzsk
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -73,259 +21,46 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -342,16 +77,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index d554468557a3..a9b22b711ff5 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -515,34 +515,34 @@ func copyFileContents(sourceFile, destinationFile string) error {
 // files.
 func createProviders(tmpFiles *tmpCredsFiles) (certprovider.Provider, certprovider.Provider, certprovider.Provider, certprovider.Provider, error) {
 	clientIdentityOptions := pemfile.Options{
-		CertFile:            tmpFiles.clientCertTmp.Name(),
-		KeyFile:             tmpFiles.clientKeyTmp.Name(),
-		CertRefreshDuration: credRefreshingInterval,
+		CertFile:        tmpFiles.clientCertTmp.Name(),
+		KeyFile:         tmpFiles.clientKeyTmp.Name(),
+		RefreshDuration: credRefreshingInterval,
 	}
 	clientIdentityProvider, err := pemfile.NewProvider(clientIdentityOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 	clientRootOptions := pemfile.Options{
-		RootFile:            tmpFiles.clientTrustTmp.Name(),
-		RootRefreshDuration: credRefreshingInterval,
+		RootFile:        tmpFiles.clientTrustTmp.Name(),
+		RefreshDuration: credRefreshingInterval,
 	}
 	clientRootProvider, err := pemfile.NewProvider(clientRootOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 	serverIdentityOptions := pemfile.Options{
-		CertFile:            tmpFiles.serverCertTmp.Name(),
-		KeyFile:             tmpFiles.serverKeyTmp.Name(),
-		CertRefreshDuration: credRefreshingInterval,
+		CertFile:        tmpFiles.serverCertTmp.Name(),
+		KeyFile:         tmpFiles.serverKeyTmp.Name(),
+		RefreshDuration: credRefreshingInterval,
 	}
 	serverIdentityProvider, err := pemfile.NewProvider(serverIdentityOptions)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 	serverRootOptions := pemfile.Options{
-		RootFile:            tmpFiles.serverTrustTmp.Name(),
-		RootRefreshDuration: credRefreshingInterval,
+		RootFile:        tmpFiles.serverTrustTmp.Name(),
+		RefreshDuration: credRefreshingInterval,
 	}
 	serverRootProvider, err := pemfile.NewProvider(serverRootOptions)
 	if err != nil {
diff --git a/security/advancedtls/examples/credential_reloading_from_files/client/main.go b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
index 2aa4dace587e..6f88994cfc51 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/client/main.go
+++ b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
@@ -47,17 +47,17 @@ func main() {
 	flag.Parse()
 
 	identityOptions := pemfile.Options{
-		CertFile:            testdata.Path("client_cert_1.pem"),
-		KeyFile:             testdata.Path("client_key_1.pem"),
-		CertRefreshDuration: credRefreshingInterval,
+		CertFile:        testdata.Path("client_cert_1.pem"),
+		KeyFile:         testdata.Path("client_key_1.pem"),
+		RefreshDuration: credRefreshingInterval,
 	}
 	identityProvider, err := pemfile.NewProvider(identityOptions)
 	if err != nil {
 		log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err)
 	}
 	rootOptions := pemfile.Options{
-		RootFile:            testdata.Path("client_trust_cert_1.pem"),
-		RootRefreshDuration: credRefreshingInterval,
+		RootFile:        testdata.Path("client_trust_cert_1.pem"),
+		RefreshDuration: credRefreshingInterval,
 	}
 	rootProvider, err := pemfile.NewProvider(rootOptions)
 	if err != nil {
diff --git a/security/advancedtls/examples/credential_reloading_from_files/server/main.go b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
index c1431866f7b2..3c14757fcfcc 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/server/main.go
+++ b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
@@ -55,9 +55,9 @@ func main() {
 	fmt.Printf("server starting on port %s...\n", port)
 
 	identityOptions := pemfile.Options{
-		CertFile:            testdata.Path("server_cert_1.pem"),
-		KeyFile:             testdata.Path("server_key_1.pem"),
-		CertRefreshDuration: credRefreshingInterval,
+		CertFile:        testdata.Path("server_cert_1.pem"),
+		KeyFile:         testdata.Path("server_key_1.pem"),
+		RefreshDuration: credRefreshingInterval,
 	}
 	identityProvider, err := pemfile.NewProvider(identityOptions)
 	if err != nil {
@@ -65,8 +65,8 @@ func main() {
 	}
 	defer identityProvider.Close()
 	rootOptions := pemfile.Options{
-		RootFile:            testdata.Path("server_trust_cert_1.pem"),
-		RootRefreshDuration: credRefreshingInterval,
+		RootFile:        testdata.Path("server_trust_cert_1.pem"),
+		RefreshDuration: credRefreshingInterval,
 	}
 	rootProvider, err := pemfile.NewProvider(rootOptions)
 	if err != nil {
diff --git a/security/advancedtls/examples/go.mod b/security/advancedtls/examples/go.mod
index 05cf3fc75c80..936aa476893b 100644
--- a/security/advancedtls/examples/go.mod
+++ b/security/advancedtls/examples/go.mod
@@ -4,8 +4,8 @@ go 1.15
 
 require (
 	google.golang.org/grpc v1.33.1
-	google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d
-	google.golang.org/grpc/security/advancedtls v0.0.0-20201102215344-15ae9fc2b247
+	google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b
+	google.golang.org/grpc/security/advancedtls v0.0.0-20201112215255-90f1b3ee835b
 )
 
 replace google.golang.org/grpc => ../../..
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
index e99f515c1397..81cb0f18bc00 100644
--- a/security/advancedtls/examples/go.sum
+++ b/security/advancedtls/examples/go.sum
@@ -1,64 +1,12 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -67,250 +15,43 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -325,16 +66,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index 7df0f3fc56c1..c641a9d60695 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/google/go-cmp v0.5.1
 	google.golang.org/grpc v1.31.0
-	google.golang.org/grpc/examples v0.0.0-20201020200225-9519efffeb5d
+	google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b
 )
 
 replace google.golang.org/grpc => ../../
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 84e6ee891384..81cb0f18bc00 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,64 +1,12 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -67,249 +15,43 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -324,16 +66,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

From 8f126961c40727a392bcf48e4e5bcf8516eea889 Mon Sep 17 00:00:00 2001
From: Pierre Fenoll <pierrefenoll@gmail.com>
Date: Wed, 18 Nov 2020 00:54:06 +0100
Subject: [PATCH 283/481] Fix typo in encoding.md (#4020)

---
 Documentation/encoding.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Documentation/encoding.md b/Documentation/encoding.md
index 31436609d51c..dd49f55e5d44 100644
--- a/Documentation/encoding.md
+++ b/Documentation/encoding.md
@@ -132,7 +132,7 @@ As a reminder, all `CallOption`s may be converted into `DialOption`s that become
 the default for all RPCs sent through a client using `grpc.WithDefaultCallOptions`:
 
 ```go
-	myclient := grpc.Dial(ctx, target, grpc.WithDefaultCallOptions(grpc.UseCompresor("gzip")))
+	myclient := grpc.Dial(ctx, target, grpc.WithDefaultCallOptions(grpc.UseCompressor("gzip")))
 ```
 
 When specified in either of these ways, messages will be compressed using this

From c5cf5144d7ad7471a3fae6fb7b2c6d3ae9d3b42d Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 17 Nov 2020 16:02:43 -0800
Subject: [PATCH 284/481] github acitons: always run (#4045)

Even for version.go and doc changes. Otherwise the PRs with only those changes will be waiting for the tests to pass.
---
 .github/workflows/testing.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index dbfc9db328da..9784f9e09bf3 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -4,9 +4,6 @@ name: Testing
 on:
   push:
   pull_request:
-    paths-ignore:
-      - 'Documentation/**'
-      - 'version.go'
   schedule:
     - cron: 0 0 * * * # daily at 00:00
 

From 230166b6df4ad811bddc77ff5f54251448236ff9 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 18 Nov 2020 09:46:14 -0800
Subject: [PATCH 285/481] xds: Use management server from go-control-plane.
 (#4039)

---
 go.sum                                        |   5 +
 .../test/xds_server_integration_test.go       | 148 +++++++-----------
 xds/internal/testutils/e2e/bootstrap.go       | 123 +++++++++++++++
 xds/internal/testutils/e2e/server.go          | 142 +++++++++++++++++
 4 files changed, 326 insertions(+), 92 deletions(-)
 create mode 100644 xds/internal/testutils/e2e/bootstrap.go
 create mode 100644 xds/internal/testutils/e2e/server.go

diff --git a/go.sum b/go.sum
index 293f5490353f..a98f8b16482b 100644
--- a/go.sum
+++ b/go.sum
@@ -8,6 +8,7 @@ github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJ
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -36,9 +37,11 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -87,7 +90,9 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 3f218eb458ca..294ded07a53b 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -1,3 +1,5 @@
+// +build !386
+
 /*
  *
  * Copyright 2020 gRPC authors.
@@ -22,19 +24,17 @@ package xds_test
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	v2discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpctest"
@@ -42,7 +42,7 @@ import (
 	"google.golang.org/grpc/xds"
 	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/testutils"
-	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -58,94 +58,36 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-func setupListenerResponse(respCh chan *fakeserver.Response, name string) {
-	respCh <- &fakeserver.Response{
-		Resp: &v2discoverypb.DiscoveryResponse{
-			Resources: []*anypb.Any{
-				{
-					TypeUrl: version.V2ListenerURL,
-					Value: func() []byte {
-						l := &v3listenerpb.Listener{
-							// This needs to match the name we are querying for.
-							Name: name,
-							ApiListener: &v3listenerpb.ApiListener{
-								ApiListener: &anypb.Any{
-									TypeUrl: version.V2HTTPConnManagerURL,
-									Value: func() []byte {
-										cm := &v3httppb.HttpConnectionManager{
-											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-												Rds: &v3httppb.Rds{
-													ConfigSource: &v3corepb.ConfigSource{
-														ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-													},
-													RouteConfigName: "route-config",
-												},
-											},
-										}
-										mcm, _ := proto.Marshal(cm)
-										return mcm
-									}(),
-								},
-							},
-						}
-						ml, _ := proto.Marshal(l)
-						return ml
-					}(),
-				},
-			},
-			TypeUrl: version.V2ListenerURL,
-		},
-	}
-}
-
-func setupBootstrapFile(t *testing.T, serverURI string) func() {
-	// Create a bootstrap file in a temporary directory.
-	tmpdir, err := ioutil.TempDir("", "xds-server-test*")
-	if err != nil {
-		t.Fatalf("failed to create tempdir: %v", err)
-	}
-	bootstrapContents := fmt.Sprintf(`
-		{
-			"node": {
-				"id": "ENVOY_NODE_ID",
-				"metadata": {
-				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
-			    }
-			},
-			"xds_servers" : [{
-				"server_uri": "%s",
-				"channel_creds": [
-					{ "type": "insecure" }
-				]
-			}]
-		}`, serverURI)
-	bootstrapFileName := path.Join(tmpdir, "bootstrap")
-	if err := ioutil.WriteFile(bootstrapFileName, []byte(bootstrapContents), os.ModePerm); err != nil {
-		t.Fatalf("failed to write bootstrap file: %v", err)
-	}
-
-	origBootstrapFileName := env.BootstrapFileName
-	env.BootstrapFileName = bootstrapFileName
-	t.Logf("Create bootstrap file at %s with contents\n%s", bootstrapFileName, bootstrapContents)
-	return func() { env.BootstrapFileName = origBootstrapFileName }
-}
-
 // TestServerSideXDS is an e2e tests for xDS use on the server. This does not
 // use any xDS features because we have not implemented any on the server side.
 func (s) TestServerSideXDS(t *testing.T) {
-	// Spin up a fake xDS management server on a local port.
-	// TODO(easwars): Switch to using the server from envoy-go-control-plane.
-	fs, cleanup, err := fakeserver.StartServer()
+	origV3Support := env.V3Support
+	env.V3Support = true
+	defer func() { env.V3Support = origV3Support }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Spin up a xDS management server on a local port.
+	nodeID := uuid.New().String()
+	fs, err := e2e.StartManagementServer()
 	if err != nil {
-		t.Fatalf("failed to start fake xDS server: %v", err)
+		t.Fatal(err)
 	}
-	defer cleanup()
-	t.Logf("Started xDS management server at %s", fs.Address)
+	defer fs.Stop()
 
 	// Create a bootstrap file in a temporary directory.
-	defer setupBootstrapFile(t, fs.Address)()
+	cleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
+		Version:   e2e.TransportV3,
+		NodeID:    nodeID,
+		ServerURI: fs.Address,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
 
-	// Initialize a gRPC server which uses xDS, and register stubServer on it.
+	// Initialize an xDS-enabled gRPC server and register the stubServer on it.
 	server := xds.NewGRPCServer()
 	testpb.RegisterTestServiceServer(server, &testService{})
 	defer server.Stop()
@@ -162,18 +104,40 @@ func (s) TestServerSideXDS(t *testing.T) {
 		}
 	}()
 
-	// Create a ClientConn and make a successful RPC.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-
 	// Setup the fake management server to respond with a Listener resource.
 	go func() {
-		if _, err := fs.XDSRequestChan.Receive(ctx); err != nil {
-			t.Errorf("timeout when waiting for listener request: %v", err)
+		listener := &v3listenerpb.Listener{
+			// This needs to match the name we are querying for.
+			Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr),
+			ApiListener: &v3listenerpb.ApiListener{
+				ApiListener: &anypb.Any{
+					TypeUrl: version.V2HTTPConnManagerURL,
+					Value: func() []byte {
+						cm := &v3httppb.HttpConnectionManager{
+							RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+								Rds: &v3httppb.Rds{
+									ConfigSource: &v3corepb.ConfigSource{
+										ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+									},
+									RouteConfigName: "route-config",
+								},
+							},
+						}
+						mcm, _ := proto.Marshal(cm)
+						return mcm
+					}(),
+				},
+			},
+		}
+		if err := fs.Update(e2e.UpdateOptions{
+			NodeID:    nodeID,
+			Listeners: []*v3listenerpb.Listener{listener},
+		}); err != nil {
+			t.Error(err)
 		}
-		setupListenerResponse(fs.XDSResponseChan, fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr))
 	}()
 
+	// Create a ClientConn and make a successful RPC.
 	cc, err := grpc.DialContext(ctx, localAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go
new file mode 100644
index 000000000000..a6a9f22fb44f
--- /dev/null
+++ b/xds/internal/testutils/e2e/bootstrap.go
@@ -0,0 +1,123 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"google.golang.org/grpc/xds/internal/env"
+)
+
+// TransportAPI refers to the API version for xDS transport protocol.
+type TransportAPI int
+
+const (
+	// TransportV2 refers to the v2 xDS transport protocol.
+	TransportV2 TransportAPI = iota
+	// TransportV3 refers to the v3 xDS transport protocol.
+	TransportV3
+)
+
+// BootstrapOptions wraps the parameters passed to SetupBootstrapFile.
+type BootstrapOptions struct {
+	// Version is the xDS transport protocol version.
+	Version TransportAPI
+	// NodeID is the node identifier of the gRPC client/server node in the
+	// proxyless service mesh.
+	NodeID string
+	// ServerURI is the address of the management server.
+	ServerURI string
+}
+
+// SetupBootstrapFile creates a temporary file with bootstrap contents, based on
+// the passed in options, and updates the bootstrap environment variable to
+// point to this file.
+//
+// Returns a cleanup function which will be non-nil if the setup process was
+// completed successfully. It is the responsibility of the caller to invoke the
+// cleanup function at the end of the test.
+func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
+	f, err := ioutil.TempFile("", "test_xds_bootstrap_*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
+	}
+
+	cfg := &bootstrapConfig{
+		XdsServers: []server{
+			{
+				ServerURI: opts.ServerURI,
+				ChannelCreds: []creds{
+					{
+						Type: "insecure",
+					},
+				},
+			},
+		},
+		Node: node{
+			ID: opts.NodeID,
+		},
+	}
+	switch opts.Version {
+	case TransportV2:
+		// TODO: Add any v2 specific fields.
+	case TransportV3:
+		cfg.ServerFeatures = append(cfg.ServerFeatures, "xds_v3")
+	default:
+		return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version)
+	}
+
+	bootstrapContents, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
+	}
+	if err := ioutil.WriteFile(f.Name(), bootstrapContents, 0644); err != nil {
+		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
+	}
+	logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents)
+
+	origBootstrapFileName := env.BootstrapFileName
+	env.BootstrapFileName = f.Name()
+	return func() {
+		os.Remove(f.Name())
+		env.BootstrapFileName = origBootstrapFileName
+	}, nil
+}
+
+type bootstrapConfig struct {
+	XdsServers     []server `json:"xds_servers,omitempty"`
+	Node           node     `json:"node,omitempty"`
+	ServerFeatures []string `json:"server_features,omitempty"`
+}
+
+type server struct {
+	ServerURI    string  `json:"server_uri,omitempty"`
+	ChannelCreds []creds `json:"channel_creds,omitempty"`
+}
+
+type creds struct {
+	Type   string      `json:"type,omitempty"`
+	Config interface{} `json:"config,omitempty"`
+}
+
+type node struct {
+	ID string `json:"id,omitempty"`
+}
diff --git a/xds/internal/testutils/e2e/server.go b/xds/internal/testutils/e2e/server.go
new file mode 100644
index 000000000000..eb6045946366
--- /dev/null
+++ b/xds/internal/testutils/e2e/server.go
@@ -0,0 +1,142 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package e2e provides utilities for end2end testing of xDS functionality.
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/envoyproxy/go-control-plane/pkg/cache/types"
+	v3cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
+	v3server "github.com/envoyproxy/go-control-plane/pkg/server/v3"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/grpclog"
+)
+
+var logger = grpclog.Component("xds-e2e")
+
+// serverLogger implements the Logger interface defined at
+// envoyproxy/go-control-plane/pkg/log. This is passed to the Snapshot cache.
+type serverLogger struct{}
+
+func (l serverLogger) Debugf(format string, args ...interface{}) { logger.Infof(format, args...) }
+func (l serverLogger) Infof(format string, args ...interface{})  { logger.Infof(format, args...) }
+func (l serverLogger) Warnf(format string, args ...interface{})  { logger.Warningf(format, args...) }
+func (l serverLogger) Errorf(format string, args ...interface{}) { logger.Errorf(format, args...) }
+
+// ManagementServer is a thin wrapper around the xDS control plane
+// implementation provided by envoyproxy/go-control-plane.
+type ManagementServer struct {
+	// Address is the host:port on which the management server is listening for
+	// new connections.
+	Address string
+
+	cancel  context.CancelFunc    // To stop the v3 ADS service.
+	xs      v3server.Server       // v3 implementation of ADS.
+	gs      *grpc.Server          // gRPC server which exports the ADS service.
+	cache   v3cache.SnapshotCache // Resource snapshot.
+	version int                   // Version of resource snapshot.
+}
+
+// StartManagementServer initializes a management server which implements the
+// AggregatedDiscoveryService endpoint. The management server is initialized
+// with no resources. Tests should call the Update() method to change the
+// resource snapshot held by the management server, as required by the test
+// logic. When the test is done, it should call the Stop() method to cleanup
+// resources allocated by the management server.
+func StartManagementServer() (*ManagementServer, error) {
+	// Create a snapshot cache.
+	cache := v3cache.NewSnapshotCache(true, v3cache.IDHash{}, serverLogger{})
+	logger.Infof("Created new snapshot cache...")
+
+	lis, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return nil, fmt.Errorf("failed to start xDS management server: %v", err)
+	}
+
+	// Create an xDS management server and register the ADS implementation
+	// provided by it on a gRPC server. Cancelling the context passed to the
+	// server is the only way of stopping it at the end of the test.
+	ctx, cancel := context.WithCancel(context.Background())
+	xs := v3server.NewServer(ctx, cache, v3server.CallbackFuncs{})
+	gs := grpc.NewServer()
+	v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(gs, xs)
+	logger.Infof("Registered Aggregated Discovery Service (ADS)...")
+
+	// Start serving.
+	go gs.Serve(lis)
+	logger.Infof("xDS management server serving at: %v...", lis.Addr().String())
+
+	return &ManagementServer{
+		Address: lis.Addr().String(),
+		cancel:  cancel,
+		version: 0,
+		gs:      gs,
+		xs:      xs,
+		cache:   cache,
+	}, nil
+}
+
+// UpdateOptions wraps parameters to be passed to the Update() method.
+type UpdateOptions struct {
+	// NodeID is the id of the client to which this update is to be pushed.
+	NodeID string
+	// Listeners is the updated list of listener resources.
+	Listeners []*v3listenerpb.Listener
+	// TODO(easwars): Add support for other resource types.
+}
+
+// Update changes the resource snapshot held by the management server, which
+// updates connected clients as required.
+func (s *ManagementServer) Update(opts UpdateOptions) error {
+	s.version++
+
+	// Create a snapshot with the passed in resources.
+	var listeners []types.Resource
+	for _, l := range opts.Listeners {
+		listeners = append(listeners, l)
+	}
+	snapshot := v3cache.NewSnapshot(strconv.Itoa(s.version), nil, nil, nil, listeners, nil, nil)
+	if err := snapshot.Consistent(); err != nil {
+		return fmt.Errorf("failed to create new resource snapshot: %v", err)
+	}
+	logger.Infof("Created new resource snapshot...")
+
+	// Update the cache with the new resource snapshot.
+	if err := s.cache.SetSnapshot(opts.NodeID, snapshot); err != nil {
+		return fmt.Errorf("failed to update resource snapshot in management server: %v", err)
+	}
+	logger.Infof("Updated snapshot cache with resource snapshot...")
+	return nil
+}
+
+// Stop stops the management server.
+func (s *ManagementServer) Stop() {
+	if s.cancel != nil {
+		s.cancel()
+	}
+	s.gs.Stop()
+	logger.Infof("Stopped the xDS management server...")
+}

From 78864797b87780af8d88b2d4d64721020abbe7ea Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 18 Nov 2020 17:12:51 -0800
Subject: [PATCH 286/481] balancer: set RPC metadata in address attributes,
 instead of Metadata field (#4041)

This metadata will be sent with all RPCs on the created SubConn
---
 balancer/grpclb/grpclb_remote_balancer.go | 18 +++--
 balancer/grpclb/grpclb_test.go            |  2 +-
 balancer/grpclb/grpclb_util.go            | 10 +--
 internal/hierarchy/hierarchy.go           | 10 +--
 internal/metadata/metadata.go             | 50 +++++++++++++
 internal/metadata/metadata_test.go        | 86 +++++++++++++++++++++++
 internal/transport/http2_client.go        | 24 ++++---
 test/balancer_test.go                     | 71 +++++++++++++++++++
 8 files changed, 236 insertions(+), 35 deletions(-)
 create mode 100644 internal/metadata/metadata.go
 create mode 100644 internal/metadata/metadata_test.go

diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index 8fdda09034f5..08d326ab40b0 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/channelz"
+	imetadata "google.golang.org/grpc/internal/metadata"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
@@ -76,10 +77,7 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
 			// net.SplitHostPort() will return too many colons error.
 			ipStr = fmt.Sprintf("[%s]", ipStr)
 		}
-		addr := resolver.Address{
-			Addr:     fmt.Sprintf("%s:%d", ipStr, s.Port),
-			Metadata: &md,
-		}
+		addr := imetadata.Set(resolver.Address{Addr: fmt.Sprintf("%s:%d", ipStr, s.Port)}, md)
 		if logger.V(2) {
 			logger.Infof("lbBalancer: server list entry[%d]: ipStr:|%s|, port:|%d|, load balancer token:|%v|",
 				i, ipStr, s.Port, s.LoadBalanceToken)
@@ -163,19 +161,19 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback
 	addrsSet := make(map[resolver.Address]struct{})
 	// Create new SubConns.
 	for _, addr := range backendAddrs {
-		addrWithoutMD := addr
-		addrWithoutMD.Metadata = nil
-		addrsSet[addrWithoutMD] = struct{}{}
-		lb.backendAddrsWithoutMetadata = append(lb.backendAddrsWithoutMetadata, addrWithoutMD)
+		addrWithoutAttrs := addr
+		addrWithoutAttrs.Attributes = nil
+		addrsSet[addrWithoutAttrs] = struct{}{}
+		lb.backendAddrsWithoutMetadata = append(lb.backendAddrsWithoutMetadata, addrWithoutAttrs)
 
-		if _, ok := lb.subConns[addrWithoutMD]; !ok {
+		if _, ok := lb.subConns[addrWithoutAttrs]; !ok {
 			// Use addrWithMD to create the SubConn.
 			sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts)
 			if err != nil {
 				logger.Warningf("grpclb: failed to create new SubConn: %v", err)
 				continue
 			}
-			lb.subConns[addrWithoutMD] = sc // Use the addr without MD as key for the map.
+			lb.subConns[addrWithoutAttrs] = sc // Use the addr without MD as key for the map.
 			if _, ok := lb.scStates[sc]; !ok {
 				// Only set state of new sc to IDLE. The state could already be
 				// READY for cached SubConns.
diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go
index dc94ca877843..9cbb338c2415 100644
--- a/balancer/grpclb/grpclb_test.go
+++ b/balancer/grpclb/grpclb_test.go
@@ -303,7 +303,7 @@ func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.E
 	if !ok {
 		return nil, status.Error(codes.Internal, "failed to receive metadata")
 	}
-	if !s.fallback && (md == nil || md["lb-token"][0] != lbToken) {
+	if !s.fallback && (md == nil || len(md["lb-token"]) == 0 || md["lb-token"][0] != lbToken) {
 		return nil, status.Errorf(codes.Internal, "received unexpected metadata: %v", md)
 	}
 	grpc.SetTrailer(ctx, metadata.Pairs(testmdkey, s.addr))
diff --git a/balancer/grpclb/grpclb_util.go b/balancer/grpclb/grpclb_util.go
index 636725e5416d..373f04b98d37 100644
--- a/balancer/grpclb/grpclb_util.go
+++ b/balancer/grpclb/grpclb_util.go
@@ -124,16 +124,16 @@ func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer
 	if len(addrs) != 1 {
 		return nil, fmt.Errorf("grpclb calling NewSubConn with addrs of length %v", len(addrs))
 	}
-	addrWithoutMD := addrs[0]
-	addrWithoutMD.Metadata = nil
+	addrWithoutAttrs := addrs[0]
+	addrWithoutAttrs.Attributes = nil
 
 	ccc.mu.Lock()
 	defer ccc.mu.Unlock()
-	if entry, ok := ccc.subConnCache[addrWithoutMD]; ok {
+	if entry, ok := ccc.subConnCache[addrWithoutAttrs]; ok {
 		// If entry is in subConnCache, the SubConn was being deleted.
 		// cancel function will never be nil.
 		entry.cancel()
-		delete(ccc.subConnCache, addrWithoutMD)
+		delete(ccc.subConnCache, addrWithoutAttrs)
 		return entry.sc, nil
 	}
 
@@ -142,7 +142,7 @@ func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer
 		return nil, err
 	}
 
-	ccc.subConnToAddr[scNew] = addrWithoutMD
+	ccc.subConnToAddr[scNew] = addrWithoutAttrs
 	return scNew, nil
 }
 
diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go
index 17185d95d38e..a2f990f552e6 100644
--- a/internal/hierarchy/hierarchy.go
+++ b/internal/hierarchy/hierarchy.go
@@ -23,7 +23,6 @@
 package hierarchy
 
 import (
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -37,19 +36,12 @@ func Get(addr resolver.Address) []string {
 	if attrs == nil {
 		return nil
 	}
-	path, ok := attrs.Value(pathKey).([]string)
-	if !ok {
-		return nil
-	}
+	path, _ := attrs.Value(pathKey).([]string)
 	return path
 }
 
 // Set overrides the hierarchical path in addr with path.
 func Set(addr resolver.Address, path []string) resolver.Address {
-	if addr.Attributes == nil {
-		addr.Attributes = attributes.New(pathKey, path)
-		return addr
-	}
 	addr.Attributes = addr.Attributes.WithValues(pathKey, path)
 	return addr
 }
diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go
new file mode 100644
index 000000000000..302262613a02
--- /dev/null
+++ b/internal/metadata/metadata.go
@@ -0,0 +1,50 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package metadata contains functions to set and get metadata from addresses.
+//
+// This package is experimental.
+package metadata
+
+import (
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+)
+
+type mdKeyType string
+
+const mdKey = mdKeyType("grpc.internal.address.metadata")
+
+// Get returns the metadata of addr.
+func Get(addr resolver.Address) metadata.MD {
+	attrs := addr.Attributes
+	if attrs == nil {
+		return nil
+	}
+	md, _ := attrs.Value(mdKey).(metadata.MD)
+	return md
+}
+
+// Set sets (overrides) the metadata in addr.
+//
+// When a SubConn is created with this address, the RPCs sent on it will all
+// have this metadata.
+func Set(addr resolver.Address, md metadata.MD) resolver.Address {
+	addr.Attributes = addr.Attributes.WithValues(mdKey, md)
+	return addr
+}
diff --git a/internal/metadata/metadata_test.go b/internal/metadata/metadata_test.go
new file mode 100644
index 000000000000..68c2ca5808c7
--- /dev/null
+++ b/internal/metadata/metadata_test.go
@@ -0,0 +1,86 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package metadata
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+)
+
+func TestGet(t *testing.T) {
+	tests := []struct {
+		name string
+		addr resolver.Address
+		want metadata.MD
+	}{
+		{
+			name: "not set",
+			addr: resolver.Address{},
+			want: nil,
+		},
+		{
+			name: "not set",
+			addr: resolver.Address{
+				Attributes: attributes.New(mdKey, metadata.Pairs("k", "v")),
+			},
+			want: metadata.Pairs("k", "v"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Get(tt.addr); !cmp.Equal(got, tt.want) {
+				t.Errorf("Get() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSet(t *testing.T) {
+	tests := []struct {
+		name string
+		addr resolver.Address
+		md   metadata.MD
+	}{
+		{
+			name: "unset before",
+			addr: resolver.Address{},
+			md:   metadata.Pairs("k", "v"),
+		},
+		{
+			name: "set before",
+			addr: resolver.Address{
+				Attributes: attributes.New(mdKey, metadata.Pairs("bef", "ore")),
+			},
+			md: metadata.Pairs("k", "v"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			newAddr := Set(tt.addr, tt.md)
+			newMD := Get(newAddr)
+			if !cmp.Equal(newMD, tt.md) {
+				t.Errorf("md after Set() = %v, want %v", newMD, tt.md)
+			}
+		})
+	}
+}
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index fef365c0d281..4778ed16252b 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -33,6 +33,7 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 	"google.golang.org/grpc/internal/grpcutil"
+	imetadata "google.golang.org/grpc/internal/metadata"
 	"google.golang.org/grpc/internal/transport/networktype"
 
 	"google.golang.org/grpc/codes"
@@ -60,7 +61,7 @@ type http2Client struct {
 	cancel     context.CancelFunc
 	ctxDone    <-chan struct{} // Cache the ctx.Done() chan.
 	userAgent  string
-	md         interface{}
+	md         metadata.MD
 	conn       net.Conn // underlying communication channel
 	loopy      *loopyWriter
 	remoteAddr net.Addr
@@ -268,7 +269,6 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		ctxDone:               ctx.Done(), // Cache Done chan.
 		cancel:                cancel,
 		userAgent:             opts.UserAgent,
-		md:                    addr.Metadata,
 		conn:                  conn,
 		remoteAddr:            conn.RemoteAddr(),
 		localAddr:             conn.LocalAddr(),
@@ -296,6 +296,12 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		keepaliveEnabled:      keepaliveEnabled,
 		bufferPool:            newBufferPool(),
 	}
+
+	if md, ok := addr.Metadata.(*metadata.MD); ok {
+		t.md = *md
+	} else if md := imetadata.Get(addr); md != nil {
+		t.md = md
+	}
 	t.controlBuf = newControlBuffer(t.ctxDone)
 	if opts.InitialWindowSize >= defaultWindowSize {
 		t.initialWindowSize = opts.InitialWindowSize
@@ -512,14 +518,12 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 			}
 		}
 	}
-	if md, ok := t.md.(*metadata.MD); ok {
-		for k, vv := range *md {
-			if isReservedHeader(k) {
-				continue
-			}
-			for _, v := range vv {
-				headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
-			}
+	for k, vv := range t.md {
+		if isReservedHeader(k) {
+			continue
+		}
+		for _, v := range vv {
+			headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
 		}
 	}
 	return headerFields, nil
diff --git a/test/balancer_test.go b/test/balancer_test.go
index f0189cf26444..7af5c81a0011 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -39,6 +39,7 @@ import (
 	"google.golang.org/grpc/internal/balancerload"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpcutil"
+	imetadata "google.golang.org/grpc/internal/metadata"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
@@ -543,6 +544,76 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) {
 	}
 }
 
+// TestMetadataInAddressAttributes verifies that the metadata added to
+// address.Attributes will be sent with the RPCs.
+func (s) TestMetadataInAddressAttributes(t *testing.T) {
+	const (
+		testMDKey      = "test-md"
+		testMDValue    = "test-md-value"
+		mdBalancerName = "metadata-balancer"
+	)
+
+	// Register a stub balancer which adds metadata to the first address that it
+	// receives and then calls NewSubConn on it.
+	bf := stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			addrs := ccs.ResolverState.Addresses
+			if len(addrs) == 0 {
+				return nil
+			}
+			// Only use the first address.
+			sc, err := bd.ClientConn.NewSubConn([]resolver.Address{
+				imetadata.Set(addrs[0], metadata.Pairs(testMDKey, testMDValue)),
+			}, balancer.NewSubConnOptions{})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+		UpdateSubConnState: func(bd *stub.BalancerData, sc balancer.SubConn, state balancer.SubConnState) {
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}})
+		},
+	}
+	stub.Register(mdBalancerName, bf)
+	t.Logf("Registered balancer %s...", mdBalancerName)
+
+	testMDChan := make(chan []string, 1)
+	ss := &stubServer{
+		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			md, ok := metadata.FromIncomingContext(ctx)
+			if ok {
+				select {
+				case testMDChan <- md[testMDKey]:
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				}
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil, grpc.WithDefaultServiceConfig(
+		fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, mdBalancerName),
+	)); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	// The RPC should succeed with the expected md.
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() = _, %v, want _, <nil>", err)
+	}
+	t.Log("Made an RPC which succeeded...")
+
+	// The server should receive the test metadata.
+	md1 := <-testMDChan
+	if len(md1) == 0 || md1[0] != testMDValue {
+		t.Fatalf("got md: %v, want %v", md1, []string{testMDValue})
+	}
+}
+
 // TestServersSwap creates two servers and verifies the client switches between
 // them when the name resolver reports the first and then the second.
 func (s) TestServersSwap(t *testing.T) {

From 2af65e80c682ba79ae3468cda13c40da63c6f14b Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 19 Nov 2020 11:57:06 -0800
Subject: [PATCH 287/481] advancedtls: Make test happy on Go1.15 (#4053)

---
 security/advancedtls/advancedtls_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index a7ecf276da14..67c0b9dba42a 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -867,8 +867,8 @@ func (s) TestGetCertificatesSNI(t *testing.T) {
 			if err != nil {
 				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
 			}
-			if !cmp.Equal(*gotCertificate, test.wantCert, cmp.AllowUnexported(big.Int{})) {
-				t.Errorf("GetCertificates() = %v, want %v", *gotCertificate, test.wantCert)
+			if !gotCertificate.Leaf.Equal(test.wantCert.Leaf) {
+				t.Errorf("GetCertificates() returned leaf certificate does not match expected (-want +got):\n%s", cmp.Diff(test.wantCert, *gotCertificate, cmp.AllowUnexported(big.Int{})))
 			}
 		})
 	}

From 40076094f63b809cebb5fad728a08cac98373c7c Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 19 Nov 2020 13:15:38 -0800
Subject: [PATCH 288/481] xds: call xdsclient.New instead of getting xds_client
 from attributes (#4032)

---
 .../balancer/cdsbalancer/cdsbalancer.go       | 34 ++++------
 .../cdsbalancer/cdsbalancer_security_test.go  | 25 ++++----
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 63 +++++++------------
 xds/internal/balancer/edsbalancer/eds.go      | 12 +++-
 xds/internal/balancer/edsbalancer/eds_test.go | 29 ++++-----
 .../edsbalancer/xds_client_wrapper.go         | 56 ++---------------
 .../edsbalancer/xds_client_wrapper_test.go    | 54 ++--------------
 .../balancer/edsbalancer/xds_lrs_test.go      | 10 +--
 xds/internal/balancer/lrs/balancer.go         | 36 +++++------
 xds/internal/balancer/lrs/balancer_test.go    | 10 +--
 xds/internal/internal.go                      |  7 ---
 xds/internal/resolver/xds_resolver.go         |  9 +--
 xds/internal/resolver/xds_resolver_test.go    | 12 ----
 13 files changed, 107 insertions(+), 250 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index b37cc733d966..5a869e016f2f 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/connectivity"
@@ -37,7 +36,6 @@ import (
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
@@ -61,6 +59,7 @@ var (
 		// not deal with subConns.
 		return builder.Build(cc, opts), nil
 	}
+	newXDSClient  = func() (xdsClientInterface, error) { return xdsclient.New() }
 	buildProvider = buildProviderFunc
 )
 
@@ -86,6 +85,13 @@ func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.
 	b.logger = prefixLogger((b))
 	b.logger.Infof("Created")
 
+	client, err := newXDSClient()
+	if err != nil {
+		b.logger.Errorf("failed to create xds-client: %v", err)
+		return nil
+	}
+	b.xdsClient = client
+
 	var creds credentials.TransportCredentials
 	switch {
 	case opts.DialCreds != nil:
@@ -141,7 +147,6 @@ type xdsClientInterface interface {
 // watcher with the xdsClient, while a non-nil error causes it to cancel the
 // existing watch and propagate the error to the underlying edsBalancer.
 type ccUpdate struct {
-	client      xdsClientInterface
 	clusterName string
 	err         error
 }
@@ -196,16 +201,9 @@ func (b *cdsBalancer) handleClientConnUpdate(update *ccUpdate) {
 	if err := update.err; err != nil {
 		b.handleErrorFromUpdate(err, true)
 	}
-	if b.xdsClient == update.client && b.clusterToWatch == update.clusterName {
+	if b.clusterToWatch == update.clusterName {
 		return
 	}
-	if update.client != nil {
-		// Since the cdsBalancer doesn't own the xdsClient object, we don't have
-		// to bother about closing the old client here, but we still need to
-		// cancel the watch on the old client.
-		b.cancelWatch()
-		b.xdsClient = update.client
-	}
 	if update.clusterName != "" {
 		cancelWatch := b.xdsClient.WatchCluster(update.clusterName, b.handleClusterUpdate)
 		b.logger.Infof("Watch started on resource name %v with xds-client %p", update.clusterName, b.xdsClient)
@@ -351,7 +349,6 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 
 	}
 	ccState := balancer.ClientConnState{
-		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, b.xdsClient)},
 		BalancerConfig: lbCfg,
 	}
 	if err := b.edsLB.UpdateClientConnState(ccState); err != nil {
@@ -469,17 +466,7 @@ func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) erro
 		b.logger.Warningf("xds: no clusterName found in LoadBalancingConfig: %+v", lbCfg)
 		return balancer.ErrBadResolverState
 	}
-	client := state.ResolverState.Attributes.Value(xdsinternal.XDSClientID)
-	if client == nil {
-		b.logger.Warningf("xds: no xdsClient found in resolver state attributes")
-		return balancer.ErrBadResolverState
-	}
-	newClient, ok := client.(xdsClientInterface)
-	if !ok {
-		b.logger.Warningf("xds: unexpected xdsClient type: %T", client)
-		return balancer.ErrBadResolverState
-	}
-	b.updateCh.Put(&ccUpdate{client: newClient, clusterName: lbCfg.ClusterName})
+	b.updateCh.Put(&ccUpdate{clusterName: lbCfg.ClusterName})
 	return nil
 }
 
@@ -504,6 +491,7 @@ func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 // Close closes the cdsBalancer and the underlying edsBalancer.
 func (b *cdsBalancer) Close() {
 	b.closed.Fire()
+	b.xdsClient.Close()
 }
 
 // ccWrapper wraps the balancer.ClientConn that was passed in to the CDS
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 972d16b7fd30..52c7f23281ee 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -115,6 +115,10 @@ func (p *fakeProvider) Close() {
 func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
 	t.Helper()
 
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+
 	builder := balancer.Get(cdsName)
 	if builder == nil {
 		t.Fatalf("balancer.Get(%q) returned nil", cdsName)
@@ -140,10 +144,8 @@ func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDS
 		return edsB, nil
 	}
 
-	// Create a fake xDS client and push a ClientConnState update to the CDS
-	// balancer with a cluster name and the fake xDS client in the attributes.
-	xdsC := fakeclient.NewClient()
-	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
+	// Push a ClientConnState update to the CDS balancer with a cluster name.
+	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
 
@@ -160,6 +162,7 @@ func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDS
 	}
 
 	return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() {
+		newXDSClient = oldNewXDSClient
 		newEDSBalancer = oldEDSBalancerBuilder
 	}
 }
@@ -229,7 +232,7 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -285,7 +288,7 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// newEDSBalancer function as part of test setup. No security config is
 	// passed to the CDS balancer as part of this update.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -441,7 +444,7 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -475,7 +478,7 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
@@ -506,7 +509,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
@@ -556,7 +559,7 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
@@ -633,7 +636,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 			RootInstanceName: "default1",
 		},
 	}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 2257929e40cd..ccd1699afaac 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -34,7 +34,6 @@ import (
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
@@ -172,7 +171,7 @@ func (tb *testEDSBalancer) waitForClose(ctx context.Context) error {
 
 // cdsCCS is a helper function to construct a good update passed from the
 // xdsResolver to the cdsBalancer.
-func cdsCCS(cluster string, xdsClient interface{}) balancer.ClientConnState {
+func cdsCCS(cluster string) balancer.ClientConnState {
 	const cdsLBConfig = `{
       "loadBalancingConfig":[
         {
@@ -186,7 +185,6 @@ func cdsCCS(cluster string, xdsClient interface{}) balancer.ClientConnState {
 	return balancer.ClientConnState{
 		ResolverState: resolver.State{
 			ServiceConfig: internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(jsonSC),
-			Attributes:    attributes.New(xdsinternal.XDSClientID, xdsClient),
 		},
 		BalancerConfig: &lbConfig{ClusterName: clusterName},
 	}
@@ -194,22 +192,25 @@ func cdsCCS(cluster string, xdsClient interface{}) balancer.ClientConnState {
 
 // edsCCS is a helper function to construct a good update passed from the
 // cdsBalancer to the edsBalancer.
-func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.ClientConnState {
+func edsCCS(service string, enableLRS bool) balancer.ClientConnState {
 	lbCfg := &edsbalancer.EDSConfig{EDSServiceName: service}
 	if enableLRS {
 		lbCfg.LrsLoadReportingServerName = new(string)
 	}
 	return balancer.ClientConnState{
-		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsClient)},
 		BalancerConfig: lbCfg,
 	}
 }
 
 // setup creates a cdsBalancer and an edsBalancer (and overrides the
 // newEDSBalancer function to return it), and also returns a cleanup function.
-func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
+func setup(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
 	t.Helper()
 
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+
 	builder := balancer.Get(cdsName)
 	if builder == nil {
 		t.Fatalf("balancer.Get(%q) returned nil", cdsName)
@@ -224,8 +225,9 @@ func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClie
 		return edsB, nil
 	}
 
-	return cdsB.(*cdsBalancer), edsB, tcc, func() {
+	return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() {
 		newEDSBalancer = oldEDSBalancerBuilder
+		newXDSClient = oldNewXDSClient
 	}
 }
 
@@ -234,9 +236,8 @@ func setup(t *testing.T) (*cdsBalancer, *testEDSBalancer, *xdstestutils.TestClie
 func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
 	t.Helper()
 
-	xdsC := fakeclient.NewClient()
-	cdsB, edsB, tcc, cancel := setup(t)
-	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
+	xdsC, cdsB, edsB, tcc, cancel := setup(t)
+	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
 
@@ -256,8 +257,6 @@ func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBal
 // cdsBalancer with different inputs and verifies that the CDS watch API on the
 // provided xdsClient is invoked appropriately.
 func (s) TestUpdateClientConnState(t *testing.T) {
-	xdsC := fakeclient.NewClient()
-
 	tests := []struct {
 		name        string
 		ccs         balancer.ClientConnState
@@ -274,36 +273,16 @@ func (s) TestUpdateClientConnState(t *testing.T) {
 			ccs:     balancer.ClientConnState{BalancerConfig: &lbConfig{ClusterName: ""}},
 			wantErr: balancer.ErrBadResolverState,
 		},
-		{
-			name: "no-xdsClient-in-attributes",
-			ccs: balancer.ClientConnState{
-				ResolverState: resolver.State{
-					Attributes: attributes.New("key", "value"),
-				},
-				BalancerConfig: &lbConfig{ClusterName: clusterName},
-			},
-			wantErr: balancer.ErrBadResolverState,
-		},
-		{
-			name: "bad-xdsClient-in-attributes",
-			ccs: balancer.ClientConnState{
-				ResolverState: resolver.State{
-					Attributes: attributes.New(xdsinternal.XDSClientID, "value"),
-				},
-				BalancerConfig: &lbConfig{ClusterName: clusterName},
-			},
-			wantErr: balancer.ErrBadResolverState,
-		},
 		{
 			name:        "happy-good-case",
-			ccs:         cdsCCS(clusterName, xdsC),
+			ccs:         cdsCCS(clusterName),
 			wantCluster: clusterName,
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			cdsB, _, _, cancel := setup(t)
+			xdsC, cdsB, _, _, cancel := setup(t)
 			defer func() {
 				cancel()
 				cdsB.Close()
@@ -340,7 +319,7 @@ func (s) TestUpdateClientConnStateWithSameState(t *testing.T) {
 	}()
 
 	// This is the same clientConn update sent in setupWithWatch().
-	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
+	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != nil {
 		t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
 	}
 	// The above update should not result in a new watch being registered.
@@ -370,12 +349,12 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 		{
 			name:      "happy-case-with-lrs",
 			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
-			wantCCS:   edsCCS(serviceName, true, xdsC),
+			wantCCS:   edsCCS(serviceName, true),
 		},
 		{
 			name:      "happy-case-without-lrs",
 			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
-			wantCCS:   edsCCS(serviceName, false, xdsC),
+			wantCCS:   edsCCS(serviceName, false),
 		},
 	}
 
@@ -443,7 +422,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -528,7 +507,7 @@ func (s) TestResolverError(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -577,7 +556,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -609,7 +588,7 @@ func (s) TestClose(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false, xdsC)
+	wantCCS := edsCCS(serviceName, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -640,7 +619,7 @@ func (s) TestClose(t *testing.T) {
 
 	// Make sure that the UpdateClientConnState() method on the CDS balancer
 	// returns error.
-	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, fakeclient.NewClient())); err != errBalancerClosed {
+	if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != errBalancerClosed {
 		t.Fatalf("UpdateClientConnState() after close returned %v, want %v", err, errBalancerClosed)
 	}
 
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 9312b0532cbf..1358dd723cca 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -43,6 +43,7 @@ var (
 	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
 		return newEDSBalancerImpl(cc, enqueueState, xdsClient, logger)
 	}
+	newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
 )
 
 func init() {
@@ -61,7 +62,14 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptio
 		childPolicyUpdate: buffer.NewUnbounded(),
 	}
 	x.logger = prefixLogger((x))
-	x.client = newXDSClientWrapper(x.handleEDSUpdate, x.logger)
+
+	client, err := newXDSClient()
+	if err != nil {
+		x.logger.Errorf("xds: failed to create xds-client: %v", err)
+		return nil
+	}
+
+	x.client = newXDSClientWrapper(client, x.handleEDSUpdate, x.logger)
 	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.client, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
@@ -177,7 +185,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			return
 		}
 
-		if err := x.client.handleUpdate(cfg, u.ResolverState.Attributes); err != nil {
+		if err := x.client.handleUpdate(cfg); err != nil {
 			x.logger.Warningf("failed to update xds clients: %v", err)
 		}
 
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 085310d44b39..eeeaacae21aa 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -30,9 +30,6 @@ import (
 	"github.com/golang/protobuf/jsonpb"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/attributes"
-	xdsinternal "google.golang.org/grpc/xds/internal"
-
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpclog"
@@ -192,15 +189,20 @@ func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
 // edsLB, creates fake version of them and makes them available on the provided
 // channels. The returned cancel function should be called by the test for
 // cleanup.
-func setup(edsLBCh *testutils.Channel) func() {
+func setup(edsLBCh *testutils.Channel) (*fakeclient.Client, func()) {
+	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+
 	origNewEDSBalancer := newEDSBalancer
 	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
 		edsLB := newFakeEDSBalancer(cc)
 		defer func() { edsLBCh.Send(edsLB) }()
 		return edsLB
 	}
-	return func() {
+	return xdsC, func() {
 		newEDSBalancer = origNewEDSBalancer
+		newXDSClient = oldNewXDSClient
 	}
 }
 
@@ -261,9 +263,8 @@ func (b *fakeBalancer) Close() {}
 //   This time around, we expect no new xdsClient or edsLB to be created.
 //   Instead, we expect the existing edsLB to receive the new child policy.
 func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
-	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	cancel := setup(edsLBCh)
+	xdsC, cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -275,7 +276,6 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 	defer edsB.Close()
 
 	edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{
 			ChildPolicy: &loadBalancingConfig{
 				Name:   fakeBalancerA,
@@ -298,7 +298,6 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 	})
 
 	edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{
 			ChildPolicy: &loadBalancingConfig{
 				Name:   fakeBalancerB,
@@ -316,9 +315,8 @@ func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
 // TestXDSSubConnStateChange verifies if the top-level edsBalancer passes on
 // the subConnStateChange to appropriate child balancers.
 func (s) TestXDSSubConnStateChange(t *testing.T) {
-	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	cancel := setup(edsLBCh)
+	xdsC, cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -330,7 +328,6 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 	defer edsB.Close()
 
 	edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
 	})
 
@@ -357,9 +354,8 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 // If it's connection error, nothing will happen. This will need to change to
 // handle fallback.
 func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
-	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	cancel := setup(edsLBCh)
+	xdsC, cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -371,7 +367,6 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	defer edsB.Close()
 
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
 	}); err != nil {
 		t.Fatal(err)
@@ -421,9 +416,8 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 // If it's connection error, nothing will happen. This will need to change to
 // handle fallback.
 func (s) TestErrorFromResolver(t *testing.T) {
-	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 	edsLBCh := testutils.NewChannel()
-	cancel := setup(edsLBCh)
+	xdsC, cancel := setup(edsLBCh)
 	defer cancel()
 
 	builder := balancer.Get(edsName)
@@ -435,7 +429,6 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	defer edsB.Close()
 
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState:  resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
 	}); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
index 16466016bbcd..78902284c7cb 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
@@ -19,12 +19,9 @@
 package edsbalancer
 
 import (
-	"fmt"
 	"sync"
 
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/internal/grpclog"
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
@@ -126,50 +123,15 @@ type xdsClientWrapper struct {
 //
 // The given callbacks won't be called until the underlying xds_client is
 // working and sends updates.
-func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), logger *grpclog.PrefixLogger) *xdsClientWrapper {
+func newXDSClientWrapper(xdsClient xdsClientInterface, newEDSUpdate func(xdsclient.EndpointsUpdate, error), logger *grpclog.PrefixLogger) *xdsClientWrapper {
 	return &xdsClientWrapper{
 		logger:       logger,
 		newEDSUpdate: newEDSUpdate,
+		xdsClient:    xdsClient,
 		loadWrapper:  &loadStoreWrapper{},
 	}
 }
 
-// updateXDSClient sets xdsClient in wrapper to the correct one based on the
-// attributes and service config.
-//
-// If client is found in attributes, it will be used, but we also need to decide
-// whether to close the old client.
-// - if old client was created locally (balancerName is not ""), close it and
-// replace it
-// - if old client was from previous attributes, only replace it, but don't
-// close it
-//
-// If client is not found in attributes, will need to create a new one only if
-// the balancerName (from bootstrap file or from service config) changed.
-// - if balancer names are the same, do nothing, and return false
-// - if balancer names are different, create new one, and return true
-func (c *xdsClientWrapper) updateXDSClient(attr *attributes.Attributes) (bool, error) {
-	if attr == nil {
-		return false, fmt.Errorf("unexported nil attributes, want attributes with xdsClient")
-	}
-	// TODO: change the way xdsClient is retrieved from attributes. One option
-	// is to add helper functions.
-	//
-	// Or, since xdsClient will become a singleton, this can just call
-	// xdsclient.New() instead. And if we decide to do this, do it in Build
-	// instead of when handling updates.
-	clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface)
-	if clientFromAttr == nil {
-		return false, fmt.Errorf("no xdsClient found in attributes")
-	}
-
-	if c.xdsClient == clientFromAttr {
-		return false, nil
-	}
-	c.xdsClient = clientFromAttr
-	return true, nil
-}
-
 // startEndpointsWatch starts the EDS watch. Caller can call this when the
 // xds_client is updated, or the edsServiceName is updated.
 //
@@ -221,16 +183,9 @@ func (c *xdsClientWrapper) loadStore() load.PerClusterReporter {
 
 // handleUpdate applies the service config and attributes updates to the client,
 // including updating the xds_client to use, and updating the EDS name to watch.
-func (c *xdsClientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) error {
-	clientChanged, err := c.updateXDSClient(attr)
-	if err != nil {
-		return err
-	}
-
-	// Need to restart EDS watch when one of the following happens:
-	// - the xds_client is updated
-	// - the xds_client didn't change, but the edsServiceName changed
-	if clientChanged || c.edsServiceName != config.EDSServiceName {
+func (c *xdsClientWrapper) handleUpdate(config *EDSConfig) error {
+	// Need to restart EDS watch when the edsServiceName changed
+	if c.edsServiceName != config.EDSServiceName {
 		c.edsServiceName = config.EDSServiceName
 		c.startEndpointsWatch()
 		// TODO: this update for the LRS service name is too early. It should
@@ -266,6 +221,7 @@ func (c *xdsClientWrapper) cancelWatch() {
 
 func (c *xdsClientWrapper) close() {
 	c.cancelWatch()
+	c.xdsClient.Close()
 }
 
 // equalStringPointers returns true if
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
index 38162d109ffa..60faf6fc3136 100644
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
@@ -25,9 +25,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/internal/testutils"
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
@@ -75,13 +73,13 @@ func verifyExpectedRequests(fc *fakeclient.Client, resourceNames ...string) erro
 func (s) TestClientWrapperWatchEDS(t *testing.T) {
 	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
 
-	cw := newXDSClientWrapper(nil, nil)
+	cw := newXDSClientWrapper(xdsC, nil, nil)
 	defer cw.close()
 	t.Logf("Started xDS client wrapper for endpoint %s...", testServiceName)
 
 	// Update with an non-empty edsServiceName should trigger an EDS watch for
 	// the same.
-	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-1"}, attributes.New(xdsinternal.XDSClientID, xdsC))
+	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-1"})
 	if err := verifyExpectedRequests(xdsC, "foobar-1"); err != nil {
 		t.Fatal(err)
 	}
@@ -90,7 +88,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 	// name to another, and make sure a new watch is registered. The previously
 	// registered watch will be cancelled, which will result in an EDS request
 	// with no resource names being sent to the server.
-	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-2"}, attributes.New(xdsinternal.XDSClientID, xdsC))
+	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-2"})
 	if err := verifyExpectedRequests(xdsC, "", "foobar-2"); err != nil {
 		t.Fatal(err)
 	}
@@ -112,11 +110,11 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 		edsRespChan.Send(&edsUpdate{resp: update, err: err})
 	}
 
-	cw := newXDSClientWrapper(newEDS, nil)
+	xdsC := fakeclient.NewClient()
+	cw := newXDSClientWrapper(xdsC, newEDS, nil)
 	defer cw.close()
 
-	xdsC := fakeclient.NewClient()
-	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC))
+	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName})
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -145,43 +143,3 @@ func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
 		t.Fatalf("want update {nil, %v}, got %+v", watchErr, update)
 	}
 }
-
-// TestClientWrapperGetsXDSClientInAttributes verfies the case where the
-// clientWrapper receives the xdsClient to use in the attributes section of the
-// update.
-func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) {
-	cw := newXDSClientWrapper(nil, nil)
-	defer cw.close()
-
-	xdsC1 := fakeclient.NewClient()
-	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC1))
-
-	// Verify that the eds watch is registered for the expected resource name.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	gotCluster, err := xdsC1.WaitForWatchEDS(ctx)
-	if err != nil {
-		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
-	}
-	if gotCluster != testEDSClusterName {
-		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
-	}
-
-	// Pass a new client in the attributes. Verify that the watch is
-	// re-registered on the new client, and that the old client is not closed
-	// (because clientWrapper only closes clients that it creates, it does not
-	// close client that are passed through attributes).
-	xdsC2 := fakeclient.NewClient()
-	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC2))
-	gotCluster, err = xdsC2.WaitForWatchEDS(ctx)
-	if err != nil {
-		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
-	}
-	if gotCluster != testEDSClusterName {
-		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
-	}
-
-	if err := xdsC1.WaitForClose(ctx); err != context.DeadlineExceeded {
-		t.Fatalf("clientWrapper closed xdsClient received in attributes")
-	}
-}
diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
index 955f54401c86..a9108a2281ed 100644
--- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
@@ -22,10 +22,7 @@ import (
 	"context"
 	"testing"
 
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/resolver"
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
@@ -33,6 +30,11 @@ import (
 // stream when the lbConfig passed to it contains a valid value for the LRS
 // server (empty string).
 func (s) TestXDSLoadReporting(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
 	builder := balancer.Get(edsName)
 	cc := newNoopTestClientConn()
 	edsB, ok := builder.Build(cc, balancer.BuildOptions{}).(*edsBalancer)
@@ -41,9 +43,7 @@ func (s) TestXDSLoadReporting(t *testing.T) {
 	}
 	defer edsB.Close()
 
-	xdsC := fakeclient.NewClient()
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)},
 		BalancerConfig: &EDSConfig{
 			EDSServiceName:             testEDSClusterName,
 			LrsLoadReportingServerName: new(string),
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index f8e7673f7d8e..7bf672b81389 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -24,12 +24,11 @@ import (
 	"fmt"
 	"sync"
 
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal"
-	xdsinternal "google.golang.org/grpc/xds/internal"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
@@ -37,6 +36,8 @@ func init() {
 	balancer.Register(&lrsBB{})
 }
 
+var newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
+
 const lrsBalancerName = "lrs_experimental"
 
 type lrsBB struct{}
@@ -46,9 +47,16 @@ func (l *lrsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balanc
 		cc:        cc,
 		buildOpts: opts,
 	}
-	b.client = newXDSClientWrapper()
 	b.logger = prefixLogger(b)
 	b.logger.Infof("Created")
+
+	client, err := newXDSClient()
+	if err != nil {
+		b.logger.Errorf("failed to create xds-client: %v", err)
+		return nil
+	}
+	b.client = newXDSClientWrapper(client)
+
 	return b
 }
 
@@ -80,7 +88,7 @@ func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	// Update load reporting config or xds client. This needs to be done before
 	// updating the child policy because we need the loadStore from the updated
 	// client to be passed to the ccWrapper.
-	if err := b.client.update(newConfig, s.ResolverState.Attributes); err != nil {
+	if err := b.client.update(newConfig); err != nil {
 		return err
 	}
 
@@ -219,34 +227,21 @@ type xdsClientWrapper struct {
 	loadWrapper *loadStoreWrapper
 }
 
-func newXDSClientWrapper() *xdsClientWrapper {
+func newXDSClientWrapper(c xdsClientInterface) *xdsClientWrapper {
 	return &xdsClientWrapper{
+		c:           c,
 		loadWrapper: &loadStoreWrapper{},
 	}
 }
 
 // update checks the config and xdsclient, and decides whether it needs to
 // restart the load reporting stream.
-func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attributes) error {
+func (w *xdsClientWrapper) update(newConfig *lbConfig) error {
 	var (
 		restartLoadReport           bool
 		updateLoadClusterAndService bool
 	)
 
-	if attr == nil {
-		return fmt.Errorf("lrs: failed to get xdsClient from attributes: attributes is nil")
-	}
-	clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface)
-	if clientFromAttr == nil {
-		return fmt.Errorf("lrs: failed to get xdsClient from attributes: xdsClient not found in attributes")
-	}
-
-	if w.c != clientFromAttr {
-		// xds client is different, restart.
-		restartLoadReport = true
-		w.c = clientFromAttr
-	}
-
 	// ClusterName is different, restart. ClusterName is from ClusterName and
 	// EdsServiceName.
 	if w.clusterName != newConfig.ClusterName {
@@ -301,4 +296,5 @@ func (w *xdsClientWrapper) close() {
 		w.cancelLoadReport()
 		w.cancelLoadReport = nil
 	}
+	w.c.Close()
 }
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index 38dd573ef14b..0794a7767214 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -25,7 +25,6 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
@@ -53,16 +52,19 @@ var (
 // stream when the lbConfig passed to it contains a valid value for the LRS
 // server (empty string).
 func TestLoadReporting(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
 	builder := balancer.Get(lrsBalancerName)
 	cc := testutils.NewTestClientConn(t)
 	lrsB := builder.Build(cc, balancer.BuildOptions{})
 	defer lrsB.Close()
 
-	xdsC := fakeclient.NewClient()
 	if err := lrsB.UpdateClientConnState(balancer.ClientConnState{
 		ResolverState: resolver.State{
-			Addresses:  testBackendAddrs,
-			Attributes: attributes.New(xdsinternal.XDSClientID, xdsC),
+			Addresses: testBackendAddrs,
 		},
 		BalancerConfig: &lbConfig{
 			ClusterName:                testClusterName,
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index 69243147a35a..5c12e6f23d26 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -24,13 +24,6 @@ import (
 	"strings"
 )
 
-type clientID string
-
-// XDSClientID is the attributes key used to pass the address of the xdsClient
-// object shared between the resolver and the balancer. The xdsClient object is
-// created by the resolver and passed to the balancer.
-const XDSClientID = clientID("xdsClientID")
-
 // LocalityID is xds.Locality without XXX fields, so it can be used as map
 // keys.
 //
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 667e7aea3cfe..3e12bb5fc29b 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -22,23 +22,17 @@ package resolver
 import (
 	"fmt"
 
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
 
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
 const xdsScheme = "xds"
 
 // For overriding in unittests.
-var (
-	newXDSClient = func() (xdsClientInterface, error) {
-		return xdsclient.New()
-	}
-)
+var newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
 
 func init() {
 	resolver.Register(&xdsResolverBuilder{})
@@ -163,7 +157,6 @@ func (r *xdsResolver) run() {
 			r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc)
 			r.cc.UpdateState(resolver.State{
 				ServiceConfig: r.cc.ParseServiceConfig(sc),
-				Attributes:    attributes.New(xdsinternal.XDSClientID, r.client),
 			})
 		}
 	}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 6e2715b64cd1..3f92ffdc75e5 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -31,7 +31,6 @@ import (
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
-	xdsinternal "google.golang.org/grpc/xds/internal"
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -310,9 +309,6 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 			t.Fatalf("ClientConn.UpdateState returned error: %v", err)
 		}
 		rState := gotState.(resolver.State)
-		if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != xdsC {
-			t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want %v", gotClient, xdsC)
-		}
 		if err := rState.ServiceConfig.Err; err != nil {
 			t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
 		}
@@ -368,9 +364,6 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
 	}
 	rState := gotState.(resolver.State)
-	if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != xdsC {
-		t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want %v", gotClient, xdsC)
-	}
 	if err := rState.ServiceConfig.Err; err != nil {
 		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
 	}
@@ -419,11 +412,6 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
 	}
 	rState := gotState.(resolver.State)
-	// This update shouldn't have xds-client in it, because it doesn't pick an
-	// xds balancer.
-	if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != nil {
-		t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want <nil>", gotClient)
-	}
 	wantParsedConfig := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)("{}")
 	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantParsedConfig.Config) {
 		t.Error("ClientConn.UpdateState got wrong service config")

From f6c335c7c19d817390ac7b807971a788fe7d748d Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 20 Nov 2020 08:49:47 -0800
Subject: [PATCH 289/481] github: Bump up Go version in testing workflow
 (#4051)

---
 .github/workflows/testing.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 9784f9e09bf3..15d6311d0c07 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -20,7 +20,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
       - name: Checkout repo
         uses: actions/checkout@v2
 
@@ -38,19 +38,19 @@ jobs:
       matrix:
         include:
           - type: vet
-            goversion: 1.14
+            goversion: 1.15
           - type: race
-            goversion: 1.14
+            goversion: 1.15
           - type: 386
-            goversion: 1.14
+            goversion: 1.15
           - type: retry
-            goversion: 1.14
+            goversion: 1.15
           - type: extras
+            goversion: 1.15
+          - type: tests
             goversion: 1.14
           - type: tests
             goversion: 1.13
-          - type: tests
-            goversion: 1.12
           - type: tests
             goversion: 1.11  # Keep until interop tests no longer require Go1.11
 

From 83af8533813d8eb98432201c8ca2ee39bf79b46a Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 20 Nov 2020 12:29:28 -0800
Subject: [PATCH 290/481] trim the Makefile (#4054)

Remove sections which deal with `deps`. These are not required with the
move to Go modules, and create unnecessary diffs in go.mod and go.sum
files.

Left the rest as is, as some are used from `testing.yaml` and others
serve as a handy reference when one wants to build/test something
specific.
---
 Makefile | 30 ++++--------------------------
 1 file changed, 4 insertions(+), 26 deletions(-)

diff --git a/Makefile b/Makefile
index 3f661a7879d6..050941a09169 100644
--- a/Makefile
+++ b/Makefile
@@ -1,14 +1,11 @@
 all: vet test testrace
 
-build: deps
+build:
 	go build google.golang.org/grpc/...
 
 clean:
 	go clean -i google.golang.org/grpc/...
 
-deps:
-	go get -d -v google.golang.org/grpc/...
-
 proto:
 	@ if ! which protoc > /dev/null; then \
 		echo "error: protoc not installed" >&2; \
@@ -16,31 +13,16 @@ proto:
 	fi
 	go generate google.golang.org/grpc/...
 
-test: testdeps
+test:
 	go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
-testsubmodule: testdeps
+testsubmodule:
 	cd security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
 	cd security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/...
 
-testappengine: testappenginedeps
-	goapp test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
-
-testappenginedeps:
-	goapp get -d -v -t -tags 'appengine appenginevm' google.golang.org/grpc/...
-
-testdeps:
-	go get -d -v -t google.golang.org/grpc/...
-
-testrace: testdeps
+testrace:
 	go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
-updatedeps:
-	go get -d -v -u -f google.golang.org/grpc/...
-
-updatetestdeps:
-	go get -d -v -t -u -f google.golang.org/grpc/...
-
 vet: vetdeps
 	./vet.sh
 
@@ -51,14 +33,10 @@ vetdeps:
 	all \
 	build \
 	clean \
-	deps \
 	proto \
 	test \
 	testappengine \
 	testappenginedeps \
-	testdeps \
 	testrace \
-	updatedeps \
-	updatetestdeps \
 	vet \
 	vetdeps

From 8736dcd05b735913226c0c8bf787db48a13a7129 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 20 Nov 2020 12:42:06 -0800
Subject: [PATCH 291/481] xds: Accept a net.Listener in Serve(). (#4052)

---
 .../test/xds_server_integration_test.go       | 13 ++-
 xds/internal/testutils/local_address.go       | 40 ----------
 xds/internal/testutils/local_listener.go      | 26 ++++++
 xds/server.go                                 | 66 ++++------------
 xds/server_test.go                            | 79 ++++---------------
 5 files changed, 62 insertions(+), 162 deletions(-)
 delete mode 100644 xds/internal/testutils/local_address.go
 create mode 100644 xds/internal/testutils/local_listener.go

diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 294ded07a53b..a934067a4339 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -92,15 +92,14 @@ func (s) TestServerSideXDS(t *testing.T) {
 	testpb.RegisterTestServiceServer(server, &testService{})
 	defer server.Stop()
 
-	localAddr, err := testutils.AvailableHostPort()
+	lis, err := testutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
 	}
 
 	go func() {
-		opts := xds.ServeOptions{Address: localAddr}
-		if err := server.Serve(opts); err != nil {
-			t.Errorf("Serve(%+v) failed: %v", opts, err)
+		if err := server.Serve(lis); err != nil {
+			t.Errorf("Serve() failed: %v", err)
 		}
 	}()
 
@@ -108,7 +107,7 @@ func (s) TestServerSideXDS(t *testing.T) {
 	go func() {
 		listener := &v3listenerpb.Listener{
 			// This needs to match the name we are querying for.
-			Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr),
+			Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String()),
 			ApiListener: &v3listenerpb.ApiListener{
 				ApiListener: &anypb.Any{
 					TypeUrl: version.V2HTTPConnManagerURL,
@@ -138,7 +137,7 @@ func (s) TestServerSideXDS(t *testing.T) {
 	}()
 
 	// Create a ClientConn and make a successful RPC.
-	cc, err := grpc.DialContext(ctx, localAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
diff --git a/xds/internal/testutils/local_address.go b/xds/internal/testutils/local_address.go
deleted file mode 100644
index 5ada7368271c..000000000000
--- a/xds/internal/testutils/local_address.go
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package testutils
-
-import "net"
-
-// AvailableHostPort returns a local address to listen on. This will be of the
-// form "host:port", where the host will be a literal IP address, and port
-// must be a literal port number. If the host is a literal IPv6 address it
-// will be enclosed in square brackets, as in "[2001:db8::1]:80.
-//
-// This is useful for tests which need to call the Serve() method on
-// xds.GRPCServer which needs to be passed an IP:Port to listen on, where the IP
-// must be a literal IP and not localhost. This approach will work on support
-// one or both of IPv4 or IPv6.
-func AvailableHostPort() (string, error) {
-	l, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		return "", err
-	}
-	addr := l.Addr().String()
-	l.Close()
-	return addr, nil
-}
diff --git a/xds/internal/testutils/local_listener.go b/xds/internal/testutils/local_listener.go
new file mode 100644
index 000000000000..7da1bc1ce416
--- /dev/null
+++ b/xds/internal/testutils/local_listener.go
@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import "net"
+
+// LocalTCPListener returns a net.Listener listening on local address and port.
+func LocalTCPListener() (net.Listener, error) {
+	return net.Listen("tcp", ":0")
+}
diff --git a/xds/server.go b/xds/server.go
index 51acf08a8694..fc499848b245 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -22,7 +22,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"strconv"
 	"sync"
 
 	"google.golang.org/grpc"
@@ -75,34 +74,6 @@ type grpcServerInterface interface {
 	GracefulStop()
 }
 
-// ServeOptions contains parameters to configure the Serve() method.
-//
-// Experimental
-//
-// Notice: This type is EXPERIMENTAL and may be changed or removed in a
-// later release.
-type ServeOptions struct {
-	// Address contains the local address to listen on. This should be of the
-	// form "host:port", where the host must be a literal IP address, and port
-	// must be a literal port number. If the host is a literal IPv6 address it
-	// must be enclosed in square brackets, as in "[2001:db8::1]:80.
-	Address string
-}
-
-func (so *ServeOptions) validate() error {
-	addr, port, err := net.SplitHostPort(so.Address)
-	if err != nil {
-		return fmt.Errorf("xds: unsupported address %q for server listener", so.Address)
-	}
-	if net.ParseIP(addr) == nil {
-		return fmt.Errorf("xds: failed to parse %q as a valid literal IP address", addr)
-	}
-	if _, err := strconv.Atoi(port); err != nil {
-		return fmt.Errorf("%q is not a valid listener port", port)
-	}
-	return nil
-}
-
 // GRPCServer wraps a gRPC server and provides server-side xDS functionality, by
 // communication with a management server using xDS APIs. It implements the
 // grpc.ServiceRegistrar interface and can be passed to service registration
@@ -173,16 +144,16 @@ func (s *GRPCServer) initXDSClient() error {
 }
 
 // Serve gets the underlying gRPC server to accept incoming connections on the
-// listening address in opts. A connection to the management server, to receive
-// xDS configuration, is initiated here.
+// listener lis, which is expected to be listening on a TCP port.
+//
+// A connection to the management server, to receive xDS configuration, is
+// initiated here.
 //
 // Serve will return a non-nil error unless Stop or GracefulStop is called.
-func (s *GRPCServer) Serve(opts ServeOptions) error {
-	s.logger.Infof("Serve() called with options: %+v", opts)
-
-	// Validate the listening address in opts.
-	if err := opts.validate(); err != nil {
-		return err
+func (s *GRPCServer) Serve(lis net.Listener) error {
+	s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String())
+	if _, ok := lis.Addr().(*net.TCPAddr); !ok {
+		return fmt.Errorf("xds: GRPCServer expects listener to return a net.TCPAddr. Got %T", lis.Addr())
 	}
 
 	// If this is the first time Serve() is being called, we need to initialize
@@ -190,7 +161,7 @@ func (s *GRPCServer) Serve(opts ServeOptions) error {
 	if err := s.initXDSClient(); err != nil {
 		return err
 	}
-	lw, err := s.newListenerWrapper(opts)
+	lw, err := s.newListenerWrapper(lis)
 	if lw == nil {
 		// Error returned can be nil (when Stop/GracefulStop() is called). So,
 		// we need to check the returned listenerWrapper instead.
@@ -199,20 +170,15 @@ func (s *GRPCServer) Serve(opts ServeOptions) error {
 	return s.gs.Serve(lw)
 }
 
-// newListenerWrapper starts a net.Listener on the address specified in opts. It
-// then registers a watch for a Listener resource and blocks until a good
+// newListenerWrapper creates and returns a listenerWrapper, which is a thin
+// wrapper around the passed in listener lis, that can be passed to
+// grpcServer.Serve().
+//
+// It then registers a watch for a Listener resource and blocks until a good
 // response is received or the server is stopped by a call to
 // Stop/GracefulStop().
-//
-// Returns a listenerWrapper, which implements the net.Listener interface, that
-// can be passed to grpcServer.Serve().
-func (s *GRPCServer) newListenerWrapper(opts ServeOptions) (*listenerWrapper, error) {
-	lis, err := net.Listen("tcp", opts.Address)
-	if err != nil {
-		return nil, fmt.Errorf("xds: failed to listen on %+v: %v", opts, err)
-	}
+func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, error) {
 	lw := &listenerWrapper{Listener: lis}
-	s.logger.Infof("Started a net.Listener on %s", lis.Addr().String())
 
 	// This is used to notify that a good update has been received and that
 	// Serve() can be invoked on the underlying gRPC server. Using a
@@ -224,7 +190,7 @@ func (s *GRPCServer) newListenerWrapper(opts ServeOptions) (*listenerWrapper, er
 	// Register an LDS watch using our xdsClient, and specify the listening
 	// address as the resource name.
 	// TODO(easwars): Check if literal IPv6 addresses need an enclosing [].
-	name := fmt.Sprintf(listenerResourceNameFormat, opts.Address)
+	name := fmt.Sprintf(listenerResourceNameFormat, lis.Addr().String())
 	cancelWatch := s.xdsC.WatchListener(name, func(update xdsclient.ListenerUpdate, err error) {
 		if err != nil {
 			// We simply log an error here and hope we get a successful update
diff --git a/xds/server_test.go b/xds/server_test.go
index 189edd880111..91583becc12b 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -50,57 +50,6 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-func (s) TestServeOptions_Validate(t *testing.T) {
-	tests := []struct {
-		desc    string
-		opts    ServeOptions
-		wantErr bool
-	}{
-		{
-			desc:    "empty options",
-			opts:    ServeOptions{},
-			wantErr: true,
-		},
-		{
-			desc:    "bad address",
-			opts:    ServeOptions{Address: "I'm a bad IP address"},
-			wantErr: true,
-		},
-		{
-			desc:    "no port",
-			opts:    ServeOptions{Address: "1.2.3.4"},
-			wantErr: true,
-		},
-		{
-			desc:    "empty hostname",
-			opts:    ServeOptions{Address: ":1234"},
-			wantErr: true,
-		},
-		{
-			desc:    "localhost",
-			opts:    ServeOptions{Address: "localhost:1234"},
-			wantErr: true,
-		},
-		{
-			desc: "ipv4",
-			opts: ServeOptions{Address: "1.2.3.4:1234"},
-		},
-		{
-			desc: "ipv6",
-			opts: ServeOptions{Address: "[1:2::3:4]:1234"},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.desc, func(t *testing.T) {
-			err := test.opts.validate()
-			if (err != nil) != test.wantErr {
-				t.Errorf("ServeOptions.validate(%+v) returned err %v, wantErr: %v", test.opts, err, test.wantErr)
-			}
-		})
-	}
-}
-
 type fakeGRPCServer struct {
 	done              chan struct{}
 	registerServiceCh *testutils.Channel
@@ -229,15 +178,15 @@ func (s) TestServeSuccess(t *testing.T) {
 	server := NewGRPCServer()
 	defer server.Stop()
 
-	localAddr, err := xdstestutils.AvailableHostPort()
+	lis, err := xdstestutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
 	}
 
 	// Call Serve() in a goroutine, and push on a channel when Serve returns.
 	serveDone := testutils.NewChannel()
 	go func() {
-		if err := server.Serve(ServeOptions{Address: localAddr}); err != nil {
+		if err := server.Serve(lis); err != nil {
 			t.Error(err)
 		}
 		serveDone.Send(nil)
@@ -257,7 +206,7 @@ func (s) TestServeSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr)
+	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String())
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -290,15 +239,15 @@ func (s) TestServeWithStop(t *testing.T) {
 	// it after the LDS watch has been registered.
 	server := NewGRPCServer()
 
-	localAddr, err := xdstestutils.AvailableHostPort()
+	lis, err := xdstestutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
 	}
 
 	// Call Serve() in a goroutine, and push on a channel when Serve returns.
 	serveDone := testutils.NewChannel()
 	go func() {
-		if err := server.Serve(ServeOptions{Address: localAddr}); err != nil {
+		if err := server.Serve(lis); err != nil {
 			t.Error(err)
 		}
 		serveDone.Send(nil)
@@ -319,7 +268,7 @@ func (s) TestServeWithStop(t *testing.T) {
 		server.Stop()
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr)
+	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String())
 	if name != wantName {
 		server.Stop()
 		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantName)
@@ -349,14 +298,14 @@ func (s) TestServeBootstrapFailure(t *testing.T) {
 	server := NewGRPCServer()
 	defer server.Stop()
 
-	localAddr, err := xdstestutils.AvailableHostPort()
+	lis, err := xdstestutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
 	}
 
 	serveDone := testutils.NewChannel()
 	go func() {
-		err := server.Serve(ServeOptions{Address: localAddr})
+		err := server.Serve(lis)
 		serveDone.Send(err)
 	}()
 
@@ -393,14 +342,14 @@ func (s) TestServeNewClientFailure(t *testing.T) {
 	server := NewGRPCServer()
 	defer server.Stop()
 
-	localAddr, err := xdstestutils.AvailableHostPort()
+	lis, err := xdstestutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.AvailableHostPort() failed: %v", err)
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
 	}
 
 	serveDone := testutils.NewChannel()
 	go func() {
-		err := server.Serve(ServeOptions{Address: localAddr})
+		err := server.Serve(lis)
 		serveDone.Send(err)
 	}()
 

From 9da74c039bbf0dd892ecf0a3c35c781509d69541 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 20 Nov 2020 16:46:45 -0800
Subject: [PATCH 292/481] xds: Blank import supported certificate provider
 plugins. (#4047)

---
 examples/go.sum |  1 +
 xds/go113.go    | 25 +++++++++++++++++++++++++
 xds/xds.go      |  9 +++++----
 3 files changed, 31 insertions(+), 4 deletions(-)
 create mode 100644 xds/go113.go

diff --git a/examples/go.sum b/examples/go.sum
index b7082ec1e78d..60d10ce19c5b 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -27,6 +27,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
diff --git a/xds/go113.go b/xds/go113.go
new file mode 100644
index 000000000000..40f82cde5c1e
--- /dev/null
+++ b/xds/go113.go
@@ -0,0 +1,25 @@
+// +build go1.13
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	_ "google.golang.org/grpc/credentials/tls/certprovider/meshca" // Register the MeshCA certificate provider plugin.
+)
diff --git a/xds/xds.go b/xds/xds.go
index 3cd0fc20b871..5deafd130a25 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -28,8 +28,9 @@
 package xds
 
 import (
-	_ "google.golang.org/grpc/xds/internal/balancer"  // Register the balancers.
-	_ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 xDS API client.
-	_ "google.golang.org/grpc/xds/internal/client/v3" // Register the v3 xDS API client.
-	_ "google.golang.org/grpc/xds/internal/resolver"  // Register the xds_resolver.
+	_ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin.
+	_ "google.golang.org/grpc/xds/internal/balancer"                // Register the balancers.
+	_ "google.golang.org/grpc/xds/internal/client/v2"               // Register the v2 xDS API client.
+	_ "google.golang.org/grpc/xds/internal/client/v3"               // Register the v3 xDS API client.
+	_ "google.golang.org/grpc/xds/internal/resolver"                // Register the xds_resolver.
 )

From 6d0f0110bf693386c7c2ef8dcad774c8f6b25697 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 23 Nov 2020 09:44:03 -0800
Subject: [PATCH 293/481] meshca: Don't use the config proto from grpc-proto
 (#4056)

---
 .../tls/certprovider/meshca/builder_test.go   |  13 +-
 credentials/tls/certprovider/meshca/config.go |  80 +++--
 .../tls/certprovider/meshca/config_test.go    | 294 +++++++---------
 .../internal/meshca_experimental/config.pb.go | 331 ------------------
 .../tls/certprovider/meshca/plugin_test.go    |  16 +-
 regenerate.sh                                 |  23 +-
 6 files changed, 197 insertions(+), 560 deletions(-)
 delete mode 100644 credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go

diff --git a/credentials/tls/certprovider/meshca/builder_test.go b/credentials/tls/certprovider/meshca/builder_test.go
index d21307f4a95b..79035d008d9e 100644
--- a/credentials/tls/certprovider/meshca/builder_test.go
+++ b/credentials/tls/certprovider/meshca/builder_test.go
@@ -22,15 +22,13 @@ package meshca
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"testing"
 
-	"github.com/golang/protobuf/proto"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials/tls/certprovider"
-	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
 	"google.golang.org/grpc/internal/testutils"
 )
 
@@ -69,11 +67,10 @@ func (s) TestBuildSameConfig(t *testing.T) {
 
 	// Parse a good config to generate a stable config which will be passed to
 	// invocations of Build().
-	inputConfig := makeJSONConfig(t, goodConfigFullySpecified)
 	builder := newPluginBuilder()
-	buildableConfig, err := builder.ParseConfig(inputConfig)
+	buildableConfig, err := builder.ParseConfig(goodConfigFullySpecified)
 	if err != nil {
-		t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
+		t.Fatalf("builder.ParseConfig(%q) failed: %v", goodConfigFullySpecified, err)
 	}
 
 	// Create multiple providers with the same config. All these providers must
@@ -143,9 +140,7 @@ func (s) TestBuildDifferentConfig(t *testing.T) {
 	for i := 0; i < cnt; i++ {
 		// Copy the good test config and modify the serverURI to make sure that
 		// a new provider is created for the config.
-		cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
-		cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = fmt.Sprintf("test-mesh-ca:%d", i)
-		inputConfig := makeJSONConfig(t, cfg)
+		inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, fmt.Sprintf("test-mesh-ca:%d", i)))
 		buildableConfig, err := builder.ParseConfig(inputConfig)
 		if err != nil {
 			t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
diff --git a/credentials/tls/certprovider/meshca/config.go b/credentials/tls/certprovider/meshca/config.go
index 029181a38fc1..c0772b3bb7ea 100644
--- a/credentials/tls/certprovider/meshca/config.go
+++ b/credentials/tls/certprovider/meshca/config.go
@@ -21,7 +21,6 @@
 package meshca
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -33,11 +32,11 @@ import (
 	"time"
 
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/types/known/durationpb"
 
 	"google.golang.org/grpc/credentials/sts"
-	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
 )
 
 const (
@@ -46,12 +45,12 @@ const (
 	mdsRequestTimeout = 5 * time.Second
 
 	// The following are default values used in the interaction with MeshCA.
-	defaultMeshCaEndpoint = "meshca.googleapis.com"
-	defaultCallTimeout    = 10 * time.Second
-	defaultCertLifetime   = 24 * time.Hour
-	defaultCertGraceTime  = 12 * time.Hour
-	defaultKeyTypeRSA     = "RSA"
-	defaultKeySize        = 2048
+	defaultMeshCaEndpoint    = "meshca.googleapis.com"
+	defaultCallTimeout       = 10 * time.Second
+	defaultCertLifetimeSecs  = 86400 // 24h in seconds
+	defaultCertGraceTimeSecs = 43200 // 12h in seconds
+	defaultKeyTypeRSA        = "RSA"
+	defaultKeySize           = 2048
 
 	// The following are default values used in the interaction with STS or
 	// Secure Token Service, which is used to exchange the JWT token for an
@@ -80,6 +79,12 @@ type pluginConfig struct {
 	location      string
 }
 
+// Type of key to be embedded in CSRs sent to the MeshCA.
+const (
+	keyTypeUnknown = 0
+	keyTypeRSA     = 1
+)
+
 // pluginConfigFromJSON parses the provided config in JSON.
 //
 // For certain values missing in the config, we use default values defined at
@@ -89,18 +94,49 @@ type pluginConfig struct {
 // GKE Metadata server and try to infer these values. If this attempt does not
 // succeed, we let those fields have empty values.
 func pluginConfigFromJSON(data json.RawMessage) (*pluginConfig, error) {
-	cfgProto := &configpb.GoogleMeshCaConfig{}
-	m := jsonpb.Unmarshaler{AllowUnknownFields: true}
-	if err := m.Unmarshal(bytes.NewReader(data), cfgProto); err != nil {
+	// This anonymous struct corresponds to the expected JSON config.
+	cfgJSON := &struct {
+		Server              json.RawMessage `json:"server,omitempty"`               // Expect a v3corepb.ApiConfigSource
+		CertificateLifetime json.RawMessage `json:"certificate_lifetime,omitempty"` // Expect a durationpb.Duration
+		RenewalGracePeriod  json.RawMessage `json:"renewal_grace_period,omitempty"` // Expect a durationpb.Duration
+		KeyType             int             `json:"key_type,omitempty"`
+		KeySize             int             `json:"key_size,omitempty"`
+		Location            string          `json:"location,omitempty"`
+	}{}
+	if err := json.Unmarshal(data, cfgJSON); err != nil {
 		return nil, fmt.Errorf("meshca: failed to unmarshal config: %v", err)
 	}
 
-	if api := cfgProto.GetServer().GetApiType(); api != v3corepb.ApiConfigSource_GRPC {
+	// Further unmarshal fields represented as json.RawMessage in the above
+	// anonymous struct, and use default values if not specified.
+	server := &v3corepb.ApiConfigSource{}
+	if cfgJSON.Server != nil {
+		if err := protojson.Unmarshal(cfgJSON.Server, server); err != nil {
+			return nil, fmt.Errorf("meshca: protojson.Unmarshal(%+v) failed: %v", cfgJSON.Server, err)
+		}
+	}
+	certLifetime := &durationpb.Duration{Seconds: defaultCertLifetimeSecs}
+	if cfgJSON.CertificateLifetime != nil {
+		if err := protojson.Unmarshal(cfgJSON.CertificateLifetime, certLifetime); err != nil {
+			return nil, fmt.Errorf("meshca: protojson.Unmarshal(%+v) failed: %v", cfgJSON.CertificateLifetime, err)
+		}
+	}
+	certGraceTime := &durationpb.Duration{Seconds: defaultCertGraceTimeSecs}
+	if cfgJSON.RenewalGracePeriod != nil {
+		if err := protojson.Unmarshal(cfgJSON.RenewalGracePeriod, certGraceTime); err != nil {
+			return nil, fmt.Errorf("meshca: protojson.Unmarshal(%+v) failed: %v", cfgJSON.RenewalGracePeriod, err)
+		}
+	}
+
+	if api := server.GetApiType(); api != v3corepb.ApiConfigSource_GRPC {
 		return nil, fmt.Errorf("meshca: server has apiType %s, want %s", api, v3corepb.ApiConfigSource_GRPC)
 	}
 
-	pc := &pluginConfig{}
-	gs := cfgProto.GetServer().GetGrpcServices()
+	pc := &pluginConfig{
+		certLifetime:  certLifetime.AsDuration(),
+		certGraceTime: certGraceTime.AsDuration(),
+	}
+	gs := server.GetGrpcServices()
 	if l := len(gs); l != 1 {
 		return nil, fmt.Errorf("meshca: number of gRPC services in config is %d, expected 1", l)
 	}
@@ -136,23 +172,17 @@ func pluginConfigFromJSON(data json.RawMessage) (*pluginConfig, error) {
 	if pc.callTimeout, err = ptypes.Duration(grpcService.GetTimeout()); err != nil {
 		pc.callTimeout = defaultCallTimeout
 	}
-	if pc.certLifetime, err = ptypes.Duration(cfgProto.GetCertificateLifetime()); err != nil {
-		pc.certLifetime = defaultCertLifetime
-	}
-	if pc.certGraceTime, err = ptypes.Duration(cfgProto.GetRenewalGracePeriod()); err != nil {
-		pc.certGraceTime = defaultCertGraceTime
-	}
-	switch cfgProto.GetKeyType() {
-	case configpb.GoogleMeshCaConfig_KEY_TYPE_UNKNOWN, configpb.GoogleMeshCaConfig_KEY_TYPE_RSA:
+	switch cfgJSON.KeyType {
+	case keyTypeUnknown, keyTypeRSA:
 		pc.keyType = defaultKeyTypeRSA
 	default:
 		return nil, fmt.Errorf("meshca: unsupported key type: %s, only support RSA keys", pc.keyType)
 	}
-	pc.keySize = int(cfgProto.GetKeySize())
+	pc.keySize = cfgJSON.KeySize
 	if pc.keySize == 0 {
 		pc.keySize = defaultKeySize
 	}
-	pc.location = cfgProto.GetLocation()
+	pc.location = cfgJSON.Location
 	if pc.location == "" {
 		pc.location = readZoneFunc(makeHTTPDoer())
 	}
diff --git a/credentials/tls/certprovider/meshca/config_test.go b/credentials/tls/certprovider/meshca/config_test.go
index f5e9b41747aa..5deb484f341c 100644
--- a/credentials/tls/certprovider/meshca/config_test.go
+++ b/credentials/tls/certprovider/meshca/config_test.go
@@ -30,12 +30,8 @@ import (
 	"strings"
 	"testing"
 
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	"github.com/golang/protobuf/jsonpb"
-	durationpb "github.com/golang/protobuf/ptypes/duration"
 	"github.com/google/go-cmp/cmp"
 
-	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 )
@@ -55,84 +51,66 @@ func Test(t *testing.T) {
 }
 
 var (
-	goodConfigFullySpecified = &configpb.GoogleMeshCaConfig{
-		Server: &v3corepb.ApiConfigSource{
-			ApiType: v3corepb.ApiConfigSource_GRPC,
-			GrpcServices: []*v3corepb.GrpcService{
+	goodConfigFormatStr = `
+	{
+		"server": {
+			"api_type": 2,
+			"grpc_services": [
 				{
-					TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
-						GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
-							TargetUri: "test-meshca",
-							CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
-								// This call creds should be ignored.
-								{
-									CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_AccessToken{},
-								},
-								{
-									CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService_{
-										StsService: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService{
-											TokenExchangeServiceUri: "http://test-sts",
-											Resource:                "test-resource",
-											Audience:                "test-audience",
-											Scope:                   "test-scope",
-											RequestedTokenType:      "test-requested-token-type",
-											SubjectTokenPath:        "test-subject-token-path",
-											SubjectTokenType:        "test-subject-token-type",
-											ActorTokenPath:          "test-actor-token-path",
-											ActorTokenType:          "test-actor-token-type",
-										},
-									},
-								},
+					"googleGrpc": {
+						"target_uri": %q,
+						"call_credentials": [
+							{
+								"access_token": "foo"
 							},
-						},
+							{
+								"sts_service": {
+									"token_exchange_service_uri": "http://test-sts",
+									"resource": "test-resource",
+									"audience": "test-audience",
+									"scope": "test-scope",
+									"requested_token_type": "test-requested-token-type",
+									"subject_token_path": "test-subject-token-path",
+									"subject_token_type": "test-subject-token-type",
+									"actor_token_path": "test-actor-token-path",
+									"actor_token_type": "test-actor-token-type"
+								}
+							}
+						]
 					},
-					Timeout: &durationpb.Duration{Seconds: 10}, // 10s
-				},
-			},
+					"timeout": "10s"
+				}
+			]
 		},
-		CertificateLifetime: &durationpb.Duration{Seconds: 86400}, // 1d
-		RenewalGracePeriod:  &durationpb.Duration{Seconds: 43200}, //12h
-		KeyType:             configpb.GoogleMeshCaConfig_KEY_TYPE_RSA,
-		KeySize:             uint32(2048),
-		Location:            "us-west1-b",
-	}
-	goodConfigWithDefaults = &configpb.GoogleMeshCaConfig{
-		Server: &v3corepb.ApiConfigSource{
-			ApiType: v3corepb.ApiConfigSource_GRPC,
-			GrpcServices: []*v3corepb.GrpcService{
+		"certificate_lifetime": "86400s",
+		"renewal_grace_period":  "43200s",
+		"key_type": 1,
+		"key_size": 2048,
+		"location": "us-west1-b"
+	}`
+	goodConfigWithDefaults = json.RawMessage(`
+	{
+		"server": {
+			"api_type": 2,
+			"grpc_services": [
 				{
-					TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
-						GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
-							CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
-								{
-									CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService_{
-										StsService: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService{
-											SubjectTokenPath: "test-subject-token-path",
-										},
-									},
-								},
-							},
-						},
+					"googleGrpc": {
+						"call_credentials": [
+							{
+								"sts_service": {
+									"subject_token_path": "test-subject-token-path"
+								}
+							}
+						]
 					},
-				},
-			},
-		},
-	}
+					"timeout": "10s"
+				}
+			]
+		}
+	}`)
 )
 
-// makeJSONConfig marshals the provided config proto into JSON. This makes it
-// possible for tests to specify the config in proto form, which is much easier
-// than specifying the config in JSON form.
-func makeJSONConfig(t *testing.T, cfg *configpb.GoogleMeshCaConfig) json.RawMessage {
-	t.Helper()
-
-	b := &bytes.Buffer{}
-	m := &jsonpb.Marshaler{EnumsAsInts: true}
-	if err := m.Marshal(b, cfg); err != nil {
-		t.Fatalf("jsonpb.Marshal(%+v) failed: %v", cfg, err)
-	}
-	return json.RawMessage(b.Bytes())
-}
+var goodConfigFullySpecified = json.RawMessage(fmt.Sprintf(goodConfigFormatStr, "test-meshca"))
 
 // verifyReceivedRequest reads the HTTP request received by the fake client
 // (exposed through a channel), and verifies that it matches the expected
@@ -157,23 +135,21 @@ func verifyReceivedRequest(fc *testutils.FakeHTTPClient, wantURI string) error {
 // TestParseConfigSuccessFullySpecified tests the case where the config is fully
 // specified and no defaults are required.
 func (s) TestParseConfigSuccessFullySpecified(t *testing.T) {
-	inputConfig := makeJSONConfig(t, goodConfigFullySpecified)
 	wantConfig := "test-meshca:http://test-sts:test-resource:test-audience:test-scope:test-requested-token-type:test-subject-token-path:test-subject-token-type:test-actor-token-path:test-actor-token-type:10s:24h0m0s:12h0m0s:RSA:2048:us-west1-b"
 
-	cfg, err := pluginConfigFromJSON(inputConfig)
+	cfg, err := pluginConfigFromJSON(goodConfigFullySpecified)
 	if err != nil {
-		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", inputConfig, err)
+		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", goodConfigFullySpecified, err)
 	}
 	gotConfig := cfg.canonical()
 	if diff := cmp.Diff(wantConfig, string(gotConfig)); diff != "" {
-		t.Errorf("pluginConfigFromJSON(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
+		t.Errorf("pluginConfigFromJSON(%q) returned config does not match expected (-want +got):\n%s", string(goodConfigFullySpecified), diff)
 	}
 }
 
 // TestParseConfigSuccessWithDefaults tests cases where the config is not fully
 // specified, and we end up using some sane defaults.
 func (s) TestParseConfigSuccessWithDefaults(t *testing.T) {
-	inputConfig := makeJSONConfig(t, goodConfigWithDefaults)
 	wantConfig := fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s",
 		"meshca.googleapis.com",      // Mesh CA Server URI.
 		"securetoken.googleapis.com", // STS Server URI.
@@ -248,13 +224,13 @@ func (s) TestParseConfigSuccessWithDefaults(t *testing.T) {
 		errCh <- nil
 	}()
 
-	cfg, err := pluginConfigFromJSON(inputConfig)
+	cfg, err := pluginConfigFromJSON(goodConfigWithDefaults)
 	if err != nil {
-		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", inputConfig, err)
+		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", goodConfigWithDefaults, err)
 	}
 	gotConfig := cfg.canonical()
 	if diff := cmp.Diff(wantConfig, string(gotConfig)); diff != "" {
-		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", inputConfig, diff)
+		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", goodConfigWithDefaults, diff)
 	}
 
 	if err := <-errCh; err != nil {
@@ -277,113 +253,109 @@ func (s) TestParseConfigFailureCases(t *testing.T) {
 		},
 		{
 			desc: "bad apiType",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType: v3corepb.ApiConfigSource_REST,
-				},
-			}),
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 1
+				}
+			}`),
 			wantErr: "server has apiType REST, want GRPC",
 		},
 		{
 			desc: "no grpc services",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType: v3corepb.ApiConfigSource_GRPC,
-				},
-			}),
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 2
+				}
+			}`),
 			wantErr: "number of gRPC services in config is 0, expected 1",
 		},
 		{
 			desc: "too many grpc services",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType:      v3corepb.ApiConfigSource_GRPC,
-					GrpcServices: []*v3corepb.GrpcService{nil, nil},
-				},
-			}),
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 2,
+					"grpc_services": [{}, {}]
+				}
+			}`),
 			wantErr: "number of gRPC services in config is 2, expected 1",
 		},
 		{
 			desc: "missing google grpc service",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType: v3corepb.ApiConfigSource_GRPC,
-					GrpcServices: []*v3corepb.GrpcService{
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 2,
+					"grpc_services": [
 						{
-							TargetSpecifier: &v3corepb.GrpcService_EnvoyGrpc_{
-								EnvoyGrpc: &v3corepb.GrpcService_EnvoyGrpc{
-									ClusterName: "foo",
-								},
-							},
-						},
-					},
-				},
-			}),
+							"envoyGrpc": {}
+						}
+					]
+				}
+			}`),
 			wantErr: "missing google gRPC service in config",
 		},
 		{
 			desc: "missing call credentials",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType: v3corepb.ApiConfigSource_GRPC,
-					GrpcServices: []*v3corepb.GrpcService{
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 2,
+					"grpc_services": [
 						{
-							TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
-								GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
-									TargetUri: "foo",
-								},
-							},
-						},
-					},
-				},
-			}),
+							"googleGrpc": {
+								"target_uri": "foo"
+							}
+						}
+					]
+				}
+			}`),
 			wantErr: "missing call credentials in config",
 		},
 		{
 			desc: "missing STS call credentials",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType: v3corepb.ApiConfigSource_GRPC,
-					GrpcServices: []*v3corepb.GrpcService{
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 2,
+					"grpc_services": [
 						{
-							TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
-								GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
-									TargetUri: "foo",
-									CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
-										{
-											CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_AccessToken{},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			}),
+							"googleGrpc": {
+								"target_uri": "foo",
+								"call_credentials": [
+									{
+										"access_token": "foo"
+									}
+								]
+							}
+						}
+					]
+				}
+			}`),
 			wantErr: "missing STS call credentials in config",
 		},
 		{
 			desc: "with no defaults",
-			inputConfig: makeJSONConfig(t, &configpb.GoogleMeshCaConfig{
-				Server: &v3corepb.ApiConfigSource{
-					ApiType: v3corepb.ApiConfigSource_GRPC,
-					GrpcServices: []*v3corepb.GrpcService{
+			inputConfig: json.RawMessage(`
+			{
+				"server": {
+					"api_type": 2,
+					"grpc_services": [
 						{
-							TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{
-								GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{
-									CallCredentials: []*v3corepb.GrpcService_GoogleGrpc_CallCredentials{
-										{
-											CredentialSpecifier: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService_{
-												StsService: &v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService{},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			}),
+							"googleGrpc": {
+								"target_uri": "foo",
+								"call_credentials": [
+									{
+										"sts_service": {}
+									}
+								]
+							}
+						}
+					]
+				}
+			}`),
 			wantErr: "missing subjectTokenPath in STS call credentials config",
 		},
 	}
diff --git a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
deleted file mode 100644
index 846586782137..000000000000
--- a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go
+++ /dev/null
@@ -1,331 +0,0 @@
-// Copyright 2020 The gRPC Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: grpc/tls/provider/meshca/experimental/config.proto
-
-// NOTE: This proto will very likely move to a different namespace and a
-// different git repo in the future.
-
-package meshca_experimental
-
-import (
-	v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	durationpb "google.golang.org/protobuf/types/known/durationpb"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-// Type of key to be embedded in CSRs sent to the MeshCA.
-type GoogleMeshCaConfig_KeyType int32
-
-const (
-	GoogleMeshCaConfig_KEY_TYPE_UNKNOWN GoogleMeshCaConfig_KeyType = 0
-	GoogleMeshCaConfig_KEY_TYPE_RSA     GoogleMeshCaConfig_KeyType = 1
-)
-
-// Enum value maps for GoogleMeshCaConfig_KeyType.
-var (
-	GoogleMeshCaConfig_KeyType_name = map[int32]string{
-		0: "KEY_TYPE_UNKNOWN",
-		1: "KEY_TYPE_RSA",
-	}
-	GoogleMeshCaConfig_KeyType_value = map[string]int32{
-		"KEY_TYPE_UNKNOWN": 0,
-		"KEY_TYPE_RSA":     1,
-	}
-)
-
-func (x GoogleMeshCaConfig_KeyType) Enum() *GoogleMeshCaConfig_KeyType {
-	p := new(GoogleMeshCaConfig_KeyType)
-	*p = x
-	return p
-}
-
-func (x GoogleMeshCaConfig_KeyType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (GoogleMeshCaConfig_KeyType) Descriptor() protoreflect.EnumDescriptor {
-	return file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes[0].Descriptor()
-}
-
-func (GoogleMeshCaConfig_KeyType) Type() protoreflect.EnumType {
-	return &file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes[0]
-}
-
-func (x GoogleMeshCaConfig_KeyType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use GoogleMeshCaConfig_KeyType.Descriptor instead.
-func (GoogleMeshCaConfig_KeyType) EnumDescriptor() ([]byte, []int) {
-	return file_grpc_tls_provider_meshca_experimental_config_proto_rawDescGZIP(), []int{0, 0}
-}
-
-// GoogleMeshCaConfig contains all configuration parameters required by the
-// MeshCA CertificateProvider plugin implementation.
-type GoogleMeshCaConfig struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// GoogleMeshCA server endpoint to get CSRs signed via the *CreateCertificate*
-	// unary call. This must have :ref:`api_type
-	// <envoy_api_field_config.core.v3.ApiConfigSource.api_type>` :ref:`GRPC
-	// <envoy_api_enum_value_config.core.v3.ApiConfigSource.ApiType.GRPC>`.
-	// STS based call credentials need to be supplied in :ref:`call_credentials
-	// <envoy_api_field_config.core.v3.GrpcService.GoogleGrpc.call_credentials>`.
-	//
-	// If :ref:`timeout envoy_api_field_config.core.v3.GrpcService.timeout` is
-	// left unspecified, a default value of 10s will be used.
-	Server *v3.ApiConfigSource `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"`
-	// Certificate lifetime to request in CSRs sent to the MeshCA.
-	//
-	// A default value of 24h will be used if left unspecified.
-	CertificateLifetime *durationpb.Duration `protobuf:"bytes,2,opt,name=certificate_lifetime,json=certificateLifetime,proto3" json:"certificate_lifetime,omitempty"`
-	// How long before certificate expiration should the certificate be renewed.
-	//
-	// A default value of 12h will be used if left unspecified.
-	RenewalGracePeriod *durationpb.Duration `protobuf:"bytes,3,opt,name=renewal_grace_period,json=renewalGracePeriod,proto3" json:"renewal_grace_period,omitempty"`
-	// Type of key.
-	//
-	// RSA keys will be used if left unspecified.
-	KeyType GoogleMeshCaConfig_KeyType `protobuf:"varint,4,opt,name=key_type,json=keyType,proto3,enum=grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig_KeyType" json:"key_type,omitempty"`
-	// Size of the key in bits.
-	//
-	// 2048 bit keys will be used if left unspecified.
-	KeySize uint32 `protobuf:"varint,5,opt,name=key_size,json=keySize,proto3" json:"key_size,omitempty"`
-	// GCE location (region/zone) where the workload is located.
-	//
-	// GCE/GKE Metadata Server will be contacted if left unspecified.
-	Location string `protobuf:"bytes,6,opt,name=location,proto3" json:"location,omitempty"`
-}
-
-func (x *GoogleMeshCaConfig) Reset() {
-	*x = GoogleMeshCaConfig{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *GoogleMeshCaConfig) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*GoogleMeshCaConfig) ProtoMessage() {}
-
-func (x *GoogleMeshCaConfig) ProtoReflect() protoreflect.Message {
-	mi := &file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use GoogleMeshCaConfig.ProtoReflect.Descriptor instead.
-func (*GoogleMeshCaConfig) Descriptor() ([]byte, []int) {
-	return file_grpc_tls_provider_meshca_experimental_config_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *GoogleMeshCaConfig) GetServer() *v3.ApiConfigSource {
-	if x != nil {
-		return x.Server
-	}
-	return nil
-}
-
-func (x *GoogleMeshCaConfig) GetCertificateLifetime() *durationpb.Duration {
-	if x != nil {
-		return x.CertificateLifetime
-	}
-	return nil
-}
-
-func (x *GoogleMeshCaConfig) GetRenewalGracePeriod() *durationpb.Duration {
-	if x != nil {
-		return x.RenewalGracePeriod
-	}
-	return nil
-}
-
-func (x *GoogleMeshCaConfig) GetKeyType() GoogleMeshCaConfig_KeyType {
-	if x != nil {
-		return x.KeyType
-	}
-	return GoogleMeshCaConfig_KEY_TYPE_UNKNOWN
-}
-
-func (x *GoogleMeshCaConfig) GetKeySize() uint32 {
-	if x != nil {
-		return x.KeySize
-	}
-	return 0
-}
-
-func (x *GoogleMeshCaConfig) GetLocation() string {
-	if x != nil {
-		return x.Location
-	}
-	return ""
-}
-
-var File_grpc_tls_provider_meshca_experimental_config_proto protoreflect.FileDescriptor
-
-var file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc = []byte{
-	0x0a, 0x32, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x6c, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69,
-	0x64, 0x65, 0x72, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2f, 0x65, 0x78, 0x70, 0x65, 0x72,
-	0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x25, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x6c, 0x73, 0x2e, 0x70,
-	0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x65,
-	0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x1a, 0x28, 0x65, 0x6e, 0x76,
-	0x6f, 0x79, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76,
-	0x33, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb6, 0x03, 0x0a, 0x12, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x4d, 0x65, 0x73, 0x68, 0x43, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3d, 0x0a, 0x06,
-	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x65,
-	0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x72, 0x65,
-	0x2e, 0x76, 0x33, 0x2e, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x4c, 0x0a, 0x14, 0x63,
-	0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74,
-	0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
-	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74,
-	0x65, 0x4c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x14, 0x72, 0x65, 0x6e,
-	0x65, 0x77, 0x61, 0x6c, 0x5f, 0x67, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f,
-	0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x52, 0x12, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x61, 0x6c, 0x47, 0x72, 0x61, 0x63, 0x65,
-	0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x5c, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x74, 0x79,
-	0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x41, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
-	0x74, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x6d, 0x65, 0x73,
-	0x68, 0x63, 0x61, 0x2e, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c,
-	0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x61, 0x43, 0x6f, 0x6e,
-	0x66, 0x69, 0x67, 0x2e, 0x4b, 0x65, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x6b, 0x65, 0x79,
-	0x54, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x69, 0x7a, 0x65,
-	0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x53, 0x69, 0x7a, 0x65, 0x12,
-	0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x31, 0x0a, 0x07, 0x4b,
-	0x65, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4b, 0x45, 0x59, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c,
-	0x4b, 0x45, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x10, 0x01, 0x42, 0x98,
-	0x01, 0x0a, 0x28, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x6c, 0x73, 0x2e, 0x70,
-	0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x65,
-	0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x42, 0x11, 0x4d, 0x65, 0x73,
-	0x68, 0x43, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
-	0x5a, 0x57, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e,
-	0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
-	0x69, 0x61, 0x6c, 0x73, 0x2f, 0x74, 0x6c, 0x73, 0x2f, 0x63, 0x65, 0x72, 0x74, 0x70, 0x72, 0x6f,
-	0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2f, 0x69, 0x6e, 0x74,
-	0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x5f, 0x65, 0x78, 0x70,
-	0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
-}
-
-var (
-	file_grpc_tls_provider_meshca_experimental_config_proto_rawDescOnce sync.Once
-	file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData = file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc
-)
-
-func file_grpc_tls_provider_meshca_experimental_config_proto_rawDescGZIP() []byte {
-	file_grpc_tls_provider_meshca_experimental_config_proto_rawDescOnce.Do(func() {
-		file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData)
-	})
-	return file_grpc_tls_provider_meshca_experimental_config_proto_rawDescData
-}
-
-var file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
-var file_grpc_tls_provider_meshca_experimental_config_proto_goTypes = []interface{}{
-	(GoogleMeshCaConfig_KeyType)(0), // 0: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.KeyType
-	(*GoogleMeshCaConfig)(nil),      // 1: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig
-	(*v3.ApiConfigSource)(nil),      // 2: envoy.config.core.v3.ApiConfigSource
-	(*durationpb.Duration)(nil),     // 3: google.protobuf.Duration
-}
-var file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs = []int32{
-	2, // 0: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.server:type_name -> envoy.config.core.v3.ApiConfigSource
-	3, // 1: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.certificate_lifetime:type_name -> google.protobuf.Duration
-	3, // 2: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.renewal_grace_period:type_name -> google.protobuf.Duration
-	0, // 3: grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.key_type:type_name -> grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig.KeyType
-	4, // [4:4] is the sub-list for method output_type
-	4, // [4:4] is the sub-list for method input_type
-	4, // [4:4] is the sub-list for extension type_name
-	4, // [4:4] is the sub-list for extension extendee
-	0, // [0:4] is the sub-list for field type_name
-}
-
-func init() { file_grpc_tls_provider_meshca_experimental_config_proto_init() }
-func file_grpc_tls_provider_meshca_experimental_config_proto_init() {
-	if File_grpc_tls_provider_meshca_experimental_config_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GoogleMeshCaConfig); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc,
-			NumEnums:      1,
-			NumMessages:   1,
-			NumExtensions: 0,
-			NumServices:   0,
-		},
-		GoTypes:           file_grpc_tls_provider_meshca_experimental_config_proto_goTypes,
-		DependencyIndexes: file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs,
-		EnumInfos:         file_grpc_tls_provider_meshca_experimental_config_proto_enumTypes,
-		MessageInfos:      file_grpc_tls_provider_meshca_experimental_config_proto_msgTypes,
-	}.Build()
-	File_grpc_tls_provider_meshca_experimental_config_proto = out.File
-	file_grpc_tls_provider_meshca_experimental_config_proto_rawDesc = nil
-	file_grpc_tls_provider_meshca_experimental_config_proto_goTypes = nil
-	file_grpc_tls_provider_meshca_experimental_config_proto_depIdxs = nil
-}
diff --git a/credentials/tls/certprovider/meshca/plugin_test.go b/credentials/tls/certprovider/meshca/plugin_test.go
index 5b3f068d61e5..51f545d6a0e3 100644
--- a/credentials/tls/certprovider/meshca/plugin_test.go
+++ b/credentials/tls/certprovider/meshca/plugin_test.go
@@ -26,6 +26,7 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"encoding/json"
 	"encoding/pem"
 	"errors"
 	"fmt"
@@ -35,11 +36,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/protobuf/proto"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/tls/certprovider"
-	configpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/meshca_experimental"
 	meshgrpc "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
 	meshpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
 	"google.golang.org/grpc/internal/testutils"
@@ -298,9 +296,7 @@ func (s) TestCreateCertificate(t *testing.T) {
 	defer cancel()
 
 	// Set the MeshCA targetURI to point to our fake MeshCA.
-	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
-	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
-	inputConfig := makeJSONConfig(t, cfg)
+	inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, addr))
 
 	// Lookup MeshCA plugin builder, parse config and start the plugin.
 	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
@@ -341,9 +337,7 @@ func (s) TestCreateCertificateWithBackoff(t *testing.T) {
 	defer cancel()
 
 	// Set the MeshCA targetURI to point to our fake MeshCA.
-	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
-	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
-	inputConfig := makeJSONConfig(t, cfg)
+	inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, addr))
 
 	// Lookup MeshCA plugin builder, parse config and start the plugin.
 	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
@@ -397,9 +391,7 @@ func (s) TestCreateCertificateWithRefresh(t *testing.T) {
 	defer cancel()
 
 	// Set the MeshCA targetURI to point to our fake MeshCA.
-	cfg := proto.Clone(goodConfigFullySpecified).(*configpb.GoogleMeshCaConfig)
-	cfg.Server.GrpcServices[0].GetGoogleGrpc().TargetUri = addr
-	inputConfig := makeJSONConfig(t, cfg)
+	inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, addr))
 
 	// Lookup MeshCA plugin builder, parse config and start the plugin.
 	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
diff --git a/regenerate.sh b/regenerate.sh
index 3443ad975913..6d3ec324a260 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -45,19 +45,6 @@ mkdir -p ${WORKDIR}/googleapis/google/rpc
 echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
 curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > ${WORKDIR}/googleapis/google/rpc/code.proto
 
-# Pull in the following repos to build the MeshCA config proto.
-ENVOY_API_REPOS=(
-  "https://github.com/envoyproxy/data-plane-api"
-  "https://github.com/cncf/udpa"
-  "https://github.com/envoyproxy/protoc-gen-validate"
-)
-for repo in ${ENVOY_API_REPOS[@]}; do
-  dirname=$(basename ${repo})
-  mkdir -p ${WORKDIR}/${dirname}
-  echo "git clone ${repo}"
-  git clone --quiet ${repo} ${WORKDIR}/${dirname}
-done
-
 # Pull in the MeshCA service proto.
 mkdir -p ${WORKDIR}/istio/istio/google/security/meshca/v1
 echo "curl https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto"
@@ -84,15 +71,13 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
-  ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto
   ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
 )
 
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
 # import path of 'bar' in the generated code when 'foo.proto' is imported in
 # one of the sources.
-OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config,\
-Menvoy/config/core/v3/config_source.proto=github.com/envoyproxy/go-control-plane/envoy/config/core/v3
+OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config
 
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
@@ -100,9 +85,6 @@ for src in ${SOURCES[@]}; do
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
-    -I${WORKDIR}/data-plane-api \
-    -I${WORKDIR}/udpa \
-    -I${WORKDIR}/protoc-gen-validate \
     -I${WORKDIR}/istio \
     ${src}
 done
@@ -113,9 +95,6 @@ for src in ${LEGACY_SOURCES[@]}; do
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
-    -I${WORKDIR}/data-plane-api \
-    -I${WORKDIR}/udpa \
-    -I${WORKDIR}/protoc-gen-validate \
     -I${WORKDIR}/istio \
     ${src}
 done

From 8f3cc6cc26958590a64f12d80fb48f601204b63f Mon Sep 17 00:00:00 2001
From: Edmond <resec0109@gmail.com>
Date: Tue, 24 Nov 2020 06:56:57 +0800
Subject: [PATCH 294/481] protoc-gen-go-grpc: export grpc.ServiceDesc (#4035)

---
 .../grpc_lb_v1/load_balancer_grpc.pb.go       |  9 ++++--
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       |  7 +++--
 benchmark/grpc_testing/services_grpc.pb.go    | 22 +++++++++-----
 channelz/grpc_channelz_v1/channelz_grpc.pb.go |  7 +++--
 cmd/protoc-gen-go-grpc/grpc.go                |  7 +++--
 .../proto/grpc_gcp/handshaker_grpc.pb.go      |  9 ++++--
 .../meshca/internal/v1/meshca_grpc.pb.go      |  7 +++--
 examples/features/proto/echo/echo_grpc.pb.go  | 13 +++++----
 .../helloworld/helloworld_grpc.pb.go          |  7 +++--
 .../routeguide/route_guide_grpc.pb.go         | 13 +++++----
 health/grpc_health_v1/health_grpc.pb.go       |  9 ++++--
 interop/grpc_testing/test_grpc.pb.go          | 29 ++++++++++++-------
 profiling/proto/service_grpc.pb.go            |  7 +++--
 .../reflection_grpc.pb.go                     |  9 ++++--
 reflection/grpc_testing/test_grpc.pb.go       |  9 ++++--
 stats/grpc_testing/test_grpc.pb.go            | 13 +++++----
 stress/grpc_testing/metrics_grpc.pb.go        |  9 ++++--
 test/grpc_testing/test_grpc.pb.go             | 15 ++++++----
 18 files changed, 132 insertions(+), 69 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index a0061631daa9..41e7274a0f83 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -30,7 +30,7 @@ func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient {
 }
 
 func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
+	stream, err := c.cc.NewStream(ctx, &LoadBalancer_ServiceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -84,7 +84,7 @@ type UnsafeLoadBalancerServer interface {
 }
 
 func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) {
-	s.RegisterService(&_LoadBalancer_serviceDesc, srv)
+	s.RegisterService(&LoadBalancer_ServiceDesc, srv)
 }
 
 func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error {
@@ -113,7 +113,10 @@ func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) {
 	return m, nil
 }
 
-var _LoadBalancer_serviceDesc = grpc.ServiceDesc{
+// LoadBalancer_ServiceDesc is the grpc.ServiceDesc for LoadBalancer service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var LoadBalancer_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.lb.v1.LoadBalancer",
 	HandlerType: (*LoadBalancerServer)(nil),
 	Methods:     []grpc.MethodDesc{},
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index 9a4f10b784e6..fe86d21cb107 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -64,7 +64,7 @@ type UnsafeRouteLookupServiceServer interface {
 }
 
 func RegisterRouteLookupServiceServer(s grpc.ServiceRegistrar, srv RouteLookupServiceServer) {
-	s.RegisterService(&_RouteLookupService_serviceDesc, srv)
+	s.RegisterService(&RouteLookupService_ServiceDesc, srv)
 }
 
 func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -85,7 +85,10 @@ func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Contex
 	return interceptor(ctx, in, info, handler)
 }
 
-var _RouteLookupService_serviceDesc = grpc.ServiceDesc{
+// RouteLookupService_ServiceDesc is the grpc.ServiceDesc for RouteLookupService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var RouteLookupService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.lookup.v1.RouteLookupService",
 	HandlerType: (*RouteLookupServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go
index 0afc725ab3f0..5dfe0e578dff 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/benchmark/grpc_testing/services_grpc.pb.go
@@ -46,7 +46,7 @@ func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleReques
 }
 
 func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +77,7 @@ func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
 }
 
 func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_BenchmarkService_serviceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -146,7 +146,7 @@ type UnsafeBenchmarkServiceServer interface {
 }
 
 func RegisterBenchmarkServiceServer(s grpc.ServiceRegistrar, srv BenchmarkServiceServer) {
-	s.RegisterService(&_BenchmarkService_serviceDesc, srv)
+	s.RegisterService(&BenchmarkService_ServiceDesc, srv)
 }
 
 func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -219,7 +219,10 @@ func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleReques
 	return m, nil
 }
 
-var _BenchmarkService_serviceDesc = grpc.ServiceDesc{
+// BenchmarkService_ServiceDesc is the grpc.ServiceDesc for BenchmarkService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var BenchmarkService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.BenchmarkService",
 	HandlerType: (*BenchmarkServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
@@ -278,7 +281,7 @@ func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient {
 }
 
 func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[0], "/grpc.testing.WorkerService/RunServer", opts...)
+	stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[0], "/grpc.testing.WorkerService/RunServer", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -309,7 +312,7 @@ func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) {
 }
 
 func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_WorkerService_serviceDesc.Streams[1], "/grpc.testing.WorkerService/RunClient", opts...)
+	stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[1], "/grpc.testing.WorkerService/RunClient", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -408,7 +411,7 @@ type UnsafeWorkerServiceServer interface {
 }
 
 func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServer) {
-	s.RegisterService(&_WorkerService_serviceDesc, srv)
+	s.RegisterService(&WorkerService_ServiceDesc, srv)
 }
 
 func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error {
@@ -499,7 +502,10 @@ func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
-var _WorkerService_serviceDesc = grpc.ServiceDesc{
+// WorkerService_ServiceDesc is the grpc.ServiceDesc for WorkerService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var WorkerService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.WorkerService",
 	HandlerType: (*WorkerServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index 447af4059929..15d664a4c64d 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -160,7 +160,7 @@ type UnsafeChannelzServer interface {
 }
 
 func RegisterChannelzServer(s grpc.ServiceRegistrar, srv ChannelzServer) {
-	s.RegisterService(&_Channelz_serviceDesc, srv)
+	s.RegisterService(&Channelz_ServiceDesc, srv)
 }
 
 func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -289,7 +289,10 @@ func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(
 	return interceptor(ctx, in, info, handler)
 }
 
-var _Channelz_serviceDesc = grpc.ServiceDesc{
+// Channelz_ServiceDesc is the grpc.ServiceDesc for Channelz service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Channelz_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.channelz.v1.Channelz",
 	HandlerType: (*ChannelzServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 5a3011e1162d..0816ae4bd874 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -180,7 +180,7 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 	if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() {
 		g.P(deprecationComment)
 	}
-	serviceDescVar := "_" + service.GoName + "_serviceDesc"
+	serviceDescVar := service.GoName + "_ServiceDesc"
 	g.P("func Register", service.GoName, "Server(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv ", serverType, ") {")
 	g.P("s.RegisterService(&", serviceDescVar, `, srv)`)
 	g.P("}")
@@ -194,6 +194,9 @@ func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.Generated
 	}
 
 	// Service descriptor.
+	g.P("// ", serviceDescVar, " is the ", grpcPackage.Ident("ServiceDesc"), " for ", service.GoName, " service.")
+	g.P("// It's only intended for direct use with ", grpcPackage.Ident("RegisterService"), ",")
+	g.P("// and not to be introspected or modified (even as a copy)")
 	g.P("var ", serviceDescVar, " = ", grpcPackage.Ident("ServiceDesc"), " {")
 	g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",")
 	g.P("HandlerType: (*", serverType, ")(nil),")
@@ -263,7 +266,7 @@ func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.Gene
 		return
 	}
 	streamType := unexport(service.GoName) + method.GoName + "Client"
-	serviceDescVar := "_" + service.GoName + "_serviceDesc"
+	serviceDescVar := service.GoName + "_ServiceDesc"
 	g.P("stream, err := c.cc.NewStream(ctx, &", serviceDescVar, ".Streams[", index, `], "`, sname, `", opts...)`)
 	g.P("if err != nil { return nil, err }")
 	g.P("x := &", streamType, "{stream}")
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index 6dd4158546a7..75bbf87dd55c 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -35,7 +35,7 @@ func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceCl
 }
 
 func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
+	stream, err := c.cc.NewStream(ctx, &HandshakerService_ServiceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -96,7 +96,7 @@ type UnsafeHandshakerServiceServer interface {
 }
 
 func RegisterHandshakerServiceServer(s grpc.ServiceRegistrar, srv HandshakerServiceServer) {
-	s.RegisterService(&_HandshakerService_serviceDesc, srv)
+	s.RegisterService(&HandshakerService_ServiceDesc, srv)
 }
 
 func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error {
@@ -125,7 +125,10 @@ func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) {
 	return m, nil
 }
 
-var _HandshakerService_serviceDesc = grpc.ServiceDesc{
+// HandshakerService_ServiceDesc is the grpc.ServiceDesc for HandshakerService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var HandshakerService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.gcp.HandshakerService",
 	HandlerType: (*HandshakerServiceServer)(nil),
 	Methods:     []grpc.MethodDesc{},
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index 424a21bf494e..7b37a7c01b79 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -67,7 +67,7 @@ type UnsafeMeshCertificateServiceServer interface {
 }
 
 func RegisterMeshCertificateServiceServer(s grpc.ServiceRegistrar, srv MeshCertificateServiceServer) {
-	s.RegisterService(&_MeshCertificateService_serviceDesc, srv)
+	s.RegisterService(&MeshCertificateService_ServiceDesc, srv)
 }
 
 func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -88,7 +88,10 @@ func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx cont
 	return interceptor(ctx, in, info, handler)
 }
 
-var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{
+// MeshCertificateService_ServiceDesc is the grpc.ServiceDesc for MeshCertificateService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var MeshCertificateService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "google.security.meshca.v1.MeshCertificateService",
 	HandlerType: (*MeshCertificateServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index fb9820f35c98..7657b8bfb143 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -45,7 +45,7 @@ func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grp
 }
 
 func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[0], "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[0], "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +77,7 @@ func (x *echoServerStreamingEchoClient) Recv() (*EchoResponse, error) {
 }
 
 func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[1], "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[1], "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -111,7 +111,7 @@ func (x *echoClientStreamingEchoClient) CloseAndRecv() (*EchoResponse, error) {
 }
 
 func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Echo_serviceDesc.Streams[2], "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
+	stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[2], "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -182,7 +182,7 @@ type UnsafeEchoServer interface {
 }
 
 func RegisterEchoServer(s grpc.ServiceRegistrar, srv EchoServer) {
-	s.RegisterService(&_Echo_serviceDesc, srv)
+	s.RegisterService(&Echo_ServiceDesc, srv)
 }
 
 func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -276,7 +276,10 @@ func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) {
 	return m, nil
 }
 
-var _Echo_serviceDesc = grpc.ServiceDesc{
+// Echo_ServiceDesc is the grpc.ServiceDesc for Echo service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Echo_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.examples.echo.Echo",
 	HandlerType: (*EchoServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 3c8c7d87df32..51c64f65b3d0 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -64,7 +64,7 @@ type UnsafeGreeterServer interface {
 }
 
 func RegisterGreeterServer(s grpc.ServiceRegistrar, srv GreeterServer) {
-	s.RegisterService(&_Greeter_serviceDesc, srv)
+	s.RegisterService(&Greeter_ServiceDesc, srv)
 }
 
 func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -85,7 +85,10 @@ func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(in
 	return interceptor(ctx, in, info, handler)
 }
 
-var _Greeter_serviceDesc = grpc.ServiceDesc{
+// Greeter_ServiceDesc is the grpc.ServiceDesc for Greeter service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Greeter_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "helloworld.Greeter",
 	HandlerType: (*GreeterServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index fd72beaea51f..dd98bf5a89e0 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -61,7 +61,7 @@ func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...gr
 }
 
 func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[0], "/routeguide.RouteGuide/ListFeatures", opts...)
+	stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[0], "/routeguide.RouteGuide/ListFeatures", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +93,7 @@ func (x *routeGuideListFeaturesClient) Recv() (*Feature, error) {
 }
 
 func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[1], "/routeguide.RouteGuide/RecordRoute", opts...)
+	stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[1], "/routeguide.RouteGuide/RecordRoute", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +127,7 @@ func (x *routeGuideRecordRouteClient) CloseAndRecv() (*RouteSummary, error) {
 }
 
 func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_RouteGuide_serviceDesc.Streams[2], "/routeguide.RouteGuide/RouteChat", opts...)
+	stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[2], "/routeguide.RouteGuide/RouteChat", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -214,7 +214,7 @@ type UnsafeRouteGuideServer interface {
 }
 
 func RegisterRouteGuideServer(s grpc.ServiceRegistrar, srv RouteGuideServer) {
-	s.RegisterService(&_RouteGuide_serviceDesc, srv)
+	s.RegisterService(&RouteGuide_ServiceDesc, srv)
 }
 
 func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -308,7 +308,10 @@ func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) {
 	return m, nil
 }
 
-var _RouteGuide_serviceDesc = grpc.ServiceDesc{
+// RouteGuide_ServiceDesc is the grpc.ServiceDesc for RouteGuide service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var RouteGuide_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "routeguide.RouteGuide",
 	HandlerType: (*RouteGuideServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index dfa6791a7da6..8c3872664c22 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -56,7 +56,7 @@ func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts .
 }
 
 func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
+	stream, err := c.cc.NewStream(ctx, &Health_ServiceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -131,7 +131,7 @@ type UnsafeHealthServer interface {
 }
 
 func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) {
-	s.RegisterService(&_Health_serviceDesc, srv)
+	s.RegisterService(&Health_ServiceDesc, srv)
 }
 
 func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -173,7 +173,10 @@ func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-var _Health_serviceDesc = grpc.ServiceDesc{
+// Health_ServiceDesc is the grpc.ServiceDesc for Health service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Health_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.health.v1.Health",
 	HandlerType: (*HealthServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index 1a1c910782b6..482d309f7b4a 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -66,7 +66,7 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 }
 
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +98,7 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo
 }
 
 func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -132,7 +132,7 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal
 }
 
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -163,7 +163,7 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 }
 
 func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -252,7 +252,7 @@ type UnsafeTestServiceServer interface {
 }
 
 func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
+	s.RegisterService(&TestService_ServiceDesc, srv)
 }
 
 func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -390,7 +390,10 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-var _TestService_serviceDesc = grpc.ServiceDesc{
+// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var TestService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.TestService",
 	HandlerType: (*TestServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
@@ -481,7 +484,7 @@ type UnsafeUnimplementedServiceServer interface {
 }
 
 func RegisterUnimplementedServiceServer(s grpc.ServiceRegistrar, srv UnimplementedServiceServer) {
-	s.RegisterService(&_UnimplementedService_serviceDesc, srv)
+	s.RegisterService(&UnimplementedService_ServiceDesc, srv)
 }
 
 func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -502,7 +505,10 @@ func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx contex
 	return interceptor(ctx, in, info, handler)
 }
 
-var _UnimplementedService_serviceDesc = grpc.ServiceDesc{
+// UnimplementedService_ServiceDesc is the grpc.ServiceDesc for UnimplementedService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var UnimplementedService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.UnimplementedService",
 	HandlerType: (*UnimplementedServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
@@ -567,7 +573,7 @@ type UnsafeLoadBalancerStatsServiceServer interface {
 }
 
 func RegisterLoadBalancerStatsServiceServer(s grpc.ServiceRegistrar, srv LoadBalancerStatsServiceServer) {
-	s.RegisterService(&_LoadBalancerStatsService_serviceDesc, srv)
+	s.RegisterService(&LoadBalancerStatsService_ServiceDesc, srv)
 }
 
 func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -588,7 +594,10 @@ func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx conte
 	return interceptor(ctx, in, info, handler)
 }
 
-var _LoadBalancerStatsService_serviceDesc = grpc.ServiceDesc{
+// LoadBalancerStatsService_ServiceDesc is the grpc.ServiceDesc for LoadBalancerStatsService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var LoadBalancerStatsService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.LoadBalancerStatsService",
 	HandlerType: (*LoadBalancerStatsServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index 11f9630db653..37d2004eb599 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -80,7 +80,7 @@ type UnsafeProfilingServer interface {
 }
 
 func RegisterProfilingServer(s grpc.ServiceRegistrar, srv ProfilingServer) {
-	s.RegisterService(&_Profiling_serviceDesc, srv)
+	s.RegisterService(&Profiling_ServiceDesc, srv)
 }
 
 func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -119,7 +119,10 @@ func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
-var _Profiling_serviceDesc = grpc.ServiceDesc{
+// Profiling_ServiceDesc is the grpc.ServiceDesc for Profiling service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Profiling_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.go.profiling.v1alpha.Profiling",
 	HandlerType: (*ProfilingServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index a821e5154e66..75b88186b20b 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -31,7 +31,7 @@ func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClie
 }
 
 func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
+	stream, err := c.cc.NewStream(ctx, &ServerReflection_ServiceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -86,7 +86,7 @@ type UnsafeServerReflectionServer interface {
 }
 
 func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) {
-	s.RegisterService(&_ServerReflection_serviceDesc, srv)
+	s.RegisterService(&ServerReflection_ServiceDesc, srv)
 }
 
 func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error {
@@ -115,7 +115,10 @@ func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRe
 	return m, nil
 }
 
-var _ServerReflection_serviceDesc = grpc.ServiceDesc{
+// ServerReflection_ServiceDesc is the grpc.ServiceDesc for ServerReflection service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var ServerReflection_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.reflection.v1alpha.ServerReflection",
 	HandlerType: (*ServerReflectionServer)(nil),
 	Methods:     []grpc.MethodDesc{},
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 608cf8874c17..cc322d75b2c8 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -39,7 +39,7 @@ func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opt
 }
 
 func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_SearchService_serviceDesc.Streams[0], "/grpc.testing.SearchService/StreamingSearch", opts...)
+	stream, err := c.cc.NewStream(ctx, &SearchService_ServiceDesc.Streams[0], "/grpc.testing.SearchService/StreamingSearch", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +98,7 @@ type UnsafeSearchServiceServer interface {
 }
 
 func RegisterSearchServiceServer(s grpc.ServiceRegistrar, srv SearchServiceServer) {
-	s.RegisterService(&_SearchService_serviceDesc, srv)
+	s.RegisterService(&SearchService_ServiceDesc, srv)
 }
 
 func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -145,7 +145,10 @@ func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) {
 	return m, nil
 }
 
-var _SearchService_serviceDesc = grpc.ServiceDesc{
+// SearchService_ServiceDesc is the grpc.ServiceDesc for SearchService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SearchService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.SearchService",
 	HandlerType: (*SearchServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
index 0c825cbff0cd..3a6f2b8e1a65 100644
--- a/stats/grpc_testing/test_grpc.pb.go
+++ b/stats/grpc_testing/test_grpc.pb.go
@@ -48,7 +48,7 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 }
 
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +79,7 @@ func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) {
 }
 
 func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -113,7 +113,7 @@ func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, err
 }
 
 func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -188,7 +188,7 @@ type UnsafeTestServiceServer interface {
 }
 
 func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
+	s.RegisterService(&TestService_ServiceDesc, srv)
 }
 
 func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -282,7 +282,10 @@ func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-var _TestService_serviceDesc = grpc.ServiceDesc{
+// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var TestService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.TestService",
 	HandlerType: (*TestServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 8262193ea4e4..4bcd6c19c28f 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -33,7 +33,7 @@ func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient {
 }
 
 func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_MetricsService_serviceDesc.Streams[0], "/grpc.testing.MetricsService/GetAllGauges", opts...)
+	stream, err := c.cc.NewStream(ctx, &MetricsService_ServiceDesc.Streams[0], "/grpc.testing.MetricsService/GetAllGauges", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -105,7 +105,7 @@ type UnsafeMetricsServiceServer interface {
 }
 
 func RegisterMetricsServiceServer(s grpc.ServiceRegistrar, srv MetricsServiceServer) {
-	s.RegisterService(&_MetricsService_serviceDesc, srv)
+	s.RegisterService(&MetricsService_ServiceDesc, srv)
 }
 
 func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error {
@@ -147,7 +147,10 @@ func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
-var _MetricsService_serviceDesc = grpc.ServiceDesc{
+// MetricsService_ServiceDesc is the grpc.ServiceDesc for MetricsService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var MetricsService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.MetricsService",
 	HandlerType: (*MetricsServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index d68f2cf84a37..98e33ba3ded5 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -66,7 +66,7 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 }
 
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +98,7 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo
 }
 
 func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[1], "/grpc.testing.TestService/StreamingInputCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -132,7 +132,7 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal
 }
 
 func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[2], "/grpc.testing.TestService/FullDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -163,7 +163,7 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 }
 
 func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_TestService_serviceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
+	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[3], "/grpc.testing.TestService/HalfDuplexCall", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -252,7 +252,7 @@ type UnsafeTestServiceServer interface {
 }
 
 func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
-	s.RegisterService(&_TestService_serviceDesc, srv)
+	s.RegisterService(&TestService_ServiceDesc, srv)
 }
 
 func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -390,7 +390,10 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
-var _TestService_serviceDesc = grpc.ServiceDesc{
+// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var TestService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "grpc.testing.TestService",
 	HandlerType: (*TestServiceServer)(nil),
 	Methods: []grpc.MethodDesc{

From 21570d76d6c5496692fd40c527679e6a92d8a99c Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 23 Nov 2020 16:40:36 -0800
Subject: [PATCH 295/481] xds: Read server_resource_name_id from bootstrap
 config (#4049)

---
 xds/server.go      | 25 ++++++++++++++-----------
 xds/server_test.go | 35 +++++++++++------------------------
 2 files changed, 25 insertions(+), 35 deletions(-)

diff --git a/xds/server.go b/xds/server.go
index fc499848b245..ce5e7d565486 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -32,21 +32,13 @@ import (
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
 
-const (
-	serverPrefix = "[xds-server %p] "
-
-	// The resource_name in the LDS request sent by the xDS-enabled gRPC server
-	// is of this format where the formatting directive at the end is replaced
-	// with the IP:Port specified by the user application.
-	listenerResourceNameFormat = "grpc/server?udpa.resource.listening_address=%s"
-)
+const serverPrefix = "[xds-server %p] "
 
 var (
 	// These new functions will be overridden in unit tests.
 	newXDSClient = func() (xdsClientInterface, error) {
 		return xdsclient.New()
 	}
-	newXDSConfig  = bootstrap.NewConfig
 	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface {
 		return grpc.NewServer(opts...)
 	}
@@ -62,6 +54,7 @@ func prefixLogger(p *GRPCServer) *internalgrpclog.PrefixLogger {
 // the server. This is useful for overriding in unit tests.
 type xdsClientInterface interface {
 	WatchListener(string, func(xdsclient.ListenerUpdate, error)) func()
+	BootstrapConfig() *bootstrap.Config
 	Close()
 }
 
@@ -187,10 +180,20 @@ func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, err
 	// or not.
 	goodUpdate := grpcsync.NewEvent()
 
+	// The resource_name in the LDS request sent by the xDS-enabled gRPC server
+	// is of the following format:
+	// "/path/to/resource?udpa.resource.listening_address=IP:Port". The
+	// `/path/to/resource` part of the name is sourced from the bootstrap config
+	// field `grpc_server_resource_name_id`. If this field is not specified in
+	// the bootstrap file, we will use a default of `grpc/server`.
+	path := "grpc/server"
+	if cfg := s.xdsC.BootstrapConfig(); cfg != nil && cfg.ServerResourceNameID != "" {
+		path = cfg.ServerResourceNameID
+	}
+	name := fmt.Sprintf("%s?udpa.resource.listening_address=%s", path, lis.Addr().String())
+
 	// Register an LDS watch using our xdsClient, and specify the listening
 	// address as the resource name.
-	// TODO(easwars): Check if literal IPv6 addresses need an enclosing [].
-	name := fmt.Sprintf(listenerResourceNameFormat, lis.Addr().String())
 	cancelWatch := s.xdsC.WatchListener(name, func(update xdsclient.ListenerUpdate, err error) {
 		if err != nil {
 			// We simply log an error here and hope we get a successful update
diff --git a/xds/server_test.go b/xds/server_test.go
index 91583becc12b..cefb0261984e 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -38,8 +38,9 @@ import (
 )
 
 const (
-	defaultTestTimeout      = 5 * time.Second
-	defaultTestShortTimeout = 10 * time.Millisecond
+	defaultTestTimeout       = 5 * time.Second
+	defaultTestShortTimeout  = 10 * time.Millisecond
+	testServerResourceNameID = "/path/to/resource"
 )
 
 type s struct {
@@ -135,19 +136,16 @@ func (s) TestRegisterService(t *testing.T) {
 func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func()) {
 	t.Helper()
 
-	origNewXDSConfig := newXDSConfig
-	newXDSConfig = func() (*bootstrap.Config, error) {
-		return &bootstrap.Config{
-			BalancerName: "dummyBalancer",
-			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
-			NodeProto:    xdstestutils.EmptyNodeProtoV3,
-		}, nil
-	}
-
 	clientCh := testutils.NewChannel()
 	origNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) {
 		c := fakeclient.NewClient()
+		c.SetBootstrapConfig(&bootstrap.Config{
+			BalancerName:         "dummyBalancer",
+			Creds:                grpc.WithTransportCredentials(insecure.NewCredentials()),
+			NodeProto:            xdstestutils.EmptyNodeProtoV3,
+			ServerResourceNameID: testServerResourceNameID,
+		})
 		clientCh.Send(c)
 		return c, nil
 	}
@@ -157,7 +155,6 @@ func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func())
 	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface { return fs }
 
 	return fs, clientCh, func() {
-		newXDSConfig = origNewXDSConfig
 		newXDSClient = origNewXDSClient
 		newGRPCServer = origNewGRPCServer
 	}
@@ -206,7 +203,7 @@ func (s) TestServeSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String())
+	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -268,7 +265,7 @@ func (s) TestServeWithStop(t *testing.T) {
 		server.Stop()
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String())
+	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
 	if name != wantName {
 		server.Stop()
 		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantName)
@@ -323,16 +320,6 @@ func (s) TestServeBootstrapFailure(t *testing.T) {
 // TestServeNewClientFailure tests the case where xds client creation fails and
 // verifies that Server() exits with a non-nil error.
 func (s) TestServeNewClientFailure(t *testing.T) {
-	origNewXDSConfig := newXDSConfig
-	newXDSConfig = func() (*bootstrap.Config, error) {
-		return &bootstrap.Config{
-			BalancerName: "dummyBalancer",
-			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
-			NodeProto:    xdstestutils.EmptyNodeProtoV3,
-		}, nil
-	}
-	defer func() { newXDSConfig = origNewXDSConfig }()
-
 	origNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) {
 		return nil, errors.New("xdsClient creation failed")

From 8238a33b57d718da4375780be741e199c85223e6 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Tue, 24 Nov 2020 09:13:08 -0800
Subject: [PATCH 296/481] advancedtls: make sni test check common name only
 (#4055)

* advancedtls: make sni test check common name only
---
 security/advancedtls/advancedtls_test.go | 34 +++++++++++++-----------
 security/advancedtls/go.mod              |  2 +-
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 67c0b9dba42a..2d6ec47b7202 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -24,11 +24,9 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"math/big"
 	"net"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/grpctest"
@@ -820,25 +818,21 @@ func (s) TestGetCertificatesSNI(t *testing.T) {
 	tests := []struct {
 		desc       string
 		serverName string
-		wantCert   tls.Certificate
+		// Use Common Name on the certificate to differentiate if we choose the right cert. The common name on all of the three certs are different.
+		wantCommonName string
 	}{
 		{
 			desc: "Select ServerCert1",
 			// "foo.bar.com" is the common name on server certificate server_cert_1.pem.
-			serverName: "foo.bar.com",
-			wantCert:   cs.ServerCert1,
-		},
-		{
-			desc: "Select ServerCert2",
-			// "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem.
-			serverName: "foo.bar.server2.com",
-			wantCert:   cs.ServerCert2,
+			serverName:     "foo.bar.com",
+			wantCommonName: "foo.bar.com",
 		},
 		{
 			desc: "Select serverCert3",
+			// "foo.bar.server3.com" is the common name on server certificate server_cert_3.pem.
 			// "google.com" is one of the DNS names on server certificate server_cert_3.pem.
-			serverName: "google.com",
-			wantCert:   cs.ServerPeer3,
+			serverName:     "google.com",
+			wantCommonName: "foo.bar.server3.com",
 		},
 	}
 	for _, test := range tests {
@@ -867,8 +861,18 @@ func (s) TestGetCertificatesSNI(t *testing.T) {
 			if err != nil {
 				t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err)
 			}
-			if !gotCertificate.Leaf.Equal(test.wantCert.Leaf) {
-				t.Errorf("GetCertificates() returned leaf certificate does not match expected (-want +got):\n%s", cmp.Diff(test.wantCert, *gotCertificate, cmp.AllowUnexported(big.Int{})))
+			if gotCertificate == nil || len(gotCertificate.Certificate) == 0 {
+				t.Fatalf("Got nil or empty Certificate after calling serverConfig.GetCertificate.")
+			}
+			parsedCert, err := x509.ParseCertificate(gotCertificate.Certificate[0])
+			if err != nil {
+				t.Fatalf("x509.ParseCertificate(%v) failed: %v", gotCertificate.Certificate[0], err)
+			}
+			if parsedCert == nil {
+				t.Fatalf("Got nil Certificate after calling x509.ParseCertificate.")
+			}
+			if parsedCert.Subject.CommonName != test.wantCommonName {
+				t.Errorf("Common name mismatch, got %v, want %v", parsedCert.Subject.CommonName, test.wantCommonName)
 			}
 		})
 	}
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index c641a9d60695..be35029503da 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -3,7 +3,7 @@ module google.golang.org/grpc/security/advancedtls
 go 1.14
 
 require (
-	github.com/google/go-cmp v0.5.1
+	github.com/google/go-cmp v0.5.1 // indirect
 	google.golang.org/grpc v1.31.0
 	google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b
 )

From a82c291b618dd554024b934df12661270cfb1e0c Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 24 Nov 2020 11:56:20 -0800
Subject: [PATCH 297/481] github: Don't use `make` in testing.yaml (#4059)

---
 .github/workflows/testing.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 15d6311d0c07..91e66efd7d2f 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -77,12 +77,12 @@ jobs:
       # Main tests run for everything except when testing "extras" and the race detector.
       - name: Run tests
         if: ${{ matrix.type != 'extras' && matrix.type != 'race' }}
-        run: make test
+        run: go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Race detector tests
       - name: Run test race
         if: ${{ matrix.TYPE == 'race' }}
-        run: make testrace
+        run: go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Non-core gRPC tests (examples, interop, etc)
       - name: Run extras tests
@@ -91,4 +91,5 @@ jobs:
           examples/examples_test.sh
           security/advancedtls/examples/examples_test.sh
           interop/interop_test.sh
-          make testsubmodule
+          cd ${GITHUB_WORKSPACE}/security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
+          cd ${GITHUB_WORKSPACE}/security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/...

From 53c8623768ef366129e4d5b2fb32bba3610f98fb Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 24 Nov 2020 11:56:47 -0800
Subject: [PATCH 298/481] Makefile: Restore `deps` and `testdeps` (#4060)

---
 Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Makefile b/Makefile
index 050941a09169..e62d0385e6b1 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,9 @@ build:
 clean:
 	go clean -i google.golang.org/grpc/...
 
+deps:
+	go get -d -v google.golang.org/grpc/...
+
 proto:
 	@ if ! which protoc > /dev/null; then \
 		echo "error: protoc not installed" >&2; \
@@ -23,6 +26,9 @@ testsubmodule:
 testrace:
 	go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
+testdeps:
+	go get -d -v -t google.golang.org/grpc/...
+
 vet: vetdeps
 	./vet.sh
 

From 44e408dab41e7c08c1d9369c84a3df5b5c11d886 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 24 Nov 2020 16:53:57 -0800
Subject: [PATCH 299/481] Makefile: enable module support in deps (#4062)

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index e62d0385e6b1..1f0722f16243 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ clean:
 	go clean -i google.golang.org/grpc/...
 
 deps:
-	go get -d -v google.golang.org/grpc/...
+	GO111MODULE=on go get -d -v google.golang.org/grpc/...
 
 proto:
 	@ if ! which protoc > /dev/null; then \
@@ -27,7 +27,7 @@ testrace:
 	go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
 testdeps:
-	go get -d -v -t google.golang.org/grpc/...
+	GO111MODULE=on go get -d -v -t google.golang.org/grpc/...
 
 vet: vetdeps
 	./vet.sh

From c456688b18606b43dcc5a817a05b7585614932d8 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <garrettgutierrez1@gmail.com>
Date: Mon, 30 Nov 2020 10:04:47 -0800
Subject: [PATCH 300/481] client: fix "unix" scheme handling for some corner
 cases (#4021)

---
 internal/grpcutil/target.go          |  9 +--
 internal/grpcutil/target_test.go     | 12 ++--
 internal/resolver/unix/unix.go       |  5 ++
 internal/transport/http2_client.go   | 21 ++++--
 internal/transport/http_util.go      | 29 +++++++++
 internal/transport/http_util_test.go | 29 +++++++++
 test/authority_test.go               | 97 ++++++++++++++--------------
 7 files changed, 138 insertions(+), 64 deletions(-)

diff --git a/internal/grpcutil/target.go b/internal/grpcutil/target.go
index 3e1b22f5a8c0..baa3479f12ca 100644
--- a/internal/grpcutil/target.go
+++ b/internal/grpcutil/target.go
@@ -48,9 +48,10 @@ func ParseTarget(target string, skipUnixColonParsing bool) (ret resolver.Target)
 	ret.Scheme, ret.Endpoint, ok = split2(target, "://")
 	if !ok {
 		if strings.HasPrefix(target, "unix:") && !skipUnixColonParsing {
-			// Handle the "unix:[path]" case, because splitting on :// only
-			// handles the "unix://[/absolute/path]" case. Only handle if the
-			// dialer is nil, to avoid a behavior change with custom dialers.
+			// Handle the "unix:[local/path]" and "unix:[/absolute/path]" cases,
+			// because splitting on :// only handles the
+			// "unix://[/absolute/path]" case. Only handle if the dialer is nil,
+			// to avoid a behavior change with custom dialers.
 			return resolver.Target{Scheme: "unix", Endpoint: target[len("unix:"):]}
 		}
 		return resolver.Target{Endpoint: target}
@@ -61,7 +62,7 @@ func ParseTarget(target string, skipUnixColonParsing bool) (ret resolver.Target)
 	}
 	if ret.Scheme == "unix" {
 		// Add the "/" back in the unix case, so the unix resolver receives the
-		// actual endpoint.
+		// actual endpoint in the "unix://[/absolute/path]" case.
 		ret.Endpoint = "/" + ret.Endpoint
 	}
 	return ret
diff --git a/internal/grpcutil/target_test.go b/internal/grpcutil/target_test.go
index 562390bfe381..6083e84d75ae 100644
--- a/internal/grpcutil/target_test.go
+++ b/internal/grpcutil/target_test.go
@@ -70,6 +70,7 @@ func TestParseTargetString(t *testing.T) {
 		// If we can only parse part of the target.
 		{targetStr: "://", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "://"}},
 		{targetStr: "unix://domain", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix://domain"}},
+		{targetStr: "unix://a/b/c", want: resolver.Target{Scheme: "unix", Authority: "a", Endpoint: "/b/c"}},
 		{targetStr: "a:b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a:b"}},
 		{targetStr: "a/b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a/b"}},
 		{targetStr: "a:/b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a:/b"}},
@@ -77,10 +78,13 @@ func TestParseTargetString(t *testing.T) {
 		{targetStr: "a://b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a://b"}},
 
 		// Unix cases without custom dialer.
-		// unix:[local_path] and unix:[/absolute] have different behaviors with
-		// a custom dialer, to prevent behavior changes with custom dialers.
-		{targetStr: "unix:domain", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "domain"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:domain"}},
-		{targetStr: "unix:/domain", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "/domain"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:/domain"}},
+		// unix:[local_path], unix:[/absolute], and unix://[/absolute] have different
+		// behaviors with a custom dialer, to prevent behavior changes with custom dialers.
+		{targetStr: "unix:a/b/c", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "a/b/c"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:a/b/c"}},
+		{targetStr: "unix:/a/b/c", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "/a/b/c"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:/a/b/c"}},
+		{targetStr: "unix:///a/b/c", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "/a/b/c"}},
+
+		{targetStr: "passthrough:///unix:///a/b/c", want: resolver.Target{Scheme: "passthrough", Authority: "", Endpoint: "unix:///a/b/c"}},
 	} {
 		got := ParseTarget(test.targetStr, false)
 		if got != test.want {
diff --git a/internal/resolver/unix/unix.go b/internal/resolver/unix/unix.go
index d046e50613d0..8e78d47197c8 100644
--- a/internal/resolver/unix/unix.go
+++ b/internal/resolver/unix/unix.go
@@ -20,6 +20,8 @@
 package unix
 
 import (
+	"fmt"
+
 	"google.golang.org/grpc/internal/transport/networktype"
 	"google.golang.org/grpc/resolver"
 )
@@ -29,6 +31,9 @@ const scheme = "unix"
 type builder struct{}
 
 func (*builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
+	if target.Authority != "" {
+		return nil, fmt.Errorf("invalid (non-empty) authority: %v", target.Authority)
+	}
 	cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(resolver.Address{Addr: target.Endpoint}, "unix")}})
 	return &nopResolver{}, nil
 }
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 4778ed16252b..4c9f9740ec71 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -140,17 +140,26 @@ type http2Client struct {
 }
 
 func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr resolver.Address, useProxy bool, grpcUA string) (net.Conn, error) {
+	address := addr.Addr
+	networkType, ok := networktype.Get(addr)
 	if fn != nil {
-		return fn(ctx, addr.Addr)
+		if networkType == "unix" {
+			// For backward compatibility, if the user dialed "unix:///path",
+			// the passthrough resolver would be used and the user's custom
+			// dialer would see "unix:///path". Since the unix resolver is used
+			// and the address is now "/path", prepend "unix://" so the user's
+			// custom dialer sees the same address.
+			return fn(ctx, "unix://"+address)
+		}
+		return fn(ctx, address)
 	}
-	networkType := "tcp"
-	if n, ok := networktype.Get(addr); ok {
-		networkType = n
+	if !ok {
+		networkType, address = parseDialTarget(address)
 	}
 	if networkType == "tcp" && useProxy {
-		return proxyDial(ctx, addr.Addr, grpcUA)
+		return proxyDial(ctx, address, grpcUA)
 	}
-	return (&net.Dialer{}).DialContext(ctx, networkType, addr.Addr)
+	return (&net.Dialer{}).DialContext(ctx, networkType, address)
 }
 
 func isTemporary(err error) bool {
diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go
index 4d15afbf73f1..7e41d1183f93 100644
--- a/internal/transport/http_util.go
+++ b/internal/transport/http_util.go
@@ -27,6 +27,7 @@ import (
 	"math"
 	"net"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -598,3 +599,31 @@ func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, maxHeaderList
 	f.fr.ReadMetaHeaders = hpack.NewDecoder(http2InitHeaderTableSize, nil)
 	return f
 }
+
+// parseDialTarget returns the network and address to pass to dialer.
+func parseDialTarget(target string) (string, string) {
+	net := "tcp"
+	m1 := strings.Index(target, ":")
+	m2 := strings.Index(target, ":/")
+	// handle unix:addr which will fail with url.Parse
+	if m1 >= 0 && m2 < 0 {
+		if n := target[0:m1]; n == "unix" {
+			return n, target[m1+1:]
+		}
+	}
+	if m2 >= 0 {
+		t, err := url.Parse(target)
+		if err != nil {
+			return net, target
+		}
+		scheme := t.Scheme
+		addr := t.Path
+		if scheme == "unix" {
+			if addr == "" {
+				addr = t.Host
+			}
+			return scheme, addr
+		}
+	}
+	return net, target
+}
diff --git a/internal/transport/http_util_test.go b/internal/transport/http_util_test.go
index 85a083f6c8a8..2205050acea0 100644
--- a/internal/transport/http_util_test.go
+++ b/internal/transport/http_util_test.go
@@ -250,3 +250,32 @@ func (s) TestDecodeHeaderH2ErrCode(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestParseDialTarget(t *testing.T) {
+	for _, test := range []struct {
+		target, wantNet, wantAddr string
+	}{
+		{"unix:a", "unix", "a"},
+		{"unix:a/b/c", "unix", "a/b/c"},
+		{"unix:/a", "unix", "/a"},
+		{"unix:/a/b/c", "unix", "/a/b/c"},
+		{"unix://a", "unix", "a"},
+		{"unix://a/b/c", "unix", "/b/c"},
+		{"unix:///a", "unix", "/a"},
+		{"unix:///a/b/c", "unix", "/a/b/c"},
+		{"unix:etcd:0", "unix", "etcd:0"},
+		{"unix:///tmp/unix-3", "unix", "/tmp/unix-3"},
+		{"unix://domain", "unix", "domain"},
+		{"unix://etcd:0", "unix", "etcd:0"},
+		{"unix:///etcd:0", "unix", "/etcd:0"},
+		{"passthrough://unix://domain", "tcp", "passthrough://unix://domain"},
+		{"https://google.com:443", "tcp", "https://google.com:443"},
+		{"dns:///google.com", "tcp", "dns:///google.com"},
+		{"/unix/socket/address", "tcp", "/unix/socket/address"},
+	} {
+		gotNet, gotAddr := parseDialTarget(test.target)
+		if gotNet != test.wantNet || gotAddr != test.wantAddr {
+			t.Errorf("parseDialTarget(%q) = %s, %s want %s, %s", test.target, gotNet, gotAddr, test.wantNet, test.wantAddr)
+		}
+	}
+}
diff --git a/test/authority_test.go b/test/authority_test.go
index e8307fee225d..e6599b2fde04 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -80,35 +80,46 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer
 	}
 }
 
+type authorityTest struct {
+	name           string
+	address        string
+	target         string
+	authority      string
+	dialTargetWant string
+}
+
+var authorityTests = []authorityTest{
+	{
+		name:      "UnixRelative",
+		address:   "sock.sock",
+		target:    "unix:sock.sock",
+		authority: "localhost",
+	},
+	{
+		name:      "UnixAbsolute",
+		address:   "/tmp/sock.sock",
+		target:    "unix:/tmp/sock.sock",
+		authority: "localhost",
+	},
+	{
+		name:      "UnixAbsoluteAlternate",
+		address:   "/tmp/sock.sock",
+		target:    "unix:///tmp/sock.sock",
+		authority: "localhost",
+	},
+	{
+		name:           "UnixPassthrough",
+		address:        "/tmp/sock.sock",
+		target:         "passthrough:///unix:///tmp/sock.sock",
+		authority:      "unix:///tmp/sock.sock",
+		dialTargetWant: "unix:///tmp/sock.sock",
+	},
+}
+
 // TestUnix does end to end tests with the various supported unix target
-// formats, ensuring that the authority is set to localhost in every case.
+// formats, ensuring that the authority is set as expected.
 func (s) TestUnix(t *testing.T) {
-	tests := []struct {
-		name      string
-		address   string
-		target    string
-		authority string
-	}{
-		{
-			name:      "UnixRelative",
-			address:   "sock.sock",
-			target:    "unix:sock.sock",
-			authority: "localhost",
-		},
-		{
-			name:      "UnixAbsolute",
-			address:   "/tmp/sock.sock",
-			target:    "unix:/tmp/sock.sock",
-			authority: "localhost",
-		},
-		{
-			name:      "UnixAbsoluteAlternate",
-			address:   "/tmp/sock.sock",
-			target:    "unix:///tmp/sock.sock",
-			authority: "localhost",
-		},
-	}
-	for _, test := range tests {
+	for _, test := range authorityTests {
 		t.Run(test.name, func(t *testing.T) {
 			runUnixTest(t, test.address, test.target, test.authority, nil)
 		})
@@ -119,30 +130,14 @@ func (s) TestUnix(t *testing.T) {
 // formats, ensuring that the target sent to the dialer does NOT have the
 // "unix:" prefix stripped.
 func (s) TestUnixCustomDialer(t *testing.T) {
-	tests := []struct {
-		name      string
-		address   string
-		target    string
-		authority string
-	}{
-		{
-			name:      "UnixRelative",
-			address:   "sock.sock",
-			target:    "unix:sock.sock",
-			authority: "localhost",
-		},
-		{
-			name:      "UnixAbsolute",
-			address:   "/tmp/sock.sock",
-			target:    "unix:/tmp/sock.sock",
-			authority: "localhost",
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
+	for _, test := range authorityTests {
+		t.Run(test.name+"WithDialer", func(t *testing.T) {
+			if test.dialTargetWant == "" {
+				test.dialTargetWant = test.target
+			}
 			dialer := func(ctx context.Context, address string) (net.Conn, error) {
-				if address != test.target {
-					return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.target, address)
+				if address != test.dialTargetWant {
+					return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.dialTargetWant, address)
 				}
 				address = address[len("unix:"):]
 				return (&net.Dialer{}).DialContext(ctx, "unix", address)
@@ -152,6 +147,8 @@ func (s) TestUnixCustomDialer(t *testing.T) {
 	}
 }
 
+// TestColonPortAuthority does an end to end test with the target for grpc.Dial
+// being ":[port]". Ensures authority is "localhost:[port]".
 func (s) TestColonPortAuthority(t *testing.T) {
 	expectedAuthority := ""
 	var authorityMu sync.Mutex

From 4a0125ac580883d82f77d6fdc836b5966d79b439 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 30 Nov 2020 14:20:03 -0800
Subject: [PATCH 301/481] roundrobin: strip attributes from addresses (#4024)

---
 attributes/attributes_test.go          | 12 ++++
 balancer/base/balancer.go              | 32 +++++++++--
 balancer/base/balancer_test.go         | 70 ++++++++++++++++++++++
 balancer/roundrobin/roundrobin_test.go | 80 +++++++++++++++++++++++++-
 xds/internal/testutils/balancer.go     |  4 +-
 5 files changed, 189 insertions(+), 9 deletions(-)
 create mode 100644 balancer/base/balancer_test.go

diff --git a/attributes/attributes_test.go b/attributes/attributes_test.go
index 4cca17b55e93..1174e2371a5f 100644
--- a/attributes/attributes_test.go
+++ b/attributes/attributes_test.go
@@ -20,6 +20,8 @@ package attributes_test
 
 import (
 	"fmt"
+	"reflect"
+	"testing"
 
 	"google.golang.org/grpc/attributes"
 )
@@ -46,3 +48,13 @@ func ExampleAttributes_WithValues() {
 	// Key one: 1
 	// Key two: two
 }
+
+// Test that two attributes with the same content are `reflect.DeepEqual`.
+func TestDeepEqual(t *testing.T) {
+	type keyOne struct{}
+	a1 := attributes.New(keyOne{}, 1)
+	a2 := attributes.New(keyOne{}, 1)
+	if !reflect.DeepEqual(a1, a2) {
+		t.Fatalf("reflect.DeepEqual(%+v, %+v), want true, got false", a1, a2)
+	}
+}
diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go
index 32d782f1cf5c..e0d34288ccfc 100644
--- a/balancer/base/balancer.go
+++ b/balancer/base/balancer.go
@@ -64,7 +64,7 @@ type baseBalancer struct {
 	csEvltr *balancer.ConnectivityStateEvaluator
 	state   connectivity.State
 
-	subConns map[resolver.Address]balancer.SubConn
+	subConns map[resolver.Address]balancer.SubConn // `attributes` is stripped from the keys of this map (the addresses)
 	scStates map[balancer.SubConn]connectivity.State
 	picker   balancer.Picker
 	config   Config
@@ -101,17 +101,41 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 	// addrsSet is the set converted from addrs, it's used for quick lookup of an address.
 	addrsSet := make(map[resolver.Address]struct{})
 	for _, a := range s.ResolverState.Addresses {
-		addrsSet[a] = struct{}{}
-		if _, ok := b.subConns[a]; !ok {
+		// Strip attributes from addresses before using them as map keys. So
+		// that when two addresses only differ in attributes pointers (but with
+		// the same attribute content), they are considered the same address.
+		//
+		// Note that this doesn't handle the case where the attribute content is
+		// different. So if users want to set different attributes to create
+		// duplicate connections to the same backend, it doesn't work. This is
+		// fine for now, because duplicate is done by setting Metadata today.
+		//
+		// TODO: read attributes to handle duplicate connections.
+		aNoAttrs := a
+		aNoAttrs.Attributes = nil
+		addrsSet[aNoAttrs] = struct{}{}
+		if sc, ok := b.subConns[aNoAttrs]; !ok {
 			// a is a new address (not existing in b.subConns).
+			//
+			// When creating SubConn, the original address with attributes is
+			// passed through. So that connection configurations in attributes
+			// (like creds) will be used.
 			sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: b.config.HealthCheck})
 			if err != nil {
 				logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err)
 				continue
 			}
-			b.subConns[a] = sc
+			b.subConns[aNoAttrs] = sc
 			b.scStates[sc] = connectivity.Idle
 			sc.Connect()
+		} else {
+			// Always update the subconn's address in case the attributes
+			// changed.
+			//
+			// The SubConn does a reflect.DeepEqual of the new and old
+			// addresses. So this is a noop if the current address is the same
+			// as the old one (including attributes).
+			sc.UpdateAddresses([]resolver.Address{a})
 		}
 	}
 	for a, sc := range b.subConns {
diff --git a/balancer/base/balancer_test.go b/balancer/base/balancer_test.go
new file mode 100644
index 000000000000..03114251a048
--- /dev/null
+++ b/balancer/base/balancer_test.go
@@ -0,0 +1,70 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package base
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/resolver"
+)
+
+type testClientConn struct {
+	balancer.ClientConn
+	newSubConn func([]resolver.Address, balancer.NewSubConnOptions) (balancer.SubConn, error)
+}
+
+func (c *testClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	return c.newSubConn(addrs, opts)
+}
+
+type testSubConn struct{}
+
+func (sc *testSubConn) UpdateAddresses(addresses []resolver.Address) {}
+
+func (sc *testSubConn) Connect() {}
+
+func TestBaseBalancerStripAttributes(t *testing.T) {
+	b := (&baseBuilder{}).Build(&testClientConn{
+		newSubConn: func(addrs []resolver.Address, _ balancer.NewSubConnOptions) (balancer.SubConn, error) {
+			for _, addr := range addrs {
+				if addr.Attributes == nil {
+					t.Errorf("in NewSubConn, got address %+v with nil attributes, want not nil", addr)
+				}
+			}
+			return &testSubConn{}, nil
+		},
+	}, balancer.BuildOptions{}).(*baseBalancer)
+
+	b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				{Addr: "1.1.1.1", Attributes: &attributes.Attributes{}},
+				{Addr: "2.2.2.2", Attributes: &attributes.Attributes{}},
+			},
+		},
+	})
+
+	for addr := range b.subConns {
+		if addr.Attributes != nil {
+			t.Errorf("in b.subConns, got address %+v with nil attributes, want not nil", addr)
+		}
+	}
+}
diff --git a/balancer/roundrobin/roundrobin_test.go b/balancer/roundrobin/roundrobin_test.go
index 5a8ba481c9f0..b89cdb4a30f3 100644
--- a/balancer/roundrobin/roundrobin_test.go
+++ b/balancer/roundrobin/roundrobin_test.go
@@ -32,6 +32,8 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpctest"
+	imetadata "google.golang.org/grpc/internal/metadata"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
@@ -39,6 +41,10 @@ import (
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
+const (
+	testMDKey = "test-md"
+)
+
 type s struct {
 	grpctest.Tester
 }
@@ -49,9 +55,23 @@ func Test(t *testing.T) {
 
 type testServer struct {
 	testpb.UnimplementedTestServiceServer
+
+	testMDChan chan []string
+}
+
+func newTestServer() *testServer {
+	return &testServer{testMDChan: make(chan []string, 1)}
 }
 
 func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if ok && len(md[testMDKey]) != 0 {
+		select {
+		case s.testMDChan <- md[testMDKey]:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
 	return &testpb.Empty{}, nil
 }
 
@@ -60,8 +80,9 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ
 }
 
 type test struct {
-	servers   []*grpc.Server
-	addresses []string
+	servers     []*grpc.Server
+	serverImpls []*testServer
+	addresses   []string
 }
 
 func (t *test) cleanup() {
@@ -85,8 +106,10 @@ func startTestServers(count int) (_ *test, err error) {
 		}
 
 		s := grpc.NewServer()
-		testpb.RegisterTestServiceServer(s, &testServer{})
+		sImpl := newTestServer()
+		testpb.RegisterTestServiceServer(s, sImpl)
 		t.servers = append(t.servers, s)
+		t.serverImpls = append(t.serverImpls, sImpl)
 		t.addresses = append(t.addresses, lis.Addr().String())
 
 		go func(s *grpc.Server, l net.Listener) {
@@ -473,3 +496,54 @@ func (s) TestAllServersDown(t *testing.T) {
 	}
 	t.Fatalf("Failfast RPCs didn't fail with Unavailable after all servers are stopped")
 }
+
+func (s) TestUpdateAddressAttributes(t *testing.T) {
+	r := manual.NewBuilderWithScheme("whatever")
+
+	test, err := startTestServers(1)
+	if err != nil {
+		t.Fatalf("failed to start servers: %v", err)
+	}
+	defer test.cleanup()
+
+	cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name))
+	if err != nil {
+		t.Fatalf("failed to dial: %v", err)
+	}
+	defer cc.Close()
+	testc := testpb.NewTestServiceClient(cc)
+	// The first RPC should fail because there's no address.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
+	defer cancel()
+	if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded {
+		t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err)
+	}
+
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: test.addresses[0]}}})
+	// The second RPC should succeed.
+	if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() = _, %v, want _, <nil>", err)
+	}
+	// The second RPC should not set metadata, so there's no md in the channel.
+	select {
+	case md1 := <-test.serverImpls[0].testMDChan:
+		t.Fatalf("got md: %v, want empty metadata", md1)
+	case <-time.After(time.Microsecond * 100):
+	}
+
+	const testMDValue = "test-md-value"
+	// Update metadata in address.
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{
+		imetadata.Set(resolver.Address{Addr: test.addresses[0]}, metadata.Pairs(testMDKey, testMDValue)),
+	}})
+	// The third RPC should succeed.
+	if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() = _, %v, want _, <nil>", err)
+	}
+
+	// The third RPC should send metadata with it.
+	md2 := <-test.serverImpls[0].testMDChan
+	if len(md2) == 0 || md2[0] != testMDValue {
+		t.Fatalf("got md: %v, want %v", md2, []string{testMDValue})
+	}
+}
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index cd81b8fa4166..6811e415a4c6 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -56,8 +56,8 @@ type TestSubConn struct {
 	id string
 }
 
-// UpdateAddresses panics.
-func (tsc *TestSubConn) UpdateAddresses([]resolver.Address) { panic("not implemented") }
+// UpdateAddresses is a no-op.
+func (tsc *TestSubConn) UpdateAddresses([]resolver.Address) {}
 
 // Connect is a no-op.
 func (tsc *TestSubConn) Connect() {}

From 5b1874ab74ee9d72fdf99a2f85d7397fb3e10093 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 3 Dec 2020 09:00:33 -0800
Subject: [PATCH 302/481] xds: Handle child policy configs properly (#4071)

---
 xds/internal/balancer/edsbalancer/eds.go      | 163 ++++++--
 xds/internal/balancer/edsbalancer/eds_impl.go |  16 +-
 .../edsbalancer/eds_impl_priority_test.go     |  18 +-
 .../balancer/edsbalancer/eds_impl_test.go     |   5 +-
 xds/internal/balancer/edsbalancer/eds_test.go | 395 +++++++++++-------
 .../edsbalancer/load_store_wrapper.go         |  80 ++++
 .../edsbalancer/xds_client_wrapper.go         | 238 -----------
 .../edsbalancer/xds_client_wrapper_test.go    | 145 -------
 .../balancer/edsbalancer/xds_lrs_test.go      |   7 +-
 xds/internal/testutils/balancer.go            |   6 -
 10 files changed, 477 insertions(+), 596 deletions(-)
 create mode 100644 xds/internal/balancer/edsbalancer/load_store_wrapper.go
 delete mode 100644 xds/internal/balancer/edsbalancer/xds_client_wrapper.go
 delete mode 100644 xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 1358dd723cca..1bcfa1d9f0fa 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -33,15 +34,22 @@ import (
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/serviceconfig"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 )
 
-const (
-	edsName = "eds_experimental"
-)
+const edsName = "eds_experimental"
+
+// xdsClientInterface contains only the xds_client methods needed by EDS
+// balancer. It's defined so we can override xdsclient.New function in tests.
+type xdsClientInterface interface {
+	WatchEndpoints(clusterName string, edsCb func(xdsclient.EndpointsUpdate, error)) (cancel func())
+	ReportLoad(server string) (loadStore *load.Store, cancel func())
+	Close()
+}
 
 var (
-	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
-		return newEDSBalancerImpl(cc, enqueueState, xdsClient, logger)
+	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), lw load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+		return newEDSBalancerImpl(cc, enqueueState, lw, logger)
 	}
 	newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
 )
@@ -60,8 +68,10 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptio
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
+		lsw:               &loadStoreWrapper{},
+		config:            &EDSConfig{},
 	}
-	x.logger = prefixLogger((x))
+	x.logger = prefixLogger(x)
 
 	client, err := newXDSClient()
 	if err != nil {
@@ -69,8 +79,8 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptio
 		return nil
 	}
 
-	x.client = newXDSClientWrapper(client, x.handleEDSUpdate, x.logger)
-	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.client, x.logger)
+	x.xdsClient = client
+	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.lsw, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
@@ -93,9 +103,11 @@ func (b *edsBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadB
 //
 // It's implemented by the real eds balancer and a fake testing eds balancer.
 type edsBalancerImplInterface interface {
-	// handleEDSResponse passes the received EDS message from traffic director to eds balancer.
+	// handleEDSResponse passes the received EDS message from traffic director
+	// to eds balancer.
 	handleEDSResponse(edsResp xdsclient.EndpointsUpdate)
-	// handleChildPolicy updates the eds balancer the intra-cluster load balancing policy to use.
+	// handleChildPolicy updates the eds balancer the intra-cluster load
+	// balancing policy to use.
 	handleChildPolicy(name string, config json.RawMessage)
 	// handleSubConnStateChange handles state change for SubConn.
 	handleSubConnStateChange(sc balancer.SubConn, state connectivity.State)
@@ -110,23 +122,33 @@ type edsBalancerImplInterface interface {
 //
 // It currently has only an edsBalancer. Later, we may add fallback.
 type edsBalancer struct {
-	cc     balancer.ClientConn // *xdsClientConn
+	cc     balancer.ClientConn
 	closed *grpcsync.Event
 	logger *grpclog.PrefixLogger
 
-	// edsBalancer continuously monitor the channels below, and will handle events from them in sync.
+	// edsBalancer continuously monitors the channels below, and will handle
+	// events from them in sync.
 	grpcUpdate        chan interface{}
 	xdsClientUpdate   chan *edsUpdate
 	childPolicyUpdate *buffer.Unbounded
 
-	client  *xdsClientWrapper // may change when passed a different service config
-	config  *EDSConfig        // may change when passed a different service config
-	edsImpl edsBalancerImplInterface
+	xdsClient xdsClientInterface
+	lsw       *loadStoreWrapper
+	config    *EDSConfig // may change when passed a different service config
+	edsImpl   edsBalancerImplInterface
+
+	// edsServiceName is the edsServiceName currently being watched, not
+	// necessary the edsServiceName from service config.
+	edsServiceName       string
+	cancelEndpointsWatch func()
+	loadReportServer     *string // LRS is disabled if loadReporterServer is nil.
+	cancelLoadReport     func()
 }
 
-// run gets executed in a goroutine once edsBalancer is created. It monitors updates from grpc,
-// xdsClient and load balancer. It synchronizes the operations that happen inside edsBalancer. It
-// exits when edsBalancer is closed.
+// run gets executed in a goroutine once edsBalancer is created. It monitors
+// updates from grpc, xdsClient and load balancer. It synchronizes the
+// operations that happen inside edsBalancer. It exits when edsBalancer is
+// closed.
 func (x *edsBalancer) run() {
 	for {
 		select {
@@ -139,7 +161,8 @@ func (x *edsBalancer) run() {
 			u := update.(*balancerStateWithPriority)
 			x.edsImpl.updateState(u.priority, u.s)
 		case <-x.closed.Done():
-			x.client.close()
+			x.cancelWatch()
+			x.xdsClient.Close()
 			x.edsImpl.close()
 			return
 		}
@@ -167,7 +190,7 @@ func (x *edsBalancer) handleErrorFromUpdate(err error, fromParent bool) {
 			// CDS balancer), and is a resource-not-found error. This means the
 			// resource (can be either LDS or CDS) was removed. Stop the EDS
 			// watch.
-			x.client.cancelWatch()
+			x.cancelWatch()
 		}
 		x.edsImpl.handleEDSResponse(xdsclient.EndpointsUpdate{})
 	}
@@ -185,25 +208,19 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			return
 		}
 
-		if err := x.client.handleUpdate(cfg); err != nil {
-			x.logger.Warningf("failed to update xds clients: %v", err)
-		}
-
-		if x.config == nil {
-			x.config = cfg
-			return
+		if err := x.handleServiceConfigUpdate(cfg); err != nil {
+			x.logger.Warningf("failed to update xDS client: %v", err)
 		}
 
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
-		if !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy) {
+		if !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy, cmpopts.EquateEmpty()) {
 			if cfg.ChildPolicy != nil {
 				x.edsImpl.handleChildPolicy(cfg.ChildPolicy.Name, cfg.ChildPolicy.Config)
 			} else {
 				x.edsImpl.handleChildPolicy(roundrobin.Name, nil)
 			}
 		}
-
 		x.config = cfg
 	case error:
 		x.handleErrorFromUpdate(u, true)
@@ -213,6 +230,81 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 	}
 }
 
+// handleServiceConfigUpdate applies the service config update, watching a new
+// EDS service name and restarting LRS stream, as required.
+func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
+	// Restart EDS watch when the edsServiceName has changed.
+	if x.edsServiceName != config.EDSServiceName {
+		x.edsServiceName = config.EDSServiceName
+		x.startEndpointsWatch()
+		// TODO: this update for the LRS service name is too early. It should
+		// only apply to the new EDS response. But this is applied to the RPCs
+		// before the new EDS response. To fully fix this, the EDS balancer
+		// needs to do a graceful switch to another EDS implementation.
+		//
+		// This is OK for now, because we don't actually expect edsServiceName
+		// to change. Fix this (a bigger change) will happen later.
+		x.lsw.updateServiceName(x.edsServiceName)
+	}
+
+	// Restart load reporting when the loadReportServer name has changed.
+	if !equalStringPointers(x.loadReportServer, config.LrsLoadReportingServerName) {
+		loadStore := x.startLoadReport(config.LrsLoadReportingServerName)
+		x.lsw.updateLoadStore(loadStore)
+	}
+
+	return nil
+}
+
+// startEndpointsWatch starts the EDS watch.
+//
+// This usually means load report needs to be restarted, but this function does
+// NOT do that. Caller needs to call startLoadReport separately.
+func (x *edsBalancer) startEndpointsWatch() {
+	if x.cancelEndpointsWatch != nil {
+		x.cancelEndpointsWatch()
+	}
+	cancelEDSWatch := x.xdsClient.WatchEndpoints(x.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
+		x.logger.Infof("Watch update from xds-client %p, content: %+v", x.xdsClient, update)
+		x.handleEDSUpdate(update, err)
+	})
+	x.logger.Infof("Watch started on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
+	x.cancelEndpointsWatch = func() {
+		cancelEDSWatch()
+		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
+	}
+}
+
+func (x *edsBalancer) cancelWatch() {
+	x.loadReportServer = nil
+	if x.cancelLoadReport != nil {
+		x.cancelLoadReport()
+	}
+	x.edsServiceName = ""
+	if x.cancelEndpointsWatch != nil {
+		x.cancelEndpointsWatch()
+	}
+}
+
+// startLoadReport starts load reporting. If there's already a load reporting in
+// progress, it cancels that.
+//
+// Caller can cal this when the loadReportServer name changes, but
+// edsServiceName doesn't (so we only need to restart load reporting, not EDS
+// watch).
+func (x *edsBalancer) startLoadReport(loadReportServer *string) *load.Store {
+	x.loadReportServer = loadReportServer
+	if x.cancelLoadReport != nil {
+		x.cancelLoadReport()
+	}
+	if loadReportServer == nil {
+		return nil
+	}
+	ls, cancel := x.xdsClient.ReportLoad(*loadReportServer)
+	x.cancelLoadReport = cancel
+	return ls
+}
+
 func (x *edsBalancer) handleXDSClientUpdate(update *edsUpdate) {
 	if err := update.err; err != nil {
 		x.handleErrorFromUpdate(err, false)
@@ -280,3 +372,16 @@ func (x *edsBalancer) Close() {
 	x.closed.Fire()
 	x.logger.Infof("Shutdown")
 }
+
+// equalStringPointers returns true if
+// - a and b are both nil OR
+// - *a == *b (and a and b are both non-nil)
+func equalStringPointers(a, b *string) bool {
+	if a == nil && b == nil {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	return *a == *b
+}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 1ce79bf61bb6..1d996e8219ee 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -62,9 +62,9 @@ type balancerGroupWithConfig struct {
 // The localities are picked as weighted round robin. A configurable child
 // policy is used to manage endpoints in each locality.
 type edsBalancerImpl struct {
-	cc        balancer.ClientConn
-	logger    *grpclog.PrefixLogger
-	xdsClient *xdsClientWrapper // To fetch the load.Store from.
+	cc           balancer.ClientConn
+	logger       *grpclog.PrefixLogger
+	loadReporter load.PerClusterReporter
 
 	enqueueChildBalancerStateUpdate func(priorityType, balancer.State)
 
@@ -99,12 +99,12 @@ type edsBalancerImpl struct {
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
-func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), xdsClient *xdsClientWrapper, logger *grpclog.PrefixLogger) *edsBalancerImpl {
+func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), lr load.PerClusterReporter, logger *grpclog.PrefixLogger) *edsBalancerImpl {
 	edsImpl := &edsBalancerImpl{
 		cc:                 cc,
 		logger:             logger,
 		subBalancerBuilder: balancer.Get(roundrobin.Name),
-		xdsClient:          xdsClient,
+		loadReporter:       lr,
 
 		enqueueChildBalancerStateUpdate: enqueueState,
 
@@ -170,7 +170,7 @@ func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropC
 		// Update picker with old inner picker, new drops.
 		edsImpl.cc.UpdateState(balancer.State{
 			ConnectivityState: edsImpl.innerState.ConnectivityState,
-			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.xdsClient.loadStore())},
+			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadReporter)},
 		)
 	}
 	edsImpl.pickerMu.Unlock()
@@ -240,7 +240,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
 			ccPriorityWrapper := edsImpl.ccWrapperWithPriority(priority)
 			stateAggregator := weightedaggregator.New(ccPriorityWrapper, edsImpl.logger, newRandomWRR)
 			bgwc = &balancerGroupWithConfig{
-				bg:              balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.xdsClient.loadStore(), edsImpl.logger),
+				bg:              balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.loadReporter, edsImpl.logger),
 				stateAggregator: stateAggregator,
 				configs:         make(map[internal.LocalityID]*localityConfig),
 			}
@@ -403,7 +403,7 @@ func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.St
 		defer edsImpl.pickerMu.Unlock()
 		edsImpl.innerState = s
 		// Don't reset drops when it's a state change.
-		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.xdsClient.loadStore())})
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadReporter)})
 	}
 }
 
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index 255665e3d4c1..1ce4f36ca73f 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -18,6 +18,7 @@
 package edsbalancer
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -74,7 +75,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 		t.Fatalf("got unexpected new SubConn")
 	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
-	case <-time.After(time.Millisecond * 100):
+	case <-time.After(defaultTestShortTimeout):
 	}
 
 	// Remove p2, no updates.
@@ -90,7 +91,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 		t.Fatalf("got unexpected new SubConn")
 	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
-	case <-time.After(time.Millisecond * 100):
+	case <-time.After(defaultTestShortTimeout):
 	}
 }
 
@@ -159,7 +160,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 		t.Fatalf("got unexpected new SubConn")
 	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
-	case <-time.After(time.Millisecond * 100):
+	case <-time.After(defaultTestShortTimeout):
 	}
 
 	// Turn down 1, use 2
@@ -269,8 +270,6 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 //
 // Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2.
 func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
-	defer time.Sleep(10 * time.Millisecond)
-
 	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
@@ -627,7 +626,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 		t.Fatalf("got unexpected new SubConn")
 	case <-cc.RemoveSubConnCh:
 		t.Fatalf("got unexpected remove SubConn")
-	case <-time.After(time.Millisecond * 100):
+	case <-time.After(defaultTestShortTimeout):
 	}
 }
 
@@ -836,6 +835,9 @@ func (s) TestEDSPriority_FirstPriorityUnavailable(t *testing.T) {
 	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
 	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
 
-	// Wait after double the init timer timeout, to ensure it doesn't fail.
-	time.Sleep(testPriorityInitTimeout * 2)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := cc.WaitForErrPicker(ctx); err != nil {
+		t.Fatal(err)
+	}
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 292ea4f80a70..fde00d225ee4 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -690,12 +690,9 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	lsWrapper := &loadStoreWrapper{}
 	lsWrapper.updateServiceName(testClusterNames[0])
 	lsWrapper.updateLoadStore(loadStore)
-	cw := &xdsClientWrapper{
-		loadWrapper: lsWrapper,
-	}
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, cw, nil)
+	edsb := newEDSBalancerImpl(cc, nil, lsWrapper, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index eeeaacae21aa..cd59bba1e799 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -38,13 +38,35 @@ import (
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 
 	_ "google.golang.org/grpc/xds/internal/client/v2" // V2 client registration.
 )
 
-const defaultTestTimeout = 1 * time.Second
+const (
+	defaultTestTimeout      = 1 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+	testServiceName         = "test/foo"
+	testEDSClusterName      = "test/service/eds"
+)
+
+var (
+	// A non-empty endpoints update which is expected to be accepted by the EDS
+	// LB policy.
+	defaultEndpointsUpdate = xdsclient.EndpointsUpdate{
+		Localities: []xdsclient.Locality{
+			{
+				Endpoints: []xdsclient.Endpoint{{Address: "endpoint1"}},
+				ID:        internal.LocalityID{Zone: "zone"},
+				Priority:  1,
+				Weight:    100,
+			},
+		},
+	}
+)
 
 func init() {
 	balancer.Register(&edsBalancerBuilder{})
@@ -77,7 +99,7 @@ type noopTestClientConn struct {
 	balancer.ClientConn
 }
 
-func (t *noopTestClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+func (t *noopTestClientConn) NewSubConn([]resolver.Address, balancer.NewSubConnOptions) (balancer.SubConn, error) {
 	return nil, nil
 }
 
@@ -111,13 +133,10 @@ func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {
 
 func (f *fakeEDSBalancer) close() {}
 
-func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-
+func (f *fakeEDSBalancer) waitForChildPolicy(ctx context.Context, wantPolicy *loadBalancingConfig) error {
 	val, err := f.childPolicy.Receive(ctx)
 	if err != nil {
-		return fmt.Errorf("error waiting for childPolicy: %v", err)
+		return err
 	}
 	gotPolicy := val.(*loadBalancingConfig)
 	if !cmp.Equal(gotPolicy, wantPolicy) {
@@ -126,13 +145,10 @@ func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) er
 	return nil
 }
 
-func (f *fakeEDSBalancer) waitForSubConnStateChange(wantState *scStateChange) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-
+func (f *fakeEDSBalancer) waitForSubConnStateChange(ctx context.Context, wantState *scStateChange) error {
 	val, err := f.subconnStateChange.Receive(ctx)
 	if err != nil {
-		return fmt.Errorf("error waiting for subconnStateChange: %v", err)
+		return err
 	}
 	gotState := val.(*scStateChange)
 	if !cmp.Equal(gotState, wantState, cmp.AllowUnexported(scStateChange{})) {
@@ -141,13 +157,10 @@ func (f *fakeEDSBalancer) waitForSubConnStateChange(wantState *scStateChange) er
 	return nil
 }
 
-func (f *fakeEDSBalancer) waitForEDSResponse(wantUpdate xdsclient.EndpointsUpdate) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-
+func (f *fakeEDSBalancer) waitForEDSResponse(ctx context.Context, wantUpdate xdsclient.EndpointsUpdate) error {
 	val, err := f.edsUpdate.Receive(ctx)
 	if err != nil {
-		return fmt.Errorf("error waiting for edsUpdate: %v", err)
+		return err
 	}
 	gotUpdate := val.(xdsclient.EndpointsUpdate)
 	if !reflect.DeepEqual(gotUpdate, wantUpdate) {
@@ -172,17 +185,12 @@ func (*fakeSubConn) Connect()                           { panic("implement me")
 
 // waitForNewEDSLB makes sure that a new edsLB is created by the top-level
 // edsBalancer.
-func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
+func waitForNewEDSLB(ctx context.Context, ch *testutils.Channel) (*fakeEDSBalancer, error) {
 	val, err := ch.Receive(ctx)
 	if err != nil {
-		t.Fatalf("error when waiting for a new edsLB: %v", err)
-		return nil
+		return nil, fmt.Errorf("error when waiting for a new edsLB: %v", err)
 	}
-	return val.(*fakeEDSBalancer)
+	return val.(*fakeEDSBalancer), nil
 }
 
 // setup overrides the functions which are used to create the xdsClient and the
@@ -195,7 +203,7 @@ func setup(edsLBCh *testutils.Channel) (*fakeclient.Client, func()) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 
 	origNewEDSBalancer := newEDSBalancer
-	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ *xdsClientWrapper, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
 		edsLB := newFakeEDSBalancer(cc)
 		defer func() { edsLBCh.Send(edsLB) }()
 		return edsLB
@@ -214,7 +222,6 @@ const (
 // Install two fake balancers for service config update tests.
 //
 // ParseConfig only accepts the json if the balancer specified is registered.
-
 func init() {
 	balancer.Register(&fakeBalancerBuilder{name: fakeBalancerA})
 	balancer.Register(&fakeBalancerBuilder{name: fakeBalancerB})
@@ -250,102 +257,115 @@ func (b *fakeBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnStat
 
 func (b *fakeBalancer) Close() {}
 
-// TestXDSConnfigChildPolicyUpdate verifies scenarios where the childPolicy
+// TestConfigChildPolicyUpdate verifies scenarios where the childPolicy
 // section of the lbConfig is updated.
 //
 // The test does the following:
-// * Builds a new xds balancer.
+// * Builds a new EDS balancer.
 // * Pushes a new ClientConnState with a childPolicy set to fakeBalancerA.
-//   Verifies that a new xdsClient is created. It then pushes a new edsUpdate
+//   Verifies that an EDS watch is registered. It then pushes a new edsUpdate
 //   through the fakexds client. Verifies that a new edsLB is created and it
 //   receives the expected childPolicy.
 // * Pushes a new ClientConnState with a childPolicy set to fakeBalancerB.
-//   This time around, we expect no new xdsClient or edsLB to be created.
-//   Instead, we expect the existing edsLB to receive the new child policy.
-func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) {
+//   Verifies that the existing edsLB receives the new child policy.
+func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	edsLBCh := testutils.NewChannel()
-	xdsC, cancel := setup(edsLBCh)
-	defer cancel()
+	xdsC, cleanup := setup(edsLBCh)
+	defer cleanup()
 
 	builder := balancer.Get(edsName)
-	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}).(*edsBalancer)
-	if !ok {
-		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
 	}
 	defer edsB.Close()
 
-	edsB.UpdateClientConnState(balancer.ClientConnState{
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	edsLB, err := waitForNewEDSLB(ctx, edsLBCh)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lbCfgA := &loadBalancingConfig{
+		Name:   fakeBalancerA,
+		Config: json.RawMessage("{}"),
+	}
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			ChildPolicy: &loadBalancingConfig{
-				Name:   fakeBalancerA,
-				Config: json.RawMessage("{}"),
-			},
-			EDSServiceName: testEDSClusterName,
+			ChildPolicy:    lbCfgA,
+			EDSServiceName: testServiceName,
 		},
-	})
+	}); err != nil {
+		t.Fatalf("edsB.UpdateClientConnState() failed: %v", err)
+	}
 
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
-	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
-	edsLB := waitForNewEDSLB(t, edsLBCh)
-	edsLB.waitForChildPolicy(&loadBalancingConfig{
-		Name:   string(fakeBalancerA),
-		Config: json.RawMessage(`{}`),
-	})
+	xdsC.InvokeWatchEDSCallback(defaultEndpointsUpdate, nil)
+	if err := edsLB.waitForChildPolicy(ctx, lbCfgA); err != nil {
+		t.Fatal(err)
+	}
 
-	edsB.UpdateClientConnState(balancer.ClientConnState{
+	lbCfgB := &loadBalancingConfig{
+		Name:   fakeBalancerB,
+		Config: json.RawMessage("{}"),
+	}
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			ChildPolicy: &loadBalancingConfig{
-				Name:   fakeBalancerB,
-				Config: json.RawMessage("{}"),
-			},
-			EDSServiceName: testEDSClusterName,
+			ChildPolicy:    lbCfgB,
+			EDSServiceName: testServiceName,
 		},
-	})
-	edsLB.waitForChildPolicy(&loadBalancingConfig{
-		Name:   string(fakeBalancerA),
-		Config: json.RawMessage(`{}`),
-	})
+	}); err != nil {
+		t.Fatalf("edsB.UpdateClientConnState() failed: %v", err)
+	}
+	if err := edsLB.waitForChildPolicy(ctx, lbCfgB); err != nil {
+		t.Fatal(err)
+	}
 }
 
-// TestXDSSubConnStateChange verifies if the top-level edsBalancer passes on
-// the subConnStateChange to appropriate child balancers.
-func (s) TestXDSSubConnStateChange(t *testing.T) {
+// TestSubConnStateChange verifies if the top-level edsBalancer passes on
+// the subConnStateChange to appropriate child balancer.
+func (s) TestSubConnStateChange(t *testing.T) {
 	edsLBCh := testutils.NewChannel()
-	xdsC, cancel := setup(edsLBCh)
-	defer cancel()
+	xdsC, cleanup := setup(edsLBCh)
+	defer cleanup()
 
 	builder := balancer.Get(edsName)
-	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
-	if !ok {
-		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
 	}
 	defer edsB.Close()
 
-	edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
-	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	edsLB, err := waitForNewEDSLB(ctx, edsLBCh)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
+	}); err != nil {
+		t.Fatalf("edsB.UpdateClientConnState() failed: %v", err)
+	}
 
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
-	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
-	edsLB := waitForNewEDSLB(t, edsLBCh)
+	xdsC.InvokeWatchEDSCallback(defaultEndpointsUpdate, nil)
 
 	fsc := &fakeSubConn{}
 	state := connectivity.Ready
 	edsB.UpdateSubConnState(fsc, balancer.SubConnState{ConnectivityState: state})
-	edsLB.waitForSubConnStateChange(&scStateChange{sc: fsc, state: state})
+	if err := edsLB.waitForSubConnStateChange(ctx, &scStateChange{sc: fsc, state: state}); err != nil {
+		t.Fatal(err)
+	}
 }
 
-// TestErrorFromXDSClientUpdate verifies that errros from xdsClient update are
+// TestErrorFromXDSClientUpdate verifies that an error from xdsClient update is
 // handled correctly.
 //
 // If it's resource-not-found, watch will NOT be canceled, the EDS impl will
@@ -355,42 +375,50 @@ func (s) TestXDSSubConnStateChange(t *testing.T) {
 // handle fallback.
 func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	edsLBCh := testutils.NewChannel()
-	xdsC, cancel := setup(edsLBCh)
-	defer cancel()
+	xdsC, cleanup := setup(edsLBCh)
+	defer cleanup()
 
 	builder := balancer.Get(edsName)
-	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
-	if !ok {
-		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
 	}
 	defer edsB.Close()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	edsLB, err := waitForNewEDSLB(ctx, edsLBCh)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
 	}); err != nil {
 		t.Fatal(err)
 	}
 
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
-	edsLB := waitForNewEDSLB(t, edsLBCh)
-	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
 	}
 
 	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, connectionErr)
 
-	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelEDSWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
-	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil {
-		t.Fatal("eds impl got EDS resp, want timeout error")
+
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsLB.waitForEDSResponse(sCtx, xdsclient.EndpointsUpdate{}); err != context.DeadlineExceeded {
+		t.Fatal(err)
 	}
 
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error")
@@ -398,12 +426,12 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	// Even if error is resource not found, watch shouldn't be canceled, because
 	// this is an EDS resource removed (and xds client actually never sends this
 	// error, but we still handles it).
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
-	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelEDSWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
-	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("eds impl expecting empty update, got %v", err)
 	}
 }
@@ -417,57 +445,128 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 // handle fallback.
 func (s) TestErrorFromResolver(t *testing.T) {
 	edsLBCh := testutils.NewChannel()
-	xdsC, cancel := setup(edsLBCh)
-	defer cancel()
+	xdsC, cleanup := setup(edsLBCh)
+	defer cleanup()
 
 	builder := balancer.Get(edsName)
-	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer)
-	if !ok {
-		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
 	}
 	defer edsB.Close()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	edsLB, err := waitForNewEDSLB(ctx, edsLBCh)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{EDSServiceName: testEDSClusterName},
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
 	}); err != nil {
 		t.Fatal(err)
 	}
 
-	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
 		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
 	}
 	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil)
-	edsLB := waitForNewEDSLB(t, edsLBCh)
-	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
 	}
 
 	connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error")
 	edsB.ResolverError(connectionErr)
 
-	if err := xdsC.WaitForCancelEDSWatch(ctx); err == nil {
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := xdsC.WaitForCancelEDSWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("watch was canceled, want not canceled (timeout error)")
 	}
-	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil {
+
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := edsLB.waitForEDSResponse(sCtx, xdsclient.EndpointsUpdate{}); err != context.DeadlineExceeded {
 		t.Fatal("eds impl got EDS resp, want timeout error")
 	}
 
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error")
 	edsB.ResolverError(resourceErr)
-	ctx, ctxCancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer ctxCancel()
 	if err := xdsC.WaitForCancelEDSWatch(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, waitForCancel failed: %v", err)
 	}
-	if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil {
+	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
 	}
 }
 
-func (s) TestXDSBalancerConfigParsing(t *testing.T) {
+// Given a list of resource names, verifies that EDS requests for the same are
+// sent by the EDS balancer, through the fake xDS client.
+func verifyExpectedRequests(ctx context.Context, fc *fakeclient.Client, resourceNames ...string) error {
+	for _, name := range resourceNames {
+		if name == "" {
+			// ResourceName empty string indicates a cancel.
+			if err := fc.WaitForCancelEDSWatch(ctx); err != nil {
+				return fmt.Errorf("timed out when expecting resource %q", name)
+			}
+			return nil
+		}
+
+		resName, err := fc.WaitForWatchEDS(ctx)
+		if err != nil {
+			return fmt.Errorf("timed out when expecting resource %q, %p", name, fc)
+		}
+		if resName != name {
+			return fmt.Errorf("got EDS request for resource %q, expected: %q", resName, name)
+		}
+	}
+	return nil
+}
+
+// TestClientWatchEDS verifies that the xdsClient inside the top-level EDS LB
+// policy registers an EDS watch for expected resource upon receiving an update
+// from gRPC.
+func (s) TestClientWatchEDS(t *testing.T) {
+	edsLBCh := testutils.NewChannel()
+	xdsC, cleanup := setup(edsLBCh)
+	defer cleanup()
+
+	builder := balancer.Get(edsName)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
+	}
+	defer edsB.Close()
+
+	// Update with an non-empty edsServiceName should trigger an EDS watch for
+	// the same.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: "foobar-1"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := verifyExpectedRequests(ctx, xdsC, "foobar-1"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Also test the case where the edsServerName changes from one non-empty
+	// name to another, and make sure a new watch is registered. The previously
+	// registered watch will be cancelled, which will result in an EDS request
+	// with no resource names being sent to the server.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: "foobar-2"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyExpectedRequests(ctx, xdsC, "", "foobar-2"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestBalancerConfigParsing(t *testing.T) {
 	const testEDSName = "eds.service"
 	var testLRSName = "lrs.server"
 	b := bytes.NewBuffer(nil)
@@ -496,6 +595,16 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 		want    serviceconfig.LoadBalancingConfig
 		wantErr bool
 	}{
+		{
+			name:    "bad json",
+			js:      json.RawMessage(`i am not JSON`),
+			wantErr: true,
+		},
+		{
+			name: "empty",
+			js:   json.RawMessage(`{}`),
+			want: &EDSConfig{},
+		},
 		{
 			name: "jsonpb-generated",
 			js:   b.Bytes(),
@@ -511,7 +620,6 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 				EDSServiceName:             testEDSName,
 				LrsLoadReportingServerName: &testLRSName,
 			},
-			wantErr: false,
 		},
 		{
 			// json with random balancers, and the first is not registered.
@@ -543,7 +651,6 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 				EDSServiceName:             testEDSName,
 				LrsLoadReportingServerName: &testLRSName,
 			},
-			wantErr: false,
 		},
 		{
 			// json with no lrs server name, LrsLoadReportingServerName should
@@ -557,37 +664,10 @@ func (s) TestXDSBalancerConfigParsing(t *testing.T) {
 				EDSServiceName:             testEDSName,
 				LrsLoadReportingServerName: nil,
 			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			b := &edsBalancerBuilder{}
-			got, err := b.ParseConfig(tt.js)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("edsBalancerBuilder.ParseConfig() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !cmp.Equal(got, tt.want) {
-				t.Errorf(cmp.Diff(got, tt.want))
-			}
-		})
-	}
-}
-func (s) TestLoadbalancingConfigParsing(t *testing.T) {
-	tests := []struct {
-		name string
-		s    string
-		want *EDSConfig
-	}{
-		{
-			name: "empty",
-			s:    "{}",
-			want: &EDSConfig{},
 		},
 		{
-			name: "success1",
-			s:    `{"childPolicy":[{"pick_first":{}}]}`,
+			name: "good child policy",
+			js:   json.RawMessage(`{"childPolicy":[{"pick_first":{}}]}`),
 			want: &EDSConfig{
 				ChildPolicy: &loadBalancingConfig{
 					Name:   "pick_first",
@@ -596,8 +676,8 @@ func (s) TestLoadbalancingConfigParsing(t *testing.T) {
 			},
 		},
 		{
-			name: "success2",
-			s:    `{"childPolicy":[{"round_robin":{}},{"pick_first":{}}]}`,
+			name: "multiple good child policies",
+			js:   json.RawMessage(`{"childPolicy":[{"round_robin":{}},{"pick_first":{}}]}`),
 			want: &EDSConfig{
 				ChildPolicy: &loadBalancingConfig{
 					Name:   "round_robin",
@@ -608,9 +688,16 @@ func (s) TestLoadbalancingConfigParsing(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			var cfg EDSConfig
-			if err := json.Unmarshal([]byte(tt.s), &cfg); err != nil || !cmp.Equal(&cfg, tt.want) {
-				t.Errorf("test name: %s, parseFullServiceConfig() = %+v, err: %v, want %+v, <nil>", tt.name, cfg, err, tt.want)
+			b := &edsBalancerBuilder{}
+			got, err := b.ParseConfig(tt.js)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("edsBalancerBuilder.ParseConfig() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
+				return
+			}
+			if !cmp.Equal(got, tt.want) {
+				t.Errorf(cmp.Diff(got, tt.want))
 			}
 		})
 	}
diff --git a/xds/internal/balancer/edsbalancer/load_store_wrapper.go b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
new file mode 100644
index 000000000000..d47a6f6a069c
--- /dev/null
+++ b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
@@ -0,0 +1,80 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package edsbalancer
+
+import (
+	"sync"
+
+	"google.golang.org/grpc/xds/internal/client/load"
+)
+
+type loadStoreWrapper struct {
+	mu      sync.RWMutex
+	service string
+	// Both store and perCluster will be nil if load reporting is disabled (EDS
+	// response doesn't have LRS server name). Note that methods on Store and
+	// perCluster all handle nil, so there's no need to check nil before calling
+	// them.
+	store      *load.Store
+	perCluster load.PerClusterReporter
+}
+
+func (lsw *loadStoreWrapper) updateServiceName(service string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if lsw.service == service {
+		return
+	}
+	lsw.service = service
+	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
+}
+
+func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
+}
+
+func (lsw *loadStoreWrapper) CallStarted(locality string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.perCluster.CallStarted(locality)
+}
+
+func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.perCluster.CallFinished(locality, err)
+}
+
+func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.perCluster.CallServerLoad(locality, name, val)
+}
+
+func (lsw *loadStoreWrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	lsw.perCluster.CallDropped(category)
+}
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
deleted file mode 100644
index 78902284c7cb..000000000000
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package edsbalancer
-
-import (
-	"sync"
-
-	"google.golang.org/grpc/internal/grpclog"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/load"
-)
-
-// xdsClientInterface contains only the xds_client methods needed by EDS
-// balancer. It's defined so we can override xdsclientNew function in tests.
-type xdsClientInterface interface {
-	WatchEndpoints(clusterName string, edsCb func(xdsclient.EndpointsUpdate, error)) (cancel func())
-	ReportLoad(server string) (loadStore *load.Store, cancel func())
-	Close()
-}
-
-type loadStoreWrapper struct {
-	mu      sync.RWMutex
-	service string
-	// Both store and perCluster will be nil if load reporting is disabled (EDS
-	// response doesn't have LRS server name). Note that methods on Store and
-	// perCluster all handle nil, so there's no need to check nil before calling
-	// them.
-	store      *load.Store
-	perCluster load.PerClusterReporter
-}
-
-func (lsw *loadStoreWrapper) updateServiceName(service string) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if lsw.service == service {
-		return
-	}
-	lsw.service = service
-	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
-}
-
-func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if store == lsw.store {
-		return
-	}
-	lsw.store = store
-	lsw.perCluster = nil
-	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
-}
-
-func (lsw *loadStoreWrapper) CallStarted(locality string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallStarted(locality)
-}
-
-func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallFinished(locality, err)
-}
-
-func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallServerLoad(locality, name, val)
-}
-
-func (lsw *loadStoreWrapper) CallDropped(category string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallDropped(category)
-}
-
-// xdsclientWrapper is responsible for getting the xds client from attributes or
-// creating a new xds client, and start watching EDS. The given callbacks will
-// be called with EDS updates or errors.
-type xdsClientWrapper struct {
-	logger *grpclog.PrefixLogger
-
-	newEDSUpdate func(xdsclient.EndpointsUpdate, error)
-
-	// xdsClient could come from attributes, or created with balancerName.
-	xdsClient xdsClientInterface
-
-	// loadWrapper is a wrapper with loadOriginal, with clusterName and
-	// edsServiceName. It's used children to report loads.
-	loadWrapper *loadStoreWrapper
-	// edsServiceName is the edsServiceName currently being watched, not
-	// necessary the edsServiceName from service config.
-	//
-	// If edsServiceName from service config is an empty, this will be user's
-	// dial target (because that's what we use to watch EDS).
-	//
-	// TODO: remove the empty string related behavior, when we switch to always
-	// do CDS.
-	edsServiceName       string
-	cancelEndpointsWatch func()
-	loadReportServer     *string // LRS is disabled if loadReporterServer is nil.
-	cancelLoadReport     func()
-}
-
-// newXDSClientWrapper creates an empty xds_client wrapper that does nothing. It
-// can accept xds_client configs, to new/switch xds_client to use.
-//
-// The given callbacks won't be called until the underlying xds_client is
-// working and sends updates.
-func newXDSClientWrapper(xdsClient xdsClientInterface, newEDSUpdate func(xdsclient.EndpointsUpdate, error), logger *grpclog.PrefixLogger) *xdsClientWrapper {
-	return &xdsClientWrapper{
-		logger:       logger,
-		newEDSUpdate: newEDSUpdate,
-		xdsClient:    xdsClient,
-		loadWrapper:  &loadStoreWrapper{},
-	}
-}
-
-// startEndpointsWatch starts the EDS watch. Caller can call this when the
-// xds_client is updated, or the edsServiceName is updated.
-//
-// Note that if there's already a watch in progress, it's not explicitly
-// canceled. Because for each xds_client, there should be only one EDS watch in
-// progress. So a new EDS watch implicitly cancels the previous one.
-//
-// This usually means load report needs to be restarted, but this function does
-// NOT do that. Caller needs to call startLoadReport separately.
-func (c *xdsClientWrapper) startEndpointsWatch() {
-	if c.cancelEndpointsWatch != nil {
-		c.cancelEndpointsWatch()
-	}
-	cancelEDSWatch := c.xdsClient.WatchEndpoints(c.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
-		c.logger.Infof("Watch update from xds-client %p, content: %+v", c.xdsClient, update)
-		c.newEDSUpdate(update, err)
-	})
-	c.logger.Infof("Watch started on resource name %v with xds-client %p", c.edsServiceName, c.xdsClient)
-	c.cancelEndpointsWatch = func() {
-		cancelEDSWatch()
-		c.logger.Infof("Watch cancelled on resource name %v with xds-client %p", c.edsServiceName, c.xdsClient)
-	}
-}
-
-// startLoadReport starts load reporting. If there's already a load reporting in
-// progress, it cancels that.
-//
-// Caller can cal this when the loadReportServer name changes, but
-// edsServiceName doesn't (so we only need to restart load reporting, not EDS
-// watch).
-func (c *xdsClientWrapper) startLoadReport(loadReportServer *string) *load.Store {
-	if c.cancelLoadReport != nil {
-		c.cancelLoadReport()
-	}
-	c.loadReportServer = loadReportServer
-	var loadStore *load.Store
-	if c.loadReportServer != nil {
-		loadStore, c.cancelLoadReport = c.xdsClient.ReportLoad(*c.loadReportServer)
-	}
-	return loadStore
-}
-
-func (c *xdsClientWrapper) loadStore() load.PerClusterReporter {
-	if c == nil {
-		return nil
-	}
-	return c.loadWrapper
-}
-
-// handleUpdate applies the service config and attributes updates to the client,
-// including updating the xds_client to use, and updating the EDS name to watch.
-func (c *xdsClientWrapper) handleUpdate(config *EDSConfig) error {
-	// Need to restart EDS watch when the edsServiceName changed
-	if c.edsServiceName != config.EDSServiceName {
-		c.edsServiceName = config.EDSServiceName
-		c.startEndpointsWatch()
-		// TODO: this update for the LRS service name is too early. It should
-		// only apply to the new EDS response. But this is applied to the RPCs
-		// before the new EDS response. To fully fix this, the EDS balancer
-		// needs to do a graceful switch to another EDS implementation.
-		//
-		// This is OK for now, because we don't actually expect edsServiceName
-		// to change. Fix this (a bigger change) will happen later.
-		c.loadWrapper.updateServiceName(c.edsServiceName)
-	}
-
-	// Only need to restart load reporting when:
-	// - the loadReportServer name changed
-	if !equalStringPointers(c.loadReportServer, config.LrsLoadReportingServerName) {
-		loadStore := c.startLoadReport(config.LrsLoadReportingServerName)
-		c.loadWrapper.updateLoadStore(loadStore)
-	}
-
-	return nil
-}
-
-func (c *xdsClientWrapper) cancelWatch() {
-	c.loadReportServer = nil
-	if c.cancelLoadReport != nil {
-		c.cancelLoadReport()
-	}
-	c.edsServiceName = ""
-	if c.cancelEndpointsWatch != nil {
-		c.cancelEndpointsWatch()
-	}
-}
-
-func (c *xdsClientWrapper) close() {
-	c.cancelWatch()
-	c.xdsClient.Close()
-}
-
-// equalStringPointers returns true if
-// - a and b are both nil OR
-// - *a == *b (and a and b are both non-nil)
-func equalStringPointers(a, b *string) bool {
-	if a == nil && b == nil {
-		return true
-	}
-	if a == nil || b == nil {
-		return false
-	}
-	return *a == *b
-}
diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
deleted file mode 100644
index 60faf6fc3136..000000000000
--- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package edsbalancer
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/internal/testutils"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
-)
-
-var (
-	testServiceName    = "test/foo"
-	testEDSClusterName = "test/service/eds"
-)
-
-// Given a list of resource names, verifies that EDS requests for the same are
-// received at the fake server.
-func verifyExpectedRequests(fc *fakeclient.Client, resourceNames ...string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-
-	for _, name := range resourceNames {
-		if name == "" {
-			// ResourceName empty string indicates a cancel.
-			if err := fc.WaitForCancelEDSWatch(ctx); err != nil {
-				return fmt.Errorf("timed out when expecting resource %q", name)
-			}
-			return nil
-		}
-
-		resName, err := fc.WaitForWatchEDS(ctx)
-		if err != nil {
-			return fmt.Errorf("timed out when expecting resource %q, %p", name, fc)
-		}
-		if resName != name {
-			return fmt.Errorf("got EDS request for resource %q, expected: %q", resName, name)
-		}
-	}
-	return nil
-}
-
-// TestClientWrapperWatchEDS verifies that the clientWrapper registers an
-// EDS watch for expected resource upon receiving an update from the top-level
-// edsBalancer.
-//
-// The test does the following:
-// * Starts a fake xDS server.
-// * Creates a clientWrapper.
-// * Sends updates with different edsServiceNames and expects new watches to be
-//   registered.
-func (s) TestClientWrapperWatchEDS(t *testing.T) {
-	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
-
-	cw := newXDSClientWrapper(xdsC, nil, nil)
-	defer cw.close()
-	t.Logf("Started xDS client wrapper for endpoint %s...", testServiceName)
-
-	// Update with an non-empty edsServiceName should trigger an EDS watch for
-	// the same.
-	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-1"})
-	if err := verifyExpectedRequests(xdsC, "foobar-1"); err != nil {
-		t.Fatal(err)
-	}
-
-	// Also test the case where the edsServerName changes from one non-empty
-	// name to another, and make sure a new watch is registered. The previously
-	// registered watch will be cancelled, which will result in an EDS request
-	// with no resource names being sent to the server.
-	cw.handleUpdate(&EDSConfig{EDSServiceName: "foobar-2"})
-	if err := verifyExpectedRequests(xdsC, "", "foobar-2"); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// TestClientWrapperHandleUpdateError verifies that the clientWrapper handles
-// errors from the edsWatch callback appropriately.
-//
-// The test does the following:
-// * Creates a clientWrapper.
-// * Creates a fakeclient.Client and passes it to the clientWrapper in attributes.
-// * Verifies the clientWrapper registers an EDS watch.
-// * Forces the fakeclient.Client to invoke the registered EDS watch callback with
-//   an error. Verifies that the wrapper does not invoke the top-level
-//   edsBalancer with the received error.
-func (s) TestClientWrapperHandleUpdateError(t *testing.T) {
-	edsRespChan := testutils.NewChannel()
-	newEDS := func(update xdsclient.EndpointsUpdate, err error) {
-		edsRespChan.Send(&edsUpdate{resp: update, err: err})
-	}
-
-	xdsC := fakeclient.NewClient()
-	cw := newXDSClientWrapper(xdsC, newEDS, nil)
-	defer cw.close()
-
-	cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName})
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	gotCluster, err := xdsC.WaitForWatchEDS(ctx)
-	if err != nil {
-		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
-	}
-	if gotCluster != testEDSClusterName {
-		t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName)
-	}
-	watchErr := errors.New("EDS watch callback error")
-	xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, watchErr)
-
-	// The callback is called with an error, expect no update from edsRespChan.
-	//
-	// TODO: check for loseContact() when errors indicating "lose contact" are
-	// handled correctly.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	gotUpdate, err := edsRespChan.Receive(ctx)
-	if err != nil {
-		t.Fatalf("edsBalancer failed to get edsUpdate %v", err)
-	}
-	update := gotUpdate.(*edsUpdate)
-	if !cmp.Equal(update.resp, (xdsclient.EndpointsUpdate{})) || update.err != watchErr {
-		t.Fatalf("want update {nil, %v}, got %+v", watchErr, update)
-	}
-}
diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
index a9108a2281ed..9f93e0b42f08 100644
--- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
@@ -36,10 +36,9 @@ func (s) TestXDSLoadReporting(t *testing.T) {
 	defer func() { newXDSClient = oldNewXDSClient }()
 
 	builder := balancer.Get(edsName)
-	cc := newNoopTestClientConn()
-	edsB, ok := builder.Build(cc, balancer.BuildOptions{}).(*edsBalancer)
-	if !ok {
-		t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
 	}
 	defer edsB.Close()
 
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 6811e415a4c6..0643a060c97e 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -123,12 +123,6 @@ func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
 	}
 }
 
-// UpdateBalancerState implements balancer.Balancer API. It will be removed when
-// switching to the new balancer interface.
-func (tcc *TestClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) {
-	panic("not implemented")
-}
-
 // UpdateState updates connectivity state and picker.
 func (tcc *TestClientConn) UpdateState(bs balancer.State) {
 	tcc.logger.Logf("testClientConn: UpdateState(%v)", bs)

From f1149b9954aa892d89202c124c5b7569df72682d Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 3 Dec 2020 09:12:22 -0800
Subject: [PATCH 303/481] xds: support server-side listeners in LDS response
 (#4046)

---
 xds/internal/client/client.go          |  16 +-
 xds/internal/client/client_lds_test.go | 728 ++++++++++++++++++++++++-
 xds/internal/client/client_rds_test.go | 147 +----
 xds/internal/client/client_xds.go      | 139 ++++-
 xds/internal/version/version.go        |  13 +-
 5 files changed, 859 insertions(+), 184 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 09729ec3e281..5d8a7ad12281 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -69,9 +69,9 @@ func getAPIClientBuilder(version version.TransportAPI) APIClientBuilder {
 
 // BuildOptions contains options to be passed to client builders.
 type BuildOptions struct {
-	// Parent is a top-level xDS client or server which has the intelligence to
-	// take appropriate action based on xDS responses received from the
-	// management server.
+	// Parent is a top-level xDS client which has the intelligence to take
+	// appropriate action based on xDS responses received from the management
+	// server.
 	Parent UpdateHandler
 	// NodeProto contains the Node proto to be used in xDS requests. The actual
 	// type depends on the transport protocol version used.
@@ -145,6 +145,8 @@ type ListenerUpdate struct {
 	// RouteConfigName is the route configuration name corresponding to the
 	// target which is being watched through LDS.
 	RouteConfigName string
+	// SecurityCfg contains security configuration sent by the control plane.
+	SecurityCfg *SecurityConfig
 }
 
 // RouteConfigUpdate contains information received in an RDS response, which is
@@ -195,7 +197,8 @@ type Int64Range struct {
 }
 
 // SecurityConfig contains the security configuration received as part of the
-// Cluster resource.
+// Cluster resource on the client-side, and as part of the Listener resource on
+// the server-side.
 type SecurityConfig struct {
 	// RootInstanceName identifies the certProvider plugin to be used to fetch
 	// root certificates. This instance name will be resolved to the plugin name
@@ -215,7 +218,8 @@ type SecurityConfig struct {
 	IdentityCertName string
 	// AcceptedSANs is a list of Subject Alternative Names. During the TLS
 	// handshake, the SAN present in the peer certificate is compared against
-	// this list, and the handshake succeeds only if a match is found.
+	// this list, and the handshake succeeds only if a match is found. Used only
+	// on the client-side.
 	AcceptedSANs []string
 }
 
@@ -227,7 +231,7 @@ type ClusterUpdate struct {
 	ServiceName string
 	// EnableLRS indicates whether or not load should be reported through LRS.
 	EnableLRS bool
-	// SecurityCfg contains security configuration sent by the xDS server.
+	// SecurityCfg contains security configuration sent by the control plane.
 	SecurityCfg *SecurityConfig
 }
 
diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
index eedfe72f94e2..b227f96e4ece 100644
--- a/xds/internal/client/client_lds_test.go
+++ b/xds/internal/client/client_lds_test.go
@@ -19,6 +19,7 @@
 package client
 
 import (
+	"strings"
 	"testing"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
@@ -28,6 +29,7 @@ import (
 	v2listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
@@ -35,7 +37,7 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-func (s) TestUnmarshalListener(t *testing.T) {
+func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 	const (
 		v2LDSTarget       = "lds.target.good:2222"
 		v3LDSTarget       = "lds.target.good:3333"
@@ -133,6 +135,136 @@ func (s) TestUnmarshalListener(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "wrong type in apiListener",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V2ListenerURL,
+									Value: func() []byte {
+										cm := &v3httppb.HttpConnectionManager{
+											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+												Rds: &v3httppb.Rds{
+													ConfigSource: &v3corepb.ConfigSource{
+														ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+													},
+													RouteConfigName: v3RouteConfigName,
+												},
+											},
+										}
+										mcm, _ := proto.Marshal(cm)
+										return mcm
+									}(),
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty httpConnMgr in apiListener",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V2ListenerURL,
+									Value: func() []byte {
+										cm := &v3httppb.HttpConnectionManager{
+											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+												Rds: &v3httppb.Rds{},
+											},
+										}
+										mcm, _ := proto.Marshal(cm)
+										return mcm
+									}(),
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "scopedRoutes routeConfig in apiListener",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V2ListenerURL,
+									Value: func() []byte {
+										cm := &v3httppb.HttpConnectionManager{
+											RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+										}
+										mcm, _ := proto.Marshal(cm)
+										return mcm
+									}(),
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "rds.ConfigSource in apiListener is not ADS",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V2ListenerURL,
+									Value: func() []byte {
+										cm := &v3httppb.HttpConnectionManager{
+											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+												Rds: &v3httppb.Rds{
+													ConfigSource: &v3corepb.ConfigSource{
+														ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{
+															Path: "/some/path",
+														},
+													},
+													RouteConfigName: v3RouteConfigName,
+												},
+											},
+										}
+										mcm, _ := proto.Marshal(cm)
+										return mcm
+									}(),
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "empty resource list",
 		},
@@ -169,3 +301,597 @@ func (s) TestUnmarshalListener(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
+	const v3LDSTarget = "grpc/server?udpa.resource.listening_address=0.0.0.0:9999"
+
+	tests := []struct {
+		name       string
+		resources  []*anypb.Any
+		wantUpdate map[string]ListenerUpdate
+		wantErr    string
+	}{
+		{
+			name: "no address field",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "no address field in LDS response",
+		},
+		{
+			name: "no socket address field",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name:    v3LDSTarget,
+							Address: &v3corepb.Address{},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "no socket_address field in LDS response",
+		},
+		{
+			name: "listener name does not match expected format",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: "foo",
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "no host:port in name field of LDS response",
+		},
+		{
+			name: "host mismatch",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "1.2.3.4",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "socket_address host does not match the one in name",
+		},
+		{
+			name: "port mismatch",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 1234,
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "socket_address port does not match the one in name",
+		},
+		{
+			name: "unexpected number of filter chains",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{Name: "filter-chain-1"},
+								{Name: "filter-chain-2"},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "filter chains count in LDS response does not match expected",
+		},
+		{
+			name: "unexpected application protocol value",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"h2"},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "application_protocols in LDS response does not match expected",
+		},
+		{
+			name: "unexpected transport socket name",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "unsupported-transport-socket-name",
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "transport_socket field has unexpected name",
+		},
+		{
+			name: "unexpected transport socket typedConfig URL",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3UpstreamTLSContextURL,
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "transport_socket field has unexpected typeURL",
+		},
+		{
+			name: "badly marshaled transport socket",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value:   []byte{1, 2, 3, 4},
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "failed to unmarshal DownstreamTlsContext in LDS response",
+		},
+		{
+			name: "missing CommonTlsContext",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value: func() []byte {
+													tls := &v3tlspb.DownstreamTlsContext{}
+													mtls, _ := proto.Marshal(tls)
+													return mtls
+												}(),
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext",
+		},
+		{
+			name: "unsupported validation context in transport socket",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value: func() []byte {
+													tls := &v3tlspb.DownstreamTlsContext{
+														CommonTlsContext: &v3tlspb.CommonTlsContext{
+															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
+																ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
+																	Name: "foo-sds-secret",
+																},
+															},
+														},
+													}
+													mtls, _ := proto.Marshal(tls)
+													return mtls
+												}(),
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "validation context contains unexpected type",
+		},
+		{
+			name: "empty transport socket",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {},
+			},
+		},
+		{
+			name: "happy case with no identity certs",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value: func() []byte {
+													tls := &v3tlspb.DownstreamTlsContext{
+														CommonTlsContext: &v3tlspb.CommonTlsContext{
+															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+																ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+																	InstanceName:    "rootPluginInstance",
+																	CertificateName: "rootCertName",
+																},
+															},
+														},
+													}
+													mtls, _ := proto.Marshal(tls)
+													return mtls
+												}(),
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					SecurityCfg: &SecurityConfig{
+						RootInstanceName: "rootPluginInstance",
+						RootCertName:     "rootCertName",
+					},
+				},
+			},
+		},
+		{
+			name: "happy case with validation context provider instance",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value: func() []byte {
+													tls := &v3tlspb.DownstreamTlsContext{
+														CommonTlsContext: &v3tlspb.CommonTlsContext{
+															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+																InstanceName:    "identityPluginInstance",
+																CertificateName: "identityCertName",
+															},
+															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+																ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+																	InstanceName:    "rootPluginInstance",
+																	CertificateName: "rootCertName",
+																},
+															},
+														},
+													}
+													mtls, _ := proto.Marshal(tls)
+													return mtls
+												}(),
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					SecurityCfg: &SecurityConfig{
+						RootInstanceName:     "rootPluginInstance",
+						RootCertName:         "rootCertName",
+						IdentityInstanceName: "identityPluginInstance",
+						IdentityCertName:     "identityCertName",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotUpdate, err := UnmarshalListener(test.resources, nil)
+			if err != nil && !strings.Contains(err.Error(), test.wantErr) {
+				t.Errorf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
+			}
+			if test.wantErr != "" {
+				return
+			}
+			if !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("UnmarshalListener(%v) = %v want %v", test.resources, gotUpdate, test.wantUpdate)
+			}
+		})
+	}
+}
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index 7b39158b8e36..5e9ee7758657 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -24,162 +24,17 @@ import (
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
-	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-func (s) TestGetRouteConfigFromListener(t *testing.T) {
-	const (
-		goodLDSTarget       = "lds.target.good:1111"
-		goodRouteConfigName = "GoodRouteConfig"
-	)
-
-	tests := []struct {
-		name      string
-		lis       *v3listenerpb.Listener
-		wantRoute string
-		wantErr   bool
-	}{
-		{
-			name:      "no-apiListener-field",
-			lis:       &v3listenerpb.Listener{},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "badly-marshaled-apiListener",
-			lis: &v3listenerpb.Listener{
-				Name: goodLDSTarget,
-				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V3HTTPConnManagerURL,
-						Value:   []byte{1, 2, 3, 4},
-					},
-				},
-			},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "wrong-type-in-apiListener",
-			lis: &v3listenerpb.Listener{
-				Name: goodLDSTarget,
-				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V2ListenerURL,
-						Value: func() []byte {
-							cm := &v3httppb.HttpConnectionManager{
-								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-									Rds: &v3httppb.Rds{
-										ConfigSource: &v3corepb.ConfigSource{
-											ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-										},
-										RouteConfigName: goodRouteConfigName}}}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}()}}},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "empty-httpConnMgr-in-apiListener",
-			lis: &v3listenerpb.Listener{
-				Name: goodLDSTarget,
-				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V3HTTPConnManagerURL,
-						Value: func() []byte {
-							cm := &v3httppb.HttpConnectionManager{
-								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-									Rds: &v3httppb.Rds{},
-								},
-							}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}()}}},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "scopedRoutes-routeConfig-in-apiListener",
-			lis: &v3listenerpb.Listener{
-				Name: goodLDSTarget,
-				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V3HTTPConnManagerURL,
-						Value: func() []byte {
-							cm := &v3httppb.HttpConnectionManager{
-								RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
-							}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}()}}},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "rds.ConfigSource-in-apiListener-is-not-ADS",
-			lis: &v3listenerpb.Listener{
-				Name: goodLDSTarget,
-				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V3HTTPConnManagerURL,
-						Value: func() []byte {
-							cm := &v3httppb.HttpConnectionManager{
-								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-									Rds: &v3httppb.Rds{
-										ConfigSource: &v3corepb.ConfigSource{
-											ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{
-												Path: "/some/path",
-											},
-										},
-										RouteConfigName: goodRouteConfigName}}}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}()}}},
-			wantRoute: "",
-			wantErr:   true,
-		},
-		{
-			name: "goodListener",
-			lis: &v3listenerpb.Listener{
-				Name: goodLDSTarget,
-				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V3HTTPConnManagerURL,
-						Value: func() []byte {
-							cm := &v3httppb.HttpConnectionManager{
-								RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-									Rds: &v3httppb.Rds{
-										ConfigSource: &v3corepb.ConfigSource{
-											ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-										},
-										RouteConfigName: goodRouteConfigName}}}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}()}}},
-			wantRoute: goodRouteConfigName,
-			wantErr:   false,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			gotRoute, err := getRouteConfigNameFromListener(test.lis, nil)
-			if (err != nil) != test.wantErr || gotRoute != test.wantRoute {
-				t.Errorf("getRouteConfigNameFromListener(%+v) = (%s, %v), want (%s, %v)", test.lis, gotRoute, err, test.wantRoute, test.wantErr)
-			}
-		})
-	}
-}
-
 func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 	const (
 		uninterestingDomain      = "uninteresting.domain"
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index af25ce226bd4..22585ed78bdd 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -19,9 +19,11 @@
 package client
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -39,6 +41,10 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
+// TransportSocket proto message has a `name` field which is expected to be set
+// to this value by the management server.
+const transportSocketName = "envoy.transport_sockets.tls"
+
 // UnmarshalListener processes resources received in an LDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
@@ -53,52 +59,131 @@ func UnmarshalListener(resources []*anypb.Any, logger *grpclog.PrefixLogger) (ma
 			return nil, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
-		routeName, err := getRouteConfigNameFromListener(lis, logger)
+
+		lu, err := processListener(lis)
 		if err != nil {
 			return nil, err
 		}
-		update[lis.GetName()] = ListenerUpdate{RouteConfigName: routeName}
+		update[lis.GetName()] = *lu
 	}
 	return update, nil
 }
 
-// getRouteConfigNameFromListener checks if the provided Listener proto meets
-// the expected criteria. If so, it returns a non-empty routeConfigName.
-func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger) (string, error) {
-	if lis.GetApiListener() == nil {
-		return "", fmt.Errorf("xds: no api_listener field in LDS response %+v", lis)
+func processListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
+	if lis.GetApiListener() != nil {
+		return processClientSideListener(lis)
 	}
+	return processServerSideListener(lis)
+}
+
+// processClientSideListener checks if the provided Listener proto meets
+// the expected criteria. If so, it returns a non-empty routeConfigName.
+func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
 	apiLisAny := lis.GetApiListener().GetApiListener()
 	if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
-		return "", fmt.Errorf("xds: unexpected resource type: %q in LDS response", apiLisAny.GetTypeUrl())
+		return nil, fmt.Errorf("xds: unexpected resource type: %q in LDS response", apiLisAny.GetTypeUrl())
 	}
 	apiLis := &v3httppb.HttpConnectionManager{}
 	if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
-		return "", fmt.Errorf("xds: failed to unmarshal api_listner in LDS response: %v", err)
+		return nil, fmt.Errorf("xds: failed to unmarshal api_listner in LDS response: %v", err)
 	}
 
-	logger.Infof("Resource with type %T, contains %v", apiLis, apiLis)
 	switch apiLis.RouteSpecifier.(type) {
 	case *v3httppb.HttpConnectionManager_Rds:
 		if apiLis.GetRds().GetConfigSource().GetAds() == nil {
-			return "", fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis)
+			return nil, fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis)
 		}
 		name := apiLis.GetRds().GetRouteConfigName()
 		if name == "" {
-			return "", fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
+			return nil, fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
 		}
-		return name, nil
+		return &ListenerUpdate{RouteConfigName: name}, nil
 	case *v3httppb.HttpConnectionManager_RouteConfig:
 		// TODO: Add support for specifying the RouteConfiguration inline
 		// in the LDS response.
-		return "", fmt.Errorf("xds: LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
+		return nil, fmt.Errorf("xds: LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
 	case nil:
-		return "", fmt.Errorf("xds: no RouteSpecifier in received LDS response: %+v", apiLis)
+		return nil, fmt.Errorf("xds: no RouteSpecifier in received LDS response: %+v", apiLis)
 	default:
-		return "", fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
+		return nil, fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
 	}
 }
 
+func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
+	// Make sure that an address encoded in the received listener resource, and
+	// that it matches the one specified in the name. Listener names on the
+	// server-side as in the following format:
+	// grpc/server?udpa.resource.listening_address=IP:Port.
+	addr := lis.GetAddress()
+	if addr == nil {
+		return nil, fmt.Errorf("xds: no address field in LDS response: %+v", lis)
+	}
+	sockAddr := addr.GetSocketAddress()
+	if sockAddr == nil {
+		return nil, fmt.Errorf("xds: no socket_address field in LDS response: %+v", lis)
+	}
+	host, port, err := getAddressFromName(lis.GetName())
+	if err != nil {
+		return nil, fmt.Errorf("xds: no host:port in name field of LDS response: %+v, error: %v", lis, err)
+	}
+	if h := sockAddr.GetAddress(); host != h {
+		return nil, fmt.Errorf("xds: socket_address host does not match the one in name. Got %q, want %q", h, host)
+	}
+	if p := strconv.Itoa(int(sockAddr.GetPortValue())); port != p {
+		return nil, fmt.Errorf("xds: socket_address port does not match the one in name. Got %q, want %q", p, port)
+	}
+
+	// Make sure that the listener resource contains a single filter chain with
+	// the application_protocols field set to "managed-mtls", and the
+	// "transport_socket" field containing the appropriate security
+	// configuration.
+	fcs := lis.GetFilterChains()
+	if n := len(fcs); n != 1 {
+		return nil, fmt.Errorf("xds: filter chains count in LDS response does not match expected. Got %d, want 1", n)
+	}
+	fc := fcs[0]
+	aps := fc.GetFilterChainMatch().GetApplicationProtocols()
+	if len(aps) != 1 || aps[0] != "managed-mtls" {
+		return nil, fmt.Errorf("xds: application_protocols in LDS response does not match expected. Got %v, want %q", aps, "managed-mtls")
+	}
+
+	// If the transport_socket field is not specified, it means that the control
+	// plane has not sent us any security config. This is fine and the server
+	// will use the fallback credentials configured as part of the
+	// xdsCredentials.
+	ts := fc.GetTransportSocket()
+	if ts == nil {
+		return &ListenerUpdate{}, nil
+	}
+	if name := ts.GetName(); name != transportSocketName {
+		return nil, fmt.Errorf("xds: transport_socket field has unexpected name: %s", name)
+	}
+	any := ts.GetTypedConfig()
+	if any == nil || any.TypeUrl != version.V3DownstreamTLSContextURL {
+		return nil, fmt.Errorf("xds: transport_socket field has unexpected typeURL: %s", any.TypeUrl)
+	}
+	downstreamCtx := &v3tlspb.DownstreamTlsContext{}
+	if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil {
+		return nil, fmt.Errorf("xds: failed to unmarshal DownstreamTlsContext in LDS response: %v", err)
+	}
+	if downstreamCtx.GetCommonTlsContext() == nil {
+		return nil, errors.New("xds: DownstreamTlsContext in LDS response does not contain a CommonTlsContext")
+	}
+	sc, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext())
+	if err != nil {
+		return nil, err
+	}
+	return &ListenerUpdate{SecurityCfg: sc}, nil
+}
+
+func getAddressFromName(name string) (host string, port string, err error) {
+	parts := strings.SplitN(name, "udpa.resource.listening_address=", 2)
+	if len(parts) != 2 {
+		return "", "", fmt.Errorf("udpa.resource_listening_address not found in name: %v", name)
+	}
+	return net.SplitHostPort(parts[1])
+}
+
 // UnmarshalRouteConfig processes resources received in an RDS response,
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
@@ -259,10 +344,6 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 	return routesRet, nil
 }
 
-// TransportSocket proto message has a `name` field which is expected to be set
-// to this value by the management server.
-const transportSocketName = "envoy.transport_sockets.tls"
-
 // UnmarshalCluster processes resources received in an CDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
@@ -337,13 +418,21 @@ func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, e
 	if err := proto.Unmarshal(any.GetValue(), upstreamCtx); err != nil {
 		return nil, fmt.Errorf("xds: failed to unmarshal UpstreamTlsContext in CDS response: %v", err)
 	}
+	if upstreamCtx.GetCommonTlsContext() == nil {
+		return nil, errors.New("xds: UpstreamTlsContext in CDS response does not contain a CommonTlsContext")
+	}
+
+	return securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext())
+}
 
-	// The `UpstreamTlsContext` has a `CommonTlsContext` which contains a
+// common is expected to be not nil.
+func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext) (*SecurityConfig, error) {
+	// The `CommonTlsContext` contains a
 	// `tls_certificate_certificate_provider_instance` field of type
 	// `CertificateProviderInstance`, which contains the provider instance name
 	// and the certificate name to fetch identity certs.
 	sc := &SecurityConfig{}
-	if identity := upstreamCtx.GetCommonTlsContext().GetTlsCertificateCertificateProviderInstance(); identity != nil {
+	if identity := common.GetTlsCertificateCertificateProviderInstance(); identity != nil {
 		sc.IdentityInstanceName = identity.GetInstanceName()
 		sc.IdentityCertName = identity.GetCertificateName()
 	}
@@ -357,9 +446,9 @@ func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, e
 	//    - contains certificate provider instance configuration
 	//  - certificate provider instance configuration
 	//    - in this case, we do not get a list of accepted SANs.
-	switch t := upstreamCtx.GetCommonTlsContext().GetValidationContextType().(type) {
+	switch t := common.GetValidationContextType().(type) {
 	case *v3tlspb.CommonTlsContext_CombinedValidationContext:
-		combined := upstreamCtx.GetCommonTlsContext().GetCombinedValidationContext()
+		combined := common.GetCombinedValidationContext()
 		if def := combined.GetDefaultValidationContext(); def != nil {
 			for _, matcher := range def.GetMatchSubjectAltNames() {
 				// We only support exact matches for now.
@@ -373,7 +462,7 @@ func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, e
 			sc.RootCertName = pi.GetCertificateName()
 		}
 	case *v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance:
-		pi := upstreamCtx.GetCommonTlsContext().GetValidationContextCertificateProviderInstance()
+		pi := common.GetValidationContextCertificateProviderInstance()
 		sc.RootInstanceName = pi.GetInstanceName()
 		sc.RootCertName = pi.GetCertificateName()
 	default:
diff --git a/xds/internal/version/version.go b/xds/internal/version/version.go
index 39344187d9e4..db1929b76b96 100644
--- a/xds/internal/version/version.go
+++ b/xds/internal/version/version.go
@@ -41,10 +41,11 @@ const (
 	V2EndpointsURL       = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
 	V2HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"
 
-	V3ListenerURL           = "type.googleapis.com/envoy.config.listener.v3.Listener"
-	V3RouteConfigURL        = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"
-	V3ClusterURL            = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
-	V3EndpointsURL          = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"
-	V3HTTPConnManagerURL    = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v3.HttpConnectionManager"
-	V3UpstreamTLSContextURL = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext"
+	V3ListenerURL             = "type.googleapis.com/envoy.config.listener.v3.Listener"
+	V3RouteConfigURL          = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"
+	V3ClusterURL              = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
+	V3EndpointsURL            = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"
+	V3HTTPConnManagerURL      = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v3.HttpConnectionManager"
+	V3UpstreamTLSContextURL   = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext"
+	V3DownstreamTLSContextURL = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext"
 )

From 2efef8fd1214261d09879e6c62d1f8a9bdd67fff Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Thu, 3 Dec 2020 09:52:30 -0800
Subject: [PATCH 304/481] advancedtls: fix default host name check issue
 (#4069)

* advancedtls: fix default hostname check issue
---
 security/advancedtls/advancedtls.go           |   8 +-
 security/advancedtls/advancedtls_test.go      |  23 +++-
 .../internal/testutils/testutils.go           |   7 +
 security/advancedtls/testdata/README.md       |   8 +-
 .../testdata/localhost-openssl.cnf            |  24 ++++
 security/advancedtls/testdata/openssl-ca.cnf  |  87 ++++++++++++
 .../testdata/server_cert_localhost_1.pem      | 124 ++++++++++++++++++
 .../testdata/server_key_localhost_1.pem       |  51 +++++++
 8 files changed, 326 insertions(+), 6 deletions(-)
 create mode 100644 security/advancedtls/testdata/localhost-openssl.cnf
 create mode 100644 security/advancedtls/testdata/openssl-ca.cnf
 create mode 100644 security/advancedtls/testdata/server_cert_localhost_1.pem
 create mode 100644 security/advancedtls/testdata/server_key_localhost_1.pem

diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go
index ea93d64069f5..534a3ed417ba 100644
--- a/security/advancedtls/advancedtls.go
+++ b/security/advancedtls/advancedtls.go
@@ -492,7 +492,13 @@ func buildVerifyFunc(c *advancedTLSCreds,
 			}
 			// Perform default hostname check if specified.
 			if c.isClient && c.vType == CertAndHostVerification && serverName != "" {
-				opts.DNSName = serverName
+				parsedName, _, err := net.SplitHostPort(serverName)
+				if err != nil {
+					// If the serverName had no host port or if the serverName cannot be
+					// parsed, use it as-is.
+					parsedName = serverName
+				}
+				opts.DNSName = parsedName
 			}
 			var err error
 			chains, err = certs[0].Verify(opts)
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 2d6ec47b7202..4cb4748c669d 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -377,8 +377,8 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: server side failure and client handshake failure
 		// Reason: client side sets vType to CertAndHostVerification, and will do
-		// default hostname check. All the default hostname checks will fail in
-		// this test suites.
+		// default hostname check. Server uses a cert without "localhost" or
+		// "127.0.0.1" as common name or SAN names, and will hence fail.
 		{
 			desc:                       "Client has root cert; server sends peer cert",
 			clientRoot:                 cs.ClientTrust1,
@@ -392,8 +392,8 @@ func (s) TestClientServerHandshake(t *testing.T) {
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: server side failure and client handshake failure
 		// Reason: client side sets vType to CertAndHostVerification, and will do
-		// default hostname check. All the default hostname checks will fail in
-		// this test suites.
+		// default hostname check. Server uses a cert without "localhost" or
+		// "127.0.0.1" as common name or SAN names, and will hence fail.
 		{
 			desc:                       "Client sets reload root function; server sends peer cert",
 			clientGetRoot:              getRootCAsForClient,
@@ -403,6 +403,21 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			serverVType:                CertAndHostVerification,
 			serverExpectError:          true,
 		},
+		// Client: only set clientGetRoot and CertAndHostVerification
+		// Server: only set serverCert with mutual TLS off
+		// Expected Behavior: success
+		// Reason: client side sets vType to CertAndHostVerification, and will do
+		// default hostname check. Server uses a certificate with "localhost" as
+		// common name, and will hence pass the default hostname check.
+		{
+			desc:                       "Client sets CertAndHostVerification; server uses localhost certs",
+			clientRoot:                 cs.ClientTrust1,
+			clientVType:                CertAndHostVerification,
+			clientExpectHandshakeError: false,
+			serverCert:                 []tls.Certificate{cs.ServerPeerLocalhost1},
+			serverVType:                CertAndHostVerification,
+			serverExpectError:          false,
+		},
 		// Client: set clientGetRoot and clientVerifyFunc
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: success
diff --git a/security/advancedtls/internal/testutils/testutils.go b/security/advancedtls/internal/testutils/testutils.go
index 665cc6026389..a2c048882b7a 100644
--- a/security/advancedtls/internal/testutils/testutils.go
+++ b/security/advancedtls/internal/testutils/testutils.go
@@ -43,6 +43,10 @@ type CertStore struct {
 	ServerCert2 tls.Certificate
 	// ServerPeer3 is the certificate sent by server to prove its identity.
 	ServerPeer3 tls.Certificate
+	// ServerPeerLocalhost1 is the certificate sent by server to prove its
+	// identity. It has "localhost" as its common name, and is trusted by
+	// ClientTrust1.
+	ServerPeerLocalhost1 tls.Certificate
 	// ClientTrust1 is the root certificate used on the client side.
 	ClientTrust1 *x509.CertPool
 	// ClientTrust2 is the root certificate used on the client side.
@@ -84,6 +88,9 @@ func (cs *CertStore) LoadCerts() error {
 	if cs.ServerPeer3, err = tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem")); err != nil {
 		return err
 	}
+	if cs.ServerPeerLocalhost1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_localhost_1.pem"), testdata.Path("server_key_localhost_1.pem")); err != nil {
+		return err
+	}
 	if cs.ClientTrust1, err = readTrustCert(testdata.Path("client_trust_cert_1.pem")); err != nil {
 		return err
 	}
diff --git a/security/advancedtls/testdata/README.md b/security/advancedtls/testdata/README.md
index 48a1c135c44a..3e4057ef3f1f 100644
--- a/security/advancedtls/testdata/README.md
+++ b/security/advancedtls/testdata/README.md
@@ -19,6 +19,11 @@ commands we run:
    ```
    $ openssl req -new -key subject_key.pem -out csr.pem
    ```
+   For some cases, we might want to add some extra SAN fields in `subject_cert.pem`.
+   In those cases, we can create a configuration file(for example, localhost-openssl.cnf), and do the following:
+   ```
+   $ openssl req -new -key subject_key.pem -out csr.pem -config $CONFIG_FILE_NAME
+   ```
 
 3. Generate a private key `subject_key.pem` for the subject: 
    
@@ -28,11 +33,12 @@ commands we run:
 
 4. Use `ca_key.pem` and `ca_cert.pem` to sign `csr.pem`, and get a certificate, `subject_cert.pem`, for the subject:
    
-   This step requires some additional files and please check out [this answer from StackOverflow](https://stackoverflow.com/a/21340898) for more.
+   This step requires some additional configuration steps and please check out [this answer from StackOverflow](https://stackoverflow.com/a/21340898) for more.
 
    ```
    $ openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out subject_cert.pem -in csr.pem -keyfile ca_key.pem -cert ca_cert.pem
    ```
+   Please see an example configuration template at `openssl-ca.cnf`.
 5. Verify the `subject_cert.pem` is trusted by `ca_cert.pem`:
    
 
diff --git a/security/advancedtls/testdata/localhost-openssl.cnf b/security/advancedtls/testdata/localhost-openssl.cnf
new file mode 100644
index 000000000000..b415cef6ae1c
--- /dev/null
+++ b/security/advancedtls/testdata/localhost-openssl.cnf
@@ -0,0 +1,24 @@
+[req]
+distinguished_name  = req_distinguished_name
+req_extensions     = v3_req
+
+[req_distinguished_name]
+countryName           = Country Name (2 letter code)
+countryName_default   = US
+stateOrProvinceName   = State or Province Name (full name)
+stateOrProvinceName_default = Illinois
+localityName          = Locality Name (eg, city)
+localityName_default  = Chicago
+organizationName          = Organization Name (eg, company)
+organizationName_default  = Example, Co.
+commonName            = Common Name (eg, YOUR name)
+commonName_max        = 64
+
+[v3_req]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+IP.1 = "127.0.0.1"
diff --git a/security/advancedtls/testdata/openssl-ca.cnf b/security/advancedtls/testdata/openssl-ca.cnf
new file mode 100644
index 000000000000..196a50c26471
--- /dev/null
+++ b/security/advancedtls/testdata/openssl-ca.cnf
@@ -0,0 +1,87 @@
+base_dir      = .
+certificate   = $base_dir/cacert.pem   # The CA certifcate
+private_key   = $base_dir/cakey.pem    # The CA private key
+new_certs_dir = $base_dir              # Location for new certs after signing
+database      = $base_dir/index.txt    # Database index file
+serial        = $base_dir/serial.txt   # The current serial number
+
+unique_subject = no  # Set to 'no' to allow creation of
+                     # several certificates with same subject.
+
+HOME            = .
+RANDFILE        = $ENV::HOME/.rnd
+
+####################################################################
+[ ca ]
+default_ca    = CA_default      # The default ca section
+
+[ CA_default ]
+
+default_days     = 10000         # How long to certify for
+default_crl_days = 30           # How long before next CRL
+default_md       = sha256       # Use public key default MD
+preserve         = no           # Keep passed DN ordering
+
+x509_extensions = ca_extensions # The extensions to add to the cert
+
+email_in_dn     = no            # Don't concat the email in the DN
+copy_extensions = copy          # Required to copy SANs from CSR to cert
+
+####################################################################
+[ req ]
+default_bits       = 4096
+default_keyfile    = cakey.pem
+distinguished_name = ca_distinguished_name
+x509_extensions    = ca_extensions
+string_mask        = utf8only
+
+####################################################################
+[ ca_distinguished_name ]
+countryName         = Country Name (2 letter code)
+countryName_default = US
+
+stateOrProvinceName         = State or Province Name (full name)
+stateOrProvinceName_default = Maryland
+
+localityName                = Locality Name (eg, city)
+localityName_default        = Baltimore
+
+organizationName            = Organization Name (eg, company)
+organizationName_default    = Test CA, Limited
+
+organizationalUnitName         = Organizational Unit (eg, division)
+organizationalUnitName_default = Server Research Department
+
+commonName         = Common Name (e.g. server FQDN or YOUR name)
+commonName_default = Test CA
+
+emailAddress         = Email Address
+emailAddress_default = test@example.com
+
+####################################################################
+[ ca_extensions ]
+
+subjectKeyIdentifier   = hash
+authorityKeyIdentifier = keyid:always, issuer
+basicConstraints       = critical, CA:true
+keyUsage               = keyCertSign, cRLSign
+
+
+
+
+####################################################################
+[ signing_policy ]
+countryName            = optional
+stateOrProvinceName    = optional
+localityName           = optional
+organizationName       = optional
+organizationalUnitName = optional
+commonName             = supplied
+emailAddress           = optional
+
+####################################################################
+[ signing_req ]
+subjectKeyIdentifier   = hash
+authorityKeyIdentifier = keyid,issuer
+basicConstraints       = CA:FALSE
+keyUsage               = digitalSignature, keyEncipherment
diff --git a/security/advancedtls/testdata/server_cert_localhost_1.pem b/security/advancedtls/testdata/server_cert_localhost_1.pem
new file mode 100644
index 000000000000..798656198f1d
--- /dev/null
+++ b/security/advancedtls/testdata/server_cert_localhost_1.pem
@@ -0,0 +1,124 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 3 (0x3)
+        Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C=US, ST=CA, L=SVL, O=Internet Widgits Pty Ltd
+        Validity
+            Not Before: Dec  2 21:51:53 2020 GMT
+            Not After : Apr 19 21:51:53 2048 GMT
+        Subject: C=US, ST=Illinois, L=Chicago, O=Example, Co., CN=localhost
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                RSA Public-Key: (4096 bit)
+                Modulus:
+                    00:a0:a1:5e:05:53:c5:a7:71:fb:ff:22:d5:d2:49:
+                    da:14:8b:54:9b:fb:40:3d:33:38:7d:3e:69:16:1c:
+                    1f:c1:88:42:99:d2:ac:8e:3e:60:ca:ac:63:e2:5b:
+                    e8:90:e5:4d:1b:e5:0b:40:6d:7d:90:6a:de:7f:ad:
+                    97:81:ae:38:2c:07:44:ca:ac:54:d7:dd:41:99:36:
+                    ca:38:44:9e:cc:e0:34:c2:44:71:2e:59:60:bf:45:
+                    e5:57:af:7d:d2:ca:c5:53:ff:8a:ab:5f:68:cf:74:
+                    88:29:99:59:fc:25:8c:06:47:26:77:14:1e:10:c1:
+                    78:d6:a1:a5:7f:0d:8d:4a:be:c8:fd:6f:a1:60:3d:
+                    e8:0a:27:f9:fe:e1:45:81:7c:e6:be:65:b8:62:ed:
+                    ca:1f:fb:7f:08:c0:b4:a6:57:f1:56:32:53:bf:d6:
+                    43:75:57:1c:e2:69:8d:96:41:a6:46:a0:49:1d:e8:
+                    ab:0a:92:7b:b8:33:b2:c3:7b:6e:2b:53:13:be:91:
+                    34:71:e3:d9:52:e4:d5:63:64:f8:a7:b0:a0:23:5c:
+                    07:3d:82:9a:6e:bc:0a:95:f1:51:ff:08:75:1c:32:
+                    38:67:9a:98:5c:94:08:90:c2:23:cd:bb:ad:86:10:
+                    4f:34:b2:28:f8:83:0f:c3:cb:3a:80:a9:2e:9a:3e:
+                    45:bf:2c:55:ee:22:d7:e4:a3:d7:b4:0b:7c:fb:30:
+                    fa:a7:02:24:04:ba:60:79:2d:1d:01:6c:b1:17:b9:
+                    b9:f3:4f:c7:20:26:6d:48:31:b2:6d:27:b8:5d:fc:
+                    8a:20:be:29:30:26:18:a4:51:45:3c:47:06:22:fa:
+                    9e:c7:43:ac:34:5b:6e:23:08:c9:ea:93:b7:de:5e:
+                    36:9a:85:51:8a:01:86:c0:47:52:4f:9b:d6:16:ce:
+                    ff:3a:5f:33:ff:f5:24:20:5d:ef:f9:ea:21:fd:05:
+                    04:b9:c6:b5:75:67:67:d5:48:70:d0:e8:53:f2:ff:
+                    f7:a0:8b:1c:09:bf:5d:08:b2:5d:08:34:00:a3:f1:
+                    fc:6d:44:e5:73:eb:6d:24:a2:0f:aa:4a:d1:8f:fe:
+                    40:a4:77:94:63:99:82:26:d0:66:a4:a2:97:59:2b:
+                    30:18:4b:d1:22:a9:99:02:4c:06:50:ff:12:6e:83:
+                    11:ff:08:7f:c2:8e:ae:83:0f:a9:b6:99:3e:cc:8e:
+                    c7:a6:17:65:65:0e:9f:7d:48:28:6d:e7:b6:e5:7a:
+                    b1:36:04:fe:db:0e:62:3a:e2:a1:bf:b9:76:23:bf:
+                    b8:4f:a4:68:bf:03:1c:53:34:55:d0:28:2d:89:10:
+                    28:2d:2c:ca:8b:b9:5a:82:f4:5f:b9:eb:ef:aa:f2:
+                    31:f0:d5
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Key Identifier: 
+                B2:87:7F:58:33:AD:24:C6:F2:07:5D:8E:88:18:BE:61:08:20:29:D1
+            X509v3 Authority Key Identifier: 
+                keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06
+
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            X509v3 Key Usage: 
+                Digital Signature, Key Encipherment
+            X509v3 Subject Alternative Name: 
+                DNS:localhost, IP Address:127.0.0.1
+    Signature Algorithm: sha256WithRSAEncryption
+         bd:79:29:a5:d1:35:3d:34:61:3e:88:d7:37:66:c0:48:2e:ba:
+         8f:ed:56:b6:48:3f:1b:7d:eb:b7:91:0f:13:c3:96:d3:bd:82:
+         0c:00:49:3f:1a:44:12:4c:27:f5:48:39:04:bd:66:d8:64:e1:
+         5f:c7:01:43:82:38:1e:7f:94:6c:96:00:97:06:b7:d6:d1:0d:
+         0a:34:a9:f1:3d:ac:0b:4f:0c:2e:ff:69:3d:53:63:9b:c1:22:
+         e8:5a:b9:cd:00:92:a0:b6:cd:c0:21:b2:fc:2e:70:8e:2d:e3:
+         ed:4a:f0:84:24:3f:7e:b9:a7:cf:05:1d:15:cc:a3:8a:6e:0d:
+         44:28:02:bf:03:31:02:00:eb:1d:6c:60:8e:9c:73:0a:b6:f7:
+         ad:61:c4:a5:60:dc:4e:44:79:3e:95:40:44:5a:9b:58:61:3d:
+         b6:5f:68:8f:c0:98:3d:a4:b1:41:fb:b4:ad:d3:b6:8b:72:f0:
+         c2:ec:8a:65:ac:a7:2e:ea:f7:e5:2c:8d:3a:18:36:d2:48:36:
+         5d:a2:1a:52:22:40:a6:da:7d:52:e5:ba:d7:fd:75:63:8b:9b:
+         ff:ad:20:bf:a0:f9:d4:e7:f1:b2:cb:a7:09:c7:d0:36:bc:ea:
+         be:f1:29:08:22:25:2a:16:3f:22:3b:2b:53:47:73:01:68:d2:
+         1e:3d:87:07:6f:ec:c4:33:3e:44:ec:a3:de:32:7e:12:57:15:
+         30:d4:6d:be:16:e7:ba:28:0e:8a:79:8a:ab:84:f8:1d:8c:d9:
+         da:06:1b:7e:37:10:3f:24:7c:1f:74:bc:0e:44:cf:a6:56:97:
+         02:19:b7:a7:f3:5d:46:c8:56:7b:19:b3:42:17:f2:97:cf:15:
+         d6:78:8d:90:f9:88:f7:28:71:7d:8d:ff:50:bb:d1:71:bb:23:
+         da:53:63:92:7c:d4:a8:be:45:17:d4:6a:e7:e3:75:93:b5:bb:
+         fd:38:30:11:ab:ec:f5:7b:6e:f5:17:0b:cc:69:44:79:3f:95:
+         85:e8:6b:da:e7:af:b3:f6:e8:73:22:db:75:f8:40:20:fd:cb:
+         24:b6:49:b6:69:79:12:8e:73:05:75:a0:de:67:5c:29:05:11:
+         3a:b2:8d:f8:ec:2f:fa:99:5e:3a:2b:df:3b:69:1a:c3:05:6c:
+         44:3c:a1:12:ab:3c:5b:87:52:a5:44:54:6c:c8:e5:f6:50:dd:
+         d7:d4:93:15:67:31:66:58:10:c1:23:98:35:08:6b:56:34:5a:
+         71:db:67:e5:41:a8:60:77:c3:30:2a:09:78:37:83:32:37:b1:
+         c5:34:b6:54:79:fd:e0:13:a3:16:f5:5f:25:87:8e:5e:c3:93:
+         f9:1f:4c:79:51:0f:23:e6
+-----BEGIN CERTIFICATE-----
+MIIFmTCCA4GgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBLMQswCQYDVQQGEwJVUzEL
+MAkGA1UECAwCQ0ExDDAKBgNVBAcMA1NWTDEhMB8GA1UECgwYSW50ZXJuZXQgV2lk
+Z2l0cyBQdHkgTHRkMB4XDTIwMTIwMjIxNTE1M1oXDTQ4MDQxOTIxNTE1M1owXTEL
+MAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdv
+MRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKChXgVTxadx+/8i1dJJ2hSLVJv7
+QD0zOH0+aRYcH8GIQpnSrI4+YMqsY+Jb6JDlTRvlC0BtfZBq3n+tl4GuOCwHRMqs
+VNfdQZk2yjhEnszgNMJEcS5ZYL9F5VevfdLKxVP/iqtfaM90iCmZWfwljAZHJncU
+HhDBeNahpX8NjUq+yP1voWA96Aon+f7hRYF85r5luGLtyh/7fwjAtKZX8VYyU7/W
+Q3VXHOJpjZZBpkagSR3oqwqSe7gzssN7bitTE76RNHHj2VLk1WNk+KewoCNcBz2C
+mm68CpXxUf8IdRwyOGeamFyUCJDCI827rYYQTzSyKPiDD8PLOoCpLpo+Rb8sVe4i
+1+Sj17QLfPsw+qcCJAS6YHktHQFssRe5ufNPxyAmbUgxsm0nuF38iiC+KTAmGKRR
+RTxHBiL6nsdDrDRbbiMIyeqTt95eNpqFUYoBhsBHUk+b1hbO/zpfM//1JCBd7/nq
+If0FBLnGtXVnZ9VIcNDoU/L/96CLHAm/XQiyXQg0AKPx/G1E5XPrbSSiD6pK0Y/+
+QKR3lGOZgibQZqSil1krMBhL0SKpmQJMBlD/Em6DEf8If8KOroMPqbaZPsyOx6YX
+ZWUOn31IKG3ntuV6sTYE/tsOYjriob+5diO/uE+kaL8DHFM0VdAoLYkQKC0syou5
+WoL0X7nr76ryMfDVAgMBAAGjdjB0MB0GA1UdDgQWBBSyh39YM60kxvIHXY6IGL5h
+CCAp0TAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7QnxBjAJBgNVHRMEAjAA
+MAsGA1UdDwQEAwIFoDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI
+hvcNAQELBQADggIBAL15KaXRNT00YT6I1zdmwEguuo/tVrZIPxt967eRDxPDltO9
+ggwAST8aRBJMJ/VIOQS9Zthk4V/HAUOCOB5/lGyWAJcGt9bRDQo0qfE9rAtPDC7/
+aT1TY5vBIuhauc0AkqC2zcAhsvwucI4t4+1K8IQkP365p88FHRXMo4puDUQoAr8D
+MQIA6x1sYI6ccwq2961hxKVg3E5EeT6VQERam1hhPbZfaI/AmD2ksUH7tK3Ttoty
+8MLsimWspy7q9+UsjToYNtJINl2iGlIiQKbafVLlutf9dWOLm/+tIL+g+dTn8bLL
+pwnH0Da86r7xKQgiJSoWPyI7K1NHcwFo0h49hwdv7MQzPkTso94yfhJXFTDUbb4W
+57ooDop5iquE+B2M2doGG343ED8kfB90vA5Ez6ZWlwIZt6fzXUbIVnsZs0IX8pfP
+FdZ4jZD5iPcocX2N/1C70XG7I9pTY5J81Ki+RRfUaufjdZO1u/04MBGr7PV7bvUX
+C8xpRHk/lYXoa9rnr7P26HMi23X4QCD9yyS2SbZpeRKOcwV1oN5nXCkFETqyjfjs
+L/qZXjor3ztpGsMFbEQ8oRKrPFuHUqVEVGzI5fZQ3dfUkxVnMWZYEMEjmDUIa1Y0
+WnHbZ+VBqGB3wzAqCXg3gzI3scU0tlR5/eAToxb1XyWHjl7Dk/kfTHlRDyPm
+-----END CERTIFICATE-----
diff --git a/security/advancedtls/testdata/server_key_localhost_1.pem b/security/advancedtls/testdata/server_key_localhost_1.pem
new file mode 100644
index 000000000000..17d9f981564b
--- /dev/null
+++ b/security/advancedtls/testdata/server_key_localhost_1.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAoKFeBVPFp3H7/yLV0knaFItUm/tAPTM4fT5pFhwfwYhCmdKs
+jj5gyqxj4lvokOVNG+ULQG19kGref62Xga44LAdEyqxU191BmTbKOESezOA0wkRx
+Lllgv0XlV6990srFU/+Kq19oz3SIKZlZ/CWMBkcmdxQeEMF41qGlfw2NSr7I/W+h
+YD3oCif5/uFFgXzmvmW4Yu3KH/t/CMC0plfxVjJTv9ZDdVcc4mmNlkGmRqBJHeir
+CpJ7uDOyw3tuK1MTvpE0cePZUuTVY2T4p7CgI1wHPYKabrwKlfFR/wh1HDI4Z5qY
+XJQIkMIjzbuthhBPNLIo+IMPw8s6gKkumj5FvyxV7iLX5KPXtAt8+zD6pwIkBLpg
+eS0dAWyxF7m580/HICZtSDGybSe4XfyKIL4pMCYYpFFFPEcGIvqex0OsNFtuIwjJ
+6pO33l42moVRigGGwEdST5vWFs7/Ol8z//UkIF3v+eoh/QUEuca1dWdn1Uhw0OhT
+8v/3oIscCb9dCLJdCDQAo/H8bUTlc+ttJKIPqkrRj/5ApHeUY5mCJtBmpKKXWSsw
+GEvRIqmZAkwGUP8SboMR/wh/wo6ugw+ptpk+zI7HphdlZQ6ffUgobee25XqxNgT+
+2w5iOuKhv7l2I7+4T6RovwMcUzRV0CgtiRAoLSzKi7lagvRfuevvqvIx8NUCAwEA
+AQKCAgBnlCagoMhPlTy95KSkmWK65K2Gd5mQ3TqL6Hay/yerEEaCEkua3bZkeo1e
+JY3uAS6b0jJTNUdGnOMkybdss/8cxQMi/cUn/VCTj7UOW5Fa4yiiLKgfDxtHu7aL
+uGoWRxK/e4TbxQY84BP9Xxmbckq8sZyoJJzOiTN2k324U/DMRgItCpKxELpT8jtO
+k8zSFsxj8gvYHyW7Qd1Es57JtOO2hXVjurJ9M9M4XIAkZ+jkme8MDkBc7OBCg3O+
+ghUkcsnElLWQyzAUN+Mx2KZO26InquwwSctzpGXfEmGhZr69k9SzWgjtibeMQOP9
+ggv+6v1oKYop1bmQs7fhxzZ517X47wKJMx740OcM2oXNVqHwGmRvQxX10I4iopej
+c23hj1HYjXVnX8q6md636Pe1CO33NQhoxGsIEeZYwCBRI7w2CYZoejaHMAoAAOyS
+eDmNLGyX2cO0lxY5ru996gzS2NzIcuPG+XkXdsaxMuYYp6GpjBxhs414UeJ1dLL5
+tviBmO9WI2votjMglcU2/cqv2yr4w1azIrSSu4vebuYVhMkAYcvauBc2A2egRUIF
+RP2zBl55ZefapuPqHJdHyv9JWngrl5xWEoaOTUt8crzvRKxrnkLSBJ3rcZwhnsyr
+ZjuxsuiJ7AA/E6vOcOrmGixhFIA3B9oUOwPqNJN9hB1Ld50ZAQKCAQEAy9+C+fHn
+JklMsmWMqfyVFS9sPOuRM0NS1cAQ6U5RJdreiyq3pUbghDhYxiRTHWkoLE3IEYKE
+mPoMKwhu06qIQSKl/G7QKHEY9gwDW9/9pqvDnDbH7xaCTmsgXIcMdu+YisNqlufx
+yy7Q+oLdqbqBLio+CqJcWOwcasTGPdQlR4bnAxCRguCki9MeQTEnkPOff2KIZQci
+2Lv2s2yrf+8Lw+mDB43/clMFcpjdIR4ezaf9QLkJ2mWsKDBHt7Hl+WgY0Y/r2F5G
+MTotqYEqXKsBNPqy6LprYb/bDDhYHsnRAzyfRpKZ+6nwgdy4VjKOs0jduIewznTW
+P1iTJudf0eC3UQKCAQEAybNoGLw9mzc7/UOtpXSEwQuztE7h4Pcm1+pHLVy/6X8S
+h6toLCl7NoaHF6BuFNi+8yGYo8NSAEvyIm0rXh2sKjZAAbl5ab4h7VxEgVpynBMO
+RR9FgeQXakEzzpxlXkVjKlsMOmO9WTOcUKwpgP0dKQt81ygHt9H4uxO93EpYRkQr
+AyD+VXLtCill23YiAf8cX1GGj2mxZzg4+rOJpERLDBxTlUn005g2e3zShZXiKGSO
+QuWwDomkBvdcKasSydiOLrdE4NzO7YXRgiY/AJGGC8935RgFQT7KMZQ1HRC8leTY
+opc/Joq5j4xQatBu1ewbbg0idwYcy+p5Ot/tXwYIRQKCAQBDuB2genrGW+ivBU5B
+FJZMsDDq13Cmr4EvYRn89Te9NENhxLG1o6JmKPVL87rr9QcUGE4RiuISklRCYw21
+H1sdD65E+GYKWO7qo7jl5rQxjbJvDD9DKp3kAG+CbJV2WEW6KgkY0TievhFKdPe+
+LiZEuGFdVOsJ2nvh9zTGStaLOMM5YGKFL6tYiqrtCq/S1SmwvYEC1ej8Rws+NCWP
+XE7zJ3iPpNoqFmuj0iT5oDCpLVjRC+W69rTFsKvR17TFMI+15HF5sG7uYR3TxQTW
+PTMsbu3IokuS75CKMZkLuQvFYHijj4S4dI1gBXnxn9+Iq/aCGghfu62C4yAV9xr7
+8wHRAoIBAGIpfRTkr/rVU827XUwzu9QTtN6gsU+CGRZlv0Q1anTh0gvTALzVZ1Cv
+AhoeitR8c9nx1M6GZWcdjvbwOHXybPKSOm5cbNlonixdhj2J3lNU9tHvGS3Q6xBc
+MTFxbegGTu+zJe1Y0zMRahbc4soS5VkvbQ9tPOxaNPoe7nzCddmknWZFbWH6r6AN
+a7P19zEPjihZjepH3v3EH/7q16bpUbjQJGF4f71my8Unh3FZ85oC7jVigV9h30FA
+q0rgJiGz0easbMoezFpOkRsNMAY/zIP88XW+Tfhl7ZNZdMvzdERi/oeKokJIq2xQ
+Nmb1j6tu4B6cJ9TTVbpsH5nmlyhy0B0CggEBAJPicCHPN828eVLp0kCOY3akPfLc
+M88O/3XN4rf8btM2wSJdKANcF88WYh6wmMR4RF26/GIn/LfVRz/y23hj+KGPW2Nz
+4anf4+iLqIoI0QuIHAzV6/WdukBhkjfXJlHESlNp9vsdDQymT+NookXfXVvQQxHm
+21x41hlvKe0Ldd7MWj/JNow4sLfiyGntsaCYZBL6SBBrL4D42AmXFiftLQ4Tx9wl
+YY/37p9hid5/PrF67Qk1jhQj1Z5cs2I+rsCo7FqDDnbbYDE69R/q9yTRDLBnEr93
+ce3EAO6RL5tKGQqc+m8gl/7eyJ5kriySBibaiZifJA4KFMwFCgwvxmsHJgA=
+-----END RSA PRIVATE KEY-----

From c7df457e12e0efe21a70b95bb554779020401255 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 4 Dec 2020 08:42:31 -0800
Subject: [PATCH 305/481] credentials/xds: Rename test file (#4077)

---
 credentials/xds/{xds_test.go => xds_client_test.go} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename credentials/xds/{xds_test.go => xds_client_test.go} (100%)

diff --git a/credentials/xds/xds_test.go b/credentials/xds/xds_client_test.go
similarity index 100%
rename from credentials/xds/xds_test.go
rename to credentials/xds/xds_client_test.go

From 9db56a08b4cdc76a6d1bc90919a69a0a192450a3 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Fri, 4 Dec 2020 15:47:27 -0800
Subject: [PATCH 306/481] advancedtls: add examples demonstrating reloading
 behaviors (#4018)

* advancedtls: add examples demonstrating reloading behaviors
---
 .../credential_reloading_from_files/README.md |   3 -
 .../client/main.go                            |  45 ++++---
 .../server/main.go                            |  10 +-
 .../advancedtls/examples/examples_test.sh     |  88 ++++++-------
 security/advancedtls/testdata/README.md       |  16 +--
 .../testdata/another_client_cert_1.pem        | 122 ++++++++++++++++++
 .../testdata/another_client_key_1.pem         |  51 ++++++++
 7 files changed, 257 insertions(+), 78 deletions(-)
 create mode 100644 security/advancedtls/testdata/another_client_cert_1.pem
 create mode 100644 security/advancedtls/testdata/another_client_key_1.pem

diff --git a/security/advancedtls/examples/credential_reloading_from_files/README.md b/security/advancedtls/examples/credential_reloading_from_files/README.md
index afb13d44cfb6..be9e06e76e4c 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/README.md
+++ b/security/advancedtls/examples/credential_reloading_from_files/README.md
@@ -7,9 +7,6 @@ This example demonstrates how to set the reloading fields in advancedtls API.
 Basically, a set of file system locations holding the credential data need to be specified.
 Once the credential data needs to be updated, users just change the credential data in the file system, and gRPC will pick up the changes automatically.
 
-This example only shows how to set the API, without demonstrating the way to reload credentials on file system. 
-To learn more about how to do that in Go, please see `advancedtls_integration_test.go`.
-
 A couple of things to note:
  1. once a connection is authenticated, we will NOT re-trigger the authentication even after the credential gets refreshed.
  2. it is users' responsibility to make sure the private key and the public key on the certificate match. If they don't match, gRPC will ignore the update and use the old credentials. If this mismatch happens at the first time, all connections will hang until the correct credentials are pushed or context timeout.  
diff --git a/security/advancedtls/examples/credential_reloading_from_files/client/main.go b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
index 6f88994cfc51..51dcb2a30ab9 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/client/main.go
+++ b/security/advancedtls/examples/credential_reloading_from_files/client/main.go
@@ -23,7 +23,6 @@ package main
 import (
 	"context"
 	"flag"
-	"fmt"
 	"log"
 	"time"
 
@@ -38,17 +37,27 @@ var address = "localhost:50051"
 
 const (
 	// Default timeout for normal connections.
-	defaultConnTimeout = 10 * time.Second
+	defaultTimeout = 2 * time.Second
 	// Intervals that set to monitor the credential updates.
-	credRefreshingInterval = 1 * time.Minute
+	credRefreshingInterval = 500 * time.Millisecond
 )
 
 func main() {
+	tmpKeyFile := flag.String("key", "", "temporary key file path")
+	tmpCertFile := flag.String("cert", "", "temporary cert file path")
 	flag.Parse()
 
+	if tmpKeyFile == nil || *tmpKeyFile == "" {
+		log.Fatalf("tmpKeyFile is nil or empty.")
+	}
+	if tmpCertFile == nil || *tmpCertFile == "" {
+		log.Fatalf("tmpCertFile is nil or empty.")
+	}
+
+	// Initialize credential struct using reloading API.
 	identityOptions := pemfile.Options{
-		CertFile:        testdata.Path("client_cert_1.pem"),
-		KeyFile:         testdata.Path("client_key_1.pem"),
+		CertFile:        *tmpCertFile,
+		KeyFile:         *tmpKeyFile,
 		RefreshDuration: credRefreshingInterval,
 	}
 	identityProvider, err := pemfile.NewProvider(identityOptions)
@@ -63,7 +72,6 @@ func main() {
 	if err != nil {
 		log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err)
 	}
-
 	options := &advancedtls.ClientOptions{
 		IdentityOptions: advancedtls.IdentityCertificateOptions{
 			IdentityProvider: identityProvider,
@@ -81,18 +89,23 @@ func main() {
 		log.Fatalf("advancedtls.NewClientCreds(%v) failed: %v", options, err)
 	}
 
-	// At initialization, the connection should be good.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultConnTimeout)
-	defer cancel()
-	conn, err := grpc.DialContext(ctx, address, grpc.WithTransportCredentials(clientTLSCreds))
+	// Make a connection using the credentials.
+	conn, err := grpc.Dial(address, grpc.WithTransportCredentials(clientTLSCreds))
 	if err != nil {
 		log.Fatalf("grpc.DialContext to %s failed: %v", address, err)
 	}
-	greetClient := pb.NewGreeterClient(conn)
-	reply, err := greetClient.SayHello(ctx, &pb.HelloRequest{Name: "gRPC"}, grpc.WaitForReady(true))
-	if err != nil {
-		log.Fatalf("greetClient.SayHello failed: %v", err)
+	client := pb.NewGreeterClient(conn)
+
+	// Send the requests every 0.5s. The credential is expected to be changed in
+	// the bash script. We don't cancel the context nor call conn.Close() here,
+	// since the bash script is expected to close the client goroutine.
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
+		_, err = client.SayHello(ctx, &pb.HelloRequest{Name: "gRPC"}, grpc.WaitForReady(true))
+		if err != nil {
+			log.Fatalf("client.SayHello failed: %v", err)
+		}
+		cancel()
+		time.Sleep(500 * time.Millisecond)
 	}
-	defer conn.Close()
-	fmt.Printf("Getting message from server: %s...\n", reply.Message)
 }
diff --git a/security/advancedtls/examples/credential_reloading_from_files/server/main.go b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
index 3c14757fcfcc..164ef093be74 100644
--- a/security/advancedtls/examples/credential_reloading_from_files/server/main.go
+++ b/security/advancedtls/examples/credential_reloading_from_files/server/main.go
@@ -30,6 +30,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
+	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/security/advancedtls"
 	"google.golang.org/grpc/security/advancedtls/testdata"
 
@@ -84,6 +85,8 @@ func main() {
 		},
 		RequireClientCert: true,
 		VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
+			// This message is to show the certificate under the hood is actually reloaded.
+			fmt.Printf("Client common name: %s.\n", params.Leaf.Subject.CommonName)
 			return &advancedtls.VerificationResults{}, nil
 		},
 		VType: advancedtls.CertVerification,
@@ -92,7 +95,12 @@ func main() {
 	if err != nil {
 		log.Fatalf("advancedtls.NewServerCreds(%v) failed: %v", options, err)
 	}
-	s := grpc.NewServer(grpc.Creds(serverTLSCreds))
+	s := grpc.NewServer(grpc.Creds(serverTLSCreds), grpc.KeepaliveParams(keepalive.ServerParameters{
+		// Set the max connection time to be 0.5 s to force the client to
+		// re-establish the connection, and hence re-invoke the verification
+		// callback.
+		MaxConnectionAge: 500 * time.Millisecond,
+	}))
 	lis, err := net.Listen("tcp", port)
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
diff --git a/security/advancedtls/examples/examples_test.sh b/security/advancedtls/examples/examples_test.sh
index 44dbadd5ab17..2b911523d8eb 100755
--- a/security/advancedtls/examples/examples_test.sh
+++ b/security/advancedtls/examples/examples_test.sh
@@ -32,6 +32,10 @@ clean () {
   echo "$(tput setaf 1) clean failed to kill tests $(tput sgr 0)"
   jobs
   pstree
+  rm ${CLIENT_LOG}
+  rm ${SERVER_LOG}
+  rm ${KEY_FILE_PATH}
+  rm ${CERT_FILE_PATH}
   exit 1
 }
 
@@ -49,78 +53,62 @@ EXAMPLES=(
     "credential_reloading_from_files"
 )
 
-declare -A EXPECTED_SERVER_OUTPUT=(
-    ["credential_reloading_from_files"]=""
-)
-
-declare -A EXPECTED_CLIENT_OUTPUT=(
-    ["credential_reloading_from_files"]="Getting message from server: Hello gRPC..."
-)
+declare -a EXPECTED_SERVER_OUTPUT=("Client common name: foo.bar.hoo.com" "Client common name: foo.bar.another.client.com")
 
 cd ./security/advancedtls/examples
 
 for example in ${EXAMPLES[@]}; do
     echo "$(tput setaf 4) testing: ${example} $(tput sgr 0)"
 
-    # Build server
+    KEY_FILE_PATH=$(mktemp)
+    cat ../testdata/client_key_1.pem > ${KEY_FILE_PATH}
+
+    CERT_FILE_PATH=$(mktemp)
+    cat ../testdata/client_cert_1.pem > ${CERT_FILE_PATH}
+
+    # Build server.
     if ! go build -o /dev/null ./${example}/*server/*.go; then
         fail "failed to build server"
     else
         pass "successfully built server"
     fi
 
-    # Build client
+    # Build client.
     if ! go build -o /dev/null ./${example}/*client/*.go; then
         fail "failed to build client"
     else
         pass "successfully built client"
     fi
 
-    # Start server
+    # Start server.
     SERVER_LOG="$(mktemp)"
     go run ./$example/*server/*.go &> $SERVER_LOG  &
 
+    # Run client binary.
     CLIENT_LOG="$(mktemp)"
-    if ! timeout 20 go run ${example}/*client/*.go &> $CLIENT_LOG; then
-        fail "client failed to communicate with server
-        got server log:
-        $(cat $SERVER_LOG)
-        got client log:
-        $(cat $CLIENT_LOG)
-        "
-    else
-        pass "client successfully communitcated with server"
-    fi
+    go run ${example}/*client/*.go -key=${KEY_FILE_PATH} -cert=${CERT_FILE_PATH} &> $CLIENT_LOG  &
+
+    # Wait for the client to send some requests using old credentials.
+    sleep 4s
+
+    # Switch to the new credentials.
+    cat ../testdata/another_client_key_1.pem > ${KEY_FILE_PATH}
+    cat ../testdata/another_client_cert_1.pem > ${CERT_FILE_PATH}
+
+    # Wait for the client to send some requests using new credentials.
+    sleep 4s
+
+    # Check server log for expected output.
+    for output in "${EXPECTED_SERVER_OUTPUT[@]}"; do
+      if ! grep -q "$output" $SERVER_LOG; then
+          fail "server log missing output: $output
+          got server log:
+          $(cat $SERVER_LOG)
+          "
+      else
+          pass "server log contains expected output: $output"
+      fi
+    done
 
-    # Check server log for expected output if expecting an
-    # output
-    if [ -n "${EXPECTED_SERVER_OUTPUT[$example]}" ]; then
-        if ! grep -q "${EXPECTED_SERVER_OUTPUT[$example]}" $SERVER_LOG; then
-            fail "server log missing output: ${EXPECTED_SERVER_OUTPUT[$example]}
-            got server log:
-            $(cat $SERVER_LOG)
-            got client log:
-            $(cat $CLIENT_LOG)
-            "
-        else
-            pass "server log contains expected output: ${EXPECTED_SERVER_OUTPUT[$example]}"
-        fi
-    fi
-
-    # Check client log for expected output if expecting an
-    # output
-    if [ -n "${EXPECTED_CLIENT_OUTPUT[$example]}" ]; then
-        if ! grep -q "${EXPECTED_CLIENT_OUTPUT[$example]}" $CLIENT_LOG; then
-            fail "client log missing output: ${EXPECTED_CLIENT_OUTPUT[$example]}
-            got server log:
-            $(cat $SERVER_LOG)
-            got client log:
-            $(cat $CLIENT_LOG)
-            "
-        else
-            pass "client log contains expected output: ${EXPECTED_CLIENT_OUTPUT[$example]}"
-        fi
-    fi
     clean
-    echo ""
 done
diff --git a/security/advancedtls/testdata/README.md b/security/advancedtls/testdata/README.md
index 3e4057ef3f1f..12a6c6b1c1f8 100644
--- a/security/advancedtls/testdata/README.md
+++ b/security/advancedtls/testdata/README.md
@@ -14,7 +14,13 @@ commands we run:
    $ openssl req -x509 -newkey rsa:4096 -keyout ca_key.pem -out ca_cert.pem -nodes -days $DURATION_DAYS
    ```
 
-2. Generate a CSR `csr.pem` using `subject_key.pem`:
+2. Generate a private key `subject_key.pem` for the subject: 
+      
+      ```
+      $ openssl genrsa -out subject_key.pem 4096
+      ```
+   
+3. Generate a CSR `csr.pem` using `subject_key.pem`:
 
    ```
    $ openssl req -new -key subject_key.pem -out csr.pem
@@ -25,12 +31,6 @@ commands we run:
    $ openssl req -new -key subject_key.pem -out csr.pem -config $CONFIG_FILE_NAME
    ```
 
-3. Generate a private key `subject_key.pem` for the subject: 
-   
-   ```
-   $ openssl genrsa -out subject_key.pem 4096
-   ```
-
 4. Use `ca_key.pem` and `ca_cert.pem` to sign `csr.pem`, and get a certificate, `subject_cert.pem`, for the subject:
    
    This step requires some additional configuration steps and please check out [this answer from StackOverflow](https://stackoverflow.com/a/21340898) for more.
@@ -45,4 +45,4 @@ commands we run:
    ```
    $ openssl verify -verbose -CAfile ca_cert.pem  subject_cert.pem
 
-   ```
+   ```
\ No newline at end of file
diff --git a/security/advancedtls/testdata/another_client_cert_1.pem b/security/advancedtls/testdata/another_client_cert_1.pem
new file mode 100644
index 000000000000..603f9d772c11
--- /dev/null
+++ b/security/advancedtls/testdata/another_client_cert_1.pem
@@ -0,0 +1,122 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 1 (0x1)
+        Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C=US, ST=VA, O=Internet Widgits Pty Ltd, CN=foo.bar.hoo.ca.com
+        Validity
+            Not Before: Nov  7 17:11:57 2020 GMT
+            Not After : Mar 25 17:11:57 2048 GMT
+        Subject: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.another.client.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                RSA Public-Key: (4096 bit)
+                Modulus:
+                    00:ec:00:a0:69:4b:11:ae:30:21:15:64:4d:c3:cb:
+                    94:67:58:44:9a:5e:ca:c3:82:75:eb:6f:8d:b7:33:
+                    ca:e0:69:5f:98:1f:17:e5:7d:04:dd:f9:af:63:3b:
+                    84:0e:48:2e:23:fc:b4:de:02:64:ad:0c:8f:6b:70:
+                    15:fe:00:98:47:cb:c3:ff:5c:fb:b4:22:cd:36:18:
+                    6e:4a:6d:de:0e:7a:90:d8:3d:af:76:1d:17:07:67:
+                    79:7f:ab:59:af:1f:37:99:b9:46:ea:db:b3:a4:a2:
+                    8a:e9:b3:41:83:04:71:82:2f:88:51:5d:b3:ba:7d:
+                    72:d3:ef:97:47:89:b4:d0:78:f1:d6:ef:8a:09:94:
+                    19:54:ae:08:8c:77:49:2f:6b:c6:c9:56:0c:fa:ec:
+                    77:76:f0:83:d8:83:d0:4d:b5:f5:d9:e3:12:85:5c:
+                    64:c5:82:60:73:20:fa:8a:36:00:f4:f2:bc:cf:48:
+                    08:94:5f:2b:39:88:4c:56:f5:65:30:67:41:78:99:
+                    7e:26:f5:ab:e7:3d:b0:a3:8c:55:c5:e1:12:39:23:
+                    00:68:88:c2:b1:43:2f:61:7c:d4:08:35:52:28:5d:
+                    93:3e:84:c9:8d:0a:37:df:75:06:f4:ae:1e:2a:1d:
+                    e3:f9:0f:26:80:ad:4e:6a:c3:6a:0c:2e:0f:31:0f:
+                    b2:39:b0:98:aa:1e:96:c7:0b:a3:70:6c:35:52:82:
+                    56:0f:27:e1:07:d5:89:a6:97:58:97:6f:9f:4f:db:
+                    0e:e6:ef:fd:62:f4:c3:d5:bb:af:ee:f7:5b:26:6d:
+                    79:c0:46:71:94:d1:6f:ea:2a:61:78:1c:c5:09:4d:
+                    93:35:bf:6d:1d:26:49:2e:84:9e:8f:48:9c:93:e0:
+                    ef:bb:c7:85:f1:20:2b:8e:1e:da:18:76:db:c6:42:
+                    4f:87:ed:e9:44:90:ab:99:ea:d7:3a:89:f9:be:b5:
+                    3a:b8:5b:50:ec:7c:54:ce:d9:ff:9e:94:30:97:25:
+                    e5:ce:13:b4:ee:06:56:39:0b:d4:51:77:a2:e8:3f:
+                    b4:e7:60:2a:03:44:3a:93:ed:8a:f0:d7:90:f7:92:
+                    fe:e9:19:57:37:24:20:f5:9c:64:d0:fc:13:d1:3e:
+                    a5:1d:ea:14:23:8d:f6:ee:c0:5c:f4:27:82:87:90:
+                    6c:80:6d:ff:6f:1d:ab:2e:83:0f:73:4a:78:3d:40:
+                    7f:e6:f9:55:b4:ea:e8:83:f7:86:e0:c8:c1:d9:96:
+                    a0:7d:38:08:72:88:d2:c5:90:e3:3e:2a:f9:64:26:
+                    52:6e:20:79:20:ea:99:1b:75:65:4b:12:93:22:09:
+                    f9:ec:1b:af:74:a4:3c:a3:df:07:d0:50:95:ad:7e:
+                    55:60:25
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Key Identifier: 
+                D5:43:51:8B:A8:4C:84:D0:C8:DE:29:14:1B:15:7A:62:01:ED:FF:EC
+            X509v3 Authority Key Identifier: 
+                keyid:B4:19:08:1C:FC:10:23:C5:30:86:22:BC:CB:B1:5F:AD:EA:7A:5D:F1
+
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            X509v3 Key Usage: 
+                Digital Signature, Key Encipherment
+    Signature Algorithm: sha256WithRSAEncryption
+         61:a0:89:19:3e:e8:3d:35:bf:6e:5d:0c:d0:ec:36:85:d4:27:
+         41:6b:21:0e:5e:ed:e9:e2:17:ef:2b:19:5d:1b:4a:52:7f:e5:
+         66:db:24:40:4d:cb:f8:9d:92:5e:10:67:68:d0:f8:56:ab:da:
+         3c:8c:4f:02:07:9e:22:49:45:de:bd:d2:92:e8:ef:9d:f2:68:
+         97:0d:10:43:c7:b7:73:9e:76:14:7b:1d:b5:8e:df:df:7b:0d:
+         b3:48:36:21:98:67:3f:41:55:c7:26:09:99:fc:bc:92:8c:9a:
+         ac:64:76:9d:b2:b1:ca:41:e5:59:79:8d:76:f1:b4:bb:2e:28:
+         8b:bb:73:c4:83:0e:f9:6e:62:2c:49:d8:00:e5:87:c7:53:1b:
+         45:ec:29:a4:1c:20:82:bf:d1:f4:a4:23:98:b2:1e:41:a0:ee:
+         1a:0c:7e:bd:00:84:f5:17:05:1e:a6:7b:fd:75:ee:b9:6d:6b:
+         31:2a:0d:97:fa:57:86:57:25:44:8c:5e:e9:bc:78:30:13:77:
+         70:87:78:e1:7a:fb:26:db:b9:95:d0:04:c9:93:26:a2:96:b5:
+         2b:d5:d3:61:22:b3:ca:31:51:15:6d:51:0a:fb:22:6e:ca:16:
+         61:a5:91:9c:e0:39:de:cb:ad:fe:23:51:15:34:42:ae:ac:82:
+         80:9a:12:f6:f4:a6:8b:65:b5:f3:21:7b:78:ab:e2:53:83:1c:
+         08:83:a5:1a:26:f4:a8:f2:cd:19:d1:85:99:99:45:c8:46:35:
+         48:c4:fa:44:80:b0:04:67:48:36:c9:5e:44:fa:41:6e:a7:f2:
+         eb:22:9e:f8:d3:f2:0d:79:1b:33:78:c4:d0:60:e0:93:5f:69:
+         6f:cf:f0:df:04:3d:5b:b3:ac:09:30:ae:32:ea:0f:9f:7b:2c:
+         69:bf:f3:fd:6d:7c:a2:25:dc:15:82:df:e0:a5:84:64:39:26:
+         43:28:0f:cb:2e:7f:fe:9e:c2:7e:20:d3:ca:6d:96:1c:0e:e1:
+         b1:ac:0d:2e:4d:97:6f:14:39:65:1a:65:9f:13:2d:0a:28:f3:
+         2e:6d:30:af:c6:82:cb:33:20:86:17:de:52:0c:9b:a1:fb:2a:
+         59:ca:b1:18:b7:56:06:52:46:9c:8d:36:c6:2e:61:f2:02:2b:
+         ec:72:15:40:3a:1f:e9:ad:ca:5e:19:aa:1a:73:b5:d4:4d:b9:
+         d5:b2:24:8c:b6:eb:e6:cc:b2:0e:12:c6:26:52:f7:7a:e7:2c:
+         0b:1f:bc:24:1c:76:66:bb:2f:1d:ec:26:9e:53:5b:12:0a:e4:
+         ae:e3:bc:dd:21:a0:aa:cc:e4:9a:1d:8e:f7:16:36:b2:ed:97:
+         81:5a:48:22:a8:f6:6a:2d
+-----BEGIN CERTIFICATE-----
+MIIFkTCCA3mgAwIBAgIBATANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJVUzEL
+MAkGA1UECAwCVkExITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEb
+MBkGA1UEAwwSZm9vLmJhci5ob28uY2EuY29tMB4XDTIwMTEwNzE3MTE1N1oXDTQ4
+MDMyNTE3MTE1N1owYjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSEwHwYDVQQK
+DBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxIzAhBgNVBAMMGmZvby5iYXIuYW5v
+dGhlci5jbGllbnQuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA
+7ACgaUsRrjAhFWRNw8uUZ1hEml7Kw4J162+NtzPK4GlfmB8X5X0E3fmvYzuEDkgu
+I/y03gJkrQyPa3AV/gCYR8vD/1z7tCLNNhhuSm3eDnqQ2D2vdh0XB2d5f6tZrx83
+mblG6tuzpKKK6bNBgwRxgi+IUV2zun1y0++XR4m00Hjx1u+KCZQZVK4IjHdJL2vG
+yVYM+ux3dvCD2IPQTbX12eMShVxkxYJgcyD6ijYA9PK8z0gIlF8rOYhMVvVlMGdB
+eJl+JvWr5z2wo4xVxeESOSMAaIjCsUMvYXzUCDVSKF2TPoTJjQo333UG9K4eKh3j
++Q8mgK1OasNqDC4PMQ+yObCYqh6WxwujcGw1UoJWDyfhB9WJppdYl2+fT9sO5u/9
+YvTD1buv7vdbJm15wEZxlNFv6ipheBzFCU2TNb9tHSZJLoSej0ick+Dvu8eF8SAr
+jh7aGHbbxkJPh+3pRJCrmerXOon5vrU6uFtQ7HxUztn/npQwlyXlzhO07gZWOQvU
+UXei6D+052AqA0Q6k+2K8NeQ95L+6RlXNyQg9Zxk0PwT0T6lHeoUI4327sBc9CeC
+h5BsgG3/bx2rLoMPc0p4PUB/5vlVtOrog/eG4MjB2ZagfTgIcojSxZDjPir5ZCZS
+biB5IOqZG3VlSxKTIgn57BuvdKQ8o98H0FCVrX5VYCUCAwEAAaNaMFgwHQYDVR0O
+BBYEFNVDUYuoTITQyN4pFBsVemIB7f/sMB8GA1UdIwQYMBaAFLQZCBz8ECPFMIYi
+vMuxX63qel3xMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMA0GCSqGSIb3DQEBCwUA
+A4ICAQBhoIkZPug9Nb9uXQzQ7DaF1CdBayEOXu3p4hfvKxldG0pSf+Vm2yRATcv4
+nZJeEGdo0PhWq9o8jE8CB54iSUXevdKS6O+d8miXDRBDx7dznnYUex21jt/few2z
+SDYhmGc/QVXHJgmZ/LySjJqsZHadsrHKQeVZeY128bS7LiiLu3PEgw75bmIsSdgA
+5YfHUxtF7CmkHCCCv9H0pCOYsh5BoO4aDH69AIT1FwUepnv9de65bWsxKg2X+leG
+VyVEjF7pvHgwE3dwh3jhevsm27mV0ATJkyailrUr1dNhIrPKMVEVbVEK+yJuyhZh
+pZGc4Dney63+I1EVNEKurIKAmhL29KaLZbXzIXt4q+JTgxwIg6UaJvSo8s0Z0YWZ
+mUXIRjVIxPpEgLAEZ0g2yV5E+kFup/LrIp740/INeRszeMTQYOCTX2lvz/DfBD1b
+s6wJMK4y6g+feyxpv/P9bXyiJdwVgt/gpYRkOSZDKA/LLn/+nsJ+INPKbZYcDuGx
+rA0uTZdvFDllGmWfEy0KKPMubTCvxoLLMyCGF95SDJuh+ypZyrEYt1YGUkacjTbG
+LmHyAivschVAOh/prcpeGaoac7XUTbnVsiSMtuvmzLIOEsYmUvd65ywLH7wkHHZm
+uy8d7CaeU1sSCuSu47zdIaCqzOSaHY73Fjay7ZeBWkgiqPZqLQ==
+-----END CERTIFICATE-----
diff --git a/security/advancedtls/testdata/another_client_key_1.pem b/security/advancedtls/testdata/another_client_key_1.pem
new file mode 100644
index 000000000000..ad61ccfeabea
--- /dev/null
+++ b/security/advancedtls/testdata/another_client_key_1.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEA7ACgaUsRrjAhFWRNw8uUZ1hEml7Kw4J162+NtzPK4GlfmB8X
+5X0E3fmvYzuEDkguI/y03gJkrQyPa3AV/gCYR8vD/1z7tCLNNhhuSm3eDnqQ2D2v
+dh0XB2d5f6tZrx83mblG6tuzpKKK6bNBgwRxgi+IUV2zun1y0++XR4m00Hjx1u+K
+CZQZVK4IjHdJL2vGyVYM+ux3dvCD2IPQTbX12eMShVxkxYJgcyD6ijYA9PK8z0gI
+lF8rOYhMVvVlMGdBeJl+JvWr5z2wo4xVxeESOSMAaIjCsUMvYXzUCDVSKF2TPoTJ
+jQo333UG9K4eKh3j+Q8mgK1OasNqDC4PMQ+yObCYqh6WxwujcGw1UoJWDyfhB9WJ
+ppdYl2+fT9sO5u/9YvTD1buv7vdbJm15wEZxlNFv6ipheBzFCU2TNb9tHSZJLoSe
+j0ick+Dvu8eF8SArjh7aGHbbxkJPh+3pRJCrmerXOon5vrU6uFtQ7HxUztn/npQw
+lyXlzhO07gZWOQvUUXei6D+052AqA0Q6k+2K8NeQ95L+6RlXNyQg9Zxk0PwT0T6l
+HeoUI4327sBc9CeCh5BsgG3/bx2rLoMPc0p4PUB/5vlVtOrog/eG4MjB2ZagfTgI
+cojSxZDjPir5ZCZSbiB5IOqZG3VlSxKTIgn57BuvdKQ8o98H0FCVrX5VYCUCAwEA
+AQKCAgAzIaOfjHMlMSpJzzSGAjqB9X7Pj1AQ8dgIjV+/3InM+yeJ9tqfjumaCjm0
+nzVqPrs4cszg+NXFJF6CYYNyR8C2dXBeiE/EZHHfkYV7vLgKnQV6xEqapYzSvtl1
+DrPcnD/Yn2q9AaK3PbwpC/xanYDWOuQm9M02z20se9Fj33L8Y+fJsJZQovSmAxq5
+DDMgAhLMlkczqj3r2ApIw65C1/SPI4JkwHLY0/l/mBqQDUlByMGdizbIpqHf0ibw
+BDTLOuPVdDP/zuRSsmvt0z7WI4BmPq4c99xuuWavkXMC4EKPmk6Hkg907kzSrjE2
+m+7PIzC8SksGQAYoXXRBdU03TPZI3PaHb/Lyb6TCVK4gmhAF+QsSobOkFsM72QpF
+JBdcRgNsy/fZLFsPFH+g4zCRLrRcPrP1ifiqXOrDJ128JD5PqrEBMyXKQMkUB6qq
++0Hdm2kPWhs5tM4XY7X3xIQJ8AmZ/VPSHwecIUIu4SwdBlaDfQAZgTFER0vzweS5
+PPuJXp/bLW/gOa7hG7bKwmMTHR9ge+i/aMN1JEY6t2pY8AGH9RiH0WRcgiIC6eev
+YTmpb8dd4GhatqueKN48UpzCK6CRmvLeBINAyOLybV4RM9cQb6g2sWTc0sgQR4tL
+QMB0XAug6QOD7pXFa36JCCrkDDDojjzPaOLCCVtY03si6Yps1QKCAQEA9sKM7nr1
+QoZjo1C7KAu35l1vHVZoXPXm4z8UVNzHqRs+Bd8cut1nNm/kDE5ur5ceGkbSqGs2
+ktqGF/nZYVpUJBKyIXVO4wamUeSFdojllUkXJKOyT+qav1J9/FtOUaONRuc8u452
+HkBclijh4fRkkuM2SROvM7+5YGhQFR/CZN/RXenOD+7wPWa3/3cmV7fZff3dQjDb
+7K7vzq/BQVsbfjVSzd3Zwuo0teFz0AEWWt2LE455v/gYP9laIRWxGbIxpRgIyu2e
+M/CwRRQT1SOJ9dwGGYls2yTHTfD2hYJnw0fXXo7YyMntMhRazKvQllZF0CdpHZO2
+OBk7UxwNwnMLnwKCAQEA9Nb0RUAR2gDoDQSWoMrRZ3cXCbA1efzP82A/TcppB1En
+qqPwuoDd+nTH6KuXaZJYzbGSADyHWzMUaokWz28wC/2ffOhIxSdbU5su/20QfuIN
+6rb9BZv6NGjh35nH8ftfyx5tW26iV7mcveig2ixQLg62xTHB8p0FJM3QRIflZ3OK
+FWCC/+vZVyiAtGL6xS3XKPHukSeD5zOFKcF3GB9KdSbwED7bdxQ06HGfoWOgCmYb
+FnlUWaUZl/wIePYxUSjAmGLkctwH2x83MiRcvPLGI/lW3xzZFHxNj+K9/aJPAbyY
+JvZmBeezXphzrU3GjStzrwwd9SQpM4TDkDMIGBk9uwKCAQEAg/KcMZmGNEBwXw/4
+Q/2gJIqps+JUhADpqXI9iPNVwFNU4wbe8f0aB73lD7+Q6EvCSQK9+lj6IaTAN2ne
+l3QZsgBdSA7WVAdmQDwWMcAaI62ltm3iF2G3xb5yp9KbGoR+Mv/LNe+DscFwwMqz
+noN0lCbzDDh+YwmOMsMUr3cAF7im17UB/vshc3PNx8kKs7UXk4uAGLjPoMwaZ0cL
+68qv9NjGolaS7usVrHwV1Y//SC9XAuoYqFIdhWbQDwjuXnMuoL0tVnWhNtzpJMcL
+o9kRGGrCyDz3/Ga6PC8xY0rL+VwdCe8QdK2lLDY+J1toejs/sYKhbrNhqLW1R0el
+A+lIuQKCAQAWkoKup7t9l7vNB3FDna80lLwg/ofPmUkqrOLpLxIDxK2dg8O7zgmo
+/382qispZn6daBOHxgzMkab+M2lQ8nVBhb5ga6HZ20kGKjZpAgsVR430563oCHtG
+vaylSq4uVvh753A5j7eT0t7qeznpI1C5Dk43W+D/lw5UWE0tJEI4CWTfl6g8I+hD
+qs5C0yU/bHx7n+JYq4XzmMJcGSP7q1bX+iEDvmfJUKmYDHGlFWQ50TQKHGF0ak4z
+vt6hGEFvtAwdgHCDTlnDD9us2cFbAh7WTjR+GVDCHLuh2kudyIr0JAj6/phlTvkw
+bWmsvpDhjvH5X2qboRvTThghgTLr1dflAoIBAQDn4g3+8Aqf7LTR5EbU64BU7j5R
+1CddpJDBafV+lmhkO5oFPFXh72tJu+P1zIRBhCwWqBGCZZQK0LUzezrenyGJH+dk
+ZlEr3axKfiv9WKRe1AB6lJSxtOgPy9Kzb5pMFEXipqFQ70i49MjRjJ6WuMRMznug
+q/dkVq7q7K+4Dv+/PDHyXols5LsMTNs8bypS0wCubklAk12NxRdeHu4JXJPjtdyS
+dQvQj3oheh4km5Q7EqxPnMRvxg2FK7RNwvZ/Q70dCXb8jh7u4Ty4sq5tK4RGyAkL
+Zfbng79N0J/+kvRFczaVIvCwSbfexPx/BtGsA1HnfAb2ADug8RlhEwHM9E8G
+-----END RSA PRIVATE KEY-----

From 0d6a24f68a5f9a38c64be00364587becb0e40518 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 4 Dec 2020 15:56:07 -0800
Subject: [PATCH 307/481] test: move stubServer to separate package in internal
 (#4081)

---
 internal/stubserver/stubserver.go | 165 ++++++++++++++++++++++
 test/authority_test.go            |  23 +--
 test/balancer_test.go             |  13 +-
 test/channelz_test.go             |   3 +-
 test/context_canceled_test.go     |  13 +-
 test/end2end_test.go              | 226 ++++++++----------------------
 test/goaway_test.go               |   5 +-
 test/gracefulstop_test.go         |   5 +-
 test/insecure_creds_test.go       |   9 +-
 test/local_creds_test.go          |   9 +-
 test/resolver_test.go             |  15 +-
 test/retry_test.go                |  41 +++---
 test/server_test.go               |  23 +--
 test/stream_cleanup_test.go       |  19 +--
 14 files changed, 316 insertions(+), 253 deletions(-)
 create mode 100644 internal/stubserver/stubserver.go

diff --git a/internal/stubserver/stubserver.go b/internal/stubserver/stubserver.go
new file mode 100644
index 000000000000..c97010dfe9a6
--- /dev/null
+++ b/internal/stubserver/stubserver.go
@@ -0,0 +1,165 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package stubserver is a stubbable implementation of
+// google.golang.org/grpc/test/grpc_testing for testing purposes.
+package stubserver
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
+
+// StubServer is a server that is easy to customize within individual test
+// cases.
+type StubServer struct {
+	// Guarantees we satisfy this interface; panics if unimplemented methods are called.
+	testpb.TestServiceServer
+
+	// Customizable implementations of server handlers.
+	EmptyCallF      func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error)
+	UnaryCallF      func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
+	FullDuplexCallF func(stream testpb.TestService_FullDuplexCallServer) error
+
+	// A client connected to this service the test may use.  Created in Start().
+	Client testpb.TestServiceClient
+	CC     *grpc.ClientConn
+	S      *grpc.Server
+
+	// Parameters for Listen and Dial. Defaults will be used if these are empty
+	// before Start.
+	Network string
+	Address string
+	Target  string
+
+	cleanups []func() // Lambdas executed in Stop(); populated by Start().
+
+	// Set automatically if Target == ""
+	R *manual.Resolver
+}
+
+// EmptyCall is the handler for testpb.EmptyCall
+func (ss *StubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	return ss.EmptyCallF(ctx, in)
+}
+
+// UnaryCall is the handler for testpb.UnaryCall
+func (ss *StubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	return ss.UnaryCallF(ctx, in)
+}
+
+// FullDuplexCall is the handler for testpb.FullDuplexCall
+func (ss *StubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+	return ss.FullDuplexCallF(stream)
+}
+
+// Start starts the server and creates a client connected to it.
+func (ss *StubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error {
+	if ss.Network == "" {
+		ss.Network = "tcp"
+	}
+	if ss.Address == "" {
+		ss.Address = "localhost:0"
+	}
+	if ss.Target == "" {
+		ss.R = manual.NewBuilderWithScheme("whatever")
+	}
+
+	lis, err := net.Listen(ss.Network, ss.Address)
+	if err != nil {
+		return fmt.Errorf("net.Listen(%q, %q) = %v", ss.Network, ss.Address, err)
+	}
+	ss.Address = lis.Addr().String()
+	ss.cleanups = append(ss.cleanups, func() { lis.Close() })
+
+	s := grpc.NewServer(sopts...)
+	testpb.RegisterTestServiceServer(s, ss)
+	go s.Serve(lis)
+	ss.cleanups = append(ss.cleanups, s.Stop)
+	ss.S = s
+
+	opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...)
+	if ss.R != nil {
+		ss.Target = ss.R.Scheme() + ":///" + ss.Address
+		opts = append(opts, grpc.WithResolvers(ss.R))
+	}
+
+	cc, err := grpc.Dial(ss.Target, opts...)
+	if err != nil {
+		return fmt.Errorf("grpc.Dial(%q) = %v", ss.Target, err)
+	}
+	ss.CC = cc
+	if ss.R != nil {
+		ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}})
+	}
+	if err := waitForReady(cc); err != nil {
+		return err
+	}
+
+	ss.cleanups = append(ss.cleanups, func() { cc.Close() })
+
+	ss.Client = testpb.NewTestServiceClient(cc)
+	return nil
+}
+
+// NewServiceConfig applies sc to ss.Client using the resolver (if present).
+func (ss *StubServer) NewServiceConfig(sc string) {
+	if ss.R != nil {
+		ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}, ServiceConfig: parseCfg(ss.R, sc)})
+	}
+}
+
+func waitForReady(cc *grpc.ClientConn) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	for {
+		s := cc.GetState()
+		if s == connectivity.Ready {
+			return nil
+		}
+		if !cc.WaitForStateChange(ctx, s) {
+			// ctx got timeout or canceled.
+			return ctx.Err()
+		}
+	}
+}
+
+// Stop stops ss and cleans up all resources it consumed.
+func (ss *StubServer) Stop() {
+	for i := len(ss.cleanups) - 1; i >= 0; i-- {
+		ss.cleanups[i]()
+	}
+}
+
+func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult {
+	g := r.CC.ParseServiceConfig(s)
+	if g.Err != nil {
+		panic(fmt.Sprintf("Error parsing config %q: %v", s, g.Err))
+	}
+	return g
+}
diff --git a/test/authority_test.go b/test/authority_test.go
index e6599b2fde04..f537ee450a42 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -29,6 +29,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
@@ -56,13 +57,13 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer
 	if err := os.RemoveAll(address); err != nil {
 		t.Fatalf("Error removing socket file %v: %v\n", address, err)
 	}
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			return authorityChecker(ctx, expectedAuthority)
 		},
-		network: "unix",
-		address: address,
-		target:  target,
+		Network: "unix",
+		Address: address,
+		Target:  target,
 	}
 	opts := []grpc.DialOption{}
 	if dialer != nil {
@@ -74,7 +75,7 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer
 	defer ss.Stop()
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	_, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+	_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 	if err != nil {
 		t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err)
 	}
@@ -152,19 +153,19 @@ func (s) TestUnixCustomDialer(t *testing.T) {
 func (s) TestColonPortAuthority(t *testing.T) {
 	expectedAuthority := ""
 	var authorityMu sync.Mutex
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			authorityMu.Lock()
 			defer authorityMu.Unlock()
 			return authorityChecker(ctx, expectedAuthority)
 		},
-		network: "tcp",
+		Network: "tcp",
 	}
 	if err := ss.Start(nil); err != nil {
 		t.Fatalf("Error starting endpoint server: %v", err)
 	}
 	defer ss.Stop()
-	_, port, err := net.SplitHostPort(ss.address)
+	_, port, err := net.SplitHostPort(ss.Address)
 	if err != nil {
 		t.Fatalf("Failed splitting host from post: %v", err)
 	}
@@ -180,7 +181,7 @@ func (s) TestColonPortAuthority(t *testing.T) {
 		return (&net.Dialer{}).DialContext(ctx, "tcp", "localhost"+addr)
 	}))
 	if err != nil {
-		t.Fatalf("grpc.Dial(%q) = %v", ss.target, err)
+		t.Fatalf("grpc.Dial(%q) = %v", ss.Target, err)
 	}
 	defer cc.Close()
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
diff --git a/test/balancer_test.go b/test/balancer_test.go
index 7af5c81a0011..bc22036dbac3 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -40,6 +40,7 @@ import (
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpcutil"
 	imetadata "google.golang.org/grpc/internal/metadata"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
@@ -301,8 +302,8 @@ func testDoneLoads(t *testing.T, e env) {
 
 	const testLoad = "test-load-,-should-be-orca"
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			grpc.SetTrailer(ctx, metadata.Pairs(loadMDKey, testLoad))
 			return &testpb.Empty{}, nil
 		},
@@ -312,7 +313,7 @@ func testDoneLoads(t *testing.T, e env) {
 	}
 	defer ss.Stop()
 
-	tc := testpb.NewTestServiceClient(ss.cc)
+	tc := testpb.NewTestServiceClient(ss.CC)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -579,8 +580,8 @@ func (s) TestMetadataInAddressAttributes(t *testing.T) {
 	t.Logf("Registered balancer %s...", mdBalancerName)
 
 	testMDChan := make(chan []string, 1)
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			md, ok := metadata.FromIncomingContext(ctx)
 			if ok {
 				select {
@@ -602,7 +603,7 @@ func (s) TestMetadataInAddressAttributes(t *testing.T) {
 	// The RPC should succeed with the expected md.
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("EmptyCall() = _, %v, want _, <nil>", err)
 	}
 	t.Log("Made an RPC which succeeded...")
diff --git a/test/channelz_test.go b/test/channelz_test.go
index 7c074961d771..47e7eb927169 100644
--- a/test/channelz_test.go
+++ b/test/channelz_test.go
@@ -38,6 +38,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
@@ -969,7 +970,7 @@ func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *t
 	// Avoid overflowing connection level flow control window, which will lead to
 	// transport being closed.
 	te.serverInitialConnWindowSize = 65536 * 2
-	ts := &stubServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ts := &stubserver.StubServer{FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 		stream.Send(&testpb.StreamingOutputCallResponse{})
 		<-stream.Context().Done()
 		return status.Errorf(codes.DeadlineExceeded, "deadline exceeded or cancelled")
diff --git a/test/context_canceled_test.go b/test/context_canceled_test.go
index 781f63f0c04e..96ee69d8d521 100644
--- a/test/context_canceled_test.go
+++ b/test/context_canceled_test.go
@@ -26,14 +26,15 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/encoding/gzip"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
 func (s) TestContextCanceled(t *testing.T) {
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			stream.SetTrailer(metadata.New(map[string]string{"a": "b"}))
 			return status.Error(codes.PermissionDenied, "perm denied")
 		},
@@ -51,7 +52,7 @@ func (s) TestContextCanceled(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), delay)
 			defer cancel()
 
-			str, err := ss.client.FullDuplexCall(ctx)
+			str, err := ss.Client.FullDuplexCall(ctx)
 			if err != nil {
 				continue
 			}
@@ -121,8 +122,8 @@ func (s) TestContextCanceled(t *testing.T) {
 // first one, but `case ctx.Done()` wins the second one, the compression info
 // will be inconsistent, and it causes internal error.
 func (s) TestCancelWhileRecvingWithCompression(t *testing.T) {
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			for {
 				if err := stream.Send(&testpb.StreamingOutputCallResponse{
 					Payload: nil,
@@ -139,7 +140,7 @@ func (s) TestCancelWhileRecvingWithCompression(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-		s, err := ss.client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name))
+		s, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name))
 		if err != nil {
 			t.Fatalf("failed to start bidi streaming RPC: %v", err)
 		}
diff --git a/test/end2end_test.go b/test/end2end_test.go
index a3850d20874d..902e94241048 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -58,6 +58,7 @@ import (
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/metadata"
@@ -5129,127 +5130,12 @@ func (fw *filterWriter) Write(p []byte) (n int, err error) {
 	return fw.dst.Write(p)
 }
 
-// stubServer is a server that is easy to customize within individual test
-// cases.
-type stubServer struct {
-	// Guarantees we satisfy this interface; panics if unimplemented methods are called.
-	testpb.TestServiceServer
-
-	// Customizable implementations of server handlers.
-	emptyCall      func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error)
-	unaryCall      func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
-	fullDuplexCall func(stream testpb.TestService_FullDuplexCallServer) error
-
-	// A client connected to this service the test may use.  Created in Start().
-	client testpb.TestServiceClient
-	cc     *grpc.ClientConn
-	s      *grpc.Server
-
-	// Parameters for Listen and Dial. Defaults will be used if these are empty
-	// before Start.
-	network string
-	address string
-	target  string
-
-	cleanups []func() // Lambdas executed in Stop(); populated by Start().
-
-	r *manual.Resolver
-}
-
-func (ss *stubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-	return ss.emptyCall(ctx, in)
-}
-
-func (ss *stubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
-	return ss.unaryCall(ctx, in)
-}
-
-func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
-	return ss.fullDuplexCall(stream)
-}
-
-// Start starts the server and creates a client connected to it.
-func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error {
-	if ss.network == "" {
-		ss.network = "tcp"
-	}
-	if ss.address == "" {
-		ss.address = "localhost:0"
-	}
-	if ss.target == "" {
-		ss.r = manual.NewBuilderWithScheme("whatever")
-	}
-
-	lis, err := net.Listen(ss.network, ss.address)
-	if err != nil {
-		return fmt.Errorf("net.Listen(%q, %q) = %v", ss.network, ss.address, err)
-	}
-	ss.address = lis.Addr().String()
-	ss.cleanups = append(ss.cleanups, func() { lis.Close() })
-
-	s := grpc.NewServer(sopts...)
-	testpb.RegisterTestServiceServer(s, ss)
-	go s.Serve(lis)
-	ss.cleanups = append(ss.cleanups, s.Stop)
-	ss.s = s
-
-	opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...)
-	if ss.r != nil {
-		ss.target = ss.r.Scheme() + ":///" + ss.address
-		opts = append(opts, grpc.WithResolvers(ss.r))
-	}
-
-	cc, err := grpc.Dial(ss.target, opts...)
-	if err != nil {
-		return fmt.Errorf("grpc.Dial(%q) = %v", ss.target, err)
-	}
-	ss.cc = cc
-	if ss.r != nil {
-		ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}})
-	}
-	if err := ss.waitForReady(cc); err != nil {
-		return err
-	}
-
-	ss.cleanups = append(ss.cleanups, func() { cc.Close() })
-
-	ss.client = testpb.NewTestServiceClient(cc)
-	return nil
-}
-
-func (ss *stubServer) newServiceConfig(sc string) {
-	if ss.r != nil {
-		ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}, ServiceConfig: parseCfg(ss.r, sc)})
-	}
-}
-
-func (ss *stubServer) waitForReady(cc *grpc.ClientConn) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	for {
-		s := cc.GetState()
-		if s == connectivity.Ready {
-			return nil
-		}
-		if !cc.WaitForStateChange(ctx, s) {
-			// ctx got timeout or canceled.
-			return ctx.Err()
-		}
-	}
-}
-
-func (ss *stubServer) Stop() {
-	for i := len(ss.cleanups) - 1; i >= 0; i-- {
-		ss.cleanups[i]()
-	}
-}
-
 func (s) TestGRPCMethod(t *testing.T) {
 	var method string
 	var ok bool
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			method, ok = grpc.Method(ctx)
 			return &testpb.Empty{}, nil
 		},
@@ -5262,8 +5148,8 @@ func (s) TestGRPCMethod(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 
-	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
-		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
 	}
 
 	if want := "/grpc.testing.TestService/EmptyCall"; !ok || method != want {
@@ -5275,8 +5161,8 @@ func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) {
 	const mdkey = "somedata"
 
 	// endpoint ensures mdkey is NOT in metadata and returns an error if it is.
-	endpoint := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	endpoint := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] != nil {
 				return nil, status.Errorf(codes.Internal, "endpoint: md=%v; want !contains(%q)", md, mdkey)
 			}
@@ -5290,12 +5176,12 @@ func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) {
 
 	// proxy ensures mdkey IS in metadata, then forwards the RPC to endpoint
 	// without explicitly copying the metadata.
-	proxy := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	proxy := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] == nil {
 				return nil, status.Errorf(codes.Internal, "proxy: md=%v; want contains(%q)", md, mdkey)
 			}
-			return endpoint.client.EmptyCall(ctx, in)
+			return endpoint.Client.EmptyCall(ctx, in)
 		},
 	}
 	if err := proxy.Start(nil); err != nil {
@@ -5309,12 +5195,12 @@ func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) {
 	ctx = metadata.NewOutgoingContext(ctx, md)
 
 	// Sanity check that endpoint properly errors when it sees mdkey.
-	_, err := endpoint.client.EmptyCall(ctx, &testpb.Empty{})
+	_, err := endpoint.Client.EmptyCall(ctx, &testpb.Empty{})
 	if s, ok := status.FromError(err); !ok || s.Code() != codes.Internal {
-		t.Fatalf("endpoint.client.EmptyCall(_, _) = _, %v; want _, <status with Code()=Internal>", err)
+		t.Fatalf("endpoint.Client.EmptyCall(_, _) = _, %v; want _, <status with Code()=Internal>", err)
 	}
 
-	if _, err := proxy.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+	if _, err := proxy.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatal(err.Error())
 	}
 }
@@ -5337,8 +5223,8 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) {
 	}
 
 	// endpoint ensures mdkey is NOT in metadata and returns an error if it is.
-	endpoint := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	endpoint := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			ctx := stream.Context()
 			if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] != nil {
 				return status.Errorf(codes.Internal, "endpoint: md=%v; want !contains(%q)", md, mdkey)
@@ -5353,13 +5239,13 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) {
 
 	// proxy ensures mdkey IS in metadata, then forwards the RPC to endpoint
 	// without explicitly copying the metadata.
-	proxy := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	proxy := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			ctx := stream.Context()
 			if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] == nil {
 				return status.Errorf(codes.Internal, "endpoint: md=%v; want !contains(%q)", md, mdkey)
 			}
-			return doFDC(ctx, endpoint.client)
+			return doFDC(ctx, endpoint.Client)
 		},
 	}
 	if err := proxy.Start(nil); err != nil {
@@ -5373,13 +5259,13 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) {
 	ctx = metadata.NewOutgoingContext(ctx, md)
 
 	// Sanity check that endpoint properly errors when it sees mdkey in ctx.
-	err := doFDC(ctx, endpoint.client)
+	err := doFDC(ctx, endpoint.Client)
 	if s, ok := status.FromError(err); !ok || s.Code() != codes.Internal {
 		t.Fatalf("stream.Recv() = _, %v; want _, <status with Code()=Internal>", err)
 	}
 
-	if err := doFDC(ctx, proxy.client); err != nil {
-		t.Fatalf("doFDC(_, proxy.client) = %v; want nil", err)
+	if err := doFDC(ctx, proxy.Client); err != nil {
+		t.Fatalf("doFDC(_, proxy.Client) = %v; want nil", err)
 	}
 }
 
@@ -5390,8 +5276,8 @@ func (s) TestStatsTagsAndTrace(t *testing.T) {
 
 	// endpoint ensures Tags() and Trace() in context match those that were added
 	// by the client and returns an error if not.
-	endpoint := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	endpoint := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			md, _ := metadata.FromIncomingContext(ctx)
 			if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, tags) {
 				return nil, status.Errorf(codes.Internal, "stats.Tags(%v)=%v; want %v", ctx, tg, tags)
@@ -5428,12 +5314,12 @@ func (s) TestStatsTagsAndTrace(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		_, err := endpoint.client.EmptyCall(tc.ctx, &testpb.Empty{})
+		_, err := endpoint.Client.EmptyCall(tc.ctx, &testpb.Empty{})
 		if tc.want == codes.OK && err != nil {
-			t.Fatalf("endpoint.client.EmptyCall(%v, _) = _, %v; want _, nil", tc.ctx, err)
+			t.Fatalf("endpoint.Client.EmptyCall(%v, _) = _, %v; want _, nil", tc.ctx, err)
 		}
 		if s, ok := status.FromError(err); !ok || s.Code() != tc.want {
-			t.Fatalf("endpoint.client.EmptyCall(%v, _) = _, %v; want _, <status with Code()=%v>", tc.ctx, err, tc.want)
+			t.Fatalf("endpoint.Client.EmptyCall(%v, _) = _, %v; want _, <status with Code()=%v>", tc.ctx, err, tc.want)
 		}
 	}
 }
@@ -5450,8 +5336,8 @@ func (s) TestTapTimeout(t *testing.T) {
 		}),
 	}
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			<-ctx.Done()
 			return nil, status.Errorf(codes.Canceled, ctx.Err().Error())
 		},
@@ -5465,18 +5351,18 @@ func (s) TestTapTimeout(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		// Set our own deadline in case the server hangs.
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-		res, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+		res, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 		cancel()
 		if s, ok := status.FromError(err); !ok || s.Code() != codes.Canceled {
-			t.Fatalf("ss.client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=Canceled>", res, err)
+			t.Fatalf("ss.Client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=Canceled>", res, err)
 		}
 	}
 
 }
 
 func (s) TestClientWriteFailsAfterServerClosesStream(t *testing.T) {
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			return status.Errorf(codes.Internal, "")
 		},
 	}
@@ -5487,7 +5373,7 @@ func (s) TestClientWriteFailsAfterServerClosesStream(t *testing.T) {
 	defer ss.Stop()
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	stream, err := ss.client.FullDuplexCall(ctx)
+	stream, err := ss.Client.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("Error while creating stream: %v", err)
 	}
@@ -6337,8 +6223,8 @@ func testCompressorRegister(t *testing.T, e env) {
 }
 
 func (s) TestServeExitsWhenListenerClosed(t *testing.T) {
-	ss := &stubServer{
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
 	}
@@ -6390,8 +6276,8 @@ func (s) TestStatusInvalidUTF8Message(t *testing.T) {
 		wantMsg = "���"
 	)
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			return nil, status.Errorf(codes.Internal, origMsg)
 		},
 	}
@@ -6403,8 +6289,8 @@ func (s) TestStatusInvalidUTF8Message(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 
-	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMsg {
-		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, status.Convert(err).Message(), wantMsg)
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMsg {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, status.Convert(err).Message(), wantMsg)
 	}
 }
 
@@ -6419,8 +6305,8 @@ func (s) TestStatusInvalidUTF8Details(t *testing.T) {
 		wantMsg = "���"
 	)
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			st := status.New(codes.Internal, origMsg)
 			st, err := st.WithDetails(&testpb.Empty{})
 			if err != nil {
@@ -6437,10 +6323,10 @@ func (s) TestStatusInvalidUTF8Details(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 
-	_, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+	_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 	st := status.Convert(err)
 	if st.Message() != wantMsg {
-		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, st.Message(), wantMsg)
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, st.Message(), wantMsg)
 	}
 	if len(st.Details()) != 0 {
 		// Details should be dropped on the server side.
@@ -6539,8 +6425,8 @@ func (s) TestDisabledIOBuffers(t *testing.T) {
 		Payload: payload,
 	}
 
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			for {
 				in, err := stream.Recv()
 				if err == io.EOF {
@@ -7220,8 +7106,8 @@ func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult {
 func (s) TestClientCancellationPropagatesUnary(t *testing.T) {
 	wg := &sync.WaitGroup{}
 	called, done := make(chan struct{}), make(chan struct{})
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			close(called)
 			<-ctx.Done()
 			err := ctx.Err()
@@ -7241,8 +7127,8 @@ func (s) TestClientCancellationPropagatesUnary(t *testing.T) {
 
 	wg.Add(1)
 	go func() {
-		if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Canceled {
-			t.Errorf("ss.client.EmptyCall() = _, %v; want _, Code()=codes.Canceled", err)
+		if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Canceled {
+			t.Errorf("ss.Client.EmptyCall() = _, %v; want _, Code()=codes.Canceled", err)
 		}
 		wg.Done()
 	}()
@@ -7283,8 +7169,8 @@ func (badGzipCompressor) Type() string {
 }
 
 func (s) TestGzipBadChecksum(t *testing.T) {
-	ss := &stubServer{
-		unaryCall: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{}, nil
 		},
 	}
@@ -7300,18 +7186,18 @@ func (s) TestGzipBadChecksum(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unexpected error from newPayload: %v", err)
 	}
-	if _, err := ss.client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: p}); err == nil ||
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: p}); err == nil ||
 		status.Code(err) != codes.Internal ||
 		!strings.Contains(status.Convert(err).Message(), gzip.ErrChecksum.Error()) {
-		t.Errorf("ss.client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum)
+		t.Errorf("ss.Client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum)
 	}
 }
 
 // When an RPC is canceled, it's possible that the last Recv() returns before
 // all call options' after are executed.
 func (s) TestCanceledRPCCallOptionRace(t *testing.T) {
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			err := stream.Send(&testpb.StreamingOutputCallResponse{})
 			if err != nil {
 				return err
@@ -7337,7 +7223,7 @@ func (s) TestCanceledRPCCallOptionRace(t *testing.T) {
 			var p peer.Peer
 			ctx, cancel := context.WithCancel(ctx)
 			defer cancel()
-			stream, err := ss.client.FullDuplexCall(ctx, grpc.Peer(&p))
+			stream, err := ss.Client.FullDuplexCall(ctx, grpc.Peer(&p))
 			if err != nil {
 				t.Errorf("_.FullDuplexCall(_) = _, %v", err)
 				return
diff --git a/test/goaway_test.go b/test/goaway_test.go
index 55f79ebc8548..6ef11e26419d 100644
--- a/test/goaway_test.go
+++ b/test/goaway_test.go
@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/keepalive"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
@@ -40,8 +41,8 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) {
 	const maxConnAge = 100 * time.Millisecond
 	const testTime = maxConnAge * 10
 
-	ss := &stubServer{
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
 	}
diff --git a/test/gracefulstop_test.go b/test/gracefulstop_test.go
index 3da75ea1b51b..6058fb8b333c 100644
--- a/test/gracefulstop_test.go
+++ b/test/gracefulstop_test.go
@@ -28,6 +28,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
@@ -107,8 +108,8 @@ func (s) TestGracefulStop(t *testing.T) {
 	}
 	d := func(ctx context.Context, _ string) (net.Conn, error) { return dlis.Dial(ctx) }
 
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			_, err := stream.Recv()
 			if err != nil {
 				return err
diff --git a/test/insecure_creds_test.go b/test/insecure_creds_test.go
index dd56a8c46ee5..19f8bb8b791b 100644
--- a/test/insecure_creds_test.go
+++ b/test/insecure_creds_test.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 
@@ -84,8 +85,8 @@ func (s) TestInsecureCreds(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			ss := &stubServer{
-				emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 					if !test.serverInsecureCreds {
 						return &testpb.Empty{}, nil
 					}
@@ -167,8 +168,8 @@ func (s) TestInsecureCredsWithPerRPCCredentials(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			ss := &stubServer{
-				emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 					return &testpb.Empty{}, nil
 				},
 			}
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index b9115f0d5ac8..3933bb39635b 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/local"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 
@@ -37,8 +38,8 @@ import (
 )
 
 func testLocalCredsE2ESucceed(network, address string) error {
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			pr, ok := peer.FromContext(ctx)
 			if !ok {
 				return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
@@ -159,8 +160,8 @@ func spoofDialer(addr net.Addr) func(target string, t time.Duration) (net.Conn,
 }
 
 func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
-	ss := &stubServer{
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
 	}
diff --git a/test/resolver_test.go b/test/resolver_test.go
index 6f50047edadc..ab154f0ea29a 100644
--- a/test/resolver_test.go
+++ b/test/resolver_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
@@ -45,13 +46,13 @@ func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) *iresolver.RPCConf
 func (s) TestConfigSelector(t *testing.T) {
 	gotContextChan := testutils.NewChannelWithSize(1)
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			gotContextChan.SendContext(ctx, ctx)
 			return &testpb.Empty{}, nil
 		},
 	}
-	ss.r = manual.NewBuilderWithScheme("confSel")
+	ss.R = manual.NewBuilderWithScheme("confSel")
 
 	if err := ss.Start(nil); err != nil {
 		t.Fatalf("Error starting endpoint server: %v", err)
@@ -134,8 +135,8 @@ func (s) TestConfigSelector(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			var gotInfo *iresolver.RPCInfo
 			state := iresolver.SetConfigSelector(resolver.State{
-				Addresses:     []resolver.Address{{Addr: ss.address}},
-				ServiceConfig: parseCfg(ss.r, "{}"),
+				Addresses:     []resolver.Address{{Addr: ss.Address}},
+				ServiceConfig: parseCfg(ss.R, "{}"),
 			}, funcConfigSelector{
 				f: func(i iresolver.RPCInfo) *iresolver.RPCConfig {
 					gotInfo = &i
@@ -146,12 +147,12 @@ func (s) TestConfigSelector(t *testing.T) {
 					return cfg
 				},
 			})
-			ss.r.UpdateState(state) // Blocks until config selector is applied
+			ss.R.UpdateState(state) // Blocks until config selector is applied
 
 			onCommittedCalled = false
 			ctx := metadata.NewOutgoingContext(ctx, tc.md)
 			startTime := time.Now()
-			if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, nil", err)
 			}
 
diff --git a/test/retry_test.go b/test/retry_test.go
index f0ab380e96bc..f93c9ac053f7 100644
--- a/test/retry_test.go
+++ b/test/retry_test.go
@@ -32,6 +32,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
@@ -46,8 +47,8 @@ func enableRetry() func() {
 func (s) TestRetryUnary(t *testing.T) {
 	defer enableRetry()()
 	i := -1
-	ss := &stubServer{
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			i++
 			switch i {
 			case 0, 2, 5:
@@ -62,7 +63,7 @@ func (s) TestRetryUnary(t *testing.T) {
 		t.Fatalf("Error starting endpoint server: %v", err)
 	}
 	defer ss.Stop()
-	ss.newServiceConfig(`{
+	ss.NewServiceConfig(`{
     "methodConfig": [{
       "name": [{"service": "grpc.testing.TestService"}],
       "waitForReady": true,
@@ -79,7 +80,7 @@ func (s) TestRetryUnary(t *testing.T) {
 		if ctx.Err() != nil {
 			t.Fatalf("Timed out waiting for service config update")
 		}
-		if ss.cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil {
+		if ss.CC.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil {
 			break
 		}
 		time.Sleep(time.Millisecond)
@@ -100,7 +101,7 @@ func (s) TestRetryUnary(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		_, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+		_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 		cancel()
 		if status.Code(err) != tc.code {
 			t.Fatalf("EmptyCall(_, _) = _, %v; want _, <Code() = %v>", err, tc.code)
@@ -116,8 +117,8 @@ func (s) TestRetryDisabledByDefault(t *testing.T) {
 		return
 	}
 	i := -1
-	ss := &stubServer{
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			i++
 			switch i {
 			case 0:
@@ -130,7 +131,7 @@ func (s) TestRetryDisabledByDefault(t *testing.T) {
 		t.Fatalf("Error starting endpoint server: %v", err)
 	}
 	defer ss.Stop()
-	ss.newServiceConfig(`{
+	ss.NewServiceConfig(`{
     "methodConfig": [{
       "name": [{"service": "grpc.testing.TestService"}],
       "waitForReady": true,
@@ -147,7 +148,7 @@ func (s) TestRetryDisabledByDefault(t *testing.T) {
 		if ctx.Err() != nil {
 			t.Fatalf("Timed out waiting for service config update")
 		}
-		if ss.cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil {
+		if ss.CC.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil {
 			break
 		}
 		time.Sleep(time.Millisecond)
@@ -162,7 +163,7 @@ func (s) TestRetryDisabledByDefault(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		_, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+		_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 		cancel()
 		if status.Code(err) != tc.code {
 			t.Fatalf("EmptyCall(_, _) = _, %v; want _, <Code() = %v>", err, tc.code)
@@ -176,8 +177,8 @@ func (s) TestRetryDisabledByDefault(t *testing.T) {
 func (s) TestRetryThrottling(t *testing.T) {
 	defer enableRetry()()
 	i := -1
-	ss := &stubServer{
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			i++
 			switch i {
 			case 0, 3, 6, 10, 11, 12, 13, 14, 16, 18:
@@ -190,7 +191,7 @@ func (s) TestRetryThrottling(t *testing.T) {
 		t.Fatalf("Error starting endpoint server: %v", err)
 	}
 	defer ss.Stop()
-	ss.newServiceConfig(`{
+	ss.NewServiceConfig(`{
     "methodConfig": [{
       "name": [{"service": "grpc.testing.TestService"}],
       "waitForReady": true,
@@ -212,7 +213,7 @@ func (s) TestRetryThrottling(t *testing.T) {
 		if ctx.Err() != nil {
 			t.Fatalf("Timed out waiting for service config update")
 		}
-		if ss.cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil {
+		if ss.CC.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil {
 			break
 		}
 		time.Sleep(time.Millisecond)
@@ -238,7 +239,7 @@ func (s) TestRetryThrottling(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		_, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+		_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 		cancel()
 		if status.Code(err) != tc.code {
 			t.Errorf("EmptyCall(_, _) = _, %v; want _, <Code() = %v>", err, tc.code)
@@ -485,8 +486,8 @@ func (s) TestRetryStreaming(t *testing.T) {
 
 	var serverOpIter int
 	var serverOps []serverOp
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			for serverOpIter < len(serverOps) {
 				op := serverOps[serverOpIter]
 				serverOpIter++
@@ -501,7 +502,7 @@ func (s) TestRetryStreaming(t *testing.T) {
 		t.Fatalf("Error starting endpoint server: %v", err)
 	}
 	defer ss.Stop()
-	ss.newServiceConfig(`{
+	ss.NewServiceConfig(`{
     "methodConfig": [{
       "name": [{"service": "grpc.testing.TestService"}],
       "waitForReady": true,
@@ -518,7 +519,7 @@ func (s) TestRetryStreaming(t *testing.T) {
 		if ctx.Err() != nil {
 			t.Fatalf("Timed out waiting for service config update")
 		}
-		if ss.cc.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").WaitForReady != nil {
+		if ss.CC.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").WaitForReady != nil {
 			break
 		}
 		time.Sleep(time.Millisecond)
@@ -532,7 +533,7 @@ func (s) TestRetryStreaming(t *testing.T) {
 
 			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 			defer cancel()
-			stream, err := ss.client.FullDuplexCall(ctx)
+			stream, err := ss.Client.FullDuplexCall(ctx)
 			if err != nil {
 				t.Fatalf("%v: Error while creating stream: %v", tc.desc, err)
 			}
diff --git a/test/server_test.go b/test/server_test.go
index 41466157a19f..97f352328873 100644
--- a/test/server_test.go
+++ b/test/server_test.go
@@ -25,6 +25,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
@@ -118,8 +119,8 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) {
 		grpc.ChainUnaryInterceptor(firstInt, secondInt, lastInt),
 	}
 
-	ss := &stubServer{
-		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 0)
 			if err != nil {
 				return nil, status.Errorf(codes.Aborted, "failed to make payload: %v", err)
@@ -137,9 +138,9 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	resp, err := ss.client.UnaryCall(ctx, &testpb.SimpleRequest{})
+	resp, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{})
 	if s, ok := status.FromError(err); !ok || s.Code() != codes.OK {
-		t.Fatalf("ss.client.UnaryCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
+		t.Fatalf("ss.Client.UnaryCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
 	}
 
 	respBytes := resp.Payload.GetBody()
@@ -173,8 +174,8 @@ func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) {
 		grpc.ChainUnaryInterceptor(chainInt),
 	}
 
-	ss := &stubServer{
-		emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
 	}
@@ -185,9 +186,9 @@ func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	resp, err := ss.client.EmptyCall(ctx, &testpb.Empty{})
+	resp, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
 	if s, ok := status.FromError(err); !ok || s.Code() != codes.OK {
-		t.Fatalf("ss.client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
+		t.Fatalf("ss.Client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
 	}
 }
 
@@ -249,8 +250,8 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) {
 		grpc.ChainStreamInterceptor(firstInt, secondInt, lastInt),
 	}
 
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			if callCounts[0] != 1 {
 				return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0])
 			}
@@ -274,7 +275,7 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	stream, err := ss.client.FullDuplexCall(ctx)
+	stream, err := ss.Client.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("failed to FullDuplexCall: %v", err)
 	}
diff --git a/test/stream_cleanup_test.go b/test/stream_cleanup_test.go
index 77d9477cf17e..83dd68549e99 100644
--- a/test/stream_cleanup_test.go
+++ b/test/stream_cleanup_test.go
@@ -26,6 +26,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
@@ -35,13 +36,13 @@ func (s) TestStreamCleanup(t *testing.T) {
 	const bodySize = 2 * initialWindowSize   // Something that is not going to fit in a single window
 	const callRecvMsgSize uint = 1           // The maximum message size the client can receive
 
-	ss := &stubServer{
-		unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{Payload: &testpb.Payload{
 				Body: make([]byte, bodySize),
 			}}, nil
 		},
-		emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
 	}
@@ -52,10 +53,10 @@ func (s) TestStreamCleanup(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	if _, err := ss.client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted {
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted {
 		t.Fatalf("should fail with ResourceExhausted, message's body size: %v, maximum message size the client can receive: %v", bodySize, callRecvMsgSize)
 	}
-	if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
 		t.Fatalf("should succeed, err: %v", err)
 	}
 }
@@ -66,8 +67,8 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) {
 
 	serverReturnedStatus := make(chan struct{})
 
-	ss := &stubServer{
-		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
 			defer func() {
 				close(serverReturnedStatus)
 			}()
@@ -90,7 +91,7 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) {
 	// empty.
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
-	stream, err := ss.client.FullDuplexCall(ctx)
+	stream, err := ss.Client.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("FullDuplexCall= _, %v; want _, <nil>", err)
 	}
@@ -115,7 +116,7 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) {
 	gracefulStopDone := make(chan struct{})
 	go func() {
 		defer close(gracefulStopDone)
-		ss.s.GracefulStop()
+		ss.S.GracefulStop()
 	}()
 
 	// 4. Make sure the stream is not broken.

From bce1fdfad97e977caa8d0ff9fb7b42dbeddecd28 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 7 Dec 2020 15:10:33 -0800
Subject: [PATCH 308/481] eds/lrs: handle nil when LRS is disabled (#4086)

---
 .../balancer/edsbalancer/eds_impl_test.go     | 27 +++++++++++++++++++
 .../edsbalancer/load_store_wrapper.go         | 16 ++++++++---
 xds/internal/balancer/lrs/balancer.go         | 16 ++++++++---
 3 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index fde00d225ee4..d2a45655efc1 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -743,3 +743,30 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 		t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
 	}
 }
+
+// TestEDS_LoadReportDisabled covers the case that LRS is disabled. It makes
+// sure the EDS implementation isn't broken (doesn't panic).
+func (s) TestEDS_LoadReportDisabled(t *testing.T) {
+	lsWrapper := &loadStoreWrapper{}
+	lsWrapper.updateServiceName(testClusterNames[0])
+	// Not calling lsWrapper.updateLoadStore(loadStore) because LRS is disabled.
+
+	cc := testutils.NewTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, nil, lsWrapper, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+
+	// One localities, with one backend.
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
+	sc1 := <-cc.NewSubConnCh
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
+
+	// Test roundrobin with two subconns.
+	p1 := <-cc.NewPickerCh
+	// We call picks to make sure they don't panic.
+	for i := 0; i < 10; i++ {
+		p1.Pick(balancer.PickInfo{})
+	}
+}
diff --git a/xds/internal/balancer/edsbalancer/load_store_wrapper.go b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
index d47a6f6a069c..18904e47a42e 100644
--- a/xds/internal/balancer/edsbalancer/load_store_wrapper.go
+++ b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
@@ -58,23 +58,31 @@ func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
 func (lsw *loadStoreWrapper) CallStarted(locality string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallStarted(locality)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallStarted(locality)
+	}
 }
 
 func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallFinished(locality, err)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallFinished(locality, err)
+	}
 }
 
 func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallServerLoad(locality, name, val)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallServerLoad(locality, name, val)
+	}
 }
 
 func (lsw *loadStoreWrapper) CallDropped(category string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallDropped(category)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallDropped(category)
+	}
 }
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index 7bf672b81389..01bb14662865 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -195,25 +195,33 @@ func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
 func (lsw *loadStoreWrapper) CallStarted(locality string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallStarted(locality)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallStarted(locality)
+	}
 }
 
 func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallFinished(locality, err)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallFinished(locality, err)
+	}
 }
 
 func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallServerLoad(locality, name, val)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallServerLoad(locality, name, val)
+	}
 }
 
 func (lsw *loadStoreWrapper) CallDropped(category string) {
 	lsw.mu.RLock()
 	defer lsw.mu.RUnlock()
-	lsw.perCluster.CallDropped(category)
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallDropped(category)
+	}
 }
 
 type xdsClientWrapper struct {

From 96a834a12f8bfdf3a2c27b72e7ef0c51a795710e Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 8 Dec 2020 10:27:46 -0800
Subject: [PATCH 309/481] xds: `server_features` should be a child of
 `xds_servers` and not a sibling (#4087)

---
 xds/internal/client/bootstrap/bootstrap.go    | 14 ++++-----
 .../client/bootstrap/bootstrap_test.go        | 30 +++++++++----------
 xds/internal/testutils/e2e/bootstrap.go       | 12 ++++----
 3 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index e9699717c956..e833a4c91f4f 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -91,8 +91,9 @@ type channelCreds struct {
 }
 
 type xdsServer struct {
-	ServerURI    string         `json:"server_uri"`
-	ChannelCreds []channelCreds `json:"channel_creds"`
+	ServerURI      string         `json:"server_uri"`
+	ChannelCreds   []channelCreds `json:"channel_creds"`
+	ServerFeatures []string       `json:"server_features"`
 }
 
 // NewConfig returns a new instance of Config initialized by reading the
@@ -108,9 +109,9 @@ type xdsServer struct {
 //          "config": <JSON object containing config for the type>
 //        }
 //      ],
+//      "server_features": [ ... ],
 //    },
 //    "node": <JSON form of Node proto>,
-//    "server_features": [ ... ],
 //    "certificate_providers" : {
 //      "default": {
 //        "plugin_name": "default-plugin-name",
@@ -188,12 +189,7 @@ func NewConfig() (*Config, error) {
 					break
 				}
 			}
-		case "server_features":
-			var features []string
-			if err := json.Unmarshal(v, &features); err != nil {
-				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
-			}
-			for _, f := range features {
+			for _, f := range xs.ServerFeatures {
 				switch f {
 				case serverFeaturesV3:
 					serverSupportsV3 = true
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index b9d2b9ab63d9..9d5aec26df71 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -163,9 +163,9 @@ var (
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
 					{ "type": "google_default" }
-				]
-			}],
-			"server_features" : ["foo", "bar"]
+				],
+				"server_features" : ["foo", "bar"]
+			}]
 		}`,
 		"serverSupportsV3": `
 		{
@@ -179,9 +179,9 @@ var (
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
 					{ "type": "google_default" }
-				]
-			}],
-			"server_features" : ["foo", "bar", "xds_v3"]
+				],
+				"server_features" : ["foo", "bar", "xds_v3"]
+			}]
 		}`,
 	}
 	metadata = &structpb.Struct{
@@ -548,9 +548,9 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
 					{ "type": "google_default" }
-				]
+				],
+				"server_features" : ["foo", "bar", "xds_v3"],
 			}],
-			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": "bad JSON"
 		}`,
 		"allUnknownCertProviders": `
@@ -565,9 +565,9 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
 					{ "type": "google_default" }
-				]
+				],
+				"server_features" : ["foo", "bar", "xds_v3"]
 			}],
-			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": {
 				"unknownProviderInstance1": {
 					"plugin_name": "foo",
@@ -591,9 +591,9 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
 					{ "type": "google_default" }
-				]
+				],
+				"server_features" : ["foo", "bar", "xds_v3"],
 			}],
-			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": {
 				"unknownProviderInstance": {
 					"plugin_name": "foo",
@@ -617,9 +617,9 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 				"server_uri": "trafficdirector.googleapis.com:443",
 				"channel_creds": [
 					{ "type": "google_default" }
-				]
+				],
+				"server_features" : ["foo", "bar", "xds_v3"]
 			}],
-			"server_features" : ["foo", "bar", "xds_v3"],
 			"certificate_providers": {
 				"unknownProviderInstance": {
 					"plugin_name": "foo",
@@ -692,7 +692,7 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 
 			c, err := NewConfig()
 			if (err != nil) != test.wantErr {
-				t.Fatalf("NewConfig() returned: (%+v, %v), wantErr: %v", c.CertProviderConfigs, err, test.wantErr)
+				t.Fatalf("NewConfig() returned: %v, wantErr: %v", err, test.wantErr)
 			}
 			if test.wantErr {
 				return
diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go
index a6a9f22fb44f..9cd5c51ab187 100644
--- a/xds/internal/testutils/e2e/bootstrap.go
+++ b/xds/internal/testutils/e2e/bootstrap.go
@@ -80,7 +80,7 @@ func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
 	case TransportV2:
 		// TODO: Add any v2 specific fields.
 	case TransportV3:
-		cfg.ServerFeatures = append(cfg.ServerFeatures, "xds_v3")
+		cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "xds_v3")
 	default:
 		return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version)
 	}
@@ -103,14 +103,14 @@ func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
 }
 
 type bootstrapConfig struct {
-	XdsServers     []server `json:"xds_servers,omitempty"`
-	Node           node     `json:"node,omitempty"`
-	ServerFeatures []string `json:"server_features,omitempty"`
+	XdsServers []server `json:"xds_servers,omitempty"`
+	Node       node     `json:"node,omitempty"`
 }
 
 type server struct {
-	ServerURI    string  `json:"server_uri,omitempty"`
-	ChannelCreds []creds `json:"channel_creds,omitempty"`
+	ServerURI      string   `json:"server_uri,omitempty"`
+	ChannelCreds   []creds  `json:"channel_creds,omitempty"`
+	ServerFeatures []string `json:"server_features,omitempty"`
 }
 
 type creds struct {

From f64220f6e12a3d380613aa1b1617cc7e07d56407 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 8 Dec 2020 13:25:59 -0800
Subject: [PATCH 310/481] xds: Process `require_client_cert` field in security
 config (#4090)

---
 xds/internal/client/client.go          | 4 ++++
 xds/internal/client/client_lds_test.go | 9 +++++++--
 xds/internal/client/client_xds.go      | 1 +
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 5d8a7ad12281..f37a6ed3832e 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -221,6 +221,10 @@ type SecurityConfig struct {
 	// this list, and the handshake succeeds only if a match is found. Used only
 	// on the client-side.
 	AcceptedSANs []string
+	// RequireClientCert indicates if the server handshake process expects the
+	// client to present a certificate. Set to true when performing mTLS. Used
+	// only on the server-side.
+	RequireClientCert bool
 }
 
 // ClusterUpdate contains information from a received CDS response, which is of
diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
index b227f96e4ece..e5c7b1e48709 100644
--- a/xds/internal/client/client_lds_test.go
+++ b/xds/internal/client/client_lds_test.go
@@ -32,6 +32,7 @@ import (
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/xds/internal/version"
@@ -777,6 +778,7 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 												TypeUrl: version.V3DownstreamTLSContextURL,
 												Value: func() []byte {
 													tls := &v3tlspb.DownstreamTlsContext{
+														RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 														CommonTlsContext: &v3tlspb.CommonTlsContext{
 															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
 																ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -803,8 +805,9 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
 					SecurityCfg: &SecurityConfig{
-						RootInstanceName: "rootPluginInstance",
-						RootCertName:     "rootCertName",
+						RootInstanceName:  "rootPluginInstance",
+						RootCertName:      "rootCertName",
+						RequireClientCert: true,
 					},
 				},
 			},
@@ -840,6 +843,7 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 												TypeUrl: version.V3DownstreamTLSContextURL,
 												Value: func() []byte {
 													tls := &v3tlspb.DownstreamTlsContext{
+														RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 														CommonTlsContext: &v3tlspb.CommonTlsContext{
 															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 																InstanceName:    "identityPluginInstance",
@@ -874,6 +878,7 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						RootCertName:         "rootCertName",
 						IdentityInstanceName: "identityPluginInstance",
 						IdentityCertName:     "identityCertName",
+						RequireClientCert:    true,
 					},
 				},
 			},
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 22585ed78bdd..a2f18b1d5d0f 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -173,6 +173,7 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	if err != nil {
 		return nil, err
 	}
+	sc.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue()
 	return &ListenerUpdate{SecurityCfg: sc}, nil
 }
 

From 750abe8f95cd270ab68f7298a2854148d0c33030 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 8 Dec 2020 13:32:37 -0800
Subject: [PATCH 311/481] resolver: allow config selector to return an RPC
 error (#4082)

---
 clientconn.go                             |  4 ++--
 internal/resolver/config_selector.go      |  8 ++++---
 internal/resolver/config_selector_test.go | 21 +++++++++---------
 stream.go                                 |  5 ++++-
 test/resolver_test.go                     | 27 ++++++++++++++++-------
 5 files changed, 41 insertions(+), 24 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index b35ba94f292d..abfd20098c80 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -109,11 +109,11 @@ type defaultConfigSelector struct {
 	sc *ServiceConfig
 }
 
-func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) *iresolver.RPCConfig {
+func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
 	return &iresolver.RPCConfig{
 		Context:      rpcInfo.Context,
 		MethodConfig: getMethodConfig(dcs.sc, rpcInfo.Method),
-	}
+	}, nil
 }
 
 // DialContext creates a client connection to the given target. By default, it's
diff --git a/internal/resolver/config_selector.go b/internal/resolver/config_selector.go
index 5ef9262e51dc..e69900400564 100644
--- a/internal/resolver/config_selector.go
+++ b/internal/resolver/config_selector.go
@@ -29,8 +29,10 @@ import (
 
 // ConfigSelector controls what configuration to use for every RPC.
 type ConfigSelector interface {
-	// Selects the configuration for the RPC.
-	SelectConfig(RPCInfo) *RPCConfig
+	// Selects the configuration for the RPC, or terminates it using the error.
+	// This error will be converted by the gRPC library to a status error with
+	// code UNKNOWN if it is not returned as a status error.
+	SelectConfig(RPCInfo) (*RPCConfig, error)
 }
 
 // RPCInfo contains RPC information needed by a ConfigSelector.
@@ -86,7 +88,7 @@ func (scs *SafeConfigSelector) UpdateConfigSelector(cs ConfigSelector) {
 }
 
 // SelectConfig defers to the current ConfigSelector in scs.
-func (scs *SafeConfigSelector) SelectConfig(r RPCInfo) *RPCConfig {
+func (scs *SafeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) {
 	scs.mu.RLock()
 	defer scs.mu.RUnlock()
 	return scs.cs.SelectConfig(r)
diff --git a/internal/resolver/config_selector_test.go b/internal/resolver/config_selector_test.go
index f41b8eb0ed08..e5a50995df11 100644
--- a/internal/resolver/config_selector_test.go
+++ b/internal/resolver/config_selector_test.go
@@ -36,10 +36,10 @@ func Test(t *testing.T) {
 }
 
 type fakeConfigSelector struct {
-	selectConfig func(RPCInfo) *RPCConfig
+	selectConfig func(RPCInfo) (*RPCConfig, error)
 }
 
-func (f *fakeConfigSelector) SelectConfig(r RPCInfo) *RPCConfig {
+func (f *fakeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) {
 	return f.selectConfig(r)
 }
 
@@ -59,21 +59,21 @@ func (s) TestSafeConfigSelector(t *testing.T) {
 	cs2Called := make(chan struct{})
 
 	cs1 := &fakeConfigSelector{
-		selectConfig: func(r RPCInfo) *RPCConfig {
+		selectConfig: func(r RPCInfo) (*RPCConfig, error) {
 			cs1Called <- struct{}{}
 			if diff := cmp.Diff(r, testRPCInfo); diff != "" {
 				t.Errorf("SelectConfig(%v) called; want %v\n  Diffs:\n%s", r, testRPCInfo, diff)
 			}
-			return <-retChan1
+			return <-retChan1, nil
 		},
 	}
 	cs2 := &fakeConfigSelector{
-		selectConfig: func(r RPCInfo) *RPCConfig {
+		selectConfig: func(r RPCInfo) (*RPCConfig, error) {
 			cs2Called <- struct{}{}
 			if diff := cmp.Diff(r, testRPCInfo); diff != "" {
 				t.Errorf("SelectConfig(%v) called; want %v\n  Diffs:\n%s", r, testRPCInfo, diff)
 			}
-			return <-retChan2
+			return <-retChan2, nil
 		},
 	}
 
@@ -82,9 +82,9 @@ func (s) TestSafeConfigSelector(t *testing.T) {
 
 	cs1Returned := make(chan struct{})
 	go func() {
-		got := scs.SelectConfig(testRPCInfo) // blocks until send to retChan1
-		if got != resp1 {
-			t.Errorf("SelectConfig(%v) = %v; want %v", testRPCInfo, got, resp1)
+		got, err := scs.SelectConfig(testRPCInfo) // blocks until send to retChan1
+		if err != nil || got != resp1 {
+			t.Errorf("SelectConfig(%v) = %v, %v; want %v, nil", testRPCInfo, got, err, resp1)
 		}
 		close(cs1Returned)
 	}()
@@ -112,7 +112,8 @@ func (s) TestSafeConfigSelector(t *testing.T) {
 	for dl := time.Now().Add(150 * time.Millisecond); !time.Now().After(dl); {
 		gotConfigChan := make(chan *RPCConfig)
 		go func() {
-			gotConfigChan <- scs.SelectConfig(testRPCInfo)
+			cfg, _ := scs.SelectConfig(testRPCInfo)
+			gotConfigChan <- cfg
 		}()
 		select {
 		case <-time.After(500 * time.Millisecond):
diff --git a/stream.go b/stream.go
index 8d4694a33f90..eda1248d60ce 100644
--- a/stream.go
+++ b/stream.go
@@ -175,7 +175,10 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 
 	var mc serviceconfig.MethodConfig
 	var onCommit func()
-	rpcConfig := cc.safeConfigSelector.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: method})
+	rpcConfig, err := cc.safeConfigSelector.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: method})
+	if err != nil {
+		return nil, status.Convert(err).Err()
+	}
 	if rpcConfig != nil {
 		if rpcConfig.Context != nil {
 			ctx = rpcConfig.Context
diff --git a/test/resolver_test.go b/test/resolver_test.go
index ab154f0ea29a..648245aef9c3 100644
--- a/test/resolver_test.go
+++ b/test/resolver_test.go
@@ -20,11 +20,13 @@ package test
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/codes"
 	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/internal/stubserver"
@@ -32,14 +34,15 @@ import (
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
 type funcConfigSelector struct {
-	f func(iresolver.RPCInfo) *iresolver.RPCConfig
+	f func(iresolver.RPCInfo) (*iresolver.RPCConfig, error)
 }
 
-func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) *iresolver.RPCConfig {
+func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
 	return f.f(i)
 }
 
@@ -75,12 +78,14 @@ func (s) TestConfigSelector(t *testing.T) {
 
 	testCases := []struct {
 		name   string
-		md     metadata.MD
-		config *iresolver.RPCConfig
+		md     metadata.MD          // MD sent with RPC
+		config *iresolver.RPCConfig // config returned by config selector
+		csErr  error                // error returned by config selector
 
 		wantMD       metadata.MD
 		wantDeadline time.Time
 		wantTimeout  time.Duration
+		wantErr      error
 	}{{
 		name:         "basic",
 		md:           testMD,
@@ -95,6 +100,10 @@ func (s) TestConfigSelector(t *testing.T) {
 		},
 		wantMD:       mdOut,
 		wantDeadline: ctxDeadline,
+	}, {
+		name:    "erroring SelectConfig",
+		csErr:   status.Errorf(codes.Unavailable, "cannot send RPC"),
+		wantErr: status.Errorf(codes.Unavailable, "cannot send RPC"),
 	}, {
 		name: "alter timeout; remove MD",
 		md:   testMD,
@@ -138,13 +147,13 @@ func (s) TestConfigSelector(t *testing.T) {
 				Addresses:     []resolver.Address{{Addr: ss.Address}},
 				ServiceConfig: parseCfg(ss.R, "{}"),
 			}, funcConfigSelector{
-				f: func(i iresolver.RPCInfo) *iresolver.RPCConfig {
+				f: func(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
 					gotInfo = &i
 					cfg := tc.config
 					if cfg != nil && cfg.Context == nil {
 						cfg.Context = i.Context
 					}
-					return cfg
+					return cfg, tc.csErr
 				},
 			})
 			ss.R.UpdateState(state) // Blocks until config selector is applied
@@ -152,8 +161,10 @@ func (s) TestConfigSelector(t *testing.T) {
 			onCommittedCalled = false
 			ctx := metadata.NewOutgoingContext(ctx, tc.md)
 			startTime := time.Now()
-			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
-				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, nil", err)
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); fmt.Sprint(err) != fmt.Sprint(tc.wantErr) {
+				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.wantErr)
+			} else if err != nil {
+				return // remaining checks are invalid
 			}
 
 			if gotInfo == nil {

From ff16195790fedd9f08f6adb4503e72e223942569 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <garrettgutierrez1@gmail.com>
Date: Tue, 8 Dec 2020 14:33:48 -0800
Subject: [PATCH 312/481] xds: Implement circuit breaking support. (#4050)

---
 .../balancer/cdsbalancer/cdsbalancer.go       |   3 +
 .../balancer/cdsbalancer/cdsbalancer_test.go  |  37 +++++
 xds/internal/balancer/edsbalancer/eds.go      |   5 +
 xds/internal/balancer/edsbalancer/eds_impl.go |  47 +++++-
 .../balancer/edsbalancer/eds_impl_test.go     |  65 +++++++-
 xds/internal/balancer/edsbalancer/eds_test.go |  53 ++++++
 xds/internal/client/client.go                 |   2 +
 xds/internal/client/client_cds_test.go        |  39 +++++
 .../client/client_requests_counter.go         |  89 ++++++++++
 .../client/client_requests_counter_test.go    | 156 ++++++++++++++++++
 xds/internal/client/client_xds.go             |  23 +++
 xds/internal/env/env.go                       |   9 +-
 12 files changed, 518 insertions(+), 10 deletions(-)
 create mode 100644 xds/internal/client/client_requests_counter.go
 create mode 100644 xds/internal/client/client_requests_counter_test.go

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 5a869e016f2f..3a276c034f2a 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
+	"google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -328,6 +329,8 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		return
 	}
 
+	client.SetMaxRequests(update.cds.ServiceName, update.cds.MaxRequests)
+
 	// The first good update from the watch API leads to the instantiation of an
 	// edsBalancer. Further updates/errors are propagated to the existing
 	// edsBalancer.
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index ccd1699afaac..9d7c4a7bff55 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
+	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
@@ -574,6 +575,42 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	}
 }
 
+// TestCircuitBreaking verifies that the CDS balancer correctly updates a
+// service's counter on watch updates.
+func (s) TestCircuitBreaking(t *testing.T) {
+	// This creates a CDS balancer, pushes a ClientConnState update with a fake
+	// xdsClient, and makes sure that the CDS balancer registers a watch on the
+	// provided xdsClient.
+	xdsC, cdsB, edsB, _, cancel := setupWithXDSCreds(t)
+	defer func() {
+		cancel()
+		cdsB.Close()
+	}()
+
+	// Here we invoke the watch callback registered on the fake xdsClient. This
+	// will trigger the watch handler on the CDS balancer, which will update
+	// the service's counter with the new max requests.
+	var maxRequests uint32 = 1
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName, MaxRequests: &maxRequests}
+	wantCCS := edsCCS(serviceName, false)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
+		t.Fatal(err)
+	}
+
+	// Since the counter's max requests was set to 1, the first request should
+	// succeed and the second should fail.
+	counter := client.GetServiceRequestsCounter(serviceName)
+	if err := counter.StartRequest(); err != nil {
+		t.Fatal(err)
+	}
+	if err := counter.StartRequest(); err == nil {
+		t.Fatal("unexpected success on start request over max")
+	}
+	counter.EndRequest()
+}
+
 // TestClose verifies the Close() method in the the CDS balancer.
 func (s) TestClose(t *testing.T) {
 	// This creates a CDS balancer, pushes a ClientConnState update with a fake
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 1bcfa1d9f0fa..12a1251abe9f 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -113,6 +113,9 @@ type edsBalancerImplInterface interface {
 	handleSubConnStateChange(sc balancer.SubConn, state connectivity.State)
 	// updateState handle a balancer state update from the priority.
 	updateState(priority priorityType, s balancer.State)
+	// updateServiceRequestsCounter updates the service requests counter to the
+	// one for the given service name.
+	updateServiceRequestsCounter(serviceName string)
 	// close closes the eds balancer.
 	close()
 }
@@ -212,6 +215,8 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			x.logger.Warningf("failed to update xDS client: %v", err)
 		}
 
+		x.edsImpl.updateServiceRequestsCounter(cfg.EDSServiceName)
+
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
 		if !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy, cmpopts.EquateEmpty()) {
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 1d996e8219ee..a6e950b88eea 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -35,8 +35,10 @@ import (
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
+	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
+	"google.golang.org/grpc/xds/internal/env"
 )
 
 // TODO: make this a environment variable?
@@ -92,10 +94,11 @@ type edsBalancerImpl struct {
 	subConnMu         sync.Mutex
 	subConnToPriority map[balancer.SubConn]priorityType
 
-	pickerMu   sync.Mutex
-	dropConfig []xdsclient.OverloadDropConfig
-	drops      []*dropper
-	innerState balancer.State // The state of the picker without drop support.
+	pickerMu               sync.Mutex
+	dropConfig             []xdsclient.OverloadDropConfig
+	drops                  []*dropper
+	innerState             balancer.State // The state of the picker without drop support.
+	serviceRequestsCounter *client.ServiceRequestsCounter
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
@@ -170,7 +173,7 @@ func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropC
 		// Update picker with old inner picker, new drops.
 		edsImpl.cc.UpdateState(balancer.State{
 			ConnectivityState: edsImpl.innerState.ConnectivityState,
-			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadReporter)},
+			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter)},
 		)
 	}
 	edsImpl.pickerMu.Unlock()
@@ -389,6 +392,16 @@ func (edsImpl *edsBalancerImpl) handleSubConnStateChange(sc balancer.SubConn, s
 	}
 }
 
+// updateConfig handles changes to the circuit breaking configuration.
+func (edsImpl *edsBalancerImpl) updateServiceRequestsCounter(serviceName string) {
+	if !env.CircuitBreakingSupport {
+		return
+	}
+	if edsImpl.serviceRequestsCounter == nil || edsImpl.serviceRequestsCounter.ServiceName != serviceName {
+		edsImpl.serviceRequestsCounter = client.GetServiceRequestsCounter(serviceName)
+	}
+}
+
 // updateState first handles priority, and then wraps picker in a drop picker
 // before forwarding the update.
 func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.State) {
@@ -403,7 +416,7 @@ func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.St
 		defer edsImpl.pickerMu.Unlock()
 		edsImpl.innerState = s
 		// Don't reset drops when it's a state change.
-		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadReporter)})
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter)})
 	}
 }
 
@@ -455,13 +468,15 @@ type dropPicker struct {
 	drops     []*dropper
 	p         balancer.Picker
 	loadStore load.PerClusterReporter
+	counter   *client.ServiceRequestsCounter
 }
 
-func newDropPicker(p balancer.Picker, drops []*dropper, loadStore load.PerClusterReporter) *dropPicker {
+func newDropPicker(p balancer.Picker, drops []*dropper, loadStore load.PerClusterReporter, counter *client.ServiceRequestsCounter) *dropPicker {
 	return &dropPicker{
 		drops:     drops,
 		p:         p,
 		loadStore: loadStore,
+		counter:   counter,
 	}
 }
 
@@ -483,6 +498,24 @@ func (d *dropPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 		}
 		return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
 	}
+	if d.counter != nil {
+		if err := d.counter.StartRequest(); err != nil {
+			return balancer.PickResult{}, status.Errorf(codes.Unavailable, err.Error())
+		}
+		pr, err := d.p.Pick(info)
+		if err != nil {
+			d.counter.EndRequest()
+			return pr, err
+		}
+		oldDone := pr.Done
+		pr.Done = func(doneInfo balancer.DoneInfo) {
+			d.counter.EndRequest()
+			if oldDone != nil {
+				oldDone(doneInfo)
+			}
+		}
+		return pr, err
+	}
 	// TODO: (eds) don't drop unless the inner picker is READY. Similar to
 	// https://github.com/grpc/grpc-go/issues/2622.
 	return d.p.Pick(info)
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index d2a45655efc1..9f28f252d9b5 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -32,8 +32,10 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
+	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
@@ -550,6 +552,67 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 	}
 }
 
+func (s) TestEDS_CircuitBreaking(t *testing.T) {
+	origCircuitBreakingSupport := env.CircuitBreakingSupport
+	env.CircuitBreakingSupport = true
+	defer func() { env.CircuitBreakingSupport = origCircuitBreakingSupport }()
+
+	cc := testutils.NewTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+	edsb.updateServiceRequestsCounter("test")
+	var maxRequests uint32 = 50
+	client.SetMaxRequests("test", &maxRequests)
+
+	// One locality with one backend.
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
+	sc1 := <-cc.NewSubConnCh
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
+
+	// Picks with drops.
+	dones := []func(){}
+	p := <-cc.NewPickerCh
+	for i := 0; i < 100; i++ {
+		pr, err := p.Pick(balancer.PickInfo{})
+		if i < 50 && err != nil {
+			t.Errorf("The first 50%% picks should be non-drops, got error %v", err)
+		} else if i > 50 && err == nil {
+			t.Errorf("The second 50%% picks should be drops, got error <nil>")
+		}
+		dones = append(dones, func() {
+			if pr.Done != nil {
+				pr.Done(balancer.DoneInfo{})
+			}
+		})
+	}
+
+	for _, done := range dones {
+		done()
+	}
+	dones = []func(){}
+
+	// Pick without drops.
+	for i := 0; i < 50; i++ {
+		pr, err := p.Pick(balancer.PickInfo{})
+		if err != nil {
+			t.Errorf("The third 50%% picks should be non-drops, got error %v", err)
+		}
+		dones = append(dones, func() {
+			if pr.Done != nil {
+				pr.Done(balancer.DoneInfo{})
+			}
+		})
+	}
+
+	// Without this, future tests with the same service name will fail.
+	for _, done := range dones {
+		done()
+	}
+}
+
 func init() {
 	balancer.Register(&testInlineUpdateBalancerBuilder{})
 }
@@ -656,7 +719,7 @@ func (s) TestDropPicker(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			p := newDropPicker(constPicker, tt.drops, nil)
+			p := newDropPicker(constPicker, tt.drops, nil, nil)
 
 			// scCount is the number of sc's returned by pick. The opposite of
 			// drop-count.
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index cd59bba1e799..3ee19a00debe 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -115,6 +115,7 @@ type fakeEDSBalancer struct {
 	childPolicy        *testutils.Channel
 	subconnStateChange *testutils.Channel
 	edsUpdate          *testutils.Channel
+	serviceName        *testutils.Channel
 }
 
 func (f *fakeEDSBalancer) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
@@ -131,6 +132,10 @@ func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {
 
 func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {}
 
+func (f *fakeEDSBalancer) updateServiceRequestsCounter(serviceName string) {
+	f.serviceName.Send(serviceName)
+}
+
 func (f *fakeEDSBalancer) close() {}
 
 func (f *fakeEDSBalancer) waitForChildPolicy(ctx context.Context, wantPolicy *loadBalancingConfig) error {
@@ -169,12 +174,25 @@ func (f *fakeEDSBalancer) waitForEDSResponse(ctx context.Context, wantUpdate xds
 	return nil
 }
 
+func (f *fakeEDSBalancer) waitForCounterUpdate(ctx context.Context, wantServiceName string) error {
+	val, err := f.serviceName.Receive(ctx)
+	if err != nil {
+		return err
+	}
+	gotServiceName := val.(string)
+	if gotServiceName != wantServiceName {
+		return fmt.Errorf("got serviceName %v, want %v", gotServiceName, wantServiceName)
+	}
+	return nil
+}
+
 func newFakeEDSBalancer(cc balancer.ClientConn) edsBalancerImplInterface {
 	return &fakeEDSBalancer{
 		cc:                 cc,
 		childPolicy:        testutils.NewChannelWithSize(10),
 		subconnStateChange: testutils.NewChannelWithSize(10),
 		edsUpdate:          testutils.NewChannelWithSize(10),
+		serviceName:        testutils.NewChannelWithSize(10),
 	}
 }
 
@@ -307,6 +325,9 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgA); err != nil {
 		t.Fatal(err)
 	}
+	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
+		t.Fatal(err)
+	}
 
 	lbCfgB := &loadBalancingConfig{
 		Name:   fakeBalancerB,
@@ -323,6 +344,11 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgB); err != nil {
 		t.Fatal(err)
 	}
+	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
+		// Counter is updated even though the service name didn't change. The
+		// eds_impl will compare the service names, and skip if it didn't change.
+		t.Fatal(err)
+	}
 }
 
 // TestSubConnStateChange verifies if the top-level edsBalancer passes on
@@ -566,6 +592,33 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}
 }
 
+// TestCounterUpdate verifies that the counter update is triggered with the
+// service name from an update's config.
+func (s) TestCounterUpdate(t *testing.T) {
+	edsLBCh := testutils.NewChannel()
+	_, cleanup := setup(edsLBCh)
+	defer cleanup()
+
+	builder := balancer.Get(edsName)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
+	}
+	defer edsB.Close()
+
+	// Update should trigger counter update with provided service name.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: "foobar-1"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := edsB.(*edsBalancer).edsImpl.(*fakeEDSBalancer).waitForCounterUpdate(ctx, "foobar-1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func (s) TestBalancerConfigParsing(t *testing.T) {
 	const testEDSName = "eds.service"
 	var testLRSName = "lrs.server"
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index f37a6ed3832e..b0e27d4fd754 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -237,6 +237,8 @@ type ClusterUpdate struct {
 	EnableLRS bool
 	// SecurityCfg contains security configuration sent by the control plane.
 	SecurityCfg *SecurityConfig
+	// MaxRequests for circuit breaking, if any (otherwise nil).
+	MaxRequests *uint32
 }
 
 // OverloadDropConfig contains the config to drop overloads.
diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go
index 6cba7ef12a08..4d0801dd8058 100644
--- a/xds/internal/client/client_cds_test.go
+++ b/xds/internal/client/client_cds_test.go
@@ -31,7 +31,9 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
 const (
@@ -169,8 +171,45 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 			},
 			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
 		},
+		{
+			name: "happiest-case-with-circuitbreakers",
+			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				CircuitBreakers: &v3clusterpb.CircuitBreakers{
+					Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+						{
+							Priority:    v3corepb.RoutingPriority_DEFAULT,
+							MaxRequests: wrapperspb.UInt32(512),
+						},
+						{
+							Priority:    v3corepb.RoutingPriority_HIGH,
+							MaxRequests: nil,
+						},
+					},
+				},
+				LrsServer: &v3corepb.ConfigSource{
+					ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{
+						Self: &v3corepb.SelfConfigSource{},
+					},
+				},
+			},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
+		},
 	}
 
+	origCircuitBreakingSupport := env.CircuitBreakingSupport
+	env.CircuitBreakingSupport = true
+	defer func() { env.CircuitBreakingSupport = origCircuitBreakingSupport }()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			update, err := validateCluster(test.cluster)
diff --git a/xds/internal/client/client_requests_counter.go b/xds/internal/client/client_requests_counter.go
new file mode 100644
index 000000000000..1e28fc003ff3
--- /dev/null
+++ b/xds/internal/client/client_requests_counter.go
@@ -0,0 +1,89 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+)
+
+const defaultMaxRequests uint32 = 1024
+
+type servicesRequestsCounter struct {
+	mu       sync.Mutex
+	services map[string]*ServiceRequestsCounter
+}
+
+var src = &servicesRequestsCounter{
+	services: make(map[string]*ServiceRequestsCounter),
+}
+
+// ServiceRequestsCounter is used to track the total inflight requests for a
+// service with the provided name.
+type ServiceRequestsCounter struct {
+	ServiceName string
+	maxRequests uint32
+	numRequests uint32
+}
+
+// GetServiceRequestsCounter returns the ServiceRequestsCounter with the
+// provided serviceName. If one does not exist, it creates it.
+func GetServiceRequestsCounter(serviceName string) *ServiceRequestsCounter {
+	src.mu.Lock()
+	defer src.mu.Unlock()
+	c, ok := src.services[serviceName]
+	if !ok {
+		c = &ServiceRequestsCounter{ServiceName: serviceName, maxRequests: defaultMaxRequests}
+		src.services[serviceName] = c
+	}
+	return c
+}
+
+// SetMaxRequests updates the max requests for a service's counter.
+func SetMaxRequests(serviceName string, maxRequests *uint32) {
+	src.mu.Lock()
+	defer src.mu.Unlock()
+	c, ok := src.services[serviceName]
+	if !ok {
+		c = &ServiceRequestsCounter{ServiceName: serviceName}
+		src.services[serviceName] = c
+	}
+	if maxRequests != nil {
+		c.maxRequests = *maxRequests
+	} else {
+		c.maxRequests = defaultMaxRequests
+	}
+}
+
+// StartRequest starts a request for a service, incrementing its number of
+// requests by 1. Returns an error if the max number of requests is exceeded.
+func (c *ServiceRequestsCounter) StartRequest() error {
+	if atomic.LoadUint32(&c.numRequests) >= atomic.LoadUint32(&c.maxRequests) {
+		return fmt.Errorf("max requests %v exceeded on service %v", c.maxRequests, c.ServiceName)
+	}
+	atomic.AddUint32(&c.numRequests, 1)
+	return nil
+}
+
+// EndRequest ends a request for a service, decrementing its number of requests
+// by 1.
+func (c *ServiceRequestsCounter) EndRequest() {
+	atomic.AddUint32(&c.numRequests, ^uint32(0))
+}
diff --git a/xds/internal/client/client_requests_counter_test.go b/xds/internal/client/client_requests_counter_test.go
new file mode 100644
index 000000000000..09295b982ecd
--- /dev/null
+++ b/xds/internal/client/client_requests_counter_test.go
@@ -0,0 +1,156 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+type counterTest struct {
+	name              string
+	maxRequests       uint32
+	numRequests       uint32
+	expectedSuccesses uint32
+	expectedErrors    uint32
+}
+
+var tests = []counterTest{
+	{
+		name:              "does-not-exceed-max-requests",
+		maxRequests:       1024,
+		numRequests:       1024,
+		expectedSuccesses: 1024,
+		expectedErrors:    0,
+	},
+	{
+		name:              "exceeds-max-requests",
+		maxRequests:       32,
+		numRequests:       64,
+		expectedSuccesses: 32,
+		expectedErrors:    32,
+	},
+}
+
+func resetServiceRequestsCounter() {
+	src = &servicesRequestsCounter{
+		services: make(map[string]*ServiceRequestsCounter),
+	}
+}
+
+func testCounter(t *testing.T, test counterTest) {
+	SetMaxRequests(test.name, &test.maxRequests)
+	requestsStarted := make(chan struct{})
+	requestsSent := sync.WaitGroup{}
+	requestsSent.Add(int(test.numRequests))
+	requestsDone := sync.WaitGroup{}
+	requestsDone.Add(int(test.numRequests))
+	var lastError atomic.Value
+	var successes, errors uint32
+	for i := 0; i < int(test.numRequests); i++ {
+		go func() {
+			counter := GetServiceRequestsCounter(test.name)
+			defer requestsDone.Done()
+			err := counter.StartRequest()
+			if err == nil {
+				atomic.AddUint32(&successes, 1)
+			} else {
+				atomic.AddUint32(&errors, 1)
+				lastError.Store(err)
+			}
+			requestsSent.Done()
+			if err == nil {
+				<-requestsStarted
+				counter.EndRequest()
+			}
+		}()
+	}
+	requestsSent.Wait()
+	close(requestsStarted)
+	requestsDone.Wait()
+	loadedError := lastError.Load()
+	if test.expectedErrors > 0 && loadedError == nil {
+		t.Error("no error when error expected")
+	}
+	if test.expectedErrors == 0 && loadedError != nil {
+		t.Errorf("error starting request: %v", loadedError.(error))
+	}
+	if successes != test.expectedSuccesses || errors != test.expectedErrors {
+		t.Errorf("unexpected number of (successes, errors), expected (%v, %v), encountered (%v, %v)", test.expectedSuccesses, test.expectedErrors, successes, errors)
+	}
+}
+
+func (s) TestRequestsCounter(t *testing.T) {
+	resetServiceRequestsCounter()
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testCounter(t, test)
+		})
+	}
+}
+
+func (s) TestGetServiceRequestsCounter(t *testing.T) {
+	resetServiceRequestsCounter()
+	for _, test := range tests {
+		counterA := GetServiceRequestsCounter(test.name)
+		counterB := GetServiceRequestsCounter(test.name)
+		if counterA != counterB {
+			t.Errorf("counter %v %v != counter %v %v", counterA, *counterA, counterB, *counterB)
+		}
+	}
+}
+
+func startRequests(t *testing.T, n uint32, max uint32, counter *ServiceRequestsCounter) {
+	SetMaxRequests(counter.ServiceName, &max)
+	for i := uint32(0); i < n; i++ {
+		if err := counter.StartRequest(); err != nil {
+			t.Fatalf("error starting initial request: %v", err)
+		}
+	}
+}
+
+func (s) TestSetMaxRequestsIncreased(t *testing.T) {
+	resetServiceRequestsCounter()
+	const serviceName string = "set-max-requests-increased"
+	var initialMax uint32 = 16
+	counter := GetServiceRequestsCounter(serviceName)
+	startRequests(t, initialMax, initialMax, counter)
+	if err := counter.StartRequest(); err == nil {
+		t.Fatal("unexpected success on start request after max met")
+	}
+	newMax := initialMax + 1
+	SetMaxRequests(counter.ServiceName, &newMax)
+	if err := counter.StartRequest(); err != nil {
+		t.Fatalf("unexpected error on start request after max increased: %v", err)
+	}
+}
+
+func (s) TestSetMaxRequestsDecreased(t *testing.T) {
+	resetServiceRequestsCounter()
+	const serviceName string = "set-max-requests-decreased"
+	var initialMax uint32 = 16
+	counter := GetServiceRequestsCounter(serviceName)
+	startRequests(t, initialMax-1, initialMax, counter)
+	newMax := initialMax - 1
+	SetMaxRequests(counter.ServiceName, &newMax)
+	if err := counter.StartRequest(); err == nil {
+		t.Fatalf("unexpected success on start request after max decreased: %v", err)
+	}
+}
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index a2f18b1d5d0f..89b2afb4b9e5 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -38,6 +38,7 @@ import (
 
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/xds/internal"
+	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -395,6 +396,7 @@ func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
 		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
 		SecurityCfg: sc,
+		MaxRequests: circuitBreakersFromCluster(cluster),
 	}, nil
 }
 
@@ -472,6 +474,27 @@ func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext) (*Secu
 	return sc, nil
 }
 
+// circuitBreakersFromCluster extracts the circuit breakers configuration from
+// the received cluster resource. Returns nil if no CircuitBreakers or no
+// Thresholds in CircuitBreakers.
+func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 {
+	if !env.CircuitBreakingSupport {
+		return nil
+	}
+	for _, threshold := range cluster.GetCircuitBreakers().GetThresholds() {
+		if threshold.GetPriority() != v3corepb.RoutingPriority_DEFAULT {
+			continue
+		}
+		maxRequestsPb := threshold.GetMaxRequests()
+		if maxRequestsPb == nil {
+			return nil
+		}
+		maxRequests := maxRequestsPb.GetValue()
+		return &maxRequests
+	}
+	return nil
+}
+
 // UnmarshalEndpoints processes resources received in an EDS response,
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in.
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index 3b338eae05fd..c0fa0e65b7a3 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -26,8 +26,9 @@ import (
 )
 
 const (
-	bootstrapFileNameEnv = "GRPC_XDS_BOOTSTRAP"
-	xdsV3SupportEnv      = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
+	bootstrapFileNameEnv      = "GRPC_XDS_BOOTSTRAP"
+	xdsV3SupportEnv           = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
+	circuitBreakingSupportEnv = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
 )
 
 var (
@@ -39,4 +40,8 @@ var (
 	// done by setting the environment variable
 	// "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" to "true".
 	V3Support = strings.EqualFold(os.Getenv(xdsV3SupportEnv), "true")
+	// CircuitBreakingSupport indicates whether circuit breaking support is
+	// enabled, which can be done by setting the environment variable
+	// "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING" to "true".
+	CircuitBreakingSupport = strings.EqualFold(os.Getenv(circuitBreakingSupportEnv), "true")
 )

From fd32f6a4fefe08a37b2fcfbbbd8fe61e8be0e33d Mon Sep 17 00:00:00 2001
From: Tobias Klauser <tobias.klauser@gmail.com>
Date: Wed, 9 Dec 2020 02:14:39 +0100
Subject: [PATCH 313/481] internal/syscall: make Rusage a type alias (#4088)

Making Rusage a type alias rather than a type definition allows to avoid
type conversions in several places.

Type aliases are supported since Go 1.9, go.mod states Go 1.11 to be the
minimum supported version.
---
 internal/syscall/syscall_linux.go    | 20 +++++++++-----------
 internal/syscall/syscall_nonlinux.go |  2 +-
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go
index 2fdcb76e6c74..4b2964f2a1e3 100644
--- a/internal/syscall/syscall_linux.go
+++ b/internal/syscall/syscall_linux.go
@@ -44,25 +44,23 @@ func GetCPUTime() int64 {
 }
 
 // Rusage is an alias for syscall.Rusage under linux environment.
-type Rusage syscall.Rusage
+type Rusage = syscall.Rusage
 
 // GetRusage returns the resource usage of current process.
-func GetRusage() (rusage *Rusage) {
-	rusage = new(Rusage)
-	syscall.Getrusage(syscall.RUSAGE_SELF, (*syscall.Rusage)(rusage))
-	return
+func GetRusage() *Rusage {
+	rusage := new(Rusage)
+	syscall.Getrusage(syscall.RUSAGE_SELF, rusage)
+	return rusage
 }
 
 // CPUTimeDiff returns the differences of user CPU time and system CPU time used
 // between two Rusage structs.
 func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) {
-	f := (*syscall.Rusage)(first)
-	l := (*syscall.Rusage)(latest)
 	var (
-		utimeDiffs  = l.Utime.Sec - f.Utime.Sec
-		utimeDiffus = l.Utime.Usec - f.Utime.Usec
-		stimeDiffs  = l.Stime.Sec - f.Stime.Sec
-		stimeDiffus = l.Stime.Usec - f.Stime.Usec
+		utimeDiffs  = latest.Utime.Sec - first.Utime.Sec
+		utimeDiffus = latest.Utime.Usec - first.Utime.Usec
+		stimeDiffs  = latest.Stime.Sec - first.Stime.Sec
+		stimeDiffus = latest.Stime.Usec - first.Stime.Usec
 	)
 
 	uTimeElapsed := float64(utimeDiffs) + float64(utimeDiffus)*1.0e-6
diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go
index adae60d65188..7913ef1dbfb5 100644
--- a/internal/syscall/syscall_nonlinux.go
+++ b/internal/syscall/syscall_nonlinux.go
@@ -50,7 +50,7 @@ func GetCPUTime() int64 {
 type Rusage struct{}
 
 // GetRusage is a no-op function under non-linux or appengine environment.
-func GetRusage() (rusage *Rusage) {
+func GetRusage() *Rusage {
 	log()
 	return nil
 }

From c638ab8ccda6cefc15eb9a25357fc98b018ede37 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 11 Dec 2020 09:14:07 -0800
Subject: [PATCH 314/481] Update envoyproxy/go-control-plane (#4098)

---
 examples/go.sum                      | 8 ++++----
 go.mod                               | 4 ++--
 go.sum                               | 8 ++++----
 security/advancedtls/examples/go.sum | 4 ++--
 security/advancedtls/go.sum          | 4 ++--
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/examples/go.sum b/examples/go.sum
index 60d10ce19c5b..1bc27f276662 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -2,11 +2,11 @@ cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.7 h1:EARl0OvqMoxq/UMgMSCLnXzkaXbxzskluEBlMQCJPms=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
diff --git a/go.mod b/go.mod
index 6feb697bbdab..cab74e55774c 100644
--- a/go.mod
+++ b/go.mod
@@ -3,8 +3,8 @@ module google.golang.org/grpc
 go 1.11
 
 require (
-	github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354
-	github.com/envoyproxy/go-control-plane v0.9.7
+	github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403
+	github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-cmp v0.5.0
diff --git a/go.sum b/go.sum
index a98f8b16482b..e437e1300b21 100644
--- a/go.sum
+++ b/go.sum
@@ -6,14 +6,14 @@ github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.7 h1:EARl0OvqMoxq/UMgMSCLnXzkaXbxzskluEBlMQCJPms=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
index 81cb0f18bc00..3d64689329fe 100644
--- a/security/advancedtls/examples/go.sum
+++ b/security/advancedtls/examples/go.sum
@@ -1,8 +1,8 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 81cb0f18bc00..3d64689329fe 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,8 +1,8 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

From 81b95b1854d7caf3cc21aed316fc222e1749cf31 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 11 Dec 2020 16:06:04 -0800
Subject: [PATCH 315/481] github: Run only a subset of tests for Go1.11 (#4102)

---
 .github/workflows/testing.yml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 91e66efd7d2f..b30cd53db5c2 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -51,7 +51,7 @@ jobs:
             goversion: 1.14
           - type: tests
             goversion: 1.13
-          - type: tests
+          - type: tests111
             goversion: 1.11  # Keep until interop tests no longer require Go1.11
 
     steps:
@@ -74,9 +74,10 @@ jobs:
         if: ${{ matrix.type == 'vet' }}
         run: ./vet.sh -install && ./vet.sh
 
-      # Main tests run for everything except when testing "extras" and the race detector.
+      # Main tests run for everything except when testing "extras", the race
+      # detector and Go1.11 (where we run a reduced set of tests).
       - name: Run tests
-        if: ${{ matrix.type != 'extras' && matrix.type != 'race' }}
+        if: ${{ matrix.type != 'extras' && matrix.type != 'race' && matrix.type != 'test111' }}
         run: go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Race detector tests
@@ -93,3 +94,10 @@ jobs:
           interop/interop_test.sh
           cd ${GITHUB_WORKSPACE}/security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
           cd ${GITHUB_WORKSPACE}/security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/...
+
+      # Reduced set of tests for Go 1.11
+      - name: Run Go1.11 tests
+        if: ${{ matrix.type == 'tests111' }}
+        run: |
+          tests=$(find ${GITHUB_WORKSPACE} -name '*_test.go' | xargs -n1 dirname | sort -u | sed "s:^${GITHUB_WORKSPACE}:.:" | sed "s:\/$::" | grep -v ^./security | grep -v ^./credentials/sts | grep -v ^./credentials/tls/certprovider | grep -v ^./credentials/xds | grep -v ^./xds | grep -v ^./security)
+          go test -cpu 1,4 -timeout 7m ${tests}

From 03d4b8878b3ce2abbcd9b57fb8125c25623806b2 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 14 Dec 2020 08:07:12 -0800
Subject: [PATCH 316/481] github: Get tests111 to actually run subset of tests
 (#4104)

---
 .github/workflows/testing.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index b30cd53db5c2..91c5d452e875 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -77,7 +77,7 @@ jobs:
       # Main tests run for everything except when testing "extras", the race
       # detector and Go1.11 (where we run a reduced set of tests).
       - name: Run tests
-        if: ${{ matrix.type != 'extras' && matrix.type != 'race' && matrix.type != 'test111' }}
+        if: ${{ matrix.type != 'extras' && matrix.type != 'race' && matrix.type != 'tests111' }}
         run: go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Race detector tests
@@ -99,5 +99,6 @@ jobs:
       - name: Run Go1.11 tests
         if: ${{ matrix.type == 'tests111' }}
         run: |
-          tests=$(find ${GITHUB_WORKSPACE} -name '*_test.go' | xargs -n1 dirname | sort -u | sed "s:^${GITHUB_WORKSPACE}:.:" | sed "s:\/$::" | grep -v ^./security | grep -v ^./credentials/sts | grep -v ^./credentials/tls/certprovider | grep -v ^./credentials/xds | grep -v ^./xds | grep -v ^./security)
+          tests=$(find ${GITHUB_WORKSPACE} -name '*_test.go' | xargs -n1 dirname | sort -u | sed "s:^${GITHUB_WORKSPACE}:.:" | sed "s:\/$::" | grep -v ^./security | grep -v ^./credentials/sts | grep -v ^./credentials/tls/certprovider | grep -v ^./credentials/xds | grep -v ^./xds )
+          echo "Running tests for " ${tests}
           go test -cpu 1,4 -timeout 7m ${tests}

From 17e2cbe88713844032125f97f1174fcb2fe7faf0 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 14 Dec 2020 09:00:45 -0800
Subject: [PATCH 317/481] credentials/xds: ServerHandshake() implementation
 (#4089)

---
 credentials/xds/xds.go             | 167 +++++++---
 credentials/xds/xds_client_test.go |  87 +++--
 credentials/xds/xds_server_test.go | 492 +++++++++++++++++++++++++++++
 go.sum                             |   1 +
 4 files changed, 680 insertions(+), 67 deletions(-)
 create mode 100644 credentials/xds/xds_server_test.go

diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go
index 2cc52ce6b798..666da9918e62 100644
--- a/credentials/xds/xds.go
+++ b/credentials/xds/xds.go
@@ -33,6 +33,7 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"time"
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials"
@@ -50,9 +51,9 @@ func init() {
 // credentials implementation.
 type ClientOptions struct {
 	// FallbackCreds specifies the fallback credentials to be used when either
-	// the `xds` scheme is not used in the user's dial target or when the xDS
-	// server does not return any security configuration. Attempts to create
-	// client credentials without a fallback credentials will fail.
+	// the `xds` scheme is not used in the user's dial target or when the
+	// management server does not return any security configuration. Attempts to
+	// create client credentials without fallback credentials will fail.
 	FallbackCreds credentials.TransportCredentials
 }
 
@@ -68,6 +69,27 @@ func NewClientCredentials(opts ClientOptions) (credentials.TransportCredentials,
 	}, nil
 }
 
+// ServerOptions contains parameters to configure a new server-side xDS
+// credentials implementation.
+type ServerOptions struct {
+	// FallbackCreds specifies the fallback credentials to be used when the
+	// management server does not return any security configuration. Attempts to
+	// create server credentials without fallback credentials will fail.
+	FallbackCreds credentials.TransportCredentials
+}
+
+// NewServerCredentials returns a new server-side transport credentials
+// implementation which uses xDS APIs to fetch its security configuration.
+func NewServerCredentials(opts ServerOptions) (credentials.TransportCredentials, error) {
+	if opts.FallbackCreds == nil {
+		return nil, errors.New("missing fallback credentials")
+	}
+	return &credsImpl{
+		isClient: false,
+		fallback: opts.FallbackCreds,
+	}, nil
+}
+
 // credsImpl is an implementation of the credentials.TransportCredentials
 // interface which uses xDS APIs to fetch its security configuration.
 type credsImpl struct {
@@ -98,11 +120,15 @@ func getHandshakeInfo(attr *attributes.Attributes) *HandshakeInfo {
 // responsible for populating these fields.
 //
 // Safe for concurrent access.
+//
+// TODO(easwars): Move this type and any other non-user functionality to an
+// internal package.
 type HandshakeInfo struct {
-	mu               sync.Mutex
-	rootProvider     certprovider.Provider
-	identityProvider certprovider.Provider
-	acceptedSANs     map[string]bool // Only on the client side.
+	mu                sync.Mutex
+	rootProvider      certprovider.Provider
+	identityProvider  certprovider.Provider
+	acceptedSANs      map[string]bool // Only on the client side.
+	requireClientCert bool            // Only on server side.
 }
 
 // SetRootCertProvider updates the root certificate provider.
@@ -129,6 +155,14 @@ func (hi *HandshakeInfo) SetAcceptedSANs(sans []string) {
 	hi.mu.Unlock()
 }
 
+// SetRequireClientCert updates whether a client cert is required during the
+// ServerHandshake(). A value of true indicates that we are performing mTLS.
+func (hi *HandshakeInfo) SetRequireClientCert(require bool) {
+	hi.mu.Lock()
+	hi.requireClientCert = require
+	hi.mu.Unlock()
+}
+
 // UseFallbackCreds returns true when fallback credentials are to be used based
 // on the contents of the HandshakeInfo.
 func (hi *HandshakeInfo) UseFallbackCreds() bool {
@@ -141,27 +175,13 @@ func (hi *HandshakeInfo) UseFallbackCreds() bool {
 	return hi.identityProvider == nil && hi.rootProvider == nil
 }
 
-func (hi *HandshakeInfo) validate(isClient bool) error {
+func (hi *HandshakeInfo) makeClientSideTLSConfig(ctx context.Context) (*tls.Config, error) {
 	hi.mu.Lock()
-	defer hi.mu.Unlock()
-
 	// On the client side, rootProvider is mandatory. IdentityProvider is
 	// optional based on whether the client is doing TLS or mTLS.
-	if isClient && hi.rootProvider == nil {
-		return errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server")
+	if hi.rootProvider == nil {
+		return nil, errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server")
 	}
-
-	// On the server side, identityProvider is mandatory. RootProvider is
-	// optional based on whether the server is doing TLS or mTLS.
-	if !isClient && hi.identityProvider == nil {
-		return errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server")
-	}
-
-	return nil
-}
-
-func (hi *HandshakeInfo) makeTLSConfig(ctx context.Context) (*tls.Config, error) {
-	hi.mu.Lock()
 	// Since the call to KeyMaterial() can block, we read the providers under
 	// the lock but call the actual function after releasing the lock.
 	rootProv, idProv := hi.rootProvider, hi.identityProvider
@@ -173,13 +193,13 @@ func (hi *HandshakeInfo) makeTLSConfig(ctx context.Context) (*tls.Config, error)
 	// includes hostname verification) or none. We are forced to go with the
 	// latter and perform the normal cert validation ourselves.
 	cfg := &tls.Config{InsecureSkipVerify: true}
-	if rootProv != nil {
-		km, err := rootProv.KeyMaterial(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
-		}
-		cfg.RootCAs = km.Roots
+
+	km, err := rootProv.KeyMaterial(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
 	}
+	cfg.RootCAs = km.Roots
+
 	if idProv != nil {
 		km, err := idProv.KeyMaterial(ctx)
 		if err != nil {
@@ -190,6 +210,39 @@ func (hi *HandshakeInfo) makeTLSConfig(ctx context.Context) (*tls.Config, error)
 	return cfg, nil
 }
 
+func (hi *HandshakeInfo) makeServerSideTLSConfig(ctx context.Context) (*tls.Config, error) {
+	cfg := &tls.Config{ClientAuth: tls.NoClientCert}
+	hi.mu.Lock()
+	// On the server side, identityProvider is mandatory. RootProvider is
+	// optional based on whether the server is doing TLS or mTLS.
+	if hi.identityProvider == nil {
+		return nil, errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server")
+	}
+	// Since the call to KeyMaterial() can block, we read the providers under
+	// the lock but call the actual function after releasing the lock.
+	rootProv, idProv := hi.rootProvider, hi.identityProvider
+	if hi.requireClientCert {
+		cfg.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+	hi.mu.Unlock()
+
+	// identityProvider is mandatory on the server side.
+	km, err := idProv.KeyMaterial(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err)
+	}
+	cfg.Certificates = km.Certs
+
+	if rootProv != nil {
+		km, err := rootProv.KeyMaterial(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
+		}
+		cfg.ClientCAs = km.Roots
+	}
+	return cfg, nil
+}
+
 func (hi *HandshakeInfo) matchingSANExists(cert *x509.Certificate) bool {
 	if len(hi.acceptedSANs) == 0 {
 		// An empty list of acceptedSANs means "accept everything".
@@ -265,9 +318,6 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 	if hi.UseFallbackCreds() {
 		return c.fallback.ClientHandshake(ctx, authority, rawConn)
 	}
-	if err := hi.validate(c.isClient); err != nil {
-		return nil, nil, err
-	}
 
 	// We build the tls.Config with the following values
 	// 1. Root certificate as returned by the root provider.
@@ -281,7 +331,7 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 	// 4. Key usage to match whether client/server usage.
 	// 5. A `VerifyPeerCertificate` function which performs normal peer
 	// 	  cert verification using configured roots, and the custom SAN checks.
-	cfg, err := hi.makeTLSConfig(ctx)
+	cfg, err := hi.makeClientSideTLSConfig(ctx)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -349,12 +399,55 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 }
 
 // ServerHandshake performs the TLS handshake on the server-side.
-func (c *credsImpl) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) {
+func (c *credsImpl) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
 	if c.isClient {
 		return nil, nil, errors.New("ServerHandshake is not supported for client credentials")
 	}
-	// TODO(easwars): Implement along with server side xDS implementation.
-	return nil, nil, errors.New("not implemented")
+
+	// An xds-enabled gRPC server wraps the underlying raw net.Conn in a type
+	// that provides a way to retrieve `HandshakeInfo`, which contains the
+	// certificate providers to be used during the handshake. If the net.Conn
+	// passed to this function does not implement this interface, or if the
+	// `HandshakeInfo` does not contain the information we are looking for, we
+	// delegate the handshake to the fallback credentials.
+	hiConn, ok := rawConn.(interface{ XDSHandshakeInfo() *HandshakeInfo })
+	if !ok {
+		return c.fallback.ServerHandshake(rawConn)
+	}
+	hi := hiConn.XDSHandshakeInfo()
+	if hi.UseFallbackCreds() {
+		return c.fallback.ServerHandshake(rawConn)
+	}
+
+	// An xds-enabled gRPC server is expected to wrap the underlying raw
+	// net.Conn in a type which provides a way to retrieve the deadline set on
+	// it. If we cannot retrieve the deadline here, we fail (by setting deadline
+	// to time.Now()), instead of using a default deadline and possibly taking
+	// longer to eventually fail.
+	deadline := time.Now()
+	if dConn, ok := rawConn.(interface{ GetDeadline() time.Time }); ok {
+		deadline = dConn.GetDeadline()
+	}
+	ctx, cancel := context.WithDeadline(context.Background(), deadline)
+	defer cancel()
+	cfg, err := hi.makeServerSideTLSConfig(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	conn := tls.Server(rawConn, cfg)
+	if err := conn.Handshake(); err != nil {
+		conn.Close()
+		return nil, nil, err
+	}
+	info := credentials.TLSInfo{
+		State: conn.ConnectionState(),
+		CommonAuthInfo: credentials.CommonAuthInfo{
+			SecurityLevel: credentials.PrivacyAndIntegrity,
+		},
+	}
+	info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState())
+	return credinternal.WrapSyscallConn(rawConn, conn), info, nil
 }
 
 // Info provides the ProtocolInfo of this TransportCredentials.
diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index 07bc48b3f64d..f22579c6b8a1 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -40,9 +40,10 @@ import (
 )
 
 const (
-	defaultTestTimeout = 1 * time.Second
-	defaultTestCertSAN = "*.test.example.com"
-	authority          = "authority"
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+	defaultTestCertSAN      = "*.test.example.com"
+	authority               = "authority"
 )
 
 type s struct {
@@ -133,17 +134,6 @@ func (ts *testServer) stop() {
 	ts.lis.Close()
 }
 
-// A handshake function which simulates a handshake timeout. Tests usually pass
-// `defaultTestTimeout` to the ClientHandshake() method. This function just
-// hangs around for twice that duration, thus making sure that the context
-// passes to the credentials code times out.
-func testServerTLSHandshakeTimeout(_ net.Conn) handshakeResult {
-	ctx, cancel := context.WithTimeout(context.Background(), 2*defaultTestTimeout)
-	<-ctx.Done()
-	cancel()
-	return handshakeResult{err: ctx.Err()}
-}
-
 // A handshake function which simulates a successful handshake without client
 // authentication (server does not request for client certificate during the
 // handshake here).
@@ -239,7 +229,7 @@ func newTestContextWithHandshakeInfo(parent context.Context, root, identity cert
 
 // compareAuthInfo compares the AuthInfo received on the client side after a
 // successful handshake with the authInfo available on the testServer.
-func compareAuthInfo(ts *testServer, ai credentials.AuthInfo) error {
+func compareAuthInfo(ctx context.Context, ts *testServer, ai credentials.AuthInfo) error {
 	if ai.AuthType() != "tls" {
 		return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls")
 	}
@@ -251,8 +241,6 @@ func compareAuthInfo(ts *testServer, ai credentials.AuthInfo) error {
 
 	// Read the handshake result from the testServer which contains the TLS
 	// connection state and compare it with the one received on the client-side.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	val, err := ts.hsResult.Receive(ctx)
 	if err != nil {
 		return fmt.Errorf("testServer failed to return handshake result: %v", err)
@@ -341,7 +329,7 @@ func (s) TestClientCredsProviderFailure(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
 			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider)
-			if _, _, err := creds.ClientHandshake(ctx, authority, nil); !strings.Contains(err.Error(), test.wantErr) {
+			if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) {
 				t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
 			}
 		})
@@ -410,13 +398,59 @@ func (s) TestClientCredsSuccess(t *testing.T) {
 			if err != nil {
 				t.Fatalf("ClientHandshake() returned failed: %q", err)
 			}
-			if err := compareAuthInfo(ts, ai); err != nil {
+			if err := compareAuthInfo(ctx, ts, ai); err != nil {
 				t.Fatal(err)
 			}
 		})
 	}
 }
 
+func (s) TestClientCredsHandshakeTimeout(t *testing.T) {
+	clientDone := make(chan struct{})
+	// A handshake function which simulates a handshake timeout from the
+	// server-side by simply blocking on the client-side handshake to timeout
+	// and not writing any handshake data.
+	hErr := errors.New("server handshake error")
+	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
+		<-clientDone
+		return handshakeResult{err: hErr}
+	})
+	defer ts.stop()
+
+	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
+	creds, err := NewClientCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
+	}
+
+	conn, err := net.Dial("tcp", ts.address)
+	if err != nil {
+		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+	}
+	defer conn.Close()
+
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	ctx := newTestContextWithHandshakeInfo(sCtx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
+	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
+		t.Fatal("ClientHandshake() succeeded when expected to timeout")
+	}
+	close(clientDone)
+
+	// Read the handshake result from the testServer and make sure the expected
+	// error is returned.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ts.hsResult.Receive(ctx)
+	if err != nil {
+		t.Fatalf("testServer failed to return handshake result: %v", err)
+	}
+	hsr := val.(handshakeResult)
+	if hsr.err != hErr {
+		t.Fatalf("testServer handshake returned error: %v, want: %v", hsr.err, hErr)
+	}
+}
+
 // TestClientCredsHandshakeFailure verifies different handshake failure cases.
 func (s) TestClientCredsHandshakeFailure(t *testing.T) {
 	tests := []struct {
@@ -433,13 +467,6 @@ func (s) TestClientCredsHandshakeFailure(t *testing.T) {
 			san:           defaultTestCertSAN,
 			wantErr:       "x509: certificate signed by unknown authority",
 		},
-		{
-			desc:          "handshake times out",
-			handshakeFunc: testServerTLSHandshakeTimeout,
-			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
-			san:           defaultTestCertSAN,
-			wantErr:       "context deadline exceeded",
-		},
 		{
 			desc:          "SAN mismatch",
 			handshakeFunc: testServerTLSHandshake,
@@ -534,13 +561,13 @@ func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	if err != nil {
 		t.Fatalf("ClientHandshake() returned failed: %q", err)
 	}
-	if err := compareAuthInfo(ts, ai); err != nil {
+	if err := compareAuthInfo(ctx, ts, ai); err != nil {
 		t.Fatal(err)
 	}
 }
 
-// TestClone verifies the Clone() method.
-func (s) TestClone(t *testing.T) {
+// TestClientClone verifies the Clone() method on client credentials.
+func (s) TestClientClone(t *testing.T) {
 	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 	orig, err := NewClientCredentials(opts)
 	if err != nil {
@@ -549,7 +576,7 @@ func (s) TestClone(t *testing.T) {
 
 	// The credsImpl does not have any exported fields, and it does not make
 	// sense to use any cmp options to look deep into. So, all we make sure here
-	// is that the cloned object points to a different locaiton in memory.
+	// is that the cloned object points to a different location in memory.
 	if clone := orig.Clone(); clone == orig {
 		t.Fatal("return value from Clone() doesn't point to new credentials instance")
 	}
diff --git a/credentials/xds/xds_server_test.go b/credentials/xds/xds_server_test.go
new file mode 100644
index 000000000000..b9c62dbd5553
--- /dev/null
+++ b/credentials/xds/xds_server_test.go
@@ -0,0 +1,492 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/testdata"
+)
+
+func makeClientTLSConfig(t *testing.T, mTLS bool) *tls.Config {
+	t.Helper()
+
+	pemData, err := ioutil.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+
+	var certs []tls.Certificate
+	if mTLS {
+		cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		certs = append(certs, cert)
+	}
+
+	return &tls.Config{
+		Certificates: certs,
+		RootCAs:      roots,
+		ServerName:   "*.test.example.com",
+		// Setting this to true completely turns off the certificate validation
+		// on the client side. So, the client side handshake always seems to
+		// succeed. But if we want to turn this ON, we will need to generate
+		// certificates which work with localhost, or supply a custom
+		// verification function. So, the server credentials tests will rely
+		// solely on the success/failure of the server-side handshake.
+		InsecureSkipVerify: true,
+	}
+}
+
+// Helper function to create a real TLS server credentials which is used as
+// fallback credentials from multiple tests.
+func makeFallbackServerCreds(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return creds
+}
+
+type errorCreds struct {
+	credentials.TransportCredentials
+}
+
+// TestServerCredsWithoutFallback verifies that the call to
+// NewServerCredentials() fails when no fallback is specified.
+func (s) TestServerCredsWithoutFallback(t *testing.T) {
+	if _, err := NewServerCredentials(ServerOptions{}); err == nil {
+		t.Fatal("NewServerCredentials() succeeded without specifying fallback")
+	}
+}
+
+type wrapperConn struct {
+	net.Conn
+	xdsHI    *HandshakeInfo
+	deadline time.Time
+}
+
+func (wc *wrapperConn) XDSHandshakeInfo() *HandshakeInfo {
+	return wc.xdsHI
+}
+
+func (wc *wrapperConn) GetDeadline() time.Time {
+	return wc.deadline
+}
+
+func newWrappedConn(conn net.Conn, xdsHI *HandshakeInfo, deadline time.Time) *wrapperConn {
+	return &wrapperConn{Conn: conn, xdsHI: xdsHI, deadline: deadline}
+}
+
+// TestServerCredsInvalidHandshakeInfo verifies scenarios where the passed in
+// HandshakeInfo is invalid because it does not contain the expected certificate
+// providers.
+func (s) TestServerCredsInvalidHandshakeInfo(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: &errorCreds{}}
+	creds, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	info := NewHandshakeInfo(&fakeProvider{}, nil)
+	conn := newWrappedConn(nil, info, time.Time{})
+	if _, _, err := creds.ServerHandshake(conn); err == nil {
+		t.Fatal("ServerHandshake succeeded without identity certificate provider in HandshakeInfo")
+	}
+}
+
+// TestServerCredsProviderFailure verifies the cases where an expected
+// certificate provider is missing in the HandshakeInfo value in the context.
+func (s) TestServerCredsProviderFailure(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: &errorCreds{}}
+	creds, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	tests := []struct {
+		desc             string
+		rootProvider     certprovider.Provider
+		identityProvider certprovider.Provider
+		wantErr          string
+	}{
+		{
+			desc:             "erroring identity provider",
+			identityProvider: &fakeProvider{err: errors.New("identity provider error")},
+			wantErr:          "identity provider error",
+		},
+		{
+			desc:             "erroring root provider",
+			identityProvider: &fakeProvider{km: &certprovider.KeyMaterial{}},
+			rootProvider:     &fakeProvider{err: errors.New("root provider error")},
+			wantErr:          "root provider error",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			info := NewHandshakeInfo(test.rootProvider, test.identityProvider)
+			conn := newWrappedConn(nil, info, time.Time{})
+			if _, _, err := creds.ServerHandshake(conn); err == nil || !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("ServerHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
+			}
+		})
+	}
+}
+
+// TestServerCredsHandshakeTimeout verifies the case where the client does not
+// send required handshake data before the deadline set on the net.Conn passed
+// to ServerHandshake().
+func (s) TestServerCredsHandshakeTimeout(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: &errorCreds{}}
+	creds, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	// Create a test server which uses the xDS server credentials created above
+	// to perform TLS handshake on incoming connections.
+	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
+		hi := NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"))
+		hi.SetRequireClientCert(true)
+
+		// Create a wrapped conn which can return the HandshakeInfo created
+		// above with a very small deadline.
+		d := time.Now().Add(defaultTestShortTimeout)
+		rawConn.SetDeadline(d)
+		conn := newWrappedConn(rawConn, hi, d)
+
+		// ServerHandshake() on the xDS credentials is expected to fail.
+		if _, _, err := creds.ServerHandshake(conn); err == nil {
+			return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to timeout")}
+		}
+		return handshakeResult{}
+	})
+	defer ts.stop()
+
+	// Dial the test server, but don't trigger the TLS handshake. This will
+	// cause ServerHandshake() to fail.
+	rawConn, err := net.Dial("tcp", ts.address)
+	if err != nil {
+		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+	}
+	defer rawConn.Close()
+
+	// Read handshake result from the testServer and expect a failure result.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ts.hsResult.Receive(ctx)
+	if err != nil {
+		t.Fatalf("testServer failed to return handshake result: %v", err)
+	}
+	hsr := val.(handshakeResult)
+	if hsr.err != nil {
+		t.Fatalf("testServer handshake failure: %v", hsr.err)
+	}
+}
+
+// TestServerCredsHandshakeFailure verifies the case where the server-side
+// credentials uses a root certificate which does not match the certificate
+// presented by the client, and hence the handshake must fail.
+func (s) TestServerCredsHandshakeFailure(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: &errorCreds{}}
+	creds, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	// Create a test server which uses the xDS server credentials created above
+	// to perform TLS handshake on incoming connections.
+	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
+		// Create a HandshakeInfo which has a root provider which does not match
+		// the certificate sent by the client.
+		hi := NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"))
+		hi.SetRequireClientCert(true)
+
+		// Create a wrapped conn which can return the HandshakeInfo and
+		// configured deadline to the xDS credentials' ServerHandshake()
+		// method.
+		conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout))
+
+		// ServerHandshake() on the xDS credentials is expected to fail.
+		if _, _, err := creds.ServerHandshake(conn); err == nil {
+			return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to fail")}
+		}
+		return handshakeResult{}
+	})
+	defer ts.stop()
+
+	// Dial the test server, and trigger the TLS handshake.
+	rawConn, err := net.Dial("tcp", ts.address)
+	if err != nil {
+		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+	}
+	defer rawConn.Close()
+	tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, true))
+	tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout))
+	if err := tlsConn.Handshake(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read handshake result from the testServer which will return an error if
+	// the handshake succeeded.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ts.hsResult.Receive(ctx)
+	if err != nil {
+		t.Fatalf("testServer failed to return handshake result: %v", err)
+	}
+	hsr := val.(handshakeResult)
+	if hsr.err != nil {
+		t.Fatalf("testServer handshake failure: %v", hsr.err)
+	}
+}
+
+// TestServerCredsHandshakeSuccess verifies success handshake cases.
+func (s) TestServerCredsHandshakeSuccess(t *testing.T) {
+	tests := []struct {
+		desc              string
+		fallbackCreds     credentials.TransportCredentials
+		rootProvider      certprovider.Provider
+		identityProvider  certprovider.Provider
+		requireClientCert bool
+	}{
+		{
+			desc:          "fallback",
+			fallbackCreds: makeFallbackServerCreds(t),
+		},
+		{
+			desc:             "TLS",
+			fallbackCreds:    &errorCreds{},
+			identityProvider: makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"),
+		},
+		{
+			desc:              "mTLS",
+			fallbackCreds:     &errorCreds{},
+			identityProvider:  makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"),
+			rootProvider:      makeRootProvider(t, "x509/client_ca_cert.pem"),
+			requireClientCert: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			// Create an xDS server credentials.
+			opts := ServerOptions{FallbackCreds: test.fallbackCreds}
+			creds, err := NewServerCredentials(opts)
+			if err != nil {
+				t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+			}
+
+			// Create a test server which uses the xDS server credentials
+			// created above to perform TLS handshake on incoming connections.
+			ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
+				// Create a HandshakeInfo with information from the test table.
+				hi := NewHandshakeInfo(test.rootProvider, test.identityProvider)
+				hi.SetRequireClientCert(test.requireClientCert)
+
+				// Create a wrapped conn which can return the HandshakeInfo and
+				// configured deadline to the xDS credentials' ServerHandshake()
+				// method.
+				conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout))
+
+				// Invoke the ServerHandshake() method on the xDS credentials
+				// and make some sanity checks before pushing the result for
+				// inspection by the main test body.
+				_, ai, err := creds.ServerHandshake(conn)
+				if err != nil {
+					return handshakeResult{err: fmt.Errorf("ServerHandshake() failed: %v", err)}
+				}
+				if ai.AuthType() != "tls" {
+					return handshakeResult{err: fmt.Errorf("ServerHandshake returned authType %q, want %q", ai.AuthType(), "tls")}
+				}
+				info, ok := ai.(credentials.TLSInfo)
+				if !ok {
+					return handshakeResult{err: fmt.Errorf("ServerHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})}
+				}
+				return handshakeResult{connState: info.State}
+			})
+			defer ts.stop()
+
+			// Dial the test server, and trigger the TLS handshake.
+			rawConn, err := net.Dial("tcp", ts.address)
+			if err != nil {
+				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+			}
+			defer rawConn.Close()
+			tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, test.requireClientCert))
+			tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout))
+			if err := tlsConn.Handshake(); err != nil {
+				t.Fatal(err)
+			}
+
+			// Read the handshake result from the testServer which contains the
+			// TLS connection state on the server-side and compare it with the
+			// one received on the client-side.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			val, err := ts.hsResult.Receive(ctx)
+			if err != nil {
+				t.Fatalf("testServer failed to return handshake result: %v", err)
+			}
+			hsr := val.(handshakeResult)
+			if hsr.err != nil {
+				t.Fatalf("testServer handshake failure: %v", hsr.err)
+			}
+
+			// AuthInfo contains a variety of information. We only verify a
+			// subset here. This is the same subset which is verified in TLS
+			// credentials tests.
+			if err := compareConnState(tlsConn.ConnectionState(), hsr.connState); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func (s) TestServerCredsProviderSwitch(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: &errorCreds{}}
+	creds, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	// The first time the handshake function is invoked, it returns a
+	// HandshakeInfo which is expected to fail. Further invocations return a
+	// HandshakeInfo which is expected to succeed.
+	cnt := 0
+	// Create a test server which uses the xDS server credentials created above
+	// to perform TLS handshake on incoming connections.
+	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
+		cnt++
+		var hi *HandshakeInfo
+		if cnt == 1 {
+			// Create a HandshakeInfo which has a root provider which does not match
+			// the certificate sent by the client.
+			hi = NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"))
+			hi.SetRequireClientCert(true)
+
+			// Create a wrapped conn which can return the HandshakeInfo and
+			// configured deadline to the xDS credentials' ServerHandshake()
+			// method.
+			conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout))
+
+			// ServerHandshake() on the xDS credentials is expected to fail.
+			if _, _, err := creds.ServerHandshake(conn); err == nil {
+				return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to fail")}
+			}
+			return handshakeResult{}
+		}
+
+		hi = NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
+		hi.SetRequireClientCert(true)
+
+		// Create a wrapped conn which can return the HandshakeInfo and
+		// configured deadline to the xDS credentials' ServerHandshake()
+		// method.
+		conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout))
+
+		// Invoke the ServerHandshake() method on the xDS credentials
+		// and make some sanity checks before pushing the result for
+		// inspection by the main test body.
+		_, ai, err := creds.ServerHandshake(conn)
+		if err != nil {
+			return handshakeResult{err: fmt.Errorf("ServerHandshake() failed: %v", err)}
+		}
+		if ai.AuthType() != "tls" {
+			return handshakeResult{err: fmt.Errorf("ServerHandshake returned authType %q, want %q", ai.AuthType(), "tls")}
+		}
+		info, ok := ai.(credentials.TLSInfo)
+		if !ok {
+			return handshakeResult{err: fmt.Errorf("ServerHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})}
+		}
+		return handshakeResult{connState: info.State}
+	})
+	defer ts.stop()
+
+	for i := 0; i < 5; i++ {
+		// Dial the test server, and trigger the TLS handshake.
+		rawConn, err := net.Dial("tcp", ts.address)
+		if err != nil {
+			t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+		}
+		defer rawConn.Close()
+		tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, true))
+		tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout))
+		if err := tlsConn.Handshake(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Read the handshake result from the testServer which contains the
+		// TLS connection state on the server-side and compare it with the
+		// one received on the client-side.
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		val, err := ts.hsResult.Receive(ctx)
+		if err != nil {
+			t.Fatalf("testServer failed to return handshake result: %v", err)
+		}
+		hsr := val.(handshakeResult)
+		if hsr.err != nil {
+			t.Fatalf("testServer handshake failure: %v", hsr.err)
+		}
+		if i == 0 {
+			// We expect the first handshake to fail. So, we skip checks which
+			// compare connection state.
+			continue
+		}
+		// AuthInfo contains a variety of information. We only verify a
+		// subset here. This is the same subset which is verified in TLS
+		// credentials tests.
+		if err := compareConnState(tlsConn.ConnectionState(), hsr.connState); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// TestServerClone verifies the Clone() method on client credentials.
+func (s) TestServerClone(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: makeFallbackServerCreds(t)}
+	orig, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	// The credsImpl does not have any exported fields, and it does not make
+	// sense to use any cmp options to look deep into. So, all we make sure here
+	// is that the cloned object points to a different location in memory.
+	if clone := orig.Clone(); clone == orig {
+		t.Fatal("return value from Clone() doesn't point to new credentials instance")
+	}
+}
diff --git a/go.sum b/go.sum
index e437e1300b21..77ee70b44354 100644
--- a/go.sum
+++ b/go.sum
@@ -43,6 +43,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

From 4cd551eca760d21ed6c9aac11d04373830b5c054 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 15 Dec 2020 14:01:04 -0800
Subject: [PATCH 318/481] xds: split routing balancer into config selector +
 cluster manager (#4083)

---
 xds/internal/balancer/balancer.go             |   2 +-
 .../balancerstateaggregator.go                | 101 +++--
 .../balancer/clustermanager/clustermanager.go | 143 +++++++
 .../clustermanager_test.go}                   | 352 ++++-------------
 .../balancer/clustermanager/config.go         |  46 +++
 .../balancer/clustermanager/config_test.go    | 144 +++++++
 .../picker.go}                                |  48 ++-
 xds/internal/balancer/xdsrouting/doc.go       |  20 -
 xds/internal/balancer/xdsrouting/routing.go   | 243 ------------
 .../balancer/xdsrouting/routing_config.go     | 207 ----------
 .../xdsrouting/routing_config_test.go         | 369 ------------------
 .../xdsrouting/routing_picker_test.go         | 176 ---------
 .../xdsrouting => resolver}/matcher.go        | 109 +++---
 .../xdsrouting => resolver}/matcher_header.go |  59 +--
 .../matcher_header_test.go                    |   2 +-
 .../xdsrouting => resolver}/matcher_path.go   |  27 +-
 .../matcher_path_test.go                      |   2 +-
 .../xdsrouting => resolver}/matcher_test.go   |  81 +---
 xds/internal/resolver/serviceconfig.go        | 227 +++++++----
 xds/internal/resolver/serviceconfig_action.go | 186 ---------
 .../resolver/serviceconfig_action_test.go     | 356 -----------------
 xds/internal/resolver/serviceconfig_test.go   | 352 +----------------
 xds/internal/resolver/watch_service_test.go   |   4 +
 xds/internal/resolver/xds_resolver.go         |  73 +++-
 xds/internal/resolver/xds_resolver_test.go    | 297 +++++++++++++-
 .../xds_integration_test.go}                  |  33 +-
 .../test/xds_server_integration_test.go       |  24 +-
 xds/internal/testutils/balancer.go            |   2 +-
 28 files changed, 1116 insertions(+), 2569 deletions(-)
 rename xds/internal/balancer/{xdsrouting => clustermanager}/balancerstateaggregator.go (71%)
 create mode 100644 xds/internal/balancer/clustermanager/clustermanager.go
 rename xds/internal/balancer/{xdsrouting/routing_test.go => clustermanager/clustermanager_test.go} (59%)
 create mode 100644 xds/internal/balancer/clustermanager/config.go
 create mode 100644 xds/internal/balancer/clustermanager/config_test.go
 rename xds/internal/balancer/{xdsrouting/routing_picker.go => clustermanager/picker.go} (51%)
 delete mode 100644 xds/internal/balancer/xdsrouting/doc.go
 delete mode 100644 xds/internal/balancer/xdsrouting/routing.go
 delete mode 100644 xds/internal/balancer/xdsrouting/routing_config.go
 delete mode 100644 xds/internal/balancer/xdsrouting/routing_config_test.go
 delete mode 100644 xds/internal/balancer/xdsrouting/routing_picker_test.go
 rename xds/internal/{balancer/xdsrouting => resolver}/matcher.go (50%)
 rename xds/internal/{balancer/xdsrouting => resolver}/matcher_header.go (76%)
 rename xds/internal/{balancer/xdsrouting => resolver}/matcher_header_test.go (99%)
 rename xds/internal/{balancer/xdsrouting => resolver}/matcher_path.go (78%)
 rename xds/internal/{balancer/xdsrouting => resolver}/matcher_path_test.go (99%)
 rename xds/internal/{balancer/xdsrouting => resolver}/matcher_test.go (62%)
 delete mode 100644 xds/internal/resolver/serviceconfig_action.go
 delete mode 100644 xds/internal/resolver/serviceconfig_action_test.go
 rename xds/internal/{balancer/xdsrouting/logging.go => test/xds_integration_test.go} (54%)

diff --git a/xds/internal/balancer/balancer.go b/xds/internal/balancer/balancer.go
index d0b1bb786c7d..5883027a2c52 100644
--- a/xds/internal/balancer/balancer.go
+++ b/xds/internal/balancer/balancer.go
@@ -21,7 +21,7 @@ package balancer
 
 import (
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"    // Register the CDS balancer
+	_ "google.golang.org/grpc/xds/internal/balancer/clustermanager" // Register the xds_cluster_manager balancer
 	_ "google.golang.org/grpc/xds/internal/balancer/edsbalancer"    // Register the EDS balancer
 	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget" // Register the weighted_target balancer
-	_ "google.golang.org/grpc/xds/internal/balancer/xdsrouting"     // Register the xds_routing balancer
 )
diff --git a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go b/xds/internal/balancer/clustermanager/balancerstateaggregator.go
similarity index 71%
rename from xds/internal/balancer/xdsrouting/balancerstateaggregator.go
rename to xds/internal/balancer/clustermanager/balancerstateaggregator.go
index dd613993629b..35eb86c3590c 100644
--- a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go
+++ b/xds/internal/balancer/clustermanager/balancerstateaggregator.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xdsrouting
+package clustermanager
 
 import (
 	"fmt"
@@ -47,8 +47,6 @@ type balancerStateAggregator struct {
 	logger *grpclog.PrefixLogger
 
 	mu sync.Mutex
-	// routes, one for each matcher.
-	routes []route
 	// If started is false, no updates should be sent to the parent cc. A closed
 	// sub-balancer could still send pickers to this aggregator. This makes sure
 	// that no updates will be forwarded to parent when the whole balancer group
@@ -71,29 +69,29 @@ func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLo
 
 // Start starts the aggregator. It can be called after Close to restart the
 // aggretator.
-func (rbsa *balancerStateAggregator) start() {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	rbsa.started = true
+func (bsa *balancerStateAggregator) start() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.started = true
 }
 
 // Close closes the aggregator. When the aggregator is closed, it won't call
 // parent ClientConn to update balancer state.
-func (rbsa *balancerStateAggregator) close() {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	rbsa.started = false
-	rbsa.clearStates()
+func (bsa *balancerStateAggregator) close() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.started = false
+	bsa.clearStates()
 }
 
 // add adds a sub-balancer state with weight. It adds a place holder, and waits
 // for the real sub-balancer to update state.
 //
-// This is called when there's a new action.
-func (rbsa *balancerStateAggregator) add(id string) {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	rbsa.idToPickerState[id] = &subBalancerState{
+// This is called when there's a new child.
+func (bsa *balancerStateAggregator) add(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.idToPickerState[id] = &subBalancerState{
 		// Start everything in CONNECTING, so if one of the sub-balancers
 		// reports TransientFailure, the RPCs will still wait for the other
 		// sub-balancers.
@@ -108,35 +106,26 @@ func (rbsa *balancerStateAggregator) add(id string) {
 // remove removes the sub-balancer state. Future updates from this sub-balancer,
 // if any, will be ignored.
 //
-// This is called when an action is removed.
-func (rbsa *balancerStateAggregator) remove(id string) {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	if _, ok := rbsa.idToPickerState[id]; !ok {
+// This is called when a child is removed.
+func (bsa *balancerStateAggregator) remove(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	if _, ok := bsa.idToPickerState[id]; !ok {
 		return
 	}
 	// Remove id and picker from picker map. This also results in future updates
 	// for this ID to be ignored.
-	delete(rbsa.idToPickerState, id)
-}
-
-// updateRoutes updates the routes. Note that it doesn't trigger an update to
-// the parent ClientConn. The caller should decide when it's necessary, and call
-// buildAndUpdate.
-func (rbsa *balancerStateAggregator) updateRoutes(newRoutes []route) {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	rbsa.routes = newRoutes
+	delete(bsa.idToPickerState, id)
 }
 
 // UpdateState is called to report a balancer state change from sub-balancer.
 // It's usually called by the balancer group.
 //
 // It calls parent ClientConn's UpdateState with the new aggregated state.
-func (rbsa *balancerStateAggregator) UpdateState(id string, state balancer.State) {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	pickerSt, ok := rbsa.idToPickerState[id]
+func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	pickerSt, ok := bsa.idToPickerState[id]
 	if !ok {
 		// All state starts with an entry in pickStateMap. If ID is not in map,
 		// it's either removed, or never existed.
@@ -151,18 +140,18 @@ func (rbsa *balancerStateAggregator) UpdateState(id string, state balancer.State
 	}
 	pickerSt.state = state
 
-	if !rbsa.started {
+	if !bsa.started {
 		return
 	}
-	rbsa.cc.UpdateState(rbsa.build())
+	bsa.cc.UpdateState(bsa.build())
 }
 
 // clearState Reset everything to init state (Connecting) but keep the entry in
 // map (to keep the weight).
 //
-// Caller must hold rbsa.mu.
-func (rbsa *balancerStateAggregator) clearStates() {
-	for _, pState := range rbsa.idToPickerState {
+// Caller must hold bsa.mu.
+func (bsa *balancerStateAggregator) clearStates() {
+	for _, pState := range bsa.idToPickerState {
 		pState.state = balancer.State{
 			ConnectivityState: connectivity.Connecting,
 			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
@@ -173,19 +162,19 @@ func (rbsa *balancerStateAggregator) clearStates() {
 
 // buildAndUpdate combines the sub-state from each sub-balancer into one state,
 // and update it to parent ClientConn.
-func (rbsa *balancerStateAggregator) buildAndUpdate() {
-	rbsa.mu.Lock()
-	defer rbsa.mu.Unlock()
-	if !rbsa.started {
+func (bsa *balancerStateAggregator) buildAndUpdate() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	if !bsa.started {
 		return
 	}
-	rbsa.cc.UpdateState(rbsa.build())
+	bsa.cc.UpdateState(bsa.build())
 }
 
-// build combines sub-states into one. The picker will do routing pick.
+// build combines sub-states into one. The picker will do a child pick.
 //
-// Caller must hold rbsa.mu.
-func (rbsa *balancerStateAggregator) build() balancer.State {
+// Caller must hold bsa.mu.
+func (bsa *balancerStateAggregator) build() balancer.State {
 	// TODO: the majority of this function (and UpdateState) is exactly the same
 	// as weighted_target's state aggregator. Try to make a general utility
 	// function/struct to handle the logic.
@@ -195,7 +184,7 @@ func (rbsa *balancerStateAggregator) build() balancer.State {
 	// function to calculate the aggregated connectivity state as in this
 	// function.
 	var readyN, connectingN int
-	for _, ps := range rbsa.idToPickerState {
+	for _, ps := range bsa.idToPickerState {
 		switch ps.stateToAggregate {
 		case connectivity.Ready:
 			readyN++
@@ -214,13 +203,13 @@ func (rbsa *balancerStateAggregator) build() balancer.State {
 	}
 
 	// The picker's return error might not be consistent with the
-	// aggregatedState. Because for routing, we want to always build picker with
-	// all sub-pickers (not even ready sub-pickers), so even if the overall
-	// state is Ready, pick for certain RPCs can behave like Connecting or
-	// TransientFailure.
-	rbsa.logger.Infof("Child pickers with routes: %s, actions: %+v", rbsa.routes, rbsa.idToPickerState)
+	// aggregatedState. Because for this LB policy, we want to always build
+	// picker with all sub-pickers (not only ready sub-pickers), so even if the
+	// overall state is Ready, pick for certain RPCs can behave like Connecting
+	// or TransientFailure.
+	bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState)
 	return balancer.State{
 		ConnectivityState: aggregatedState,
-		Picker:            newPickerGroup(rbsa.routes, rbsa.idToPickerState),
+		Picker:            newPickerGroup(bsa.idToPickerState),
 	}
 }
diff --git a/xds/internal/balancer/clustermanager/clustermanager.go b/xds/internal/balancer/clustermanager/clustermanager.go
new file mode 100644
index 000000000000..da5900ac75cf
--- /dev/null
+++ b/xds/internal/balancer/clustermanager/clustermanager.go
@@ -0,0 +1,143 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clustermanager implements the cluster manager LB policy for xds.
+package clustermanager
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+)
+
+const balancerName = "xds_cluster_manager_experimental"
+
+func init() {
+	balancer.Register(builder{})
+}
+
+type builder struct{}
+
+func (builder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+	b := &bal{}
+	b.logger = prefixLogger(b)
+	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
+	b.stateAggregator.start()
+	b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
+	b.bg.Start()
+	b.logger.Infof("Created")
+	return b
+}
+
+func (builder) Name() string {
+	return balancerName
+}
+
+func (builder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type bal struct {
+	logger *internalgrpclog.PrefixLogger
+
+	// TODO: make this package not dependent on xds specific code. Same as for
+	// weighted target balancer.
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *balancerStateAggregator
+
+	children map[string]childConfig
+}
+
+func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) {
+	update := false
+	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
+
+	// Remove sub-pickers and sub-balancers that are not in the new cluster list.
+	for name := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			b.stateAggregator.remove(name)
+			b.bg.Remove(name)
+			update = true
+		}
+	}
+
+	// For sub-balancers in the new cluster list,
+	// - add to balancer group if it's new,
+	// - forward the address/balancer config update.
+	for name, newT := range newConfig.Children {
+		if _, ok := b.children[name]; !ok {
+			// If this is a new sub-balancer, add it to the picker map.
+			b.stateAggregator.add(name)
+			// Then add to the balancer group.
+			b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name))
+		}
+		// TODO: handle error? How to aggregate errors and return?
+		_ = b.bg.UpdateClientConnState(name, balancer.ClientConnState{
+			ResolverState: resolver.State{
+				Addresses:     addressesSplit[name],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			},
+			BalancerConfig: newT.ChildPolicy.Config,
+		})
+	}
+
+	b.children = newConfig.Children
+	if update {
+		b.stateAggregator.buildAndUpdate()
+	}
+}
+
+func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error {
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+	b.logger.Infof("update with config %+v, resolver state %+v", s.BalancerConfig, s.ResolverState)
+
+	b.updateChildren(s, newConfig)
+	return nil
+}
+
+func (b *bal) ResolverError(err error) {
+	b.bg.ResolverError(err)
+}
+
+func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.bg.UpdateSubConnState(sc, state)
+}
+
+func (b *bal) Close() {
+	b.stateAggregator.close()
+	b.bg.Close()
+}
+
+const prefix = "[xds-cluster-manager-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *bal) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}
diff --git a/xds/internal/balancer/xdsrouting/routing_test.go b/xds/internal/balancer/clustermanager/clustermanager_test.go
similarity index 59%
rename from xds/internal/balancer/xdsrouting/routing_test.go
rename to xds/internal/balancer/clustermanager/clustermanager_test.go
index 90607bc18607..86c377937f36 100644
--- a/xds/internal/balancer/xdsrouting/routing_test.go
+++ b/xds/internal/balancer/clustermanager/clustermanager_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xdsrouting
+package clustermanager
 
 import (
 	"context"
@@ -27,11 +27,12 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/hierarchy"
-	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
@@ -94,7 +95,7 @@ func init() {
 	for i := 0; i < testBackendAddrsCount; i++ {
 		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
 	}
-	rtBuilder = balancer.Get(xdsRoutingName)
+	rtBuilder = balancer.Get(balancerName)
 	rtParser = rtBuilder.(balancer.ConfigParser)
 
 	balancer.Register(&ignoreAttrsRRBuilder{balancer.Get(roundrobin.Name)})
@@ -106,7 +107,7 @@ func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC ba
 	t.Helper()
 	for i := 0; i < 5; i++ {
 		gotSCSt, err := p.Pick(info)
-		if err != wantErr {
+		if fmt.Sprint(err) != fmt.Sprint(wantErr) {
 			t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr)
 		}
 		if !cmp.Equal(gotSCSt.SubConn, wantSC, cmp.AllowUnexported(testutils.TestSubConn{})) {
@@ -115,19 +116,15 @@ func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC ba
 	}
 }
 
-func TestRouting(t *testing.T) {
+func TestClusterPicks(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
 	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
 
 	configJSON1 := `{
-"Action": {
+"children": {
 	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
 	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
-},
-"Route": [
-	{"prefix":"/a/", "action":"cds:cluster_1"},
-	{"prefix":"", "headers":[{"name":"header-1", "exactMatch":"value-1"}], "action":"cds:cluster_2"}
-]
+}
 }`
 
 	config1, err := rtParser.ParseConfig([]byte(configJSON1))
@@ -173,51 +170,39 @@ func TestRouting(t *testing.T) {
 		wantErr  error
 	}{
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
 		},
 		{
 			pickInfo: balancer.PickInfo{
-				FullMethodName: "/z/y",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")),
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"),
 			},
-			wantSC:  m1[wantAddrs[1]],
-			wantErr: nil,
+			wantSC: m1[wantAddrs[1]],
 		},
 		{
 			pickInfo: balancer.PickInfo{
-				FullMethodName: "/z/y",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")),
+				Ctx: SetPickedCluster(context.Background(), "notacluster"),
 			},
-			wantSC:  nil,
-			wantErr: errNoMatchedRouteFound,
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "notacluster"`),
 		},
 	} {
 		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
 	}
 }
 
-// TestRoutingConfigUpdateAddRoute covers the cases the routing balancer
-// receives config update with extra route, but the same actions.
-func TestRoutingConfigUpdateAddRoute(t *testing.T) {
+// TestConfigUpdateAddCluster covers the cases the balancer receives config
+// update with extra clusters.
+func TestConfigUpdateAddCluster(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
 	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
 
 	configJSON1 := `{
-"Action": {
+"children": {
 	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
 	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
-},
-"Route": [
-	{"prefix":"/a/", "action":"cds:cluster_1"},
-	{"path":"/z/y", "action":"cds:cluster_2"}
-]
+}
 }`
 
 	config1, err := rtParser.ParseConfig([]byte(configJSON1))
@@ -262,186 +247,23 @@ func TestRoutingConfigUpdateAddRoute(t *testing.T) {
 		wantSC   balancer.SubConn
 		wantErr  error
 	}{
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
-			wantSC:   m1[wantAddrs[1]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
-			wantSC:   nil,
-			wantErr:  errNoMatchedRouteFound,
-		},
-	} {
-		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
-	}
-
-	// A config update with different routes, but the same actions. Expect a
-	// picker update, but no subconn changes.
-	configJSON2 := `{
-"Action": {
-	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
-	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
-},
-"Route": [
-	{"prefix":"", "headers":[{"name":"header-1", "presentMatch":true}], "action":"cds:cluster_2"},
-	{"prefix":"/a/", "action":"cds:cluster_1"},
-	{"path":"/z/y", "action":"cds:cluster_2"}
-]
-}`
-	config2, err := rtParser.ParseConfig([]byte(configJSON2))
-	if err != nil {
-		t.Fatalf("failed to parse balancer config: %v", err)
-	}
-	// Send update with the same addresses.
-	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState: resolver.State{Addresses: []resolver.Address{
-			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
-			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
-		}},
-		BalancerConfig: config2,
-	}); err != nil {
-		t.Fatalf("failed to update ClientConn state: %v", err)
-	}
-
-	// New change to actions, expect no newSubConn.
-	select {
-	case <-time.After(time.Millisecond * 500):
-	case <-cc.NewSubConnCh:
-		addrs := <-cc.NewSubConnAddrsCh
-		t.Fatalf("unexpected NewSubConn with address %v", addrs)
-	}
-
-	p2 := <-cc.NewPickerCh
-	for _, tt := range []struct {
-		pickInfo balancer.PickInfo
-		wantSC   balancer.SubConn
-		wantErr  error
-	}{
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
-			wantSC:   m1[wantAddrs[1]],
-			wantErr:  nil,
-		},
 		{
 			pickInfo: balancer.PickInfo{
-				FullMethodName: "/a/z",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")),
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"),
 			},
-			wantSC:  m1[wantAddrs[1]],
-			wantErr: nil,
+			wantSC: m1[wantAddrs[0]],
 		},
 		{
 			pickInfo: balancer.PickInfo{
-				FullMethodName: "/c/d",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")),
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"),
 			},
-			wantSC:  nil,
-			wantErr: errNoMatchedRouteFound,
-		},
-	} {
-		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
-	}
-}
-
-// TestRoutingConfigUpdateAddRouteAndAction covers the cases the routing
-// balancer receives config update with extra route and actions.
-func TestRoutingConfigUpdateAddRouteAndAction(t *testing.T) {
-	cc := testutils.NewTestClientConn(t)
-	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
-
-	configJSON1 := `{
-"Action": {
-	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
-	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
-},
-"Route": [
-	{"prefix":"/a/", "action":"cds:cluster_1"},
-	{"path":"/z/y", "action":"cds:cluster_2"}
-]
-}`
-
-	config1, err := rtParser.ParseConfig([]byte(configJSON1))
-	if err != nil {
-		t.Fatalf("failed to parse balancer config: %v", err)
-	}
-
-	// Send the config, and an address with hierarchy path ["cluster_1"].
-	wantAddrs := []resolver.Address{
-		{Addr: testBackendAddrStrs[0], Attributes: nil},
-		{Addr: testBackendAddrStrs[1], Attributes: nil},
-	}
-	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
-		ResolverState: resolver.State{Addresses: []resolver.Address{
-			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
-			hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}),
-		}},
-		BalancerConfig: config1,
-	}); err != nil {
-		t.Fatalf("failed to update ClientConn state: %v", err)
-	}
-
-	m1 := make(map[resolver.Address]balancer.SubConn)
-	// Verify that a subconn is created with the address, and the hierarchy path
-	// in the address is cleared.
-	for range wantAddrs {
-		addrs := <-cc.NewSubConnAddrsCh
-		if len(hierarchy.Get(addrs[0])) != 0 {
-			t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes)
-		}
-		sc := <-cc.NewSubConnCh
-		// Clear the attributes before adding to map.
-		addrs[0].Attributes = nil
-		m1[addrs[0]] = sc
-		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
-		rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
-	}
-
-	p1 := <-cc.NewPickerCh
-	for _, tt := range []struct {
-		pickInfo balancer.PickInfo
-		wantSC   balancer.SubConn
-		wantErr  error
-	}{
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
-			wantSC:   m1[wantAddrs[1]],
-			wantErr:  nil,
+			wantSC: m1[wantAddrs[1]],
 		},
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
-			wantSC:   nil,
-			wantErr:  errNoMatchedRouteFound,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
 		},
 	} {
 		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
@@ -450,16 +272,11 @@ func TestRoutingConfigUpdateAddRouteAndAction(t *testing.T) {
 	// A config update with different routes, and different actions. Expect a
 	// new subconn and a picker update.
 	configJSON2 := `{
-"Action": {
+"children": {
 	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
 	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
 	"cds:cluster_3":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
-},
-"Route": [
-	{"prefix":"", "headers":[{"name":"header-1", "presentMatch":false, "invertMatch":true}], "action":"cds:cluster_3"},
-	{"prefix":"/a/", "action":"cds:cluster_1"},
-	{"path":"/z/y", "action":"cds:cluster_2"}
-]
+}
 }`
 	config2, err := rtParser.ParseConfig([]byte(configJSON2))
 	if err != nil {
@@ -504,56 +321,45 @@ func TestRoutingConfigUpdateAddRouteAndAction(t *testing.T) {
 		wantErr  error
 	}{
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
 		},
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
-			wantSC:   m1[wantAddrs[1]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
 		},
 		{
 			pickInfo: balancer.PickInfo{
-				FullMethodName: "/a/z",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")),
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_3"),
 			},
-			wantSC:  m1[wantAddrs[2]],
-			wantErr: nil,
+			wantSC: m1[wantAddrs[2]],
 		},
 		{
 			pickInfo: balancer.PickInfo{
-				FullMethodName: "/c/d",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")),
+				Ctx: SetPickedCluster(context.Background(), "cds:notacluster"),
 			},
-			wantSC:  nil,
-			wantErr: errNoMatchedRouteFound,
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
 		},
 	} {
 		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
 	}
 }
 
-// TestRoutingConfigUpdateDeleteAll covers the cases the routing balancer receives config
-// update with no routes. Pick should fail with details in error.
+// TestRoutingConfigUpdateDeleteAll covers the cases the balancer receives
+// config update with no clusters. Pick should fail with details in error.
 func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
 	rtb := rtBuilder.Build(cc, balancer.BuildOptions{})
 
 	configJSON1 := `{
-"Action": {
+"children": {
 	"cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] },
 	"cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }
-},
-"Route": [
-	{"prefix":"/a/", "action":"cds:cluster_1"},
-	{"path":"/z/y", "action":"cds:cluster_2"}
-]
+}
 }`
 
 	config1, err := rtParser.ParseConfig([]byte(configJSON1))
@@ -599,30 +405,28 @@ func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
 		wantErr  error
 	}{
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m1[wantAddrs[0]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
 		},
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
-			wantSC:   m1[wantAddrs[1]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
 		},
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
-			wantSC:   nil,
-			wantErr:  errNoMatchedRouteFound,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
 		},
 	} {
 		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
 	}
 
-	// A config update with no routes.
+	// A config update with no clusters.
 	configJSON2 := `{}`
 	config2, err := rtParser.ParseConfig([]byte(configJSON2))
 	if err != nil {
@@ -634,7 +438,7 @@ func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
 		t.Fatalf("failed to update ClientConn state: %v", err)
 	}
 
-	// Expect two remove subconn.
+	// Expect two removed subconns.
 	for range wantAddrs {
 		select {
 		case <-time.After(time.Millisecond * 500):
@@ -645,13 +449,13 @@ func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
 
 	p2 := <-cc.NewPickerCh
 	for i := 0; i < 5; i++ {
-		gotSCSt, err := p2.Pick(balancer.PickInfo{})
-		if err != errNoMatchedRouteFound {
-			t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, errNoMatchedRouteFound)
+		gotSCSt, err := p2.Pick(balancer.PickInfo{Ctx: SetPickedCluster(context.Background(), "cds:notacluster")})
+		if fmt.Sprint(err) != status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`).Error() {
+			t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, `unknown cluster selected for RPC: "cds:notacluster"`)
 		}
 	}
 
-	// Resend the previous config with routes and actions.
+	// Resend the previous config with clusters
 	if err := rtb.UpdateClientConnState(balancer.ClientConnState{
 		ResolverState: resolver.State{Addresses: []resolver.Address{
 			hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}),
@@ -685,24 +489,22 @@ func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
 		wantErr  error
 	}{
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/0"},
-			wantSC:   m2[wantAddrs[0]],
-			wantErr:  nil,
-		},
-		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/a/1"},
-			wantSC:   m2[wantAddrs[0]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"),
+			},
+			wantSC: m2[wantAddrs[0]],
 		},
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/z/y"},
-			wantSC:   m2[wantAddrs[1]],
-			wantErr:  nil,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"),
+			},
+			wantSC: m2[wantAddrs[1]],
 		},
 		{
-			pickInfo: balancer.PickInfo{FullMethodName: "/c/d"},
-			wantSC:   nil,
-			wantErr:  errNoMatchedRouteFound,
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(context.Background(), "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
 		},
 	} {
 		testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
diff --git a/xds/internal/balancer/clustermanager/config.go b/xds/internal/balancer/clustermanager/config.go
new file mode 100644
index 000000000000..3a5625ff19e1
--- /dev/null
+++ b/xds/internal/balancer/clustermanager/config.go
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type childConfig struct {
+	// ChildPolicy is the child policy and it's config.
+	ChildPolicy *internalserviceconfig.BalancerConfig
+}
+
+// lbConfig is the balancer config for xds routing policy.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	Children map[string]childConfig
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	cfg := &lbConfig{}
+	if err := json.Unmarshal(c, cfg); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
diff --git a/xds/internal/balancer/clustermanager/config_test.go b/xds/internal/balancer/clustermanager/config_test.go
new file mode 100644
index 000000000000..3328ba1d300f
--- /dev/null
+++ b/xds/internal/balancer/clustermanager/config_test.go
@@ -0,0 +1,144 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"
+	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
+)
+
+const (
+	testJSONConfig = `{
+      "children":{
+        "cds:cluster_1":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_1"}
+          }]
+        },
+        "weighted:cluster_1_cluster_2_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1" : {
+                  "weight":75,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_2" : {
+                  "weight":25,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+                }
+              }
+            }
+          }]
+        },
+        "weighted:cluster_1_cluster_3_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1": {
+                  "weight":99,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_3": {
+                  "weight":1,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
+                }
+              }
+            }
+          }]
+        }
+      }
+}
+`
+
+	cdsName = "cds_experimental"
+	wtName  = "weighted_target_experimental"
+)
+
+var (
+	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
+	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
+	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
+
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON1  = `{
+	"targets": {
+	  "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+	  "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] }
+	} }`
+	wtConfig1, _  = wtConfigParser.ParseConfig([]byte(wtConfigJSON1))
+	wtConfigJSON2 = `{
+    "targets": {
+      "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+      "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] }
+    } }`
+	wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2))
+)
+
+func Test_parseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				Children: map[string]childConfig{
+					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig1},
+					},
+					"weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig1},
+					},
+					"weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig2},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if d := cmp.Diff(got, tt.want, cmp.AllowUnexported(lbConfig{})); d != "" {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", d)
+			}
+		})
+	}
+}
diff --git a/xds/internal/balancer/xdsrouting/routing_picker.go b/xds/internal/balancer/clustermanager/picker.go
similarity index 51%
rename from xds/internal/balancer/xdsrouting/routing_picker.go
rename to xds/internal/balancer/clustermanager/picker.go
index 2309f144802c..015cd2b7af49 100644
--- a/xds/internal/balancer/xdsrouting/routing_picker.go
+++ b/xds/internal/balancer/clustermanager/picker.go
@@ -16,45 +16,55 @@
  *
  */
 
-package xdsrouting
+package clustermanager
 
 import (
+	"context"
+
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
 
-// pickerGroup contains a list of route matchers and their corresponding
-// pickers. For each pick, the first matched picker is used. If the picker isn't
-// ready, the pick will be queued.
+// pickerGroup contains a list of pickers. If the picker isn't ready, the pick
+// will be queued.
 type pickerGroup struct {
-	routes  []route
 	pickers map[string]balancer.Picker
 }
 
-func newPickerGroup(routes []route, idToPickerState map[string]*subBalancerState) *pickerGroup {
+func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup {
 	pickers := make(map[string]balancer.Picker)
 	for id, st := range idToPickerState {
 		pickers[id] = st.state.Picker
 	}
 	return &pickerGroup{
-		routes:  routes,
 		pickers: pickers,
 	}
 }
 
-var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found")
-
 func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
-	for _, rt := range pg.routes {
-		if rt.m.match(info) {
-			// action from route is the ID for the sub-balancer to use.
-			p, ok := pg.pickers[rt.action]
-			if !ok {
-				return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
-			}
-			return p.Pick(info)
-		}
+	cluster := getPickedCluster(info.Ctx)
+	if p := pg.pickers[cluster]; p != nil {
+		return p.Pick(info)
 	}
-	return balancer.PickResult{}, errNoMatchedRouteFound
+	return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster)
+}
+
+type clusterKey struct{}
+
+func getPickedCluster(ctx context.Context) string {
+	cluster, _ := ctx.Value(clusterKey{}).(string)
+	return cluster
+}
+
+// GetPickedClusterForTesting returns the cluster in the context; to be used
+// for testing only.
+func GetPickedClusterForTesting(ctx context.Context) string {
+	return getPickedCluster(ctx)
+}
+
+// SetPickedCluster adds the selected cluster to the context for the
+// xds_cluster_manager LB policy to pick.
+func SetPickedCluster(ctx context.Context, cluster string) context.Context {
+	return context.WithValue(ctx, clusterKey{}, cluster)
 }
diff --git a/xds/internal/balancer/xdsrouting/doc.go b/xds/internal/balancer/xdsrouting/doc.go
deleted file mode 100644
index 2f00649a0eb0..000000000000
--- a/xds/internal/balancer/xdsrouting/doc.go
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Package xdsrouting implements the routing balancer for xds.
-package xdsrouting
diff --git a/xds/internal/balancer/xdsrouting/routing.go b/xds/internal/balancer/xdsrouting/routing.go
deleted file mode 100644
index fd92fd91b8a8..000000000000
--- a/xds/internal/balancer/xdsrouting/routing.go
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package xdsrouting
-
-import (
-	"encoding/json"
-	"fmt"
-	"regexp"
-
-	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/internal/hierarchy"
-	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/serviceconfig"
-	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
-)
-
-const xdsRoutingName = "xds_routing_experimental"
-
-func init() {
-	balancer.Register(&routingBB{})
-}
-
-type routingBB struct{}
-
-func (rbb *routingBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
-	b := &routingBalancer{}
-	b.logger = prefixLogger(b)
-	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
-	b.stateAggregator.start()
-	b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
-	b.bg.Start()
-	b.logger.Infof("Created")
-	return b
-}
-
-func (rbb *routingBB) Name() string {
-	return xdsRoutingName
-}
-
-func (rbb *routingBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
-	return parseConfig(c)
-}
-
-type route struct {
-	action string
-	m      *compositeMatcher
-}
-
-func (r route) String() string {
-	return r.m.String() + "->" + r.action
-}
-
-type routingBalancer struct {
-	logger *grpclog.PrefixLogger
-
-	// TODO: make this package not dependent on xds specific code. Same as for
-	// weighted target balancer.
-	bg              *balancergroup.BalancerGroup
-	stateAggregator *balancerStateAggregator
-
-	actions map[string]actionConfig
-	routes  []route
-}
-
-func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *lbConfig) (needRebuild bool) {
-	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
-	var rebuildStateAndPicker bool
-
-	// Remove sub-pickers and sub-balancers that are not in the new action list.
-	for name := range rb.actions {
-		if _, ok := newConfig.actions[name]; !ok {
-			rb.stateAggregator.remove(name)
-			rb.bg.Remove(name)
-			// Trigger a state/picker update, because we don't want `ClientConn`
-			// to pick this sub-balancer anymore.
-			rebuildStateAndPicker = true
-		}
-	}
-
-	// For sub-balancers in the new action list,
-	// - add to balancer group if it's new,
-	// - forward the address/balancer config update.
-	for name, newT := range newConfig.actions {
-		if _, ok := rb.actions[name]; !ok {
-			// If this is a new sub-balancer, add weights to the picker map.
-			rb.stateAggregator.add(name)
-			// Then add to the balancer group.
-			rb.bg.Add(name, balancer.Get(newT.ChildPolicy.Name))
-			// Not trigger a state/picker update. Wait for the new sub-balancer
-			// to send its updates.
-		}
-		// Forwards all the update:
-		// - Addresses are from the map after splitting with hierarchy path,
-		// - Top level service config and attributes are the same,
-		// - Balancer config comes from the targets map.
-		//
-		// TODO: handle error? How to aggregate errors and return?
-		_ = rb.bg.UpdateClientConnState(name, balancer.ClientConnState{
-			ResolverState: resolver.State{
-				Addresses:     addressesSplit[name],
-				ServiceConfig: s.ResolverState.ServiceConfig,
-				Attributes:    s.ResolverState.Attributes,
-			},
-			BalancerConfig: newT.ChildPolicy.Config,
-		})
-	}
-
-	rb.actions = newConfig.actions
-	return rebuildStateAndPicker
-}
-
-func routeToMatcher(r routeConfig) (*compositeMatcher, error) {
-	var pathMatcher pathMatcherInterface
-	switch {
-	case r.regex != "":
-		re, err := regexp.Compile(r.regex)
-		if err != nil {
-			return nil, fmt.Errorf("failed to compile regex %q", r.regex)
-		}
-		pathMatcher = newPathRegexMatcher(re)
-	case r.path != "":
-		pathMatcher = newPathExactMatcher(r.path, r.caseInsensitive)
-	default:
-		pathMatcher = newPathPrefixMatcher(r.prefix, r.caseInsensitive)
-	}
-
-	var headerMatchers []headerMatcherInterface
-	for _, h := range r.headers {
-		var matcherT headerMatcherInterface
-		switch {
-		case h.exactMatch != "":
-			matcherT = newHeaderExactMatcher(h.name, h.exactMatch)
-		case h.regexMatch != "":
-			re, err := regexp.Compile(h.regexMatch)
-			if err != nil {
-				return nil, fmt.Errorf("failed to compile regex %q, skipping this matcher", h.regexMatch)
-			}
-			matcherT = newHeaderRegexMatcher(h.name, re)
-		case h.prefixMatch != "":
-			matcherT = newHeaderPrefixMatcher(h.name, h.prefixMatch)
-		case h.suffixMatch != "":
-			matcherT = newHeaderSuffixMatcher(h.name, h.suffixMatch)
-		case h.rangeMatch != nil:
-			matcherT = newHeaderRangeMatcher(h.name, h.rangeMatch.start, h.rangeMatch.end)
-		default:
-			matcherT = newHeaderPresentMatcher(h.name, h.presentMatch)
-		}
-		if h.invertMatch {
-			matcherT = newInvertMatcher(matcherT)
-		}
-		headerMatchers = append(headerMatchers, matcherT)
-	}
-
-	var fractionMatcher *fractionMatcher
-	if r.fraction != nil {
-		fractionMatcher = newFractionMatcher(*r.fraction)
-	}
-	return newCompositeMatcher(pathMatcher, headerMatchers, fractionMatcher), nil
-}
-
-func routesEqual(a, b []route) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		aa := a[i]
-		bb := b[i]
-		if aa.action != bb.action {
-			return false
-		}
-		if !aa.m.equal(bb.m) {
-			return false
-		}
-	}
-	return true
-}
-
-func (rb *routingBalancer) updateRoutes(newConfig *lbConfig) (needRebuild bool, _ error) {
-	var newRoutes []route
-	for _, rt := range newConfig.routes {
-		newMatcher, err := routeToMatcher(rt)
-		if err != nil {
-			return false, err
-		}
-		newRoutes = append(newRoutes, route{action: rt.action, m: newMatcher})
-	}
-	rebuildStateAndPicker := !routesEqual(newRoutes, rb.routes)
-	rb.routes = newRoutes
-
-	if rebuildStateAndPicker {
-		rb.stateAggregator.updateRoutes(rb.routes)
-	}
-	return rebuildStateAndPicker, nil
-}
-
-func (rb *routingBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	newConfig, ok := s.BalancerConfig.(*lbConfig)
-	if !ok {
-		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
-	}
-	rb.logger.Infof("update with config %+v, resolver state %+v", s.BalancerConfig, s.ResolverState)
-
-	rebuildForActions := rb.updateActions(s, newConfig)
-	rebuildForRoutes, err := rb.updateRoutes(newConfig)
-	if err != nil {
-		return fmt.Errorf("xds_routing balancer: failed to update routes: %v", err)
-	}
-
-	if rebuildForActions || rebuildForRoutes {
-		rb.stateAggregator.buildAndUpdate()
-	}
-	return nil
-}
-
-func (rb *routingBalancer) ResolverError(err error) {
-	rb.bg.ResolverError(err)
-}
-
-func (rb *routingBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
-	rb.bg.UpdateSubConnState(sc, state)
-}
-
-func (rb *routingBalancer) Close() {
-	rb.stateAggregator.close()
-	rb.bg.Close()
-}
diff --git a/xds/internal/balancer/xdsrouting/routing_config.go b/xds/internal/balancer/xdsrouting/routing_config.go
deleted file mode 100644
index 526fe27692f0..000000000000
--- a/xds/internal/balancer/xdsrouting/routing_config.go
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package xdsrouting
-
-import (
-	"encoding/json"
-	"fmt"
-
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
-	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
-	"google.golang.org/grpc/serviceconfig"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
-)
-
-type actionConfig struct {
-	// ChildPolicy is the child policy and it's config.
-	ChildPolicy *internalserviceconfig.BalancerConfig
-}
-
-type int64Range struct {
-	start, end int64
-}
-
-type headerMatcher struct {
-	name string
-	// matchSpecifiers
-	invertMatch bool
-
-	// At most one of the following has non-default value.
-	exactMatch, regexMatch, prefixMatch, suffixMatch string
-	rangeMatch                                       *int64Range
-	presentMatch                                     bool
-}
-
-type routeConfig struct {
-	// Path, Prefix and Regex can have at most one set. This is guaranteed by
-	// config parsing.
-	path, prefix, regex string
-	// Indicates if prefix/path matching should be case insensitive. The default
-	// is false (case sensitive).
-	caseInsensitive bool
-
-	headers  []headerMatcher
-	fraction *uint32
-
-	// Action is the name from the action list.
-	action string
-}
-
-// lbConfig is the balancer config for xds routing policy.
-type lbConfig struct {
-	serviceconfig.LoadBalancingConfig
-	routes  []routeConfig
-	actions map[string]actionConfig
-}
-
-// The following structs with `JSON` in name are temporary structs to unmarshal
-// json into. The fields will be read into lbConfig, to be used by the balancer.
-
-// routeJSON is temporary struct for json unmarshal.
-type routeJSON struct {
-	// Path, Prefix and Regex can have at most one non-nil.
-	Path, Prefix, Regex *string
-	CaseInsensitive     bool
-	// Zero or more header matchers.
-	Headers       []*xdsclient.HeaderMatcher
-	MatchFraction *wrapperspb.UInt32Value
-	// Action is the name from the action list.
-	Action string
-}
-
-// lbConfigJSON is temporary struct for json unmarshal.
-type lbConfigJSON struct {
-	Route  []routeJSON
-	Action map[string]actionConfig
-}
-
-func (jc lbConfigJSON) toLBConfig() *lbConfig {
-	var ret lbConfig
-	for _, r := range jc.Route {
-		var tempR routeConfig
-		switch {
-		case r.Path != nil:
-			tempR.path = *r.Path
-		case r.Prefix != nil:
-			tempR.prefix = *r.Prefix
-		case r.Regex != nil:
-			tempR.regex = *r.Regex
-		}
-		tempR.caseInsensitive = r.CaseInsensitive
-		for _, h := range r.Headers {
-			var tempHeader headerMatcher
-			switch {
-			case h.ExactMatch != nil:
-				tempHeader.exactMatch = *h.ExactMatch
-			case h.RegexMatch != nil:
-				tempHeader.regexMatch = *h.RegexMatch
-			case h.PrefixMatch != nil:
-				tempHeader.prefixMatch = *h.PrefixMatch
-			case h.SuffixMatch != nil:
-				tempHeader.suffixMatch = *h.SuffixMatch
-			case h.RangeMatch != nil:
-				tempHeader.rangeMatch = &int64Range{
-					start: h.RangeMatch.Start,
-					end:   h.RangeMatch.End,
-				}
-			case h.PresentMatch != nil:
-				tempHeader.presentMatch = *h.PresentMatch
-			}
-			tempHeader.name = h.Name
-			if h.InvertMatch != nil {
-				tempHeader.invertMatch = *h.InvertMatch
-			}
-			tempR.headers = append(tempR.headers, tempHeader)
-		}
-		if r.MatchFraction != nil {
-			tempR.fraction = &r.MatchFraction.Value
-		}
-		tempR.action = r.Action
-		ret.routes = append(ret.routes, tempR)
-	}
-	ret.actions = jc.Action
-	return &ret
-}
-
-func parseConfig(c json.RawMessage) (*lbConfig, error) {
-	var tempConfig lbConfigJSON
-	if err := json.Unmarshal(c, &tempConfig); err != nil {
-		return nil, err
-	}
-
-	// For each route:
-	// - at most one of path/prefix/regex.
-	// - action is in action list.
-
-	allRouteActions := make(map[string]bool)
-	for _, r := range tempConfig.Route {
-		var oneOfCount int
-		if r.Path != nil {
-			oneOfCount++
-		}
-		if r.Prefix != nil {
-			oneOfCount++
-		}
-		if r.Regex != nil {
-			oneOfCount++
-		}
-		if oneOfCount != 1 {
-			return nil, fmt.Errorf("%d (not exactly one) of path/prefix/regex is set in route %+v", oneOfCount, r)
-		}
-
-		for _, h := range r.Headers {
-			var oneOfCountH int
-			if h.ExactMatch != nil {
-				oneOfCountH++
-			}
-			if h.RegexMatch != nil {
-				oneOfCountH++
-			}
-			if h.PrefixMatch != nil {
-				oneOfCountH++
-			}
-			if h.SuffixMatch != nil {
-				oneOfCountH++
-			}
-			if h.RangeMatch != nil {
-				oneOfCountH++
-			}
-			if h.PresentMatch != nil {
-				oneOfCountH++
-			}
-			if oneOfCountH != 1 {
-				return nil, fmt.Errorf("%d (not exactly one) of header matcher specifier is set in route %+v", oneOfCountH, h)
-			}
-		}
-
-		if _, ok := tempConfig.Action[r.Action]; !ok {
-			return nil, fmt.Errorf("action %q from route %+v is not found in action list", r.Action, r)
-		}
-		allRouteActions[r.Action] = true
-	}
-
-	// Verify that actions are used by at least one route.
-	for n := range tempConfig.Action {
-		if _, ok := allRouteActions[n]; !ok {
-			return nil, fmt.Errorf("action %q is not used by any route", n)
-		}
-	}
-
-	return tempConfig.toLBConfig(), nil
-}
diff --git a/xds/internal/balancer/xdsrouting/routing_config_test.go b/xds/internal/balancer/xdsrouting/routing_config_test.go
deleted file mode 100644
index 8cb4d22724e3..000000000000
--- a/xds/internal/balancer/xdsrouting/routing_config_test.go
+++ /dev/null
@@ -1,369 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package xdsrouting
-
-import (
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/balancer"
-	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
-	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"
-	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
-)
-
-const (
-	testJSONConfig = `{
-      "action":{
-        "cds:cluster_1":{
-          "childPolicy":[{
-            "cds_experimental":{"cluster":"cluster_1"}
-          }]
-        },
-        "weighted:cluster_1_cluster_2_1":{
-          "childPolicy":[{
-            "weighted_target_experimental":{
-              "targets": {
-                "cluster_1" : {
-                  "weight":75,
-                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
-                },
-                "cluster_2" : {
-                  "weight":25,
-                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
-                }
-              }
-            }
-          }]
-        },
-        "weighted:cluster_1_cluster_3_1":{
-          "childPolicy":[{
-            "weighted_target_experimental":{
-              "targets": {
-                "cluster_1": {
-                  "weight":99,
-                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
-                },
-                "cluster_3": {
-                  "weight":1,
-                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
-                }
-              }
-            }
-          }]
-        }
-      },
-
-      "route":[{
-        "path":"/service_1/method_1",
-        "action":"cds:cluster_1"
-      },
-      {
-        "path":"/service_1/method_2",
-        "action":"cds:cluster_1"
-      },
-      {
-        "prefix":"/service_2/method_1",
-        "action":"weighted:cluster_1_cluster_2_1"
-      },
-      {
-        "prefix":"/service_2",
-        "action":"weighted:cluster_1_cluster_2_1"
-      },
-      {
-        "regex":"^/service_2/method_3$",
-        "action":"weighted:cluster_1_cluster_3_1"
-      }]
-    }
-`
-
-	testJSONConfigWithAllMatchers = `{
-      "action":{
-        "cds:cluster_1":{
-          "childPolicy":[{
-            "cds_experimental":{"cluster":"cluster_1"}
-          }]
-        },
-        "cds:cluster_2":{
-          "childPolicy":[{
-            "cds_experimental":{"cluster":"cluster_2"}
-          }]
-        },
-        "cds:cluster_3":{
-          "childPolicy":[{
-            "cds_experimental":{"cluster":"cluster_3"}
-          }]
-        }
-      },
-
-      "route":[{
-        "path":"/service_1/method_1",
-        "action":"cds:cluster_1"
-      },
-      {
-        "prefix":"/service_2/method_1",
-        "action":"cds:cluster_1"
-      },
-      {
-        "regex":"^/service_2/method_3$",
-        "action":"cds:cluster_1"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "exactMatch":"value-1", "invertMatch":true}],
-        "action":"cds:cluster_2"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "regexMatch":"^value-1$"}],
-        "action":"cds:cluster_2"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "rangeMatch":{"start":-1, "end":7}}],
-        "action":"cds:cluster_3"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "presentMatch":true}],
-        "action":"cds:cluster_3"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "prefixMatch":"value-1"}],
-        "action":"cds:cluster_2"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "suffixMatch":"value-1"}],
-        "action":"cds:cluster_2"
-      },
-      {
-        "prefix":"",
-        "matchFraction":{"value": 31415},
-        "action":"cds:cluster_3"
-      }]
-    }
-`
-
-	cdsName = "cds_experimental"
-	wtName  = "weighted_target_experimental"
-)
-
-var (
-	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
-	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
-	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
-	cdsConfigJSON2  = `{"cluster":"cluster_2"}`
-	cdsConfig2, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON2))
-	cdsConfigJSON3  = `{"cluster":"cluster_3"}`
-	cdsConfig3, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON3))
-
-	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
-	wtConfigJSON1  = `{
-	"targets": {
-	  "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
-	  "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] }
-	} }`
-	wtConfig1, _  = wtConfigParser.ParseConfig([]byte(wtConfigJSON1))
-	wtConfigJSON2 = `{
-    "targets": {
-      "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
-      "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] }
-    } }`
-	wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2))
-)
-
-func Test_parseConfig(t *testing.T) {
-	tests := []struct {
-		name    string
-		js      string
-		want    *lbConfig
-		wantErr bool
-	}{
-		{
-			name:    "empty json",
-			js:      "",
-			want:    nil,
-			wantErr: true,
-		},
-		{
-			name: "more than one path matcher", // Path matcher is oneof, so this is an error.
-			js: `{
-              "Action":{
-                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
-              },
-              "Route": [{
-                "path":"/service_1/method_1",
-                "prefix":"/service_1/",
-                "action":"cds:cluster_1"
-              }]
-            }`,
-			want:    nil,
-			wantErr: true,
-		},
-		{
-			name: "no path matcher",
-			js: `{
-              "Action":{
-                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
-              },
-              "Route": [{
-                "action":"cds:cluster_1"
-              }]
-            }`,
-			want:    nil,
-			wantErr: true,
-		},
-		{
-			name: "route action not found in action list",
-			js: `{
-              "Action":{},
-              "Route": [{
-                "path":"/service_1/method_1",
-                "action":"cds:cluster_1"
-              }]
-            }`,
-			want:    nil,
-			wantErr: true,
-		},
-		{
-			name: "action list contains action not used",
-			js: `{
-              "Action":{
-                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]},
-                "cds:cluster_not_used":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
-              },
-              "Route": [{
-                "path":"/service_1/method_1",
-                "action":"cds:cluster_1"
-              }]
-            }`,
-			want:    nil,
-			wantErr: true,
-		},
-
-		{
-			name: "no header specifier in header matcher",
-			js: `{
-              "Action":{
-                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
-              },
-              "Route": [{
-                "path":"/service_1/method_1",
-                "headers":[{"name":"header-1"}],
-                "action":"cds:cluster_1"
-              }]
-            }`,
-			want:    nil,
-			wantErr: true,
-		},
-		{
-			name: "more than one header specifier in header matcher",
-			js: `{
-              "Action":{
-                "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}
-              },
-              "Route": [{
-                "path":"/service_1/method_1",
-                "headers":[{"name":"header-1", "prefixMatch":"a", "suffixMatch":"b"}],
-                "action":"cds:cluster_1"
-              }]
-            }`,
-			want:    nil,
-			wantErr: true,
-		},
-
-		{
-			name: "OK with path matchers only",
-			js:   testJSONConfig,
-			want: &lbConfig{
-				routes: []routeConfig{
-					{path: "/service_1/method_1", action: "cds:cluster_1"},
-					{path: "/service_1/method_2", action: "cds:cluster_1"},
-					{prefix: "/service_2/method_1", action: "weighted:cluster_1_cluster_2_1"},
-					{prefix: "/service_2", action: "weighted:cluster_1_cluster_2_1"},
-					{regex: "^/service_2/method_3$", action: "weighted:cluster_1_cluster_3_1"},
-				},
-				actions: map[string]actionConfig{
-					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
-						Name: cdsName, Config: cdsConfig1},
-					},
-					"weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
-						Name: wtName, Config: wtConfig1},
-					},
-					"weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
-						Name: wtName, Config: wtConfig2},
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "OK with all matchers",
-			js:   testJSONConfigWithAllMatchers,
-			want: &lbConfig{
-				routes: []routeConfig{
-					{path: "/service_1/method_1", action: "cds:cluster_1"},
-					{prefix: "/service_2/method_1", action: "cds:cluster_1"},
-					{regex: "^/service_2/method_3$", action: "cds:cluster_1"},
-
-					{prefix: "", headers: []headerMatcher{{name: "header-1", exactMatch: "value-1", invertMatch: true}}, action: "cds:cluster_2"},
-					{prefix: "", headers: []headerMatcher{{name: "header-1", regexMatch: "^value-1$"}}, action: "cds:cluster_2"},
-					{prefix: "", headers: []headerMatcher{{name: "header-1", rangeMatch: &int64Range{start: -1, end: 7}}}, action: "cds:cluster_3"},
-					{prefix: "", headers: []headerMatcher{{name: "header-1", presentMatch: true}}, action: "cds:cluster_3"},
-					{prefix: "", headers: []headerMatcher{{name: "header-1", prefixMatch: "value-1"}}, action: "cds:cluster_2"},
-					{prefix: "", headers: []headerMatcher{{name: "header-1", suffixMatch: "value-1"}}, action: "cds:cluster_2"},
-					{prefix: "", fraction: newUInt32P(31415), action: "cds:cluster_3"},
-				},
-				actions: map[string]actionConfig{
-					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
-						Name: cdsName, Config: cdsConfig1},
-					},
-					"cds:cluster_2": {ChildPolicy: &internalserviceconfig.BalancerConfig{
-						Name: cdsName, Config: cdsConfig2},
-					},
-					"cds:cluster_3": {ChildPolicy: &internalserviceconfig.BalancerConfig{
-						Name: cdsName, Config: cdsConfig3},
-					},
-				},
-			},
-			wantErr: false,
-		},
-	}
-
-	cmpOptions := []cmp.Option{cmp.AllowUnexported(lbConfig{}, routeConfig{}, headerMatcher{}, int64Range{})}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseConfig([]byte(tt.js))
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !cmp.Equal(got, tt.want, cmpOptions...) {
-				t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want, cmpOptions...))
-			}
-		})
-	}
-}
-
-func newUInt32P(i uint32) *uint32 {
-	return &i
-}
diff --git a/xds/internal/balancer/xdsrouting/routing_picker_test.go b/xds/internal/balancer/xdsrouting/routing_picker_test.go
deleted file mode 100644
index fd03c3b975cb..000000000000
--- a/xds/internal/balancer/xdsrouting/routing_picker_test.go
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package xdsrouting
-
-import (
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/xds/internal/testutils"
-)
-
-var (
-	testPickers = []*testutils.TestConstPicker{
-		{SC: testutils.TestSubConns[0]},
-		{SC: testutils.TestSubConns[1]},
-	}
-)
-
-func (s) TestRoutingPickerGroupPick(t *testing.T) {
-	tests := []struct {
-		name string
-
-		routes  []route
-		pickers map[string]*subBalancerState
-		info    balancer.PickInfo
-
-		want    balancer.PickResult
-		wantErr error
-	}{
-		{
-			name:    "empty",
-			wantErr: errNoMatchedRouteFound,
-		},
-		{
-			name: "one route no match",
-			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
-			},
-			pickers: map[string]*subBalancerState{
-				"action-0": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[0],
-				}},
-			},
-			info:    balancer.PickInfo{FullMethodName: "/z/y"},
-			wantErr: errNoMatchedRouteFound,
-		},
-		{
-			name: "one route one match",
-			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
-			},
-			pickers: map[string]*subBalancerState{
-				"action-0": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[0],
-				}},
-			},
-			info: balancer.PickInfo{FullMethodName: "/a/b"},
-			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
-		},
-		{
-			name: "two routes first match",
-			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathPrefixMatcher("/z/", false), nil, nil), action: "action-1"},
-			},
-			pickers: map[string]*subBalancerState{
-				"action-0": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[0],
-				}},
-				"action-1": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[1],
-				}},
-			},
-			info: balancer.PickInfo{FullMethodName: "/a/b"},
-			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
-		},
-		{
-			name: "two routes second match",
-			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathPrefixMatcher("/z/", false), nil, nil), action: "action-1"},
-			},
-			pickers: map[string]*subBalancerState{
-				"action-0": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[0],
-				}},
-				"action-1": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[1],
-				}},
-			},
-			info: balancer.PickInfo{FullMethodName: "/z/y"},
-			want: balancer.PickResult{SubConn: testutils.TestSubConns[1]},
-		},
-		{
-			name: "two routes both match former more specific",
-			routes: []route{
-				{m: newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-1"},
-			},
-			pickers: map[string]*subBalancerState{
-				"action-0": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[0],
-				}},
-				"action-1": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[1],
-				}},
-			},
-			info: balancer.PickInfo{FullMethodName: "/a/b"},
-			// First route is a match, so first action is picked.
-			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
-		},
-		{
-			name: "tow routes both match latter more specific",
-			routes: []route{
-				{m: newCompositeMatcher(newPathPrefixMatcher("/a/", false), nil, nil), action: "action-0"},
-				{m: newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil), action: "action-1"},
-			},
-			pickers: map[string]*subBalancerState{
-				"action-0": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[0],
-				}},
-				"action-1": {state: balancer.State{
-					ConnectivityState: connectivity.Ready,
-					Picker:            testPickers[1],
-				}},
-			},
-			info: balancer.PickInfo{FullMethodName: "/a/b"},
-			// First route is a match, so first action is picked, even though
-			// second is an exact match.
-			want: balancer.PickResult{SubConn: testutils.TestSubConns[0]},
-		},
-	}
-	cmpOpts := []cmp.Option{cmp.AllowUnexported(testutils.TestSubConn{})}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			pg := newPickerGroup(tt.routes, tt.pickers)
-			got, err := pg.Pick(tt.info)
-			t.Logf("Pick(%+v) = {%+v, %+v}", tt.info, got, err)
-			if err != tt.wantErr {
-				t.Errorf("Pick() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !cmp.Equal(got, tt.want, cmpOpts...) {
-				t.Errorf("Pick() got = %v, want %v, diff %s", got, tt.want, cmp.Diff(got, tt.want, cmpOpts...))
-			}
-		})
-	}
-
-}
diff --git a/xds/internal/balancer/xdsrouting/matcher.go b/xds/internal/resolver/matcher.go
similarity index 50%
rename from xds/internal/balancer/xdsrouting/matcher.go
rename to xds/internal/resolver/matcher.go
index 196aefae564b..b7b5f3db0e3e 100644
--- a/xds/internal/balancer/xdsrouting/matcher.go
+++ b/xds/internal/resolver/matcher.go
@@ -16,18 +16,73 @@
  *
  */
 
-package xdsrouting
+package resolver
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 
-	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpcutil"
+	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/metadata"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
+func routeToMatcher(r *xdsclient.Route) (*compositeMatcher, error) {
+	var pathMatcher pathMatcherInterface
+	switch {
+	case r.Regex != nil:
+		re, err := regexp.Compile(*r.Regex)
+		if err != nil {
+			return nil, fmt.Errorf("failed to compile regex %q", *r.Regex)
+		}
+		pathMatcher = newPathRegexMatcher(re)
+	case r.Path != nil:
+		pathMatcher = newPathExactMatcher(*r.Path, r.CaseInsensitive)
+	case r.Prefix != nil:
+		pathMatcher = newPathPrefixMatcher(*r.Prefix, r.CaseInsensitive)
+	default:
+		return nil, fmt.Errorf("illegal route: missing path_matcher")
+	}
+
+	var headerMatchers []headerMatcherInterface
+	for _, h := range r.Headers {
+		var matcherT headerMatcherInterface
+		switch {
+		case h.ExactMatch != nil && *h.ExactMatch != "":
+			matcherT = newHeaderExactMatcher(h.Name, *h.ExactMatch)
+		case h.RegexMatch != nil && *h.RegexMatch != "":
+			re, err := regexp.Compile(*h.RegexMatch)
+			if err != nil {
+				return nil, fmt.Errorf("failed to compile regex %q, skipping this matcher", *h.RegexMatch)
+			}
+			matcherT = newHeaderRegexMatcher(h.Name, re)
+		case h.PrefixMatch != nil && *h.PrefixMatch != "":
+			matcherT = newHeaderPrefixMatcher(h.Name, *h.PrefixMatch)
+		case h.SuffixMatch != nil && *h.SuffixMatch != "":
+			matcherT = newHeaderSuffixMatcher(h.Name, *h.SuffixMatch)
+		case h.RangeMatch != nil:
+			matcherT = newHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End)
+		case h.PresentMatch != nil:
+			matcherT = newHeaderPresentMatcher(h.Name, *h.PresentMatch)
+		default:
+			return nil, fmt.Errorf("illegal route: missing header_match_specifier")
+		}
+		if h.InvertMatch != nil && *h.InvertMatch {
+			matcherT = newInvertMatcher(matcherT)
+		}
+		headerMatchers = append(headerMatchers, matcherT)
+	}
+
+	var fractionMatcher *fractionMatcher
+	if r.Fraction != nil {
+		fractionMatcher = newFractionMatcher(*r.Fraction)
+	}
+	return newCompositeMatcher(pathMatcher, headerMatchers, fractionMatcher), nil
+}
+
 // compositeMatcher.match returns true if all matchers return true.
 type compositeMatcher struct {
 	pm  pathMatcherInterface
@@ -39,17 +94,17 @@ func newCompositeMatcher(pm pathMatcherInterface, hms []headerMatcherInterface,
 	return &compositeMatcher{pm: pm, hms: hms, fm: fm}
 }
 
-func (a *compositeMatcher) match(info balancer.PickInfo) bool {
-	if a.pm != nil && !a.pm.match(info.FullMethodName) {
+func (a *compositeMatcher) match(info iresolver.RPCInfo) bool {
+	if a.pm != nil && !a.pm.match(info.Method) {
 		return false
 	}
 
 	// Call headerMatchers even if md is nil, because routes may match
 	// non-presence of some headers.
 	var md metadata.MD
-	if info.Ctx != nil {
-		md, _ = metadata.FromOutgoingContext(info.Ctx)
-		if extraMD, ok := grpcutil.ExtraMetadata(info.Ctx); ok {
+	if info.Context != nil {
+		md, _ = metadata.FromOutgoingContext(info.Context)
+		if extraMD, ok := grpcutil.ExtraMetadata(info.Context); ok {
 			md = metadata.Join(md, extraMD)
 			// Remove all binary headers. They are hard to match with. May need
 			// to add back if asked by users.
@@ -72,35 +127,6 @@ func (a *compositeMatcher) match(info balancer.PickInfo) bool {
 	return true
 }
 
-func (a *compositeMatcher) equal(mm *compositeMatcher) bool {
-	if a == mm {
-		return true
-	}
-
-	if a == nil || mm == nil {
-		return false
-	}
-
-	if (a.pm != nil || mm.pm != nil) && (a.pm == nil || !a.pm.equal(mm.pm)) {
-		return false
-	}
-
-	if len(a.hms) != len(mm.hms) {
-		return false
-	}
-	for i := range a.hms {
-		if !a.hms[i].equal(mm.hms[i]) {
-			return false
-		}
-	}
-
-	if (a.fm != nil || mm.fm != nil) && (a.fm == nil || !a.fm.equal(mm.fm)) {
-		return false
-	}
-
-	return true
-}
-
 func (a *compositeMatcher) String() string {
 	var ret string
 	if a.pm != nil {
@@ -130,17 +156,6 @@ func (fm *fractionMatcher) match() bool {
 	return t <= fm.fraction
 }
 
-func (fm *fractionMatcher) equal(m *fractionMatcher) bool {
-	if fm == m {
-		return true
-	}
-	if fm == nil || m == nil {
-		return false
-	}
-
-	return fm.fraction == m.fraction
-}
-
 func (fm *fractionMatcher) String() string {
 	return fmt.Sprintf("fraction:%v", fm.fraction)
 }
diff --git a/xds/internal/balancer/xdsrouting/matcher_header.go b/xds/internal/resolver/matcher_header.go
similarity index 76%
rename from xds/internal/balancer/xdsrouting/matcher_header.go
rename to xds/internal/resolver/matcher_header.go
index a900e43f5492..05a92788d7bf 100644
--- a/xds/internal/balancer/xdsrouting/matcher_header.go
+++ b/xds/internal/resolver/matcher_header.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xdsrouting
+package resolver
 
 import (
 	"fmt"
@@ -29,7 +29,6 @@ import (
 
 type headerMatcherInterface interface {
 	match(metadata.MD) bool
-	equal(headerMatcherInterface) bool
 	String() string
 }
 
@@ -62,14 +61,6 @@ func (hem *headerExactMatcher) match(md metadata.MD) bool {
 	return v == hem.exact
 }
 
-func (hem *headerExactMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*headerExactMatcher)
-	if !ok {
-		return false
-	}
-	return hem.key == mm.key && hem.exact == mm.exact
-}
-
 func (hem *headerExactMatcher) String() string {
 	return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact)
 }
@@ -91,14 +82,6 @@ func (hrm *headerRegexMatcher) match(md metadata.MD) bool {
 	return hrm.re.MatchString(v)
 }
 
-func (hrm *headerRegexMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*headerRegexMatcher)
-	if !ok {
-		return false
-	}
-	return hrm.key == mm.key && hrm.re.String() == mm.re.String()
-}
-
 func (hrm *headerRegexMatcher) String() string {
 	return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String())
 }
@@ -123,14 +106,6 @@ func (hrm *headerRangeMatcher) match(md metadata.MD) bool {
 	return false
 }
 
-func (hrm *headerRangeMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*headerRangeMatcher)
-	if !ok {
-		return false
-	}
-	return hrm.key == mm.key && hrm.start == mm.start && hrm.end == mm.end
-}
-
 func (hrm *headerRangeMatcher) String() string {
 	return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end)
 }
@@ -150,14 +125,6 @@ func (hpm *headerPresentMatcher) match(md metadata.MD) bool {
 	return present == hpm.present
 }
 
-func (hpm *headerPresentMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*headerPresentMatcher)
-	if !ok {
-		return false
-	}
-	return hpm.key == mm.key && hpm.present == mm.present
-}
-
 func (hpm *headerPresentMatcher) String() string {
 	return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present)
 }
@@ -179,14 +146,6 @@ func (hpm *headerPrefixMatcher) match(md metadata.MD) bool {
 	return strings.HasPrefix(v, hpm.prefix)
 }
 
-func (hpm *headerPrefixMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*headerPrefixMatcher)
-	if !ok {
-		return false
-	}
-	return hpm.key == mm.key && hpm.prefix == mm.prefix
-}
-
 func (hpm *headerPrefixMatcher) String() string {
 	return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix)
 }
@@ -208,14 +167,6 @@ func (hsm *headerSuffixMatcher) match(md metadata.MD) bool {
 	return strings.HasSuffix(v, hsm.suffix)
 }
 
-func (hsm *headerSuffixMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*headerSuffixMatcher)
-	if !ok {
-		return false
-	}
-	return hsm.key == mm.key && hsm.suffix == mm.suffix
-}
-
 func (hsm *headerSuffixMatcher) String() string {
 	return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix)
 }
@@ -232,14 +183,6 @@ func (i *invertMatcher) match(md metadata.MD) bool {
 	return !i.m.match(md)
 }
 
-func (i *invertMatcher) equal(m headerMatcherInterface) bool {
-	mm, ok := m.(*invertMatcher)
-	if !ok {
-		return false
-	}
-	return i.m.equal(mm.m)
-}
-
 func (i *invertMatcher) String() string {
 	return fmt.Sprintf("invert{%s}", i.m)
 }
diff --git a/xds/internal/balancer/xdsrouting/matcher_header_test.go b/xds/internal/resolver/matcher_header_test.go
similarity index 99%
rename from xds/internal/balancer/xdsrouting/matcher_header_test.go
rename to xds/internal/resolver/matcher_header_test.go
index 3ec73ee90f78..fb87cc5dd329 100644
--- a/xds/internal/balancer/xdsrouting/matcher_header_test.go
+++ b/xds/internal/resolver/matcher_header_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xdsrouting
+package resolver
 
 import (
 	"regexp"
diff --git a/xds/internal/balancer/xdsrouting/matcher_path.go b/xds/internal/resolver/matcher_path.go
similarity index 78%
rename from xds/internal/balancer/xdsrouting/matcher_path.go
rename to xds/internal/resolver/matcher_path.go
index f6f4b7ddd9e1..011d1a94c49c 100644
--- a/xds/internal/balancer/xdsrouting/matcher_path.go
+++ b/xds/internal/resolver/matcher_path.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xdsrouting
+package resolver
 
 import (
 	"regexp"
@@ -25,7 +25,6 @@ import (
 
 type pathMatcherInterface interface {
 	match(path string) bool
-	equal(pathMatcherInterface) bool
 	String() string
 }
 
@@ -53,14 +52,6 @@ func (pem *pathExactMatcher) match(path string) bool {
 	return pem.fullPath == path
 }
 
-func (pem *pathExactMatcher) equal(m pathMatcherInterface) bool {
-	mm, ok := m.(*pathExactMatcher)
-	if !ok {
-		return false
-	}
-	return pem.fullPath == mm.fullPath && pem.caseInsensitive == mm.caseInsensitive
-}
-
 func (pem *pathExactMatcher) String() string {
 	return "pathExact:" + pem.fullPath
 }
@@ -89,14 +80,6 @@ func (ppm *pathPrefixMatcher) match(path string) bool {
 	return strings.HasPrefix(path, ppm.prefix)
 }
 
-func (ppm *pathPrefixMatcher) equal(m pathMatcherInterface) bool {
-	mm, ok := m.(*pathPrefixMatcher)
-	if !ok {
-		return false
-	}
-	return ppm.prefix == mm.prefix && ppm.caseInsensitive == mm.caseInsensitive
-}
-
 func (ppm *pathPrefixMatcher) String() string {
 	return "pathPrefix:" + ppm.prefix
 }
@@ -113,14 +96,6 @@ func (prm *pathRegexMatcher) match(path string) bool {
 	return prm.re.MatchString(path)
 }
 
-func (prm *pathRegexMatcher) equal(m pathMatcherInterface) bool {
-	mm, ok := m.(*pathRegexMatcher)
-	if !ok {
-		return false
-	}
-	return prm.re.String() == mm.re.String()
-}
-
 func (prm *pathRegexMatcher) String() string {
 	return "pathRegex:" + prm.re.String()
 }
diff --git a/xds/internal/balancer/xdsrouting/matcher_path_test.go b/xds/internal/resolver/matcher_path_test.go
similarity index 99%
rename from xds/internal/balancer/xdsrouting/matcher_path_test.go
rename to xds/internal/resolver/matcher_path_test.go
index b50d5d84d8bc..263a049108e4 100644
--- a/xds/internal/balancer/xdsrouting/matcher_path_test.go
+++ b/xds/internal/resolver/matcher_path_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xdsrouting
+package resolver
 
 import (
 	"regexp"
diff --git a/xds/internal/balancer/xdsrouting/matcher_test.go b/xds/internal/resolver/matcher_test.go
similarity index 62%
rename from xds/internal/balancer/xdsrouting/matcher_test.go
rename to xds/internal/resolver/matcher_test.go
index 74c570a1a5b7..7657b87bf45f 100644
--- a/xds/internal/balancer/xdsrouting/matcher_test.go
+++ b/xds/internal/resolver/matcher_test.go
@@ -16,15 +16,15 @@
  *
  */
 
-package xdsrouting
+package resolver
 
 import (
 	"context"
 	"testing"
 
-	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpcutil"
+	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -33,16 +33,16 @@ func TestAndMatcherMatch(t *testing.T) {
 		name string
 		pm   pathMatcherInterface
 		hm   headerMatcherInterface
-		info balancer.PickInfo
+		info iresolver.RPCInfo
 		want bool
 	}{
 		{
 			name: "both match",
 			pm:   newPathExactMatcher("/a/b", false),
 			hm:   newHeaderExactMatcher("th", "tv"),
-			info: balancer.PickInfo{
-				FullMethodName: "/a/b",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			info: iresolver.RPCInfo{
+				Method:  "/a/b",
+				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
 			},
 			want: true,
 		},
@@ -50,9 +50,9 @@ func TestAndMatcherMatch(t *testing.T) {
 			name: "both match with path case insensitive",
 			pm:   newPathExactMatcher("/A/B", true),
 			hm:   newHeaderExactMatcher("th", "tv"),
-			info: balancer.PickInfo{
-				FullMethodName: "/a/b",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			info: iresolver.RPCInfo{
+				Method:  "/a/b",
+				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
 			},
 			want: true,
 		},
@@ -60,9 +60,9 @@ func TestAndMatcherMatch(t *testing.T) {
 			name: "only one match",
 			pm:   newPathExactMatcher("/a/b", false),
 			hm:   newHeaderExactMatcher("th", "tv"),
-			info: balancer.PickInfo{
-				FullMethodName: "/z/y",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			info: iresolver.RPCInfo{
+				Method:  "/z/y",
+				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
 			},
 			want: false,
 		},
@@ -70,9 +70,9 @@ func TestAndMatcherMatch(t *testing.T) {
 			name: "both not match",
 			pm:   newPathExactMatcher("/z/y", false),
 			hm:   newHeaderExactMatcher("th", "abc"),
-			info: balancer.PickInfo{
-				FullMethodName: "/a/b",
-				Ctx:            metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
+			info: iresolver.RPCInfo{
+				Method:  "/a/b",
+				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
 			},
 			want: false,
 		},
@@ -80,9 +80,9 @@ func TestAndMatcherMatch(t *testing.T) {
 			name: "fake header",
 			pm:   newPathPrefixMatcher("/", false),
 			hm:   newHeaderExactMatcher("content-type", "fake"),
-			info: balancer.PickInfo{
-				FullMethodName: "/a/b",
-				Ctx: grpcutil.WithExtraMetadata(context.Background(), metadata.Pairs(
+			info: iresolver.RPCInfo{
+				Method: "/a/b",
+				Context: grpcutil.WithExtraMetadata(context.Background(), metadata.Pairs(
 					"content-type", "fake",
 				)),
 			},
@@ -92,9 +92,9 @@ func TestAndMatcherMatch(t *testing.T) {
 			name: "binary header",
 			pm:   newPathPrefixMatcher("/", false),
 			hm:   newHeaderPresentMatcher("t-bin", true),
-			info: balancer.PickInfo{
-				FullMethodName: "/a/b",
-				Ctx: grpcutil.WithExtraMetadata(
+			info: iresolver.RPCInfo{
+				Method: "/a/b",
+				Context: grpcutil.WithExtraMetadata(
 					metadata.NewOutgoingContext(context.Background(), metadata.Pairs("t-bin", "123")), metadata.Pairs(
 						"content-type", "fake",
 					)),
@@ -144,42 +144,3 @@ func TestFractionMatcherMatch(t *testing.T) {
 		t.Errorf("match() = %v, want match", matched)
 	}
 }
-
-func TestCompositeMatcherEqual(t *testing.T) {
-	tests := []struct {
-		name string
-		pm   pathMatcherInterface
-		hms  []headerMatcherInterface
-		fm   *fractionMatcher
-		mm   *compositeMatcher
-		want bool
-	}{
-		{
-			name: "equal",
-			pm:   newPathExactMatcher("/a/b", false),
-			mm:   newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil),
-			want: true,
-		},
-		{
-			name: "no path matcher",
-			pm:   nil,
-			mm:   newCompositeMatcher(nil, nil, nil),
-			want: true,
-		},
-		{
-			name: "not equal",
-			pm:   newPathExactMatcher("/a/b", false),
-			fm:   newFractionMatcher(123),
-			mm:   newCompositeMatcher(newPathExactMatcher("/a/b", false), nil, nil),
-			want: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			a := newCompositeMatcher(tt.pm, tt.hms, tt.fm)
-			if got := a.equal(tt.mm); got != tt.want {
-				t.Errorf("equal() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index f585e4379678..50514303e894 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -21,15 +21,18 @@ package resolver
 import (
 	"encoding/json"
 	"fmt"
+	"sync/atomic"
 
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/codes"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/wrr"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
 )
 
 const (
-	cdsName            = "cds_experimental"
-	weightedTargetName = "weighted_target_experimental"
-	xdsRoutingName     = "xds_routing_experimental"
+	cdsName               = "cds_experimental"
+	xdsClusterManagerName = "xds_cluster_manager_experimental"
 )
 
 type serviceConfig struct {
@@ -42,74 +45,43 @@ func newBalancerConfig(name string, config interface{}) balancerConfig {
 	return []map[string]interface{}{{name: config}}
 }
 
-type weightedCDSBalancerConfig struct {
-	Targets map[string]cdsWithWeight `json:"targets"`
-}
-
-type cdsWithWeight struct {
-	Weight      uint32         `json:"weight"`
-	ChildPolicy balancerConfig `json:"childPolicy"`
-}
-
 type cdsBalancerConfig struct {
 	Cluster string `json:"cluster"`
 }
 
-type route struct {
-	Path            *string                    `json:"path,omitempty"`
-	Prefix          *string                    `json:"prefix,omitempty"`
-	Regex           *string                    `json:"regex,omitempty"`
-	CaseInsensitive bool                       `json:"caseInsensitive"`
-	Headers         []*xdsclient.HeaderMatcher `json:"headers,omitempty"`
-	Fraction        *wrapperspb.UInt32Value    `json:"matchFraction,omitempty"`
-	Action          string                     `json:"action"`
-}
-
-type xdsActionConfig struct {
+type xdsChildConfig struct {
 	ChildPolicy balancerConfig `json:"childPolicy"`
 }
 
-type xdsRoutingBalancerConfig struct {
-	Action map[string]xdsActionConfig `json:"action"`
-	Route  []*route                   `json:"route"`
+type xdsClusterManagerConfig struct {
+	Children map[string]xdsChildConfig `json:"children"`
 }
 
-func (r *xdsResolver) routesToJSON(routes []*xdsclient.Route) (string, error) {
-	r.updateActions(newActionsFromRoutes(routes))
-
-	// Generate routes.
-	var rts []*route
-	for _, rt := range routes {
-		t := &route{
-			Path:            rt.Path,
-			Prefix:          rt.Prefix,
-			Regex:           rt.Regex,
-			Headers:         rt.Headers,
-			CaseInsensitive: rt.CaseInsensitive,
+// pruneActiveClusters deletes entries in r.activeClusters with zero
+// references.
+func (r *xdsResolver) pruneActiveClusters() {
+	for cluster, ci := range r.activeClusters {
+		if atomic.LoadInt32(&ci.refCount) == 0 {
+			delete(r.activeClusters, cluster)
 		}
-
-		if f := rt.Fraction; f != nil {
-			t.Fraction = &wrapperspb.UInt32Value{Value: *f}
-		}
-
-		t.Action = r.getActionAssignedName(rt.Action)
-		rts = append(rts, t)
 	}
+}
 
-	// Generate actions.
-	action := make(map[string]xdsActionConfig)
-	for _, act := range r.actions {
-		action[act.assignedName] = xdsActionConfig{
-			ChildPolicy: weightedClusterToBalancerConfig(act.clustersWithWeights),
+// serviceConfigJSON produces a service config in JSON format representing all
+// the clusters referenced in activeClusters.  This includes clusters with zero
+// references, so they must be pruned first.
+func serviceConfigJSON(activeClusters map[string]*clusterInfo) (string, error) {
+	// Generate children (all entries in activeClusters).
+	children := make(map[string]xdsChildConfig)
+	for cluster := range activeClusters {
+		children[cluster] = xdsChildConfig{
+			ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: cluster}),
 		}
 	}
 
 	sc := serviceConfig{
 		LoadBalancingConfig: newBalancerConfig(
-			xdsRoutingName, xdsRoutingBalancerConfig{
-				Route:  rts,
-				Action: action,
-			},
+			xdsClusterManagerName, xdsClusterManagerConfig{Children: children},
 		),
 	}
 
@@ -120,25 +92,136 @@ func (r *xdsResolver) routesToJSON(routes []*xdsclient.Route) (string, error) {
 	return string(bs), nil
 }
 
-func weightedClusterToBalancerConfig(wc map[string]uint32) balancerConfig {
-	// Even if WeightedCluster has only one entry, we still use weighted_target
-	// as top level balancer, to avoid switching top policy between CDS and
-	// weighted_target, causing TCP connection to be recreated.
-	targets := make(map[string]cdsWithWeight)
-	for name, weight := range wc {
-		targets[name] = cdsWithWeight{
-			Weight:      weight,
-			ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: name}),
+type route struct {
+	action wrr.WRR
+	m      *compositeMatcher // converted from route matchers
+}
+
+func (r route) String() string {
+	return r.m.String() + "->" + fmt.Sprint(r.action)
+}
+
+type configSelector struct {
+	r        *xdsResolver
+	routes   []route
+	clusters map[string]*clusterInfo
+}
+
+var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found")
+
+func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
+	var action wrr.WRR
+	// Loop through routes in order and select first match.
+	for _, rt := range cs.routes {
+		if rt.m.match(rpcInfo) {
+			action = rt.action
+			break
 		}
 	}
-	bc := newBalancerConfig(
-		weightedTargetName, weightedCDSBalancerConfig{
-			Targets: targets,
+	if action == nil {
+		return nil, errNoMatchedRouteFound
+	}
+	cluster, ok := action.Next().(string)
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "error retrieving cluster for match: %v (%T)", cluster, cluster)
+	}
+	// Add a ref to the selected cluster, as this RPC needs this cluster until
+	// it is committed.
+	ref := &cs.clusters[cluster].refCount
+	atomic.AddInt32(ref, 1)
+	return &iresolver.RPCConfig{
+		// Communicate to the LB policy the chosen cluster.
+		Context: clustermanager.SetPickedCluster(rpcInfo.Context, cluster),
+		OnCommitted: func() {
+			// When the RPC is committed, the cluster is no longer required.
+			// Decrease its ref.
+			if v := atomic.AddInt32(ref, -1); v == 0 {
+				// This entry will be removed from activeClusters when
+				// producing the service config for the empty update.
+				select {
+				case cs.r.updateCh <- suWithError{emptyUpdate: true}:
+				default:
+				}
+			}
 		},
-	)
-	return bc
+	}, nil
+}
+
+// incRefs increments refs of all clusters referenced by this config selector.
+func (cs *configSelector) incRefs() {
+	// Loops over cs.clusters, but these are pointers to entries in
+	// activeClusters.
+	for _, ci := range cs.clusters {
+		atomic.AddInt32(&ci.refCount, 1)
+	}
+}
+
+// decRefs decrements refs of all clusters referenced by this config selector.
+func (cs *configSelector) decRefs() {
+	// The resolver's old configSelector may be nil.  Handle that here.
+	if cs == nil {
+		return
+	}
+	// If any refs drop to zero, we'll need a service config update to delete
+	// the cluster.
+	needUpdate := false
+	// Loops over cs.clusters, but these are pointers to entries in
+	// activeClusters.
+	for _, ci := range cs.clusters {
+		if v := atomic.AddInt32(&ci.refCount, -1); v == 0 {
+			needUpdate = true
+		}
+	}
+	// We stop the old config selector immediately after sending a new config
+	// selector; we need another update to delete clusters from the config (if
+	// we don't have another update pending already).
+	if needUpdate {
+		select {
+		case cs.r.updateCh <- suWithError{emptyUpdate: true}:
+		default:
+		}
+	}
+}
+
+// A global for testing.
+var newWRR = wrr.NewRandom
+
+// newConfigSelector creates the config selector for su; may add entries to
+// r.activeClusters for previously-unseen clusters.
+func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, error) {
+	cs := &configSelector{
+		r:        r,
+		routes:   make([]route, len(su.Routes)),
+		clusters: make(map[string]*clusterInfo),
+	}
+
+	for i, rt := range su.Routes {
+		action := newWRR()
+		for cluster, weight := range rt.Action {
+			action.Add(cluster, int64(weight))
+
+			// Initialize entries in cs.clusters map, creating entries in
+			// r.activeClusters as necessary.  Set to zero as they will be
+			// incremented by incRefs.
+			ci := r.activeClusters[cluster]
+			if ci == nil {
+				ci = &clusterInfo{refCount: 0}
+				r.activeClusters[cluster] = ci
+			}
+			cs.clusters[cluster] = ci
+		}
+		cs.routes[i].action = action
+
+		var err error
+		cs.routes[i].m, err = routeToMatcher(rt)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return cs, nil
 }
 
-func (r *xdsResolver) serviceUpdateToJSON(su serviceUpdate) (string, error) {
-	return r.routesToJSON(su.Routes)
+type clusterInfo struct {
+	// number of references to this cluster; accessed atomically
+	refCount int32
 }
diff --git a/xds/internal/resolver/serviceconfig_action.go b/xds/internal/resolver/serviceconfig_action.go
deleted file mode 100644
index d582048fda09..000000000000
--- a/xds/internal/resolver/serviceconfig_action.go
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package resolver
-
-import (
-	"fmt"
-	"math"
-	"sort"
-	"strconv"
-
-	"google.golang.org/grpc/internal/grpcrand"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
-)
-
-type actionWithAssignedName struct {
-	// cluster:weight, "A":40, "B":60
-	clustersWithWeights map[string]uint32
-	// clusterNames, without weights, sorted and hashed, "A_B_"
-	clusterNames string
-	// The assigned name, clusters plus a random number, "A_B_1"
-	assignedName string
-	// randomNumber is the number appended to assignedName.
-	randomNumber int64
-}
-
-// newActionsFromRoutes gets actions from the routes, and turns them into a map
-// keyed by the hash of the clusters.
-//
-// In the returned map, all actions don't have assignedName. The assignedName
-// will be filled in after comparing the new actions with the existing actions,
-// so when a new and old action only diff in weights, the new action can reuse
-// the old action's name.
-//
-// from
-//   {B:60, A:40}, {A:30, B:70}, {B:90, C:10}
-//
-// to
-//   A40_B60_: {{A:40, B:60}, "A_B_", ""}
-//   A30_B70_: {{A:30, B:70}, "A_B_", ""}
-//   B90_C10_: {{B:90, C:10}, "B_C_", ""}
-func newActionsFromRoutes(routes []*xdsclient.Route) map[string]actionWithAssignedName {
-	newActions := make(map[string]actionWithAssignedName)
-	for _, route := range routes {
-		var clusterNames []string
-		for n := range route.Action {
-			clusterNames = append(clusterNames, n)
-		}
-
-		// Sort names to be consistent.
-		sort.Strings(clusterNames)
-		clustersOnly := ""
-		clustersWithWeight := ""
-		for _, c := range clusterNames {
-			// Generates A_B_
-			clustersOnly = clustersOnly + c + "_"
-			// Generates A40_B60_
-			clustersWithWeight = clustersWithWeight + c + strconv.FormatUint(uint64(route.Action[c]), 10) + "_"
-		}
-
-		if _, ok := newActions[clustersWithWeight]; !ok {
-			newActions[clustersWithWeight] = actionWithAssignedName{
-				clustersWithWeights: route.Action,
-				clusterNames:        clustersOnly,
-			}
-		}
-	}
-	return newActions
-}
-
-// updateActions takes a new map of actions, and updates the existing action map in the resolver.
-//
-// In the old map, all actions have assignedName set.
-// In the new map, all actions have no assignedName.
-//
-// After the update, the action map is updated to have all actions from the new
-// map, with assignedName:
-// - if the new action exists in old, get the old name
-// - if the new action doesn't exist in old
-//   - if there is an old action that will be removed, and has the same set of
-//   clusters, reuse the old action's name
-//   - otherwise, generate a new name
-func (r *xdsResolver) updateActions(newActions map[string]actionWithAssignedName) {
-	if r.actions == nil {
-		r.actions = make(map[string]actionWithAssignedName)
-	}
-
-	// Delete actions from existingActions if they are not in newActions. Keep
-	// the removed actions in a map, with key as clusterNames without weights,
-	// so their assigned names can be reused.
-	existingActions := r.actions
-	actionsRemoved := make(map[string][]string)
-	for actionHash, act := range existingActions {
-		if _, ok := newActions[actionHash]; !ok {
-			actionsRemoved[act.clusterNames] = append(actionsRemoved[act.clusterNames], act.assignedName)
-			delete(existingActions, actionHash)
-		}
-	}
-
-	// Find actions in newActions but not in oldActions. Add them, and try to
-	// reuse assigned names from actionsRemoved.
-	if r.usedActionNameRandomNumber == nil {
-		r.usedActionNameRandomNumber = make(map[int64]bool)
-	}
-	for actionHash, act := range newActions {
-		if _, ok := existingActions[actionHash]; !ok {
-			if assignedNamed, ok := actionsRemoved[act.clusterNames]; ok {
-				// Reuse the first assigned name from actionsRemoved.
-				act.assignedName = assignedNamed[0]
-				// If there are more names to reuse after this, update the slice
-				// in the map. Otherwise, remove the entry from the map.
-				if len(assignedNamed) > 1 {
-					actionsRemoved[act.clusterNames] = assignedNamed[1:]
-				} else {
-					delete(actionsRemoved, act.clusterNames)
-				}
-				existingActions[actionHash] = act
-				continue
-			}
-			// Generate a new name.
-			act.randomNumber = r.nextAssignedNameRandomNumber()
-			act.assignedName = fmt.Sprintf("%s%d", act.clusterNames, act.randomNumber)
-			existingActions[actionHash] = act
-		}
-	}
-
-	// Delete entry from nextIndex if all actions with the clusters are removed.
-	remainingRandomNumbers := make(map[int64]bool)
-	for _, act := range existingActions {
-		remainingRandomNumbers[act.randomNumber] = true
-	}
-	r.usedActionNameRandomNumber = remainingRandomNumbers
-}
-
-var grpcrandInt63n = grpcrand.Int63n
-
-func (r *xdsResolver) nextAssignedNameRandomNumber() int64 {
-	for {
-		t := grpcrandInt63n(math.MaxInt32)
-		if !r.usedActionNameRandomNumber[t] {
-			return t
-		}
-	}
-}
-
-// getActionAssignedName hashes the clusters from the action, and find the
-// assigned action name. The assigned action names are kept in r.actions, with
-// the clusters name hash as map key.
-//
-// The assigned action name is not simply the hash. For example, the hash can be
-// "A40_B60_", but the assigned name can be "A_B_0". It's this way so the action
-// can be reused if only weights are changing.
-func (r *xdsResolver) getActionAssignedName(action map[string]uint32) string {
-	var clusterNames []string
-	for n := range action {
-		clusterNames = append(clusterNames, n)
-	}
-	// Hash cluster names. Sort names to be consistent.
-	sort.Strings(clusterNames)
-	clustersWithWeight := ""
-	for _, c := range clusterNames {
-		// Generates hash "A40_B60_".
-		clustersWithWeight = clustersWithWeight + c + strconv.FormatUint(uint64(action[c]), 10) + "_"
-	}
-	// Look in r.actions for the assigned action name.
-	if act, ok := r.actions[clustersWithWeight]; ok {
-		return act.assignedName
-	}
-	r.logger.Warningf("no assigned name found for action %v", action)
-	return ""
-}
diff --git a/xds/internal/resolver/serviceconfig_action_test.go b/xds/internal/resolver/serviceconfig_action_test.go
deleted file mode 100644
index bfc0e6830155..000000000000
--- a/xds/internal/resolver/serviceconfig_action_test.go
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package resolver
-
-import (
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
-)
-
-func TestNewActionsFromRoutes(t *testing.T) {
-	tests := []struct {
-		name   string
-		routes []*xdsclient.Route
-		want   map[string]actionWithAssignedName
-	}{
-		{
-			name: "temp",
-			routes: []*xdsclient.Route{
-				{Action: map[string]uint32{"B": 60, "A": 40}},
-				{Action: map[string]uint32{"A": 30, "B": 70}},
-				{Action: map[string]uint32{"B": 90, "C": 10}},
-			},
-			want: map[string]actionWithAssignedName{
-				"A40_B60_": {map[string]uint32{"A": 40, "B": 60}, "A_B_", "", 0},
-				"A30_B70_": {map[string]uint32{"A": 30, "B": 70}, "A_B_", "", 0},
-				"B90_C10_": {map[string]uint32{"B": 90, "C": 10}, "B_C_", "", 0},
-			},
-		},
-	}
-
-	cmpOpts := []cmp.Option{cmp.AllowUnexported(actionWithAssignedName{})}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := newActionsFromRoutes(tt.routes); !cmp.Equal(got, tt.want, cmpOpts...) {
-				t.Errorf("newActionsFromRoutes() got unexpected result, diff %v", cmp.Diff(got, tt.want, cmpOpts...))
-			}
-		})
-	}
-}
-
-func TestRemoveOrReuseName(t *testing.T) {
-	tests := []struct {
-		name         string
-		oldActions   map[string]actionWithAssignedName
-		oldRandNums  map[int64]bool
-		newActions   map[string]actionWithAssignedName
-		wantActions  map[string]actionWithAssignedName
-		wantRandNums map[int64]bool
-	}{
-		{
-			name: "add same cluster",
-			oldActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-			},
-			oldRandNums: map[int64]bool{
-				0: true,
-			},
-			newActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-				},
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-				},
-			},
-			wantActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_1000",
-					randomNumber:        1000,
-				},
-			},
-			wantRandNums: map[int64]bool{
-				0:    true,
-				1000: true,
-			},
-		},
-		{
-			name: "delete same cluster",
-			oldActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_1",
-					randomNumber:        1,
-				},
-			},
-			oldRandNums: map[int64]bool{
-				0: true,
-				1: true,
-			},
-			newActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-				},
-			},
-			wantActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-			},
-			wantRandNums: map[int64]bool{
-				0: true,
-			},
-		},
-		{
-			name: "add new clusters",
-			oldActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-			},
-			oldRandNums: map[int64]bool{
-				0: true,
-			},
-			newActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-				},
-				"a50_b50_": {
-					clustersWithWeights: map[string]uint32{"a": 50, "b": 50},
-					clusterNames:        "a_b_",
-				},
-			},
-			wantActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-				"a50_b50_": {
-					clustersWithWeights: map[string]uint32{"a": 50, "b": 50},
-					clusterNames:        "a_b_",
-					assignedName:        "a_b_1000",
-					randomNumber:        1000,
-				},
-			},
-			wantRandNums: map[int64]bool{
-				0:    true,
-				1000: true,
-			},
-		},
-		{
-			name: "reuse",
-			oldActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-			},
-			oldRandNums: map[int64]bool{
-				0: true,
-			},
-			newActions: map[string]actionWithAssignedName{
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-				},
-			},
-			wantActions: map[string]actionWithAssignedName{
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-			},
-			wantRandNums: map[int64]bool{
-				0: true,
-			},
-		},
-		{
-			name: "add and reuse",
-			oldActions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_1",
-					randomNumber:        1,
-				},
-				"a50_b50_": {
-					clustersWithWeights: map[string]uint32{"a": 50, "b": 50},
-					clusterNames:        "a_b_",
-					assignedName:        "a_b_2",
-					randomNumber:        2,
-				},
-			},
-			oldRandNums: map[int64]bool{
-				0: true,
-				1: true,
-				2: true,
-			},
-			newActions: map[string]actionWithAssignedName{
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-				},
-				"a30_b30_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 30, "b": 30, "c": 40},
-					clusterNames:        "a_b_c_",
-				},
-				"c50_d50_": {
-					clustersWithWeights: map[string]uint32{"c": 50, "d": 50},
-					clusterNames:        "c_d_",
-				},
-			},
-			wantActions: map[string]actionWithAssignedName{
-				"a10_b50_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_1",
-					randomNumber:        1,
-				},
-				"a30_b30_c40_": {
-					clustersWithWeights: map[string]uint32{"a": 30, "b": 30, "c": 40},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-					randomNumber:        0,
-				},
-				"c50_d50_": {
-					clustersWithWeights: map[string]uint32{"c": 50, "d": 50},
-					clusterNames:        "c_d_",
-					assignedName:        "c_d_1000",
-					randomNumber:        1000,
-				},
-			},
-			wantRandNums: map[int64]bool{
-				0:    true,
-				1:    true,
-				1000: true,
-			},
-		},
-	}
-	cmpOpts := []cmp.Option{cmp.AllowUnexported(actionWithAssignedName{})}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			defer replaceRandNumGenerator(1000)()
-			r := &xdsResolver{
-				actions:                    tt.oldActions,
-				usedActionNameRandomNumber: tt.oldRandNums,
-			}
-			r.updateActions(tt.newActions)
-			if !cmp.Equal(r.actions, tt.wantActions, cmpOpts...) {
-				t.Errorf("removeOrReuseName() got unexpected actions, diff %v", cmp.Diff(r.actions, tt.wantActions, cmpOpts...))
-			}
-			if !cmp.Equal(r.usedActionNameRandomNumber, tt.wantRandNums) {
-				t.Errorf("removeOrReuseName() got unexpected nextIndex, diff %v", cmp.Diff(r.usedActionNameRandomNumber, tt.wantRandNums))
-			}
-		})
-	}
-}
-
-func TestGetActionAssignedName(t *testing.T) {
-	tests := []struct {
-		name    string
-		actions map[string]actionWithAssignedName
-		action  map[string]uint32
-		want    string
-	}{
-		{
-			name: "good",
-			actions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-				},
-			},
-			action: map[string]uint32{"a": 20, "b": 30, "c": 50},
-			want:   "a_b_c_0",
-		},
-		{
-			name: "two",
-			actions: map[string]actionWithAssignedName{
-				"a20_b30_c50_": {
-					clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50},
-					clusterNames:        "a_b_c_",
-					assignedName:        "a_b_c_0",
-				},
-				"c50_d50_": {
-					clustersWithWeights: map[string]uint32{"c": 50, "d": 50},
-					clusterNames:        "c_d_",
-					assignedName:        "c_d_0",
-				},
-			},
-			action: map[string]uint32{"c": 50, "d": 50},
-			want:   "c_d_0",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := &xdsResolver{
-				actions: tt.actions,
-			}
-			if got := r.getActionAssignedName(tt.action); got != tt.want {
-				t.Errorf("getActionAssignedName() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index cce30c183910..1e253841e801 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -22,346 +22,22 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/internal"
-	"google.golang.org/grpc/internal/grpcrand"
-	"google.golang.org/grpc/serviceconfig"
-	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
-	_ "google.golang.org/grpc/xds/internal/balancer/xdsrouting"
-	xdsclient "google.golang.org/grpc/xds/internal/client"
+	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config
 )
 
-const (
-	testCluster1           = "test-cluster-1"
-	testOneClusterOnlyJSON = `{"loadBalancingConfig":[{
-    "xds_routing_experimental":{
-      "action":{
-        "test-cluster-1_0":{
-          "childPolicy":[{
-            "weighted_target_experimental":{
-              "targets":{
-                "test-cluster-1":{
-                  "weight":1,
-                  "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
-                }
-              }}}]
-        }
-      },
-      "route":[{"prefix":"","action":"test-cluster-1_0"}]
-    }}]}`
-	testWeightedCDSJSON = `{"loadBalancingConfig":[{
-    "xds_routing_experimental":{
-      "action":{
-        "cluster_1_cluster_2_1":{
-          "childPolicy":[{
-            "weighted_target_experimental":{
-              "targets":{
-                "cluster_1":{
-                  "weight":75,
-                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
-                },
-                "cluster_2":{
-                  "weight":25,
-                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
-                }
-              }}}]
-        }
-      },
-      "route":[{"prefix":"","action":"cluster_1_cluster_2_1"}]
-    }}]}`
-
-	testRoutingJSON = `{"loadBalancingConfig":[{
-    "xds_routing_experimental": {
-      "action":{
-        "cluster_1_cluster_2_0":{
-          "childPolicy":[{
-			"weighted_target_experimental": {
-			  "targets": {
-				"cluster_1" : {
-				  "weight":75,
-				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
-				},
-				"cluster_2" : {
-				  "weight":25,
-				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
-				}
-			  }
-			}
-          }]
-        }
-      },
-
-      "route":[{
-        "path":"/service_1/method_1",
-        "action":"cluster_1_cluster_2_0"
-      }]
-    }
-    }]}
-`
-	testRoutingAllMatchersJSON = `{"loadBalancingConfig":[{
-    "xds_routing_experimental": {
-      "action":{
-        "cluster_1_0":{
-          "childPolicy":[{
-			"weighted_target_experimental": {
-			  "targets": {
-				"cluster_1" : {
-				  "weight":1,
-				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
-				}
-			  }
-			}
-          }]
-        },
-        "cluster_2_0":{
-          "childPolicy":[{
-			"weighted_target_experimental": {
-			  "targets": {
-				"cluster_2" : {
-				  "weight":1,
-				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
-				}
-			  }
-			}
-          }]
-        },
-        "cluster_3_0":{
-          "childPolicy":[{
-			"weighted_target_experimental": {
-			  "targets": {
-				"cluster_3" : {
-				  "weight":1,
-				  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
-				}
-			  }
-			}
-          }]
-        }
-      },
-
-      "route":[{
-        "path":"/service_1/method_1",
-        "action":"cluster_1_0"
-      },
-      {
-        "prefix":"/service_2/method_1",
-        "caseInsensitive":true,
-        "action":"cluster_1_0"
-      },
-      {
-        "regex":"^/service_2/method_3$",
-        "action":"cluster_1_0"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "exactMatch":"value-1", "invertMatch":true}],
-        "action":"cluster_2_0"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "regexMatch":"^value-1$"}],
-        "action":"cluster_2_0"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "rangeMatch":{"start":-1, "end":7}}],
-        "action":"cluster_3_0"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "presentMatch":true}],
-        "action":"cluster_3_0"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "prefixMatch":"value-1"}],
-        "action":"cluster_2_0"
-      },
-      {
-        "prefix":"",
-        "headers":[{"name":"header-1", "suffixMatch":"value-1"}],
-        "action":"cluster_2_0"
-      },
-      {
-        "prefix":"",
-        "matchFraction":{"value": 31415},
-        "action":"cluster_3_0"
-      }]
-    }
-    }]}
-`
-)
-
-func TestRoutesToJSON(t *testing.T) {
-	tests := []struct {
-		name     string
-		routes   []*xdsclient.Route
-		wantJSON string
-		wantErr  bool
-	}{
-		{
-			name: "one route",
-			routes: []*xdsclient.Route{{
-				Path:   newStringP("/service_1/method_1"),
-				Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25},
-			}},
-			wantJSON: testRoutingJSON,
-			wantErr:  false,
-		},
-		{
-			name: "all matchers",
-			routes: []*xdsclient.Route{
-				{
-					Path:   newStringP("/service_1/method_1"),
-					Action: map[string]uint32{"cluster_1": 1},
-				},
-				{
-					Prefix:          newStringP("/service_2/method_1"),
-					CaseInsensitive: true,
-					Action:          map[string]uint32{"cluster_1": 1},
-				},
-				{
-					Regex:  newStringP("^/service_2/method_3$"),
-					Action: map[string]uint32{"cluster_1": 1},
-				},
-				{
-					Prefix: newStringP(""),
-					Headers: []*xdsclient.HeaderMatcher{{
-						Name:        "header-1",
-						InvertMatch: newBoolP(true),
-						ExactMatch:  newStringP("value-1"),
-					}},
-					Action: map[string]uint32{"cluster_2": 1},
-				},
-				{
-					Prefix: newStringP(""),
-					Headers: []*xdsclient.HeaderMatcher{{
-						Name:       "header-1",
-						RegexMatch: newStringP("^value-1$"),
-					}},
-					Action: map[string]uint32{"cluster_2": 1},
-				},
-				{
-					Prefix: newStringP(""),
-					Headers: []*xdsclient.HeaderMatcher{{
-						Name:       "header-1",
-						RangeMatch: &xdsclient.Int64Range{Start: -1, End: 7},
-					}},
-					Action: map[string]uint32{"cluster_3": 1},
-				},
-				{
-					Prefix: newStringP(""),
-					Headers: []*xdsclient.HeaderMatcher{{
-						Name:         "header-1",
-						PresentMatch: newBoolP(true),
-					}},
-					Action: map[string]uint32{"cluster_3": 1},
-				},
-				{
-					Prefix: newStringP(""),
-					Headers: []*xdsclient.HeaderMatcher{{
-						Name:        "header-1",
-						PrefixMatch: newStringP("value-1"),
-					}},
-					Action: map[string]uint32{"cluster_2": 1},
-				},
-				{
-					Prefix: newStringP(""),
-					Headers: []*xdsclient.HeaderMatcher{{
-						Name:        "header-1",
-						SuffixMatch: newStringP("value-1"),
-					}},
-					Action: map[string]uint32{"cluster_2": 1},
-				},
-				{
-					Prefix:   newStringP(""),
-					Fraction: newUint32P(31415),
-					Action:   map[string]uint32{"cluster_3": 1},
-				},
-			},
-			wantJSON: testRoutingAllMatchersJSON,
-			wantErr:  false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			// Note this random number function only generates 0. This is
-			// because the test doesn't handle action update, and there's only
-			// one action for each cluster bundle.
-			//
-			// This is necessary so the output is deterministic.
-			grpcrandInt63n = func(int64) int64 { return 0 }
-			defer func() { grpcrandInt63n = grpcrand.Int63n }()
-
-			gotJSON, err := (&xdsResolver{}).routesToJSON(tt.routes)
-			if err != nil {
-				t.Errorf("routesToJSON returned error: %v", err)
-				return
-			}
-
-			gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON)
-			wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
-
-			if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
-				t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON)
-				t.Error("gotParsed: ", cmp.Diff(nil, gotParsed))
-				t.Error("wantParsed: ", cmp.Diff(nil, wantParsed))
-			}
-		})
-	}
-}
-
-func TestServiceUpdateToJSON(t *testing.T) {
-	tests := []struct {
-		name     string
-		su       serviceUpdate
-		wantJSON string
-		wantErr  bool
-	}{
-		{
-			name: "routing",
-			su: serviceUpdate{
-				Routes: []*xdsclient.Route{{
-					Path:   newStringP("/service_1/method_1"),
-					Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25},
-				}},
-			},
-			wantJSON: testRoutingJSON,
-			wantErr:  false,
-		},
+func (s) TestPruneActiveClusters(t *testing.T) {
+	r := &xdsResolver{activeClusters: map[string]*clusterInfo{
+		"zero":        {refCount: 0},
+		"one":         {refCount: 1},
+		"two":         {refCount: 2},
+		"anotherzero": {refCount: 0},
+	}}
+	want := map[string]*clusterInfo{
+		"one": {refCount: 1},
+		"two": {refCount: 2},
 	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			defer replaceRandNumGenerator(0)()
-			gotJSON, err := (&xdsResolver{}).serviceUpdateToJSON(tt.su)
-			if err != nil {
-				t.Errorf("serviceUpdateToJSON returned error: %v", err)
-				return
-			}
-
-			gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON)
-			wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON)
-
-			if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
-				t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON)
-				t.Error("gotParsed: ", cmp.Diff(nil, gotParsed))
-				t.Error("wantParsed: ", cmp.Diff(nil, wantParsed))
-			}
-		})
+	r.pruneActiveClusters()
+	if d := cmp.Diff(r.activeClusters, want, cmp.AllowUnexported(clusterInfo{})); d != "" {
+		t.Fatalf("r.activeClusters = %v; want %v\nDiffs: %v", r.activeClusters, want, d)
 	}
 }
-
-// Two updates to the same resolver, test that action names are reused.
-func TestServiceUpdateToJSON_TwoConfig_UpdateActions(t *testing.T) {
-}
-
-func newStringP(s string) *string {
-	return &s
-}
-
-func newBoolP(b bool) *bool {
-	return &b
-}
-
-func newUint32P(i uint32) *uint32 {
-	return &i
-}
diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go
index 6fd2103bcd58..2291d485b8b2 100644
--- a/xds/internal/resolver/watch_service_test.go
+++ b/xds/internal/resolver/watch_service_test.go
@@ -144,6 +144,10 @@ func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantU
 	return nil
 }
 
+func newStringP(s string) *string {
+	return &s
+}
+
 // TestServiceWatch covers the cases:
 // - an update is received after a watch()
 // - an update with routes received
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index 3e12bb5fc29b..faddc2f7bc34 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -26,6 +26,7 @@ import (
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
 
+	iresolver "google.golang.org/grpc/internal/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
@@ -46,10 +47,11 @@ type xdsResolverBuilder struct{}
 // time an xds resolver is built.
 func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
 	r := &xdsResolver{
-		target:   t,
-		cc:       cc,
-		closed:   grpcsync.NewEvent(),
-		updateCh: make(chan suWithError, 1),
+		target:         t,
+		cc:             cc,
+		closed:         grpcsync.NewEvent(),
+		updateCh:       make(chan suWithError, 1),
+		activeClusters: make(map[string]*clusterInfo),
 	}
 	r.logger = prefixLogger((r))
 	r.logger.Infof("Creating resolver for target: %+v", t)
@@ -88,8 +90,9 @@ type xdsClientInterface interface {
 // suWithError wraps the ServiceUpdate and error received through a watch API
 // callback, so that it can pushed onto the update channel as a single entity.
 type suWithError struct {
-	su  serviceUpdate
-	err error
+	su          serviceUpdate
+	emptyUpdate bool
+	err         error
 }
 
 // xdsResolver implements the resolver.Resolver interface.
@@ -112,15 +115,11 @@ type xdsResolver struct {
 	// cancelWatch is the function to cancel the watcher.
 	cancelWatch func()
 
-	// actions is a map from hash of weighted cluster, to the weighted cluster
-	// map, and it's assigned name. E.g.
-	//   "A40_B60_": {{A:40, B:60}, "A_B_", "A_B_0"}
-	//   "A30_B70_": {{A:30, B:70}, "A_B_", "A_B_1"}
-	//   "B90_C10_": {{B:90, C:10}, "B_C_", "B_C_0"}
-	actions map[string]actionWithAssignedName
-	// usedActionNameRandomNumber contains random numbers that have been used in
-	// assigned names, to avoid collision.
-	usedActionNameRandomNumber map[int64]bool
+	// activeClusters is a map from cluster name to a ref count.  Only read or
+	// written during a service update (synchronous).
+	activeClusters map[string]*clusterInfo
+
+	curConfigSelector *configSelector
 }
 
 // run is a long running goroutine which blocks on receiving service updates
@@ -141,23 +140,55 @@ func (r *xdsResolver) run() {
 					r.cc.UpdateState(resolver.State{
 						ServiceConfig: r.cc.ParseServiceConfig("{}"),
 					})
+					// Dereference the active config selector, if one exists.
+					r.curConfigSelector.decRefs()
+					r.curConfigSelector = nil
 					continue
 				}
 				// Send error to ClientConn, and balancers, if error is not
-				// resource not found.
+				// resource not found.  No need to update resolver state if we
+				// can keep using the old config.
 				r.cc.ReportError(update.err)
 				continue
 			}
-			sc, err := r.serviceUpdateToJSON(update.su)
+			var cs *configSelector
+			if !update.emptyUpdate {
+				// Create the config selector for this update.
+				var err error
+				if cs, err = r.newConfigSelector(update.su); err != nil {
+					r.logger.Warningf("Error parsing update on resource %v from xds-client %p: %v", r.target.Endpoint, r.client, err)
+					r.cc.ReportError(err)
+					continue
+				}
+			} else {
+				// Empty update; use the existing config selector.
+				cs = r.curConfigSelector
+			}
+			// Account for this config selector's clusters.
+			cs.incRefs()
+			// Delete entries from r.activeClusters with zero references;
+			// otherwise serviceConfigJSON will generate a config including
+			// them.
+			r.pruneActiveClusters()
+			// Produce the service config.
+			sc, err := serviceConfigJSON(r.activeClusters)
 			if err != nil {
-				r.logger.Warningf("failed to convert update to service config: %v", err)
+				// JSON marshal error; should never happen.
+				r.logger.Errorf("%v", err)
 				r.cc.ReportError(err)
+				cs.decRefs()
 				continue
 			}
 			r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc)
-			r.cc.UpdateState(resolver.State{
+			// Send the update to the ClientConn.
+			state := iresolver.SetConfigSelector(resolver.State{
 				ServiceConfig: r.cc.ParseServiceConfig(sc),
-			})
+			}, cs)
+			r.cc.UpdateState(state)
+			// Decrement references to the old config selector and assign the
+			// new one as the current one.
+			r.curConfigSelector.decRefs()
+			r.curConfigSelector = cs
 		}
 	}
 }
@@ -170,7 +201,7 @@ func (r *xdsResolver) handleServiceUpdate(su serviceUpdate, err error) {
 		// Do not pass updates to the ClientConn once the resolver is closed.
 		return
 	}
-	r.updateCh <- suWithError{su, err}
+	r.updateCh <- suWithError{su: su, err: err}
 }
 
 // ResolveNow is a no-op at this point.
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 3f92ffdc75e5..5f373c036f4c 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -21,6 +21,7 @@ package resolver
 import (
 	"context"
 	"errors"
+	"reflect"
 	"testing"
 	"time"
 
@@ -28,12 +29,16 @@ import (
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpctest"
+	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config
+	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
 
@@ -276,19 +281,66 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	defer replaceRandNumGenerator(0)()
 
 	for _, tt := range []struct {
-		routes   []*xdsclient.Route
-		wantJSON string
+		routes       []*xdsclient.Route
+		wantJSON     string
+		wantClusters map[string]bool
 	}{
 		{
-			routes:   []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}},
-			wantJSON: testOneClusterOnlyJSON,
+			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+			wantJSON: `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "test-cluster-1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
+        }
+      }
+    }}]}`,
+			wantClusters: map[string]bool{"test-cluster-1": true},
 		},
 		{
 			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
 				"cluster_1": 75,
 				"cluster_2": 25,
 			}}},
-			wantJSON: testWeightedCDSJSON,
+			// This update contains the cluster from the previous update as
+			// well as this update, as the previous config selector still
+			// references the old cluster when the new one is pushed.
+			wantJSON: `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "test-cluster-1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
+        },
+        "cluster_1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+        },
+        "cluster_2":{
+          "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+        }
+      }
+    }}]}`,
+			wantClusters: map[string]bool{"cluster_1": true, "cluster_2": true},
+		},
+		{
+			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
+				"cluster_1": 75,
+				"cluster_2": 25,
+			}}},
+			// With this redundant update, the old config selector has been
+			// stopped, so there are no more references to the first cluster.
+			// Only the second update's clusters should remain.
+			wantJSON: `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "cluster_1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+        },
+        "cluster_2":{
+          "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+        }
+      }
+    }}]}`,
+			wantClusters: map[string]bool{"cluster_1": true, "cluster_2": true},
 		},
 	} {
 		// Invoke the watchAPI callback with a good service update and wait for the
@@ -319,6 +371,241 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 			t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
 			t.Error("want: ", cmp.Diff(nil, wantSCParsed.Config))
 		}
+
+		cs := iresolver.GetConfigSelector(rState)
+		if cs == nil {
+			t.Error("received nil config selector")
+			continue
+		}
+
+		pickedClusters := make(map[string]bool)
+		// Odds of picking 75% cluster 100 times in a row: 1 in 3E-13.  And
+		// with the random number generator stubbed out, we can rely on this
+		// to be 100% reproducible.
+		for i := 0; i < 100; i++ {
+			res, err := cs.SelectConfig(iresolver.RPCInfo{Context: context.Background()})
+			if err != nil {
+				t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
+			}
+			cluster := clustermanager.GetPickedClusterForTesting(res.Context)
+			pickedClusters[cluster] = true
+			res.OnCommitted()
+		}
+		if !reflect.DeepEqual(pickedClusters, tt.wantClusters) {
+			t.Errorf("Picked clusters: %v; want: %v", pickedClusters, tt.wantClusters)
+		}
+	}
+}
+
+func (s) TestXDSResolverWRR(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer func() {
+		cancel()
+		xdsR.Close()
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR)
+	newWRR = xdstestutils.NewTestWRR
+
+	// Invoke the watchAPI callback with a good service update and wait for the
+	// UpdateState method to be called on the ClientConn.
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
+					"A": 5,
+					"B": 10,
+				}}},
+			},
+		},
+	}, nil)
+
+	gotState, err := tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState := gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+
+	cs := iresolver.GetConfigSelector(rState)
+	if cs == nil {
+		t.Fatal("received nil config selector")
+	}
+
+	picks := map[string]int{}
+	for i := 0; i < 30; i++ {
+		res, err := cs.SelectConfig(iresolver.RPCInfo{Context: context.Background()})
+		if err != nil {
+			t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
+		}
+		picks[clustermanager.GetPickedClusterForTesting(res.Context)]++
+		res.OnCommitted()
+	}
+	want := map[string]int{"A": 10, "B": 20}
+	if !reflect.DeepEqual(picks, want) {
+		t.Errorf("picked clusters = %v; want %v", picks, want)
+	}
+}
+
+// TestXDSResolverDelayedOnCommitted tests that clusters remain in service
+// config if RPCs are in flight.
+func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer func() {
+		cancel()
+		xdsR.Close()
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	// Invoke the watchAPI callback with a good service update and wait for the
+	// UpdateState method to be called on the ClientConn.
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+			},
+		},
+	}, nil)
+
+	gotState, err := tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState := gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+
+	wantJSON := `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "test-cluster-1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
+        }
+      }
+    }}]}`
+	wantSCParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(wantJSON)
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) {
+		t.Errorf("ClientConn.UpdateState received different service config")
+		t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Fatal("want: ", cmp.Diff(nil, wantSCParsed.Config))
+	}
+
+	cs := iresolver.GetConfigSelector(rState)
+	if cs == nil {
+		t.Fatal("received nil config selector")
+	}
+
+	res, err := cs.SelectConfig(iresolver.RPCInfo{Context: context.Background()})
+	if err != nil {
+		t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
+	}
+	cluster := clustermanager.GetPickedClusterForTesting(res.Context)
+	if cluster != "test-cluster-1" {
+		t.Fatalf("")
+	}
+	// delay res.OnCommitted()
+
+	// Perform TWO updates to ensure the old config selector does not hold a
+	// reference to test-cluster-1.
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"NEW": 1}}},
+			},
+		},
+	}, nil)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"NEW": 1}}},
+			},
+		},
+	}, nil)
+
+	tcc.stateCh.Receive(ctx) // Ignore the first update
+	gotState, err = tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState = gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+	wantJSON2 := `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "test-cluster-1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
+        },
+        "NEW":{
+          "childPolicy":[{"cds_experimental":{"cluster":"NEW"}}]
+        }
+      }
+    }}]}`
+	wantSCParsed2 := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(wantJSON2)
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed2.Config) {
+		t.Errorf("ClientConn.UpdateState received different service config")
+		t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Fatal("want: ", cmp.Diff(nil, wantSCParsed2.Config))
+	}
+
+	// Invoke OnCommitted; should lead to a service config update that deletes
+	// test-cluster-1.
+	res.OnCommitted()
+
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"NEW": 1}}},
+			},
+		},
+	}, nil)
+	gotState, err = tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState = gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+	wantJSON3 := `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "NEW":{
+          "childPolicy":[{"cds_experimental":{"cluster":"NEW"}}]
+        }
+      }
+    }}]}`
+	wantSCParsed3 := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(wantJSON3)
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed3.Config) {
+		t.Errorf("ClientConn.UpdateState received different service config")
+		t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Fatal("want: ", cmp.Diff(nil, wantSCParsed3.Config))
 	}
 }
 
diff --git a/xds/internal/balancer/xdsrouting/logging.go b/xds/internal/test/xds_integration_test.go
similarity index 54%
rename from xds/internal/balancer/xdsrouting/logging.go
rename to xds/internal/test/xds_integration_test.go
index 5c4a6b3cb410..ae306ae7864e 100644
--- a/xds/internal/balancer/xdsrouting/logging.go
+++ b/xds/internal/test/xds_integration_test.go
@@ -1,3 +1,5 @@
+// +build !386
+
 /*
  *
  * Copyright 2020 gRPC authors.
@@ -16,19 +18,34 @@
  *
  */
 
-package xdsrouting
+// Package xds_test contains e2e tests for xDS use.
+package xds_test
 
 import (
-	"fmt"
+	"context"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
 
-	"google.golang.org/grpc/grpclog"
-	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+const (
+	defaultTestTimeout = 10 * time.Second
 )
 
-const prefix = "[xds-routing-lb %p] "
+type s struct {
+	grpctest.Tester
+}
 
-var logger = grpclog.Component("xds")
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type testService struct {
+	testpb.TestServiceServer
+}
 
-func prefixLogger(p *routingBalancer) *internalgrpclog.PrefixLogger {
-	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	return &testpb.Empty{}, nil
 }
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index a934067a4339..cb1072e162e7 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -18,14 +18,13 @@
  *
  */
 
-// Package xds_test contains e2e tests for xDS use on the server.
+// Package xds_test contains e2e tests for xDS use.
 package xds_test
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/uuid"
 	"google.golang.org/protobuf/proto"
@@ -37,7 +36,6 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/grpc/internal/grpctest"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/xds"
 	"google.golang.org/grpc/xds/internal/env"
@@ -46,18 +44,6 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-const (
-	defaultTestTimeout = 10 * time.Second
-)
-
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
 // TestServerSideXDS is an e2e tests for xDS use on the server. This does not
 // use any xDS features because we have not implemented any on the server side.
 func (s) TestServerSideXDS(t *testing.T) {
@@ -148,11 +134,3 @@ func (s) TestServerSideXDS(t *testing.T) {
 		t.Fatalf("rpc EmptyCall() failed: %v", err)
 	}
 }
-
-type testService struct {
-	testpb.TestServiceServer
-}
-
-func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
-	return &testpb.Empty{}, nil
-}
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 0643a060c97e..69e355e353c2 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -116,7 +116,7 @@ func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubCon
 
 // RemoveSubConn removes the SubConn.
 func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
-	tcc.logger.Logf("testClientCOnn: RemoveSubConn(%p)", sc)
+	tcc.logger.Logf("testClientConn: RemoveSubConn(%p)", sc)
 	select {
 	case tcc.RemoveSubConnCh <- sc:
 	default:

From 73671aeeee1ddbd921b3c06a51709aa4c58c0615 Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <garrettgutierrez1@gmail.com>
Date: Tue, 15 Dec 2020 14:35:14 -0800
Subject: [PATCH 319/481] Updated test.proto to facilitate xds interop testing
 (#4109)

---
 interop/grpc_testing/test.pb.go      | 716 ++++++++++++++++++++++-----
 interop/grpc_testing/test.proto      |  46 ++
 interop/grpc_testing/test_grpc.pb.go | 127 +++++
 3 files changed, 770 insertions(+), 119 deletions(-)

diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 8b8178f3ce53..8b4017a81fe9 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -153,6 +153,53 @@ func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) {
 	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{1}
 }
 
+// Type of RPCs to send.
+type ClientConfigureRequest_RpcType int32
+
+const (
+	ClientConfigureRequest_EMPTY_CALL ClientConfigureRequest_RpcType = 0
+	ClientConfigureRequest_UNARY_CALL ClientConfigureRequest_RpcType = 1
+)
+
+// Enum value maps for ClientConfigureRequest_RpcType.
+var (
+	ClientConfigureRequest_RpcType_name = map[int32]string{
+		0: "EMPTY_CALL",
+		1: "UNARY_CALL",
+	}
+	ClientConfigureRequest_RpcType_value = map[string]int32{
+		"EMPTY_CALL": 0,
+		"UNARY_CALL": 1,
+	}
+)
+
+func (x ClientConfigureRequest_RpcType) Enum() *ClientConfigureRequest_RpcType {
+	p := new(ClientConfigureRequest_RpcType)
+	*p = x
+	return p
+}
+
+func (x ClientConfigureRequest_RpcType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ClientConfigureRequest_RpcType) Descriptor() protoreflect.EnumDescriptor {
+	return file_interop_grpc_testing_test_proto_enumTypes[2].Descriptor()
+}
+
+func (ClientConfigureRequest_RpcType) Type() protoreflect.EnumType {
+	return &file_interop_grpc_testing_test_proto_enumTypes[2]
+}
+
+func (x ClientConfigureRequest_RpcType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ClientConfigureRequest_RpcType.Descriptor instead.
+func (ClientConfigureRequest_RpcType) EnumDescriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{14, 0}
+}
+
 type Empty struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -925,6 +972,209 @@ func (x *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerSt
 	return nil
 }
 
+// Request for retrieving a test client's accumulated stats.
+type LoadBalancerAccumulatedStatsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *LoadBalancerAccumulatedStatsRequest) Reset() {
+	*x = LoadBalancerAccumulatedStatsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerAccumulatedStatsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerAccumulatedStatsRequest) ProtoMessage() {}
+
+func (x *LoadBalancerAccumulatedStatsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerAccumulatedStatsRequest.ProtoReflect.Descriptor instead.
+func (*LoadBalancerAccumulatedStatsRequest) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{12}
+}
+
+// Accumulated stats for RPCs sent by a test client.
+type LoadBalancerAccumulatedStatsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The total number of RPCs have ever issued for each type.
+	NumRpcsStartedByMethod map[string]int32 `protobuf:"bytes,1,rep,name=num_rpcs_started_by_method,json=numRpcsStartedByMethod,proto3" json:"num_rpcs_started_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	// The total number of RPCs have ever completed successfully for each type.
+	NumRpcsSucceededByMethod map[string]int32 `protobuf:"bytes,2,rep,name=num_rpcs_succeeded_by_method,json=numRpcsSucceededByMethod,proto3" json:"num_rpcs_succeeded_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	// The total number of RPCs have ever failed for each type.
+	NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) Reset() {
+	*x = LoadBalancerAccumulatedStatsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerAccumulatedStatsResponse) ProtoMessage() {}
+
+func (x *LoadBalancerAccumulatedStatsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerAccumulatedStatsResponse.ProtoReflect.Descriptor instead.
+func (*LoadBalancerAccumulatedStatsResponse) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{13}
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[string]int32 {
+	if x != nil {
+		return x.NumRpcsStartedByMethod
+	}
+	return nil
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map[string]int32 {
+	if x != nil {
+		return x.NumRpcsSucceededByMethod
+	}
+	return nil
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[string]int32 {
+	if x != nil {
+		return x.NumRpcsFailedByMethod
+	}
+	return nil
+}
+
+// Configurations for a test client.
+type ClientConfigureRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The types of RPCs the client sends.
+	Types []ClientConfigureRequest_RpcType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"types,omitempty"`
+	// The collection of custom metadata to be attached to RPCs sent by the client.
+	Metadata []*ClientConfigureRequest_Metadata `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"`
+}
+
+func (x *ClientConfigureRequest) Reset() {
+	*x = ClientConfigureRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfigureRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfigureRequest) ProtoMessage() {}
+
+func (x *ClientConfigureRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfigureRequest.ProtoReflect.Descriptor instead.
+func (*ClientConfigureRequest) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{14}
+}
+
+func (x *ClientConfigureRequest) GetTypes() []ClientConfigureRequest_RpcType {
+	if x != nil {
+		return x.Types
+	}
+	return nil
+}
+
+func (x *ClientConfigureRequest) GetMetadata() []*ClientConfigureRequest_Metadata {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+// Response for updating a test client's configuration.
+type ClientConfigureResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *ClientConfigureResponse) Reset() {
+	*x = ClientConfigureResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfigureResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfigureResponse) ProtoMessage() {}
+
+func (x *ClientConfigureResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfigureResponse.ProtoReflect.Descriptor instead.
+func (*ClientConfigureResponse) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{15}
+}
+
 type LoadBalancerStatsResponse_RpcsByPeer struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -937,7 +1187,7 @@ type LoadBalancerStatsResponse_RpcsByPeer struct {
 func (x *LoadBalancerStatsResponse_RpcsByPeer) Reset() {
 	*x = LoadBalancerStatsResponse_RpcsByPeer{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[16]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -950,7 +1200,7 @@ func (x *LoadBalancerStatsResponse_RpcsByPeer) String() string {
 func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {}
 
 func (x *LoadBalancerStatsResponse_RpcsByPeer) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[16]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -973,6 +1223,70 @@ func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32
 	return nil
 }
 
+// Metadata to be attached for the given type of RPCs.
+type ClientConfigureRequest_Metadata struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type  ClientConfigureRequest_RpcType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"type,omitempty"`
+	Key   string                         `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	Value string                         `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (x *ClientConfigureRequest_Metadata) Reset() {
+	*x = ClientConfigureRequest_Metadata{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_interop_grpc_testing_test_proto_msgTypes[23]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfigureRequest_Metadata) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfigureRequest_Metadata) ProtoMessage() {}
+
+func (x *ClientConfigureRequest_Metadata) ProtoReflect() protoreflect.Message {
+	mi := &file_interop_grpc_testing_test_proto_msgTypes[23]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfigureRequest_Metadata.ProtoReflect.Descriptor instead.
+func (*ClientConfigureRequest_Metadata) Descriptor() ([]byte, []int) {
+	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{14, 0}
+}
+
+func (x *ClientConfigureRequest_Metadata) GetType() ClientConfigureRequest_RpcType {
+	if x != nil {
+		return x.Type
+	}
+	return ClientConfigureRequest_EMPTY_CALL
+}
+
+func (x *ClientConfigureRequest_Metadata) GetKey() string {
+	if x != nil {
+		return x.Key
+	}
+	return ""
+}
+
+func (x *ClientConfigureRequest_Metadata) GetValue() string {
+	if x != nil {
+		return x.Value
+	}
+	return ""
+}
+
 var File_interop_grpc_testing_test_proto protoreflect.FileDescriptor
 
 var file_interop_grpc_testing_test_proto_rawDesc = []byte{
@@ -1111,71 +1425,156 @@ var file_interop_grpc_testing_test_proto_rawDesc = []byte{
 	0x32, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c,
 	0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73,
 	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50,
-	0x65, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x3f,
-	0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a,
-	0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x12,
-	0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c,
-	0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10, 0x02, 0x2a,
-	0x6f, 0x0a, 0x0f, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79,
-	0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55,
-	0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10,
-	0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10,
-	0x01, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, 0x02,
-	0x32, 0xbb, 0x04, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
-	0x12, 0x35, 0x0a, 0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e,
+	0x65, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25,
+	0x0a, 0x23, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
+	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb2, 0x05, 0x0a, 0x24, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
+	0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65,
+	0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a,
+	0x01, 0x0a, 0x1a, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72,
+	0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41,
+	0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53,
+	0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x52, 0x16, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72,
+	0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x90, 0x01, 0x0a, 0x1c,
+	0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
+	0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x50, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
+	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75,
+	0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x52, 0x18, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63,
+	0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x87,
+	0x01, 0x0a, 0x19, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c,
+	0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
+	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61,
+	0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x15, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64,
+	0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0x49, 0x0a, 0x1b, 0x4e, 0x75, 0x6d, 0x52,
+	0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68,
+	0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
+	0x02, 0x38, 0x01, 0x1a, 0x4b, 0x0a, 0x1d, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75,
+	0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
+	0x1a, 0x48, 0x0a, 0x1a, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65,
+	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
+	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
+	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc8, 0x02, 0x0a, 0x16, 0x43,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01,
+	0x20, 0x03, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79,
+	0x70, 0x65, 0x52, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61,
+	0x64, 0x61, 0x74, 0x61, 0x1a, 0x74, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x12, 0x40, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c,
+	0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
+	0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x07, 0x52, 0x70,
+	0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x43,
+	0x41, 0x4c, 0x4c, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x5f, 0x43,
+	0x41, 0x4c, 0x4c, 0x10, 0x01, 0x22, 0x19, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x2a, 0x3f, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10,
+	0x00, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41,
+	0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10,
+	0x02, 0x2a, 0x6f, 0x0a, 0x0f, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52,
+	0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57,
+	0x4e, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f,
+	0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43,
+	0x4b, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f,
+	0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44,
+	0x10, 0x02, 0x32, 0xbb, 0x04, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x12, 0x35, 0x0a, 0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12,
+	0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45,
+	0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61,
+	0x72, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x6c, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
+	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e,
+	0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75,
+	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12,
+	0x69, 0x0a, 0x12, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75,
+	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e,
+	0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75,
+	0x6c, 0x6c, 0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f,
+	0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70,
+	0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67,
+	0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74,
+	0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01,
+	0x32, 0x55, 0x0a, 0x14, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65,
+	0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x11, 0x55, 0x6e, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e,
 	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70,
 	0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79,
-	0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
-	0x6c, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70,
-	0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f,
-	0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
-	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43,
-	0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x69, 0x0a,
-	0x12, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43,
-	0x61, 0x6c, 0x6c, 0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75,
-	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67,
-	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65,
-	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c,
-	0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
-	0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74,
-	0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28,
-	0x01, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70, 0x6c, 0x65,
-	0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
-	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
-	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61,
-	0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x32, 0x55,
-	0x0a, 0x14, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x53,
-	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x11, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c,
-	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
-	0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
-	0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x7f, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c,
-	0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
-	0x65, 0x12, 0x63, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74,
-	0x61, 0x74, 0x73, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53,
-	0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42,
-	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x2d, 0x5a, 0x2b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6f, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x86, 0x02, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64,
+	0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x12, 0x63, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x84, 0x01, 0x0a, 0x19, 0x47, 0x65,
+	0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74,
+	0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e,
+	0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74,
+	0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
+	0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65,
+	0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
+	0x32, 0x7b, 0x0a, 0x1f, 0x58, 0x64, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
+	0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66,
+	0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x2d, 0x5a,
+	0x2b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f,
+	0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6f, 0x70, 0x2f,
+	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -1190,66 +1589,85 @@ func file_interop_grpc_testing_test_proto_rawDescGZIP() []byte {
 	return file_interop_grpc_testing_test_proto_rawDescData
 }
 
-var file_interop_grpc_testing_test_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_interop_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 16)
+var file_interop_grpc_testing_test_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_interop_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 24)
 var file_interop_grpc_testing_test_proto_goTypes = []interface{}{
 	(PayloadType)(0),                             // 0: grpc.testing.PayloadType
 	(GrpclbRouteType)(0),                         // 1: grpc.testing.GrpclbRouteType
-	(*Empty)(nil),                                // 2: grpc.testing.Empty
-	(*Payload)(nil),                              // 3: grpc.testing.Payload
-	(*EchoStatus)(nil),                           // 4: grpc.testing.EchoStatus
-	(*SimpleRequest)(nil),                        // 5: grpc.testing.SimpleRequest
-	(*SimpleResponse)(nil),                       // 6: grpc.testing.SimpleResponse
-	(*StreamingInputCallRequest)(nil),            // 7: grpc.testing.StreamingInputCallRequest
-	(*StreamingInputCallResponse)(nil),           // 8: grpc.testing.StreamingInputCallResponse
-	(*ResponseParameters)(nil),                   // 9: grpc.testing.ResponseParameters
-	(*StreamingOutputCallRequest)(nil),           // 10: grpc.testing.StreamingOutputCallRequest
-	(*StreamingOutputCallResponse)(nil),          // 11: grpc.testing.StreamingOutputCallResponse
-	(*LoadBalancerStatsRequest)(nil),             // 12: grpc.testing.LoadBalancerStatsRequest
-	(*LoadBalancerStatsResponse)(nil),            // 13: grpc.testing.LoadBalancerStatsResponse
-	(*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 14: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
-	nil, // 15: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
-	nil, // 16: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
-	nil, // 17: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	(ClientConfigureRequest_RpcType)(0),          // 2: grpc.testing.ClientConfigureRequest.RpcType
+	(*Empty)(nil),                                // 3: grpc.testing.Empty
+	(*Payload)(nil),                              // 4: grpc.testing.Payload
+	(*EchoStatus)(nil),                           // 5: grpc.testing.EchoStatus
+	(*SimpleRequest)(nil),                        // 6: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil),                       // 7: grpc.testing.SimpleResponse
+	(*StreamingInputCallRequest)(nil),            // 8: grpc.testing.StreamingInputCallRequest
+	(*StreamingInputCallResponse)(nil),           // 9: grpc.testing.StreamingInputCallResponse
+	(*ResponseParameters)(nil),                   // 10: grpc.testing.ResponseParameters
+	(*StreamingOutputCallRequest)(nil),           // 11: grpc.testing.StreamingOutputCallRequest
+	(*StreamingOutputCallResponse)(nil),          // 12: grpc.testing.StreamingOutputCallResponse
+	(*LoadBalancerStatsRequest)(nil),             // 13: grpc.testing.LoadBalancerStatsRequest
+	(*LoadBalancerStatsResponse)(nil),            // 14: grpc.testing.LoadBalancerStatsResponse
+	(*LoadBalancerAccumulatedStatsRequest)(nil),  // 15: grpc.testing.LoadBalancerAccumulatedStatsRequest
+	(*LoadBalancerAccumulatedStatsResponse)(nil), // 16: grpc.testing.LoadBalancerAccumulatedStatsResponse
+	(*ClientConfigureRequest)(nil),               // 17: grpc.testing.ClientConfigureRequest
+	(*ClientConfigureResponse)(nil),              // 18: grpc.testing.ClientConfigureResponse
+	(*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 19: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	nil,                                     // 20: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	nil,                                     // 21: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	nil,                                     // 22: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	nil,                                     // 23: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
+	nil,                                     // 24: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
+	nil,                                     // 25: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
+	(*ClientConfigureRequest_Metadata)(nil), // 26: grpc.testing.ClientConfigureRequest.Metadata
 }
 var file_interop_grpc_testing_test_proto_depIdxs = []int32{
 	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
 	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
-	3,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
-	4,  // 3: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
-	3,  // 4: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
+	4,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
+	5,  // 3: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
+	4,  // 4: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
 	1,  // 5: grpc.testing.SimpleResponse.grpclb_route_type:type_name -> grpc.testing.GrpclbRouteType
-	3,  // 6: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
+	4,  // 6: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
 	0,  // 7: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
-	9,  // 8: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
-	3,  // 9: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
-	4,  // 10: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
-	3,  // 11: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
-	15, // 12: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
-	16, // 13: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
-	17, // 14: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
-	14, // 15: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
-	2,  // 16: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty
-	5,  // 17: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
-	10, // 18: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest
-	7,  // 19: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest
-	10, // 20: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
-	10, // 21: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
-	2,  // 22: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty
-	12, // 23: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest
-	2,  // 24: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty
-	6,  // 25: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
-	11, // 26: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse
-	8,  // 27: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse
-	11, // 28: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
-	11, // 29: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
-	2,  // 30: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty
-	13, // 31: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse
-	24, // [24:32] is the sub-list for method output_type
-	16, // [16:24] is the sub-list for method input_type
-	16, // [16:16] is the sub-list for extension type_name
-	16, // [16:16] is the sub-list for extension extendee
-	0,  // [0:16] is the sub-list for field type_name
+	10, // 8: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
+	4,  // 9: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
+	5,  // 10: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
+	4,  // 11: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
+	20, // 12: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	21, // 13: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	23, // 14: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_started_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
+	24, // 15: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_succeeded_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
+	25, // 16: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_failed_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
+	2,  // 17: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType
+	26, // 18: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata
+	22, // 19: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	19, // 20: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	2,  // 21: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType
+	3,  // 22: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty
+	6,  // 23: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	11, // 24: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	8,  // 25: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest
+	11, // 26: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	11, // 27: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	3,  // 28: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty
+	13, // 29: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest
+	15, // 30: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:input_type -> grpc.testing.LoadBalancerAccumulatedStatsRequest
+	17, // 31: grpc.testing.XdsUpdateClientConfigureService.Configure:input_type -> grpc.testing.ClientConfigureRequest
+	3,  // 32: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty
+	7,  // 33: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	12, // 34: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	9,  // 35: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse
+	12, // 36: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	12, // 37: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	3,  // 38: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty
+	14, // 39: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse
+	16, // 40: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:output_type -> grpc.testing.LoadBalancerAccumulatedStatsResponse
+	18, // 41: grpc.testing.XdsUpdateClientConfigureService.Configure:output_type -> grpc.testing.ClientConfigureResponse
+	32, // [32:42] is the sub-list for method output_type
+	22, // [22:32] is the sub-list for method input_type
+	22, // [22:22] is the sub-list for extension type_name
+	22, // [22:22] is the sub-list for extension extendee
+	0,  // [0:22] is the sub-list for field type_name
 }
 
 func init() { file_interop_grpc_testing_test_proto_init() }
@@ -1403,6 +1821,54 @@ func file_interop_grpc_testing_test_proto_init() {
 			}
 		}
 		file_interop_grpc_testing_test_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerAccumulatedStatsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerAccumulatedStatsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfigureRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfigureResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_interop_grpc_testing_test_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*LoadBalancerStatsResponse_RpcsByPeer); i {
 			case 0:
 				return &v.state
@@ -1414,16 +1880,28 @@ func file_interop_grpc_testing_test_proto_init() {
 				return nil
 			}
 		}
+		file_interop_grpc_testing_test_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfigureRequest_Metadata); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_interop_grpc_testing_test_proto_rawDesc,
-			NumEnums:      2,
-			NumMessages:   16,
+			NumEnums:      3,
+			NumMessages:   24,
 			NumExtensions: 0,
-			NumServices:   3,
+			NumServices:   4,
 		},
 		GoTypes:           file_interop_grpc_testing_test_proto_goTypes,
 		DependencyIndexes: file_interop_grpc_testing_test_proto_depIdxs,
diff --git a/interop/grpc_testing/test.proto b/interop/grpc_testing/test.proto
index 21451047e513..3d7f018d3bb5 100644
--- a/interop/grpc_testing/test.proto
+++ b/interop/grpc_testing/test.proto
@@ -227,9 +227,55 @@ message LoadBalancerStatsResponse {
   map<string, RpcsByPeer> rpcs_by_method = 3;
 }
 
+// Request for retrieving a test client's accumulated stats.
+message LoadBalancerAccumulatedStatsRequest {}
+
+// Accumulated stats for RPCs sent by a test client.
+message LoadBalancerAccumulatedStatsResponse {
+  // The total number of RPCs have ever issued for each type.
+  map<string, int32> num_rpcs_started_by_method = 1;
+  // The total number of RPCs have ever completed successfully for each type.
+  map<string, int32> num_rpcs_succeeded_by_method = 2;
+  // The total number of RPCs have ever failed for each type.
+  map<string, int32> num_rpcs_failed_by_method = 3;
+}
+
 // A service used to obtain stats for verifying LB behavior.
 service LoadBalancerStatsService {
   // Gets the backend distribution for RPCs sent by a test client.
   rpc GetClientStats(LoadBalancerStatsRequest)
       returns (LoadBalancerStatsResponse) {}
+  // Gets the accumulated stats for RPCs sent by a test client.
+  rpc GetClientAccumulatedStats(LoadBalancerAccumulatedStatsRequest)
+    returns (LoadBalancerAccumulatedStatsResponse) {}
+}
+
+// Configurations for a test client.
+message ClientConfigureRequest {
+  // Type of RPCs to send.
+  enum RpcType {
+    EMPTY_CALL = 0;
+    UNARY_CALL = 1;
+  }
+
+  // Metadata to be attached for the given type of RPCs.
+  message Metadata {
+    RpcType type = 1;
+    string key = 2;
+    string value = 3;
+  }
+
+  // The types of RPCs the client sends.
+  repeated RpcType types = 1;
+  // The collection of custom metadata to be attached to RPCs sent by the client.
+  repeated Metadata metadata = 2;
+}
+
+// Response for updating a test client's configuration.
+message ClientConfigureResponse {}
+
+// A service to dynamically update the configuration of an xDS test client.
+service XdsUpdateClientConfigureService {
+  // Update the tes client's configuration.
+  rpc Configure(ClientConfigureRequest) returns (ClientConfigureResponse);
 }
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index 482d309f7b4a..a8a2e67375be 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -527,6 +527,8 @@ var UnimplementedService_ServiceDesc = grpc.ServiceDesc{
 type LoadBalancerStatsServiceClient interface {
 	// Gets the backend distribution for RPCs sent by a test client.
 	GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error)
+	// Gets the accumulated stats for RPCs sent by a test client.
+	GetClientAccumulatedStats(ctx context.Context, in *LoadBalancerAccumulatedStatsRequest, opts ...grpc.CallOption) (*LoadBalancerAccumulatedStatsResponse, error)
 }
 
 type loadBalancerStatsServiceClient struct {
@@ -546,12 +548,23 @@ func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in
 	return out, nil
 }
 
+func (c *loadBalancerStatsServiceClient) GetClientAccumulatedStats(ctx context.Context, in *LoadBalancerAccumulatedStatsRequest, opts ...grpc.CallOption) (*LoadBalancerAccumulatedStatsResponse, error) {
+	out := new(LoadBalancerAccumulatedStatsResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.LoadBalancerStatsService/GetClientAccumulatedStats", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service.
 // All implementations must embed UnimplementedLoadBalancerStatsServiceServer
 // for forward compatibility
 type LoadBalancerStatsServiceServer interface {
 	// Gets the backend distribution for RPCs sent by a test client.
 	GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error)
+	// Gets the accumulated stats for RPCs sent by a test client.
+	GetClientAccumulatedStats(context.Context, *LoadBalancerAccumulatedStatsRequest) (*LoadBalancerAccumulatedStatsResponse, error)
 	mustEmbedUnimplementedLoadBalancerStatsServiceServer()
 }
 
@@ -562,6 +575,9 @@ type UnimplementedLoadBalancerStatsServiceServer struct {
 func (UnimplementedLoadBalancerStatsServiceServer) GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented")
 }
+func (UnimplementedLoadBalancerStatsServiceServer) GetClientAccumulatedStats(context.Context, *LoadBalancerAccumulatedStatsRequest) (*LoadBalancerAccumulatedStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetClientAccumulatedStats not implemented")
+}
 func (UnimplementedLoadBalancerStatsServiceServer) mustEmbedUnimplementedLoadBalancerStatsServiceServer() {
 }
 
@@ -594,6 +610,24 @@ func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx conte
 	return interceptor(ctx, in, info, handler)
 }
 
+func _LoadBalancerStatsService_GetClientAccumulatedStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LoadBalancerAccumulatedStatsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(LoadBalancerStatsServiceServer).GetClientAccumulatedStats(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.LoadBalancerStatsService/GetClientAccumulatedStats",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(LoadBalancerStatsServiceServer).GetClientAccumulatedStats(ctx, req.(*LoadBalancerAccumulatedStatsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // LoadBalancerStatsService_ServiceDesc is the grpc.ServiceDesc for LoadBalancerStatsService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -605,6 +639,99 @@ var LoadBalancerStatsService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "GetClientStats",
 			Handler:    _LoadBalancerStatsService_GetClientStats_Handler,
 		},
+		{
+			MethodName: "GetClientAccumulatedStats",
+			Handler:    _LoadBalancerStatsService_GetClientAccumulatedStats_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "interop/grpc_testing/test.proto",
+}
+
+// XdsUpdateClientConfigureServiceClient is the client API for XdsUpdateClientConfigureService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type XdsUpdateClientConfigureServiceClient interface {
+	// Update the tes client's configuration.
+	Configure(ctx context.Context, in *ClientConfigureRequest, opts ...grpc.CallOption) (*ClientConfigureResponse, error)
+}
+
+type xdsUpdateClientConfigureServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewXdsUpdateClientConfigureServiceClient(cc grpc.ClientConnInterface) XdsUpdateClientConfigureServiceClient {
+	return &xdsUpdateClientConfigureServiceClient{cc}
+}
+
+func (c *xdsUpdateClientConfigureServiceClient) Configure(ctx context.Context, in *ClientConfigureRequest, opts ...grpc.CallOption) (*ClientConfigureResponse, error) {
+	out := new(ClientConfigureResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.XdsUpdateClientConfigureService/Configure", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// XdsUpdateClientConfigureServiceServer is the server API for XdsUpdateClientConfigureService service.
+// All implementations must embed UnimplementedXdsUpdateClientConfigureServiceServer
+// for forward compatibility
+type XdsUpdateClientConfigureServiceServer interface {
+	// Update the tes client's configuration.
+	Configure(context.Context, *ClientConfigureRequest) (*ClientConfigureResponse, error)
+	mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer()
+}
+
+// UnimplementedXdsUpdateClientConfigureServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedXdsUpdateClientConfigureServiceServer struct {
+}
+
+func (UnimplementedXdsUpdateClientConfigureServiceServer) Configure(context.Context, *ClientConfigureRequest) (*ClientConfigureResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
+}
+func (UnimplementedXdsUpdateClientConfigureServiceServer) mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() {
+}
+
+// UnsafeXdsUpdateClientConfigureServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to XdsUpdateClientConfigureServiceServer will
+// result in compilation errors.
+type UnsafeXdsUpdateClientConfigureServiceServer interface {
+	mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer()
+}
+
+func RegisterXdsUpdateClientConfigureServiceServer(s grpc.ServiceRegistrar, srv XdsUpdateClientConfigureServiceServer) {
+	s.RegisterService(&XdsUpdateClientConfigureService_ServiceDesc, srv)
+}
+
+func _XdsUpdateClientConfigureService_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ClientConfigureRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(XdsUpdateClientConfigureServiceServer).Configure(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.XdsUpdateClientConfigureService/Configure",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(XdsUpdateClientConfigureServiceServer).Configure(ctx, req.(*ClientConfigureRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// XdsUpdateClientConfigureService_ServiceDesc is the grpc.ServiceDesc for XdsUpdateClientConfigureService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var XdsUpdateClientConfigureService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.XdsUpdateClientConfigureService",
+	HandlerType: (*XdsUpdateClientConfigureServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Configure",
+			Handler:    _XdsUpdateClientConfigureService_Configure_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "interop/grpc_testing/test.proto",

From 20f4a846104776b596e5b2eeacc17c4515384eec Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 15 Dec 2020 14:46:55 -0800
Subject: [PATCH 320/481] xds: Expect certprovider bootstrap configs when
 xdsCreds are in use (#4111)

---
 xds/internal/resolver/xds_resolver.go      | 24 +++++++++++++-
 xds/internal/resolver/xds_resolver_test.go | 37 ++++++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index faddc2f7bc34..a99403edcfa9 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -20,11 +20,14 @@
 package resolver
 
 import (
+	"errors"
 	"fmt"
 
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
 	iresolver "google.golang.org/grpc/internal/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -45,7 +48,7 @@ type xdsResolverBuilder struct{}
 //
 // The xds bootstrap process is performed (and a new xds client is built) every
 // time an xds resolver is built.
-func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
+func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
 	r := &xdsResolver{
 		target:         t,
 		cc:             cc,
@@ -62,6 +65,24 @@ func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, _
 	}
 	r.client = client
 
+	// If xds credentials were specified by the user, but bootstrap configs do
+	// not contain any certificate provider configuration, it is better to fail
+	// right now rather than failing when attempting to create certificate
+	// providers after receiving an CDS response with security configuration.
+	var creds credentials.TransportCredentials
+	switch {
+	case opts.DialCreds != nil:
+		creds = opts.DialCreds
+	case opts.CredsBundle != nil:
+		creds = opts.CredsBundle.TransportCredentials()
+	}
+	if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() {
+		bc := client.BootstrapConfig()
+		if len(bc.CertProviderConfigs) == 0 {
+			return nil, errors.New("xds: xdsCreds specified but certificate_providers config missing in bootstrap file")
+		}
+	}
+
 	// Register a watch on the xdsClient for the user's dial target.
 	cancelWatch := watchService(r.client, r.target.Endpoint, r.handleServiceUpdate, r.logger)
 	r.logger.Infof("Watch started on resource name %v with xds-client %p", r.target.Endpoint, r.client)
@@ -84,6 +105,7 @@ func (*xdsResolverBuilder) Scheme() string {
 type xdsClientInterface interface {
 	WatchListener(serviceName string, cb func(xdsclient.ListenerUpdate, error)) func()
 	WatchRouteConfig(routeName string, cb func(xdsclient.RouteConfigUpdate, error)) func()
+	BootstrapConfig() *bootstrap.Config
 	Close()
 }
 
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 5f373c036f4c..f3bdc57c0ba0 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -26,6 +26,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpctest"
@@ -38,6 +40,7 @@ import (
 	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
@@ -145,6 +148,40 @@ func (s) TestResolverBuilder(t *testing.T) {
 	}
 }
 
+// TestResolverBuilder_xdsCredsBootstrapMismatch tests the case where an xds
+// resolver is built with xds credentials being specified by the user. The
+// bootstrap file does not contain any certificate provider configuration
+// though, and therefore we expect the resolver build to fail.
+func (s) TestResolverBuilder_xdsCredsBootstrapMismatch(t *testing.T) {
+	// Fake out the xdsClient creation process by providing a fake, which does
+	// not have any certificate provider configuration.
+	oldClientMaker := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) {
+		fc := fakeclient.NewClient()
+		fc.SetBootstrapConfig(&bootstrap.Config{})
+		return fc, nil
+	}
+	defer func() { newXDSClient = oldClientMaker }()
+
+	builder := resolver.Get(xdsScheme)
+	if builder == nil {
+		t.Fatalf("resolver.Get(%v) returned nil", xdsScheme)
+	}
+
+	// Create xds credentials to be passed to resolver.Build().
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("xds.NewClientCredentials() failed: %v", err)
+	}
+
+	// Since the fake xds client is not configured with any certificate provider
+	// configs, and we are specifying xds credentials in the call to
+	// resolver.Build(), we expect it to fail.
+	if _, err := builder.Build(target, newTestClientConn(), resolver.BuildOptions{DialCreds: creds}); err == nil {
+		t.Fatal("builder.Build() succeeded when expected to fail")
+	}
+}
+
 type setupOpts struct {
 	xdsClientFunc func() (xdsClientInterface, error)
 }

From 668e3ae6d589183c5d4f0e723e5657e6989c5c1e Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 15 Dec 2020 15:56:06 -0800
Subject: [PATCH 321/481] testing: stop listening on externally-visible devices
 (#4112)

---
 security/advancedtls/advancedtls_integration_test.go | 5 ++---
 xds/internal/testutils/e2e/server.go                 | 2 +-
 xds/internal/testutils/local_listener.go             | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index a9b22b711ff5..34f80532d4aa 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -41,7 +41,6 @@ import (
 
 var (
 	address = "localhost:50051"
-	port    = ":50051"
 )
 
 const (
@@ -363,7 +362,7 @@ func (s) TestEnd2End(t *testing.T) {
 			}
 			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
 			defer s.Stop()
-			lis, err := net.Listen("tcp", port)
+			lis, err := net.Listen("tcp", address)
 			if err != nil {
 				t.Fatalf("failed to listen: %v", err)
 			}
@@ -652,7 +651,7 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 			}
 			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
 			defer s.Stop()
-			lis, err := net.Listen("tcp", port)
+			lis, err := net.Listen("tcp", address)
 			if err != nil {
 				t.Fatalf("failed to listen: %v", err)
 			}
diff --git a/xds/internal/testutils/e2e/server.go b/xds/internal/testutils/e2e/server.go
index eb6045946366..e3020614702b 100644
--- a/xds/internal/testutils/e2e/server.go
+++ b/xds/internal/testutils/e2e/server.go
@@ -71,7 +71,7 @@ func StartManagementServer() (*ManagementServer, error) {
 	cache := v3cache.NewSnapshotCache(true, v3cache.IDHash{}, serverLogger{})
 	logger.Infof("Created new snapshot cache...")
 
-	lis, err := net.Listen("tcp", ":0")
+	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		return nil, fmt.Errorf("failed to start xDS management server: %v", err)
 	}
diff --git a/xds/internal/testutils/local_listener.go b/xds/internal/testutils/local_listener.go
index 7da1bc1ce416..f831b95f4133 100644
--- a/xds/internal/testutils/local_listener.go
+++ b/xds/internal/testutils/local_listener.go
@@ -22,5 +22,5 @@ import "net"
 
 // LocalTCPListener returns a net.Listener listening on local address and port.
 func LocalTCPListener() (net.Listener, error) {
-	return net.Listen("tcp", ":0")
+	return net.Listen("tcp", "localhost:0")
 }

From e2d2703c48fc959e80ac70a950368c369e8d47c1 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 16 Dec 2020 09:28:14 -0800
Subject: [PATCH 322/481] xds: NACK resource when security config is not as
 expected (#4113)

- On the client side, we always expect a root provider.
- On the server side, we always expect an identity provider. If the
  `require_client_cert` field is set, a root provider is also expected.

This simplifies the upper layers which can now expect a security config
to be populated correctly, the cds balancer on the client-side and the
xds.GRPCServer on the server-side. And this is what the other languages
are doing as well.
---
 xds/internal/client/client_cds_test.go |  30 +++++++
 xds/internal/client/client_lds_test.go | 118 +++++++++++++++++++++++--
 xds/internal/client/client_xds.go      |  17 +++-
 3 files changed, 155 insertions(+), 10 deletions(-)

diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go
index 4d0801dd8058..5d2366d64888 100644
--- a/xds/internal/client/client_cds_test.go
+++ b/xds/internal/client/client_cds_test.go
@@ -347,6 +347,36 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "transport-socket-without-validation-context",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "happy-case-with-no-identity-certs",
 			cluster: &v3clusterpb.Cluster{
diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
index e5c7b1e48709..35d8201f124b 100644
--- a/xds/internal/client/client_lds_test.go
+++ b/xds/internal/client/client_lds_test.go
@@ -748,7 +748,7 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			},
 		},
 		{
-			name: "happy case with no identity certs",
+			name: "no identity and root certificate providers",
 			resources: []*anypb.Any{
 				{
 					TypeUrl: version.V3ListenerURL,
@@ -780,11 +780,112 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 													tls := &v3tlspb.DownstreamTlsContext{
 														RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 														CommonTlsContext: &v3tlspb.CommonTlsContext{
-															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-																ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-																	InstanceName:    "rootPluginInstance",
-																	CertificateName: "rootCertName",
-																},
+															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+																InstanceName:    "identityPluginInstance",
+																CertificateName: "identityCertName",
+															},
+														},
+													}
+													mtls, _ := proto.Marshal(tls)
+													return mtls
+												}(),
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set",
+		},
+		{
+			name: "no identity certificate provider with require_client_cert",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value: func() []byte {
+													tls := &v3tlspb.DownstreamTlsContext{
+														CommonTlsContext: &v3tlspb.CommonTlsContext{},
+													}
+													mtls, _ := proto.Marshal(tls)
+													return mtls
+												}(),
+											},
+										},
+									},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantErr: "security configuration on the server-side does not contain identity certificate provider instance name",
+		},
+		{
+			name: "happy case with no validation context",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									Name: "filter-chain-1",
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{
+										ApplicationProtocols: []string{"managed-mtls"},
+									},
+									TransportSocket: &v3corepb.TransportSocket{
+										Name: "envoy.transport_sockets.tls",
+										ConfigType: &v3corepb.TransportSocket_TypedConfig{
+											TypedConfig: &anypb.Any{
+												TypeUrl: version.V3DownstreamTLSContextURL,
+												Value: func() []byte {
+													tls := &v3tlspb.DownstreamTlsContext{
+														CommonTlsContext: &v3tlspb.CommonTlsContext{
+															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+																InstanceName:    "identityPluginInstance",
+																CertificateName: "identityCertName",
 															},
 														},
 													}
@@ -805,9 +906,8 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
 					SecurityCfg: &SecurityConfig{
-						RootInstanceName:  "rootPluginInstance",
-						RootCertName:      "rootCertName",
-						RequireClientCert: true,
+						IdentityInstanceName: "identityPluginInstance",
+						IdentityCertName:     "identityCertName",
 					},
 				},
 			},
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 89b2afb4b9e5..ac32b99a3608 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -174,7 +174,13 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	if err != nil {
 		return nil, err
 	}
+	if sc.IdentityInstanceName == "" {
+		return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name")
+	}
 	sc.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue()
+	if sc.RequireClientCert && sc.RootInstanceName == "" {
+		return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set")
+	}
 	return &ListenerUpdate{SecurityCfg: sc}, nil
 }
 
@@ -425,7 +431,14 @@ func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, e
 		return nil, errors.New("xds: UpstreamTlsContext in CDS response does not contain a CommonTlsContext")
 	}
 
-	return securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext())
+	sc, err := securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext())
+	if err != nil {
+		return nil, err
+	}
+	if sc.RootInstanceName == "" {
+		return nil, errors.New("security configuration on the client-side does not contain root certificate provider instance name")
+	}
+	return sc, nil
 }
 
 // common is expected to be not nil.
@@ -468,6 +481,8 @@ func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext) (*Secu
 		pi := common.GetValidationContextCertificateProviderInstance()
 		sc.RootInstanceName = pi.GetInstanceName()
 		sc.RootCertName = pi.GetCertificateName()
+	case nil:
+		// It is valid for the validation context to be nil on the server side.
 	default:
 		return nil, fmt.Errorf("xds: validation context contains unexpected type: %T", t)
 	}

From cfef8c7e035d0bd3340436c000b50e1b46668612 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 16 Dec 2020 09:28:28 -0800
Subject: [PATCH 323/481] pemfile: Update distributor only inside of the for
 loop (#4114)

---
 credentials/tls/certprovider/pemfile/watcher.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/credentials/tls/certprovider/pemfile/watcher.go b/credentials/tls/certprovider/pemfile/watcher.go
index a46b149dd03d..e154030cbe8a 100644
--- a/credentials/tls/certprovider/pemfile/watcher.go
+++ b/credentials/tls/certprovider/pemfile/watcher.go
@@ -214,11 +214,6 @@ func (w *watcher) updateRootDistributor() {
 // changes, and pushes new key material into the appropriate distributors which
 // is returned from calls to KeyMaterial().
 func (w *watcher) run(ctx context.Context) {
-	// Update both root and identity certs at the beginning. Subsequently,
-	// update only the appropriate file whose ticker has fired.
-	w.updateIdentityDistributor()
-	w.updateRootDistributor()
-
 	ticker := time.NewTicker(w.opts.RefreshDuration)
 	for {
 		w.updateIdentityDistributor()

From 2fad6bf4dad121c9c4f28f85d8a274c95fa91029 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 16 Dec 2020 10:27:18 -0800
Subject: [PATCH 324/481] xds: Implement server-side security (#4092)

---
 internal/internal.go                          |   6 +-
 server.go                                     |   7 +
 xds/internal/client/client.go                 |   4 +
 .../test/xds_server_integration_test.go       | 374 +++++++++++++--
 xds/internal/testutils/e2e/bootstrap.go       |  30 +-
 xds/server.go                                 | 258 +++++++++--
 xds/server_test.go                            | 432 ++++++++++++++++--
 7 files changed, 1007 insertions(+), 104 deletions(-)

diff --git a/internal/internal.go b/internal/internal.go
index f83d6625988d..1e2834c70f67 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -60,7 +60,11 @@ var (
 	// GetXDSHandshakeInfoForTesting returns a pointer to the xds.HandshakeInfo
 	// stored in the passed in attributes. This is set by
 	// credentials/xds/xds.go.
-	GetXDSHandshakeInfoForTesting interface{} // func (attr *attributes.Attributes) *xds.HandshakeInfo
+	GetXDSHandshakeInfoForTesting interface{} // func (*attributes.Attributes) *xds.HandshakeInfo
+	// GetServerCredentials returns the transport credentials configured on a
+	// gRPC server. An xDS-enabled server needs to know what type of credentials
+	// is configured on the underlying gRPC server. This is set by server.go.
+	GetServerCredentials interface{} // func (*grpc.Server) credentials.TransportCredentials
 )
 
 // HealthChecker defines the signature of the client-side LB channel health checking function.
diff --git a/server.go b/server.go
index 968eb598e1e5..7a2aa28a1147 100644
--- a/server.go
+++ b/server.go
@@ -40,6 +40,7 @@ import (
 	"google.golang.org/grpc/encoding"
 	"google.golang.org/grpc/encoding/proto"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/binarylog"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcrand"
@@ -58,6 +59,12 @@ const (
 	defaultServerMaxSendMessageSize    = math.MaxInt32
 )
 
+func init() {
+	internal.GetServerCredentials = func(srv *Server) credentials.TransportCredentials {
+		return srv.opts.creds
+	}
+}
+
 var statusOK = status.New(codes.OK, "")
 var logger = grpclog.Component("core")
 
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index b0e27d4fd754..d78122ca5b9d 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -149,6 +149,10 @@ type ListenerUpdate struct {
 	SecurityCfg *SecurityConfig
 }
 
+func (lu *ListenerUpdate) String() string {
+	return fmt.Sprintf("{RouteConfigName: %q, SecurityConfig: %+v", lu.RouteConfigName, lu.SecurityCfg)
+}
+
 // RouteConfigUpdate contains information received in an RDS response, which is
 // of interest to the registered RDS watcher.
 type RouteConfigUpdate struct {
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index cb1072e162e7..8a9c8f53c74a 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -23,20 +23,32 @@ package xds_test
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path"
+	"strconv"
 	"testing"
 
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/uuid"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/status"
 	testpb "google.golang.org/grpc/test/grpc_testing"
+	"google.golang.org/grpc/testdata"
 	"google.golang.org/grpc/xds"
 	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/testutils"
@@ -44,15 +56,89 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-// TestServerSideXDS is an e2e tests for xDS use on the server. This does not
-// use any xDS features because we have not implemented any on the server side.
-func (s) TestServerSideXDS(t *testing.T) {
+const (
+	// Names of files inside tempdir, for certprovider plugin to watch.
+	certFile = "cert.pem"
+	keyFile  = "key.pem"
+	rootFile = "ca.pem"
+)
+
+func createTmpFile(t *testing.T, src, dst string) {
+	t.Helper()
+
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		t.Fatalf("ioutil.ReadFile(%q) failed: %v", src, err)
+	}
+	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
+		t.Fatalf("ioutil.WriteFile(%q) failed: %v", dst, err)
+	}
+	t.Logf("Wrote file at: %s", dst)
+	t.Logf("%s", string(data))
+}
+
+// createTempDirWithFiles creates a temporary directory under the system default
+// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
+// rootSrc files are creates appropriate files under the newly create tempDir.
+// Returns the name of the created tempDir.
+func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc string) string {
+	t.Helper()
+
+	// Create a temp directory. Passing an empty string for the first argument
+	// uses the system temp directory.
+	dir, err := ioutil.TempDir("", dirSuffix)
+	if err != nil {
+		t.Fatalf("ioutil.TempDir() failed: %v", err)
+	}
+	t.Logf("Using tmpdir: %s", dir)
+
+	createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile))
+	createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile))
+	createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile))
+	return dir
+}
+
+// createClientTLSCredentials creates client-side TLS transport credentials.
+func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err)
+	}
+	b, err := ioutil.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("ioutil.ReadFile(x509/server_ca_cert.pem) failed: %v", err)
+	}
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(b) {
+		t.Fatal("failed to append certificates")
+	}
+	return credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      roots,
+		ServerName:   "x.test.example.com",
+	})
+}
+
+// commonSetup performs a bunch of steps common to all xDS server tests here:
+// - turn on V3 support by setting the environment variable
+// - spin up an xDS management server on a local port
+// - set up certificates for consumption by the file_watcher plugin
+// - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and
+//   register the test service on it
+// - create a local TCP listener and start serving on it
+//
+// Returns the following:
+// - the management server: tests use this to configure resources
+// - nodeID expected by the management server: this is set in the Node proto
+//   sent by the xdsClient used on the xDS-enabled gRPC server
+// - local listener on which the xDS-enabled gRPC server is serving on
+// - cleanup function to be invoked by the tests when done
+func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, func()) {
+	t.Helper()
+
+	// Turn on xDS V3 support.
 	origV3Support := env.V3Support
 	env.V3Support = true
-	defer func() { env.V3Support = origV3Support }()
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 
 	// Spin up a xDS management server on a local port.
 	nodeID := uuid.New().String()
@@ -60,24 +146,37 @@ func (s) TestServerSideXDS(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.Stop()
+
+	// Create certificate and key files in a temporary directory and generate
+	// certificate provider configuration for a file_watcher plugin.
+	tmpdir := createTmpDirWithFiles(t, "testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile))
 
 	// Create a bootstrap file in a temporary directory.
-	cleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:   e2e.TransportV3,
-		NodeID:    nodeID,
-		ServerURI: fs.Address,
+	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
+		Version:              e2e.TransportV3,
+		NodeID:               nodeID,
+		ServerURI:            fs.Address,
+		CertificateProviders: cpc,
+		ServerResourceNameID: "grpc/server",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS credentials to be used on the server-side.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
+		FallbackCreds: insecure.NewCredentials(),
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer cleanup()
 
 	// Initialize an xDS-enabled gRPC server and register the stubServer on it.
-	server := xds.NewGRPCServer()
+	server := xds.NewGRPCServer(grpc.Creds(creds))
 	testpb.RegisterTestServiceServer(server, &testService{})
-	defer server.Stop()
 
+	// Create a local listener and pass it to Serve().
 	lis, err := testutils.LocalTCPListener()
 	if err != nil {
 		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
@@ -89,40 +188,124 @@ func (s) TestServerSideXDS(t *testing.T) {
 		}
 	}()
 
-	// Setup the fake management server to respond with a Listener resource.
-	go func() {
-		listener := &v3listenerpb.Listener{
-			// This needs to match the name we are querying for.
-			Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String()),
-			ApiListener: &v3listenerpb.ApiListener{
-				ApiListener: &anypb.Any{
-					TypeUrl: version.V2HTTPConnManagerURL,
-					Value: func() []byte {
-						cm := &v3httppb.HttpConnectionManager{
-							RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-								Rds: &v3httppb.Rds{
-									ConfigSource: &v3corepb.ConfigSource{
-										ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-									},
-									RouteConfigName: "route-config",
-								},
-							},
-						}
-						mcm, _ := proto.Marshal(cm)
-						return mcm
-					}(),
+	return fs, nodeID, lis, func() {
+		env.V3Support = origV3Support
+		fs.Stop()
+		bootstrapCleanup()
+		server.Stop()
+	}
+}
+
+func hostPortFromListener(t *testing.T, lis net.Listener) (string, uint32) {
+	t.Helper()
+
+	host, p, err := net.SplitHostPort(lis.Addr().String())
+	if err != nil {
+		t.Fatalf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err)
+	}
+	port, err := strconv.ParseInt(p, 10, 32)
+	if err != nil {
+		t.Fatalf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err)
+	}
+	return host, uint32(port)
+
+}
+
+// listenerResourceWithoutSecurityConfig returns a listener resource with no
+// security configuration, and name and address fields matching the passed in
+// net.Listener.
+func listenerResourceWithoutSecurityConfig(t *testing.T, lis net.Listener) *v3listenerpb.Listener {
+	host, port := hostPortFromListener(t, lis)
+	return &v3listenerpb.Listener{
+		// This needs to match the name we are querying for.
+		Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String()),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
 				},
 			},
-		}
-		if err := fs.Update(e2e.UpdateOptions{
-			NodeID:    nodeID,
-			Listeners: []*v3listenerpb.Listener{listener},
-		}); err != nil {
-			t.Error(err)
-		}
-	}()
+		},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "filter-chain-1",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					ApplicationProtocols: []string{"managed-mtls"},
+				},
+			},
+		},
+	}
+}
+
+// listenerResourceWithSecurityConfig returns a listener resource with security
+// configuration pointing to the use of the file_watcher certificate provider
+// plugin, and name and address fields matching the passed in net.Listener.
+func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3listenerpb.Listener {
+	host, port := hostPortFromListener(t, lis)
+	return &v3listenerpb.Listener{
+		// This needs to match the name we are querying for.
+		Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String()),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					}}}},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "filter-chain-1",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					ApplicationProtocols: []string{"managed-mtls"},
+				},
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3DownstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.DownstreamTlsContext{
+									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName: "google_cloud_private_spiffe",
+										},
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+												InstanceName: "google_cloud_private_spiffe",
+											}}}}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						}}}}},
+	}
+}
+
+// TestServerSideXDS_Fallback is an e2e test where xDS is enabled on the
+// server-side and xdsCredentials are configured for security. The control plane
+// does not provide any security configuration and therefore the xdsCredentials
+// uses fallback credentials, which in this case is insecure creds.
+func (s) TestServerSideXDS_Fallback(t *testing.T) {
+	fs, nodeID, lis, cleanup := commonSetup(t)
+	defer cleanup()
+
+	// Setup the fake management server to respond with a Listener resource that
+	// does not contain any security configuration. This should force the
+	// server-side xdsCredentials to use fallback.
+	listener := listenerResourceWithoutSecurityConfig(t, lis)
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener},
+	}); err != nil {
+		t.Error(err)
+	}
 
 	// Create a ClientConn and make a successful RPC.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
@@ -134,3 +317,96 @@ func (s) TestServerSideXDS(t *testing.T) {
 		t.Fatalf("rpc EmptyCall() failed: %v", err)
 	}
 }
+
+// TestServerSideXDS_FileWatcherCerts is an e2e test where xDS is enabled on the
+// server-side and xdsCredentials are configured for security. The control plane
+// sends security configuration pointing to the use of the file_watcher plugin,
+// and we verify that a client connecting with TLS creds is able to successfully
+// make an RPC.
+func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
+	fs, nodeID, lis, cleanup := commonSetup(t)
+	defer cleanup()
+
+	// Setup the fake management server to respond with a Listener resource with
+	// security configuration pointing to the file watcher plugin and requiring
+	// mTLS.
+	listener := listenerResourceWithSecurityConfig(t, lis)
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener},
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Create a ClientConn with TLS creds and make a successful RPC.
+	clientCreds := createClientTLSCredentials(t)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(clientCreds))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testpb.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on
+// the server-side and xdsCredentials are configured for security. The control
+// plane initially does not any security configuration. This forces the
+// xdsCredentials to use fallback creds, which is this case is insecure creds.
+// We verify that a client connecting with TLS creds is not able to successfully
+// make an RPC. The control plan then sends a listener resource with security
+// configuration pointing to the use of the file_watcher plugin and we verify
+// that the same client is now able to successfully make an RPC.
+func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
+	fs, nodeID, lis, cleanup := commonSetup(t)
+	defer cleanup()
+
+	// Setup the fake management server to respond with a Listener resource that
+	// does not contain any security configuration. This should force the
+	// server-side xdsCredentials to use fallback.
+	listener := listenerResourceWithoutSecurityConfig(t, lis)
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener},
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Create a ClientConn with TLS creds. This should fail since the server is
+	// using fallback credentials which in this case in insecure creds.
+	clientCreds := createClientTLSCredentials(t)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(clientCreds))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// We don't set 'waitForReady` here since we want this call to failfast.
+	client := testpb.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Code() != codes.Unavailable {
+		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
+	}
+
+	// Setup the fake management server to respond with a Listener resource with
+	// security configuration pointing to the file watcher plugin and requiring
+	// mTLS.
+	listener = listenerResourceWithSecurityConfig(t, lis)
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener},
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Make another RPC with `waitForReady` set and expect this to succeed.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go
index 9cd5c51ab187..25993f19fc38 100644
--- a/xds/internal/testutils/e2e/bootstrap.go
+++ b/xds/internal/testutils/e2e/bootstrap.go
@@ -46,6 +46,10 @@ type BootstrapOptions struct {
 	NodeID string
 	// ServerURI is the address of the management server.
 	ServerURI string
+	// ServerResourceNameID is the Listener resource name to fetch.
+	ServerResourceNameID string
+	// CertificateProviders is the certificate providers configuration.
+	CertificateProviders map[string]json.RawMessage
 }
 
 // SetupBootstrapFile creates a temporary file with bootstrap contents, based on
@@ -75,6 +79,8 @@ func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
 		Node: node{
 			ID: opts.NodeID,
 		},
+		CertificateProviders:     opts.CertificateProviders,
+		GRPCServerResourceNameID: opts.ServerResourceNameID,
 	}
 	switch opts.Version {
 	case TransportV2:
@@ -102,9 +108,29 @@ func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
 	}, nil
 }
 
+// DefaultFileWatcherConfig is a helper function to create a default certificate
+// provider plugin configuration. The test is expected to have setup the files
+// appropriately before this configuration is used to instantiate providers.
+func DefaultFileWatcherConfig(certPath, keyPath, caPath string) map[string]json.RawMessage {
+	cfg := fmt.Sprintf(`{
+			"plugin_name": "file_watcher",
+			"config": {
+				"certificate_file": %q,
+				"private_key_file": %q,
+				"ca_certificate_file": %q,
+				"refresh_interval": "600s"
+			}
+		}`, certPath, keyPath, caPath)
+	return map[string]json.RawMessage{
+		"google_cloud_private_spiffe": json.RawMessage(cfg),
+	}
+}
+
 type bootstrapConfig struct {
-	XdsServers []server `json:"xds_servers,omitempty"`
-	Node       node     `json:"node,omitempty"`
+	XdsServers               []server                   `json:"xds_servers,omitempty"`
+	Node                     node                       `json:"node,omitempty"`
+	CertificateProviders     map[string]json.RawMessage `json:"certificate_providers,omitempty"`
+	GRPCServerResourceNameID string                     `json:"grpc_server_resource_name_id,omitempty"`
 }
 
 type server struct {
diff --git a/xds/server.go b/xds/server.go
index ce5e7d565486..d193b8be13ac 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -20,12 +20,18 @@ package xds
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"sync"
+	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -43,7 +49,10 @@ var (
 		return grpc.NewServer(opts...)
 	}
 
-	logger = grpclog.Component("xds")
+	// Unexported function to retrieve transport credentials from a gRPC server.
+	grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials)
+	buildProvider      = buildProviderFunc
+	logger             = grpclog.Component("xds")
 )
 
 func prefixLogger(p *GRPCServer) *internalgrpclog.PrefixLogger {
@@ -77,9 +86,10 @@ type grpcServerInterface interface {
 // Notice: This type is EXPERIMENTAL and may be changed or removed in a
 // later release.
 type GRPCServer struct {
-	gs     grpcServerInterface
-	quit   *grpcsync.Event
-	logger *internalgrpclog.PrefixLogger
+	gs            grpcServerInterface
+	quit          *grpcsync.Event
+	logger        *internalgrpclog.PrefixLogger
+	xdsCredsInUse bool
 
 	// clientMu is used only in initXDSClient(), which is called at the
 	// beginning of Serve(), where we have to decide if we have to create a
@@ -108,6 +118,20 @@ func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer {
 	}
 	s.logger = prefixLogger(s)
 	s.logger.Infof("Created xds.GRPCServer")
+
+	// We type assert our underlying gRPC server to the real grpc.Server here
+	// before trying to retrieve the configured credentials. This approach
+	// avoids performing the same type assertion in the grpc package which
+	// provides the implementation for internal.GetServerCredentials, and allows
+	// us to use a fake gRPC server in tests.
+	if gs, ok := s.gs.(*grpc.Server); ok {
+		creds := grpcGetServerCreds(gs)
+		if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() {
+			s.xdsCredsInUse = true
+		}
+	}
+
+	s.logger.Infof("xDS credentials in use: %v", s.xdsCredsInUse)
 	return s
 }
 
@@ -154,6 +178,18 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 	if err := s.initXDSClient(); err != nil {
 		return err
 	}
+
+	// If xds credentials were specified by the user, but bootstrap configs do
+	// not contain any certificate provider configuration, it is better to fail
+	// right now rather than failing when attempting to create certificate
+	// providers after receiving an LDS response with security configuration.
+	if s.xdsCredsInUse {
+		bc := s.xdsC.BootstrapConfig()
+		if bc == nil || len(bc.CertProviderConfigs) == 0 {
+			return errors.New("xds: certificate_providers config missing in bootstrap file")
+		}
+	}
+
 	lw, err := s.newListenerWrapper(lis)
 	if lw == nil {
 		// Error returned can be nil (when Stop/GracefulStop() is called). So,
@@ -171,7 +207,11 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 // response is received or the server is stopped by a call to
 // Stop/GracefulStop().
 func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, error) {
-	lw := &listenerWrapper{Listener: lis}
+	lw := &listenerWrapper{
+		Listener: lis,
+		closed:   grpcsync.NewEvent(),
+		xdsHI:    xds.NewHandshakeInfo(nil, nil),
+	}
 
 	// This is used to notify that a good update has been received and that
 	// Serve() can be invoked on the underlying gRPC server. Using a
@@ -195,25 +235,14 @@ func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, err
 	// Register an LDS watch using our xdsClient, and specify the listening
 	// address as the resource name.
 	cancelWatch := s.xdsC.WatchListener(name, func(update xdsclient.ListenerUpdate, err error) {
-		if err != nil {
-			// We simply log an error here and hope we get a successful update
-			// in the future. The error could be because of a timeout or an
-			// actual error, like the requested resource not found. In any case,
-			// it is fine for the server to hang indefinitely until Stop() is
-			// called.
-			s.logger.Warningf("Received error for resource %q: %+v", name, err)
-			return
-		}
-
-		s.logger.Infof("Received update for resource %q: %+v", name, update)
-
-		// TODO(easwars): Handle security configuration, create appropriate
-		// certificate providers and update the listenerWrapper before firing
-		// the event. Errors encountered during any of these steps should result
-		// in an early exit, and the update event should not fire.
-		goodUpdate.Fire()
+		s.handleListenerUpdate(listenerUpdate{
+			lw:         lw,
+			name:       name,
+			lds:        update,
+			err:        err,
+			goodUpdate: goodUpdate,
+		})
 	})
-
 	s.logger.Infof("Watch started on resource name %v", name)
 	lw.cancelWatch = func() {
 		cancelWatch()
@@ -233,6 +262,119 @@ func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, err
 	return lw, nil
 }
 
+// listenerUpdate wraps the information received from a registered LDS watcher.
+type listenerUpdate struct {
+	lw         *listenerWrapper         // listener associated with this watch
+	name       string                   // resource name being watched
+	lds        xdsclient.ListenerUpdate // received update
+	err        error                    // received error
+	goodUpdate *grpcsync.Event          // event to fire upon a good update
+}
+
+func (s *GRPCServer) handleListenerUpdate(update listenerUpdate) {
+	if update.lw.closed.HasFired() {
+		s.logger.Warningf("Resource %q received update: %v with error: %v, after for listener was closed", update.name, update.lds, update.err)
+		return
+	}
+
+	if update.err != nil {
+		// We simply log an error here and hope we get a successful update
+		// in the future. The error could be because of a timeout or an
+		// actual error, like the requested resource not found. In any case,
+		// it is fine for the server to hang indefinitely until Stop() is
+		// called.
+		s.logger.Warningf("Received error for resource %q: %+v", update.name, update.err)
+		return
+	}
+	s.logger.Infof("Received update for resource %q: %+v", update.name, update.lds.String())
+
+	if err := s.handleSecurityConfig(update.lds.SecurityCfg, update.lw); err != nil {
+		s.logger.Warningf("Invalid security config update: %v", err)
+		return
+	}
+
+	// If we got all the way here, it means the received update was a good one.
+	update.goodUpdate.Fire()
+}
+
+func (s *GRPCServer) handleSecurityConfig(config *xdsclient.SecurityConfig, lw *listenerWrapper) error {
+	// If xdsCredentials are not in use, i.e, the user did not want to get
+	// security configuration from the control plane, we should not be acting on
+	// the received security config here. Doing so poses a security threat.
+	if !s.xdsCredsInUse {
+		return nil
+	}
+
+	// Security config being nil is a valid case where the control plane has
+	// not sent any security configuration. The xdsCredentials implementation
+	// handles this by delegating to its fallback credentials.
+	if config == nil {
+		// We need to explicitly set the fields to nil here since this might be
+		// a case of switching from a good security configuration to an empty
+		// one where fallback credentials are to be used.
+		lw.xdsHI.SetRootCertProvider(nil)
+		lw.xdsHI.SetIdentityCertProvider(nil)
+		lw.xdsHI.SetRequireClientCert(false)
+		return nil
+	}
+
+	cpc := s.xdsC.BootstrapConfig().CertProviderConfigs
+	// Identity provider is mandatory on the server side.
+	identityProvider, err := buildProvider(cpc, config.IdentityInstanceName, config.IdentityCertName, true, false)
+	if err != nil {
+		return err
+	}
+
+	// A root provider is required only when doing mTLS.
+	var rootProvider certprovider.Provider
+	if config.RootInstanceName != "" {
+		rootProvider, err = buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Close the old providers and cache the new ones.
+	lw.providerMu.Lock()
+	if lw.cachedIdentity != nil {
+		lw.cachedIdentity.Close()
+	}
+	if lw.cachedRoot != nil {
+		lw.cachedRoot.Close()
+	}
+	lw.cachedRoot = rootProvider
+	lw.cachedIdentity = identityProvider
+
+	// We set all fields here, even if some of them are nil, since they
+	// could have been non-nil earlier.
+	lw.xdsHI.SetRootCertProvider(rootProvider)
+	lw.xdsHI.SetIdentityCertProvider(identityProvider)
+	lw.xdsHI.SetRequireClientCert(config.RequireClientCert)
+	lw.providerMu.Unlock()
+
+	return nil
+}
+
+func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
+	cfg, ok := configs[instanceName]
+	if !ok {
+		return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName)
+	}
+	provider, err := cfg.Build(certprovider.BuildOptions{
+		CertName:     certName,
+		WantIdentity: wantIdentity,
+		WantRoot:     wantRoot,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return nil, fmt.Errorf("failed to get security plugin instance (%+v): %v", cfg, err)
+	}
+	return provider, nil
+}
+
 // Stop stops the underlying gRPC server. It immediately closes all open
 // connections. It cancels all active RPCs on the server side and the
 // corresponding pending RPCs on the client side will get notified by connection
@@ -279,7 +421,22 @@ type listenerWrapper struct {
 	net.Listener
 	cancelWatch func()
 
-	// TODO(easwars): Add fields for certificate providers.
+	// A small race exists in the xdsClient code where an xDS response is
+	// received while the user is calling cancel(). In this small window the
+	// registered callback can be called after the watcher is canceled. We avoid
+	// processing updates received in callbacks once the listener is closed, to
+	// make sure that we do not process updates received during this race
+	// window.
+	closed *grpcsync.Event
+
+	// The certificate providers are cached here to that they can be closed when
+	// a new provider is to be created.
+	providerMu     sync.Mutex
+	cachedRoot     certprovider.Provider
+	cachedIdentity certprovider.Provider
+
+	// Wraps all information required by the xds handshaker.
+	xdsHI *xds.HandshakeInfo
 }
 
 // Accept blocks on an Accept() on the underlying listener, and wraps the
@@ -289,17 +446,30 @@ func (l *listenerWrapper) Accept() (net.Conn, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &conn{Conn: c}, nil
+	return &conn{Conn: c, xdsHI: l.xdsHI}, nil
 }
 
 // Close closes the underlying listener. It also cancels the xDS watch
 // registered in Serve() and closes any certificate provider instances created
 // based on security configuration received in the LDS response.
 func (l *listenerWrapper) Close() error {
+	l.closed.Fire()
 	l.Listener.Close()
 	if l.cancelWatch != nil {
 		l.cancelWatch()
 	}
+
+	l.providerMu.Lock()
+	if l.cachedIdentity != nil {
+		l.cachedIdentity.Close()
+		l.cachedIdentity = nil
+	}
+	if l.cachedRoot != nil {
+		l.cachedRoot.Close()
+		l.cachedRoot = nil
+	}
+	l.providerMu.Unlock()
+
 	return nil
 }
 
@@ -307,5 +477,41 @@ func (l *listenerWrapper) Close() error {
 type conn struct {
 	net.Conn
 
-	// TODO(easwars): Add fields for certificate providers.
+	// This is the same HandshakeInfo as stored in the listenerWrapper that
+	// created this conn. The former updates the HandshakeInfo whenever it
+	// receives new security configuration.
+	xdsHI *xds.HandshakeInfo
+
+	// The connection deadline as configured by the grpc.Server on the rawConn
+	// that is returned by a call to Accept(). This is set to the connection
+	// timeout value configured by the user (or to a default value) before
+	// initiating the transport credential handshake, and set to zero after
+	// completing the HTTP2 handshake.
+	deadlineMu sync.Mutex
+	deadline   time.Time
+}
+
+// SetDeadline makes a copy of the passed in deadline and forwards the call to
+// the underlying rawConn.
+func (c *conn) SetDeadline(t time.Time) error {
+	c.deadlineMu.Lock()
+	c.deadline = t
+	c.deadlineMu.Unlock()
+	return c.Conn.SetDeadline(t)
+}
+
+// GetDeadline returns the configured deadline. This will be invoked by the
+// ServerHandshake() method of the XdsCredentials, which needs a deadline to
+// pass to the certificate provider.
+func (c *conn) GetDeadline() time.Time {
+	c.deadlineMu.Lock()
+	t := c.deadline
+	c.deadlineMu.Unlock()
+	return t
+}
+
+// XDSHandshakeInfo returns a pointer to the HandshakeInfo stored in conn. This
+// will be invoked by the ServerHandshake() method of the XdsCredentials.
+func (c *conn) XDSHandshakeInfo() *xds.HandshakeInfo {
+	return c.xdsHI
 }
diff --git a/xds/server_test.go b/xds/server_test.go
index cefb0261984e..cde963079262 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -29,6 +29,8 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -89,28 +91,56 @@ func newFakeGRPCServer() *fakeGRPCServer {
 }
 
 func (s) TestNewServer(t *testing.T) {
-	// The xds package adds a couple of server options (unary and stream
-	// interceptors) to the server options passed in by the user.
-	serverOpts := []grpc.ServerOption{grpc.Creds(insecure.NewCredentials())}
-	wantServerOpts := len(serverOpts) + 2
+	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("failed to create xds server credentials: %v", err)
+	}
 
-	origNewGRPCServer := newGRPCServer
-	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface {
-		if got := len(opts); got != wantServerOpts {
-			t.Fatalf("%d ServerOptions passed to grpc.Server, want %d", got, wantServerOpts)
-		}
-		// Verify that the user passed ServerOptions are forwarded as is.
-		if !reflect.DeepEqual(opts[2:], serverOpts) {
-			t.Fatalf("got ServerOptions %v, want %v", opts[2:], serverOpts)
-		}
-		return newFakeGRPCServer()
+	tests := []struct {
+		desc              string
+		serverOpts        []grpc.ServerOption
+		wantXDSCredsInUse bool
+	}{
+		{
+			desc:       "without_xds_creds",
+			serverOpts: []grpc.ServerOption{grpc.Creds(insecure.NewCredentials())},
+		},
+		{
+			desc:              "with_xds_creds",
+			serverOpts:        []grpc.ServerOption{grpc.Creds(xdsCreds)},
+			wantXDSCredsInUse: true,
+		},
 	}
-	defer func() {
-		newGRPCServer = origNewGRPCServer
-	}()
 
-	s := NewGRPCServer(serverOpts...)
-	defer s.Stop()
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			// The xds package adds a couple of server options (unary and stream
+			// interceptors) to the server options passed in by the user.
+			wantServerOpts := len(test.serverOpts) + 2
+
+			origNewGRPCServer := newGRPCServer
+			newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface {
+				if got := len(opts); got != wantServerOpts {
+					t.Fatalf("%d ServerOptions passed to grpc.Server, want %d", got, wantServerOpts)
+				}
+				// Verify that the user passed ServerOptions are forwarded as is.
+				if !reflect.DeepEqual(opts[2:], test.serverOpts) {
+					t.Fatalf("got ServerOptions %v, want %v", opts[2:], test.serverOpts)
+				}
+				return grpc.NewServer(opts...)
+			}
+			defer func() {
+				newGRPCServer = origNewGRPCServer
+			}()
+
+			s := NewGRPCServer(test.serverOpts...)
+			defer s.Stop()
+
+			if s.xdsCredsInUse != test.wantXDSCredsInUse {
+				t.Fatalf("xdsCredsInUse is %v, want %v", s.xdsCredsInUse, test.wantXDSCredsInUse)
+			}
+		})
+	}
 }
 
 func (s) TestRegisterService(t *testing.T) {
@@ -131,11 +161,68 @@ func (s) TestRegisterService(t *testing.T) {
 	}
 }
 
-// setupOverrides sets up overrides for bootstrap config, new xdsClient creation
-// and new gRPC.Server creation.
-func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func()) {
-	t.Helper()
+const (
+	fakeProvider1Name = "fake-certificate-provider-1"
+	fakeProvider2Name = "fake-certificate-provider-2"
+	fakeConfig        = "my fake config"
+)
+
+var (
+	fpb1, fpb2          *fakeProviderBuilder
+	certProviderConfigs map[string]*certprovider.BuildableConfig
+)
+
+func init() {
+	fpb1 = &fakeProviderBuilder{name: fakeProvider1Name}
+	fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
+	cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111")
+	cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222")
+	certProviderConfigs = map[string]*certprovider.BuildableConfig{
+		"default1": cfg1,
+		"default2": cfg2,
+	}
+	certprovider.Register(fpb1)
+	certprovider.Register(fpb2)
+}
+
+// fakeProviderBuilder builds new instances of fakeProvider and interprets the
+// config provided to it as a string.
+type fakeProviderBuilder struct {
+	name string
+}
+
+func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.BuildableConfig, error) {
+	s, ok := config.(string)
+	if !ok {
+		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
+	}
+	return certprovider.NewBuildableConfig(b.name, []byte(s), func(certprovider.BuildOptions) certprovider.Provider {
+		return &fakeProvider{
+			Distributor: certprovider.NewDistributor(),
+			config:      s,
+		}
+	}), nil
+}
+
+func (b *fakeProviderBuilder) Name() string {
+	return b.name
+}
+
+// fakeProvider is an implementation of the Provider interface which provides a
+// method for tests to invoke to push new key materials.
+type fakeProvider struct {
+	*certprovider.Distributor
+	config string
+}
+
+// Close helps implement the Provider interface.
+func (p *fakeProvider) Close() {
+	p.Distributor.Stop()
+}
 
+// setupOverrides sets up overrides for bootstrap config, new xdsClient creation,
+// new gRPC.Server creation, and certificate provider creation.
+func setupOverrides() (*fakeGRPCServer, *testutils.Channel, *testutils.Channel, func()) {
 	clientCh := testutils.NewChannel()
 	origNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) {
@@ -145,6 +232,7 @@ func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func())
 			Creds:                grpc.WithTransportCredentials(insecure.NewCredentials()),
 			NodeProto:            xdstestutils.EmptyNodeProtoV3,
 			ServerResourceNameID: testServerResourceNameID,
+			CertProviderConfigs:  certProviderConfigs,
 		})
 		clientCh.Send(c)
 		return c, nil
@@ -154,9 +242,55 @@ func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func())
 	origNewGRPCServer := newGRPCServer
 	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface { return fs }
 
-	return fs, clientCh, func() {
+	providerCh := testutils.NewChannel()
+	origBuildProvider := buildProvider
+	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
+		p, err := origBuildProvider(c, id, cert, wi, wr)
+		providerCh.Send(nil)
+		return p, err
+	}
+
+	return fs, clientCh, providerCh, func() {
 		newXDSClient = origNewXDSClient
 		newGRPCServer = origNewGRPCServer
+		buildProvider = origBuildProvider
+	}
+}
+
+// setupOverridesForXDSCreds overrides only the xdsClient creation with a fake
+// one. Tests that use xdsCredentials need a real grpc.Server instead of a fake
+// one, because the xDS-enabled server needs to read configured creds from the
+// underlying grpc.Server to confirm whether xdsCreds were configured.
+func setupOverridesForXDSCreds(includeCertProviderCfg bool) (*testutils.Channel, *testutils.Channel, func()) {
+	clientCh := testutils.NewChannel()
+	origNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) {
+		c := fakeclient.NewClient()
+		bc := &bootstrap.Config{
+			BalancerName:         "dummyBalancer",
+			Creds:                grpc.WithTransportCredentials(insecure.NewCredentials()),
+			NodeProto:            xdstestutils.EmptyNodeProtoV3,
+			ServerResourceNameID: testServerResourceNameID,
+		}
+		if includeCertProviderCfg {
+			bc.CertProviderConfigs = certProviderConfigs
+		}
+		c.SetBootstrapConfig(bc)
+		clientCh.Send(c)
+		return c, nil
+	}
+
+	providerCh := testutils.NewChannel()
+	origBuildProvider := buildProvider
+	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
+		p, err := origBuildProvider(c, id, cert, wi, wr)
+		providerCh.Send(nil)
+		return p, err
+	}
+
+	return clientCh, providerCh, func() {
+		newXDSClient = origNewXDSClient
+		buildProvider = origBuildProvider
 	}
 }
 
@@ -169,7 +303,7 @@ func setupOverrides(t *testing.T) (*fakeGRPCServer, *testutils.Channel, func())
 // 4. Push a good response from the xdsClient, and make sure that Serve() on the
 // 	  underlying grpc.Server is called.
 func (s) TestServeSuccess(t *testing.T) {
-	fs, clientCh, cleanup := setupOverrides(t)
+	fs, clientCh, _, cleanup := setupOverrides()
 	defer cleanup()
 
 	server := NewGRPCServer()
@@ -229,7 +363,7 @@ func (s) TestServeSuccess(t *testing.T) {
 // is received. This should cause Serve() to exit before calling Serve() on the
 // underlying grpc.Server.
 func (s) TestServeWithStop(t *testing.T) {
-	fs, clientCh, cleanup := setupOverrides(t)
+	fs, clientCh, _, cleanup := setupOverrides()
 	defer cleanup()
 
 	// Note that we are not deferring the Stop() here since we explicitly call
@@ -300,6 +434,39 @@ func (s) TestServeBootstrapFailure(t *testing.T) {
 		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
 	}
 
+	serveDone := testutils.NewChannel()
+	go func() { serveDone.Send(server.Serve(lis)) }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	v, err := serveDone.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for Serve() to exit: %v", err)
+	}
+	if err, ok := v.(error); !ok || err == nil {
+		t.Fatal("Serve() did not exit with error")
+	}
+}
+
+// TestServeBootstrapWithMissingCertProviders tests the case where the bootstrap
+// config does not contain certificate provider configuration, but xdsCreds are
+// passed to the server. Verifies that the call to Serve() fails.
+func (s) TestServeBootstrapWithMissingCertProviders(t *testing.T) {
+	_, _, cleanup := setupOverridesForXDSCreds(false)
+	defer cleanup()
+
+	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("failed to create xds server credentials: %v", err)
+	}
+	server := NewGRPCServer(grpc.Creds(xdsCreds))
+	defer server.Stop()
+
+	lis, err := xdstestutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
+	}
+
 	serveDone := testutils.NewChannel()
 	go func() {
 		err := server.Serve(lis)
@@ -350,3 +517,216 @@ func (s) TestServeNewClientFailure(t *testing.T) {
 		t.Fatal("Serve() did not exit with error")
 	}
 }
+
+// TestHandleListenerUpdate_NoXDSCreds tests the case where an xds-enabled gRPC
+// server is not configured with xDS credentials. Verifies that the security
+// config received as part of a Listener update is not acted upon.
+func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
+	fs, clientCh, providerCh, cleanup := setupOverrides()
+	defer cleanup()
+
+	server := NewGRPCServer()
+	defer server.Stop()
+
+	lis, err := xdstestutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
+	}
+
+	// Call Serve() in a goroutine, and push on a channel when Serve returns.
+	serveDone := testutils.NewChannel()
+	go func() {
+		if err := server.Serve(lis); err != nil {
+			t.Error(err)
+		}
+		serveDone.Send(nil)
+	}()
+
+	// Wait for an xdsClient to be created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := clientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for new xdsClient to be created: %v", err)
+	}
+	client := c.(*fakeclient.Client)
+
+	// Wait for a listener watch to be registered on the xdsClient.
+	name, err := client.WaitForWatchListener(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
+	}
+	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	if name != wantName {
+		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
+	}
+
+	// Push a good LDS response with security config, and wait for Serve() to be
+	// invoked on the underlying grpc.Server. Also make sure that certificate
+	// providers are not created.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		SecurityCfg: &xdsclient.SecurityConfig{
+			RootInstanceName:     "default1",
+			IdentityInstanceName: "default2",
+			RequireClientCert:    true,
+		},
+	}, nil)
+	if _, err := fs.serveCh.Receive(ctx); err != nil {
+		t.Fatalf("error when waiting for Serve() to be invoked on the grpc.Server")
+	}
+
+	// Make sure the security configuration is not acted upon.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("certificate provider created when no xDS creds were specified")
+	}
+}
+
+// TestHandleListenerUpdate_ErrorUpdate tests the case where an xds-enabled gRPC
+// server is configured with xDS credentials, but receives a Listener update
+// with an error. Verifies that no certificate providers are created.
+func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
+	clientCh, providerCh, cleanup := setupOverridesForXDSCreds(true)
+	defer cleanup()
+
+	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("failed to create xds server credentials: %v", err)
+	}
+
+	server := NewGRPCServer(grpc.Creds(xdsCreds))
+	defer server.Stop()
+
+	lis, err := xdstestutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
+	}
+
+	// Call Serve() in a goroutine, and push on a channel when Serve returns.
+	serveDone := testutils.NewChannel()
+	go func() {
+		if err := server.Serve(lis); err != nil {
+			t.Error(err)
+		}
+		serveDone.Send(nil)
+	}()
+
+	// Wait for an xdsClient to be created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := clientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for new xdsClient to be created: %v", err)
+	}
+	client := c.(*fakeclient.Client)
+
+	// Wait for a listener watch to be registered on the xdsClient.
+	name, err := client.WaitForWatchListener(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
+	}
+	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	if name != wantName {
+		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
+	}
+
+	// Push an error to the registered listener watch callback and make sure
+	// that Serve does not return.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		SecurityCfg: &xdsclient.SecurityConfig{
+			RootInstanceName:     "default1",
+			IdentityInstanceName: "default2",
+			RequireClientCert:    true,
+		},
+	}, errors.New("LDS error"))
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Serve() returned after a bad LDS response")
+	}
+
+	// Also make sure that no certificate providers are created.
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("certificate provider created when no xDS creds were specified")
+	}
+}
+
+func (s) TestHandleListenerUpdate_ClosedListener(t *testing.T) {
+	clientCh, providerCh, cleanup := setupOverridesForXDSCreds(true)
+	defer cleanup()
+
+	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("failed to create xds server credentials: %v", err)
+	}
+
+	server := NewGRPCServer(grpc.Creds(xdsCreds))
+	defer server.Stop()
+
+	lis, err := xdstestutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
+	}
+
+	// Call Serve() in a goroutine, and push on a channel when Serve returns.
+	serveDone := testutils.NewChannel()
+	go func() { serveDone.Send(server.Serve(lis)) }()
+
+	// Wait for an xdsClient to be created.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	c, err := clientCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for new xdsClient to be created: %v", err)
+	}
+	client := c.(*fakeclient.Client)
+
+	// Wait for a listener watch to be registered on the xdsClient.
+	name, err := client.WaitForWatchListener(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
+	}
+	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	if name != wantName {
+		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
+	}
+
+	// Push a good update to the registered listener watch callback. This will
+	// unblock the xds-enabled server which is waiting for a good listener
+	// update before calling Serve() on the underlying grpc.Server.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		SecurityCfg:     &xdsclient.SecurityConfig{IdentityInstanceName: "default2"},
+	}, nil)
+	if _, err := providerCh.Receive(ctx); err != nil {
+		t.Fatal("error when waiting for certificate provider to be created")
+	}
+
+	// Close the listener passed to Serve(), and wait for the latter to return a
+	// non-nil error.
+	lis.Close()
+	v, err := serveDone.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for Serve() to exit: %v", err)
+	}
+	if err, ok := v.(error); !ok || err == nil {
+		t.Fatal("Serve() did not exit with error")
+	}
+
+	// Push another listener update and make sure that no certificate providers
+	// are created.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		SecurityCfg:     &xdsclient.SecurityConfig{IdentityInstanceName: "default1"},
+	}, nil)
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("certificate provider created when no xDS creds were specified")
+	}
+}

From 644d506ebbbf2347d3f01f99976e796f92f1bd0f Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 16 Dec 2020 14:42:32 -0800
Subject: [PATCH 325/481] xds: Ignore application_protocols field for now
 (#4115)

---
 xds/internal/client/client_lds_test.go        | 72 ++-----------------
 xds/internal/client/client_xds.go             | 10 +--
 .../test/xds_server_integration_test.go       |  6 --
 3 files changed, 6 insertions(+), 82 deletions(-)

diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
index 35d8201f124b..6c0ec32106ea 100644
--- a/xds/internal/client/client_lds_test.go
+++ b/xds/internal/client/client_lds_test.go
@@ -453,40 +453,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			},
 			wantErr: "filter chains count in LDS response does not match expected",
 		},
-		{
-			name: "unexpected application protocol value",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"h2"},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
-			wantErr: "application_protocols in LDS response does not match expected",
-		},
 		{
 			name: "unexpected transport socket name",
 			resources: []*anypb.Any{
@@ -508,9 +474,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "unsupported-transport-socket-name",
 									},
@@ -545,9 +508,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -587,9 +547,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -630,9 +587,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -677,9 +631,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -732,9 +683,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 								},
 							},
 						}
@@ -768,9 +716,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -823,9 +768,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -872,9 +814,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -933,9 +872,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							FilterChains: []*v3listenerpb.FilterChain{
 								{
 									Name: "filter-chain-1",
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{
-										ApplicationProtocols: []string{"managed-mtls"},
-									},
 									TransportSocket: &v3corepb.TransportSocket{
 										Name: "envoy.transport_sockets.tls",
 										ConfigType: &v3corepb.TransportSocket_TypedConfig{
@@ -988,11 +924,11 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			gotUpdate, err := UnmarshalListener(test.resources, nil)
-			if err != nil && !strings.Contains(err.Error(), test.wantErr) {
-				t.Errorf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
+			if (err != nil) != (test.wantErr != "") {
+				t.Fatalf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
 			}
-			if test.wantErr != "" {
-				return
+			if err != nil && !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
 			}
 			if !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("UnmarshalListener(%v) = %v want %v", test.resources, gotUpdate, test.wantUpdate)
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index ac32b99a3608..66376571546f 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -134,19 +134,13 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 		return nil, fmt.Errorf("xds: socket_address port does not match the one in name. Got %q, want %q", p, port)
 	}
 
-	// Make sure that the listener resource contains a single filter chain with
-	// the application_protocols field set to "managed-mtls", and the
-	// "transport_socket" field containing the appropriate security
-	// configuration.
+	// Make sure the listener resource contains a single filter chain. We do not
+	// support multiple filter chains and picking the best match from the list.
 	fcs := lis.GetFilterChains()
 	if n := len(fcs); n != 1 {
 		return nil, fmt.Errorf("xds: filter chains count in LDS response does not match expected. Got %d, want 1", n)
 	}
 	fc := fcs[0]
-	aps := fc.GetFilterChainMatch().GetApplicationProtocols()
-	if len(aps) != 1 || aps[0] != "managed-mtls" {
-		return nil, fmt.Errorf("xds: application_protocols in LDS response does not match expected. Got %v, want %q", aps, "managed-mtls")
-	}
 
 	// If the transport_socket field is not specified, it means that the control
 	// plane has not sent us any security config. This is fine and the server
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 8a9c8f53c74a..300aab3a737b 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -232,9 +232,6 @@ func listenerResourceWithoutSecurityConfig(t *testing.T, lis net.Listener) *v3li
 		FilterChains: []*v3listenerpb.FilterChain{
 			{
 				Name: "filter-chain-1",
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					ApplicationProtocols: []string{"managed-mtls"},
-				},
 			},
 		},
 	}
@@ -258,9 +255,6 @@ func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3liste
 		FilterChains: []*v3listenerpb.FilterChain{
 			{
 				Name: "filter-chain-1",
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					ApplicationProtocols: []string{"managed-mtls"},
-				},
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{

From d79063fdde284ef7722591e56c72143eea59c256 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 16 Dec 2020 15:46:56 -0800
Subject: [PATCH 326/481] credentials/xds: Move non-user facing functionality
 to an internal package (#4117)

---
 credentials/xds/xds.go                        | 209 +---------------
 credentials/xds/xds_client_test.go            |   9 +-
 credentials/xds/xds_server_test.go            |  23 +-
 internal/credentials/xds/handshake_info.go    | 230 ++++++++++++++++++
 .../balancer/cdsbalancer/cdsbalancer.go       |  13 +-
 .../cdsbalancer/cdsbalancer_security_test.go  |   3 +-
 xds/server.go                                 |  10 +-
 7 files changed, 268 insertions(+), 229 deletions(-)
 create mode 100644 internal/credentials/xds/handshake_info.go

diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go
index 666da9918e62..ede0806d70db 100644
--- a/credentials/xds/xds.go
+++ b/credentials/xds/xds.go
@@ -32,21 +32,13 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync"
 	"time"
 
-	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/internal"
 	credinternal "google.golang.org/grpc/internal/credentials"
-	"google.golang.org/grpc/resolver"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 )
 
-func init() {
-	internal.GetXDSHandshakeInfoForTesting = getHandshakeInfo
-}
-
 // ClientOptions contains parameters to configure a new client-side xDS
 // credentials implementation.
 type ClientOptions struct {
@@ -97,193 +89,6 @@ type credsImpl struct {
 	fallback credentials.TransportCredentials
 }
 
-// handshakeAttrKey is the type used as the key to store HandshakeInfo in
-// the Attributes field of resolver.Address.
-type handshakeAttrKey struct{}
-
-// SetHandshakeInfo returns a copy of addr in which the Attributes field is
-// updated with hInfo.
-func SetHandshakeInfo(addr resolver.Address, hInfo *HandshakeInfo) resolver.Address {
-	addr.Attributes = addr.Attributes.WithValues(handshakeAttrKey{}, hInfo)
-	return addr
-}
-
-// getHandshakeInfo returns a pointer to the HandshakeInfo stored in attr.
-func getHandshakeInfo(attr *attributes.Attributes) *HandshakeInfo {
-	v := attr.Value(handshakeAttrKey{})
-	hi, _ := v.(*HandshakeInfo)
-	return hi
-}
-
-// HandshakeInfo wraps all the security configuration required by client and
-// server handshake methods in credsImpl. The xDS implementation will be
-// responsible for populating these fields.
-//
-// Safe for concurrent access.
-//
-// TODO(easwars): Move this type and any other non-user functionality to an
-// internal package.
-type HandshakeInfo struct {
-	mu                sync.Mutex
-	rootProvider      certprovider.Provider
-	identityProvider  certprovider.Provider
-	acceptedSANs      map[string]bool // Only on the client side.
-	requireClientCert bool            // Only on server side.
-}
-
-// SetRootCertProvider updates the root certificate provider.
-func (hi *HandshakeInfo) SetRootCertProvider(root certprovider.Provider) {
-	hi.mu.Lock()
-	hi.rootProvider = root
-	hi.mu.Unlock()
-}
-
-// SetIdentityCertProvider updates the identity certificate provider.
-func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider) {
-	hi.mu.Lock()
-	hi.identityProvider = identity
-	hi.mu.Unlock()
-}
-
-// SetAcceptedSANs updates the list of accepted SANs.
-func (hi *HandshakeInfo) SetAcceptedSANs(sans []string) {
-	hi.mu.Lock()
-	hi.acceptedSANs = make(map[string]bool, len(sans))
-	for _, san := range sans {
-		hi.acceptedSANs[san] = true
-	}
-	hi.mu.Unlock()
-}
-
-// SetRequireClientCert updates whether a client cert is required during the
-// ServerHandshake(). A value of true indicates that we are performing mTLS.
-func (hi *HandshakeInfo) SetRequireClientCert(require bool) {
-	hi.mu.Lock()
-	hi.requireClientCert = require
-	hi.mu.Unlock()
-}
-
-// UseFallbackCreds returns true when fallback credentials are to be used based
-// on the contents of the HandshakeInfo.
-func (hi *HandshakeInfo) UseFallbackCreds() bool {
-	if hi == nil {
-		return true
-	}
-
-	hi.mu.Lock()
-	defer hi.mu.Unlock()
-	return hi.identityProvider == nil && hi.rootProvider == nil
-}
-
-func (hi *HandshakeInfo) makeClientSideTLSConfig(ctx context.Context) (*tls.Config, error) {
-	hi.mu.Lock()
-	// On the client side, rootProvider is mandatory. IdentityProvider is
-	// optional based on whether the client is doing TLS or mTLS.
-	if hi.rootProvider == nil {
-		return nil, errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server")
-	}
-	// Since the call to KeyMaterial() can block, we read the providers under
-	// the lock but call the actual function after releasing the lock.
-	rootProv, idProv := hi.rootProvider, hi.identityProvider
-	hi.mu.Unlock()
-
-	// InsecureSkipVerify needs to be set to true because we need to perform
-	// custom verification to check the SAN on the received certificate.
-	// Currently the Go stdlib does complete verification of the cert (which
-	// includes hostname verification) or none. We are forced to go with the
-	// latter and perform the normal cert validation ourselves.
-	cfg := &tls.Config{InsecureSkipVerify: true}
-
-	km, err := rootProv.KeyMaterial(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
-	}
-	cfg.RootCAs = km.Roots
-
-	if idProv != nil {
-		km, err := idProv.KeyMaterial(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err)
-		}
-		cfg.Certificates = km.Certs
-	}
-	return cfg, nil
-}
-
-func (hi *HandshakeInfo) makeServerSideTLSConfig(ctx context.Context) (*tls.Config, error) {
-	cfg := &tls.Config{ClientAuth: tls.NoClientCert}
-	hi.mu.Lock()
-	// On the server side, identityProvider is mandatory. RootProvider is
-	// optional based on whether the server is doing TLS or mTLS.
-	if hi.identityProvider == nil {
-		return nil, errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server")
-	}
-	// Since the call to KeyMaterial() can block, we read the providers under
-	// the lock but call the actual function after releasing the lock.
-	rootProv, idProv := hi.rootProvider, hi.identityProvider
-	if hi.requireClientCert {
-		cfg.ClientAuth = tls.RequireAndVerifyClientCert
-	}
-	hi.mu.Unlock()
-
-	// identityProvider is mandatory on the server side.
-	km, err := idProv.KeyMaterial(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err)
-	}
-	cfg.Certificates = km.Certs
-
-	if rootProv != nil {
-		km, err := rootProv.KeyMaterial(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
-		}
-		cfg.ClientCAs = km.Roots
-	}
-	return cfg, nil
-}
-
-func (hi *HandshakeInfo) matchingSANExists(cert *x509.Certificate) bool {
-	if len(hi.acceptedSANs) == 0 {
-		// An empty list of acceptedSANs means "accept everything".
-		return true
-	}
-
-	var sans []string
-	// SANs can be specified in any of these four fields on the parsed cert.
-	sans = append(sans, cert.DNSNames...)
-	sans = append(sans, cert.EmailAddresses...)
-	for _, ip := range cert.IPAddresses {
-		sans = append(sans, ip.String())
-	}
-	for _, uri := range cert.URIs {
-		sans = append(sans, uri.String())
-	}
-
-	hi.mu.Lock()
-	defer hi.mu.Unlock()
-	for _, san := range sans {
-		if hi.acceptedSANs[san] {
-			return true
-		}
-	}
-	return false
-}
-
-// NewHandshakeInfo returns a new instance of HandshakeInfo with the given root
-// and identity certificate providers.
-func NewHandshakeInfo(root, identity certprovider.Provider, sans ...string) *HandshakeInfo {
-	acceptedSANs := make(map[string]bool, len(sans))
-	for _, san := range sans {
-		acceptedSANs[san] = true
-	}
-	return &HandshakeInfo{
-		rootProvider:     root,
-		identityProvider: identity,
-		acceptedSANs:     acceptedSANs,
-	}
-}
-
 // ClientHandshake performs the TLS handshake on the client-side.
 //
 // It looks for the presence of a HandshakeInfo value in the passed in context
@@ -314,7 +119,7 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 	if chi.Attributes == nil {
 		return c.fallback.ClientHandshake(ctx, authority, rawConn)
 	}
-	hi := getHandshakeInfo(chi.Attributes)
+	hi := xdsinternal.GetHandshakeInfo(chi.Attributes)
 	if hi.UseFallbackCreds() {
 		return c.fallback.ClientHandshake(ctx, authority, rawConn)
 	}
@@ -331,7 +136,7 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 	// 4. Key usage to match whether client/server usage.
 	// 5. A `VerifyPeerCertificate` function which performs normal peer
 	// 	  cert verification using configured roots, and the custom SAN checks.
-	cfg, err := hi.makeClientSideTLSConfig(ctx)
+	cfg, err := hi.ClientSideTLSConfig(ctx)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -362,7 +167,7 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 		}
 		// The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to
 		// only look at the SANs on the leaf cert.
-		if !hi.matchingSANExists(certs[0]) {
+		if !hi.MatchingSANExists(certs[0]) {
 			return fmt.Errorf("SANs received in leaf certificate %+v does not match any of the accepted SANs", certs[0])
 		}
 		return nil
@@ -410,7 +215,9 @@ func (c *credsImpl) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.Aut
 	// passed to this function does not implement this interface, or if the
 	// `HandshakeInfo` does not contain the information we are looking for, we
 	// delegate the handshake to the fallback credentials.
-	hiConn, ok := rawConn.(interface{ XDSHandshakeInfo() *HandshakeInfo })
+	hiConn, ok := rawConn.(interface {
+		XDSHandshakeInfo() *xdsinternal.HandshakeInfo
+	})
 	if !ok {
 		return c.fallback.ServerHandshake(rawConn)
 	}
@@ -430,7 +237,7 @@ func (c *credsImpl) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.Aut
 	}
 	ctx, cancel := context.WithDeadline(context.Background(), deadline)
 	defer cancel()
-	cfg, err := hi.makeServerSideTLSConfig(ctx)
+	cfg, err := hi.ServerSideTLSConfig(ctx)
 	if err != nil {
 		return nil, nil, err
 	}
diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index f22579c6b8a1..219d0aefcba6 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -33,6 +33,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
@@ -217,8 +218,8 @@ func newTestContextWithHandshakeInfo(parent context.Context, root, identity cert
 	// Creating the HandshakeInfo and adding it to the attributes is very
 	// similar to what the CDS balancer would do when it intercepts calls to
 	// NewSubConn().
-	info := NewHandshakeInfo(root, identity, sans...)
-	addr := SetHandshakeInfo(resolver.Address{}, info)
+	info := xdsinternal.NewHandshakeInfo(root, identity, sans...)
+	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)
 
 	// Moving the attributes from the resolver.Address to the context passed to
 	// the handshaker is done in the transport layer. Since we directly call the
@@ -529,12 +530,12 @@ func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	// Create a root provider which will fail the handshake because it does not
 	// use the correct trust roots.
 	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
-	handshakeInfo := NewHandshakeInfo(root1, nil, defaultTestCertSAN)
+	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil, defaultTestCertSAN)
 
 	// We need to repeat most of what newTestContextWithHandshakeInfo() does
 	// here because we need access to the underlying HandshakeInfo so that we
 	// can update it before the next call to ClientHandshake().
-	addr := SetHandshakeInfo(resolver.Address{}, handshakeInfo)
+	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, handshakeInfo)
 	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
 	ctx = contextWithHandshakeInfo(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
diff --git a/credentials/xds/xds_server_test.go b/credentials/xds/xds_server_test.go
index b9c62dbd5553..68d92b28e286 100644
--- a/credentials/xds/xds_server_test.go
+++ b/credentials/xds/xds_server_test.go
@@ -32,6 +32,7 @@ import (
 
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/testdata"
 )
 
@@ -94,11 +95,11 @@ func (s) TestServerCredsWithoutFallback(t *testing.T) {
 
 type wrapperConn struct {
 	net.Conn
-	xdsHI    *HandshakeInfo
+	xdsHI    *xdsinternal.HandshakeInfo
 	deadline time.Time
 }
 
-func (wc *wrapperConn) XDSHandshakeInfo() *HandshakeInfo {
+func (wc *wrapperConn) XDSHandshakeInfo() *xdsinternal.HandshakeInfo {
 	return wc.xdsHI
 }
 
@@ -106,7 +107,7 @@ func (wc *wrapperConn) GetDeadline() time.Time {
 	return wc.deadline
 }
 
-func newWrappedConn(conn net.Conn, xdsHI *HandshakeInfo, deadline time.Time) *wrapperConn {
+func newWrappedConn(conn net.Conn, xdsHI *xdsinternal.HandshakeInfo, deadline time.Time) *wrapperConn {
 	return &wrapperConn{Conn: conn, xdsHI: xdsHI, deadline: deadline}
 }
 
@@ -120,7 +121,7 @@ func (s) TestServerCredsInvalidHandshakeInfo(t *testing.T) {
 		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
 	}
 
-	info := NewHandshakeInfo(&fakeProvider{}, nil)
+	info := xdsinternal.NewHandshakeInfo(&fakeProvider{}, nil)
 	conn := newWrappedConn(nil, info, time.Time{})
 	if _, _, err := creds.ServerHandshake(conn); err == nil {
 		t.Fatal("ServerHandshake succeeded without identity certificate provider in HandshakeInfo")
@@ -156,7 +157,7 @@ func (s) TestServerCredsProviderFailure(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			info := NewHandshakeInfo(test.rootProvider, test.identityProvider)
+			info := xdsinternal.NewHandshakeInfo(test.rootProvider, test.identityProvider)
 			conn := newWrappedConn(nil, info, time.Time{})
 			if _, _, err := creds.ServerHandshake(conn); err == nil || !strings.Contains(err.Error(), test.wantErr) {
 				t.Fatalf("ServerHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
@@ -178,7 +179,7 @@ func (s) TestServerCredsHandshakeTimeout(t *testing.T) {
 	// Create a test server which uses the xDS server credentials created above
 	// to perform TLS handshake on incoming connections.
 	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
-		hi := NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"))
+		hi := xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"))
 		hi.SetRequireClientCert(true)
 
 		// Create a wrapped conn which can return the HandshakeInfo created
@@ -231,7 +232,7 @@ func (s) TestServerCredsHandshakeFailure(t *testing.T) {
 	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
 		// Create a HandshakeInfo which has a root provider which does not match
 		// the certificate sent by the client.
-		hi := NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"))
+		hi := xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"))
 		hi.SetRequireClientCert(true)
 
 		// Create a wrapped conn which can return the HandshakeInfo and
@@ -313,7 +314,7 @@ func (s) TestServerCredsHandshakeSuccess(t *testing.T) {
 			// created above to perform TLS handshake on incoming connections.
 			ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
 				// Create a HandshakeInfo with information from the test table.
-				hi := NewHandshakeInfo(test.rootProvider, test.identityProvider)
+				hi := xdsinternal.NewHandshakeInfo(test.rootProvider, test.identityProvider)
 				hi.SetRequireClientCert(test.requireClientCert)
 
 				// Create a wrapped conn which can return the HandshakeInfo and
@@ -390,11 +391,11 @@ func (s) TestServerCredsProviderSwitch(t *testing.T) {
 	// to perform TLS handshake on incoming connections.
 	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
 		cnt++
-		var hi *HandshakeInfo
+		var hi *xdsinternal.HandshakeInfo
 		if cnt == 1 {
 			// Create a HandshakeInfo which has a root provider which does not match
 			// the certificate sent by the client.
-			hi = NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"))
+			hi = xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"))
 			hi.SetRequireClientCert(true)
 
 			// Create a wrapped conn which can return the HandshakeInfo and
@@ -409,7 +410,7 @@ func (s) TestServerCredsProviderSwitch(t *testing.T) {
 			return handshakeResult{}
 		}
 
-		hi = NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
+		hi = xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
 		hi.SetRequireClientCert(true)
 
 		// Create a wrapped conn which can return the HandshakeInfo and
diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go
new file mode 100644
index 000000000000..8b2035660639
--- /dev/null
+++ b/internal/credentials/xds/handshake_info.go
@@ -0,0 +1,230 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds contains non-user facing functionality of the xds credentials.
+package xds
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/resolver"
+)
+
+func init() {
+	internal.GetXDSHandshakeInfoForTesting = GetHandshakeInfo
+}
+
+// handshakeAttrKey is the type used as the key to store HandshakeInfo in
+// the Attributes field of resolver.Address.
+type handshakeAttrKey struct{}
+
+// SetHandshakeInfo returns a copy of addr in which the Attributes field is
+// updated with hInfo.
+func SetHandshakeInfo(addr resolver.Address, hInfo *HandshakeInfo) resolver.Address {
+	addr.Attributes = addr.Attributes.WithValues(handshakeAttrKey{}, hInfo)
+	return addr
+}
+
+// GetHandshakeInfo returns a pointer to the HandshakeInfo stored in attr.
+func GetHandshakeInfo(attr *attributes.Attributes) *HandshakeInfo {
+	v := attr.Value(handshakeAttrKey{})
+	hi, _ := v.(*HandshakeInfo)
+	return hi
+}
+
+// HandshakeInfo wraps all the security configuration required by client and
+// server handshake methods in xds credentials. The xDS implementation will be
+// responsible for populating these fields.
+//
+// Safe for concurrent access.
+type HandshakeInfo struct {
+	mu                sync.Mutex
+	rootProvider      certprovider.Provider
+	identityProvider  certprovider.Provider
+	acceptedSANs      map[string]bool // Only on the client side.
+	requireClientCert bool            // Only on server side.
+}
+
+// SetRootCertProvider updates the root certificate provider.
+func (hi *HandshakeInfo) SetRootCertProvider(root certprovider.Provider) {
+	hi.mu.Lock()
+	hi.rootProvider = root
+	hi.mu.Unlock()
+}
+
+// SetIdentityCertProvider updates the identity certificate provider.
+func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider) {
+	hi.mu.Lock()
+	hi.identityProvider = identity
+	hi.mu.Unlock()
+}
+
+// SetAcceptedSANs updates the list of accepted SANs.
+func (hi *HandshakeInfo) SetAcceptedSANs(sans []string) {
+	hi.mu.Lock()
+	hi.acceptedSANs = make(map[string]bool, len(sans))
+	for _, san := range sans {
+		hi.acceptedSANs[san] = true
+	}
+	hi.mu.Unlock()
+}
+
+// SetRequireClientCert updates whether a client cert is required during the
+// ServerHandshake(). A value of true indicates that we are performing mTLS.
+func (hi *HandshakeInfo) SetRequireClientCert(require bool) {
+	hi.mu.Lock()
+	hi.requireClientCert = require
+	hi.mu.Unlock()
+}
+
+// UseFallbackCreds returns true when fallback credentials are to be used based
+// on the contents of the HandshakeInfo.
+func (hi *HandshakeInfo) UseFallbackCreds() bool {
+	if hi == nil {
+		return true
+	}
+
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+	return hi.identityProvider == nil && hi.rootProvider == nil
+}
+
+// ClientSideTLSConfig constructs a tls.Config to be used in a client-side
+// handshake based on the contents of the HandshakeInfo.
+func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config, error) {
+	hi.mu.Lock()
+	// On the client side, rootProvider is mandatory. IdentityProvider is
+	// optional based on whether the client is doing TLS or mTLS.
+	if hi.rootProvider == nil {
+		return nil, errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server")
+	}
+	// Since the call to KeyMaterial() can block, we read the providers under
+	// the lock but call the actual function after releasing the lock.
+	rootProv, idProv := hi.rootProvider, hi.identityProvider
+	hi.mu.Unlock()
+
+	// InsecureSkipVerify needs to be set to true because we need to perform
+	// custom verification to check the SAN on the received certificate.
+	// Currently the Go stdlib does complete verification of the cert (which
+	// includes hostname verification) or none. We are forced to go with the
+	// latter and perform the normal cert validation ourselves.
+	cfg := &tls.Config{InsecureSkipVerify: true}
+
+	km, err := rootProv.KeyMaterial(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
+	}
+	cfg.RootCAs = km.Roots
+
+	if idProv != nil {
+		km, err := idProv.KeyMaterial(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err)
+		}
+		cfg.Certificates = km.Certs
+	}
+	return cfg, nil
+}
+
+// ServerSideTLSConfig constructs a tls.Config to be used in a server-side
+// handshake based on the contents of the HandshakeInfo.
+func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) {
+	cfg := &tls.Config{ClientAuth: tls.NoClientCert}
+	hi.mu.Lock()
+	// On the server side, identityProvider is mandatory. RootProvider is
+	// optional based on whether the server is doing TLS or mTLS.
+	if hi.identityProvider == nil {
+		return nil, errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server")
+	}
+	// Since the call to KeyMaterial() can block, we read the providers under
+	// the lock but call the actual function after releasing the lock.
+	rootProv, idProv := hi.rootProvider, hi.identityProvider
+	if hi.requireClientCert {
+		cfg.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+	hi.mu.Unlock()
+
+	// identityProvider is mandatory on the server side.
+	km, err := idProv.KeyMaterial(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err)
+	}
+	cfg.Certificates = km.Certs
+
+	if rootProv != nil {
+		km, err := rootProv.KeyMaterial(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
+		}
+		cfg.ClientCAs = km.Roots
+	}
+	return cfg, nil
+}
+
+// MatchingSANExists returns true if the SAN contained in the passed in
+// certificate is present in the list of accepted SANs in the HandshakeInfo.
+//
+// If the list of accepted SANs in the HandshakeInfo is empty, this function
+// returns true for all input certificates.
+func (hi *HandshakeInfo) MatchingSANExists(cert *x509.Certificate) bool {
+	if len(hi.acceptedSANs) == 0 {
+		return true
+	}
+
+	var sans []string
+	// SANs can be specified in any of these four fields on the parsed cert.
+	sans = append(sans, cert.DNSNames...)
+	sans = append(sans, cert.EmailAddresses...)
+	for _, ip := range cert.IPAddresses {
+		sans = append(sans, ip.String())
+	}
+	for _, uri := range cert.URIs {
+		sans = append(sans, uri.String())
+	}
+
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+	for _, san := range sans {
+		if hi.acceptedSANs[san] {
+			return true
+		}
+	}
+	return false
+}
+
+// NewHandshakeInfo returns a new instance of HandshakeInfo with the given root
+// and identity certificate providers.
+func NewHandshakeInfo(root, identity certprovider.Provider, sans ...string) *HandshakeInfo {
+	acceptedSANs := make(map[string]bool, len(sans))
+	for _, san := range sans {
+		acceptedSANs[san] = true
+	}
+	return &HandshakeInfo{
+		rootProvider:     root,
+		identityProvider: identity,
+		acceptedSANs:     acceptedSANs,
+	}
+}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 3a276c034f2a..5a8a3c35d79c 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -27,17 +27,16 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/buffer"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
 	"google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/client/bootstrap"
-
 	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
 
 const (
@@ -81,7 +80,7 @@ func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.
 		updateCh:    buffer.NewUnbounded(),
 		closed:      grpcsync.NewEvent(),
 		cancelWatch: func() {}, // No-op at this point.
-		xdsHI:       xds.NewHandshakeInfo(nil, nil),
+		xdsHI:       xdsinternal.NewHandshakeInfo(nil, nil),
 	}
 	b.logger = prefixLogger((b))
 	b.logger.Infof("Created")
@@ -188,7 +187,7 @@ type cdsBalancer struct {
 	// a new provider is to be created.
 	cachedRoot     certprovider.Provider
 	cachedIdentity certprovider.Provider
-	xdsHI          *xds.HandshakeInfo
+	xdsHI          *xdsinternal.HandshakeInfo
 	xdsCredsInUse  bool
 }
 
@@ -506,7 +505,7 @@ type ccWrapper struct {
 
 	// The certificate providers in this HandshakeInfo are updated based on the
 	// received security configuration in the Cluster resource.
-	xdsHI *xds.HandshakeInfo
+	xdsHI *xdsinternal.HandshakeInfo
 }
 
 // NewSubConn intercepts NewSubConn() calls from the child policy and adds an
@@ -515,7 +514,7 @@ type ccWrapper struct {
 func (ccw *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
 	newAddrs := make([]resolver.Address, len(addrs))
 	for i, addr := range addrs {
-		newAddrs[i] = xds.SetHandshakeInfo(addr, ccw.xdsHI)
+		newAddrs[i] = xdsinternal.SetHandshakeInfo(addr, ccw.xdsHI)
 	}
 	return ccw.ClientConn.NewSubConn(newAddrs, opts)
 }
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 52c7f23281ee..b39b2abf06bc 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -28,6 +28,7 @@ import (
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -188,7 +189,7 @@ func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xd
 		if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
 			return fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want)
 		}
-		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xds.HandshakeInfo)
+		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdsinternal.HandshakeInfo)
 		hi := getHI(gotAddrs[0].Attributes)
 		if hi == nil {
 			return errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes")
diff --git a/xds/server.go b/xds/server.go
index d193b8be13ac..f1c1e4181b8c 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -29,9 +29,9 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -210,7 +210,7 @@ func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, err
 	lw := &listenerWrapper{
 		Listener: lis,
 		closed:   grpcsync.NewEvent(),
-		xdsHI:    xds.NewHandshakeInfo(nil, nil),
+		xdsHI:    xdsinternal.NewHandshakeInfo(nil, nil),
 	}
 
 	// This is used to notify that a good update has been received and that
@@ -436,7 +436,7 @@ type listenerWrapper struct {
 	cachedIdentity certprovider.Provider
 
 	// Wraps all information required by the xds handshaker.
-	xdsHI *xds.HandshakeInfo
+	xdsHI *xdsinternal.HandshakeInfo
 }
 
 // Accept blocks on an Accept() on the underlying listener, and wraps the
@@ -480,7 +480,7 @@ type conn struct {
 	// This is the same HandshakeInfo as stored in the listenerWrapper that
 	// created this conn. The former updates the HandshakeInfo whenever it
 	// receives new security configuration.
-	xdsHI *xds.HandshakeInfo
+	xdsHI *xdsinternal.HandshakeInfo
 
 	// The connection deadline as configured by the grpc.Server on the rawConn
 	// that is returned by a call to Accept(). This is set to the connection
@@ -512,6 +512,6 @@ func (c *conn) GetDeadline() time.Time {
 
 // XDSHandshakeInfo returns a pointer to the HandshakeInfo stored in conn. This
 // will be invoked by the ServerHandshake() method of the XdsCredentials.
-func (c *conn) XDSHandshakeInfo() *xds.HandshakeInfo {
+func (c *conn) XDSHandshakeInfo() *xdsinternal.HandshakeInfo {
 	return c.xdsHI
 }

From 15458d28204f6a78654de64b61d7c4a65117265a Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 17 Dec 2020 10:40:27 -0800
Subject: [PATCH 327/481] xds: implement RouteAction timeout support (#4116)

---
 internal/wrr/random.go                     |  9 ++
 xds/internal/client/client.go              |  5 +-
 xds/internal/client/client_rds_test.go     | 92 ++++++++++++++++++++
 xds/internal/client/client_xds.go          | 10 ++-
 xds/internal/env/env.go                    |  5 ++
 xds/internal/resolver/serviceconfig.go     | 40 +++++----
 xds/internal/resolver/xds_resolver_test.go | 98 ++++++++++++++++++++++
 xds/internal/testutils/wrr.go              |  5 ++
 8 files changed, 248 insertions(+), 16 deletions(-)

diff --git a/internal/wrr/random.go b/internal/wrr/random.go
index a43652dcb735..ccf5113e9f32 100644
--- a/internal/wrr/random.go
+++ b/internal/wrr/random.go
@@ -18,6 +18,7 @@
 package wrr
 
 import (
+	"fmt"
 	"sync"
 
 	"google.golang.org/grpc/internal/grpcrand"
@@ -29,6 +30,10 @@ type weightedItem struct {
 	Weight int64
 }
 
+func (w *weightedItem) String() string {
+	return fmt.Sprint(*w)
+}
+
 // randomWRR is a struct that contains weighted items implement weighted random algorithm.
 type randomWRR struct {
 	mu           sync.RWMutex
@@ -68,3 +73,7 @@ func (rw *randomWRR) Add(item interface{}, weight int64) {
 	rw.items = append(rw.items, rItem)
 	rw.sumOfWeights += weight
 }
+
+func (rw *randomWRR) String() string {
+	return fmt.Sprint(rw.items)
+}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index d78122ca5b9d..4007a420f484 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -179,7 +179,10 @@ type Route struct {
 	CaseInsensitive bool
 	Headers         []*HeaderMatcher
 	Fraction        *uint32
-	Action          map[string]uint32 // action is weighted clusters.
+
+	// If the matchers above indicate a match, the below configuration is used.
+	Action            map[string]uint32 // action is weighted clusters.
+	MaxStreamDuration time.Duration
 }
 
 // HeaderMatcher represents header matchers.
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index 5e9ee7758657..ab4737376e54 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -20,6 +20,7 @@ package client
 
 import (
 	"testing"
+	"time"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
@@ -33,6 +34,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/types/known/durationpb"
 )
 
 func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
@@ -290,6 +292,96 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "good-route-config-with-max-stream-duration",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier:  &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+										MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{MaxStreamDuration: durationpb.New(time.Second)},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: time.Second}},
+					},
+				},
+			},
+		},
+		{
+			name: "good-route-config-with-grpc-timeout-header-max",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier:  &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+										MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{GrpcTimeoutHeaderMax: durationpb.New(time.Second)},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: time.Second}},
+					},
+				},
+			},
+		},
+		{
+			name: "good-route-config-with-both-timeouts",
+			rc: &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes: []*v3routepb.Route{
+							{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+								Action: &v3routepb.Route_Route{
+									Route: &v3routepb.RouteAction{
+										ClusterSpecifier:  &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+										MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{MaxStreamDuration: durationpb.New(2 * time.Second), GrpcTimeoutHeaderMax: durationpb.New(0)},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdate: RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{
+					{
+						Domains: []string{ldsTarget},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: 0}},
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 66376571546f..f31b6009b6e1 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -322,7 +322,8 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 		}
 
 		clusters := make(map[string]uint32)
-		switch a := r.GetRoute().GetClusterSpecifier().(type) {
+		action := r.GetRoute()
+		switch a := action.GetClusterSpecifier().(type) {
 		case *v3routepb.RouteAction_Cluster:
 			clusters[a.Cluster] = 1
 		case *v3routepb.RouteAction_WeightedClusters:
@@ -341,6 +342,13 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 		}
 
 		route.Action = clusters
+		msd := action.GetMaxStreamDuration()
+		// Prefer grpc_timeout_header_max, if set.
+		if dur := msd.GetGrpcTimeoutHeaderMax(); dur != nil {
+			route.MaxStreamDuration = dur.AsDuration()
+		} else {
+			route.MaxStreamDuration = msd.GetMaxStreamDuration().AsDuration()
+		}
 		routesRet = append(routesRet, &route)
 	}
 	return routesRet, nil
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index c0fa0e65b7a3..c4b46bae171b 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -29,6 +29,7 @@ const (
 	bootstrapFileNameEnv      = "GRPC_XDS_BOOTSTRAP"
 	xdsV3SupportEnv           = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
 	circuitBreakingSupportEnv = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
+	timeoutSupportEnv         = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
 )
 
 var (
@@ -44,4 +45,8 @@ var (
 	// enabled, which can be done by setting the environment variable
 	// "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING" to "true".
 	CircuitBreakingSupport = strings.EqualFold(os.Getenv(circuitBreakingSupportEnv), "true")
+	// TimeoutSupport indicates whether support for max_stream_duration in
+	// route actions is enabled.  This can be enabled by setting the
+	// environment variable "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT" to "true".
+	TimeoutSupport = strings.EqualFold(os.Getenv(timeoutSupportEnv), "true")
 )
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 50514303e894..13d3f2a095fc 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -22,12 +22,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync/atomic"
+	"time"
 
 	"google.golang.org/grpc/codes"
 	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
+	"google.golang.org/grpc/xds/internal/env"
 )
 
 const (
@@ -93,12 +95,13 @@ func serviceConfigJSON(activeClusters map[string]*clusterInfo) (string, error) {
 }
 
 type route struct {
-	action wrr.WRR
-	m      *compositeMatcher // converted from route matchers
+	m                 *compositeMatcher // converted from route matchers
+	clusters          wrr.WRR
+	maxStreamDuration time.Duration
 }
 
 func (r route) String() string {
-	return r.m.String() + "->" + fmt.Sprint(r.action)
+	return fmt.Sprintf("%s -> { clusters: %v, maxStreamDuration: %v }", r.m.String(), r.clusters, r.maxStreamDuration)
 }
 
 type configSelector struct {
@@ -110,18 +113,18 @@ type configSelector struct {
 var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found")
 
 func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
-	var action wrr.WRR
+	var rt *route
 	// Loop through routes in order and select first match.
-	for _, rt := range cs.routes {
-		if rt.m.match(rpcInfo) {
-			action = rt.action
+	for _, r := range cs.routes {
+		if r.m.match(rpcInfo) {
+			rt = &r
 			break
 		}
 	}
-	if action == nil {
+	if rt == nil || rt.clusters == nil {
 		return nil, errNoMatchedRouteFound
 	}
-	cluster, ok := action.Next().(string)
+	cluster, ok := rt.clusters.Next().(string)
 	if !ok {
 		return nil, status.Errorf(codes.Internal, "error retrieving cluster for match: %v (%T)", cluster, cluster)
 	}
@@ -129,7 +132,8 @@ func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RP
 	// it is committed.
 	ref := &cs.clusters[cluster].refCount
 	atomic.AddInt32(ref, 1)
-	return &iresolver.RPCConfig{
+
+	config := &iresolver.RPCConfig{
 		// Communicate to the LB policy the chosen cluster.
 		Context: clustermanager.SetPickedCluster(rpcInfo.Context, cluster),
 		OnCommitted: func() {
@@ -144,7 +148,13 @@ func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RP
 				}
 			}
 		},
-	}, nil
+	}
+
+	if env.TimeoutSupport && rt.maxStreamDuration != 0 {
+		config.MethodConfig.Timeout = &rt.maxStreamDuration
+	}
+
+	return config, nil
 }
 
 // incRefs increments refs of all clusters referenced by this config selector.
@@ -196,9 +206,9 @@ func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, erro
 	}
 
 	for i, rt := range su.Routes {
-		action := newWRR()
+		clusters := newWRR()
 		for cluster, weight := range rt.Action {
-			action.Add(cluster, int64(weight))
+			clusters.Add(cluster, int64(weight))
 
 			// Initialize entries in cs.clusters map, creating entries in
 			// r.activeClusters as necessary.  Set to zero as they will be
@@ -210,14 +220,16 @@ func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, erro
 			}
 			cs.clusters[cluster] = ci
 		}
-		cs.routes[i].action = action
+		cs.routes[i].clusters = clusters
 
 		var err error
 		cs.routes[i].m, err = routeToMatcher(rt)
 		if err != nil {
 			return nil, err
 		}
+		cs.routes[i].maxStreamDuration = rt.MaxStreamDuration
 	}
+
 	return cs, nil
 }
 
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index f3bdc57c0ba0..a3c0f2866353 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -41,6 +41,7 @@ import (
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/env"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
@@ -496,6 +497,103 @@ func (s) TestXDSResolverWRR(t *testing.T) {
 	}
 }
 
+func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
+	defer func(old bool) { env.TimeoutSupport = old }(env.TimeoutSupport)
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer func() {
+		cancel()
+		xdsR.Close()
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR)
+	newWRR = xdstestutils.NewTestWRR
+
+	// Invoke the watchAPI callback with a good service update and wait for the
+	// UpdateState method to be called on the ClientConn.
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes: []*client.Route{{
+					Prefix:            newStringP("/foo"),
+					Action:            map[string]uint32{"A": 1},
+					MaxStreamDuration: 5 * time.Second,
+				}, {
+					Prefix:            newStringP("/bar"),
+					Action:            map[string]uint32{"B": 1},
+					MaxStreamDuration: time.Duration(0),
+				}, {
+					Prefix: newStringP(""),
+					Action: map[string]uint32{"C": 1},
+				}},
+			},
+		},
+	}, nil)
+
+	gotState, err := tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState := gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+
+	cs := iresolver.GetConfigSelector(rState)
+	if cs == nil {
+		t.Fatal("received nil config selector")
+	}
+
+	testCases := []struct {
+		method         string
+		timeoutSupport bool
+		want           *time.Duration
+	}{{
+		method:         "/foo/method",
+		timeoutSupport: true,
+		want:           func() *time.Duration { x := 5 * time.Second; return &x }(),
+	}, {
+		method:         "/foo/method",
+		timeoutSupport: false,
+		want:           nil,
+	}, {
+		method:         "/bar/method",
+		timeoutSupport: true,
+		want:           nil,
+	}, {
+		method:         "/baz/method",
+		timeoutSupport: true,
+		want:           nil,
+	}}
+
+	for _, tc := range testCases {
+		env.TimeoutSupport = tc.timeoutSupport
+		req := iresolver.RPCInfo{
+			Method:  tc.method,
+			Context: context.Background(),
+		}
+		res, err := cs.SelectConfig(req)
+		if err != nil {
+			t.Errorf("Unexpected error from cs.SelectConfig(%v): %v", req, err)
+			continue
+		}
+		res.OnCommitted()
+		got := res.MethodConfig.Timeout
+		if !reflect.DeepEqual(got, tc.want) {
+			t.Errorf("For method %q: res.MethodConfig.Timeout = %v; want %v", tc.method, got, tc.want)
+		}
+	}
+}
+
 // TestXDSResolverDelayedOnCommitted tests that clusters remain in service
 // config if RPCs are in flight.
 func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
diff --git a/xds/internal/testutils/wrr.go b/xds/internal/testutils/wrr.go
index a4df5fc6050e..6c9486329d41 100644
--- a/xds/internal/testutils/wrr.go
+++ b/xds/internal/testutils/wrr.go
@@ -19,6 +19,7 @@
 package testutils
 
 import (
+	"fmt"
 	"sync"
 
 	"google.golang.org/grpc/internal/wrr"
@@ -66,3 +67,7 @@ func (twrr *testWRR) Next() interface{} {
 	twrr.mu.Unlock()
 	return iww.item
 }
+
+func (twrr *testWRR) String() string {
+	return fmt.Sprint(twrr.itemsWithWeight)
+}

From 666aea1fb34c9ceeab997f7a9c37c2cb51deb6e5 Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Fri, 18 Dec 2020 11:05:59 -0800
Subject: [PATCH 328/481] advancedtls: add IPv6 address to certificate SAN
 names (#4101)

* advancedtls: add IPv6 address to certificate SAN names
---
 .../advancedtls_integration_test.go           | 113 +++++++++--
 security/advancedtls/advancedtls_test.go      |  45 ----
 .../testdata/localhost-openssl.cnf            |   1 -
 .../testdata/server_cert_localhost_1.pem      | 192 +++++++++---------
 .../testdata/server_key_localhost_1.pem       |  98 ++++-----
 5 files changed, 244 insertions(+), 205 deletions(-)

diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 34f80532d4aa..4bb9e645b0a1 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -39,10 +39,6 @@ import (
 	"google.golang.org/grpc/security/advancedtls/testdata"
 )
 
-var (
-	address = "localhost:50051"
-)
-
 const (
 	// Default timeout for normal connections.
 	defaultTestTimeout = 5 * time.Second
@@ -105,7 +101,7 @@ func callAndVerify(msg string, client pb.GreeterClient, shouldFail bool) error {
 
 // TODO(ZhenLian): remove shouldFail and add ...DialOption to the function
 // signature to provider cleaner tests.
-func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) {
+func callAndVerifyWithClientConn(connCtx context.Context, address string, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) {
 	var conn *grpc.ClientConn
 	var err error
 	// If we want the test to fail, we establish a non-blocking connection to
@@ -362,11 +358,12 @@ func (s) TestEnd2End(t *testing.T) {
 			}
 			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
 			defer s.Stop()
-			lis, err := net.Listen("tcp", address)
+			lis, err := net.Listen("tcp", "localhost:0")
 			if err != nil {
 				t.Fatalf("failed to listen: %v", err)
 			}
 			defer lis.Close()
+			addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port)
 			pb.RegisterGreeterServer(s, greeterServer{})
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{
@@ -389,7 +386,7 @@ func (s) TestEnd2End(t *testing.T) {
 			// stage = 0, initial connection should succeed
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
-			conn, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 1", clientTLSCreds, false)
+			conn, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -406,7 +403,7 @@ func (s) TestEnd2End(t *testing.T) {
 			// stage = 1, new connection should fail
 			shortCtx, shortCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 			defer shortCancel()
-			conn2, greetClient, err := callAndVerifyWithClientConn(shortCtx, "rpc call 3", clientTLSCreds, true)
+			conn2, greetClient, err := callAndVerifyWithClientConn(shortCtx, addr, "rpc call 3", clientTLSCreds, true)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -415,7 +412,7 @@ func (s) TestEnd2End(t *testing.T) {
 			stage.increase()
 			// ------------------------Scenario 4------------------------------------
 			// stage = 2,  new connection should succeed
-			conn3, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 4", clientTLSCreds, false)
+			conn3, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 4", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -651,11 +648,12 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 			}
 			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
 			defer s.Stop()
-			lis, err := net.Listen("tcp", address)
+			lis, err := net.Listen("tcp", "localhost:0")
 			if err != nil {
 				t.Fatalf("failed to listen: %v", err)
 			}
 			defer lis.Close()
+			addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port)
 			pb.RegisterGreeterServer(s, greeterServer{})
 			go s.Serve(lis)
 			clientOptions := &ClientOptions{
@@ -678,7 +676,7 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 			// At initialization, the connection should be good.
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
-			conn, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 1", clientTLSCreds, false)
+			conn, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -694,7 +692,7 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 			}
 			// New connections should still be good, because the Provider didn't pick
 			// up the changes due to key-cert mismatch.
-			conn2, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 3", clientTLSCreds, false)
+			conn2, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 3", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -708,7 +706,7 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 			// other side.
 			shortCtx, shortCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 			defer shortCancel()
-			conn3, greetClient, err := callAndVerifyWithClientConn(shortCtx, "rpc call 4", clientTLSCreds, true)
+			conn3, greetClient, err := callAndVerifyWithClientConn(shortCtx, addr, "rpc call 4", clientTLSCreds, true)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -719,7 +717,7 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 			time.Sleep(sleepInterval)
 			// New connections should be good, because the other side is using
 			// *_trust_cert_2.pem now.
-			conn4, greetClient, err := callAndVerifyWithClientConn(ctx, "rpc call 5", clientTLSCreds, false)
+			conn4, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 5", clientTLSCreds, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -727,3 +725,90 @@ func (s) TestPEMFileProviderEnd2End(t *testing.T) {
 		})
 	}
 }
+
+func (s) TestDefaultHostNameCheck(t *testing.T) {
+	cs := &testutils.CertStore{}
+	if err := cs.LoadCerts(); err != nil {
+		t.Fatalf("cs.LoadCerts() failed, err: %v", err)
+	}
+	for _, test := range []struct {
+		desc             string
+		clientRoot       *x509.CertPool
+		clientVerifyFunc CustomVerificationFunc
+		clientVType      VerificationType
+		serverCert       []tls.Certificate
+		serverVType      VerificationType
+		expectError      bool
+	}{
+		// Client side sets vType to CertAndHostVerification, and will do
+		// default hostname check. Server uses a cert without "localhost" or
+		// "127.0.0.1" as common name or SAN names, and will hence fail.
+		{
+			desc:        "Bad default hostname check",
+			clientRoot:  cs.ClientTrust1,
+			clientVType: CertAndHostVerification,
+			serverCert:  []tls.Certificate{cs.ServerCert1},
+			serverVType: CertAndHostVerification,
+			expectError: true,
+		},
+		// Client side sets vType to CertAndHostVerification, and will do
+		// default hostname check. Server uses a certificate with "localhost" as
+		// common name, and will hence pass the default hostname check.
+		{
+			desc:        "Good default hostname check",
+			clientRoot:  cs.ClientTrust1,
+			clientVType: CertAndHostVerification,
+			serverCert:  []tls.Certificate{cs.ServerPeerLocalhost1},
+			serverVType: CertAndHostVerification,
+			expectError: false,
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			// Start a server using ServerOptions in another goroutine.
+			serverOptions := &ServerOptions{
+				IdentityOptions: IdentityCertificateOptions{
+					Certificates: test.serverCert,
+				},
+				RequireClientCert: false,
+				VType:             test.serverVType,
+			}
+			serverTLSCreds, err := NewServerCreds(serverOptions)
+			if err != nil {
+				t.Fatalf("failed to create server creds: %v", err)
+			}
+			s := grpc.NewServer(grpc.Creds(serverTLSCreds))
+			defer s.Stop()
+			lis, err := net.Listen("tcp", "localhost:0")
+			if err != nil {
+				t.Fatalf("failed to listen: %v", err)
+			}
+			defer lis.Close()
+			addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port)
+			pb.RegisterGreeterServer(s, greeterServer{})
+			go s.Serve(lis)
+			clientOptions := &ClientOptions{
+				VerifyPeer: test.clientVerifyFunc,
+				RootOptions: RootCertificateOptions{
+					RootCACerts: test.clientRoot,
+				},
+				VType: test.clientVType,
+			}
+			clientTLSCreds, err := NewClientCreds(clientOptions)
+			if err != nil {
+				t.Fatalf("clientTLSCreds failed to create")
+			}
+			shouldFail := false
+			if test.expectError {
+				shouldFail = true
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			conn, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, shouldFail)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer conn.Close()
+		})
+	}
+}
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 4cb4748c669d..64da81a1700c 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -373,51 +373,6 @@ func (s) TestClientServerHandshake(t *testing.T) {
 			serverCert:       []tls.Certificate{cs.ServerCert1},
 			serverVType:      CertAndHostVerification,
 		},
-		// Client: only set clientRoot
-		// Server: only set serverCert with mutual TLS off
-		// Expected Behavior: server side failure and client handshake failure
-		// Reason: client side sets vType to CertAndHostVerification, and will do
-		// default hostname check. Server uses a cert without "localhost" or
-		// "127.0.0.1" as common name or SAN names, and will hence fail.
-		{
-			desc:                       "Client has root cert; server sends peer cert",
-			clientRoot:                 cs.ClientTrust1,
-			clientVType:                CertAndHostVerification,
-			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{cs.ServerCert1},
-			serverVType:                CertAndHostVerification,
-			serverExpectError:          true,
-		},
-		// Client: only set clientGetRoot
-		// Server: only set serverCert with mutual TLS off
-		// Expected Behavior: server side failure and client handshake failure
-		// Reason: client side sets vType to CertAndHostVerification, and will do
-		// default hostname check. Server uses a cert without "localhost" or
-		// "127.0.0.1" as common name or SAN names, and will hence fail.
-		{
-			desc:                       "Client sets reload root function; server sends peer cert",
-			clientGetRoot:              getRootCAsForClient,
-			clientVType:                CertAndHostVerification,
-			clientExpectHandshakeError: true,
-			serverCert:                 []tls.Certificate{cs.ServerCert1},
-			serverVType:                CertAndHostVerification,
-			serverExpectError:          true,
-		},
-		// Client: only set clientGetRoot and CertAndHostVerification
-		// Server: only set serverCert with mutual TLS off
-		// Expected Behavior: success
-		// Reason: client side sets vType to CertAndHostVerification, and will do
-		// default hostname check. Server uses a certificate with "localhost" as
-		// common name, and will hence pass the default hostname check.
-		{
-			desc:                       "Client sets CertAndHostVerification; server uses localhost certs",
-			clientRoot:                 cs.ClientTrust1,
-			clientVType:                CertAndHostVerification,
-			clientExpectHandshakeError: false,
-			serverCert:                 []tls.Certificate{cs.ServerPeerLocalhost1},
-			serverVType:                CertAndHostVerification,
-			serverExpectError:          false,
-		},
 		// Client: set clientGetRoot and clientVerifyFunc
 		// Server: only set serverCert with mutual TLS off
 		// Expected Behavior: success
diff --git a/security/advancedtls/testdata/localhost-openssl.cnf b/security/advancedtls/testdata/localhost-openssl.cnf
index b415cef6ae1c..5f46206d2889 100644
--- a/security/advancedtls/testdata/localhost-openssl.cnf
+++ b/security/advancedtls/testdata/localhost-openssl.cnf
@@ -21,4 +21,3 @@ subjectAltName = @alt_names
 
 [alt_names]
 DNS.1 = localhost
-IP.1 = "127.0.0.1"
diff --git a/security/advancedtls/testdata/server_cert_localhost_1.pem b/security/advancedtls/testdata/server_cert_localhost_1.pem
index 798656198f1d..b6364f23ac9b 100644
--- a/security/advancedtls/testdata/server_cert_localhost_1.pem
+++ b/security/advancedtls/testdata/server_cert_localhost_1.pem
@@ -1,56 +1,56 @@
 Certificate:
     Data:
         Version: 3 (0x2)
-        Serial Number: 3 (0x3)
+        Serial Number: 5 (0x5)
         Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=US, ST=CA, L=SVL, O=Internet Widgits Pty Ltd
         Validity
-            Not Before: Dec  2 21:51:53 2020 GMT
-            Not After : Apr 19 21:51:53 2048 GMT
+            Not Before: Dec 15 18:05:59 2020 GMT
+            Not After : May  2 18:05:59 2048 GMT
         Subject: C=US, ST=Illinois, L=Chicago, O=Example, Co., CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
                 RSA Public-Key: (4096 bit)
                 Modulus:
-                    00:a0:a1:5e:05:53:c5:a7:71:fb:ff:22:d5:d2:49:
-                    da:14:8b:54:9b:fb:40:3d:33:38:7d:3e:69:16:1c:
-                    1f:c1:88:42:99:d2:ac:8e:3e:60:ca:ac:63:e2:5b:
-                    e8:90:e5:4d:1b:e5:0b:40:6d:7d:90:6a:de:7f:ad:
-                    97:81:ae:38:2c:07:44:ca:ac:54:d7:dd:41:99:36:
-                    ca:38:44:9e:cc:e0:34:c2:44:71:2e:59:60:bf:45:
-                    e5:57:af:7d:d2:ca:c5:53:ff:8a:ab:5f:68:cf:74:
-                    88:29:99:59:fc:25:8c:06:47:26:77:14:1e:10:c1:
-                    78:d6:a1:a5:7f:0d:8d:4a:be:c8:fd:6f:a1:60:3d:
-                    e8:0a:27:f9:fe:e1:45:81:7c:e6:be:65:b8:62:ed:
-                    ca:1f:fb:7f:08:c0:b4:a6:57:f1:56:32:53:bf:d6:
-                    43:75:57:1c:e2:69:8d:96:41:a6:46:a0:49:1d:e8:
-                    ab:0a:92:7b:b8:33:b2:c3:7b:6e:2b:53:13:be:91:
-                    34:71:e3:d9:52:e4:d5:63:64:f8:a7:b0:a0:23:5c:
-                    07:3d:82:9a:6e:bc:0a:95:f1:51:ff:08:75:1c:32:
-                    38:67:9a:98:5c:94:08:90:c2:23:cd:bb:ad:86:10:
-                    4f:34:b2:28:f8:83:0f:c3:cb:3a:80:a9:2e:9a:3e:
-                    45:bf:2c:55:ee:22:d7:e4:a3:d7:b4:0b:7c:fb:30:
-                    fa:a7:02:24:04:ba:60:79:2d:1d:01:6c:b1:17:b9:
-                    b9:f3:4f:c7:20:26:6d:48:31:b2:6d:27:b8:5d:fc:
-                    8a:20:be:29:30:26:18:a4:51:45:3c:47:06:22:fa:
-                    9e:c7:43:ac:34:5b:6e:23:08:c9:ea:93:b7:de:5e:
-                    36:9a:85:51:8a:01:86:c0:47:52:4f:9b:d6:16:ce:
-                    ff:3a:5f:33:ff:f5:24:20:5d:ef:f9:ea:21:fd:05:
-                    04:b9:c6:b5:75:67:67:d5:48:70:d0:e8:53:f2:ff:
-                    f7:a0:8b:1c:09:bf:5d:08:b2:5d:08:34:00:a3:f1:
-                    fc:6d:44:e5:73:eb:6d:24:a2:0f:aa:4a:d1:8f:fe:
-                    40:a4:77:94:63:99:82:26:d0:66:a4:a2:97:59:2b:
-                    30:18:4b:d1:22:a9:99:02:4c:06:50:ff:12:6e:83:
-                    11:ff:08:7f:c2:8e:ae:83:0f:a9:b6:99:3e:cc:8e:
-                    c7:a6:17:65:65:0e:9f:7d:48:28:6d:e7:b6:e5:7a:
-                    b1:36:04:fe:db:0e:62:3a:e2:a1:bf:b9:76:23:bf:
-                    b8:4f:a4:68:bf:03:1c:53:34:55:d0:28:2d:89:10:
-                    28:2d:2c:ca:8b:b9:5a:82:f4:5f:b9:eb:ef:aa:f2:
-                    31:f0:d5
+                    00:d3:4d:01:08:9d:80:de:15:fa:0a:52:ed:2c:f6:
+                    4b:9d:c5:67:2d:b6:41:33:d7:5b:b4:fd:f0:a8:5c:
+                    dd:8a:8d:1f:2d:12:5d:47:88:09:d4:96:ee:63:10:
+                    f9:9d:9a:6c:34:c4:ec:a9:0d:95:6c:48:bc:17:d0:
+                    77:53:93:f8:44:8a:0b:0b:a2:4d:1d:53:f7:55:a7:
+                    ed:6a:35:ad:1d:af:79:bd:d5:c6:5b:96:24:9e:4d:
+                    d8:e9:21:93:e1:93:3b:5d:c4:e3:90:1a:36:d0:f7:
+                    bb:8c:22:e9:d2:f8:29:19:bf:24:c6:52:21:0e:d6:
+                    3a:73:48:ba:e7:81:34:ba:23:21:57:c3:51:0e:59:
+                    51:9f:a6:07:56:17:c4:aa:1b:7b:6c:36:6b:ab:32:
+                    5e:2e:f7:70:dc:eb:f3:da:3b:39:4e:a4:8c:bf:40:
+                    72:ef:00:1e:46:c9:07:6a:93:4c:36:b7:c3:2e:7c:
+                    c5:c1:85:9f:6b:4d:2d:fd:3c:9b:9a:cf:52:b2:fa:
+                    ba:f8:ce:cc:8b:dc:57:7b:ed:37:69:c9:80:dc:a6:
+                    2c:a7:e4:4b:8c:77:cb:2e:28:65:64:61:a4:c8:33:
+                    d9:f8:7d:b7:7c:4f:b7:f4:07:5a:89:ae:2a:59:8c:
+                    ac:00:c7:ce:b0:d0:9b:cd:4d:83:39:55:bb:72:0f:
+                    62:d4:5f:8b:c3:c7:e2:79:95:53:eb:a0:26:0d:52:
+                    7b:4d:40:56:66:2b:55:67:f5:1a:c9:e8:a8:49:bd:
+                    e7:e4:31:9a:e1:8d:80:f2:cc:ab:3d:70:f6:fe:75:
+                    cb:aa:1b:12:d0:6f:d3:c8:df:f1:bd:ce:2b:21:42:
+                    e5:2c:bd:c6:c8:c4:bb:4a:3d:92:48:0d:49:a8:96:
+                    c0:51:ed:30:64:81:dd:ce:05:d4:ff:07:87:59:0b:
+                    13:41:8e:1d:58:e4:47:0c:00:97:12:e4:67:94:24:
+                    67:ef:ed:1e:85:89:df:85:78:10:1f:7f:b2:e8:af:
+                    e7:0f:c7:ec:aa:01:67:b6:0a:9a:23:83:90:1d:0a:
+                    2a:37:80:c3:26:d7:f1:24:29:b3:d8:37:7c:5c:f8:
+                    e3:08:96:1a:34:2d:fa:ff:75:e8:70:25:e2:ab:51:
+                    9d:f7:8a:23:52:89:02:c8:71:ea:cd:a2:89:b6:eb:
+                    6e:c9:fd:fb:dc:63:e8:f9:db:d7:57:d2:0c:fc:56:
+                    9c:16:a8:67:fe:17:88:07:e8:f8:c5:7a:33:72:ad:
+                    b4:5a:43:5b:49:be:79:8b:00:18:83:f7:19:00:07:
+                    94:2c:38:96:61:a7:55:a8:30:db:7e:07:f6:c5:a3:
+                    65:1e:93:2d:6b:5e:d9:a6:0f:fa:c2:19:9b:59:4c:
+                    41:ba:07
                 Exponent: 65537 (0x10001)
         X509v3 extensions:
             X509v3 Subject Key Identifier: 
-                B2:87:7F:58:33:AD:24:C6:F2:07:5D:8E:88:18:BE:61:08:20:29:D1
+                F3:DC:6A:5B:B7:CE:E9:E1:4D:3E:C4:AE:B7:8E:39:E3:6D:CA:AF:C7
             X509v3 Authority Key Identifier: 
                 keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06
 
@@ -59,66 +59,66 @@ Certificate:
             X509v3 Key Usage: 
                 Digital Signature, Key Encipherment
             X509v3 Subject Alternative Name: 
-                DNS:localhost, IP Address:127.0.0.1
+                DNS:localhost
     Signature Algorithm: sha256WithRSAEncryption
-         bd:79:29:a5:d1:35:3d:34:61:3e:88:d7:37:66:c0:48:2e:ba:
-         8f:ed:56:b6:48:3f:1b:7d:eb:b7:91:0f:13:c3:96:d3:bd:82:
-         0c:00:49:3f:1a:44:12:4c:27:f5:48:39:04:bd:66:d8:64:e1:
-         5f:c7:01:43:82:38:1e:7f:94:6c:96:00:97:06:b7:d6:d1:0d:
-         0a:34:a9:f1:3d:ac:0b:4f:0c:2e:ff:69:3d:53:63:9b:c1:22:
-         e8:5a:b9:cd:00:92:a0:b6:cd:c0:21:b2:fc:2e:70:8e:2d:e3:
-         ed:4a:f0:84:24:3f:7e:b9:a7:cf:05:1d:15:cc:a3:8a:6e:0d:
-         44:28:02:bf:03:31:02:00:eb:1d:6c:60:8e:9c:73:0a:b6:f7:
-         ad:61:c4:a5:60:dc:4e:44:79:3e:95:40:44:5a:9b:58:61:3d:
-         b6:5f:68:8f:c0:98:3d:a4:b1:41:fb:b4:ad:d3:b6:8b:72:f0:
-         c2:ec:8a:65:ac:a7:2e:ea:f7:e5:2c:8d:3a:18:36:d2:48:36:
-         5d:a2:1a:52:22:40:a6:da:7d:52:e5:ba:d7:fd:75:63:8b:9b:
-         ff:ad:20:bf:a0:f9:d4:e7:f1:b2:cb:a7:09:c7:d0:36:bc:ea:
-         be:f1:29:08:22:25:2a:16:3f:22:3b:2b:53:47:73:01:68:d2:
-         1e:3d:87:07:6f:ec:c4:33:3e:44:ec:a3:de:32:7e:12:57:15:
-         30:d4:6d:be:16:e7:ba:28:0e:8a:79:8a:ab:84:f8:1d:8c:d9:
-         da:06:1b:7e:37:10:3f:24:7c:1f:74:bc:0e:44:cf:a6:56:97:
-         02:19:b7:a7:f3:5d:46:c8:56:7b:19:b3:42:17:f2:97:cf:15:
-         d6:78:8d:90:f9:88:f7:28:71:7d:8d:ff:50:bb:d1:71:bb:23:
-         da:53:63:92:7c:d4:a8:be:45:17:d4:6a:e7:e3:75:93:b5:bb:
-         fd:38:30:11:ab:ec:f5:7b:6e:f5:17:0b:cc:69:44:79:3f:95:
-         85:e8:6b:da:e7:af:b3:f6:e8:73:22:db:75:f8:40:20:fd:cb:
-         24:b6:49:b6:69:79:12:8e:73:05:75:a0:de:67:5c:29:05:11:
-         3a:b2:8d:f8:ec:2f:fa:99:5e:3a:2b:df:3b:69:1a:c3:05:6c:
-         44:3c:a1:12:ab:3c:5b:87:52:a5:44:54:6c:c8:e5:f6:50:dd:
-         d7:d4:93:15:67:31:66:58:10:c1:23:98:35:08:6b:56:34:5a:
-         71:db:67:e5:41:a8:60:77:c3:30:2a:09:78:37:83:32:37:b1:
-         c5:34:b6:54:79:fd:e0:13:a3:16:f5:5f:25:87:8e:5e:c3:93:
-         f9:1f:4c:79:51:0f:23:e6
+         54:13:3d:55:d3:4b:d8:85:f0:54:a8:33:5c:a1:9f:87:79:31:
+         34:7e:52:b9:46:ea:97:43:fd:0a:9b:ae:2c:8b:74:cd:51:d5:
+         63:b3:b0:b4:19:a1:da:73:b4:e2:47:0c:d9:33:4a:c6:aa:27:
+         41:bc:4d:15:74:42:59:eb:41:8e:28:49:f0:55:89:13:f8:f0:
+         35:1f:e9:b2:ae:48:79:e7:15:a2:aa:59:e0:fd:91:c8:7f:ba:
+         aa:a4:2e:77:2a:e4:62:fe:c2:83:51:dc:58:83:f8:7f:b2:47:
+         68:8b:1f:9d:9b:12:25:f2:0f:7d:06:a4:9a:be:a3:af:2d:27:
+         32:4a:20:fe:5f:98:d0:5d:6a:10:c9:04:49:54:26:60:20:6e:
+         38:f2:91:a4:24:e7:ed:d2:e4:aa:88:7e:9c:6d:0a:1e:30:97:
+         a9:9c:45:35:09:fc:45:6c:32:d0:db:79:04:fa:03:7c:36:e5:
+         27:72:1d:77:a1:d4:12:86:53:bf:93:4e:f7:da:7a:a1:4b:5b:
+         42:e4:6b:9b:a5:58:f2:ef:30:2e:a6:6f:f7:63:fd:16:b0:35:
+         81:99:3d:41:dc:da:34:46:ea:1f:02:cc:5e:dc:67:1c:62:68:
+         76:45:5b:ea:f3:2f:a5:0f:2e:b8:d0:df:ad:8f:3b:03:b4:36:
+         21:d4:4d:99:09:76:22:a4:61:b4:59:ca:8b:42:5d:64:2a:9a:
+         4f:7f:67:65:38:4a:c6:e7:8b:ff:36:0d:42:4b:60:03:77:99:
+         71:1f:ac:f4:83:bb:06:5c:22:fa:ed:4c:c5:37:22:22:11:85:
+         1a:ab:06:e3:14:c8:71:bf:79:1f:b4:22:64:ea:65:5e:99:c5:
+         ca:3a:1e:d2:22:96:24:df:65:b1:42:7e:72:21:6b:de:49:cd:
+         09:45:4d:3e:ff:45:22:f4:01:d5:09:5b:dd:22:cc:25:8f:b2:
+         5e:b3:35:f8:b3:6d:6e:2b:bd:98:f2:eb:5b:0e:58:31:f3:90:
+         ec:d9:41:9d:e4:4d:e7:6a:ff:32:63:d1:45:f0:fd:a6:7d:34:
+         16:e4:c8:0c:79:39:5d:46:e1:28:f1:13:49:a6:02:07:a2:ca:
+         de:ae:4a:3a:f7:0b:e4:27:49:33:c8:29:2f:cd:6f:0e:5a:be:
+         e0:5d:c0:98:7e:15:97:ea:a5:ba:84:3b:ab:8d:aa:81:d7:7f:
+         df:7c:b9:e7:11:6b:7f:82:6e:62:57:d9:ba:25:60:1d:93:49:
+         eb:48:91:88:2c:22:74:dc:50:cf:6c:44:bd:04:84:fb:97:53:
+         57:62:56:8b:b1:5c:a2:06:ef:c4:01:21:d6:73:4f:58:79:60:
+         75:09:53:7d:cb:8c:a2:86
 -----BEGIN CERTIFICATE-----
-MIIFmTCCA4GgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBLMQswCQYDVQQGEwJVUzEL
+MIIFkzCCA3ugAwIBAgIBBTANBgkqhkiG9w0BAQsFADBLMQswCQYDVQQGEwJVUzEL
 MAkGA1UECAwCQ0ExDDAKBgNVBAcMA1NWTDEhMB8GA1UECgwYSW50ZXJuZXQgV2lk
-Z2l0cyBQdHkgTHRkMB4XDTIwMTIwMjIxNTE1M1oXDTQ4MDQxOTIxNTE1M1owXTEL
+Z2l0cyBQdHkgTHRkMB4XDTIwMTIxNTE4MDU1OVoXDTQ4MDUwMjE4MDU1OVowXTEL
 MAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdv
 MRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAiIw
-DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKChXgVTxadx+/8i1dJJ2hSLVJv7
-QD0zOH0+aRYcH8GIQpnSrI4+YMqsY+Jb6JDlTRvlC0BtfZBq3n+tl4GuOCwHRMqs
-VNfdQZk2yjhEnszgNMJEcS5ZYL9F5VevfdLKxVP/iqtfaM90iCmZWfwljAZHJncU
-HhDBeNahpX8NjUq+yP1voWA96Aon+f7hRYF85r5luGLtyh/7fwjAtKZX8VYyU7/W
-Q3VXHOJpjZZBpkagSR3oqwqSe7gzssN7bitTE76RNHHj2VLk1WNk+KewoCNcBz2C
-mm68CpXxUf8IdRwyOGeamFyUCJDCI827rYYQTzSyKPiDD8PLOoCpLpo+Rb8sVe4i
-1+Sj17QLfPsw+qcCJAS6YHktHQFssRe5ufNPxyAmbUgxsm0nuF38iiC+KTAmGKRR
-RTxHBiL6nsdDrDRbbiMIyeqTt95eNpqFUYoBhsBHUk+b1hbO/zpfM//1JCBd7/nq
-If0FBLnGtXVnZ9VIcNDoU/L/96CLHAm/XQiyXQg0AKPx/G1E5XPrbSSiD6pK0Y/+
-QKR3lGOZgibQZqSil1krMBhL0SKpmQJMBlD/Em6DEf8If8KOroMPqbaZPsyOx6YX
-ZWUOn31IKG3ntuV6sTYE/tsOYjriob+5diO/uE+kaL8DHFM0VdAoLYkQKC0syou5
-WoL0X7nr76ryMfDVAgMBAAGjdjB0MB0GA1UdDgQWBBSyh39YM60kxvIHXY6IGL5h
-CCAp0TAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7QnxBjAJBgNVHRMEAjAA
-MAsGA1UdDwQEAwIFoDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI
-hvcNAQELBQADggIBAL15KaXRNT00YT6I1zdmwEguuo/tVrZIPxt967eRDxPDltO9
-ggwAST8aRBJMJ/VIOQS9Zthk4V/HAUOCOB5/lGyWAJcGt9bRDQo0qfE9rAtPDC7/
-aT1TY5vBIuhauc0AkqC2zcAhsvwucI4t4+1K8IQkP365p88FHRXMo4puDUQoAr8D
-MQIA6x1sYI6ccwq2961hxKVg3E5EeT6VQERam1hhPbZfaI/AmD2ksUH7tK3Ttoty
-8MLsimWspy7q9+UsjToYNtJINl2iGlIiQKbafVLlutf9dWOLm/+tIL+g+dTn8bLL
-pwnH0Da86r7xKQgiJSoWPyI7K1NHcwFo0h49hwdv7MQzPkTso94yfhJXFTDUbb4W
-57ooDop5iquE+B2M2doGG343ED8kfB90vA5Ez6ZWlwIZt6fzXUbIVnsZs0IX8pfP
-FdZ4jZD5iPcocX2N/1C70XG7I9pTY5J81Ki+RRfUaufjdZO1u/04MBGr7PV7bvUX
-C8xpRHk/lYXoa9rnr7P26HMi23X4QCD9yyS2SbZpeRKOcwV1oN5nXCkFETqyjfjs
-L/qZXjor3ztpGsMFbEQ8oRKrPFuHUqVEVGzI5fZQ3dfUkxVnMWZYEMEjmDUIa1Y0
-WnHbZ+VBqGB3wzAqCXg3gzI3scU0tlR5/eAToxb1XyWHjl7Dk/kfTHlRDyPm
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANNNAQidgN4V+gpS7Sz2S53FZy22
+QTPXW7T98Khc3YqNHy0SXUeICdSW7mMQ+Z2abDTE7KkNlWxIvBfQd1OT+ESKCwui
+TR1T91Wn7Wo1rR2veb3VxluWJJ5N2Okhk+GTO13E45AaNtD3u4wi6dL4KRm/JMZS
+IQ7WOnNIuueBNLojIVfDUQ5ZUZ+mB1YXxKobe2w2a6syXi73cNzr89o7OU6kjL9A
+cu8AHkbJB2qTTDa3wy58xcGFn2tNLf08m5rPUrL6uvjOzIvcV3vtN2nJgNymLKfk
+S4x3yy4oZWRhpMgz2fh9t3xPt/QHWomuKlmMrADHzrDQm81NgzlVu3IPYtRfi8PH
+4nmVU+ugJg1Se01AVmYrVWf1GsnoqEm95+QxmuGNgPLMqz1w9v51y6obEtBv08jf
+8b3OKyFC5Sy9xsjEu0o9kkgNSaiWwFHtMGSB3c4F1P8Hh1kLE0GOHVjkRwwAlxLk
+Z5QkZ+/tHoWJ34V4EB9/suiv5w/H7KoBZ7YKmiODkB0KKjeAwybX8SQps9g3fFz4
+4wiWGjQt+v916HAl4qtRnfeKI1KJAshx6s2iibbrbsn9+9xj6Pnb11fSDPxWnBao
+Z/4XiAfo+MV6M3KttFpDW0m+eYsAGIP3GQAHlCw4lmGnVagw234H9sWjZR6TLWte
+2aYP+sIZm1lMQboHAgMBAAGjcDBuMB0GA1UdDgQWBBTz3Gpbt87p4U0+xK63jjnj
+bcqvxzAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7QnxBjAJBgNVHRMEAjAA
+MAsGA1UdDwQEAwIFoDAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQEL
+BQADggIBAFQTPVXTS9iF8FSoM1yhn4d5MTR+UrlG6pdD/QqbriyLdM1R1WOzsLQZ
+odpztOJHDNkzSsaqJ0G8TRV0QlnrQY4oSfBViRP48DUf6bKuSHnnFaKqWeD9kch/
+uqqkLncq5GL+woNR3FiD+H+yR2iLH52bEiXyD30GpJq+o68tJzJKIP5fmNBdahDJ
+BElUJmAgbjjykaQk5+3S5KqIfpxtCh4wl6mcRTUJ/EVsMtDbeQT6A3w25SdyHXeh
+1BKGU7+TTvfaeqFLW0Lka5ulWPLvMC6mb/dj/RawNYGZPUHc2jRG6h8CzF7cZxxi
+aHZFW+rzL6UPLrjQ362POwO0NiHUTZkJdiKkYbRZyotCXWQqmk9/Z2U4Ssbni/82
+DUJLYAN3mXEfrPSDuwZcIvrtTMU3IiIRhRqrBuMUyHG/eR+0ImTqZV6Zxco6HtIi
+liTfZbFCfnIha95JzQlFTT7/RSL0AdUJW90izCWPsl6zNfizbW4rvZjy61sOWDHz
+kOzZQZ3kTedq/zJj0UXw/aZ9NBbkyAx5OV1G4SjxE0mmAgeiyt6uSjr3C+QnSTPI
+KS/Nbw5avuBdwJh+FZfqpbqEO6uNqoHXf998uecRa3+CbmJX2bolYB2TSetIkYgs
+InTcUM9sRL0EhPuXU1diVouxXKIG78QBIdZzT1h5YHUJU33LjKKG
 -----END CERTIFICATE-----
diff --git a/security/advancedtls/testdata/server_key_localhost_1.pem b/security/advancedtls/testdata/server_key_localhost_1.pem
index 17d9f981564b..1185eff04ba4 100644
--- a/security/advancedtls/testdata/server_key_localhost_1.pem
+++ b/security/advancedtls/testdata/server_key_localhost_1.pem
@@ -1,51 +1,51 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIJKAIBAAKCAgEAoKFeBVPFp3H7/yLV0knaFItUm/tAPTM4fT5pFhwfwYhCmdKs
-jj5gyqxj4lvokOVNG+ULQG19kGref62Xga44LAdEyqxU191BmTbKOESezOA0wkRx
-Lllgv0XlV6990srFU/+Kq19oz3SIKZlZ/CWMBkcmdxQeEMF41qGlfw2NSr7I/W+h
-YD3oCif5/uFFgXzmvmW4Yu3KH/t/CMC0plfxVjJTv9ZDdVcc4mmNlkGmRqBJHeir
-CpJ7uDOyw3tuK1MTvpE0cePZUuTVY2T4p7CgI1wHPYKabrwKlfFR/wh1HDI4Z5qY
-XJQIkMIjzbuthhBPNLIo+IMPw8s6gKkumj5FvyxV7iLX5KPXtAt8+zD6pwIkBLpg
-eS0dAWyxF7m580/HICZtSDGybSe4XfyKIL4pMCYYpFFFPEcGIvqex0OsNFtuIwjJ
-6pO33l42moVRigGGwEdST5vWFs7/Ol8z//UkIF3v+eoh/QUEuca1dWdn1Uhw0OhT
-8v/3oIscCb9dCLJdCDQAo/H8bUTlc+ttJKIPqkrRj/5ApHeUY5mCJtBmpKKXWSsw
-GEvRIqmZAkwGUP8SboMR/wh/wo6ugw+ptpk+zI7HphdlZQ6ffUgobee25XqxNgT+
-2w5iOuKhv7l2I7+4T6RovwMcUzRV0CgtiRAoLSzKi7lagvRfuevvqvIx8NUCAwEA
-AQKCAgBnlCagoMhPlTy95KSkmWK65K2Gd5mQ3TqL6Hay/yerEEaCEkua3bZkeo1e
-JY3uAS6b0jJTNUdGnOMkybdss/8cxQMi/cUn/VCTj7UOW5Fa4yiiLKgfDxtHu7aL
-uGoWRxK/e4TbxQY84BP9Xxmbckq8sZyoJJzOiTN2k324U/DMRgItCpKxELpT8jtO
-k8zSFsxj8gvYHyW7Qd1Es57JtOO2hXVjurJ9M9M4XIAkZ+jkme8MDkBc7OBCg3O+
-ghUkcsnElLWQyzAUN+Mx2KZO26InquwwSctzpGXfEmGhZr69k9SzWgjtibeMQOP9
-ggv+6v1oKYop1bmQs7fhxzZ517X47wKJMx740OcM2oXNVqHwGmRvQxX10I4iopej
-c23hj1HYjXVnX8q6md636Pe1CO33NQhoxGsIEeZYwCBRI7w2CYZoejaHMAoAAOyS
-eDmNLGyX2cO0lxY5ru996gzS2NzIcuPG+XkXdsaxMuYYp6GpjBxhs414UeJ1dLL5
-tviBmO9WI2votjMglcU2/cqv2yr4w1azIrSSu4vebuYVhMkAYcvauBc2A2egRUIF
-RP2zBl55ZefapuPqHJdHyv9JWngrl5xWEoaOTUt8crzvRKxrnkLSBJ3rcZwhnsyr
-ZjuxsuiJ7AA/E6vOcOrmGixhFIA3B9oUOwPqNJN9hB1Ld50ZAQKCAQEAy9+C+fHn
-JklMsmWMqfyVFS9sPOuRM0NS1cAQ6U5RJdreiyq3pUbghDhYxiRTHWkoLE3IEYKE
-mPoMKwhu06qIQSKl/G7QKHEY9gwDW9/9pqvDnDbH7xaCTmsgXIcMdu+YisNqlufx
-yy7Q+oLdqbqBLio+CqJcWOwcasTGPdQlR4bnAxCRguCki9MeQTEnkPOff2KIZQci
-2Lv2s2yrf+8Lw+mDB43/clMFcpjdIR4ezaf9QLkJ2mWsKDBHt7Hl+WgY0Y/r2F5G
-MTotqYEqXKsBNPqy6LprYb/bDDhYHsnRAzyfRpKZ+6nwgdy4VjKOs0jduIewznTW
-P1iTJudf0eC3UQKCAQEAybNoGLw9mzc7/UOtpXSEwQuztE7h4Pcm1+pHLVy/6X8S
-h6toLCl7NoaHF6BuFNi+8yGYo8NSAEvyIm0rXh2sKjZAAbl5ab4h7VxEgVpynBMO
-RR9FgeQXakEzzpxlXkVjKlsMOmO9WTOcUKwpgP0dKQt81ygHt9H4uxO93EpYRkQr
-AyD+VXLtCill23YiAf8cX1GGj2mxZzg4+rOJpERLDBxTlUn005g2e3zShZXiKGSO
-QuWwDomkBvdcKasSydiOLrdE4NzO7YXRgiY/AJGGC8935RgFQT7KMZQ1HRC8leTY
-opc/Joq5j4xQatBu1ewbbg0idwYcy+p5Ot/tXwYIRQKCAQBDuB2genrGW+ivBU5B
-FJZMsDDq13Cmr4EvYRn89Te9NENhxLG1o6JmKPVL87rr9QcUGE4RiuISklRCYw21
-H1sdD65E+GYKWO7qo7jl5rQxjbJvDD9DKp3kAG+CbJV2WEW6KgkY0TievhFKdPe+
-LiZEuGFdVOsJ2nvh9zTGStaLOMM5YGKFL6tYiqrtCq/S1SmwvYEC1ej8Rws+NCWP
-XE7zJ3iPpNoqFmuj0iT5oDCpLVjRC+W69rTFsKvR17TFMI+15HF5sG7uYR3TxQTW
-PTMsbu3IokuS75CKMZkLuQvFYHijj4S4dI1gBXnxn9+Iq/aCGghfu62C4yAV9xr7
-8wHRAoIBAGIpfRTkr/rVU827XUwzu9QTtN6gsU+CGRZlv0Q1anTh0gvTALzVZ1Cv
-AhoeitR8c9nx1M6GZWcdjvbwOHXybPKSOm5cbNlonixdhj2J3lNU9tHvGS3Q6xBc
-MTFxbegGTu+zJe1Y0zMRahbc4soS5VkvbQ9tPOxaNPoe7nzCddmknWZFbWH6r6AN
-a7P19zEPjihZjepH3v3EH/7q16bpUbjQJGF4f71my8Unh3FZ85oC7jVigV9h30FA
-q0rgJiGz0easbMoezFpOkRsNMAY/zIP88XW+Tfhl7ZNZdMvzdERi/oeKokJIq2xQ
-Nmb1j6tu4B6cJ9TTVbpsH5nmlyhy0B0CggEBAJPicCHPN828eVLp0kCOY3akPfLc
-M88O/3XN4rf8btM2wSJdKANcF88WYh6wmMR4RF26/GIn/LfVRz/y23hj+KGPW2Nz
-4anf4+iLqIoI0QuIHAzV6/WdukBhkjfXJlHESlNp9vsdDQymT+NookXfXVvQQxHm
-21x41hlvKe0Ldd7MWj/JNow4sLfiyGntsaCYZBL6SBBrL4D42AmXFiftLQ4Tx9wl
-YY/37p9hid5/PrF67Qk1jhQj1Z5cs2I+rsCo7FqDDnbbYDE69R/q9yTRDLBnEr93
-ce3EAO6RL5tKGQqc+m8gl/7eyJ5kriySBibaiZifJA4KFMwFCgwvxmsHJgA=
+MIIJKAIBAAKCAgEA000BCJ2A3hX6ClLtLPZLncVnLbZBM9dbtP3wqFzdio0fLRJd
+R4gJ1JbuYxD5nZpsNMTsqQ2VbEi8F9B3U5P4RIoLC6JNHVP3VaftajWtHa95vdXG
+W5Yknk3Y6SGT4ZM7XcTjkBo20Pe7jCLp0vgpGb8kxlIhDtY6c0i654E0uiMhV8NR
+DllRn6YHVhfEqht7bDZrqzJeLvdw3Ovz2js5TqSMv0By7wAeRskHapNMNrfDLnzF
+wYWfa00t/Tybms9Ssvq6+M7Mi9xXe+03acmA3KYsp+RLjHfLLihlZGGkyDPZ+H23
+fE+39Adaia4qWYysAMfOsNCbzU2DOVW7cg9i1F+Lw8fieZVT66AmDVJ7TUBWZitV
+Z/UayeioSb3n5DGa4Y2A8syrPXD2/nXLqhsS0G/TyN/xvc4rIULlLL3GyMS7Sj2S
+SA1JqJbAUe0wZIHdzgXU/weHWQsTQY4dWORHDACXEuRnlCRn7+0ehYnfhXgQH3+y
+6K/nD8fsqgFntgqaI4OQHQoqN4DDJtfxJCmz2Dd8XPjjCJYaNC36/3XocCXiq1Gd
+94ojUokCyHHqzaKJtutuyf373GPo+dvXV9IM/FacFqhn/heIB+j4xXozcq20WkNb
+Sb55iwAYg/cZAAeULDiWYadVqDDbfgf2xaNlHpMta17Zpg/6whmbWUxBugcCAwEA
+AQKCAgBswO5uQ7qnE8Kc+6+M+7tRmd+QHIUUrJxL3IO39AwmmpnYNeKCxZbhr0lE
+/eCr6GYXBuAT5qTolcsRqr8v6jHW/QHQXBm6pZPgp0y/5J6Ub9OGDHhKfU2dmM2y
+uBCIAqKEkajaa1OZXFhQOUwFxKpK0SGZXX4cR9DPszhXnR3JS/mGVUXrz7b+J5MR
+EaysLPbqbFwgQg1NuReC7YKV6POG8ZRrfz1om7P5lNBXXzbT1uMDkz6pax/xN0kb
+VM118Y1MB1aiZrXKqn7wjth9fzPu3SyQwSTNSH7v4+TDtKn+TQm8JuCAf/tbA0nr
+IRQ1AP0qbayJPuVh1qpaoTCX9SlU29QwahuXmjqZzbtXlre64Psfmx6fGpTrFVY4
+Irhd4If/VwtbHCK6hU4c8GsIori+5rgquKBQgawQgDXCcF4671RJlwQnYm9YT5cb
+q84wgBfGbXODHV1A+qJL3J+K/YzwxZKi7w9hXE+MsBoDiiUPekTBZA3GkFRCXwpN
+nnb/BpnHnf3Fvy+BcmjzlzaPJZ1mwLJJEH0/R820U59S5m2fh7Gwy9+ZZsSNsJVW
+fGaMZBGhOgAiBLd7zl7If4Eza14U499P8Rl8+4RiaMjnQC8xyI5eb49DtKcPdDp9
+d+ZQiXTQeFWriCeXTQlEV8uUCpRosfUtJC7xjljQWHeNeMpnAQKCAQEA7u13pfYd
+XH4BpO+wlRwjb4ADc6wLWFOm0nySNB3fzDrOix0vAGvm7sjtIGVYjcHAbjG2oflH
+siqgFaT5iQUdvo5AGZU9rp6f4qctJQuSuti4eXE1P+YSGyNALISE49lpi/z3lwJ7
+K3zxT0FywS2pvHEMc3+s+T0otik285/TJ7yg9d5UTS3TAgLzkan6wzGCAos935Ok
+0/pJXWk3jjmc7q9cMGPDbX2kb67j4BuqbDbiCNTdfL2BOErwZw6+lissp8TPVNf2
+03Hj24GQGO/A9H8U47M2bcY63UgcflmBwKvXVSnCVpYl9UjY/A4j45w2zjAX/lm1
+3mjTBiwQGVR1hwKCAQEA4mYt2SuyS6AkdpANUrzISS0FvLTy/Qb3iU7iBkSBqShe
+XV/a520uTJ+5v0m6ifGsafrjKJ0Rb3GYAug/mzWItTuRArIH5hTNd+FaAeC9XBEv
+dvUnAnffvocvAnmpQdO/kJLY54rA1KnuIJ30Xy5653LgQLur/b2EG7Fljc9vHqI8
+Emp7sLwPPFzelR+ryTak8GimUHm7psPJryvAeB8ak68TAmlBkLKrdO6oE3O1fJoO
+cFbZG3r2KdRPZgBpegskOOgk0GQIG7z3G0auZpiwViRJLTBda1lLLYnVBcT2Qq10
+AQebj1pDC7Op33kZMNtQtrb6bvEdPhEsaWqcfZm3gQKCAQAvgcojlq8539gl2n7q
+9yBYoESPcGsFEgT+n0RW1oXUTvEYmiHpXIsbeZokseIMtbS0dHAS/sTxuSYBh78S
+LpE+fXxjWdhc6y9xWrpQPl/bhRIRG6Bx5yY8fSLadzMRNv6UliUIwraI7BvzHVla
+7eBtFrFaGc3j9PQuXD2P7XyHzyrWGHH8sprdMIcLtJemziZCqTsRRIMmnwKNb0lb
+nzsD/pw/Bucp0yyqBEVNH1Mglz0Ucnbjwa566fOpGjZtF4KWjTyIazSp0GB1Gerz
++mAMfWRC7jRpWVwE+byoptV04PY8+cOpgctkXSq/23PpYvtGvitXKLFP2tnyxToi
+PzfrAoIBAGk9+HgosOQo2GppAliAu1YQ4MbdEst+bplcmwMw21lIE72yLm9AOLKT
+2WPLoTQ4rN5DK0+Y3B8DHhfT4KWE2DzvKLSpD7Tr3KuqjQ2sbDodHwRcZ7rlAJRw
+APFUntKj3TwWl0/jF0qEh9aPtqZ8U9O9efN9ijEU5RF+gGfQkqYZ4nTpHQCGG0sD
+HNETfOa3SSscapukSw/1mY6ddwYf51nZm6uWRE1AUSW1P1pzgl0evDGKnbgBi+bb
+8+DFtkJuZXMyrtJUfdRvHiuGytGUjvwsN/wSrIqXYrQTi3v4GEXcnb1QzQZxfhM1
+fHUOtSAaA0Y8fuQNn3tXvl5umbplN4ECggEBAIPDR2/Igmze9sCqZF06fexBZUcB
+j6gsjY7zJYppAO6tniyLkQaOHoCZGvMA2xxcR1+4F+QQvLOkRTPxXHqZgNwze9oK
+Y9QLkhsu0Q3A0+C/YC3HGL90HKndRx6pLHVhrBg3vxoTjQpPkpVT6UYnMGnRr5D5
+qluURPPYR56WX76rMud0yDTB/MeNsNjdBn8ukr1LwR40ircFgmy/O+FdR9mpPOVY
+3DIDysyzug3IeIoa0QSy7aKHHGd9pTHJ3EA/tfoTCmr9iywKxsgPp1fcSjt8Nfg7
+mVo2qbVoqchrp0tFG2/MoxqiOlRMtWX6N37btlMxifj28BDE63hG53W4bGA=
 -----END RSA PRIVATE KEY-----

From 39a500abb9b14b9d9bedb07a8323810a46690c09 Mon Sep 17 00:00:00 2001
From: Edmond <edmond@edison.tech>
Date: Wed, 23 Dec 2020 00:56:49 +0800
Subject: [PATCH 329/481] support unix-abstract schema (#4079)

---
 clientconn.go                      |  2 +-
 internal/grpcutil/target.go        | 24 ++++++++++++++++++++++--
 internal/grpcutil/target_test.go   | 12 ++++++++++++
 internal/resolver/unix/unix.go     | 23 ++++++++++++++++-------
 internal/transport/http2_client.go |  2 +-
 test/authority_test.go             | 18 +++++++++++++++---
 6 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index abfd20098c80..77a08fd33bf8 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -270,7 +270,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		cc.authority = creds.Info().ServerName
 	} else if cc.dopts.insecure && cc.dopts.authority != "" {
 		cc.authority = cc.dopts.authority
-	} else if strings.HasPrefix(cc.target, "unix:") {
+	} else if strings.HasPrefix(cc.target, "unix:") || strings.HasPrefix(cc.target, "unix-abstract:") {
 		cc.authority = "localhost"
 	} else if strings.HasPrefix(cc.parsedTarget.Endpoint, ":") {
 		cc.authority = "localhost" + cc.parsedTarget.Endpoint
diff --git a/internal/grpcutil/target.go b/internal/grpcutil/target.go
index baa3479f12ca..8833021da02e 100644
--- a/internal/grpcutil/target.go
+++ b/internal/grpcutil/target.go
@@ -41,10 +41,30 @@ func split2(s, sep string) (string, string, bool) {
 // not parse "unix:[path]" cases. This should be true in cases where a custom
 // dialer is present, to prevent a behavior change.
 //
-// If target is not a valid scheme://authority/endpoint, it returns {Endpoint:
-// target}.
+// If target is not a valid scheme://authority/endpoint as specified in
+// https://github.com/grpc/grpc/blob/master/doc/naming.md,
+// it returns {Endpoint: target}.
 func ParseTarget(target string, skipUnixColonParsing bool) (ret resolver.Target) {
 	var ok bool
+	if strings.HasPrefix(target, "unix-abstract:") {
+		if strings.HasPrefix(target, "unix-abstract://") {
+			// Maybe, with Authority specified, try to parse it
+			var remain string
+			ret.Scheme, remain, _ = split2(target, "://")
+			ret.Authority, ret.Endpoint, ok = split2(remain, "/")
+			if !ok {
+				// No Authority, add the "//" back
+				ret.Endpoint = "//" + remain
+			} else {
+				// Found Authority, add the "/" back
+				ret.Endpoint = "/" + ret.Endpoint
+			}
+		} else {
+			// Without Authority specified, split target on ":"
+			ret.Scheme, ret.Endpoint, _ = split2(target, ":")
+		}
+		return ret
+	}
 	ret.Scheme, ret.Endpoint, ok = split2(target, "://")
 	if !ok {
 		if strings.HasPrefix(target, "unix:") && !skipUnixColonParsing {
diff --git a/internal/grpcutil/target_test.go b/internal/grpcutil/target_test.go
index 6083e84d75ae..f6c586dd0808 100644
--- a/internal/grpcutil/target_test.go
+++ b/internal/grpcutil/target_test.go
@@ -84,6 +84,18 @@ func TestParseTargetString(t *testing.T) {
 		{targetStr: "unix:/a/b/c", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "/a/b/c"}, wantWithDialer: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix:/a/b/c"}},
 		{targetStr: "unix:///a/b/c", want: resolver.Target{Scheme: "unix", Authority: "", Endpoint: "/a/b/c"}},
 
+		{targetStr: "unix-abstract:a/b/c", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "a/b/c"}},
+		{targetStr: "unix-abstract:a b", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "a b"}},
+		{targetStr: "unix-abstract:a:b", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "a:b"}},
+		{targetStr: "unix-abstract:a-b", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "a-b"}},
+		{targetStr: "unix-abstract:/ a///://::!@#$%^&*()b", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "/ a///://::!@#$%^&*()b"}},
+		{targetStr: "unix-abstract:passthrough:abc", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "passthrough:abc"}},
+		{targetStr: "unix-abstract:unix:///abc", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "unix:///abc"}},
+		{targetStr: "unix-abstract:///a/b/c", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "/a/b/c"}},
+		{targetStr: "unix-abstract://authority/a/b/c", want: resolver.Target{Scheme: "unix-abstract", Authority: "authority", Endpoint: "/a/b/c"}},
+		{targetStr: "unix-abstract:///", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "/"}},
+		{targetStr: "unix-abstract://authority", want: resolver.Target{Scheme: "unix-abstract", Authority: "", Endpoint: "//authority"}},
+
 		{targetStr: "passthrough:///unix:///a/b/c", want: resolver.Target{Scheme: "passthrough", Authority: "", Endpoint: "unix:///a/b/c"}},
 	} {
 		got := ParseTarget(test.targetStr, false)
diff --git a/internal/resolver/unix/unix.go b/internal/resolver/unix/unix.go
index 8e78d47197c8..0d5a811ddfad 100644
--- a/internal/resolver/unix/unix.go
+++ b/internal/resolver/unix/unix.go
@@ -26,20 +26,28 @@ import (
 	"google.golang.org/grpc/resolver"
 )
 
-const scheme = "unix"
+const unixScheme = "unix"
+const unixAbstractScheme = "unix-abstract"
 
-type builder struct{}
+type builder struct {
+	scheme string
+}
 
-func (*builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
+func (b *builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
 	if target.Authority != "" {
 		return nil, fmt.Errorf("invalid (non-empty) authority: %v", target.Authority)
 	}
-	cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(resolver.Address{Addr: target.Endpoint}, "unix")}})
+	addr := resolver.Address{Addr: target.Endpoint}
+	if b.scheme == unixAbstractScheme {
+		// prepend "\x00" to address for unix-abstract
+		addr.Addr = "\x00" + addr.Addr
+	}
+	cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(addr, "unix")}})
 	return &nopResolver{}, nil
 }
 
-func (*builder) Scheme() string {
-	return scheme
+func (b *builder) Scheme() string {
+	return b.scheme
 }
 
 type nopResolver struct {
@@ -50,5 +58,6 @@ func (*nopResolver) ResolveNow(resolver.ResolveNowOptions) {}
 func (*nopResolver) Close() {}
 
 func init() {
-	resolver.Register(&builder{})
+	resolver.Register(&builder{scheme: unixScheme})
+	resolver.Register(&builder{scheme: unixAbstractScheme})
 }
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 4c9f9740ec71..8902b7f90d9d 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -143,7 +143,7 @@ func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error
 	address := addr.Addr
 	networkType, ok := networktype.Get(addr)
 	if fn != nil {
-		if networkType == "unix" {
+		if networkType == "unix" && !strings.HasPrefix(address, "\x00") {
 			// For backward compatibility, if the user dialed "unix:///path",
 			// the passthrough resolver would be used and the user's custom
 			// dialer would see "unix:///path". Since the unix resolver is used
diff --git a/test/authority_test.go b/test/authority_test.go
index f537ee450a42..2839f01e228e 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -54,8 +55,10 @@ func authorityChecker(ctx context.Context, expectedAuthority string) (*testpb.Em
 }
 
 func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer func(context.Context, string) (net.Conn, error)) {
-	if err := os.RemoveAll(address); err != nil {
-		t.Fatalf("Error removing socket file %v: %v\n", address, err)
+	if !strings.HasPrefix(target, "unix-abstract:") {
+		if err := os.RemoveAll(address); err != nil {
+			t.Fatalf("Error removing socket file %v: %v\n", address, err)
+		}
 	}
 	ss := &stubserver.StubServer{
 		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
@@ -115,6 +118,13 @@ var authorityTests = []authorityTest{
 		authority:      "unix:///tmp/sock.sock",
 		dialTargetWant: "unix:///tmp/sock.sock",
 	},
+	{
+		name:           "UnixAbstract",
+		address:        "\x00abc efg",
+		target:         "unix-abstract:abc efg",
+		authority:      "localhost",
+		dialTargetWant: "\x00abc efg",
+	},
 }
 
 // TestUnix does end to end tests with the various supported unix target
@@ -140,7 +150,9 @@ func (s) TestUnixCustomDialer(t *testing.T) {
 				if address != test.dialTargetWant {
 					return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.dialTargetWant, address)
 				}
-				address = address[len("unix:"):]
+				if !strings.HasPrefix(test.target, "unix-abstract:") {
+					address = address[len("unix:"):]
+				}
 				return (&net.Dialer{}).DialContext(ctx, "unix", address)
 			}
 			runUnixTest(t, test.address, test.target, test.authority, dialer)

From 66c1393796be6e46f820359f798c8a24c5b9a535 Mon Sep 17 00:00:00 2001
From: John Howard <howardjohn@google.com>
Date: Tue, 22 Dec 2020 13:01:38 -0800
Subject: [PATCH 330/481] github: improve security policy documentation (#4130)

---
 SECURITY.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 SECURITY.md

diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 000000000000..be6e108705c4
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,3 @@
+# Security Policy
+
+For information on gRPC Security Policy and reporting potentional security issues, please see [gRPC CVE Process](https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md).

From 53788aa5dcb46078eb29b05869a7472a5cd886e8 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Sat, 26 Dec 2020 10:11:54 -0800
Subject: [PATCH 331/481] github: update bug template to link to the CVE
 process (#4131)

---
 .github/ISSUE_TEMPLATE/bug.md | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md
index 1c4663918c18..5384f2d93114 100644
--- a/.github/ISSUE_TEMPLATE/bug.md
+++ b/.github/ISSUE_TEMPLATE/bug.md
@@ -1,12 +1,17 @@
 ---
 name: Bug Report
-about: Create a report to help us improve
+about: Report a non-security bug.  For suspected security vulnerabilities or crashes, please use "Report a Security Vulnerability", below.
 labels: 'Type: Bug'
 
 ---
 
-Please see the FAQ in our main README.md, then answer the questions below before
-submitting your issue.
+NOTE: if you are reporting is a potential security vulnerability or a crash,
+please follow our CVE process at
+https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md instead of
+filing an issue here.
+
+Please see the FAQ in our main README.md, then answer the questions below
+before submitting your issue.
 
 ### What version of gRPC are you using?
 

From f4a20d2f414ffe9ca43416e6980831bde3140404 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 5 Jan 2021 13:53:57 -0800
Subject: [PATCH 332/481] xds: NACK more invalid RDS responses (#4120)

---
 xds/internal/client/client_rds_test.go | 64 ++++++++++++++++++++++++++
 xds/internal/client/client_xds.go      | 12 +++--
 2 files changed, 72 insertions(+), 4 deletions(-)

diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index ab4737376e54..481030fee211 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -707,6 +707,70 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			}},
 			wantErr: false,
 		},
+		{
+			name: "unrecognized path specifier",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_ConnectMatcher_{},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "unrecognized header match specifier",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"},
+						Headers: []*v3routepb.HeaderMatcher{
+							{
+								Name:                 "th",
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_HiddenEnvoyDeprecatedRegexMatch{},
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "no cluster in weighted clusters action",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &v3routepb.WeightedCluster{}}}},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "all 0-weight clusters in weighted clusters action",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &v3routepb.WeightedCluster{
+									Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 0}},
+										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 0}},
+									},
+									TotalWeight: &wrapperspb.UInt32Value{Value: 0},
+								}}}},
+				},
+			},
+			wantErr: true,
+		},
 	}
 
 	cmpOpts := []cmp.Option{
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index f31b6009b6e1..68f65c082412 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -272,8 +272,7 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 		case *v3routepb.RouteMatch_SafeRegex:
 			route.Regex = &pt.SafeRegex.Regex
 		default:
-			logger.Warningf("route %+v has an unrecognized path specifier: %+v", r, pt)
-			continue
+			return nil, fmt.Errorf("route %+v has an unrecognized path specifier: %+v", r, pt)
 		}
 
 		if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil {
@@ -299,8 +298,7 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			case *v3routepb.HeaderMatcher_SuffixMatch:
 				header.SuffixMatch = &ht.SuffixMatch
 			default:
-				logger.Warningf("route %+v has an unrecognized header matcher: %+v", r, ht)
-				continue
+				return nil, fmt.Errorf("route %+v has an unrecognized header matcher: %+v", r, ht)
 			}
 			header.Name = h.GetName()
 			invert := h.GetInvertMatch()
@@ -331,12 +329,18 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			var totalWeight uint32
 			for _, c := range wcs.Clusters {
 				w := c.GetWeight().GetValue()
+				if w == 0 {
+					continue
+				}
 				clusters[c.GetName()] = w
 				totalWeight += w
 			}
 			if totalWeight != wcs.GetTotalWeight().GetValue() {
 				return nil, fmt.Errorf("route %+v, action %+v, weights of clusters do not add up to total total weight, got: %v, want %v", r, a, wcs.GetTotalWeight().GetValue(), totalWeight)
 			}
+			if totalWeight == 0 {
+				return nil, fmt.Errorf("route %+v, action %+v, has no valid cluster in WeightedCluster action", r, a)
+			}
 		case *v3routepb.RouteAction_ClusterHeader:
 			continue
 		}

From 829919d5722373b6ad9bc3fa539b73d5dbb91257 Mon Sep 17 00:00:00 2001
From: Pradeep Mamillapalli <pradeep.mamillapalli@equifax.com>
Date: Thu, 7 Jan 2021 13:06:14 -0500
Subject: [PATCH 333/481] xds client: Updated v3 type for http connection
 manager (#4137)

---
 xds/internal/client/client_lds_test.go | 8 +++-----
 xds/internal/version/version.go        | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
index 6c0ec32106ea..09dd2eea4d84 100644
--- a/xds/internal/client/client_lds_test.go
+++ b/xds/internal/client/client_lds_test.go
@@ -31,6 +31,7 @@ import (
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
@@ -87,14 +88,11 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 						},
 					},
 				}
-				mcm, _ := proto.Marshal(cm)
+				mcm, _ := ptypes.MarshalAny(cm)
 				lis := &v3listenerpb.Listener{
 					Name: v3LDSTarget,
 					ApiListener: &v3listenerpb.ApiListener{
-						ApiListener: &anypb.Any{
-							TypeUrl: version.V3HTTPConnManagerURL,
-							Value:   mcm,
-						},
+						ApiListener: mcm,
 					},
 				}
 				mLis, _ := proto.Marshal(lis)
diff --git a/xds/internal/version/version.go b/xds/internal/version/version.go
index db1929b76b96..dbcb76ffd1f1 100644
--- a/xds/internal/version/version.go
+++ b/xds/internal/version/version.go
@@ -45,7 +45,7 @@ const (
 	V3RouteConfigURL          = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"
 	V3ClusterURL              = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
 	V3EndpointsURL            = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"
-	V3HTTPConnManagerURL      = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v3.HttpConnectionManager"
+	V3HTTPConnManagerURL      = "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
 	V3UpstreamTLSContextURL   = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext"
 	V3DownstreamTLSContextURL = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext"
 )

From 4f80d77fe446b820dfe8dadd1eb8094f950a0128 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 7 Jan 2021 13:10:44 -0800
Subject: [PATCH 334/481] github: enable CodeQL checker (#4134)

---
 .github/workflows/codeql-analysis.yml | 31 +++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 .github/workflows/codeql-analysis.yml

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 000000000000..0c3806bdc23d
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,31 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '24 20 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: go
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

From ecc9a99b668778a2702fce85b95f9a278f6ace61 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 7 Jan 2021 14:19:11 -0800
Subject: [PATCH 335/481] interop: remove test.proto clones/variants and use
 grpc-proto repo instead (#4129)

---
 benchmark/benchmain/main.go                   |   10 +-
 benchmark/benchmark.go                        |   32 +-
 benchmark/client/main.go                      |    2 +-
 benchmark/grpc_testing/control.pb.go          | 1994 -------------
 benchmark/grpc_testing/control.proto          |  188 --
 benchmark/grpc_testing/messages.pb.go         | 1160 --------
 benchmark/grpc_testing/messages.proto         |  159 --
 benchmark/grpc_testing/payloads.proto         |   42 -
 benchmark/grpc_testing/services.pb.go         |  152 -
 benchmark/grpc_testing/services.proto         |   61 -
 benchmark/grpc_testing/stats.pb.go            |  473 ----
 benchmark/grpc_testing/stats.proto            |   57 -
 benchmark/worker/benchmark_client.go          |    2 +-
 benchmark/worker/benchmark_server.go          |    2 +-
 benchmark/worker/main.go                      |    2 +-
 binarylog/binarylog_end2end_test.go           |  101 +-
 interop/grpc_testing/benchmark_service.pb.go  |  127 +
 .../grpc_testing/benchmark_service_grpc.pb.go |  392 +++
 interop/grpc_testing/control.pb.go            | 2461 +++++++++++++++++
 interop/grpc_testing/core/stats.pb.go         |  412 +++
 interop/grpc_testing/empty.pb.go              |  158 ++
 interop/grpc_testing/messages.pb.go           | 2035 ++++++++++++++
 .../grpc_testing/payloads.pb.go               |  148 +-
 .../report_qps_scenario_service.pb.go         |   99 +
 .../report_qps_scenario_service_grpc.pb.go    |  103 +
 interop/grpc_testing/stats.pb.go              |  632 +++++
 interop/grpc_testing/test.pb.go               | 2030 ++------------
 interop/grpc_testing/test.proto               |  281 --
 interop/grpc_testing/test_grpc.pb.go          |  337 ++-
 interop/grpc_testing/worker_service.pb.go     |  120 +
 .../grpc_testing/worker_service_grpc.pb.go    |  237 +-
 interop/test_utils.go                         |    4 -
 regenerate.sh                                 |    8 +-
 stats/grpc_testing/test.pb.go                 |  254 --
 stats/grpc_testing/test.proto                 |   45 -
 stats/grpc_testing/test_grpc.pb.go            |  316 ---
 stats/stats_test.go                           |  125 +-
 37 files changed, 7268 insertions(+), 7493 deletions(-)
 delete mode 100644 benchmark/grpc_testing/control.pb.go
 delete mode 100644 benchmark/grpc_testing/control.proto
 delete mode 100644 benchmark/grpc_testing/messages.pb.go
 delete mode 100644 benchmark/grpc_testing/messages.proto
 delete mode 100644 benchmark/grpc_testing/payloads.proto
 delete mode 100644 benchmark/grpc_testing/services.pb.go
 delete mode 100644 benchmark/grpc_testing/services.proto
 delete mode 100644 benchmark/grpc_testing/stats.pb.go
 delete mode 100644 benchmark/grpc_testing/stats.proto
 create mode 100644 interop/grpc_testing/benchmark_service.pb.go
 create mode 100644 interop/grpc_testing/benchmark_service_grpc.pb.go
 create mode 100644 interop/grpc_testing/control.pb.go
 create mode 100644 interop/grpc_testing/core/stats.pb.go
 create mode 100644 interop/grpc_testing/empty.pb.go
 create mode 100644 interop/grpc_testing/messages.pb.go
 rename {benchmark => interop}/grpc_testing/payloads.pb.go (59%)
 create mode 100644 interop/grpc_testing/report_qps_scenario_service.pb.go
 create mode 100644 interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
 create mode 100644 interop/grpc_testing/stats.pb.go
 delete mode 100644 interop/grpc_testing/test.proto
 create mode 100644 interop/grpc_testing/worker_service.pb.go
 rename benchmark/grpc_testing/services_grpc.pb.go => interop/grpc_testing/worker_service_grpc.pb.go (54%)
 delete mode 100644 stats/grpc_testing/test.pb.go
 delete mode 100644 stats/grpc_testing/test.proto
 delete mode 100644 stats/grpc_testing/test_grpc.pb.go

diff --git a/benchmark/benchmain/main.go b/benchmark/benchmain/main.go
index 956024c7f047..509a1b905654 100644
--- a/benchmark/benchmain/main.go
+++ b/benchmark/benchmain/main.go
@@ -58,14 +58,16 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/benchmark"
 	bm "google.golang.org/grpc/benchmark"
 	"google.golang.org/grpc/benchmark/flags"
-	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/benchmark/latency"
 	"google.golang.org/grpc/benchmark/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/channelz"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/keepalive"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/test/bufconn"
 )
 
@@ -404,10 +406,12 @@ func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_Stre
 	tc, cleanup := makeClient(bf)
 
 	streams := make([]testpb.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls)
+	md := metadata.Pairs(benchmark.UnconstrainedStreamingHeader, "1")
+	ctx := metadata.NewOutgoingContext(context.Background(), md)
 	for i := 0; i < bf.MaxConcurrentCalls; i++ {
-		stream, err := tc.UnconstrainedStreamingCall(context.Background())
+		stream, err := tc.StreamingCall(ctx)
 		if err != nil {
-			logger.Fatalf("%v.UnconstrainedStreamingCall(_) = _, %v", tc, err)
+			logger.Fatalf("%v.StreamingCall(_) = _, %v", tc, err)
 		}
 		streams[i] = stream
 	}
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index 56841e172366..92fbeb888365 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -29,9 +29,10 @@ import (
 	"net"
 
 	"google.golang.org/grpc"
-	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
 
@@ -45,8 +46,6 @@ func setPayload(p *testpb.Payload, t testpb.PayloadType, size int) {
 	body := make([]byte, size)
 	switch t {
 	case testpb.PayloadType_COMPRESSABLE:
-	case testpb.PayloadType_UNCOMPRESSABLE:
-		logger.Fatalf("PayloadType UNCOMPRESSABLE is not supported")
 	default:
 		logger.Fatalf("Unsupported payload type: %d", t)
 	}
@@ -71,7 +70,15 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 	}, nil
 }
 
+// UnconstrainedStreamingHeader indicates to the StreamingCall handler that its
+// behavior should be unconstrained (constant send/receive in parallel) instead
+// of ping-pong.
+const UnconstrainedStreamingHeader = "unconstrained-streaming"
+
 func (s *testServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error {
+	if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[UnconstrainedStreamingHeader]) != 0 {
+		return s.UnconstrainedStreamingCall(stream)
+	}
 	response := &testpb.SimpleResponse{
 		Payload: new(testpb.Payload),
 	}
@@ -93,7 +100,7 @@ func (s *testServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallS
 	}
 }
 
-func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_UnconstrainedStreamingCallServer) error {
+func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error {
 	in := new(testpb.SimpleRequest)
 	// Receive a message to learn response type and size.
 	err := stream.RecvMsg(in)
@@ -171,23 +178,6 @@ func (s *byteBufServer) StreamingCall(stream testpb.BenchmarkService_StreamingCa
 	}
 }
 
-func (s *byteBufServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_UnconstrainedStreamingCallServer) error {
-	for {
-		var in []byte
-		err := stream.(grpc.ServerStream).RecvMsg(&in)
-		if err == io.EOF {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-		out := make([]byte, s.respSize)
-		if err := stream.(grpc.ServerStream).SendMsg(&out); err != nil {
-			return err
-		}
-	}
-}
-
 // ServerInfo contains the information to create a gRPC benchmark server.
 type ServerInfo struct {
 	// Type is the type of the server.
diff --git a/benchmark/client/main.go b/benchmark/client/main.go
index f750c5c30721..c744348469ed 100644
--- a/benchmark/client/main.go
+++ b/benchmark/client/main.go
@@ -50,10 +50,10 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/benchmark"
-	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/benchmark/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/syscall"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
diff --git a/benchmark/grpc_testing/control.pb.go b/benchmark/grpc_testing/control.pb.go
deleted file mode 100644
index af117985dbda..000000000000
--- a/benchmark/grpc_testing/control.pb.go
+++ /dev/null
@@ -1,1994 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: benchmark/grpc_testing/control.proto
-
-package grpc_testing
-
-import (
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-type ClientType int32
-
-const (
-	ClientType_SYNC_CLIENT  ClientType = 0
-	ClientType_ASYNC_CLIENT ClientType = 1
-)
-
-// Enum value maps for ClientType.
-var (
-	ClientType_name = map[int32]string{
-		0: "SYNC_CLIENT",
-		1: "ASYNC_CLIENT",
-	}
-	ClientType_value = map[string]int32{
-		"SYNC_CLIENT":  0,
-		"ASYNC_CLIENT": 1,
-	}
-)
-
-func (x ClientType) Enum() *ClientType {
-	p := new(ClientType)
-	*p = x
-	return p
-}
-
-func (x ClientType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (ClientType) Descriptor() protoreflect.EnumDescriptor {
-	return file_benchmark_grpc_testing_control_proto_enumTypes[0].Descriptor()
-}
-
-func (ClientType) Type() protoreflect.EnumType {
-	return &file_benchmark_grpc_testing_control_proto_enumTypes[0]
-}
-
-func (x ClientType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use ClientType.Descriptor instead.
-func (ClientType) EnumDescriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{0}
-}
-
-type ServerType int32
-
-const (
-	ServerType_SYNC_SERVER          ServerType = 0
-	ServerType_ASYNC_SERVER         ServerType = 1
-	ServerType_ASYNC_GENERIC_SERVER ServerType = 2
-)
-
-// Enum value maps for ServerType.
-var (
-	ServerType_name = map[int32]string{
-		0: "SYNC_SERVER",
-		1: "ASYNC_SERVER",
-		2: "ASYNC_GENERIC_SERVER",
-	}
-	ServerType_value = map[string]int32{
-		"SYNC_SERVER":          0,
-		"ASYNC_SERVER":         1,
-		"ASYNC_GENERIC_SERVER": 2,
-	}
-)
-
-func (x ServerType) Enum() *ServerType {
-	p := new(ServerType)
-	*p = x
-	return p
-}
-
-func (x ServerType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (ServerType) Descriptor() protoreflect.EnumDescriptor {
-	return file_benchmark_grpc_testing_control_proto_enumTypes[1].Descriptor()
-}
-
-func (ServerType) Type() protoreflect.EnumType {
-	return &file_benchmark_grpc_testing_control_proto_enumTypes[1]
-}
-
-func (x ServerType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use ServerType.Descriptor instead.
-func (ServerType) EnumDescriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{1}
-}
-
-type RpcType int32
-
-const (
-	RpcType_UNARY     RpcType = 0
-	RpcType_STREAMING RpcType = 1
-)
-
-// Enum value maps for RpcType.
-var (
-	RpcType_name = map[int32]string{
-		0: "UNARY",
-		1: "STREAMING",
-	}
-	RpcType_value = map[string]int32{
-		"UNARY":     0,
-		"STREAMING": 1,
-	}
-)
-
-func (x RpcType) Enum() *RpcType {
-	p := new(RpcType)
-	*p = x
-	return p
-}
-
-func (x RpcType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (RpcType) Descriptor() protoreflect.EnumDescriptor {
-	return file_benchmark_grpc_testing_control_proto_enumTypes[2].Descriptor()
-}
-
-func (RpcType) Type() protoreflect.EnumType {
-	return &file_benchmark_grpc_testing_control_proto_enumTypes[2]
-}
-
-func (x RpcType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use RpcType.Descriptor instead.
-func (RpcType) EnumDescriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{2}
-}
-
-// Parameters of poisson process distribution, which is a good representation
-// of activity coming in from independent identical stationary sources.
-type PoissonParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
-	OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
-}
-
-func (x *PoissonParams) Reset() {
-	*x = PoissonParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *PoissonParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*PoissonParams) ProtoMessage() {}
-
-func (x *PoissonParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use PoissonParams.ProtoReflect.Descriptor instead.
-func (*PoissonParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *PoissonParams) GetOfferedLoad() float64 {
-	if x != nil {
-		return x.OfferedLoad
-	}
-	return 0
-}
-
-type UniformParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	InterarrivalLo float64 `protobuf:"fixed64,1,opt,name=interarrival_lo,json=interarrivalLo,proto3" json:"interarrival_lo,omitempty"`
-	InterarrivalHi float64 `protobuf:"fixed64,2,opt,name=interarrival_hi,json=interarrivalHi,proto3" json:"interarrival_hi,omitempty"`
-}
-
-func (x *UniformParams) Reset() {
-	*x = UniformParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *UniformParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*UniformParams) ProtoMessage() {}
-
-func (x *UniformParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use UniformParams.ProtoReflect.Descriptor instead.
-func (*UniformParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *UniformParams) GetInterarrivalLo() float64 {
-	if x != nil {
-		return x.InterarrivalLo
-	}
-	return 0
-}
-
-func (x *UniformParams) GetInterarrivalHi() float64 {
-	if x != nil {
-		return x.InterarrivalHi
-	}
-	return 0
-}
-
-type DeterministicParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
-}
-
-func (x *DeterministicParams) Reset() {
-	*x = DeterministicParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *DeterministicParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*DeterministicParams) ProtoMessage() {}
-
-func (x *DeterministicParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use DeterministicParams.ProtoReflect.Descriptor instead.
-func (*DeterministicParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *DeterministicParams) GetOfferedLoad() float64 {
-	if x != nil {
-		return x.OfferedLoad
-	}
-	return 0
-}
-
-type ParetoParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	InterarrivalBase float64 `protobuf:"fixed64,1,opt,name=interarrival_base,json=interarrivalBase,proto3" json:"interarrival_base,omitempty"`
-	Alpha            float64 `protobuf:"fixed64,2,opt,name=alpha,proto3" json:"alpha,omitempty"`
-}
-
-func (x *ParetoParams) Reset() {
-	*x = ParetoParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ParetoParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ParetoParams) ProtoMessage() {}
-
-func (x *ParetoParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ParetoParams.ProtoReflect.Descriptor instead.
-func (*ParetoParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{3}
-}
-
-func (x *ParetoParams) GetInterarrivalBase() float64 {
-	if x != nil {
-		return x.InterarrivalBase
-	}
-	return 0
-}
-
-func (x *ParetoParams) GetAlpha() float64 {
-	if x != nil {
-		return x.Alpha
-	}
-	return 0
-}
-
-// Once an RPC finishes, immediately start a new one.
-// No configuration parameters needed.
-type ClosedLoopParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-}
-
-func (x *ClosedLoopParams) Reset() {
-	*x = ClosedLoopParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[4]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClosedLoopParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClosedLoopParams) ProtoMessage() {}
-
-func (x *ClosedLoopParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[4]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClosedLoopParams.ProtoReflect.Descriptor instead.
-func (*ClosedLoopParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{4}
-}
-
-type LoadParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Types that are assignable to Load:
-	//	*LoadParams_ClosedLoop
-	//	*LoadParams_Poisson
-	//	*LoadParams_Uniform
-	//	*LoadParams_Determ
-	//	*LoadParams_Pareto
-	Load isLoadParams_Load `protobuf_oneof:"load"`
-}
-
-func (x *LoadParams) Reset() {
-	*x = LoadParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[5]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *LoadParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*LoadParams) ProtoMessage() {}
-
-func (x *LoadParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[5]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use LoadParams.ProtoReflect.Descriptor instead.
-func (*LoadParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{5}
-}
-
-func (m *LoadParams) GetLoad() isLoadParams_Load {
-	if m != nil {
-		return m.Load
-	}
-	return nil
-}
-
-func (x *LoadParams) GetClosedLoop() *ClosedLoopParams {
-	if x, ok := x.GetLoad().(*LoadParams_ClosedLoop); ok {
-		return x.ClosedLoop
-	}
-	return nil
-}
-
-func (x *LoadParams) GetPoisson() *PoissonParams {
-	if x, ok := x.GetLoad().(*LoadParams_Poisson); ok {
-		return x.Poisson
-	}
-	return nil
-}
-
-func (x *LoadParams) GetUniform() *UniformParams {
-	if x, ok := x.GetLoad().(*LoadParams_Uniform); ok {
-		return x.Uniform
-	}
-	return nil
-}
-
-func (x *LoadParams) GetDeterm() *DeterministicParams {
-	if x, ok := x.GetLoad().(*LoadParams_Determ); ok {
-		return x.Determ
-	}
-	return nil
-}
-
-func (x *LoadParams) GetPareto() *ParetoParams {
-	if x, ok := x.GetLoad().(*LoadParams_Pareto); ok {
-		return x.Pareto
-	}
-	return nil
-}
-
-type isLoadParams_Load interface {
-	isLoadParams_Load()
-}
-
-type LoadParams_ClosedLoop struct {
-	ClosedLoop *ClosedLoopParams `protobuf:"bytes,1,opt,name=closed_loop,json=closedLoop,proto3,oneof"`
-}
-
-type LoadParams_Poisson struct {
-	Poisson *PoissonParams `protobuf:"bytes,2,opt,name=poisson,proto3,oneof"`
-}
-
-type LoadParams_Uniform struct {
-	Uniform *UniformParams `protobuf:"bytes,3,opt,name=uniform,proto3,oneof"`
-}
-
-type LoadParams_Determ struct {
-	Determ *DeterministicParams `protobuf:"bytes,4,opt,name=determ,proto3,oneof"`
-}
-
-type LoadParams_Pareto struct {
-	Pareto *ParetoParams `protobuf:"bytes,5,opt,name=pareto,proto3,oneof"`
-}
-
-func (*LoadParams_ClosedLoop) isLoadParams_Load() {}
-
-func (*LoadParams_Poisson) isLoadParams_Load() {}
-
-func (*LoadParams_Uniform) isLoadParams_Load() {}
-
-func (*LoadParams_Determ) isLoadParams_Load() {}
-
-func (*LoadParams_Pareto) isLoadParams_Load() {}
-
-// presence of SecurityParams implies use of TLS
-type SecurityParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	UseTestCa          bool   `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"`
-	ServerHostOverride string `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"`
-}
-
-func (x *SecurityParams) Reset() {
-	*x = SecurityParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[6]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SecurityParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SecurityParams) ProtoMessage() {}
-
-func (x *SecurityParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[6]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SecurityParams.ProtoReflect.Descriptor instead.
-func (*SecurityParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{6}
-}
-
-func (x *SecurityParams) GetUseTestCa() bool {
-	if x != nil {
-		return x.UseTestCa
-	}
-	return false
-}
-
-func (x *SecurityParams) GetServerHostOverride() string {
-	if x != nil {
-		return x.ServerHostOverride
-	}
-	return ""
-}
-
-type ClientConfig struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// List of targets to connect to. At least one target needs to be specified.
-	ServerTargets  []string        `protobuf:"bytes,1,rep,name=server_targets,json=serverTargets,proto3" json:"server_targets,omitempty"`
-	ClientType     ClientType      `protobuf:"varint,2,opt,name=client_type,json=clientType,proto3,enum=grpc.testing.ClientType" json:"client_type,omitempty"`
-	SecurityParams *SecurityParams `protobuf:"bytes,3,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"`
-	// How many concurrent RPCs to start for each channel.
-	// For synchronous client, use a separate thread for each outstanding RPC.
-	OutstandingRpcsPerChannel int32 `protobuf:"varint,4,opt,name=outstanding_rpcs_per_channel,json=outstandingRpcsPerChannel,proto3" json:"outstanding_rpcs_per_channel,omitempty"`
-	// Number of independent client channels to create.
-	// i-th channel will connect to server_target[i % server_targets.size()]
-	ClientChannels int32 `protobuf:"varint,5,opt,name=client_channels,json=clientChannels,proto3" json:"client_channels,omitempty"`
-	// Only for async client. Number of threads to use to start/manage RPCs.
-	AsyncClientThreads int32   `protobuf:"varint,7,opt,name=async_client_threads,json=asyncClientThreads,proto3" json:"async_client_threads,omitempty"`
-	RpcType            RpcType `protobuf:"varint,8,opt,name=rpc_type,json=rpcType,proto3,enum=grpc.testing.RpcType" json:"rpc_type,omitempty"`
-	// The requested load for the entire client (aggregated over all the threads).
-	LoadParams      *LoadParams      `protobuf:"bytes,10,opt,name=load_params,json=loadParams,proto3" json:"load_params,omitempty"`
-	PayloadConfig   *PayloadConfig   `protobuf:"bytes,11,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"`
-	HistogramParams *HistogramParams `protobuf:"bytes,12,opt,name=histogram_params,json=histogramParams,proto3" json:"histogram_params,omitempty"`
-	// Specify the cores we should run the client on, if desired
-	CoreList  []int32 `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
-	CoreLimit int32   `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"`
-}
-
-func (x *ClientConfig) Reset() {
-	*x = ClientConfig{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[7]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientConfig) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientConfig) ProtoMessage() {}
-
-func (x *ClientConfig) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[7]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientConfig.ProtoReflect.Descriptor instead.
-func (*ClientConfig) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{7}
-}
-
-func (x *ClientConfig) GetServerTargets() []string {
-	if x != nil {
-		return x.ServerTargets
-	}
-	return nil
-}
-
-func (x *ClientConfig) GetClientType() ClientType {
-	if x != nil {
-		return x.ClientType
-	}
-	return ClientType_SYNC_CLIENT
-}
-
-func (x *ClientConfig) GetSecurityParams() *SecurityParams {
-	if x != nil {
-		return x.SecurityParams
-	}
-	return nil
-}
-
-func (x *ClientConfig) GetOutstandingRpcsPerChannel() int32 {
-	if x != nil {
-		return x.OutstandingRpcsPerChannel
-	}
-	return 0
-}
-
-func (x *ClientConfig) GetClientChannels() int32 {
-	if x != nil {
-		return x.ClientChannels
-	}
-	return 0
-}
-
-func (x *ClientConfig) GetAsyncClientThreads() int32 {
-	if x != nil {
-		return x.AsyncClientThreads
-	}
-	return 0
-}
-
-func (x *ClientConfig) GetRpcType() RpcType {
-	if x != nil {
-		return x.RpcType
-	}
-	return RpcType_UNARY
-}
-
-func (x *ClientConfig) GetLoadParams() *LoadParams {
-	if x != nil {
-		return x.LoadParams
-	}
-	return nil
-}
-
-func (x *ClientConfig) GetPayloadConfig() *PayloadConfig {
-	if x != nil {
-		return x.PayloadConfig
-	}
-	return nil
-}
-
-func (x *ClientConfig) GetHistogramParams() *HistogramParams {
-	if x != nil {
-		return x.HistogramParams
-	}
-	return nil
-}
-
-func (x *ClientConfig) GetCoreList() []int32 {
-	if x != nil {
-		return x.CoreList
-	}
-	return nil
-}
-
-func (x *ClientConfig) GetCoreLimit() int32 {
-	if x != nil {
-		return x.CoreLimit
-	}
-	return 0
-}
-
-type ClientStatus struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Stats *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
-}
-
-func (x *ClientStatus) Reset() {
-	*x = ClientStatus{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[8]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientStatus) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientStatus) ProtoMessage() {}
-
-func (x *ClientStatus) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[8]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientStatus.ProtoReflect.Descriptor instead.
-func (*ClientStatus) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{8}
-}
-
-func (x *ClientStatus) GetStats() *ClientStats {
-	if x != nil {
-		return x.Stats
-	}
-	return nil
-}
-
-// Request current stats
-type Mark struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// if true, the stats will be reset after taking their snapshot.
-	Reset_ bool `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"`
-}
-
-func (x *Mark) Reset() {
-	*x = Mark{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[9]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Mark) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Mark) ProtoMessage() {}
-
-func (x *Mark) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[9]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Mark.ProtoReflect.Descriptor instead.
-func (*Mark) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{9}
-}
-
-func (x *Mark) GetReset_() bool {
-	if x != nil {
-		return x.Reset_
-	}
-	return false
-}
-
-type ClientArgs struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Types that are assignable to Argtype:
-	//	*ClientArgs_Setup
-	//	*ClientArgs_Mark
-	Argtype isClientArgs_Argtype `protobuf_oneof:"argtype"`
-}
-
-func (x *ClientArgs) Reset() {
-	*x = ClientArgs{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[10]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientArgs) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientArgs) ProtoMessage() {}
-
-func (x *ClientArgs) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[10]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientArgs.ProtoReflect.Descriptor instead.
-func (*ClientArgs) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{10}
-}
-
-func (m *ClientArgs) GetArgtype() isClientArgs_Argtype {
-	if m != nil {
-		return m.Argtype
-	}
-	return nil
-}
-
-func (x *ClientArgs) GetSetup() *ClientConfig {
-	if x, ok := x.GetArgtype().(*ClientArgs_Setup); ok {
-		return x.Setup
-	}
-	return nil
-}
-
-func (x *ClientArgs) GetMark() *Mark {
-	if x, ok := x.GetArgtype().(*ClientArgs_Mark); ok {
-		return x.Mark
-	}
-	return nil
-}
-
-type isClientArgs_Argtype interface {
-	isClientArgs_Argtype()
-}
-
-type ClientArgs_Setup struct {
-	Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
-}
-
-type ClientArgs_Mark struct {
-	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
-}
-
-func (*ClientArgs_Setup) isClientArgs_Argtype() {}
-
-func (*ClientArgs_Mark) isClientArgs_Argtype() {}
-
-type ServerConfig struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	ServerType     ServerType      `protobuf:"varint,1,opt,name=server_type,json=serverType,proto3,enum=grpc.testing.ServerType" json:"server_type,omitempty"`
-	SecurityParams *SecurityParams `protobuf:"bytes,2,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"`
-	// Port on which to listen. Zero means pick unused port.
-	Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"`
-	// Only for async server. Number of threads used to serve the requests.
-	AsyncServerThreads int32 `protobuf:"varint,7,opt,name=async_server_threads,json=asyncServerThreads,proto3" json:"async_server_threads,omitempty"`
-	// Specify the number of cores to limit server to, if desired
-	CoreLimit int32 `protobuf:"varint,8,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"`
-	// payload config, used in generic server
-	PayloadConfig *PayloadConfig `protobuf:"bytes,9,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"`
-	// Specify the cores we should run the server on, if desired
-	CoreList []int32 `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
-}
-
-func (x *ServerConfig) Reset() {
-	*x = ServerConfig{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[11]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ServerConfig) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ServerConfig) ProtoMessage() {}
-
-func (x *ServerConfig) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[11]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ServerConfig.ProtoReflect.Descriptor instead.
-func (*ServerConfig) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{11}
-}
-
-func (x *ServerConfig) GetServerType() ServerType {
-	if x != nil {
-		return x.ServerType
-	}
-	return ServerType_SYNC_SERVER
-}
-
-func (x *ServerConfig) GetSecurityParams() *SecurityParams {
-	if x != nil {
-		return x.SecurityParams
-	}
-	return nil
-}
-
-func (x *ServerConfig) GetPort() int32 {
-	if x != nil {
-		return x.Port
-	}
-	return 0
-}
-
-func (x *ServerConfig) GetAsyncServerThreads() int32 {
-	if x != nil {
-		return x.AsyncServerThreads
-	}
-	return 0
-}
-
-func (x *ServerConfig) GetCoreLimit() int32 {
-	if x != nil {
-		return x.CoreLimit
-	}
-	return 0
-}
-
-func (x *ServerConfig) GetPayloadConfig() *PayloadConfig {
-	if x != nil {
-		return x.PayloadConfig
-	}
-	return nil
-}
-
-func (x *ServerConfig) GetCoreList() []int32 {
-	if x != nil {
-		return x.CoreList
-	}
-	return nil
-}
-
-type ServerArgs struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Types that are assignable to Argtype:
-	//	*ServerArgs_Setup
-	//	*ServerArgs_Mark
-	Argtype isServerArgs_Argtype `protobuf_oneof:"argtype"`
-}
-
-func (x *ServerArgs) Reset() {
-	*x = ServerArgs{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[12]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ServerArgs) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ServerArgs) ProtoMessage() {}
-
-func (x *ServerArgs) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[12]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ServerArgs.ProtoReflect.Descriptor instead.
-func (*ServerArgs) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{12}
-}
-
-func (m *ServerArgs) GetArgtype() isServerArgs_Argtype {
-	if m != nil {
-		return m.Argtype
-	}
-	return nil
-}
-
-func (x *ServerArgs) GetSetup() *ServerConfig {
-	if x, ok := x.GetArgtype().(*ServerArgs_Setup); ok {
-		return x.Setup
-	}
-	return nil
-}
-
-func (x *ServerArgs) GetMark() *Mark {
-	if x, ok := x.GetArgtype().(*ServerArgs_Mark); ok {
-		return x.Mark
-	}
-	return nil
-}
-
-type isServerArgs_Argtype interface {
-	isServerArgs_Argtype()
-}
-
-type ServerArgs_Setup struct {
-	Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
-}
-
-type ServerArgs_Mark struct {
-	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
-}
-
-func (*ServerArgs_Setup) isServerArgs_Argtype() {}
-
-func (*ServerArgs_Mark) isServerArgs_Argtype() {}
-
-type ServerStatus struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Stats *ServerStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
-	// the port bound by the server
-	Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
-	// Number of cores available to the server
-	Cores int32 `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"`
-}
-
-func (x *ServerStatus) Reset() {
-	*x = ServerStatus{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[13]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ServerStatus) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ServerStatus) ProtoMessage() {}
-
-func (x *ServerStatus) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[13]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ServerStatus.ProtoReflect.Descriptor instead.
-func (*ServerStatus) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{13}
-}
-
-func (x *ServerStatus) GetStats() *ServerStats {
-	if x != nil {
-		return x.Stats
-	}
-	return nil
-}
-
-func (x *ServerStatus) GetPort() int32 {
-	if x != nil {
-		return x.Port
-	}
-	return 0
-}
-
-func (x *ServerStatus) GetCores() int32 {
-	if x != nil {
-		return x.Cores
-	}
-	return 0
-}
-
-type CoreRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-}
-
-func (x *CoreRequest) Reset() {
-	*x = CoreRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[14]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *CoreRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*CoreRequest) ProtoMessage() {}
-
-func (x *CoreRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[14]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use CoreRequest.ProtoReflect.Descriptor instead.
-func (*CoreRequest) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{14}
-}
-
-type CoreResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Number of cores available on the server
-	Cores int32 `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"`
-}
-
-func (x *CoreResponse) Reset() {
-	*x = CoreResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[15]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *CoreResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*CoreResponse) ProtoMessage() {}
-
-func (x *CoreResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[15]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use CoreResponse.ProtoReflect.Descriptor instead.
-func (*CoreResponse) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{15}
-}
-
-func (x *CoreResponse) GetCores() int32 {
-	if x != nil {
-		return x.Cores
-	}
-	return 0
-}
-
-type Void struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-}
-
-func (x *Void) Reset() {
-	*x = Void{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[16]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Void) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Void) ProtoMessage() {}
-
-func (x *Void) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[16]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Void.ProtoReflect.Descriptor instead.
-func (*Void) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{16}
-}
-
-// A single performance scenario: input to qps_json_driver
-type Scenario struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Human readable name for this scenario
-	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	// Client configuration
-	ClientConfig *ClientConfig `protobuf:"bytes,2,opt,name=client_config,json=clientConfig,proto3" json:"client_config,omitempty"`
-	// Number of clients to start for the test
-	NumClients int32 `protobuf:"varint,3,opt,name=num_clients,json=numClients,proto3" json:"num_clients,omitempty"`
-	// Server configuration
-	ServerConfig *ServerConfig `protobuf:"bytes,4,opt,name=server_config,json=serverConfig,proto3" json:"server_config,omitempty"`
-	// Number of servers to start for the test
-	NumServers int32 `protobuf:"varint,5,opt,name=num_servers,json=numServers,proto3" json:"num_servers,omitempty"`
-	// Warmup period, in seconds
-	WarmupSeconds int32 `protobuf:"varint,6,opt,name=warmup_seconds,json=warmupSeconds,proto3" json:"warmup_seconds,omitempty"`
-	// Benchmark time, in seconds
-	BenchmarkSeconds int32 `protobuf:"varint,7,opt,name=benchmark_seconds,json=benchmarkSeconds,proto3" json:"benchmark_seconds,omitempty"`
-	// Number of workers to spawn locally (usually zero)
-	SpawnLocalWorkerCount int32 `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"`
-}
-
-func (x *Scenario) Reset() {
-	*x = Scenario{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[17]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Scenario) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Scenario) ProtoMessage() {}
-
-func (x *Scenario) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[17]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Scenario.ProtoReflect.Descriptor instead.
-func (*Scenario) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{17}
-}
-
-func (x *Scenario) GetName() string {
-	if x != nil {
-		return x.Name
-	}
-	return ""
-}
-
-func (x *Scenario) GetClientConfig() *ClientConfig {
-	if x != nil {
-		return x.ClientConfig
-	}
-	return nil
-}
-
-func (x *Scenario) GetNumClients() int32 {
-	if x != nil {
-		return x.NumClients
-	}
-	return 0
-}
-
-func (x *Scenario) GetServerConfig() *ServerConfig {
-	if x != nil {
-		return x.ServerConfig
-	}
-	return nil
-}
-
-func (x *Scenario) GetNumServers() int32 {
-	if x != nil {
-		return x.NumServers
-	}
-	return 0
-}
-
-func (x *Scenario) GetWarmupSeconds() int32 {
-	if x != nil {
-		return x.WarmupSeconds
-	}
-	return 0
-}
-
-func (x *Scenario) GetBenchmarkSeconds() int32 {
-	if x != nil {
-		return x.BenchmarkSeconds
-	}
-	return 0
-}
-
-func (x *Scenario) GetSpawnLocalWorkerCount() int32 {
-	if x != nil {
-		return x.SpawnLocalWorkerCount
-	}
-	return 0
-}
-
-// A set of scenarios to be run with qps_json_driver
-type Scenarios struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Scenarios []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"`
-}
-
-func (x *Scenarios) Reset() {
-	*x = Scenarios{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_control_proto_msgTypes[18]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Scenarios) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Scenarios) ProtoMessage() {}
-
-func (x *Scenarios) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_control_proto_msgTypes[18]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Scenarios.ProtoReflect.Descriptor instead.
-func (*Scenarios) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_control_proto_rawDescGZIP(), []int{18}
-}
-
-func (x *Scenarios) GetScenarios() []*Scenario {
-	if x != nil {
-		return x.Scenarios
-	}
-	return nil
-}
-
-var File_benchmark_grpc_testing_control_proto protoreflect.FileDescriptor
-
-var file_benchmark_grpc_testing_control_proto_rawDesc = []byte{
-	0x0a, 0x24, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x1a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f,
-	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22, 0x62, 0x65, 0x6e,
-	0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
-	0x32, 0x0a, 0x0d, 0x50, 0x6f, 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x6c, 0x6f, 0x61, 0x64,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x4c,
-	0x6f, 0x61, 0x64, 0x22, 0x61, 0x0a, 0x0d, 0x55, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x50, 0x61,
-	0x72, 0x61, 0x6d, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72,
-	0x69, 0x76, 0x61, 0x6c, 0x5f, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x69,
-	0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c, 0x4c, 0x6f, 0x12, 0x27, 0x0a,
-	0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c, 0x5f, 0x68, 0x69,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72,
-	0x69, 0x76, 0x61, 0x6c, 0x48, 0x69, 0x22, 0x38, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x65, 0x72, 0x6d,
-	0x69, 0x6e, 0x69, 0x73, 0x74, 0x69, 0x63, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x21, 0x0a,
-	0x0c, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x01, 0x52, 0x0b, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x4c, 0x6f, 0x61, 0x64,
-	0x22, 0x51, 0x0a, 0x0c, 0x50, 0x61, 0x72, 0x65, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x12, 0x2b, 0x0a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c,
-	0x5f, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x69, 0x6e, 0x74,
-	0x65, 0x72, 0x61, 0x72, 0x72, 0x69, 0x76, 0x61, 0x6c, 0x42, 0x61, 0x73, 0x65, 0x12, 0x14, 0x0a,
-	0x05, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x61, 0x6c,
-	0x70, 0x68, 0x61, 0x22, 0x12, 0x0a, 0x10, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f,
-	0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0xbc, 0x02, 0x0a, 0x0a, 0x4c, 0x6f, 0x61, 0x64,
-	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x64,
-	0x5f, 0x6c, 0x6f, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65,
-	0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x63,
-	0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x12, 0x37, 0x0a, 0x07, 0x70, 0x6f, 0x69,
-	0x73, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x6f, 0x69, 0x73, 0x73, 0x6f,
-	0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x07, 0x70, 0x6f, 0x69, 0x73, 0x73,
-	0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x03, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x55, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x48, 0x00, 0x52, 0x07, 0x75, 0x6e, 0x69, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x3b, 0x0a, 0x06, 0x64,
-	0x65, 0x74, 0x65, 0x72, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x65, 0x74, 0x65, 0x72,
-	0x6d, 0x69, 0x6e, 0x69, 0x73, 0x74, 0x69, 0x63, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00,
-	0x52, 0x06, 0x64, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x12, 0x34, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65,
-	0x74, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
-	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x72, 0x65, 0x74, 0x6f, 0x50, 0x61,
-	0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x74, 0x6f, 0x42, 0x06,
-	0x0a, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x62, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69,
-	0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x5f,
-	0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x75,
-	0x73, 0x65, 0x54, 0x65, 0x73, 0x74, 0x43, 0x61, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x6f,
-	0x73, 0x74, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x22, 0x8a, 0x05, 0x0a, 0x0c, 0x43,
-	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x25, 0x0a, 0x0e, 0x73,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20,
-	0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65,
-	0x74, 0x73, 0x12, 0x39, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70,
-	0x65, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a,
-	0x0f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61,
-	0x72, 0x61, 0x6d, 0x73, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61,
-	0x72, 0x61, 0x6d, 0x73, 0x12, 0x3f, 0x0a, 0x1c, 0x6f, 0x75, 0x74, 0x73, 0x74, 0x61, 0x6e, 0x64,
-	0x69, 0x6e, 0x67, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x68, 0x61,
-	0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x19, 0x6f, 0x75, 0x74, 0x73,
-	0x74, 0x61, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x70, 0x63, 0x73, 0x50, 0x65, 0x72, 0x43, 0x68,
-	0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f,
-	0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e,
-	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x30,
-	0x0a, 0x14, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74,
-	0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x73,
-	0x79, 0x6e, 0x63, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73,
-	0x12, 0x30, 0x0a, 0x08, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01,
-	0x28, 0x0e, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x72, 0x70, 0x63, 0x54, 0x79,
-	0x70, 0x65, 0x12, 0x39, 0x0a, 0x0b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d,
-	0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d,
-	0x73, 0x52, 0x0a, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x42, 0x0a,
-	0x0e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18,
-	0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66,
-	0x69, 0x67, 0x52, 0x0d, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69,
-	0x67, 0x12, 0x48, 0x0a, 0x10, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x5f, 0x70,
-	0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f,
-	0x67, 0x72, 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0f, 0x68, 0x69, 0x73, 0x74,
-	0x6f, 0x67, 0x72, 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63,
-	0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08,
-	0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65,
-	0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f,
-	0x72, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x3f, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
-	0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0x1c, 0x0a, 0x04, 0x4d, 0x61, 0x72, 0x6b,
-	0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x05, 0x72, 0x65, 0x73, 0x65, 0x74, 0x22, 0x75, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
-	0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48,
-	0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61, 0x72, 0x6b,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x61,
-	0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, 0x22, 0xd6, 0x02,
-	0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x39,
-	0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x73,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x73, 0x65, 0x63,
-	0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04,
-	0x70, 0x6f, 0x72, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x73, 0x65,
-	0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01,
-	0x28, 0x05, 0x52, 0x12, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54,
-	0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c,
-	0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65,
-	0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x42, 0x0a, 0x0e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
-	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0d, 0x70, 0x61, 0x79, 0x6c,
-	0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x72,
-	0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6f,
-	0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x75, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
-	0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48,
-	0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61, 0x72, 0x6b,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x61,
-	0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, 0x22, 0x69, 0x0a,
-	0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a,
-	0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67,
-	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x12, 0x12,
-	0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f,
-	0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22, 0x0d, 0x0a, 0x0b, 0x43, 0x6f, 0x72, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x0c, 0x43, 0x6f, 0x72, 0x65, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22, 0x06, 0x0a,
-	0x04, 0x56, 0x6f, 0x69, 0x64, 0x22, 0xef, 0x02, 0x0a, 0x08, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72,
-	0x69, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
-	0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69,
-	0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x63,
-	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6e, 0x75,
-	0x6d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x73, 0x65, 0x72,
-	0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, 0x6d,
-	0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
-	0x6e, 0x75, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x61,
-	0x72, 0x6d, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01,
-	0x28, 0x05, 0x52, 0x0d, 0x77, 0x61, 0x72, 0x6d, 0x75, 0x70, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64,
-	0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x5f, 0x73,
-	0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x62, 0x65,
-	0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x37,
-	0x0a, 0x18, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x77, 0x6f,
-	0x72, 0x6b, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05,
-	0x52, 0x15, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x57, 0x6f, 0x72, 0x6b,
-	0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x41, 0x0a, 0x09, 0x53, 0x63, 0x65, 0x6e, 0x61,
-	0x72, 0x69, 0x6f, 0x73, 0x12, 0x34, 0x0a, 0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f,
-	0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52,
-	0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x73, 0x2a, 0x2f, 0x0a, 0x0a, 0x43, 0x6c,
-	0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e, 0x43,
-	0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53, 0x59,
-	0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x2a, 0x49, 0x0a, 0x0a, 0x53,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e,
-	0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53,
-	0x59, 0x4e, 0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14,
-	0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x5f, 0x53, 0x45,
-	0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x23, 0x0a, 0x07, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70,
-	0x65, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09,
-	0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x42, 0x2f, 0x5a, 0x2d, 0x67,
-	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
-	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f,
-	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x33,
-}
-
-var (
-	file_benchmark_grpc_testing_control_proto_rawDescOnce sync.Once
-	file_benchmark_grpc_testing_control_proto_rawDescData = file_benchmark_grpc_testing_control_proto_rawDesc
-)
-
-func file_benchmark_grpc_testing_control_proto_rawDescGZIP() []byte {
-	file_benchmark_grpc_testing_control_proto_rawDescOnce.Do(func() {
-		file_benchmark_grpc_testing_control_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_control_proto_rawDescData)
-	})
-	return file_benchmark_grpc_testing_control_proto_rawDescData
-}
-
-var file_benchmark_grpc_testing_control_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_benchmark_grpc_testing_control_proto_msgTypes = make([]protoimpl.MessageInfo, 19)
-var file_benchmark_grpc_testing_control_proto_goTypes = []interface{}{
-	(ClientType)(0),             // 0: grpc.testing.ClientType
-	(ServerType)(0),             // 1: grpc.testing.ServerType
-	(RpcType)(0),                // 2: grpc.testing.RpcType
-	(*PoissonParams)(nil),       // 3: grpc.testing.PoissonParams
-	(*UniformParams)(nil),       // 4: grpc.testing.UniformParams
-	(*DeterministicParams)(nil), // 5: grpc.testing.DeterministicParams
-	(*ParetoParams)(nil),        // 6: grpc.testing.ParetoParams
-	(*ClosedLoopParams)(nil),    // 7: grpc.testing.ClosedLoopParams
-	(*LoadParams)(nil),          // 8: grpc.testing.LoadParams
-	(*SecurityParams)(nil),      // 9: grpc.testing.SecurityParams
-	(*ClientConfig)(nil),        // 10: grpc.testing.ClientConfig
-	(*ClientStatus)(nil),        // 11: grpc.testing.ClientStatus
-	(*Mark)(nil),                // 12: grpc.testing.Mark
-	(*ClientArgs)(nil),          // 13: grpc.testing.ClientArgs
-	(*ServerConfig)(nil),        // 14: grpc.testing.ServerConfig
-	(*ServerArgs)(nil),          // 15: grpc.testing.ServerArgs
-	(*ServerStatus)(nil),        // 16: grpc.testing.ServerStatus
-	(*CoreRequest)(nil),         // 17: grpc.testing.CoreRequest
-	(*CoreResponse)(nil),        // 18: grpc.testing.CoreResponse
-	(*Void)(nil),                // 19: grpc.testing.Void
-	(*Scenario)(nil),            // 20: grpc.testing.Scenario
-	(*Scenarios)(nil),           // 21: grpc.testing.Scenarios
-	(*PayloadConfig)(nil),       // 22: grpc.testing.PayloadConfig
-	(*HistogramParams)(nil),     // 23: grpc.testing.HistogramParams
-	(*ClientStats)(nil),         // 24: grpc.testing.ClientStats
-	(*ServerStats)(nil),         // 25: grpc.testing.ServerStats
-}
-var file_benchmark_grpc_testing_control_proto_depIdxs = []int32{
-	7,  // 0: grpc.testing.LoadParams.closed_loop:type_name -> grpc.testing.ClosedLoopParams
-	3,  // 1: grpc.testing.LoadParams.poisson:type_name -> grpc.testing.PoissonParams
-	4,  // 2: grpc.testing.LoadParams.uniform:type_name -> grpc.testing.UniformParams
-	5,  // 3: grpc.testing.LoadParams.determ:type_name -> grpc.testing.DeterministicParams
-	6,  // 4: grpc.testing.LoadParams.pareto:type_name -> grpc.testing.ParetoParams
-	0,  // 5: grpc.testing.ClientConfig.client_type:type_name -> grpc.testing.ClientType
-	9,  // 6: grpc.testing.ClientConfig.security_params:type_name -> grpc.testing.SecurityParams
-	2,  // 7: grpc.testing.ClientConfig.rpc_type:type_name -> grpc.testing.RpcType
-	8,  // 8: grpc.testing.ClientConfig.load_params:type_name -> grpc.testing.LoadParams
-	22, // 9: grpc.testing.ClientConfig.payload_config:type_name -> grpc.testing.PayloadConfig
-	23, // 10: grpc.testing.ClientConfig.histogram_params:type_name -> grpc.testing.HistogramParams
-	24, // 11: grpc.testing.ClientStatus.stats:type_name -> grpc.testing.ClientStats
-	10, // 12: grpc.testing.ClientArgs.setup:type_name -> grpc.testing.ClientConfig
-	12, // 13: grpc.testing.ClientArgs.mark:type_name -> grpc.testing.Mark
-	1,  // 14: grpc.testing.ServerConfig.server_type:type_name -> grpc.testing.ServerType
-	9,  // 15: grpc.testing.ServerConfig.security_params:type_name -> grpc.testing.SecurityParams
-	22, // 16: grpc.testing.ServerConfig.payload_config:type_name -> grpc.testing.PayloadConfig
-	14, // 17: grpc.testing.ServerArgs.setup:type_name -> grpc.testing.ServerConfig
-	12, // 18: grpc.testing.ServerArgs.mark:type_name -> grpc.testing.Mark
-	25, // 19: grpc.testing.ServerStatus.stats:type_name -> grpc.testing.ServerStats
-	10, // 20: grpc.testing.Scenario.client_config:type_name -> grpc.testing.ClientConfig
-	14, // 21: grpc.testing.Scenario.server_config:type_name -> grpc.testing.ServerConfig
-	20, // 22: grpc.testing.Scenarios.scenarios:type_name -> grpc.testing.Scenario
-	23, // [23:23] is the sub-list for method output_type
-	23, // [23:23] is the sub-list for method input_type
-	23, // [23:23] is the sub-list for extension type_name
-	23, // [23:23] is the sub-list for extension extendee
-	0,  // [0:23] is the sub-list for field type_name
-}
-
-func init() { file_benchmark_grpc_testing_control_proto_init() }
-func file_benchmark_grpc_testing_control_proto_init() {
-	if File_benchmark_grpc_testing_control_proto != nil {
-		return
-	}
-	file_benchmark_grpc_testing_payloads_proto_init()
-	file_benchmark_grpc_testing_stats_proto_init()
-	if !protoimpl.UnsafeEnabled {
-		file_benchmark_grpc_testing_control_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*PoissonParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*UniformParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*DeterministicParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ParetoParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClosedLoopParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LoadParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SecurityParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientConfig); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientStatus); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Mark); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientArgs); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ServerConfig); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ServerArgs); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ServerStatus); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*CoreRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*CoreResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Void); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Scenario); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_control_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Scenarios); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	file_benchmark_grpc_testing_control_proto_msgTypes[5].OneofWrappers = []interface{}{
-		(*LoadParams_ClosedLoop)(nil),
-		(*LoadParams_Poisson)(nil),
-		(*LoadParams_Uniform)(nil),
-		(*LoadParams_Determ)(nil),
-		(*LoadParams_Pareto)(nil),
-	}
-	file_benchmark_grpc_testing_control_proto_msgTypes[10].OneofWrappers = []interface{}{
-		(*ClientArgs_Setup)(nil),
-		(*ClientArgs_Mark)(nil),
-	}
-	file_benchmark_grpc_testing_control_proto_msgTypes[12].OneofWrappers = []interface{}{
-		(*ServerArgs_Setup)(nil),
-		(*ServerArgs_Mark)(nil),
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_benchmark_grpc_testing_control_proto_rawDesc,
-			NumEnums:      3,
-			NumMessages:   19,
-			NumExtensions: 0,
-			NumServices:   0,
-		},
-		GoTypes:           file_benchmark_grpc_testing_control_proto_goTypes,
-		DependencyIndexes: file_benchmark_grpc_testing_control_proto_depIdxs,
-		EnumInfos:         file_benchmark_grpc_testing_control_proto_enumTypes,
-		MessageInfos:      file_benchmark_grpc_testing_control_proto_msgTypes,
-	}.Build()
-	File_benchmark_grpc_testing_control_proto = out.File
-	file_benchmark_grpc_testing_control_proto_rawDesc = nil
-	file_benchmark_grpc_testing_control_proto_goTypes = nil
-	file_benchmark_grpc_testing_control_proto_depIdxs = nil
-}
diff --git a/benchmark/grpc_testing/control.proto b/benchmark/grpc_testing/control.proto
deleted file mode 100644
index e9ee3484d19f..000000000000
--- a/benchmark/grpc_testing/control.proto
+++ /dev/null
@@ -1,188 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
-
-import "benchmark/grpc_testing/payloads.proto";
-import "benchmark/grpc_testing/stats.proto";
-
-package grpc.testing;
-
-enum ClientType {
-  SYNC_CLIENT = 0;
-  ASYNC_CLIENT = 1;
-}
-
-enum ServerType {
-  SYNC_SERVER = 0;
-  ASYNC_SERVER = 1;
-  ASYNC_GENERIC_SERVER = 2;
-}
-
-enum RpcType {
-  UNARY = 0;
-  STREAMING = 1;
-}
-
-// Parameters of poisson process distribution, which is a good representation
-// of activity coming in from independent identical stationary sources.
-message PoissonParams {
-  // The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
-  double offered_load = 1;
-}
-
-message UniformParams {
-  double interarrival_lo = 1;
-  double interarrival_hi = 2;
-}
-
-message DeterministicParams {
-  double offered_load = 1;
-}
-
-message ParetoParams {
-  double interarrival_base = 1;
-  double alpha = 2;
-}
-
-// Once an RPC finishes, immediately start a new one.
-// No configuration parameters needed.
-message ClosedLoopParams {
-}
-
-message LoadParams {
-  oneof load {
-    ClosedLoopParams closed_loop = 1;
-    PoissonParams poisson = 2;
-    UniformParams uniform = 3;
-    DeterministicParams determ = 4;
-    ParetoParams pareto = 5;
-  };
-}
-
-// presence of SecurityParams implies use of TLS
-message SecurityParams {
-  bool use_test_ca = 1;
-  string server_host_override = 2;
-}
-
-message ClientConfig {
-  // List of targets to connect to. At least one target needs to be specified.
-  repeated string server_targets = 1;
-  ClientType client_type = 2;
-  SecurityParams security_params = 3;
-  // How many concurrent RPCs to start for each channel.
-  // For synchronous client, use a separate thread for each outstanding RPC.
-  int32 outstanding_rpcs_per_channel = 4;
-  // Number of independent client channels to create.
-  // i-th channel will connect to server_target[i % server_targets.size()]
-  int32 client_channels = 5;
-  // Only for async client. Number of threads to use to start/manage RPCs.
-  int32 async_client_threads = 7;
-  RpcType rpc_type = 8;
-  // The requested load for the entire client (aggregated over all the threads).
-  LoadParams load_params = 10;
-  PayloadConfig payload_config = 11;
-  HistogramParams histogram_params = 12;
-
-  // Specify the cores we should run the client on, if desired
-  repeated int32 core_list = 13;
-  int32 core_limit = 14;
-}
-
-message ClientStatus {
-  ClientStats stats = 1;
-}
-
-// Request current stats
-message Mark {
-  // if true, the stats will be reset after taking their snapshot.
-  bool reset = 1;
-}
-
-message ClientArgs {
-  oneof argtype {
-    ClientConfig setup = 1;
-    Mark mark = 2;
-  }
-}
-
-message ServerConfig {
-  ServerType server_type = 1;
-  SecurityParams security_params = 2;
-  // Port on which to listen. Zero means pick unused port.
-  int32 port = 4;
-  // Only for async server. Number of threads used to serve the requests.
-  int32 async_server_threads = 7;
-  // Specify the number of cores to limit server to, if desired
-  int32 core_limit = 8;
-  // payload config, used in generic server
-  PayloadConfig payload_config = 9;
-
-  // Specify the cores we should run the server on, if desired
-  repeated int32 core_list = 10;
-}
-
-message ServerArgs {
-  oneof argtype {
-    ServerConfig setup = 1;
-    Mark mark = 2;
-  }
-}
-
-message ServerStatus {
-  ServerStats stats = 1;
-  // the port bound by the server
-  int32 port = 2;
-  // Number of cores available to the server
-  int32 cores = 3;
-}
-
-message CoreRequest {
-}
-
-message CoreResponse {
-  // Number of cores available on the server
-  int32 cores = 1;
-}
-
-message Void {
-}
-
-// A single performance scenario: input to qps_json_driver
-message Scenario {
-  // Human readable name for this scenario
-  string name = 1;
-  // Client configuration
-  ClientConfig client_config = 2;
-  // Number of clients to start for the test
-  int32 num_clients = 3;
-  // Server configuration
-  ServerConfig server_config = 4;
-  // Number of servers to start for the test
-  int32 num_servers = 5;
-  // Warmup period, in seconds
-  int32 warmup_seconds = 6;
-  // Benchmark time, in seconds
-  int32 benchmark_seconds = 7;
-  // Number of workers to spawn locally (usually zero)
-  int32 spawn_local_worker_count = 8;
-}
-
-// A set of scenarios to be run with qps_json_driver
-message Scenarios {
-  repeated Scenario scenarios = 1;
-}
diff --git a/benchmark/grpc_testing/messages.pb.go b/benchmark/grpc_testing/messages.pb.go
deleted file mode 100644
index eb6e2975b326..000000000000
--- a/benchmark/grpc_testing/messages.pb.go
+++ /dev/null
@@ -1,1160 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Message definitions to be used by integration test service definitions.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: benchmark/grpc_testing/messages.proto
-
-package grpc_testing
-
-import (
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-// The type of payload that should be returned.
-type PayloadType int32
-
-const (
-	// Compressable text format.
-	PayloadType_COMPRESSABLE PayloadType = 0
-	// Uncompressable binary format.
-	PayloadType_UNCOMPRESSABLE PayloadType = 1
-	// Randomly chosen from all other formats defined in this enum.
-	PayloadType_RANDOM PayloadType = 2
-)
-
-// Enum value maps for PayloadType.
-var (
-	PayloadType_name = map[int32]string{
-		0: "COMPRESSABLE",
-		1: "UNCOMPRESSABLE",
-		2: "RANDOM",
-	}
-	PayloadType_value = map[string]int32{
-		"COMPRESSABLE":   0,
-		"UNCOMPRESSABLE": 1,
-		"RANDOM":         2,
-	}
-)
-
-func (x PayloadType) Enum() *PayloadType {
-	p := new(PayloadType)
-	*p = x
-	return p
-}
-
-func (x PayloadType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (PayloadType) Descriptor() protoreflect.EnumDescriptor {
-	return file_benchmark_grpc_testing_messages_proto_enumTypes[0].Descriptor()
-}
-
-func (PayloadType) Type() protoreflect.EnumType {
-	return &file_benchmark_grpc_testing_messages_proto_enumTypes[0]
-}
-
-func (x PayloadType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use PayloadType.Descriptor instead.
-func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{0}
-}
-
-// Compression algorithms
-type CompressionType int32
-
-const (
-	// No compression
-	CompressionType_NONE    CompressionType = 0
-	CompressionType_GZIP    CompressionType = 1
-	CompressionType_DEFLATE CompressionType = 2
-)
-
-// Enum value maps for CompressionType.
-var (
-	CompressionType_name = map[int32]string{
-		0: "NONE",
-		1: "GZIP",
-		2: "DEFLATE",
-	}
-	CompressionType_value = map[string]int32{
-		"NONE":    0,
-		"GZIP":    1,
-		"DEFLATE": 2,
-	}
-)
-
-func (x CompressionType) Enum() *CompressionType {
-	p := new(CompressionType)
-	*p = x
-	return p
-}
-
-func (x CompressionType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (CompressionType) Descriptor() protoreflect.EnumDescriptor {
-	return file_benchmark_grpc_testing_messages_proto_enumTypes[1].Descriptor()
-}
-
-func (CompressionType) Type() protoreflect.EnumType {
-	return &file_benchmark_grpc_testing_messages_proto_enumTypes[1]
-}
-
-func (x CompressionType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use CompressionType.Descriptor instead.
-func (CompressionType) EnumDescriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{1}
-}
-
-// A block of data, to simply increase gRPC message size.
-type Payload struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The type of data in body.
-	Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"`
-	// Primary contents of payload.
-	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
-}
-
-func (x *Payload) Reset() {
-	*x = Payload{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Payload) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Payload) ProtoMessage() {}
-
-func (x *Payload) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Payload.ProtoReflect.Descriptor instead.
-func (*Payload) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *Payload) GetType() PayloadType {
-	if x != nil {
-		return x.Type
-	}
-	return PayloadType_COMPRESSABLE
-}
-
-func (x *Payload) GetBody() []byte {
-	if x != nil {
-		return x.Body
-	}
-	return nil
-}
-
-// A protobuf representation for grpc status. This is used by test
-// clients to specify a status that the server should attempt to return.
-type EchoStatus struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Code    int32  `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
-	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-}
-
-func (x *EchoStatus) Reset() {
-	*x = EchoStatus{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *EchoStatus) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*EchoStatus) ProtoMessage() {}
-
-func (x *EchoStatus) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead.
-func (*EchoStatus) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *EchoStatus) GetCode() int32 {
-	if x != nil {
-		return x.Code
-	}
-	return 0
-}
-
-func (x *EchoStatus) GetMessage() string {
-	if x != nil {
-		return x.Message
-	}
-	return ""
-}
-
-// Unary request.
-type SimpleRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Desired payload type in the response from the server.
-	// If response_type is RANDOM, server randomly chooses one from other formats.
-	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
-	// Desired payload size in the response from the server.
-	// If response_type is COMPRESSABLE, this denotes the size before compression.
-	ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"`
-	// Optional input payload sent along with the request.
-	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
-	// Whether SimpleResponse should include username.
-	FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"`
-	// Whether SimpleResponse should include OAuth scope.
-	FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"`
-	// Compression algorithm to be used by the server for the response (stream)
-	ResponseCompression CompressionType `protobuf:"varint,6,opt,name=response_compression,json=responseCompression,proto3,enum=grpc.testing.CompressionType" json:"response_compression,omitempty"`
-	// Whether server should return a given status
-	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-}
-
-func (x *SimpleRequest) Reset() {
-	*x = SimpleRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SimpleRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SimpleRequest) ProtoMessage() {}
-
-func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *SimpleRequest) GetResponseType() PayloadType {
-	if x != nil {
-		return x.ResponseType
-	}
-	return PayloadType_COMPRESSABLE
-}
-
-func (x *SimpleRequest) GetResponseSize() int32 {
-	if x != nil {
-		return x.ResponseSize
-	}
-	return 0
-}
-
-func (x *SimpleRequest) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-func (x *SimpleRequest) GetFillUsername() bool {
-	if x != nil {
-		return x.FillUsername
-	}
-	return false
-}
-
-func (x *SimpleRequest) GetFillOauthScope() bool {
-	if x != nil {
-		return x.FillOauthScope
-	}
-	return false
-}
-
-func (x *SimpleRequest) GetResponseCompression() CompressionType {
-	if x != nil {
-		return x.ResponseCompression
-	}
-	return CompressionType_NONE
-}
-
-func (x *SimpleRequest) GetResponseStatus() *EchoStatus {
-	if x != nil {
-		return x.ResponseStatus
-	}
-	return nil
-}
-
-// Unary response, as configured by the request.
-type SimpleResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Payload to increase message size.
-	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	// The user the request came from, for verifying authentication was
-	// successful when the client expected it.
-	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
-	// OAuth scope.
-	OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
-}
-
-func (x *SimpleResponse) Reset() {
-	*x = SimpleResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SimpleResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SimpleResponse) ProtoMessage() {}
-
-func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{3}
-}
-
-func (x *SimpleResponse) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-func (x *SimpleResponse) GetUsername() string {
-	if x != nil {
-		return x.Username
-	}
-	return ""
-}
-
-func (x *SimpleResponse) GetOauthScope() string {
-	if x != nil {
-		return x.OauthScope
-	}
-	return ""
-}
-
-// Client-streaming request.
-type StreamingInputCallRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Optional input payload sent along with the request.
-	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-}
-
-func (x *StreamingInputCallRequest) Reset() {
-	*x = StreamingInputCallRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[4]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingInputCallRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingInputCallRequest) ProtoMessage() {}
-
-func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[4]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead.
-func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{4}
-}
-
-func (x *StreamingInputCallRequest) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-// Client-streaming response.
-type StreamingInputCallResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Aggregated size of payloads received from the client.
-	AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
-}
-
-func (x *StreamingInputCallResponse) Reset() {
-	*x = StreamingInputCallResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[5]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingInputCallResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingInputCallResponse) ProtoMessage() {}
-
-func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[5]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead.
-func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{5}
-}
-
-func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
-	if x != nil {
-		return x.AggregatedPayloadSize
-	}
-	return 0
-}
-
-// Configuration for a particular response.
-type ResponseParameters struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Desired payload sizes in responses from the server.
-	// If response_type is COMPRESSABLE, this denotes the size before compression.
-	Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"`
-	// Desired interval between consecutive responses in the response stream in
-	// microseconds.
-	IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
-}
-
-func (x *ResponseParameters) Reset() {
-	*x = ResponseParameters{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[6]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ResponseParameters) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ResponseParameters) ProtoMessage() {}
-
-func (x *ResponseParameters) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[6]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead.
-func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{6}
-}
-
-func (x *ResponseParameters) GetSize() int32 {
-	if x != nil {
-		return x.Size
-	}
-	return 0
-}
-
-func (x *ResponseParameters) GetIntervalUs() int32 {
-	if x != nil {
-		return x.IntervalUs
-	}
-	return 0
-}
-
-// Server-streaming request.
-type StreamingOutputCallRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Desired payload type in the response from the server.
-	// If response_type is RANDOM, the payload from each response in the stream
-	// might be of different types. This is to simulate a mixed type of payload
-	// stream.
-	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
-	// Configuration for each expected response message.
-	ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"`
-	// Optional input payload sent along with the request.
-	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
-	// Compression algorithm to be used by the server for the response (stream)
-	ResponseCompression CompressionType `protobuf:"varint,6,opt,name=response_compression,json=responseCompression,proto3,enum=grpc.testing.CompressionType" json:"response_compression,omitempty"`
-	// Whether server should return a given status
-	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-}
-
-func (x *StreamingOutputCallRequest) Reset() {
-	*x = StreamingOutputCallRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[7]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingOutputCallRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingOutputCallRequest) ProtoMessage() {}
-
-func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[7]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead.
-func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{7}
-}
-
-func (x *StreamingOutputCallRequest) GetResponseType() PayloadType {
-	if x != nil {
-		return x.ResponseType
-	}
-	return PayloadType_COMPRESSABLE
-}
-
-func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
-	if x != nil {
-		return x.ResponseParameters
-	}
-	return nil
-}
-
-func (x *StreamingOutputCallRequest) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-func (x *StreamingOutputCallRequest) GetResponseCompression() CompressionType {
-	if x != nil {
-		return x.ResponseCompression
-	}
-	return CompressionType_NONE
-}
-
-func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
-	if x != nil {
-		return x.ResponseStatus
-	}
-	return nil
-}
-
-// Server-streaming response, as configured by the request and parameters.
-type StreamingOutputCallResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Payload to increase response size.
-	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-}
-
-func (x *StreamingOutputCallResponse) Reset() {
-	*x = StreamingOutputCallResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[8]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingOutputCallResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingOutputCallResponse) ProtoMessage() {}
-
-func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[8]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead.
-func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{8}
-}
-
-func (x *StreamingOutputCallResponse) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-// For reconnect interop test only.
-// Client tells server what reconnection parameters it used.
-type ReconnectParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	MaxReconnectBackoffMs int32 `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"`
-}
-
-func (x *ReconnectParams) Reset() {
-	*x = ReconnectParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[9]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ReconnectParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ReconnectParams) ProtoMessage() {}
-
-func (x *ReconnectParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[9]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ReconnectParams.ProtoReflect.Descriptor instead.
-func (*ReconnectParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{9}
-}
-
-func (x *ReconnectParams) GetMaxReconnectBackoffMs() int32 {
-	if x != nil {
-		return x.MaxReconnectBackoffMs
-	}
-	return 0
-}
-
-// For reconnect interop test only.
-// Server tells client whether its reconnects are following the spec and the
-// reconnect backoffs it saw.
-type ReconnectInfo struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Passed    bool    `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"`
-	BackoffMs []int32 `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"`
-}
-
-func (x *ReconnectInfo) Reset() {
-	*x = ReconnectInfo{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[10]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ReconnectInfo) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ReconnectInfo) ProtoMessage() {}
-
-func (x *ReconnectInfo) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_messages_proto_msgTypes[10]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ReconnectInfo.ProtoReflect.Descriptor instead.
-func (*ReconnectInfo) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_messages_proto_rawDescGZIP(), []int{10}
-}
-
-func (x *ReconnectInfo) GetPassed() bool {
-	if x != nil {
-		return x.Passed
-	}
-	return false
-}
-
-func (x *ReconnectInfo) GetBackoffMs() []int32 {
-	if x != nil {
-		return x.BackoffMs
-	}
-	return nil
-}
-
-var File_benchmark_grpc_testing_messages_proto protoreflect.FileDescriptor
-
-var file_benchmark_grpc_testing_messages_proto_rawDesc = []byte{
-	0x0a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x4c, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
-	0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
-	0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62,
-	0x6f, 0x64, 0x79, 0x22, 0x3a, 0x0a, 0x0a, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22,
-	0x89, 0x03, 0x0a, 0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79,
-	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
-	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54,
-	0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
-	0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x69,
-	0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07,
-	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x6c, 0x5f,
-	0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c,
-	0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10,
-	0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65,
-	0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x4f, 0x61, 0x75, 0x74,
-	0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x50, 0x0a, 0x14, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06,
-	0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54,
-	0x79, 0x70, 0x65, 0x52, 0x13, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x6d,
-	0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x7e, 0x0a, 0x0e, 0x53,
-	0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a,
-	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a,
-	0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x61,
-	0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0a, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x4c, 0x0a, 0x19, 0x53,
-	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c,
-	0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c,
-	0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63,
-	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
-	0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74, 0x72,
-	0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65,
-	0x67, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x69,
-	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67,
-	0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x22,
-	0x49, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d,
-	0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x74,
-	0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
-	0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x22, 0xf5, 0x02, 0x0a, 0x1a, 0x53,
-	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61,
-	0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e,
-	0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
-	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73,
-	0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61,
-	0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x07,
-	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x50, 0x0a,
-	0x14, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65,
-	0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x72,
-	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x13, 0x72, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12,
-	0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74,
-	0x75, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
-	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74,
-	0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74,
-	0x75, 0x73, 0x22, 0x4e, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f,
-	0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
-	0x61, 0x64, 0x22, 0x4a, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x50,
-	0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x63,
-	0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d,
-	0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x63, 0x6f,
-	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x22, 0x46,
-	0x0a, 0x0d, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12,
-	0x16, 0x0a, 0x06, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x06, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x6f,
-	0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x63,
-	0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x2a, 0x3f, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53,
-	0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d,
-	0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52,
-	0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10, 0x02, 0x2a, 0x32, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x72,
-	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f,
-	0x4e, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x47, 0x5a, 0x49, 0x50, 0x10, 0x01, 0x12, 0x0b,
-	0x0a, 0x07, 0x44, 0x45, 0x46, 0x4c, 0x41, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2f, 0x5a, 0x2d, 0x67,
-	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
-	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f,
-	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x33,
-}
-
-var (
-	file_benchmark_grpc_testing_messages_proto_rawDescOnce sync.Once
-	file_benchmark_grpc_testing_messages_proto_rawDescData = file_benchmark_grpc_testing_messages_proto_rawDesc
-)
-
-func file_benchmark_grpc_testing_messages_proto_rawDescGZIP() []byte {
-	file_benchmark_grpc_testing_messages_proto_rawDescOnce.Do(func() {
-		file_benchmark_grpc_testing_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_messages_proto_rawDescData)
-	})
-	return file_benchmark_grpc_testing_messages_proto_rawDescData
-}
-
-var file_benchmark_grpc_testing_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_benchmark_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
-var file_benchmark_grpc_testing_messages_proto_goTypes = []interface{}{
-	(PayloadType)(0),                    // 0: grpc.testing.PayloadType
-	(CompressionType)(0),                // 1: grpc.testing.CompressionType
-	(*Payload)(nil),                     // 2: grpc.testing.Payload
-	(*EchoStatus)(nil),                  // 3: grpc.testing.EchoStatus
-	(*SimpleRequest)(nil),               // 4: grpc.testing.SimpleRequest
-	(*SimpleResponse)(nil),              // 5: grpc.testing.SimpleResponse
-	(*StreamingInputCallRequest)(nil),   // 6: grpc.testing.StreamingInputCallRequest
-	(*StreamingInputCallResponse)(nil),  // 7: grpc.testing.StreamingInputCallResponse
-	(*ResponseParameters)(nil),          // 8: grpc.testing.ResponseParameters
-	(*StreamingOutputCallRequest)(nil),  // 9: grpc.testing.StreamingOutputCallRequest
-	(*StreamingOutputCallResponse)(nil), // 10: grpc.testing.StreamingOutputCallResponse
-	(*ReconnectParams)(nil),             // 11: grpc.testing.ReconnectParams
-	(*ReconnectInfo)(nil),               // 12: grpc.testing.ReconnectInfo
-}
-var file_benchmark_grpc_testing_messages_proto_depIdxs = []int32{
-	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
-	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
-	2,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
-	1,  // 3: grpc.testing.SimpleRequest.response_compression:type_name -> grpc.testing.CompressionType
-	3,  // 4: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
-	2,  // 5: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
-	2,  // 6: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
-	0,  // 7: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
-	8,  // 8: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
-	2,  // 9: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
-	1,  // 10: grpc.testing.StreamingOutputCallRequest.response_compression:type_name -> grpc.testing.CompressionType
-	3,  // 11: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
-	2,  // 12: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
-	13, // [13:13] is the sub-list for method output_type
-	13, // [13:13] is the sub-list for method input_type
-	13, // [13:13] is the sub-list for extension type_name
-	13, // [13:13] is the sub-list for extension extendee
-	0,  // [0:13] is the sub-list for field type_name
-}
-
-func init() { file_benchmark_grpc_testing_messages_proto_init() }
-func file_benchmark_grpc_testing_messages_proto_init() {
-	if File_benchmark_grpc_testing_messages_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_benchmark_grpc_testing_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Payload); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*EchoStatus); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SimpleRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SimpleResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingInputCallRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingInputCallResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ResponseParameters); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingOutputCallRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingOutputCallResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ReconnectParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_messages_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ReconnectInfo); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_benchmark_grpc_testing_messages_proto_rawDesc,
-			NumEnums:      2,
-			NumMessages:   11,
-			NumExtensions: 0,
-			NumServices:   0,
-		},
-		GoTypes:           file_benchmark_grpc_testing_messages_proto_goTypes,
-		DependencyIndexes: file_benchmark_grpc_testing_messages_proto_depIdxs,
-		EnumInfos:         file_benchmark_grpc_testing_messages_proto_enumTypes,
-		MessageInfos:      file_benchmark_grpc_testing_messages_proto_msgTypes,
-	}.Build()
-	File_benchmark_grpc_testing_messages_proto = out.File
-	file_benchmark_grpc_testing_messages_proto_rawDesc = nil
-	file_benchmark_grpc_testing_messages_proto_goTypes = nil
-	file_benchmark_grpc_testing_messages_proto_depIdxs = nil
-}
diff --git a/benchmark/grpc_testing/messages.proto b/benchmark/grpc_testing/messages.proto
deleted file mode 100644
index c48cdae9a29e..000000000000
--- a/benchmark/grpc_testing/messages.proto
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Message definitions to be used by integration test service definitions.
-
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
-
-package grpc.testing;
-
-// The type of payload that should be returned.
-enum PayloadType {
-  // Compressable text format.
-  COMPRESSABLE = 0;
-
-  // Uncompressable binary format.
-  UNCOMPRESSABLE = 1;
-
-  // Randomly chosen from all other formats defined in this enum.
-  RANDOM = 2;
-}
-
-// Compression algorithms
-enum CompressionType {
-  // No compression
-  NONE = 0;
-  GZIP = 1;
-  DEFLATE = 2;
-}
-
-// A block of data, to simply increase gRPC message size.
-message Payload {
-  // The type of data in body.
-  PayloadType type = 1;
-  // Primary contents of payload.
-  bytes body = 2;
-}
-
-// A protobuf representation for grpc status. This is used by test
-// clients to specify a status that the server should attempt to return.
-message EchoStatus {
-  int32 code = 1;
-  string message = 2;
-}
-
-// Unary request.
-message SimpleRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, server randomly chooses one from other formats.
-  PayloadType response_type = 1;
-
-  // Desired payload size in the response from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  int32 response_size = 2;
-
-  // Optional input payload sent along with the request.
-  Payload payload = 3;
-
-  // Whether SimpleResponse should include username.
-  bool fill_username = 4;
-
-  // Whether SimpleResponse should include OAuth scope.
-  bool fill_oauth_scope = 5;
-
-  // Compression algorithm to be used by the server for the response (stream)
-  CompressionType response_compression = 6;
-
-  // Whether server should return a given status
-  EchoStatus response_status = 7;
-}
-
-// Unary response, as configured by the request.
-message SimpleResponse {
-  // Payload to increase message size.
-  Payload payload = 1;
-  // The user the request came from, for verifying authentication was
-  // successful when the client expected it.
-  string username = 2;
-  // OAuth scope.
-  string oauth_scope = 3;
-}
-
-// Client-streaming request.
-message StreamingInputCallRequest {
-  // Optional input payload sent along with the request.
-  Payload payload = 1;
-
-  // Not expecting any payload from the response.
-}
-
-// Client-streaming response.
-message StreamingInputCallResponse {
-  // Aggregated size of payloads received from the client.
-  int32 aggregated_payload_size = 1;
-}
-
-// Configuration for a particular response.
-message ResponseParameters {
-  // Desired payload sizes in responses from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  int32 size = 1;
-
-  // Desired interval between consecutive responses in the response stream in
-  // microseconds.
-  int32 interval_us = 2;
-}
-
-// Server-streaming request.
-message StreamingOutputCallRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, the payload from each response in the stream
-  // might be of different types. This is to simulate a mixed type of payload
-  // stream.
-  PayloadType response_type = 1;
-
-  // Configuration for each expected response message.
-  repeated ResponseParameters response_parameters = 2;
-
-  // Optional input payload sent along with the request.
-  Payload payload = 3;
-
-  // Compression algorithm to be used by the server for the response (stream)
-  CompressionType response_compression = 6;
-
-  // Whether server should return a given status
-  EchoStatus response_status = 7;
-}
-
-// Server-streaming response, as configured by the request and parameters.
-message StreamingOutputCallResponse {
-  // Payload to increase response size.
-  Payload payload = 1;
-}
-
-// For reconnect interop test only.
-// Client tells server what reconnection parameters it used.
-message ReconnectParams {
-  int32 max_reconnect_backoff_ms = 1;
-}
-
-// For reconnect interop test only.
-// Server tells client whether its reconnects are following the spec and the
-// reconnect backoffs it saw.
-message ReconnectInfo {
-  bool passed = 1;
-  repeated int32 backoff_ms = 2;
-}
diff --git a/benchmark/grpc_testing/payloads.proto b/benchmark/grpc_testing/payloads.proto
deleted file mode 100644
index 862fb71bc135..000000000000
--- a/benchmark/grpc_testing/payloads.proto
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
-
-package grpc.testing;
-
-message ByteBufferParams {
-  int32 req_size = 1;
-  int32 resp_size = 2;
-}
-
-message SimpleProtoParams {
-  int32 req_size = 1;
-  int32 resp_size = 2;
-}
-
-message ComplexProtoParams {
-  // TODO (vpai): Fill this in once the details of complex, representative
-  //              protos are decided
-}
-
-message PayloadConfig {
-  oneof payload {
-    ByteBufferParams bytebuf_params = 1;
-    SimpleProtoParams simple_params = 2;
-    ComplexProtoParams complex_params = 3;
-  }
-}
diff --git a/benchmark/grpc_testing/services.pb.go b/benchmark/grpc_testing/services.pb.go
deleted file mode 100644
index 17d33e05678b..000000000000
--- a/benchmark/grpc_testing/services.pb.go
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// An integration test service that covers all the method signature permutations
-// of unary/streaming requests/responses.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: benchmark/grpc_testing/services.proto
-
-package grpc_testing
-
-import (
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-var File_benchmark_grpc_testing_services_proto protoreflect.FileDescriptor
-
-var file_benchmark_grpc_testing_services_proto_rawDesc = []byte{
-	0x0a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
-	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b,
-	0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x24, 0x62, 0x65,
-	0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x32, 0x87, 0x02, 0x0a, 0x10, 0x42, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b,
-	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79,
-	0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
-	0x4e, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x6c, 0x6c,
-	0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
-	0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
-	0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12,
-	0x5b, 0x0a, 0x1a, 0x55, 0x6e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x64,
-	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
-	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x32, 0x97, 0x02, 0x0a,
-	0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45,
-	0x0a, 0x09, 0x52, 0x75, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65,
-	0x72, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x45, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x69, 0x65,
-	0x6e, 0x74, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, 0x67,
-	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65,
-	0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x09,
-	0x43, 0x6f, 0x72, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63,
-	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x12, 0x34, 0x0a, 0x0a, 0x51, 0x75, 0x69, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x12,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x56, 0x6f,
-	0x69, 0x64, 0x1a, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f,
-	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
-
-var file_benchmark_grpc_testing_services_proto_goTypes = []interface{}{
-	(*SimpleRequest)(nil),  // 0: grpc.testing.SimpleRequest
-	(*ServerArgs)(nil),     // 1: grpc.testing.ServerArgs
-	(*ClientArgs)(nil),     // 2: grpc.testing.ClientArgs
-	(*CoreRequest)(nil),    // 3: grpc.testing.CoreRequest
-	(*Void)(nil),           // 4: grpc.testing.Void
-	(*SimpleResponse)(nil), // 5: grpc.testing.SimpleResponse
-	(*ServerStatus)(nil),   // 6: grpc.testing.ServerStatus
-	(*ClientStatus)(nil),   // 7: grpc.testing.ClientStatus
-	(*CoreResponse)(nil),   // 8: grpc.testing.CoreResponse
-}
-var file_benchmark_grpc_testing_services_proto_depIdxs = []int32{
-	0, // 0: grpc.testing.BenchmarkService.UnaryCall:input_type -> grpc.testing.SimpleRequest
-	0, // 1: grpc.testing.BenchmarkService.StreamingCall:input_type -> grpc.testing.SimpleRequest
-	0, // 2: grpc.testing.BenchmarkService.UnconstrainedStreamingCall:input_type -> grpc.testing.SimpleRequest
-	1, // 3: grpc.testing.WorkerService.RunServer:input_type -> grpc.testing.ServerArgs
-	2, // 4: grpc.testing.WorkerService.RunClient:input_type -> grpc.testing.ClientArgs
-	3, // 5: grpc.testing.WorkerService.CoreCount:input_type -> grpc.testing.CoreRequest
-	4, // 6: grpc.testing.WorkerService.QuitWorker:input_type -> grpc.testing.Void
-	5, // 7: grpc.testing.BenchmarkService.UnaryCall:output_type -> grpc.testing.SimpleResponse
-	5, // 8: grpc.testing.BenchmarkService.StreamingCall:output_type -> grpc.testing.SimpleResponse
-	5, // 9: grpc.testing.BenchmarkService.UnconstrainedStreamingCall:output_type -> grpc.testing.SimpleResponse
-	6, // 10: grpc.testing.WorkerService.RunServer:output_type -> grpc.testing.ServerStatus
-	7, // 11: grpc.testing.WorkerService.RunClient:output_type -> grpc.testing.ClientStatus
-	8, // 12: grpc.testing.WorkerService.CoreCount:output_type -> grpc.testing.CoreResponse
-	4, // 13: grpc.testing.WorkerService.QuitWorker:output_type -> grpc.testing.Void
-	7, // [7:14] is the sub-list for method output_type
-	0, // [0:7] is the sub-list for method input_type
-	0, // [0:0] is the sub-list for extension type_name
-	0, // [0:0] is the sub-list for extension extendee
-	0, // [0:0] is the sub-list for field type_name
-}
-
-func init() { file_benchmark_grpc_testing_services_proto_init() }
-func file_benchmark_grpc_testing_services_proto_init() {
-	if File_benchmark_grpc_testing_services_proto != nil {
-		return
-	}
-	file_benchmark_grpc_testing_messages_proto_init()
-	file_benchmark_grpc_testing_control_proto_init()
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_benchmark_grpc_testing_services_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   0,
-			NumExtensions: 0,
-			NumServices:   2,
-		},
-		GoTypes:           file_benchmark_grpc_testing_services_proto_goTypes,
-		DependencyIndexes: file_benchmark_grpc_testing_services_proto_depIdxs,
-	}.Build()
-	File_benchmark_grpc_testing_services_proto = out.File
-	file_benchmark_grpc_testing_services_proto_rawDesc = nil
-	file_benchmark_grpc_testing_services_proto_goTypes = nil
-	file_benchmark_grpc_testing_services_proto_depIdxs = nil
-}
diff --git a/benchmark/grpc_testing/services.proto b/benchmark/grpc_testing/services.proto
deleted file mode 100644
index 9028c9cfe113..000000000000
--- a/benchmark/grpc_testing/services.proto
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// An integration test service that covers all the method signature permutations
-// of unary/streaming requests/responses.
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
-
-import "benchmark/grpc_testing/messages.proto";
-import "benchmark/grpc_testing/control.proto";
-
-package grpc.testing;
-
-service BenchmarkService {
-  // One request followed by one response.
-  // The server returns the client payload as-is.
-  rpc UnaryCall(SimpleRequest) returns (SimpleResponse);
-
-  // One request followed by one response.
-  // The server returns the client payload as-is.
-  rpc StreamingCall(stream SimpleRequest) returns (stream SimpleResponse);
-  // Unconstrainted streaming.
-  // Both server and client keep sending & receiving simultaneously.
-  rpc UnconstrainedStreamingCall(stream SimpleRequest) returns (stream SimpleResponse);
-}
-
-service WorkerService {
-  // Start server with specified workload.
-  // First request sent specifies the ServerConfig followed by ServerStatus
-  // response. After that, a "Mark" can be sent anytime to request the latest
-  // stats. Closing the stream will initiate shutdown of the test server
-  // and once the shutdown has finished, the OK status is sent to terminate
-  // this RPC.
-  rpc RunServer(stream ServerArgs) returns (stream ServerStatus);
-
-  // Start client with specified workload.
-  // First request sent specifies the ClientConfig followed by ClientStatus
-  // response. After that, a "Mark" can be sent anytime to request the latest
-  // stats. Closing the stream will initiate shutdown of the test client
-  // and once the shutdown has finished, the OK status is sent to terminate
-  // this RPC.
-  rpc RunClient(stream ClientArgs) returns (stream ClientStatus);
-
-  // Just return the core count - unary call
-  rpc CoreCount(CoreRequest) returns (CoreResponse);
-
-  // Quit this worker
-  rpc QuitWorker(Void) returns (Void);
-}
diff --git a/benchmark/grpc_testing/stats.pb.go b/benchmark/grpc_testing/stats.pb.go
deleted file mode 100644
index 87fc9bf2d3bc..000000000000
--- a/benchmark/grpc_testing/stats.pb.go
+++ /dev/null
@@ -1,473 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: benchmark/grpc_testing/stats.proto
-
-package grpc_testing
-
-import (
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-type ServerStats struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// wall clock time change in seconds since last reset
-	TimeElapsed float64 `protobuf:"fixed64,1,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
-	// change in user time (in seconds) used by the server since last reset
-	TimeUser float64 `protobuf:"fixed64,2,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
-	// change in server time (in seconds) used by the server process and all
-	// threads since last reset
-	TimeSystem float64 `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
-}
-
-func (x *ServerStats) Reset() {
-	*x = ServerStats{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ServerStats) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ServerStats) ProtoMessage() {}
-
-func (x *ServerStats) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ServerStats.ProtoReflect.Descriptor instead.
-func (*ServerStats) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *ServerStats) GetTimeElapsed() float64 {
-	if x != nil {
-		return x.TimeElapsed
-	}
-	return 0
-}
-
-func (x *ServerStats) GetTimeUser() float64 {
-	if x != nil {
-		return x.TimeUser
-	}
-	return 0
-}
-
-func (x *ServerStats) GetTimeSystem() float64 {
-	if x != nil {
-		return x.TimeSystem
-	}
-	return 0
-}
-
-// Histogram params based on grpc/support/histogram.c
-type HistogramParams struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Resolution  float64 `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"`                      // first bucket is [0, 1 + resolution)
-	MaxPossible float64 `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"` // use enough buckets to allow this value
-}
-
-func (x *HistogramParams) Reset() {
-	*x = HistogramParams{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *HistogramParams) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*HistogramParams) ProtoMessage() {}
-
-func (x *HistogramParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use HistogramParams.ProtoReflect.Descriptor instead.
-func (*HistogramParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *HistogramParams) GetResolution() float64 {
-	if x != nil {
-		return x.Resolution
-	}
-	return 0
-}
-
-func (x *HistogramParams) GetMaxPossible() float64 {
-	if x != nil {
-		return x.MaxPossible
-	}
-	return 0
-}
-
-// Histogram data based on grpc/support/histogram.c
-type HistogramData struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Bucket       []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"`
-	MinSeen      float64  `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"`
-	MaxSeen      float64  `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"`
-	Sum          float64  `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"`
-	SumOfSquares float64  `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"`
-	Count        float64  `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"`
-}
-
-func (x *HistogramData) Reset() {
-	*x = HistogramData{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *HistogramData) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*HistogramData) ProtoMessage() {}
-
-func (x *HistogramData) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use HistogramData.ProtoReflect.Descriptor instead.
-func (*HistogramData) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *HistogramData) GetBucket() []uint32 {
-	if x != nil {
-		return x.Bucket
-	}
-	return nil
-}
-
-func (x *HistogramData) GetMinSeen() float64 {
-	if x != nil {
-		return x.MinSeen
-	}
-	return 0
-}
-
-func (x *HistogramData) GetMaxSeen() float64 {
-	if x != nil {
-		return x.MaxSeen
-	}
-	return 0
-}
-
-func (x *HistogramData) GetSum() float64 {
-	if x != nil {
-		return x.Sum
-	}
-	return 0
-}
-
-func (x *HistogramData) GetSumOfSquares() float64 {
-	if x != nil {
-		return x.SumOfSquares
-	}
-	return 0
-}
-
-func (x *HistogramData) GetCount() float64 {
-	if x != nil {
-		return x.Count
-	}
-	return 0
-}
-
-type ClientStats struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Latency histogram. Data points are in nanoseconds.
-	Latencies *HistogramData `protobuf:"bytes,1,opt,name=latencies,proto3" json:"latencies,omitempty"`
-	// See ServerStats for details.
-	TimeElapsed float64 `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
-	TimeUser    float64 `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
-	TimeSystem  float64 `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
-}
-
-func (x *ClientStats) Reset() {
-	*x = ClientStats{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientStats) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientStats) ProtoMessage() {}
-
-func (x *ClientStats) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_stats_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead.
-func (*ClientStats) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_stats_proto_rawDescGZIP(), []int{3}
-}
-
-func (x *ClientStats) GetLatencies() *HistogramData {
-	if x != nil {
-		return x.Latencies
-	}
-	return nil
-}
-
-func (x *ClientStats) GetTimeElapsed() float64 {
-	if x != nil {
-		return x.TimeElapsed
-	}
-	return 0
-}
-
-func (x *ClientStats) GetTimeUser() float64 {
-	if x != nil {
-		return x.TimeUser
-	}
-	return 0
-}
-
-func (x *ClientStats) GetTimeSystem() float64 {
-	if x != nil {
-		return x.TimeSystem
-	}
-	return 0
-}
-
-var File_benchmark_grpc_testing_stats_proto protoreflect.FileDescriptor
-
-var file_benchmark_grpc_testing_stats_proto_rawDesc = []byte{
-	0x0a, 0x22, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x22, 0x6e, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74,
-	0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65,
-	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61,
-	0x70, 0x73, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65,
-	0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65,
-	0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74,
-	0x65, 0x6d, 0x22, 0x54, 0x0a, 0x0f, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x50,
-	0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74,
-	0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c,
-	0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x73,
-	0x73, 0x69, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6d, 0x61, 0x78,
-	0x50, 0x6f, 0x73, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x22, 0xab, 0x01, 0x0a, 0x0d, 0x48, 0x69, 0x73,
-	0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75,
-	0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b,
-	0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x6d, 0x69, 0x6e, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x19, 0x0a,
-	0x08, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52,
-	0x07, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18,
-	0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x75,
-	0x6d, 0x5f, 0x6f, 0x66, 0x5f, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01,
-	0x28, 0x01, 0x52, 0x0c, 0x73, 0x75, 0x6d, 0x4f, 0x66, 0x53, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73,
-	0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52,
-	0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xa9, 0x01, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63,
-	0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63,
-	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72,
-	0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65,
-	0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65,
-	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61,
-	0x70, 0x73, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65,
-	0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65,
-	0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
-	0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74,
-	0x65, 0x6d, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c,
-	0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e,
-	0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
-
-var (
-	file_benchmark_grpc_testing_stats_proto_rawDescOnce sync.Once
-	file_benchmark_grpc_testing_stats_proto_rawDescData = file_benchmark_grpc_testing_stats_proto_rawDesc
-)
-
-func file_benchmark_grpc_testing_stats_proto_rawDescGZIP() []byte {
-	file_benchmark_grpc_testing_stats_proto_rawDescOnce.Do(func() {
-		file_benchmark_grpc_testing_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_stats_proto_rawDescData)
-	})
-	return file_benchmark_grpc_testing_stats_proto_rawDescData
-}
-
-var file_benchmark_grpc_testing_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
-var file_benchmark_grpc_testing_stats_proto_goTypes = []interface{}{
-	(*ServerStats)(nil),     // 0: grpc.testing.ServerStats
-	(*HistogramParams)(nil), // 1: grpc.testing.HistogramParams
-	(*HistogramData)(nil),   // 2: grpc.testing.HistogramData
-	(*ClientStats)(nil),     // 3: grpc.testing.ClientStats
-}
-var file_benchmark_grpc_testing_stats_proto_depIdxs = []int32{
-	2, // 0: grpc.testing.ClientStats.latencies:type_name -> grpc.testing.HistogramData
-	1, // [1:1] is the sub-list for method output_type
-	1, // [1:1] is the sub-list for method input_type
-	1, // [1:1] is the sub-list for extension type_name
-	1, // [1:1] is the sub-list for extension extendee
-	0, // [0:1] is the sub-list for field type_name
-}
-
-func init() { file_benchmark_grpc_testing_stats_proto_init() }
-func file_benchmark_grpc_testing_stats_proto_init() {
-	if File_benchmark_grpc_testing_stats_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_benchmark_grpc_testing_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ServerStats); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*HistogramParams); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*HistogramData); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_benchmark_grpc_testing_stats_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientStats); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_benchmark_grpc_testing_stats_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   4,
-			NumExtensions: 0,
-			NumServices:   0,
-		},
-		GoTypes:           file_benchmark_grpc_testing_stats_proto_goTypes,
-		DependencyIndexes: file_benchmark_grpc_testing_stats_proto_depIdxs,
-		MessageInfos:      file_benchmark_grpc_testing_stats_proto_msgTypes,
-	}.Build()
-	File_benchmark_grpc_testing_stats_proto = out.File
-	file_benchmark_grpc_testing_stats_proto_rawDesc = nil
-	file_benchmark_grpc_testing_stats_proto_goTypes = nil
-	file_benchmark_grpc_testing_stats_proto_depIdxs = nil
-}
diff --git a/benchmark/grpc_testing/stats.proto b/benchmark/grpc_testing/stats.proto
deleted file mode 100644
index 1517e7f6d2ef..000000000000
--- a/benchmark/grpc_testing/stats.proto
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright 2016 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/benchmark/grpc_testing";
-
-package grpc.testing;
-
-message ServerStats {
-  // wall clock time change in seconds since last reset
-  double time_elapsed = 1;
-
-  // change in user time (in seconds) used by the server since last reset
-  double time_user = 2;
-
-  // change in server time (in seconds) used by the server process and all
-  // threads since last reset
-  double time_system = 3;
-}
-
-// Histogram params based on grpc/support/histogram.c
-message HistogramParams {
-  double resolution = 1;    // first bucket is [0, 1 + resolution)
-  double max_possible = 2;  // use enough buckets to allow this value
-}
-
-// Histogram data based on grpc/support/histogram.c
-message HistogramData {
-  repeated uint32 bucket = 1;
-  double min_seen = 2;
-  double max_seen = 3;
-  double sum = 4;
-  double sum_of_squares = 5;
-  double count = 6;
-}
-
-message ClientStats {
-  // Latency histogram. Data points are in nanoseconds.
-  HistogramData latencies = 1;
-
-  // See ServerStats for details.
-  double time_elapsed = 2;
-  double time_user = 3;
-  double time_system = 4;
-}
diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go
index 9253c0b48737..f760c7c36acc 100644
--- a/benchmark/worker/benchmark_client.go
+++ b/benchmark/worker/benchmark_client.go
@@ -28,11 +28,11 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/benchmark"
-	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/benchmark/stats"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/syscall"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/testdata"
 )
diff --git a/benchmark/worker/benchmark_server.go b/benchmark/worker/benchmark_server.go
index 60d52d75da38..da6288c11de4 100644
--- a/benchmark/worker/benchmark_server.go
+++ b/benchmark/worker/benchmark_server.go
@@ -30,10 +30,10 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/benchmark"
-	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/syscall"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/testdata"
 )
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 634f09e658c5..4ecf997238d8 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -33,9 +33,9 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
-	testpb "google.golang.org/grpc/benchmark/grpc_testing"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
 )
 
diff --git a/binarylog/binarylog_end2end_test.go b/binarylog/binarylog_end2end_test.go
index 89685c65a26b..e06ffec9166e 100644
--- a/binarylog/binarylog_end2end_test.go
+++ b/binarylog/binarylog_end2end_test.go
@@ -35,8 +35,8 @@ import (
 	"google.golang.org/grpc/grpclog"
 	iblog "google.golang.org/grpc/internal/binarylog"
 	"google.golang.org/grpc/internal/grpctest"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
-	testpb "google.golang.org/grpc/stats/grpc_testing"
 	"google.golang.org/grpc/status"
 )
 
@@ -114,6 +114,17 @@ var (
 	globalRPCID uint64 // RPC id starts with 1, but we do ++ at the beginning of each test.
 )
 
+func idToPayload(id int32) *testpb.Payload {
+	return &testpb.Payload{Body: []byte{byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24)}}
+}
+
+func payloadToID(p *testpb.Payload) int32 {
+	if p == nil || len(p.Body) != 4 {
+		panic("invalid payload")
+	}
+	return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24
+}
+
 type testServer struct {
 	testpb.UnimplementedTestServiceServer
 	te *test
@@ -130,11 +141,11 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 		}
 	}
 
-	if in.Id == errorID {
-		return nil, fmt.Errorf("got error id: %v", in.Id)
+	if id := payloadToID(in.Payload); id == errorID {
+		return nil, fmt.Errorf("got error id: %v", id)
 	}
 
-	return &testpb.SimpleResponse{Id: in.Id}, nil
+	return &testpb.SimpleResponse{Payload: in.Payload}, nil
 }
 
 func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
@@ -155,17 +166,17 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ
 			return err
 		}
 
-		if in.Id == errorID {
-			return fmt.Errorf("got error id: %v", in.Id)
+		if id := payloadToID(in.Payload); id == errorID {
+			return fmt.Errorf("got error id: %v", id)
 		}
 
-		if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil {
+		if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil {
 			return err
 		}
 	}
 }
 
-func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCallServer) error {
+func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
 	md, ok := metadata.FromIncomingContext(stream.Context())
 	if ok {
 		if err := stream.SendHeader(md); err != nil {
@@ -177,19 +188,19 @@ func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCall
 		in, err := stream.Recv()
 		if err == io.EOF {
 			// read done.
-			return stream.SendAndClose(&testpb.SimpleResponse{Id: int32(0)})
+			return stream.SendAndClose(&testpb.StreamingInputCallResponse{AggregatedPayloadSize: 0})
 		}
 		if err != nil {
 			return err
 		}
 
-		if in.Id == errorID {
-			return fmt.Errorf("got error id: %v", in.Id)
+		if id := payloadToID(in.Payload); id == errorID {
+			return fmt.Errorf("got error id: %v", id)
 		}
 	}
 }
 
-func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.TestService_ServerStreamCallServer) error {
+func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
 	md, ok := metadata.FromIncomingContext(stream.Context())
 	if ok {
 		if err := stream.SendHeader(md); err != nil {
@@ -198,12 +209,12 @@ func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.Te
 		stream.SetTrailer(testTrailerMetadata)
 	}
 
-	if in.Id == errorID {
-		return fmt.Errorf("got error id: %v", in.Id)
+	if id := payloadToID(in.Payload); id == errorID {
+		return fmt.Errorf("got error id: %v", id)
 	}
 
 	for i := 0; i < 5; i++ {
-		if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil {
+		if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil {
 			return err
 		}
 	}
@@ -334,9 +345,9 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	if c.success {
-		req = &testpb.SimpleRequest{Id: errorID + 1}
+		req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)}
 	} else {
-		req = &testpb.SimpleRequest{Id: errorID}
+		req = &testpb.SimpleRequest{Payload: idToPayload(errorID)}
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -346,10 +357,10 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 	return req, resp, err
 }
 
-func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest, []*testpb.SimpleResponse, error) {
+func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []proto.Message, error) {
 	var (
-		reqs  []*testpb.SimpleRequest
-		resps []*testpb.SimpleResponse
+		reqs  []proto.Message
+		resps []proto.Message
 		err   error
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -372,14 +383,14 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest
 		startID = errorID
 	}
 	for i := 0; i < c.count; i++ {
-		req := &testpb.SimpleRequest{
-			Id: int32(i) + startID,
+		req := &testpb.StreamingOutputCallRequest{
+			Payload: idToPayload(int32(i) + startID),
 		}
 		reqs = append(reqs, req)
 		if err = stream.Send(req); err != nil {
 			return reqs, resps, err
 		}
-		var resp *testpb.SimpleResponse
+		var resp *testpb.StreamingOutputCallResponse
 		if resp, err = stream.Recv(); err != nil {
 			return reqs, resps, err
 		}
@@ -395,10 +406,10 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest
 	return reqs, resps, nil
 }
 
-func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *testpb.SimpleResponse, error) {
+func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, proto.Message, error) {
 	var (
-		reqs []*testpb.SimpleRequest
-		resp *testpb.SimpleResponse
+		reqs []proto.Message
+		resp *testpb.StreamingInputCallResponse
 		err  error
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -406,7 +417,7 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test
 	defer cancel()
 	ctx = metadata.NewOutgoingContext(ctx, testMetadata)
 
-	stream, err := tc.ClientStreamCall(ctx)
+	stream, err := tc.StreamingInputCall(ctx)
 	if err != nil {
 		return reqs, resp, err
 	}
@@ -415,8 +426,8 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test
 		startID = errorID
 	}
 	for i := 0; i < c.count; i++ {
-		req := &testpb.SimpleRequest{
-			Id: int32(i) + startID,
+		req := &testpb.StreamingInputCallRequest{
+			Payload: idToPayload(int32(i) + startID),
 		}
 		reqs = append(reqs, req)
 		if err = stream.Send(req); err != nil {
@@ -427,10 +438,10 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test
 	return reqs, resp, err
 }
 
-func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*testpb.SimpleResponse, error) {
+func (te *test) doServerStreamCall(c *rpcConfig) (proto.Message, []proto.Message, error) {
 	var (
-		req   *testpb.SimpleRequest
-		resps []*testpb.SimpleResponse
+		req   *testpb.StreamingOutputCallRequest
+		resps []proto.Message
 		err   error
 	)
 
@@ -443,13 +454,13 @@ func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*test
 	if !c.success {
 		startID = errorID
 	}
-	req = &testpb.SimpleRequest{Id: startID}
-	stream, err := tc.ServerStreamCall(ctx, req)
+	req = &testpb.StreamingOutputCallRequest{Payload: idToPayload(startID)}
+	stream, err := tc.StreamingOutputCall(ctx, req)
 	if err != nil {
 		return req, resps, err
 	}
 	for {
-		var resp *testpb.SimpleResponse
+		var resp *testpb.StreamingOutputCallResponse
 		resp, err := stream.Recv()
 		if err == io.EOF {
 			return req, resps, nil
@@ -465,8 +476,8 @@ type expectedData struct {
 	cc *rpcConfig
 
 	method    string
-	requests  []*testpb.SimpleRequest
-	responses []*testpb.SimpleResponse
+	requests  []proto.Message
+	responses []proto.Message
 	err       error
 }
 
@@ -534,7 +545,7 @@ func (ed *expectedData) newServerHeaderEntry(client bool, rpcID, inRPCID uint64)
 	}
 }
 
-func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64, msg *testpb.SimpleRequest) *pb.GrpcLogEntry {
+func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64, msg proto.Message) *pb.GrpcLogEntry {
 	logger := pb.GrpcLogEntry_LOGGER_CLIENT
 	if !client {
 		logger = pb.GrpcLogEntry_LOGGER_SERVER
@@ -558,7 +569,7 @@ func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64
 	}
 }
 
-func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64, msg *testpb.SimpleResponse) *pb.GrpcLogEntry {
+func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64, msg proto.Message) *pb.GrpcLogEntry {
 	logger := pb.GrpcLogEntry_LOGGER_CLIENT
 	if !client {
 		logger = pb.GrpcLogEntry_LOGGER_SERVER
@@ -795,20 +806,20 @@ func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData {
 	case unaryRPC:
 		expect.method = "/grpc.testing.TestService/UnaryCall"
 		req, resp, err := te.doUnaryCall(cc)
-		expect.requests = []*testpb.SimpleRequest{req}
-		expect.responses = []*testpb.SimpleResponse{resp}
+		expect.requests = []proto.Message{req}
+		expect.responses = []proto.Message{resp}
 		expect.err = err
 	case clientStreamRPC:
-		expect.method = "/grpc.testing.TestService/ClientStreamCall"
+		expect.method = "/grpc.testing.TestService/StreamingInputCall"
 		reqs, resp, err := te.doClientStreamCall(cc)
 		expect.requests = reqs
-		expect.responses = []*testpb.SimpleResponse{resp}
+		expect.responses = []proto.Message{resp}
 		expect.err = err
 	case serverStreamRPC:
-		expect.method = "/grpc.testing.TestService/ServerStreamCall"
+		expect.method = "/grpc.testing.TestService/StreamingOutputCall"
 		req, resps, err := te.doServerStreamCall(cc)
 		expect.responses = resps
-		expect.requests = []*testpb.SimpleRequest{req}
+		expect.requests = []proto.Message{req}
 		expect.err = err
 	case fullDuplexStreamRPC, cancelRPC:
 		expect.method = "/grpc.testing.TestService/FullDuplexCall"
diff --git a/interop/grpc_testing/benchmark_service.pb.go b/interop/grpc_testing/benchmark_service.pb.go
new file mode 100644
index 000000000000..a1057b8a7599
--- /dev/null
+++ b/interop/grpc_testing/benchmark_service.pb.go
@@ -0,0 +1,127 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/benchmark_service.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+var File_grpc_testing_benchmark_service_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_benchmark_service_proto_rawDesc = []byte{
+	0x0a, 0x24, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x62,
+	0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x32, 0xa6, 0x03, 0x0a, 0x10, 0x42, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43,
+	0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e,
+	0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x6c, 0x6c, 0x12,
+	0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70,
+	0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x52,
+	0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x46, 0x72, 0x6f, 0x6d, 0x43,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x28, 0x01, 0x12, 0x52, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x46,
+	0x72, 0x6f, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x52, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
+	0x69, 0x6e, 0x67, 0x42, 0x6f, 0x74, 0x68, 0x57, 0x61, 0x79, 0x73, 0x12, 0x1b, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x33,
+}
+
+var file_grpc_testing_benchmark_service_proto_goTypes = []interface{}{
+	(*SimpleRequest)(nil),  // 0: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil), // 1: grpc.testing.SimpleResponse
+}
+var file_grpc_testing_benchmark_service_proto_depIdxs = []int32{
+	0, // 0: grpc.testing.BenchmarkService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	0, // 1: grpc.testing.BenchmarkService.StreamingCall:input_type -> grpc.testing.SimpleRequest
+	0, // 2: grpc.testing.BenchmarkService.StreamingFromClient:input_type -> grpc.testing.SimpleRequest
+	0, // 3: grpc.testing.BenchmarkService.StreamingFromServer:input_type -> grpc.testing.SimpleRequest
+	0, // 4: grpc.testing.BenchmarkService.StreamingBothWays:input_type -> grpc.testing.SimpleRequest
+	1, // 5: grpc.testing.BenchmarkService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	1, // 6: grpc.testing.BenchmarkService.StreamingCall:output_type -> grpc.testing.SimpleResponse
+	1, // 7: grpc.testing.BenchmarkService.StreamingFromClient:output_type -> grpc.testing.SimpleResponse
+	1, // 8: grpc.testing.BenchmarkService.StreamingFromServer:output_type -> grpc.testing.SimpleResponse
+	1, // 9: grpc.testing.BenchmarkService.StreamingBothWays:output_type -> grpc.testing.SimpleResponse
+	5, // [5:10] is the sub-list for method output_type
+	0, // [0:5] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_benchmark_service_proto_init() }
+func file_grpc_testing_benchmark_service_proto_init() {
+	if File_grpc_testing_benchmark_service_proto != nil {
+		return
+	}
+	file_grpc_testing_messages_proto_init()
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_benchmark_service_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   0,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_testing_benchmark_service_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_benchmark_service_proto_depIdxs,
+	}.Build()
+	File_grpc_testing_benchmark_service_proto = out.File
+	file_grpc_testing_benchmark_service_proto_rawDesc = nil
+	file_grpc_testing_benchmark_service_proto_goTypes = nil
+	file_grpc_testing_benchmark_service_proto_depIdxs = nil
+}
diff --git a/interop/grpc_testing/benchmark_service_grpc.pb.go b/interop/grpc_testing/benchmark_service_grpc.pb.go
new file mode 100644
index 000000000000..b4badf5fe5d3
--- /dev/null
+++ b/interop/grpc_testing/benchmark_service_grpc.pb.go
@@ -0,0 +1,392 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion7
+
+// BenchmarkServiceClient is the client API for BenchmarkService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type BenchmarkServiceClient interface {
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
+	// Repeated sequence of one request followed by one response.
+	// Should be called streaming ping-pong
+	// The server returns the client payload as-is on each response
+	StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error)
+	// Single-sided unbounded streaming from client to server
+	// The server returns the client payload as-is once the client does WritesDone
+	StreamingFromClient(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingFromClientClient, error)
+	// Single-sided unbounded streaming from server to client
+	// The server repeatedly returns the client payload as-is
+	StreamingFromServer(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (BenchmarkService_StreamingFromServerClient, error)
+	// Two-sided unbounded streaming between server to client
+	// Both sides send the content of their own choice to the other
+	StreamingBothWays(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingBothWaysClient, error)
+}
+
+type benchmarkServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient {
+	return &benchmarkServiceClient{cc}
+}
+
+func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
+	out := new(SimpleResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
+	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &benchmarkServiceStreamingCallClient{stream}
+	return x, nil
+}
+
+type BenchmarkService_StreamingCallClient interface {
+	Send(*SimpleRequest) error
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type benchmarkServiceStreamingCallClient struct {
+	grpc.ClientStream
+}
+
+func (x *benchmarkServiceStreamingCallClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *benchmarkServiceClient) StreamingFromClient(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingFromClientClient, error) {
+	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[1], "/grpc.testing.BenchmarkService/StreamingFromClient", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &benchmarkServiceStreamingFromClientClient{stream}
+	return x, nil
+}
+
+type BenchmarkService_StreamingFromClientClient interface {
+	Send(*SimpleRequest) error
+	CloseAndRecv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type benchmarkServiceStreamingFromClientClient struct {
+	grpc.ClientStream
+}
+
+func (x *benchmarkServiceStreamingFromClientClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingFromClientClient) CloseAndRecv() (*SimpleResponse, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *benchmarkServiceClient) StreamingFromServer(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (BenchmarkService_StreamingFromServerClient, error) {
+	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[2], "/grpc.testing.BenchmarkService/StreamingFromServer", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &benchmarkServiceStreamingFromServerClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type BenchmarkService_StreamingFromServerClient interface {
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type benchmarkServiceStreamingFromServerClient struct {
+	grpc.ClientStream
+}
+
+func (x *benchmarkServiceStreamingFromServerClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *benchmarkServiceClient) StreamingBothWays(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingBothWaysClient, error) {
+	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[3], "/grpc.testing.BenchmarkService/StreamingBothWays", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &benchmarkServiceStreamingBothWaysClient{stream}
+	return x, nil
+}
+
+type BenchmarkService_StreamingBothWaysClient interface {
+	Send(*SimpleRequest) error
+	Recv() (*SimpleResponse, error)
+	grpc.ClientStream
+}
+
+type benchmarkServiceStreamingBothWaysClient struct {
+	grpc.ClientStream
+}
+
+func (x *benchmarkServiceStreamingBothWaysClient) Send(m *SimpleRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingBothWaysClient) Recv() (*SimpleResponse, error) {
+	m := new(SimpleResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// BenchmarkServiceServer is the server API for BenchmarkService service.
+// All implementations must embed UnimplementedBenchmarkServiceServer
+// for forward compatibility
+type BenchmarkServiceServer interface {
+	// One request followed by one response.
+	// The server returns the client payload as-is.
+	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// Repeated sequence of one request followed by one response.
+	// Should be called streaming ping-pong
+	// The server returns the client payload as-is on each response
+	StreamingCall(BenchmarkService_StreamingCallServer) error
+	// Single-sided unbounded streaming from client to server
+	// The server returns the client payload as-is once the client does WritesDone
+	StreamingFromClient(BenchmarkService_StreamingFromClientServer) error
+	// Single-sided unbounded streaming from server to client
+	// The server repeatedly returns the client payload as-is
+	StreamingFromServer(*SimpleRequest, BenchmarkService_StreamingFromServerServer) error
+	// Two-sided unbounded streaming between server to client
+	// Both sides send the content of their own choice to the other
+	StreamingBothWays(BenchmarkService_StreamingBothWaysServer) error
+	mustEmbedUnimplementedBenchmarkServiceServer()
+}
+
+// UnimplementedBenchmarkServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedBenchmarkServiceServer struct {
+}
+
+func (UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) StreamingCall(BenchmarkService_StreamingCallServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) StreamingFromClient(BenchmarkService_StreamingFromClientServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingFromClient not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) StreamingFromServer(*SimpleRequest, BenchmarkService_StreamingFromServerServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingFromServer not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) StreamingBothWays(BenchmarkService_StreamingBothWaysServer) error {
+	return status.Errorf(codes.Unimplemented, "method StreamingBothWays not implemented")
+}
+func (UnimplementedBenchmarkServiceServer) mustEmbedUnimplementedBenchmarkServiceServer() {}
+
+// UnsafeBenchmarkServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to BenchmarkServiceServer will
+// result in compilation errors.
+type UnsafeBenchmarkServiceServer interface {
+	mustEmbedUnimplementedBenchmarkServiceServer()
+}
+
+func RegisterBenchmarkServiceServer(s grpc.ServiceRegistrar, srv BenchmarkServiceServer) {
+	s.RegisterService(&BenchmarkService_ServiceDesc, srv)
+}
+
+func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SimpleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BenchmarkServiceServer).UnaryCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.BenchmarkService/UnaryCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream})
+}
+
+type BenchmarkService_StreamingCallServer interface {
+	Send(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type benchmarkServiceStreamingCallServer struct {
+	grpc.ServerStream
+}
+
+func (x *benchmarkServiceStreamingCallServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _BenchmarkService_StreamingFromClient_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).StreamingFromClient(&benchmarkServiceStreamingFromClientServer{stream})
+}
+
+type BenchmarkService_StreamingFromClientServer interface {
+	SendAndClose(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type benchmarkServiceStreamingFromClientServer struct {
+	grpc.ServerStream
+}
+
+func (x *benchmarkServiceStreamingFromClientServer) SendAndClose(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingFromClientServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _BenchmarkService_StreamingFromServer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(SimpleRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(BenchmarkServiceServer).StreamingFromServer(m, &benchmarkServiceStreamingFromServerServer{stream})
+}
+
+type BenchmarkService_StreamingFromServerServer interface {
+	Send(*SimpleResponse) error
+	grpc.ServerStream
+}
+
+type benchmarkServiceStreamingFromServerServer struct {
+	grpc.ServerStream
+}
+
+func (x *benchmarkServiceStreamingFromServerServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _BenchmarkService_StreamingBothWays_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BenchmarkServiceServer).StreamingBothWays(&benchmarkServiceStreamingBothWaysServer{stream})
+}
+
+type BenchmarkService_StreamingBothWaysServer interface {
+	Send(*SimpleResponse) error
+	Recv() (*SimpleRequest, error)
+	grpc.ServerStream
+}
+
+type benchmarkServiceStreamingBothWaysServer struct {
+	grpc.ServerStream
+}
+
+func (x *benchmarkServiceStreamingBothWaysServer) Send(m *SimpleResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *benchmarkServiceStreamingBothWaysServer) Recv() (*SimpleRequest, error) {
+	m := new(SimpleRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// BenchmarkService_ServiceDesc is the grpc.ServiceDesc for BenchmarkService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var BenchmarkService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.BenchmarkService",
+	HandlerType: (*BenchmarkServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "UnaryCall",
+			Handler:    _BenchmarkService_UnaryCall_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "StreamingCall",
+			Handler:       _BenchmarkService_StreamingCall_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "StreamingFromClient",
+			Handler:       _BenchmarkService_StreamingFromClient_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "StreamingFromServer",
+			Handler:       _BenchmarkService_StreamingFromServer_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "StreamingBothWays",
+			Handler:       _BenchmarkService_StreamingBothWays_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "grpc/testing/benchmark_service.proto",
+}
diff --git a/interop/grpc_testing/control.pb.go b/interop/grpc_testing/control.pb.go
new file mode 100644
index 000000000000..8db13921b77e
--- /dev/null
+++ b/interop/grpc_testing/control.pb.go
@@ -0,0 +1,2461 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/control.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+type ClientType int32
+
+const (
+	// Many languages support a basic distinction between using
+	// sync or async client, and this allows the specification
+	ClientType_SYNC_CLIENT     ClientType = 0
+	ClientType_ASYNC_CLIENT    ClientType = 1
+	ClientType_OTHER_CLIENT    ClientType = 2 // used for some language-specific variants
+	ClientType_CALLBACK_CLIENT ClientType = 3
+)
+
+// Enum value maps for ClientType.
+var (
+	ClientType_name = map[int32]string{
+		0: "SYNC_CLIENT",
+		1: "ASYNC_CLIENT",
+		2: "OTHER_CLIENT",
+		3: "CALLBACK_CLIENT",
+	}
+	ClientType_value = map[string]int32{
+		"SYNC_CLIENT":     0,
+		"ASYNC_CLIENT":    1,
+		"OTHER_CLIENT":    2,
+		"CALLBACK_CLIENT": 3,
+	}
+)
+
+func (x ClientType) Enum() *ClientType {
+	p := new(ClientType)
+	*p = x
+	return p
+}
+
+func (x ClientType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ClientType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_testing_control_proto_enumTypes[0].Descriptor()
+}
+
+func (ClientType) Type() protoreflect.EnumType {
+	return &file_grpc_testing_control_proto_enumTypes[0]
+}
+
+func (x ClientType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ClientType.Descriptor instead.
+func (ClientType) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{0}
+}
+
+type ServerType int32
+
+const (
+	ServerType_SYNC_SERVER          ServerType = 0
+	ServerType_ASYNC_SERVER         ServerType = 1
+	ServerType_ASYNC_GENERIC_SERVER ServerType = 2
+	ServerType_OTHER_SERVER         ServerType = 3 // used for some language-specific variants
+	ServerType_CALLBACK_SERVER      ServerType = 4
+)
+
+// Enum value maps for ServerType.
+var (
+	ServerType_name = map[int32]string{
+		0: "SYNC_SERVER",
+		1: "ASYNC_SERVER",
+		2: "ASYNC_GENERIC_SERVER",
+		3: "OTHER_SERVER",
+		4: "CALLBACK_SERVER",
+	}
+	ServerType_value = map[string]int32{
+		"SYNC_SERVER":          0,
+		"ASYNC_SERVER":         1,
+		"ASYNC_GENERIC_SERVER": 2,
+		"OTHER_SERVER":         3,
+		"CALLBACK_SERVER":      4,
+	}
+)
+
+func (x ServerType) Enum() *ServerType {
+	p := new(ServerType)
+	*p = x
+	return p
+}
+
+func (x ServerType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ServerType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_testing_control_proto_enumTypes[1].Descriptor()
+}
+
+func (ServerType) Type() protoreflect.EnumType {
+	return &file_grpc_testing_control_proto_enumTypes[1]
+}
+
+func (x ServerType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ServerType.Descriptor instead.
+func (ServerType) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{1}
+}
+
+type RpcType int32
+
+const (
+	RpcType_UNARY                 RpcType = 0
+	RpcType_STREAMING             RpcType = 1
+	RpcType_STREAMING_FROM_CLIENT RpcType = 2
+	RpcType_STREAMING_FROM_SERVER RpcType = 3
+	RpcType_STREAMING_BOTH_WAYS   RpcType = 4
+)
+
+// Enum value maps for RpcType.
+var (
+	RpcType_name = map[int32]string{
+		0: "UNARY",
+		1: "STREAMING",
+		2: "STREAMING_FROM_CLIENT",
+		3: "STREAMING_FROM_SERVER",
+		4: "STREAMING_BOTH_WAYS",
+	}
+	RpcType_value = map[string]int32{
+		"UNARY":                 0,
+		"STREAMING":             1,
+		"STREAMING_FROM_CLIENT": 2,
+		"STREAMING_FROM_SERVER": 3,
+		"STREAMING_BOTH_WAYS":   4,
+	}
+)
+
+func (x RpcType) Enum() *RpcType {
+	p := new(RpcType)
+	*p = x
+	return p
+}
+
+func (x RpcType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (RpcType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_testing_control_proto_enumTypes[2].Descriptor()
+}
+
+func (RpcType) Type() protoreflect.EnumType {
+	return &file_grpc_testing_control_proto_enumTypes[2]
+}
+
+func (x RpcType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use RpcType.Descriptor instead.
+func (RpcType) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{2}
+}
+
+// Parameters of poisson process distribution, which is a good representation
+// of activity coming in from independent identical stationary sources.
+type PoissonParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
+	OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"`
+}
+
+func (x *PoissonParams) Reset() {
+	*x = PoissonParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PoissonParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PoissonParams) ProtoMessage() {}
+
+func (x *PoissonParams) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PoissonParams.ProtoReflect.Descriptor instead.
+func (*PoissonParams) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *PoissonParams) GetOfferedLoad() float64 {
+	if x != nil {
+		return x.OfferedLoad
+	}
+	return 0
+}
+
+// Once an RPC finishes, immediately start a new one.
+// No configuration parameters needed.
+type ClosedLoopParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *ClosedLoopParams) Reset() {
+	*x = ClosedLoopParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClosedLoopParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClosedLoopParams) ProtoMessage() {}
+
+func (x *ClosedLoopParams) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClosedLoopParams.ProtoReflect.Descriptor instead.
+func (*ClosedLoopParams) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{1}
+}
+
+type LoadParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Load:
+	//	*LoadParams_ClosedLoop
+	//	*LoadParams_Poisson
+	Load isLoadParams_Load `protobuf_oneof:"load"`
+}
+
+func (x *LoadParams) Reset() {
+	*x = LoadParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadParams) ProtoMessage() {}
+
+func (x *LoadParams) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadParams.ProtoReflect.Descriptor instead.
+func (*LoadParams) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{2}
+}
+
+func (m *LoadParams) GetLoad() isLoadParams_Load {
+	if m != nil {
+		return m.Load
+	}
+	return nil
+}
+
+func (x *LoadParams) GetClosedLoop() *ClosedLoopParams {
+	if x, ok := x.GetLoad().(*LoadParams_ClosedLoop); ok {
+		return x.ClosedLoop
+	}
+	return nil
+}
+
+func (x *LoadParams) GetPoisson() *PoissonParams {
+	if x, ok := x.GetLoad().(*LoadParams_Poisson); ok {
+		return x.Poisson
+	}
+	return nil
+}
+
+type isLoadParams_Load interface {
+	isLoadParams_Load()
+}
+
+type LoadParams_ClosedLoop struct {
+	ClosedLoop *ClosedLoopParams `protobuf:"bytes,1,opt,name=closed_loop,json=closedLoop,proto3,oneof"`
+}
+
+type LoadParams_Poisson struct {
+	Poisson *PoissonParams `protobuf:"bytes,2,opt,name=poisson,proto3,oneof"`
+}
+
+func (*LoadParams_ClosedLoop) isLoadParams_Load() {}
+
+func (*LoadParams_Poisson) isLoadParams_Load() {}
+
+// presence of SecurityParams implies use of TLS
+type SecurityParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	UseTestCa          bool   `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"`
+	ServerHostOverride string `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"`
+	CredType           string `protobuf:"bytes,3,opt,name=cred_type,json=credType,proto3" json:"cred_type,omitempty"`
+}
+
+func (x *SecurityParams) Reset() {
+	*x = SecurityParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SecurityParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SecurityParams) ProtoMessage() {}
+
+func (x *SecurityParams) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SecurityParams.ProtoReflect.Descriptor instead.
+func (*SecurityParams) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *SecurityParams) GetUseTestCa() bool {
+	if x != nil {
+		return x.UseTestCa
+	}
+	return false
+}
+
+func (x *SecurityParams) GetServerHostOverride() string {
+	if x != nil {
+		return x.ServerHostOverride
+	}
+	return ""
+}
+
+func (x *SecurityParams) GetCredType() string {
+	if x != nil {
+		return x.CredType
+	}
+	return ""
+}
+
+type ChannelArg struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// Types that are assignable to Value:
+	//	*ChannelArg_StrValue
+	//	*ChannelArg_IntValue
+	Value isChannelArg_Value `protobuf_oneof:"value"`
+}
+
+func (x *ChannelArg) Reset() {
+	*x = ChannelArg{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ChannelArg) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ChannelArg) ProtoMessage() {}
+
+func (x *ChannelArg) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ChannelArg.ProtoReflect.Descriptor instead.
+func (*ChannelArg) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ChannelArg) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (m *ChannelArg) GetValue() isChannelArg_Value {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+func (x *ChannelArg) GetStrValue() string {
+	if x, ok := x.GetValue().(*ChannelArg_StrValue); ok {
+		return x.StrValue
+	}
+	return ""
+}
+
+func (x *ChannelArg) GetIntValue() int32 {
+	if x, ok := x.GetValue().(*ChannelArg_IntValue); ok {
+		return x.IntValue
+	}
+	return 0
+}
+
+type isChannelArg_Value interface {
+	isChannelArg_Value()
+}
+
+type ChannelArg_StrValue struct {
+	StrValue string `protobuf:"bytes,2,opt,name=str_value,json=strValue,proto3,oneof"`
+}
+
+type ChannelArg_IntValue struct {
+	IntValue int32 `protobuf:"varint,3,opt,name=int_value,json=intValue,proto3,oneof"`
+}
+
+func (*ChannelArg_StrValue) isChannelArg_Value() {}
+
+func (*ChannelArg_IntValue) isChannelArg_Value() {}
+
+type ClientConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// List of targets to connect to. At least one target needs to be specified.
+	ServerTargets  []string        `protobuf:"bytes,1,rep,name=server_targets,json=serverTargets,proto3" json:"server_targets,omitempty"`
+	ClientType     ClientType      `protobuf:"varint,2,opt,name=client_type,json=clientType,proto3,enum=grpc.testing.ClientType" json:"client_type,omitempty"`
+	SecurityParams *SecurityParams `protobuf:"bytes,3,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"`
+	// How many concurrent RPCs to start for each channel.
+	// For synchronous client, use a separate thread for each outstanding RPC.
+	OutstandingRpcsPerChannel int32 `protobuf:"varint,4,opt,name=outstanding_rpcs_per_channel,json=outstandingRpcsPerChannel,proto3" json:"outstanding_rpcs_per_channel,omitempty"`
+	// Number of independent client channels to create.
+	// i-th channel will connect to server_target[i % server_targets.size()]
+	ClientChannels int32 `protobuf:"varint,5,opt,name=client_channels,json=clientChannels,proto3" json:"client_channels,omitempty"`
+	// Only for async client. Number of threads to use to start/manage RPCs.
+	AsyncClientThreads int32   `protobuf:"varint,7,opt,name=async_client_threads,json=asyncClientThreads,proto3" json:"async_client_threads,omitempty"`
+	RpcType            RpcType `protobuf:"varint,8,opt,name=rpc_type,json=rpcType,proto3,enum=grpc.testing.RpcType" json:"rpc_type,omitempty"`
+	// The requested load for the entire client (aggregated over all the threads).
+	LoadParams      *LoadParams      `protobuf:"bytes,10,opt,name=load_params,json=loadParams,proto3" json:"load_params,omitempty"`
+	PayloadConfig   *PayloadConfig   `protobuf:"bytes,11,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"`
+	HistogramParams *HistogramParams `protobuf:"bytes,12,opt,name=histogram_params,json=histogramParams,proto3" json:"histogram_params,omitempty"`
+	// Specify the cores we should run the client on, if desired
+	CoreList  []int32 `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
+	CoreLimit int32   `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"`
+	// If we use an OTHER_CLIENT client_type, this string gives more detail
+	OtherClientApi string        `protobuf:"bytes,15,opt,name=other_client_api,json=otherClientApi,proto3" json:"other_client_api,omitempty"`
+	ChannelArgs    []*ChannelArg `protobuf:"bytes,16,rep,name=channel_args,json=channelArgs,proto3" json:"channel_args,omitempty"`
+	// Number of threads that share each completion queue
+	ThreadsPerCq int32 `protobuf:"varint,17,opt,name=threads_per_cq,json=threadsPerCq,proto3" json:"threads_per_cq,omitempty"`
+	// Number of messages on a stream before it gets finished/restarted
+	MessagesPerStream int32 `protobuf:"varint,18,opt,name=messages_per_stream,json=messagesPerStream,proto3" json:"messages_per_stream,omitempty"`
+	// Use coalescing API when possible.
+	UseCoalesceApi bool `protobuf:"varint,19,opt,name=use_coalesce_api,json=useCoalesceApi,proto3" json:"use_coalesce_api,omitempty"`
+	// If 0, disabled. Else, specifies the period between gathering latency
+	// medians in milliseconds.
+	MedianLatencyCollectionIntervalMillis int32 `protobuf:"varint,20,opt,name=median_latency_collection_interval_millis,json=medianLatencyCollectionIntervalMillis,proto3" json:"median_latency_collection_interval_millis,omitempty"`
+	// Number of client processes. 0 indicates no restriction.
+	ClientProcesses int32 `protobuf:"varint,21,opt,name=client_processes,json=clientProcesses,proto3" json:"client_processes,omitempty"`
+}
+
+func (x *ClientConfig) Reset() {
+	*x = ClientConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfig) ProtoMessage() {}
+
+func (x *ClientConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfig.ProtoReflect.Descriptor instead.
+func (*ClientConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *ClientConfig) GetServerTargets() []string {
+	if x != nil {
+		return x.ServerTargets
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetClientType() ClientType {
+	if x != nil {
+		return x.ClientType
+	}
+	return ClientType_SYNC_CLIENT
+}
+
+func (x *ClientConfig) GetSecurityParams() *SecurityParams {
+	if x != nil {
+		return x.SecurityParams
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetOutstandingRpcsPerChannel() int32 {
+	if x != nil {
+		return x.OutstandingRpcsPerChannel
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetClientChannels() int32 {
+	if x != nil {
+		return x.ClientChannels
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetAsyncClientThreads() int32 {
+	if x != nil {
+		return x.AsyncClientThreads
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetRpcType() RpcType {
+	if x != nil {
+		return x.RpcType
+	}
+	return RpcType_UNARY
+}
+
+func (x *ClientConfig) GetLoadParams() *LoadParams {
+	if x != nil {
+		return x.LoadParams
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetPayloadConfig() *PayloadConfig {
+	if x != nil {
+		return x.PayloadConfig
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetHistogramParams() *HistogramParams {
+	if x != nil {
+		return x.HistogramParams
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetCoreList() []int32 {
+	if x != nil {
+		return x.CoreList
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetCoreLimit() int32 {
+	if x != nil {
+		return x.CoreLimit
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetOtherClientApi() string {
+	if x != nil {
+		return x.OtherClientApi
+	}
+	return ""
+}
+
+func (x *ClientConfig) GetChannelArgs() []*ChannelArg {
+	if x != nil {
+		return x.ChannelArgs
+	}
+	return nil
+}
+
+func (x *ClientConfig) GetThreadsPerCq() int32 {
+	if x != nil {
+		return x.ThreadsPerCq
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetMessagesPerStream() int32 {
+	if x != nil {
+		return x.MessagesPerStream
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetUseCoalesceApi() bool {
+	if x != nil {
+		return x.UseCoalesceApi
+	}
+	return false
+}
+
+func (x *ClientConfig) GetMedianLatencyCollectionIntervalMillis() int32 {
+	if x != nil {
+		return x.MedianLatencyCollectionIntervalMillis
+	}
+	return 0
+}
+
+func (x *ClientConfig) GetClientProcesses() int32 {
+	if x != nil {
+		return x.ClientProcesses
+	}
+	return 0
+}
+
+type ClientStatus struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Stats *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
+}
+
+func (x *ClientStatus) Reset() {
+	*x = ClientStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientStatus) ProtoMessage() {}
+
+func (x *ClientStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientStatus.ProtoReflect.Descriptor instead.
+func (*ClientStatus) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *ClientStatus) GetStats() *ClientStats {
+	if x != nil {
+		return x.Stats
+	}
+	return nil
+}
+
+// Request current stats
+type Mark struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// if true, the stats will be reset after taking their snapshot.
+	Reset_ bool `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"`
+}
+
+func (x *Mark) Reset() {
+	*x = Mark{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Mark) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Mark) ProtoMessage() {}
+
+func (x *Mark) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Mark.ProtoReflect.Descriptor instead.
+func (*Mark) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *Mark) GetReset_() bool {
+	if x != nil {
+		return x.Reset_
+	}
+	return false
+}
+
+type ClientArgs struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Argtype:
+	//	*ClientArgs_Setup
+	//	*ClientArgs_Mark
+	Argtype isClientArgs_Argtype `protobuf_oneof:"argtype"`
+}
+
+func (x *ClientArgs) Reset() {
+	*x = ClientArgs{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientArgs) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientArgs) ProtoMessage() {}
+
+func (x *ClientArgs) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientArgs.ProtoReflect.Descriptor instead.
+func (*ClientArgs) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{8}
+}
+
+func (m *ClientArgs) GetArgtype() isClientArgs_Argtype {
+	if m != nil {
+		return m.Argtype
+	}
+	return nil
+}
+
+func (x *ClientArgs) GetSetup() *ClientConfig {
+	if x, ok := x.GetArgtype().(*ClientArgs_Setup); ok {
+		return x.Setup
+	}
+	return nil
+}
+
+func (x *ClientArgs) GetMark() *Mark {
+	if x, ok := x.GetArgtype().(*ClientArgs_Mark); ok {
+		return x.Mark
+	}
+	return nil
+}
+
+type isClientArgs_Argtype interface {
+	isClientArgs_Argtype()
+}
+
+type ClientArgs_Setup struct {
+	Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
+}
+
+type ClientArgs_Mark struct {
+	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
+}
+
+func (*ClientArgs_Setup) isClientArgs_Argtype() {}
+
+func (*ClientArgs_Mark) isClientArgs_Argtype() {}
+
+type ServerConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	ServerType     ServerType      `protobuf:"varint,1,opt,name=server_type,json=serverType,proto3,enum=grpc.testing.ServerType" json:"server_type,omitempty"`
+	SecurityParams *SecurityParams `protobuf:"bytes,2,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"`
+	// Port on which to listen. Zero means pick unused port.
+	Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"`
+	// Only for async server. Number of threads used to serve the requests.
+	AsyncServerThreads int32 `protobuf:"varint,7,opt,name=async_server_threads,json=asyncServerThreads,proto3" json:"async_server_threads,omitempty"`
+	// Specify the number of cores to limit server to, if desired
+	CoreLimit int32 `protobuf:"varint,8,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"`
+	// payload config, used in generic server.
+	// Note this must NOT be used in proto (non-generic) servers. For proto servers,
+	// 'response sizes' must be configured from the 'response_size' field of the
+	// 'SimpleRequest' objects in RPC requests.
+	PayloadConfig *PayloadConfig `protobuf:"bytes,9,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"`
+	// Specify the cores we should run the server on, if desired
+	CoreList []int32 `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"`
+	// If we use an OTHER_SERVER client_type, this string gives more detail
+	OtherServerApi string `protobuf:"bytes,11,opt,name=other_server_api,json=otherServerApi,proto3" json:"other_server_api,omitempty"`
+	// Number of threads that share each completion queue
+	ThreadsPerCq int32 `protobuf:"varint,12,opt,name=threads_per_cq,json=threadsPerCq,proto3" json:"threads_per_cq,omitempty"`
+	// Buffer pool size (no buffer pool specified if unset)
+	ResourceQuotaSize int32         `protobuf:"varint,1001,opt,name=resource_quota_size,json=resourceQuotaSize,proto3" json:"resource_quota_size,omitempty"`
+	ChannelArgs       []*ChannelArg `protobuf:"bytes,1002,rep,name=channel_args,json=channelArgs,proto3" json:"channel_args,omitempty"`
+	// Number of server processes. 0 indicates no restriction.
+	ServerProcesses int32 `protobuf:"varint,21,opt,name=server_processes,json=serverProcesses,proto3" json:"server_processes,omitempty"`
+}
+
+func (x *ServerConfig) Reset() {
+	*x = ServerConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ServerConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ServerConfig) ProtoMessage() {}
+
+func (x *ServerConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ServerConfig.ProtoReflect.Descriptor instead.
+func (*ServerConfig) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *ServerConfig) GetServerType() ServerType {
+	if x != nil {
+		return x.ServerType
+	}
+	return ServerType_SYNC_SERVER
+}
+
+func (x *ServerConfig) GetSecurityParams() *SecurityParams {
+	if x != nil {
+		return x.SecurityParams
+	}
+	return nil
+}
+
+func (x *ServerConfig) GetPort() int32 {
+	if x != nil {
+		return x.Port
+	}
+	return 0
+}
+
+func (x *ServerConfig) GetAsyncServerThreads() int32 {
+	if x != nil {
+		return x.AsyncServerThreads
+	}
+	return 0
+}
+
+func (x *ServerConfig) GetCoreLimit() int32 {
+	if x != nil {
+		return x.CoreLimit
+	}
+	return 0
+}
+
+func (x *ServerConfig) GetPayloadConfig() *PayloadConfig {
+	if x != nil {
+		return x.PayloadConfig
+	}
+	return nil
+}
+
+func (x *ServerConfig) GetCoreList() []int32 {
+	if x != nil {
+		return x.CoreList
+	}
+	return nil
+}
+
+func (x *ServerConfig) GetOtherServerApi() string {
+	if x != nil {
+		return x.OtherServerApi
+	}
+	return ""
+}
+
+func (x *ServerConfig) GetThreadsPerCq() int32 {
+	if x != nil {
+		return x.ThreadsPerCq
+	}
+	return 0
+}
+
+func (x *ServerConfig) GetResourceQuotaSize() int32 {
+	if x != nil {
+		return x.ResourceQuotaSize
+	}
+	return 0
+}
+
+func (x *ServerConfig) GetChannelArgs() []*ChannelArg {
+	if x != nil {
+		return x.ChannelArgs
+	}
+	return nil
+}
+
+func (x *ServerConfig) GetServerProcesses() int32 {
+	if x != nil {
+		return x.ServerProcesses
+	}
+	return 0
+}
+
+type ServerArgs struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Argtype:
+	//	*ServerArgs_Setup
+	//	*ServerArgs_Mark
+	Argtype isServerArgs_Argtype `protobuf_oneof:"argtype"`
+}
+
+func (x *ServerArgs) Reset() {
+	*x = ServerArgs{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ServerArgs) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ServerArgs) ProtoMessage() {}
+
+func (x *ServerArgs) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ServerArgs.ProtoReflect.Descriptor instead.
+func (*ServerArgs) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{10}
+}
+
+func (m *ServerArgs) GetArgtype() isServerArgs_Argtype {
+	if m != nil {
+		return m.Argtype
+	}
+	return nil
+}
+
+func (x *ServerArgs) GetSetup() *ServerConfig {
+	if x, ok := x.GetArgtype().(*ServerArgs_Setup); ok {
+		return x.Setup
+	}
+	return nil
+}
+
+func (x *ServerArgs) GetMark() *Mark {
+	if x, ok := x.GetArgtype().(*ServerArgs_Mark); ok {
+		return x.Mark
+	}
+	return nil
+}
+
+type isServerArgs_Argtype interface {
+	isServerArgs_Argtype()
+}
+
+type ServerArgs_Setup struct {
+	Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"`
+}
+
+type ServerArgs_Mark struct {
+	Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"`
+}
+
+func (*ServerArgs_Setup) isServerArgs_Argtype() {}
+
+func (*ServerArgs_Mark) isServerArgs_Argtype() {}
+
+type ServerStatus struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Stats *ServerStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"`
+	// the port bound by the server
+	Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
+	// Number of cores available to the server
+	Cores int32 `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"`
+}
+
+func (x *ServerStatus) Reset() {
+	*x = ServerStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ServerStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ServerStatus) ProtoMessage() {}
+
+func (x *ServerStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ServerStatus.ProtoReflect.Descriptor instead.
+func (*ServerStatus) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{11}
+}
+
+func (x *ServerStatus) GetStats() *ServerStats {
+	if x != nil {
+		return x.Stats
+	}
+	return nil
+}
+
+func (x *ServerStatus) GetPort() int32 {
+	if x != nil {
+		return x.Port
+	}
+	return 0
+}
+
+func (x *ServerStatus) GetCores() int32 {
+	if x != nil {
+		return x.Cores
+	}
+	return 0
+}
+
+type CoreRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *CoreRequest) Reset() {
+	*x = CoreRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *CoreRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CoreRequest) ProtoMessage() {}
+
+func (x *CoreRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CoreRequest.ProtoReflect.Descriptor instead.
+func (*CoreRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{12}
+}
+
+type CoreResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Number of cores available on the server
+	Cores int32 `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"`
+}
+
+func (x *CoreResponse) Reset() {
+	*x = CoreResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *CoreResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CoreResponse) ProtoMessage() {}
+
+func (x *CoreResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CoreResponse.ProtoReflect.Descriptor instead.
+func (*CoreResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{13}
+}
+
+func (x *CoreResponse) GetCores() int32 {
+	if x != nil {
+		return x.Cores
+	}
+	return 0
+}
+
+type Void struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *Void) Reset() {
+	*x = Void{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Void) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Void) ProtoMessage() {}
+
+func (x *Void) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Void.ProtoReflect.Descriptor instead.
+func (*Void) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{14}
+}
+
+// A single performance scenario: input to qps_json_driver
+type Scenario struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Human readable name for this scenario
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// Client configuration
+	ClientConfig *ClientConfig `protobuf:"bytes,2,opt,name=client_config,json=clientConfig,proto3" json:"client_config,omitempty"`
+	// Number of clients to start for the test
+	NumClients int32 `protobuf:"varint,3,opt,name=num_clients,json=numClients,proto3" json:"num_clients,omitempty"`
+	// Server configuration
+	ServerConfig *ServerConfig `protobuf:"bytes,4,opt,name=server_config,json=serverConfig,proto3" json:"server_config,omitempty"`
+	// Number of servers to start for the test
+	NumServers int32 `protobuf:"varint,5,opt,name=num_servers,json=numServers,proto3" json:"num_servers,omitempty"`
+	// Warmup period, in seconds
+	WarmupSeconds int32 `protobuf:"varint,6,opt,name=warmup_seconds,json=warmupSeconds,proto3" json:"warmup_seconds,omitempty"`
+	// Benchmark time, in seconds
+	BenchmarkSeconds int32 `protobuf:"varint,7,opt,name=benchmark_seconds,json=benchmarkSeconds,proto3" json:"benchmark_seconds,omitempty"`
+	// Number of workers to spawn locally (usually zero)
+	SpawnLocalWorkerCount int32 `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"`
+}
+
+func (x *Scenario) Reset() {
+	*x = Scenario{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Scenario) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Scenario) ProtoMessage() {}
+
+func (x *Scenario) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Scenario.ProtoReflect.Descriptor instead.
+func (*Scenario) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{15}
+}
+
+func (x *Scenario) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *Scenario) GetClientConfig() *ClientConfig {
+	if x != nil {
+		return x.ClientConfig
+	}
+	return nil
+}
+
+func (x *Scenario) GetNumClients() int32 {
+	if x != nil {
+		return x.NumClients
+	}
+	return 0
+}
+
+func (x *Scenario) GetServerConfig() *ServerConfig {
+	if x != nil {
+		return x.ServerConfig
+	}
+	return nil
+}
+
+func (x *Scenario) GetNumServers() int32 {
+	if x != nil {
+		return x.NumServers
+	}
+	return 0
+}
+
+func (x *Scenario) GetWarmupSeconds() int32 {
+	if x != nil {
+		return x.WarmupSeconds
+	}
+	return 0
+}
+
+func (x *Scenario) GetBenchmarkSeconds() int32 {
+	if x != nil {
+		return x.BenchmarkSeconds
+	}
+	return 0
+}
+
+func (x *Scenario) GetSpawnLocalWorkerCount() int32 {
+	if x != nil {
+		return x.SpawnLocalWorkerCount
+	}
+	return 0
+}
+
+// A set of scenarios to be run with qps_json_driver
+type Scenarios struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Scenarios []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"`
+}
+
+func (x *Scenarios) Reset() {
+	*x = Scenarios{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[16]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Scenarios) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Scenarios) ProtoMessage() {}
+
+func (x *Scenarios) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[16]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Scenarios.ProtoReflect.Descriptor instead.
+func (*Scenarios) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{16}
+}
+
+func (x *Scenarios) GetScenarios() []*Scenario {
+	if x != nil {
+		return x.Scenarios
+	}
+	return nil
+}
+
+// Basic summary that can be computed from ClientStats and ServerStats
+// once the scenario has finished.
+type ScenarioResultSummary struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Total number of operations per second over all clients. What is counted as 1 'operation' depends on the benchmark scenarios:
+	// For unary benchmarks, an operation is processing of a single unary RPC.
+	// For streaming benchmarks, an operation is processing of a single ping pong of request and response.
+	Qps float64 `protobuf:"fixed64,1,opt,name=qps,proto3" json:"qps,omitempty"`
+	// QPS per server core.
+	QpsPerServerCore float64 `protobuf:"fixed64,2,opt,name=qps_per_server_core,json=qpsPerServerCore,proto3" json:"qps_per_server_core,omitempty"`
+	// The total server cpu load based on system time across all server processes, expressed as percentage of a single cpu core.
+	// For example, 85 implies 85% of a cpu core, 125 implies 125% of a cpu core. Since we are accumulating the cpu load across all the server
+	// processes, the value could > 100 when there are multiple servers or a single server using multiple threads and cores.
+	// Same explanation for the total client cpu load below.
+	ServerSystemTime float64 `protobuf:"fixed64,3,opt,name=server_system_time,json=serverSystemTime,proto3" json:"server_system_time,omitempty"`
+	// The total server cpu load based on user time across all server processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%)
+	ServerUserTime float64 `protobuf:"fixed64,4,opt,name=server_user_time,json=serverUserTime,proto3" json:"server_user_time,omitempty"`
+	// The total client cpu load based on system time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%)
+	ClientSystemTime float64 `protobuf:"fixed64,5,opt,name=client_system_time,json=clientSystemTime,proto3" json:"client_system_time,omitempty"`
+	// The total client cpu load based on user time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%)
+	ClientUserTime float64 `protobuf:"fixed64,6,opt,name=client_user_time,json=clientUserTime,proto3" json:"client_user_time,omitempty"`
+	// X% latency percentiles (in nanoseconds)
+	Latency_50  float64 `protobuf:"fixed64,7,opt,name=latency_50,json=latency50,proto3" json:"latency_50,omitempty"`
+	Latency_90  float64 `protobuf:"fixed64,8,opt,name=latency_90,json=latency90,proto3" json:"latency_90,omitempty"`
+	Latency_95  float64 `protobuf:"fixed64,9,opt,name=latency_95,json=latency95,proto3" json:"latency_95,omitempty"`
+	Latency_99  float64 `protobuf:"fixed64,10,opt,name=latency_99,json=latency99,proto3" json:"latency_99,omitempty"`
+	Latency_999 float64 `protobuf:"fixed64,11,opt,name=latency_999,json=latency999,proto3" json:"latency_999,omitempty"`
+	// server cpu usage percentage
+	ServerCpuUsage float64 `protobuf:"fixed64,12,opt,name=server_cpu_usage,json=serverCpuUsage,proto3" json:"server_cpu_usage,omitempty"`
+	// Number of requests that succeeded/failed
+	SuccessfulRequestsPerSecond float64 `protobuf:"fixed64,13,opt,name=successful_requests_per_second,json=successfulRequestsPerSecond,proto3" json:"successful_requests_per_second,omitempty"`
+	FailedRequestsPerSecond     float64 `protobuf:"fixed64,14,opt,name=failed_requests_per_second,json=failedRequestsPerSecond,proto3" json:"failed_requests_per_second,omitempty"`
+	// Number of polls called inside completion queue per request
+	ClientPollsPerRequest float64 `protobuf:"fixed64,15,opt,name=client_polls_per_request,json=clientPollsPerRequest,proto3" json:"client_polls_per_request,omitempty"`
+	ServerPollsPerRequest float64 `protobuf:"fixed64,16,opt,name=server_polls_per_request,json=serverPollsPerRequest,proto3" json:"server_polls_per_request,omitempty"`
+	// Queries per CPU-sec over all servers or clients
+	ServerQueriesPerCpuSec float64 `protobuf:"fixed64,17,opt,name=server_queries_per_cpu_sec,json=serverQueriesPerCpuSec,proto3" json:"server_queries_per_cpu_sec,omitempty"`
+	ClientQueriesPerCpuSec float64 `protobuf:"fixed64,18,opt,name=client_queries_per_cpu_sec,json=clientQueriesPerCpuSec,proto3" json:"client_queries_per_cpu_sec,omitempty"`
+}
+
+func (x *ScenarioResultSummary) Reset() {
+	*x = ScenarioResultSummary{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[17]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ScenarioResultSummary) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ScenarioResultSummary) ProtoMessage() {}
+
+func (x *ScenarioResultSummary) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[17]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ScenarioResultSummary.ProtoReflect.Descriptor instead.
+func (*ScenarioResultSummary) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{17}
+}
+
+func (x *ScenarioResultSummary) GetQps() float64 {
+	if x != nil {
+		return x.Qps
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetQpsPerServerCore() float64 {
+	if x != nil {
+		return x.QpsPerServerCore
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetServerSystemTime() float64 {
+	if x != nil {
+		return x.ServerSystemTime
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetServerUserTime() float64 {
+	if x != nil {
+		return x.ServerUserTime
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetClientSystemTime() float64 {
+	if x != nil {
+		return x.ClientSystemTime
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetClientUserTime() float64 {
+	if x != nil {
+		return x.ClientUserTime
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetLatency_50() float64 {
+	if x != nil {
+		return x.Latency_50
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetLatency_90() float64 {
+	if x != nil {
+		return x.Latency_90
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetLatency_95() float64 {
+	if x != nil {
+		return x.Latency_95
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetLatency_99() float64 {
+	if x != nil {
+		return x.Latency_99
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetLatency_999() float64 {
+	if x != nil {
+		return x.Latency_999
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetServerCpuUsage() float64 {
+	if x != nil {
+		return x.ServerCpuUsage
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetSuccessfulRequestsPerSecond() float64 {
+	if x != nil {
+		return x.SuccessfulRequestsPerSecond
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetFailedRequestsPerSecond() float64 {
+	if x != nil {
+		return x.FailedRequestsPerSecond
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetClientPollsPerRequest() float64 {
+	if x != nil {
+		return x.ClientPollsPerRequest
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetServerPollsPerRequest() float64 {
+	if x != nil {
+		return x.ServerPollsPerRequest
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetServerQueriesPerCpuSec() float64 {
+	if x != nil {
+		return x.ServerQueriesPerCpuSec
+	}
+	return 0
+}
+
+func (x *ScenarioResultSummary) GetClientQueriesPerCpuSec() float64 {
+	if x != nil {
+		return x.ClientQueriesPerCpuSec
+	}
+	return 0
+}
+
+// Results of a single benchmark scenario.
+type ScenarioResult struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Inputs used to run the scenario.
+	Scenario *Scenario `protobuf:"bytes,1,opt,name=scenario,proto3" json:"scenario,omitempty"`
+	// Histograms from all clients merged into one histogram.
+	Latencies *HistogramData `protobuf:"bytes,2,opt,name=latencies,proto3" json:"latencies,omitempty"`
+	// Client stats for each client
+	ClientStats []*ClientStats `protobuf:"bytes,3,rep,name=client_stats,json=clientStats,proto3" json:"client_stats,omitempty"`
+	// Server stats for each server
+	ServerStats []*ServerStats `protobuf:"bytes,4,rep,name=server_stats,json=serverStats,proto3" json:"server_stats,omitempty"`
+	// Number of cores available to each server
+	ServerCores []int32 `protobuf:"varint,5,rep,packed,name=server_cores,json=serverCores,proto3" json:"server_cores,omitempty"`
+	// An after-the-fact computed summary
+	Summary *ScenarioResultSummary `protobuf:"bytes,6,opt,name=summary,proto3" json:"summary,omitempty"`
+	// Information on success or failure of each worker
+	ClientSuccess []bool `protobuf:"varint,7,rep,packed,name=client_success,json=clientSuccess,proto3" json:"client_success,omitempty"`
+	ServerSuccess []bool `protobuf:"varint,8,rep,packed,name=server_success,json=serverSuccess,proto3" json:"server_success,omitempty"`
+	// Number of failed requests (one row per status code seen)
+	RequestResults []*RequestResultCount `protobuf:"bytes,9,rep,name=request_results,json=requestResults,proto3" json:"request_results,omitempty"`
+}
+
+func (x *ScenarioResult) Reset() {
+	*x = ScenarioResult{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_control_proto_msgTypes[18]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ScenarioResult) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ScenarioResult) ProtoMessage() {}
+
+func (x *ScenarioResult) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_control_proto_msgTypes[18]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ScenarioResult.ProtoReflect.Descriptor instead.
+func (*ScenarioResult) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_control_proto_rawDescGZIP(), []int{18}
+}
+
+func (x *ScenarioResult) GetScenario() *Scenario {
+	if x != nil {
+		return x.Scenario
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetLatencies() *HistogramData {
+	if x != nil {
+		return x.Latencies
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetClientStats() []*ClientStats {
+	if x != nil {
+		return x.ClientStats
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetServerStats() []*ServerStats {
+	if x != nil {
+		return x.ServerStats
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetServerCores() []int32 {
+	if x != nil {
+		return x.ServerCores
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetSummary() *ScenarioResultSummary {
+	if x != nil {
+		return x.Summary
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetClientSuccess() []bool {
+	if x != nil {
+		return x.ClientSuccess
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetServerSuccess() []bool {
+	if x != nil {
+		return x.ServerSuccess
+	}
+	return nil
+}
+
+func (x *ScenarioResult) GetRequestResults() []*RequestResultCount {
+	if x != nil {
+		return x.RequestResults
+	}
+	return nil
+}
+
+var File_grpc_testing_control_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_control_proto_rawDesc = []byte{
+	0x0a, 0x1a, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63,
+	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63,
+	0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x22, 0x32, 0x0a, 0x0d, 0x50, 0x6f, 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61,
+	0x6d, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x6c, 0x6f,
+	0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65,
+	0x64, 0x4c, 0x6f, 0x61, 0x64, 0x22, 0x12, 0x0a, 0x10, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c,
+	0x6f, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x90, 0x01, 0x0a, 0x0a, 0x4c, 0x6f,
+	0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x63, 0x6c, 0x6f, 0x73,
+	0x65, 0x64, 0x5f, 0x6c, 0x6f, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x6f,
+	0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52,
+	0x0a, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x12, 0x37, 0x0a, 0x07, 0x70,
+	0x6f, 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x6f, 0x69, 0x73,
+	0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x07, 0x70, 0x6f, 0x69,
+	0x73, 0x73, 0x6f, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x7f, 0x0a, 0x0e,
+	0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1e,
+	0x0a, 0x0b, 0x75, 0x73, 0x65, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x61, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x08, 0x52, 0x09, 0x75, 0x73, 0x65, 0x54, 0x65, 0x73, 0x74, 0x43, 0x61, 0x12, 0x30,
+	0x0a, 0x14, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x6f, 0x76,
+	0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65,
+	0x12, 0x1b, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x72, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x22, 0x67, 0x0a,
+	0x0a, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
+	0x1d, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x09, 0x48, 0x00, 0x52, 0x08, 0x73, 0x74, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d,
+	0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x05, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xf6, 0x07, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52,
+	0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x39,
+	0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x63,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x73, 0x65, 0x63,
+	0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x12, 0x3f, 0x0a, 0x1c, 0x6f, 0x75, 0x74, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f,
+	0x72, 0x70, 0x63, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x19, 0x6f, 0x75, 0x74, 0x73, 0x74, 0x61, 0x6e, 0x64,
+	0x69, 0x6e, 0x67, 0x52, 0x70, 0x63, 0x73, 0x50, 0x65, 0x72, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+	0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x68, 0x61, 0x6e,
+	0x6e, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x63, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x73,
+	0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61,
+	0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x43,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x30, 0x0a, 0x08,
+	0x72, 0x70, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x70,
+	0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x72, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39,
+	0x0a, 0x0b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0a, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0a, 0x6c,
+	0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x42, 0x0a, 0x0e, 0x70, 0x61, 0x79,
+	0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0b, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0d,
+	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x48, 0x0a,
+	0x10, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d,
+	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0f, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61,
+	0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x5f,
+	0x6c, 0x69, 0x73, 0x74, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6f, 0x72, 0x65,
+	0x4c, 0x69, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x6d,
+	0x69, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69,
+	0x6d, 0x69, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x63, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x69, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f,
+	0x74, 0x68, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x69, 0x12, 0x3b, 0x0a,
+	0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x18, 0x10, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x52, 0x0b, 0x63,
+	0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x68,
+	0x72, 0x65, 0x61, 0x64, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x71, 0x18, 0x11, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x0c, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x50, 0x65, 0x72, 0x43, 0x71,
+	0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x70, 0x65, 0x72,
+	0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x6d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
+	0x12, 0x28, 0x0a, 0x10, 0x75, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x61, 0x6c, 0x65, 0x73, 0x63, 0x65,
+	0x5f, 0x61, 0x70, 0x69, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, 0x43,
+	0x6f, 0x61, 0x6c, 0x65, 0x73, 0x63, 0x65, 0x41, 0x70, 0x69, 0x12, 0x58, 0x0a, 0x29, 0x6d, 0x65,
+	0x64, 0x69, 0x61, 0x6e, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x63, 0x6f, 0x6c,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c,
+	0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x52, 0x25, 0x6d,
+	0x65, 0x64, 0x69, 0x61, 0x6e, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6c, 0x6c,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x4d, 0x69,
+	0x6c, 0x6c, 0x69, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x70,
+	0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f,
+	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22,
+	0x3f, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12,
+	0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c,
+	0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73,
+	0x22, 0x1c, 0x0a, 0x04, 0x4d, 0x61, 0x72, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x73, 0x65,
+	0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x73, 0x65, 0x74, 0x22, 0x75,
+	0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05,
+	0x73, 0x65, 0x74, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e,
+	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70,
+	0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61,
+	0x72, 0x6b, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x61, 0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72,
+	0x67, 0x74, 0x79, 0x70, 0x65, 0x22, 0xc0, 0x04, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x39, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x45, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69,
+	0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69,
+	0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x30, 0x0a, 0x14,
+	0x61, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x68, 0x72,
+	0x65, 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x73, 0x79, 0x6e,
+	0x63, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x1d,
+	0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x42, 0x0a,
+	0x0e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18,
+	0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66,
+	0x69, 0x67, 0x52, 0x0d, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0a,
+	0x20, 0x03, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x28,
+	0x0a, 0x10, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x61,
+	0x70, 0x69, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x70, 0x69, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x65,
+	0x61, 0x64, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x71, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x0c, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x50, 0x65, 0x72, 0x43, 0x71, 0x12, 0x2f,
+	0x0a, 0x13, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x71, 0x75, 0x6f, 0x74, 0x61,
+	0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x72, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x53, 0x69, 0x7a, 0x65, 0x12,
+	0x3c, 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x18,
+	0xea, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67,
+	0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x29, 0x0a,
+	0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65,
+	0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x50,
+	0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x75, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x48, 0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61,
+	0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x04,
+	0x6d, 0x61, 0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, 0x22,
+	0x69, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12,
+	0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73,
+	0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04,
+	0x70, 0x6f, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22, 0x0d, 0x0a, 0x0b, 0x43, 0x6f,
+	0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x0c, 0x43, 0x6f, 0x72,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72,
+	0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22,
+	0x06, 0x0a, 0x04, 0x56, 0x6f, 0x69, 0x64, 0x22, 0xef, 0x02, 0x0a, 0x08, 0x53, 0x63, 0x65, 0x6e,
+	0x61, 0x72, 0x69, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x63, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, 0x6d,
+	0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
+	0x6e, 0x75, 0x6d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0d, 0x73, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x73,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e,
+	0x75, 0x6d, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e,
+	0x77, 0x61, 0x72, 0x6d, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x06,
+	0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x77, 0x61, 0x72, 0x6d, 0x75, 0x70, 0x53, 0x65, 0x63, 0x6f,
+	0x6e, 0x64, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b,
+	0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10,
+	0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73,
+	0x12, 0x37, 0x0a, 0x18, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f,
+	0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x15, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x57, 0x6f,
+	0x72, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x41, 0x0a, 0x09, 0x53, 0x63, 0x65,
+	0x6e, 0x61, 0x72, 0x69, 0x6f, 0x73, 0x12, 0x34, 0x0a, 0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72,
+	0x69, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69,
+	0x6f, 0x52, 0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x73, 0x22, 0xbb, 0x06, 0x0a,
+	0x15, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53,
+	0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x71, 0x70, 0x73, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x01, 0x52, 0x03, 0x71, 0x70, 0x73, 0x12, 0x2d, 0x0a, 0x13, 0x71, 0x70, 0x73, 0x5f,
+	0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x71, 0x70, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x43, 0x6f, 0x72, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x01, 0x52, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, 0x65,
+	0x6d, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f,
+	0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52,
+	0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x55, 0x73, 0x65, 0x72, 0x54, 0x69, 0x6d, 0x65, 0x12,
+	0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
+	0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x63, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x28, 0x0a,
+	0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x69, 0x6d,
+	0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x55,
+	0x73, 0x65, 0x72, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e,
+	0x63, 0x79, 0x5f, 0x35, 0x30, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74,
+	0x65, 0x6e, 0x63, 0x79, 0x35, 0x30, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63,
+	0x79, 0x5f, 0x39, 0x30, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65,
+	0x6e, 0x63, 0x79, 0x39, 0x30, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79,
+	0x5f, 0x39, 0x35, 0x18, 0x09, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e,
+	0x63, 0x79, 0x39, 0x35, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f,
+	0x39, 0x39, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63,
+	0x79, 0x39, 0x39, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x39,
+	0x39, 0x39, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63,
+	0x79, 0x39, 0x39, 0x39, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63,
+	0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x43,
+	0x0a, 0x1e, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x5f, 0x72, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64,
+	0x18, 0x0d, 0x20, 0x01, 0x28, 0x01, 0x52, 0x1b, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66,
+	0x75, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63,
+	0x6f, 0x6e, 0x64, 0x12, 0x3b, 0x0a, 0x1a, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e,
+	0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x01, 0x52, 0x17, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64,
+	0x12, 0x37, 0x0a, 0x18, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x73,
+	0x5f, 0x70, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x0f, 0x20, 0x01,
+	0x28, 0x01, 0x52, 0x15, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x6c, 0x6c, 0x73, 0x50,
+	0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x18, 0x73, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x72, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x01, 0x52, 0x15, 0x73, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x50, 0x6f, 0x6c, 0x6c, 0x73, 0x50, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x71, 0x75, 0x65,
+	0x72, 0x69, 0x65, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x73, 0x65, 0x63,
+	0x18, 0x11, 0x20, 0x01, 0x28, 0x01, 0x52, 0x16, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x51, 0x75,
+	0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x65, 0x72, 0x43, 0x70, 0x75, 0x53, 0x65, 0x63, 0x12, 0x3a,
+	0x0a, 0x1a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73,
+	0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x12, 0x20, 0x01,
+	0x28, 0x01, 0x52, 0x16, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65,
+	0x73, 0x50, 0x65, 0x72, 0x43, 0x70, 0x75, 0x53, 0x65, 0x63, 0x22, 0xf6, 0x03, 0x0a, 0x0e, 0x53,
+	0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x32, 0x0a,
+	0x08, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x08, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69,
+	0x6f, 0x12, 0x39, 0x0a, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74,
+	0x61, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x0c,
+	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x0b, 0x63,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x0c, 0x73, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b,
+	0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x0b, 0x73, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0b,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x07, 0x73,
+	0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e,
+	0x61, 0x72, 0x69, 0x6f, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72,
+	0x79, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c,
+	0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, 0x03,
+	0x28, 0x08, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,
+	0x73, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x63, 0x63,
+	0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x08, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x49, 0x0a, 0x0f, 0x72, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f,
+	0x75, 0x6e, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x75,
+	0x6c, 0x74, 0x73, 0x2a, 0x56, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54,
+	0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x49, 0x45,
+	0x4e, 0x54, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x54, 0x48, 0x45, 0x52, 0x5f, 0x43, 0x4c,
+	0x49, 0x45, 0x4e, 0x54, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x43, 0x41, 0x4c, 0x4c, 0x42, 0x41,
+	0x43, 0x4b, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, 0x2a, 0x70, 0x0a, 0x0a, 0x53,
+	0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e,
+	0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53,
+	0x59, 0x4e, 0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14,
+	0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x5f, 0x53, 0x45,
+	0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x54, 0x48, 0x45, 0x52, 0x5f,
+	0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x43, 0x41, 0x4c, 0x4c,
+	0x42, 0x41, 0x43, 0x4b, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x04, 0x2a, 0x72, 0x0a,
+	0x07, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x4e, 0x41, 0x52,
+	0x59, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47,
+	0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x5f,
+	0x46, 0x52, 0x4f, 0x4d, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x02, 0x12, 0x19, 0x0a,
+	0x15, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x5f, 0x46, 0x52, 0x4f, 0x4d, 0x5f,
+	0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x54, 0x52, 0x45,
+	0x41, 0x4d, 0x49, 0x4e, 0x47, 0x5f, 0x42, 0x4f, 0x54, 0x48, 0x5f, 0x57, 0x41, 0x59, 0x53, 0x10,
+	0x04, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_testing_control_proto_rawDescOnce sync.Once
+	file_grpc_testing_control_proto_rawDescData = file_grpc_testing_control_proto_rawDesc
+)
+
+func file_grpc_testing_control_proto_rawDescGZIP() []byte {
+	file_grpc_testing_control_proto_rawDescOnce.Do(func() {
+		file_grpc_testing_control_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_control_proto_rawDescData)
+	})
+	return file_grpc_testing_control_proto_rawDescData
+}
+
+var file_grpc_testing_control_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_grpc_testing_control_proto_msgTypes = make([]protoimpl.MessageInfo, 19)
+var file_grpc_testing_control_proto_goTypes = []interface{}{
+	(ClientType)(0),               // 0: grpc.testing.ClientType
+	(ServerType)(0),               // 1: grpc.testing.ServerType
+	(RpcType)(0),                  // 2: grpc.testing.RpcType
+	(*PoissonParams)(nil),         // 3: grpc.testing.PoissonParams
+	(*ClosedLoopParams)(nil),      // 4: grpc.testing.ClosedLoopParams
+	(*LoadParams)(nil),            // 5: grpc.testing.LoadParams
+	(*SecurityParams)(nil),        // 6: grpc.testing.SecurityParams
+	(*ChannelArg)(nil),            // 7: grpc.testing.ChannelArg
+	(*ClientConfig)(nil),          // 8: grpc.testing.ClientConfig
+	(*ClientStatus)(nil),          // 9: grpc.testing.ClientStatus
+	(*Mark)(nil),                  // 10: grpc.testing.Mark
+	(*ClientArgs)(nil),            // 11: grpc.testing.ClientArgs
+	(*ServerConfig)(nil),          // 12: grpc.testing.ServerConfig
+	(*ServerArgs)(nil),            // 13: grpc.testing.ServerArgs
+	(*ServerStatus)(nil),          // 14: grpc.testing.ServerStatus
+	(*CoreRequest)(nil),           // 15: grpc.testing.CoreRequest
+	(*CoreResponse)(nil),          // 16: grpc.testing.CoreResponse
+	(*Void)(nil),                  // 17: grpc.testing.Void
+	(*Scenario)(nil),              // 18: grpc.testing.Scenario
+	(*Scenarios)(nil),             // 19: grpc.testing.Scenarios
+	(*ScenarioResultSummary)(nil), // 20: grpc.testing.ScenarioResultSummary
+	(*ScenarioResult)(nil),        // 21: grpc.testing.ScenarioResult
+	(*PayloadConfig)(nil),         // 22: grpc.testing.PayloadConfig
+	(*HistogramParams)(nil),       // 23: grpc.testing.HistogramParams
+	(*ClientStats)(nil),           // 24: grpc.testing.ClientStats
+	(*ServerStats)(nil),           // 25: grpc.testing.ServerStats
+	(*HistogramData)(nil),         // 26: grpc.testing.HistogramData
+	(*RequestResultCount)(nil),    // 27: grpc.testing.RequestResultCount
+}
+var file_grpc_testing_control_proto_depIdxs = []int32{
+	4,  // 0: grpc.testing.LoadParams.closed_loop:type_name -> grpc.testing.ClosedLoopParams
+	3,  // 1: grpc.testing.LoadParams.poisson:type_name -> grpc.testing.PoissonParams
+	0,  // 2: grpc.testing.ClientConfig.client_type:type_name -> grpc.testing.ClientType
+	6,  // 3: grpc.testing.ClientConfig.security_params:type_name -> grpc.testing.SecurityParams
+	2,  // 4: grpc.testing.ClientConfig.rpc_type:type_name -> grpc.testing.RpcType
+	5,  // 5: grpc.testing.ClientConfig.load_params:type_name -> grpc.testing.LoadParams
+	22, // 6: grpc.testing.ClientConfig.payload_config:type_name -> grpc.testing.PayloadConfig
+	23, // 7: grpc.testing.ClientConfig.histogram_params:type_name -> grpc.testing.HistogramParams
+	7,  // 8: grpc.testing.ClientConfig.channel_args:type_name -> grpc.testing.ChannelArg
+	24, // 9: grpc.testing.ClientStatus.stats:type_name -> grpc.testing.ClientStats
+	8,  // 10: grpc.testing.ClientArgs.setup:type_name -> grpc.testing.ClientConfig
+	10, // 11: grpc.testing.ClientArgs.mark:type_name -> grpc.testing.Mark
+	1,  // 12: grpc.testing.ServerConfig.server_type:type_name -> grpc.testing.ServerType
+	6,  // 13: grpc.testing.ServerConfig.security_params:type_name -> grpc.testing.SecurityParams
+	22, // 14: grpc.testing.ServerConfig.payload_config:type_name -> grpc.testing.PayloadConfig
+	7,  // 15: grpc.testing.ServerConfig.channel_args:type_name -> grpc.testing.ChannelArg
+	12, // 16: grpc.testing.ServerArgs.setup:type_name -> grpc.testing.ServerConfig
+	10, // 17: grpc.testing.ServerArgs.mark:type_name -> grpc.testing.Mark
+	25, // 18: grpc.testing.ServerStatus.stats:type_name -> grpc.testing.ServerStats
+	8,  // 19: grpc.testing.Scenario.client_config:type_name -> grpc.testing.ClientConfig
+	12, // 20: grpc.testing.Scenario.server_config:type_name -> grpc.testing.ServerConfig
+	18, // 21: grpc.testing.Scenarios.scenarios:type_name -> grpc.testing.Scenario
+	18, // 22: grpc.testing.ScenarioResult.scenario:type_name -> grpc.testing.Scenario
+	26, // 23: grpc.testing.ScenarioResult.latencies:type_name -> grpc.testing.HistogramData
+	24, // 24: grpc.testing.ScenarioResult.client_stats:type_name -> grpc.testing.ClientStats
+	25, // 25: grpc.testing.ScenarioResult.server_stats:type_name -> grpc.testing.ServerStats
+	20, // 26: grpc.testing.ScenarioResult.summary:type_name -> grpc.testing.ScenarioResultSummary
+	27, // 27: grpc.testing.ScenarioResult.request_results:type_name -> grpc.testing.RequestResultCount
+	28, // [28:28] is the sub-list for method output_type
+	28, // [28:28] is the sub-list for method input_type
+	28, // [28:28] is the sub-list for extension type_name
+	28, // [28:28] is the sub-list for extension extendee
+	0,  // [0:28] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_control_proto_init() }
+func file_grpc_testing_control_proto_init() {
+	if File_grpc_testing_control_proto != nil {
+		return
+	}
+	file_grpc_testing_payloads_proto_init()
+	file_grpc_testing_stats_proto_init()
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_testing_control_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PoissonParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClosedLoopParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SecurityParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChannelArg); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Mark); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientArgs); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerArgs); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CoreRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CoreResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Void); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Scenario); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Scenarios); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ScenarioResultSummary); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_control_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ScenarioResult); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_testing_control_proto_msgTypes[2].OneofWrappers = []interface{}{
+		(*LoadParams_ClosedLoop)(nil),
+		(*LoadParams_Poisson)(nil),
+	}
+	file_grpc_testing_control_proto_msgTypes[4].OneofWrappers = []interface{}{
+		(*ChannelArg_StrValue)(nil),
+		(*ChannelArg_IntValue)(nil),
+	}
+	file_grpc_testing_control_proto_msgTypes[8].OneofWrappers = []interface{}{
+		(*ClientArgs_Setup)(nil),
+		(*ClientArgs_Mark)(nil),
+	}
+	file_grpc_testing_control_proto_msgTypes[10].OneofWrappers = []interface{}{
+		(*ServerArgs_Setup)(nil),
+		(*ServerArgs_Mark)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_control_proto_rawDesc,
+			NumEnums:      3,
+			NumMessages:   19,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_testing_control_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_control_proto_depIdxs,
+		EnumInfos:         file_grpc_testing_control_proto_enumTypes,
+		MessageInfos:      file_grpc_testing_control_proto_msgTypes,
+	}.Build()
+	File_grpc_testing_control_proto = out.File
+	file_grpc_testing_control_proto_rawDesc = nil
+	file_grpc_testing_control_proto_goTypes = nil
+	file_grpc_testing_control_proto_depIdxs = nil
+}
diff --git a/interop/grpc_testing/core/stats.pb.go b/interop/grpc_testing/core/stats.pb.go
new file mode 100644
index 000000000000..e5652dc78cc3
--- /dev/null
+++ b/interop/grpc_testing/core/stats.pb.go
@@ -0,0 +1,412 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/core/stats.proto
+
+package grpc_core
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+type Bucket struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Start float64 `protobuf:"fixed64,1,opt,name=start,proto3" json:"start,omitempty"`
+	Count uint64  `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"`
+}
+
+func (x *Bucket) Reset() {
+	*x = Bucket{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_core_stats_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Bucket) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Bucket) ProtoMessage() {}
+
+func (x *Bucket) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_core_stats_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Bucket.ProtoReflect.Descriptor instead.
+func (*Bucket) Descriptor() ([]byte, []int) {
+	return file_grpc_core_stats_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Bucket) GetStart() float64 {
+	if x != nil {
+		return x.Start
+	}
+	return 0
+}
+
+func (x *Bucket) GetCount() uint64 {
+	if x != nil {
+		return x.Count
+	}
+	return 0
+}
+
+type Histogram struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Buckets []*Bucket `protobuf:"bytes,1,rep,name=buckets,proto3" json:"buckets,omitempty"`
+}
+
+func (x *Histogram) Reset() {
+	*x = Histogram{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_core_stats_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Histogram) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Histogram) ProtoMessage() {}
+
+func (x *Histogram) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_core_stats_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Histogram.ProtoReflect.Descriptor instead.
+func (*Histogram) Descriptor() ([]byte, []int) {
+	return file_grpc_core_stats_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Histogram) GetBuckets() []*Bucket {
+	if x != nil {
+		return x.Buckets
+	}
+	return nil
+}
+
+type Metric struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// Types that are assignable to Value:
+	//	*Metric_Count
+	//	*Metric_Histogram
+	Value isMetric_Value `protobuf_oneof:"value"`
+}
+
+func (x *Metric) Reset() {
+	*x = Metric{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_core_stats_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Metric) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Metric) ProtoMessage() {}
+
+func (x *Metric) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_core_stats_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Metric.ProtoReflect.Descriptor instead.
+func (*Metric) Descriptor() ([]byte, []int) {
+	return file_grpc_core_stats_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *Metric) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (m *Metric) GetValue() isMetric_Value {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+func (x *Metric) GetCount() uint64 {
+	if x, ok := x.GetValue().(*Metric_Count); ok {
+		return x.Count
+	}
+	return 0
+}
+
+func (x *Metric) GetHistogram() *Histogram {
+	if x, ok := x.GetValue().(*Metric_Histogram); ok {
+		return x.Histogram
+	}
+	return nil
+}
+
+type isMetric_Value interface {
+	isMetric_Value()
+}
+
+type Metric_Count struct {
+	Count uint64 `protobuf:"varint,10,opt,name=count,proto3,oneof"`
+}
+
+type Metric_Histogram struct {
+	Histogram *Histogram `protobuf:"bytes,11,opt,name=histogram,proto3,oneof"`
+}
+
+func (*Metric_Count) isMetric_Value() {}
+
+func (*Metric_Histogram) isMetric_Value() {}
+
+type Stats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Metrics []*Metric `protobuf:"bytes,1,rep,name=metrics,proto3" json:"metrics,omitempty"`
+}
+
+func (x *Stats) Reset() {
+	*x = Stats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_core_stats_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Stats) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Stats) ProtoMessage() {}
+
+func (x *Stats) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_core_stats_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Stats.ProtoReflect.Descriptor instead.
+func (*Stats) Descriptor() ([]byte, []int) {
+	return file_grpc_core_stats_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *Stats) GetMetrics() []*Metric {
+	if x != nil {
+		return x.Metrics
+	}
+	return nil
+}
+
+var File_grpc_core_stats_proto protoreflect.FileDescriptor
+
+var file_grpc_core_stats_proto_rawDesc = []byte{
+	0x0a, 0x15, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x73, 0x74, 0x61, 0x74,
+	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f,
+	0x72, 0x65, 0x22, 0x34, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05,
+	0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x74, 0x61,
+	0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x38, 0x0a, 0x09, 0x48, 0x69, 0x73, 0x74,
+	0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x2b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f,
+	0x72, 0x65, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65,
+	0x74, 0x73, 0x22, 0x73, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x12, 0x0a, 0x04,
+	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x12, 0x16, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x48,
+	0x00, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74,
+	0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61,
+	0x6d, 0x48, 0x00, 0x52, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x42, 0x07,
+	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x34, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73,
+	0x12, 0x2b, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x11, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4d, 0x65,
+	0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_core_stats_proto_rawDescOnce sync.Once
+	file_grpc_core_stats_proto_rawDescData = file_grpc_core_stats_proto_rawDesc
+)
+
+func file_grpc_core_stats_proto_rawDescGZIP() []byte {
+	file_grpc_core_stats_proto_rawDescOnce.Do(func() {
+		file_grpc_core_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_core_stats_proto_rawDescData)
+	})
+	return file_grpc_core_stats_proto_rawDescData
+}
+
+var file_grpc_core_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_grpc_core_stats_proto_goTypes = []interface{}{
+	(*Bucket)(nil),    // 0: grpc.core.Bucket
+	(*Histogram)(nil), // 1: grpc.core.Histogram
+	(*Metric)(nil),    // 2: grpc.core.Metric
+	(*Stats)(nil),     // 3: grpc.core.Stats
+}
+var file_grpc_core_stats_proto_depIdxs = []int32{
+	0, // 0: grpc.core.Histogram.buckets:type_name -> grpc.core.Bucket
+	1, // 1: grpc.core.Metric.histogram:type_name -> grpc.core.Histogram
+	2, // 2: grpc.core.Stats.metrics:type_name -> grpc.core.Metric
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_grpc_core_stats_proto_init() }
+func file_grpc_core_stats_proto_init() {
+	if File_grpc_core_stats_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_core_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Bucket); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_core_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Histogram); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_core_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Metric); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_core_stats_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Stats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_grpc_core_stats_proto_msgTypes[2].OneofWrappers = []interface{}{
+		(*Metric_Count)(nil),
+		(*Metric_Histogram)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_core_stats_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_core_stats_proto_goTypes,
+		DependencyIndexes: file_grpc_core_stats_proto_depIdxs,
+		MessageInfos:      file_grpc_core_stats_proto_msgTypes,
+	}.Build()
+	File_grpc_core_stats_proto = out.File
+	file_grpc_core_stats_proto_rawDesc = nil
+	file_grpc_core_stats_proto_goTypes = nil
+	file_grpc_core_stats_proto_depIdxs = nil
+}
diff --git a/interop/grpc_testing/empty.pb.go b/interop/grpc_testing/empty.pb.go
new file mode 100644
index 000000000000..5378d2c58d0f
--- /dev/null
+++ b/interop/grpc_testing/empty.pb.go
@@ -0,0 +1,158 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/empty.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+// An empty message that you can re-use to avoid defining duplicated empty
+// messages in your project. A typical example is to use it as argument or the
+// return value of a service API. For instance:
+//
+//   service Foo {
+//     rpc Bar (grpc.testing.Empty) returns (grpc.testing.Empty) { };
+//   };
+//
+type Empty struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *Empty) Reset() {
+	*x = Empty{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_empty_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Empty) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Empty) ProtoMessage() {}
+
+func (x *Empty) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_empty_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Empty.ProtoReflect.Descriptor instead.
+func (*Empty) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_empty_proto_rawDescGZIP(), []int{0}
+}
+
+var File_grpc_testing_empty_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_empty_proto_rawDesc = []byte{
+	0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x65,
+	0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74,
+	0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_testing_empty_proto_rawDescOnce sync.Once
+	file_grpc_testing_empty_proto_rawDescData = file_grpc_testing_empty_proto_rawDesc
+)
+
+func file_grpc_testing_empty_proto_rawDescGZIP() []byte {
+	file_grpc_testing_empty_proto_rawDescOnce.Do(func() {
+		file_grpc_testing_empty_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_empty_proto_rawDescData)
+	})
+	return file_grpc_testing_empty_proto_rawDescData
+}
+
+var file_grpc_testing_empty_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_grpc_testing_empty_proto_goTypes = []interface{}{
+	(*Empty)(nil), // 0: grpc.testing.Empty
+}
+var file_grpc_testing_empty_proto_depIdxs = []int32{
+	0, // [0:0] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_empty_proto_init() }
+func file_grpc_testing_empty_proto_init() {
+	if File_grpc_testing_empty_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_testing_empty_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Empty); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_empty_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_testing_empty_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_empty_proto_depIdxs,
+		MessageInfos:      file_grpc_testing_empty_proto_msgTypes,
+	}.Build()
+	File_grpc_testing_empty_proto = out.File
+	file_grpc_testing_empty_proto_rawDesc = nil
+	file_grpc_testing_empty_proto_goTypes = nil
+	file_grpc_testing_empty_proto_depIdxs = nil
+}
diff --git a/interop/grpc_testing/messages.pb.go b/interop/grpc_testing/messages.pb.go
new file mode 100644
index 000000000000..fb141c57df04
--- /dev/null
+++ b/interop/grpc_testing/messages.pb.go
@@ -0,0 +1,2035 @@
+// Copyright 2015-2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Message definitions to be used by integration test service definitions.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/messages.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+// The type of payload that should be returned.
+type PayloadType int32
+
+const (
+	// Compressable text format.
+	PayloadType_COMPRESSABLE PayloadType = 0
+)
+
+// Enum value maps for PayloadType.
+var (
+	PayloadType_name = map[int32]string{
+		0: "COMPRESSABLE",
+	}
+	PayloadType_value = map[string]int32{
+		"COMPRESSABLE": 0,
+	}
+)
+
+func (x PayloadType) Enum() *PayloadType {
+	p := new(PayloadType)
+	*p = x
+	return p
+}
+
+func (x PayloadType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (PayloadType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_testing_messages_proto_enumTypes[0].Descriptor()
+}
+
+func (PayloadType) Type() protoreflect.EnumType {
+	return &file_grpc_testing_messages_proto_enumTypes[0]
+}
+
+func (x PayloadType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use PayloadType.Descriptor instead.
+func (PayloadType) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{0}
+}
+
+// The type of route that a client took to reach a server w.r.t. gRPCLB.
+// The server must fill in "fallback" if it detects that the RPC reached
+// the server via the "gRPCLB fallback" path, and "backend" if it detects
+// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got
+// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly
+// how this detection is done is context and server dependent.
+type GrpclbRouteType int32
+
+const (
+	// Server didn't detect the route that a client took to reach it.
+	GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN GrpclbRouteType = 0
+	// Indicates that a client reached a server via gRPCLB fallback.
+	GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK GrpclbRouteType = 1
+	// Indicates that a client reached a server as a gRPCLB-given backend.
+	GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND GrpclbRouteType = 2
+)
+
+// Enum value maps for GrpclbRouteType.
+var (
+	GrpclbRouteType_name = map[int32]string{
+		0: "GRPCLB_ROUTE_TYPE_UNKNOWN",
+		1: "GRPCLB_ROUTE_TYPE_FALLBACK",
+		2: "GRPCLB_ROUTE_TYPE_BACKEND",
+	}
+	GrpclbRouteType_value = map[string]int32{
+		"GRPCLB_ROUTE_TYPE_UNKNOWN":  0,
+		"GRPCLB_ROUTE_TYPE_FALLBACK": 1,
+		"GRPCLB_ROUTE_TYPE_BACKEND":  2,
+	}
+)
+
+func (x GrpclbRouteType) Enum() *GrpclbRouteType {
+	p := new(GrpclbRouteType)
+	*p = x
+	return p
+}
+
+func (x GrpclbRouteType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (GrpclbRouteType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_testing_messages_proto_enumTypes[1].Descriptor()
+}
+
+func (GrpclbRouteType) Type() protoreflect.EnumType {
+	return &file_grpc_testing_messages_proto_enumTypes[1]
+}
+
+func (x GrpclbRouteType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use GrpclbRouteType.Descriptor instead.
+func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{1}
+}
+
+// Type of RPCs to send.
+type ClientConfigureRequest_RpcType int32
+
+const (
+	ClientConfigureRequest_EMPTY_CALL ClientConfigureRequest_RpcType = 0
+	ClientConfigureRequest_UNARY_CALL ClientConfigureRequest_RpcType = 1
+)
+
+// Enum value maps for ClientConfigureRequest_RpcType.
+var (
+	ClientConfigureRequest_RpcType_name = map[int32]string{
+		0: "EMPTY_CALL",
+		1: "UNARY_CALL",
+	}
+	ClientConfigureRequest_RpcType_value = map[string]int32{
+		"EMPTY_CALL": 0,
+		"UNARY_CALL": 1,
+	}
+)
+
+func (x ClientConfigureRequest_RpcType) Enum() *ClientConfigureRequest_RpcType {
+	p := new(ClientConfigureRequest_RpcType)
+	*p = x
+	return p
+}
+
+func (x ClientConfigureRequest_RpcType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ClientConfigureRequest_RpcType) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_testing_messages_proto_enumTypes[2].Descriptor()
+}
+
+func (ClientConfigureRequest_RpcType) Type() protoreflect.EnumType {
+	return &file_grpc_testing_messages_proto_enumTypes[2]
+}
+
+func (x ClientConfigureRequest_RpcType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ClientConfigureRequest_RpcType.Descriptor instead.
+func (ClientConfigureRequest_RpcType) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16, 0}
+}
+
+// TODO(dgq): Go back to using well-known types once
+// https://github.com/grpc/grpc/issues/6980 has been fixed.
+// import "google/protobuf/wrappers.proto";
+type BoolValue struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The bool value.
+	Value bool `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (x *BoolValue) Reset() {
+	*x = BoolValue{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *BoolValue) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BoolValue) ProtoMessage() {}
+
+func (x *BoolValue) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BoolValue.ProtoReflect.Descriptor instead.
+func (*BoolValue) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *BoolValue) GetValue() bool {
+	if x != nil {
+		return x.Value
+	}
+	return false
+}
+
+// A block of data, to simply increase gRPC message size.
+type Payload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The type of data in body.
+	Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"`
+	// Primary contents of payload.
+	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
+}
+
+func (x *Payload) Reset() {
+	*x = Payload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Payload) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Payload) ProtoMessage() {}
+
+func (x *Payload) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Payload.ProtoReflect.Descriptor instead.
+func (*Payload) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Payload) GetType() PayloadType {
+	if x != nil {
+		return x.Type
+	}
+	return PayloadType_COMPRESSABLE
+}
+
+func (x *Payload) GetBody() []byte {
+	if x != nil {
+		return x.Body
+	}
+	return nil
+}
+
+// A protobuf representation for grpc status. This is used by test
+// clients to specify a status that the server should attempt to return.
+type EchoStatus struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Code    int32  `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
+	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+}
+
+func (x *EchoStatus) Reset() {
+	*x = EchoStatus{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *EchoStatus) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EchoStatus) ProtoMessage() {}
+
+func (x *EchoStatus) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead.
+func (*EchoStatus) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *EchoStatus) GetCode() int32 {
+	if x != nil {
+		return x.Code
+	}
+	return 0
+}
+
+func (x *EchoStatus) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+// Unary request.
+type SimpleRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Desired payload type in the response from the server.
+	// If response_type is RANDOM, server randomly chooses one from other formats.
+	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
+	// Desired payload size in the response from the server.
+	ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"`
+	// Optional input payload sent along with the request.
+	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
+	// Whether SimpleResponse should include username.
+	FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"`
+	// Whether SimpleResponse should include OAuth scope.
+	FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"`
+	// Whether to request the server to compress the response. This field is
+	// "nullable" in order to interoperate seamlessly with clients not able to
+	// implement the full compression tests by introspecting the call to verify
+	// the response's compression status.
+	ResponseCompressed *BoolValue `protobuf:"bytes,6,opt,name=response_compressed,json=responseCompressed,proto3" json:"response_compressed,omitempty"`
+	// Whether server should return a given status
+	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
+	// Whether the server should expect this request to be compressed.
+	ExpectCompressed *BoolValue `protobuf:"bytes,8,opt,name=expect_compressed,json=expectCompressed,proto3" json:"expect_compressed,omitempty"`
+	// Whether SimpleResponse should include server_id.
+	FillServerId bool `protobuf:"varint,9,opt,name=fill_server_id,json=fillServerId,proto3" json:"fill_server_id,omitempty"`
+	// Whether SimpleResponse should include grpclb_route_type.
+	FillGrpclbRouteType bool `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"`
+}
+
+func (x *SimpleRequest) Reset() {
+	*x = SimpleRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SimpleRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SimpleRequest) ProtoMessage() {}
+
+func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
+func (*SimpleRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *SimpleRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
+	}
+	return PayloadType_COMPRESSABLE
+}
+
+func (x *SimpleRequest) GetResponseSize() int32 {
+	if x != nil {
+		return x.ResponseSize
+	}
+	return 0
+}
+
+func (x *SimpleRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+func (x *SimpleRequest) GetFillUsername() bool {
+	if x != nil {
+		return x.FillUsername
+	}
+	return false
+}
+
+func (x *SimpleRequest) GetFillOauthScope() bool {
+	if x != nil {
+		return x.FillOauthScope
+	}
+	return false
+}
+
+func (x *SimpleRequest) GetResponseCompressed() *BoolValue {
+	if x != nil {
+		return x.ResponseCompressed
+	}
+	return nil
+}
+
+func (x *SimpleRequest) GetResponseStatus() *EchoStatus {
+	if x != nil {
+		return x.ResponseStatus
+	}
+	return nil
+}
+
+func (x *SimpleRequest) GetExpectCompressed() *BoolValue {
+	if x != nil {
+		return x.ExpectCompressed
+	}
+	return nil
+}
+
+func (x *SimpleRequest) GetFillServerId() bool {
+	if x != nil {
+		return x.FillServerId
+	}
+	return false
+}
+
+func (x *SimpleRequest) GetFillGrpclbRouteType() bool {
+	if x != nil {
+		return x.FillGrpclbRouteType
+	}
+	return false
+}
+
+// Unary response, as configured by the request.
+type SimpleResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Payload to increase message size.
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
+	// The user the request came from, for verifying authentication was
+	// successful when the client expected it.
+	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
+	// OAuth scope.
+	OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
+	// Server ID. This must be unique among different server instances,
+	// but the same across all RPC's made to a particular server instance.
+	ServerId string `protobuf:"bytes,4,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
+	// gRPCLB Path.
+	GrpclbRouteType GrpclbRouteType `protobuf:"varint,5,opt,name=grpclb_route_type,json=grpclbRouteType,proto3,enum=grpc.testing.GrpclbRouteType" json:"grpclb_route_type,omitempty"`
+	// Server hostname.
+	Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"`
+}
+
+func (x *SimpleResponse) Reset() {
+	*x = SimpleResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SimpleResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SimpleResponse) ProtoMessage() {}
+
+func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
+func (*SimpleResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *SimpleResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+func (x *SimpleResponse) GetUsername() string {
+	if x != nil {
+		return x.Username
+	}
+	return ""
+}
+
+func (x *SimpleResponse) GetOauthScope() string {
+	if x != nil {
+		return x.OauthScope
+	}
+	return ""
+}
+
+func (x *SimpleResponse) GetServerId() string {
+	if x != nil {
+		return x.ServerId
+	}
+	return ""
+}
+
+func (x *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType {
+	if x != nil {
+		return x.GrpclbRouteType
+	}
+	return GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN
+}
+
+func (x *SimpleResponse) GetHostname() string {
+	if x != nil {
+		return x.Hostname
+	}
+	return ""
+}
+
+// Client-streaming request.
+type StreamingInputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Optional input payload sent along with the request.
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
+	// Whether the server should expect this request to be compressed. This field
+	// is "nullable" in order to interoperate seamlessly with servers not able to
+	// implement the full compression tests by introspecting the call to verify
+	// the request's compression status.
+	ExpectCompressed *BoolValue `protobuf:"bytes,2,opt,name=expect_compressed,json=expectCompressed,proto3" json:"expect_compressed,omitempty"`
+}
+
+func (x *StreamingInputCallRequest) Reset() {
+	*x = StreamingInputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *StreamingInputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StreamingInputCallRequest) ProtoMessage() {}
+
+func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *StreamingInputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+func (x *StreamingInputCallRequest) GetExpectCompressed() *BoolValue {
+	if x != nil {
+		return x.ExpectCompressed
+	}
+	return nil
+}
+
+// Client-streaming response.
+type StreamingInputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Aggregated size of payloads received from the client.
+	AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
+}
+
+func (x *StreamingInputCallResponse) Reset() {
+	*x = StreamingInputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *StreamingInputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StreamingInputCallResponse) ProtoMessage() {}
+
+func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
+	if x != nil {
+		return x.AggregatedPayloadSize
+	}
+	return 0
+}
+
+// Configuration for a particular response.
+type ResponseParameters struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Desired payload sizes in responses from the server.
+	Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"`
+	// Desired interval between consecutive responses in the response stream in
+	// microseconds.
+	IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
+	// Whether to request the server to compress the response. This field is
+	// "nullable" in order to interoperate seamlessly with clients not able to
+	// implement the full compression tests by introspecting the call to verify
+	// the response's compression status.
+	Compressed *BoolValue `protobuf:"bytes,3,opt,name=compressed,proto3" json:"compressed,omitempty"`
+}
+
+func (x *ResponseParameters) Reset() {
+	*x = ResponseParameters{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ResponseParameters) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ResponseParameters) ProtoMessage() {}
+
+func (x *ResponseParameters) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead.
+func (*ResponseParameters) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *ResponseParameters) GetSize() int32 {
+	if x != nil {
+		return x.Size
+	}
+	return 0
+}
+
+func (x *ResponseParameters) GetIntervalUs() int32 {
+	if x != nil {
+		return x.IntervalUs
+	}
+	return 0
+}
+
+func (x *ResponseParameters) GetCompressed() *BoolValue {
+	if x != nil {
+		return x.Compressed
+	}
+	return nil
+}
+
+// Server-streaming request.
+type StreamingOutputCallRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Desired payload type in the response from the server.
+	// If response_type is RANDOM, the payload from each response in the stream
+	// might be of different types. This is to simulate a mixed type of payload
+	// stream.
+	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
+	// Configuration for each expected response message.
+	ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"`
+	// Optional input payload sent along with the request.
+	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
+	// Whether server should return a given status
+	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
+}
+
+func (x *StreamingOutputCallRequest) Reset() {
+	*x = StreamingOutputCallRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *StreamingOutputCallRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StreamingOutputCallRequest) ProtoMessage() {}
+
+func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *StreamingOutputCallRequest) GetResponseType() PayloadType {
+	if x != nil {
+		return x.ResponseType
+	}
+	return PayloadType_COMPRESSABLE
+}
+
+func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
+	if x != nil {
+		return x.ResponseParameters
+	}
+	return nil
+}
+
+func (x *StreamingOutputCallRequest) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
+	if x != nil {
+		return x.ResponseStatus
+	}
+	return nil
+}
+
+// Server-streaming response, as configured by the request and parameters.
+type StreamingOutputCallResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Payload to increase response size.
+	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
+}
+
+func (x *StreamingOutputCallResponse) Reset() {
+	*x = StreamingOutputCallResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *StreamingOutputCallResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StreamingOutputCallResponse) ProtoMessage() {}
+
+func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead.
+func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *StreamingOutputCallResponse) GetPayload() *Payload {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+// For reconnect interop test only.
+// Client tells server what reconnection parameters it used.
+type ReconnectParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	MaxReconnectBackoffMs int32 `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"`
+}
+
+func (x *ReconnectParams) Reset() {
+	*x = ReconnectParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ReconnectParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReconnectParams) ProtoMessage() {}
+
+func (x *ReconnectParams) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReconnectParams.ProtoReflect.Descriptor instead.
+func (*ReconnectParams) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *ReconnectParams) GetMaxReconnectBackoffMs() int32 {
+	if x != nil {
+		return x.MaxReconnectBackoffMs
+	}
+	return 0
+}
+
+// For reconnect interop test only.
+// Server tells client whether its reconnects are following the spec and the
+// reconnect backoffs it saw.
+type ReconnectInfo struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Passed    bool    `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"`
+	BackoffMs []int32 `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"`
+}
+
+func (x *ReconnectInfo) Reset() {
+	*x = ReconnectInfo{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ReconnectInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReconnectInfo) ProtoMessage() {}
+
+func (x *ReconnectInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReconnectInfo.ProtoReflect.Descriptor instead.
+func (*ReconnectInfo) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{11}
+}
+
+func (x *ReconnectInfo) GetPassed() bool {
+	if x != nil {
+		return x.Passed
+	}
+	return false
+}
+
+func (x *ReconnectInfo) GetBackoffMs() []int32 {
+	if x != nil {
+		return x.BackoffMs
+	}
+	return nil
+}
+
+type LoadBalancerStatsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Request stats for the next num_rpcs sent by client.
+	NumRpcs int32 `protobuf:"varint,1,opt,name=num_rpcs,json=numRpcs,proto3" json:"num_rpcs,omitempty"`
+	// If num_rpcs have not completed within timeout_sec, return partial results.
+	TimeoutSec int32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
+}
+
+func (x *LoadBalancerStatsRequest) Reset() {
+	*x = LoadBalancerStatsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[12]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerStatsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerStatsRequest) ProtoMessage() {}
+
+func (x *LoadBalancerStatsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[12]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerStatsRequest.ProtoReflect.Descriptor instead.
+func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{12}
+}
+
+func (x *LoadBalancerStatsRequest) GetNumRpcs() int32 {
+	if x != nil {
+		return x.NumRpcs
+	}
+	return 0
+}
+
+func (x *LoadBalancerStatsRequest) GetTimeoutSec() int32 {
+	if x != nil {
+		return x.TimeoutSec
+	}
+	return 0
+}
+
+type LoadBalancerStatsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The number of completed RPCs for each peer.
+	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	// The number of RPCs that failed to record a remote peer.
+	NumFailures  int32                                            `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"`
+	RpcsByMethod map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (x *LoadBalancerStatsResponse) Reset() {
+	*x = LoadBalancerStatsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[13]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerStatsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerStatsResponse) ProtoMessage() {}
+
+func (x *LoadBalancerStatsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[13]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerStatsResponse.ProtoReflect.Descriptor instead.
+func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13}
+}
+
+func (x *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 {
+	if x != nil {
+		return x.RpcsByPeer
+	}
+	return nil
+}
+
+func (x *LoadBalancerStatsResponse) GetNumFailures() int32 {
+	if x != nil {
+		return x.NumFailures
+	}
+	return 0
+}
+
+func (x *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer {
+	if x != nil {
+		return x.RpcsByMethod
+	}
+	return nil
+}
+
+// Request for retrieving a test client's accumulated stats.
+type LoadBalancerAccumulatedStatsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *LoadBalancerAccumulatedStatsRequest) Reset() {
+	*x = LoadBalancerAccumulatedStatsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[14]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerAccumulatedStatsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerAccumulatedStatsRequest) ProtoMessage() {}
+
+func (x *LoadBalancerAccumulatedStatsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[14]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerAccumulatedStatsRequest.ProtoReflect.Descriptor instead.
+func (*LoadBalancerAccumulatedStatsRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{14}
+}
+
+// Accumulated stats for RPCs sent by a test client.
+type LoadBalancerAccumulatedStatsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The total number of RPCs have ever issued for each type.
+	NumRpcsStartedByMethod map[string]int32 `protobuf:"bytes,1,rep,name=num_rpcs_started_by_method,json=numRpcsStartedByMethod,proto3" json:"num_rpcs_started_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	// The total number of RPCs have ever completed successfully for each type.
+	NumRpcsSucceededByMethod map[string]int32 `protobuf:"bytes,2,rep,name=num_rpcs_succeeded_by_method,json=numRpcsSucceededByMethod,proto3" json:"num_rpcs_succeeded_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	// The total number of RPCs have ever failed for each type.
+	NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) Reset() {
+	*x = LoadBalancerAccumulatedStatsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[15]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerAccumulatedStatsResponse) ProtoMessage() {}
+
+func (x *LoadBalancerAccumulatedStatsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[15]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerAccumulatedStatsResponse.ProtoReflect.Descriptor instead.
+func (*LoadBalancerAccumulatedStatsResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15}
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[string]int32 {
+	if x != nil {
+		return x.NumRpcsStartedByMethod
+	}
+	return nil
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map[string]int32 {
+	if x != nil {
+		return x.NumRpcsSucceededByMethod
+	}
+	return nil
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[string]int32 {
+	if x != nil {
+		return x.NumRpcsFailedByMethod
+	}
+	return nil
+}
+
+// Configurations for a test client.
+type ClientConfigureRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The types of RPCs the client sends.
+	Types []ClientConfigureRequest_RpcType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"types,omitempty"`
+	// The collection of custom metadata to be attached to RPCs sent by the client.
+	Metadata []*ClientConfigureRequest_Metadata `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"`
+}
+
+func (x *ClientConfigureRequest) Reset() {
+	*x = ClientConfigureRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[16]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfigureRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfigureRequest) ProtoMessage() {}
+
+func (x *ClientConfigureRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[16]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfigureRequest.ProtoReflect.Descriptor instead.
+func (*ClientConfigureRequest) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16}
+}
+
+func (x *ClientConfigureRequest) GetTypes() []ClientConfigureRequest_RpcType {
+	if x != nil {
+		return x.Types
+	}
+	return nil
+}
+
+func (x *ClientConfigureRequest) GetMetadata() []*ClientConfigureRequest_Metadata {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+// Response for updating a test client's configuration.
+type ClientConfigureResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *ClientConfigureResponse) Reset() {
+	*x = ClientConfigureResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[17]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfigureResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfigureResponse) ProtoMessage() {}
+
+func (x *ClientConfigureResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[17]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfigureResponse.ProtoReflect.Descriptor instead.
+func (*ClientConfigureResponse) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{17}
+}
+
+type LoadBalancerStatsResponse_RpcsByPeer struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The number of completed RPCs for each peer.
+	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+}
+
+func (x *LoadBalancerStatsResponse_RpcsByPeer) Reset() {
+	*x = LoadBalancerStatsResponse_RpcsByPeer{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[18]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerStatsResponse_RpcsByPeer) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {}
+
+func (x *LoadBalancerStatsResponse_RpcsByPeer) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[18]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerStatsResponse_RpcsByPeer.ProtoReflect.Descriptor instead.
+func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 0}
+}
+
+func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 {
+	if x != nil {
+		return x.RpcsByPeer
+	}
+	return nil
+}
+
+// Metadata to be attached for the given type of RPCs.
+type ClientConfigureRequest_Metadata struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type  ClientConfigureRequest_RpcType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"type,omitempty"`
+	Key   string                         `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	Value string                         `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (x *ClientConfigureRequest_Metadata) Reset() {
+	*x = ClientConfigureRequest_Metadata{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[25]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientConfigureRequest_Metadata) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientConfigureRequest_Metadata) ProtoMessage() {}
+
+func (x *ClientConfigureRequest_Metadata) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[25]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientConfigureRequest_Metadata.ProtoReflect.Descriptor instead.
+func (*ClientConfigureRequest_Metadata) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16, 0}
+}
+
+func (x *ClientConfigureRequest_Metadata) GetType() ClientConfigureRequest_RpcType {
+	if x != nil {
+		return x.Type
+	}
+	return ClientConfigureRequest_EMPTY_CALL
+}
+
+func (x *ClientConfigureRequest_Metadata) GetKey() string {
+	if x != nil {
+		return x.Key
+	}
+	return ""
+}
+
+func (x *ClientConfigureRequest_Metadata) GetValue() string {
+	if x != nil {
+		return x.Value
+	}
+	return ""
+}
+
+var File_grpc_testing_messages_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_messages_proto_rawDesc = []byte{
+	0x0a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x21, 0x0a, 0x09, 0x42,
+	0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x4c,
+	0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79,
+	0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x3a, 0x0a, 0x0a,
+	0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f,
+	0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18,
+	0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xa2, 0x04, 0x0a, 0x0d, 0x53, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12,
+	0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d,
+	0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65,
+	0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61,
+	0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52,
+	0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12,
+	0x48, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70,
+	0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f, 0x6c,
+	0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43,
+	0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x11,
+	0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65,
+	0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65,
+	0x52, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73,
+	0x65, 0x64, 0x12, 0x24, 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c,
+	0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x66, 0x69, 0x6c, 0x6c,
+	0x5f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79,
+	0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x6c, 0x47, 0x72,
+	0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, 0x82, 0x02,
+	0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a,
+	0x0b, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x0a, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1b,
+	0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x49, 0x0a, 0x11, 0x67,
+	0x72, 0x70, 0x63, 0x6c, 0x62, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74,
+	0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75,
+	0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61,
+	0x6d, 0x65, 0x22, 0x92, 0x01, 0x0a, 0x19, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67,
+	0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x12, 0x44, 0x0a, 0x11, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70,
+	0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f, 0x6c,
+	0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6d,
+	0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61,
+	0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74,
+	0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x82, 0x01,
+	0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65,
+	0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x65,
+	0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x69,
+	0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x12, 0x37, 0x0a, 0x0a, 0x63, 0x6f, 0x6d,
+	0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f,
+	0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73,
+	0x65, 0x64, 0x22, 0xa3, 0x02, 0x0a, 0x1a, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67,
+	0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79,
+	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54,
+	0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61,
+	0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73,
+	0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65,
+	0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63,
+	0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x4e, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x4a, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f,
+	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x6d,
+	0x61, 0x78, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x62, 0x61, 0x63,
+	0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6d,
+	0x61, 0x78, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x6f,
+	0x66, 0x66, 0x4d, 0x73, 0x22, 0x46, 0x0a, 0x0d, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
+	0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x12, 0x1d, 0x0a,
+	0x0a, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x05, 0x52, 0x09, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x22, 0x56, 0x0a, 0x18,
+	0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f,
+	0x72, 0x70, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52,
+	0x70, 0x63, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73,
+	0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75,
+	0x74, 0x53, 0x65, 0x63, 0x22, 0xe2, 0x04, 0x0a, 0x19, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c,
+	0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x12, 0x59, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x70, 0x65,
+	0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61,
+	0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x0a, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x21, 0x0a,
+	0x0c, 0x6e, 0x75, 0x6d, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73,
+	0x12, 0x5f, 0x0a, 0x0e, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68,
+	0x6f, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61,
+	0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x52, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x1a, 0xb1, 0x01, 0x0a, 0x0a, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72,
+	0x12, 0x64, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x70, 0x65, 0x65, 0x72,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e,
+	0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42,
+	0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x70, 0x63, 0x73,
+	0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79,
+	0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3d, 0x0a, 0x0f, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50,
+	0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x3a, 0x02, 0x38, 0x01, 0x1a, 0x73, 0x0a, 0x11, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65,
+	0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
+	0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x23, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c,
+	0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x22, 0xb2, 0x05, 0x0a, 0x24, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
+	0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x1a, 0x6e, 0x75,
+	0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x62,
+	0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75,
+	0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65,
+	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x16,
+	0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79,
+	0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x90, 0x01, 0x0a, 0x1c, 0x6e, 0x75, 0x6d, 0x5f, 0x72,
+	0x70, 0x63, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x5f, 0x62, 0x79,
+	0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c,
+	0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
+	0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
+	0x18, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65,
+	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x87, 0x01, 0x0a, 0x19, 0x6e, 0x75,
+	0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x62, 0x79,
+	0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
+	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c,
+	0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42,
+	0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x15, 0x6e, 0x75,
+	0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74,
+	0x68, 0x6f, 0x64, 0x1a, 0x49, 0x0a, 0x1b, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74,
+	0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74,
+	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4b,
+	0x0a, 0x1d, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
+	0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
+	0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
+	0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x48, 0x0a, 0x1a, 0x4e,
+	0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65,
+	0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc8, 0x02, 0x0a, 0x16, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x42, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32,
+	0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x74,
+	0x79, 0x70, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66,
+	0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a,
+	0x74, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, 0x04, 0x74,
+	0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e,
+	0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x07, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65,
+	0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x43, 0x41, 0x4c, 0x4c, 0x10, 0x00,
+	0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x5f, 0x43, 0x41, 0x4c, 0x4c, 0x10, 0x01,
+	0x22, 0x19, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x1f, 0x0a, 0x0b, 0x50,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f,
+	0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x2a, 0x6f, 0x0a, 0x0f,
+	0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1e,
+	0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x01, 0x12, 0x1d,
+	0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, 0x02, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_testing_messages_proto_rawDescOnce sync.Once
+	file_grpc_testing_messages_proto_rawDescData = file_grpc_testing_messages_proto_rawDesc
+)
+
+func file_grpc_testing_messages_proto_rawDescGZIP() []byte {
+	file_grpc_testing_messages_proto_rawDescOnce.Do(func() {
+		file_grpc_testing_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_messages_proto_rawDescData)
+	})
+	return file_grpc_testing_messages_proto_rawDescData
+}
+
+var file_grpc_testing_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 26)
+var file_grpc_testing_messages_proto_goTypes = []interface{}{
+	(PayloadType)(0),                             // 0: grpc.testing.PayloadType
+	(GrpclbRouteType)(0),                         // 1: grpc.testing.GrpclbRouteType
+	(ClientConfigureRequest_RpcType)(0),          // 2: grpc.testing.ClientConfigureRequest.RpcType
+	(*BoolValue)(nil),                            // 3: grpc.testing.BoolValue
+	(*Payload)(nil),                              // 4: grpc.testing.Payload
+	(*EchoStatus)(nil),                           // 5: grpc.testing.EchoStatus
+	(*SimpleRequest)(nil),                        // 6: grpc.testing.SimpleRequest
+	(*SimpleResponse)(nil),                       // 7: grpc.testing.SimpleResponse
+	(*StreamingInputCallRequest)(nil),            // 8: grpc.testing.StreamingInputCallRequest
+	(*StreamingInputCallResponse)(nil),           // 9: grpc.testing.StreamingInputCallResponse
+	(*ResponseParameters)(nil),                   // 10: grpc.testing.ResponseParameters
+	(*StreamingOutputCallRequest)(nil),           // 11: grpc.testing.StreamingOutputCallRequest
+	(*StreamingOutputCallResponse)(nil),          // 12: grpc.testing.StreamingOutputCallResponse
+	(*ReconnectParams)(nil),                      // 13: grpc.testing.ReconnectParams
+	(*ReconnectInfo)(nil),                        // 14: grpc.testing.ReconnectInfo
+	(*LoadBalancerStatsRequest)(nil),             // 15: grpc.testing.LoadBalancerStatsRequest
+	(*LoadBalancerStatsResponse)(nil),            // 16: grpc.testing.LoadBalancerStatsResponse
+	(*LoadBalancerAccumulatedStatsRequest)(nil),  // 17: grpc.testing.LoadBalancerAccumulatedStatsRequest
+	(*LoadBalancerAccumulatedStatsResponse)(nil), // 18: grpc.testing.LoadBalancerAccumulatedStatsResponse
+	(*ClientConfigureRequest)(nil),               // 19: grpc.testing.ClientConfigureRequest
+	(*ClientConfigureResponse)(nil),              // 20: grpc.testing.ClientConfigureResponse
+	(*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 21: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	nil,                                     // 22: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	nil,                                     // 23: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	nil,                                     // 24: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	nil,                                     // 25: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
+	nil,                                     // 26: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
+	nil,                                     // 27: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
+	(*ClientConfigureRequest_Metadata)(nil), // 28: grpc.testing.ClientConfigureRequest.Metadata
+}
+var file_grpc_testing_messages_proto_depIdxs = []int32{
+	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
+	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
+	4,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
+	3,  // 3: grpc.testing.SimpleRequest.response_compressed:type_name -> grpc.testing.BoolValue
+	5,  // 4: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
+	3,  // 5: grpc.testing.SimpleRequest.expect_compressed:type_name -> grpc.testing.BoolValue
+	4,  // 6: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
+	1,  // 7: grpc.testing.SimpleResponse.grpclb_route_type:type_name -> grpc.testing.GrpclbRouteType
+	4,  // 8: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
+	3,  // 9: grpc.testing.StreamingInputCallRequest.expect_compressed:type_name -> grpc.testing.BoolValue
+	3,  // 10: grpc.testing.ResponseParameters.compressed:type_name -> grpc.testing.BoolValue
+	0,  // 11: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
+	10, // 12: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
+	4,  // 13: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
+	5,  // 14: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
+	4,  // 15: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
+	22, // 16: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	23, // 17: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	25, // 18: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_started_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
+	26, // 19: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_succeeded_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
+	27, // 20: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_failed_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
+	2,  // 21: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType
+	28, // 22: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata
+	24, // 23: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	21, // 24: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	2,  // 25: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType
+	26, // [26:26] is the sub-list for method output_type
+	26, // [26:26] is the sub-list for method input_type
+	26, // [26:26] is the sub-list for extension type_name
+	26, // [26:26] is the sub-list for extension extendee
+	0,  // [0:26] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_messages_proto_init() }
+func file_grpc_testing_messages_proto_init() {
+	if File_grpc_testing_messages_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_testing_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*BoolValue); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Payload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoStatus); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SimpleResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingInputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ResponseParameters); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamingOutputCallResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ReconnectParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ReconnectInfo); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerStatsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerStatsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerAccumulatedStatsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerAccumulatedStatsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfigureRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfigureResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerStatsResponse_RpcsByPeer); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientConfigureRequest_Metadata); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_messages_proto_rawDesc,
+			NumEnums:      3,
+			NumMessages:   26,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_testing_messages_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_messages_proto_depIdxs,
+		EnumInfos:         file_grpc_testing_messages_proto_enumTypes,
+		MessageInfos:      file_grpc_testing_messages_proto_msgTypes,
+	}.Build()
+	File_grpc_testing_messages_proto = out.File
+	file_grpc_testing_messages_proto_rawDesc = nil
+	file_grpc_testing_messages_proto_goTypes = nil
+	file_grpc_testing_messages_proto_depIdxs = nil
+}
diff --git a/benchmark/grpc_testing/payloads.pb.go b/interop/grpc_testing/payloads.pb.go
similarity index 59%
rename from benchmark/grpc_testing/payloads.pb.go
rename to interop/grpc_testing/payloads.pb.go
index f8a3edbd854f..b85a289c1532 100644
--- a/benchmark/grpc_testing/payloads.pb.go
+++ b/interop/grpc_testing/payloads.pb.go
@@ -1,4 +1,4 @@
-// Copyright 2016 gRPC authors.
+// Copyright 2015 gRPC authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 // versions:
 // 	protoc-gen-go v1.25.0
 // 	protoc        v3.14.0
-// source: benchmark/grpc_testing/payloads.proto
+// source: grpc/testing/payloads.proto
 
 package grpc_testing
 
@@ -51,7 +51,7 @@ type ByteBufferParams struct {
 func (x *ByteBufferParams) Reset() {
 	*x = ByteBufferParams{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[0]
+		mi := &file_grpc_testing_payloads_proto_msgTypes[0]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -64,7 +64,7 @@ func (x *ByteBufferParams) String() string {
 func (*ByteBufferParams) ProtoMessage() {}
 
 func (x *ByteBufferParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[0]
+	mi := &file_grpc_testing_payloads_proto_msgTypes[0]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -77,7 +77,7 @@ func (x *ByteBufferParams) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ByteBufferParams.ProtoReflect.Descriptor instead.
 func (*ByteBufferParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{0}
+	return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{0}
 }
 
 func (x *ByteBufferParams) GetReqSize() int32 {
@@ -106,7 +106,7 @@ type SimpleProtoParams struct {
 func (x *SimpleProtoParams) Reset() {
 	*x = SimpleProtoParams{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[1]
+		mi := &file_grpc_testing_payloads_proto_msgTypes[1]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -119,7 +119,7 @@ func (x *SimpleProtoParams) String() string {
 func (*SimpleProtoParams) ProtoMessage() {}
 
 func (x *SimpleProtoParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[1]
+	mi := &file_grpc_testing_payloads_proto_msgTypes[1]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -132,7 +132,7 @@ func (x *SimpleProtoParams) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use SimpleProtoParams.ProtoReflect.Descriptor instead.
 func (*SimpleProtoParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{1}
+	return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{1}
 }
 
 func (x *SimpleProtoParams) GetReqSize() int32 {
@@ -158,7 +158,7 @@ type ComplexProtoParams struct {
 func (x *ComplexProtoParams) Reset() {
 	*x = ComplexProtoParams{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[2]
+		mi := &file_grpc_testing_payloads_proto_msgTypes[2]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -171,7 +171,7 @@ func (x *ComplexProtoParams) String() string {
 func (*ComplexProtoParams) ProtoMessage() {}
 
 func (x *ComplexProtoParams) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[2]
+	mi := &file_grpc_testing_payloads_proto_msgTypes[2]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -184,7 +184,7 @@ func (x *ComplexProtoParams) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ComplexProtoParams.ProtoReflect.Descriptor instead.
 func (*ComplexProtoParams) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{2}
+	return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{2}
 }
 
 type PayloadConfig struct {
@@ -202,7 +202,7 @@ type PayloadConfig struct {
 func (x *PayloadConfig) Reset() {
 	*x = PayloadConfig{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[3]
+		mi := &file_grpc_testing_payloads_proto_msgTypes[3]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -215,7 +215,7 @@ func (x *PayloadConfig) String() string {
 func (*PayloadConfig) ProtoMessage() {}
 
 func (x *PayloadConfig) ProtoReflect() protoreflect.Message {
-	mi := &file_benchmark_grpc_testing_payloads_proto_msgTypes[3]
+	mi := &file_grpc_testing_payloads_proto_msgTypes[3]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -228,7 +228,7 @@ func (x *PayloadConfig) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use PayloadConfig.ProtoReflect.Descriptor instead.
 func (*PayloadConfig) Descriptor() ([]byte, []int) {
-	return file_benchmark_grpc_testing_payloads_proto_rawDescGZIP(), []int{3}
+	return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{3}
 }
 
 func (m *PayloadConfig) GetPayload() isPayloadConfig_Payload {
@@ -281,65 +281,61 @@ func (*PayloadConfig_SimpleParams) isPayloadConfig_Payload() {}
 
 func (*PayloadConfig_ComplexParams) isPayloadConfig_Payload() {}
 
-var File_benchmark_grpc_testing_payloads_proto protoreflect.FileDescriptor
-
-var file_benchmark_grpc_testing_payloads_proto_rawDesc = []byte{
-	0x0a, 0x25, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
-	0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x4a, 0x0a, 0x10, 0x42, 0x79, 0x74, 0x65, 0x42, 0x75, 0x66,
-	0x66, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x65, 0x71,
-	0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x71,
-	0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f, 0x73, 0x69, 0x7a,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x53, 0x69, 0x7a,
-	0x65, 0x22, 0x4b, 0x0a, 0x11, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f,
-	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x5f, 0x73, 0x69,
-	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x71, 0x53, 0x69, 0x7a,
-	0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x14,
-	0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61,
-	0x72, 0x61, 0x6d, 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x0d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
-	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x47, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75,
-	0x66, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x79,
-	0x74, 0x65, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00,
-	0x52, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12,
-	0x46, 0x0a, 0x0d, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74,
-	0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x69, 0x6d, 0x70, 0x6c,
-	0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x49, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x6c,
-	0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43,
-	0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d,
-	0x73, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x61, 0x72, 0x61,
-	0x6d, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x2f, 0x5a,
-	0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f,
-	0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72,
-	0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+var File_grpc_testing_payloads_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_payloads_proto_rawDesc = []byte{
+	0x0a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x4a, 0x0a, 0x10, 0x42,
+	0x79, 0x74, 0x65, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12,
+	0x19, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x07, 0x72, 0x65, 0x71, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65,
+	0x73, 0x70, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72,
+	0x65, 0x73, 0x70, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x4b, 0x0a, 0x11, 0x53, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08,
+	0x72, 0x65, 0x71, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07,
+	0x72, 0x65, 0x71, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f,
+	0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70,
+	0x53, 0x69, 0x7a, 0x65, 0x22, 0x14, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x0d, 0x50,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x47, 0x0a, 0x0e,
+	0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x50, 0x61,
+	0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x50,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x46, 0x0a, 0x0d, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x5f,
+	0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70,
+	0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52,
+	0x0c, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x49, 0x0a,
+	0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x6c,
+	0x65, 0x78, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
-	file_benchmark_grpc_testing_payloads_proto_rawDescOnce sync.Once
-	file_benchmark_grpc_testing_payloads_proto_rawDescData = file_benchmark_grpc_testing_payloads_proto_rawDesc
+	file_grpc_testing_payloads_proto_rawDescOnce sync.Once
+	file_grpc_testing_payloads_proto_rawDescData = file_grpc_testing_payloads_proto_rawDesc
 )
 
-func file_benchmark_grpc_testing_payloads_proto_rawDescGZIP() []byte {
-	file_benchmark_grpc_testing_payloads_proto_rawDescOnce.Do(func() {
-		file_benchmark_grpc_testing_payloads_proto_rawDescData = protoimpl.X.CompressGZIP(file_benchmark_grpc_testing_payloads_proto_rawDescData)
+func file_grpc_testing_payloads_proto_rawDescGZIP() []byte {
+	file_grpc_testing_payloads_proto_rawDescOnce.Do(func() {
+		file_grpc_testing_payloads_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_payloads_proto_rawDescData)
 	})
-	return file_benchmark_grpc_testing_payloads_proto_rawDescData
+	return file_grpc_testing_payloads_proto_rawDescData
 }
 
-var file_benchmark_grpc_testing_payloads_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
-var file_benchmark_grpc_testing_payloads_proto_goTypes = []interface{}{
+var file_grpc_testing_payloads_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_grpc_testing_payloads_proto_goTypes = []interface{}{
 	(*ByteBufferParams)(nil),   // 0: grpc.testing.ByteBufferParams
 	(*SimpleProtoParams)(nil),  // 1: grpc.testing.SimpleProtoParams
 	(*ComplexProtoParams)(nil), // 2: grpc.testing.ComplexProtoParams
 	(*PayloadConfig)(nil),      // 3: grpc.testing.PayloadConfig
 }
-var file_benchmark_grpc_testing_payloads_proto_depIdxs = []int32{
+var file_grpc_testing_payloads_proto_depIdxs = []int32{
 	0, // 0: grpc.testing.PayloadConfig.bytebuf_params:type_name -> grpc.testing.ByteBufferParams
 	1, // 1: grpc.testing.PayloadConfig.simple_params:type_name -> grpc.testing.SimpleProtoParams
 	2, // 2: grpc.testing.PayloadConfig.complex_params:type_name -> grpc.testing.ComplexProtoParams
@@ -350,13 +346,13 @@ var file_benchmark_grpc_testing_payloads_proto_depIdxs = []int32{
 	0, // [0:3] is the sub-list for field type_name
 }
 
-func init() { file_benchmark_grpc_testing_payloads_proto_init() }
-func file_benchmark_grpc_testing_payloads_proto_init() {
-	if File_benchmark_grpc_testing_payloads_proto != nil {
+func init() { file_grpc_testing_payloads_proto_init() }
+func file_grpc_testing_payloads_proto_init() {
+	if File_grpc_testing_payloads_proto != nil {
 		return
 	}
 	if !protoimpl.UnsafeEnabled {
-		file_benchmark_grpc_testing_payloads_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+		file_grpc_testing_payloads_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*ByteBufferParams); i {
 			case 0:
 				return &v.state
@@ -368,7 +364,7 @@ func file_benchmark_grpc_testing_payloads_proto_init() {
 				return nil
 			}
 		}
-		file_benchmark_grpc_testing_payloads_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+		file_grpc_testing_payloads_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*SimpleProtoParams); i {
 			case 0:
 				return &v.state
@@ -380,7 +376,7 @@ func file_benchmark_grpc_testing_payloads_proto_init() {
 				return nil
 			}
 		}
-		file_benchmark_grpc_testing_payloads_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+		file_grpc_testing_payloads_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*ComplexProtoParams); i {
 			case 0:
 				return &v.state
@@ -392,7 +388,7 @@ func file_benchmark_grpc_testing_payloads_proto_init() {
 				return nil
 			}
 		}
-		file_benchmark_grpc_testing_payloads_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+		file_grpc_testing_payloads_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*PayloadConfig); i {
 			case 0:
 				return &v.state
@@ -405,7 +401,7 @@ func file_benchmark_grpc_testing_payloads_proto_init() {
 			}
 		}
 	}
-	file_benchmark_grpc_testing_payloads_proto_msgTypes[3].OneofWrappers = []interface{}{
+	file_grpc_testing_payloads_proto_msgTypes[3].OneofWrappers = []interface{}{
 		(*PayloadConfig_BytebufParams)(nil),
 		(*PayloadConfig_SimpleParams)(nil),
 		(*PayloadConfig_ComplexParams)(nil),
@@ -414,18 +410,18 @@ func file_benchmark_grpc_testing_payloads_proto_init() {
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_benchmark_grpc_testing_payloads_proto_rawDesc,
+			RawDescriptor: file_grpc_testing_payloads_proto_rawDesc,
 			NumEnums:      0,
 			NumMessages:   4,
 			NumExtensions: 0,
 			NumServices:   0,
 		},
-		GoTypes:           file_benchmark_grpc_testing_payloads_proto_goTypes,
-		DependencyIndexes: file_benchmark_grpc_testing_payloads_proto_depIdxs,
-		MessageInfos:      file_benchmark_grpc_testing_payloads_proto_msgTypes,
+		GoTypes:           file_grpc_testing_payloads_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_payloads_proto_depIdxs,
+		MessageInfos:      file_grpc_testing_payloads_proto_msgTypes,
 	}.Build()
-	File_benchmark_grpc_testing_payloads_proto = out.File
-	file_benchmark_grpc_testing_payloads_proto_rawDesc = nil
-	file_benchmark_grpc_testing_payloads_proto_goTypes = nil
-	file_benchmark_grpc_testing_payloads_proto_depIdxs = nil
+	File_grpc_testing_payloads_proto = out.File
+	file_grpc_testing_payloads_proto_rawDesc = nil
+	file_grpc_testing_payloads_proto_goTypes = nil
+	file_grpc_testing_payloads_proto_depIdxs = nil
 }
diff --git a/interop/grpc_testing/report_qps_scenario_service.pb.go b/interop/grpc_testing/report_qps_scenario_service.pb.go
new file mode 100644
index 000000000000..0f4de5984942
--- /dev/null
+++ b/interop/grpc_testing/report_qps_scenario_service.pb.go
@@ -0,0 +1,99 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/report_qps_scenario_service.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+var File_grpc_testing_report_qps_scenario_service_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_report_qps_scenario_service_proto_rawDesc = []byte{
+	0x0a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x72,
+	0x65, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x71, 0x70, 0x73, 0x5f, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72,
+	0x69, 0x6f, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x1a,
+	0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x6e,
+	0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x5e, 0x0a, 0x18, 0x52, 0x65,
+	0x70, 0x6f, 0x72, 0x74, 0x51, 0x70, 0x73, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x0e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74,
+	0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f,
+	0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x33,
+}
+
+var file_grpc_testing_report_qps_scenario_service_proto_goTypes = []interface{}{
+	(*ScenarioResult)(nil), // 0: grpc.testing.ScenarioResult
+	(*Void)(nil),           // 1: grpc.testing.Void
+}
+var file_grpc_testing_report_qps_scenario_service_proto_depIdxs = []int32{
+	0, // 0: grpc.testing.ReportQpsScenarioService.ReportScenario:input_type -> grpc.testing.ScenarioResult
+	1, // 1: grpc.testing.ReportQpsScenarioService.ReportScenario:output_type -> grpc.testing.Void
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_report_qps_scenario_service_proto_init() }
+func file_grpc_testing_report_qps_scenario_service_proto_init() {
+	if File_grpc_testing_report_qps_scenario_service_proto != nil {
+		return
+	}
+	file_grpc_testing_control_proto_init()
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_report_qps_scenario_service_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   0,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_testing_report_qps_scenario_service_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_report_qps_scenario_service_proto_depIdxs,
+	}.Build()
+	File_grpc_testing_report_qps_scenario_service_proto = out.File
+	file_grpc_testing_report_qps_scenario_service_proto_rawDesc = nil
+	file_grpc_testing_report_qps_scenario_service_proto_goTypes = nil
+	file_grpc_testing_report_qps_scenario_service_proto_depIdxs = nil
+}
diff --git a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
new file mode 100644
index 000000000000..fb6c22a00d09
--- /dev/null
+++ b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
@@ -0,0 +1,103 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package grpc_testing
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion7
+
+// ReportQpsScenarioServiceClient is the client API for ReportQpsScenarioService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ReportQpsScenarioServiceClient interface {
+	// Report results of a QPS test benchmark scenario.
+	ReportScenario(ctx context.Context, in *ScenarioResult, opts ...grpc.CallOption) (*Void, error)
+}
+
+type reportQpsScenarioServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewReportQpsScenarioServiceClient(cc grpc.ClientConnInterface) ReportQpsScenarioServiceClient {
+	return &reportQpsScenarioServiceClient{cc}
+}
+
+func (c *reportQpsScenarioServiceClient) ReportScenario(ctx context.Context, in *ScenarioResult, opts ...grpc.CallOption) (*Void, error) {
+	out := new(Void)
+	err := c.cc.Invoke(ctx, "/grpc.testing.ReportQpsScenarioService/ReportScenario", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ReportQpsScenarioServiceServer is the server API for ReportQpsScenarioService service.
+// All implementations must embed UnimplementedReportQpsScenarioServiceServer
+// for forward compatibility
+type ReportQpsScenarioServiceServer interface {
+	// Report results of a QPS test benchmark scenario.
+	ReportScenario(context.Context, *ScenarioResult) (*Void, error)
+	mustEmbedUnimplementedReportQpsScenarioServiceServer()
+}
+
+// UnimplementedReportQpsScenarioServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedReportQpsScenarioServiceServer struct {
+}
+
+func (UnimplementedReportQpsScenarioServiceServer) ReportScenario(context.Context, *ScenarioResult) (*Void, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ReportScenario not implemented")
+}
+func (UnimplementedReportQpsScenarioServiceServer) mustEmbedUnimplementedReportQpsScenarioServiceServer() {
+}
+
+// UnsafeReportQpsScenarioServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to ReportQpsScenarioServiceServer will
+// result in compilation errors.
+type UnsafeReportQpsScenarioServiceServer interface {
+	mustEmbedUnimplementedReportQpsScenarioServiceServer()
+}
+
+func RegisterReportQpsScenarioServiceServer(s grpc.ServiceRegistrar, srv ReportQpsScenarioServiceServer) {
+	s.RegisterService(&ReportQpsScenarioService_ServiceDesc, srv)
+}
+
+func _ReportQpsScenarioService_ReportScenario_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ScenarioResult)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ReportQpsScenarioServiceServer).ReportScenario(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.ReportQpsScenarioService/ReportScenario",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ReportQpsScenarioServiceServer).ReportScenario(ctx, req.(*ScenarioResult))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// ReportQpsScenarioService_ServiceDesc is the grpc.ServiceDesc for ReportQpsScenarioService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var ReportQpsScenarioService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.ReportQpsScenarioService",
+	HandlerType: (*ReportQpsScenarioServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "ReportScenario",
+			Handler:    _ReportQpsScenarioService_ReportScenario_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/testing/report_qps_scenario_service.proto",
+}
diff --git a/interop/grpc_testing/stats.pb.go b/interop/grpc_testing/stats.pb.go
new file mode 100644
index 000000000000..3ff0ccd80b28
--- /dev/null
+++ b/interop/grpc_testing/stats.pb.go
@@ -0,0 +1,632 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/stats.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	core "google.golang.org/grpc/interop/grpc_testing/core"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+type ServerStats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// wall clock time change in seconds since last reset
+	TimeElapsed float64 `protobuf:"fixed64,1,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
+	// change in user time (in seconds) used by the server since last reset
+	TimeUser float64 `protobuf:"fixed64,2,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
+	// change in server time (in seconds) used by the server process and all
+	// threads since last reset
+	TimeSystem float64 `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
+	// change in total cpu time of the server (data from proc/stat)
+	TotalCpuTime uint64 `protobuf:"varint,4,opt,name=total_cpu_time,json=totalCpuTime,proto3" json:"total_cpu_time,omitempty"`
+	// change in idle time of the server (data from proc/stat)
+	IdleCpuTime uint64 `protobuf:"varint,5,opt,name=idle_cpu_time,json=idleCpuTime,proto3" json:"idle_cpu_time,omitempty"`
+	// Number of polls called inside completion queue
+	CqPollCount uint64 `protobuf:"varint,6,opt,name=cq_poll_count,json=cqPollCount,proto3" json:"cq_poll_count,omitempty"`
+	// Core library stats
+	CoreStats *core.Stats `protobuf:"bytes,7,opt,name=core_stats,json=coreStats,proto3" json:"core_stats,omitempty"`
+}
+
+func (x *ServerStats) Reset() {
+	*x = ServerStats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_stats_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ServerStats) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ServerStats) ProtoMessage() {}
+
+func (x *ServerStats) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_stats_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ServerStats.ProtoReflect.Descriptor instead.
+func (*ServerStats) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_stats_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ServerStats) GetTimeElapsed() float64 {
+	if x != nil {
+		return x.TimeElapsed
+	}
+	return 0
+}
+
+func (x *ServerStats) GetTimeUser() float64 {
+	if x != nil {
+		return x.TimeUser
+	}
+	return 0
+}
+
+func (x *ServerStats) GetTimeSystem() float64 {
+	if x != nil {
+		return x.TimeSystem
+	}
+	return 0
+}
+
+func (x *ServerStats) GetTotalCpuTime() uint64 {
+	if x != nil {
+		return x.TotalCpuTime
+	}
+	return 0
+}
+
+func (x *ServerStats) GetIdleCpuTime() uint64 {
+	if x != nil {
+		return x.IdleCpuTime
+	}
+	return 0
+}
+
+func (x *ServerStats) GetCqPollCount() uint64 {
+	if x != nil {
+		return x.CqPollCount
+	}
+	return 0
+}
+
+func (x *ServerStats) GetCoreStats() *core.Stats {
+	if x != nil {
+		return x.CoreStats
+	}
+	return nil
+}
+
+// Histogram params based on grpc/support/histogram.c
+type HistogramParams struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Resolution  float64 `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"`                      // first bucket is [0, 1 + resolution)
+	MaxPossible float64 `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"` // use enough buckets to allow this value
+}
+
+func (x *HistogramParams) Reset() {
+	*x = HistogramParams{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_stats_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *HistogramParams) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*HistogramParams) ProtoMessage() {}
+
+func (x *HistogramParams) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_stats_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use HistogramParams.ProtoReflect.Descriptor instead.
+func (*HistogramParams) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_stats_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *HistogramParams) GetResolution() float64 {
+	if x != nil {
+		return x.Resolution
+	}
+	return 0
+}
+
+func (x *HistogramParams) GetMaxPossible() float64 {
+	if x != nil {
+		return x.MaxPossible
+	}
+	return 0
+}
+
+// Histogram data based on grpc/support/histogram.c
+type HistogramData struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Bucket       []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"`
+	MinSeen      float64  `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"`
+	MaxSeen      float64  `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"`
+	Sum          float64  `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"`
+	SumOfSquares float64  `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"`
+	Count        float64  `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"`
+}
+
+func (x *HistogramData) Reset() {
+	*x = HistogramData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_stats_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *HistogramData) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*HistogramData) ProtoMessage() {}
+
+func (x *HistogramData) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_stats_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use HistogramData.ProtoReflect.Descriptor instead.
+func (*HistogramData) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_stats_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *HistogramData) GetBucket() []uint32 {
+	if x != nil {
+		return x.Bucket
+	}
+	return nil
+}
+
+func (x *HistogramData) GetMinSeen() float64 {
+	if x != nil {
+		return x.MinSeen
+	}
+	return 0
+}
+
+func (x *HistogramData) GetMaxSeen() float64 {
+	if x != nil {
+		return x.MaxSeen
+	}
+	return 0
+}
+
+func (x *HistogramData) GetSum() float64 {
+	if x != nil {
+		return x.Sum
+	}
+	return 0
+}
+
+func (x *HistogramData) GetSumOfSquares() float64 {
+	if x != nil {
+		return x.SumOfSquares
+	}
+	return 0
+}
+
+func (x *HistogramData) GetCount() float64 {
+	if x != nil {
+		return x.Count
+	}
+	return 0
+}
+
+type RequestResultCount struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	StatusCode int32 `protobuf:"varint,1,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"`
+	Count      int64 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"`
+}
+
+func (x *RequestResultCount) Reset() {
+	*x = RequestResultCount{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_stats_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RequestResultCount) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RequestResultCount) ProtoMessage() {}
+
+func (x *RequestResultCount) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_stats_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RequestResultCount.ProtoReflect.Descriptor instead.
+func (*RequestResultCount) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_stats_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *RequestResultCount) GetStatusCode() int32 {
+	if x != nil {
+		return x.StatusCode
+	}
+	return 0
+}
+
+func (x *RequestResultCount) GetCount() int64 {
+	if x != nil {
+		return x.Count
+	}
+	return 0
+}
+
+type ClientStats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Latency histogram. Data points are in nanoseconds.
+	Latencies *HistogramData `protobuf:"bytes,1,opt,name=latencies,proto3" json:"latencies,omitempty"`
+	// See ServerStats for details.
+	TimeElapsed float64 `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"`
+	TimeUser    float64 `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"`
+	TimeSystem  float64 `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"`
+	// Number of failed requests (one row per status code seen)
+	RequestResults []*RequestResultCount `protobuf:"bytes,5,rep,name=request_results,json=requestResults,proto3" json:"request_results,omitempty"`
+	// Number of polls called inside completion queue
+	CqPollCount uint64 `protobuf:"varint,6,opt,name=cq_poll_count,json=cqPollCount,proto3" json:"cq_poll_count,omitempty"`
+	// Core library stats
+	CoreStats *core.Stats `protobuf:"bytes,7,opt,name=core_stats,json=coreStats,proto3" json:"core_stats,omitempty"`
+}
+
+func (x *ClientStats) Reset() {
+	*x = ClientStats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_stats_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ClientStats) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClientStats) ProtoMessage() {}
+
+func (x *ClientStats) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_stats_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead.
+func (*ClientStats) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_stats_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ClientStats) GetLatencies() *HistogramData {
+	if x != nil {
+		return x.Latencies
+	}
+	return nil
+}
+
+func (x *ClientStats) GetTimeElapsed() float64 {
+	if x != nil {
+		return x.TimeElapsed
+	}
+	return 0
+}
+
+func (x *ClientStats) GetTimeUser() float64 {
+	if x != nil {
+		return x.TimeUser
+	}
+	return 0
+}
+
+func (x *ClientStats) GetTimeSystem() float64 {
+	if x != nil {
+		return x.TimeSystem
+	}
+	return 0
+}
+
+func (x *ClientStats) GetRequestResults() []*RequestResultCount {
+	if x != nil {
+		return x.RequestResults
+	}
+	return nil
+}
+
+func (x *ClientStats) GetCqPollCount() uint64 {
+	if x != nil {
+		return x.CqPollCount
+	}
+	return 0
+}
+
+func (x *ClientStats) GetCoreStats() *core.Stats {
+	if x != nil {
+		return x.CoreStats
+	}
+	return nil
+}
+
+var File_grpc_testing_stats_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_stats_proto_rawDesc = []byte{
+	0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73,
+	0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x15, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63,
+	0x6f, 0x72, 0x65, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
+	0x8d, 0x02, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12,
+	0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61, 0x70, 0x73,
+	0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65, 0x72, 0x12,
+	0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x03,
+	0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d,
+	0x12, 0x24, 0x0a, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x74, 0x69,
+	0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43,
+	0x70, 0x75, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x69, 0x64, 0x6c, 0x65, 0x5f, 0x63,
+	0x70, 0x75, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x69,
+	0x64, 0x6c, 0x65, 0x43, 0x70, 0x75, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x63, 0x71,
+	0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x04, 0x52, 0x0b, 0x63, 0x71, 0x50, 0x6f, 0x6c, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2f,
+	0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53,
+	0x74, 0x61, 0x74, 0x73, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x22,
+	0x54, 0x0a, 0x0f, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61,
+	0x6d, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69,
+	0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x73, 0x73, 0x69, 0x62,
+	0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x50, 0x6f, 0x73,
+	0x73, 0x69, 0x62, 0x6c, 0x65, 0x22, 0xab, 0x01, 0x0a, 0x0d, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67,
+	0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65,
+	0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12,
+	0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x01, 0x52, 0x07, 0x6d, 0x69, 0x6e, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61,
+	0x78, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x6d, 0x61,
+	0x78, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x75, 0x6d, 0x5f, 0x6f,
+	0x66, 0x5f, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52,
+	0x0c, 0x73, 0x75, 0x6d, 0x4f, 0x66, 0x53, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73, 0x12, 0x14, 0x0a,
+	0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x63, 0x6f,
+	0x75, 0x6e, 0x74, 0x22, 0x4b, 0x0a, 0x12, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65,
+	0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61,
+	0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
+	0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f,
+	0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74,
+	0x22, 0xc9, 0x02, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73,
+	0x12, 0x39, 0x0a, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61,
+	0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74,
+	0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x12, 0x1b,
+	0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x74,
+	0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01,
+	0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x49, 0x0a, 0x0f,
+	0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18,
+	0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x75,
+	0x6c, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x63, 0x71, 0x5f, 0x70, 0x6f,
+	0x6c, 0x6c, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b,
+	0x63, 0x71, 0x50, 0x6f, 0x6c, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63,
+	0x6f, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_grpc_testing_stats_proto_rawDescOnce sync.Once
+	file_grpc_testing_stats_proto_rawDescData = file_grpc_testing_stats_proto_rawDesc
+)
+
+func file_grpc_testing_stats_proto_rawDescGZIP() []byte {
+	file_grpc_testing_stats_proto_rawDescOnce.Do(func() {
+		file_grpc_testing_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_stats_proto_rawDescData)
+	})
+	return file_grpc_testing_stats_proto_rawDescData
+}
+
+var file_grpc_testing_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_grpc_testing_stats_proto_goTypes = []interface{}{
+	(*ServerStats)(nil),        // 0: grpc.testing.ServerStats
+	(*HistogramParams)(nil),    // 1: grpc.testing.HistogramParams
+	(*HistogramData)(nil),      // 2: grpc.testing.HistogramData
+	(*RequestResultCount)(nil), // 3: grpc.testing.RequestResultCount
+	(*ClientStats)(nil),        // 4: grpc.testing.ClientStats
+	(*core.Stats)(nil),         // 5: grpc.core.Stats
+}
+var file_grpc_testing_stats_proto_depIdxs = []int32{
+	5, // 0: grpc.testing.ServerStats.core_stats:type_name -> grpc.core.Stats
+	2, // 1: grpc.testing.ClientStats.latencies:type_name -> grpc.testing.HistogramData
+	3, // 2: grpc.testing.ClientStats.request_results:type_name -> grpc.testing.RequestResultCount
+	5, // 3: grpc.testing.ClientStats.core_stats:type_name -> grpc.core.Stats
+	4, // [4:4] is the sub-list for method output_type
+	4, // [4:4] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_stats_proto_init() }
+func file_grpc_testing_stats_proto_init() {
+	if File_grpc_testing_stats_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_grpc_testing_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ServerStats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HistogramParams); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*HistogramData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_stats_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RequestResultCount); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_stats_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ClientStats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_stats_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   5,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_grpc_testing_stats_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_stats_proto_depIdxs,
+		MessageInfos:      file_grpc_testing_stats_proto_msgTypes,
+	}.Build()
+	File_grpc_testing_stats_proto = out.File
+	file_grpc_testing_stats_proto_rawDesc = nil
+	file_grpc_testing_stats_proto_goTypes = nil
+	file_grpc_testing_stats_proto_depIdxs = nil
+}
diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go
index 8b4017a81fe9..50db0950b9c5 100644
--- a/interop/grpc_testing/test.pb.go
+++ b/interop/grpc_testing/test.pb.go
@@ -1,4 +1,4 @@
-// Copyright 2017 gRPC authors.
+// Copyright 2015-2016 gRPC authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@
 // versions:
 // 	protoc-gen-go v1.25.0
 // 	protoc        v3.14.0
-// source: interop/grpc_testing/test.proto
+// source: grpc/testing/test.proto
 
 package grpc_testing
 
@@ -28,7 +28,6 @@ import (
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
-	sync "sync"
 )
 
 const (
@@ -42,1874 +41,189 @@ const (
 // of the legacy proto package is being used.
 const _ = proto.ProtoPackageIsVersion4
 
-// The type of payload that should be returned.
-type PayloadType int32
-
-const (
-	// Compressable text format.
-	PayloadType_COMPRESSABLE PayloadType = 0
-	// Uncompressable binary format.
-	PayloadType_UNCOMPRESSABLE PayloadType = 1
-	// Randomly chosen from all other formats defined in this enum.
-	PayloadType_RANDOM PayloadType = 2
-)
-
-// Enum value maps for PayloadType.
-var (
-	PayloadType_name = map[int32]string{
-		0: "COMPRESSABLE",
-		1: "UNCOMPRESSABLE",
-		2: "RANDOM",
-	}
-	PayloadType_value = map[string]int32{
-		"COMPRESSABLE":   0,
-		"UNCOMPRESSABLE": 1,
-		"RANDOM":         2,
-	}
-)
-
-func (x PayloadType) Enum() *PayloadType {
-	p := new(PayloadType)
-	*p = x
-	return p
-}
-
-func (x PayloadType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (PayloadType) Descriptor() protoreflect.EnumDescriptor {
-	return file_interop_grpc_testing_test_proto_enumTypes[0].Descriptor()
-}
-
-func (PayloadType) Type() protoreflect.EnumType {
-	return &file_interop_grpc_testing_test_proto_enumTypes[0]
-}
-
-func (x PayloadType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use PayloadType.Descriptor instead.
-func (PayloadType) EnumDescriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{0}
-}
-
-// The type of route that a client took to reach a server w.r.t. gRPCLB.
-// The server must fill in "fallback" if it detects that the RPC reached
-// the server via the "gRPCLB fallback" path, and "backend" if it detects
-// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got
-// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly
-// how this detection is done is context and server dependant.
-type GrpclbRouteType int32
-
-const (
-	// Server didn't detect the route that a client took to reach it.
-	GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN GrpclbRouteType = 0
-	// Indicates that a client reached a server via gRPCLB fallback.
-	GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK GrpclbRouteType = 1
-	// Indicates that a client reached a server as a gRPCLB-given backend.
-	GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND GrpclbRouteType = 2
-)
-
-// Enum value maps for GrpclbRouteType.
-var (
-	GrpclbRouteType_name = map[int32]string{
-		0: "GRPCLB_ROUTE_TYPE_UNKNOWN",
-		1: "GRPCLB_ROUTE_TYPE_FALLBACK",
-		2: "GRPCLB_ROUTE_TYPE_BACKEND",
-	}
-	GrpclbRouteType_value = map[string]int32{
-		"GRPCLB_ROUTE_TYPE_UNKNOWN":  0,
-		"GRPCLB_ROUTE_TYPE_FALLBACK": 1,
-		"GRPCLB_ROUTE_TYPE_BACKEND":  2,
-	}
-)
-
-func (x GrpclbRouteType) Enum() *GrpclbRouteType {
-	p := new(GrpclbRouteType)
-	*p = x
-	return p
-}
-
-func (x GrpclbRouteType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (GrpclbRouteType) Descriptor() protoreflect.EnumDescriptor {
-	return file_interop_grpc_testing_test_proto_enumTypes[1].Descriptor()
-}
-
-func (GrpclbRouteType) Type() protoreflect.EnumType {
-	return &file_interop_grpc_testing_test_proto_enumTypes[1]
-}
-
-func (x GrpclbRouteType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use GrpclbRouteType.Descriptor instead.
-func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{1}
-}
-
-// Type of RPCs to send.
-type ClientConfigureRequest_RpcType int32
-
-const (
-	ClientConfigureRequest_EMPTY_CALL ClientConfigureRequest_RpcType = 0
-	ClientConfigureRequest_UNARY_CALL ClientConfigureRequest_RpcType = 1
-)
-
-// Enum value maps for ClientConfigureRequest_RpcType.
-var (
-	ClientConfigureRequest_RpcType_name = map[int32]string{
-		0: "EMPTY_CALL",
-		1: "UNARY_CALL",
-	}
-	ClientConfigureRequest_RpcType_value = map[string]int32{
-		"EMPTY_CALL": 0,
-		"UNARY_CALL": 1,
-	}
-)
-
-func (x ClientConfigureRequest_RpcType) Enum() *ClientConfigureRequest_RpcType {
-	p := new(ClientConfigureRequest_RpcType)
-	*p = x
-	return p
-}
-
-func (x ClientConfigureRequest_RpcType) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (ClientConfigureRequest_RpcType) Descriptor() protoreflect.EnumDescriptor {
-	return file_interop_grpc_testing_test_proto_enumTypes[2].Descriptor()
-}
-
-func (ClientConfigureRequest_RpcType) Type() protoreflect.EnumType {
-	return &file_interop_grpc_testing_test_proto_enumTypes[2]
-}
-
-func (x ClientConfigureRequest_RpcType) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use ClientConfigureRequest_RpcType.Descriptor instead.
-func (ClientConfigureRequest_RpcType) EnumDescriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{14, 0}
-}
-
-type Empty struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-}
-
-func (x *Empty) Reset() {
-	*x = Empty{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Empty) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Empty) ProtoMessage() {}
-
-func (x *Empty) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Empty.ProtoReflect.Descriptor instead.
-func (*Empty) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{0}
-}
-
-// A block of data, to simply increase gRPC message size.
-type Payload struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The type of data in body.
-	Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"`
-	// Primary contents of payload.
-	Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
-}
-
-func (x *Payload) Reset() {
-	*x = Payload{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Payload) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Payload) ProtoMessage() {}
-
-func (x *Payload) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Payload.ProtoReflect.Descriptor instead.
-func (*Payload) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *Payload) GetType() PayloadType {
-	if x != nil {
-		return x.Type
-	}
-	return PayloadType_COMPRESSABLE
-}
-
-func (x *Payload) GetBody() []byte {
-	if x != nil {
-		return x.Body
-	}
-	return nil
-}
-
-// A protobuf representation for grpc status. This is used by test
-// clients to specify a status that the server should attempt to return.
-type EchoStatus struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Code    int32  `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
-	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-}
-
-func (x *EchoStatus) Reset() {
-	*x = EchoStatus{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *EchoStatus) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*EchoStatus) ProtoMessage() {}
-
-func (x *EchoStatus) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead.
-func (*EchoStatus) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *EchoStatus) GetCode() int32 {
-	if x != nil {
-		return x.Code
-	}
-	return 0
-}
-
-func (x *EchoStatus) GetMessage() string {
-	if x != nil {
-		return x.Message
-	}
-	return ""
-}
-
-// Unary request.
-type SimpleRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Desired payload type in the response from the server.
-	// If response_type is RANDOM, server randomly chooses one from other formats.
-	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
-	// Desired payload size in the response from the server.
-	// If response_type is COMPRESSABLE, this denotes the size before compression.
-	ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"`
-	// Optional input payload sent along with the request.
-	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
-	// Whether SimpleResponse should include username.
-	FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"`
-	// Whether SimpleResponse should include OAuth scope.
-	FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"`
-	// Whether server should return a given status
-	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-	// Whether SimpleResponse should include server_id.
-	FillServerId bool `protobuf:"varint,9,opt,name=fill_server_id,json=fillServerId,proto3" json:"fill_server_id,omitempty"`
-	// Whether SimpleResponse should include grpclb_route_type.
-	FillGrpclbRouteType bool `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"`
-}
-
-func (x *SimpleRequest) Reset() {
-	*x = SimpleRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[3]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SimpleRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SimpleRequest) ProtoMessage() {}
-
-func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[3]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{3}
-}
-
-func (x *SimpleRequest) GetResponseType() PayloadType {
-	if x != nil {
-		return x.ResponseType
-	}
-	return PayloadType_COMPRESSABLE
-}
-
-func (x *SimpleRequest) GetResponseSize() int32 {
-	if x != nil {
-		return x.ResponseSize
-	}
-	return 0
-}
-
-func (x *SimpleRequest) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-func (x *SimpleRequest) GetFillUsername() bool {
-	if x != nil {
-		return x.FillUsername
-	}
-	return false
-}
-
-func (x *SimpleRequest) GetFillOauthScope() bool {
-	if x != nil {
-		return x.FillOauthScope
-	}
-	return false
-}
-
-func (x *SimpleRequest) GetResponseStatus() *EchoStatus {
-	if x != nil {
-		return x.ResponseStatus
-	}
-	return nil
-}
-
-func (x *SimpleRequest) GetFillServerId() bool {
-	if x != nil {
-		return x.FillServerId
-	}
-	return false
-}
-
-func (x *SimpleRequest) GetFillGrpclbRouteType() bool {
-	if x != nil {
-		return x.FillGrpclbRouteType
-	}
-	return false
-}
-
-// Unary response, as configured by the request.
-type SimpleResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Payload to increase message size.
-	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-	// The user the request came from, for verifying authentication was
-	// successful when the client expected it.
-	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
-	// OAuth scope.
-	OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"`
-	// Server ID. This must be unique among different server instances,
-	// but the same across all RPC's made to a particular server instance.
-	ServerId string `protobuf:"bytes,4,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"`
-	// gRPCLB Path.
-	GrpclbRouteType GrpclbRouteType `protobuf:"varint,5,opt,name=grpclb_route_type,json=grpclbRouteType,proto3,enum=grpc.testing.GrpclbRouteType" json:"grpclb_route_type,omitempty"`
-	// Server hostname.
-	Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"`
-}
-
-func (x *SimpleResponse) Reset() {
-	*x = SimpleResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[4]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SimpleResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SimpleResponse) ProtoMessage() {}
-
-func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[4]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{4}
-}
-
-func (x *SimpleResponse) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-func (x *SimpleResponse) GetUsername() string {
-	if x != nil {
-		return x.Username
-	}
-	return ""
-}
-
-func (x *SimpleResponse) GetOauthScope() string {
-	if x != nil {
-		return x.OauthScope
-	}
-	return ""
-}
-
-func (x *SimpleResponse) GetServerId() string {
-	if x != nil {
-		return x.ServerId
-	}
-	return ""
-}
-
-func (x *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType {
-	if x != nil {
-		return x.GrpclbRouteType
-	}
-	return GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN
-}
-
-func (x *SimpleResponse) GetHostname() string {
-	if x != nil {
-		return x.Hostname
-	}
-	return ""
-}
-
-// Client-streaming request.
-type StreamingInputCallRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Optional input payload sent along with the request.
-	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-}
-
-func (x *StreamingInputCallRequest) Reset() {
-	*x = StreamingInputCallRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[5]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingInputCallRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingInputCallRequest) ProtoMessage() {}
-
-func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[5]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead.
-func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{5}
-}
-
-func (x *StreamingInputCallRequest) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-// Client-streaming response.
-type StreamingInputCallResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Aggregated size of payloads received from the client.
-	AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"`
-}
-
-func (x *StreamingInputCallResponse) Reset() {
-	*x = StreamingInputCallResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[6]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingInputCallResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingInputCallResponse) ProtoMessage() {}
-
-func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[6]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead.
-func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{6}
-}
-
-func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 {
-	if x != nil {
-		return x.AggregatedPayloadSize
-	}
-	return 0
-}
-
-// Configuration for a particular response.
-type ResponseParameters struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Desired payload sizes in responses from the server.
-	// If response_type is COMPRESSABLE, this denotes the size before compression.
-	Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"`
-	// Desired interval between consecutive responses in the response stream in
-	// microseconds.
-	IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"`
-}
-
-func (x *ResponseParameters) Reset() {
-	*x = ResponseParameters{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[7]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ResponseParameters) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ResponseParameters) ProtoMessage() {}
-
-func (x *ResponseParameters) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[7]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead.
-func (*ResponseParameters) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{7}
-}
-
-func (x *ResponseParameters) GetSize() int32 {
-	if x != nil {
-		return x.Size
-	}
-	return 0
-}
-
-func (x *ResponseParameters) GetIntervalUs() int32 {
-	if x != nil {
-		return x.IntervalUs
-	}
-	return 0
-}
-
-// Server-streaming request.
-type StreamingOutputCallRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Desired payload type in the response from the server.
-	// If response_type is RANDOM, the payload from each response in the stream
-	// might be of different types. This is to simulate a mixed type of payload
-	// stream.
-	ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"`
-	// Configuration for each expected response message.
-	ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"`
-	// Optional input payload sent along with the request.
-	Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
-	// Whether server should return a given status
-	ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"`
-}
-
-func (x *StreamingOutputCallRequest) Reset() {
-	*x = StreamingOutputCallRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[8]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingOutputCallRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingOutputCallRequest) ProtoMessage() {}
-
-func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[8]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead.
-func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{8}
-}
-
-func (x *StreamingOutputCallRequest) GetResponseType() PayloadType {
-	if x != nil {
-		return x.ResponseType
-	}
-	return PayloadType_COMPRESSABLE
-}
-
-func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters {
-	if x != nil {
-		return x.ResponseParameters
-	}
-	return nil
-}
-
-func (x *StreamingOutputCallRequest) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus {
-	if x != nil {
-		return x.ResponseStatus
-	}
-	return nil
-}
-
-// Server-streaming response, as configured by the request and parameters.
-type StreamingOutputCallResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Payload to increase response size.
-	Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
-}
-
-func (x *StreamingOutputCallResponse) Reset() {
-	*x = StreamingOutputCallResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[9]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *StreamingOutputCallResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*StreamingOutputCallResponse) ProtoMessage() {}
-
-func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[9]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead.
-func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{9}
-}
-
-func (x *StreamingOutputCallResponse) GetPayload() *Payload {
-	if x != nil {
-		return x.Payload
-	}
-	return nil
-}
-
-type LoadBalancerStatsRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// Request stats for the next num_rpcs sent by client.
-	NumRpcs int32 `protobuf:"varint,1,opt,name=num_rpcs,json=numRpcs,proto3" json:"num_rpcs,omitempty"`
-	// If num_rpcs have not completed within timeout_sec, return partial results.
-	TimeoutSec int32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
-}
-
-func (x *LoadBalancerStatsRequest) Reset() {
-	*x = LoadBalancerStatsRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[10]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *LoadBalancerStatsRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*LoadBalancerStatsRequest) ProtoMessage() {}
-
-func (x *LoadBalancerStatsRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[10]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use LoadBalancerStatsRequest.ProtoReflect.Descriptor instead.
-func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{10}
-}
-
-func (x *LoadBalancerStatsRequest) GetNumRpcs() int32 {
-	if x != nil {
-		return x.NumRpcs
-	}
-	return 0
-}
-
-func (x *LoadBalancerStatsRequest) GetTimeoutSec() int32 {
-	if x != nil {
-		return x.TimeoutSec
-	}
-	return 0
-}
-
-type LoadBalancerStatsResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The number of completed RPCs for each peer.
-	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-	// The number of RPCs that failed to record a remote peer.
-	NumFailures int32 `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"`
-	// The number of completed RPCs for each method (UnaryCall or EmptyCall).
-	RpcsByMethod map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-}
-
-func (x *LoadBalancerStatsResponse) Reset() {
-	*x = LoadBalancerStatsResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[11]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *LoadBalancerStatsResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*LoadBalancerStatsResponse) ProtoMessage() {}
-
-func (x *LoadBalancerStatsResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[11]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use LoadBalancerStatsResponse.ProtoReflect.Descriptor instead.
-func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{11}
-}
-
-func (x *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 {
-	if x != nil {
-		return x.RpcsByPeer
-	}
-	return nil
-}
-
-func (x *LoadBalancerStatsResponse) GetNumFailures() int32 {
-	if x != nil {
-		return x.NumFailures
-	}
-	return 0
-}
-
-func (x *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer {
-	if x != nil {
-		return x.RpcsByMethod
-	}
-	return nil
-}
-
-// Request for retrieving a test client's accumulated stats.
-type LoadBalancerAccumulatedStatsRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-}
-
-func (x *LoadBalancerAccumulatedStatsRequest) Reset() {
-	*x = LoadBalancerAccumulatedStatsRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *LoadBalancerAccumulatedStatsRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*LoadBalancerAccumulatedStatsRequest) ProtoMessage() {}
-
-func (x *LoadBalancerAccumulatedStatsRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[12]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use LoadBalancerAccumulatedStatsRequest.ProtoReflect.Descriptor instead.
-func (*LoadBalancerAccumulatedStatsRequest) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{12}
-}
-
-// Accumulated stats for RPCs sent by a test client.
-type LoadBalancerAccumulatedStatsResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The total number of RPCs have ever issued for each type.
-	NumRpcsStartedByMethod map[string]int32 `protobuf:"bytes,1,rep,name=num_rpcs_started_by_method,json=numRpcsStartedByMethod,proto3" json:"num_rpcs_started_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-	// The total number of RPCs have ever completed successfully for each type.
-	NumRpcsSucceededByMethod map[string]int32 `protobuf:"bytes,2,rep,name=num_rpcs_succeeded_by_method,json=numRpcsSucceededByMethod,proto3" json:"num_rpcs_succeeded_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-	// The total number of RPCs have ever failed for each type.
-	NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-}
-
-func (x *LoadBalancerAccumulatedStatsResponse) Reset() {
-	*x = LoadBalancerAccumulatedStatsResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[13]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *LoadBalancerAccumulatedStatsResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*LoadBalancerAccumulatedStatsResponse) ProtoMessage() {}
-
-func (x *LoadBalancerAccumulatedStatsResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[13]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use LoadBalancerAccumulatedStatsResponse.ProtoReflect.Descriptor instead.
-func (*LoadBalancerAccumulatedStatsResponse) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{13}
-}
-
-func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[string]int32 {
-	if x != nil {
-		return x.NumRpcsStartedByMethod
-	}
-	return nil
-}
-
-func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map[string]int32 {
-	if x != nil {
-		return x.NumRpcsSucceededByMethod
-	}
-	return nil
-}
-
-func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[string]int32 {
-	if x != nil {
-		return x.NumRpcsFailedByMethod
-	}
-	return nil
-}
-
-// Configurations for a test client.
-type ClientConfigureRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The types of RPCs the client sends.
-	Types []ClientConfigureRequest_RpcType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"types,omitempty"`
-	// The collection of custom metadata to be attached to RPCs sent by the client.
-	Metadata []*ClientConfigureRequest_Metadata `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"`
-}
-
-func (x *ClientConfigureRequest) Reset() {
-	*x = ClientConfigureRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[14]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientConfigureRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientConfigureRequest) ProtoMessage() {}
-
-func (x *ClientConfigureRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[14]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientConfigureRequest.ProtoReflect.Descriptor instead.
-func (*ClientConfigureRequest) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{14}
-}
-
-func (x *ClientConfigureRequest) GetTypes() []ClientConfigureRequest_RpcType {
-	if x != nil {
-		return x.Types
-	}
-	return nil
-}
-
-func (x *ClientConfigureRequest) GetMetadata() []*ClientConfigureRequest_Metadata {
-	if x != nil {
-		return x.Metadata
-	}
-	return nil
-}
-
-// Response for updating a test client's configuration.
-type ClientConfigureResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-}
-
-func (x *ClientConfigureResponse) Reset() {
-	*x = ClientConfigureResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[15]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientConfigureResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientConfigureResponse) ProtoMessage() {}
-
-func (x *ClientConfigureResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[15]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientConfigureResponse.ProtoReflect.Descriptor instead.
-func (*ClientConfigureResponse) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{15}
-}
-
-type LoadBalancerStatsResponse_RpcsByPeer struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The number of completed RPCs for each peer.
-	RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-}
-
-func (x *LoadBalancerStatsResponse_RpcsByPeer) Reset() {
-	*x = LoadBalancerStatsResponse_RpcsByPeer{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[16]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *LoadBalancerStatsResponse_RpcsByPeer) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {}
-
-func (x *LoadBalancerStatsResponse_RpcsByPeer) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[16]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use LoadBalancerStatsResponse_RpcsByPeer.ProtoReflect.Descriptor instead.
-func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{11, 0}
-}
-
-func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 {
-	if x != nil {
-		return x.RpcsByPeer
-	}
-	return nil
-}
-
-// Metadata to be attached for the given type of RPCs.
-type ClientConfigureRequest_Metadata struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Type  ClientConfigureRequest_RpcType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"type,omitempty"`
-	Key   string                         `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
-	Value string                         `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
-}
-
-func (x *ClientConfigureRequest_Metadata) Reset() {
-	*x = ClientConfigureRequest_Metadata{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_interop_grpc_testing_test_proto_msgTypes[23]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *ClientConfigureRequest_Metadata) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*ClientConfigureRequest_Metadata) ProtoMessage() {}
-
-func (x *ClientConfigureRequest_Metadata) ProtoReflect() protoreflect.Message {
-	mi := &file_interop_grpc_testing_test_proto_msgTypes[23]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use ClientConfigureRequest_Metadata.ProtoReflect.Descriptor instead.
-func (*ClientConfigureRequest_Metadata) Descriptor() ([]byte, []int) {
-	return file_interop_grpc_testing_test_proto_rawDescGZIP(), []int{14, 0}
-}
-
-func (x *ClientConfigureRequest_Metadata) GetType() ClientConfigureRequest_RpcType {
-	if x != nil {
-		return x.Type
-	}
-	return ClientConfigureRequest_EMPTY_CALL
-}
-
-func (x *ClientConfigureRequest_Metadata) GetKey() string {
-	if x != nil {
-		return x.Key
-	}
-	return ""
-}
-
-func (x *ClientConfigureRequest_Metadata) GetValue() string {
-	if x != nil {
-		return x.Value
-	}
-	return ""
-}
-
-var File_interop_grpc_testing_test_proto protoreflect.FileDescriptor
-
-var file_interop_grpc_testing_test_proto_rawDesc = []byte{
-	0x0a, 0x1f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6f, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22,
-	0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x4c, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c,
-	0x6f, 0x61, 0x64, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
-	0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c,
-	0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x3a, 0x0a, 0x0a, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74,
-	0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73,
-	0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61,
-	0x67, 0x65, 0x22, 0x92, 0x03, 0x0a, 0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f,
-	0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x72, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69,
-	0x6c, 0x6c, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28,
-	0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12,
-	0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63,
-	0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x4f,
-	0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x24, 0x0a, 0x0e,
-	0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x09,
-	0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
-	0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x67, 0x72, 0x70, 0x63, 0x6c,
-	0x62, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01,
-	0x28, 0x08, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f,
-	0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, 0x82, 0x02, 0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70,
-	0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f,
-	0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x75,
-	0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,
-	0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x61, 0x75, 0x74, 0x68,
-	0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x61,
-	0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x72,
-	0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x49, 0x0a, 0x11, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x5f,
-	0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e,
-	0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
-	0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52,
-	0x0f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x4c, 0x0a, 0x19,
+var File_grpc_testing_test_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_test_proto_rawDesc = []byte{
+	0x0a, 0x17, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74,
+	0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f,
+	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xcb,
+	0x05, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x35,
+	0x0a, 0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
+	0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, 0x61,
+	0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a,
+	0x12, 0x43, 0x61, 0x63, 0x68, 0x65, 0x61, 0x62, 0x6c, 0x65, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43,
+	0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c,
+	0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75,
+	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
+	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61,
+	0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x12,
 	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61,
-	0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74,
-	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72,
-	0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73,
-	0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65,
-	0x67, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65,
-	0x22, 0x49, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61,
-	0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e,
-	0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x22, 0xa3, 0x02, 0x0a, 0x1a,
-	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43,
-	0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72,
-	0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50,
-	0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a,
-	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x41,
-	0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x22, 0x4e, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
-	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x64, 0x22, 0x56, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
-	0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a,
-	0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x07, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65,
-	0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74,
-	0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x22, 0xe2, 0x04, 0x0a, 0x19, 0x4c, 0x6f,
-	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f,
-	0x62, 0x79, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
-	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65,
-	0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65,
-	0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x6e, 0x75, 0x6d, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72,
-	0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x46, 0x61, 0x69,
-	0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x5f, 0x0a, 0x0e, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79,
-	0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
-	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74,
-	0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79,
-	0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0xb1, 0x01, 0x0a, 0x0a, 0x52, 0x70, 0x63, 0x73, 0x42,
-	0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x64, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79,
-	0x5f, 0x70, 0x65, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42,
-	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x2e,
-	0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
-	0x0a, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x52,
-	0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3d, 0x0a, 0x0f, 0x52, 0x70,
-	0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
-	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
-	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x73, 0x0a, 0x11, 0x52, 0x70, 0x63,
-	0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x48, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x32, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c,
-	0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50,
-	0x65, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25,
-	0x0a, 0x23, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
-	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb2, 0x05, 0x0a, 0x24, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
-	0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65,
-	0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a,
-	0x01, 0x0a, 0x1a, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72,
-	0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20,
-	0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41,
-	0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53,
-	0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e,
-	0x74, 0x72, 0x79, 0x52, 0x16, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72,
-	0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x90, 0x01, 0x0a, 0x1c,
-	0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
-	0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03,
-	0x28, 0x0b, 0x32, 0x50, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
-	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75,
-	0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45,
-	0x6e, 0x74, 0x72, 0x79, 0x52, 0x18, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63,
-	0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x87,
-	0x01, 0x0a, 0x19, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c,
-	0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03,
-	0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
-	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61,
-	0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72,
-	0x79, 0x52, 0x15, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64,
-	0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0x49, 0x0a, 0x1b, 0x4e, 0x75, 0x6d, 0x52,
-	0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68,
-	0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
-	0x02, 0x38, 0x01, 0x1a, 0x4b, 0x0a, 0x1d, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75,
-	0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45,
-	0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
-	0x1a, 0x48, 0x0a, 0x1a, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65,
-	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc8, 0x02, 0x0a, 0x16, 0x43,
-	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01,
-	0x20, 0x03, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79,
-	0x70, 0x65, 0x52, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x1a, 0x74, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
-	0x12, 0x40, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c,
-	0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
-	0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x07, 0x52, 0x70,
-	0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x43,
-	0x41, 0x4c, 0x4c, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x5f, 0x43,
-	0x41, 0x4c, 0x4c, 0x10, 0x01, 0x22, 0x19, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x2a, 0x3f, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12,
-	0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10,
-	0x00, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41,
-	0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x10,
-	0x02, 0x2a, 0x6f, 0x0a, 0x0f, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65,
-	0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52,
-	0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57,
-	0x4e, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f,
-	0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43,
-	0x4b, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f,
-	0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44,
-	0x10, 0x02, 0x32, 0xbb, 0x04, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69,
-	0x63, 0x65, 0x12, 0x35, 0x0a, 0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12,
-	0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45,
-	0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61,
-	0x72, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
-	0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x12, 0x6c, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75,
-	0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
-	0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e,
-	0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75,
-	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12,
-	0x69, 0x0a, 0x12, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75,
-	0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e,
-	0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28,
+	0x6c, 0x6c, 0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74,
+	0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x72,
+	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c, 0x44,
+	0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69,
+	0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70,
+	0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01,
+	0x30, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70, 0x6c, 0x65, 0x78,
+	0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74,
+	0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29,
 	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74,
-	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75,
-	0x6c, 0x6c, 0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67,
-	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65,
-	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f,
-	0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70,
-	0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67,
-	0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
-	0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74,
-	0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01,
-	0x32, 0x55, 0x0a, 0x14, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65,
-	0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x11, 0x55, 0x6e, 0x69, 0x6d,
-	0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e,
+	0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c,
+	0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3d, 0x0a,
+	0x11, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61,
+	0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x55, 0x0a, 0x14,
+	0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x53, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x11, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d,
+	0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d,
+	0x70, 0x74, 0x79, 0x32, 0x89, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
+	0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x72,
+	0x74, 0x12, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
+	0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
+	0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x38, 0x0a, 0x04, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x13, 0x2e,
 	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70,
-	0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
-	0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x86, 0x02, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64,
-	0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x65, 0x72,
-	0x76, 0x69, 0x63, 0x65, 0x12, 0x63, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
-	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27,
+	0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x32,
+	0x86, 0x02, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72,
+	0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x63, 0x0a, 0x0e,
+	0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x26,
 	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f,
 	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x84, 0x01, 0x0a, 0x19, 0x47, 0x65,
-	0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74,
-	0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e,
-	0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74,
-	0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61,
-	0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65,
-	0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
-	0x32, 0x7b, 0x0a, 0x1f, 0x58, 0x64, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x69,
-	0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76,
-	0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
-	0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e,
-	0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
+	0x00, 0x12, 0x84, 0x01, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41,
+	0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12,
+	0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c,
+	0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d,
+	0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x32, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63,
+	0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x32, 0x8b, 0x01, 0x0a, 0x16, 0x58, 0x64, 0x73,
+	0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x53, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e,
+	0x67, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x0d, 0x53,
+	0x65, 0x74, 0x4e, 0x6f, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x12, 0x13, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74,
+	0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x7b, 0x0a, 0x1f, 0x58, 0x64, 0x73, 0x55, 0x70, 0x64,
+	0x61, 0x74, 0x65, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
+	0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
 	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66,
-	0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x2d, 0x5a,
-	0x2b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f,
-	0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6f, 0x70, 0x2f,
-	0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x33,
-}
-
-var (
-	file_interop_grpc_testing_test_proto_rawDescOnce sync.Once
-	file_interop_grpc_testing_test_proto_rawDescData = file_interop_grpc_testing_test_proto_rawDesc
-)
-
-func file_interop_grpc_testing_test_proto_rawDescGZIP() []byte {
-	file_interop_grpc_testing_test_proto_rawDescOnce.Do(func() {
-		file_interop_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_interop_grpc_testing_test_proto_rawDescData)
-	})
-	return file_interop_grpc_testing_test_proto_rawDescData
-}
-
-var file_interop_grpc_testing_test_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_interop_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 24)
-var file_interop_grpc_testing_test_proto_goTypes = []interface{}{
-	(PayloadType)(0),                             // 0: grpc.testing.PayloadType
-	(GrpclbRouteType)(0),                         // 1: grpc.testing.GrpclbRouteType
-	(ClientConfigureRequest_RpcType)(0),          // 2: grpc.testing.ClientConfigureRequest.RpcType
-	(*Empty)(nil),                                // 3: grpc.testing.Empty
-	(*Payload)(nil),                              // 4: grpc.testing.Payload
-	(*EchoStatus)(nil),                           // 5: grpc.testing.EchoStatus
-	(*SimpleRequest)(nil),                        // 6: grpc.testing.SimpleRequest
-	(*SimpleResponse)(nil),                       // 7: grpc.testing.SimpleResponse
-	(*StreamingInputCallRequest)(nil),            // 8: grpc.testing.StreamingInputCallRequest
-	(*StreamingInputCallResponse)(nil),           // 9: grpc.testing.StreamingInputCallResponse
-	(*ResponseParameters)(nil),                   // 10: grpc.testing.ResponseParameters
-	(*StreamingOutputCallRequest)(nil),           // 11: grpc.testing.StreamingOutputCallRequest
-	(*StreamingOutputCallResponse)(nil),          // 12: grpc.testing.StreamingOutputCallResponse
-	(*LoadBalancerStatsRequest)(nil),             // 13: grpc.testing.LoadBalancerStatsRequest
-	(*LoadBalancerStatsResponse)(nil),            // 14: grpc.testing.LoadBalancerStatsResponse
-	(*LoadBalancerAccumulatedStatsRequest)(nil),  // 15: grpc.testing.LoadBalancerAccumulatedStatsRequest
-	(*LoadBalancerAccumulatedStatsResponse)(nil), // 16: grpc.testing.LoadBalancerAccumulatedStatsResponse
-	(*ClientConfigureRequest)(nil),               // 17: grpc.testing.ClientConfigureRequest
-	(*ClientConfigureResponse)(nil),              // 18: grpc.testing.ClientConfigureResponse
-	(*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 19: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
-	nil,                                     // 20: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
-	nil,                                     // 21: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
-	nil,                                     // 22: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
-	nil,                                     // 23: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
-	nil,                                     // 24: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
-	nil,                                     // 25: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
-	(*ClientConfigureRequest_Metadata)(nil), // 26: grpc.testing.ClientConfigureRequest.Metadata
-}
-var file_interop_grpc_testing_test_proto_depIdxs = []int32{
-	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
-	0,  // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType
-	4,  // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload
-	5,  // 3: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus
-	4,  // 4: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload
-	1,  // 5: grpc.testing.SimpleResponse.grpclb_route_type:type_name -> grpc.testing.GrpclbRouteType
-	4,  // 6: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload
-	0,  // 7: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType
-	10, // 8: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters
-	4,  // 9: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload
-	5,  // 10: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus
-	4,  // 11: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload
-	20, // 12: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
-	21, // 13: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
-	23, // 14: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_started_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
-	24, // 15: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_succeeded_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
-	25, // 16: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_failed_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
-	2,  // 17: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType
-	26, // 18: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata
-	22, // 19: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
-	19, // 20: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
-	2,  // 21: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType
-	3,  // 22: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty
-	6,  // 23: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
-	11, // 24: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest
-	8,  // 25: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest
-	11, // 26: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
-	11, // 27: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
-	3,  // 28: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty
-	13, // 29: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest
-	15, // 30: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:input_type -> grpc.testing.LoadBalancerAccumulatedStatsRequest
-	17, // 31: grpc.testing.XdsUpdateClientConfigureService.Configure:input_type -> grpc.testing.ClientConfigureRequest
-	3,  // 32: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty
-	7,  // 33: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
-	12, // 34: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse
-	9,  // 35: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse
-	12, // 36: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
-	12, // 37: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
-	3,  // 38: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty
-	14, // 39: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse
-	16, // 40: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:output_type -> grpc.testing.LoadBalancerAccumulatedStatsResponse
-	18, // 41: grpc.testing.XdsUpdateClientConfigureService.Configure:output_type -> grpc.testing.ClientConfigureResponse
-	32, // [32:42] is the sub-list for method output_type
-	22, // [22:32] is the sub-list for method input_type
-	22, // [22:22] is the sub-list for extension type_name
-	22, // [22:22] is the sub-list for extension extendee
-	0,  // [0:22] is the sub-list for field type_name
-}
-
-func init() { file_interop_grpc_testing_test_proto_init() }
-func file_interop_grpc_testing_test_proto_init() {
-	if File_interop_grpc_testing_test_proto != nil {
+	0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var file_grpc_testing_test_proto_goTypes = []interface{}{
+	(*Empty)(nil),                                // 0: grpc.testing.Empty
+	(*SimpleRequest)(nil),                        // 1: grpc.testing.SimpleRequest
+	(*StreamingOutputCallRequest)(nil),           // 2: grpc.testing.StreamingOutputCallRequest
+	(*StreamingInputCallRequest)(nil),            // 3: grpc.testing.StreamingInputCallRequest
+	(*ReconnectParams)(nil),                      // 4: grpc.testing.ReconnectParams
+	(*LoadBalancerStatsRequest)(nil),             // 5: grpc.testing.LoadBalancerStatsRequest
+	(*LoadBalancerAccumulatedStatsRequest)(nil),  // 6: grpc.testing.LoadBalancerAccumulatedStatsRequest
+	(*ClientConfigureRequest)(nil),               // 7: grpc.testing.ClientConfigureRequest
+	(*SimpleResponse)(nil),                       // 8: grpc.testing.SimpleResponse
+	(*StreamingOutputCallResponse)(nil),          // 9: grpc.testing.StreamingOutputCallResponse
+	(*StreamingInputCallResponse)(nil),           // 10: grpc.testing.StreamingInputCallResponse
+	(*ReconnectInfo)(nil),                        // 11: grpc.testing.ReconnectInfo
+	(*LoadBalancerStatsResponse)(nil),            // 12: grpc.testing.LoadBalancerStatsResponse
+	(*LoadBalancerAccumulatedStatsResponse)(nil), // 13: grpc.testing.LoadBalancerAccumulatedStatsResponse
+	(*ClientConfigureResponse)(nil),              // 14: grpc.testing.ClientConfigureResponse
+}
+var file_grpc_testing_test_proto_depIdxs = []int32{
+	0,  // 0: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty
+	1,  // 1: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
+	1,  // 2: grpc.testing.TestService.CacheableUnaryCall:input_type -> grpc.testing.SimpleRequest
+	2,  // 3: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	3,  // 4: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest
+	2,  // 5: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	2,  // 6: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest
+	0,  // 7: grpc.testing.TestService.UnimplementedCall:input_type -> grpc.testing.Empty
+	0,  // 8: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty
+	4,  // 9: grpc.testing.ReconnectService.Start:input_type -> grpc.testing.ReconnectParams
+	0,  // 10: grpc.testing.ReconnectService.Stop:input_type -> grpc.testing.Empty
+	5,  // 11: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest
+	6,  // 12: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:input_type -> grpc.testing.LoadBalancerAccumulatedStatsRequest
+	0,  // 13: grpc.testing.XdsUpdateHealthService.SetServing:input_type -> grpc.testing.Empty
+	0,  // 14: grpc.testing.XdsUpdateHealthService.SetNotServing:input_type -> grpc.testing.Empty
+	7,  // 15: grpc.testing.XdsUpdateClientConfigureService.Configure:input_type -> grpc.testing.ClientConfigureRequest
+	0,  // 16: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty
+	8,  // 17: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
+	8,  // 18: grpc.testing.TestService.CacheableUnaryCall:output_type -> grpc.testing.SimpleResponse
+	9,  // 19: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	10, // 20: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse
+	9,  // 21: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	9,  // 22: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse
+	0,  // 23: grpc.testing.TestService.UnimplementedCall:output_type -> grpc.testing.Empty
+	0,  // 24: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty
+	0,  // 25: grpc.testing.ReconnectService.Start:output_type -> grpc.testing.Empty
+	11, // 26: grpc.testing.ReconnectService.Stop:output_type -> grpc.testing.ReconnectInfo
+	12, // 27: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse
+	13, // 28: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:output_type -> grpc.testing.LoadBalancerAccumulatedStatsResponse
+	0,  // 29: grpc.testing.XdsUpdateHealthService.SetServing:output_type -> grpc.testing.Empty
+	0,  // 30: grpc.testing.XdsUpdateHealthService.SetNotServing:output_type -> grpc.testing.Empty
+	14, // 31: grpc.testing.XdsUpdateClientConfigureService.Configure:output_type -> grpc.testing.ClientConfigureResponse
+	16, // [16:32] is the sub-list for method output_type
+	0,  // [0:16] is the sub-list for method input_type
+	0,  // [0:0] is the sub-list for extension type_name
+	0,  // [0:0] is the sub-list for extension extendee
+	0,  // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_test_proto_init() }
+func file_grpc_testing_test_proto_init() {
+	if File_grpc_testing_test_proto != nil {
 		return
 	}
-	if !protoimpl.UnsafeEnabled {
-		file_interop_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Empty); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Payload); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*EchoStatus); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SimpleRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SimpleResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingInputCallRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingInputCallResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ResponseParameters); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingOutputCallRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamingOutputCallResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LoadBalancerStatsRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LoadBalancerStatsResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LoadBalancerAccumulatedStatsRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LoadBalancerAccumulatedStatsResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientConfigureRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientConfigureResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LoadBalancerStatsResponse_RpcsByPeer); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_interop_grpc_testing_test_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ClientConfigureRequest_Metadata); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
+	file_grpc_testing_empty_proto_init()
+	file_grpc_testing_messages_proto_init()
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_interop_grpc_testing_test_proto_rawDesc,
-			NumEnums:      3,
-			NumMessages:   24,
+			RawDescriptor: file_grpc_testing_test_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   0,
 			NumExtensions: 0,
-			NumServices:   4,
+			NumServices:   6,
 		},
-		GoTypes:           file_interop_grpc_testing_test_proto_goTypes,
-		DependencyIndexes: file_interop_grpc_testing_test_proto_depIdxs,
-		EnumInfos:         file_interop_grpc_testing_test_proto_enumTypes,
-		MessageInfos:      file_interop_grpc_testing_test_proto_msgTypes,
+		GoTypes:           file_grpc_testing_test_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_test_proto_depIdxs,
 	}.Build()
-	File_interop_grpc_testing_test_proto = out.File
-	file_interop_grpc_testing_test_proto_rawDesc = nil
-	file_interop_grpc_testing_test_proto_goTypes = nil
-	file_interop_grpc_testing_test_proto_depIdxs = nil
+	File_grpc_testing_test_proto = out.File
+	file_grpc_testing_test_proto_rawDesc = nil
+	file_grpc_testing_test_proto_goTypes = nil
+	file_grpc_testing_test_proto_depIdxs = nil
 }
diff --git a/interop/grpc_testing/test.proto b/interop/grpc_testing/test.proto
deleted file mode 100644
index 3d7f018d3bb5..000000000000
--- a/interop/grpc_testing/test.proto
+++ /dev/null
@@ -1,281 +0,0 @@
-// Copyright 2017 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// An integration test service that covers all the method signature permutations
-// of unary/streaming requests/responses.
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/interop/grpc_testing";
-
-package grpc.testing;
-
-message Empty {}
-
-// The type of payload that should be returned.
-enum PayloadType {
-  // Compressable text format.
-  COMPRESSABLE = 0;
-
-  // Uncompressable binary format.
-  UNCOMPRESSABLE = 1;
-
-  // Randomly chosen from all other formats defined in this enum.
-  RANDOM = 2;
-}
-
-// A block of data, to simply increase gRPC message size.
-message Payload {
-  // The type of data in body.
-  PayloadType type = 1;
-  // Primary contents of payload.
-  bytes body = 2;
-}
-
-// A protobuf representation for grpc status. This is used by test
-// clients to specify a status that the server should attempt to return.
-message EchoStatus {
-  int32 code = 1;
-  string message = 2;
-}
-
-// The type of route that a client took to reach a server w.r.t. gRPCLB.
-// The server must fill in "fallback" if it detects that the RPC reached
-// the server via the "gRPCLB fallback" path, and "backend" if it detects
-// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got
-// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly
-// how this detection is done is context and server dependant.
-enum GrpclbRouteType {
-  // Server didn't detect the route that a client took to reach it.
-  GRPCLB_ROUTE_TYPE_UNKNOWN = 0;
-  // Indicates that a client reached a server via gRPCLB fallback.
-  GRPCLB_ROUTE_TYPE_FALLBACK = 1;
-  // Indicates that a client reached a server as a gRPCLB-given backend.
-  GRPCLB_ROUTE_TYPE_BACKEND = 2;
-}
-
-// Unary request.
-message SimpleRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, server randomly chooses one from other formats.
-  PayloadType response_type = 1;
-
-  // Desired payload size in the response from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  int32 response_size = 2;
-
-  // Optional input payload sent along with the request.
-  Payload payload = 3;
-
-  // Whether SimpleResponse should include username.
-  bool fill_username = 4;
-
-  // Whether SimpleResponse should include OAuth scope.
-  bool fill_oauth_scope = 5;
-
-  // Whether server should return a given status
-  EchoStatus response_status = 7;
-
-  // Whether SimpleResponse should include server_id.
-  bool fill_server_id = 9;
-
-  // Whether SimpleResponse should include grpclb_route_type.
-  bool fill_grpclb_route_type = 10;
-}
-
-// Unary response, as configured by the request.
-message SimpleResponse {
-  // Payload to increase message size.
-  Payload payload = 1;
-
-  // The user the request came from, for verifying authentication was
-  // successful when the client expected it.
-  string username = 2;
-
-  // OAuth scope.
-  string oauth_scope = 3;
-
-  // Server ID. This must be unique among different server instances,
-  // but the same across all RPC's made to a particular server instance.
-  string server_id = 4;
-
-  // gRPCLB Path.
-  GrpclbRouteType grpclb_route_type = 5;
-
-  // Server hostname.
-  string hostname = 6;
-}
-
-// Client-streaming request.
-message StreamingInputCallRequest {
-  // Optional input payload sent along with the request.
-  Payload payload = 1;
-
-  // Not expecting any payload from the response.
-}
-
-// Client-streaming response.
-message StreamingInputCallResponse {
-  // Aggregated size of payloads received from the client.
-  int32 aggregated_payload_size = 1;
-}
-
-// Configuration for a particular response.
-message ResponseParameters {
-  // Desired payload sizes in responses from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  int32 size = 1;
-
-  // Desired interval between consecutive responses in the response stream in
-  // microseconds.
-  int32 interval_us = 2;
-}
-
-// Server-streaming request.
-message StreamingOutputCallRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, the payload from each response in the stream
-  // might be of different types. This is to simulate a mixed type of payload
-  // stream.
-  PayloadType response_type = 1;
-
-  // Configuration for each expected response message.
-  repeated ResponseParameters response_parameters = 2;
-
-  // Optional input payload sent along with the request.
-  Payload payload = 3;
-
-  // Whether server should return a given status
-  EchoStatus response_status = 7;
-}
-
-// Server-streaming response, as configured by the request and parameters.
-message StreamingOutputCallResponse {
-  // Payload to increase response size.
-  Payload payload = 1;
-}
-
-// A simple service to test the various types of RPCs and experiment with
-// performance with various types of payload.
-service TestService {
-  // One empty request followed by one empty response.
-  rpc EmptyCall(Empty) returns (Empty);
-
-  // One request followed by one response.
-  // The server returns the client payload as-is.
-  rpc UnaryCall(SimpleRequest) returns (SimpleResponse);
-
-  // One request followed by a sequence of responses (streamed download).
-  // The server returns the payload with client desired type and sizes.
-  rpc StreamingOutputCall(StreamingOutputCallRequest)
-      returns (stream StreamingOutputCallResponse);
-
-  // A sequence of requests followed by one response (streamed upload).
-  // The server returns the aggregated size of client payload as the result.
-  rpc StreamingInputCall(stream StreamingInputCallRequest)
-      returns (StreamingInputCallResponse);
-
-  // A sequence of requests with each request served by the server immediately.
-  // As one request could lead to multiple responses, this interface
-  // demonstrates the idea of full duplexing.
-  rpc FullDuplexCall(stream StreamingOutputCallRequest)
-      returns (stream StreamingOutputCallResponse);
-
-  // A sequence of requests followed by a sequence of responses.
-  // The server buffers all the client requests and then serves them in order. A
-  // stream of responses are returned to the client when the server starts with
-  // first request.
-  rpc HalfDuplexCall(stream StreamingOutputCallRequest)
-      returns (stream StreamingOutputCallResponse);
-}
-
-// A simple service NOT implemented at servers so clients can test for
-// that case.
-service UnimplementedService {
-  // A call that no server should implement
-  rpc UnimplementedCall(grpc.testing.Empty) returns (grpc.testing.Empty);
-}
-
-message LoadBalancerStatsRequest {
-  // Request stats for the next num_rpcs sent by client.
-  int32 num_rpcs = 1;
-  // If num_rpcs have not completed within timeout_sec, return partial results.
-  int32 timeout_sec = 2;
-}
-
-message LoadBalancerStatsResponse {
-  message RpcsByPeer {
-    // The number of completed RPCs for each peer.
-    map<string, int32> rpcs_by_peer = 1;
-  }
-
-  // The number of completed RPCs for each peer.
-  map<string, int32> rpcs_by_peer = 1;
-  // The number of RPCs that failed to record a remote peer.
-  int32 num_failures = 2;
-  // The number of completed RPCs for each method (UnaryCall or EmptyCall).
-  map<string, RpcsByPeer> rpcs_by_method = 3;
-}
-
-// Request for retrieving a test client's accumulated stats.
-message LoadBalancerAccumulatedStatsRequest {}
-
-// Accumulated stats for RPCs sent by a test client.
-message LoadBalancerAccumulatedStatsResponse {
-  // The total number of RPCs have ever issued for each type.
-  map<string, int32> num_rpcs_started_by_method = 1;
-  // The total number of RPCs have ever completed successfully for each type.
-  map<string, int32> num_rpcs_succeeded_by_method = 2;
-  // The total number of RPCs have ever failed for each type.
-  map<string, int32> num_rpcs_failed_by_method = 3;
-}
-
-// A service used to obtain stats for verifying LB behavior.
-service LoadBalancerStatsService {
-  // Gets the backend distribution for RPCs sent by a test client.
-  rpc GetClientStats(LoadBalancerStatsRequest)
-      returns (LoadBalancerStatsResponse) {}
-  // Gets the accumulated stats for RPCs sent by a test client.
-  rpc GetClientAccumulatedStats(LoadBalancerAccumulatedStatsRequest)
-    returns (LoadBalancerAccumulatedStatsResponse) {}
-}
-
-// Configurations for a test client.
-message ClientConfigureRequest {
-  // Type of RPCs to send.
-  enum RpcType {
-    EMPTY_CALL = 0;
-    UNARY_CALL = 1;
-  }
-
-  // Metadata to be attached for the given type of RPCs.
-  message Metadata {
-    RpcType type = 1;
-    string key = 2;
-    string value = 3;
-  }
-
-  // The types of RPCs the client sends.
-  repeated RpcType types = 1;
-  // The collection of custom metadata to be attached to RPCs sent by the client.
-  repeated Metadata metadata = 2;
-}
-
-// Response for updating a test client's configuration.
-message ClientConfigureResponse {}
-
-// A service to dynamically update the configuration of an xDS test client.
-service XdsUpdateClientConfigureService {
-  // Update the tes client's configuration.
-  rpc Configure(ClientConfigureRequest) returns (ClientConfigureResponse);
-}
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index a8a2e67375be..6b1ae9eb4321 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -20,8 +20,11 @@ type TestServiceClient interface {
 	// One empty request followed by one empty response.
 	EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
 	// One request followed by one response.
-	// The server returns the client payload as-is.
 	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
+	// One request followed by one response. Response has cache control
+	// headers set such that a caching HTTP proxy (such as GFE) can
+	// satisfy subsequent requests.
+	CacheableUnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
 	// One request followed by a sequence of responses (streamed download).
 	// The server returns the payload with client desired type and sizes.
 	StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error)
@@ -37,6 +40,9 @@ type TestServiceClient interface {
 	// stream of responses are returned to the client when the server starts with
 	// first request.
 	HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error)
+	// The test server will not implement this method. It will be used
+	// to test the behavior when clients call unimplemented methods.
+	UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
 }
 
 type testServiceClient struct {
@@ -65,6 +71,15 @@ func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, op
 	return out, nil
 }
 
+func (c *testServiceClient) CacheableUnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
+	out := new(SimpleResponse)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/CacheableUnaryCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) {
 	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], "/grpc.testing.TestService/StreamingOutputCall", opts...)
 	if err != nil {
@@ -193,6 +208,15 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse,
 	return m, nil
 }
 
+func (c *testServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnimplementedCall", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // TestServiceServer is the server API for TestService service.
 // All implementations must embed UnimplementedTestServiceServer
 // for forward compatibility
@@ -200,8 +224,11 @@ type TestServiceServer interface {
 	// One empty request followed by one empty response.
 	EmptyCall(context.Context, *Empty) (*Empty, error)
 	// One request followed by one response.
-	// The server returns the client payload as-is.
 	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
+	// One request followed by one response. Response has cache control
+	// headers set such that a caching HTTP proxy (such as GFE) can
+	// satisfy subsequent requests.
+	CacheableUnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
 	// One request followed by a sequence of responses (streamed download).
 	// The server returns the payload with client desired type and sizes.
 	StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error
@@ -217,6 +244,9 @@ type TestServiceServer interface {
 	// stream of responses are returned to the client when the server starts with
 	// first request.
 	HalfDuplexCall(TestService_HalfDuplexCallServer) error
+	// The test server will not implement this method. It will be used
+	// to test the behavior when clients call unimplemented methods.
+	UnimplementedCall(context.Context, *Empty) (*Empty, error)
 	mustEmbedUnimplementedTestServiceServer()
 }
 
@@ -230,6 +260,9 @@ func (UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty
 func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
 }
+func (UnimplementedTestServiceServer) CacheableUnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CacheableUnaryCall not implemented")
+}
 func (UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error {
 	return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented")
 }
@@ -242,6 +275,9 @@ func (UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallS
 func (UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error {
 	return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented")
 }
+func (UnimplementedTestServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented")
+}
 func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {}
 
 // UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service.
@@ -291,6 +327,24 @@ func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec fu
 	return interceptor(ctx, in, info, handler)
 }
 
+func _TestService_CacheableUnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SimpleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).CacheableUnaryCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/CacheableUnaryCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).CacheableUnaryCall(ctx, req.(*SimpleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(StreamingOutputCallRequest)
 	if err := stream.RecvMsg(m); err != nil {
@@ -390,6 +444,24 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e
 	return m, nil
 }
 
+func _TestService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(TestServiceServer).UnimplementedCall(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.TestService/UnimplementedCall",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(TestServiceServer).UnimplementedCall(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // TestService_ServiceDesc is the grpc.ServiceDesc for TestService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -405,6 +477,14 @@ var TestService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "UnaryCall",
 			Handler:    _TestService_UnaryCall_Handler,
 		},
+		{
+			MethodName: "CacheableUnaryCall",
+			Handler:    _TestService_CacheableUnaryCall_Handler,
+		},
+		{
+			MethodName: "UnimplementedCall",
+			Handler:    _TestService_UnimplementedCall_Handler,
+		},
 	},
 	Streams: []grpc.StreamDesc{
 		{
@@ -430,7 +510,7 @@ var TestService_ServiceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "interop/grpc_testing/test.proto",
+	Metadata: "grpc/testing/test.proto",
 }
 
 // UnimplementedServiceClient is the client API for UnimplementedService service.
@@ -518,7 +598,129 @@ var UnimplementedService_ServiceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
+	Metadata: "grpc/testing/test.proto",
+}
+
+// ReconnectServiceClient is the client API for ReconnectService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type ReconnectServiceClient interface {
+	Start(ctx context.Context, in *ReconnectParams, opts ...grpc.CallOption) (*Empty, error)
+	Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ReconnectInfo, error)
+}
+
+type reconnectServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewReconnectServiceClient(cc grpc.ClientConnInterface) ReconnectServiceClient {
+	return &reconnectServiceClient{cc}
+}
+
+func (c *reconnectServiceClient) Start(ctx context.Context, in *ReconnectParams, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.ReconnectService/Start", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *reconnectServiceClient) Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ReconnectInfo, error) {
+	out := new(ReconnectInfo)
+	err := c.cc.Invoke(ctx, "/grpc.testing.ReconnectService/Stop", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ReconnectServiceServer is the server API for ReconnectService service.
+// All implementations must embed UnimplementedReconnectServiceServer
+// for forward compatibility
+type ReconnectServiceServer interface {
+	Start(context.Context, *ReconnectParams) (*Empty, error)
+	Stop(context.Context, *Empty) (*ReconnectInfo, error)
+	mustEmbedUnimplementedReconnectServiceServer()
+}
+
+// UnimplementedReconnectServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedReconnectServiceServer struct {
+}
+
+func (UnimplementedReconnectServiceServer) Start(context.Context, *ReconnectParams) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Start not implemented")
+}
+func (UnimplementedReconnectServiceServer) Stop(context.Context, *Empty) (*ReconnectInfo, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Stop not implemented")
+}
+func (UnimplementedReconnectServiceServer) mustEmbedUnimplementedReconnectServiceServer() {}
+
+// UnsafeReconnectServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to ReconnectServiceServer will
+// result in compilation errors.
+type UnsafeReconnectServiceServer interface {
+	mustEmbedUnimplementedReconnectServiceServer()
+}
+
+func RegisterReconnectServiceServer(s grpc.ServiceRegistrar, srv ReconnectServiceServer) {
+	s.RegisterService(&ReconnectService_ServiceDesc, srv)
+}
+
+func _ReconnectService_Start_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ReconnectParams)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ReconnectServiceServer).Start(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.ReconnectService/Start",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ReconnectServiceServer).Start(ctx, req.(*ReconnectParams))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ReconnectService_Stop_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ReconnectServiceServer).Stop(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.ReconnectService/Stop",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ReconnectServiceServer).Stop(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// ReconnectService_ServiceDesc is the grpc.ServiceDesc for ReconnectService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var ReconnectService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.ReconnectService",
+	HandlerType: (*ReconnectServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Start",
+			Handler:    _ReconnectService_Start_Handler,
+		},
+		{
+			MethodName: "Stop",
+			Handler:    _ReconnectService_Stop_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/testing/test.proto",
 }
 
 // LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service.
@@ -645,7 +847,130 @@ var LoadBalancerStatsService_ServiceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
+	Metadata: "grpc/testing/test.proto",
+}
+
+// XdsUpdateHealthServiceClient is the client API for XdsUpdateHealthService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type XdsUpdateHealthServiceClient interface {
+	SetServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
+	SetNotServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error)
+}
+
+type xdsUpdateHealthServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewXdsUpdateHealthServiceClient(cc grpc.ClientConnInterface) XdsUpdateHealthServiceClient {
+	return &xdsUpdateHealthServiceClient{cc}
+}
+
+func (c *xdsUpdateHealthServiceClient) SetServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.XdsUpdateHealthService/SetServing", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *xdsUpdateHealthServiceClient) SetNotServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) {
+	out := new(Empty)
+	err := c.cc.Invoke(ctx, "/grpc.testing.XdsUpdateHealthService/SetNotServing", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// XdsUpdateHealthServiceServer is the server API for XdsUpdateHealthService service.
+// All implementations must embed UnimplementedXdsUpdateHealthServiceServer
+// for forward compatibility
+type XdsUpdateHealthServiceServer interface {
+	SetServing(context.Context, *Empty) (*Empty, error)
+	SetNotServing(context.Context, *Empty) (*Empty, error)
+	mustEmbedUnimplementedXdsUpdateHealthServiceServer()
+}
+
+// UnimplementedXdsUpdateHealthServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedXdsUpdateHealthServiceServer struct {
+}
+
+func (UnimplementedXdsUpdateHealthServiceServer) SetServing(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SetServing not implemented")
+}
+func (UnimplementedXdsUpdateHealthServiceServer) SetNotServing(context.Context, *Empty) (*Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SetNotServing not implemented")
+}
+func (UnimplementedXdsUpdateHealthServiceServer) mustEmbedUnimplementedXdsUpdateHealthServiceServer() {
+}
+
+// UnsafeXdsUpdateHealthServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to XdsUpdateHealthServiceServer will
+// result in compilation errors.
+type UnsafeXdsUpdateHealthServiceServer interface {
+	mustEmbedUnimplementedXdsUpdateHealthServiceServer()
+}
+
+func RegisterXdsUpdateHealthServiceServer(s grpc.ServiceRegistrar, srv XdsUpdateHealthServiceServer) {
+	s.RegisterService(&XdsUpdateHealthService_ServiceDesc, srv)
+}
+
+func _XdsUpdateHealthService_SetServing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(XdsUpdateHealthServiceServer).SetServing(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.XdsUpdateHealthService/SetServing",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(XdsUpdateHealthServiceServer).SetServing(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _XdsUpdateHealthService_SetNotServing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(XdsUpdateHealthServiceServer).SetNotServing(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/grpc.testing.XdsUpdateHealthService/SetNotServing",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(XdsUpdateHealthServiceServer).SetNotServing(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// XdsUpdateHealthService_ServiceDesc is the grpc.ServiceDesc for XdsUpdateHealthService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var XdsUpdateHealthService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "grpc.testing.XdsUpdateHealthService",
+	HandlerType: (*XdsUpdateHealthServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "SetServing",
+			Handler:    _XdsUpdateHealthService_SetServing_Handler,
+		},
+		{
+			MethodName: "SetNotServing",
+			Handler:    _XdsUpdateHealthService_SetNotServing_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "grpc/testing/test.proto",
 }
 
 // XdsUpdateClientConfigureServiceClient is the client API for XdsUpdateClientConfigureService service.
@@ -734,5 +1059,5 @@ var XdsUpdateClientConfigureService_ServiceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams:  []grpc.StreamDesc{},
-	Metadata: "interop/grpc_testing/test.proto",
+	Metadata: "grpc/testing/test.proto",
 }
diff --git a/interop/grpc_testing/worker_service.pb.go b/interop/grpc_testing/worker_service.pb.go
new file mode 100644
index 000000000000..3effdd6533b4
--- /dev/null
+++ b/interop/grpc_testing/worker_service.pb.go
@@ -0,0 +1,120 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.14.0
+// source: grpc/testing/worker_service.proto
+
+package grpc_testing
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+var File_grpc_testing_worker_service_proto protoreflect.FileDescriptor
+
+var file_grpc_testing_worker_service_proto_rawDesc = []byte{
+	0x0a, 0x21, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x77,
+	0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e,
+	0x67, 0x1a, 0x1a, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f,
+	0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x97, 0x02,
+	0x0a, 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12,
+	0x45, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x67,
+	0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76,
+	0x65, 0x72, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74,
+	0x75, 0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x45, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e,
+	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a,
+	0x09, 0x43, 0x6f, 0x72, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x34, 0x0a, 0x0a, 0x51, 0x75, 0x69, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12,
+	0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x56,
+	0x6f, 0x69, 0x64, 0x1a, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var file_grpc_testing_worker_service_proto_goTypes = []interface{}{
+	(*ServerArgs)(nil),   // 0: grpc.testing.ServerArgs
+	(*ClientArgs)(nil),   // 1: grpc.testing.ClientArgs
+	(*CoreRequest)(nil),  // 2: grpc.testing.CoreRequest
+	(*Void)(nil),         // 3: grpc.testing.Void
+	(*ServerStatus)(nil), // 4: grpc.testing.ServerStatus
+	(*ClientStatus)(nil), // 5: grpc.testing.ClientStatus
+	(*CoreResponse)(nil), // 6: grpc.testing.CoreResponse
+}
+var file_grpc_testing_worker_service_proto_depIdxs = []int32{
+	0, // 0: grpc.testing.WorkerService.RunServer:input_type -> grpc.testing.ServerArgs
+	1, // 1: grpc.testing.WorkerService.RunClient:input_type -> grpc.testing.ClientArgs
+	2, // 2: grpc.testing.WorkerService.CoreCount:input_type -> grpc.testing.CoreRequest
+	3, // 3: grpc.testing.WorkerService.QuitWorker:input_type -> grpc.testing.Void
+	4, // 4: grpc.testing.WorkerService.RunServer:output_type -> grpc.testing.ServerStatus
+	5, // 5: grpc.testing.WorkerService.RunClient:output_type -> grpc.testing.ClientStatus
+	6, // 6: grpc.testing.WorkerService.CoreCount:output_type -> grpc.testing.CoreResponse
+	3, // 7: grpc.testing.WorkerService.QuitWorker:output_type -> grpc.testing.Void
+	4, // [4:8] is the sub-list for method output_type
+	0, // [0:4] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_grpc_testing_worker_service_proto_init() }
+func file_grpc_testing_worker_service_proto_init() {
+	if File_grpc_testing_worker_service_proto != nil {
+		return
+	}
+	file_grpc_testing_control_proto_init()
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_grpc_testing_worker_service_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   0,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_grpc_testing_worker_service_proto_goTypes,
+		DependencyIndexes: file_grpc_testing_worker_service_proto_depIdxs,
+	}.Build()
+	File_grpc_testing_worker_service_proto = out.File
+	file_grpc_testing_worker_service_proto_rawDesc = nil
+	file_grpc_testing_worker_service_proto_goTypes = nil
+	file_grpc_testing_worker_service_proto_depIdxs = nil
+}
diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/interop/grpc_testing/worker_service_grpc.pb.go
similarity index 54%
rename from benchmark/grpc_testing/services_grpc.pb.go
rename to interop/grpc_testing/worker_service_grpc.pb.go
index 5dfe0e578dff..76d24ca2e20a 100644
--- a/benchmark/grpc_testing/services_grpc.pb.go
+++ b/interop/grpc_testing/worker_service_grpc.pb.go
@@ -13,241 +13,6 @@ import (
 // is compatible with the grpc package it is being compiled against.
 const _ = grpc.SupportPackageIsVersion7
 
-// BenchmarkServiceClient is the client API for BenchmarkService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type BenchmarkServiceClient interface {
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error)
-	// Unconstrainted streaming.
-	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error)
-}
-
-type benchmarkServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient {
-	return &benchmarkServiceClient{cc}
-}
-
-func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
-	out := new(SimpleResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[0], "/grpc.testing.BenchmarkService/StreamingCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &benchmarkServiceStreamingCallClient{stream}
-	return x, nil
-}
-
-type BenchmarkService_StreamingCallClient interface {
-	Send(*SimpleRequest) error
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type benchmarkServiceStreamingCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *benchmarkServiceStreamingCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[1], "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &benchmarkServiceUnconstrainedStreamingCallClient{stream}
-	return x, nil
-}
-
-type BenchmarkService_UnconstrainedStreamingCallClient interface {
-	Send(*SimpleRequest) error
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type benchmarkServiceUnconstrainedStreamingCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// BenchmarkServiceServer is the server API for BenchmarkService service.
-// All implementations must embed UnimplementedBenchmarkServiceServer
-// for forward compatibility
-type BenchmarkServiceServer interface {
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// One request followed by one response.
-	// The server returns the client payload as-is.
-	StreamingCall(BenchmarkService_StreamingCallServer) error
-	// Unconstrainted streaming.
-	// Both server and client keep sending & receiving simultaneously.
-	UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error
-	mustEmbedUnimplementedBenchmarkServiceServer()
-}
-
-// UnimplementedBenchmarkServiceServer must be embedded to have forward compatible implementations.
-type UnimplementedBenchmarkServiceServer struct {
-}
-
-func (UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (UnimplementedBenchmarkServiceServer) StreamingCall(BenchmarkService_StreamingCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented")
-}
-func (UnimplementedBenchmarkServiceServer) UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented")
-}
-func (UnimplementedBenchmarkServiceServer) mustEmbedUnimplementedBenchmarkServiceServer() {}
-
-// UnsafeBenchmarkServiceServer may be embedded to opt out of forward compatibility for this service.
-// Use of this interface is not recommended, as added methods to BenchmarkServiceServer will
-// result in compilation errors.
-type UnsafeBenchmarkServiceServer interface {
-	mustEmbedUnimplementedBenchmarkServiceServer()
-}
-
-func RegisterBenchmarkServiceServer(s grpc.ServiceRegistrar, srv BenchmarkServiceServer) {
-	s.RegisterService(&BenchmarkService_ServiceDesc, srv)
-}
-
-func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SimpleRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(BenchmarkServiceServer).UnaryCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.BenchmarkService/UnaryCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream})
-}
-
-type BenchmarkService_StreamingCallServer interface {
-	Send(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type benchmarkServiceStreamingCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *benchmarkServiceStreamingCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _BenchmarkService_UnconstrainedStreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(BenchmarkServiceServer).UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream})
-}
-
-type BenchmarkService_UnconstrainedStreamingCallServer interface {
-	Send(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type benchmarkServiceUnconstrainedStreamingCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// BenchmarkService_ServiceDesc is the grpc.ServiceDesc for BenchmarkService service.
-// It's only intended for direct use with grpc.RegisterService,
-// and not to be introspected or modified (even as a copy)
-var BenchmarkService_ServiceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.BenchmarkService",
-	HandlerType: (*BenchmarkServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryCall",
-			Handler:    _BenchmarkService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamingCall",
-			Handler:       _BenchmarkService_StreamingCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "UnconstrainedStreamingCall",
-			Handler:       _BenchmarkService_UnconstrainedStreamingCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "benchmark/grpc_testing/services.proto",
-}
-
 // WorkerServiceClient is the client API for WorkerService service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@@ -532,5 +297,5 @@ var WorkerService_ServiceDesc = grpc.ServiceDesc{
 			ClientStreams: true,
 		},
 	},
-	Metadata: "benchmark/grpc_testing/services.proto",
+	Metadata: "grpc/testing/worker_service.proto",
 }
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 7e3aaa95f0ca..78f937a83d68 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -57,8 +57,6 @@ func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 	body := make([]byte, size)
 	switch t {
 	case testpb.PayloadType_COMPRESSABLE:
-	case testpb.PayloadType_UNCOMPRESSABLE:
-		logger.Fatalf("PayloadType UNCOMPRESSABLE is not supported")
 	default:
 		logger.Fatalf("Unsupported payload type: %d", t)
 	}
@@ -693,8 +691,6 @@ func serverNewPayload(t testpb.PayloadType, size int32) (*testpb.Payload, error)
 	body := make([]byte, size)
 	switch t {
 	case testpb.PayloadType_COMPRESSABLE:
-	case testpb.PayloadType_UNCOMPRESSABLE:
-		return nil, fmt.Errorf("payloadType UNCOMPRESSABLE is not supported")
 	default:
 		return nil, fmt.Errorf("unsupported payload type: %d", t)
 	}
diff --git a/regenerate.sh b/regenerate.sh
index 6d3ec324a260..ed52187df660 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -71,13 +71,15 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto
   ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
+  ${WORKDIR}/grpc-proto/grpc/testing/*.proto
+  ${WORKDIR}/grpc-proto/grpc/core/*.proto
   ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
 )
 
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
 # import path of 'bar' in the generated code when 'foo.proto' is imported in
 # one of the sources.
-OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config
+OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config,Mgrpc/core/stats.proto=google.golang.org/grpc/interop/grpc_testing/core
 
 for src in ${SOURCES[@]}; do
   echo "protoc ${src}"
@@ -111,6 +113,10 @@ rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/*.pb.go
 # grpc/service_config/service_config.proto does not have a go_package option.
 mv ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config
 
+# grpc/testing does not have a go_package option.
+mv ${WORKDIR}/out/grpc/testing/*.pb.go interop/grpc_testing/
+mv ${WORKDIR}/out/grpc/core/*.pb.go interop/grpc_testing/core/
+
 # istio/google/security/meshca/v1/meshca.proto does not have a go_package option.
 mkdir -p ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/
 mv ${WORKDIR}/out/istio/google/security/meshca/v1/* ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/
diff --git a/stats/grpc_testing/test.pb.go b/stats/grpc_testing/test.pb.go
deleted file mode 100644
index 7acea50d128f..000000000000
--- a/stats/grpc_testing/test.pb.go
+++ /dev/null
@@ -1,254 +0,0 @@
-// Copyright 2017 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: stats/grpc_testing/test.proto
-
-package grpc_testing
-
-import (
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-type SimpleRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Id int32 `protobuf:"varint,2,opt,name=id,proto3" json:"id,omitempty"`
-}
-
-func (x *SimpleRequest) Reset() {
-	*x = SimpleRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_stats_grpc_testing_test_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SimpleRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SimpleRequest) ProtoMessage() {}
-
-func (x *SimpleRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_stats_grpc_testing_test_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead.
-func (*SimpleRequest) Descriptor() ([]byte, []int) {
-	return file_stats_grpc_testing_test_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *SimpleRequest) GetId() int32 {
-	if x != nil {
-		return x.Id
-	}
-	return 0
-}
-
-type SimpleResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"`
-}
-
-func (x *SimpleResponse) Reset() {
-	*x = SimpleResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_stats_grpc_testing_test_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *SimpleResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*SimpleResponse) ProtoMessage() {}
-
-func (x *SimpleResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_stats_grpc_testing_test_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead.
-func (*SimpleResponse) Descriptor() ([]byte, []int) {
-	return file_stats_grpc_testing_test_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *SimpleResponse) GetId() int32 {
-	if x != nil {
-		return x.Id
-	}
-	return 0
-}
-
-var File_stats_grpc_testing_test_proto protoreflect.FileDescriptor
-
-var file_stats_grpc_testing_test_proto_rawDesc = []byte{
-	0x0a, 0x1d, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73,
-	0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
-	0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x1f, 0x0a,
-	0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e,
-	0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x22, 0x20,
-	0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64,
-	0x32, 0xc8, 0x02, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
-	0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
-	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c,
-	0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74,
-	0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x10, 0x43, 0x6c, 0x69,
-	0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d,
-	0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x4f, 0x0a, 0x10, 0x53, 0x65,
-	0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1b,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69,
-	0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c,
-	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x2b, 0x5a, 0x29, 0x67,
-	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
-	0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63,
-	0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
-
-var (
-	file_stats_grpc_testing_test_proto_rawDescOnce sync.Once
-	file_stats_grpc_testing_test_proto_rawDescData = file_stats_grpc_testing_test_proto_rawDesc
-)
-
-func file_stats_grpc_testing_test_proto_rawDescGZIP() []byte {
-	file_stats_grpc_testing_test_proto_rawDescOnce.Do(func() {
-		file_stats_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_stats_grpc_testing_test_proto_rawDescData)
-	})
-	return file_stats_grpc_testing_test_proto_rawDescData
-}
-
-var file_stats_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
-var file_stats_grpc_testing_test_proto_goTypes = []interface{}{
-	(*SimpleRequest)(nil),  // 0: grpc.testing.SimpleRequest
-	(*SimpleResponse)(nil), // 1: grpc.testing.SimpleResponse
-}
-var file_stats_grpc_testing_test_proto_depIdxs = []int32{
-	0, // 0: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest
-	0, // 1: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.SimpleRequest
-	0, // 2: grpc.testing.TestService.ClientStreamCall:input_type -> grpc.testing.SimpleRequest
-	0, // 3: grpc.testing.TestService.ServerStreamCall:input_type -> grpc.testing.SimpleRequest
-	1, // 4: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse
-	1, // 5: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.SimpleResponse
-	1, // 6: grpc.testing.TestService.ClientStreamCall:output_type -> grpc.testing.SimpleResponse
-	1, // 7: grpc.testing.TestService.ServerStreamCall:output_type -> grpc.testing.SimpleResponse
-	4, // [4:8] is the sub-list for method output_type
-	0, // [0:4] is the sub-list for method input_type
-	0, // [0:0] is the sub-list for extension type_name
-	0, // [0:0] is the sub-list for extension extendee
-	0, // [0:0] is the sub-list for field type_name
-}
-
-func init() { file_stats_grpc_testing_test_proto_init() }
-func file_stats_grpc_testing_test_proto_init() {
-	if File_stats_grpc_testing_test_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_stats_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SimpleRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_stats_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SimpleResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_stats_grpc_testing_test_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   2,
-			NumExtensions: 0,
-			NumServices:   1,
-		},
-		GoTypes:           file_stats_grpc_testing_test_proto_goTypes,
-		DependencyIndexes: file_stats_grpc_testing_test_proto_depIdxs,
-		MessageInfos:      file_stats_grpc_testing_test_proto_msgTypes,
-	}.Build()
-	File_stats_grpc_testing_test_proto = out.File
-	file_stats_grpc_testing_test_proto_rawDesc = nil
-	file_stats_grpc_testing_test_proto_goTypes = nil
-	file_stats_grpc_testing_test_proto_depIdxs = nil
-}
diff --git a/stats/grpc_testing/test.proto b/stats/grpc_testing/test.proto
deleted file mode 100644
index 2716363260a6..000000000000
--- a/stats/grpc_testing/test.proto
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-option go_package = "google.golang.org/grpc/stats/grpc_testing";
-
-package grpc.testing;
-
-message SimpleRequest {
-  int32 id = 2;
-}
-
-message SimpleResponse {
-  int32 id = 3;
-}
-
-// A simple test service.
-service TestService {
-  // One request followed by one response.
-  // The server returns the client id as-is.
-  rpc UnaryCall(SimpleRequest) returns (SimpleResponse);
-
-  // A sequence of requests with each request served by the server immediately.
-  // As one request could lead to multiple responses, this interface
-  // demonstrates the idea of full duplexing.
-  rpc FullDuplexCall(stream SimpleRequest) returns (stream SimpleResponse);
-
-  // Client stream 
-  rpc ClientStreamCall(stream SimpleRequest) returns (SimpleResponse);
-
-  // Server stream
-  rpc ServerStreamCall(SimpleRequest) returns (stream SimpleResponse);
-}
diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go
deleted file mode 100644
index 3a6f2b8e1a65..000000000000
--- a/stats/grpc_testing/test_grpc.pb.go
+++ /dev/null
@@ -1,316 +0,0 @@
-// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
-
-package grpc_testing
-
-import (
-	context "context"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
-)
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion7
-
-// TestServiceClient is the client API for TestService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type TestServiceClient interface {
-	// One request followed by one response.
-	// The server returns the client id as-is.
-	UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error)
-	// Client stream
-	ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error)
-	// Server stream
-	ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error)
-}
-
-type testServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient {
-	return &testServiceClient{cc}
-}
-
-func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) {
-	out := new(SimpleResponse)
-	err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], "/grpc.testing.TestService/FullDuplexCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceFullDuplexCallClient{stream}
-	return x, nil
-}
-
-type TestService_FullDuplexCallClient interface {
-	Send(*SimpleRequest) error
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceFullDuplexCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceFullDuplexCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[1], "/grpc.testing.TestService/ClientStreamCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceClientStreamCallClient{stream}
-	return x, nil
-}
-
-type TestService_ClientStreamCallClient interface {
-	Send(*SimpleRequest) error
-	CloseAndRecv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceClientStreamCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceClientStreamCallClient) Send(m *SimpleRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) {
-	stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[2], "/grpc.testing.TestService/ServerStreamCall", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &testServiceServerStreamCallClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type TestService_ServerStreamCallClient interface {
-	Recv() (*SimpleResponse, error)
-	grpc.ClientStream
-}
-
-type testServiceServerStreamCallClient struct {
-	grpc.ClientStream
-}
-
-func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) {
-	m := new(SimpleResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// TestServiceServer is the server API for TestService service.
-// All implementations must embed UnimplementedTestServiceServer
-// for forward compatibility
-type TestServiceServer interface {
-	// One request followed by one response.
-	// The server returns the client id as-is.
-	UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error)
-	// A sequence of requests with each request served by the server immediately.
-	// As one request could lead to multiple responses, this interface
-	// demonstrates the idea of full duplexing.
-	FullDuplexCall(TestService_FullDuplexCallServer) error
-	// Client stream
-	ClientStreamCall(TestService_ClientStreamCallServer) error
-	// Server stream
-	ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error
-	mustEmbedUnimplementedTestServiceServer()
-}
-
-// UnimplementedTestServiceServer must be embedded to have forward compatible implementations.
-type UnimplementedTestServiceServer struct {
-}
-
-func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented")
-}
-func (UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented")
-}
-func (UnimplementedTestServiceServer) ClientStreamCall(TestService_ClientStreamCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented")
-}
-func (UnimplementedTestServiceServer) ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error {
-	return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented")
-}
-func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {}
-
-// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service.
-// Use of this interface is not recommended, as added methods to TestServiceServer will
-// result in compilation errors.
-type UnsafeTestServiceServer interface {
-	mustEmbedUnimplementedTestServiceServer()
-}
-
-func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) {
-	s.RegisterService(&TestService_ServiceDesc, srv)
-}
-
-func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SimpleRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(TestServiceServer).UnaryCall(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/grpc.testing.TestService/UnaryCall",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream})
-}
-
-type TestService_FullDuplexCallServer interface {
-	Send(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceFullDuplexCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceFullDuplexCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceFullDuplexCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_ClientStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(TestServiceServer).ClientStreamCall(&testServiceClientStreamCallServer{stream})
-}
-
-type TestService_ClientStreamCallServer interface {
-	SendAndClose(*SimpleResponse) error
-	Recv() (*SimpleRequest, error)
-	grpc.ServerStream
-}
-
-type testServiceClientStreamCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceClientStreamCallServer) SendAndClose(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *testServiceClientStreamCallServer) Recv() (*SimpleRequest, error) {
-	m := new(SimpleRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _TestService_ServerStreamCall_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(SimpleRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(TestServiceServer).ServerStreamCall(m, &testServiceServerStreamCallServer{stream})
-}
-
-type TestService_ServerStreamCallServer interface {
-	Send(*SimpleResponse) error
-	grpc.ServerStream
-}
-
-type testServiceServerStreamCallServer struct {
-	grpc.ServerStream
-}
-
-func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service.
-// It's only intended for direct use with grpc.RegisterService,
-// and not to be introspected or modified (even as a copy)
-var TestService_ServiceDesc = grpc.ServiceDesc{
-	ServiceName: "grpc.testing.TestService",
-	HandlerType: (*TestServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "UnaryCall",
-			Handler:    _TestService_UnaryCall_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "FullDuplexCall",
-			Handler:       _TestService_FullDuplexCall_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "ClientStreamCall",
-			Handler:       _TestService_ClientStreamCall_Handler,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "ServerStreamCall",
-			Handler:       _TestService_ServerStreamCall_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "stats/grpc_testing/test.proto",
-}
diff --git a/stats/stats_test.go b/stats/stats_test.go
index 875a57eeddfc..aac8166f76f8 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -31,9 +31,9 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpctest"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/stats"
-	testpb "google.golang.org/grpc/stats/grpc_testing"
 	"google.golang.org/grpc/status"
 )
 
@@ -75,6 +75,17 @@ var (
 	errorID int32 = 32202
 )
 
+func idToPayload(id int32) *testpb.Payload {
+	return &testpb.Payload{Body: []byte{byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24)}}
+}
+
+func payloadToID(p *testpb.Payload) int32 {
+	if p == nil || len(p.Body) != 4 {
+		panic("invalid payload")
+	}
+	return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24
+}
+
 type testServer struct {
 	testpb.UnimplementedTestServiceServer
 }
@@ -87,11 +98,11 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 		return nil, status.Errorf(status.Code(err), "grpc.SetTrailer(_, %v) = %v, want <nil>", testTrailerMetadata, err)
 	}
 
-	if in.Id == errorID {
-		return nil, fmt.Errorf("got error id: %v", in.Id)
+	if id := payloadToID(in.Payload); id == errorID {
+		return nil, fmt.Errorf("got error id: %v", id)
 	}
 
-	return &testpb.SimpleResponse{Id: in.Id}, nil
+	return &testpb.SimpleResponse{Payload: in.Payload}, nil
 }
 
 func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
@@ -109,17 +120,17 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ
 			return err
 		}
 
-		if in.Id == errorID {
-			return fmt.Errorf("got error id: %v", in.Id)
+		if id := payloadToID(in.Payload); id == errorID {
+			return fmt.Errorf("got error id: %v", id)
 		}
 
-		if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil {
+		if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil {
 			return err
 		}
 	}
 }
 
-func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCallServer) error {
+func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
 	if err := stream.SendHeader(testHeaderMetadata); err != nil {
 		return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil)
 	}
@@ -128,30 +139,30 @@ func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCall
 		in, err := stream.Recv()
 		if err == io.EOF {
 			// read done.
-			return stream.SendAndClose(&testpb.SimpleResponse{Id: int32(0)})
+			return stream.SendAndClose(&testpb.StreamingInputCallResponse{AggregatedPayloadSize: 0})
 		}
 		if err != nil {
 			return err
 		}
 
-		if in.Id == errorID {
-			return fmt.Errorf("got error id: %v", in.Id)
+		if id := payloadToID(in.Payload); id == errorID {
+			return fmt.Errorf("got error id: %v", id)
 		}
 	}
 }
 
-func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.TestService_ServerStreamCallServer) error {
+func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
 	if err := stream.SendHeader(testHeaderMetadata); err != nil {
 		return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil)
 	}
 	stream.SetTrailer(testTrailerMetadata)
 
-	if in.Id == errorID {
-		return fmt.Errorf("got error id: %v", in.Id)
+	if id := payloadToID(in.Payload); id == errorID {
+		return fmt.Errorf("got error id: %v", id)
 	}
 
 	for i := 0; i < 5; i++ {
-		if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil {
+		if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil {
 			return err
 		}
 	}
@@ -279,9 +290,9 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	if c.success {
-		req = &testpb.SimpleRequest{Id: errorID + 1}
+		req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)}
 	} else {
-		req = &testpb.SimpleRequest{Id: errorID}
+		req = &testpb.SimpleRequest{Payload: idToPayload(errorID)}
 	}
 
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
@@ -290,10 +301,10 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 	return req, resp, err
 }
 
-func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest, []*testpb.SimpleResponse, error) {
+func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []proto.Message, error) {
 	var (
-		reqs  []*testpb.SimpleRequest
-		resps []*testpb.SimpleResponse
+		reqs  []proto.Message
+		resps []proto.Message
 		err   error
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
@@ -308,14 +319,14 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest
 		startID = errorID
 	}
 	for i := 0; i < c.count; i++ {
-		req := &testpb.SimpleRequest{
-			Id: int32(i) + startID,
+		req := &testpb.StreamingOutputCallRequest{
+			Payload: idToPayload(int32(i) + startID),
 		}
 		reqs = append(reqs, req)
 		if err = stream.Send(req); err != nil {
 			return reqs, resps, err
 		}
-		var resp *testpb.SimpleResponse
+		var resp *testpb.StreamingOutputCallResponse
 		if resp, err = stream.Recv(); err != nil {
 			return reqs, resps, err
 		}
@@ -331,16 +342,16 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest
 	return reqs, resps, nil
 }
 
-func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *testpb.SimpleResponse, error) {
+func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, *testpb.StreamingInputCallResponse, error) {
 	var (
-		reqs []*testpb.SimpleRequest
-		resp *testpb.SimpleResponse
+		reqs []proto.Message
+		resp *testpb.StreamingInputCallResponse
 		err  error
 	)
 	tc := testpb.NewTestServiceClient(te.clientConn())
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	stream, err := tc.ClientStreamCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast))
+	stream, err := tc.StreamingInputCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast))
 	if err != nil {
 		return reqs, resp, err
 	}
@@ -349,8 +360,8 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test
 		startID = errorID
 	}
 	for i := 0; i < c.count; i++ {
-		req := &testpb.SimpleRequest{
-			Id: int32(i) + startID,
+		req := &testpb.StreamingInputCallRequest{
+			Payload: idToPayload(int32(i) + startID),
 		}
 		reqs = append(reqs, req)
 		if err = stream.Send(req); err != nil {
@@ -361,10 +372,10 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test
 	return reqs, resp, err
 }
 
-func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*testpb.SimpleResponse, error) {
+func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.StreamingOutputCallRequest, []proto.Message, error) {
 	var (
-		req   *testpb.SimpleRequest
-		resps []*testpb.SimpleResponse
+		req   *testpb.StreamingOutputCallRequest
+		resps []proto.Message
 		err   error
 	)
 
@@ -374,15 +385,15 @@ func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*test
 	if !c.success {
 		startID = errorID
 	}
-	req = &testpb.SimpleRequest{Id: startID}
+	req = &testpb.StreamingOutputCallRequest{Payload: idToPayload(startID)}
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	stream, err := tc.ServerStreamCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast))
+	stream, err := tc.StreamingOutputCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast))
 	if err != nil {
 		return req, resps, err
 	}
 	for {
-		var resp *testpb.SimpleResponse
+		var resp *testpb.StreamingOutputCallResponse
 		resp, err := stream.Recv()
 		if err == io.EOF {
 			return req, resps, nil
@@ -398,9 +409,9 @@ type expectedData struct {
 	serverAddr  string
 	compression string
 	reqIdx      int
-	requests    []*testpb.SimpleRequest
+	requests    []proto.Message
 	respIdx     int
-	responses   []*testpb.SimpleResponse
+	responses   []proto.Message
 	err         error
 	failfast    bool
 }
@@ -829,13 +840,13 @@ func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []f
 	defer te.tearDown()
 
 	var (
-		reqs   []*testpb.SimpleRequest
-		resps  []*testpb.SimpleResponse
+		reqs   []proto.Message
+		resps  []proto.Message
 		err    error
 		method string
 
-		req  *testpb.SimpleRequest
-		resp *testpb.SimpleResponse
+		req  proto.Message
+		resp proto.Message
 		e    error
 	)
 
@@ -843,18 +854,18 @@ func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []f
 	case unaryRPC:
 		method = "/grpc.testing.TestService/UnaryCall"
 		req, resp, e = te.doUnaryCall(cc)
-		reqs = []*testpb.SimpleRequest{req}
-		resps = []*testpb.SimpleResponse{resp}
+		reqs = []proto.Message{req}
+		resps = []proto.Message{resp}
 		err = e
 	case clientStreamRPC:
-		method = "/grpc.testing.TestService/ClientStreamCall"
+		method = "/grpc.testing.TestService/StreamingInputCall"
 		reqs, resp, e = te.doClientStreamCall(cc)
-		resps = []*testpb.SimpleResponse{resp}
+		resps = []proto.Message{resp}
 		err = e
 	case serverStreamRPC:
-		method = "/grpc.testing.TestService/ServerStreamCall"
+		method = "/grpc.testing.TestService/StreamingOutputCall"
 		req, resps, e = te.doServerStreamCall(cc)
-		reqs = []*testpb.SimpleRequest{req}
+		reqs = []proto.Message{req}
 		err = e
 	case fullDuplexStreamRPC:
 		method = "/grpc.testing.TestService/FullDuplexCall"
@@ -1120,31 +1131,31 @@ func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map
 	defer te.tearDown()
 
 	var (
-		reqs   []*testpb.SimpleRequest
-		resps  []*testpb.SimpleResponse
+		reqs   []proto.Message
+		resps  []proto.Message
 		method string
 		err    error
 
-		req  *testpb.SimpleRequest
-		resp *testpb.SimpleResponse
+		req  proto.Message
+		resp proto.Message
 		e    error
 	)
 	switch cc.callType {
 	case unaryRPC:
 		method = "/grpc.testing.TestService/UnaryCall"
 		req, resp, e = te.doUnaryCall(cc)
-		reqs = []*testpb.SimpleRequest{req}
-		resps = []*testpb.SimpleResponse{resp}
+		reqs = []proto.Message{req}
+		resps = []proto.Message{resp}
 		err = e
 	case clientStreamRPC:
-		method = "/grpc.testing.TestService/ClientStreamCall"
+		method = "/grpc.testing.TestService/StreamingInputCall"
 		reqs, resp, e = te.doClientStreamCall(cc)
-		resps = []*testpb.SimpleResponse{resp}
+		resps = []proto.Message{resp}
 		err = e
 	case serverStreamRPC:
-		method = "/grpc.testing.TestService/ServerStreamCall"
+		method = "/grpc.testing.TestService/StreamingOutputCall"
 		req, resps, e = te.doServerStreamCall(cc)
-		reqs = []*testpb.SimpleRequest{req}
+		reqs = []proto.Message{req}
 		err = e
 	case fullDuplexStreamRPC:
 		method = "/grpc.testing.TestService/FullDuplexCall"

From 0bd76be2bb6b62dba23cef0454c86e97304ac58a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 7 Jan 2021 14:53:43 -0800
Subject: [PATCH 336/481] lrs: use JSON for locality's String representation
 (#4135)

---
 xds/internal/balancer/edsbalancer/eds_impl.go | 33 ++++++++++----
 .../balancer/edsbalancer/eds_impl_test.go     |  6 ++-
 xds/internal/balancer/lrs/balancer.go         | 22 +++++----
 xds/internal/balancer/lrs/balancer_test.go    |  3 +-
 xds/internal/balancer/lrs/picker.go           |  5 +--
 xds/internal/internal.go                      | 31 +++++++------
 xds/internal/internal_test.go                 | 45 +++++++++++++++++++
 7 files changed, 105 insertions(+), 40 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index a6e950b88eea..17a41ef18e75 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -142,12 +142,17 @@ func (edsImpl *edsBalancerImpl) handleChildPolicy(name string, config json.RawMe
 			continue
 		}
 		for lid, config := range bgwc.configs {
+			lidJSON, err := lid.ToString()
+			if err != nil {
+				edsImpl.logger.Errorf("failed to marshal LocalityID: %#v, skipping this locality", lid)
+				continue
+			}
 			// TODO: (eds) add support to balancer group to support smoothly
 			//  switching sub-balancers (keep old balancer around until new
 			//  balancer becomes ready).
-			bgwc.bg.Remove(lid.String())
-			bgwc.bg.Add(lid.String(), edsImpl.subBalancerBuilder)
-			bgwc.bg.UpdateClientConnState(lid.String(), balancer.ClientConnState{
+			bgwc.bg.Remove(lidJSON)
+			bgwc.bg.Add(lidJSON, edsImpl.subBalancerBuilder)
+			bgwc.bg.UpdateClientConnState(lidJSON, balancer.ClientConnState{
 				ResolverState: resolver.State{Addresses: config.addrs},
 			})
 			// This doesn't need to manually update picker, because the new
@@ -285,6 +290,11 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 		// One balancer for each locality.
 
 		lid := locality.ID
+		lidJSON, err := lid.ToString()
+		if err != nil {
+			edsImpl.logger.Errorf("failed to marshal LocalityID: %#v, skipping this locality", lid)
+			continue
+		}
 		newLocalitiesSet[lid] = struct{}{}
 
 		newWeight := locality.Weight
@@ -319,8 +329,8 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 		config, ok := bgwc.configs[lid]
 		if !ok {
 			// A new balancer, add it to balancer group and balancer map.
-			bgwc.stateAggregator.Add(lid.String(), newWeight)
-			bgwc.bg.Add(lid.String(), edsImpl.subBalancerBuilder)
+			bgwc.stateAggregator.Add(lidJSON, newWeight)
+			bgwc.bg.Add(lidJSON, edsImpl.subBalancerBuilder)
 			config = &localityConfig{
 				weight: newWeight,
 			}
@@ -343,13 +353,13 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 
 		if weightChanged {
 			config.weight = newWeight
-			bgwc.stateAggregator.UpdateWeight(lid.String(), newWeight)
+			bgwc.stateAggregator.UpdateWeight(lidJSON, newWeight)
 			rebuildStateAndPicker = true
 		}
 
 		if addrsChanged {
 			config.addrs = newAddrs
-			bgwc.bg.UpdateClientConnState(lid.String(), balancer.ClientConnState{
+			bgwc.bg.UpdateClientConnState(lidJSON, balancer.ClientConnState{
 				ResolverState: resolver.State{Addresses: newAddrs},
 			})
 		}
@@ -357,9 +367,14 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 
 	// Delete localities that are removed in the latest response.
 	for lid := range bgwc.configs {
+		lidJSON, err := lid.ToString()
+		if err != nil {
+			edsImpl.logger.Errorf("failed to marshal LocalityID: %#v, skipping this locality", lid)
+			continue
+		}
 		if _, ok := newLocalitiesSet[lid]; !ok {
-			bgwc.stateAggregator.Remove(lid.String())
-			bgwc.bg.Remove(lid.String())
+			bgwc.stateAggregator.Remove(lidJSON)
+			bgwc.bg.Remove(lidJSON)
 			delete(bgwc.configs, lid)
 			edsImpl.logger.Infof("Locality %v deleted", lid)
 			rebuildStateAndPicker = true
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 9f28f252d9b5..2eec6be30f10 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -786,12 +786,14 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// We expect the 10 picks to be split between the localities since they are
 	// of equal weight. And since we only mark the picks routed to sc2 as done,
 	// the picks on sc1 should show up as inProgress.
+	locality1JSON, _ := locality1.ToString()
+	locality2JSON, _ := locality2.ToString()
 	wantStoreData := []*load.Data{{
 		Cluster: testClusterNames[0],
 		Service: "",
 		LocalityStats: map[string]load.LocalityData{
-			locality1.String(): {RequestStats: load.RequestData{InProgress: 5}},
-			locality2.String(): {RequestStats: load.RequestData{Succeeded: 5}},
+			locality1JSON: {RequestStats: load.RequestData{InProgress: 5}},
+			locality2JSON: {RequestStats: load.RequestData{Succeeded: 5}},
 		},
 	}}
 	for i := 0; i < 10; i++ {
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index 01bb14662865..d60355afd25e 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -27,7 +27,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/serviceconfig"
-	"google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
@@ -101,7 +100,12 @@ func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 		if b.lb != nil {
 			b.lb.Close()
 		}
-		b.lb = bb.Build(newCCWrapper(b.cc, b.client.loadStore(), newConfig.Locality), b.buildOpts)
+		lidJSON, err := newConfig.Locality.ToString()
+		if err != nil {
+			return fmt.Errorf("failed to marshal LocalityID: %#v", newConfig.Locality)
+		}
+		ccWrapper := newCCWrapper(b.cc, b.client.loadStore(), lidJSON)
+		b.lb = bb.Build(ccWrapper, b.buildOpts)
 	}
 	b.config = newConfig
 
@@ -134,20 +138,20 @@ func (b *lrsBalancer) Close() {
 
 type ccWrapper struct {
 	balancer.ClientConn
-	loadStore  load.PerClusterReporter
-	localityID *internal.LocalityID
+	loadStore      load.PerClusterReporter
+	localityIDJSON string
 }
 
-func newCCWrapper(cc balancer.ClientConn, loadStore load.PerClusterReporter, localityID *internal.LocalityID) *ccWrapper {
+func newCCWrapper(cc balancer.ClientConn, loadStore load.PerClusterReporter, localityIDJSON string) *ccWrapper {
 	return &ccWrapper{
-		ClientConn: cc,
-		loadStore:  loadStore,
-		localityID: localityID,
+		ClientConn:     cc,
+		loadStore:      loadStore,
+		localityIDJSON: localityIDJSON,
 	}
 }
 
 func (ccw *ccWrapper) UpdateState(s balancer.State) {
-	s.Picker = newLoadReportPicker(s.Picker, *ccw.localityID, ccw.loadStore)
+	s.Picker = newLoadReportPicker(s.Picker, ccw.localityIDJSON, ccw.loadStore)
 	ccw.ClientConn.UpdateState(s)
 }
 
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index 0794a7767214..0b575b112104 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -126,7 +126,8 @@ func TestLoadReporting(t *testing.T) {
 	if sd.Cluster != testClusterName || sd.Service != testServiceName {
 		t.Fatalf("got unexpected load for %q, %q, want %q, %q", sd.Cluster, sd.Service, testClusterName, testServiceName)
 	}
-	localityData, ok := sd.LocalityStats[testLocality.String()]
+	testLocalityJSON, _ := testLocality.ToString()
+	localityData, ok := sd.LocalityStats[testLocalityJSON]
 	if !ok {
 		t.Fatalf("loads for %v not found in store", testLocality)
 	}
diff --git a/xds/internal/balancer/lrs/picker.go b/xds/internal/balancer/lrs/picker.go
index 8ca72b4fe57c..1e4ad156e5b7 100644
--- a/xds/internal/balancer/lrs/picker.go
+++ b/xds/internal/balancer/lrs/picker.go
@@ -21,7 +21,6 @@ package lrs
 import (
 	orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1"
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/xds/internal"
 )
 
 const (
@@ -43,10 +42,10 @@ type loadReportPicker struct {
 	loadStore loadReporter
 }
 
-func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore loadReporter) *loadReportPicker {
+func newLoadReportPicker(p balancer.Picker, id string, loadStore loadReporter) *loadReportPicker {
 	return &loadReportPicker{
 		p:         p,
-		locality:  id.String(),
+		locality:  id,
 		loadStore: loadStore,
 	}
 }
diff --git a/xds/internal/internal.go b/xds/internal/internal.go
index 5c12e6f23d26..e4284ee02e0c 100644
--- a/xds/internal/internal.go
+++ b/xds/internal/internal.go
@@ -20,8 +20,8 @@
 package internal
 
 import (
+	"encoding/json"
 	"fmt"
-	"strings"
 )
 
 // LocalityID is xds.Locality without XXX fields, so it can be used as map
@@ -34,23 +34,22 @@ type LocalityID struct {
 	SubZone string `json:"subZone,omitempty"`
 }
 
-// String generates a string representation of LocalityID by adding ":" between
-// the components of the LocalityID.
-func (l LocalityID) String() string {
-	return fmt.Sprintf("%s:%s:%s", l.Region, l.Zone, l.SubZone)
+// ToString generates a string representation of LocalityID by marshalling it into
+// json. Not calling it String() so printf won't call it.
+func (l LocalityID) ToString() (string, error) {
+	b, err := json.Marshal(l)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
 }
 
-// LocalityIDFromString converts a string representation of locality, of the
-// form region:zone:sub-zone (as generated by the above String() method), into a
+// LocalityIDFromString converts a json representation of locality, into a
 // LocalityID struct.
-func LocalityIDFromString(l string) (LocalityID, error) {
-	parts := strings.Split(l, ":")
-	if len(parts) != 3 {
-		return LocalityID{}, fmt.Errorf("%s is not a well formatted locality ID", l)
+func LocalityIDFromString(s string) (ret LocalityID, _ error) {
+	err := json.Unmarshal([]byte(s), &ret)
+	if err != nil {
+		return LocalityID{}, fmt.Errorf("%s is not a well formatted locality ID, error: %v", s, err)
 	}
-	return LocalityID{
-		Region:  parts[0],
-		Zone:    parts[1],
-		SubZone: parts[2],
-	}, nil
+	return ret, nil
 }
diff --git a/xds/internal/internal_test.go b/xds/internal/internal_test.go
index bdd6e85cba97..903b9db23c48 100644
--- a/xds/internal/internal_test.go
+++ b/xds/internal/internal_test.go
@@ -70,3 +70,48 @@ func (s) TestLocalityMatchProtoMessage(t *testing.T) {
 		t.Fatalf("internal type and proto message have different fields: (-got +want):\n%+v", diff)
 	}
 }
+
+func TestLocalityToAndFromJSON(t *testing.T) {
+	tests := []struct {
+		name       string
+		localityID LocalityID
+		str        string
+		wantErr    bool
+	}{
+		{
+			name:       "3 fields",
+			localityID: LocalityID{Region: "r:r", Zone: "z#z", SubZone: "s^s"},
+			str:        `{"region":"r:r","zone":"z#z","subZone":"s^s"}`,
+		},
+		{
+			name:       "2 fields",
+			localityID: LocalityID{Region: "r:r", Zone: "z#z"},
+			str:        `{"region":"r:r","zone":"z#z"}`,
+		},
+		{
+			name:       "1 field",
+			localityID: LocalityID{Region: "r:r"},
+			str:        `{"region":"r:r"}`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotStr, err := tt.localityID.ToString()
+			if err != nil {
+				t.Errorf("failed to marshal LocalityID: %#v", tt.localityID)
+			}
+			if gotStr != tt.str {
+				t.Errorf("%#v.String() = %q, want %q", tt.localityID, gotStr, tt.str)
+			}
+
+			gotID, err := LocalityIDFromString(tt.str)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("LocalityIDFromString(%q) error = %v, wantErr %v", tt.str, err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(gotID, tt.localityID); diff != "" {
+				t.Errorf("LocalityIDFromString() got = %v, want %v, diff: %s", gotID, tt.localityID, diff)
+			}
+		})
+	}
+}

From 6a318bb011c6613d6f1c98eb0b1b28edfe6b7c0d Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 8 Jan 2021 10:14:53 -0800
Subject: [PATCH 337/481] xds: add HTTP connection manager max_stream_duration
 support (#4122)

---
 xds/internal/client/client.go               | 13 ++++-
 xds/internal/client/client_lds_test.go      |  9 ++-
 xds/internal/client/client_rds_test.go      | 10 +++-
 xds/internal/client/client_xds.go           | 20 +++++--
 xds/internal/resolver/serviceconfig.go      | 10 +++-
 xds/internal/resolver/watch_service.go      | 35 +++++++++--
 xds/internal/resolver/watch_service_test.go | 65 +++++++++++++++++++--
 xds/internal/resolver/xds_resolver_test.go  | 51 +++++++++-------
 8 files changed, 166 insertions(+), 47 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 4007a420f484..10c45415a574 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -147,6 +147,10 @@ type ListenerUpdate struct {
 	RouteConfigName string
 	// SecurityCfg contains security configuration sent by the control plane.
 	SecurityCfg *SecurityConfig
+	// MaxStreamDuration contains the HTTP connection manager's
+	// common_http_protocol_options.max_stream_duration field, or zero if
+	// unset.
+	MaxStreamDuration time.Duration
 }
 
 func (lu *ListenerUpdate) String() string {
@@ -181,8 +185,13 @@ type Route struct {
 	Fraction        *uint32
 
 	// If the matchers above indicate a match, the below configuration is used.
-	Action            map[string]uint32 // action is weighted clusters.
-	MaxStreamDuration time.Duration
+	Action map[string]uint32 // action is weighted clusters.
+	// If MaxStreamDuration is nil, it indicates neither of the route action's
+	// max_stream_duration fields (grpc_timeout_header_max nor
+	// max_stream_duration) were set.  In this case, the ListenerUpdate's
+	// MaxStreamDuration field should be used.  If MaxStreamDuration is set to
+	// an explicit zero duration, the application's deadline should be used.
+	MaxStreamDuration *time.Duration
 }
 
 // HeaderMatcher represents header matchers.
diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go
index 09dd2eea4d84..5172def9a82f 100644
--- a/xds/internal/client/client_lds_test.go
+++ b/xds/internal/client/client_lds_test.go
@@ -21,6 +21,7 @@ package client
 import (
 	"strings"
 	"testing"
+	"time"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
@@ -37,6 +38,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/types/known/durationpb"
 )
 
 func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
@@ -87,6 +89,9 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 							RouteConfigName: v3RouteConfigName,
 						},
 					},
+					CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
+						MaxStreamDuration: durationpb.New(time.Second),
+					},
 				}
 				mcm, _ := ptypes.MarshalAny(cm)
 				lis := &v3listenerpb.Listener{
@@ -278,7 +283,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			name:      "v3 listener resource",
 			resources: []*anypb.Any{v3Lis},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
 			},
 		},
 		{
@@ -286,7 +291,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			resources: []*anypb.Any{v2Lis, v3Lis},
 			wantUpdate: map[string]ListenerUpdate{
 				v2LDSTarget: {RouteConfigName: v2RouteConfigName},
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
 			},
 		},
 	}
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go
index 481030fee211..c91444d4bfb6 100644
--- a/xds/internal/client/client_rds_test.go
+++ b/xds/internal/client/client_rds_test.go
@@ -317,7 +317,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: time.Second}},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: newDurationP(time.Second)}},
 					},
 				},
 			},
@@ -347,7 +347,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: time.Second}},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: newDurationP(time.Second)}},
 					},
 				},
 			},
@@ -377,7 +377,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: 0}},
+						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: newDurationP(0)}},
 					},
 				},
 			},
@@ -803,3 +803,7 @@ func newUInt32P(i uint32) *uint32 {
 func newBoolP(b bool) *bool {
 	return &b
 }
+
+func newDurationP(d time.Duration) *time.Duration {
+	return &d
+}
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go
index 68f65c082412..e2a008200a6a 100644
--- a/xds/internal/client/client_xds.go
+++ b/xds/internal/client/client_xds.go
@@ -80,6 +80,8 @@ func processListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
 // processClientSideListener checks if the provided Listener proto meets
 // the expected criteria. If so, it returns a non-empty routeConfigName.
 func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
+	update := &ListenerUpdate{}
+
 	apiLisAny := lis.GetApiListener().GetApiListener()
 	if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
 		return nil, fmt.Errorf("xds: unexpected resource type: %q in LDS response", apiLisAny.GetTypeUrl())
@@ -98,7 +100,7 @@ func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 		if name == "" {
 			return nil, fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
 		}
-		return &ListenerUpdate{RouteConfigName: name}, nil
+		update.RouteConfigName = name
 	case *v3httppb.HttpConnectionManager_RouteConfig:
 		// TODO: Add support for specifying the RouteConfiguration inline
 		// in the LDS response.
@@ -108,6 +110,10 @@ func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	default:
 		return nil, fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
 	}
+
+	update.MaxStreamDuration = apiLis.GetCommonHttpProtocolOptions().GetMaxStreamDuration().AsDuration()
+
+	return update, nil
 }
 
 func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
@@ -346,12 +352,16 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 		}
 
 		route.Action = clusters
+
 		msd := action.GetMaxStreamDuration()
 		// Prefer grpc_timeout_header_max, if set.
-		if dur := msd.GetGrpcTimeoutHeaderMax(); dur != nil {
-			route.MaxStreamDuration = dur.AsDuration()
-		} else {
-			route.MaxStreamDuration = msd.GetMaxStreamDuration().AsDuration()
+		dur := msd.GetGrpcTimeoutHeaderMax()
+		if dur == nil {
+			dur = msd.GetMaxStreamDuration()
+		}
+		if dur != nil {
+			d := dur.AsDuration()
+			route.MaxStreamDuration = &d
 		}
 		routesRet = append(routesRet, &route)
 	}
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 13d3f2a095fc..95c3c4221609 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -201,11 +201,11 @@ var newWRR = wrr.NewRandom
 func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, error) {
 	cs := &configSelector{
 		r:        r,
-		routes:   make([]route, len(su.Routes)),
+		routes:   make([]route, len(su.routes)),
 		clusters: make(map[string]*clusterInfo),
 	}
 
-	for i, rt := range su.Routes {
+	for i, rt := range su.routes {
 		clusters := newWRR()
 		for cluster, weight := range rt.Action {
 			clusters.Add(cluster, int64(weight))
@@ -227,7 +227,11 @@ func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, erro
 		if err != nil {
 			return nil, err
 		}
-		cs.routes[i].maxStreamDuration = rt.MaxStreamDuration
+		if rt.MaxStreamDuration == nil {
+			cs.routes[i].maxStreamDuration = su.ldsConfig.maxStreamDuration
+		} else {
+			cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration
+		}
 	}
 
 	return cs, nil
diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go
index 01aad899e8be..79b83e95aa3c 100644
--- a/xds/internal/resolver/watch_service.go
+++ b/xds/internal/resolver/watch_service.go
@@ -22,17 +22,28 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"google.golang.org/grpc/internal/grpclog"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
-// serviceUpdate contains information received from the RDS responses which is
-// of interested to the xds resolver. The RDS request is built by first making a
-// LDS to get the RouteConfig name.
+// serviceUpdate contains information received from the LDS/RDS responses which
+// are of interest to the xds resolver. The RDS request is built by first
+// making a LDS to get the RouteConfig name.
 type serviceUpdate struct {
-	// Routes contain matchers+actions to route RPCs.
-	Routes []*xdsclient.Route
+	// routes contain matchers+actions to route RPCs.
+	routes []*xdsclient.Route
+	// ldsConfig contains configuration that applies to all routes.
+	ldsConfig ldsConfig
+}
+
+// ldsConfig contains information received from the LDS responses which are of
+// interest to the xds resolver.
+type ldsConfig struct {
+	// maxStreamDuration is from the HTTP connection manager's
+	// common_http_protocol_options field.
+	maxStreamDuration time.Duration
 }
 
 // watchService uses LDS and RDS to discover information about the provided
@@ -61,6 +72,7 @@ type serviceUpdateWatcher struct {
 	serviceName string
 	ldsCancel   func()
 	serviceCb   func(serviceUpdate, error)
+	lastUpdate  serviceUpdate
 
 	mu        sync.Mutex
 	closed    bool
@@ -84,6 +96,7 @@ func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, er
 			w.rdsCancel()
 			w.rdsName = ""
 			w.rdsCancel = nil
+			w.lastUpdate = serviceUpdate{}
 		}
 		// The other error cases still return early without canceling the
 		// existing RDS watch.
@@ -91,9 +104,18 @@ func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, er
 		return
 	}
 
+	oldLDSConfig := w.lastUpdate.ldsConfig
+	w.lastUpdate.ldsConfig = ldsConfig{maxStreamDuration: update.MaxStreamDuration}
+
 	if w.rdsName == update.RouteConfigName {
 		// If the new RouteConfigName is same as the previous, don't cancel and
 		// restart the RDS watch.
+		if w.lastUpdate.ldsConfig != oldLDSConfig {
+			// The route name didn't change but the LDS data did; send it now.
+			// If the route name did change, then we will wait until the first
+			// RDS update before reporting this LDS config.
+			w.serviceCb(w.lastUpdate, nil)
+		}
 		return
 	}
 	w.rdsName = update.RouteConfigName
@@ -127,7 +149,8 @@ func (w *serviceUpdateWatcher) handleRDSResp(update xdsclient.RouteConfigUpdate,
 		return
 	}
 
-	w.serviceCb(serviceUpdate{Routes: matchVh.Routes}, nil)
+	w.lastUpdate.routes = matchVh.Routes
+	w.serviceCb(w.lastUpdate, nil)
 }
 
 func (w *serviceUpdateWatcher) close() {
diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go
index 2291d485b8b2..55a20372c018 100644
--- a/xds/internal/resolver/watch_service_test.go
+++ b/xds/internal/resolver/watch_service_test.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -138,7 +139,7 @@ func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantU
 		return fmt.Errorf("timeout when waiting for service update: %v", err)
 	}
 	gotUpdate := u.(serviceUpdateErr)
-	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()) {
+	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty(), cmp.AllowUnexported(serviceUpdate{}, ldsConfig{})) {
 		return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil),  diff (-want +got):\n%s", gotUpdate.u, gotUpdate.err, wantUpdate, cmp.Diff(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()))
 	}
 	return nil
@@ -165,7 +166,7 @@ func (s) TestServiceWatch(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
@@ -179,7 +180,7 @@ func (s) TestServiceWatch(t *testing.T) {
 	}
 
 	wantUpdate2 := serviceUpdate{
-		Routes: []*xdsclient.Route{{
+		routes: []*xdsclient.Route{{
 			Path:   newStringP(""),
 			Action: map[string]uint32{cluster: 1},
 		}},
@@ -219,7 +220,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
@@ -240,7 +241,7 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr+"2")
 
 	// RDS update for the new name.
-	wantUpdate2 := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}}}
+	wantUpdate2 := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
@@ -254,6 +255,58 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	}
 }
 
+// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS
+// response, the second LDS response includes a new MaxStreamDuration.  It also
+// verifies this is reported in subsequent RDS updates.
+func (s) TestServiceWatchLDSUpdateMaxStreamDuration(t *testing.T) {
+	serviceUpdateCh := testutils.NewChannel()
+	xdsC := fakeclient.NewClient()
+	cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	}, nil)
+	defer cancelWatch()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}, ldsConfig: ldsConfig{maxStreamDuration: time.Second}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Another LDS update with the same RDS_name but different MaxStreamDuration (zero in this case).
+	wantUpdate2 := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
+	}
+
+	// RDS update.
+	wantUpdate3 := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate3); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // TestServiceNotCancelRDSOnSameLDSUpdate covers the case that if the second LDS
 // update contains the same RDS name as the previous, the RDS watch isn't
 // canceled and restarted.
@@ -271,7 +324,7 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index a3c0f2866353..13f17e513df0 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -511,7 +511,7 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR)
@@ -526,11 +526,11 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 				Routes: []*client.Route{{
 					Prefix:            newStringP("/foo"),
 					Action:            map[string]uint32{"A": 1},
-					MaxStreamDuration: 5 * time.Second,
+					MaxStreamDuration: newDurationP(5 * time.Second),
 				}, {
 					Prefix:            newStringP("/bar"),
 					Action:            map[string]uint32{"B": 1},
-					MaxStreamDuration: time.Duration(0),
+					MaxStreamDuration: newDurationP(0),
 				}, {
 					Prefix: newStringP(""),
 					Action: map[string]uint32{"C": 1},
@@ -554,43 +554,50 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 	}
 
 	testCases := []struct {
+		name           string
 		method         string
 		timeoutSupport bool
 		want           *time.Duration
 	}{{
+		name:           "RDS setting",
 		method:         "/foo/method",
 		timeoutSupport: true,
-		want:           func() *time.Duration { x := 5 * time.Second; return &x }(),
+		want:           newDurationP(5 * time.Second),
 	}, {
+		name:           "timeout support disabled",
 		method:         "/foo/method",
 		timeoutSupport: false,
 		want:           nil,
 	}, {
+		name:           "explicit zero in RDS; ignore LDS",
 		method:         "/bar/method",
 		timeoutSupport: true,
 		want:           nil,
 	}, {
+		name:           "no config in RDS; fallback to LDS",
 		method:         "/baz/method",
 		timeoutSupport: true,
-		want:           nil,
+		want:           newDurationP(time.Second),
 	}}
 
 	for _, tc := range testCases {
-		env.TimeoutSupport = tc.timeoutSupport
-		req := iresolver.RPCInfo{
-			Method:  tc.method,
-			Context: context.Background(),
-		}
-		res, err := cs.SelectConfig(req)
-		if err != nil {
-			t.Errorf("Unexpected error from cs.SelectConfig(%v): %v", req, err)
-			continue
-		}
-		res.OnCommitted()
-		got := res.MethodConfig.Timeout
-		if !reflect.DeepEqual(got, tc.want) {
-			t.Errorf("For method %q: res.MethodConfig.Timeout = %v; want %v", tc.method, got, tc.want)
-		}
+		t.Run(tc.name, func(t *testing.T) {
+			env.TimeoutSupport = tc.timeoutSupport
+			req := iresolver.RPCInfo{
+				Method:  tc.method,
+				Context: context.Background(),
+			}
+			res, err := cs.SelectConfig(req)
+			if err != nil {
+				t.Errorf("Unexpected error from cs.SelectConfig(%v): %v", req, err)
+				return
+			}
+			res.OnCommitted()
+			got := res.MethodConfig.Timeout
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("For method %q: res.MethodConfig.Timeout = %v; want %v", tc.method, got, tc.want)
+			}
+		})
 	}
 }
 
@@ -856,3 +863,7 @@ func replaceRandNumGenerator(start int64) func() {
 		grpcrandInt63n = grpcrand.Int63n
 	}
 }
+
+func newDurationP(d time.Duration) *time.Duration {
+	return &d
+}

From 85e55dc558e1ad7f7d4d165480b47bf6efd3c6ff Mon Sep 17 00:00:00 2001
From: Garrett Gutierrez <garrettgutierrez1@gmail.com>
Date: Fri, 8 Jan 2021 16:09:17 -0800
Subject: [PATCH 338/481] interop: update client for xds testing support
 (#4108)

---
 interop/xds/client/client.go | 156 ++++++++++++++++++++++++++++++-----
 1 file changed, 135 insertions(+), 21 deletions(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index b119bcd2b865..7afdb20e8d72 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -38,6 +38,10 @@ import (
 	_ "google.golang.org/grpc/xds"
 )
 
+func init() {
+	rpcCfgs.Store([]*rpcConfig{{typ: unaryCall}})
+}
+
 type statsWatcherKey struct {
 	startID int32
 	endID   int32
@@ -73,21 +77,84 @@ func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse {
 	}
 }
 
+type accumulatedStats struct {
+	mu                       sync.Mutex
+	numRpcsStartedByMethod   map[string]int32
+	numRpcsSucceededByMethod map[string]int32
+	numRpcsFailedByMethod    map[string]int32
+}
+
+// copyStatsMap makes a copy of the map, and also replaces the RPC type string
+// to the proto string. E.g. "UnaryCall" -> "UNARY_CALL".
+func copyStatsMap(originalMap map[string]int32) (newMap map[string]int32) {
+	newMap = make(map[string]int32)
+	for k, v := range originalMap {
+		var kk string
+		switch k {
+		case unaryCall:
+			kk = testpb.ClientConfigureRequest_UNARY_CALL.String()
+		case emptyCall:
+			kk = testpb.ClientConfigureRequest_EMPTY_CALL.String()
+		default:
+			logger.Warningf("unrecognized rpc type: %s", k)
+		}
+		if kk == "" {
+			continue
+		}
+		newMap[kk] = v
+	}
+	return newMap
+}
+
+func (as *accumulatedStats) buildResp() *testpb.LoadBalancerAccumulatedStatsResponse {
+	as.mu.Lock()
+	defer as.mu.Unlock()
+	return &testpb.LoadBalancerAccumulatedStatsResponse{
+		NumRpcsStartedByMethod:   copyStatsMap(as.numRpcsStartedByMethod),
+		NumRpcsSucceededByMethod: copyStatsMap(as.numRpcsSucceededByMethod),
+		NumRpcsFailedByMethod:    copyStatsMap(as.numRpcsFailedByMethod),
+	}
+}
+
+func (as *accumulatedStats) startRPC(rpcType string) {
+	as.mu.Lock()
+	defer as.mu.Unlock()
+	as.numRpcsStartedByMethod[rpcType]++
+}
+
+func (as *accumulatedStats) finishRPC(rpcType string, failed bool) {
+	as.mu.Lock()
+	defer as.mu.Unlock()
+	if failed {
+		as.numRpcsFailedByMethod[rpcType]++
+		return
+	}
+	as.numRpcsSucceededByMethod[rpcType]++
+}
+
 var (
 	failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail after first success")
 	numChannels     = flag.Int("num_channels", 1, "Num of channels")
 	printResponse   = flag.Bool("print_response", false, "Write RPC response to stdout")
 	qps             = flag.Int("qps", 1, "QPS per channel, for each type of RPC")
-	rpc             = flag.String("rpc", "UnaryCall", "Types of RPCs to make, ',' separated string. RPCs can be EmptyCall or UnaryCall")
-	rpcMetadata     = flag.String("metadata", "", "The metadata to send with RPC, in format EmptyCall:key1:value1,UnaryCall:key2:value2")
+	rpc             = flag.String("rpc", "UnaryCall", "Types of RPCs to make, ',' separated string. RPCs can be EmptyCall or UnaryCall. Deprecated: Use Configure RPC to XdsUpdateClientConfigureServiceServer instead.")
+	rpcMetadata     = flag.String("metadata", "", "The metadata to send with RPC, in format EmptyCall:key1:value1,UnaryCall:key2:value2. Deprecated: Use Configure RPC to XdsUpdateClientConfigureServiceServer instead.")
 	rpcTimeout      = flag.Duration("rpc_timeout", 20*time.Second, "Per RPC timeout")
 	server          = flag.String("server", "localhost:8080", "Address of server to connect to")
 	statsPort       = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service")
 
+	rpcCfgs atomic.Value
+
 	mu               sync.Mutex
 	currentRequestID int32
 	watchers         = make(map[statsWatcherKey]*statsWatcher)
 
+	accStats = accumulatedStats{
+		numRpcsStartedByMethod:   make(map[string]int32),
+		numRpcsSucceededByMethod: make(map[string]int32),
+		numRpcsFailedByMethod:    make(map[string]int32),
+	}
+
 	// 0 or 1 representing an RPC has succeeded. Use hasRPCSucceeded and
 	// setRPCSucceeded to access in a safe manner.
 	rpcSucceeded uint32
@@ -163,6 +230,47 @@ func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalanc
 	}
 }
 
+func (s *statsService) GetClientAccumulatedStats(ctx context.Context, in *testpb.LoadBalancerAccumulatedStatsRequest) (*testpb.LoadBalancerAccumulatedStatsResponse, error) {
+	return accStats.buildResp(), nil
+}
+
+type configureService struct {
+	testpb.UnimplementedXdsUpdateClientConfigureServiceServer
+}
+
+func (s *configureService) Configure(ctx context.Context, in *testpb.ClientConfigureRequest) (*testpb.ClientConfigureResponse, error) {
+	rpcsToMD := make(map[testpb.ClientConfigureRequest_RpcType][]string)
+	for _, typ := range in.GetTypes() {
+		rpcsToMD[typ] = nil
+	}
+	for _, md := range in.GetMetadata() {
+		typ := md.GetType()
+		strs, ok := rpcsToMD[typ]
+		if !ok {
+			continue
+		}
+		rpcsToMD[typ] = append(strs, md.GetKey(), md.GetValue())
+	}
+	cfgs := make([]*rpcConfig, 0, len(rpcsToMD))
+	for typ, md := range rpcsToMD {
+		var rpcType string
+		switch typ {
+		case testpb.ClientConfigureRequest_UNARY_CALL:
+			rpcType = unaryCall
+		case testpb.ClientConfigureRequest_EMPTY_CALL:
+			rpcType = emptyCall
+		default:
+			return nil, fmt.Errorf("unsupported RPC type: %v", typ)
+		}
+		cfgs = append(cfgs, &rpcConfig{
+			typ: rpcType,
+			md:  metadata.Pairs(md...),
+		})
+	}
+	rpcCfgs.Store(cfgs)
+	return &testpb.ClientConfigureResponse{}, nil
+}
+
 const (
 	unaryCall string = "UnaryCall"
 	emptyCall string = "EmptyCall"
@@ -218,7 +326,7 @@ func parseRPCMetadata(rpcMetadataStr string, rpcs []string) []*rpcConfig {
 
 func main() {
 	flag.Parse()
-	rpcCfgs := parseRPCMetadata(*rpcMetadata, parseRPCTypes(*rpc))
+	rpcCfgs.Store(parseRPCMetadata(*rpcMetadata, parseRPCTypes(*rpc)))
 
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *statsPort))
 	if err != nil {
@@ -227,6 +335,7 @@ func main() {
 	s := grpc.NewServer()
 	defer s.Stop()
 	testpb.RegisterLoadBalancerStatsServiceServer(s, &statsService{})
+	testpb.RegisterXdsUpdateClientConfigureServiceServer(s, &configureService{})
 	go s.Serve(lis)
 
 	clients := make([]testpb.TestServiceClient, *numChannels)
@@ -240,7 +349,7 @@ func main() {
 	}
 	ticker := time.NewTicker(time.Second / time.Duration(*qps**numChannels))
 	defer ticker.Stop()
-	sendRPCs(clients, rpcCfgs, ticker)
+	sendRPCs(clients, ticker)
 }
 
 func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) {
@@ -257,6 +366,7 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf
 		header metadata.MD
 		err    error
 	)
+	accStats.startRPC(cfg.typ)
 	switch cfg.typ {
 	case unaryCall:
 		var resp *testpb.SimpleResponse
@@ -270,8 +380,10 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf
 		_, err = c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p), grpc.Header(&header))
 	}
 	if err != nil {
+		accStats.finishRPC(cfg.typ, true)
 		return nil, nil, err
 	}
+	accStats.finishRPC(cfg.typ, false)
 
 	hosts := header["hostname"]
 	if len(hosts) > 0 {
@@ -280,26 +392,28 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf
 	return &p, &info, err
 }
 
-func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *time.Ticker) {
+func sendRPCs(clients []testpb.TestServiceClient, ticker *time.Ticker) {
 	var i int
 	for range ticker.C {
-		go func(i int) {
-			// Get and increment request ID, and save a list of watchers that
-			// are interested in this RPC.
-			mu.Lock()
-			savedRequestID := currentRequestID
-			currentRequestID++
-			savedWatchers := []*statsWatcher{}
-			for key, value := range watchers {
-				if key.startID <= savedRequestID && savedRequestID < key.endID {
-					savedWatchers = append(savedWatchers, value)
-				}
+		// Get and increment request ID, and save a list of watchers that are
+		// interested in this RPC.
+		mu.Lock()
+		savedRequestID := currentRequestID
+		currentRequestID++
+		savedWatchers := []*statsWatcher{}
+		for key, value := range watchers {
+			if key.startID <= savedRequestID && savedRequestID < key.endID {
+				savedWatchers = append(savedWatchers, value)
 			}
-			mu.Unlock()
+		}
+		mu.Unlock()
 
-			c := clients[i]
+		// Get the RPC metadata configurations from the Configure RPC.
+		cfgs := rpcCfgs.Load().([]*rpcConfig)
 
-			for _, cfg := range cfgs {
+		c := clients[i]
+		for _, cfg := range cfgs {
+			go func(cfg *rpcConfig) {
 				p, info, err := makeOneRPC(c, cfg)
 
 				for _, watcher := range savedWatchers {
@@ -325,8 +439,8 @@ func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *tim
 						fmt.Printf("RPC %q, failed with %v\n", cfg.typ, err)
 					}
 				}
-			}
-		}(i)
+			}(cfg)
+		}
 		i = (i + 1) % len(clients)
 	}
 }

From 083393f287fe61b048da16074ca39bd93cd5e6bc Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 8 Jan 2021 16:47:46 -0800
Subject: [PATCH 339/481] xds/resolver: fix resource deletion (#4143)

---
 xds/internal/resolver/serviceconfig.go     |  23 +--
 xds/internal/resolver/xds_resolver.go      | 102 +++++++-----
 xds/internal/resolver/xds_resolver_test.go | 172 +++++++++++++++++++++
 3 files changed, 247 insertions(+), 50 deletions(-)

diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 95c3c4221609..d61a546a04e2 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -113,6 +113,9 @@ type configSelector struct {
 var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found")
 
 func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
+	if cs == nil {
+		return nil, status.Errorf(codes.Unavailable, "no valid clusters")
+	}
 	var rt *route
 	// Loop through routes in order and select first match.
 	for _, r := range cs.routes {
@@ -157,17 +160,8 @@ func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RP
 	return config, nil
 }
 
-// incRefs increments refs of all clusters referenced by this config selector.
-func (cs *configSelector) incRefs() {
-	// Loops over cs.clusters, but these are pointers to entries in
-	// activeClusters.
-	for _, ci := range cs.clusters {
-		atomic.AddInt32(&ci.refCount, 1)
-	}
-}
-
-// decRefs decrements refs of all clusters referenced by this config selector.
-func (cs *configSelector) decRefs() {
+// stop decrements refs of all clusters referenced by this config selector.
+func (cs *configSelector) stop() {
 	// The resolver's old configSelector may be nil.  Handle that here.
 	if cs == nil {
 		return
@@ -234,6 +228,13 @@ func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, erro
 		}
 	}
 
+	// Account for this config selector's clusters.  Do this after no further
+	// errors may occur.  Note: cs.clusters are pointers to entries in
+	// activeClusters.
+	for _, ci := range cs.clusters {
+		atomic.AddInt32(&ci.refCount, 1)
+	}
+
 	return cs, nil
 }
 
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index a99403edcfa9..b40e115299a4 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -144,6 +144,43 @@ type xdsResolver struct {
 	curConfigSelector *configSelector
 }
 
+// sendNewServiceConfig prunes active clusters, generates a new service config
+// based on the current set of active clusters, and sends an update to the
+// channel with that service config and the provided config selector.  Returns
+// false if an error occurs while generating the service config and the update
+// cannot be sent.
+func (r *xdsResolver) sendNewServiceConfig(cs *configSelector) bool {
+	// Delete entries from r.activeClusters with zero references;
+	// otherwise serviceConfigJSON will generate a config including
+	// them.
+	r.pruneActiveClusters()
+
+	if cs == nil && len(r.activeClusters) == 0 {
+		// There are no clusters and we are sending a failing configSelector.
+		// Send an empty config, which picks pick-first, with no address, and
+		// puts the ClientConn into transient failure.
+		r.cc.UpdateState(resolver.State{ServiceConfig: r.cc.ParseServiceConfig("{}")})
+		return true
+	}
+
+	// Produce the service config.
+	sc, err := serviceConfigJSON(r.activeClusters)
+	if err != nil {
+		// JSON marshal error; should never happen.
+		r.logger.Errorf("%v", err)
+		r.cc.ReportError(err)
+		return false
+	}
+	r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc)
+
+	// Send the update to the ClientConn.
+	state := iresolver.SetConfigSelector(resolver.State{
+		ServiceConfig: r.cc.ParseServiceConfig(sc),
+	}, cs)
+	r.cc.UpdateState(state)
+	return true
+}
+
 // run is a long running goroutine which blocks on receiving service updates
 // and passes it on the ClientConn.
 func (r *xdsResolver) run() {
@@ -155,15 +192,15 @@ func (r *xdsResolver) run() {
 			if update.err != nil {
 				r.logger.Warningf("Watch error on resource %v from xds-client %p, %v", r.target.Endpoint, r.client, update.err)
 				if xdsclient.ErrType(update.err) == xdsclient.ErrorTypeResourceNotFound {
-					// If error is resource-not-found, it means the LDS resource
-					// was removed. Send an empty service config, which picks
-					// pick-first, with no address, and puts the ClientConn into
-					// transient failure..
-					r.cc.UpdateState(resolver.State{
-						ServiceConfig: r.cc.ParseServiceConfig("{}"),
-					})
-					// Dereference the active config selector, if one exists.
-					r.curConfigSelector.decRefs()
+					// If error is resource-not-found, it means the LDS
+					// resource was removed. Ultimately send an empty service
+					// config, which picks pick-first, with no address, and
+					// puts the ClientConn into transient failure.  Before we
+					// can do that, we may need to send a normal service config
+					// along with an erroring (nil) config selector.
+					r.sendNewServiceConfig(nil)
+					// Stop and dereference the active config selector, if one exists.
+					r.curConfigSelector.stop()
 					r.curConfigSelector = nil
 					continue
 				}
@@ -173,43 +210,30 @@ func (r *xdsResolver) run() {
 				r.cc.ReportError(update.err)
 				continue
 			}
-			var cs *configSelector
-			if !update.emptyUpdate {
-				// Create the config selector for this update.
-				var err error
-				if cs, err = r.newConfigSelector(update.su); err != nil {
-					r.logger.Warningf("Error parsing update on resource %v from xds-client %p: %v", r.target.Endpoint, r.client, err)
-					r.cc.ReportError(err)
-					continue
-				}
-			} else {
-				// Empty update; use the existing config selector.
-				cs = r.curConfigSelector
+			if update.emptyUpdate {
+				r.sendNewServiceConfig(r.curConfigSelector)
+				continue
 			}
-			// Account for this config selector's clusters.
-			cs.incRefs()
-			// Delete entries from r.activeClusters with zero references;
-			// otherwise serviceConfigJSON will generate a config including
-			// them.
-			r.pruneActiveClusters()
-			// Produce the service config.
-			sc, err := serviceConfigJSON(r.activeClusters)
+
+			// Create the config selector for this update.
+			cs, err := r.newConfigSelector(update.su)
 			if err != nil {
-				// JSON marshal error; should never happen.
-				r.logger.Errorf("%v", err)
+				r.logger.Warningf("Error parsing update on resource %v from xds-client %p: %v", r.target.Endpoint, r.client, err)
 				r.cc.ReportError(err)
-				cs.decRefs()
 				continue
 			}
-			r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc)
-			// Send the update to the ClientConn.
-			state := iresolver.SetConfigSelector(resolver.State{
-				ServiceConfig: r.cc.ParseServiceConfig(sc),
-			}, cs)
-			r.cc.UpdateState(state)
+
+			if !r.sendNewServiceConfig(cs) {
+				// JSON error creating the service config (unexpected); erase
+				// this config selector and ignore this update, continuing with
+				// the previous config selector.
+				cs.stop()
+				continue
+			}
+
 			// Decrement references to the old config selector and assign the
 			// new one as the current one.
-			r.curConfigSelector.decRefs()
+			r.curConfigSelector.stop()
 			r.curConfigSelector = cs
 		}
 	}
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 13f17e513df0..218a7a610913 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
 	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal"
@@ -36,6 +37,7 @@ import (
 	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
 	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config
 	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
 	"google.golang.org/grpc/xds/internal/client"
@@ -435,6 +437,176 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	}
 }
 
+// TestXDSResolverRemovedWithRPCs tests the case where a config selector sends
+// an empty update to the resolver after the resource is removed.
+func (s) TestXDSResolverRemovedWithRPCs(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer cancel()
+	defer xdsR.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	// Invoke the watchAPI callback with a good service update and wait for the
+	// UpdateState method to be called on the ClientConn.
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+			},
+		},
+	}, nil)
+
+	gotState, err := tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState := gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+
+	// "Make an RPC" by invoking the config selector.
+	cs := iresolver.GetConfigSelector(rState)
+	if cs == nil {
+		t.Fatalf("received nil config selector")
+	}
+
+	res, err := cs.SelectConfig(iresolver.RPCInfo{Context: context.Background()})
+	if err != nil {
+		t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
+	}
+
+	// Delete the resource
+	suErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "resource removed error")
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
+
+	if _, err = tcc.stateCh.Receive(ctx); err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+
+	// "Finish the RPC"; this could cause a panic if the resolver doesn't
+	// handle it correctly.
+	res.OnCommitted()
+}
+
+// TestXDSResolverRemovedResource tests for proper behavior after a resource is
+// removed.
+func (s) TestXDSResolverRemovedResource(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer cancel()
+	defer xdsR.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+
+	// Invoke the watchAPI callback with a good service update and wait for the
+	// UpdateState method to be called on the ClientConn.
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+			},
+		},
+	}, nil)
+	wantJSON := `{"loadBalancingConfig":[{
+    "xds_cluster_manager_experimental":{
+      "children":{
+        "test-cluster-1":{
+          "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}]
+        }
+      }
+    }}]}`
+	wantSCParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(wantJSON)
+
+	gotState, err := tcc.stateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState := gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) {
+		t.Errorf("ClientConn.UpdateState received different service config")
+		t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Error("want: ", cmp.Diff(nil, wantSCParsed.Config))
+	}
+
+	// "Make an RPC" by invoking the config selector.
+	cs := iresolver.GetConfigSelector(rState)
+	if cs == nil {
+		t.Fatalf("received nil config selector")
+	}
+
+	res, err := cs.SelectConfig(iresolver.RPCInfo{Context: context.Background()})
+	if err != nil {
+		t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
+	}
+
+	// "Finish the RPC"; this could cause a panic if the resolver doesn't
+	// handle it correctly.
+	res.OnCommitted()
+
+	// Delete the resource.  The channel should receive a service config with the
+	// original cluster but with an erroring config selector.
+	suErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "resource removed error")
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
+
+	if gotState, err = tcc.stateCh.Receive(ctx); err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState = gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) {
+		t.Errorf("ClientConn.UpdateState received different service config")
+		t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Error("want: ", cmp.Diff(nil, wantSCParsed.Config))
+	}
+
+	// "Make another RPC" by invoking the config selector.
+	cs = iresolver.GetConfigSelector(rState)
+	if cs == nil {
+		t.Fatalf("received nil config selector")
+	}
+
+	res, err = cs.SelectConfig(iresolver.RPCInfo{Context: context.Background()})
+	if err == nil || status.Code(err) != codes.Unavailable {
+		t.Fatalf("Expected UNAVAILABLE error from cs.SelectConfig(_); got %v, %v", res, err)
+	}
+
+	// In the meantime, an empty ServiceConfig update should have been sent.
+	if gotState, err = tcc.stateCh.Receive(ctx); err != nil {
+		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+	}
+	rState = gotState.(resolver.State)
+	if err := rState.ServiceConfig.Err; err != nil {
+		t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+	}
+	wantSCParsed = internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)("{}")
+	if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) {
+		t.Errorf("ClientConn.UpdateState received different service config")
+		t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config))
+		t.Error("want: ", cmp.Diff(nil, wantSCParsed.Config))
+	}
+}
+
 func (s) TestXDSResolverWRR(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	xdsR, tcc, cancel := testSetup(t, setupOpts{

From fb40d83340e8c3830cc4907c00a1d8a003c6902c Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 8 Jan 2021 17:16:38 -0800
Subject: [PATCH 340/481] xds interop: turn on circuit breaking test (#4144)

Also need to set server image to version 3, so that the server blocks the
pending RPCs
---
 test/kokoro/xds.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index 23c9d0119425..565d55aa116c 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -27,9 +27,9 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh
 # they are added into "all".
 GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
-    --test_case="all,path_matching,header_matching" \
+    --test_case="all,path_matching,header_matching,circuit_breaking" \
     --project_id=grpc-testing \
-    --source_image=projects/grpc-testing/global/images/xds-test-server-2 \
+    --source_image=projects/grpc-testing/global/images/xds-test-server-3 \
     --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \
     --gcp_suffix=$(date '+%s') \
     --verbose \

From 4cf4a98505bcdfb27ea2a5427b6e5a2d143dd060 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 11 Jan 2021 10:09:13 -0800
Subject: [PATCH 341/481] Change version to 1.36.0-dev (#4142)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index 1e75a5c7954f..1b9f3715952d 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.35.0-dev"
+const Version = "1.36.0-dev"

From d3ae124a07fcb3eeddc18d84c5df3e82e182380c Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 12 Jan 2021 12:23:41 -0800
Subject: [PATCH 342/481] cleanup: use different import alias for services than
 messages (#4148)

---
 benchmark/benchmain/main.go            | 18 ++++----
 benchmark/benchmark.go                 | 24 ++++++-----
 benchmark/client/main.go               |  4 +-
 benchmark/worker/benchmark_client.go   | 10 +++--
 benchmark/worker/main.go               | 12 +++---
 binarylog/binarylog_end2end_test.go    | 28 +++++++------
 interop/alts/client/client.go          |  4 +-
 interop/alts/server/server.go          |  5 ++-
 interop/client/client.go               |  7 ++--
 interop/grpclb_fallback/client.go      | 14 ++++---
 interop/http2/negative_http2_client.go | 18 ++++----
 interop/server/server.go               |  5 ++-
 interop/test_utils.go                  | 58 +++++++++++++-------------
 interop/xds/client/client.go           | 20 +++++----
 interop/xds/server/server.go           |  8 ++--
 stats/stats_test.go                    | 26 ++++++------
 stress/client/main.go                  |  7 ++--
 17 files changed, 149 insertions(+), 119 deletions(-)

diff --git a/benchmark/benchmain/main.go b/benchmark/benchmain/main.go
index 509a1b905654..55427035f41b 100644
--- a/benchmark/benchmain/main.go
+++ b/benchmark/benchmain/main.go
@@ -65,10 +65,12 @@ import (
 	"google.golang.org/grpc/benchmark/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/channelz"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/test/bufconn"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
@@ -259,7 +261,7 @@ func unconstrainedStreamBenchmark(start startFunc, stop ucStopFunc, bf stats.Fea
 // service. The client is configured using the different options in the passed
 // 'bf'. Also returns a cleanup function to close the client and release
 // resources.
-func makeClient(bf stats.Features) (testpb.BenchmarkServiceClient, func()) {
+func makeClient(bf stats.Features) (testgrpc.BenchmarkServiceClient, func()) {
 	nw := &latency.Network{Kbps: bf.Kbps, Latency: bf.Latency, MTU: bf.MTU}
 	opts := []grpc.DialOption{}
 	sopts := []grpc.ServerOption{}
@@ -327,7 +329,7 @@ func makeClient(bf stats.Features) (testpb.BenchmarkServiceClient, func()) {
 	lis = nw.Listener(lis)
 	stopper := bm.StartServer(bm.ServerInfo{Type: "protobuf", Listener: lis}, sopts...)
 	conn := bm.NewClientConn("" /* target not used */, opts...)
-	return testpb.NewBenchmarkServiceClient(conn), func() {
+	return testgrpc.NewBenchmarkServiceClient(conn), func() {
 		conn.Close()
 		stopper()
 	}
@@ -351,7 +353,7 @@ func makeFuncUnary(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) {
 func makeFuncStream(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) {
 	tc, cleanup := makeClient(bf)
 
-	streams := make([]testpb.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls)
+	streams := make([]testgrpc.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls)
 	for i := 0; i < bf.MaxConcurrentCalls; i++ {
 		stream, err := tc.StreamingCall(context.Background())
 		if err != nil {
@@ -402,10 +404,10 @@ func makeFuncUnconstrainedStream(bf stats.Features) (rpcSendFunc, rpcRecvFunc, r
 		}, cleanup
 }
 
-func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_StreamingCallClient, *testpb.SimpleRequest, rpcCleanupFunc) {
+func setupUnconstrainedStream(bf stats.Features) ([]testgrpc.BenchmarkService_StreamingCallClient, *testpb.SimpleRequest, rpcCleanupFunc) {
 	tc, cleanup := makeClient(bf)
 
-	streams := make([]testpb.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls)
+	streams := make([]testgrpc.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls)
 	md := metadata.Pairs(benchmark.UnconstrainedStreamingHeader, "1")
 	ctx := metadata.NewOutgoingContext(context.Background(), md)
 	for i := 0; i < bf.MaxConcurrentCalls; i++ {
@@ -428,13 +430,13 @@ func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_Stre
 
 // Makes a UnaryCall gRPC request using the given BenchmarkServiceClient and
 // request and response sizes.
-func unaryCaller(client testpb.BenchmarkServiceClient, reqSize, respSize int) {
+func unaryCaller(client testgrpc.BenchmarkServiceClient, reqSize, respSize int) {
 	if err := bm.DoUnaryCall(client, reqSize, respSize); err != nil {
 		logger.Fatalf("DoUnaryCall failed: %v", err)
 	}
 }
 
-func streamCaller(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) {
+func streamCaller(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, respSize int) {
 	if err := bm.DoStreamingRoundTrip(stream, reqSize, respSize); err != nil {
 		logger.Fatalf("DoStreamingRoundTrip failed: %v", err)
 	}
diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go
index 92fbeb888365..a8ae40fa6ada 100644
--- a/benchmark/benchmark.go
+++ b/benchmark/benchmark.go
@@ -31,9 +31,11 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var logger = grpclog.Component("benchmark")
@@ -61,7 +63,7 @@ func NewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 }
 
 type testServer struct {
-	testpb.UnimplementedBenchmarkServiceServer
+	testgrpc.UnimplementedBenchmarkServiceServer
 }
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -75,7 +77,7 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 // of ping-pong.
 const UnconstrainedStreamingHeader = "unconstrained-streaming"
 
-func (s *testServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error {
+func (s *testServer) StreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error {
 	if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[UnconstrainedStreamingHeader]) != 0 {
 		return s.UnconstrainedStreamingCall(stream)
 	}
@@ -100,7 +102,7 @@ func (s *testServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallS
 	}
 }
 
-func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error {
+func (s *testServer) UnconstrainedStreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error {
 	in := new(testpb.SimpleRequest)
 	// Receive a message to learn response type and size.
 	err := stream.RecvMsg(in)
@@ -151,7 +153,7 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_S
 // byteBufServer is a gRPC server that sends and receives byte buffer.
 // The purpose is to benchmark the gRPC performance without protobuf serialization/deserialization overhead.
 type byteBufServer struct {
-	testpb.UnimplementedBenchmarkServiceServer
+	testgrpc.UnimplementedBenchmarkServiceServer
 	respSize int32
 }
 
@@ -161,7 +163,7 @@ func (s *byteBufServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest)
 	return &testpb.SimpleResponse{}, nil
 }
 
-func (s *byteBufServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error {
+func (s *byteBufServer) StreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error {
 	for {
 		var in []byte
 		err := stream.(grpc.ServerStream).RecvMsg(&in)
@@ -201,13 +203,13 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() {
 	s := grpc.NewServer(opts...)
 	switch info.Type {
 	case "protobuf":
-		testpb.RegisterBenchmarkServiceServer(s, &testServer{})
+		testgrpc.RegisterBenchmarkServiceServer(s, &testServer{})
 	case "bytebuf":
 		respSize, ok := info.Metadata.(int32)
 		if !ok {
 			logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type)
 		}
-		testpb.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize})
+		testgrpc.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize})
 	default:
 		logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type)
 	}
@@ -218,7 +220,7 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() {
 }
 
 // DoUnaryCall performs an unary RPC with given stub and request and response sizes.
-func DoUnaryCall(tc testpb.BenchmarkServiceClient, reqSize, respSize int) error {
+func DoUnaryCall(tc testgrpc.BenchmarkServiceClient, reqSize, respSize int) error {
 	pl := NewPayload(testpb.PayloadType_COMPRESSABLE, reqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType: pl.Type,
@@ -232,7 +234,7 @@ func DoUnaryCall(tc testpb.BenchmarkServiceClient, reqSize, respSize int) error
 }
 
 // DoStreamingRoundTrip performs a round trip for a single streaming rpc.
-func DoStreamingRoundTrip(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) error {
+func DoStreamingRoundTrip(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, respSize int) error {
 	pl := NewPayload(testpb.PayloadType_COMPRESSABLE, reqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType: pl.Type,
@@ -253,7 +255,7 @@ func DoStreamingRoundTrip(stream testpb.BenchmarkService_StreamingCallClient, re
 }
 
 // DoByteBufStreamingRoundTrip performs a round trip for a single streaming rpc, using a custom codec for byte buffer.
-func DoByteBufStreamingRoundTrip(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) error {
+func DoByteBufStreamingRoundTrip(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, respSize int) error {
 	out := make([]byte, reqSize)
 	if err := stream.(grpc.ClientStream).SendMsg(&out); err != nil {
 		return fmt.Errorf("/BenchmarkService/StreamingCall.(ClientStream).SendMsg(_) = %v, want <nil>", err)
diff --git a/benchmark/client/main.go b/benchmark/client/main.go
index c744348469ed..caf2db70a501 100644
--- a/benchmark/client/main.go
+++ b/benchmark/client/main.go
@@ -53,6 +53,8 @@ import (
 	"google.golang.org/grpc/benchmark/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/syscall"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
@@ -164,7 +166,7 @@ func runWithConn(cc *grpc.ClientConn, req *testpb.SimpleRequest, warmDeadline, e
 }
 
 func makeCaller(cc *grpc.ClientConn, req *testpb.SimpleRequest) func() {
-	client := testpb.NewBenchmarkServiceClient(cc)
+	client := testgrpc.NewBenchmarkServiceClient(cc)
 	if *rpcType == "unary" {
 		return func() {
 			if _, err := client.UnaryCall(context.Background(), req); err != nil {
diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go
index f760c7c36acc..43af38dc5f78 100644
--- a/benchmark/worker/benchmark_client.go
+++ b/benchmark/worker/benchmark_client.go
@@ -32,9 +32,11 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/syscall"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/testdata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var caFile = flag.String("ca_file", "", "The file containing the CA root cert file")
@@ -243,7 +245,7 @@ func startBenchmarkClient(config *testpb.ClientConfig) (*benchmarkClient, error)
 
 func (bc *benchmarkClient) doCloseLoopUnary(conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int) {
 	for ic, conn := range conns {
-		client := testpb.NewBenchmarkServiceClient(conn)
+		client := testgrpc.NewBenchmarkServiceClient(conn)
 		// For each connection, create rpcCountPerConn goroutines to do rpc.
 		for j := 0; j < rpcCountPerConn; j++ {
 			// Create histogram for each goroutine.
@@ -285,7 +287,7 @@ func (bc *benchmarkClient) doCloseLoopUnary(conns []*grpc.ClientConn, rpcCountPe
 }
 
 func (bc *benchmarkClient) doCloseLoopStreaming(conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int, payloadType string) {
-	var doRPC func(testpb.BenchmarkService_StreamingCallClient, int, int) error
+	var doRPC func(testgrpc.BenchmarkService_StreamingCallClient, int, int) error
 	if payloadType == "bytebuf" {
 		doRPC = benchmark.DoByteBufStreamingRoundTrip
 	} else {
@@ -294,7 +296,7 @@ func (bc *benchmarkClient) doCloseLoopStreaming(conns []*grpc.ClientConn, rpcCou
 	for ic, conn := range conns {
 		// For each connection, create rpcCountPerConn goroutines to do rpc.
 		for j := 0; j < rpcCountPerConn; j++ {
-			c := testpb.NewBenchmarkServiceClient(conn)
+			c := testgrpc.NewBenchmarkServiceClient(conn)
 			stream, err := c.StreamingCall(context.Background())
 			if err != nil {
 				logger.Fatalf("%v.StreamingCall(_) = _, %v", c, err)
diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go
index 4ecf997238d8..901341f51e7d 100644
--- a/benchmark/worker/main.go
+++ b/benchmark/worker/main.go
@@ -35,8 +35,10 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
@@ -75,12 +77,12 @@ func (byteBufCodec) String() string {
 // workerServer implements WorkerService rpc handlers.
 // It can create benchmarkServer or benchmarkClient on demand.
 type workerServer struct {
-	testpb.UnimplementedWorkerServiceServer
+	testgrpc.UnimplementedWorkerServiceServer
 	stop       chan<- bool
 	serverPort int
 }
 
-func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) error {
+func (s *workerServer) RunServer(stream testgrpc.WorkerService_RunServerServer) error {
 	var bs *benchmarkServer
 	defer func() {
 		// Close benchmark server when stream ends.
@@ -135,7 +137,7 @@ func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) er
 	}
 }
 
-func (s *workerServer) RunClient(stream testpb.WorkerService_RunClientServer) error {
+func (s *workerServer) RunClient(stream testgrpc.WorkerService_RunClientServer) error {
 	var bc *benchmarkClient
 	defer func() {
 		// Shut down benchmark client when stream ends.
@@ -209,7 +211,7 @@ func main() {
 
 	s := grpc.NewServer()
 	stop := make(chan bool)
-	testpb.RegisterWorkerServiceServer(s, &workerServer{
+	testgrpc.RegisterWorkerServiceServer(s, &workerServer{
 		stop:       stop,
 		serverPort: *serverPort,
 	})
diff --git a/binarylog/binarylog_end2end_test.go b/binarylog/binarylog_end2end_test.go
index e06ffec9166e..61eeb68edae8 100644
--- a/binarylog/binarylog_end2end_test.go
+++ b/binarylog/binarylog_end2end_test.go
@@ -31,13 +31,15 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/binarylog"
-	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
 	"google.golang.org/grpc/grpclog"
 	iblog "google.golang.org/grpc/internal/binarylog"
 	"google.golang.org/grpc/internal/grpctest"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
+
+	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var grpclogLogger = grpclog.Component("binarylog")
@@ -126,7 +128,7 @@ func payloadToID(p *testpb.Payload) int32 {
 }
 
 type testServer struct {
-	testpb.UnimplementedTestServiceServer
+	testgrpc.UnimplementedTestServiceServer
 	te *test
 }
 
@@ -148,7 +150,7 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 	return &testpb.SimpleResponse{Payload: in.Payload}, nil
 }
 
-func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error {
 	md, ok := metadata.FromIncomingContext(stream.Context())
 	if ok {
 		if err := stream.SendHeader(md); err != nil {
@@ -176,7 +178,7 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ
 	}
 }
 
-func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
+func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error {
 	md, ok := metadata.FromIncomingContext(stream.Context())
 	if ok {
 		if err := stream.SendHeader(md); err != nil {
@@ -200,7 +202,7 @@ func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInput
 	}
 }
 
-func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
+func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error {
 	md, ok := metadata.FromIncomingContext(stream.Context())
 	if ok {
 		if err := stream.SendHeader(md); err != nil {
@@ -227,7 +229,7 @@ func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest,
 type test struct {
 	t *testing.T
 
-	testService testpb.TestServiceServer // nil means none
+	testService testgrpc.TestServiceServer // nil means none
 	// srv and srvAddr are set once startServer is called.
 	srv     *grpc.Server
 	srvAddr string // Server IP without port.
@@ -282,7 +284,7 @@ func (lw *listenerWrapper) Accept() (net.Conn, error) {
 
 // startServer starts a gRPC server listening. Callers should defer a
 // call to te.tearDown to clean up.
-func (te *test) startServer(ts testpb.TestServiceServer) {
+func (te *test) startServer(ts testgrpc.TestServiceServer) {
 	te.testService = ts
 	lis, err := net.Listen("tcp", "localhost:0")
 
@@ -298,7 +300,7 @@ func (te *test) startServer(ts testpb.TestServiceServer) {
 	s := grpc.NewServer(opts...)
 	te.srv = s
 	if te.testService != nil {
-		testpb.RegisterTestServiceServer(s, te.testService)
+		testgrpc.RegisterTestServiceServer(s, te.testService)
 	}
 
 	go s.Serve(lis)
@@ -343,7 +345,7 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 		req  *testpb.SimpleRequest
 		err  error
 	)
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	if c.success {
 		req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)}
 	} else {
@@ -363,7 +365,7 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []prot
 		resps []proto.Message
 		err   error
 	)
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	ctx = metadata.NewOutgoingContext(ctx, testMetadata)
@@ -412,7 +414,7 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, proto.Message
 		resp *testpb.StreamingInputCallResponse
 		err  error
 	)
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	ctx = metadata.NewOutgoingContext(ctx, testMetadata)
@@ -445,7 +447,7 @@ func (te *test) doServerStreamCall(c *rpcConfig) (proto.Message, []proto.Message
 		err   error
 	)
 
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	ctx = metadata.NewOutgoingContext(ctx, testMetadata)
diff --git a/interop/alts/client/client.go b/interop/alts/client/client.go
index 9fc0e3c15650..aef601ff885b 100644
--- a/interop/alts/client/client.go
+++ b/interop/alts/client/client.go
@@ -27,6 +27,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/alts"
 	"google.golang.org/grpc/grpclog"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
@@ -51,7 +53,7 @@ func main() {
 		logger.Fatalf("gRPC Client: failed to dial the server at %v: %v", *serverAddr, err)
 	}
 	defer conn.Close()
-	grpcClient := testpb.NewTestServiceClient(conn)
+	grpcClient := testgrpc.NewTestServiceClient(conn)
 
 	// Call the EmptyCall API.
 	ctx := context.Background()
diff --git a/interop/alts/server/server.go b/interop/alts/server/server.go
index 0d0f375a0d1a..9db6750252e0 100644
--- a/interop/alts/server/server.go
+++ b/interop/alts/server/server.go
@@ -29,8 +29,9 @@ import (
 	"google.golang.org/grpc/credentials/alts"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/interop"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/tap"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 )
 
 const (
@@ -64,7 +65,7 @@ func main() {
 	}
 	altsTC := alts.NewServerCreds(opts)
 	grpcServer := grpc.NewServer(grpc.Creds(altsTC), grpc.InTapHandle(authz))
-	testpb.RegisterTestServiceServer(grpcServer, interop.NewTestServer())
+	testgrpc.RegisterTestServiceServer(grpcServer, interop.NewTestServer())
 	grpcServer.Serve(lis)
 }
 
diff --git a/interop/client/client.go b/interop/client/client.go
index 2d7823062309..8854ed2d76ec 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -32,9 +32,10 @@ import (
 	"google.golang.org/grpc/credentials/oauth"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/interop"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/testdata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 )
 
 const (
@@ -188,7 +189,7 @@ func main() {
 		logger.Fatalf("Fail to dial: %v", err)
 	}
 	defer conn.Close()
-	tc := testpb.NewTestServiceClient(conn)
+	tc := testgrpc.NewTestServiceClient(conn)
 	switch *testCase {
 	case "empty_unary":
 		interop.DoEmptyUnaryCall(tc)
@@ -272,7 +273,7 @@ func main() {
 		interop.DoUnimplementedMethod(conn)
 		logger.Infoln("UnimplementedMethod done")
 	case "unimplemented_service":
-		interop.DoUnimplementedService(testpb.NewUnimplementedServiceClient(conn))
+		interop.DoUnimplementedService(testgrpc.NewUnimplementedServiceClient(conn))
 		logger.Infoln("UnimplementedService done")
 	case "pick_first_unary":
 		interop.DoPickFirstUnary(tc)
diff --git a/interop/grpclb_fallback/client.go b/interop/grpclb_fallback/client.go
index 262d7458aba5..61b2fae6968e 100644
--- a/interop/grpclb_fallback/client.go
+++ b/interop/grpclb_fallback/client.go
@@ -37,6 +37,8 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/alts"
 	"google.golang.org/grpc/credentials/google"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
@@ -55,7 +57,7 @@ var (
 	errorLog = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile)
 )
 
-func doRPCAndGetPath(client testpb.TestServiceClient, timeout time.Duration) testpb.GrpclbRouteType {
+func doRPCAndGetPath(client testgrpc.TestServiceClient, timeout time.Duration) testpb.GrpclbRouteType {
 	infoLog.Printf("doRPCAndGetPath timeout:%v\n", timeout)
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
@@ -128,7 +130,7 @@ func runCmd(command string) {
 	}
 }
 
-func waitForFallbackAndDoRPCs(client testpb.TestServiceClient, fallbackDeadline time.Time) {
+func waitForFallbackAndDoRPCs(client testgrpc.TestServiceClient, fallbackDeadline time.Time) {
 	fallbackRetryCount := 0
 	fellBack := false
 	for time.Now().Before(fallbackDeadline) {
@@ -160,7 +162,7 @@ func doFastFallbackBeforeStartup() {
 	fallbackDeadline := time.Now().Add(5 * time.Second)
 	conn := createTestConn()
 	defer conn.Close()
-	client := testpb.NewTestServiceClient(conn)
+	client := testgrpc.NewTestServiceClient(conn)
 	waitForFallbackAndDoRPCs(client, fallbackDeadline)
 }
 
@@ -169,14 +171,14 @@ func doSlowFallbackBeforeStartup() {
 	fallbackDeadline := time.Now().Add(20 * time.Second)
 	conn := createTestConn()
 	defer conn.Close()
-	client := testpb.NewTestServiceClient(conn)
+	client := testgrpc.NewTestServiceClient(conn)
 	waitForFallbackAndDoRPCs(client, fallbackDeadline)
 }
 
 func doFastFallbackAfterStartup() {
 	conn := createTestConn()
 	defer conn.Close()
-	client := testpb.NewTestServiceClient(conn)
+	client := testgrpc.NewTestServiceClient(conn)
 	if g := doRPCAndGetPath(client, 20*time.Second); g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND {
 		errorLog.Fatalf("Expected RPC to take grpclb route type BACKEND. Got: %v", g)
 	}
@@ -188,7 +190,7 @@ func doFastFallbackAfterStartup() {
 func doSlowFallbackAfterStartup() {
 	conn := createTestConn()
 	defer conn.Close()
-	client := testpb.NewTestServiceClient(conn)
+	client := testgrpc.NewTestServiceClient(conn)
 	if g := doRPCAndGetPath(client, 20*time.Second); g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND {
 		errorLog.Fatalf("Expected RPC to take grpclb route type BACKEND. Got: %v", g)
 	}
diff --git a/interop/http2/negative_http2_client.go b/interop/http2/negative_http2_client.go
index 8ead7f49c6b4..9ed34f75716d 100644
--- a/interop/http2/negative_http2_client.go
+++ b/interop/http2/negative_http2_client.go
@@ -35,8 +35,10 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/interop"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
@@ -66,7 +68,7 @@ func largeSimpleRequest() *testpb.SimpleRequest {
 }
 
 // sends two unary calls. The server asserts that the calls use different connections.
-func goaway(tc testpb.TestServiceClient) {
+func goaway(tc testgrpc.TestServiceClient) {
 	interop.DoLargeUnaryCall(tc)
 	// sleep to ensure that the client has time to recv the GOAWAY.
 	// TODO(ncteisen): make this less hacky.
@@ -74,7 +76,7 @@ func goaway(tc testpb.TestServiceClient) {
 	interop.DoLargeUnaryCall(tc)
 }
 
-func rstAfterHeader(tc testpb.TestServiceClient) {
+func rstAfterHeader(tc testgrpc.TestServiceClient) {
 	req := largeSimpleRequest()
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if reply != nil {
@@ -85,7 +87,7 @@ func rstAfterHeader(tc testpb.TestServiceClient) {
 	}
 }
 
-func rstDuringData(tc testpb.TestServiceClient) {
+func rstDuringData(tc testgrpc.TestServiceClient) {
 	req := largeSimpleRequest()
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if reply != nil {
@@ -96,7 +98,7 @@ func rstDuringData(tc testpb.TestServiceClient) {
 	}
 }
 
-func rstAfterData(tc testpb.TestServiceClient) {
+func rstAfterData(tc testgrpc.TestServiceClient) {
 	req := largeSimpleRequest()
 	reply, err := tc.UnaryCall(context.Background(), req)
 	if reply != nil {
@@ -107,12 +109,12 @@ func rstAfterData(tc testpb.TestServiceClient) {
 	}
 }
 
-func ping(tc testpb.TestServiceClient) {
+func ping(tc testgrpc.TestServiceClient) {
 	// The server will assert that every ping it sends was ACK-ed by the client.
 	interop.DoLargeUnaryCall(tc)
 }
 
-func maxStreams(tc testpb.TestServiceClient) {
+func maxStreams(tc testgrpc.TestServiceClient) {
 	interop.DoLargeUnaryCall(tc)
 	var wg sync.WaitGroup
 	for i := 0; i < 15; i++ {
@@ -135,7 +137,7 @@ func main() {
 		logger.Fatalf("Fail to dial: %v", err)
 	}
 	defer conn.Close()
-	tc := testpb.NewTestServiceClient(conn)
+	tc := testgrpc.NewTestServiceClient(conn)
 	switch *testCase {
 	case "goaway":
 		goaway(tc)
diff --git a/interop/server/server.go b/interop/server/server.go
index c70e450bb108..16360abe9e7b 100644
--- a/interop/server/server.go
+++ b/interop/server/server.go
@@ -29,8 +29,9 @@ import (
 	"google.golang.org/grpc/credentials/alts"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/interop"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/testdata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
@@ -76,6 +77,6 @@ func main() {
 		opts = append(opts, grpc.Creds(altsTC))
 	}
 	server := grpc.NewServer(opts...)
-	testpb.RegisterTestServiceServer(server, interop.NewTestServer())
+	testgrpc.RegisterTestServiceServer(server, interop.NewTestServer())
 	server.Serve(lis)
 }
diff --git a/interop/test_utils.go b/interop/test_utils.go
index 78f937a83d68..cbcbcc4da173 100644
--- a/interop/test_utils.go
+++ b/interop/test_utils.go
@@ -33,9 +33,11 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
@@ -67,7 +69,7 @@ func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload {
 }
 
 // DoEmptyUnaryCall performs a unary RPC with empty request and response messages.
-func DoEmptyUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoEmptyUnaryCall(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	reply, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, args...)
 	if err != nil {
 		logger.Fatal("/TestService/EmptyCall RPC failed: ", err)
@@ -78,7 +80,7 @@ func DoEmptyUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoLargeUnaryCall performs a unary RPC with large payload in the request and response.
-func DoLargeUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoLargeUnaryCall(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType: testpb.PayloadType_COMPRESSABLE,
@@ -97,7 +99,7 @@ func DoLargeUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoClientStreaming performs a client streaming RPC.
-func DoClientStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoClientStreaming(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	stream, err := tc.StreamingInputCall(context.Background(), args...)
 	if err != nil {
 		logger.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
@@ -123,7 +125,7 @@ func DoClientStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoServerStreaming performs a server streaming RPC.
-func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoServerStreaming(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	respParam := make([]*testpb.ResponseParameters, len(respSizes))
 	for i, s := range respSizes {
 		respParam[i] = &testpb.ResponseParameters{
@@ -167,7 +169,7 @@ func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoPingPong performs ping-pong style bi-directional streaming RPC.
-func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoPingPong(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	stream, err := tc.FullDuplexCall(context.Background(), args...)
 	if err != nil {
 		logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
@@ -211,7 +213,7 @@ func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoEmptyStream sets up a bi-directional streaming with zero message.
-func DoEmptyStream(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoEmptyStream(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	stream, err := tc.FullDuplexCall(context.Background(), args...)
 	if err != nil {
 		logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
@@ -225,7 +227,7 @@ func DoEmptyStream(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoTimeoutOnSleepingServer performs an RPC on a sleep server which causes RPC timeout.
-func DoTimeoutOnSleepingServer(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoTimeoutOnSleepingServer(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
 	defer cancel()
 	stream, err := tc.FullDuplexCall(ctx, args...)
@@ -249,7 +251,7 @@ func DoTimeoutOnSleepingServer(tc testpb.TestServiceClient, args ...grpc.CallOpt
 }
 
 // DoComputeEngineCreds performs a unary RPC with compute engine auth.
-func DoComputeEngineCreds(tc testpb.TestServiceClient, serviceAccount, oauthScope string) {
+func DoComputeEngineCreds(tc testgrpc.TestServiceClient, serviceAccount, oauthScope string) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType:   testpb.PayloadType_COMPRESSABLE,
@@ -281,7 +283,7 @@ func getServiceAccountJSONKey(keyFile string) []byte {
 }
 
 // DoServiceAccountCreds performs a unary RPC with service account auth.
-func DoServiceAccountCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
+func DoServiceAccountCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType:   testpb.PayloadType_COMPRESSABLE,
@@ -306,7 +308,7 @@ func DoServiceAccountCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, o
 }
 
 // DoJWTTokenCreds performs a unary RPC with JWT token auth.
-func DoJWTTokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile string) {
+func DoJWTTokenCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile string) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType: testpb.PayloadType_COMPRESSABLE,
@@ -340,7 +342,7 @@ func GetToken(serviceAccountKeyFile string, oauthScope string) *oauth2.Token {
 }
 
 // DoOauth2TokenCreds performs a unary RPC with OAUTH2 token auth.
-func DoOauth2TokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
+func DoOauth2TokenCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType:   testpb.PayloadType_COMPRESSABLE,
@@ -365,7 +367,7 @@ func DoOauth2TokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oaut
 }
 
 // DoPerRPCCreds performs a unary RPC with per RPC OAUTH2 token.
-func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
+func DoPerRPCCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
 	jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
@@ -393,7 +395,7 @@ func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScop
 }
 
 // DoGoogleDefaultCredentials performs an unary RPC with google default credentials
-func DoGoogleDefaultCredentials(tc testpb.TestServiceClient, defaultServiceAccount string) {
+func DoGoogleDefaultCredentials(tc testgrpc.TestServiceClient, defaultServiceAccount string) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType:   testpb.PayloadType_COMPRESSABLE,
@@ -412,7 +414,7 @@ func DoGoogleDefaultCredentials(tc testpb.TestServiceClient, defaultServiceAccou
 }
 
 // DoComputeEngineChannelCredentials performs an unary RPC with compute engine channel credentials
-func DoComputeEngineChannelCredentials(tc testpb.TestServiceClient, defaultServiceAccount string) {
+func DoComputeEngineChannelCredentials(tc testgrpc.TestServiceClient, defaultServiceAccount string) {
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
 	req := &testpb.SimpleRequest{
 		ResponseType:   testpb.PayloadType_COMPRESSABLE,
@@ -436,7 +438,7 @@ var testMetadata = metadata.MD{
 }
 
 // DoCancelAfterBegin cancels the RPC after metadata has been sent but before payloads are sent.
-func DoCancelAfterBegin(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoCancelAfterBegin(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	ctx, cancel := context.WithCancel(metadata.NewOutgoingContext(context.Background(), testMetadata))
 	stream, err := tc.StreamingInputCall(ctx, args...)
 	if err != nil {
@@ -450,7 +452,7 @@ func DoCancelAfterBegin(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoCancelAfterFirstResponse cancels the RPC after receiving the first message from the server.
-func DoCancelAfterFirstResponse(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoCancelAfterFirstResponse(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	ctx, cancel := context.WithCancel(context.Background())
 	stream, err := tc.FullDuplexCall(ctx, args...)
 	if err != nil {
@@ -504,7 +506,7 @@ func validateMetadata(header, trailer metadata.MD) {
 }
 
 // DoCustomMetadata checks that metadata is echoed back to the client.
-func DoCustomMetadata(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoCustomMetadata(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	// Testing with UnaryCall.
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 1)
 	req := &testpb.SimpleRequest{
@@ -566,7 +568,7 @@ func DoCustomMetadata(tc testpb.TestServiceClient, args ...grpc.CallOption) {
 }
 
 // DoStatusCodeAndMessage checks that the status code is propagated back to the client.
-func DoStatusCodeAndMessage(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoStatusCodeAndMessage(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	var code int32 = 2
 	msg := "test status message"
 	expectedErr := status.Error(codes.Code(code), msg)
@@ -602,7 +604,7 @@ func DoStatusCodeAndMessage(tc testpb.TestServiceClient, args ...grpc.CallOption
 
 // DoSpecialStatusMessage verifies Unicode and whitespace is correctly processed
 // in status message.
-func DoSpecialStatusMessage(tc testpb.TestServiceClient, args ...grpc.CallOption) {
+func DoSpecialStatusMessage(tc testgrpc.TestServiceClient, args ...grpc.CallOption) {
 	const (
 		code int32  = 2
 		msg  string = "\t\ntest with whitespace\r\nand Unicode BMP ☺ and non-BMP 😈\t\n"
@@ -622,7 +624,7 @@ func DoSpecialStatusMessage(tc testpb.TestServiceClient, args ...grpc.CallOption
 }
 
 // DoUnimplementedService attempts to call a method from an unimplemented service.
-func DoUnimplementedService(tc testpb.UnimplementedServiceClient) {
+func DoUnimplementedService(tc testgrpc.UnimplementedServiceClient) {
 	_, err := tc.UnimplementedCall(context.Background(), &testpb.Empty{})
 	if status.Code(err) != codes.Unimplemented {
 		logger.Fatalf("%v.UnimplementedCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unimplemented)
@@ -639,7 +641,7 @@ func DoUnimplementedMethod(cc *grpc.ClientConn) {
 
 // DoPickFirstUnary runs multiple RPCs (rpcCount) and checks that all requests
 // are sent to the same backend.
-func DoPickFirstUnary(tc testpb.TestServiceClient) {
+func DoPickFirstUnary(tc testgrpc.TestServiceClient) {
 	const rpcCount = 100
 
 	pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 1)
@@ -672,11 +674,11 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) {
 }
 
 type testServer struct {
-	testpb.UnimplementedTestServiceServer
+	testgrpc.UnimplementedTestServiceServer
 }
 
 // NewTestServer creates a test server for test service.
-func NewTestServer() testpb.TestServiceServer {
+func NewTestServer() testgrpc.TestServiceServer {
 	return &testServer{}
 }
 
@@ -724,7 +726,7 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 	}, nil
 }
 
-func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
+func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error {
 	cs := args.GetResponseParameters()
 	for _, c := range cs {
 		if us := c.GetIntervalUs(); us > 0 {
@@ -743,7 +745,7 @@ func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest
 	return nil
 }
 
-func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
+func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error {
 	var sum int
 	for {
 		in, err := stream.Recv()
@@ -760,7 +762,7 @@ func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInput
 	}
 }
 
-func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error {
 	if md, ok := metadata.FromIncomingContext(stream.Context()); ok {
 		if initialMetadata, ok := md[initialMetadataKey]; ok {
 			header := metadata.Pairs(initialMetadataKey, initialMetadata[0])
@@ -802,7 +804,7 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ
 	}
 }
 
-func (s *testServer) HalfDuplexCall(stream testpb.TestService_HalfDuplexCallServer) error {
+func (s *testServer) HalfDuplexCall(stream testgrpc.TestService_HalfDuplexCallServer) error {
 	var msgBuf []*testpb.StreamingOutputCallRequest
 	for {
 		in, err := stream.Recv()
diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 7afdb20e8d72..0b8e9dee3af6 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -32,10 +32,12 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	_ "google.golang.org/grpc/xds"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 func init() {
@@ -163,7 +165,7 @@ var (
 )
 
 type statsService struct {
-	testpb.UnimplementedLoadBalancerStatsServiceServer
+	testgrpc.UnimplementedLoadBalancerStatsServiceServer
 }
 
 func hasRPCSucceeded() bool {
@@ -235,7 +237,7 @@ func (s *statsService) GetClientAccumulatedStats(ctx context.Context, in *testpb
 }
 
 type configureService struct {
-	testpb.UnimplementedXdsUpdateClientConfigureServiceServer
+	testgrpc.UnimplementedXdsUpdateClientConfigureServiceServer
 }
 
 func (s *configureService) Configure(ctx context.Context, in *testpb.ClientConfigureRequest) (*testpb.ClientConfigureResponse, error) {
@@ -334,25 +336,25 @@ func main() {
 	}
 	s := grpc.NewServer()
 	defer s.Stop()
-	testpb.RegisterLoadBalancerStatsServiceServer(s, &statsService{})
-	testpb.RegisterXdsUpdateClientConfigureServiceServer(s, &configureService{})
+	testgrpc.RegisterLoadBalancerStatsServiceServer(s, &statsService{})
+	testgrpc.RegisterXdsUpdateClientConfigureServiceServer(s, &configureService{})
 	go s.Serve(lis)
 
-	clients := make([]testpb.TestServiceClient, *numChannels)
+	clients := make([]testgrpc.TestServiceClient, *numChannels)
 	for i := 0; i < *numChannels; i++ {
 		conn, err := grpc.DialContext(context.Background(), *server, grpc.WithInsecure())
 		if err != nil {
 			logger.Fatalf("Fail to dial: %v", err)
 		}
 		defer conn.Close()
-		clients[i] = testpb.NewTestServiceClient(conn)
+		clients[i] = testgrpc.NewTestServiceClient(conn)
 	}
 	ticker := time.NewTicker(time.Second / time.Duration(*qps**numChannels))
 	defer ticker.Stop()
 	sendRPCs(clients, ticker)
 }
 
-func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) {
+func makeOneRPC(c testgrpc.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), *rpcTimeout)
 	defer cancel()
 
@@ -392,7 +394,7 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf
 	return &p, &info, err
 }
 
-func sendRPCs(clients []testpb.TestServiceClient, ticker *time.Ticker) {
+func sendRPCs(clients []testgrpc.TestServiceClient, ticker *time.Ticker) {
 	var i int
 	for range ticker.C {
 		// Get and increment request ID, and save a list of watchers that are
diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go
index 45b8448822b2..4989eb728eec 100644
--- a/interop/xds/server/server.go
+++ b/interop/xds/server/server.go
@@ -29,8 +29,10 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 var (
@@ -50,7 +52,7 @@ func getHostname() string {
 }
 
 type server struct {
-	testpb.UnimplementedTestServiceServer
+	testgrpc.UnimplementedTestServiceServer
 }
 
 func (s *server) EmptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
@@ -71,6 +73,6 @@ func main() {
 		logger.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
-	testpb.RegisterTestServiceServer(s, &server{})
+	testgrpc.RegisterTestServiceServer(s, &server{})
 	s.Serve(lis)
 }
diff --git a/stats/stats_test.go b/stats/stats_test.go
index aac8166f76f8..306f2f6b8e90 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -31,10 +31,12 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/grpctest"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
 const defaultTestTimeout = 10 * time.Second
@@ -87,7 +89,7 @@ func payloadToID(p *testpb.Payload) int32 {
 }
 
 type testServer struct {
-	testpb.UnimplementedTestServiceServer
+	testgrpc.UnimplementedTestServiceServer
 }
 
 func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
@@ -105,7 +107,7 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*
 	return &testpb.SimpleResponse{Payload: in.Payload}, nil
 }
 
-func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error {
 	if err := stream.SendHeader(testHeaderMetadata); err != nil {
 		return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil)
 	}
@@ -130,7 +132,7 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ
 	}
 }
 
-func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error {
+func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error {
 	if err := stream.SendHeader(testHeaderMetadata); err != nil {
 		return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil)
 	}
@@ -151,7 +153,7 @@ func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInput
 	}
 }
 
-func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error {
+func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error {
 	if err := stream.SendHeader(testHeaderMetadata); err != nil {
 		return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil)
 	}
@@ -178,7 +180,7 @@ type test struct {
 	clientStatsHandler stats.Handler
 	serverStatsHandler stats.Handler
 
-	testServer testpb.TestServiceServer // nil means none
+	testServer testgrpc.TestServiceServer // nil means none
 	// srv and srvAddr are set once startServer is called.
 	srv     *grpc.Server
 	srvAddr string
@@ -213,7 +215,7 @@ func newTest(t *testing.T, tc *testConfig, ch stats.Handler, sh stats.Handler) *
 
 // startServer starts a gRPC server listening. Callers should defer a
 // call to te.tearDown to clean up.
-func (te *test) startServer(ts testpb.TestServiceServer) {
+func (te *test) startServer(ts testgrpc.TestServiceServer) {
 	te.testServer = ts
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -232,7 +234,7 @@ func (te *test) startServer(ts testpb.TestServiceServer) {
 	s := grpc.NewServer(opts...)
 	te.srv = s
 	if te.testServer != nil {
-		testpb.RegisterTestServiceServer(s, te.testServer)
+		testgrpc.RegisterTestServiceServer(s, te.testServer)
 	}
 
 	go s.Serve(lis)
@@ -288,7 +290,7 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple
 		req  *testpb.SimpleRequest
 		err  error
 	)
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	if c.success {
 		req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)}
 	} else {
@@ -307,7 +309,7 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []prot
 		resps []proto.Message
 		err   error
 	)
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	stream, err := tc.FullDuplexCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast))
@@ -348,7 +350,7 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, *testpb.Strea
 		resp *testpb.StreamingInputCallResponse
 		err  error
 	)
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	stream, err := tc.StreamingInputCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast))
@@ -379,7 +381,7 @@ func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.StreamingOutputCallReq
 		err   error
 	)
 
-	tc := testpb.NewTestServiceClient(te.clientConn())
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
 
 	var startID int32
 	if !c.success {
diff --git a/stress/client/main.go b/stress/client/main.go
index c5bfffa4e519..37e2a38f42a2 100644
--- a/stress/client/main.go
+++ b/stress/client/main.go
@@ -35,10 +35,11 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/interop"
-	testpb "google.golang.org/grpc/interop/grpc_testing"
 	"google.golang.org/grpc/status"
-	metricspb "google.golang.org/grpc/stress/grpc_testing"
 	"google.golang.org/grpc/testdata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	metricspb "google.golang.org/grpc/stress/grpc_testing"
 )
 
 var (
@@ -209,7 +210,7 @@ func startServer(server *server, port int) {
 
 // performRPCs uses weightedRandomTestSelector to select test case and runs the tests.
 func performRPCs(gauge *gauge, conn *grpc.ClientConn, selector *weightedRandomTestSelector, stop <-chan bool) {
-	client := testpb.NewTestServiceClient(conn)
+	client := testgrpc.NewTestServiceClient(conn)
 	var numCalls int64
 	startTime := time.Now()
 	for {

From 5e3cbb54d230a7d9373b2d750784ed3eeb90d441 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 13 Jan 2021 13:23:43 -0800
Subject: [PATCH 343/481] interop/grpc_testing: update generated code after
 recent grpc-proto changes (#4149)

---
 interop/grpc_testing/messages.pb.go | 315 ++++++++++++++++++++--------
 1 file changed, 227 insertions(+), 88 deletions(-)

diff --git a/interop/grpc_testing/messages.pb.go b/interop/grpc_testing/messages.pb.go
index fb141c57df04..6ca348b6201d 100644
--- a/interop/grpc_testing/messages.pb.go
+++ b/interop/grpc_testing/messages.pb.go
@@ -1171,11 +1171,23 @@ type LoadBalancerAccumulatedStatsResponse struct {
 	unknownFields protoimpl.UnknownFields
 
 	// The total number of RPCs have ever issued for each type.
+	// Deprecated: use stats_per_method.rpcs_started instead.
+	//
+	// Deprecated: Do not use.
 	NumRpcsStartedByMethod map[string]int32 `protobuf:"bytes,1,rep,name=num_rpcs_started_by_method,json=numRpcsStartedByMethod,proto3" json:"num_rpcs_started_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
 	// The total number of RPCs have ever completed successfully for each type.
+	// Deprecated: use stats_per_method.result instead.
+	//
+	// Deprecated: Do not use.
 	NumRpcsSucceededByMethod map[string]int32 `protobuf:"bytes,2,rep,name=num_rpcs_succeeded_by_method,json=numRpcsSucceededByMethod,proto3" json:"num_rpcs_succeeded_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
 	// The total number of RPCs have ever failed for each type.
+	// Deprecated: use stats_per_method.result instead.
+	//
+	// Deprecated: Do not use.
 	NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	// Per-method RPC statistics.  The key is the full method path; i.e.
+	// "/proto.package.ServiceName/MethodName".
+	StatsPerMethod map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats `protobuf:"bytes,4,rep,name=stats_per_method,json=statsPerMethod,proto3" json:"stats_per_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (x *LoadBalancerAccumulatedStatsResponse) Reset() {
@@ -1210,6 +1222,7 @@ func (*LoadBalancerAccumulatedStatsResponse) Descriptor() ([]byte, []int) {
 	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15}
 }
 
+// Deprecated: Do not use.
 func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[string]int32 {
 	if x != nil {
 		return x.NumRpcsStartedByMethod
@@ -1217,6 +1230,7 @@ func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[s
 	return nil
 }
 
+// Deprecated: Do not use.
 func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map[string]int32 {
 	if x != nil {
 		return x.NumRpcsSucceededByMethod
@@ -1224,6 +1238,7 @@ func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map
 	return nil
 }
 
+// Deprecated: Do not use.
 func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[string]int32 {
 	if x != nil {
 		return x.NumRpcsFailedByMethod
@@ -1231,6 +1246,13 @@ func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[st
 	return nil
 }
 
+func (x *LoadBalancerAccumulatedStatsResponse) GetStatsPerMethod() map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats {
+	if x != nil {
+		return x.StatsPerMethod
+	}
+	return nil
+}
+
 // Configurations for a test client.
 type ClientConfigureRequest struct {
 	state         protoimpl.MessageState
@@ -1241,6 +1263,9 @@ type ClientConfigureRequest struct {
 	Types []ClientConfigureRequest_RpcType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"types,omitempty"`
 	// The collection of custom metadata to be attached to RPCs sent by the client.
 	Metadata []*ClientConfigureRequest_Metadata `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"`
+	// The deadline to use, in seconds, for all RPCs.  If unset or zero, the
+	// client will use the default from the command-line.
+	TimeoutSec int32 `protobuf:"varint,3,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
 }
 
 func (x *ClientConfigureRequest) Reset() {
@@ -1289,6 +1314,13 @@ func (x *ClientConfigureRequest) GetMetadata() []*ClientConfigureRequest_Metadat
 	return nil
 }
 
+func (x *ClientConfigureRequest) GetTimeoutSec() int32 {
+	if x != nil {
+		return x.TimeoutSec
+	}
+	return 0
+}
+
 // Response for updating a test client's configuration.
 type ClientConfigureResponse struct {
 	state         protoimpl.MessageState
@@ -1376,6 +1408,64 @@ func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32
 	return nil
 }
 
+type LoadBalancerAccumulatedStatsResponse_MethodStats struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The number of RPCs that were started for this method.
+	RpcsStarted int32 `protobuf:"varint,1,opt,name=rpcs_started,json=rpcsStarted,proto3" json:"rpcs_started,omitempty"`
+	// The number of RPCs that completed with each status for this method.  The
+	// key is the integral value of a google.rpc.Code; the value is the count.
+	Result map[int32]int32 `protobuf:"bytes,2,rep,name=result,proto3" json:"result,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) Reset() {
+	*x = LoadBalancerAccumulatedStatsResponse_MethodStats{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_testing_messages_proto_msgTypes[25]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LoadBalancerAccumulatedStatsResponse_MethodStats) ProtoMessage() {}
+
+func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_testing_messages_proto_msgTypes[25]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LoadBalancerAccumulatedStatsResponse_MethodStats.ProtoReflect.Descriptor instead.
+func (*LoadBalancerAccumulatedStatsResponse_MethodStats) Descriptor() ([]byte, []int) {
+	return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15, 3}
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) GetRpcsStarted() int32 {
+	if x != nil {
+		return x.RpcsStarted
+	}
+	return 0
+}
+
+func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) GetResult() map[int32]int32 {
+	if x != nil {
+		return x.Result
+	}
+	return nil
+}
+
 // Metadata to be attached for the given type of RPCs.
 type ClientConfigureRequest_Metadata struct {
 	state         protoimpl.MessageState
@@ -1390,7 +1480,7 @@ type ClientConfigureRequest_Metadata struct {
 func (x *ClientConfigureRequest_Metadata) Reset() {
 	*x = ClientConfigureRequest_Metadata{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_grpc_testing_messages_proto_msgTypes[25]
+		mi := &file_grpc_testing_messages_proto_msgTypes[28]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -1403,7 +1493,7 @@ func (x *ClientConfigureRequest_Metadata) String() string {
 func (*ClientConfigureRequest_Metadata) ProtoMessage() {}
 
 func (x *ClientConfigureRequest_Metadata) ProtoReflect() protoreflect.Message {
-	mi := &file_grpc_testing_messages_proto_msgTypes[25]
+	mi := &file_grpc_testing_messages_proto_msgTypes[28]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1609,82 +1699,113 @@ var file_grpc_testing_messages_proto_rawDesc = []byte{
 	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x23, 0x4c, 0x6f, 0x61,
 	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c,
 	0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x22, 0xb2, 0x05, 0x0a, 0x24, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
+	0x22, 0x86, 0x09, 0x0a, 0x24, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
 	0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74,
-	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x1a, 0x6e, 0x75,
+	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8e, 0x01, 0x0a, 0x1a, 0x6e, 0x75,
 	0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x62,
 	0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e,
 	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f,
 	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75,
 	0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
 	0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65,
-	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x16,
-	0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79,
-	0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x90, 0x01, 0x0a, 0x1c, 0x6e, 0x75, 0x6d, 0x5f, 0x72,
-	0x70, 0x63, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x5f, 0x62, 0x79,
-	0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
-	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c,
-	0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
-	0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
-	0x18, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65,
-	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x87, 0x01, 0x0a, 0x19, 0x6e, 0x75,
-	0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x62, 0x79,
-	0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, 0x2e,
-	0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61,
-	0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c,
-	0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42,
-	0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x15, 0x6e, 0x75,
-	0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74,
-	0x68, 0x6f, 0x64, 0x1a, 0x49, 0x0a, 0x1b, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74,
-	0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74,
-	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4b,
-	0x0a, 0x1d, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64,
-	0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
-	0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
-	0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05,
-	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x48, 0x0a, 0x1a, 0x4e,
-	0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65,
-	0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76,
+	0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x02,
+	0x18, 0x01, 0x52, 0x16, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74,
+	0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x94, 0x01, 0x0a, 0x1c, 0x6e,
+	0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65,
+	0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x50, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67,
+	0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63,
+	0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63,
+	0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x18, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73,
+	0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x12, 0x8b, 0x01, 0x0a, 0x19, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x66,
+	0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18,
+	0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65,
+	0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74,
+	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63,
+	0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x15, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63,
+	0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12,
+	0x70, 0x0a, 0x10, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x6d, 0x65, 0x74,
+	0x68, 0x6f, 0x64, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c,
+	0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64,
+	0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74,
+	0x61, 0x74, 0x73, 0x50, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x73, 0x50, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x1a, 0x49, 0x0a, 0x1b, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72,
+	0x74, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79,
+	0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
+	0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4b, 0x0a, 0x1d,
+	0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64,
+	0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x48, 0x0a, 0x1a, 0x4e, 0x75, 0x6d,
+	0x52, 0x70, 0x63, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68,
+	0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
+	0x02, 0x38, 0x01, 0x1a, 0xcf, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x53, 0x74,
+	0x61, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72,
+	0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x73, 0x53,
+	0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x62, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74,
+	0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63,
+	0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61,
+	0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x45, 0x6e, 0x74,
+	0x72, 0x79, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x39, 0x0a, 0x0b, 0x52, 0x65,
+	0x73, 0x75, 0x6c, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76,
 	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
-	0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc8, 0x02, 0x0a, 0x16, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x81, 0x01, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50,
+	0x65, 0x72, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x54, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f,
+	0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75,
+	0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe9, 0x02, 0x0a, 0x16, 0x43, 0x6c,
+	0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20,
+	0x03, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69,
+	0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
+	0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70,
+	0x65, 0x52, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61,
+	0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
 	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x12, 0x42, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32,
-	0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43,
-	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x74,
-	0x79, 0x70, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
-	0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65,
-	0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66,
-	0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a,
-	0x74, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, 0x04, 0x74,
-	0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63,
-	0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e,
-	0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a,
-	0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
-	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x07, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x43, 0x41, 0x4c, 0x4c, 0x10, 0x00,
-	0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x5f, 0x43, 0x41, 0x4c, 0x4c, 0x10, 0x01,
-	0x22, 0x19, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x1f, 0x0a, 0x0b, 0x50,
-	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f,
-	0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, 0x2a, 0x6f, 0x0a, 0x0f,
-	0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12,
-	0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1e,
-	0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x01, 0x12, 0x1d,
-	0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, 0x02, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
+	0x61, 0x74, 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73,
+	0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75,
+	0x74, 0x53, 0x65, 0x63, 0x1a, 0x74, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x12, 0x40, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c,
+	0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
+	0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x07, 0x52, 0x70,
+	0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x43,
+	0x41, 0x4c, 0x4c, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x5f, 0x43,
+	0x41, 0x4c, 0x4c, 0x10, 0x01, 0x22, 0x19, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x2a, 0x1f, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10,
+	0x00, 0x2a, 0x6f, 0x0a, 0x0f, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52,
+	0x4f, 0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57,
+	0x4e, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f,
+	0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43,
+	0x4b, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f,
+	0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44,
+	0x10, 0x02, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -1700,7 +1821,7 @@ func file_grpc_testing_messages_proto_rawDescGZIP() []byte {
 }
 
 var file_grpc_testing_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 26)
+var file_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
 var file_grpc_testing_messages_proto_goTypes = []interface{}{
 	(PayloadType)(0),                             // 0: grpc.testing.PayloadType
 	(GrpclbRouteType)(0),                         // 1: grpc.testing.GrpclbRouteType
@@ -1724,13 +1845,16 @@ var file_grpc_testing_messages_proto_goTypes = []interface{}{
 	(*ClientConfigureRequest)(nil),               // 19: grpc.testing.ClientConfigureRequest
 	(*ClientConfigureResponse)(nil),              // 20: grpc.testing.ClientConfigureResponse
 	(*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 21: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
-	nil,                                     // 22: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
-	nil,                                     // 23: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
-	nil,                                     // 24: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
-	nil,                                     // 25: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
-	nil,                                     // 26: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
-	nil,                                     // 27: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
-	(*ClientConfigureRequest_Metadata)(nil), // 28: grpc.testing.ClientConfigureRequest.Metadata
+	nil, // 22: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry
+	nil, // 23: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry
+	nil, // 24: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	nil, // 25: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
+	nil, // 26: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
+	nil, // 27: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
+	(*LoadBalancerAccumulatedStatsResponse_MethodStats)(nil), // 28: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats
+	nil,                                     // 29: grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry
+	nil,                                     // 30: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntry
+	(*ClientConfigureRequest_Metadata)(nil), // 31: grpc.testing.ClientConfigureRequest.Metadata
 }
 var file_grpc_testing_messages_proto_depIdxs = []int32{
 	0,  // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType
@@ -1754,16 +1878,19 @@ var file_grpc_testing_messages_proto_depIdxs = []int32{
 	25, // 18: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_started_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry
 	26, // 19: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_succeeded_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry
 	27, // 20: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_failed_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry
-	2,  // 21: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType
-	28, // 22: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata
-	24, // 23: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
-	21, // 24: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
-	2,  // 25: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType
-	26, // [26:26] is the sub-list for method output_type
-	26, // [26:26] is the sub-list for method input_type
-	26, // [26:26] is the sub-list for extension type_name
-	26, // [26:26] is the sub-list for extension extendee
-	0,  // [0:26] is the sub-list for field type_name
+	29, // 21: grpc.testing.LoadBalancerAccumulatedStatsResponse.stats_per_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry
+	2,  // 22: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType
+	31, // 23: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata
+	24, // 24: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry
+	21, // 25: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer
+	30, // 26: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.result:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntry
+	28, // 27: grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry.value:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats
+	2,  // 28: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType
+	29, // [29:29] is the sub-list for method output_type
+	29, // [29:29] is the sub-list for method input_type
+	29, // [29:29] is the sub-list for extension type_name
+	29, // [29:29] is the sub-list for extension extendee
+	0,  // [0:29] is the sub-list for field type_name
 }
 
 func init() { file_grpc_testing_messages_proto_init() }
@@ -2001,6 +2128,18 @@ func file_grpc_testing_messages_proto_init() {
 			}
 		}
 		file_grpc_testing_messages_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LoadBalancerAccumulatedStatsResponse_MethodStats); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_grpc_testing_messages_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*ClientConfigureRequest_Metadata); i {
 			case 0:
 				return &v.state
@@ -2019,7 +2158,7 @@ func file_grpc_testing_messages_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_grpc_testing_messages_proto_rawDesc,
 			NumEnums:      3,
-			NumMessages:   26,
+			NumMessages:   29,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

From 938f6e2f7550e542bd78f3b9e8812665db109e02 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 14 Jan 2021 14:28:25 -0800
Subject: [PATCH 344/481] cmd/protoc-gen-go-grpc: add gRPC-Go version comment
 and update release version (#4152)

---
 balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go            | 1 +
 balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go      | 1 +
 channelz/grpc_channelz_v1/channelz_grpc.pb.go                  | 1 +
 cmd/protoc-gen-go-grpc/grpc.go                                 | 3 ++-
 cmd/protoc-gen-go-grpc/main.go                                 | 2 +-
 credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go | 1 +
 .../tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go      | 1 +
 examples/features/proto/echo/echo_grpc.pb.go                   | 1 +
 examples/helloworld/helloworld/helloworld_grpc.pb.go           | 1 +
 examples/route_guide/routeguide/route_guide_grpc.pb.go         | 1 +
 health/grpc_health_v1/health_grpc.pb.go                        | 1 +
 interop/grpc_testing/benchmark_service_grpc.pb.go              | 1 +
 interop/grpc_testing/report_qps_scenario_service_grpc.pb.go    | 1 +
 interop/grpc_testing/test_grpc.pb.go                           | 1 +
 interop/grpc_testing/worker_service_grpc.pb.go                 | 1 +
 profiling/proto/service_grpc.pb.go                             | 1 +
 reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go       | 1 +
 reflection/grpc_testing/test_grpc.pb.go                        | 1 +
 rpc_util.go                                                    | 3 +--
 stress/grpc_testing/metrics_grpc.pb.go                         | 1 +
 test/grpc_testing/test_grpc.pb.go                              | 1 +
 21 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index 41e7274a0f83..d56b77cca634 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // LoadBalancerClient is the client API for LoadBalancer service.
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index fe86d21cb107..b469089ed570 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // RouteLookupServiceClient is the client API for RouteLookupService service.
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index 15d664a4c64d..051d1ac440c7 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // ChannelzClient is the client API for Channelz service.
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 0816ae4bd874..1e787344ebcc 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -58,7 +58,8 @@ func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.
 
 	g.P("// This is a compile-time assertion to ensure that this generated file")
 	g.P("// is compatible with the grpc package it is being compiled against.")
-	g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion7"))
+	g.P("// Requires gRPC-Go v1.32.0 or later.")
+	g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion7")) // When changing, update version number above.
 	g.P()
 	for _, service := range file.Services {
 		genService(gen, file, g, service)
diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go
index 7e22561745e9..7f104da7d068 100644
--- a/cmd/protoc-gen-go-grpc/main.go
+++ b/cmd/protoc-gen-go-grpc/main.go
@@ -38,7 +38,7 @@ import (
 	"google.golang.org/protobuf/types/pluginpb"
 )
 
-const version = "1.0.1"
+const version = "1.1.0"
 
 var requireUnimplemented *bool
 
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index 75bbf87dd55c..efdbd13fa304 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // HandshakerServiceClient is the client API for HandshakerService service.
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index 7b37a7c01b79..e53a61598aba 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // MeshCertificateServiceClient is the client API for MeshCertificateService service.
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index 7657b8bfb143..052087dae369 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // EchoClient is the client API for Echo service.
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 51c64f65b3d0..39a0301c16b2 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // GreeterClient is the client API for Greeter service.
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index dd98bf5a89e0..66860e63c476 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // RouteGuideClient is the client API for RouteGuide service.
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index 8c3872664c22..386d16ce62d1 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // HealthClient is the client API for Health service.
diff --git a/interop/grpc_testing/benchmark_service_grpc.pb.go b/interop/grpc_testing/benchmark_service_grpc.pb.go
index b4badf5fe5d3..1dcba4587d29 100644
--- a/interop/grpc_testing/benchmark_service_grpc.pb.go
+++ b/interop/grpc_testing/benchmark_service_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // BenchmarkServiceClient is the client API for BenchmarkService service.
diff --git a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
index fb6c22a00d09..b0fe8c8f5ee5 100644
--- a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
+++ b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // ReportQpsScenarioServiceClient is the client API for ReportQpsScenarioService service.
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index 6b1ae9eb4321..ad5310aed623 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // TestServiceClient is the client API for TestService service.
diff --git a/interop/grpc_testing/worker_service_grpc.pb.go b/interop/grpc_testing/worker_service_grpc.pb.go
index 76d24ca2e20a..cc49b22b9261 100644
--- a/interop/grpc_testing/worker_service_grpc.pb.go
+++ b/interop/grpc_testing/worker_service_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // WorkerServiceClient is the client API for WorkerService service.
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index 37d2004eb599..bfdcc69bffb8 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // ProfilingClient is the client API for Profiling service.
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index 75b88186b20b..c2b7429a06b0 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // ServerReflectionClient is the client API for ServerReflection service.
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index cc322d75b2c8..76ec8d52d684 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SearchServiceClient is the client API for SearchService service.
diff --git a/rpc_util.go b/rpc_util.go
index ea5bb8d0c2b9..c0a1208f2f30 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -888,8 +888,7 @@ type channelzData struct {
 // buffer files to ensure compatibility with the gRPC version used.  The latest
 // support package version is 7.
 //
-// Older versions are kept for compatibility. They may be removed if
-// compatibility cannot be maintained.
+// Older versions are kept for compatibility.
 //
 // These constants should not be referenced from any other code.
 const (
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 4bcd6c19c28f..2ece03255630 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // MetricsServiceClient is the client API for MetricsService service.
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index 98e33ba3ded5..ab3b68a92bcc 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -11,6 +11,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // TestServiceClient is the client API for TestService service.

From ef9850d4ff3e9e6e9af871a3cab00c8caa5c4d85 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 15 Jan 2021 11:20:58 -0800
Subject: [PATCH 345/481] xds bootstrap: support config content in env variable
 (#4153)

---
 xds/internal/client/bootstrap/bootstrap.go    |  35 +++-
 .../client/bootstrap/bootstrap_test.go        | 185 +++++++++++-------
 xds/internal/env/env.go                       |  23 ++-
 3 files changed, 157 insertions(+), 86 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index e833a4c91f4f..385ba87cd7df 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -96,6 +96,29 @@ type xdsServer struct {
 	ServerFeatures []string       `json:"server_features"`
 }
 
+func bootstrapConfigFromEnvVariable() ([]byte, error) {
+	fName := env.BootstrapFileName
+	fContent := env.BootstrapFileContent
+
+	// Bootstrap file name has higher priority than bootstrap content.
+	if fName != "" {
+		// If file name is set
+		// - If file not found (or other errors), fail
+		// - Otherwise, use the content.
+		//
+		// Note that even if the content is invalid, we don't failover to the
+		// file content env variable.
+		logger.Debugf("xds: using bootstrap file with name %q", fName)
+		return bootstrapFileReadFunc(fName)
+	}
+
+	if fContent != "" {
+		return []byte(fContent), nil
+	}
+
+	return nil, fmt.Errorf("none of the bootstrap environment variables (%q or %q) defined", env.BootstrapFileNameEnv, env.BootstrapFileContentEnv)
+}
+
 // NewConfig returns a new instance of Config initialized by reading the
 // bootstrap file found at ${GRPC_XDS_BOOTSTRAP}.
 //
@@ -136,21 +159,15 @@ type xdsServer struct {
 func NewConfig() (*Config, error) {
 	config := &Config{}
 
-	fName := env.BootstrapFileName
-	if fName == "" {
-		return nil, fmt.Errorf("xds: Environment variable %q not defined", "GRPC_XDS_BOOTSTRAP")
-	}
-	logger.Infof("Got bootstrap file location %q", fName)
-
-	data, err := bootstrapFileReadFunc(fName)
+	data, err := bootstrapConfigFromEnvVariable()
 	if err != nil {
-		return nil, fmt.Errorf("xds: Failed to read bootstrap file %s with error %v", fName, err)
+		return nil, fmt.Errorf("xds: Failed to read bootstrap config: %v", err)
 	}
 	logger.Debugf("Bootstrap content: %s", data)
 
 	var jsonData map[string]json.RawMessage
 	if err := json.Unmarshal(data, &jsonData); err != nil {
-		return nil, fmt.Errorf("xds: Failed to parse file %s (content %v) with error: %v", fName, string(data), err)
+		return nil, fmt.Errorf("xds: Failed to parse bootstrap config: %v", err)
 	}
 
 	serverSupportsV3 := false
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 9d5aec26df71..3368af1f6a58 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -264,13 +264,17 @@ func (c *Config) compare(want *Config) error {
 	return nil
 }
 
+func fileReadFromFileMap(bootstrapFileMap map[string]string, name string) ([]byte, error) {
+	if b, ok := bootstrapFileMap[name]; ok {
+		return []byte(b), nil
+	}
+	return nil, os.ErrNotExist
+}
+
 func setupBootstrapOverride(bootstrapFileMap map[string]string) func() {
 	oldFileReadFunc := bootstrapFileReadFunc
-	bootstrapFileReadFunc = func(name string) ([]byte, error) {
-		if b, ok := bootstrapFileMap[name]; ok {
-			return []byte(b), nil
-		}
-		return nil, os.ErrNotExist
+	bootstrapFileReadFunc = func(filename string) ([]byte, error) {
+		return fileReadFromFileMap(bootstrapFileMap, filename)
 	}
 	return func() { bootstrapFileReadFunc = oldFileReadFunc }
 }
@@ -278,6 +282,49 @@ func setupBootstrapOverride(bootstrapFileMap map[string]string) func() {
 // TODO: enable leak check for this package when
 // https://github.com/googleapis/google-cloud-go/issues/2417 is fixed.
 
+// This function overrides the bootstrap file NAME env variable, to test the
+// code that reads file with the given fileName.
+func testNewConfigWithFileNameEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) {
+	origBootstrapFileName := env.BootstrapFileName
+	env.BootstrapFileName = fileName
+	defer func() { env.BootstrapFileName = origBootstrapFileName }()
+
+	c, err := NewConfig()
+	if (err != nil) != wantError {
+		t.Fatalf("NewConfig() returned error %v, wantError: %v", err, wantError)
+	}
+	if wantError {
+		return
+	}
+	if err := c.compare(wantConfig); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// This function overrides the bootstrap file CONTENT env variable, to test the
+// code that uses the content from env directly.
+func testNewConfigWithFileContentEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) {
+	b, err := bootstrapFileReadFunc(fileName)
+	if err != nil {
+		// If file reading failed, skip this test.
+		return
+	}
+	origBootstrapContent := env.BootstrapFileContent
+	env.BootstrapFileContent = string(b)
+	defer func() { env.BootstrapFileContent = origBootstrapContent }()
+
+	c, err := NewConfig()
+	if (err != nil) != wantError {
+		t.Fatalf("NewConfig() returned error %v, wantError: %v", err, wantError)
+	}
+	if wantError {
+		return
+	}
+	if err := c.compare(wantConfig); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // TestNewConfigV2ProtoFailure exercises the functionality in NewConfig with
 // different bootstrap file contents which are expected to fail.
 func TestNewConfigV2ProtoFailure(t *testing.T) {
@@ -338,13 +385,8 @@ func TestNewConfigV2ProtoFailure(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			origBootstrapFileName := env.BootstrapFileName
-			env.BootstrapFileName = test.name
-			defer func() { env.BootstrapFileName = origBootstrapFileName }()
-
-			if _, err := NewConfig(); err == nil {
-				t.Fatalf("NewConfig() returned nil error, expected to fail")
-			}
+			testNewConfigWithFileNameEnv(t, test.name, true, nil)
+			testNewConfigWithFileContentEnv(t, test.name, true, nil)
 		})
 	}
 }
@@ -382,17 +424,8 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			origBootstrapFileName := env.BootstrapFileName
-			env.BootstrapFileName = test.name
-			defer func() { env.BootstrapFileName = origBootstrapFileName }()
-
-			c, err := NewConfig()
-			if err != nil {
-				t.Fatalf("NewConfig() failed: %v", err)
-			}
-			if err := c.compare(test.wantConfig); err != nil {
-				t.Fatal(err)
-			}
+			testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig)
+			testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig)
 		})
 	}
 }
@@ -419,17 +452,8 @@ func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			origBootstrapFileName := env.BootstrapFileName
-			env.BootstrapFileName = test.name
-			defer func() { env.BootstrapFileName = origBootstrapFileName }()
-
-			c, err := NewConfig()
-			if err != nil {
-				t.Fatalf("NewConfig() failed: %v", err)
-			}
-			if err := c.compare(test.wantConfig); err != nil {
-				t.Fatal(err)
-			}
+			testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig)
+			testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig)
 		})
 	}
 }
@@ -456,31 +480,66 @@ func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			origBootstrapFileName := env.BootstrapFileName
-			env.BootstrapFileName = test.name
-			defer func() { env.BootstrapFileName = origBootstrapFileName }()
-
-			c, err := NewConfig()
-			if err != nil {
-				t.Fatalf("NewConfig() failed: %v", err)
-			}
-			if err := c.compare(test.wantConfig); err != nil {
-				t.Fatal(err)
-			}
+			testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig)
+			testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig)
 		})
 	}
 }
 
-// TestNewConfigBootstrapFileEnvNotSet tests the case where the bootstrap file
+// TestNewConfigBootstrapEnvPriority tests that the two env variables are read in correct priority
+//
+// the case where the bootstrap file
 // environment variable is not set.
-func TestNewConfigBootstrapFileEnvNotSet(t *testing.T) {
+func TestNewConfigBootstrapEnvPriority(t *testing.T) {
+	origV3Support := env.V3Support
+	env.V3Support = true
+	defer func() { env.V3Support = origV3Support }()
+
+	oldFileReadFunc := bootstrapFileReadFunc
+	bootstrapFileReadFunc = func(filename string) ([]byte, error) {
+		return fileReadFromFileMap(v2BootstrapFileMap, filename)
+	}
+	defer func() { bootstrapFileReadFunc = oldFileReadFunc }()
+
+	goodFileName1 := "goodBootstrap"
+	goodConfig1 := nonNilCredsConfigV2
+
+	goodFileName2 := "serverSupportsV3"
+	goodFileContent2 := v3BootstrapFileMap[goodFileName2]
+	goodConfig2 := nonNilCredsConfigV3
+
 	origBootstrapFileName := env.BootstrapFileName
 	env.BootstrapFileName = ""
 	defer func() { env.BootstrapFileName = origBootstrapFileName }()
 
+	origBootstrapContent := env.BootstrapFileContent
+	env.BootstrapFileContent = ""
+	defer func() { env.BootstrapFileContent = origBootstrapContent }()
+
+	// When both env variables are empty, NewConfig should fail.
 	if _, err := NewConfig(); err == nil {
 		t.Errorf("NewConfig() returned nil error, expected to fail")
 	}
+
+	// When one of them is set, it should be used.
+	env.BootstrapFileName = goodFileName1
+	env.BootstrapFileContent = ""
+	if c, err := NewConfig(); err != nil || c.compare(goodConfig1) != nil {
+		t.Errorf("NewConfig() = %v, %v, want: %v, %v", c, err, goodConfig1, nil)
+	}
+
+	env.BootstrapFileName = ""
+	env.BootstrapFileContent = goodFileContent2
+	if c, err := NewConfig(); err != nil || c.compare(goodConfig2) != nil {
+		t.Errorf("NewConfig() = %v, %v, want: %v, %v", c, err, goodConfig1, nil)
+	}
+
+	// Set both, file name should be read.
+	env.BootstrapFileName = goodFileName1
+	env.BootstrapFileContent = goodFileContent2
+	if c, err := NewConfig(); err != nil || c.compare(goodConfig1) != nil {
+		t.Errorf("NewConfig() = %v, %v, want: %v, %v", c, err, goodConfig1, nil)
+	}
 }
 
 func init() {
@@ -686,20 +745,8 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			origBootstrapFileName := env.BootstrapFileName
-			env.BootstrapFileName = test.name
-			defer func() { env.BootstrapFileName = origBootstrapFileName }()
-
-			c, err := NewConfig()
-			if (err != nil) != test.wantErr {
-				t.Fatalf("NewConfig() returned: %v, wantErr: %v", err, test.wantErr)
-			}
-			if test.wantErr {
-				return
-			}
-			if err := c.compare(test.wantConfig); err != nil {
-				t.Fatal(err)
-			}
+			testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig)
+			testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig)
 		})
 	}
 }
@@ -764,20 +811,8 @@ func TestNewConfigWithServerResourceNameID(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			origBootstrapFileName := env.BootstrapFileName
-			env.BootstrapFileName = test.name
-			defer func() { env.BootstrapFileName = origBootstrapFileName }()
-
-			c, err := NewConfig()
-			if (err != nil) != test.wantErr {
-				t.Fatalf("NewConfig() returned (%+v, %v), wantErr: %v", c, err, test.wantErr)
-			}
-			if test.wantErr {
-				return
-			}
-			if err := c.compare(test.wantConfig); err != nil {
-				t.Fatal(err)
-			}
+			testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig)
+			testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig)
 		})
 	}
 }
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index c4b46bae171b..38b62808fa39 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -26,7 +26,18 @@ import (
 )
 
 const (
-	bootstrapFileNameEnv      = "GRPC_XDS_BOOTSTRAP"
+	// BootstrapFileNameEnv is the env variable to set bootstrap file name.
+	// Do not use this and read from env directly. Its value is read and kept in
+	// variable BootstrapFileName.
+	//
+	// When both bootstrap FileName and FileContent are set, FileName is used.
+	BootstrapFileNameEnv = "GRPC_XDS_BOOTSTRAP"
+	// BootstrapFileContentEnv is the env variable to set bootstrapp file
+	// content. Do not use this and read from env directly. Its value is read
+	// and kept in variable BootstrapFileName.
+	//
+	// When both bootstrap FileName and FileContent are set, FileName is used.
+	BootstrapFileContentEnv   = "GRPC_XDS_BOOTSTRAP_CONFIG"
 	xdsV3SupportEnv           = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
 	circuitBreakingSupportEnv = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
 	timeoutSupportEnv         = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
@@ -36,7 +47,15 @@ var (
 	// BootstrapFileName holds the name of the file which contains xDS bootstrap
 	// configuration. Users can specify the location of the bootstrap file by
 	// setting the environment variable "GRPC_XDS_BOOSTRAP".
-	BootstrapFileName = os.Getenv(bootstrapFileNameEnv)
+	//
+	// When both bootstrap FileName and FileContent are set, FileName is used.
+	BootstrapFileName = os.Getenv(BootstrapFileNameEnv)
+	// BootstrapFileContent holds the content of the xDS bootstrap
+	// configuration. Users can specify the bootstrap config by
+	// setting the environment variable "GRPC_XDS_BOOSTRAP_CONFIG".
+	//
+	// When both bootstrap FileName and FileContent are set, FileName is used.
+	BootstrapFileContent = os.Getenv(BootstrapFileContentEnv)
 	// V3Support indicates whether xDS v3 API support is enabled, which can be
 	// done by setting the environment variable
 	// "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" to "true".

From 269d25366610aeef2aa632d1d88020ddba34aa12 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 15 Jan 2021 16:07:34 -0800
Subject: [PATCH 346/481] interop/xds: implement new stats response field
 (#4156)

---
 interop/xds/client/client.go | 107 ++++++++++++++++++++++-------------
 1 file changed, 68 insertions(+), 39 deletions(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 0b8e9dee3af6..300cbc9122d4 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
 	_ "google.golang.org/grpc/xds"
 
 	testgrpc "google.golang.org/grpc/interop/grpc_testing"
@@ -60,7 +61,7 @@ type statsWatcher struct {
 	rpcsByPeer    map[string]int32
 	rpcsByType    map[string]map[string]int32
 	numFailures   int32
-	remainingRpcs int32
+	remainingRPCs int32
 	chanHosts     chan *rpcInfo
 }
 
@@ -73,7 +74,7 @@ func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse {
 	}
 
 	return &testpb.LoadBalancerStatsResponse{
-		NumFailures:  watcher.numFailures + watcher.remainingRpcs,
+		NumFailures:  watcher.numFailures + watcher.remainingRPCs,
 		RpcsByPeer:   watcher.rpcsByPeer,
 		RpcsByMethod: rpcsByType,
 	}
@@ -81,57 +82,85 @@ func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse {
 
 type accumulatedStats struct {
 	mu                       sync.Mutex
-	numRpcsStartedByMethod   map[string]int32
-	numRpcsSucceededByMethod map[string]int32
-	numRpcsFailedByMethod    map[string]int32
+	numRPCsStartedByMethod   map[string]int32
+	numRPCsSucceededByMethod map[string]int32
+	numRPCsFailedByMethod    map[string]int32
+	rpcStatusByMethod        map[string]map[int32]int32
 }
 
-// copyStatsMap makes a copy of the map, and also replaces the RPC type string
-// to the proto string. E.g. "UnaryCall" -> "UNARY_CALL".
-func copyStatsMap(originalMap map[string]int32) (newMap map[string]int32) {
-	newMap = make(map[string]int32)
+func convertRPCName(in string) string {
+	switch in {
+	case unaryCall:
+		return testpb.ClientConfigureRequest_UNARY_CALL.String()
+	case emptyCall:
+		return testpb.ClientConfigureRequest_EMPTY_CALL.String()
+	}
+	logger.Warningf("unrecognized rpc type: %s", in)
+	return in
+}
+
+// copyStatsMap makes a copy of the map.
+func copyStatsMap(originalMap map[string]int32) map[string]int32 {
+	newMap := make(map[string]int32, len(originalMap))
 	for k, v := range originalMap {
-		var kk string
-		switch k {
-		case unaryCall:
-			kk = testpb.ClientConfigureRequest_UNARY_CALL.String()
-		case emptyCall:
-			kk = testpb.ClientConfigureRequest_EMPTY_CALL.String()
-		default:
-			logger.Warningf("unrecognized rpc type: %s", k)
-		}
-		if kk == "" {
-			continue
-		}
-		newMap[kk] = v
+		newMap[k] = v
 	}
 	return newMap
 }
 
+// copyStatsIntMap makes a copy of the map.
+func copyStatsIntMap(originalMap map[int32]int32) map[int32]int32 {
+	newMap := make(map[int32]int32, len(originalMap))
+	for k, v := range originalMap {
+		newMap[k] = v
+	}
+	return newMap
+}
+
+func (as *accumulatedStats) makeStatsMap() map[string]*testpb.LoadBalancerAccumulatedStatsResponse_MethodStats {
+	m := make(map[string]*testpb.LoadBalancerAccumulatedStatsResponse_MethodStats)
+	for k, v := range as.numRPCsStartedByMethod {
+		m[k] = &testpb.LoadBalancerAccumulatedStatsResponse_MethodStats{RpcsStarted: v}
+	}
+	for k, v := range as.rpcStatusByMethod {
+		if m[k] == nil {
+			m[k] = &testpb.LoadBalancerAccumulatedStatsResponse_MethodStats{}
+		}
+		m[k].Result = copyStatsIntMap(v)
+	}
+	return m
+}
+
 func (as *accumulatedStats) buildResp() *testpb.LoadBalancerAccumulatedStatsResponse {
 	as.mu.Lock()
 	defer as.mu.Unlock()
 	return &testpb.LoadBalancerAccumulatedStatsResponse{
-		NumRpcsStartedByMethod:   copyStatsMap(as.numRpcsStartedByMethod),
-		NumRpcsSucceededByMethod: copyStatsMap(as.numRpcsSucceededByMethod),
-		NumRpcsFailedByMethod:    copyStatsMap(as.numRpcsFailedByMethod),
+		NumRpcsStartedByMethod:   copyStatsMap(as.numRPCsStartedByMethod),
+		NumRpcsSucceededByMethod: copyStatsMap(as.numRPCsSucceededByMethod),
+		NumRpcsFailedByMethod:    copyStatsMap(as.numRPCsFailedByMethod),
+		StatsPerMethod:           as.makeStatsMap(),
 	}
 }
 
 func (as *accumulatedStats) startRPC(rpcType string) {
 	as.mu.Lock()
 	defer as.mu.Unlock()
-	as.numRpcsStartedByMethod[rpcType]++
+	as.numRPCsStartedByMethod[convertRPCName(rpcType)]++
 }
 
-func (as *accumulatedStats) finishRPC(rpcType string, failed bool) {
+func (as *accumulatedStats) finishRPC(rpcType string, err error) {
 	as.mu.Lock()
 	defer as.mu.Unlock()
-	if failed {
-		as.numRpcsFailedByMethod[rpcType]++
+	name := convertRPCName(rpcType)
+	if as.rpcStatusByMethod[name] == nil {
+		as.rpcStatusByMethod[name] = make(map[int32]int32)
+	}
+	as.rpcStatusByMethod[name][int32(status.Convert(err).Code())]++
+	if err != nil {
+		as.numRPCsFailedByMethod[name]++
 		return
 	}
-	as.numRpcsSucceededByMethod[rpcType]++
+	as.numRPCsSucceededByMethod[name]++
 }
 
 var (
@@ -152,9 +181,10 @@ var (
 	watchers         = make(map[statsWatcherKey]*statsWatcher)
 
 	accStats = accumulatedStats{
-		numRpcsStartedByMethod:   make(map[string]int32),
-		numRpcsSucceededByMethod: make(map[string]int32),
-		numRpcsFailedByMethod:    make(map[string]int32),
+		numRPCsStartedByMethod:   make(map[string]int32),
+		numRPCsSucceededByMethod: make(map[string]int32),
+		numRPCsFailedByMethod:    make(map[string]int32),
+		rpcStatusByMethod:        make(map[string]map[int32]int32),
 	}
 
 	// 0 or 1 representing an RPC has succeeded. Use hasRPCSucceeded and
@@ -189,7 +219,7 @@ func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalanc
 			rpcsByPeer:    make(map[string]int32),
 			rpcsByType:    make(map[string]map[string]int32),
 			numFailures:   0,
-			remainingRpcs: in.GetNumRpcs(),
+			remainingRPCs: in.GetNumRpcs(),
 			chanHosts:     make(chan *rpcInfo),
 		}
 		watchers[watcherKey] = watcher
@@ -221,8 +251,8 @@ func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalanc
 			} else {
 				watcher.numFailures++
 			}
-			watcher.remainingRpcs--
-			if watcher.remainingRpcs == 0 {
+			watcher.remainingRPCs--
+			if watcher.remainingRPCs == 0 {
 				return watcher.buildResp(), nil
 			}
 		case <-ctx.Done():
@@ -342,7 +372,7 @@ func main() {
 
 	clients := make([]testgrpc.TestServiceClient, *numChannels)
 	for i := 0; i < *numChannels; i++ {
-		conn, err := grpc.DialContext(context.Background(), *server, grpc.WithInsecure())
+		conn, err := grpc.Dial(*server, grpc.WithInsecure())
 		if err != nil {
 			logger.Fatalf("Fail to dial: %v", err)
 		}
@@ -381,11 +411,10 @@ func makeOneRPC(c testgrpc.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcI
 	case emptyCall:
 		_, err = c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p), grpc.Header(&header))
 	}
+	accStats.finishRPC(cfg.typ, err)
 	if err != nil {
-		accStats.finishRPC(cfg.typ, true)
 		return nil, nil, err
 	}
-	accStats.finishRPC(cfg.typ, false)
 
 	hosts := header["hostname"]
 	if len(hosts) > 0 {

From 504caa93c53934ac0367ba2636aa1c8147372a43 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 15 Jan 2021 16:07:52 -0800
Subject: [PATCH 347/481] interop/xds: support
 ClientConfigureRequest.timeout_sec field (#4157)

---
 interop/xds/client/client.go | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 300cbc9122d4..5b755272d3e7 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -295,8 +295,9 @@ func (s *configureService) Configure(ctx context.Context, in *testpb.ClientConfi
 			return nil, fmt.Errorf("unsupported RPC type: %v", typ)
 		}
 		cfgs = append(cfgs, &rpcConfig{
-			typ: rpcType,
-			md:  metadata.Pairs(md...),
+			typ:     rpcType,
+			md:      metadata.Pairs(md...),
+			timeout: in.GetTimeoutSec(),
 		})
 	}
 	rpcCfgs.Store(cfgs)
@@ -327,8 +328,9 @@ func parseRPCTypes(rpcStr string) (ret []string) {
 }
 
 type rpcConfig struct {
-	typ string
-	md  metadata.MD
+	typ     string
+	md      metadata.MD
+	timeout int32
 }
 
 // parseRPCMetadata turns EmptyCall:key1:value1 into
@@ -385,7 +387,11 @@ func main() {
 }
 
 func makeOneRPC(c testgrpc.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), *rpcTimeout)
+	timeout := *rpcTimeout
+	if cfg.timeout != 0 {
+		timeout = time.Duration(cfg.timeout) * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
 	if len(cfg.md) != 0 {

From f579b61a694cfecb7287d6833347128422006b55 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 19 Jan 2021 18:00:32 -0800
Subject: [PATCH 348/481] test: Run e2e authority tests only on linux. (#4160)

Although darwin supports UDS, these tests need some non-trivial amount
of work to get them to pass on darwin. This build constraint will make a
top-level `go test` happy on darwin.
---
 test/authority_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/authority_test.go b/test/authority_test.go
index 2839f01e228e..17ae178b73c9 100644
--- a/test/authority_test.go
+++ b/test/authority_test.go
@@ -1,3 +1,5 @@
+// +build linux
+
 /*
  *
  * Copyright 2020 gRPC authors.

From 7f2581f910fc21497091c4109b56d310276fc943 Mon Sep 17 00:00:00 2001
From: ZHANG Dapeng <zdapeng@google.com>
Date: Wed, 20 Jan 2021 17:24:22 -0800
Subject: [PATCH 349/481] xds interop: add xds v3 kokoro tests (#4165)

---
 test/kokoro/xds.sh     |  2 ++
 test/kokoro/xds_v3.cfg | 11 +++++++++++
 test/kokoro/xds_v3.sh  |  3 +++
 3 files changed, 16 insertions(+)
 create mode 100644 test/kokoro/xds_v3.cfg
 create mode 100755 test/kokoro/xds_v3.sh

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index 565d55aa116c..ee98399319bc 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -29,10 +29,12 @@ GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
     --test_case="all,path_matching,header_matching,circuit_breaking" \
     --project_id=grpc-testing \
+    --project_num=830293263384 \
     --source_image=projects/grpc-testing/global/images/xds-test-server-3 \
     --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \
     --gcp_suffix=$(date '+%s') \
     --verbose \
+    ${XDS_V3_OPT-} \
     --client_cmd="grpc-go/interop/xds/client/client \
       --server=xds:///{server_uri} \
       --stats_port={stats_port} \
diff --git a/test/kokoro/xds_v3.cfg b/test/kokoro/xds_v3.cfg
new file mode 100644
index 000000000000..fc4949e280d9
--- /dev/null
+++ b/test/kokoro/xds_v3.cfg
@@ -0,0 +1,11 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/xds_v3.sh"
+timeout_mins: 90
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
+}
diff --git a/test/kokoro/xds_v3.sh b/test/kokoro/xds_v3.sh
new file mode 100755
index 000000000000..73eb50d248a2
--- /dev/null
+++ b/test/kokoro/xds_v3.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+XDS_V3_OPT="--xds_v3_support" `dirname $0`/xds.sh

From 2c42474aca0cd9410911a4dbcb4ae7e13cecdb14 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 21 Jan 2021 17:21:34 -0800
Subject: [PATCH 350/481] pemfile: Make test happy with Go1.16 (#4164)

Go1.16 adds a new unexported field to x509.CertPool which causes our
tests to fail because cmp.Equal() isn't happy. This change introduces a
helper function which compares certprovider.KeyMaterial in a way that
makes the test happy with the new Go version.
---
 .../tls/certprovider/pemfile/watcher_test.go  | 39 +++++++++++++++----
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/credentials/tls/certprovider/pemfile/watcher_test.go b/credentials/tls/certprovider/pemfile/watcher_test.go
index d5ce5ab7e94d..e43cf7358eca 100644
--- a/credentials/tls/certprovider/pemfile/watcher_test.go
+++ b/credentials/tls/certprovider/pemfile/watcher_test.go
@@ -20,7 +20,7 @@ package pemfile
 
 import (
 	"context"
-	"crypto/x509"
+	"fmt"
 	"io/ioutil"
 	"math/big"
 	"os"
@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/grpctest"
@@ -55,6 +56,30 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
+func compareKeyMaterial(got, want *certprovider.KeyMaterial) error {
+	// x509.Certificate type defines an Equal() method, but does not check for
+	// nil. This has been fixed in
+	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
+	// but this is only available starting go1.14.
+	// TODO(easwars): Remove this check once we remove support for go1.13.
+	if (got.Certs == nil && want.Certs != nil) || (want.Certs == nil && got.Certs != nil) {
+		return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
+	}
+	if !cmp.Equal(got.Certs, want.Certs, cmp.AllowUnexported(big.Int{})) {
+		return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
+	}
+	// x509.CertPool contains only unexported fields some of which contain other
+	// unexported fields. So usage of cmp.AllowUnexported() or
+	// cmpopts.IgnoreUnexported() does not help us much here. Also, the standard
+	// library does not provide a way to compare CertPool values. Comparing the
+	// subjects field of the certs in the CertPool seems like a reasonable
+	// approach.
+	if gotR, wantR := got.Roots.Subjects(), want.Roots.Subjects(); !cmp.Equal(gotR, wantR, cmpopts.EquateEmpty()) {
+		return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR)
+	}
+	return nil
+}
+
 // TestNewProvider tests the NewProvider() function with different inputs.
 func (s) TestNewProvider(t *testing.T) {
 	tests := []struct {
@@ -263,7 +288,7 @@ func (s) TestProvider_UpdateSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
-	if cmp.Equal(km1, km2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+	if err := compareKeyMaterial(km1, km2); err == nil {
 		t.Fatal("expected provider to return new key material after update to underlying file")
 	}
 
@@ -279,7 +304,7 @@ func (s) TestProvider_UpdateSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
-	if cmp.Equal(km2, km3, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+	if err := compareKeyMaterial(km2, km3); err == nil {
 		t.Fatal("expected provider to return new key material after update to underlying file")
 	}
 }
@@ -363,7 +388,7 @@ func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
 
-	if cmp.Equal(km1, km2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+	if err := compareKeyMaterial(km1, km2); err == nil {
 		t.Fatal("expected provider to return new key material after symlink update")
 	}
 }
@@ -403,8 +428,8 @@ func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
-	if !cmp.Equal(km1, km2, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
-		t.Fatal("expected provider to not update key material")
+	if err := compareKeyMaterial(km1, km2); err != nil {
+		t.Fatalf("expected provider to not update key material: %v", err)
 	}
 
 	// Update the key file to match the cert file.
@@ -418,7 +443,7 @@ func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
-	if cmp.Equal(km2, km3, cmp.AllowUnexported(big.Int{}, x509.CertPool{})) {
+	if err := compareKeyMaterial(km2, km3); err == nil {
 		t.Fatal("expected provider to return new key material after update to underlying file")
 	}
 }

From 7bb497f78432890caa760b51fcfc077498b1fee2 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 27 Jan 2021 16:36:06 -0800
Subject: [PATCH 351/481] grpc: Update protobuf regenrate script (#4177)

- Pull in protobuf repo since one of our protos pulls in duration.proto
- Update grpc_testing/messages.pb.go to pull in a recent change
---
 interop/grpc_testing/messages.pb.go | 4 ++--
 regenerate.sh                       | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/interop/grpc_testing/messages.pb.go b/interop/grpc_testing/messages.pb.go
index 6ca348b6201d..f956a5ad771f 100644
--- a/interop/grpc_testing/messages.pb.go
+++ b/interop/grpc_testing/messages.pb.go
@@ -1185,8 +1185,8 @@ type LoadBalancerAccumulatedStatsResponse struct {
 	//
 	// Deprecated: Do not use.
 	NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
-	// Per-method RPC statistics.  The key is the full method path; i.e.
-	// "/proto.package.ServiceName/MethodName".
+	// Per-method RPC statistics.  The key is the RpcType in string form; e.g.
+	// 'EMPTY_CALL' or 'UNARY_CALL'
 	StatsPerMethod map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats `protobuf:"bytes,4,rep,name=stats_per_method,json=statsPerMethod,proto3" json:"stats_per_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
diff --git a/regenerate.sh b/regenerate.sh
index ed52187df660..fc6725b89f84 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -40,6 +40,9 @@ echo "go install cmd/protoc-gen-go-grpc"
 echo "git clone https://github.com/grpc/grpc-proto"
 git clone --quiet https://github.com/grpc/grpc-proto ${WORKDIR}/grpc-proto
 
+echo "git clone https://github.com/protocolbuffers/protobuf"
+git clone --quiet https://github.com/protocolbuffers/protobuf ${WORKDIR}/protobuf
+
 # Pull in code.proto as a proto dependency
 mkdir -p ${WORKDIR}/googleapis/google/rpc
 echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
@@ -87,6 +90,7 @@ for src in ${SOURCES[@]}; do
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
+    -I${WORKDIR}/protobuf/src \
     -I${WORKDIR}/istio \
     ${src}
 done
@@ -97,6 +101,7 @@ for src in ${LEGACY_SOURCES[@]}; do
     -I"." \
     -I${WORKDIR}/grpc-proto \
     -I${WORKDIR}/googleapis \
+    -I${WORKDIR}/protobuf/src \
     -I${WORKDIR}/istio \
     ${src}
 done

From e526a29227471c90813ef4d5c1b2df98e627bb28 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 27 Jan 2021 16:48:32 -0800
Subject: [PATCH 352/481] xds: Remove v3Support environment variable (#4174)

---
 xds/internal/client/bootstrap/bootstrap.go    | 12 ++---
 .../client/bootstrap/bootstrap_test.go        | 52 +++----------------
 xds/internal/env/env.go                       |  5 --
 .../test/xds_server_integration_test.go       |  6 ---
 4 files changed, 10 insertions(+), 65 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index 385ba87cd7df..c48bb797f36a 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -258,14 +258,10 @@ func NewConfig() (*Config, error) {
 		return nil, fmt.Errorf("xds: Required field %q doesn't contain valid value in bootstrap %s", "xds_servers.channel_creds", jsonData["xds_servers"])
 	}
 
-	// We end up using v3 transport protocol version only if the following
-	// conditions are met:
-	// 1. Server supports v3, indicated by the presence of "xds_v3" in
-	//    server_features.
-	// 2. Environment variable "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" is set to
-	//    true.
-	// The default value of the enum type "version.TransportAPI" is v2.
-	if env.V3Support && serverSupportsV3 {
+	// We end up using v3 transport protocol version only if the server supports
+	// v3, indicated by the presence of "xds_v3" in server_features. The default
+	// value of the enum type "version.TransportAPI" is v2.
+	if serverSupportsV3 {
 		config.TransportAPI = version.TransportV3
 	}
 
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 3368af1f6a58..e881594b8770 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -430,43 +430,10 @@ func TestNewConfigV2ProtoSuccess(t *testing.T) {
 	}
 }
 
-// TestNewConfigV3SupportNotEnabledOnClient verifies bootstrap functionality
-// when the GRPC_XDS_EXPERIMENTAL_V3_SUPPORT environment variable is not enabled
-// on the client. In this case, whether the server supports v3 or not, the
-// client will end up using v2.
-func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) {
-	origV3Support := env.V3Support
-	env.V3Support = false
-	defer func() { env.V3Support = origV3Support }()
-
-	cancel := setupBootstrapOverride(v3BootstrapFileMap)
-	defer cancel()
-
-	tests := []struct {
-		name       string
-		wantConfig *Config
-	}{
-		{"serverDoesNotSupportsV3", nonNilCredsConfigV2},
-		{"serverSupportsV3", nonNilCredsConfigV2},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig)
-			testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig)
-		})
-	}
-}
-
-// TestNewConfigV3SupportEnabledOnClient verifies bootstrap functionality when
-// the GRPC_XDS_EXPERIMENTAL_V3_SUPPORT environment variable is enabled on the
-// client. Here the client ends up using v2 or v3 based on what the server
-// supports.
-func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
-	origV3Support := env.V3Support
-	env.V3Support = true
-	defer func() { env.V3Support = origV3Support }()
-
+// TestNewConfigV3Support verifies bootstrap functionality involving support for
+// the xDS v3 transport protocol. Here the client ends up using v2 or v3 based
+// on what the server supports.
+func TestNewConfigV3Support(t *testing.T) {
 	cancel := setupBootstrapOverride(v3BootstrapFileMap)
 	defer cancel()
 
@@ -486,15 +453,12 @@ func TestNewConfigV3SupportEnabledOnClient(t *testing.T) {
 	}
 }
 
-// TestNewConfigBootstrapEnvPriority tests that the two env variables are read in correct priority
+// TestNewConfigBootstrapEnvPriority tests that the two env variables are read
+// in correct priority.
 //
 // the case where the bootstrap file
 // environment variable is not set.
 func TestNewConfigBootstrapEnvPriority(t *testing.T) {
-	origV3Support := env.V3Support
-	env.V3Support = true
-	defer func() { env.V3Support = origV3Support }()
-
 	oldFileReadFunc := bootstrapFileReadFunc
 	bootstrapFileReadFunc = func(filename string) ([]byte, error) {
 		return fileReadFromFileMap(v2BootstrapFileMap, filename)
@@ -702,10 +666,6 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 		t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err)
 	}
 
-	origV3Support := env.V3Support
-	env.V3Support = true
-	defer func() { env.V3Support = origV3Support }()
-
 	cancel := setupBootstrapOverride(bootstrapFileMap)
 	defer cancel()
 
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index 38b62808fa39..3d75bc3e86f4 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -38,7 +38,6 @@ const (
 	//
 	// When both bootstrap FileName and FileContent are set, FileName is used.
 	BootstrapFileContentEnv   = "GRPC_XDS_BOOTSTRAP_CONFIG"
-	xdsV3SupportEnv           = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
 	circuitBreakingSupportEnv = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
 	timeoutSupportEnv         = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
 )
@@ -56,10 +55,6 @@ var (
 	//
 	// When both bootstrap FileName and FileContent are set, FileName is used.
 	BootstrapFileContent = os.Getenv(BootstrapFileContentEnv)
-	// V3Support indicates whether xDS v3 API support is enabled, which can be
-	// done by setting the environment variable
-	// "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" to "true".
-	V3Support = strings.EqualFold(os.Getenv(xdsV3SupportEnv), "true")
 	// CircuitBreakingSupport indicates whether circuit breaking support is
 	// enabled, which can be done by setting the environment variable
 	// "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING" to "true".
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 300aab3a737b..e55e048e6a36 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -50,7 +50,6 @@ import (
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/testdata"
 	"google.golang.org/grpc/xds"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 	"google.golang.org/grpc/xds/internal/version"
@@ -136,10 +135,6 @@ func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
 func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, func()) {
 	t.Helper()
 
-	// Turn on xDS V3 support.
-	origV3Support := env.V3Support
-	env.V3Support = true
-
 	// Spin up a xDS management server on a local port.
 	nodeID := uuid.New().String()
 	fs, err := e2e.StartManagementServer()
@@ -189,7 +184,6 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 	}()
 
 	return fs, nodeID, lis, func() {
-		env.V3Support = origV3Support
 		fs.Stop()
 		bootstrapCleanup()
 		server.Stop()

From 0bc741730b8171fc51cdaf826caea5119c411009 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 28 Jan 2021 16:47:07 -0800
Subject: [PATCH 353/481] xds: report drops by circuit breaking (#4171)

---
 xds/internal/balancer/edsbalancer/eds_impl.go |  5 ++
 .../balancer/edsbalancer/eds_impl_test.go     | 48 ++++++++++++++++---
 .../client/client_requests_counter.go         | 13 +++++
 xds/internal/client/load/store.go             |  7 ++-
 xds/internal/client/load/store_test.go        |  3 +-
 xds/internal/testutils/protos.go              |  7 ++-
 6 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 17a41ef18e75..8c34e4b9d436 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -515,6 +515,11 @@ func (d *dropPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 	}
 	if d.counter != nil {
 		if err := d.counter.StartRequest(); err != nil {
+			// Drops by circuit breaking are reported with empty category. They
+			// will be reported only in total drops, but not in per category.
+			if d.loadStore != nil {
+				d.loadStore.CallDropped("")
+			}
 			return balancer.PickResult{}, status.Errorf(codes.Unavailable, err.Error())
 		}
 		pr, err := d.p.Pick(info)
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 2eec6be30f10..b4349a021dca 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -142,7 +142,7 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 	}
 
 	// The same locality, different drop rate, dropping 50%.
-	clab5 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], []uint32{50})
+	clab5 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], map[string]uint32{"test-drop": 50})
 	clab5.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
 	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab5.Build()))
 
@@ -746,6 +746,10 @@ func (s) TestDropPicker(t *testing.T) {
 }
 
 func (s) TestEDS_LoadReport(t *testing.T) {
+	origCircuitBreakingSupport := env.CircuitBreakingSupport
+	env.CircuitBreakingSupport = true
+	defer func() { env.CircuitBreakingSupport = origCircuitBreakingSupport }()
+
 	// We create an xdsClientWrapper with a dummy xdsClientInterface which only
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
@@ -758,10 +762,20 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	edsb := newEDSBalancerImpl(cc, nil, lsWrapper, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
+	const (
+		testServiceName = "test-service"
+		cbMaxRequests   = 20
+	)
+	var maxRequestsTemp uint32 = cbMaxRequests
+	client.SetMaxRequests(testServiceName, &maxRequestsTemp)
+	defer client.ClearCounterForTesting(testServiceName)
+	edsb.updateServiceRequestsCounter(testServiceName)
+
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
+	const testDropCategory = "test-drop"
 	// Two localities, each with one backend.
-	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], map[string]uint32{testDropCategory: 50})
 	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
 	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
 	sc1 := <-cc.NewSubConnCh
@@ -788,20 +802,42 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// the picks on sc1 should show up as inProgress.
 	locality1JSON, _ := locality1.ToString()
 	locality2JSON, _ := locality2.ToString()
+	const (
+		rpcCount = 100
+		// 50% will be dropped with category testDropCategory.
+		dropWithCategory = rpcCount / 2
+		// In the remaining RPCs, only cbMaxRequests are allowed by circuit
+		// breaking. Others will be dropped by CB.
+		dropWithCB = rpcCount - dropWithCategory - cbMaxRequests
+
+		rpcInProgress = cbMaxRequests / 2 // 50% of RPCs will be never done.
+		rpcSucceeded  = cbMaxRequests / 2 // 50% of RPCs will succeed.
+	)
 	wantStoreData := []*load.Data{{
 		Cluster: testClusterNames[0],
 		Service: "",
 		LocalityStats: map[string]load.LocalityData{
-			locality1JSON: {RequestStats: load.RequestData{InProgress: 5}},
-			locality2JSON: {RequestStats: load.RequestData{Succeeded: 5}},
+			locality1JSON: {RequestStats: load.RequestData{InProgress: rpcInProgress}},
+			locality2JSON: {RequestStats: load.RequestData{Succeeded: rpcSucceeded}},
+		},
+		TotalDrops: dropWithCategory + dropWithCB,
+		Drops: map[string]uint64{
+			testDropCategory: dropWithCategory,
 		},
 	}}
-	for i := 0; i < 10; i++ {
+
+	var rpcsToBeDone []balancer.PickResult
+	// Run the picks, but only pick with sc1 will be done later.
+	for i := 0; i < rpcCount; i++ {
 		scst, _ := p1.Pick(balancer.PickInfo{})
 		if scst.Done != nil && scst.SubConn != sc1 {
-			scst.Done(balancer.DoneInfo{})
+			rpcsToBeDone = append(rpcsToBeDone, scst)
 		}
 	}
+	// Call done on those sc1 picks.
+	for _, scst := range rpcsToBeDone {
+		scst.Done(balancer.DoneInfo{})
+	}
 
 	gotStoreData := loadStore.Stats(testClusterNames[0:1])
 	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(load.Data{}, "ReportInterval")); diff != "" {
diff --git a/xds/internal/client/client_requests_counter.go b/xds/internal/client/client_requests_counter.go
index 1e28fc003ff3..74b80f1c3f7c 100644
--- a/xds/internal/client/client_requests_counter.go
+++ b/xds/internal/client/client_requests_counter.go
@@ -87,3 +87,16 @@ func (c *ServiceRequestsCounter) StartRequest() error {
 func (c *ServiceRequestsCounter) EndRequest() {
 	atomic.AddUint32(&c.numRequests, ^uint32(0))
 }
+
+// ClearCounterForTesting clears the counter for the service. Should be only
+// used in tests.
+func ClearCounterForTesting(serviceName string) {
+	src.mu.Lock()
+	defer src.mu.Unlock()
+	c, ok := src.services[serviceName]
+	if !ok {
+		return
+	}
+	c.maxRequests = defaultMaxRequests
+	c.numRequests = 0
+}
diff --git a/xds/internal/client/load/store.go b/xds/internal/client/load/store.go
index a6ec1ec337cd..551a5147b6bd 100644
--- a/xds/internal/client/load/store.go
+++ b/xds/internal/client/load/store.go
@@ -283,7 +283,12 @@ func (ls *perClusterStore) stats() *Data {
 			return true
 		}
 		sd.TotalDrops += d
-		sd.Drops[key.(string)] = d
+		keyStr := key.(string)
+		if keyStr != "" {
+			// Skip drops without category. They are counted in total_drops, but
+			// not in per category. One example is drops by circuit breaking.
+			sd.Drops[keyStr] = d
+		}
 		return true
 	})
 	ls.localityRPCCount.Range(func(key, val interface{}) bool {
diff --git a/xds/internal/client/load/store_test.go b/xds/internal/client/load/store_test.go
index 4d62ebc4c621..46568591f9e4 100644
--- a/xds/internal/client/load/store_test.go
+++ b/xds/internal/client/load/store_test.go
@@ -47,9 +47,10 @@ func TestDrops(t *testing.T) {
 		drops = map[string]int{
 			dropCategories[0]: 30,
 			dropCategories[1]: 40,
+			"":                10,
 		}
 		wantStoreData = &Data{
-			TotalDrops: 70,
+			TotalDrops: 80,
 			Drops: map[string]uint64{
 				dropCategories[0]: 30,
 				dropCategories[1]: 40,
diff --git a/xds/internal/testutils/protos.go b/xds/internal/testutils/protos.go
index 25a0944d96dc..e0dba0e2b301 100644
--- a/xds/internal/testutils/protos.go
+++ b/xds/internal/testutils/protos.go
@@ -18,7 +18,6 @@
 package testutils
 
 import (
-	"fmt"
 	"net"
 	"strconv"
 
@@ -59,11 +58,11 @@ type ClusterLoadAssignmentBuilder struct {
 }
 
 // NewClusterLoadAssignmentBuilder creates a ClusterLoadAssignmentBuilder.
-func NewClusterLoadAssignmentBuilder(clusterName string, dropPercents []uint32) *ClusterLoadAssignmentBuilder {
+func NewClusterLoadAssignmentBuilder(clusterName string, dropPercents map[string]uint32) *ClusterLoadAssignmentBuilder {
 	var drops []*v2xdspb.ClusterLoadAssignment_Policy_DropOverload
-	for i, d := range dropPercents {
+	for n, d := range dropPercents {
 		drops = append(drops, &v2xdspb.ClusterLoadAssignment_Policy_DropOverload{
-			Category: fmt.Sprintf("test-drop-%d", i),
+			Category: n,
 			DropPercentage: &v2typepb.FractionalPercent{
 				Numerator:   d,
 				Denominator: v2typepb.FractionalPercent_HUNDRED,

From f005af03c24d49a4ec379884392cc4a557a0d047 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 3 Feb 2021 13:21:42 -0800
Subject: [PATCH 354/481] examples: delete profiling example since profiling
 support was rolled back (#4182)

---
 examples/features/profiling/README.md      | 261 ---------------------
 examples/features/profiling/client/main.go |  89 -------
 examples/features/profiling/server/main.go |  69 ------
 3 files changed, 419 deletions(-)
 delete mode 100644 examples/features/profiling/README.md
 delete mode 100644 examples/features/profiling/client/main.go
 delete mode 100644 examples/features/profiling/server/main.go

diff --git a/examples/features/profiling/README.md b/examples/features/profiling/README.md
deleted file mode 100644
index 1fd80e9ada2d..000000000000
--- a/examples/features/profiling/README.md
+++ /dev/null
@@ -1,261 +0,0 @@
-# gRPC-Go Profiling
-
-- Author(s): adtac
-- Status: Experimental
-- Availability: gRPC-Go >= 1.27
-- Last updated: December 17, 2019
-
-gRPC-Go has built-in profiling that can be used to generate a detailed timeline
-of the lifecycle of an RPC request. This can be done on the client-side and the
-server-side. This directory contains an example client-server implementation
-with profiling enabled and some example commands you can run to remotely manage
-profiling.
-
-Typically, there are three logically separate parts involved in integrating
-profiling into your application:
-
-1. Register the `Profiling` service: this requires a simple code change in your
-   application.
-1. Enable profiling when required: profiling is disabled by default and must be
-   enabled remotely or at server initialization.
-1. Download and process profiling data: once your application has collected
-   enough profiling data, you must use a bundled command-line application to
-   download your data and process it to generate human-friendly visualization.
-
-## Registering the `Profiling` Service
-
-### Server-Side
-
-Typically, you would create and register a server like so (some Go is shortened
-in the interest of brevity; please see the `server` subdirectory for a full
-implementation):
-
-```go
-import (
-	"google.golang.org/grpc"
-	profsvc "google.golang.org/grpc/profiling/service"
-	pb "google.golang.org/grpc/examples/features/proto/echo"
-)
-
-type server struct{}
-
-func main() error {
-	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
-
-	// Include this to register a profiling-specific service within your server.
-	if err := profsvc.Init(&profsvc.ProfilingConfig{Server: s}); err != nil {
-		fmt.Printf("error calling profsvc.Init: %v\n", err)
-		return
-	}
-
-	lis, _ := net.Listen("tcp", address)
-	s.Serve(lis)
-}
-```
-
-To register your server for profiling, simply call the `profsvc.Init` method
-as shown above. The passed `ProfilingConfig` parameter must set the `Server`
-field to a server that is being served on a TCP address.
-
-### Client-Side
-
-To register profiling on the client-side, you must create a server to expose
-your profiling data in order for it to be retrievable. To do this, it is
-recommended that you create a dummy, dedicated server with no service other
-than profiling's. See the `client` directory for an example client.
-
-## Enabling/Disabling Profiling
-
-Once profiling is baked into your server (unless otherwise specified, from here
-on, the word "server" will be used to refer to a `grpc.Server`, not the
-server/client distinction from the previous subsection), you need to enable
-profiling. There are three ways to do this -- at initialization, remotely
-post-initialization, or programmatically within Go.
-
-### Enabling Profiling at Initialization
-
-To force profiling to start measuring data right from the first RPC, set the
-`Enabled` attribute of the `ProfilingConfig` struct to `true` when you are
-initializing profiling.
-
-```go
-	// Set Enabled: true to turn profiling on at initialization time.
-	profsvc.Init(&profsvc.ProfilingConfig{
-		Server:  s,
-		Enabled: true,
-	})
-```
-
-### Enabling/Disabling Remotely
-
-Alternatively, you can enable/disable profiling any time after server
-initialization by using a bundled command-line tool designed for remote
-profiling management. Assuming `example.com:50051` is the address of the server
-that you would like to enable profiling in, do the following:
-
-```bash
-$ go run google.golang.org/grpc/profiling/cmd \
-    -address example.com:50051                \
-    -enable-profiling
-```
-
-Similarly, running the command with `-disable-profiling` can be used to disable
-profiling remotely.
-
-
-### Enabling/Disabling Within Go
-
-In addition to the remote service that is exposed, you may enable/disable
-profiling within your application in Go:
-
-```go
-import (
-	"google.golang.org/grpc/profiling"
-)
-
-func setProfiling(enable bool) {
-	profiling.Enable(true)
-}
-```
-
-The `profiling.Enable` function can be safely accessed and called concurrently.
-
-## Downloading and Processing Profiling Data
-
-Once your server has collected enough profiling data, you may want to download
-that data and perform some analysis on the retrieved data. The aforementioned
-command-line application within gRPC comes bundled with support for both
-operations.
-
-To retrieve profiling data from a remote server, run the following command:
-
-```bash
-$ go run google.golang.org/grpc/profiling/cmd \
-    -address example.com:50051                \
-    -retrieve-snapshot                        \
-    -snapshot /path/to/snapshot
-```
-
-You must provide a path to `-snapshot` that can be written to. This file will
-store the retrieved data in a raw and binary form.
-
-To process this data into a human-consumable such as
-[Catapult's trace-viewer format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview):
-
-```bash
-$ go run google.golang.org/grpc/profiling/cmd \
-    -snapshot /path/to/snapshot               \
-    -stream-stats-catapult-json /path/to/json
-```
-
-This would read the data stored in `/path/to/snapshot` and process it to
-generate a JSON format that is understood by Chromium's
-[Catapult project](https://chromium.googlesource.com/catapult).
-The Catapult project comes with a utility called
-[trace-viewer](https://chromium.googlesource.com/catapult/+/HEAD/tracing/README.md),
-which can be used to generate human-readable visualizations:
-
-```bash
-$ git clone https://chromium.googlesource.com/catapult /path/to/catapult
-$ /path/to/catapult/tracing/bin/trace2html /path/to/json --output=/path/to/html
-```
-
-When the generated `/path/to/html` file is opened with a browser, you will be
-presented with a detailed visualization of the lifecycle of all RPC requests.
-To learn more about trace-viewer and how to navigate the generated HTML, see
-[this](https://chromium.googlesource.com/catapult/+/HEAD/tracing/README.md).
-
-## Frequently Asked Questions
-
-##### I have multiple `grpc.Server`s in my application. Can I register profiling with just one of them?
-
-You may not call `profsvc.Init` more than once -- all calls except for the
-first one will return an error. As a corollary, it is also not possible to
-register or enable/disable profiling for just one `grpc.Server` or operation.
-That is, you can enable/disable profiling globally for all gRPC operations or
-none at all.
-
-##### Is `google.golang.org/grpc/profiling/cmd` the canonical implementation of a client that can talk to the profiling service?
-
-No, the command-line tool is simply provided as a reference implementation and
-as a convenience. You are free to write your own tool as long as it can
-communicate using the underlying protocol buffers.
-
-##### Is Catapult's `trace-viewer` the only option that is supported?
-
-Currently, yes. However, support for other (or better) visualization tools is
-welcome.
-
-##### What is the impact of profiling on application performance?
-
-When turned off, profiling has virtually no impact on the performance (QPS,
-latency, memory footprint) of your application. However, when turned on, expect
-a 5-10% throughput/latency penalty and double the memory footprint.
-
-Profiling is mostly used by gRPC-Go devs. However, if you foresee using
-profiling in production machines, because of the negligible impact of profiling
-when turned off, you may want to register/initialize your applications with
-profiling (but leave it turned off). This will be useful in the off-chance you
-want to debug an application later -- in such an event, you can simply remotely
-toggle profiling using the `go run` command previously described to enable
-profiling data collection. Once you're confident that enough profiling data has
-been measured, you can turn it off again and retrieve the data for
-post-processing (see previous section).
-
-##### How many RPCs worth of data is stored by profiling? I'd like to restrict the memory footprint of gRPC's profiling framework to a fixed amount.
-
-By default, at any given time, the last 2<sup>14</sup> RPCs worth of data is
-stored by profiling. Newly generated profiling data overwrites older data. Note
-that the internal data structure is not strictly LIFO in order to be performant
-(but is approximately LIFO). All profiling data is timestamped anyway, so
-a LIFO property is unnecessary.
-
-This number is configurable. When registering your server with profiling, you
-may specify the number of samples that should be stored, like so:
-
-```go
-	// Setting StreamStatsSize: 1024 will make profiling store the last 1024
-	// RPCs' data (if profiling is enabled, of course).
-	profsvc.Init(&profsvc.ProfilingConfig{
-		Server:          s,
-		StreamStatsSize: 1024,
-	})
-```
-
-As an estimate, a typical unary RPC is expected produce ~2-3 KiB of profiling
-data in memory. This may be useful in estimating how many RPCs worth of data
-you can afford depending on your memory capacity. For more complex RPCs such as
-streaming RPCs, each RPC will consume more data. The amount of memory consumed
-by profiling is mostly independent of the size of messages your application
-handles.
-
-##### The generated visualization is flat and has no flows/arrows. How do I distinguish between different RPCs?
-
-Unfortunately, there isn't any way to do this without some changes to the way
-your application is compiled. This is because gRPC's profiling relies on the
-Goroutine ID to uniquely identify different components.
-
-To enable this, first apply the following patch to your Go runtime installation
-directory:
-
-```diff
-diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
---- a/src/runtime/runtime2.go
-+++ b/src/runtime/runtime2.go
-@@ -392,6 +392,10 @@ type stack struct {
- 	hi uintptr
- }
- 
-+func Goid() int64 {
-+	return getg().goid
-+}
-+
- type g struct {
- 	// Stack parameters.
- 	// stack describes the actual stack memory: [stack.lo, stack.hi).
-```
-
-Then, recompile your application with `-tags grpcgoid` to generate a new
-binary. This binary should produce profiling data that is much nicer when
-visualized.
diff --git a/examples/features/profiling/client/main.go b/examples/features/profiling/client/main.go
deleted file mode 100644
index d899e47dadb0..000000000000
--- a/examples/features/profiling/client/main.go
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Binary client is an example client.
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"net"
-	"time"
-
-	"google.golang.org/grpc"
-	pb "google.golang.org/grpc/examples/features/proto/echo"
-	profsvc "google.golang.org/grpc/profiling/service"
-)
-
-var addr = flag.String("addr", "localhost:50051", "the address to connect to")
-var profilingPort = flag.Int("profilingPort", 50052, "port to expose the profiling service on")
-
-func setupClientProfiling() error {
-	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *profilingPort))
-	if err != nil {
-		log.Printf("failed to listen: %v\n", err)
-		return err
-	}
-	fmt.Printf("server listening at %v\n", lis.Addr())
-
-	s := grpc.NewServer()
-
-	// Register this grpc.Server with profiling.
-	pc := &profsvc.ProfilingConfig{
-		Server:          s,
-		Enabled:         true,
-		StreamStatsSize: 1024,
-	}
-	if err = profsvc.Init(pc); err != nil {
-		fmt.Printf("error calling profsvc.Init: %v\n", err)
-		return err
-	}
-
-	go s.Serve(lis)
-	return nil
-}
-
-func main() {
-	flag.Parse()
-
-	if err := setupClientProfiling(); err != nil {
-		log.Fatalf("error setting up profiling: %v\n", err)
-	}
-
-	// Set up a connection to the server.
-	conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock())
-	if err != nil {
-		log.Fatalf("did not connect: %v", err)
-	}
-	defer conn.Close()
-
-	c := pb.NewEchoClient(conn)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	res, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "hello, profiling"})
-	fmt.Printf("UnaryEcho call returned %q, %v\n", res.GetMessage(), err)
-	if err != nil {
-		log.Fatalf("error calling UnaryEcho: %v", err)
-	}
-
-	log.Printf("sleeping for 30 seconds with exposed profiling service on :%d\n", *profilingPort)
-	time.Sleep(30 * time.Second)
-}
diff --git a/examples/features/profiling/server/main.go b/examples/features/profiling/server/main.go
deleted file mode 100644
index ee0a4f55ab2e..000000000000
--- a/examples/features/profiling/server/main.go
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Binary server is an example server.
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"net"
-
-	"google.golang.org/grpc"
-	pb "google.golang.org/grpc/examples/features/proto/echo"
-	profsvc "google.golang.org/grpc/profiling/service"
-)
-
-var port = flag.Int("port", 50051, "the port to serve on")
-
-type server struct {
-	pb.UnimplementedEchoServer
-}
-
-func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) {
-	fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage())
-	return &pb.EchoResponse{Message: in.Message}, nil
-}
-
-func main() {
-	flag.Parse()
-
-	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
-	if err != nil {
-		log.Fatalf("failed to listen: %v", err)
-	}
-	fmt.Printf("server listening at %v\n", lis.Addr())
-
-	s := grpc.NewServer()
-	pb.RegisterEchoServer(s, &server{})
-
-	// Register your grpc.Server with profiling.
-	pc := &profsvc.ProfilingConfig{
-		Server:          s,
-		Enabled:         true,
-		StreamStatsSize: 1024,
-	}
-	if err = profsvc.Init(pc); err != nil {
-		fmt.Printf("error calling profsvc.Init: %v\n", err)
-		return
-	}
-
-	s.Serve(lis)
-}

From 7b8d65a7bc4446f61cceb8ee679847c100a07695 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 4 Feb 2021 14:11:50 -0800
Subject: [PATCH 355/481] xds: rename internal/client files to remove client
 prefix (#4188)

---
 xds/internal/client/{client_callback.go => callback.go}           | 0
 xds/internal/client/{client_cds_test.go => cds_test.go}           | 0
 xds/internal/client/{client_eds_test.go => eds_test.go}           | 0
 xds/internal/client/{client_lds_test.go => lds_test.go}           | 0
 xds/internal/client/{client_loadreport.go => loadreport.go}       | 0
 .../client/{client_loadreport_test.go => loadreport_test.go}      | 0
 xds/internal/client/{client_logging.go => logging.go}             | 0
 xds/internal/client/{client_rds_test.go => rds_test.go}           | 0
 .../client/{client_requests_counter.go => requests_counter.go}    | 0
 .../{client_requests_counter_test.go => requests_counter_test.go} | 0
 xds/internal/client/{client_singleton.go => singleton.go}         | 0
 xds/internal/client/v2/{client_ack_test.go => ack_test.go}        | 0
 xds/internal/client/v2/{client_cds_test.go => cds_test.go}        | 0
 xds/internal/client/v2/{client_eds_test.go => eds_test.go}        | 0
 xds/internal/client/v2/{client_lds_test.go => lds_test.go}        | 0
 xds/internal/client/v2/{client_rds_test.go => rds_test.go}        | 0
 xds/internal/client/{client_watchers.go => watchers.go}           | 0
 .../{client_watchers_cluster_test.go => watchers_cluster_test.go} | 0
 ...ient_watchers_endpoints_test.go => watchers_endpoints_test.go} | 0
 ...client_watchers_listener_test.go => watchers_listener_test.go} | 0
 .../{client_watchers_route_test.go => watchers_route_test.go}     | 0
 xds/internal/client/{client_xds.go => xds.go}                     | 0
 22 files changed, 0 insertions(+), 0 deletions(-)
 rename xds/internal/client/{client_callback.go => callback.go} (100%)
 rename xds/internal/client/{client_cds_test.go => cds_test.go} (100%)
 rename xds/internal/client/{client_eds_test.go => eds_test.go} (100%)
 rename xds/internal/client/{client_lds_test.go => lds_test.go} (100%)
 rename xds/internal/client/{client_loadreport.go => loadreport.go} (100%)
 rename xds/internal/client/{client_loadreport_test.go => loadreport_test.go} (100%)
 rename xds/internal/client/{client_logging.go => logging.go} (100%)
 rename xds/internal/client/{client_rds_test.go => rds_test.go} (100%)
 rename xds/internal/client/{client_requests_counter.go => requests_counter.go} (100%)
 rename xds/internal/client/{client_requests_counter_test.go => requests_counter_test.go} (100%)
 rename xds/internal/client/{client_singleton.go => singleton.go} (100%)
 rename xds/internal/client/v2/{client_ack_test.go => ack_test.go} (100%)
 rename xds/internal/client/v2/{client_cds_test.go => cds_test.go} (100%)
 rename xds/internal/client/v2/{client_eds_test.go => eds_test.go} (100%)
 rename xds/internal/client/v2/{client_lds_test.go => lds_test.go} (100%)
 rename xds/internal/client/v2/{client_rds_test.go => rds_test.go} (100%)
 rename xds/internal/client/{client_watchers.go => watchers.go} (100%)
 rename xds/internal/client/{client_watchers_cluster_test.go => watchers_cluster_test.go} (100%)
 rename xds/internal/client/{client_watchers_endpoints_test.go => watchers_endpoints_test.go} (100%)
 rename xds/internal/client/{client_watchers_listener_test.go => watchers_listener_test.go} (100%)
 rename xds/internal/client/{client_watchers_route_test.go => watchers_route_test.go} (100%)
 rename xds/internal/client/{client_xds.go => xds.go} (100%)

diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/callback.go
similarity index 100%
rename from xds/internal/client/client_callback.go
rename to xds/internal/client/callback.go
diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/cds_test.go
similarity index 100%
rename from xds/internal/client/client_cds_test.go
rename to xds/internal/client/cds_test.go
diff --git a/xds/internal/client/client_eds_test.go b/xds/internal/client/eds_test.go
similarity index 100%
rename from xds/internal/client/client_eds_test.go
rename to xds/internal/client/eds_test.go
diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/lds_test.go
similarity index 100%
rename from xds/internal/client/client_lds_test.go
rename to xds/internal/client/lds_test.go
diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/loadreport.go
similarity index 100%
rename from xds/internal/client/client_loadreport.go
rename to xds/internal/client/loadreport.go
diff --git a/xds/internal/client/client_loadreport_test.go b/xds/internal/client/loadreport_test.go
similarity index 100%
rename from xds/internal/client/client_loadreport_test.go
rename to xds/internal/client/loadreport_test.go
diff --git a/xds/internal/client/client_logging.go b/xds/internal/client/logging.go
similarity index 100%
rename from xds/internal/client/client_logging.go
rename to xds/internal/client/logging.go
diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/rds_test.go
similarity index 100%
rename from xds/internal/client/client_rds_test.go
rename to xds/internal/client/rds_test.go
diff --git a/xds/internal/client/client_requests_counter.go b/xds/internal/client/requests_counter.go
similarity index 100%
rename from xds/internal/client/client_requests_counter.go
rename to xds/internal/client/requests_counter.go
diff --git a/xds/internal/client/client_requests_counter_test.go b/xds/internal/client/requests_counter_test.go
similarity index 100%
rename from xds/internal/client/client_requests_counter_test.go
rename to xds/internal/client/requests_counter_test.go
diff --git a/xds/internal/client/client_singleton.go b/xds/internal/client/singleton.go
similarity index 100%
rename from xds/internal/client/client_singleton.go
rename to xds/internal/client/singleton.go
diff --git a/xds/internal/client/v2/client_ack_test.go b/xds/internal/client/v2/ack_test.go
similarity index 100%
rename from xds/internal/client/v2/client_ack_test.go
rename to xds/internal/client/v2/ack_test.go
diff --git a/xds/internal/client/v2/client_cds_test.go b/xds/internal/client/v2/cds_test.go
similarity index 100%
rename from xds/internal/client/v2/client_cds_test.go
rename to xds/internal/client/v2/cds_test.go
diff --git a/xds/internal/client/v2/client_eds_test.go b/xds/internal/client/v2/eds_test.go
similarity index 100%
rename from xds/internal/client/v2/client_eds_test.go
rename to xds/internal/client/v2/eds_test.go
diff --git a/xds/internal/client/v2/client_lds_test.go b/xds/internal/client/v2/lds_test.go
similarity index 100%
rename from xds/internal/client/v2/client_lds_test.go
rename to xds/internal/client/v2/lds_test.go
diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/rds_test.go
similarity index 100%
rename from xds/internal/client/v2/client_rds_test.go
rename to xds/internal/client/v2/rds_test.go
diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/watchers.go
similarity index 100%
rename from xds/internal/client/client_watchers.go
rename to xds/internal/client/watchers.go
diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/watchers_cluster_test.go
similarity index 100%
rename from xds/internal/client/client_watchers_cluster_test.go
rename to xds/internal/client/watchers_cluster_test.go
diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/watchers_endpoints_test.go
similarity index 100%
rename from xds/internal/client/client_watchers_endpoints_test.go
rename to xds/internal/client/watchers_endpoints_test.go
diff --git a/xds/internal/client/client_watchers_listener_test.go b/xds/internal/client/watchers_listener_test.go
similarity index 100%
rename from xds/internal/client/client_watchers_listener_test.go
rename to xds/internal/client/watchers_listener_test.go
diff --git a/xds/internal/client/client_watchers_route_test.go b/xds/internal/client/watchers_route_test.go
similarity index 100%
rename from xds/internal/client/client_watchers_route_test.go
rename to xds/internal/client/watchers_route_test.go
diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/xds.go
similarity index 100%
rename from xds/internal/client/client_xds.go
rename to xds/internal/client/xds.go

From b753f4903c1b09ced50ae223b439a771aff798f9 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 4 Feb 2021 20:13:54 -0800
Subject: [PATCH 356/481] xds testing: increase timeout from 90 minutes to 120
 minutes (#4191)

This is the timeout for all the tests, after which the VM will be termiated.
Since now we have more tests, we need more time.

The current average is around 85-90 minutes. The extra 30 minutes should be
enough for several new tests.
---
 test/kokoro/xds.cfg    | 2 +-
 test/kokoro/xds_v3.cfg | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/kokoro/xds.cfg b/test/kokoro/xds.cfg
index 09ab0dacc463..d1a078217b84 100644
--- a/test/kokoro/xds.cfg
+++ b/test/kokoro/xds.cfg
@@ -2,7 +2,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc-go/test/kokoro/xds.sh"
-timeout_mins: 90
+timeout_mins: 120
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
diff --git a/test/kokoro/xds_v3.cfg b/test/kokoro/xds_v3.cfg
index fc4949e280d9..c4c8aad9e6f2 100644
--- a/test/kokoro/xds_v3.cfg
+++ b/test/kokoro/xds_v3.cfg
@@ -2,7 +2,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc-go/test/kokoro/xds_v3.sh"
-timeout_mins: 90
+timeout_mins: 120
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"

From 9280052d36656451dd7568a18a836c2a74edaf6c Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Sun, 7 Feb 2021 19:55:33 -0800
Subject: [PATCH 357/481] balancergroup: Propagate balancer.BuildOptions to
 child policies (#4184)

---
 .../balancer/balancergroup/balancergroup.go   | 11 +++-
 .../balancergroup/balancergroup_test.go       | 58 +++++++++++++++++--
 .../balancer/clustermanager/clustermanager.go |  4 +-
 .../clustermanager/clustermanager_test.go     | 55 ++++++++++++++++++
 xds/internal/balancer/edsbalancer/eds.go      |  8 +--
 xds/internal/balancer/edsbalancer/eds_impl.go |  7 ++-
 .../edsbalancer/eds_impl_priority_test.go     | 20 +++----
 .../balancer/edsbalancer/eds_impl_test.go     | 49 +++++++++++-----
 xds/internal/balancer/edsbalancer/eds_test.go |  2 +-
 .../balancer/weightedtarget/weightedtarget.go |  4 +-
 xds/internal/client/v3/client.go              |  2 +-
 xds/internal/testutils/balancer.go            | 40 -------------
 12 files changed, 175 insertions(+), 85 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index d646355191c3..2ec576a4b572 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -60,6 +60,8 @@ type subBalancerWrapper struct {
 	// The static part of sub-balancer. Keeps balancerBuilders and addresses.
 	// To be used when restarting sub-balancer.
 	builder balancer.Builder
+	// Options to be passed to sub-balancer at the time of creation.
+	buildOpts balancer.BuildOptions
 	// ccState is a cache of the addresses/balancer config, so when the balancer
 	// is restarted after close, it will get the previous update. It's a pointer
 	// and is set to nil at init, so when the balancer is built for the first
@@ -94,7 +96,7 @@ func (sbc *subBalancerWrapper) updateBalancerStateWithCachedPicker() {
 }
 
 func (sbc *subBalancerWrapper) startBalancer() {
-	b := sbc.builder.Build(sbc, balancer.BuildOptions{})
+	b := sbc.builder.Build(sbc, sbc.buildOpts)
 	sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name())
 	sbc.balancer = b
 	if sbc.ccState != nil {
@@ -179,6 +181,7 @@ func (sbc *subBalancerWrapper) stopBalancer() {
 // balancer group.
 type BalancerGroup struct {
 	cc        balancer.ClientConn
+	buildOpts balancer.BuildOptions
 	logger    *grpclog.PrefixLogger
 	loadStore load.PerClusterReporter
 
@@ -235,9 +238,12 @@ var DefaultSubBalancerCloseTimeout = 15 * time.Minute
 
 // New creates a new BalancerGroup. Note that the BalancerGroup
 // needs to be started to work.
-func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore load.PerClusterReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
+//
+// TODO(easwars): Pass an options struct instead of N args.
+func New(cc balancer.ClientConn, bOpts balancer.BuildOptions, stateAggregator BalancerStateAggregator, loadStore load.PerClusterReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
 	return &BalancerGroup{
 		cc:        cc,
+		buildOpts: bOpts,
 		logger:    logger,
 		loadStore: loadStore,
 
@@ -305,6 +311,7 @@ func (bg *BalancerGroup) Add(id string, builder balancer.Builder) {
 			id:         id,
 			group:      bg,
 			builder:    builder,
+			buildOpts:  bg.buildOpts,
 		}
 		if bg.outgoingStarted {
 			// Only start the balancer if bg is started. Otherwise, we only keep the
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 0474e7c722e8..0ad4bf8df10f 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -38,6 +38,8 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	"google.golang.org/grpc/xds/internal/client/load"
@@ -74,7 +76,7 @@ func newTestBalancerGroup(t *testing.T, loadStore load.PerClusterReporter) (*tes
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
-	bg := New(cc, gator, loadStore, nil)
+	bg := New(cc, balancer.BuildOptions{}, gator, loadStore, nil)
 	bg.Start()
 	return cc, gator, bg
 }
@@ -501,7 +503,7 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
-	bg := New(cc, gator, nil, nil)
+	bg := New(cc, balancer.BuildOptions{}, gator, nil, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
@@ -590,16 +592,20 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
 // whenever it gets an address update. It's expected that start() doesn't block
 // because of deadlock.
 func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
+	const balancerName = "stub-TestBalancerGroup_start_close_deadlock"
+	stub.Register(balancerName, stub.BalancerFuncs{})
+	builder := balancer.Get(balancerName)
+
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
-	bg := New(cc, gator, nil, nil)
+	bg := New(cc, balancer.BuildOptions{}, gator, nil, nil)
 
 	gator.Add(testBalancerIDs[0], 2)
-	bg.Add(testBalancerIDs[0], &testutils.TestConstBalancerBuilder{})
+	bg.Add(testBalancerIDs[0], builder)
 	bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
 	gator.Add(testBalancerIDs[1], 1)
-	bg.Add(testBalancerIDs[1], &testutils.TestConstBalancerBuilder{})
+	bg.Add(testBalancerIDs[1], builder)
 	bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
 
 	bg.Start()
@@ -695,7 +701,7 @@ func initBalancerGroupForCachingTest(t *testing.T) (*weightedaggregator.Aggregat
 	cc := testutils.NewTestClientConn(t)
 	gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
 	gator.Start()
-	bg := New(cc, gator, nil, nil)
+	bg := New(cc, balancer.BuildOptions{}, gator, nil, nil)
 
 	// Add two balancers to group and send two resolved addresses to both
 	// balancers.
@@ -931,3 +937,43 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 		t.Fatalf("want %v, got %v", want, err)
 	}
 }
+
+// TestBalancerGroupBuildOptions verifies that the balancer.BuildOptions passed
+// to the balancergroup at creation time is passed to child policies.
+func (s) TestBalancerGroupBuildOptions(t *testing.T) {
+	const (
+		balancerName       = "stubBalancer-TestBalancerGroupBuildOptions"
+		parent             = int64(1234)
+		userAgent          = "ua"
+		defaultTestTimeout = 1 * time.Second
+	)
+
+	// Setup the stub balancer such that we can read the build options passed to
+	// it in the UpdateClientConnState method.
+	bOpts := balancer.BuildOptions{
+		DialCreds:        insecure.NewCredentials(),
+		ChannelzParentID: parent,
+		CustomUserAgent:  userAgent,
+	}
+	stub.Register(balancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			if !cmp.Equal(bd.BuildOptions, bOpts) {
+				return fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
+			}
+			return nil
+		},
+	})
+	cc := testutils.NewTestClientConn(t)
+	bg := New(cc, bOpts, nil, nil, nil)
+	bg.Start()
+
+	// Add the stub balancer build above as a child policy.
+	balancerBuilder := balancer.Get(balancerName)
+	bg.Add(testBalancerIDs[0], balancerBuilder)
+
+	// Send an empty clientConn state change. This should trigger the
+	// verification of the buildOptions being passed to the child policy.
+	if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{}); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/xds/internal/balancer/clustermanager/clustermanager.go b/xds/internal/balancer/clustermanager/clustermanager.go
index da5900ac75cf..1e4dee7f5d3a 100644
--- a/xds/internal/balancer/clustermanager/clustermanager.go
+++ b/xds/internal/balancer/clustermanager/clustermanager.go
@@ -40,12 +40,12 @@ func init() {
 
 type builder struct{}
 
-func (builder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+func (builder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
 	b := &bal{}
 	b.logger = prefixLogger(b)
 	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
 	b.stateAggregator.start()
-	b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
+	b.bg = balancergroup.New(cc, opts, b.stateAggregator, nil, b.logger)
 	b.bg.Start()
 	b.logger.Infof("Created")
 	return b
diff --git a/xds/internal/balancer/clustermanager/clustermanager_test.go b/xds/internal/balancer/clustermanager/clustermanager_test.go
index 86c377937f36..a40d954ad64f 100644
--- a/xds/internal/balancer/clustermanager/clustermanager_test.go
+++ b/xds/internal/balancer/clustermanager/clustermanager_test.go
@@ -29,8 +29,11 @@ import (
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/hierarchy"
+	itestutils "google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
@@ -510,3 +513,55 @@ func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
 		testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
 	}
 }
+
+func TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) {
+	const (
+		balancerName       = "stubBalancer-TestClusterManagerForwardsBalancerBuildOptions"
+		parent             = int64(1234)
+		userAgent          = "ua"
+		defaultTestTimeout = 1 * time.Second
+	)
+
+	// Setup the stub balancer such that we can read the build options passed to
+	// it in the UpdateClientConnState method.
+	ccsCh := itestutils.NewChannel()
+	bOpts := balancer.BuildOptions{
+		DialCreds:        insecure.NewCredentials(),
+		ChannelzParentID: parent,
+		CustomUserAgent:  userAgent,
+	}
+	stub.Register(balancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			if !cmp.Equal(bd.BuildOptions, bOpts) {
+				err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
+				ccsCh.Send(err)
+				return err
+			}
+			ccsCh.Send(nil)
+			return nil
+		},
+	})
+
+	cc := testutils.NewTestClientConn(t)
+	rtb := rtBuilder.Build(cc, bOpts)
+
+	configJSON1 := fmt.Sprintf(`{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"%s":""}] }
+}
+}`, balancerName)
+	config1, err := rtParser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	if err := rtb.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if v, err := ccsCh.Receive(ctx); err != nil {
+		err2 := v.(error)
+		t.Fatal(err2)
+	}
+}
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 12a1251abe9f..a6b37f6277a1 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -48,8 +48,8 @@ type xdsClientInterface interface {
 }
 
 var (
-	newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), lw load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
-		return newEDSBalancerImpl(cc, enqueueState, lw, logger)
+	newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions, enqueueState func(priorityType, balancer.State), lw load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+		return newEDSBalancerImpl(cc, opts, enqueueState, lw, logger)
 	}
 	newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
 )
@@ -61,7 +61,7 @@ func init() {
 type edsBalancerBuilder struct{}
 
 // Build helps implement the balancer.Builder interface.
-func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
 	x := &edsBalancer{
 		cc:                cc,
 		closed:            grpcsync.NewEvent(),
@@ -80,7 +80,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptio
 	}
 
 	x.xdsClient = client
-	x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.lsw, x.logger)
+	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.lsw, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 8c34e4b9d436..499ea5243b64 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -65,6 +66,7 @@ type balancerGroupWithConfig struct {
 // policy is used to manage endpoints in each locality.
 type edsBalancerImpl struct {
 	cc           balancer.ClientConn
+	buildOpts    balancer.BuildOptions
 	logger       *grpclog.PrefixLogger
 	loadReporter load.PerClusterReporter
 
@@ -102,9 +104,10 @@ type edsBalancerImpl struct {
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
-func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), lr load.PerClusterReporter, logger *grpclog.PrefixLogger) *edsBalancerImpl {
+func newEDSBalancerImpl(cc balancer.ClientConn, bOpts balancer.BuildOptions, enqueueState func(priorityType, balancer.State), lr load.PerClusterReporter, logger *grpclog.PrefixLogger) *edsBalancerImpl {
 	edsImpl := &edsBalancerImpl{
 		cc:                 cc,
+		buildOpts:          bOpts,
 		logger:             logger,
 		subBalancerBuilder: balancer.Get(roundrobin.Name),
 		loadReporter:       lr,
@@ -248,7 +251,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
 			ccPriorityWrapper := edsImpl.ccWrapperWithPriority(priority)
 			stateAggregator := weightedaggregator.New(ccPriorityWrapper, edsImpl.logger, newRandomWRR)
 			bgwc = &balancerGroupWithConfig{
-				bg:              balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.loadReporter, edsImpl.logger),
+				bg:              balancergroup.New(ccPriorityWrapper, edsImpl.buildOpts, stateAggregator, edsImpl.loadReporter, edsImpl.logger),
 				stateAggregator: stateAggregator,
 				configs:         make(map[internal.LocalityID]*localityConfig),
 			}
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index 1ce4f36ca73f..7696feb5bd04 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -35,7 +35,7 @@ import (
 // Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0.
 func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
@@ -101,7 +101,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
 // down, use 2; remove 2, use 1.
 func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
@@ -208,7 +208,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
 // Init 0 and 1; 0 and 1 both down; add 2, use 2.
 func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
@@ -271,7 +271,7 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
 // Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2.
 func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0,1,2], each with one backend.
@@ -353,7 +353,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 	}()()
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
@@ -403,7 +403,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
 //  - add localities to existing p0 and p1
 func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
@@ -514,7 +514,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
 	}()()
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with different priorities, each with one backend.
@@ -698,7 +698,7 @@ func (s) TestPriorityTypeEqual(t *testing.T) {
 // will be used.
 func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
@@ -757,7 +757,7 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
 // priority will be used.
 func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, with priorities [0, 1], each with one backend.
@@ -823,7 +823,7 @@ func (s) TestEDSPriority_FirstPriorityUnavailable(t *testing.T) {
 	defaultPriorityInitTimeout = testPriorityInitTimeout
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// One localities, with priorities [0], each with one backend.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index b4349a021dca..69ce6dcac1da 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/client"
@@ -61,7 +62,7 @@ func init() {
 //  - change drop rate
 func (s) TestEDS_OneLocality(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// One locality with one backend.
@@ -182,7 +183,7 @@ func (s) TestEDS_OneLocality(t *testing.T) {
 //  - update locality weight
 func (s) TestEDS_TwoLocalities(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, each with one backend.
@@ -313,7 +314,7 @@ func (s) TestEDS_TwoLocalities(t *testing.T) {
 // healthy ones are used.
 func (s) TestEDS_EndpointsHealth(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// Two localities, each 3 backend, one Healthy, one Unhealthy, one Unknown.
@@ -385,7 +386,7 @@ func (s) TestEDS_EndpointsHealth(t *testing.T) {
 }
 
 func (s) TestClose(t *testing.T) {
-	edsb := newEDSBalancerImpl(nil, nil, nil, nil)
+	edsb := newEDSBalancerImpl(nil, balancer.BuildOptions{}, nil, nil, nil)
 	// This is what could happen when switching between fallback and eds. This
 	// make sure it doesn't panic.
 	edsb.close()
@@ -396,7 +397,7 @@ func (s) TestClose(t *testing.T) {
 // It should send an error picker with transient failure to the parent.
 func (s) TestEDS_EmptyUpdate(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// The first update is an empty update.
@@ -456,15 +457,33 @@ func (s) TestEDS_EmptyUpdate(t *testing.T) {
 }
 
 // Create XDS balancer, and update sub-balancer before handling eds responses.
-// Then switch between round-robin and test-const-balancer after handling first
+// Then switch between round-robin and a test stub-balancer after handling first
 // eds response.
 func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
+	const balancerName = "stubBalancer-TestEDS_UpdateSubBalancerName"
+
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	stub.Register(balancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, s balancer.ClientConnState) error {
+			if len(s.ResolverState.Addresses) == 0 {
+				return nil
+			}
+			bd.ClientConn.NewSubConn(s.ResolverState.Addresses, balancer.NewSubConnOptions{})
+			return nil
+		},
+		UpdateSubConnState: func(bd *stub.BalancerData, sc balancer.SubConn, state balancer.SubConnState) {
+			bd.ClientConn.UpdateState(balancer.State{
+				ConnectivityState: state.ConnectivityState,
+				Picker:            &testutils.TestConstPicker{Err: testutils.ErrTestConstPicker},
+			})
+		},
+	})
+
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
-	t.Logf("update sub-balancer to test-const-balancer")
-	edsb.handleChildPolicy("test-const-balancer", nil)
+	t.Logf("update sub-balancer to stub-balancer")
+	edsb.handleChildPolicy(balancerName, nil)
 
 	// Two localities, each with one backend.
 	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
@@ -506,8 +525,8 @@ func (s) TestEDS_UpdateSubBalancerName(t *testing.T) {
 		t.Fatalf("want %v, got %v", want, err)
 	}
 
-	t.Logf("update sub-balancer to test-const-balancer")
-	edsb.handleChildPolicy("test-const-balancer", nil)
+	t.Logf("update sub-balancer to stub-balancer")
+	edsb.handleChildPolicy(balancerName, nil)
 
 	for i := 0; i < 2; i++ {
 		scToRemove := <-cc.RemoveSubConnCh
@@ -558,7 +577,7 @@ func (s) TestEDS_CircuitBreaking(t *testing.T) {
 	defer func() { env.CircuitBreakingSupport = origCircuitBreakingSupport }()
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 	edsb.updateServiceRequestsCounter("test")
 	var maxRequests uint32 = 50
@@ -658,7 +677,7 @@ func (*testInlineUpdateBalancer) Close() {
 // by acquiring a locked mutex.
 func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, nil, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = func(p priorityType, state balancer.State) {
 		// For this test, euqueue needs to happen asynchronously (like in the
 		// real implementation).
@@ -759,7 +778,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	lsWrapper.updateLoadStore(loadStore)
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, lsWrapper, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, lsWrapper, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	const (
@@ -853,7 +872,7 @@ func (s) TestEDS_LoadReportDisabled(t *testing.T) {
 	// Not calling lsWrapper.updateLoadStore(loadStore) because LRS is disabled.
 
 	cc := testutils.NewTestClientConn(t)
-	edsb := newEDSBalancerImpl(cc, nil, lsWrapper, nil)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, lsWrapper, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
 
 	// One localities, with one backend.
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 3ee19a00debe..ea5ec39c4568 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -221,7 +221,7 @@ func setup(edsLBCh *testutils.Channel) (*fakeclient.Client, func()) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 
 	origNewEDSBalancer := newEDSBalancer
-	newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), _ load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
+	newEDSBalancer = func(cc balancer.ClientConn, _ balancer.BuildOptions, _ func(priorityType, balancer.State), _ load.PerClusterReporter, _ *grpclog.PrefixLogger) edsBalancerImplInterface {
 		edsLB := newFakeEDSBalancer(cc)
 		defer func() { edsLBCh.Send(edsLB) }()
 		return edsLB
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index b0310660fb29..02b199258cd2 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -45,12 +45,12 @@ func init() {
 
 type weightedTargetBB struct{}
 
-func (wt *weightedTargetBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+func (wt *weightedTargetBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
 	b := &weightedTargetBalancer{}
 	b.logger = prefixLogger(b)
 	b.stateAggregator = weightedaggregator.New(cc, b.logger, newRandomWRR)
 	b.stateAggregator.Start()
-	b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
+	b.bg = balancergroup.New(cc, bOpts, b.stateAggregator, nil, b.logger)
 	b.bg.Start()
 	b.logger.Infof("Created")
 	return b
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index de34da819639..85de8d584a4f 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -143,7 +143,7 @@ func (v3c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
 		return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err)
 	}
 	v3c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
-	v3c.logger.Debugf("ADS response received: %v", resp)
+	v3c.logger.Debugf("ADS response received: %+v", resp)
 	return resp, nil
 }
 
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 69e355e353c2..f49ba85606ff 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -232,49 +232,9 @@ func (tc *testClosure) next() balancer.SubConn {
 	return ret
 }
 
-func init() {
-	balancer.Register(&TestConstBalancerBuilder{})
-}
-
 // ErrTestConstPicker is error returned by test const picker.
 var ErrTestConstPicker = fmt.Errorf("const picker error")
 
-// TestConstBalancerBuilder is a balancer builder for tests.
-type TestConstBalancerBuilder struct{}
-
-// Build builds a test const balancer.
-func (*TestConstBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
-	return &testConstBalancer{cc: cc}
-}
-
-// Name returns test-const-balancer name.
-func (*TestConstBalancerBuilder) Name() string {
-	return "test-const-balancer"
-}
-
-type testConstBalancer struct {
-	cc balancer.ClientConn
-}
-
-func (tb *testConstBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
-	tb.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &TestConstPicker{Err: ErrTestConstPicker}})
-}
-
-func (tb *testConstBalancer) ResolverError(error) {
-	panic("not implemented")
-}
-
-func (tb *testConstBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	if len(s.ResolverState.Addresses) == 0 {
-		return nil
-	}
-	tb.cc.NewSubConn(s.ResolverState.Addresses, balancer.NewSubConnOptions{})
-	return nil
-}
-
-func (*testConstBalancer) Close() {
-}
-
 // TestConstPicker is a const picker for tests.
 type TestConstPicker struct {
 	Err error

From 61962d0e8e4ee228bd2ec73e02b19f90785310d0 Mon Sep 17 00:00:00 2001
From: Gaurav Gahlot <gauravgahlot0107@gmail.com>
Date: Tue, 9 Feb 2021 23:17:07 +0530
Subject: [PATCH 358/481] status: document nil error handling of FromError
 (#4196)

Signed-off-by: Gaurav Gahlot <gauravgahlot0107@gmail.com>
---
 status/status.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/status/status.go b/status/status.go
index 01e182c306c2..54d187186b8f 100644
--- a/status/status.go
+++ b/status/status.go
@@ -73,9 +73,11 @@ func FromProto(s *spb.Status) *Status {
 	return status.FromProto(s)
 }
 
-// FromError returns a Status representing err if it was produced from this
-// package or has a method `GRPCStatus() *Status`. Otherwise, ok is false and a
-// Status is returned with codes.Unknown and the original error message.
+// FromError returns a Status representing err if it was produced by this
+// package or has a method `GRPCStatus() *Status`.
+// If err is nil, a Status is returned with codes.OK and no message.
+// Otherwise, ok is false and a Status is returned with codes.Unknown and
+// the original error message.
 func FromError(err error) (s *Status, ok bool) {
 	if err == nil {
 		return nil, true

From ce29c77c5fa13ec2b424e310e84dfa3188618df8 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 10 Feb 2021 09:13:50 -0800
Subject: [PATCH 359/481] Change version to 1.37.0-dev (#4200)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index 1b9f3715952d..c149b22ad8a1 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.36.0-dev"
+const Version = "1.37.0-dev"

From ad24ab52b1624d64554867408e11e74d546829ea Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 10 Feb 2021 10:38:04 -0800
Subject: [PATCH 360/481] priority: the implementation (#4070)

---
 xds/internal/balancer/priority/balancer.go    |  241 +++
 .../balancer/priority/balancer_child.go       |  104 ++
 .../balancer/priority/balancer_priority.go    |  354 ++++
 .../balancer/priority/balancer_test.go        | 1618 +++++++++++++++++
 xds/internal/balancer/priority/logging.go     |   34 +
 .../balancer/priority/{doc.go => utils.go}    |   19 +-
 xds/internal/balancer/priority/utils_test.go  |   62 +
 xds/internal/testutils/balancer.go            |    2 +-
 8 files changed, 2427 insertions(+), 7 deletions(-)
 create mode 100644 xds/internal/balancer/priority/balancer.go
 create mode 100644 xds/internal/balancer/priority/balancer_child.go
 create mode 100644 xds/internal/balancer/priority/balancer_priority.go
 create mode 100644 xds/internal/balancer/priority/balancer_test.go
 create mode 100644 xds/internal/balancer/priority/logging.go
 rename xds/internal/balancer/priority/{doc.go => utils.go} (68%)
 create mode 100644 xds/internal/balancer/priority/utils_test.go

diff --git a/xds/internal/balancer/priority/balancer.go b/xds/internal/balancer/priority/balancer.go
new file mode 100644
index 000000000000..c1b761972c55
--- /dev/null
+++ b/xds/internal/balancer/priority/balancer.go
@@ -0,0 +1,241 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package priority implements the priority balancer.
+//
+// This balancer will be kept in internal until we use it in the xds balancers,
+// and are confident its functionalities are stable. It will then be exported
+// for more users.
+package priority
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+)
+
+const priorityBalancerName = "priority_experimental"
+
+func init() {
+	balancer.Register(priorityBB{})
+}
+
+type priorityBB struct{}
+
+func (priorityBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+	b := &priorityBalancer{
+		cc:                       cc,
+		done:                     grpcsync.NewEvent(),
+		childToPriority:          make(map[string]int),
+		children:                 make(map[string]*childBalancer),
+		childBalancerStateUpdate: buffer.NewUnbounded(),
+	}
+
+	b.logger = prefixLogger(b)
+	b.bg = balancergroup.New(cc, b, nil, b.logger)
+	b.bg.Start()
+	go b.run()
+	b.logger.Infof("Created")
+	return b
+
+}
+
+func (priorityBB) Name() string {
+	return priorityBalancerName
+}
+
+// timerWrapper wraps a timer with a boolean. So that when a race happens
+// between AfterFunc and Stop, the func is guaranteed to not execute.
+type timerWrapper struct {
+	stopped bool
+	timer   *time.Timer
+}
+
+type priorityBalancer struct {
+	logger                   *grpclog.PrefixLogger
+	cc                       balancer.ClientConn
+	bg                       *balancergroup.BalancerGroup
+	done                     *grpcsync.Event
+	childBalancerStateUpdate *buffer.Unbounded
+
+	mu         sync.Mutex
+	childInUse string
+	// priority of the child that's current in use. Int starting from 0, and 0
+	// is the higher priority.
+	priorityInUse int
+	// priorities is a list of child names from higher to lower priority.
+	priorities []string
+	// childToPriority is a map from the child name to it's priority. Priority
+	// is an int start from 0, and 0 is the higher priority.
+	childToPriority map[string]int
+	// children is a map from child name to sub-balancers.
+	children map[string]*childBalancer
+	// The timer to give a priority some time to connect. And if the priority
+	// doesn't go into Ready/Failure, the next priority will be started.
+	//
+	// One timer is enough because there can be at most one priority in init
+	// state.
+	priorityInitTimer *timerWrapper
+}
+
+func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	// Create and remove children, since we know all children from the config
+	// are used by some priority.
+	for name, newSubConfig := range newConfig.Children {
+		bb := balancer.Get(newSubConfig.Config.Name)
+		if bb == nil {
+			b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name)
+			continue
+		}
+
+		currentChild, ok := b.children[name]
+		if !ok {
+			// This is a new child, add it to the children list. But note that
+			// the balancer isn't built, because this child can be a low
+			// priority. If necessary, it will be built when syncing priorities.
+			cb := newChildBalancer(name, b, bb)
+			cb.updateConfig(newSubConfig.Config.Config, resolver.State{
+				Addresses:     addressesSplit[name],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			})
+			b.children[name] = cb
+			continue
+		}
+
+		// This is not a new child. But the config/addresses could change.
+
+		// The balancing policy name is changed, close the old child. But don't
+		// rebuild, rebuild will happen when syncing priorities.
+		if currentChild.bb.Name() != bb.Name() {
+			currentChild.stop()
+			currentChild.bb = bb
+		}
+
+		// Update config and address, but note that this doesn't send the
+		// updates to child balancer (the child balancer might not be built, if
+		// it's a low priority).
+		currentChild.updateConfig(newSubConfig.Config.Config, resolver.State{
+			Addresses:     addressesSplit[name],
+			ServiceConfig: s.ResolverState.ServiceConfig,
+			Attributes:    s.ResolverState.Attributes,
+		})
+	}
+
+	// Remove child from children if it's not in new config.
+	for name, oldChild := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			oldChild.stop()
+		}
+	}
+
+	// Update priorities and handle priority changes.
+	b.priorities = newConfig.Priorities
+	b.childToPriority = make(map[string]int, len(newConfig.Priorities))
+	for pi, pName := range newConfig.Priorities {
+		b.childToPriority[pName] = pi
+	}
+	// Sync the states of all children to the new updated priorities. This
+	// include starting/stopping child balancers when necessary.
+	b.syncPriority()
+
+	return nil
+}
+
+func (b *priorityBalancer) ResolverError(err error) {
+	b.bg.ResolverError(err)
+}
+
+func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.bg.UpdateSubConnState(sc, state)
+}
+
+func (b *priorityBalancer) Close() {
+	b.bg.Close()
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.done.Fire()
+	// Clear states of the current child in use, so if there's a race in picker
+	// update, it will be dropped.
+	b.childInUse = ""
+	b.stopPriorityInitTimer()
+}
+
+// stopPriorityInitTimer stops the priorityInitTimer if it's not nil, and set it
+// to nil.
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) stopPriorityInitTimer() {
+	timerW := b.priorityInitTimer
+	if timerW == nil {
+		return
+	}
+	b.priorityInitTimer = nil
+	timerW.stopped = true
+	timerW.timer.Stop()
+}
+
+// UpdateState implements balancergroup.BalancerStateAggregator interface. The
+// balancer group sends new connectivity state and picker here.
+func (b *priorityBalancer) UpdateState(childName string, state balancer.State) {
+	b.childBalancerStateUpdate.Put(&childBalancerState{
+		name: childName,
+		s:    state,
+	})
+}
+
+type childBalancerState struct {
+	name string
+	s    balancer.State
+}
+
+// run handles child update in a separate goroutine, so if the child sends
+// updates inline (when called by parent), it won't cause deadlocks (by trying
+// to hold the same mutex).
+func (b *priorityBalancer) run() {
+	for {
+		select {
+		case u := <-b.childBalancerStateUpdate.Get():
+			b.childBalancerStateUpdate.Load()
+			s := u.(*childBalancerState)
+			// Needs to handle state update in a goroutine, because each state
+			// update needs to start/close child policy, could result in
+			// deadlock.
+			b.handleChildStateUpdate(s.name, s.s)
+		case <-b.done.Done():
+			return
+		}
+	}
+}
diff --git a/xds/internal/balancer/priority/balancer_child.go b/xds/internal/balancer/priority/balancer_child.go
new file mode 100644
index 000000000000..d012ad4e4593
--- /dev/null
+++ b/xds/internal/balancer/priority/balancer_child.go
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type childBalancer struct {
+	name   string
+	parent *priorityBalancer
+	bb     balancer.Builder
+
+	config serviceconfig.LoadBalancingConfig
+	rState resolver.State
+
+	started bool
+	state   balancer.State
+}
+
+// newChildBalancer creates a child balancer place holder, but doesn't
+// build/start the child balancer.
+func newChildBalancer(name string, parent *priorityBalancer, bb balancer.Builder) *childBalancer {
+	return &childBalancer{
+		name:    name,
+		parent:  parent,
+		bb:      bb,
+		started: false,
+		// Start with the connecting state and picker with re-pick error, so
+		// that when a priority switch causes this child picked before it's
+		// balancing policy is created, a re-pick will happen.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+	}
+}
+
+// updateConfig sets childBalancer's config and state, but doesn't send update to
+// the child balancer.
+func (cb *childBalancer) updateConfig(config serviceconfig.LoadBalancingConfig, rState resolver.State) {
+	cb.config = config
+	cb.rState = rState
+}
+
+// start builds the child balancer if it's not already started.
+//
+// It doesn't do it directly. It asks the balancer group to build it.
+func (cb *childBalancer) start() {
+	if cb.started {
+		return
+	}
+	cb.started = true
+	cb.parent.bg.Add(cb.name, cb.bb)
+}
+
+// sendUpdate sends the addresses and config to the child balancer.
+func (cb *childBalancer) sendUpdate() {
+	// TODO: return and aggregate the returned error in the parent.
+	err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{
+		ResolverState:  cb.rState,
+		BalancerConfig: cb.config,
+	})
+	if err != nil {
+		cb.parent.logger.Warningf("failed to update ClientConn state for child %v: %v", cb.name, err)
+	}
+}
+
+// stop stops the child balancer and resets the state.
+//
+// It doesn't do it directly. It asks the balancer group to remove it.
+//
+// Note that the underlying balancer group could keep the child in a cache.
+func (cb *childBalancer) stop() {
+	if !cb.started {
+		return
+	}
+	cb.parent.bg.Remove(cb.name)
+	cb.started = false
+	cb.state = balancer.State{
+		ConnectivityState: connectivity.Connecting,
+		Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+	}
+}
diff --git a/xds/internal/balancer/priority/balancer_priority.go b/xds/internal/balancer/priority/balancer_priority.go
new file mode 100644
index 000000000000..ea2f4f04184c
--- /dev/null
+++ b/xds/internal/balancer/priority/balancer_priority.go
@@ -0,0 +1,354 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"errors"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+)
+
+var (
+	errAllPrioritiesRemoved    = errors.New("no locality is provided, all priorities are removed")
+	defaultPriorityInitTimeout = 10 * time.Second
+)
+
+// syncPriority handles priority after a config update. It makes sure the
+// balancer state (started or not) is in sync with the priorities (even in
+// tricky cases where a child is moved from a priority to another).
+//
+// It's guaranteed that after this function returns:
+// - If some child is READY, it is childInUse, and all lower priorities are
+// closed.
+// - If some child is newly started(in Connecting for the first time), it is
+// childInUse, and all lower priorities are closed.
+// - Otherwise, the lowest priority is childInUse (none of the children is
+// ready, and the overall state is not ready).
+//
+// Steps:
+// - If all priorities were deleted, unset childInUse (to an empty string), and
+// set parent ClientConn to TransientFailure
+// - Otherwise, Scan all children from p0, and check balancer stats:
+//   - For any of the following cases:
+// 	   - If balancer is not started (not built), this is either a new child
+//       with high priority, or a new builder for an existing child.
+// 	   - If balancer is READY
+// 	   - If this is the lowest priority
+//   - do the following:
+//     - if this is not the old childInUse, override picker so old picker is no
+//       longer used.
+//     - switch to it (because all higher priorities are neither new or Ready)
+//     - forward the new addresses and config
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) syncPriority() {
+	// Everything was removed by the update.
+	if len(b.priorities) == 0 {
+		b.childInUse = ""
+		b.priorityInUse = 0
+		// Stop the init timer. This can happen if the only priority is removed
+		// shortly after it's added.
+		b.stopPriorityInitTimer()
+		b.cc.UpdateState(balancer.State{
+			ConnectivityState: connectivity.TransientFailure,
+			Picker:            base.NewErrPicker(errAllPrioritiesRemoved),
+		})
+		return
+	}
+
+	for p, name := range b.priorities {
+		child, ok := b.children[name]
+		if !ok {
+			b.logger.Errorf("child with name %q is not found in children", name)
+			continue
+		}
+
+		if !child.started ||
+			child.state.ConnectivityState == connectivity.Ready ||
+			p == len(b.priorities)-1 {
+			if b.childInUse != "" && b.childInUse != child.name {
+				// childInUse was set and is different from this child, will
+				// change childInUse later. We need to update picker here
+				// immediately so parent stops using the old picker.
+				b.cc.UpdateState(child.state)
+			}
+			b.logger.Infof("switching to (%q, %v) in syncPriority", child.name, p)
+			b.switchToChild(child, p)
+			child.sendUpdate()
+			break
+		}
+	}
+}
+
+// Stop priorities [p+1, lowest].
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) stopSubBalancersLowerThanPriority(p int) {
+	for i := p + 1; i < len(b.priorities); i++ {
+		name := b.priorities[i]
+		child, ok := b.children[name]
+		if !ok {
+			b.logger.Errorf("child with name %q is not found in children", name)
+			continue
+		}
+		child.stop()
+	}
+}
+
+// switchToChild does the following:
+// - stop all child with lower priorities
+// - if childInUse is not this child
+//   - set childInUse to this child
+//   - stops init timer
+//   - if this child is not started, start it, and start a init timer
+//
+// Note that it does NOT send the current child state (picker) to the parent
+// ClientConn. The caller needs to send it if necessary.
+//
+// this can be called when
+// 1. first update, start p0
+// 2. an update moves a READY child from a lower priority to higher
+// 2. a different builder is updated for this child
+// 3. a high priority goes Failure, start next
+// 4. a high priority init timeout, start next
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) switchToChild(child *childBalancer, priority int) {
+	// Stop lower priorities even if childInUse is same as this child. It's
+	// possible this child was moved from a priority to another.
+	b.stopSubBalancersLowerThanPriority(priority)
+
+	// If this child is already in use, do nothing.
+	//
+	// This can happen:
+	// - all priorities are not READY, an config update always triggers switch
+	// to the lowest. In this case, the lowest child could still be connecting,
+	// so we don't stop the init timer.
+	// - a high priority is READY, an config update always triggers switch to
+	// it.
+	if b.childInUse == child.name && child.started {
+		return
+	}
+	b.childInUse = child.name
+	b.priorityInUse = priority
+
+	// Init timer is always for childInUse. Since we are switching to a
+	// different child, we will stop the init timer no matter what. If this
+	// child is not started, we will start the init timer later.
+	b.stopPriorityInitTimer()
+
+	if !child.started {
+		child.start()
+		// Need this local variable to capture timerW in the AfterFunc closure
+		// to check the stopped boolean.
+		timerW := &timerWrapper{}
+		b.priorityInitTimer = timerW
+		timerW.timer = time.AfterFunc(defaultPriorityInitTimeout, func() {
+			b.mu.Lock()
+			defer b.mu.Unlock()
+			if timerW.stopped {
+				return
+			}
+			b.priorityInitTimer = nil
+			// Switch to the next priority if there's any.
+			if pNext := priority + 1; pNext < len(b.priorities) {
+				nameNext := b.priorities[pNext]
+				if childNext, ok := b.children[nameNext]; ok {
+					b.switchToChild(childNext, pNext)
+					childNext.sendUpdate()
+				}
+			}
+		})
+	}
+}
+
+// handleChildStateUpdate start/close priorities based on the connectivity
+// state.
+func (b *priorityBalancer) handleChildStateUpdate(childName string, s balancer.State) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.done.HasFired() {
+		return
+	}
+
+	priority, ok := b.childToPriority[childName]
+	if !ok {
+		b.logger.Errorf("priority: received picker update with unknown child %v", childName)
+		return
+	}
+
+	if b.childInUse == "" {
+		b.logger.Errorf("priority: no child is in use when picker update is received")
+		return
+	}
+
+	// priorityInUse is higher than this priority.
+	if b.priorityInUse < priority {
+		// Lower priorities should all be closed, this is an unexpected update.
+		// Can happen if the child policy sends an update after we tell it to
+		// close.
+		b.logger.Warningf("priority: received picker update from priority %v,  lower than priority in use %v", priority, b.priorityInUse)
+		return
+	}
+
+	// Update state in child. The updated picker will be sent to parent later if
+	// necessary.
+	child, ok := b.children[childName]
+	if !ok {
+		b.logger.Errorf("priority: child balancer not found for child %v, priority %v", childName, priority)
+		return
+	}
+	oldState := child.state.ConnectivityState
+	child.state = s
+
+	switch s.ConnectivityState {
+	case connectivity.Ready:
+		b.handlePriorityWithNewStateReady(child, priority)
+	case connectivity.TransientFailure:
+		b.handlePriorityWithNewStateTransientFailure(child, priority)
+	case connectivity.Connecting:
+		b.handlePriorityWithNewStateConnecting(child, priority, oldState)
+	default:
+		// New state is Idle, should never happen. Don't forward.
+	}
+}
+
+// handlePriorityWithNewStateReady handles state Ready from a higher or equal
+// priority.
+//
+// An update with state Ready:
+// - If it's from higher priority:
+//   - Switch to this priority
+//   - Forward the update
+// - If it's from priorityInUse:
+//   - Forward only
+//
+// Caller must make sure priorityInUse is not higher than priority.
+//
+// Caller must hold mu.
+func (b *priorityBalancer) handlePriorityWithNewStateReady(child *childBalancer, priority int) {
+	// If one priority higher or equal to priorityInUse goes Ready, stop the
+	// init timer. If update is from higher than priorityInUse, priorityInUse
+	// will be closed, and the init timer will become useless.
+	b.stopPriorityInitTimer()
+
+	// priorityInUse is lower than this priority, switch to this.
+	if b.priorityInUse > priority {
+		b.logger.Infof("Switching priority from %v to %v, because latter became Ready", b.priorityInUse, priority)
+		b.switchToChild(child, priority)
+	}
+	// Forward the update since it's READY.
+	b.cc.UpdateState(child.state)
+}
+
+// handlePriorityWithNewStateTransientFailure handles state TransientFailure
+// from a higher or equal priority.
+//
+// An update with state TransientFailure:
+// - If it's from a higher priority:
+//   - Do not forward, and do nothing
+// - If it's from priorityInUse:
+//   - If there's no lower:
+//     - Forward and do nothing else
+//   - If there's a lower priority:
+//     - Switch to the lower
+//     - Forward the lower child's state
+//     - Do NOT forward this update
+//
+// Caller must make sure priorityInUse is not higher than priority.
+//
+// Caller must hold mu.
+func (b *priorityBalancer) handlePriorityWithNewStateTransientFailure(child *childBalancer, priority int) {
+	// priorityInUse is lower than this priority, do nothing.
+	if b.priorityInUse > priority {
+		return
+	}
+	// priorityInUse sends a failure. Stop its init timer.
+	b.stopPriorityInitTimer()
+	priorityNext := priority + 1
+	if priorityNext >= len(b.priorities) {
+		// Forward this update.
+		b.cc.UpdateState(child.state)
+		return
+	}
+	b.logger.Infof("Switching priority from %v to %v, because former became TransientFailure", priority, priorityNext)
+	nameNext := b.priorities[priorityNext]
+	childNext := b.children[nameNext]
+	b.switchToChild(childNext, priorityNext)
+	b.cc.UpdateState(childNext.state)
+	childNext.sendUpdate()
+}
+
+// handlePriorityWithNewStateConnecting handles state Connecting from a higher
+// than or equal priority.
+//
+// An update with state Connecting:
+// - If it's from a higher priority
+//   - Do nothing
+// - If it's from priorityInUse, the behavior depends on previous state.
+//
+// When new state is Connecting, the behavior depends on previous state. If the
+// previous state was Ready, this is a transition out from Ready to Connecting.
+// Assuming there are multiple backends in the same priority, this mean we are
+// in a bad situation and we should failover to the next priority (Side note:
+// the current connectivity state aggregating algorithm (e.g. round-robin) is
+// not handling this right, because if many backends all go from Ready to
+// Connecting, the overall situation is more like TransientFailure, not
+// Connecting).
+//
+// If the previous state was Idle, we don't do anything special with failure,
+// and simply forward the update. The init timer should be in process, will
+// handle failover if it timeouts. If the previous state was TransientFailure,
+// we do not forward, because the lower priority is in use.
+//
+// Caller must make sure priorityInUse is not higher than priority.
+//
+// Caller must hold mu.
+func (b *priorityBalancer) handlePriorityWithNewStateConnecting(child *childBalancer, priority int, oldState connectivity.State) {
+	// priorityInUse is lower than this priority, do nothing.
+	if b.priorityInUse > priority {
+		return
+	}
+
+	switch oldState {
+	case connectivity.Ready:
+		// Handling transition from Ready to Connecting, is same as handling
+		// TransientFailure. There's no need to stop the init timer, because it
+		// should have been stopped when state turned Ready.
+		priorityNext := priority + 1
+		if priorityNext >= len(b.priorities) {
+			// Forward this update.
+			b.cc.UpdateState(child.state)
+			return
+		}
+		b.logger.Infof("Switching priority from %v to %v, because former became TransientFailure", priority, priorityNext)
+		nameNext := b.priorities[priorityNext]
+		childNext := b.children[nameNext]
+		b.switchToChild(childNext, priorityNext)
+		b.cc.UpdateState(childNext.state)
+		childNext.sendUpdate()
+	case connectivity.Idle:
+		b.cc.UpdateState(child.state)
+	default:
+		// Old state is Connecting, TransientFailure or Shutdown. Don't forward.
+	}
+}
diff --git a/xds/internal/balancer/priority/balancer_test.go b/xds/internal/balancer/priority/balancer_test.go
new file mode 100644
index 000000000000..be14231dcb3f
--- /dev/null
+++ b/xds/internal/balancer/priority/balancer_test.go
@@ -0,0 +1,1618 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/hierarchy"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+var testBackendAddrStrs []string
+
+const (
+	testBackendAddrsCount = 12
+	testRRBalancerName    = "another-round-robin"
+)
+
+type anotherRR struct {
+	balancer.Builder
+}
+
+func (*anotherRR) Name() string {
+	return testRRBalancerName
+}
+
+func init() {
+	for i := 0; i < testBackendAddrsCount; i++ {
+		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
+	}
+	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond
+	balancer.Register(&anotherRR{Builder: balancer.Get(roundrobin.Name)})
+}
+
+func subConnFromPicker(t *testing.T, p balancer.Picker) func() balancer.SubConn {
+	return func() balancer.SubConn {
+		scst, err := p.Pick(balancer.PickInfo{})
+		if err != nil {
+			t.Fatalf("unexpected error from picker.Pick: %v", err)
+		}
+		return scst.SubConn
+	}
+}
+
+// When a high priority is ready, adding/removing lower locality doesn't cause
+// changes.
+//
+// Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0.
+func (s) TestPriority_HighPriorityReady(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two children, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Add p2, it shouldn't cause any updates.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1", "child-2"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("got unexpected new picker")
+	case sc := <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn: %s", sc)
+	case sc := <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn: %v", sc)
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// Remove p2, no updates.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("got unexpected new picker")
+	case <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn")
+	case <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn")
+	case <-time.After(time.Millisecond * 100):
+	}
+}
+
+// Lower priority is used when higher priority is not ready.
+//
+// Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is
+// down, use 2; remove 2, use 1.
+func (s) TestPriority_SwitchPriority(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two localities, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p0 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc0}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p0)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Turn down 0, will start and use 1.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := p1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	// Handle SubConn creation from 1.
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with 1.
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+	}
+
+	// Add p2, it shouldn't cause any udpates.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1", "child-2"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("got unexpected new picker")
+	case sc := <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn, %s", sc)
+	case <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn")
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// Turn down 1, use 2
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	// Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	p3 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := p3.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with 2.
+	p4 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p4.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
+		}
+	}
+
+	// Remove 2, use 1.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// p2 SubConns are removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
+	}
+
+	// Should get an update with 1's old transient failure picker, to override
+	// 2's old picker.
+	p5 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := p5.Pick(balancer.PickInfo{}); err == nil {
+			t.Fatalf("want pick error non-nil, got nil")
+		}
+	}
+
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	p6 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p6.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
+		}
+	}
+}
+
+// Lower priority is used when higher priority turns Connecting from Ready.
+// Because changing from Ready to Connecting is a failure.
+//
+// Init 0 and 1; 0 is up, use 0; 0 is connecting, 1 is up, use 1; 0 is ready,
+// use 0.
+func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two localities, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p0 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc0}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p0)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Turn 0 to Connecting, will start and use 1. Because 0 changing from Ready
+	// to Connecting is a failure.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := p1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	// Handle SubConn creation from 1.
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with 1.
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+	}
+
+	// Turn 0 back to Ready.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// p1 subconn should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc0, scToRemove)
+	}
+
+	p3 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p3.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0)
+		}
+	}
+}
+
+// Add a lower priority while the higher priority is down.
+//
+// Init 0 and 1; 0 and 1 both down; add 2, use 2.
+func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two localities, with different priorities, each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// Turn down 0, 1 is used.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	pFail0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail0.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+	// Turn down 1, pick should error.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	// Test pick failure.
+	pFail1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail1.Pick(balancer.PickInfo{}); err == nil {
+			t.Fatalf("want pick error non-nil, got nil")
+		}
+	}
+
+	// Add p2, it should create a new SubConn.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1", "child-2"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// A new connecting picker should be updated for the new priority.
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := p0.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with 2.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
+		}
+	}
+}
+
+// When a higher priority becomes available, all lower priorities are closed.
+//
+// Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2.
+func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) {
+	// defer time.Sleep(10 * time.Millisecond)
+
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Three localities, with priorities [0,1,2], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1", "child-2"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// Turn down 0, 1 is used.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	pFail0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail0.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	// Turn down 1, 2 is used.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	// Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	pFail1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with 2.
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
+		}
+	}
+
+	// When 0 becomes ready, 0 should be used, 1 and 2 should all be closed.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// sc1 and sc2 should be removed.
+	//
+	// With localities caching, the lower priorities are closed after a timeout,
+	// in goroutines. The order is no longer guaranteed.
+	scToRemove := []balancer.SubConn{<-cc.RemoveSubConnCh, <-cc.RemoveSubConnCh}
+	if !(cmp.Equal(scToRemove[0], sc1, cmp.AllowUnexported(testutils.TestSubConn{})) &&
+		cmp.Equal(scToRemove[1], sc2, cmp.AllowUnexported(testutils.TestSubConn{}))) &&
+		!(cmp.Equal(scToRemove[0], sc2, cmp.AllowUnexported(testutils.TestSubConn{})) &&
+			cmp.Equal(scToRemove[1], sc1, cmp.AllowUnexported(testutils.TestSubConn{}))) {
+		t.Errorf("RemoveSubConn, want [%v, %v], got %v", sc1, sc2, scToRemove)
+	}
+
+	// Test pick with 0.
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p0.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0)
+		}
+	}
+}
+
+// At init, start the next lower priority after timeout if the higher priority
+// doesn't get ready.
+//
+// Init 0,1; 0 is not ready (in connecting), after timeout, use 1.
+func (s) TestPriority_InitTimeout(t *testing.T) {
+	const testPriorityInitTimeout = time.Second
+	defer func() func() {
+		old := defaultPriorityInitTimeout
+		defaultPriorityInitTimeout = testPriorityInitTimeout
+		return func() {
+			defaultPriorityInitTimeout = old
+		}
+	}()()
+
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two localities, with different priorities, each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// Keep 0 in connecting, 1 will be used after init timeout.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+
+	// Make sure new SubConn is created before timeout.
+	select {
+	case <-time.After(testPriorityInitTimeout * 3 / 4):
+	case <-cc.NewSubConnAddrsCh:
+		t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout")
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with 1.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+	}
+}
+
+// EDS removes all priorities, and re-adds them.
+func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
+	const testPriorityInitTimeout = time.Second
+	defer func() func() {
+		old := defaultPriorityInitTimeout
+		defaultPriorityInitTimeout = testPriorityInitTimeout
+		return func() {
+			defaultPriorityInitTimeout = old
+		}
+	}()()
+
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two localities, with different priorities, each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p0 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc0}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p0)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Remove all priorities.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: nil,
+		},
+		BalancerConfig: &lbConfig{
+			Children:   nil,
+			Priorities: nil,
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// p0 subconn should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc0, scToRemove)
+	}
+
+	// Test pick return TransientFailure.
+	pFail := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail.Pick(balancer.PickInfo{}); err != errAllPrioritiesRemoved {
+			t.Fatalf("want pick error %v, got %v", errAllPrioritiesRemoved, err)
+		}
+	}
+
+	// Re-add two localities, with previous priorities, but different backends.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[3]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs01 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs01[0].Addr, testBackendAddrStrs[2]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc01 := <-cc.NewSubConnCh
+
+	// Don't send any update to p0, so to not override the old state of p0.
+	// Later, connect to p1 and then remove p1. This will fallback to p0, and
+	// will send p0's old picker if they are not correctly removed.
+
+	// p1 will be used after priority init timeout.
+	addrs11 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs11[0].Addr, testBackendAddrStrs[3]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc11 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc11, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc11, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p1 subconns.
+	p1 := <-cc.NewPickerCh
+	want = []balancer.SubConn{sc11}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Remove p1, to fallback to p0.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// p1 subconn should be removed.
+	scToRemove1 := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove1, sc11, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc11, scToRemove1)
+	}
+
+	// Test pick return NoSubConn.
+	pFail1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if scst, err := pFail1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error _, %v, got %v, _ ,%v", balancer.ErrNoSubConnAvailable, scst, err)
+		}
+	}
+
+	// Send an ready update for the p0 sc that was received when re-adding
+	// priorities.
+	pb.UpdateSubConnState(sc01, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc01, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p2 := <-cc.NewPickerCh
+	want = []balancer.SubConn{sc01}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p2)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("got unexpected new picker")
+	case <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn")
+	case <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn")
+	case <-time.After(time.Millisecond * 100):
+	}
+}
+
+// Test the case where the high priority contains no backends. The low priority
+// will be used.
+func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two localities, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Remove addresses from priority 0, should use p1.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// p0 will remove the subconn, and ClientConn will send a sc update to
+	// shutdown.
+	scToRemove := <-cc.RemoveSubConnCh
+	pb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.NewSubConnCh
+
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	pFail1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	// p1 is ready.
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p1 subconns.
+	p2 := <-cc.NewPickerCh
+	want = []balancer.SubConn{sc2}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p2)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+}
+
+// Test the case where the first and only priority is removed.
+func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
+	const testPriorityInitTimeout = time.Second
+	defer func(t time.Duration) {
+		defaultPriorityInitTimeout = t
+	}(defaultPriorityInitTimeout)
+	defaultPriorityInitTimeout = testPriorityInitTimeout
+
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// One localities, with priorities [0], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Remove the only localities.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: nil,
+		},
+		BalancerConfig: &lbConfig{
+			Children:   nil,
+			Priorities: nil,
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Wait after double the init timer timeout, to ensure it doesn't panic.
+	time.Sleep(testPriorityInitTimeout * 2)
+}
+
+// When a child is moved from low priority to high.
+//
+// Init a(p0) and b(p1); a(p0) is up, use a; move b to p0, a to p1, use b.
+func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two children, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Swap child with p0 and p1, the child at lower priority should now be the
+	// higher priority, and be used. The old SubConn should be closed.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-1", "child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// When the new child for p0 is changed from the previous child, the
+	// balancer should immediately update the picker so the picker from old
+	// child is not used. In this case, the picker becomes a
+	// no-subconn-available picker because this child is just started.
+	pFail := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	// Old subconn should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.NewSubConnCh
+
+	// New p0 child is ready.
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only new subconns.
+	p2 := <-cc.NewPickerCh
+	want2 := []balancer.SubConn{sc2}
+	if err := testutils.IsRoundRobin(want2, subConnFromPicker(t, p2)); err != nil {
+		t.Fatalf("want %v, got %v", want2, err)
+	}
+}
+
+// When a child is in lower priority, and in use (because higher is down),
+// move it from low priority to high.
+//
+// Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b.
+func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two children, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// p0 is down.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	pFail0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail0.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p1 subconns.
+	p0 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p0)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Swap child with p0 and p1, the child at lower priority should now be the
+	// higher priority, and be used. The old SubConn should be closed.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-1", "child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Old subconn from child-0 should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc0, scToRemove)
+	}
+
+	// Because this was a ready child moved to a higher priority, no new subconn
+	// or picker should be updated.
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("got unexpected new picker")
+	case <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn")
+	case <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn")
+	case <-time.After(time.Millisecond * 100):
+	}
+}
+
+// When the lowest child is in use, and is removed, should use the higher
+// priority child even though it's not ready.
+//
+// Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b.
+func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// Two children, with priorities [0, 1], each with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+
+	// p0 is down.
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
+	// will retry.
+	pFail0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail0.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
+		}
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p1 subconns.
+	p0 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p0)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Remove child with p1, the child at higher priority should now be used.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Old subconn from child-1 should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
+	}
+
+	pFail := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail.Pick(balancer.PickInfo{}); err == nil {
+			t.Fatalf("want pick error <non-nil>, got %v", err)
+		}
+	}
+
+	// Because there was no new child, no new subconn should be created.
+	select {
+	case <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn")
+	case <-time.After(time.Millisecond * 100):
+	}
+}
+
+// When a ready child is removed, it's kept in cache. Re-adding doesn't create subconns.
+//
+// Init 0; 0 is up, use 0; remove 0, only picker is updated, no subconn is
+// removed; re-add 0, picker is updated.
+func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
+	const testChildCacheTimeout = time.Second
+	defer func() func() {
+		old := balancergroup.DefaultSubBalancerCloseTimeout
+		balancergroup.DefaultSubBalancerCloseTimeout = testChildCacheTimeout
+		return func() {
+			balancergroup.DefaultSubBalancerCloseTimeout = old
+		}
+	}()()
+
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// One children, with priorities [0], with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Remove the child, it shouldn't cause any conn changed, but picker should
+	// be different.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  resolver.State{},
+		BalancerConfig: &lbConfig{},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	pFail := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		if _, err := pFail.Pick(balancer.PickInfo{}); err != errAllPrioritiesRemoved {
+			t.Fatalf("want pick error %v, got %v", errAllPrioritiesRemoved, err)
+		}
+	}
+
+	// But no conn changes should happen. Child balancer is in cache.
+	select {
+	case sc := <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn: %s", sc)
+	case sc := <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn: %v", sc)
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// Re-add the child, shouldn't create new connections.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Test roundrobin with only p0 subconns.
+	p2 := <-cc.NewPickerCh
+	want2 := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want2, subConnFromPicker(t, p2)); err != nil {
+		t.Fatalf("want %v, got %v", want2, err)
+	}
+
+	// But no conn changes should happen. Child balancer is just taken out from
+	// the cache.
+	select {
+	case sc := <-cc.NewSubConnCh:
+		t.Fatalf("got unexpected new SubConn: %s", sc)
+	case sc := <-cc.RemoveSubConnCh:
+		t.Fatalf("got unexpected remove SubConn: %v", sc)
+	case <-time.After(time.Millisecond * 100):
+	}
+}
+
+// When the policy of a child is changed.
+//
+// Init 0; 0 is up, use 0; change 0's policy, 0 is used.
+func (s) TestPriority_ChildPolicyChange(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// One children, with priorities [0], with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc1 := <-cc.NewSubConnCh
+
+	// p0 is ready.
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test roundrobin with only p0 subconns.
+	p1 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(t, p1)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	// Change the policy for the child (still roundrobin, but with a different
+	// name).
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: testRRBalancerName}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Old subconn should be removed.
+	scToRemove := <-cc.RemoveSubConnCh
+	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
+	}
+
+	// A new subconn should be created.
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc2 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	pb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pickfirst with the new subconns.
+	p2 := <-cc.NewPickerCh
+	want2 := []balancer.SubConn{sc2}
+	if err := testutils.IsRoundRobin(want2, subConnFromPicker(t, p2)); err != nil {
+		t.Fatalf("want %v, got %v", want2, err)
+	}
+}
+
+const inlineUpdateBalancerName = "test-inline-update-balancer"
+
+var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not")
+
+func init() {
+	stub.Register(inlineUpdateBalancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{
+				ConnectivityState: connectivity.Ready,
+				Picker:            &testutils.TestConstPicker{Err: errTestInlineStateUpdate},
+			})
+			return nil
+		},
+	})
+}
+
+// When the child policy update picker inline in a handleClientUpdate call
+// (e.g., roundrobin handling empty addresses). There could be deadlock caused
+// by acquiring a locked mutex.
+func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(priorityBalancerName)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// One children, with priorities [0], with one backend.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &lbConfig{
+			Children: map[string]*child{
+				"child-0": {&internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		_, err := p0.Pick(balancer.PickInfo{})
+		if err != errTestInlineStateUpdate {
+			t.Fatalf("picker.Pick, got err %q, want err %q", err, errTestInlineStateUpdate)
+		}
+	}
+}
diff --git a/xds/internal/balancer/priority/logging.go b/xds/internal/balancer/priority/logging.go
new file mode 100644
index 000000000000..2fb8d2d204c5
--- /dev/null
+++ b/xds/internal/balancer/priority/logging.go
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[priority-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *priorityBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}
diff --git a/xds/internal/balancer/priority/doc.go b/xds/internal/balancer/priority/utils.go
similarity index 68%
rename from xds/internal/balancer/priority/doc.go
rename to xds/internal/balancer/priority/utils.go
index 53bd270cb10e..45fbe764434a 100644
--- a/xds/internal/balancer/priority/doc.go
+++ b/xds/internal/balancer/priority/utils.go
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2020 gRPC authors.
+ * Copyright 2021 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,16 @@
  *
  */
 
-// Package priority implements the priority balancer.
-//
-// This balancer will be kept in internal until we use it in the xds balancers,
-// and are confident its functionalities are stable. It will then be exported
-// for more users.
 package priority
+
+func equalStringSlice(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
diff --git a/xds/internal/balancer/priority/utils_test.go b/xds/internal/balancer/priority/utils_test.go
new file mode 100644
index 000000000000..c80a89b080f9
--- /dev/null
+++ b/xds/internal/balancer/priority/utils_test.go
@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import "testing"
+
+func TestCompareStringSlice(t *testing.T) {
+	tests := []struct {
+		name string
+		a    []string
+		b    []string
+		want bool
+	}{
+		{
+			name: "equal",
+			a:    []string{"a", "b"},
+			b:    []string{"a", "b"},
+			want: true,
+		},
+		{
+			name: "not equal",
+			a:    []string{"a", "b"},
+			b:    []string{"a", "b", "c"},
+			want: false,
+		},
+		{
+			name: "both empty",
+			a:    nil,
+			b:    nil,
+			want: true,
+		},
+		{
+			name: "one empty",
+			a:    []string{"a", "b"},
+			b:    nil,
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := equalStringSlice(tt.a, tt.b); got != tt.want {
+				t.Errorf("equalStringSlice(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want)
+			}
+		})
+	}
+}
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index f49ba85606ff..673c4ee9a9b6 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -116,7 +116,7 @@ func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubCon
 
 // RemoveSubConn removes the SubConn.
 func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
-	tcc.logger.Logf("testClientConn: RemoveSubConn(%p)", sc)
+	tcc.logger.Logf("testClientConn: RemoveSubConn(%s)", sc)
 	select {
 	case tcc.RemoveSubConnCh <- sc:
 	default:

From c9217c719557b3398af8fead8f4138c59dfa088e Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 11 Feb 2021 14:30:32 -0800
Subject: [PATCH 361/481] priority: pass build options to balancergroup (#4202)

---
 xds/internal/balancer/priority/balancer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xds/internal/balancer/priority/balancer.go b/xds/internal/balancer/priority/balancer.go
index c1b761972c55..6c4ff08378ec 100644
--- a/xds/internal/balancer/priority/balancer.go
+++ b/xds/internal/balancer/priority/balancer.go
@@ -45,7 +45,7 @@ func init() {
 
 type priorityBB struct{}
 
-func (priorityBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+func (priorityBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
 	b := &priorityBalancer{
 		cc:                       cc,
 		done:                     grpcsync.NewEvent(),
@@ -55,7 +55,7 @@ func (priorityBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balance
 	}
 
 	b.logger = prefixLogger(b)
-	b.bg = balancergroup.New(cc, b, nil, b.logger)
+	b.bg = balancergroup.New(cc, bOpts, b, nil, b.logger)
 	b.bg.Start()
 	go b.run()
 	b.logger.Infof("Created")

From 9f3606cd0f76e99139ca8dc15a78cfc08a737822 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 11 Feb 2021 15:03:39 -0800
Subject: [PATCH 362/481] xds: xds_cluster_impl_balancer part 1 (#4154)

Part of C2P fallback. To support fallback to a DNS cluster.

This PR adds implementation of xds_cluster_impl_balancer, which will be responsible for circuit breaking and rpc dropping.

This PR only added RPC dropping. Circuit breaking will be done in a followup PR, after some necessary refactoring.
---
 vet.sh                                        |  16 +-
 .../balancer/clusterimpl/balancer_test.go     | 216 ++++++++++++
 .../balancer/clusterimpl/clusterimpl.go       | 312 ++++++++++++++++++
 xds/internal/balancer/clusterimpl/config.go   |  63 ++++
 .../balancer/clusterimpl/config_test.go       | 144 ++++++++
 xds/internal/balancer/clusterimpl/logging.go  |  34 ++
 xds/internal/balancer/clusterimpl/picker.go   | 104 ++++++
 .../balancer/loadstore/load_store_wrapper.go  | 120 +++++++
 xds/internal/balancer/lrs/balancer.go         |  76 +----
 9 files changed, 1007 insertions(+), 78 deletions(-)
 create mode 100644 xds/internal/balancer/clusterimpl/balancer_test.go
 create mode 100644 xds/internal/balancer/clusterimpl/clusterimpl.go
 create mode 100644 xds/internal/balancer/clusterimpl/config.go
 create mode 100644 xds/internal/balancer/clusterimpl/config_test.go
 create mode 100644 xds/internal/balancer/clusterimpl/logging.go
 create mode 100644 xds/internal/balancer/clusterimpl/picker.go
 create mode 100644 xds/internal/balancer/loadstore/load_store_wrapper.go

diff --git a/vet.sh b/vet.sh
index b41df6dc8607..605d7a80cfe6 100755
--- a/vet.sh
+++ b/vet.sh
@@ -141,8 +141,11 @@ not grep -Fv '.CredsBundle
 .NewAddress
 .NewServiceConfig
 .Type is deprecated: use Attributes
+BuildVersion is deprecated
 balancer.ErrTransientFailure
 balancer.Picker
+extDesc.Filename is deprecated
+github.com/golang/protobuf/jsonpb is deprecated
 grpc.CallCustomCodec
 grpc.Code
 grpc.Compressor
@@ -164,13 +167,7 @@ grpc.WithServiceConfig
 grpc.WithTimeout
 http.CloseNotifier
 info.SecurityVersion
-resolver.Backend
-resolver.GRPCLB
-extDesc.Filename is deprecated
-BuildVersion is deprecated
-github.com/golang/protobuf/jsonpb is deprecated
 proto is deprecated
-xxx_messageInfo_
 proto.InternalMessageInfo is deprecated
 proto.EnumName is deprecated
 proto.ErrInternalBadWireType is deprecated
@@ -184,7 +181,12 @@ proto.RegisterExtension is deprecated
 proto.RegisteredExtension is deprecated
 proto.RegisteredExtensions is deprecated
 proto.RegisterMapType is deprecated
-proto.Unmarshaler is deprecated' "${SC_OUT}"
+proto.Unmarshaler is deprecated
+resolver.Backend
+resolver.GRPCLB
+Target is deprecated: Use the Target field in the BuildOptions instead.
+xxx_messageInfo_
+' "${SC_OUT}"
 
 # - special golint on package comments.
 lint_package_comment_per_package() {
diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
new file mode 100644
index 000000000000..ac2d18821f01
--- /dev/null
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -0,0 +1,216 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/client/load"
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+)
+
+const (
+	defaultTestTimeout = 1 * time.Second
+	testClusterName    = "test-cluster"
+	testServiceName    = "test-eds-service"
+	testLRSServerName  = "test-lrs-name"
+)
+
+var (
+	testBackendAddrs = []resolver.Address{
+		{Addr: "1.1.1.1:1"},
+	}
+
+	cmpOpts = cmp.Options{
+		cmpopts.EquateEmpty(),
+		cmpopts.IgnoreFields(load.Data{}, "ReportInterval"),
+	}
+)
+
+func init() {
+	newRandomWRR = testutils.NewTestWRR
+}
+
+// TestDrop verifies that the balancer correctly drops the picks, and that
+// the drops are reported.
+func TestDrop(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
+	builder := balancer.Get(clusterImplName)
+	cc := testutils.NewTestClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	const (
+		dropReason      = "test-dropping-category"
+		dropNumerator   = 1
+		dropDenominator = 2
+	)
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: testBackendAddrs,
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:                    testClusterName,
+			EDSServiceName:             testServiceName,
+			LRSLoadReportingServerName: newString(testLRSServerName),
+			DropCategories: []dropCategory{{
+				Category:           dropReason,
+				RequestsPerMillion: million * dropNumerator / dropDenominator,
+			}},
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	got, err := xdsC.WaitForReportLoad(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
+	}
+	if got.Server != testLRSServerName {
+		t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerName)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 10; i++ {
+		_, err := p0.Pick(balancer.PickInfo{})
+		if err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("picker.Pick, got _,%v, want Err=%v", err, balancer.ErrNoSubConnAvailable)
+		}
+	}
+
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	p1 := <-cc.NewPickerCh
+	const rpcCount = 20
+	for i := 0; i < rpcCount; i++ {
+		gotSCSt, err := p1.Pick(balancer.PickInfo{})
+		// Even RPCs are dropped.
+		if i%2 == 0 {
+			if err == nil || !strings.Contains(err.Error(), "dropped") {
+				t.Fatalf("pick.Pick, got %v, %v, want error RPC dropped", gotSCSt, err)
+			}
+			continue
+		}
+		if err != nil || !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1)
+		}
+		if gotSCSt.Done != nil {
+			gotSCSt.Done(balancer.DoneInfo{})
+		}
+	}
+
+	// Dump load data from the store and compare with expected counts.
+	loadStore := xdsC.LoadStore()
+	if loadStore == nil {
+		t.Fatal("loadStore is nil in xdsClient")
+	}
+	const dropCount = rpcCount * dropNumerator / dropDenominator
+	wantStatsData0 := []*load.Data{{
+		Cluster:    testClusterName,
+		Service:    testServiceName,
+		TotalDrops: dropCount,
+		Drops:      map[string]uint64{dropReason: dropCount},
+	}}
+
+	gotStatsData0 := loadStore.Stats([]string{testClusterName})
+	if diff := cmp.Diff(gotStatsData0, wantStatsData0, cmpOpts); diff != "" {
+		t.Fatalf("got unexpected reports, diff (-got, +want): %v", diff)
+	}
+
+	// Send an update with new drop configs.
+	const (
+		dropReason2      = "test-dropping-category-2"
+		dropNumerator2   = 1
+		dropDenominator2 = 4
+	)
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: testBackendAddrs,
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:                    testClusterName,
+			EDSServiceName:             testServiceName,
+			LRSLoadReportingServerName: newString(testLRSServerName),
+			DropCategories: []dropCategory{{
+				Category:           dropReason2,
+				RequestsPerMillion: million * dropNumerator2 / dropDenominator2,
+			}},
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < rpcCount; i++ {
+		gotSCSt, err := p2.Pick(balancer.PickInfo{})
+		// Even RPCs are dropped.
+		if i%4 == 0 {
+			if err == nil || !strings.Contains(err.Error(), "dropped") {
+				t.Fatalf("pick.Pick, got %v, %v, want error RPC dropped", gotSCSt, err)
+			}
+			continue
+		}
+		if err != nil || !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1)
+		}
+		if gotSCSt.Done != nil {
+			gotSCSt.Done(balancer.DoneInfo{})
+		}
+	}
+
+	const dropCount2 = rpcCount * dropNumerator2 / dropDenominator2
+	wantStatsData1 := []*load.Data{{
+		Cluster:    testClusterName,
+		Service:    testServiceName,
+		TotalDrops: dropCount2,
+		Drops:      map[string]uint64{dropReason2: dropCount2},
+	}}
+
+	gotStatsData1 := loadStore.Stats([]string{testClusterName})
+	if diff := cmp.Diff(gotStatsData1, wantStatsData1, cmpOpts); diff != "" {
+		t.Fatalf("got unexpected reports, diff (-got, +want): %v", diff)
+	}
+}
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
new file mode 100644
index 000000000000..e9201c10bdc9
--- /dev/null
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -0,0 +1,312 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clusterimpl implements the xds_cluster_impl balancing policy. It
+// handles the cluster features (e.g. circuit_breaking, RPC dropping).
+//
+// Note that it doesn't handle name resolution, which is done by policy
+// xds_cluster_resolver.
+package clusterimpl
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal/balancer/loadstore"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/load"
+)
+
+const (
+	clusterImplName = "xds_cluster_impl_experimental"
+	// TODO: define defaultRequestCountMax = 1024
+)
+
+func init() {
+	balancer.Register(clusterImplBB{})
+}
+
+var newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
+
+type clusterImplBB struct{}
+
+func (clusterImplBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &clusterImplBalancer{
+		ClientConn:     cc,
+		bOpts:          bOpts,
+		closed:         grpcsync.NewEvent(),
+		loadWrapper:    loadstore.NewWrapper(),
+		pickerUpdateCh: buffer.NewUnbounded(),
+	}
+	b.logger = prefixLogger(b)
+
+	client, err := newXDSClient()
+	if err != nil {
+		b.logger.Errorf("failed to create xds-client: %v", err)
+		return nil
+	}
+	b.xdsC = client
+	go b.run()
+
+	b.logger.Infof("Created")
+	return b
+}
+
+func (clusterImplBB) Name() string {
+	return clusterImplName
+}
+
+func (clusterImplBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+// xdsClientInterface contains only the xds_client methods needed by LRS
+// balancer. It's defined so we can override xdsclient in tests.
+type xdsClientInterface interface {
+	ReportLoad(server string) (*load.Store, func())
+	Close()
+}
+
+type clusterImplBalancer struct {
+	balancer.ClientConn
+	bOpts  balancer.BuildOptions
+	closed *grpcsync.Event
+	logger *grpclog.PrefixLogger
+	xdsC   xdsClientInterface
+
+	config           *lbConfig
+	childLB          balancer.Balancer
+	cancelLoadReport func()
+	clusterName      string
+	edsServiceName   string
+	lrsServerName    string
+	loadWrapper      *loadstore.Wrapper
+
+	// childState/drops/requestCounter can only be accessed in run(). And run()
+	// is the only goroutine that sends picker to the parent ClientConn. All
+	// requests to update picker need to be sent to pickerUpdateCh.
+	childState balancer.State
+	drops      []*dropper
+	// TODO: add serviceRequestCount and maxRequestCount for circuit breaking.
+	pickerUpdateCh *buffer.Unbounded
+}
+
+// updateLoadStore checks the config for load store, and decides whether it
+// needs to restart the load reporting stream.
+func (cib *clusterImplBalancer) updateLoadStore(newConfig *lbConfig) error {
+	var updateLoadClusterAndService bool
+
+	// ClusterName is different, restart. ClusterName is from ClusterName and
+	// EdsServiceName.
+	if cib.clusterName != newConfig.Cluster {
+		updateLoadClusterAndService = true
+		cib.clusterName = newConfig.Cluster
+	}
+	if cib.edsServiceName != newConfig.EDSServiceName {
+		updateLoadClusterAndService = true
+		cib.edsServiceName = newConfig.EDSServiceName
+	}
+	if updateLoadClusterAndService {
+		// This updates the clusterName and serviceName that will be reported
+		// for the loads. The update here is too early, the perfect timing is
+		// when the picker is updated with the new connection. But from this
+		// balancer's point of view, it's impossible to tell.
+		//
+		// On the other hand, this will almost never happen. Each LRS policy
+		// shouldn't get updated config. The parent should do a graceful switch
+		// when the clusterName or serviceName is changed.
+		cib.loadWrapper.UpdateClusterAndService(cib.clusterName, cib.edsServiceName)
+	}
+
+	// Check if it's necessary to restart load report.
+	var newLRSServerName string
+	if newConfig.LRSLoadReportingServerName != nil {
+		newLRSServerName = *newConfig.LRSLoadReportingServerName
+	}
+	if cib.lrsServerName != newLRSServerName {
+		// LrsLoadReportingServerName is different, load should be report to a
+		// different server, restart.
+		cib.lrsServerName = newLRSServerName
+		if cib.cancelLoadReport != nil {
+			cib.cancelLoadReport()
+			cib.cancelLoadReport = nil
+		}
+		var loadStore *load.Store
+		if cib.xdsC != nil {
+			loadStore, cib.cancelLoadReport = cib.xdsC.ReportLoad(cib.lrsServerName)
+		}
+		cib.loadWrapper.UpdateLoadStore(loadStore)
+	}
+
+	return nil
+}
+
+func (cib *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	if cib.closed.HasFired() {
+		cib.logger.Warningf("xds: received ClientConnState {%+v} after clusterImplBalancer was closed", s)
+		return nil
+	}
+
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	// Need to check for potential errors at the beginning of this function, so
+	// that on errors, we reject the whole config, instead of applying part of
+	// it.
+	bb := balancer.Get(newConfig.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("balancer %q not registered", newConfig.ChildPolicy.Name)
+	}
+
+	// Update load reporting config. This needs to be done before updating the
+	// child policy because we need the loadStore from the updated client to be
+	// passed to the ccWrapper, so that the next picker from the child policy
+	// will pick up the new loadStore.
+	if err := cib.updateLoadStore(newConfig); err != nil {
+		return err
+	}
+
+	// Compare new drop config. And update picker if it's changed.
+	var updatePicker bool
+	if cib.config == nil || !equalDropCategories(cib.config.DropCategories, newConfig.DropCategories) {
+		cib.drops = make([]*dropper, 0, len(newConfig.DropCategories))
+		for _, c := range newConfig.DropCategories {
+			cib.drops = append(cib.drops, newDropper(c))
+		}
+		updatePicker = true
+	}
+
+	// TODO: compare cluster name. And update picker if it's changed, because
+	// circuit breaking's stream counter will be different.
+	//
+	// Set `updatePicker` to manually update the picker.
+
+	// TODO: compare upper bound of stream count. And update picker if it's
+	// changed. This is also for circuit breaking.
+	//
+	// Set `updatePicker` to manually update the picker.
+
+	if updatePicker {
+		cib.pickerUpdateCh.Put(&dropConfigs{
+			drops: cib.drops,
+		})
+	}
+
+	// If child policy is a different type, recreate the sub-balancer.
+	if cib.config == nil || cib.config.ChildPolicy.Name != newConfig.ChildPolicy.Name {
+		if cib.childLB != nil {
+			cib.childLB.Close()
+		}
+		cib.childLB = bb.Build(cib, cib.bOpts)
+	}
+	cib.config = newConfig
+
+	if cib.childLB == nil {
+		// This is not an expected situation, and should be super rare in
+		// practice.
+		//
+		// When this happens, we already applied all the other configurations
+		// (drop/circuit breaking), but there's no child policy. This balancer
+		// will be stuck, and we report the error to the parent.
+		return fmt.Errorf("child policy is nil, this means balancer %q's Build() returned nil", newConfig.ChildPolicy.Name)
+	}
+
+	// Addresses and sub-balancer config are sent to sub-balancer.
+	return cib.childLB.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: cib.config.ChildPolicy.Config,
+	})
+}
+
+func (cib *clusterImplBalancer) ResolverError(err error) {
+	if cib.closed.HasFired() {
+		cib.logger.Warningf("xds: received resolver error {%+v} after clusterImplBalancer was closed", err)
+		return
+	}
+
+	if cib.childLB != nil {
+		cib.childLB.ResolverError(err)
+	}
+}
+
+func (cib *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	if cib.closed.HasFired() {
+		cib.logger.Warningf("xds: received subconn state change {%+v, %+v} after clusterImplBalancer was closed", sc, s)
+		return
+	}
+
+	if cib.childLB != nil {
+		cib.childLB.UpdateSubConnState(sc, s)
+	}
+}
+
+func (cib *clusterImplBalancer) Close() {
+	if cib.childLB != nil {
+		cib.childLB.Close()
+		cib.childLB = nil
+	}
+	cib.xdsC.Close()
+	cib.closed.Fire()
+	cib.logger.Infof("Shutdown")
+}
+
+// Override methods to accept updates from the child LB.
+
+func (cib *clusterImplBalancer) UpdateState(state balancer.State) {
+	// Instead of updating parent ClientConn inline, send state to run().
+	cib.pickerUpdateCh.Put(state)
+}
+
+type dropConfigs struct {
+	drops []*dropper
+}
+
+func (cib *clusterImplBalancer) run() {
+	for {
+		select {
+		case update := <-cib.pickerUpdateCh.Get():
+			cib.pickerUpdateCh.Load()
+			switch u := update.(type) {
+			case balancer.State:
+				cib.childState = u
+				cib.ClientConn.UpdateState(balancer.State{
+					ConnectivityState: cib.childState.ConnectivityState,
+					Picker:            newDropPicker(cib.childState, cib.drops, cib.loadWrapper),
+				})
+			case *dropConfigs:
+				cib.drops = u.drops
+				// cib.requestCounter = u.requestCounter
+				if cib.childState.Picker != nil {
+					cib.ClientConn.UpdateState(balancer.State{
+						ConnectivityState: cib.childState.ConnectivityState,
+						Picker:            newDropPicker(cib.childState, cib.drops, cib.loadWrapper),
+					})
+				}
+			}
+		case <-cib.closed.Done():
+			return
+		}
+	}
+}
diff --git a/xds/internal/balancer/clusterimpl/config.go b/xds/internal/balancer/clusterimpl/config.go
new file mode 100644
index 000000000000..548ab34bce4d
--- /dev/null
+++ b/xds/internal/balancer/clusterimpl/config.go
@@ -0,0 +1,63 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type dropCategory struct {
+	Category           string
+	RequestsPerMillion uint32
+}
+
+// lbConfig is the balancer config for weighted_target.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+
+	Cluster                    string
+	EDSServiceName             string
+	LRSLoadReportingServerName *string
+	MaxConcurrentRequests      *uint32
+	DropCategories             []dropCategory
+	ChildPolicy                *internalserviceconfig.BalancerConfig
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	var cfg lbConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+func equalDropCategories(a, b []dropCategory) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
diff --git a/xds/internal/balancer/clusterimpl/config_test.go b/xds/internal/balancer/clusterimpl/config_test.go
new file mode 100644
index 000000000000..89696981e2a0
--- /dev/null
+++ b/xds/internal/balancer/clusterimpl/config_test.go
@@ -0,0 +1,144 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/roundrobin"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/xds/internal/balancer/weightedtarget"
+)
+
+const (
+	testJSONConfig = `{
+  "cluster": "test_cluster",
+  "edsServiceName": "test-eds",
+  "lrsLoadReportingServerName": "lrs_server",
+  "maxConcurrentRequests": 123,
+  "dropCategories": [
+    {
+      "category": "drop-1",
+      "requestsPerMillion": 314
+    },
+    {
+      "category": "drop-2",
+      "requestsPerMillion": 159
+    }
+  ],
+  "childPolicy": [
+    {
+      "weighted_target_experimental": {
+        "targets": {
+          "wt-child-1": {
+            "weight": 75,
+            "childPolicy":[{"round_robin":{}}]
+          },
+          "wt-child-2": {
+            "weight": 25,
+            "childPolicy":[{"round_robin":{}}]
+          }
+        }
+      }
+    }
+  ]
+}`
+
+	wtName = "weighted_target_experimental"
+)
+
+var (
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON   = `{
+  "targets": {
+    "wt-child-1": {
+      "weight": 75,
+      "childPolicy":[{"round_robin":{}}]
+    },
+    "wt-child-2": {
+      "weight": 25,
+      "childPolicy":[{"round_robin":{}}]
+    }
+  }
+}`
+
+	wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON))
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "bad json",
+			js:      "{",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				Cluster:                    "test_cluster",
+				EDSServiceName:             "test-eds",
+				LRSLoadReportingServerName: newString("lrs_server"),
+				MaxConcurrentRequests:      newUint32(123),
+				DropCategories: []dropCategory{
+					{Category: "drop-1", RequestsPerMillion: 314},
+					{Category: "drop-2", RequestsPerMillion: 159},
+				},
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name:   wtName,
+					Config: wtConfig,
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !cmp.Equal(got, tt.want) {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want))
+			}
+		})
+	}
+}
+
+func newString(s string) *string {
+	return &s
+}
+
+func newUint32(i uint32) *uint32 {
+	return &i
+}
diff --git a/xds/internal/balancer/clusterimpl/logging.go b/xds/internal/balancer/clusterimpl/logging.go
new file mode 100644
index 000000000000..3bbd1b0d7837
--- /dev/null
+++ b/xds/internal/balancer/clusterimpl/logging.go
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[xds-cluster-impl-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}
diff --git a/xds/internal/balancer/clusterimpl/picker.go b/xds/internal/balancer/clusterimpl/picker.go
new file mode 100644
index 000000000000..05e9f89786fd
--- /dev/null
+++ b/xds/internal/balancer/clusterimpl/picker.go
@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/wrr"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/client/load"
+)
+
+var newRandomWRR = wrr.NewRandom
+
+const million = 1000000
+
+type dropper struct {
+	category string
+	w        wrr.WRR
+}
+
+// greatest common divisor (GCD) via Euclidean algorithm
+func gcd(a, b uint32) uint32 {
+	for b != 0 {
+		t := b
+		b = a % b
+		a = t
+	}
+	return a
+}
+
+func newDropper(c dropCategory) *dropper {
+	w := newRandomWRR()
+	gcdv := gcd(c.RequestsPerMillion, million)
+	// Return true for RequestPerMillion, false for the rest.
+	w.Add(true, int64(c.RequestsPerMillion/gcdv))
+	w.Add(false, int64((million-c.RequestsPerMillion)/gcdv))
+
+	return &dropper{
+		category: c.Category,
+		w:        w,
+	}
+}
+
+func (d *dropper) drop() (ret bool) {
+	return d.w.Next().(bool)
+}
+
+// loadReporter wraps the methods from the loadStore that are used here.
+type loadReporter interface {
+	CallDropped(locality string)
+}
+
+type dropPicker struct {
+	drops     []*dropper
+	s         balancer.State
+	loadStore loadReporter
+	// TODO: add serviceRequestCount and maxRequestCount for circuit breaking.
+}
+
+func newDropPicker(s balancer.State, drops []*dropper, loadStore load.PerClusterReporter) *dropPicker {
+	return &dropPicker{
+		drops:     drops,
+		s:         s,
+		loadStore: loadStore,
+	}
+}
+
+func (d *dropPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	// Don't drop unless the inner picker is READY. Similar to
+	// https://github.com/grpc/grpc-go/issues/2622.
+	if d.s.ConnectivityState != connectivity.Ready {
+		return d.s.Picker.Pick(info)
+	}
+
+	for _, dp := range d.drops {
+		if dp.drop() {
+			if d.loadStore != nil {
+				d.loadStore.CallDropped(dp.category)
+			}
+			return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
+		}
+	}
+	// TODO: support circuit breaking, check if d.maxRequestCount >=
+	// d.counter.StartRequestWithMax().
+	return d.s.Picker.Pick(info)
+}
diff --git a/xds/internal/balancer/loadstore/load_store_wrapper.go b/xds/internal/balancer/loadstore/load_store_wrapper.go
new file mode 100644
index 000000000000..88fa344118cc
--- /dev/null
+++ b/xds/internal/balancer/loadstore/load_store_wrapper.go
@@ -0,0 +1,120 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package loadstore contains the loadStoreWrapper shared by the balancers.
+package loadstore
+
+import (
+	"sync"
+
+	"google.golang.org/grpc/xds/internal/client/load"
+)
+
+// NewWrapper creates a Wrapper.
+func NewWrapper() *Wrapper {
+	return &Wrapper{}
+}
+
+// Wrapper wraps a load store with cluster and edsService.
+//
+// It's store and cluster/edsService can be updated separately. And it will
+// update its internal perCluster store so that new stats will be added to the
+// correct perCluster.
+//
+// Note that this struct is a temporary walkaround before we implement graceful
+// switch for EDS. Any update to the clusterName and serviceName is too early,
+// the perfect timing is when the picker is updated with the new connection.
+// This early update could cause picks for the old SubConn being reported to the
+// new services.
+//
+// When the graceful switch in EDS is done, there should be no need for this
+// struct. The policies that record/report load shouldn't need to handle update
+// of lrsServerName/cluster/edsService. Its parent should do a graceful switch
+// of the whole tree when one of that changes.
+type Wrapper struct {
+	mu         sync.RWMutex
+	cluster    string
+	edsService string
+	// store and perCluster are initialized as nil. They are only set by the
+	// balancer when LRS is enabled. Before that, all functions to record loads
+	// are no-op.
+	store      *load.Store
+	perCluster load.PerClusterReporter
+}
+
+// UpdateClusterAndService updates the cluster name and eds service for this
+// wrapper. If any one of them is changed from before, the perCluster store in
+// this wrapper will also be updated.
+func (lsw *Wrapper) UpdateClusterAndService(cluster, edsService string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if cluster == lsw.cluster && edsService == lsw.edsService {
+		return
+	}
+	lsw.cluster = cluster
+	lsw.edsService = edsService
+	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
+}
+
+// UpdateLoadStore updates the load store for this wrapper. If it is changed
+// from before, the perCluster store in this wrapper will also be updated.
+func (lsw *Wrapper) UpdateLoadStore(store *load.Store) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
+}
+
+// CallStarted records a call started in the store.
+func (lsw *Wrapper) CallStarted(locality string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallStarted(locality)
+	}
+}
+
+// CallFinished records a call finished in the store.
+func (lsw *Wrapper) CallFinished(locality string, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallFinished(locality, err)
+	}
+}
+
+// CallServerLoad records the server load in the store.
+func (lsw *Wrapper) CallServerLoad(locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallServerLoad(locality, name, val)
+	}
+}
+
+// CallDropped records a call dropped in the store.
+func (lsw *Wrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallDropped(category)
+	}
+}
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index d60355afd25e..ab9ee7109db1 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -22,11 +22,11 @@ package lrs
 import (
 	"encoding/json"
 	"fmt"
-	"sync"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
@@ -162,72 +162,6 @@ type xdsClientInterface interface {
 	Close()
 }
 
-type loadStoreWrapper struct {
-	mu         sync.RWMutex
-	cluster    string
-	edsService string
-	// Both store and perCluster will be nil if load reporting is disabled (EDS
-	// response doesn't have LRS server name). Note that methods on Store and
-	// perCluster all handle nil, so there's no need to check nil before calling
-	// them.
-	store      *load.Store
-	perCluster load.PerClusterReporter
-}
-
-func (lsw *loadStoreWrapper) updateClusterAndService(cluster, edsService string) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if cluster == lsw.cluster && edsService == lsw.edsService {
-		return
-	}
-	lsw.cluster = cluster
-	lsw.edsService = edsService
-	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
-}
-
-func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if store == lsw.store {
-		return
-	}
-	lsw.store = store
-	lsw.perCluster = nil
-	lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService)
-}
-
-func (lsw *loadStoreWrapper) CallStarted(locality string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallStarted(locality)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallFinished(locality, err)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallServerLoad(locality, name, val)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallDropped(category string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallDropped(category)
-	}
-}
-
 type xdsClientWrapper struct {
 	c                xdsClientInterface
 	cancelLoadReport func()
@@ -236,13 +170,13 @@ type xdsClientWrapper struct {
 	lrsServerName    string
 	// loadWrapper is a wrapper with loadOriginal, with clusterName and
 	// edsServiceName. It's used children to report loads.
-	loadWrapper *loadStoreWrapper
+	loadWrapper *loadstore.Wrapper
 }
 
 func newXDSClientWrapper(c xdsClientInterface) *xdsClientWrapper {
 	return &xdsClientWrapper{
 		c:           c,
-		loadWrapper: &loadStoreWrapper{},
+		loadWrapper: loadstore.NewWrapper(),
 	}
 }
 
@@ -274,7 +208,7 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig) error {
 		// On the other hand, this will almost never happen. Each LRS policy
 		// shouldn't get updated config. The parent should do a graceful switch when
 		// the clusterName or serviceName is changed.
-		w.loadWrapper.updateClusterAndService(w.clusterName, w.edsServiceName)
+		w.loadWrapper.UpdateClusterAndService(w.clusterName, w.edsServiceName)
 	}
 
 	if w.lrsServerName != newConfig.LrsLoadReportingServerName {
@@ -293,7 +227,7 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig) error {
 		if w.c != nil {
 			loadStore, w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName)
 		}
-		w.loadWrapper.updateLoadStore(loadStore)
+		w.loadWrapper.UpdateLoadStore(loadStore)
 	}
 
 	return nil

From 425d405f392835ed7ad2f5f91c32443961e29246 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 12 Feb 2021 14:24:30 -0800
Subject: [PATCH 363/481] test: add timeout to regular xds test runs (#4201)

---
 test/kokoro/xds.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index ee98399319bc..36a3f4563cfe 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -27,10 +27,10 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh
 # they are added into "all".
 GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
-    --test_case="all,path_matching,header_matching,circuit_breaking" \
+    --test_case="all,path_matching,header_matching,circuit_breaking,timeout" \
     --project_id=grpc-testing \
     --project_num=830293263384 \
-    --source_image=projects/grpc-testing/global/images/xds-test-server-3 \
+    --source_image=projects/grpc-testing/global/images/xds-test-server-4 \
     --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \
     --gcp_suffix=$(date '+%s') \
     --verbose \

From 1b75f7144df29e6ae87043e972b5673a0e03cbbf Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 17 Feb 2021 10:46:07 -0800
Subject: [PATCH 364/481] circuit breaking: keep max_count per picker, instead
 of globally, and add support in cluster_impl balancer (#4203)

Also changed circuit breaking counter implementation to move max_count into the
picker, because this is how cluster_impl is designed. Implementation in EDS is
also modified to keep max_count in picker.
---
 .../balancer/cdsbalancer/cdsbalancer.go       |   8 +-
 .../cdsbalancer/cdsbalancer_security_test.go  |  14 +--
 .../balancer/cdsbalancer/cdsbalancer_test.go  |  25 ++--
 .../balancer/clusterimpl/balancer_test.go     | 116 +++++++++++++++++-
 .../balancer/clusterimpl/clusterimpl.go       |  67 ++++++----
 xds/internal/balancer/clusterimpl/picker.go   |  37 +++++-
 xds/internal/balancer/edsbalancer/config.go   |  11 ++
 xds/internal/balancer/edsbalancer/eds.go      |   6 +-
 xds/internal/balancer/edsbalancer/eds_impl.go |  27 ++--
 .../balancer/edsbalancer/eds_impl_test.go     |   8 +-
 xds/internal/balancer/edsbalancer/eds_test.go |  51 +++++++-
 xds/internal/client/requests_counter.go       |  28 +----
 xds/internal/client/requests_counter_test.go  |  26 ++--
 13 files changed, 310 insertions(+), 114 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 5a8a3c35d79c..04286d715901 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -34,7 +34,6 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
-	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
@@ -328,8 +327,6 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		return
 	}
 
-	client.SetMaxRequests(update.cds.ServiceName, update.cds.MaxRequests)
-
 	// The first good update from the watch API leads to the instantiation of an
 	// edsBalancer. Further updates/errors are propagated to the existing
 	// edsBalancer.
@@ -342,7 +339,10 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		b.edsLB = edsLB
 		b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
 	}
-	lbCfg := &edsbalancer.EDSConfig{EDSServiceName: update.cds.ServiceName}
+	lbCfg := &edsbalancer.EDSConfig{
+		EDSServiceName:        update.cds.ServiceName,
+		MaxConcurrentRequests: update.cds.MaxRequests,
+	}
 	if update.cds.EnableLRS {
 		// An empty string here indicates that the edsBalancer should use the
 		// same xDS server for load reporting as it does for EDS
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index b39b2abf06bc..e862f99cd4e1 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -233,7 +233,7 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -289,7 +289,7 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// newEDSBalancer function as part of test setup. No security config is
 	// passed to the CDS balancer as part of this update.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -445,7 +445,7 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -479,7 +479,7 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
@@ -510,7 +510,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
@@ -560,7 +560,7 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
@@ -637,7 +637,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 			RootInstanceName: "default1",
 		},
 	}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 9d7c4a7bff55..9c7bc2362ab7 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -193,8 +193,11 @@ func cdsCCS(cluster string) balancer.ClientConnState {
 
 // edsCCS is a helper function to construct a good update passed from the
 // cdsBalancer to the edsBalancer.
-func edsCCS(service string, enableLRS bool) balancer.ClientConnState {
-	lbCfg := &edsbalancer.EDSConfig{EDSServiceName: service}
+func edsCCS(service string, countMax *uint32, enableLRS bool) balancer.ClientConnState {
+	lbCfg := &edsbalancer.EDSConfig{
+		EDSServiceName:        service,
+		MaxConcurrentRequests: countMax,
+	}
 	if enableLRS {
 		lbCfg.LrsLoadReportingServerName = new(string)
 	}
@@ -350,12 +353,12 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 		{
 			name:      "happy-case-with-lrs",
 			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
-			wantCCS:   edsCCS(serviceName, true),
+			wantCCS:   edsCCS(serviceName, nil, true),
 		},
 		{
 			name:      "happy-case-without-lrs",
 			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
-			wantCCS:   edsCCS(serviceName, false),
+			wantCCS:   edsCCS(serviceName, nil, false),
 		},
 	}
 
@@ -423,7 +426,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -508,7 +511,7 @@ func (s) TestResolverError(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -557,7 +560,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -592,7 +595,7 @@ func (s) TestCircuitBreaking(t *testing.T) {
 	// the service's counter with the new max requests.
 	var maxRequests uint32 = 1
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName, MaxRequests: &maxRequests}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, &maxRequests, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
@@ -602,10 +605,10 @@ func (s) TestCircuitBreaking(t *testing.T) {
 	// Since the counter's max requests was set to 1, the first request should
 	// succeed and the second should fail.
 	counter := client.GetServiceRequestsCounter(serviceName)
-	if err := counter.StartRequest(); err != nil {
+	if err := counter.StartRequest(maxRequests); err != nil {
 		t.Fatal(err)
 	}
-	if err := counter.StartRequest(); err == nil {
+	if err := counter.StartRequest(maxRequests); err == nil {
 		t.Fatal("unexpected success on start request over max")
 	}
 	counter.EndRequest()
@@ -625,7 +628,7 @@ func (s) TestClose(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
-	wantCCS := edsCCS(serviceName, false)
+	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index ac2d18821f01..3e6ac0fd2900 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -58,9 +58,9 @@ func init() {
 	newRandomWRR = testutils.NewTestWRR
 }
 
-// TestDrop verifies that the balancer correctly drops the picks, and that
-// the drops are reported.
-func TestDrop(t *testing.T) {
+// TestDropByCategory verifies that the balancer correctly drops the picks, and
+// that the drops are reported.
+func TestDropByCategory(t *testing.T) {
 	xdsC := fakeclient.NewClient()
 	oldNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
@@ -214,3 +214,113 @@ func TestDrop(t *testing.T) {
 		t.Fatalf("got unexpected reports, diff (-got, +want): %v", diff)
 	}
 }
+
+// TestDropCircuitBreaking verifies that the balancer correctly drops the picks
+// due to circuit breaking, and that the drops are reported.
+func TestDropCircuitBreaking(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
+	builder := balancer.Get(clusterImplName)
+	cc := testutils.NewTestClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	var maxRequest uint32 = 50
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: testBackendAddrs,
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:                    testClusterName,
+			EDSServiceName:             testServiceName,
+			LRSLoadReportingServerName: newString(testLRSServerName),
+			MaxConcurrentRequests:      &maxRequest,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	got, err := xdsC.WaitForReportLoad(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.ReportLoad failed with error: %v", err)
+	}
+	if got.Server != testLRSServerName {
+		t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerName)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 10; i++ {
+		_, err := p0.Pick(balancer.PickInfo{})
+		if err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("picker.Pick, got _,%v, want Err=%v", err, balancer.ErrNoSubConnAvailable)
+		}
+	}
+
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	dones := []func(){}
+	p1 := <-cc.NewPickerCh
+	const rpcCount = 100
+	for i := 0; i < rpcCount; i++ {
+		gotSCSt, err := p1.Pick(balancer.PickInfo{})
+		if i < 50 && err != nil {
+			t.Errorf("The first 50%% picks should be non-drops, got error %v", err)
+		} else if i > 50 && err == nil {
+			t.Errorf("The second 50%% picks should be drops, got error <nil>")
+		}
+		dones = append(dones, func() {
+			if gotSCSt.Done != nil {
+				gotSCSt.Done(balancer.DoneInfo{})
+			}
+		})
+	}
+	for _, done := range dones {
+		done()
+	}
+
+	dones = []func(){}
+	// Pick without drops.
+	for i := 0; i < 50; i++ {
+		gotSCSt, err := p1.Pick(balancer.PickInfo{})
+		if err != nil {
+			t.Errorf("The third 50%% picks should be non-drops, got error %v", err)
+		}
+		dones = append(dones, func() {
+			if gotSCSt.Done != nil {
+				gotSCSt.Done(balancer.DoneInfo{})
+			}
+		})
+	}
+	for _, done := range dones {
+		done()
+	}
+
+	// Dump load data from the store and compare with expected counts.
+	loadStore := xdsC.LoadStore()
+	if loadStore == nil {
+		t.Fatal("loadStore is nil in xdsClient")
+	}
+
+	wantStatsData0 := []*load.Data{{
+		Cluster:    testClusterName,
+		Service:    testServiceName,
+		TotalDrops: uint64(maxRequest),
+	}}
+
+	gotStatsData0 := loadStore.Stats([]string{testClusterName})
+	if diff := cmp.Diff(gotStatsData0, wantStatsData0, cmpOpts); diff != "" {
+		t.Fatalf("got unexpected drop reports, diff (-got, +want): %v", diff)
+	}
+}
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index e9201c10bdc9..4e4af5a02b45 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -38,8 +38,8 @@ import (
 )
 
 const (
-	clusterImplName = "xds_cluster_impl_experimental"
-	// TODO: define defaultRequestCountMax = 1024
+	clusterImplName        = "xds_cluster_impl_experimental"
+	defaultRequestCountMax = 1024
 )
 
 func init() {
@@ -52,11 +52,12 @@ type clusterImplBB struct{}
 
 func (clusterImplBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
 	b := &clusterImplBalancer{
-		ClientConn:     cc,
-		bOpts:          bOpts,
-		closed:         grpcsync.NewEvent(),
-		loadWrapper:    loadstore.NewWrapper(),
-		pickerUpdateCh: buffer.NewUnbounded(),
+		ClientConn:      cc,
+		bOpts:           bOpts,
+		closed:          grpcsync.NewEvent(),
+		loadWrapper:     loadstore.NewWrapper(),
+		pickerUpdateCh:  buffer.NewUnbounded(),
+		requestCountMax: defaultRequestCountMax,
 	}
 	b.logger = prefixLogger(b)
 
@@ -105,10 +106,11 @@ type clusterImplBalancer struct {
 	// childState/drops/requestCounter can only be accessed in run(). And run()
 	// is the only goroutine that sends picker to the parent ClientConn. All
 	// requests to update picker need to be sent to pickerUpdateCh.
-	childState balancer.State
-	drops      []*dropper
-	// TODO: add serviceRequestCount and maxRequestCount for circuit breaking.
-	pickerUpdateCh *buffer.Unbounded
+	childState      balancer.State
+	drops           []*dropper
+	requestCounter  *xdsclient.ServiceRequestsCounter
+	requestCountMax uint32
+	pickerUpdateCh  *buffer.Unbounded
 }
 
 // updateLoadStore checks the config for load store, and decides whether it
@@ -198,19 +200,28 @@ func (cib *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState
 		updatePicker = true
 	}
 
-	// TODO: compare cluster name. And update picker if it's changed, because
-	// circuit breaking's stream counter will be different.
-	//
-	// Set `updatePicker` to manually update the picker.
-
-	// TODO: compare upper bound of stream count. And update picker if it's
-	// changed. This is also for circuit breaking.
-	//
-	// Set `updatePicker` to manually update the picker.
+	// Compare cluster name. And update picker if it's changed, because circuit
+	// breaking's stream counter will be different.
+	if cib.config == nil || cib.config.Cluster != newConfig.Cluster {
+		cib.requestCounter = xdsclient.GetServiceRequestsCounter(newConfig.Cluster)
+		updatePicker = true
+	}
+	// Compare upper bound of stream count. And update picker if it's changed.
+	// This is also for circuit breaking.
+	var newRequestCountMax uint32 = 1024
+	if newConfig.MaxConcurrentRequests != nil {
+		newRequestCountMax = *newConfig.MaxConcurrentRequests
+	}
+	if cib.requestCountMax != newRequestCountMax {
+		cib.requestCountMax = newRequestCountMax
+		updatePicker = true
+	}
 
 	if updatePicker {
 		cib.pickerUpdateCh.Put(&dropConfigs{
-			drops: cib.drops,
+			drops:           cib.drops,
+			requestCounter:  cib.requestCounter,
+			requestCountMax: cib.requestCountMax,
 		})
 	}
 
@@ -280,7 +291,9 @@ func (cib *clusterImplBalancer) UpdateState(state balancer.State) {
 }
 
 type dropConfigs struct {
-	drops []*dropper
+	drops           []*dropper
+	requestCounter  *xdsclient.ServiceRequestsCounter
+	requestCountMax uint32
 }
 
 func (cib *clusterImplBalancer) run() {
@@ -293,15 +306,19 @@ func (cib *clusterImplBalancer) run() {
 				cib.childState = u
 				cib.ClientConn.UpdateState(balancer.State{
 					ConnectivityState: cib.childState.ConnectivityState,
-					Picker:            newDropPicker(cib.childState, cib.drops, cib.loadWrapper),
+					Picker: newDropPicker(cib.childState, &dropConfigs{
+						drops:           cib.drops,
+						requestCounter:  cib.requestCounter,
+						requestCountMax: cib.requestCountMax,
+					}, cib.loadWrapper),
 				})
 			case *dropConfigs:
 				cib.drops = u.drops
-				// cib.requestCounter = u.requestCounter
+				cib.requestCounter = u.requestCounter
 				if cib.childState.Picker != nil {
 					cib.ClientConn.UpdateState(balancer.State{
 						ConnectivityState: cib.childState.ConnectivityState,
-						Picker:            newDropPicker(cib.childState, cib.drops, cib.loadWrapper),
+						Picker:            newDropPicker(cib.childState, u, cib.loadWrapper),
 					})
 				}
 			}
diff --git a/xds/internal/balancer/clusterimpl/picker.go b/xds/internal/balancer/clusterimpl/picker.go
index 05e9f89786fd..6e9d27911534 100644
--- a/xds/internal/balancer/clusterimpl/picker.go
+++ b/xds/internal/balancer/clusterimpl/picker.go
@@ -24,6 +24,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
 )
 
@@ -72,14 +73,17 @@ type dropPicker struct {
 	drops     []*dropper
 	s         balancer.State
 	loadStore loadReporter
-	// TODO: add serviceRequestCount and maxRequestCount for circuit breaking.
+	counter   *client.ServiceRequestsCounter
+	countMax  uint32
 }
 
-func newDropPicker(s balancer.State, drops []*dropper, loadStore load.PerClusterReporter) *dropPicker {
+func newDropPicker(s balancer.State, config *dropConfigs, loadStore load.PerClusterReporter) *dropPicker {
 	return &dropPicker{
-		drops:     drops,
+		drops:     config.drops,
 		s:         s,
 		loadStore: loadStore,
+		counter:   config.requestCounter,
+		countMax:  config.requestCountMax,
 	}
 }
 
@@ -98,7 +102,30 @@ func (d *dropPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 			return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
 		}
 	}
-	// TODO: support circuit breaking, check if d.maxRequestCount >=
-	// d.counter.StartRequestWithMax().
+
+	if d.counter != nil {
+		if err := d.counter.StartRequest(d.countMax); err != nil {
+			// Drops by circuit breaking are reported with empty category. They
+			// will be reported only in total drops, but not in per category.
+			if d.loadStore != nil {
+				d.loadStore.CallDropped("")
+			}
+			return balancer.PickResult{}, status.Errorf(codes.Unavailable, err.Error())
+		}
+		pr, err := d.s.Picker.Pick(info)
+		if err != nil {
+			d.counter.EndRequest()
+			return pr, err
+		}
+		oldDone := pr.Done
+		pr.Done = func(doneInfo balancer.DoneInfo) {
+			d.counter.EndRequest()
+			if oldDone != nil {
+				oldDone(doneInfo)
+			}
+		}
+		return pr, err
+	}
+
 	return d.s.Picker.Pick(info)
 }
diff --git a/xds/internal/balancer/edsbalancer/config.go b/xds/internal/balancer/edsbalancer/config.go
index 9b59cfa01ab3..11c1338c81f7 100644
--- a/xds/internal/balancer/edsbalancer/config.go
+++ b/xds/internal/balancer/edsbalancer/config.go
@@ -38,6 +38,15 @@ type EDSConfig struct {
 	// Name to use in EDS query.  If not present, defaults to the server
 	// name from the target URI.
 	EDSServiceName string
+	// MaxConcurrentRequests is the max number of concurrent request allowed for
+	// this service. If unset, default value 1024 is used.
+	//
+	// Note that this is not defined in the service config proto. And the reason
+	// is, we are dropping EDS and moving the features into cluster_impl. But in
+	// the mean time, to keep things working, we need to add this field. And it
+	// should be fine to add this extra field here, because EDS is only used in
+	// CDS today, so we have full control.
+	MaxConcurrentRequests *uint32
 	// LRS server to send load reports to.  If not present, load reporting
 	// will be disabled.  If set to the empty string, load reporting will
 	// be sent to the same server that we obtained CDS data from.
@@ -51,6 +60,7 @@ type edsConfigJSON struct {
 	ChildPolicy                []*loadBalancingConfig
 	FallbackPolicy             []*loadBalancingConfig
 	EDSServiceName             string
+	MaxConcurrentRequests      *uint32
 	LRSLoadReportingServerName *string
 }
 
@@ -64,6 +74,7 @@ func (l *EDSConfig) UnmarshalJSON(data []byte) error {
 	}
 
 	l.EDSServiceName = configJSON.EDSServiceName
+	l.MaxConcurrentRequests = configJSON.MaxConcurrentRequests
 	l.LrsLoadReportingServerName = configJSON.LRSLoadReportingServerName
 
 	for _, lbcfg := range configJSON.ChildPolicy {
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index a6b37f6277a1..423df7aed95e 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -113,9 +113,9 @@ type edsBalancerImplInterface interface {
 	handleSubConnStateChange(sc balancer.SubConn, state connectivity.State)
 	// updateState handle a balancer state update from the priority.
 	updateState(priority priorityType, s balancer.State)
-	// updateServiceRequestsCounter updates the service requests counter to the
+	// updateServiceRequestsConfig updates the service requests counter to the
 	// one for the given service name.
-	updateServiceRequestsCounter(serviceName string)
+	updateServiceRequestsConfig(serviceName string, max *uint32)
 	// close closes the eds balancer.
 	close()
 }
@@ -215,7 +215,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			x.logger.Warningf("failed to update xDS client: %v", err)
 		}
 
-		x.edsImpl.updateServiceRequestsCounter(cfg.EDSServiceName)
+		x.edsImpl.updateServiceRequestsConfig(cfg.EDSServiceName, cfg.MaxConcurrentRequests)
 
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 499ea5243b64..8d0cc8898bcf 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -45,6 +45,8 @@ import (
 // TODO: make this a environment variable?
 var defaultPriorityInitTimeout = 10 * time.Second
 
+const defaultServiceRequestCountMax = 1024
+
 type localityConfig struct {
 	weight uint32
 	addrs  []resolver.Address
@@ -101,6 +103,7 @@ type edsBalancerImpl struct {
 	drops                  []*dropper
 	innerState             balancer.State // The state of the picker without drop support.
 	serviceRequestsCounter *client.ServiceRequestsCounter
+	serviceRequestCountMax uint32
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
@@ -114,9 +117,10 @@ func newEDSBalancerImpl(cc balancer.ClientConn, bOpts balancer.BuildOptions, enq
 
 		enqueueChildBalancerStateUpdate: enqueueState,
 
-		priorityToLocalities: make(map[priorityType]*balancerGroupWithConfig),
-		priorityToState:      make(map[priorityType]*balancer.State),
-		subConnToPriority:    make(map[balancer.SubConn]priorityType),
+		priorityToLocalities:   make(map[priorityType]*balancerGroupWithConfig),
+		priorityToState:        make(map[priorityType]*balancer.State),
+		subConnToPriority:      make(map[balancer.SubConn]priorityType),
+		serviceRequestCountMax: defaultServiceRequestCountMax,
 	}
 	// Don't start balancer group here. Start it when handling the first EDS
 	// response. Otherwise the balancer group will be started with round-robin,
@@ -181,7 +185,7 @@ func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropC
 		// Update picker with old inner picker, new drops.
 		edsImpl.cc.UpdateState(balancer.State{
 			ConnectivityState: edsImpl.innerState.ConnectivityState,
-			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter)},
+			Picker:            newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter, edsImpl.serviceRequestCountMax)},
 		)
 	}
 	edsImpl.pickerMu.Unlock()
@@ -410,14 +414,17 @@ func (edsImpl *edsBalancerImpl) handleSubConnStateChange(sc balancer.SubConn, s
 	}
 }
 
-// updateConfig handles changes to the circuit breaking configuration.
-func (edsImpl *edsBalancerImpl) updateServiceRequestsCounter(serviceName string) {
+// updateServiceRequestsConfig handles changes to the circuit breaking configuration.
+func (edsImpl *edsBalancerImpl) updateServiceRequestsConfig(serviceName string, max *uint32) {
 	if !env.CircuitBreakingSupport {
 		return
 	}
 	if edsImpl.serviceRequestsCounter == nil || edsImpl.serviceRequestsCounter.ServiceName != serviceName {
 		edsImpl.serviceRequestsCounter = client.GetServiceRequestsCounter(serviceName)
 	}
+	if max != nil {
+		edsImpl.serviceRequestCountMax = *max
+	}
 }
 
 // updateState first handles priority, and then wraps picker in a drop picker
@@ -434,7 +441,7 @@ func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.St
 		defer edsImpl.pickerMu.Unlock()
 		edsImpl.innerState = s
 		// Don't reset drops when it's a state change.
-		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter)})
+		edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter, edsImpl.serviceRequestCountMax)})
 	}
 }
 
@@ -487,14 +494,16 @@ type dropPicker struct {
 	p         balancer.Picker
 	loadStore load.PerClusterReporter
 	counter   *client.ServiceRequestsCounter
+	countMax  uint32
 }
 
-func newDropPicker(p balancer.Picker, drops []*dropper, loadStore load.PerClusterReporter, counter *client.ServiceRequestsCounter) *dropPicker {
+func newDropPicker(p balancer.Picker, drops []*dropper, loadStore load.PerClusterReporter, counter *client.ServiceRequestsCounter, countMax uint32) *dropPicker {
 	return &dropPicker{
 		drops:     drops,
 		p:         p,
 		loadStore: loadStore,
 		counter:   counter,
+		countMax:  countMax,
 	}
 }
 
@@ -517,7 +526,7 @@ func (d *dropPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
 		return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
 	}
 	if d.counter != nil {
-		if err := d.counter.StartRequest(); err != nil {
+		if err := d.counter.StartRequest(d.countMax); err != nil {
 			// Drops by circuit breaking are reported with empty category. They
 			// will be reported only in total drops, but not in per category.
 			if d.loadStore != nil {
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 69ce6dcac1da..7f8a0d3aa82f 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -579,9 +579,8 @@ func (s) TestEDS_CircuitBreaking(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
 	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
-	edsb.updateServiceRequestsCounter("test")
 	var maxRequests uint32 = 50
-	client.SetMaxRequests("test", &maxRequests)
+	edsb.updateServiceRequestsConfig("test", &maxRequests)
 
 	// One locality with one backend.
 	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
@@ -738,7 +737,7 @@ func (s) TestDropPicker(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			p := newDropPicker(constPicker, tt.drops, nil, nil)
+			p := newDropPicker(constPicker, tt.drops, nil, nil, defaultServiceRequestCountMax)
 
 			// scCount is the number of sc's returned by pick. The opposite of
 			// drop-count.
@@ -786,9 +785,8 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 		cbMaxRequests   = 20
 	)
 	var maxRequestsTemp uint32 = cbMaxRequests
-	client.SetMaxRequests(testServiceName, &maxRequestsTemp)
+	edsb.updateServiceRequestsConfig(testServiceName, &maxRequestsTemp)
 	defer client.ClearCounterForTesting(testServiceName)
-	edsb.updateServiceRequestsCounter(testServiceName)
 
 	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
 
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index ea5ec39c4568..5fe1f2ef6b90 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -116,6 +116,7 @@ type fakeEDSBalancer struct {
 	subconnStateChange *testutils.Channel
 	edsUpdate          *testutils.Channel
 	serviceName        *testutils.Channel
+	serviceRequestMax  *testutils.Channel
 }
 
 func (f *fakeEDSBalancer) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
@@ -132,8 +133,9 @@ func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) {
 
 func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {}
 
-func (f *fakeEDSBalancer) updateServiceRequestsCounter(serviceName string) {
+func (f *fakeEDSBalancer) updateServiceRequestsConfig(serviceName string, max *uint32) {
 	f.serviceName.Send(serviceName)
+	f.serviceRequestMax.Send(max)
 }
 
 func (f *fakeEDSBalancer) close() {}
@@ -186,6 +188,25 @@ func (f *fakeEDSBalancer) waitForCounterUpdate(ctx context.Context, wantServiceN
 	return nil
 }
 
+func (f *fakeEDSBalancer) waitForCountMaxUpdate(ctx context.Context, want *uint32) error {
+	val, err := f.serviceRequestMax.Receive(ctx)
+	if err != nil {
+		return err
+	}
+	got := val.(*uint32)
+
+	if got == nil && want == nil {
+		return nil
+	}
+	if got != nil && want != nil {
+		if *got != *want {
+			return fmt.Errorf("got countMax %v, want %v", *got, *want)
+		}
+		return nil
+	}
+	return fmt.Errorf("got countMax %+v, want %+v", got, want)
+}
+
 func newFakeEDSBalancer(cc balancer.ClientConn) edsBalancerImplInterface {
 	return &fakeEDSBalancer{
 		cc:                 cc,
@@ -193,6 +214,7 @@ func newFakeEDSBalancer(cc balancer.ClientConn) edsBalancerImplInterface {
 		subconnStateChange: testutils.NewChannelWithSize(10),
 		edsUpdate:          testutils.NewChannelWithSize(10),
 		serviceName:        testutils.NewChannelWithSize(10),
+		serviceRequestMax:  testutils.NewChannelWithSize(10),
 	}
 }
 
@@ -328,15 +350,20 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
 		t.Fatal(err)
 	}
+	if err := edsLB.waitForCountMaxUpdate(ctx, nil); err != nil {
+		t.Fatal(err)
+	}
 
+	var testCountMax uint32 = 100
 	lbCfgB := &loadBalancingConfig{
 		Name:   fakeBalancerB,
 		Config: json.RawMessage("{}"),
 	}
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			ChildPolicy:    lbCfgB,
-			EDSServiceName: testServiceName,
+			ChildPolicy:           lbCfgB,
+			EDSServiceName:        testServiceName,
+			MaxConcurrentRequests: &testCountMax,
 		},
 	}); err != nil {
 		t.Fatalf("edsB.UpdateClientConnState() failed: %v", err)
@@ -349,6 +376,9 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 		// eds_impl will compare the service names, and skip if it didn't change.
 		t.Fatal(err)
 	}
+	if err := edsLB.waitForCountMaxUpdate(ctx, &testCountMax); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // TestSubConnStateChange verifies if the top-level edsBalancer passes on
@@ -606,15 +636,23 @@ func (s) TestCounterUpdate(t *testing.T) {
 	}
 	defer edsB.Close()
 
+	var testCountMax uint32 = 100
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{EDSServiceName: "foobar-1"},
+		BalancerConfig: &EDSConfig{
+			EDSServiceName:        "foobar-1",
+			MaxConcurrentRequests: &testCountMax,
+		},
 	}); err != nil {
 		t.Fatal(err)
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	if err := edsB.(*edsBalancer).edsImpl.(*fakeEDSBalancer).waitForCounterUpdate(ctx, "foobar-1"); err != nil {
+	edsI := edsB.(*edsBalancer).edsImpl.(*fakeEDSBalancer)
+	if err := edsI.waitForCounterUpdate(ctx, "foobar-1"); err != nil {
+		t.Fatal(err)
+	}
+	if err := edsI.waitForCountMaxUpdate(ctx, &testCountMax); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -642,6 +680,7 @@ func (s) TestBalancerConfigParsing(t *testing.T) {
 		t.Fatalf("%v", err)
 	}
 
+	var testMaxConcurrentRequests uint32 = 123
 	tests := []struct {
 		name    string
 		js      json.RawMessage
@@ -690,6 +729,7 @@ func (s) TestBalancerConfigParsing(t *testing.T) {
     {"fake_balancer_A": {}}
   ],
   "edsServiceName": "eds.service",
+  "maxConcurrentRequests": 123,
   "lrsLoadReportingServerName": "lrs.server"
 }`),
 			want: &EDSConfig{
@@ -702,6 +742,7 @@ func (s) TestBalancerConfigParsing(t *testing.T) {
 					Config: json.RawMessage("{}"),
 				},
 				EDSServiceName:             testEDSName,
+				MaxConcurrentRequests:      &testMaxConcurrentRequests,
 				LrsLoadReportingServerName: &testLRSName,
 			},
 		},
diff --git a/xds/internal/client/requests_counter.go b/xds/internal/client/requests_counter.go
index 74b80f1c3f7c..7ef18345ed6c 100644
--- a/xds/internal/client/requests_counter.go
+++ b/xds/internal/client/requests_counter.go
@@ -24,8 +24,6 @@ import (
 	"sync/atomic"
 )
 
-const defaultMaxRequests uint32 = 1024
-
 type servicesRequestsCounter struct {
 	mu       sync.Mutex
 	services map[string]*ServiceRequestsCounter
@@ -39,25 +37,12 @@ var src = &servicesRequestsCounter{
 // service with the provided name.
 type ServiceRequestsCounter struct {
 	ServiceName string
-	maxRequests uint32
 	numRequests uint32
 }
 
 // GetServiceRequestsCounter returns the ServiceRequestsCounter with the
 // provided serviceName. If one does not exist, it creates it.
 func GetServiceRequestsCounter(serviceName string) *ServiceRequestsCounter {
-	src.mu.Lock()
-	defer src.mu.Unlock()
-	c, ok := src.services[serviceName]
-	if !ok {
-		c = &ServiceRequestsCounter{ServiceName: serviceName, maxRequests: defaultMaxRequests}
-		src.services[serviceName] = c
-	}
-	return c
-}
-
-// SetMaxRequests updates the max requests for a service's counter.
-func SetMaxRequests(serviceName string, maxRequests *uint32) {
 	src.mu.Lock()
 	defer src.mu.Unlock()
 	c, ok := src.services[serviceName]
@@ -65,18 +50,14 @@ func SetMaxRequests(serviceName string, maxRequests *uint32) {
 		c = &ServiceRequestsCounter{ServiceName: serviceName}
 		src.services[serviceName] = c
 	}
-	if maxRequests != nil {
-		c.maxRequests = *maxRequests
-	} else {
-		c.maxRequests = defaultMaxRequests
-	}
+	return c
 }
 
 // StartRequest starts a request for a service, incrementing its number of
 // requests by 1. Returns an error if the max number of requests is exceeded.
-func (c *ServiceRequestsCounter) StartRequest() error {
-	if atomic.LoadUint32(&c.numRequests) >= atomic.LoadUint32(&c.maxRequests) {
-		return fmt.Errorf("max requests %v exceeded on service %v", c.maxRequests, c.ServiceName)
+func (c *ServiceRequestsCounter) StartRequest(max uint32) error {
+	if atomic.LoadUint32(&c.numRequests) >= max {
+		return fmt.Errorf("max requests %v exceeded on service %v", max, c.ServiceName)
 	}
 	atomic.AddUint32(&c.numRequests, 1)
 	return nil
@@ -97,6 +78,5 @@ func ClearCounterForTesting(serviceName string) {
 	if !ok {
 		return
 	}
-	c.maxRequests = defaultMaxRequests
 	c.numRequests = 0
 }
diff --git a/xds/internal/client/requests_counter_test.go b/xds/internal/client/requests_counter_test.go
index 09295b982ecd..fe532724d14e 100644
--- a/xds/internal/client/requests_counter_test.go
+++ b/xds/internal/client/requests_counter_test.go
@@ -56,7 +56,6 @@ func resetServiceRequestsCounter() {
 }
 
 func testCounter(t *testing.T, test counterTest) {
-	SetMaxRequests(test.name, &test.maxRequests)
 	requestsStarted := make(chan struct{})
 	requestsSent := sync.WaitGroup{}
 	requestsSent.Add(int(test.numRequests))
@@ -68,7 +67,7 @@ func testCounter(t *testing.T, test counterTest) {
 		go func() {
 			counter := GetServiceRequestsCounter(test.name)
 			defer requestsDone.Done()
-			err := counter.StartRequest()
+			err := counter.StartRequest(test.maxRequests)
 			if err == nil {
 				atomic.AddUint32(&successes, 1)
 			} else {
@@ -98,7 +97,7 @@ func testCounter(t *testing.T, test counterTest) {
 }
 
 func (s) TestRequestsCounter(t *testing.T) {
-	resetServiceRequestsCounter()
+	defer resetServiceRequestsCounter()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			testCounter(t, test)
@@ -107,7 +106,7 @@ func (s) TestRequestsCounter(t *testing.T) {
 }
 
 func (s) TestGetServiceRequestsCounter(t *testing.T) {
-	resetServiceRequestsCounter()
+	defer resetServiceRequestsCounter()
 	for _, test := range tests {
 		counterA := GetServiceRequestsCounter(test.name)
 		counterB := GetServiceRequestsCounter(test.name)
@@ -118,39 +117,40 @@ func (s) TestGetServiceRequestsCounter(t *testing.T) {
 }
 
 func startRequests(t *testing.T, n uint32, max uint32, counter *ServiceRequestsCounter) {
-	SetMaxRequests(counter.ServiceName, &max)
 	for i := uint32(0); i < n; i++ {
-		if err := counter.StartRequest(); err != nil {
+		if err := counter.StartRequest(max); err != nil {
 			t.Fatalf("error starting initial request: %v", err)
 		}
 	}
 }
 
 func (s) TestSetMaxRequestsIncreased(t *testing.T) {
-	resetServiceRequestsCounter()
+	defer resetServiceRequestsCounter()
 	const serviceName string = "set-max-requests-increased"
 	var initialMax uint32 = 16
+
 	counter := GetServiceRequestsCounter(serviceName)
 	startRequests(t, initialMax, initialMax, counter)
-	if err := counter.StartRequest(); err == nil {
+	if err := counter.StartRequest(initialMax); err == nil {
 		t.Fatal("unexpected success on start request after max met")
 	}
+
 	newMax := initialMax + 1
-	SetMaxRequests(counter.ServiceName, &newMax)
-	if err := counter.StartRequest(); err != nil {
+	if err := counter.StartRequest(newMax); err != nil {
 		t.Fatalf("unexpected error on start request after max increased: %v", err)
 	}
 }
 
 func (s) TestSetMaxRequestsDecreased(t *testing.T) {
-	resetServiceRequestsCounter()
+	defer resetServiceRequestsCounter()
 	const serviceName string = "set-max-requests-decreased"
 	var initialMax uint32 = 16
+
 	counter := GetServiceRequestsCounter(serviceName)
 	startRequests(t, initialMax-1, initialMax, counter)
+
 	newMax := initialMax - 1
-	SetMaxRequests(counter.ServiceName, &newMax)
-	if err := counter.StartRequest(); err == nil {
+	if err := counter.StartRequest(newMax); err == nil {
 		t.Fatalf("unexpected success on start request after max decreased: %v", err)
 	}
 }

From 26c143bd5f59344a4b8a1e491e0f5e18aa97abc7 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 18 Feb 2021 10:12:25 -0800
Subject: [PATCH 365/481] circuit breaking: update picker inline when there's a
 counter update (#4212)

---
 xds/internal/balancer/edsbalancer/eds_impl.go | 19 +++++++-
 .../balancer/edsbalancer/eds_impl_test.go     | 45 +++++++++++++++++++
 2 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 8d0cc8898bcf..894b9a0152bc 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -419,12 +419,29 @@ func (edsImpl *edsBalancerImpl) updateServiceRequestsConfig(serviceName string,
 	if !env.CircuitBreakingSupport {
 		return
 	}
+	edsImpl.pickerMu.Lock()
+	var updatePicker bool
 	if edsImpl.serviceRequestsCounter == nil || edsImpl.serviceRequestsCounter.ServiceName != serviceName {
 		edsImpl.serviceRequestsCounter = client.GetServiceRequestsCounter(serviceName)
+		updatePicker = true
 	}
+
+	var newMax uint32 = defaultServiceRequestCountMax
 	if max != nil {
-		edsImpl.serviceRequestCountMax = *max
+		newMax = *max
+	}
+	if edsImpl.serviceRequestCountMax != newMax {
+		edsImpl.serviceRequestCountMax = newMax
+		updatePicker = true
+	}
+	if updatePicker && edsImpl.innerState.Picker != nil {
+		// Update picker with old inner picker, new counter and counterMax.
+		edsImpl.cc.UpdateState(balancer.State{
+			ConnectivityState: edsImpl.innerState.ConnectivityState,
+			Picker:            newDropPicker(edsImpl.innerState.Picker, edsImpl.drops, edsImpl.loadReporter, edsImpl.serviceRequestsCounter, edsImpl.serviceRequestCountMax)},
+		)
 	}
+	edsImpl.pickerMu.Unlock()
 }
 
 // updateState first handles priority, and then wraps picker in a drop picker
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 7f8a0d3aa82f..3334376f4a9a 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -629,6 +629,51 @@ func (s) TestEDS_CircuitBreaking(t *testing.T) {
 	for _, done := range dones {
 		done()
 	}
+
+	// Send another update, with only circuit breaking update (and no picker
+	// update afterwards). Make sure the new picker uses the new configs.
+	var maxRequests2 uint32 = 10
+	edsb.updateServiceRequestsConfig("test", &maxRequests2)
+
+	// Picks with drops.
+	dones = []func(){}
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 100; i++ {
+		pr, err := p2.Pick(balancer.PickInfo{})
+		if i < 10 && err != nil {
+			t.Errorf("The first 10%% picks should be non-drops, got error %v", err)
+		} else if i > 10 && err == nil {
+			t.Errorf("The next 90%% picks should be drops, got error <nil>")
+		}
+		dones = append(dones, func() {
+			if pr.Done != nil {
+				pr.Done(balancer.DoneInfo{})
+			}
+		})
+	}
+
+	for _, done := range dones {
+		done()
+	}
+	dones = []func(){}
+
+	// Pick without drops.
+	for i := 0; i < 10; i++ {
+		pr, err := p2.Pick(balancer.PickInfo{})
+		if err != nil {
+			t.Errorf("The next 10%% picks should be non-drops, got error %v", err)
+		}
+		dones = append(dones, func() {
+			if pr.Done != nil {
+				pr.Done(balancer.DoneInfo{})
+			}
+		})
+	}
+
+	// Without this, future tests with the same service name will fail.
+	for _, done := range dones {
+		done()
+	}
 }
 
 func init() {

From 25cf9393fa212572bd9376383a1d4b9675c9e940 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Mon, 22 Feb 2021 16:04:34 -0800
Subject: [PATCH 366/481] vet: allow golint to run on generated protos (#4220)

---
 vet.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vet.sh b/vet.sh
index 605d7a80cfe6..85b32bb6dd5e 100755
--- a/vet.sh
+++ b/vet.sh
@@ -107,7 +107,7 @@ go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go
 # - gofmt, goimports, golint (with exceptions for generated code), go vet.
 gofmt -s -d -l . 2>&1 | fail_on_output
 goimports -l . 2>&1 | not grep -vE "\.pb\.go"
-golint ./... 2>&1 | not grep -vE "\.pb\.go:"
+golint ./... 2>&1 | not grep -vE "/testv3\.pb\.go:"
 go vet -all ./...
 
 misspell -error .

From dabedfb38b74d175936dd0b1b256cdbc0bff09ff Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 23 Feb 2021 09:47:33 -0800
Subject: [PATCH 367/481] encoding/proto: do not panic when types do not match
 (#4218)

---
 encoding/proto/proto.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index e1230fdd358a..3009b35afe7d 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -21,6 +21,8 @@
 package proto
 
 import (
+	"fmt"
+
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc/encoding"
 )
@@ -36,11 +38,19 @@ func init() {
 type codec struct{}
 
 func (codec) Marshal(v interface{}) ([]byte, error) {
-	return proto.Marshal(v.(proto.Message))
+	vv, ok := v.(proto.Message)
+	if !ok {
+		return nil, fmt.Errorf("failed to marshal, message is %T, want proto.Message", v)
+	}
+	return proto.Marshal(vv)
 }
 
 func (codec) Unmarshal(data []byte, v interface{}) error {
-	return proto.Unmarshal(data, v.(proto.Message))
+	vv, ok := v.(proto.Message)
+	if !ok {
+		return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v)
+	}
+	return proto.Unmarshal(data, vv)
 }
 
 func (codec) Name() string {

From c8cef768c7f2342791a42babe34c06fe8ee74852 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 25 Feb 2021 12:57:45 -0800
Subject: [PATCH 368/481] csds: update client resource cache to keep and dump
 metadata (#4217)

---
 xds/internal/client/callback.go               | 104 +++-
 xds/internal/client/cds_test.go               |   2 +-
 xds/internal/client/client.go                 |  85 +++-
 xds/internal/client/client_test.go            |  19 +-
 xds/internal/client/dump.go                   | 123 +++++
 xds/internal/client/dump_test.go              | 474 ++++++++++++++++++
 xds/internal/client/eds_test.go               |   2 +-
 xds/internal/client/lds_test.go               |   4 +-
 xds/internal/client/rds_test.go               |   2 +-
 xds/internal/client/v2/ack_test.go            |   2 +-
 xds/internal/client/v2/cds_test.go            |   2 +-
 xds/internal/client/v2/client.go              |  16 +-
 xds/internal/client/v2/client_test.go         |  47 +-
 xds/internal/client/v2/eds_test.go            |   2 +-
 xds/internal/client/v2/lds_test.go            |   2 +-
 xds/internal/client/v2/rds_test.go            |   2 +-
 xds/internal/client/v3/client.go              |  16 +-
 xds/internal/client/watchers.go               |  14 +-
 xds/internal/client/watchers_cluster_test.go  |  22 +-
 .../client/watchers_endpoints_test.go         |  14 +-
 xds/internal/client/watchers_listener_test.go |  20 +-
 xds/internal/client/watchers_route_test.go    |  14 +-
 xds/internal/client/xds.go                    |  40 +-
 23 files changed, 909 insertions(+), 119 deletions(-)
 create mode 100644 xds/internal/client/dump.go
 create mode 100644 xds/internal/client/dump_test.go

diff --git a/xds/internal/client/callback.go b/xds/internal/client/callback.go
index 90ffcdec6219..da8e2f62d6c0 100644
--- a/xds/internal/client/callback.go
+++ b/xds/internal/client/callback.go
@@ -74,26 +74,50 @@ func (c *clientImpl) callCallback(wiu *watcherInfoWithUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *clientImpl) NewListeners(updates map[string]ListenerUpdate) {
+func (c *clientImpl) NewListeners(updates map[string]ListenerUpdate, metadata UpdateMetadata) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	if metadata.ErrState != nil {
+		// On NACK, update overall version to the NACKed resp.
+		c.ldsVersion = metadata.ErrState.Version
+		for name := range updates {
+			if _, ok := c.ldsWatchers[name]; ok {
+				// On error, keep previous version for each resource. But update
+				// status and error.
+				mdCopy := c.ldsMD[name]
+				mdCopy.ErrState = metadata.ErrState
+				mdCopy.Status = metadata.Status
+				c.ldsMD[name] = mdCopy
+				// TODO: send the NACK error to the watcher.
+			}
+		}
+		return
+	}
+
+	// If no error received, the status is ACK.
+	c.ldsVersion = metadata.Version
 	for name, update := range updates {
 		if s, ok := c.ldsWatchers[name]; ok {
+			// Only send the update if this is not an error.
 			for wi := range s {
 				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, update)
 			c.ldsCache[name] = update
+			c.ldsMD[name] = metadata
 		}
 	}
+	// Resources not in the new update were removed by the server, so delete
+	// them.
 	for name := range c.ldsCache {
 		if _, ok := updates[name]; !ok {
-			// If resource exists in cache, but not in the new update, delete it
-			// from cache, and also send an resource not found error to indicate
-			// resource removed.
+			// If resource exists in cache, but not in the new update, delete
+			// the resource from cache, and also send an resource not found
+			// error to indicate resource removed.
 			delete(c.ldsCache, name)
+			c.ldsMD[name] = UpdateMetadata{Status: ServiceStatusNotExist}
 			for wi := range c.ldsWatchers[name] {
 				wi.resourceNotFound()
 			}
@@ -109,18 +133,39 @@ func (c *clientImpl) NewListeners(updates map[string]ListenerUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *clientImpl) NewRouteConfigs(updates map[string]RouteConfigUpdate) {
+func (c *clientImpl) NewRouteConfigs(updates map[string]RouteConfigUpdate, metadata UpdateMetadata) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	if metadata.ErrState != nil {
+		// On NACK, update overall version to the NACKed resp.
+		c.rdsVersion = metadata.ErrState.Version
+		for name := range updates {
+			if _, ok := c.rdsWatchers[name]; ok {
+				// On error, keep previous version for each resource. But update
+				// status and error.
+				mdCopy := c.rdsMD[name]
+				mdCopy.ErrState = metadata.ErrState
+				mdCopy.Status = metadata.Status
+				c.rdsMD[name] = mdCopy
+				// TODO: send the NACK error to the watcher.
+			}
+		}
+		return
+	}
+
+	// If no error received, the status is ACK.
+	c.rdsVersion = metadata.Version
 	for name, update := range updates {
 		if s, ok := c.rdsWatchers[name]; ok {
+			// Only send the update if this is not an error.
 			for wi := range s {
 				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, update)
 			c.rdsCache[name] = update
+			c.rdsMD[name] = metadata
 		}
 	}
 }
@@ -130,26 +175,50 @@ func (c *clientImpl) NewRouteConfigs(updates map[string]RouteConfigUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *clientImpl) NewClusters(updates map[string]ClusterUpdate) {
+func (c *clientImpl) NewClusters(updates map[string]ClusterUpdate, metadata UpdateMetadata) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	if metadata.ErrState != nil {
+		// On NACK, update overall version to the NACKed resp.
+		c.cdsVersion = metadata.ErrState.Version
+		for name := range updates {
+			if _, ok := c.cdsWatchers[name]; ok {
+				// On error, keep previous version for each resource. But update
+				// status and error.
+				mdCopy := c.cdsMD[name]
+				mdCopy.ErrState = metadata.ErrState
+				mdCopy.Status = metadata.Status
+				c.cdsMD[name] = mdCopy
+				// TODO: send the NACK error to the watcher.
+			}
+		}
+		return
+	}
+
+	// If no error received, the status is ACK.
+	c.cdsVersion = metadata.Version
 	for name, update := range updates {
 		if s, ok := c.cdsWatchers[name]; ok {
+			// Only send the update if this is not an error.
 			for wi := range s {
 				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, update)
 			c.cdsCache[name] = update
+			c.cdsMD[name] = metadata
 		}
 	}
+	// Resources not in the new update were removed by the server, so delete
+	// them.
 	for name := range c.cdsCache {
 		if _, ok := updates[name]; !ok {
 			// If resource exists in cache, but not in the new update, delete it
 			// from cache, and also send an resource not found error to indicate
 			// resource removed.
 			delete(c.cdsCache, name)
+			c.ldsMD[name] = UpdateMetadata{Status: ServiceStatusNotExist}
 			for wi := range c.cdsWatchers[name] {
 				wi.resourceNotFound()
 			}
@@ -165,18 +234,39 @@ func (c *clientImpl) NewClusters(updates map[string]ClusterUpdate) {
 //
 // A response can contain multiple resources. They will be parsed and put in a
 // map from resource name to the resource content.
-func (c *clientImpl) NewEndpoints(updates map[string]EndpointsUpdate) {
+func (c *clientImpl) NewEndpoints(updates map[string]EndpointsUpdate, metadata UpdateMetadata) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	if metadata.ErrState != nil {
+		// On NACK, update overall version to the NACKed resp.
+		c.edsVersion = metadata.ErrState.Version
+		for name := range updates {
+			if _, ok := c.edsWatchers[name]; ok {
+				// On error, keep previous version for each resource. But update
+				// status and error.
+				mdCopy := c.edsMD[name]
+				mdCopy.ErrState = metadata.ErrState
+				mdCopy.Status = metadata.Status
+				c.edsMD[name] = mdCopy
+				// TODO: send the NACK error to the watcher.
+			}
+		}
+		return
+	}
+
+	// If no error received, the status is ACK.
+	c.edsVersion = metadata.Version
 	for name, update := range updates {
 		if s, ok := c.edsWatchers[name]; ok {
+			// Only send the update if this is not an error.
 			for wi := range s {
 				wi.newUpdate(update)
 			}
 			// Sync cache.
 			c.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, update)
 			c.edsCache[name] = update
+			c.edsMD[name] = metadata
 		}
 	}
 }
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 5d2366d64888..9839660ff0e7 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -685,7 +685,7 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := UnmarshalCluster(test.resources, nil)
+			update, _, err := UnmarshalCluster("", test.resources, nil)
 			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("UnmarshalCluster(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
 			}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 10c45415a574..7c7ebf3e4cd2 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -30,6 +30,7 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 
 	"google.golang.org/grpc/xds/internal/client/load"
 
@@ -129,14 +130,61 @@ type loadReportingOptions struct {
 // resource updates from an APIClient for a specific version.
 type UpdateHandler interface {
 	// NewListeners handles updates to xDS listener resources.
-	NewListeners(map[string]ListenerUpdate)
+	NewListeners(map[string]ListenerUpdate, UpdateMetadata)
 	// NewRouteConfigs handles updates to xDS RouteConfiguration resources.
-	NewRouteConfigs(map[string]RouteConfigUpdate)
+	NewRouteConfigs(map[string]RouteConfigUpdate, UpdateMetadata)
 	// NewClusters handles updates to xDS Cluster resources.
-	NewClusters(map[string]ClusterUpdate)
+	NewClusters(map[string]ClusterUpdate, UpdateMetadata)
 	// NewEndpoints handles updates to xDS ClusterLoadAssignment (or tersely
 	// referred to as Endpoints) resources.
-	NewEndpoints(map[string]EndpointsUpdate)
+	NewEndpoints(map[string]EndpointsUpdate, UpdateMetadata)
+}
+
+// ServiceStatus is the status of the update.
+type ServiceStatus int
+
+const (
+	// ServiceStatusUnknown is the default state, before a watch is started for
+	// the resource.
+	ServiceStatusUnknown ServiceStatus = iota
+	// ServiceStatusRequested is when the watch is started, but before and
+	// response is received.
+	ServiceStatusRequested
+	// ServiceStatusNotExist is when the resource doesn't exist in
+	// state-of-the-world responses (e.g. LDS and CDS), which means the resource
+	// is removed by the management server.
+	ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS.
+	// ServiceStatusACKed is when the resource is ACKed.
+	ServiceStatusACKed
+	// ServiceStatusNACKed is when the resource is NACKed.
+	ServiceStatusNACKed
+)
+
+// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state
+// when a response is NACKed.
+type UpdateErrorMetadata struct {
+	// Version is the version of the NACKed response.
+	Version string
+	// Err contains why the response was NACKed.
+	Err error
+	// Timestamp is when the NACKed response was received.
+	Timestamp time.Time
+}
+
+// UpdateMetadata contains the metadata for each update, including timestamp,
+// raw message, and so on.
+type UpdateMetadata struct {
+	// Status is the status of this resource, e.g. ACKed, NACKed, or
+	// Not_exist(removed).
+	Status ServiceStatus
+	// Version is the version of the xds response. Note that this is the version
+	// of the resource in use (previous ACKed). If a response is NACKed, the
+	// NACKed version is in ErrState.
+	Version string
+	// Timestamp is when the response is received.
+	Timestamp time.Time
+	// ErrState is set when the update is NACKed.
+	ErrState *UpdateErrorMetadata
 }
 
 // ListenerUpdate contains information received in an LDS response, which is of
@@ -151,6 +199,9 @@ type ListenerUpdate struct {
 	// common_http_protocol_options.max_stream_duration field, or zero if
 	// unset.
 	MaxStreamDuration time.Duration
+
+	// Raw is the resource from the xds response.
+	Raw *anypb.Any
 }
 
 func (lu *ListenerUpdate) String() string {
@@ -161,6 +212,9 @@ func (lu *ListenerUpdate) String() string {
 // of interest to the registered RDS watcher.
 type RouteConfigUpdate struct {
 	VirtualHosts []*VirtualHost
+
+	// Raw is the resource from the xds response.
+	Raw *anypb.Any
 }
 
 // VirtualHost contains the routes for a list of Domains.
@@ -255,6 +309,9 @@ type ClusterUpdate struct {
 	SecurityCfg *SecurityConfig
 	// MaxRequests for circuit breaking, if any (otherwise nil).
 	MaxRequests *uint32
+
+	// Raw is the resource from the xds response.
+	Raw *anypb.Any
 }
 
 // OverloadDropConfig contains the config to drop overloads.
@@ -301,6 +358,9 @@ type Locality struct {
 type EndpointsUpdate struct {
 	Drops      []OverloadDropConfig
 	Localities []Locality
+
+	// Raw is the resource from the xds response.
+	Raw *anypb.Any
 }
 
 // Function to be overridden in tests.
@@ -328,16 +388,27 @@ type clientImpl struct {
 
 	logger *grpclog.PrefixLogger
 
-	updateCh    *buffer.Unbounded // chan *watcherInfoWithUpdate
+	updateCh *buffer.Unbounded // chan *watcherInfoWithUpdate
+	// All the following maps are to keep the updates/metadata in a cache.
+	// TODO: move them to a separate struct/package, to cleanup the xds_client.
+	// And CSDS handler can be implemented directly by the cache.
 	mu          sync.Mutex
 	ldsWatchers map[string]map[*watchInfo]bool
+	ldsVersion  string // Only used in CSDS.
 	ldsCache    map[string]ListenerUpdate
+	ldsMD       map[string]UpdateMetadata
 	rdsWatchers map[string]map[*watchInfo]bool
+	rdsVersion  string // Only used in CSDS.
 	rdsCache    map[string]RouteConfigUpdate
+	rdsMD       map[string]UpdateMetadata
 	cdsWatchers map[string]map[*watchInfo]bool
+	cdsVersion  string // Only used in CSDS.
 	cdsCache    map[string]ClusterUpdate
+	cdsMD       map[string]UpdateMetadata
 	edsWatchers map[string]map[*watchInfo]bool
+	edsVersion  string // Only used in CSDS.
 	edsCache    map[string]EndpointsUpdate
+	edsMD       map[string]UpdateMetadata
 
 	// Changes to map lrsClients and the lrsClient inside the map need to be
 	// protected by lrsMu.
@@ -383,12 +454,16 @@ func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration) (
 		updateCh:    buffer.NewUnbounded(),
 		ldsWatchers: make(map[string]map[*watchInfo]bool),
 		ldsCache:    make(map[string]ListenerUpdate),
+		ldsMD:       make(map[string]UpdateMetadata),
 		rdsWatchers: make(map[string]map[*watchInfo]bool),
 		rdsCache:    make(map[string]RouteConfigUpdate),
+		rdsMD:       make(map[string]UpdateMetadata),
 		cdsWatchers: make(map[string]map[*watchInfo]bool),
 		cdsCache:    make(map[string]ClusterUpdate),
+		cdsMD:       make(map[string]UpdateMetadata),
 		edsWatchers: make(map[string]map[*watchInfo]bool),
 		edsCache:    make(map[string]EndpointsUpdate),
+		edsMD:       make(map[string]UpdateMetadata),
 		lrsClients:  make(map[string]*lrsClient),
 	}
 
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 721492e5f430..f3651d6f5122 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/protobuf/testing/protocmp"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -58,6 +59,20 @@ const (
 	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
 )
 
+var (
+	cmpOpts = cmp.Options{
+		cmpopts.EquateEmpty(),
+		cmp.Comparer(func(a, b time.Time) bool { return true }),
+		cmp.Comparer(func(x, y error) bool {
+			if x == nil || y == nil {
+				return x == nil && y == nil
+			}
+			return x.Error() == y.Error()
+		}),
+		protocmp.Transform(),
+	}
+)
+
 func clientOpts(balancerName string, overrideWatchExpiryTimeout bool) (*bootstrap.Config, time.Duration) {
 	watchExpiryTimeout := defaultWatchExpiryTimeout
 	if overrideWatchExpiryTimeout {
@@ -159,13 +174,13 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
 	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
-	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate2})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate2}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate2); err != nil {
 		t.Fatal(err)
 	}
diff --git a/xds/internal/client/dump.go b/xds/internal/client/dump.go
new file mode 100644
index 000000000000..3fd18f6103b3
--- /dev/null
+++ b/xds/internal/client/dump.go
@@ -0,0 +1,123 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import anypb "github.com/golang/protobuf/ptypes/any"
+
+// UpdateWithMD contains the raw message of the update and the metadata,
+// including version, raw message, timestamp.
+//
+// This is to be used for config dump and CSDS, not directly by users (like
+// resolvers/balancers).
+type UpdateWithMD struct {
+	MD  UpdateMetadata
+	Raw *anypb.Any
+}
+
+func rawFromCache(s string, cache interface{}) *anypb.Any {
+	switch c := cache.(type) {
+	case map[string]ListenerUpdate:
+		v, ok := c[s]
+		if !ok {
+			return nil
+		}
+		return v.Raw
+	case map[string]RouteConfigUpdate:
+		v, ok := c[s]
+		if !ok {
+			return nil
+		}
+		return v.Raw
+	case map[string]ClusterUpdate:
+		v, ok := c[s]
+		if !ok {
+			return nil
+		}
+		return v.Raw
+	case map[string]EndpointsUpdate:
+		v, ok := c[s]
+		if !ok {
+			return nil
+		}
+		return v.Raw
+	default:
+		return nil
+	}
+}
+
+func (c *clientImpl) dump(t ResourceType) (string, map[string]UpdateWithMD) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	var (
+		version string
+		md      map[string]UpdateMetadata
+		cache   interface{}
+	)
+	switch t {
+	case ListenerResource:
+		version = c.ldsVersion
+		md = c.ldsMD
+		cache = c.ldsCache
+	case RouteConfigResource:
+		version = c.rdsVersion
+		md = c.rdsMD
+		cache = c.rdsCache
+	case ClusterResource:
+		version = c.cdsVersion
+		md = c.cdsMD
+		cache = c.cdsCache
+	case EndpointsResource:
+		version = c.edsVersion
+		md = c.edsMD
+		cache = c.edsCache
+	default:
+		c.logger.Errorf("dumping resource of unknown type: %v", t)
+		return "", nil
+	}
+
+	ret := make(map[string]UpdateWithMD, len(md))
+	for s, md := range md {
+		ret[s] = UpdateWithMD{
+			MD:  md,
+			Raw: rawFromCache(s, cache),
+		}
+	}
+	return version, ret
+}
+
+// DumpLDS returns the status and contents of LDS.
+func (c *clientImpl) DumpLDS() (string, map[string]UpdateWithMD) {
+	return c.dump(ListenerResource)
+}
+
+// DumpRDS returns the status and contents of RDS.
+func (c *clientImpl) DumpRDS() (string, map[string]UpdateWithMD) {
+	return c.dump(RouteConfigResource)
+}
+
+// DumpCDS returns the status and contents of CDS.
+func (c *clientImpl) DumpCDS() (string, map[string]UpdateWithMD) {
+	return c.dump(ClusterResource)
+}
+
+// DumpEDS returns the status and contents of EDS.
+func (c *clientImpl) DumpEDS() (string, map[string]UpdateWithMD) {
+	return c.dump(EndpointsResource)
+}
diff --git a/xds/internal/client/dump_test.go b/xds/internal/client/dump_test.go
new file mode 100644
index 000000000000..d460da0bbd44
--- /dev/null
+++ b/xds/internal/client/dump_test.go
@@ -0,0 +1,474 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+func (s) TestLDSConfigDump(t *testing.T) {
+	const testVersion = "test-version-lds"
+	var (
+		ldsTargets       = []string{"lds.target.good:0000", "lds.target.good:1111"}
+		routeConfigNames = []string{"route-config-0", "route-config-1"}
+		listenerRaws     = make(map[string]*anypb.Any, len(ldsTargets))
+	)
+
+	for i := range ldsTargets {
+		listenersT := &v3listenerpb.Listener{
+			Name: ldsTargets[i],
+			ApiListener: &v3listenerpb.ApiListener{
+				ApiListener: func() *anypb.Any {
+					mcm, _ := ptypes.MarshalAny(&v3httppb.HttpConnectionManager{
+						RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+							Rds: &v3httppb.Rds{
+								ConfigSource: &v3corepb.ConfigSource{
+									ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+								},
+								RouteConfigName: routeConfigNames[i],
+							},
+						},
+						CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
+							MaxStreamDuration: durationpb.New(time.Second),
+						},
+					})
+					return mcm
+				}(),
+			},
+		}
+		anyT, err := ptypes.MarshalAny(listenersT)
+		if err != nil {
+			t.Fatalf("failed to marshal proto to any: %v", err)
+		}
+		listenerRaws[ldsTargets[i]] = anyT
+	}
+
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Expected unknown.
+	if err := compareDump(client.DumpLDS, "", map[string]UpdateWithMD{}); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	wantRequested := make(map[string]UpdateWithMD)
+	for _, n := range ldsTargets {
+		cancel := client.WatchListener(n, func(update ListenerUpdate, err error) {})
+		defer cancel()
+		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+	}
+	// Expected requested.
+	if err := compareDump(client.DumpLDS, "", wantRequested); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	update0 := make(map[string]ListenerUpdate)
+	want0 := make(map[string]UpdateWithMD)
+	for n, r := range listenerRaws {
+		update0[n] = ListenerUpdate{Raw: r}
+		want0[n] = UpdateWithMD{
+			MD:  UpdateMetadata{Version: testVersion},
+			Raw: r,
+		}
+	}
+	client.NewListeners(update0, UpdateMetadata{Version: testVersion})
+
+	// Expect ACK.
+	if err := compareDump(client.DumpLDS, testVersion, want0); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	const nackVersion = "lds-version-nack"
+	var nackErr = fmt.Errorf("lds nack error")
+	client.NewListeners(
+		map[string]ListenerUpdate{
+			ldsTargets[0]: {},
+		},
+		UpdateMetadata{
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+	)
+
+	// Expect NACK for [0], but old ACK for [1].
+	wantDump := make(map[string]UpdateWithMD)
+	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
+	// message, as well as the NACK error.
+	wantDump[ldsTargets[0]] = UpdateWithMD{
+		MD: UpdateMetadata{
+			Version: testVersion,
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+		Raw: listenerRaws[ldsTargets[0]],
+	}
+
+	wantDump[ldsTargets[1]] = UpdateWithMD{
+		MD:  UpdateMetadata{Version: testVersion},
+		Raw: listenerRaws[ldsTargets[1]],
+	}
+	if err := compareDump(client.DumpLDS, nackVersion, wantDump); err != nil {
+		t.Fatalf(err.Error())
+	}
+}
+
+func (s) TestRDSConfigDump(t *testing.T) {
+	const testVersion = "test-version-rds"
+	var (
+		listenerNames = []string{"lds.target.good:0000", "lds.target.good:1111"}
+		rdsTargets    = []string{"route-config-0", "route-config-1"}
+		clusterNames  = []string{"cluster-0", "cluster-1"}
+		routeRaws     = make(map[string]*anypb.Any, len(rdsTargets))
+	)
+
+	for i := range rdsTargets {
+		routeConfigT := &v3routepb.RouteConfiguration{
+			Name: rdsTargets[i],
+			VirtualHosts: []*v3routepb.VirtualHost{
+				{
+					Domains: []string{listenerNames[i]},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{
+							Route: &v3routepb.RouteAction{
+								ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterNames[i]},
+							},
+						},
+					}},
+				},
+			},
+		}
+
+		anyT, err := ptypes.MarshalAny(routeConfigT)
+		if err != nil {
+			t.Fatalf("failed to marshal proto to any: %v", err)
+		}
+		routeRaws[rdsTargets[i]] = anyT
+	}
+
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Expected unknown.
+	if err := compareDump(client.DumpRDS, "", map[string]UpdateWithMD{}); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	wantRequested := make(map[string]UpdateWithMD)
+	for _, n := range rdsTargets {
+		cancel := client.WatchRouteConfig(n, func(update RouteConfigUpdate, err error) {})
+		defer cancel()
+		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+	}
+	// Expected requested.
+	if err := compareDump(client.DumpRDS, "", wantRequested); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	update0 := make(map[string]RouteConfigUpdate)
+	want0 := make(map[string]UpdateWithMD)
+	for n, r := range routeRaws {
+		update0[n] = RouteConfigUpdate{Raw: r}
+		want0[n] = UpdateWithMD{
+			MD:  UpdateMetadata{Version: testVersion},
+			Raw: r,
+		}
+	}
+	client.NewRouteConfigs(update0, UpdateMetadata{Version: testVersion})
+
+	// Expect ACK.
+	if err := compareDump(client.DumpRDS, testVersion, want0); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	const nackVersion = "rds-version-nack"
+	var nackErr = fmt.Errorf("rds nack error")
+	client.NewRouteConfigs(
+		map[string]RouteConfigUpdate{
+			rdsTargets[0]: {},
+		},
+		UpdateMetadata{
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+	)
+
+	// Expect NACK for [0], but old ACK for [1].
+	wantDump := make(map[string]UpdateWithMD)
+	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
+	// message, as well as the NACK error.
+	wantDump[rdsTargets[0]] = UpdateWithMD{
+		MD: UpdateMetadata{
+			Version: testVersion,
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+		Raw: routeRaws[rdsTargets[0]],
+	}
+	wantDump[rdsTargets[1]] = UpdateWithMD{
+		MD:  UpdateMetadata{Version: testVersion},
+		Raw: routeRaws[rdsTargets[1]],
+	}
+	if err := compareDump(client.DumpRDS, nackVersion, wantDump); err != nil {
+		t.Fatalf(err.Error())
+	}
+}
+
+func (s) TestCDSConfigDump(t *testing.T) {
+	const testVersion = "test-version-cds"
+	var (
+		cdsTargets   = []string{"cluster-0", "cluster-1"}
+		serviceNames = []string{"service-0", "service-1"}
+		clusterRaws  = make(map[string]*anypb.Any, len(cdsTargets))
+	)
+
+	for i := range cdsTargets {
+		clusterT := &v3clusterpb.Cluster{
+			Name:                 cdsTargets[i],
+			ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+			EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+				EdsConfig: &v3corepb.ConfigSource{
+					ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+						Ads: &v3corepb.AggregatedConfigSource{},
+					},
+				},
+				ServiceName: serviceNames[i],
+			},
+			LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+			LrsServer: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{
+					Self: &v3corepb.SelfConfigSource{},
+				},
+			},
+		}
+
+		anyT, err := ptypes.MarshalAny(clusterT)
+		if err != nil {
+			t.Fatalf("failed to marshal proto to any: %v", err)
+		}
+		clusterRaws[cdsTargets[i]] = anyT
+	}
+
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Expected unknown.
+	if err := compareDump(client.DumpCDS, "", map[string]UpdateWithMD{}); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	wantRequested := make(map[string]UpdateWithMD)
+	for _, n := range cdsTargets {
+		cancel := client.WatchCluster(n, func(update ClusterUpdate, err error) {})
+		defer cancel()
+		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+	}
+	// Expected requested.
+	if err := compareDump(client.DumpCDS, "", wantRequested); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	update0 := make(map[string]ClusterUpdate)
+	want0 := make(map[string]UpdateWithMD)
+	for n, r := range clusterRaws {
+		update0[n] = ClusterUpdate{Raw: r}
+		want0[n] = UpdateWithMD{
+			MD:  UpdateMetadata{Version: testVersion},
+			Raw: r,
+		}
+	}
+	client.NewClusters(update0, UpdateMetadata{Version: testVersion})
+
+	// Expect ACK.
+	if err := compareDump(client.DumpCDS, testVersion, want0); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	const nackVersion = "cds-version-nack"
+	var nackErr = fmt.Errorf("cds nack error")
+	client.NewClusters(
+		map[string]ClusterUpdate{
+			cdsTargets[0]: {},
+		},
+		UpdateMetadata{
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+	)
+
+	// Expect NACK for [0], but old ACK for [1].
+	wantDump := make(map[string]UpdateWithMD)
+	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
+	// message, as well as the NACK error.
+	wantDump[cdsTargets[0]] = UpdateWithMD{
+		MD: UpdateMetadata{
+			Version: testVersion,
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+		Raw: clusterRaws[cdsTargets[0]],
+	}
+	wantDump[cdsTargets[1]] = UpdateWithMD{
+		MD:  UpdateMetadata{Version: testVersion},
+		Raw: clusterRaws[cdsTargets[1]],
+	}
+	if err := compareDump(client.DumpCDS, nackVersion, wantDump); err != nil {
+		t.Fatalf(err.Error())
+	}
+}
+
+func (s) TestEDSConfigDump(t *testing.T) {
+	const testVersion = "test-version-cds"
+	var (
+		edsTargets    = []string{"cluster-0", "cluster-1"}
+		localityNames = []string{"locality-0", "locality-1"}
+		addrs         = []string{"addr0:123", "addr1:456"}
+		endpointRaws  = make(map[string]*anypb.Any, len(edsTargets))
+	)
+
+	for i := range edsTargets {
+		clab0 := newClaBuilder(edsTargets[i], nil)
+		clab0.addLocality(localityNames[i], 1, 1, []string{addrs[i]}, nil)
+		claT := clab0.Build()
+
+		anyT, err := ptypes.MarshalAny(claT)
+		if err != nil {
+			t.Fatalf("failed to marshal proto to any: %v", err)
+		}
+		endpointRaws[edsTargets[i]] = anyT
+	}
+
+	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Expected unknown.
+	if err := compareDump(client.DumpEDS, "", map[string]UpdateWithMD{}); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	wantRequested := make(map[string]UpdateWithMD)
+	for _, n := range edsTargets {
+		cancel := client.WatchEndpoints(n, func(update EndpointsUpdate, err error) {})
+		defer cancel()
+		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+	}
+	// Expected requested.
+	if err := compareDump(client.DumpEDS, "", wantRequested); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	update0 := make(map[string]EndpointsUpdate)
+	want0 := make(map[string]UpdateWithMD)
+	for n, r := range endpointRaws {
+		update0[n] = EndpointsUpdate{Raw: r}
+		want0[n] = UpdateWithMD{
+			MD:  UpdateMetadata{Version: testVersion},
+			Raw: r,
+		}
+	}
+	client.NewEndpoints(update0, UpdateMetadata{Version: testVersion})
+
+	// Expect ACK.
+	if err := compareDump(client.DumpEDS, testVersion, want0); err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	const nackVersion = "eds-version-nack"
+	var nackErr = fmt.Errorf("eds nack error")
+	client.NewEndpoints(
+		map[string]EndpointsUpdate{
+			edsTargets[0]: {},
+		},
+		UpdateMetadata{
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+	)
+
+	// Expect NACK for [0], but old ACK for [1].
+	wantDump := make(map[string]UpdateWithMD)
+	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
+	// message, as well as the NACK error.
+	wantDump[edsTargets[0]] = UpdateWithMD{
+		MD: UpdateMetadata{
+			Version: testVersion,
+			ErrState: &UpdateErrorMetadata{
+				Version: nackVersion,
+				Err:     nackErr,
+			},
+		},
+		Raw: endpointRaws[edsTargets[0]],
+	}
+	wantDump[edsTargets[1]] = UpdateWithMD{
+		MD:  UpdateMetadata{Version: testVersion},
+		Raw: endpointRaws[edsTargets[1]],
+	}
+	if err := compareDump(client.DumpEDS, nackVersion, wantDump); err != nil {
+		t.Fatalf(err.Error())
+	}
+}
+
+func compareDump(dumpFunc func() (string, map[string]UpdateWithMD), wantVersion string, wantDump interface{}) error {
+	v, dump := dumpFunc()
+	if v != wantVersion {
+		return fmt.Errorf("Dump returned version %q, want %q", v, wantVersion)
+	}
+	if diff := cmp.Diff(dump, wantDump, cmpOpts); diff != "" {
+		return fmt.Errorf("Dump returned unexpected dump, diff (-got +want): %s", diff)
+	}
+	return nil
+}
diff --git a/xds/internal/client/eds_test.go b/xds/internal/client/eds_test.go
index 01df225083b5..e51c8938f8c9 100644
--- a/xds/internal/client/eds_test.go
+++ b/xds/internal/client/eds_test.go
@@ -211,7 +211,7 @@ func (s) TestUnmarshalEndpoints(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := UnmarshalEndpoints(test.resources, nil)
+			update, _, err := UnmarshalEndpoints("", test.resources, nil)
 			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("UnmarshalEndpoints(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
 			}
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 5172def9a82f..2a31b7e2e508 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -298,7 +298,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := UnmarshalListener(test.resources, nil)
+			update, _, err := UnmarshalListener("", test.resources, nil)
 			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("UnmarshalListener(%v) = (%v, %v) want (%v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
 			}
@@ -926,7 +926,7 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, err := UnmarshalListener(test.resources, nil)
+			gotUpdate, _, err := UnmarshalListener("", test.resources, nil)
 			if (err != nil) != (test.wantErr != "") {
 				t.Fatalf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
 			}
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index c91444d4bfb6..1a82e0304e74 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -579,7 +579,7 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := UnmarshalRouteConfig(test.resources, nil)
+			update, _, err := UnmarshalRouteConfig("", test.resources, nil)
 			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
 				t.Errorf("UnmarshalRouteConfig(%v, %v) = (%v, %v) want (%v, %v)", test.resources, ldsTarget, update, err, test.wantUpdate, test.wantErr)
 			}
diff --git a/xds/internal/client/v2/ack_test.go b/xds/internal/client/v2/ack_test.go
index 57a6d1c200dc..813d8baa79d9 100644
--- a/xds/internal/client/v2/ack_test.go
+++ b/xds/internal/client/v2/ack_test.go
@@ -47,7 +47,7 @@ func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cb
 	cbCDS = testutils.NewChannel()
 	cbEDS = testutils.NewChannel()
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}, md xdsclient.UpdateMetadata) {
 			t.Logf("Received %v callback with {%+v}", rType, d)
 			switch rType {
 			case xdsclient.ListenerResource:
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index 8538f4e46afc..1f0acad6d1e4 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -172,7 +172,7 @@ func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(xdsclient.ResourceType, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}, xdsclient.UpdateMetadata) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index faf9c6e082e3..fcfa6f7b74a5 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -185,11 +185,11 @@ func (v2c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, stri
 // server. On receipt of a good response, it also invokes the registered watcher
 // callback.
 func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalListener(resp.GetResources(), v2c.logger)
+	update, md, err := xdsclient.UnmarshalListener(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
 	if err != nil {
 		return err
 	}
-	v2c.parent.NewListeners(update)
+	v2c.parent.NewListeners(update, md)
 	return nil
 }
 
@@ -197,11 +197,11 @@ func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 // server. On receipt of a good response, it caches validated resources and also
 // invokes the registered watcher callback.
 func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), v2c.logger)
+	update, md, err := xdsclient.UnmarshalRouteConfig(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
 	if err != nil {
 		return err
 	}
-	v2c.parent.NewRouteConfigs(update)
+	v2c.parent.NewRouteConfigs(update, md)
 	return nil
 }
 
@@ -209,19 +209,19 @@ func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 // server. On receipt of a good response, it also invokes the registered watcher
 // callback.
 func (v2c *client) handleCDSResponse(resp *v2xdspb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v2c.logger)
+	update, md, err := xdsclient.UnmarshalCluster(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
 	if err != nil {
 		return err
 	}
-	v2c.parent.NewClusters(update)
+	v2c.parent.NewClusters(update, md)
 	return nil
 }
 
 func (v2c *client) handleEDSResponse(resp *v2xdspb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalEndpoints(resp.GetResources(), v2c.logger)
+	update, md, err := xdsclient.UnmarshalEndpoints(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
 	if err != nil {
 		return err
 	}
-	v2c.parent.NewEndpoints(update)
+	v2c.parent.NewEndpoints(update, md)
 	return nil
 }
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index db97788843ae..e79d7bf47f2a 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -25,7 +25,14 @@ import (
 	"testing"
 	"time"
 
+	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
+	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
 	"github.com/golang/protobuf/proto"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	structpb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -37,14 +44,6 @@ import (
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
-
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
-	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
-	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
-	anypb "github.com/golang/protobuf/ptypes/any"
-	structpb "github.com/golang/protobuf/ptypes/struct"
 )
 
 type s struct {
@@ -344,43 +343,43 @@ type watchHandleTestcase struct {
 }
 
 type testUpdateReceiver struct {
-	f func(rType xdsclient.ResourceType, d map[string]interface{})
+	f func(rType xdsclient.ResourceType, d map[string]interface{}, md xdsclient.UpdateMetadata)
 }
 
-func (t *testUpdateReceiver) NewListeners(d map[string]xdsclient.ListenerUpdate) {
+func (t *testUpdateReceiver) NewListeners(d map[string]xdsclient.ListenerUpdate, metadata xdsclient.UpdateMetadata) {
 	dd := make(map[string]interface{})
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(xdsclient.ListenerResource, dd)
+	t.newUpdate(xdsclient.ListenerResource, dd, metadata)
 }
 
-func (t *testUpdateReceiver) NewRouteConfigs(d map[string]xdsclient.RouteConfigUpdate) {
+func (t *testUpdateReceiver) NewRouteConfigs(d map[string]xdsclient.RouteConfigUpdate, metadata xdsclient.UpdateMetadata) {
 	dd := make(map[string]interface{})
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(xdsclient.RouteConfigResource, dd)
+	t.newUpdate(xdsclient.RouteConfigResource, dd, metadata)
 }
 
-func (t *testUpdateReceiver) NewClusters(d map[string]xdsclient.ClusterUpdate) {
+func (t *testUpdateReceiver) NewClusters(d map[string]xdsclient.ClusterUpdate, metadata xdsclient.UpdateMetadata) {
 	dd := make(map[string]interface{})
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(xdsclient.ClusterResource, dd)
+	t.newUpdate(xdsclient.ClusterResource, dd, metadata)
 }
 
-func (t *testUpdateReceiver) NewEndpoints(d map[string]xdsclient.EndpointsUpdate) {
+func (t *testUpdateReceiver) NewEndpoints(d map[string]xdsclient.EndpointsUpdate, metadata xdsclient.UpdateMetadata) {
 	dd := make(map[string]interface{})
 	for k, v := range d {
 		dd[k] = v
 	}
-	t.newUpdate(xdsclient.EndpointsResource, dd)
+	t.newUpdate(xdsclient.EndpointsResource, dd, metadata)
 }
 
-func (t *testUpdateReceiver) newUpdate(rType xdsclient.ResourceType, d map[string]interface{}) {
-	t.f(rType, d)
+func (t *testUpdateReceiver) newUpdate(rType xdsclient.ResourceType, d map[string]interface{}, metadata xdsclient.UpdateMetadata) {
+	t.f(rType, d, metadata)
 }
 
 // testWatchHandle is called to test response handling for each xDS.
@@ -390,6 +389,8 @@ func (t *testUpdateReceiver) newUpdate(rType xdsclient.ResourceType, d map[strin
 // handleXDSResp with responseToHandle (if it's set). It then compares the
 // update received by watch callback with the expected results.
 func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
+	t.Helper()
+
 	fakeServer, cc, cleanup := startServerAndGetCC(t)
 	defer cleanup()
 
@@ -400,7 +401,7 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 	gotUpdateCh := testutils.NewChannel()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}, _ xdsclient.UpdateMetadata) {
 			if rType == test.rType {
 				if u, ok := d[test.resourceName]; ok {
 					gotUpdateCh.Send(updateErr{u, nil})
@@ -526,7 +527,7 @@ func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) {
 
 	callbackCh := make(chan struct{})
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(xdsclient.ResourceType, map[string]interface{}) { close(callbackCh) },
+		f: func(xdsclient.ResourceType, map[string]interface{}, xdsclient.UpdateMetadata) { close(callbackCh) },
 	}, cc, goodNodeProto, clientBackoff, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -571,7 +572,7 @@ func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}, md xdsclient.UpdateMetadata) {
 			if rType == xdsclient.ListenerResource {
 				if u, ok := d[goodLDSTarget1]; ok {
 					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
@@ -643,7 +644,7 @@ func (s) TestV2ClientWatchWithoutStream(t *testing.T) {
 
 	callbackCh := testutils.NewChannel()
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(rType xdsclient.ResourceType, d map[string]interface{}) {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}, md xdsclient.UpdateMetadata) {
 			if rType == xdsclient.ListenerResource {
 				if u, ok := d[goodLDSTarget1]; ok {
 					t.Logf("Received LDS callback with ldsUpdate {%+v}", u)
diff --git a/xds/internal/client/v2/eds_test.go b/xds/internal/client/v2/eds_test.go
index 7af74f0b6393..9d04433de746 100644
--- a/xds/internal/client/v2/eds_test.go
+++ b/xds/internal/client/v2/eds_test.go
@@ -153,7 +153,7 @@ func (s) TestEDSHandleResponseWithoutWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(xdsclient.ResourceType, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}, xdsclient.UpdateMetadata) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/lds_test.go b/xds/internal/client/v2/lds_test.go
index 854cf3ccee73..28e5633b244a 100644
--- a/xds/internal/client/v2/lds_test.go
+++ b/xds/internal/client/v2/lds_test.go
@@ -131,7 +131,7 @@ func (s) TestLDSHandleResponseWithoutWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(xdsclient.ResourceType, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}, xdsclient.UpdateMetadata) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/rds_test.go b/xds/internal/client/v2/rds_test.go
index c685d4696985..94508f9e5cf1 100644
--- a/xds/internal/client/v2/rds_test.go
+++ b/xds/internal/client/v2/rds_test.go
@@ -129,7 +129,7 @@ func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) {
 	defer cleanup()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(xdsclient.ResourceType, map[string]interface{}) {},
+		f: func(xdsclient.ResourceType, map[string]interface{}, xdsclient.UpdateMetadata) {},
 	}, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil)
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 85de8d584a4f..23bb1bfd478b 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -185,11 +185,11 @@ func (v3c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, stri
 // server. On receipt of a good response, it also invokes the registered watcher
 // callback.
 func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalListener(resp.GetResources(), v3c.logger)
+	update, md, err := xdsclient.UnmarshalListener(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
 	if err != nil {
 		return err
 	}
-	v3c.parent.NewListeners(update)
+	v3c.parent.NewListeners(update, md)
 	return nil
 }
 
@@ -197,11 +197,11 @@ func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 // server. On receipt of a good response, it caches validated resources and also
 // invokes the registered watcher callback.
 func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), v3c.logger)
+	update, md, err := xdsclient.UnmarshalRouteConfig(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
 	if err != nil {
 		return err
 	}
-	v3c.parent.NewRouteConfigs(update)
+	v3c.parent.NewRouteConfigs(update, md)
 	return nil
 }
 
@@ -209,19 +209,19 @@ func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 // server. On receipt of a good response, it also invokes the registered watcher
 // callback.
 func (v3c *client) handleCDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v3c.logger)
+	update, md, err := xdsclient.UnmarshalCluster(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
 	if err != nil {
 		return err
 	}
-	v3c.parent.NewClusters(update)
+	v3c.parent.NewClusters(update, md)
 	return nil
 }
 
 func (v3c *client) handleEDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
-	update, err := xdsclient.UnmarshalEndpoints(resp.GetResources(), v3c.logger)
+	update, md, err := xdsclient.UnmarshalEndpoints(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
 	if err != nil {
 		return err
 	}
-	v3c.parent.NewEndpoints(update)
+	v3c.parent.NewEndpoints(update, md)
 	return nil
 }
diff --git a/xds/internal/client/watchers.go b/xds/internal/client/watchers.go
index 31704afc0790..9fafe5a60f83 100644
--- a/xds/internal/client/watchers.go
+++ b/xds/internal/client/watchers.go
@@ -117,16 +117,26 @@ func (c *clientImpl) watch(wi *watchInfo) (cancel func()) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.logger.Debugf("new watch for type %v, resource name %v", wi.rType, wi.target)
-	var watchers map[string]map[*watchInfo]bool
+	var (
+		watchers map[string]map[*watchInfo]bool
+		mds      map[string]UpdateMetadata
+	)
 	switch wi.rType {
 	case ListenerResource:
 		watchers = c.ldsWatchers
+		mds = c.ldsMD
 	case RouteConfigResource:
 		watchers = c.rdsWatchers
+		mds = c.rdsMD
 	case ClusterResource:
 		watchers = c.cdsWatchers
+		mds = c.cdsMD
 	case EndpointsResource:
 		watchers = c.edsWatchers
+		mds = c.edsMD
+	default:
+		c.logger.Errorf("unknown watch type: %v", wi.rType)
+		return nil
 	}
 
 	resourceName := wi.target
@@ -140,6 +150,7 @@ func (c *clientImpl) watch(wi *watchInfo) (cancel func()) {
 		c.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.rType, wi.target)
 		s = make(map[*watchInfo]bool)
 		watchers[resourceName] = s
+		mds[resourceName] = UpdateMetadata{Status: ServiceStatusRequested}
 		c.apiClient.AddWatch(wi.rType, resourceName)
 	}
 	// No matter what, add the new watcher to the set, so it's callback will be
@@ -184,6 +195,7 @@ func (c *clientImpl) watch(wi *watchInfo) (cancel func()) {
 				// If this was the last watcher, also tell xdsv2Client to stop
 				// watching this resource.
 				delete(watchers, resourceName)
+				delete(mds, resourceName)
 				c.apiClient.RemoveWatch(wi.rType, resourceName)
 				// Remove the resource from cache. When a watch for this
 				// resource is added later, it will trigger a xDS request with
diff --git a/xds/internal/client/watchers_cluster_test.go b/xds/internal/client/watchers_cluster_test.go
index b6b8d61216df..fdef0cf61649 100644
--- a/xds/internal/client/watchers_cluster_test.go
+++ b/xds/internal/client/watchers_cluster_test.go
@@ -63,7 +63,7 @@ func (s) TestClusterWatch(t *testing.T) {
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
@@ -72,14 +72,14 @@ func (s) TestClusterWatch(t *testing.T) {
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName:  wantUpdate,
 		"randomName": {},
-	})
+	}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
 	if u, err := clusterUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
@@ -127,7 +127,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 	}
 
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
-	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -136,7 +136,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count-1; i++ {
 		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -203,7 +203,7 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
-	})
+	}, UpdateMetadata{})
 
 	for i := 0; i < count; i++ {
 		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate1); err != nil {
@@ -246,7 +246,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
-	})
+	}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
@@ -346,7 +346,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
-	})
+	}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
@@ -405,7 +405,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
-	})
+	}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh1, wantUpdate1); err != nil {
 		t.Fatal(err)
 	}
@@ -414,7 +414,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	}
 
 	// Send another update to remove resource 1.
-	client.NewClusters(map[string]ClusterUpdate{testCDSName + "2": wantUpdate2})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName + "2": wantUpdate2}, UpdateMetadata{})
 
 	// Watcher 1 should get an error.
 	if u, err := clusterUpdateCh1.Receive(ctx); err != nil || ErrType(u.(clusterUpdateErr).err) != ErrorTypeResourceNotFound {
@@ -427,7 +427,7 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 	}
 
 	// Send one more update without resource 1.
-	client.NewClusters(map[string]ClusterUpdate{testCDSName + "2": wantUpdate2})
+	client.NewClusters(map[string]ClusterUpdate{testCDSName + "2": wantUpdate2}, UpdateMetadata{})
 
 	// Watcher 1 should not see an update.
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
diff --git a/xds/internal/client/watchers_endpoints_test.go b/xds/internal/client/watchers_endpoints_test.go
index 2bf6abab234a..b79397414d4a 100644
--- a/xds/internal/client/watchers_endpoints_test.go
+++ b/xds/internal/client/watchers_endpoints_test.go
@@ -81,13 +81,13 @@ func (s) TestEndpointsWatch(t *testing.T) {
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyEndpointsUpdate(ctx, endpointsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
 	// Another update for a different resource name.
-	client.NewEndpoints(map[string]EndpointsUpdate{"randomName": {}})
+	client.NewEndpoints(map[string]EndpointsUpdate{"randomName": {}}, UpdateMetadata{})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
 	if u, err := endpointsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
@@ -96,7 +96,7 @@ func (s) TestEndpointsWatch(t *testing.T) {
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
 	if u, err := endpointsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
@@ -146,7 +146,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyEndpointsUpdate(ctx, endpointsUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -155,7 +155,7 @@ func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) {
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count-1; i++ {
 		if err := verifyEndpointsUpdate(ctx, endpointsUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -222,7 +222,7 @@ func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) {
 	client.NewEndpoints(map[string]EndpointsUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
-	})
+	}, UpdateMetadata{})
 
 	for i := 0; i < count; i++ {
 		if err := verifyEndpointsUpdate(ctx, endpointsUpdateChs[i], wantUpdate1); err != nil {
@@ -263,7 +263,7 @@ func (s) TestEndpointsWatchAfterCache(t *testing.T) {
 	}
 
 	wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}}
-	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate})
+	client.NewEndpoints(map[string]EndpointsUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyEndpointsUpdate(ctx, endpointsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
diff --git a/xds/internal/client/watchers_listener_test.go b/xds/internal/client/watchers_listener_test.go
index a233cd305417..bf3a122da075 100644
--- a/xds/internal/client/watchers_listener_test.go
+++ b/xds/internal/client/watchers_listener_test.go
@@ -61,7 +61,7 @@ func (s) TestLDSWatch(t *testing.T) {
 	}
 
 	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyListenerUpdate(ctx, ldsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
@@ -70,14 +70,14 @@ func (s) TestLDSWatch(t *testing.T) {
 	client.NewListeners(map[string]ListenerUpdate{
 		testLDSName:  wantUpdate,
 		"randomName": {},
-	})
+	}, UpdateMetadata{})
 	if err := verifyListenerUpdate(ctx, ldsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate}, UpdateMetadata{})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
 	if u, err := ldsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
@@ -128,7 +128,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 	}
 
 	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyListenerUpdate(ctx, ldsUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -137,7 +137,7 @@ func (s) TestLDSTwoWatchSameResourceName(t *testing.T) {
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count-1; i++ {
 		if err := verifyListenerUpdate(ctx, ldsUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -205,7 +205,7 @@ func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) {
 	client.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "1": wantUpdate1,
 		testLDSName + "2": wantUpdate2,
-	})
+	}, UpdateMetadata{})
 
 	for i := 0; i < count; i++ {
 		if err := verifyListenerUpdate(ctx, ldsUpdateChs[i], wantUpdate1); err != nil {
@@ -246,7 +246,7 @@ func (s) TestLDSWatchAfterCache(t *testing.T) {
 	}
 
 	wantUpdate := ListenerUpdate{RouteConfigName: testRDSName}
-	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyListenerUpdate(ctx, ldsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
@@ -320,7 +320,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	client.NewListeners(map[string]ListenerUpdate{
 		testLDSName + "1": wantUpdate1,
 		testLDSName + "2": wantUpdate2,
-	})
+	}, UpdateMetadata{})
 	if err := verifyListenerUpdate(ctx, ldsUpdateCh1, wantUpdate1); err != nil {
 		t.Fatal(err)
 	}
@@ -329,7 +329,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	}
 
 	// Send another update to remove resource 1.
-	client.NewListeners(map[string]ListenerUpdate{testLDSName + "2": wantUpdate2})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName + "2": wantUpdate2}, UpdateMetadata{})
 
 	// Watcher 1 should get an error.
 	if u, err := ldsUpdateCh1.Receive(ctx); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound {
@@ -342,7 +342,7 @@ func (s) TestLDSResourceRemoved(t *testing.T) {
 	}
 
 	// Send one more update without resource 1.
-	client.NewListeners(map[string]ListenerUpdate{testLDSName + "2": wantUpdate2})
+	client.NewListeners(map[string]ListenerUpdate{testLDSName + "2": wantUpdate2}, UpdateMetadata{})
 
 	// Watcher 1 should not see an update.
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
diff --git a/xds/internal/client/watchers_route_test.go b/xds/internal/client/watchers_route_test.go
index ec04383c2df1..93a47e9a3423 100644
--- a/xds/internal/client/watchers_route_test.go
+++ b/xds/internal/client/watchers_route_test.go
@@ -70,13 +70,13 @@ func (s) TestRDSWatch(t *testing.T) {
 			},
 		},
 	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
 	// Another update for a different resource name.
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{"randomName": {}})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{"randomName": {}}, UpdateMetadata{})
 	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
 	if u, err := rdsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
@@ -85,7 +85,7 @@ func (s) TestRDSWatch(t *testing.T) {
 
 	// Cancel watch, and send update again.
 	cancelWatch()
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate}, UpdateMetadata{})
 	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer sCancel()
 	if u, err := rdsUpdateCh.Receive(sCtx); err != context.DeadlineExceeded {
@@ -142,7 +142,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 			},
 		},
 	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -151,7 +151,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 
 	// Cancel the last watch, and send update again.
 	cancelLastWatch()
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count-1; i++ {
 		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate); err != nil {
 			t.Fatal(err)
@@ -232,7 +232,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 	client.NewRouteConfigs(map[string]RouteConfigUpdate{
 		testRDSName + "1": wantUpdate1,
 		testRDSName + "2": wantUpdate2,
-	})
+	}, UpdateMetadata{})
 
 	for i := 0; i < count; i++ {
 		if err := verifyRouteConfigUpdate(ctx, rdsUpdateChs[i], wantUpdate1); err != nil {
@@ -280,7 +280,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 			},
 		},
 	}
-	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate})
+	client.NewRouteConfigs(map[string]RouteConfigUpdate{testRDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyRouteConfigUpdate(ctx, rdsUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index e2a008200a6a..caeb42856e09 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -49,25 +49,25 @@ const transportSocketName = "envoy.transport_sockets.tls"
 // UnmarshalListener processes resources received in an LDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
-func UnmarshalListener(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, error) {
+func UnmarshalListener(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, UpdateMetadata, error) {
 	update := make(map[string]ListenerUpdate)
 	for _, r := range resources {
 		if !IsListenerResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %q in LDS response", r.GetTypeUrl())
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in LDS response", r.GetTypeUrl())
 		}
 		lis := &v3listenerpb.Listener{}
 		if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
-			return nil, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
 
 		lu, err := processListener(lis)
 		if err != nil {
-			return nil, err
+			return nil, UpdateMetadata{}, err
 		}
 		update[lis.GetName()] = *lu
 	}
-	return update, nil
+	return update, UpdateMetadata{}, nil
 }
 
 func processListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
@@ -196,26 +196,26 @@ func getAddressFromName(name string) (host string, port string, err error) {
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
 // configuration resources of interest.
-func UnmarshalRouteConfig(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, error) {
+func UnmarshalRouteConfig(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, UpdateMetadata, error) {
 	update := make(map[string]RouteConfigUpdate)
 	for _, r := range resources {
 		if !IsRouteConfigResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %q in RDS response", r.GetTypeUrl())
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in RDS response", r.GetTypeUrl())
 		}
 		rc := &v3routepb.RouteConfiguration{}
 		if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
-			return nil, fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err)
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err)
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, rc)
 
 		// Use the hostname (resourceName for LDS) to find the routes.
 		u, err := generateRDSUpdateFromRouteConfiguration(rc, logger)
 		if err != nil {
-			return nil, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
 		}
 		update[rc.GetName()] = u
 	}
-	return update, nil
+	return update, UpdateMetadata{}, nil
 }
 
 // generateRDSUpdateFromRouteConfiguration checks if the provided
@@ -371,21 +371,21 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 // UnmarshalCluster processes resources received in an CDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
-func UnmarshalCluster(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ClusterUpdate, error) {
+func UnmarshalCluster(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ClusterUpdate, UpdateMetadata, error) {
 	update := make(map[string]ClusterUpdate)
 	for _, r := range resources {
 		if !IsClusterResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %q in CDS response", r.GetTypeUrl())
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in CDS response", r.GetTypeUrl())
 		}
 
 		cluster := &v3clusterpb.Cluster{}
 		if err := proto.Unmarshal(r.GetValue(), cluster); err != nil {
-			return nil, fmt.Errorf("xds: failed to unmarshal resource in CDS response: %v", err)
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in CDS response: %v", err)
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
 		cu, err := validateCluster(cluster)
 		if err != nil {
-			return nil, err
+			return nil, UpdateMetadata{}, err
 		}
 
 		// If the Cluster message in the CDS response did not contain a
@@ -396,7 +396,7 @@ func UnmarshalCluster(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map
 		logger.Debugf("Resource with name %v, value %+v added to cache", cluster.GetName(), cu)
 		update[cluster.GetName()] = cu
 	}
-	return update, nil
+	return update, UpdateMetadata{}, nil
 }
 
 func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
@@ -529,26 +529,26 @@ func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 {
 // UnmarshalEndpoints processes resources received in an EDS response,
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in.
-func UnmarshalEndpoints(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]EndpointsUpdate, error) {
+func UnmarshalEndpoints(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]EndpointsUpdate, UpdateMetadata, error) {
 	update := make(map[string]EndpointsUpdate)
 	for _, r := range resources {
 		if !IsEndpointsResource(r.GetTypeUrl()) {
-			return nil, fmt.Errorf("xds: unexpected resource type: %q in EDS response", r.GetTypeUrl())
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in EDS response", r.GetTypeUrl())
 		}
 
 		cla := &v3endpointpb.ClusterLoadAssignment{}
 		if err := proto.Unmarshal(r.GetValue(), cla); err != nil {
-			return nil, fmt.Errorf("xds: failed to unmarshal resource in EDS response: %v", err)
+			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in EDS response: %v", err)
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
 
 		u, err := parseEDSRespProto(cla)
 		if err != nil {
-			return nil, err
+			return nil, UpdateMetadata{}, err
 		}
 		update[cla.GetClusterName()] = u
 	}
-	return update, nil
+	return update, UpdateMetadata{}, nil
 }
 
 func parseAddress(socketAddress *v3corepb.SocketAddress) string {

From 60843b10662720f0b1c8b5f64856bafe21656b29 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 25 Feb 2021 14:04:15 -0800
Subject: [PATCH 369/481] xds: add support for HTTP filters (gRFC A39) (#4206)

---
 internal/resolver/config_selector.go        |  32 ++
 internal/transport/http2_client.go          |   4 +
 internal/transport/transport.go             |   3 +
 stream.go                                   |  14 +-
 xds/internal/client/client.go               |  42 ++-
 xds/internal/client/lds_test.go             | 317 +++++++++++++++++--
 xds/internal/client/rds_test.go             | 203 ++++++++++--
 xds/internal/client/v2/rds_test.go          |   4 +-
 xds/internal/client/watchers_route_test.go  |  10 +-
 xds/internal/client/xds.go                  | 160 ++++++++--
 xds/internal/env/env.go                     |   4 +
 xds/internal/httpfilter/httpfilter.go       | 102 ++++++
 xds/internal/httpfilter/router/router.go    |  95 ++++++
 xds/internal/resolver/serviceconfig.go      | 111 ++++++-
 xds/internal/resolver/watch_service.go      |  23 +-
 xds/internal/resolver/watch_service_test.go |  52 ++--
 xds/internal/resolver/xds_resolver.go       |   5 +
 xds/internal/resolver/xds_resolver_test.go  | 326 ++++++++++++++++++--
 18 files changed, 1342 insertions(+), 165 deletions(-)
 create mode 100644 xds/internal/httpfilter/httpfilter.go
 create mode 100644 xds/internal/httpfilter/router/router.go

diff --git a/internal/resolver/config_selector.go b/internal/resolver/config_selector.go
index e69900400564..1b20c7592e33 100644
--- a/internal/resolver/config_selector.go
+++ b/internal/resolver/config_selector.go
@@ -51,6 +51,38 @@ type RPCConfig struct {
 	Context      context.Context
 	MethodConfig serviceconfig.MethodConfig // configuration to use for this RPC
 	OnCommitted  func()                     // Called when the RPC has been committed (retries no longer possible)
+	Interceptor  ClientInterceptor
+}
+
+// ClientStream will ultimately be a superset of grpc.ClientStream as
+// operations become necessary to support.
+type ClientStream interface {
+	// Done is invoked when the RPC is finished using its connection, or could
+	// not be assigned a connection.  RPC operations may still occur on
+	// ClientStream after done is called, since the interceptor is invoked by
+	// application-layer operations.
+	Done()
+}
+
+// NOPClientStream is a ClientStream that does nothing
+type NOPClientStream struct{}
+
+// Done is a nop.
+func (NOPClientStream) Done() {}
+
+var _ ClientStream = NOPClientStream{}
+
+// ClientInterceptor is an interceptor for gRPC client streams.
+type ClientInterceptor interface {
+	// NewStream can intercept ClientStream calls.  The provided ClientStream
+	// should not be used during NewStream.  RPCInfo.Context should not be used
+	// (will be nil).
+	NewStream(context.Context, RPCInfo, ClientStream) (context.Context, ClientStream, error)
+}
+
+// ServerInterceptor is unimplementable; do not use.
+type ServerInterceptor interface {
+	notDefined()
 }
 
 type csKeyType string
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 8902b7f90d9d..d5bbe720db54 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -414,6 +414,7 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream {
 		buf:            newRecvBuffer(),
 		headerChan:     make(chan struct{}),
 		contentSubtype: callHdr.ContentSubtype,
+		doneFunc:       callHdr.DoneFunc,
 	}
 	s.wq = newWriteQuota(defaultWriteQuota, s.done)
 	s.requestRead = func(n int) {
@@ -832,6 +833,9 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2.
 	t.controlBuf.executeAndPut(addBackStreamQuota, cleanup)
 	// This will unblock write.
 	close(s.done)
+	if s.doneFunc != nil {
+		s.doneFunc()
+	}
 }
 
 // Close kicks off the shutdown process of the transport. This should be called
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index 9c8f79cb4b29..5cf7c5f80fe1 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -241,6 +241,7 @@ type Stream struct {
 	ctx          context.Context    // the associated context of the stream
 	cancel       context.CancelFunc // always nil for client side Stream
 	done         chan struct{}      // closed at the end of stream to unblock writers. On the client side.
+	doneFunc     func()             // invoked at the end of stream on client side.
 	ctxDone      <-chan struct{}    // same as done chan but for server side. Cache of ctx.Done() (for performance)
 	method       string             // the associated RPC method of the stream
 	recvCompress string
@@ -611,6 +612,8 @@ type CallHdr struct {
 	ContentSubtype string
 
 	PreviousAttempts int // value of grpc-previous-rpc-attempts header to set
+
+	DoneFunc func() // called when the stream is finished
 }
 
 // ClientTransport is the common interface for all gRPC client-side transport
diff --git a/stream.go b/stream.go
index eda1248d60ce..ebc74cd030fb 100644
--- a/stream.go
+++ b/stream.go
@@ -175,16 +175,27 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 
 	var mc serviceconfig.MethodConfig
 	var onCommit func()
-	rpcConfig, err := cc.safeConfigSelector.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: method})
+	rpcInfo := iresolver.RPCInfo{Context: ctx, Method: method}
+	rpcConfig, err := cc.safeConfigSelector.SelectConfig(rpcInfo)
 	if err != nil {
 		return nil, status.Convert(err).Err()
 	}
+	var doneFunc func()
 	if rpcConfig != nil {
 		if rpcConfig.Context != nil {
 			ctx = rpcConfig.Context
 		}
 		mc = rpcConfig.MethodConfig
 		onCommit = rpcConfig.OnCommitted
+		if rpcConfig.Interceptor != nil {
+			rpcInfo.Context = nil
+			newCtx, cs, err := rpcConfig.Interceptor.NewStream(ctx, rpcInfo, iresolver.NOPClientStream{})
+			if err != nil {
+				return nil, status.Convert(err).Err()
+			}
+			ctx = newCtx
+			doneFunc = cs.Done
+		}
 	}
 
 	if mc.WaitForReady != nil {
@@ -223,6 +234,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 		Host:           cc.authority,
 		Method:         method,
 		ContentSubtype: c.contentSubtype,
+		DoneFunc:       doneFunc,
 	}
 
 	// Set our outgoing compression according to the UseCompressor CallOption, if
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 7c7ebf3e4cd2..21881cc6eae6 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -16,8 +16,8 @@
  *
  */
 
-// Package client implementation a full fledged gRPC client for the xDS API
-// used by the xds resolver and balancer implementations.
+// Package client implements a full fledged gRPC client for the xDS API used by
+// the xds resolver and balancer implementations.
 package client
 
 import (
@@ -33,6 +33,7 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"google.golang.org/grpc/xds/internal/client/load"
+	"google.golang.org/grpc/xds/internal/httpfilter"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/backoff"
@@ -199,11 +200,27 @@ type ListenerUpdate struct {
 	// common_http_protocol_options.max_stream_duration field, or zero if
 	// unset.
 	MaxStreamDuration time.Duration
+	// HTTPFilters is a list of HTTP filters (name, config) from the LDS
+	// response.
+	HTTPFilters []HTTPFilter
 
 	// Raw is the resource from the xds response.
 	Raw *anypb.Any
 }
 
+// HTTPFilter represents one HTTP filter from an LDS response's HTTP connection
+// manager field.
+type HTTPFilter struct {
+	// Name is an arbitrary name of the filter.  Used for applying override
+	// settings in virtual host / route / weighted cluster configuration (not
+	// yet supported).
+	Name string
+	// Filter is the HTTP filter found in the registry for the config type.
+	Filter httpfilter.Filter
+	// Config contains the filter's configuration
+	Config httpfilter.FilterConfig
+}
+
 func (lu *ListenerUpdate) String() string {
 	return fmt.Sprintf("{RouteConfigName: %q, SecurityConfig: %+v", lu.RouteConfigName, lu.SecurityCfg)
 }
@@ -226,6 +243,11 @@ type VirtualHost struct {
 	// Routes contains a list of routes, each containing matchers and
 	// corresponding action.
 	Routes []*Route
+	// HTTPFilterConfigOverride contains any HTTP filter config overrides for
+	// the virtual host which may be present.  An individual filter's override
+	// may be unused if the matching Route contains an override for that
+	// filter.
+	HTTPFilterConfigOverride map[string]httpfilter.FilterConfig
 }
 
 // Route is both a specification of how to match a request as well as an
@@ -239,13 +261,27 @@ type Route struct {
 	Fraction        *uint32
 
 	// If the matchers above indicate a match, the below configuration is used.
-	Action map[string]uint32 // action is weighted clusters.
+	WeightedClusters map[string]WeightedCluster
 	// If MaxStreamDuration is nil, it indicates neither of the route action's
 	// max_stream_duration fields (grpc_timeout_header_max nor
 	// max_stream_duration) were set.  In this case, the ListenerUpdate's
 	// MaxStreamDuration field should be used.  If MaxStreamDuration is set to
 	// an explicit zero duration, the application's deadline should be used.
 	MaxStreamDuration *time.Duration
+	// HTTPFilterConfigOverride contains any HTTP filter config overrides for
+	// the route which may be present.  An individual filter's override may be
+	// unused if the matching WeightedCluster contains an override for that
+	// filter.
+	HTTPFilterConfigOverride map[string]httpfilter.FilterConfig
+}
+
+// WeightedCluster contains settings for an xds RouteAction.WeightedCluster.
+type WeightedCluster struct {
+	// Weight is the relative weight of the cluster.  It will never be zero.
+	Weight uint32
+	// HTTPFilterConfigOverride contains any HTTP filter config overrides for
+	// the weighted cluster which may be present.
+	HTTPFilterConfigOverride map[string]httpfilter.FilterConfig
 }
 
 // HeaderMatcher represents header matchers.
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 2a31b7e2e508..bdce958f30ca 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -19,10 +19,22 @@
 package client
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
+	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v1typepb "github.com/cncf/udpa/go/udpa/type/v1"
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -31,14 +43,9 @@ import (
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
 	anypb "github.com/golang/protobuf/ptypes/any"
+	spb "github.com/golang/protobuf/ptypes/struct"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"google.golang.org/grpc/xds/internal/version"
-	"google.golang.org/protobuf/types/known/durationpb"
 )
 
 func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
@@ -77,32 +84,59 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 				return mLis
 			}(),
 		}
-		v3Lis = &anypb.Any{
-			TypeUrl: version.V3ListenerURL,
-			Value: func() []byte {
-				cm := &v3httppb.HttpConnectionManager{
-					RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-						Rds: &v3httppb.Rds{
-							ConfigSource: &v3corepb.ConfigSource{
-								ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-							},
-							RouteConfigName: v3RouteConfigName,
+		customFilter = &v3httppb.HttpFilter{
+			Name:       "customFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig},
+		}
+		typedStructFilter = &v3httppb.HttpFilter{
+			Name:       "customFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: wrappedCustomFilterTypedStructConfig},
+		}
+		customFilter2 = &v3httppb.HttpFilter{
+			Name:       "customFilter2",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig},
+		}
+		errFilter = &v3httppb.HttpFilter{
+			Name:       "errFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig},
+		}
+		clientOnlyCustomFilter = &v3httppb.HttpFilter{
+			Name:       "clientOnlyCustomFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig},
+		}
+		serverOnlyCustomFilter = &v3httppb.HttpFilter{
+			Name:       "serverOnlyCustomFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig},
+		}
+		v3LisWithFilters = func(fs ...*v3httppb.HttpFilter) *anypb.Any {
+			hcm := &v3httppb.HttpConnectionManager{
+				RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+					Rds: &v3httppb.Rds{
+						ConfigSource: &v3corepb.ConfigSource{
+							ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
 						},
+						RouteConfigName: v3RouteConfigName,
 					},
-					CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
-						MaxStreamDuration: durationpb.New(time.Second),
-					},
-				}
-				mcm, _ := ptypes.MarshalAny(cm)
-				lis := &v3listenerpb.Listener{
-					Name: v3LDSTarget,
-					ApiListener: &v3listenerpb.ApiListener{
-						ApiListener: mcm,
-					},
-				}
-				mLis, _ := proto.Marshal(lis)
-				return mLis
-			}(),
+				},
+				CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
+					MaxStreamDuration: durationpb.New(time.Second),
+				},
+				HttpFilters: fs,
+			}
+			return &anypb.Any{
+				TypeUrl: version.V3ListenerURL,
+				Value: func() []byte {
+					mcm, _ := ptypes.MarshalAny(hcm)
+					lis := &v3listenerpb.Listener{
+						Name: v3LDSTarget,
+						ApiListener: &v3listenerpb.ApiListener{
+							ApiListener: mcm,
+						},
+					}
+					mLis, _ := proto.Marshal(lis)
+					return mLis
+				}(),
+			}
 		}
 	)
 
@@ -111,6 +145,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 		resources  []*anypb.Any
 		wantUpdate map[string]ListenerUpdate
 		wantErr    bool
+		disableFI  bool // disable fault injection
 	}{
 		{
 			name:      "non-listener resource",
@@ -272,6 +307,104 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 		{
 			name: "empty resource list",
 		},
+		{
+			name:      "v3 with no filters",
+			resources: []*anypb.Any{v3LisWithFilters()},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+			},
+		},
+		{
+			name:      "v3 with custom filter",
+			resources: []*anypb.Any{v3LisWithFilters(customFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+					HTTPFilters: []HTTPFilter{{
+						Name:   "customFilter",
+						Filter: httpFilter{},
+						Config: filterConfig{Cfg: customFilterConfig},
+					}},
+				},
+			},
+		},
+		{
+			name:      "v3 with custom filter in typed struct",
+			resources: []*anypb.Any{v3LisWithFilters(typedStructFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+					HTTPFilters: []HTTPFilter{{
+						Name:   "customFilter",
+						Filter: httpFilter{},
+						Config: filterConfig{Cfg: customFilterTypedStructConfig},
+					}},
+				},
+			},
+		},
+		{
+			name:      "v3 with custom filter, fault injection disabled",
+			resources: []*anypb.Any{v3LisWithFilters(customFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+			},
+			disableFI: true,
+		},
+		{
+			name:      "v3 with two filters with same name",
+			resources: []*anypb.Any{v3LisWithFilters(customFilter, customFilter)},
+			wantErr:   true,
+		},
+		{
+			name:      "v3 with two filters - same type different name",
+			resources: []*anypb.Any{v3LisWithFilters(customFilter, customFilter2)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+					HTTPFilters: []HTTPFilter{{
+						Name:   "customFilter",
+						Filter: httpFilter{},
+						Config: filterConfig{Cfg: customFilterConfig},
+					}, {
+						Name:   "customFilter2",
+						Filter: httpFilter{},
+						Config: filterConfig{Cfg: customFilterConfig},
+					}},
+				},
+			},
+		},
+		{
+			name:      "v3 with server-only filter",
+			resources: []*anypb.Any{v3LisWithFilters(serverOnlyCustomFilter)},
+			wantErr:   true,
+		},
+		{
+			name:      "v3 with client-only filter",
+			resources: []*anypb.Any{v3LisWithFilters(clientOnlyCustomFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+					HTTPFilters: []HTTPFilter{{
+						Name:   "clientOnlyCustomFilter",
+						Filter: clientOnlyHTTPFilter{},
+						Config: filterConfig{Cfg: clientOnlyCustomFilterConfig},
+					}},
+				},
+			},
+		},
+		{
+			name:      "v3 with err filter",
+			resources: []*anypb.Any{v3LisWithFilters(errFilter)},
+			wantErr:   true,
+		},
+		{
+			name:      "v3 with error filter, fault injection disabled",
+			resources: []*anypb.Any{v3LisWithFilters(errFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+			},
+			disableFI: true,
+		},
 		{
 			name:      "v2 listener resource",
 			resources: []*anypb.Any{v2Lis},
@@ -281,14 +414,14 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 		},
 		{
 			name:      "v3 listener resource",
-			resources: []*anypb.Any{v3Lis},
+			resources: []*anypb.Any{v3LisWithFilters()},
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
 			},
 		},
 		{
 			name:      "multiple listener resources",
-			resources: []*anypb.Any{v2Lis, v3Lis},
+			resources: []*anypb.Any{v2Lis, v3LisWithFilters()},
 			wantUpdate: map[string]ListenerUpdate{
 				v2LDSTarget: {RouteConfigName: v2RouteConfigName},
 				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
@@ -298,10 +431,16 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			oldFI := env.FaultInjectionSupport
+			env.FaultInjectionSupport = !test.disableFI
+
 			update, _, err := UnmarshalListener("", test.resources, nil)
-			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+			if ((err != nil) != test.wantErr) ||
+				!cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty(), protocmp.Transform()) {
 				t.Errorf("UnmarshalListener(%v) = (%v, %v) want (%v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
 			}
+
+			env.FaultInjectionSupport = oldFI
 		})
 	}
 }
@@ -939,3 +1078,115 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 		})
 	}
 }
+
+type filterConfig struct {
+	httpfilter.FilterConfig
+	Cfg      proto.Message
+	Override proto.Message
+}
+
+// httpFilter allows testing the http filter registry and parsing functionality.
+type httpFilter struct {
+	httpfilter.ClientInterceptorBuilder
+	httpfilter.ServerInterceptorBuilder
+}
+
+func (httpFilter) TypeURLs() []string { return []string{"custom.filter"} }
+
+func (httpFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	return filterConfig{Cfg: cfg}, nil
+}
+
+func (httpFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) {
+	return filterConfig{Override: override}, nil
+}
+
+// errHTTPFilter returns errors no matter what is passed to ParseFilterConfig.
+type errHTTPFilter struct {
+	httpfilter.ClientInterceptorBuilder
+}
+
+func (errHTTPFilter) TypeURLs() []string { return []string{"err.custom.filter"} }
+
+func (errHTTPFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	return nil, fmt.Errorf("error from ParseFilterConfig")
+}
+
+func (errHTTPFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) {
+	return nil, fmt.Errorf("error from ParseFilterConfigOverride")
+}
+
+func init() {
+	httpfilter.Register(httpFilter{})
+	httpfilter.Register(errHTTPFilter{})
+	httpfilter.Register(serverOnlyHTTPFilter{})
+	httpfilter.Register(clientOnlyHTTPFilter{})
+}
+
+// serverOnlyHTTPFilter does not implement ClientInterceptorBuilder
+type serverOnlyHTTPFilter struct {
+	httpfilter.ServerInterceptorBuilder
+}
+
+func (serverOnlyHTTPFilter) TypeURLs() []string { return []string{"serverOnly.custom.filter"} }
+
+func (serverOnlyHTTPFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	return filterConfig{Cfg: cfg}, nil
+}
+
+func (serverOnlyHTTPFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) {
+	return filterConfig{Override: override}, nil
+}
+
+// clientOnlyHTTPFilter does not implement ServerInterceptorBuilder
+type clientOnlyHTTPFilter struct {
+	httpfilter.ClientInterceptorBuilder
+}
+
+func (clientOnlyHTTPFilter) TypeURLs() []string { return []string{"clientOnly.custom.filter"} }
+
+func (clientOnlyHTTPFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	return filterConfig{Cfg: cfg}, nil
+}
+
+func (clientOnlyHTTPFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) {
+	return filterConfig{Override: override}, nil
+}
+
+var customFilterConfig = &anypb.Any{
+	TypeUrl: "custom.filter",
+	Value:   []byte{1, 2, 3},
+}
+
+var errFilterConfig = &anypb.Any{
+	TypeUrl: "err.custom.filter",
+	Value:   []byte{1, 2, 3},
+}
+
+var serverOnlyCustomFilterConfig = &anypb.Any{
+	TypeUrl: "serverOnly.custom.filter",
+	Value:   []byte{1, 2, 3},
+}
+
+var clientOnlyCustomFilterConfig = &anypb.Any{
+	TypeUrl: "clientOnly.custom.filter",
+	Value:   []byte{1, 2, 3},
+}
+
+var customFilterTypedStructConfig = &v1typepb.TypedStruct{
+	TypeUrl: "custom.filter",
+	Value: &spb.Struct{
+		Fields: map[string]*spb.Value{
+			"foo": {Kind: &spb.Value_StringValue{StringValue: "bar"}},
+		},
+	},
+}
+var wrappedCustomFilterTypedStructConfig *anypb.Any
+
+func init() {
+	var err error
+	wrappedCustomFilterTypedStructConfig, err = ptypes.MarshalAny(customFilterTypedStructConfig)
+	if err != nil {
+		panic(err.Error())
+	}
+}
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index 1a82e0304e74..1a33c90a7dbc 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -19,6 +19,7 @@
 package client
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -33,6 +34,8 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/durationpb"
 )
@@ -46,11 +49,42 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 		clusterName              = "clusterName"
 	)
 
+	var (
+		goodRouteConfigWithFilterConfigs = func(cfgs map[string]*anypb.Any) *v3routepb.RouteConfiguration {
+			return &v3routepb.RouteConfiguration{
+				Name: routeName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{ldsTarget},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+						Action: &v3routepb.Route_Route{
+							Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}},
+						},
+					}},
+					TypedPerFilterConfig: cfgs,
+				}},
+			}
+		}
+		goodUpdateWithFilterConfigs = func(cfgs map[string]httpfilter.FilterConfig) RouteConfigUpdate {
+			return RouteConfigUpdate{
+				VirtualHosts: []*VirtualHost{{
+					Domains: []string{ldsTarget},
+					Routes: []*Route{{
+						Prefix:           newStringP("/"),
+						WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}},
+					}},
+					HTTPFilterConfigOverride: cfgs,
+				}},
+			}
+		}
+	)
+
 	tests := []struct {
 		name       string
 		rc         *v3routepb.RouteConfiguration
 		wantUpdate RouteConfigUpdate
 		wantError  bool
+		disableFI  bool // disable fault injection
 	}{
 		{
 			name: "default-route-match-field-is-nil",
@@ -141,7 +175,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), CaseInsensitive: true, Action: map[string]uint32{clusterName: 1}}},
+						Routes:  []*Route{{Prefix: newStringP("/"), CaseInsensitive: true, WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}}},
 					},
 				},
 			},
@@ -183,11 +217,11 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{uninterestingDomain},
-						Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+						Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
 					},
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{clusterName: 1}}},
+						Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}}},
 					},
 				},
 			},
@@ -217,7 +251,7 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}}},
+						Routes:  []*Route{{Prefix: newStringP("/"), WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}}},
 					},
 				},
 			},
@@ -287,7 +321,14 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}},
+						Routes: []*Route{{
+							Prefix: newStringP("/"),
+							WeightedClusters: map[string]WeightedCluster{
+								"a": {Weight: 2},
+								"b": {Weight: 3},
+								"c": {Weight: 5},
+							},
+						}},
 					},
 				},
 			},
@@ -317,7 +358,11 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: newDurationP(time.Second)}},
+						Routes: []*Route{{
+							Prefix:            newStringP("/"),
+							WeightedClusters:  map[string]WeightedCluster{clusterName: {Weight: 1}},
+							MaxStreamDuration: newDurationP(time.Second),
+						}},
 					},
 				},
 			},
@@ -347,7 +392,11 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: newDurationP(time.Second)}},
+						Routes: []*Route{{
+							Prefix:            newStringP("/"),
+							WeightedClusters:  map[string]WeightedCluster{clusterName: {Weight: 1}},
+							MaxStreamDuration: newDurationP(time.Second),
+						}},
 					},
 				},
 			},
@@ -377,18 +426,52 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 				VirtualHosts: []*VirtualHost{
 					{
 						Domains: []string{ldsTarget},
-						Routes:  []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}, MaxStreamDuration: newDurationP(0)}},
+						Routes: []*Route{{
+							Prefix:            newStringP("/"),
+							WeightedClusters:  map[string]WeightedCluster{clusterName: {Weight: 1}},
+							MaxStreamDuration: newDurationP(0),
+						}},
 					},
 				},
 			},
 		},
+		{
+			name:       "good-route-config-with-http-filter-config",
+			rc:         goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}),
+			wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}),
+		},
+		{
+			name:       "good-route-config-with-http-filter-config-typed-struct",
+			rc:         goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterTypedStructConfig}),
+			wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterTypedStructConfig}}),
+		},
+		{
+			name:      "good-route-config-with-http-err-filter-config",
+			rc:        goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
+			wantError: true,
+		},
+		{
+			name:       "good-route-config-with-http-err-filter-config-fi-disabled",
+			disableFI:  true,
+			rc:         goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
+			wantUpdate: goodUpdateWithFilterConfigs(nil),
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, nil)
-			if (gotError != nil) != test.wantError || !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
+			oldFI := env.FaultInjectionSupport
+			env.FaultInjectionSupport = !test.disableFI
+
+			gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, nil, false)
+			if (gotError != nil) != test.wantError ||
+				!cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty(),
+					cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string {
+						return fmt.Sprint(fc)
+					})) {
 				t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) returned unexpected, diff (-want +got):\\n%s", test.rc, ldsTarget, cmp.Diff(test.wantUpdate, gotUpdate, cmpopts.EquateEmpty()))
+
+				env.FaultInjectionSupport = oldFI
 			}
 		})
 	}
@@ -518,11 +601,11 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 					VirtualHosts: []*VirtualHost{
 						{
 							Domains: []string{uninterestingDomain},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
 						},
 						{
 							Domains: []string{ldsTarget},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v2ClusterName: {Weight: 1}}}},
 						},
 					},
 				},
@@ -536,11 +619,11 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 					VirtualHosts: []*VirtualHost{
 						{
 							Domains: []string{uninterestingDomain},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
 						},
 						{
 							Domains: []string{ldsTarget},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}}},
 						},
 					},
 				},
@@ -554,11 +637,11 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 					VirtualHosts: []*VirtualHost{
 						{
 							Domains: []string{uninterestingDomain},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
 						},
 						{
 							Domains: []string{ldsTarget},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}}},
 						},
 					},
 				},
@@ -566,11 +649,11 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 					VirtualHosts: []*VirtualHost{
 						{
 							Domains: []string{uninterestingDomain},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
 						},
 						{
 							Domains: []string{ldsTarget},
-							Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v2ClusterName: {Weight: 1}}}},
 						},
 					},
 				},
@@ -588,11 +671,44 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 }
 
 func (s) TestRoutesProtoToSlice(t *testing.T) {
+	var (
+		goodRouteWithFilterConfigs = func(cfgs map[string]*anypb.Any) []*v3routepb.Route {
+			// Sets per-filter config in cluster "B" and in the route.
+			return []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{
+					PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+					CaseSensitive: &wrapperspb.BoolValue{Value: false},
+				},
+				Action: &v3routepb.Route_Route{
+					Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+							WeightedClusters: &v3routepb.WeightedCluster{
+								Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+									{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}, TypedPerFilterConfig: cfgs},
+									{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+								},
+								TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+							}}}},
+				TypedPerFilterConfig: cfgs,
+			}}
+		}
+		goodUpdateWithFilterConfigs = func(cfgs map[string]httpfilter.FilterConfig) []*Route {
+			// Sets per-filter config in cluster "B" and in the route.
+			return []*Route{{
+				Prefix:                   newStringP("/"),
+				CaseInsensitive:          true,
+				WeightedClusters:         map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60, HTTPFilterConfigOverride: cfgs}},
+				HTTPFilterConfigOverride: cfgs,
+			}}
+		}
+	)
+
 	tests := []struct {
 		name       string
 		routes     []*v3routepb.Route
 		wantRoutes []*Route
 		wantErr    bool
+		disableFI  bool // disable fault injection
 	}{
 		{
 			name: "no path",
@@ -620,9 +736,9 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 							}}}},
 			}},
 			wantRoutes: []*Route{{
-				Prefix:          newStringP("/"),
-				CaseInsensitive: true,
-				Action:          map[string]uint32{"A": 40, "B": 60},
+				Prefix:           newStringP("/"),
+				CaseInsensitive:  true,
+				WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}},
 			}},
 		},
 		{
@@ -668,8 +784,8 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 						PrefixMatch: newStringP("tv"),
 					},
 				},
-				Fraction: newUInt32P(10000),
-				Action:   map[string]uint32{"A": 40, "B": 60},
+				Fraction:         newUInt32P(10000),
+				WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}},
 			}},
 			wantErr: false,
 		},
@@ -702,8 +818,8 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			// Only one route in the result, because the second one with query
 			// parameters is ignored.
 			wantRoutes: []*Route{{
-				Prefix: newStringP("/a/"),
-				Action: map[string]uint32{"A": 40, "B": 60},
+				Prefix:           newStringP("/a/"),
+				WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}},
 			}},
 			wantErr: false,
 		},
@@ -771,16 +887,49 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name:       "with custom HTTP filter config",
+			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}),
+			wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}),
+		},
+		{
+			name:       "with custom HTTP filter config in typed struct",
+			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterTypedStructConfig}),
+			wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterTypedStructConfig}}),
+		},
+		{
+			name:       "with custom HTTP filter config, FI disabled",
+			disableFI:  true,
+			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}),
+			wantRoutes: goodUpdateWithFilterConfigs(nil),
+		},
+		{
+			name:    "with erroring custom HTTP filter config",
+			routes:  goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
+			wantErr: true,
+		},
+		{
+			name:       "with erroring custom HTTP filter config, FI disabled",
+			disableFI:  true,
+			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
+			wantRoutes: goodUpdateWithFilterConfigs(nil),
+		},
 	}
 
 	cmpOpts := []cmp.Option{
 		cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}),
 		cmpopts.EquateEmpty(),
+		cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string {
+			return fmt.Sprint(fc)
+		}),
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := routesProtoToSlice(tt.routes, nil)
+			oldFI := env.FaultInjectionSupport
+			env.FaultInjectionSupport = !tt.disableFI
+
+			got, err := routesProtoToSlice(tt.routes, nil, false)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -788,6 +937,8 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			if !cmp.Equal(got, tt.wantRoutes, cmpOpts...) {
 				t.Errorf("routesProtoToSlice() got = %v, want %v, diff: %v", got, tt.wantRoutes, cmp.Diff(got, tt.wantRoutes, cmpOpts...))
 			}
+
+			env.FaultInjectionSupport = oldFI
 		})
 	}
 }
diff --git a/xds/internal/client/v2/rds_test.go b/xds/internal/client/v2/rds_test.go
index 94508f9e5cf1..ed5058836ce6 100644
--- a/xds/internal/client/v2/rds_test.go
+++ b/xds/internal/client/v2/rds_test.go
@@ -97,11 +97,11 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 				VirtualHosts: []*xdsclient.VirtualHost{
 					{
 						Domains: []string{uninterestingDomain},
-						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{uninterestingClusterName: 1}}},
+						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
 					},
 					{
 						Domains: []string{goodLDSTarget1},
-						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}},
+						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{goodClusterName1: {Weight: 1}}}},
 					},
 				},
 			},
diff --git a/xds/internal/client/watchers_route_test.go b/xds/internal/client/watchers_route_test.go
index 93a47e9a3423..5f44e5493330 100644
--- a/xds/internal/client/watchers_route_test.go
+++ b/xds/internal/client/watchers_route_test.go
@@ -66,7 +66,7 @@ func (s) TestRDSWatch(t *testing.T) {
 		VirtualHosts: []*VirtualHost{
 			{
 				Domains: []string{testLDSName},
-				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{testCDSName: {Weight: 1}}}},
 			},
 		},
 	}
@@ -138,7 +138,7 @@ func (s) TestRDSTwoWatchSameResourceName(t *testing.T) {
 		VirtualHosts: []*VirtualHost{
 			{
 				Domains: []string{testLDSName},
-				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{testCDSName: {Weight: 1}}}},
 			},
 		},
 	}
@@ -217,7 +217,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		VirtualHosts: []*VirtualHost{
 			{
 				Domains: []string{testLDSName},
-				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}},
+				Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{testCDSName + "1": {Weight: 1}}}},
 			},
 		},
 	}
@@ -225,7 +225,7 @@ func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) {
 		VirtualHosts: []*VirtualHost{
 			{
 				Domains: []string{testLDSName},
-				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}},
+				Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{testCDSName + "2": {Weight: 1}}}},
 			},
 		},
 	}
@@ -276,7 +276,7 @@ func (s) TestRDSWatchAfterCache(t *testing.T) {
 		VirtualHosts: []*VirtualHost{
 			{
 				Domains: []string{testLDSName},
-				Routes:  []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}},
+				Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{testCDSName: {Weight: 1}}}},
 			},
 		},
 	}
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index caeb42856e09..055a782d468b 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 
+	v1typepb "github.com/cncf/udpa/go/udpa/type/v1"
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
@@ -34,11 +35,13 @@ import (
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/golang/protobuf/proto"
-	anypb "github.com/golang/protobuf/ptypes/any"
+	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/protobuf/types/known/anypb"
 
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -49,19 +52,21 @@ const transportSocketName = "envoy.transport_sockets.tls"
 // UnmarshalListener processes resources received in an LDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
-func UnmarshalListener(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, UpdateMetadata, error) {
+func UnmarshalListener(_ string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, UpdateMetadata, error) {
 	update := make(map[string]ListenerUpdate)
 	for _, r := range resources {
 		if !IsListenerResource(r.GetTypeUrl()) {
 			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in LDS response", r.GetTypeUrl())
 		}
+		// TODO: Pass version.TransportAPI instead of relying upon the type URL
+		v2 := r.GetTypeUrl() == version.V2ListenerURL
 		lis := &v3listenerpb.Listener{}
 		if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
 			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
 
-		lu, err := processListener(lis)
+		lu, err := processListener(lis, v2)
 		if err != nil {
 			return nil, UpdateMetadata{}, err
 		}
@@ -70,16 +75,16 @@ func UnmarshalListener(version string, resources []*anypb.Any, logger *grpclog.P
 	return update, UpdateMetadata{}, nil
 }
 
-func processListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
+func processListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUpdate, error) {
 	if lis.GetApiListener() != nil {
-		return processClientSideListener(lis)
+		return processClientSideListener(lis, v2)
 	}
 	return processServerSideListener(lis)
 }
 
 // processClientSideListener checks if the provided Listener proto meets
 // the expected criteria. If so, it returns a non-empty routeConfigName.
-func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
+func processClientSideListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUpdate, error) {
 	update := &ListenerUpdate{}
 
 	apiLisAny := lis.GetApiListener().GetApiListener()
@@ -111,11 +116,106 @@ func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 		return nil, fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
 	}
 
+	if v2 {
+		return update, nil
+	}
+
+	// The following checks and fields only apply to xDS protocol versions v3+.
+
 	update.MaxStreamDuration = apiLis.GetCommonHttpProtocolOptions().GetMaxStreamDuration().AsDuration()
 
+	var err error
+	if update.HTTPFilters, err = processHTTPFilters(apiLis.GetHttpFilters(), false); err != nil {
+		return nil, fmt.Errorf("xds: %v", err)
+	}
+
 	return update, nil
 }
 
+func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) {
+	if typeURL := config.GetTypeUrl(); typeURL != "type.googleapis.com/udpa.type.v1.TypedStruct" {
+		return config, typeURL, nil
+	}
+	// The real type name is inside the TypedStruct.
+	s := new(v1typepb.TypedStruct)
+	if err := ptypes.UnmarshalAny(config, s); err != nil {
+		return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err)
+	}
+	return s, s.GetTypeUrl(), nil
+}
+
+func validateHTTPFilterConfig(cfg *anypb.Any, lds bool) (httpfilter.Filter, httpfilter.FilterConfig, error) {
+	config, typeURL, err := unwrapHTTPFilterConfig(cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+	filterBuilder := httpfilter.Get(typeURL)
+	if filterBuilder == nil {
+		return nil, nil, fmt.Errorf("no filter implementation found for %q", typeURL)
+	}
+	parseFunc := filterBuilder.ParseFilterConfig
+	if !lds {
+		parseFunc = filterBuilder.ParseFilterConfigOverride
+	}
+	filterConfig, err := parseFunc(config)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing config for filter %q: %v", typeURL, err)
+	}
+	return filterBuilder, filterConfig, nil
+}
+
+func processHTTPFilterOverrides(cfgs map[string]*anypb.Any) (map[string]httpfilter.FilterConfig, error) {
+	if !env.FaultInjectionSupport || len(cfgs) == 0 {
+		return nil, nil
+	}
+	m := make(map[string]httpfilter.FilterConfig)
+	for name, cfg := range cfgs {
+		_, config, err := validateHTTPFilterConfig(cfg, false)
+		if err != nil {
+			return nil, err
+		}
+		m[name] = config
+	}
+	return m, nil
+}
+
+func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool) ([]HTTPFilter, error) {
+	if !env.FaultInjectionSupport {
+		return nil, nil
+	}
+
+	ret := make([]HTTPFilter, 0, len(filters))
+	seenNames := make(map[string]bool, len(filters))
+	for _, filter := range filters {
+		name := filter.GetName()
+		if name == "" {
+			return nil, errors.New("filter missing name field")
+		}
+		if seenNames[name] {
+			return nil, fmt.Errorf("duplicate filter name %q", name)
+		}
+		seenNames[name] = true
+
+		httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true)
+		if err != nil {
+			return nil, err
+		}
+		if server {
+			if _, ok := httpFilter.(httpfilter.ServerInterceptorBuilder); !ok {
+				return nil, fmt.Errorf("httpFilter %q not supported server-side", name)
+			}
+		} else {
+			if _, ok := httpFilter.(httpfilter.ClientInterceptorBuilder); !ok {
+				return nil, fmt.Errorf("httpFilter %q not supported client-side", name)
+			}
+		}
+
+		// Save name/config
+		ret = append(ret, HTTPFilter{Name: name, Filter: httpFilter, Config: config})
+	}
+	return ret, nil
+}
+
 func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
 	// Make sure that an address encoded in the received listener resource, and
 	// that it matches the one specified in the name. Listener names on the
@@ -196,7 +296,7 @@ func getAddressFromName(name string) (host string, port string, err error) {
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
 // configuration resources of interest.
-func UnmarshalRouteConfig(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, UpdateMetadata, error) {
+func UnmarshalRouteConfig(_ string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, UpdateMetadata, error) {
 	update := make(map[string]RouteConfigUpdate)
 	for _, r := range resources {
 		if !IsRouteConfigResource(r.GetTypeUrl()) {
@@ -208,8 +308,10 @@ func UnmarshalRouteConfig(version string, resources []*anypb.Any, logger *grpclo
 		}
 		logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, rc)
 
+		// TODO: Pass version.TransportAPI instead of relying upon the type URL
+		v2 := r.GetTypeUrl() == version.V2RouteConfigURL
 		// Use the hostname (resourceName for LDS) to find the routes.
-		u, err := generateRDSUpdateFromRouteConfiguration(rc, logger)
+		u, err := generateRDSUpdateFromRouteConfiguration(rc, logger, v2)
 		if err != nil {
 			return nil, UpdateMetadata{}, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
 		}
@@ -234,22 +336,30 @@ func UnmarshalRouteConfig(version string, resources []*anypb.Any, logger *grpclo
 // field must be empty and whose route field must be set.  Inside that route
 // message, the cluster field will contain the clusterName or weighted clusters
 // we are looking for.
-func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, logger *grpclog.PrefixLogger) (RouteConfigUpdate, error) {
+func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, logger *grpclog.PrefixLogger, v2 bool) (RouteConfigUpdate, error) {
 	var vhs []*VirtualHost
 	for _, vh := range rc.GetVirtualHosts() {
-		routes, err := routesProtoToSlice(vh.Routes, logger)
+		routes, err := routesProtoToSlice(vh.Routes, logger, v2)
 		if err != nil {
 			return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
 		}
-		vhs = append(vhs, &VirtualHost{
+		vhOut := &VirtualHost{
 			Domains: vh.GetDomains(),
 			Routes:  routes,
-		})
+		}
+		if !v2 {
+			cfgs, err := processHTTPFilterOverrides(vh.GetTypedPerFilterConfig())
+			if err != nil {
+				return RouteConfigUpdate{}, fmt.Errorf("virtual host %+v: %v", vh, err)
+			}
+			vhOut.HTTPFilterConfigOverride = cfgs
+		}
+		vhs = append(vhs, vhOut)
 	}
 	return RouteConfigUpdate{VirtualHosts: vhs}, nil
 }
 
-func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) {
+func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger, v2 bool) ([]*Route, error) {
 	var routesRet []*Route
 
 	for _, r := range routes {
@@ -325,11 +435,11 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			route.Fraction = &n
 		}
 
-		clusters := make(map[string]uint32)
+		route.WeightedClusters = make(map[string]WeightedCluster)
 		action := r.GetRoute()
 		switch a := action.GetClusterSpecifier().(type) {
 		case *v3routepb.RouteAction_Cluster:
-			clusters[a.Cluster] = 1
+			route.WeightedClusters[a.Cluster] = WeightedCluster{Weight: 1}
 		case *v3routepb.RouteAction_WeightedClusters:
 			wcs := a.WeightedClusters
 			var totalWeight uint32
@@ -338,7 +448,15 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 				if w == 0 {
 					continue
 				}
-				clusters[c.GetName()] = w
+				wc := WeightedCluster{Weight: w}
+				if !v2 {
+					cfgs, err := processHTTPFilterOverrides(c.GetTypedPerFilterConfig())
+					if err != nil {
+						return nil, fmt.Errorf("route %+v, action %+v: %v", r, a, err)
+					}
+					wc.HTTPFilterConfigOverride = cfgs
+				}
+				route.WeightedClusters[c.GetName()] = wc
 				totalWeight += w
 			}
 			if totalWeight != wcs.GetTotalWeight().GetValue() {
@@ -351,8 +469,6 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			continue
 		}
 
-		route.Action = clusters
-
 		msd := action.GetMaxStreamDuration()
 		// Prefer grpc_timeout_header_max, if set.
 		dur := msd.GetGrpcTimeoutHeaderMax()
@@ -363,6 +479,14 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger)
 			d := dur.AsDuration()
 			route.MaxStreamDuration = &d
 		}
+
+		if !v2 {
+			cfgs, err := processHTTPFilterOverrides(r.GetTypedPerFilterConfig())
+			if err != nil {
+				return nil, fmt.Errorf("route %+v: %v", r, err)
+			}
+			route.HTTPFilterConfigOverride = cfgs
+		}
 		routesRet = append(routesRet, &route)
 	}
 	return routesRet, nil
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index 3d75bc3e86f4..065ce6145638 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -40,6 +40,7 @@ const (
 	BootstrapFileContentEnv   = "GRPC_XDS_BOOTSTRAP_CONFIG"
 	circuitBreakingSupportEnv = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
 	timeoutSupportEnv         = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
+	faultInjectionSupportEnv  = "GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION"
 )
 
 var (
@@ -63,4 +64,7 @@ var (
 	// route actions is enabled.  This can be enabled by setting the
 	// environment variable "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT" to "true".
 	TimeoutSupport = strings.EqualFold(os.Getenv(timeoutSupportEnv), "true")
+	// FaultInjectionSupport is used to control both fault injection and HTTP
+	// filter support.
+	FaultInjectionSupport = strings.EqualFold(os.Getenv(faultInjectionSupportEnv), "true")
 )
diff --git a/xds/internal/httpfilter/httpfilter.go b/xds/internal/httpfilter/httpfilter.go
new file mode 100644
index 000000000000..6650241fab71
--- /dev/null
+++ b/xds/internal/httpfilter/httpfilter.go
@@ -0,0 +1,102 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package httpfilter contains the HTTPFilter interface and a registry for
+// storing and retrieving their implementations.
+package httpfilter
+
+import (
+	"github.com/golang/protobuf/proto"
+	iresolver "google.golang.org/grpc/internal/resolver"
+)
+
+// FilterConfig represents an opaque data structure holding configuration for a
+// filter.  Embed this interface to implement it.
+type FilterConfig interface {
+	isFilterConfig()
+}
+
+// Filter defines the parsing functionality of an HTTP filter.  A Filter may
+// optionally implement either ClientInterceptorBuilder or
+// ServerInterceptorBuilder or both, indicating it is capable of working on the
+// client side or server side or both, respectively.
+type Filter interface {
+	// TypeURLs are the proto message types supported by this filter.  A filter
+	// will be registered by each of its supported message types.
+	TypeURLs() []string
+	// ParseFilterConfig parses the provided configuration proto.Message from
+	// the LDS configuration of this filter.  This may be an anypb.Any or a
+	// udpa.type.v1.TypedStruct for filters that do not accept a custom type.
+	// The resulting FilterConfig will later be passed to Build.
+	ParseFilterConfig(proto.Message) (FilterConfig, error)
+	// ParseFilterConfigOverride parses the provided override configuration
+	// proto.Message from the RDS override configuration of this filter.  This
+	// may be an anypb.Any or a udpa.type.v1.TypedStruct for filters that do
+	// not accept a custom type.  The resulting FilterConfig will later be
+	// passed to Build.
+	ParseFilterConfigOverride(proto.Message) (FilterConfig, error)
+}
+
+// ClientInterceptorBuilder constructs a Client Interceptor.  If this type is
+// implemented by a Filter, it is capable of working on a client.
+type ClientInterceptorBuilder interface {
+	// BuildClientInterceptor uses the FilterConfigs produced above to produce
+	// an HTTP filter interceptor for clients.  config will always be non-nil,
+	// but override may be nil if no override config exists for the filter.  It
+	// is valid for Build to return a nil Interceptor and a nil error.  In this
+	// case, the RPC will not be intercepted by this filter.
+	BuildClientInterceptor(config, override FilterConfig) (iresolver.ClientInterceptor, error)
+}
+
+// ServerInterceptorBuilder constructs a Server Interceptor.  If this type is
+// implemented by a Filter, it is capable of working on a server.
+//
+// Server side filters are not currently supported, but this interface is
+// defined for clarity.
+type ServerInterceptorBuilder interface {
+	// BuildServerInterceptor uses the FilterConfigs produced above to produce
+	// an HTTP filter interceptor for servers.  config will always be non-nil,
+	// but override may be nil if no override config exists for the filter.  It
+	// is valid for Build to return a nil Interceptor and a nil error.  In this
+	// case, the RPC will not be intercepted by this filter.
+	BuildServerInterceptor(config, override FilterConfig) (iresolver.ServerInterceptor, error)
+}
+
+var (
+	// m is a map from scheme to filter.
+	m = make(map[string]Filter)
+)
+
+// Register registers the HTTP filter Builder to the filter map. b.TypeURLs()
+// will be used as the types for this filter.
+//
+// NOTE: this function must only be called during initialization time (i.e. in
+// an init() function), and is not thread-safe. If multiple filters are
+// registered with the same type URL, the one registered last will take effect.
+func Register(b Filter) {
+	for _, u := range b.TypeURLs() {
+		m[u] = b
+	}
+}
+
+// Get returns the HTTPFilter registered with typeURL.
+//
+// If no filter is register with typeURL, nil will be returned.
+func Get(typeURL string) Filter {
+	return m[typeURL]
+}
diff --git a/xds/internal/httpfilter/router/router.go b/xds/internal/httpfilter/router/router.go
new file mode 100644
index 000000000000..26e3acb5a4f4
--- /dev/null
+++ b/xds/internal/httpfilter/router/router.go
@@ -0,0 +1,95 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package router implements the Envoy Router HTTP filter.
+package router
+
+import (
+	"fmt"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/xds/internal/httpfilter"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+)
+
+// TypeURL is the message type for the Router configuration.
+const TypeURL = "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
+
+func init() {
+	httpfilter.Register(builder{})
+}
+
+// IsRouterFilter returns true iff a HTTP filter is a Router filter.
+func IsRouterFilter(b httpfilter.Filter) bool {
+	_, ok := b.(builder)
+	return ok
+}
+
+type builder struct {
+}
+
+func (builder) TypeURLs() []string { return []string{TypeURL} }
+
+func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	// The gRPC router filter does not currently use any fields from the
+	// config.  Verify type only.
+	if cfg == nil {
+		return nil, fmt.Errorf("router: nil configuration message provided")
+	}
+	any, ok := cfg.(*anypb.Any)
+	if !ok {
+		return nil, fmt.Errorf("router: error parsing config %v: unknown type %T", cfg, cfg)
+	}
+	msg := new(pb.Router)
+	if err := ptypes.UnmarshalAny(any, msg); err != nil {
+		return nil, fmt.Errorf("router: error parsing config %v: %v", cfg, err)
+	}
+	return config{}, nil
+}
+
+func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) {
+	if override != nil {
+		return nil, fmt.Errorf("router: unexpected config override specified: %v", override)
+	}
+	return config{}, nil
+}
+
+var _ httpfilter.ClientInterceptorBuilder = builder{}
+
+func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) {
+	if _, ok := cfg.(config); !ok {
+		return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg)
+	}
+	if override != nil {
+		return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override)
+	}
+	// The gRPC router is implemented within the xds resolver's config
+	// selector, not as a separate plugin.  So we return a nil HTTPFilter,
+	// which will not be invoked.
+	return nil, nil
+}
+
+// The gRPC router filter does not currently support any configuration.  Verify
+// type only.
+type config struct {
+	httpfilter.FilterConfig
+}
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index d61a546a04e2..95025fc8a423 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -19,6 +19,7 @@
 package resolver
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"sync/atomic"
@@ -29,7 +30,10 @@ import (
 	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
+	"google.golang.org/grpc/xds/internal/httpfilter/router"
 )
 
 const (
@@ -94,10 +98,24 @@ func serviceConfigJSON(activeClusters map[string]*clusterInfo) (string, error) {
 	return string(bs), nil
 }
 
+type virtualHost struct {
+	// map from filter name to its config
+	httpFilterConfigOverride map[string]httpfilter.FilterConfig
+}
+
+// routeCluster holds information about a cluster as referenced by a route.
+type routeCluster struct {
+	name string
+	// map from filter name to its config
+	httpFilterConfigOverride map[string]httpfilter.FilterConfig
+}
+
 type route struct {
 	m                 *compositeMatcher // converted from route matchers
-	clusters          wrr.WRR
+	clusters          wrr.WRR           // holds *routeCluster entries
 	maxStreamDuration time.Duration
+	// map from filter name to its config
+	httpFilterConfigOverride map[string]httpfilter.FilterConfig
 }
 
 func (r route) String() string {
@@ -105,9 +123,11 @@ func (r route) String() string {
 }
 
 type configSelector struct {
-	r        *xdsResolver
-	routes   []route
-	clusters map[string]*clusterInfo
+	r                *xdsResolver
+	virtualHost      virtualHost
+	routes           []route
+	clusters         map[string]*clusterInfo
+	httpFilterConfig []xdsclient.HTTPFilter
 }
 
 var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found")
@@ -127,18 +147,23 @@ func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RP
 	if rt == nil || rt.clusters == nil {
 		return nil, errNoMatchedRouteFound
 	}
-	cluster, ok := rt.clusters.Next().(string)
+	cluster, ok := rt.clusters.Next().(*routeCluster)
 	if !ok {
 		return nil, status.Errorf(codes.Internal, "error retrieving cluster for match: %v (%T)", cluster, cluster)
 	}
 	// Add a ref to the selected cluster, as this RPC needs this cluster until
 	// it is committed.
-	ref := &cs.clusters[cluster].refCount
+	ref := &cs.clusters[cluster.name].refCount
 	atomic.AddInt32(ref, 1)
 
+	interceptor, err := cs.newInterceptor(rt, cluster)
+	if err != nil {
+		return nil, err
+	}
+
 	config := &iresolver.RPCConfig{
 		// Communicate to the LB policy the chosen cluster.
-		Context: clustermanager.SetPickedCluster(rpcInfo.Context, cluster),
+		Context: clustermanager.SetPickedCluster(rpcInfo.Context, cluster.name),
 		OnCommitted: func() {
 			// When the RPC is committed, the cluster is no longer required.
 			// Decrease its ref.
@@ -151,6 +176,7 @@ func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RP
 				}
 			}
 		},
+		Interceptor: interceptor,
 	}
 
 	if env.TimeoutSupport && rt.maxStreamDuration != 0 {
@@ -160,6 +186,40 @@ func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RP
 	return config, nil
 }
 
+func (cs *configSelector) newInterceptor(rt *route, cluster *routeCluster) (iresolver.ClientInterceptor, error) {
+	if len(cs.httpFilterConfig) == 0 {
+		return nil, nil
+	}
+	interceptors := make([]iresolver.ClientInterceptor, 0, len(cs.httpFilterConfig))
+	for _, filter := range cs.httpFilterConfig {
+		if router.IsRouterFilter(filter.Filter) {
+			// Ignore any filters after the router filter.  The router itself
+			// is currently a nop.
+			return &interceptorList{interceptors: interceptors}, nil
+		}
+		override := cluster.httpFilterConfigOverride[filter.Name] // cluster is highest priority
+		if override == nil {
+			override = rt.httpFilterConfigOverride[filter.Name] // route is second priority
+		}
+		if override == nil {
+			override = cs.virtualHost.httpFilterConfigOverride[filter.Name] // VH is third & lowest priority
+		}
+		ib, ok := filter.Filter.(httpfilter.ClientInterceptorBuilder)
+		if !ok {
+			// Should not happen if it passed xdsClient validation.
+			return nil, fmt.Errorf("filter does not support use in client")
+		}
+		i, err := ib.BuildClientInterceptor(filter.Config, override)
+		if err != nil {
+			return nil, fmt.Errorf("error constructing filter: %v", err)
+		}
+		if i != nil {
+			interceptors = append(interceptors, i)
+		}
+	}
+	return nil, fmt.Errorf("error in xds config: no router filter present")
+}
+
 // stop decrements refs of all clusters referenced by this config selector.
 func (cs *configSelector) stop() {
 	// The resolver's old configSelector may be nil.  Handle that here.
@@ -194,15 +254,20 @@ var newWRR = wrr.NewRandom
 // r.activeClusters for previously-unseen clusters.
 func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, error) {
 	cs := &configSelector{
-		r:        r,
-		routes:   make([]route, len(su.routes)),
-		clusters: make(map[string]*clusterInfo),
+		r:                r,
+		virtualHost:      virtualHost{httpFilterConfigOverride: su.virtualHost.HTTPFilterConfigOverride},
+		routes:           make([]route, len(su.virtualHost.Routes)),
+		clusters:         make(map[string]*clusterInfo),
+		httpFilterConfig: su.ldsConfig.httpFilterConfig,
 	}
 
-	for i, rt := range su.routes {
+	for i, rt := range su.virtualHost.Routes {
 		clusters := newWRR()
-		for cluster, weight := range rt.Action {
-			clusters.Add(cluster, int64(weight))
+		for cluster, wc := range rt.WeightedClusters {
+			clusters.Add(&routeCluster{
+				name:                     cluster,
+				httpFilterConfigOverride: wc.HTTPFilterConfigOverride,
+			}, int64(wc.Weight))
 
 			// Initialize entries in cs.clusters map, creating entries in
 			// r.activeClusters as necessary.  Set to zero as they will be
@@ -226,6 +291,8 @@ func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, erro
 		} else {
 			cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration
 		}
+
+		cs.routes[i].httpFilterConfigOverride = rt.HTTPFilterConfigOverride
 	}
 
 	// Account for this config selector's clusters.  Do this after no further
@@ -242,3 +309,21 @@ type clusterInfo struct {
 	// number of references to this cluster; accessed atomically
 	refCount int32
 }
+
+type interceptorList struct {
+	interceptors []iresolver.ClientInterceptor
+}
+
+func (il *interceptorList) NewStream(ctx context.Context, ri iresolver.RPCInfo, cs iresolver.ClientStream) (context.Context, iresolver.ClientStream, error) {
+	for _, i := range il.interceptors {
+		var err error
+		newCTX, newCS, err := i.NewStream(ctx, ri, cs)
+		if err != nil {
+			cs.Done()
+			return nil, nil, err
+		}
+		cs = newCS
+		ctx = newCTX
+	}
+	return ctx, cs, nil
+}
diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go
index 79b83e95aa3c..913ac4ced15c 100644
--- a/xds/internal/resolver/watch_service.go
+++ b/xds/internal/resolver/watch_service.go
@@ -32,8 +32,8 @@ import (
 // are of interest to the xds resolver. The RDS request is built by first
 // making a LDS to get the RouteConfig name.
 type serviceUpdate struct {
-	// routes contain matchers+actions to route RPCs.
-	routes []*xdsclient.Route
+	// virtualHost contains routes and other configuration to route RPCs.
+	virtualHost *xdsclient.VirtualHost
 	// ldsConfig contains configuration that applies to all routes.
 	ldsConfig ldsConfig
 }
@@ -44,6 +44,7 @@ type ldsConfig struct {
 	// maxStreamDuration is from the HTTP connection manager's
 	// common_http_protocol_options field.
 	maxStreamDuration time.Duration
+	httpFilterConfig  []xdsclient.HTTPFilter
 }
 
 // watchService uses LDS and RDS to discover information about the provided
@@ -104,18 +105,18 @@ func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, er
 		return
 	}
 
-	oldLDSConfig := w.lastUpdate.ldsConfig
-	w.lastUpdate.ldsConfig = ldsConfig{maxStreamDuration: update.MaxStreamDuration}
+	w.lastUpdate.ldsConfig = ldsConfig{
+		maxStreamDuration: update.MaxStreamDuration,
+		httpFilterConfig:  update.HTTPFilters,
+	}
 
 	if w.rdsName == update.RouteConfigName {
 		// If the new RouteConfigName is same as the previous, don't cancel and
 		// restart the RDS watch.
-		if w.lastUpdate.ldsConfig != oldLDSConfig {
-			// The route name didn't change but the LDS data did; send it now.
-			// If the route name did change, then we will wait until the first
-			// RDS update before reporting this LDS config.
-			w.serviceCb(w.lastUpdate, nil)
-		}
+		//
+		// If the route name did change, then we must wait until the first RDS
+		// update before reporting this LDS config.
+		w.serviceCb(w.lastUpdate, nil)
 		return
 	}
 	w.rdsName = update.RouteConfigName
@@ -149,7 +150,7 @@ func (w *serviceUpdateWatcher) handleRDSResp(update xdsclient.RouteConfigUpdate,
 		return
 	}
 
-	w.lastUpdate.routes = matchVh.Routes
+	w.lastUpdate.virtualHost = matchVh
 	w.serviceCb(w.lastUpdate, nil)
 }
 
diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go
index 55a20372c018..705a3d35ae1b 100644
--- a/xds/internal/resolver/watch_service_test.go
+++ b/xds/internal/resolver/watch_service_test.go
@@ -140,7 +140,7 @@ func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantU
 	}
 	gotUpdate := u.(serviceUpdateErr)
 	if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty(), cmp.AllowUnexported(serviceUpdate{}, ldsConfig{})) {
-		return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil),  diff (-want +got):\n%s", gotUpdate.u, gotUpdate.err, wantUpdate, cmp.Diff(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty()))
+		return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil),  diff (-want +got):\n%s", gotUpdate.u, gotUpdate.err, wantUpdate, cmp.Diff(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty(), cmp.AllowUnexported(serviceUpdate{}, ldsConfig{})))
 	}
 	return nil
 }
@@ -166,12 +166,12 @@ func (s) TestServiceWatch(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -179,22 +179,22 @@ func (s) TestServiceWatch(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	wantUpdate2 := serviceUpdate{
-		routes: []*xdsclient.Route{{
-			Path:   newStringP(""),
-			Action: map[string]uint32{cluster: 1},
+	wantUpdate2 := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"},
+		Routes: []*xdsclient.Route{{
+			Path:             newStringP(""),
+			WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}},
 		}},
-	}
+	}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Path: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*xdsclient.Route{{Path: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 			{
 				// Another virtual host, with different domains.
 				Domains: []string{"random"},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -220,12 +220,12 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -241,12 +241,12 @@ func (s) TestServiceWatchLDSUpdate(t *testing.T) {
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr+"2")
 
 	// RDS update for the new name.
-	wantUpdate2 := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}}}
+	wantUpdate2 := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster + "2": {Weight: 1}}}}}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster + "2": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -272,12 +272,16 @@ func (s) TestServiceWatchLDSUpdateMaxStreamDuration(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}, ldsConfig: ldsConfig{maxStreamDuration: time.Second}}
+	wantUpdate := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{
+		Prefix:           newStringP(""),
+		WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}}},
+		ldsConfig: ldsConfig{maxStreamDuration: time.Second},
+	}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -286,19 +290,22 @@ func (s) TestServiceWatchLDSUpdateMaxStreamDuration(t *testing.T) {
 	}
 
 	// Another LDS update with the same RDS_name but different MaxStreamDuration (zero in this case).
-	wantUpdate2 := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate2 := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}}}}
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
 		t.Fatal(err)
 	}
 
 	// RDS update.
-	wantUpdate3 := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}}}
+	wantUpdate3 := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{
+		Prefix:           newStringP(""),
+		WeightedClusters: map[string]xdsclient.WeightedCluster{cluster + "2": {Weight: 1}}}},
+	}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster + "2": 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster + "2": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -324,12 +331,15 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
-	wantUpdate := serviceUpdate{routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}
+	wantUpdate := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{
+		Prefix:           newStringP(""),
+		WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
+	}}
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index b40e115299a4..d8c09db69b5a 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -247,6 +247,11 @@ func (r *xdsResolver) handleServiceUpdate(su serviceUpdate, err error) {
 		// Do not pass updates to the ClientConn once the resolver is closed.
 		return
 	}
+	// Remove any existing entry in updateCh and replace with the new one.
+	select {
+	case <-r.updateCh:
+	default:
+	}
 	r.updateCh <- suWithError{su: su, err: err}
 }
 
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 218a7a610913..e800d433150f 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"errors"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -44,6 +45,8 @@ import (
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
+	"google.golang.org/grpc/xds/internal/httpfilter/router"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
 )
@@ -58,6 +61,9 @@ const (
 
 var target = resolver.Target{Endpoint: targetStr}
 
+var routerFilter = xdsclient.HTTPFilter{Name: "rtr", Filter: httpfilter.Get(router.TypeURL)}
+var routerFilterList = []xdsclient.HTTPFilter{routerFilter}
+
 type s struct {
 	grpctest.Tester
 }
@@ -253,7 +259,7 @@ func (s) TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Call the watchAPI callback after closing the resolver, and make sure no
@@ -263,7 +269,7 @@ func (s) TestXDSResolverWatchCallbackAfterClose(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -288,7 +294,7 @@ func (s) TestXDSResolverBadServiceUpdate(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a bad service update and wait for the
@@ -316,7 +322,7 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 	defer replaceRandNumGenerator(0)()
 
@@ -326,7 +332,7 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 		wantClusters map[string]bool
 	}{
 		{
-			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+			routes: []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"test-cluster-1": {Weight: 1}}}},
 			wantJSON: `{"loadBalancingConfig":[{
     "xds_cluster_manager_experimental":{
       "children":{
@@ -338,9 +344,9 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 			wantClusters: map[string]bool{"test-cluster-1": true},
 		},
 		{
-			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
-				"cluster_1": 75,
-				"cluster_2": 25,
+			routes: []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{
+				"cluster_1": {Weight: 75},
+				"cluster_2": {Weight: 25},
 			}}},
 			// This update contains the cluster from the previous update as
 			// well as this update, as the previous config selector still
@@ -362,9 +368,9 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 			wantClusters: map[string]bool{"cluster_1": true, "cluster_2": true},
 		},
 		{
-			routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
-				"cluster_1": 75,
-				"cluster_2": 25,
+			routes: []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{
+				"cluster_1": {Weight: 75},
+				"cluster_2": {Weight: 25},
 			}}},
 			// With this redundant update, the old config selector has been
 			// stopped, so there are no more references to the first cluster.
@@ -450,7 +456,7 @@ func (s) TestXDSResolverRemovedWithRPCs(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a good service update and wait for the
@@ -459,7 +465,7 @@ func (s) TestXDSResolverRemovedWithRPCs(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"test-cluster-1": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -510,7 +516,7 @@ func (s) TestXDSResolverRemovedResource(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a good service update and wait for the
@@ -519,7 +525,7 @@ func (s) TestXDSResolverRemovedResource(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"test-cluster-1": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -620,7 +626,7 @@ func (s) TestXDSResolverWRR(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR)
@@ -632,9 +638,9 @@ func (s) TestXDSResolverWRR(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{
-					"A": 5,
-					"B": 10,
+				Routes: []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{
+					"A": {Weight: 5},
+					"B": {Weight: 10},
 				}}},
 			},
 		},
@@ -683,7 +689,7 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR)
@@ -697,15 +703,15 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 				Domains: []string{targetStr},
 				Routes: []*client.Route{{
 					Prefix:            newStringP("/foo"),
-					Action:            map[string]uint32{"A": 1},
+					WeightedClusters:  map[string]xdsclient.WeightedCluster{"A": {Weight: 1}},
 					MaxStreamDuration: newDurationP(5 * time.Second),
 				}, {
 					Prefix:            newStringP("/bar"),
-					Action:            map[string]uint32{"B": 1},
+					WeightedClusters:  map[string]xdsclient.WeightedCluster{"B": {Weight: 1}},
 					MaxStreamDuration: newDurationP(0),
 				}, {
-					Prefix: newStringP(""),
-					Action: map[string]uint32{"C": 1},
+					Prefix:           newStringP(""),
+					WeightedClusters: map[string]xdsclient.WeightedCluster{"C": {Weight: 1}},
 				}},
 			},
 		},
@@ -788,7 +794,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a good service update and wait for the
@@ -797,7 +803,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"test-cluster-1": 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"test-cluster-1": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -847,7 +853,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"NEW": 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"NEW": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -855,7 +861,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"NEW": 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"NEW": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -895,7 +901,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{"NEW": 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{"NEW": {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -938,7 +944,7 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a bad service update and wait for the
@@ -956,7 +962,7 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
 				Domains: []string{targetStr},
-				Routes:  []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}},
+				Routes:  []*client.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
 			},
 		},
 	}, nil)
@@ -994,7 +1000,7 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	waitForWatchListener(ctx, t, xdsC, targetStr)
-	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
 	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
 
 	// Invoke the watchAPI callback with a bad service update and wait for the
@@ -1024,6 +1030,262 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	}
 }
 
+type filterBuilder struct {
+	httpfilter.Filter // embedded as we do not need to implement registry / parsing in this test.
+	path              *[]string
+}
+
+var _ httpfilter.ClientInterceptorBuilder = &filterBuilder{}
+
+func (fb *filterBuilder) BuildClientInterceptor(config, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) {
+	if config == nil {
+		panic("unexpected missing config")
+	}
+	*fb.path = append(*fb.path, "build:"+config.(filterCfg).s)
+	err := config.(filterCfg).newStreamErr
+	if override != nil {
+		*fb.path = append(*fb.path, "override:"+override.(filterCfg).s)
+		err = override.(filterCfg).newStreamErr
+	}
+
+	return &filterInterceptor{path: fb.path, s: config.(filterCfg).s, err: err}, nil
+}
+
+type filterInterceptor struct {
+	path *[]string
+	s    string
+	err  error
+}
+
+func (fi *filterInterceptor) NewStream(ctx context.Context, i iresolver.RPCInfo, cs iresolver.ClientStream) (context.Context, iresolver.ClientStream, error) {
+	*fi.path = append(*fi.path, "newstream:"+fi.s)
+	if fi.err != nil {
+		return nil, nil, fi.err
+	}
+	return ctx, &clientStream{cs: cs, path: fi.path, s: fi.s}, nil
+}
+
+type clientStream struct {
+	cs   iresolver.ClientStream
+	path *[]string
+	s    string
+}
+
+func (cs *clientStream) Done() {
+	*cs.path = append(*cs.path, "done:"+cs.s)
+	cs.cs.Done()
+}
+
+type filterCfg struct {
+	httpfilter.FilterConfig
+	s            string
+	newStreamErr error
+}
+
+func (s) TestXDSResolverHTTPFilters(t *testing.T) {
+	var path []string
+	testCases := []struct {
+		name         string
+		ldsFilters   []xdsclient.HTTPFilter
+		vhOverrides  map[string]httpfilter.FilterConfig
+		rtOverrides  map[string]httpfilter.FilterConfig
+		clOverrides  map[string]httpfilter.FilterConfig
+		rpcRes       map[string][][]string
+		selectErr    string
+		newStreamErr string
+	}{
+		{
+			name: "no router filter",
+			ldsFilters: []xdsclient.HTTPFilter{
+				{Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1"}},
+			},
+			rpcRes: map[string][][]string{
+				"1": {
+					{"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+				},
+			},
+			selectErr: "no router filter present",
+		},
+		{
+			name: "ignored after router filter",
+			ldsFilters: []xdsclient.HTTPFilter{
+				{Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1"}},
+				routerFilter,
+				{Name: "foo2", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo2"}},
+			},
+			rpcRes: map[string][][]string{
+				"1": {
+					{"build:foo1", "newstream:foo1", "done:foo1"},
+				},
+				"2": {
+					{"build:foo1", "newstream:foo1", "done:foo1"},
+					{"build:foo1", "newstream:foo1", "done:foo1"},
+					{"build:foo1", "newstream:foo1", "done:foo1"},
+				},
+			},
+		},
+		{
+			name: "NewStream error; ensure earlier interceptor Done is still called",
+			ldsFilters: []xdsclient.HTTPFilter{
+				{Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1"}},
+				{Name: "bar", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "bar1", newStreamErr: errors.New("bar newstream err")}},
+				routerFilter,
+			},
+			rpcRes: map[string][][]string{
+				"1": {
+					{"build:foo1", "build:bar1", "newstream:foo1", "newstream:bar1" /* <err in bar1 NewStream> */, "done:foo1"},
+				},
+				"2": {
+					{"build:foo1", "build:bar1", "newstream:foo1", "newstream:bar1" /* <err in bar1 NewSteam> */, "done:foo1"},
+				},
+			},
+			newStreamErr: "bar newstream err",
+		},
+		{
+			name: "all overrides",
+			ldsFilters: []xdsclient.HTTPFilter{
+				{Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1", newStreamErr: errors.New("this is overridden to nil")}},
+				{Name: "bar", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "bar1"}},
+				routerFilter,
+			},
+			vhOverrides: map[string]httpfilter.FilterConfig{"foo": filterCfg{s: "foo2"}, "bar": filterCfg{s: "bar2"}},
+			rtOverrides: map[string]httpfilter.FilterConfig{"foo": filterCfg{s: "foo3"}, "bar": filterCfg{s: "bar3"}},
+			clOverrides: map[string]httpfilter.FilterConfig{"foo": filterCfg{s: "foo4"}, "bar": filterCfg{s: "bar4"}},
+			rpcRes: map[string][][]string{
+				"1": {
+					{"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+					{"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+				},
+				"2": {
+					{"build:foo1", "override:foo3", "build:bar1", "override:bar3", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+					{"build:foo1", "override:foo4", "build:bar1", "override:bar4", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+					{"build:foo1", "override:foo3", "build:bar1", "override:bar3", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+					{"build:foo1", "override:foo4", "build:bar1", "override:bar4", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			xdsC := fakeclient.NewClient()
+			xdsR, tcc, cancel := testSetup(t, setupOpts{
+				xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+			})
+			defer func() {
+				cancel()
+				xdsR.Close()
+			}()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			waitForWatchListener(ctx, t, xdsC, targetStr)
+
+			xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+				RouteConfigName: routeStr,
+				HTTPFilters:     tc.ldsFilters,
+			}, nil)
+			if i == 0 {
+				waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+			}
+
+			defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR)
+			newWRR = xdstestutils.NewTestWRR
+
+			// Invoke the watchAPI callback with a good service update and wait for the
+			// UpdateState method to be called on the ClientConn.
+			xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+				VirtualHosts: []*xdsclient.VirtualHost{
+					{
+						Domains: []string{targetStr},
+						Routes: []*client.Route{{
+							Prefix: newStringP("1"), WeightedClusters: map[string]xdsclient.WeightedCluster{
+								"A": {Weight: 1},
+								"B": {Weight: 1},
+							},
+						}, {
+							Prefix: newStringP("2"), WeightedClusters: map[string]xdsclient.WeightedCluster{
+								"A": {Weight: 1},
+								"B": {Weight: 1, HTTPFilterConfigOverride: tc.clOverrides},
+							},
+							HTTPFilterConfigOverride: tc.rtOverrides,
+						}},
+						HTTPFilterConfigOverride: tc.vhOverrides,
+					},
+				},
+			}, nil)
+
+			gotState, err := tcc.stateCh.Receive(ctx)
+			if err != nil {
+				t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+			}
+			rState := gotState.(resolver.State)
+			if err := rState.ServiceConfig.Err; err != nil {
+				t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err)
+			}
+
+			cs := iresolver.GetConfigSelector(rState)
+			if cs == nil {
+				t.Fatal("received nil config selector")
+			}
+
+			for method, wants := range tc.rpcRes {
+				// Order of wants is non-deterministic.
+				remainingWant := make([][]string, len(wants))
+				copy(remainingWant, wants)
+				for n := range wants {
+					path = nil
+
+					res, err := cs.SelectConfig(iresolver.RPCInfo{Method: method, Context: context.Background()})
+					if tc.selectErr != "" {
+						if err == nil || !strings.Contains(err.Error(), tc.selectErr) {
+							t.Errorf("SelectConfig(_) = _, %v; want _, Contains(%v)", err, tc.selectErr)
+						}
+						if err == nil {
+							res.OnCommitted()
+						}
+						continue
+					}
+					if err != nil {
+						t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
+					}
+					_, cs, err := res.Interceptor.NewStream(context.Background(), iresolver.RPCInfo{}, iresolver.NOPClientStream{})
+					if tc.newStreamErr != "" {
+						if err == nil || !strings.Contains(err.Error(), tc.newStreamErr) {
+							t.Errorf("NewStream(...) = _, %v; want _, Contains(%v)", err, tc.newStreamErr)
+						}
+						if err == nil {
+							res.OnCommitted()
+							cs.Done()
+						}
+						continue
+					}
+					if err != nil {
+						t.Fatalf("unexpected error from Interceptor.NewStream: %v", err)
+
+					}
+					res.OnCommitted()
+					cs.Done()
+
+					// Confirm the desired path is found in remainingWant, and remove it.
+					pass := false
+					for i := range remainingWant {
+						if reflect.DeepEqual(path, remainingWant[i]) {
+							remainingWant[i] = remainingWant[len(remainingWant)-1]
+							remainingWant = remainingWant[:len(remainingWant)-1]
+							pass = true
+							break
+						}
+					}
+					if !pass {
+						t.Errorf("%q:%v - path:\n%v\nwant one of:\n%v", method, n, path, remainingWant)
+					}
+				}
+			}
+		})
+	}
+}
+
 func replaceRandNumGenerator(start int64) func() {
 	nextInt := start
 	grpcrandInt63n = func(int64) (ret int64) {

From 9dfe677337d567bdea82c6c11603127a7d83cdac Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 25 Feb 2021 15:48:39 -0800
Subject: [PATCH 370/481] balancer: Add UpdateAddresses() to
 balancer.ClientConn interface (#4215)

---
 balancer/balancer.go                      | 10 ++++++++++
 balancer/base/balancer.go                 |  2 +-
 balancer/grpclb/grpclb_remote_balancer.go |  2 +-
 balancer_conn_wrappers.go                 |  8 ++++++++
 pickfirst.go                              |  2 +-
 xds/internal/testutils/balancer.go        |  3 +++
 6 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/balancer/balancer.go b/balancer/balancer.go
index 788759bde4b5..ab531f4c0b80 100644
--- a/balancer/balancer.go
+++ b/balancer/balancer.go
@@ -101,6 +101,9 @@ type SubConn interface {
 	// a new connection will be created.
 	//
 	// This will trigger a state transition for the SubConn.
+	//
+	// Deprecated: This method is now part of the ClientConn interface and will
+	// eventually be removed from here.
 	UpdateAddresses([]resolver.Address)
 	// Connect starts the connecting for this SubConn.
 	Connect()
@@ -143,6 +146,13 @@ type ClientConn interface {
 	// RemoveSubConn removes the SubConn from ClientConn.
 	// The SubConn will be shutdown.
 	RemoveSubConn(SubConn)
+	// UpdateAddresses updates the addresses used in the passed in SubConn.
+	// gRPC checks if the currently connected address is still in the new list.
+	// If so, the connection will be kept. Else, the connection will be
+	// gracefully closed, and a new connection will be created.
+	//
+	// This will trigger a state transition for the SubConn.
+	UpdateAddresses(SubConn, []resolver.Address)
 
 	// UpdateState notifies gRPC that the balancer's internal state has
 	// changed.
diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go
index e0d34288ccfc..383d02ec2bf5 100644
--- a/balancer/base/balancer.go
+++ b/balancer/base/balancer.go
@@ -135,7 +135,7 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 			// The SubConn does a reflect.DeepEqual of the new and old
 			// addresses. So this is a noop if the current address is the same
 			// as the old one (including attributes).
-			sc.UpdateAddresses([]resolver.Address{a})
+			b.cc.UpdateAddresses(sc, []resolver.Address{a})
 		}
 	}
 	for a, sc := range b.subConns {
diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go
index 08d326ab40b0..5ac8d86bd570 100644
--- a/balancer/grpclb/grpclb_remote_balancer.go
+++ b/balancer/grpclb/grpclb_remote_balancer.go
@@ -140,7 +140,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback
 			break
 		}
 		if sc != nil {
-			sc.UpdateAddresses(backendAddrs)
+			lb.cc.cc.UpdateAddresses(sc, backendAddrs)
 			sc.Connect()
 			return
 		}
diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go
index 11e592aabb01..41061d6d3dc5 100644
--- a/balancer_conn_wrappers.go
+++ b/balancer_conn_wrappers.go
@@ -163,6 +163,14 @@ func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) {
 	ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain)
 }
 
+func (ccb *ccBalancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {
+	acbw, ok := sc.(*acBalancerWrapper)
+	if !ok {
+		return
+	}
+	acbw.UpdateAddresses(addrs)
+}
+
 func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) {
 	ccb.mu.Lock()
 	defer ccb.mu.Unlock()
diff --git a/pickfirst.go b/pickfirst.go
index 56e33f6c76b7..b858c2a5e63b 100644
--- a/pickfirst.go
+++ b/pickfirst.go
@@ -84,7 +84,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) e
 		b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: &picker{result: balancer.PickResult{SubConn: b.sc}}})
 		b.sc.Connect()
 	} else {
-		b.sc.UpdateAddresses(cs.ResolverState.Addresses)
+		b.cc.UpdateAddresses(b.sc, cs.ResolverState.Addresses)
 		b.sc.Connect()
 	}
 	return nil
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 673c4ee9a9b6..4973cbc96e5a 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -123,6 +123,9 @@ func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
 	}
 }
 
+// UpdateAddresses updates the addresses on the SubConn.
+func (tcc *TestClientConn) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {}
+
 // UpdateState updates connectivity state and picker.
 func (tcc *TestClientConn) UpdateState(bs balancer.State) {
 	tcc.logger.Logf("testClientConn: UpdateState(%v)", bs)

From c949703b4b98fb798b026b4f28b1ff42b3460e8d Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 26 Feb 2021 08:45:26 -0800
Subject: [PATCH 371/481] xds/cdsbalancer: Override UpdateAddresses() (#4227)

---
 .../balancer/cdsbalancer/cdsbalancer.go       | 18 ++++--
 .../cdsbalancer/cdsbalancer_security_test.go  | 61 +++++++++++++------
 xds/internal/testutils/balancer.go            | 22 ++++---
 3 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 04286d715901..e4d349753e13 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -496,10 +496,12 @@ func (b *cdsBalancer) Close() {
 	b.xdsClient.Close()
 }
 
-// ccWrapper wraps the balancer.ClientConn that was passed in to the CDS
-// balancer during creation and intercepts the NewSubConn() call from the child
-// policy. Other methods of the balancer.ClientConn interface are not overridden
-// and hence get the original implementation.
+// ccWrapper wraps the balancer.ClientConn passed to the CDS balancer at
+// creation and intercepts the NewSubConn() and UpdateAddresses() call from the
+// child policy to add security configuration required by xDS credentials.
+//
+// Other methods of the balancer.ClientConn interface are not overridden and
+// hence get the original implementation.
 type ccWrapper struct {
 	balancer.ClientConn
 
@@ -518,3 +520,11 @@ func (ccw *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubC
 	}
 	return ccw.ClientConn.NewSubConn(newAddrs, opts)
 }
+
+func (ccw *ccWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetHandshakeInfo(addr, ccw.xdsHI)
+	}
+	ccw.ClientConn.UpdateAddresses(sc, newAddrs)
+}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index e862f99cd4e1..fee48c262ebe 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -171,34 +171,35 @@ func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDS
 // makeNewSubConn invokes the NewSubConn() call on the balancer.ClientConn
 // passed to the EDS balancer, and verifies that the CDS balancer forwards the
 // call appropriately to its parent balancer.ClientConn with or without
-// attributes bases on the value of wantAttributes.
-func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xdstestutils.TestClientConn, wantFallback bool) error {
+// attributes bases on the value of wantFallback.
+func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xdstestutils.TestClientConn, wantFallback bool) (balancer.SubConn, error) {
 	dummyAddr := "foo-address"
 	addrs := []resolver.Address{{Addr: dummyAddr}}
-	if _, err := edsCC.NewSubConn(addrs, balancer.NewSubConnOptions{}); err != nil {
-		return fmt.Errorf("NewSubConn(%+v) on parent ClientConn failed: %v", addrs, err)
+	sc, err := edsCC.NewSubConn(addrs, balancer.NewSubConnOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("NewSubConn(%+v) on parent ClientConn failed: %v", addrs, err)
 	}
 
 	select {
 	case <-ctx.Done():
-		return errors.New("timeout when waiting for new SubConn")
+		return nil, errors.New("timeout when waiting for new SubConn")
 	case gotAddrs := <-parentCC.NewSubConnAddrsCh:
 		if len(gotAddrs) != 1 {
-			return fmt.Errorf("NewSubConn expected 1 address, got %d", len(gotAddrs))
+			return nil, fmt.Errorf("NewSubConn expected 1 address, got %d", len(gotAddrs))
 		}
 		if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
-			return fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want)
+			return nil, fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want)
 		}
 		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdsinternal.HandshakeInfo)
 		hi := getHI(gotAddrs[0].Attributes)
 		if hi == nil {
-			return errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes")
+			return nil, errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes")
 		}
 		if gotFallback := hi.UseFallbackCreds(); gotFallback != wantFallback {
-			return fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback)
+			return nil, fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback)
 		}
 	}
-	return nil
+	return sc, nil
 }
 
 // TestSecurityConfigWithoutXDSCreds tests the case where xdsCredentials are not
@@ -243,7 +244,7 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 	// Make a NewSubConn and verify that the HandshakeInfo does not contain any
 	// certificate providers, forcing the credentials implementation to use
 	// fallback creds.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
 		t.Fatal(err)
 	}
 
@@ -299,7 +300,7 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// Make a NewSubConn and verify that the HandshakeInfo does not contain any
 	// certificate providers, forcing the credentials implementation to use
 	// fallback creds.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
 		t.Fatal(err)
 	}
 
@@ -451,7 +452,7 @@ func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that attributes are added.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -487,9 +488,31 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that attributes are added.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+	sc, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false)
+	if err != nil {
 		t.Fatal(err)
 	}
+
+	// Invoke UpdateAddresses and verify that attributes are added.
+	dummyAddr := "bar-address"
+	addrs := []resolver.Address{{Addr: dummyAddr}}
+	edsB.parentCC.UpdateAddresses(sc, addrs)
+	select {
+	case <-ctx.Done():
+		t.Fatal("timeout when waiting for addresses to be updated on the subConn")
+	case gotAddrs := <-tcc.UpdateAddressesAddrsCh:
+		if len(gotAddrs) != 1 {
+			t.Fatalf("UpdateAddresses expected 1 address, got %d", len(gotAddrs))
+		}
+		if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
+			t.Fatalf("resolver.Address passed to parent ClientConn through UpdateAddresses() has address %q, want %q", got, want)
+		}
+		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdsinternal.HandshakeInfo)
+		hi := getHI(gotAddrs[0].Attributes)
+		if hi == nil {
+			t.Fatal("resolver.Address passed to parent ClientConn through UpdateAddresses() doesn't contain attributes")
+		}
+	}
 }
 
 func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
@@ -518,7 +541,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that attributes are added.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -532,7 +555,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that fallback creds are used.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -568,7 +591,7 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that attributes are added.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -650,7 +673,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that attributes are added.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -671,7 +694,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	}
 
 	// Make a NewSubConn and verify that attributes are added.
-	if err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
+	if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index 4973cbc96e5a..dab84a84e072 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -71,9 +71,10 @@ func (tsc *TestSubConn) String() string {
 type TestClientConn struct {
 	logger testingLogger
 
-	NewSubConnAddrsCh chan []resolver.Address // the last 10 []Address to create subconn.
-	NewSubConnCh      chan balancer.SubConn   // the last 10 subconn created.
-	RemoveSubConnCh   chan balancer.SubConn   // the last 10 subconn removed.
+	NewSubConnAddrsCh      chan []resolver.Address // the last 10 []Address to create subconn.
+	NewSubConnCh           chan balancer.SubConn   // the last 10 subconn created.
+	RemoveSubConnCh        chan balancer.SubConn   // the last 10 subconn removed.
+	UpdateAddressesAddrsCh chan []resolver.Address // last updated address via UpdateAddresses().
 
 	NewPickerCh chan balancer.Picker    // the last picker updated.
 	NewStateCh  chan connectivity.State // the last state.
@@ -86,9 +87,10 @@ func NewTestClientConn(t *testing.T) *TestClientConn {
 	return &TestClientConn{
 		logger: t,
 
-		NewSubConnAddrsCh: make(chan []resolver.Address, 10),
-		NewSubConnCh:      make(chan balancer.SubConn, 10),
-		RemoveSubConnCh:   make(chan balancer.SubConn, 10),
+		NewSubConnAddrsCh:      make(chan []resolver.Address, 10),
+		NewSubConnCh:           make(chan balancer.SubConn, 10),
+		RemoveSubConnCh:        make(chan balancer.SubConn, 10),
+		UpdateAddressesAddrsCh: make(chan []resolver.Address, 1),
 
 		NewPickerCh: make(chan balancer.Picker, 1),
 		NewStateCh:  make(chan connectivity.State, 1),
@@ -124,7 +126,13 @@ func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) {
 }
 
 // UpdateAddresses updates the addresses on the SubConn.
-func (tcc *TestClientConn) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {}
+func (tcc *TestClientConn) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {
+	tcc.logger.Logf("testClientConn: UpdateAddresses(%v, %+v)", sc, addrs)
+	select {
+	case tcc.UpdateAddressesAddrsCh <- addrs:
+	default:
+	}
+}
 
 // UpdateState updates connectivity state and picker.
 func (tcc *TestClientConn) UpdateState(bs balancer.State) {

From fc8f38cccf75306a1ae0e7998930e027a0033fdb Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Mon, 1 Mar 2021 13:02:55 -0800
Subject: [PATCH 372/481] xds: add support for more resources to e2e server
 (#4230)

---
 .../test/xds_client_integration_test.go       | 112 +++++++++++++
 .../test/xds_server_integration_test.go       |   1 -
 xds/internal/testutils/e2e/clientresources.go | 148 ++++++++++++++++++
 xds/internal/testutils/e2e/server.go          |  28 +++-
 4 files changed, 281 insertions(+), 8 deletions(-)
 create mode 100644 xds/internal/test/xds_client_integration_test.go
 create mode 100644 xds/internal/testutils/e2e/clientresources.go

diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go
new file mode 100644
index 000000000000..c3ea71acaa35
--- /dev/null
+++ b/xds/internal/test/xds_client_integration_test.go
@@ -0,0 +1,112 @@
+// +build !386
+
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
+
+	testpb "google.golang.org/grpc/test/grpc_testing"
+)
+
+// clientSetup performs a bunch of steps common to all xDS client tests here:
+// - spin up an xDS management server on a local port
+// - spin up a gRPC server and register the test service on it
+// - create a local TCP listener and start serving on it
+//
+// Returns the following:
+// - the management server: tests use this to configure resources
+// - nodeID expected by the management server: this is set in the Node proto
+//   sent by the xdsClient for queries.
+// - the port the server is listening on
+// - cleanup function to be invoked by the tests when done
+func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
+	// Spin up a xDS management server on a local port.
+	nodeID := uuid.New().String()
+	fs, err := e2e.StartManagementServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a bootstrap file in a temporary directory.
+	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
+		Version:              e2e.TransportV3,
+		NodeID:               nodeID,
+		ServerURI:            fs.Address,
+		ServerResourceNameID: "grpc/server",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Initialize a gRPC server and register the stubServer on it.
+	server := grpc.NewServer()
+	testpb.RegisterTestServiceServer(server, &testService{})
+
+	// Create a local listener and pass it to Serve().
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	go func() {
+		if err := server.Serve(lis); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+
+	return fs, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), func() {
+		fs.Stop()
+		bootstrapCleanup()
+		server.Stop()
+	}
+}
+
+func (s) TestClientSideXDS(t *testing.T) {
+	fs, nodeID, port, cleanup := clientSetup(t)
+	defer cleanup()
+
+	resources := e2e.DefaultClientResources("myservice", nodeID, "localhost", port)
+	if err := fs.Update(resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.Dial("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index e55e048e6a36..169daf19d263 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -119,7 +119,6 @@ func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
 }
 
 // commonSetup performs a bunch of steps common to all xDS server tests here:
-// - turn on V3 support by setting the environment variable
 // - spin up an xDS management server on a local port
 // - set up certificates for consumption by the file_watcher plugin
 // - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and
diff --git a/xds/internal/testutils/e2e/clientresources.go b/xds/internal/testutils/e2e/clientresources.go
new file mode 100644
index 000000000000..79424b13b918
--- /dev/null
+++ b/xds/internal/testutils/e2e/clientresources.go
@@ -0,0 +1,148 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e
+
+import (
+	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
+)
+
+func any(m proto.Message) *anypb.Any {
+	a, err := ptypes.MarshalAny(m)
+	if err != nil {
+		panic("error marshalling any: " + err.Error())
+	}
+	return a
+}
+
+// DefaultClientResources returns a set of resources (LDS, RDS, CDS, EDS) for a
+// client to generically connect to one server.
+func DefaultClientResources(target, nodeID, host string, port uint32) UpdateOptions {
+	const routeConfigName = "route"
+	const clusterName = "cluster"
+	const endpointsName = "endpoints"
+
+	return UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{DefaultListener(target, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{DefaultRouteConfig(routeConfigName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{DefaultCluster(clusterName, endpointsName)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{DefaultEndpoint(endpointsName, host, port)},
+	}
+}
+
+// DefaultListener returns a basic xds Listener resource.
+func DefaultListener(target, routeName string) *v3listenerpb.Listener {
+	hcm := any(&v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{
+			ConfigSource: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+			},
+			RouteConfigName: routeName,
+		}},
+		HttpFilters: []*v3httppb.HttpFilter{HTTPFilter("router", &v3routerpb.Router{})}, // router fields are unused by grpc
+	})
+	return &v3listenerpb.Listener{
+		Name:        target,
+		ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
+		FilterChains: []*v3listenerpb.FilterChain{{
+			Name: "filter-chain-name",
+			Filters: []*v3listenerpb.Filter{{
+				Name:       wellknown.HTTPConnectionManager,
+				ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm},
+			}},
+		}},
+	}
+}
+
+// HTTPFilter constructs an xds HttpFilter with the provided name and config.
+func HTTPFilter(name string, config proto.Message) *v3httppb.HttpFilter {
+	return &v3httppb.HttpFilter{
+		Name: name,
+		ConfigType: &v3httppb.HttpFilter_TypedConfig{
+			TypedConfig: any(config),
+		},
+	}
+}
+
+// DefaultRouteConfig returns a basic xds RouteConfig resource.
+func DefaultRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration {
+	return &v3routepb.RouteConfiguration{
+		Name: routeName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{ldsTarget},
+			Routes: []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+				Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+					ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+				}},
+			}},
+		}},
+	}
+}
+
+// DefaultCluster returns a basic xds Cluster resource.
+func DefaultCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster {
+	return &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: edsServiceName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+	}
+}
+
+// DefaultEndpoint returns a basic xds Endpoint resource.
+func DefaultEndpoint(clusterName string, host string, port uint32) *v3endpointpb.ClusterLoadAssignment {
+	return &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: clusterName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{{
+			Locality: &v3corepb.Locality{SubZone: "subzone"},
+			LbEndpoints: []*v3endpointpb.LbEndpoint{{
+				HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{Endpoint: &v3endpointpb.Endpoint{
+					Address: &v3corepb.Address{Address: &v3corepb.Address_SocketAddress{
+						SocketAddress: &v3corepb.SocketAddress{
+							Protocol:      v3corepb.SocketAddress_TCP,
+							Address:       host,
+							PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(port)}},
+					}},
+				}},
+			}},
+			LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+			Priority:            0,
+		}},
+	}
+}
diff --git a/xds/internal/testutils/e2e/server.go b/xds/internal/testutils/e2e/server.go
index e3020614702b..cc55c595cae7 100644
--- a/xds/internal/testutils/e2e/server.go
+++ b/xds/internal/testutils/e2e/server.go
@@ -23,9 +23,13 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"reflect"
 	"strconv"
 
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	"github.com/envoyproxy/go-control-plane/pkg/cache/types"
 	v3cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
@@ -103,9 +107,12 @@ func StartManagementServer() (*ManagementServer, error) {
 type UpdateOptions struct {
 	// NodeID is the id of the client to which this update is to be pushed.
 	NodeID string
-	// Listeners is the updated list of listener resources.
+	// Endpoints, Clusters, Routes, and Listeners are the updated list of xds
+	// resources for the server.  All must be provided with each Update.
+	Endpoints []*v3endpointpb.ClusterLoadAssignment
+	Clusters  []*v3clusterpb.Cluster
+	Routes    []*v3routepb.RouteConfiguration
 	Listeners []*v3listenerpb.Listener
-	// TODO(easwars): Add support for other resource types.
 }
 
 // Update changes the resource snapshot held by the management server, which
@@ -114,11 +121,7 @@ func (s *ManagementServer) Update(opts UpdateOptions) error {
 	s.version++
 
 	// Create a snapshot with the passed in resources.
-	var listeners []types.Resource
-	for _, l := range opts.Listeners {
-		listeners = append(listeners, l)
-	}
-	snapshot := v3cache.NewSnapshot(strconv.Itoa(s.version), nil, nil, nil, listeners, nil, nil)
+	snapshot := v3cache.NewSnapshot(strconv.Itoa(s.version), resourceSlice(opts.Endpoints), resourceSlice(opts.Clusters), resourceSlice(opts.Routes), resourceSlice(opts.Listeners), nil /*runtimes*/, nil /*secrets*/)
 	if err := snapshot.Consistent(); err != nil {
 		return fmt.Errorf("failed to create new resource snapshot: %v", err)
 	}
@@ -140,3 +143,14 @@ func (s *ManagementServer) Stop() {
 	s.gs.Stop()
 	logger.Infof("Stopped the xDS management server...")
 }
+
+// resourceSlice accepts a slice of any type of proto messages and returns a
+// slice of types.Resource.  Will panic if there is an input type mismatch.
+func resourceSlice(i interface{}) []types.Resource {
+	v := reflect.ValueOf(i)
+	rs := make([]types.Resource, v.Len())
+	for i := 0; i < v.Len(); i++ {
+		rs[i] = v.Index(i).Interface().(types.Resource)
+	}
+	return rs
+}

From 29bf29e0ed38f06baa61a81788ddcea89ccfca24 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 2 Mar 2021 09:11:35 -0800
Subject: [PATCH 373/481] xds: add HTTP filter is_optional support (gRFC A39)
 (#4221)

---
 examples/go.sum                      |  4 +-
 go.mod                               |  2 +-
 go.sum                               |  4 +-
 security/advancedtls/examples/go.sum |  2 +-
 security/advancedtls/go.sum          |  2 +-
 xds/internal/client/lds_test.go      | 85 ++++++++++++++++++++++++++++
 xds/internal/client/rds_test.go      | 40 +++++++++++++
 xds/internal/client/xds.go           | 47 +++++++++++----
 8 files changed, 168 insertions(+), 18 deletions(-)

diff --git a/examples/go.sum b/examples/go.sum
index 1bc27f276662..c6aa163b01ad 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -5,8 +5,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
diff --git a/go.mod b/go.mod
index cab74e55774c..b177cfa66df5 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.11
 
 require (
 	github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403
-	github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad
+	github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-cmp v0.5.0
diff --git a/go.sum b/go.sum
index 77ee70b44354..bb25cd49156d 100644
--- a/go.sum
+++ b/go.sum
@@ -12,8 +12,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
index 3d64689329fe..519267dbc278 100644
--- a/security/advancedtls/examples/go.sum
+++ b/security/advancedtls/examples/go.sum
@@ -2,7 +2,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 3d64689329fe..519267dbc278 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -2,7 +2,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index bdce958f30ca..32b44da1e4a3 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -41,6 +41,7 @@ import (
 	v2httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
 	v2listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	anypb "github.com/golang/protobuf/ptypes/any"
@@ -92,6 +93,11 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			Name:       "customFilter",
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: wrappedCustomFilterTypedStructConfig},
 		}
+		customOptionalFilter = &v3httppb.HttpFilter{
+			Name:       "customFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig},
+			IsOptional: true,
+		}
 		customFilter2 = &v3httppb.HttpFilter{
 			Name:       "customFilter2",
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig},
@@ -100,6 +106,11 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			Name:       "errFilter",
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig},
 		}
+		errOptionalFilter = &v3httppb.HttpFilter{
+			Name:       "errFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig},
+			IsOptional: true,
+		}
 		clientOnlyCustomFilter = &v3httppb.HttpFilter{
 			Name:       "clientOnlyCustomFilter",
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig},
@@ -108,6 +119,20 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			Name:       "serverOnlyCustomFilter",
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig},
 		}
+		serverOnlyOptionalCustomFilter = &v3httppb.HttpFilter{
+			Name:       "serverOnlyOptionalCustomFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig},
+			IsOptional: true,
+		}
+		unknownFilter = &v3httppb.HttpFilter{
+			Name:       "unknownFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig},
+		}
+		unknownOptionalFilter = &v3httppb.HttpFilter{
+			Name:       "unknownFilter",
+			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig},
+			IsOptional: true,
+		}
 		v3LisWithFilters = func(fs ...*v3httppb.HttpFilter) *anypb.Any {
 			hcm := &v3httppb.HttpConnectionManager{
 				RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
@@ -342,6 +367,20 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:      "v3 with optional custom filter",
+			resources: []*anypb.Any{v3LisWithFilters(customOptionalFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+					HTTPFilters: []HTTPFilter{{
+						Name:   "customFilter",
+						Filter: httpFilter{},
+						Config: filterConfig{Cfg: customFilterConfig},
+					}},
+				},
+			},
+		},
 		{
 			name:      "v3 with custom filter, fault injection disabled",
 			resources: []*anypb.Any{v3LisWithFilters(customFilter)},
@@ -378,6 +417,15 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			resources: []*anypb.Any{v3LisWithFilters(serverOnlyCustomFilter)},
 			wantErr:   true,
 		},
+		{
+			name:      "v3 with optional server-only filter",
+			resources: []*anypb.Any{v3LisWithFilters(serverOnlyOptionalCustomFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+				},
+			},
+		},
 		{
 			name:      "v3 with client-only filter",
 			resources: []*anypb.Any{v3LisWithFilters(clientOnlyCustomFilter)},
@@ -397,6 +445,23 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			resources: []*anypb.Any{v3LisWithFilters(errFilter)},
 			wantErr:   true,
 		},
+		{
+			name:      "v3 with optional err filter",
+			resources: []*anypb.Any{v3LisWithFilters(errOptionalFilter)},
+			wantErr:   true,
+		},
+		{
+			name:      "v3 with unknown filter",
+			resources: []*anypb.Any{v3LisWithFilters(unknownFilter)},
+			wantErr:   true,
+		},
+		{
+			name:      "v3 with unknown filter (optional)",
+			resources: []*anypb.Any{v3LisWithFilters(unknownOptionalFilter)},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+			},
+		},
 		{
 			name:      "v3 with error filter, fault injection disabled",
 			resources: []*anypb.Any{v3LisWithFilters(errFilter)},
@@ -1190,3 +1255,23 @@ func init() {
 		panic(err.Error())
 	}
 }
+
+var unknownFilterConfig = &anypb.Any{
+	TypeUrl: "unknown.custom.filter",
+	Value:   []byte{1, 2, 3},
+}
+
+func wrappedOptionalFilter(name string) *anypb.Any {
+	filter := &v3routepb.FilterConfig{
+		IsOptional: true,
+		Config: &anypb.Any{
+			TypeUrl: name,
+			Value:   []byte{1, 2, 3},
+		},
+	}
+	w, err := ptypes.MarshalAny(filter)
+	if err != nil {
+		panic("error marshalling any: " + err.Error())
+	}
+	return w
+}
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index 1a33c90a7dbc..9bc068fd9d96 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -445,11 +445,31 @@ func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
 			rc:         goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterTypedStructConfig}),
 			wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterTypedStructConfig}}),
 		},
+		{
+			name:       "good-route-config-with-optional-http-filter-config",
+			rc:         goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("custom.filter")}),
+			wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}),
+		},
 		{
 			name:      "good-route-config-with-http-err-filter-config",
 			rc:        goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
 			wantError: true,
 		},
+		{
+			name:      "good-route-config-with-http-optional-err-filter-config",
+			rc:        goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("err.custom.filter")}),
+			wantError: true,
+		},
+		{
+			name:      "good-route-config-with-http-unknown-filter-config",
+			rc:        goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": unknownFilterConfig}),
+			wantError: true,
+		},
+		{
+			name:       "good-route-config-with-http-optional-unknown-filter-config",
+			rc:         goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("unknown.custom.filter")}),
+			wantUpdate: goodUpdateWithFilterConfigs(nil),
+		},
 		{
 			name:       "good-route-config-with-http-err-filter-config-fi-disabled",
 			disableFI:  true,
@@ -897,6 +917,11 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterTypedStructConfig}),
 			wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterTypedStructConfig}}),
 		},
+		{
+			name:       "with optional custom HTTP filter config",
+			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("custom.filter")}),
+			wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}),
+		},
 		{
 			name:       "with custom HTTP filter config, FI disabled",
 			disableFI:  true,
@@ -908,12 +933,27 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			routes:  goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
 			wantErr: true,
 		},
+		{
+			name:    "with optional erroring custom HTTP filter config",
+			routes:  goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("err.custom.filter")}),
+			wantErr: true,
+		},
 		{
 			name:       "with erroring custom HTTP filter config, FI disabled",
 			disableFI:  true,
 			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}),
 			wantRoutes: goodUpdateWithFilterConfigs(nil),
 		},
+		{
+			name:    "with unknown custom HTTP filter config",
+			routes:  goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": unknownFilterConfig}),
+			wantErr: true,
+		},
+		{
+			name:       "with optional unknown custom HTTP filter config",
+			routes:     goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("unknown.custom.filter")}),
+			wantRoutes: goodUpdateWithFilterConfigs(nil),
+		},
 	}
 
 	cmpOpts := []cmp.Option{
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 055a782d468b..cdecbe29694e 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -133,24 +133,27 @@ func processClientSideListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUp
 }
 
 func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) {
-	if typeURL := config.GetTypeUrl(); typeURL != "type.googleapis.com/udpa.type.v1.TypedStruct" {
-		return config, typeURL, nil
-	}
 	// The real type name is inside the TypedStruct.
 	s := new(v1typepb.TypedStruct)
+	if !ptypes.Is(config, s) {
+		return config, config.GetTypeUrl(), nil
+	}
 	if err := ptypes.UnmarshalAny(config, s); err != nil {
 		return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err)
 	}
 	return s, s.GetTypeUrl(), nil
 }
 
-func validateHTTPFilterConfig(cfg *anypb.Any, lds bool) (httpfilter.Filter, httpfilter.FilterConfig, error) {
+func validateHTTPFilterConfig(cfg *anypb.Any, lds, optional bool) (httpfilter.Filter, httpfilter.FilterConfig, error) {
 	config, typeURL, err := unwrapHTTPFilterConfig(cfg)
 	if err != nil {
 		return nil, nil, err
 	}
 	filterBuilder := httpfilter.Get(typeURL)
 	if filterBuilder == nil {
+		if optional {
+			return nil, nil, nil
+		}
 		return nil, nil, fmt.Errorf("no filter implementation found for %q", typeURL)
 	}
 	parseFunc := filterBuilder.ParseFilterConfig
@@ -170,9 +173,23 @@ func processHTTPFilterOverrides(cfgs map[string]*anypb.Any) (map[string]httpfilt
 	}
 	m := make(map[string]httpfilter.FilterConfig)
 	for name, cfg := range cfgs {
-		_, config, err := validateHTTPFilterConfig(cfg, false)
+		optional := false
+		s := new(v3routepb.FilterConfig)
+		if ptypes.Is(cfg, s) {
+			if err := ptypes.UnmarshalAny(cfg, s); err != nil {
+				return nil, fmt.Errorf("filter override %q: error unmarshalling FilterConfig: %v", name, err)
+			}
+			cfg = s.GetConfig()
+			optional = s.GetIsOptional()
+		}
+
+		httpFilter, config, err := validateHTTPFilterConfig(cfg, false, optional)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("filter override %q: %v", name, err)
+		}
+		if httpFilter == nil {
+			// Optional configs are ignored.
+			continue
 		}
 		m[name] = config
 	}
@@ -196,18 +213,26 @@ func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool) ([]HTTPFilt
 		}
 		seenNames[name] = true
 
-		httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true)
+		httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true, filter.GetIsOptional())
 		if err != nil {
 			return nil, err
 		}
+		if httpFilter == nil {
+			// Optional configs are ignored.
+			continue
+		}
 		if server {
 			if _, ok := httpFilter.(httpfilter.ServerInterceptorBuilder); !ok {
-				return nil, fmt.Errorf("httpFilter %q not supported server-side", name)
+				if filter.GetIsOptional() {
+					continue
+				}
+				return nil, fmt.Errorf("HTTP filter %q not supported server-side", name)
 			}
-		} else {
-			if _, ok := httpFilter.(httpfilter.ClientInterceptorBuilder); !ok {
-				return nil, fmt.Errorf("httpFilter %q not supported client-side", name)
+		} else if _, ok := httpFilter.(httpfilter.ClientInterceptorBuilder); !ok {
+			if filter.GetIsOptional() {
+				continue
 			}
+			return nil, fmt.Errorf("HTTP filter %q not supported client-side", name)
 		}
 
 		// Save name/config

From c275c3599a6f8b50696b8ebd632d4ff110eb4666 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 2 Mar 2021 16:34:04 -0800
Subject: [PATCH 374/481] xds: Move tests requiring API client to `tests`
 directory (#4238)

---
 xds/internal/client/client_test.go            |  17 +-
 xds/internal/client/tests/README.md           |   1 +
 xds/internal/client/{ => tests}/dump_test.go  | 209 +++++++++++-------
 .../client/{ => tests}/loadreport_test.go     |  16 +-
 4 files changed, 128 insertions(+), 115 deletions(-)
 create mode 100644 xds/internal/client/tests/README.md
 rename xds/internal/client/{ => tests}/dump_test.go (59%)
 rename xds/internal/client/{ => tests}/loadreport_test.go (94%)

diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index f3651d6f5122..688c80adf7a7 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -26,11 +26,10 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"google.golang.org/grpc/internal/grpcsync"
-	"google.golang.org/protobuf/testing/protocmp"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
@@ -59,20 +58,6 @@ const (
 	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
 )
 
-var (
-	cmpOpts = cmp.Options{
-		cmpopts.EquateEmpty(),
-		cmp.Comparer(func(a, b time.Time) bool { return true }),
-		cmp.Comparer(func(x, y error) bool {
-			if x == nil || y == nil {
-				return x == nil && y == nil
-			}
-			return x.Error() == y.Error()
-		}),
-		protocmp.Transform(),
-	}
-)
-
 func clientOpts(balancerName string, overrideWatchExpiryTimeout bool) (*bootstrap.Config, time.Duration) {
 	watchExpiryTimeout := defaultWatchExpiryTimeout
 	if overrideWatchExpiryTimeout {
diff --git a/xds/internal/client/tests/README.md b/xds/internal/client/tests/README.md
new file mode 100644
index 000000000000..6dc940c103f7
--- /dev/null
+++ b/xds/internal/client/tests/README.md
@@ -0,0 +1 @@
+This package contains tests which cannot live in the `client` package because they need to import one of the API client packages (which itself has a dependency on the `client` package).
diff --git a/xds/internal/client/dump_test.go b/xds/internal/client/tests/dump_test.go
similarity index 59%
rename from xds/internal/client/dump_test.go
rename to xds/internal/client/tests/dump_test.go
index d460da0bbd44..58220866eb19 100644
--- a/xds/internal/client/dump_test.go
+++ b/xds/internal/client/tests/dump_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package client
+package tests_test
 
 import (
 	"fmt"
@@ -30,10 +30,20 @@ import (
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	"github.com/golang/protobuf/ptypes"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/durationpb"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 )
 
+const defaultTestWatchExpiryTimeout = 500 * time.Millisecond
+
 func (s) TestLDSConfigDump(t *testing.T) {
 	const testVersion = "test-version-lds"
 	var (
@@ -71,38 +81,42 @@ func (s) TestLDSConfigDump(t *testing.T) {
 		listenerRaws[ldsTargets[i]] = anyT
 	}
 
-	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
+		BalancerName: testXDSServer,
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
+		NodeProto:    xdstestutils.EmptyNodeProtoV2,
+	}, defaultTestWatchExpiryTimeout)
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
 	defer client.Close()
 
 	// Expected unknown.
-	if err := compareDump(client.DumpLDS, "", map[string]UpdateWithMD{}); err != nil {
+	if err := compareDump(client.DumpLDS, "", map[string]xdsclient.UpdateWithMD{}); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	wantRequested := make(map[string]UpdateWithMD)
+	wantRequested := make(map[string]xdsclient.UpdateWithMD)
 	for _, n := range ldsTargets {
-		cancel := client.WatchListener(n, func(update ListenerUpdate, err error) {})
+		cancel := client.WatchListener(n, func(update xdsclient.ListenerUpdate, err error) {})
 		defer cancel()
-		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+		wantRequested[n] = xdsclient.UpdateWithMD{MD: xdsclient.UpdateMetadata{Status: xdsclient.ServiceStatusRequested}}
 	}
 	// Expected requested.
 	if err := compareDump(client.DumpLDS, "", wantRequested); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	update0 := make(map[string]ListenerUpdate)
-	want0 := make(map[string]UpdateWithMD)
+	update0 := make(map[string]xdsclient.ListenerUpdate)
+	want0 := make(map[string]xdsclient.UpdateWithMD)
 	for n, r := range listenerRaws {
-		update0[n] = ListenerUpdate{Raw: r}
-		want0[n] = UpdateWithMD{
-			MD:  UpdateMetadata{Version: testVersion},
+		update0[n] = xdsclient.ListenerUpdate{Raw: r}
+		want0[n] = xdsclient.UpdateWithMD{
+			MD:  xdsclient.UpdateMetadata{Version: testVersion},
 			Raw: r,
 		}
 	}
-	client.NewListeners(update0, UpdateMetadata{Version: testVersion})
+	client.NewListeners(update0, xdsclient.UpdateMetadata{Version: testVersion})
 
 	// Expect ACK.
 	if err := compareDump(client.DumpLDS, testVersion, want0); err != nil {
@@ -112,11 +126,11 @@ func (s) TestLDSConfigDump(t *testing.T) {
 	const nackVersion = "lds-version-nack"
 	var nackErr = fmt.Errorf("lds nack error")
 	client.NewListeners(
-		map[string]ListenerUpdate{
+		map[string]xdsclient.ListenerUpdate{
 			ldsTargets[0]: {},
 		},
-		UpdateMetadata{
-			ErrState: &UpdateErrorMetadata{
+		xdsclient.UpdateMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
@@ -124,13 +138,13 @@ func (s) TestLDSConfigDump(t *testing.T) {
 	)
 
 	// Expect NACK for [0], but old ACK for [1].
-	wantDump := make(map[string]UpdateWithMD)
+	wantDump := make(map[string]xdsclient.UpdateWithMD)
 	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
 	// message, as well as the NACK error.
-	wantDump[ldsTargets[0]] = UpdateWithMD{
-		MD: UpdateMetadata{
+	wantDump[ldsTargets[0]] = xdsclient.UpdateWithMD{
+		MD: xdsclient.UpdateMetadata{
 			Version: testVersion,
-			ErrState: &UpdateErrorMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
@@ -138,8 +152,8 @@ func (s) TestLDSConfigDump(t *testing.T) {
 		Raw: listenerRaws[ldsTargets[0]],
 	}
 
-	wantDump[ldsTargets[1]] = UpdateWithMD{
-		MD:  UpdateMetadata{Version: testVersion},
+	wantDump[ldsTargets[1]] = xdsclient.UpdateWithMD{
+		MD:  xdsclient.UpdateMetadata{Version: testVersion},
 		Raw: listenerRaws[ldsTargets[1]],
 	}
 	if err := compareDump(client.DumpLDS, nackVersion, wantDump); err != nil {
@@ -181,38 +195,42 @@ func (s) TestRDSConfigDump(t *testing.T) {
 		routeRaws[rdsTargets[i]] = anyT
 	}
 
-	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
+		BalancerName: testXDSServer,
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
+		NodeProto:    xdstestutils.EmptyNodeProtoV2,
+	}, defaultTestWatchExpiryTimeout)
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
 	defer client.Close()
 
 	// Expected unknown.
-	if err := compareDump(client.DumpRDS, "", map[string]UpdateWithMD{}); err != nil {
+	if err := compareDump(client.DumpRDS, "", map[string]xdsclient.UpdateWithMD{}); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	wantRequested := make(map[string]UpdateWithMD)
+	wantRequested := make(map[string]xdsclient.UpdateWithMD)
 	for _, n := range rdsTargets {
-		cancel := client.WatchRouteConfig(n, func(update RouteConfigUpdate, err error) {})
+		cancel := client.WatchRouteConfig(n, func(update xdsclient.RouteConfigUpdate, err error) {})
 		defer cancel()
-		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+		wantRequested[n] = xdsclient.UpdateWithMD{MD: xdsclient.UpdateMetadata{Status: xdsclient.ServiceStatusRequested}}
 	}
 	// Expected requested.
 	if err := compareDump(client.DumpRDS, "", wantRequested); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	update0 := make(map[string]RouteConfigUpdate)
-	want0 := make(map[string]UpdateWithMD)
+	update0 := make(map[string]xdsclient.RouteConfigUpdate)
+	want0 := make(map[string]xdsclient.UpdateWithMD)
 	for n, r := range routeRaws {
-		update0[n] = RouteConfigUpdate{Raw: r}
-		want0[n] = UpdateWithMD{
-			MD:  UpdateMetadata{Version: testVersion},
+		update0[n] = xdsclient.RouteConfigUpdate{Raw: r}
+		want0[n] = xdsclient.UpdateWithMD{
+			MD:  xdsclient.UpdateMetadata{Version: testVersion},
 			Raw: r,
 		}
 	}
-	client.NewRouteConfigs(update0, UpdateMetadata{Version: testVersion})
+	client.NewRouteConfigs(update0, xdsclient.UpdateMetadata{Version: testVersion})
 
 	// Expect ACK.
 	if err := compareDump(client.DumpRDS, testVersion, want0); err != nil {
@@ -222,11 +240,11 @@ func (s) TestRDSConfigDump(t *testing.T) {
 	const nackVersion = "rds-version-nack"
 	var nackErr = fmt.Errorf("rds nack error")
 	client.NewRouteConfigs(
-		map[string]RouteConfigUpdate{
+		map[string]xdsclient.RouteConfigUpdate{
 			rdsTargets[0]: {},
 		},
-		UpdateMetadata{
-			ErrState: &UpdateErrorMetadata{
+		xdsclient.UpdateMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
@@ -234,21 +252,21 @@ func (s) TestRDSConfigDump(t *testing.T) {
 	)
 
 	// Expect NACK for [0], but old ACK for [1].
-	wantDump := make(map[string]UpdateWithMD)
+	wantDump := make(map[string]xdsclient.UpdateWithMD)
 	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
 	// message, as well as the NACK error.
-	wantDump[rdsTargets[0]] = UpdateWithMD{
-		MD: UpdateMetadata{
+	wantDump[rdsTargets[0]] = xdsclient.UpdateWithMD{
+		MD: xdsclient.UpdateMetadata{
 			Version: testVersion,
-			ErrState: &UpdateErrorMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
 		},
 		Raw: routeRaws[rdsTargets[0]],
 	}
-	wantDump[rdsTargets[1]] = UpdateWithMD{
-		MD:  UpdateMetadata{Version: testVersion},
+	wantDump[rdsTargets[1]] = xdsclient.UpdateWithMD{
+		MD:  xdsclient.UpdateMetadata{Version: testVersion},
 		Raw: routeRaws[rdsTargets[1]],
 	}
 	if err := compareDump(client.DumpRDS, nackVersion, wantDump); err != nil {
@@ -291,38 +309,42 @@ func (s) TestCDSConfigDump(t *testing.T) {
 		clusterRaws[cdsTargets[i]] = anyT
 	}
 
-	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
+		BalancerName: testXDSServer,
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
+		NodeProto:    xdstestutils.EmptyNodeProtoV2,
+	}, defaultTestWatchExpiryTimeout)
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
 	defer client.Close()
 
 	// Expected unknown.
-	if err := compareDump(client.DumpCDS, "", map[string]UpdateWithMD{}); err != nil {
+	if err := compareDump(client.DumpCDS, "", map[string]xdsclient.UpdateWithMD{}); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	wantRequested := make(map[string]UpdateWithMD)
+	wantRequested := make(map[string]xdsclient.UpdateWithMD)
 	for _, n := range cdsTargets {
-		cancel := client.WatchCluster(n, func(update ClusterUpdate, err error) {})
+		cancel := client.WatchCluster(n, func(update xdsclient.ClusterUpdate, err error) {})
 		defer cancel()
-		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+		wantRequested[n] = xdsclient.UpdateWithMD{MD: xdsclient.UpdateMetadata{Status: xdsclient.ServiceStatusRequested}}
 	}
 	// Expected requested.
 	if err := compareDump(client.DumpCDS, "", wantRequested); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	update0 := make(map[string]ClusterUpdate)
-	want0 := make(map[string]UpdateWithMD)
+	update0 := make(map[string]xdsclient.ClusterUpdate)
+	want0 := make(map[string]xdsclient.UpdateWithMD)
 	for n, r := range clusterRaws {
-		update0[n] = ClusterUpdate{Raw: r}
-		want0[n] = UpdateWithMD{
-			MD:  UpdateMetadata{Version: testVersion},
+		update0[n] = xdsclient.ClusterUpdate{Raw: r}
+		want0[n] = xdsclient.UpdateWithMD{
+			MD:  xdsclient.UpdateMetadata{Version: testVersion},
 			Raw: r,
 		}
 	}
-	client.NewClusters(update0, UpdateMetadata{Version: testVersion})
+	client.NewClusters(update0, xdsclient.UpdateMetadata{Version: testVersion})
 
 	// Expect ACK.
 	if err := compareDump(client.DumpCDS, testVersion, want0); err != nil {
@@ -332,11 +354,11 @@ func (s) TestCDSConfigDump(t *testing.T) {
 	const nackVersion = "cds-version-nack"
 	var nackErr = fmt.Errorf("cds nack error")
 	client.NewClusters(
-		map[string]ClusterUpdate{
+		map[string]xdsclient.ClusterUpdate{
 			cdsTargets[0]: {},
 		},
-		UpdateMetadata{
-			ErrState: &UpdateErrorMetadata{
+		xdsclient.UpdateMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
@@ -344,21 +366,21 @@ func (s) TestCDSConfigDump(t *testing.T) {
 	)
 
 	// Expect NACK for [0], but old ACK for [1].
-	wantDump := make(map[string]UpdateWithMD)
+	wantDump := make(map[string]xdsclient.UpdateWithMD)
 	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
 	// message, as well as the NACK error.
-	wantDump[cdsTargets[0]] = UpdateWithMD{
-		MD: UpdateMetadata{
+	wantDump[cdsTargets[0]] = xdsclient.UpdateWithMD{
+		MD: xdsclient.UpdateMetadata{
 			Version: testVersion,
-			ErrState: &UpdateErrorMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
 		},
 		Raw: clusterRaws[cdsTargets[0]],
 	}
-	wantDump[cdsTargets[1]] = UpdateWithMD{
-		MD:  UpdateMetadata{Version: testVersion},
+	wantDump[cdsTargets[1]] = xdsclient.UpdateWithMD{
+		MD:  xdsclient.UpdateMetadata{Version: testVersion},
 		Raw: clusterRaws[cdsTargets[1]],
 	}
 	if err := compareDump(client.DumpCDS, nackVersion, wantDump); err != nil {
@@ -376,8 +398,8 @@ func (s) TestEDSConfigDump(t *testing.T) {
 	)
 
 	for i := range edsTargets {
-		clab0 := newClaBuilder(edsTargets[i], nil)
-		clab0.addLocality(localityNames[i], 1, 1, []string{addrs[i]}, nil)
+		clab0 := xdstestutils.NewClusterLoadAssignmentBuilder(edsTargets[i], nil)
+		clab0.AddLocality(localityNames[i], 1, 1, []string{addrs[i]}, nil)
 		claT := clab0.Build()
 
 		anyT, err := ptypes.MarshalAny(claT)
@@ -387,38 +409,42 @@ func (s) TestEDSConfigDump(t *testing.T) {
 		endpointRaws[edsTargets[i]] = anyT
 	}
 
-	client, err := newWithConfig(clientOpts(testXDSServer, false))
+	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
+		BalancerName: testXDSServer,
+		Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
+		NodeProto:    xdstestutils.EmptyNodeProtoV2,
+	}, defaultTestWatchExpiryTimeout)
 	if err != nil {
 		t.Fatalf("failed to create client: %v", err)
 	}
 	defer client.Close()
 
 	// Expected unknown.
-	if err := compareDump(client.DumpEDS, "", map[string]UpdateWithMD{}); err != nil {
+	if err := compareDump(client.DumpEDS, "", map[string]xdsclient.UpdateWithMD{}); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	wantRequested := make(map[string]UpdateWithMD)
+	wantRequested := make(map[string]xdsclient.UpdateWithMD)
 	for _, n := range edsTargets {
-		cancel := client.WatchEndpoints(n, func(update EndpointsUpdate, err error) {})
+		cancel := client.WatchEndpoints(n, func(update xdsclient.EndpointsUpdate, err error) {})
 		defer cancel()
-		wantRequested[n] = UpdateWithMD{MD: UpdateMetadata{Status: ServiceStatusRequested}}
+		wantRequested[n] = xdsclient.UpdateWithMD{MD: xdsclient.UpdateMetadata{Status: xdsclient.ServiceStatusRequested}}
 	}
 	// Expected requested.
 	if err := compareDump(client.DumpEDS, "", wantRequested); err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	update0 := make(map[string]EndpointsUpdate)
-	want0 := make(map[string]UpdateWithMD)
+	update0 := make(map[string]xdsclient.EndpointsUpdate)
+	want0 := make(map[string]xdsclient.UpdateWithMD)
 	for n, r := range endpointRaws {
-		update0[n] = EndpointsUpdate{Raw: r}
-		want0[n] = UpdateWithMD{
-			MD:  UpdateMetadata{Version: testVersion},
+		update0[n] = xdsclient.EndpointsUpdate{Raw: r}
+		want0[n] = xdsclient.UpdateWithMD{
+			MD:  xdsclient.UpdateMetadata{Version: testVersion},
 			Raw: r,
 		}
 	}
-	client.NewEndpoints(update0, UpdateMetadata{Version: testVersion})
+	client.NewEndpoints(update0, xdsclient.UpdateMetadata{Version: testVersion})
 
 	// Expect ACK.
 	if err := compareDump(client.DumpEDS, testVersion, want0); err != nil {
@@ -428,11 +454,11 @@ func (s) TestEDSConfigDump(t *testing.T) {
 	const nackVersion = "eds-version-nack"
 	var nackErr = fmt.Errorf("eds nack error")
 	client.NewEndpoints(
-		map[string]EndpointsUpdate{
+		map[string]xdsclient.EndpointsUpdate{
 			edsTargets[0]: {},
 		},
-		UpdateMetadata{
-			ErrState: &UpdateErrorMetadata{
+		xdsclient.UpdateMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
@@ -440,21 +466,21 @@ func (s) TestEDSConfigDump(t *testing.T) {
 	)
 
 	// Expect NACK for [0], but old ACK for [1].
-	wantDump := make(map[string]UpdateWithMD)
+	wantDump := make(map[string]xdsclient.UpdateWithMD)
 	// Though resource 0 was NACKed, the dump should show the previous ACKed raw
 	// message, as well as the NACK error.
-	wantDump[edsTargets[0]] = UpdateWithMD{
-		MD: UpdateMetadata{
+	wantDump[edsTargets[0]] = xdsclient.UpdateWithMD{
+		MD: xdsclient.UpdateMetadata{
 			Version: testVersion,
-			ErrState: &UpdateErrorMetadata{
+			ErrState: &xdsclient.UpdateErrorMetadata{
 				Version: nackVersion,
 				Err:     nackErr,
 			},
 		},
 		Raw: endpointRaws[edsTargets[0]],
 	}
-	wantDump[edsTargets[1]] = UpdateWithMD{
-		MD:  UpdateMetadata{Version: testVersion},
+	wantDump[edsTargets[1]] = xdsclient.UpdateWithMD{
+		MD:  xdsclient.UpdateMetadata{Version: testVersion},
 		Raw: endpointRaws[edsTargets[1]],
 	}
 	if err := compareDump(client.DumpEDS, nackVersion, wantDump); err != nil {
@@ -462,13 +488,24 @@ func (s) TestEDSConfigDump(t *testing.T) {
 	}
 }
 
-func compareDump(dumpFunc func() (string, map[string]UpdateWithMD), wantVersion string, wantDump interface{}) error {
+func compareDump(dumpFunc func() (string, map[string]xdsclient.UpdateWithMD), wantVersion string, wantDump interface{}) error {
 	v, dump := dumpFunc()
 	if v != wantVersion {
-		return fmt.Errorf("Dump returned version %q, want %q", v, wantVersion)
+		return fmt.Errorf("Dump() returned version %q, want %q", v, wantVersion)
+	}
+	cmpOpts := cmp.Options{
+		cmpopts.EquateEmpty(),
+		cmp.Comparer(func(a, b time.Time) bool { return true }),
+		cmp.Comparer(func(x, y error) bool {
+			if x == nil || y == nil {
+				return x == nil && y == nil
+			}
+			return x.Error() == y.Error()
+		}),
+		protocmp.Transform(),
 	}
 	if diff := cmp.Diff(dump, wantDump, cmpOpts); diff != "" {
-		return fmt.Errorf("Dump returned unexpected dump, diff (-got +want): %s", diff)
+		return fmt.Errorf("Dump() returned unexpected dump, diff (-got +want): %s", diff)
 	}
 	return nil
 }
diff --git a/xds/internal/client/loadreport_test.go b/xds/internal/client/tests/loadreport_test.go
similarity index 94%
rename from xds/internal/client/loadreport_test.go
rename to xds/internal/client/tests/loadreport_test.go
index e4979505f363..af145e7f2a92 100644
--- a/xds/internal/client/loadreport_test.go
+++ b/xds/internal/client/tests/loadreport_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package client_test
+package tests_test
 
 import (
 	"context"
@@ -31,7 +31,6 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
@@ -43,20 +42,11 @@ import (
 )
 
 const (
-	defaultTestTimeout      = 5 * time.Second
-	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
-
+	defaultTestTimeout              = 5 * time.Second
+	defaultTestShortTimeout         = 10 * time.Millisecond // For events expected to *not* happen.
 	defaultClientWatchExpiryTimeout = 15 * time.Second
 )
 
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
 func (s) TestLRSClient(t *testing.T) {
 	fs, sCleanup, err := fakeserver.StartServer()
 	if err != nil {

From 930c79186c9963385a7345aa6cdb4511c8f65c4a Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 3 Mar 2021 18:06:50 -0800
Subject: [PATCH 375/481] csds: update xds_client to populate update metadata
 (#4226)

---
 xds/internal/client/cds_test.go       | 149 ++++--
 xds/internal/client/client_test.go    |  25 +
 xds/internal/client/eds_test.go       | 124 ++++-
 xds/internal/client/lds_test.go       | 695 +++++++++++++++++++-------
 xds/internal/client/rds_test.go       | 112 ++++-
 xds/internal/client/v2/cds_test.go    |  85 ++--
 xds/internal/client/v2/client.go      |  20 +-
 xds/internal/client/v2/client_test.go | 138 ++---
 xds/internal/client/v2/eds_test.go    | 116 +++--
 xds/internal/client/v2/lds_test.go    | 116 +++--
 xds/internal/client/v2/rds_test.go    |  91 +++-
 xds/internal/client/v3/client.go      |  20 +-
 xds/internal/client/xds.go            | 336 +++++++++----
 13 files changed, 1473 insertions(+), 554 deletions(-)

diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 9839660ff0e7..6ad0b0fa88a0 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -571,6 +571,13 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 				},
 			},
 		}
+		v2ClusterAny = &anypb.Any{
+			TypeUrl: version.V2ClusterURL,
+			Value: func() []byte {
+				mcl, _ := proto.Marshal(v2Cluster)
+				return mcl
+			}(),
+		}
 
 		v3Cluster = &v3clusterpb.Cluster{
 			Name:                 v3ClusterName,
@@ -590,18 +597,35 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 				},
 			},
 		}
+		v3ClusterAny = &anypb.Any{
+			TypeUrl: version.V3ClusterURL,
+			Value: func() []byte {
+				mcl, _ := proto.Marshal(v3Cluster)
+				return mcl
+			}(),
+		}
 	)
+	const testVersion = "test-version-cds"
 
 	tests := []struct {
 		name       string
 		resources  []*anypb.Any
 		wantUpdate map[string]ClusterUpdate
+		wantMD     UpdateMetadata
 		wantErr    bool
 	}{
 		{
 			name:      "non-cluster resource type",
 			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
-			wantErr:   true,
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name: "badly marshaled cluster resource",
@@ -611,6 +635,14 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 					Value:   []byte{1, 2, 3, 4},
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -620,6 +652,7 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 					TypeUrl: version.V3ClusterURL,
 					Value: func() []byte {
 						cl := &v3clusterpb.Cluster{
+							Name:                 "test",
 							ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
 						}
 						mcl, _ := proto.Marshal(cl)
@@ -627,67 +660,115 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ClusterUpdate{"test": {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
-			name: "v2 cluster",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ClusterURL,
-					Value: func() []byte {
-						mcl, _ := proto.Marshal(v2Cluster)
-						return mcl
-					}(),
+			name:      "v2 cluster",
+			resources: []*anypb.Any{v2ClusterAny},
+			wantUpdate: map[string]ClusterUpdate{
+				v2ClusterName: {
+					ServiceName: v2Service, EnableLRS: true,
+					Raw: v2ClusterAny,
 				},
 			},
-			wantUpdate: map[string]ClusterUpdate{
-				v2ClusterName: {ServiceName: v2Service, EnableLRS: true},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
-			name: "v3 cluster",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ClusterURL,
-					Value: func() []byte {
-						mcl, _ := proto.Marshal(v3Cluster)
-						return mcl
-					}(),
+			name:      "v3 cluster",
+			resources: []*anypb.Any{v3ClusterAny},
+			wantUpdate: map[string]ClusterUpdate{
+				v3ClusterName: {
+					ServiceName: v3Service, EnableLRS: true,
+					Raw: v3ClusterAny,
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
+		},
+		{
+			name:      "multiple clusters",
+			resources: []*anypb.Any{v2ClusterAny, v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
-				v3ClusterName: {ServiceName: v3Service, EnableLRS: true},
+				v2ClusterName: {
+					ServiceName: v2Service, EnableLRS: true,
+					Raw: v2ClusterAny,
+				},
+				v3ClusterName: {
+					ServiceName: v3Service, EnableLRS: true,
+					Raw: v3ClusterAny,
+				},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
-			name: "multiple clusters",
+			// To test that unmarshal keeps processing on errors.
+			name: "good and bad clusters",
 			resources: []*anypb.Any{
+				v2ClusterAny,
 				{
+					// bad cluster resource
 					TypeUrl: version.V3ClusterURL,
 					Value: func() []byte {
-						mcl, _ := proto.Marshal(v2Cluster)
-						return mcl
-					}(),
-				},
-				{
-					TypeUrl: version.V3ClusterURL,
-					Value: func() []byte {
-						mcl, _ := proto.Marshal(v3Cluster)
+						cl := &v3clusterpb.Cluster{
+							Name:                 "bad",
+							ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
+						}
+						mcl, _ := proto.Marshal(cl)
 						return mcl
 					}(),
 				},
+				v3ClusterAny,
 			},
 			wantUpdate: map[string]ClusterUpdate{
-				v2ClusterName: {ServiceName: v2Service, EnableLRS: true},
-				v3ClusterName: {ServiceName: v3Service, EnableLRS: true},
+				v2ClusterName: {
+					ServiceName: v2Service, EnableLRS: true,
+					Raw: v2ClusterAny,
+				},
+				v3ClusterName: {
+					ServiceName: v3Service, EnableLRS: true,
+					Raw: v3ClusterAny,
+				},
+				"bad": {},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, _, err := UnmarshalCluster("", test.resources, nil)
-			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("UnmarshalCluster(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
+			update, md, err := UnmarshalCluster(testVersion, test.resources, nil)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("UnmarshalCluster(), got err: %v, wantErr: %v", err, test.wantErr)
+			}
+			if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" {
+				t.Errorf("got unexpected update, diff (-got +want): %v", diff)
+			}
+			if diff := cmp.Diff(md, test.wantMD, cmpOptsIgnoreDetails); diff != "" {
+				t.Errorf("got unexpected metadata, diff (-got +want): %v", diff)
 			}
 		})
 	}
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 688c80adf7a7..8275ea60e0dc 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/testing/protocmp"
 )
 
 type s struct {
@@ -58,6 +59,30 @@ const (
 	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
 )
 
+var (
+	cmpOpts = cmp.Options{
+		cmpopts.EquateEmpty(),
+		cmp.Comparer(func(a, b time.Time) bool { return true }),
+		cmp.Comparer(func(x, y error) bool {
+			if x == nil || y == nil {
+				return x == nil && y == nil
+			}
+			return x.Error() == y.Error()
+		}),
+		protocmp.Transform(),
+	}
+
+	// When comparing NACK UpdateMetadata, we only care if error is nil, but not
+	// the details in error.
+	errPlaceHolder       = fmt.Errorf("error whose details don't matter")
+	cmpOptsIgnoreDetails = cmp.Options{
+		cmp.Comparer(func(a, b time.Time) bool { return true }),
+		cmp.Comparer(func(x, y error) bool {
+			return (x == nil) == (y == nil)
+		}),
+	}
+)
+
 func clientOpts(balancerName string, overrideWatchExpiryTimeout bool) (*bootstrap.Config, time.Duration) {
 	watchExpiryTimeout := defaultWatchExpiryTimeout
 	if overrideWatchExpiryTimeout {
diff --git a/xds/internal/client/eds_test.go b/xds/internal/client/eds_test.go
index e51c8938f8c9..daa5d6525e19 100644
--- a/xds/internal/client/eds_test.go
+++ b/xds/internal/client/eds_test.go
@@ -31,7 +31,6 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/version"
 )
@@ -121,16 +120,44 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 }
 
 func (s) TestUnmarshalEndpoints(t *testing.T) {
+	var v3EndpointsAny = &anypb.Any{
+		TypeUrl: version.V3EndpointsURL,
+		Value: func() []byte {
+			clab0 := newClaBuilder("test", nil)
+			clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{
+				Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY},
+				Weight: []uint32{271},
+			})
+			clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{
+				Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING},
+				Weight: []uint32{828},
+			})
+			e := clab0.Build()
+			me, _ := proto.Marshal(e)
+			return me
+		}(),
+	}
+	const testVersion = "test-version-eds"
+
 	tests := []struct {
 		name       string
 		resources  []*anypb.Any
 		wantUpdate map[string]EndpointsUpdate
+		wantMD     UpdateMetadata
 		wantErr    bool
 	}{
 		{
 			name:      "non-clusterLoadAssignment resource type",
 			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
-			wantErr:   true,
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name: "badly marshaled clusterLoadAssignment resource",
@@ -140,6 +167,14 @@ func (s) TestUnmarshalEndpoints(t *testing.T) {
 					Value:   []byte{1, 2, 3, 4},
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -157,23 +192,65 @@ func (s) TestUnmarshalEndpoints(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]EndpointsUpdate{"test": {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
-			name: "v3 endpoints",
+			name:      "v3 endpoints",
+			resources: []*anypb.Any{v3EndpointsAny},
+			wantUpdate: map[string]EndpointsUpdate{
+				"test": {
+					Drops: nil,
+					Localities: []Locality{
+						{
+							Endpoints: []Endpoint{{
+								Address:      "addr1:314",
+								HealthStatus: EndpointHealthStatusUnhealthy,
+								Weight:       271,
+							}},
+							ID:       internal.LocalityID{SubZone: "locality-1"},
+							Priority: 1,
+							Weight:   1,
+						},
+						{
+							Endpoints: []Endpoint{{
+								Address:      "addr2:159",
+								HealthStatus: EndpointHealthStatusDraining,
+								Weight:       828,
+							}},
+							ID:       internal.LocalityID{SubZone: "locality-2"},
+							Priority: 0,
+							Weight:   1,
+						},
+					},
+					Raw: v3EndpointsAny,
+				},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
+		},
+		{
+			// To test that unmarshal keeps processing on errors.
+			name: "good and bad endpoints",
 			resources: []*anypb.Any{
+				v3EndpointsAny,
 				{
+					// bad endpoints resource
 					TypeUrl: version.V3EndpointsURL,
 					Value: func() []byte {
-						clab0 := newClaBuilder("test", nil)
-						clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{
-							Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY},
-							Weight: []uint32{271},
-						})
-						clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{
-							Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING},
-							Weight: []uint32{828},
-						})
+						clab0 := newClaBuilder("bad", nil)
+						clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
+						clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
 						e := clab0.Build()
 						me, _ := proto.Marshal(e)
 						return me
@@ -205,15 +282,32 @@ func (s) TestUnmarshalEndpoints(t *testing.T) {
 							Weight:   1,
 						},
 					},
+					Raw: v3EndpointsAny,
 				},
+				"bad": {},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, _, err := UnmarshalEndpoints("", test.resources, nil)
-			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("UnmarshalEndpoints(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
+			update, md, err := UnmarshalEndpoints(testVersion, test.resources, nil)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("UnmarshalEndpoints(), got err: %v, wantErr: %v", err, test.wantErr)
+			}
+			if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" {
+				t.Errorf("got unexpected update, diff (-got +want): %v", diff)
+			}
+			if diff := cmp.Diff(md, test.wantMD, cmpOptsIgnoreDetails); diff != "" {
+				t.Errorf("got unexpected metadata, diff (-got +want): %v", diff)
 			}
 		})
 	}
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 32b44da1e4a3..26e79b78d133 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -24,28 +24,26 @@ import (
 	"testing"
 	"time"
 
+	v1typepb "github.com/cncf/udpa/go/udpa/type/v1"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	spb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
-	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/durationpb"
 
-	v1typepb "github.com/cncf/udpa/go/udpa/type/v1"
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v2httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
 	v2listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	anypb "github.com/golang/protobuf/ptypes/any"
-	spb "github.com/golang/protobuf/ptypes/struct"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 )
 
@@ -164,18 +162,28 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			}
 		}
 	)
+	const testVersion = "test-version-lds-client"
 
 	tests := []struct {
 		name       string
 		resources  []*anypb.Any
 		wantUpdate map[string]ListenerUpdate
+		wantMD     UpdateMetadata
 		wantErr    bool
 		disableFI  bool // disable fault injection
 	}{
 		{
 			name:      "non-listener resource",
 			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
-			wantErr:   true,
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name: "badly marshaled listener resource",
@@ -197,6 +205,15 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -232,6 +249,15 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -262,6 +288,15 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -290,6 +325,15 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -327,16 +371,33 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
 			name: "empty resource list",
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v3 with no filters",
 			resources: []*anypb.Any{v3LisWithFilters()},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, Raw: v3LisWithFilters()},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
@@ -350,8 +411,13 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 						Filter: httpFilter{},
 						Config: filterConfig{Cfg: customFilterConfig},
 					}},
+					Raw: v3LisWithFilters(customFilter),
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v3 with custom filter in typed struct",
@@ -364,8 +430,13 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 						Filter: httpFilter{},
 						Config: filterConfig{Cfg: customFilterTypedStructConfig},
 					}},
+					Raw: v3LisWithFilters(typedStructFilter),
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v3 with optional custom filter",
@@ -378,21 +449,39 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 						Filter: httpFilter{},
 						Config: filterConfig{Cfg: customFilterConfig},
 					}},
+					Raw: v3LisWithFilters(customOptionalFilter),
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v3 with custom filter, fault injection disabled",
 			resources: []*anypb.Any{v3LisWithFilters(customFilter)},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, Raw: v3LisWithFilters(customFilter)},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 			disableFI: true,
 		},
 		{
-			name:      "v3 with two filters with same name",
-			resources: []*anypb.Any{v3LisWithFilters(customFilter, customFilter)},
-			wantErr:   true,
+			name:       "v3 with two filters with same name",
+			resources:  []*anypb.Any{v3LisWithFilters(customFilter, customFilter)},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name:      "v3 with two filters - same type different name",
@@ -409,22 +498,42 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 						Filter: httpFilter{},
 						Config: filterConfig{Cfg: customFilterConfig},
 					}},
+					Raw: v3LisWithFilters(customFilter, customFilter2),
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
-			name:      "v3 with server-only filter",
-			resources: []*anypb.Any{v3LisWithFilters(serverOnlyCustomFilter)},
-			wantErr:   true,
+			name:       "v3 with server-only filter",
+			resources:  []*anypb.Any{v3LisWithFilters(serverOnlyCustomFilter)},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name:      "v3 with optional server-only filter",
 			resources: []*anypb.Any{v3LisWithFilters(serverOnlyOptionalCustomFilter)},
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
-					RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second,
+					RouteConfigName:   v3RouteConfigName,
+					MaxStreamDuration: time.Second,
+					Raw:               v3LisWithFilters(serverOnlyOptionalCustomFilter),
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v3 with client-only filter",
@@ -437,36 +546,84 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 						Filter: clientOnlyHTTPFilter{},
 						Config: filterConfig{Cfg: clientOnlyCustomFilterConfig},
 					}},
+					Raw: v3LisWithFilters(clientOnlyCustomFilter),
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
-			name:      "v3 with err filter",
-			resources: []*anypb.Any{v3LisWithFilters(errFilter)},
-			wantErr:   true,
+			name:       "v3 with err filter",
+			resources:  []*anypb.Any{v3LisWithFilters(errFilter)},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
-			name:      "v3 with optional err filter",
-			resources: []*anypb.Any{v3LisWithFilters(errOptionalFilter)},
-			wantErr:   true,
+			name:       "v3 with optional err filter",
+			resources:  []*anypb.Any{v3LisWithFilters(errOptionalFilter)},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
-			name:      "v3 with unknown filter",
-			resources: []*anypb.Any{v3LisWithFilters(unknownFilter)},
-			wantErr:   true,
+			name:       "v3 with unknown filter",
+			resources:  []*anypb.Any{v3LisWithFilters(unknownFilter)},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name:      "v3 with unknown filter (optional)",
 			resources: []*anypb.Any{v3LisWithFilters(unknownOptionalFilter)},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+				v3LDSTarget: {
+					RouteConfigName:   v3RouteConfigName,
+					MaxStreamDuration: time.Second,
+					Raw:               v3LisWithFilters(unknownOptionalFilter),
+				},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
 			name:      "v3 with error filter, fault injection disabled",
 			resources: []*anypb.Any{v3LisWithFilters(errFilter)},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+				v3LDSTarget: {
+					RouteConfigName:   v3RouteConfigName,
+					MaxStreamDuration: time.Second,
+					Raw:               v3LisWithFilters(errFilter),
+				},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 			disableFI: true,
 		},
@@ -474,24 +631,77 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			name:      "v2 listener resource",
 			resources: []*anypb.Any{v2Lis},
 			wantUpdate: map[string]ListenerUpdate{
-				v2LDSTarget: {RouteConfigName: v2RouteConfigName},
+				v2LDSTarget: {RouteConfigName: v2RouteConfigName, Raw: v2Lis},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
 			name:      "v3 listener resource",
 			resources: []*anypb.Any{v3LisWithFilters()},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, Raw: v3LisWithFilters()},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
 			name:      "multiple listener resources",
 			resources: []*anypb.Any{v2Lis, v3LisWithFilters()},
 			wantUpdate: map[string]ListenerUpdate{
-				v2LDSTarget: {RouteConfigName: v2RouteConfigName},
-				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second},
+				v2LDSTarget: {RouteConfigName: v2RouteConfigName, Raw: v2Lis},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, Raw: v3LisWithFilters()},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
+		{
+			// To test that unmarshal keeps processing on errors.
+			name: "good and bad listener resources",
+			resources: []*anypb.Any{
+				v2Lis,
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: "bad",
+							ApiListener: &v3listenerpb.ApiListener{
+								ApiListener: &anypb.Any{
+									TypeUrl: version.V2ListenerURL,
+									Value: func() []byte {
+										cm := &v3httppb.HttpConnectionManager{
+											RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+										}
+										mcm, _ := proto.Marshal(cm)
+										return mcm
+									}()}}}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+				v3LisWithFilters(),
+			},
+			wantUpdate: map[string]ListenerUpdate{
+				v2LDSTarget: {RouteConfigName: v2RouteConfigName, Raw: v2Lis},
+				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, Raw: v3LisWithFilters()},
+				"bad":       {},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
+		},
 	}
 
 	for _, test := range tests {
@@ -499,12 +709,16 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			oldFI := env.FaultInjectionSupport
 			env.FaultInjectionSupport = !test.disableFI
 
-			update, _, err := UnmarshalListener("", test.resources, nil)
-			if ((err != nil) != test.wantErr) ||
-				!cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty(), protocmp.Transform()) {
-				t.Errorf("UnmarshalListener(%v) = (%v, %v) want (%v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr)
+			update, md, err := UnmarshalListener(testVersion, test.resources, nil)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("UnmarshalListener(), got err: %v, wantErr: %v", err, test.wantErr)
+			}
+			if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" {
+				t.Errorf("got unexpected update, diff (-got +want): %v", diff)
+			}
+			if diff := cmp.Diff(md, test.wantMD, cmpOptsIgnoreDetails); diff != "" {
+				t.Errorf("got unexpected metadata, diff (-got +want): %v", diff)
 			}
-
 			env.FaultInjectionSupport = oldFI
 		})
 	}
@@ -513,10 +727,138 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 	const v3LDSTarget = "grpc/server?udpa.resource.listening_address=0.0.0.0:9999"
 
+	var (
+		listenerEmptyTransportSocket = &anypb.Any{
+			TypeUrl: version.V3ListenerURL,
+			Value: func() []byte {
+				lis := &v3listenerpb.Listener{
+					Name: v3LDSTarget,
+					Address: &v3corepb.Address{
+						Address: &v3corepb.Address_SocketAddress{
+							SocketAddress: &v3corepb.SocketAddress{
+								Address: "0.0.0.0",
+								PortSpecifier: &v3corepb.SocketAddress_PortValue{
+									PortValue: 9999,
+								},
+							},
+						},
+					},
+					FilterChains: []*v3listenerpb.FilterChain{
+						{
+							Name: "filter-chain-1",
+						},
+					},
+				}
+				mLis, _ := proto.Marshal(lis)
+				return mLis
+			}(),
+		}
+		listenerNoValidationContext = &anypb.Any{
+			TypeUrl: version.V3ListenerURL,
+			Value: func() []byte {
+				lis := &v3listenerpb.Listener{
+					Name: v3LDSTarget,
+					Address: &v3corepb.Address{
+						Address: &v3corepb.Address_SocketAddress{
+							SocketAddress: &v3corepb.SocketAddress{
+								Address: "0.0.0.0",
+								PortSpecifier: &v3corepb.SocketAddress_PortValue{
+									PortValue: 9999,
+								},
+							},
+						},
+					},
+					FilterChains: []*v3listenerpb.FilterChain{
+						{
+							Name: "filter-chain-1",
+							TransportSocket: &v3corepb.TransportSocket{
+								Name: "envoy.transport_sockets.tls",
+								ConfigType: &v3corepb.TransportSocket_TypedConfig{
+									TypedConfig: &anypb.Any{
+										TypeUrl: version.V3DownstreamTLSContextURL,
+										Value: func() []byte {
+											tls := &v3tlspb.DownstreamTlsContext{
+												CommonTlsContext: &v3tlspb.CommonTlsContext{
+													TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+														InstanceName:    "identityPluginInstance",
+														CertificateName: "identityCertName",
+													},
+												},
+											}
+											mtls, _ := proto.Marshal(tls)
+											return mtls
+										}(),
+									},
+								},
+							},
+						},
+					},
+				}
+				mLis, _ := proto.Marshal(lis)
+				return mLis
+			}(),
+		}
+		listenerWithValidationContext = &anypb.Any{
+			TypeUrl: version.V3ListenerURL,
+			Value: func() []byte {
+				lis := &v3listenerpb.Listener{
+					Name: v3LDSTarget,
+					Address: &v3corepb.Address{
+						Address: &v3corepb.Address_SocketAddress{
+							SocketAddress: &v3corepb.SocketAddress{
+								Address: "0.0.0.0",
+								PortSpecifier: &v3corepb.SocketAddress_PortValue{
+									PortValue: 9999,
+								},
+							},
+						},
+					},
+					FilterChains: []*v3listenerpb.FilterChain{
+						{
+							Name: "filter-chain-1",
+							TransportSocket: &v3corepb.TransportSocket{
+								Name: "envoy.transport_sockets.tls",
+								ConfigType: &v3corepb.TransportSocket_TypedConfig{
+									TypedConfig: &anypb.Any{
+										TypeUrl: version.V3DownstreamTLSContextURL,
+										Value: func() []byte {
+											tls := &v3tlspb.DownstreamTlsContext{
+												RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+												CommonTlsContext: &v3tlspb.CommonTlsContext{
+													TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+														InstanceName:    "identityPluginInstance",
+														CertificateName: "identityCertName",
+													},
+													ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+														ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+															InstanceName:    "rootPluginInstance",
+															CertificateName: "rootCertName",
+														},
+													},
+												},
+											}
+											mtls, _ := proto.Marshal(tls)
+											return mtls
+										}(),
+									},
+								},
+							},
+						},
+					},
+				}
+				mLis, _ := proto.Marshal(lis)
+				return mLis
+			}(),
+		}
+	)
+
+	const testVersion = "test-version-lds-server"
+
 	tests := []struct {
 		name       string
 		resources  []*anypb.Any
 		wantUpdate map[string]ListenerUpdate
+		wantMD     UpdateMetadata
 		wantErr    string
 	}{
 		{
@@ -533,6 +875,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "no address field in LDS response",
 		},
 		{
@@ -550,6 +901,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "no socket_address field in LDS response",
 		},
 		{
@@ -576,6 +936,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{"foo": {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "no host:port in name field of LDS response",
 		},
 		{
@@ -602,6 +971,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "socket_address host does not match the one in name",
 		},
 		{
@@ -628,6 +1006,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "socket_address port does not match the one in name",
 		},
 		{
@@ -658,6 +1045,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "filter chains count in LDS response does not match expected",
 		},
 		{
@@ -692,6 +1088,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "transport_socket field has unexpected name",
 		},
 		{
@@ -731,6 +1136,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "transport_socket field has unexpected typeURL",
 		},
 		{
@@ -771,6 +1185,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "failed to unmarshal DownstreamTlsContext in LDS response",
 		},
 		{
@@ -815,6 +1238,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext",
 		},
 		{
@@ -867,39 +1299,26 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "validation context contains unexpected type",
 		},
 		{
-			name: "empty transport socket",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
+			name:      "empty transport socket",
+			resources: []*anypb.Any{listenerEmptyTransportSocket},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {},
+				v3LDSTarget: {Raw: listenerEmptyTransportSocket},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
 			},
 		},
 		{
@@ -952,6 +1371,15 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set",
 		},
 		{
@@ -998,122 +1426,37 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					}(),
 				},
 			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: "security configuration on the server-side does not contain identity certificate provider instance name",
 		},
 		{
-			name: "happy case with no validation context",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value: func() []byte {
-													tls := &v3tlspb.DownstreamTlsContext{
-														CommonTlsContext: &v3tlspb.CommonTlsContext{
-															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-																InstanceName:    "identityPluginInstance",
-																CertificateName: "identityCertName",
-															},
-														},
-													}
-													mtls, _ := proto.Marshal(tls)
-													return mtls
-												}(),
-											},
-										},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
+			name:      "happy case with no validation context",
+			resources: []*anypb.Any{listenerNoValidationContext},
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
 					SecurityCfg: &SecurityConfig{
 						IdentityInstanceName: "identityPluginInstance",
 						IdentityCertName:     "identityCertName",
 					},
+					Raw: listenerNoValidationContext,
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
-			name: "happy case with validation context provider instance",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value: func() []byte {
-													tls := &v3tlspb.DownstreamTlsContext{
-														RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-														CommonTlsContext: &v3tlspb.CommonTlsContext{
-															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-																InstanceName:    "identityPluginInstance",
-																CertificateName: "identityCertName",
-															},
-															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-																ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-																	InstanceName:    "rootPluginInstance",
-																	CertificateName: "rootCertName",
-																},
-															},
-														},
-													}
-													mtls, _ := proto.Marshal(tls)
-													return mtls
-												}(),
-											},
-										},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
+			name:      "happy case with validation context provider instance",
+			resources: []*anypb.Any{listenerWithValidationContext},
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
 					SecurityCfg: &SecurityConfig{
@@ -1123,22 +1466,30 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						IdentityCertName:     "identityCertName",
 						RequireClientCert:    true,
 					},
+					Raw: listenerWithValidationContext,
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			gotUpdate, _, err := UnmarshalListener("", test.resources, nil)
+			gotUpdate, md, err := UnmarshalListener(testVersion, test.resources, nil)
 			if (err != nil) != (test.wantErr != "") {
-				t.Fatalf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
+				t.Fatalf("UnmarshalListener(), got err: %v, wantErr: %v", err, test.wantErr)
 			}
 			if err != nil && !strings.Contains(err.Error(), test.wantErr) {
-				t.Fatalf("UnmarshalListener(%v) = %v wantErr: %q", test.resources, err, test.wantErr)
+				t.Fatalf("UnmarshalListener() = %v wantErr: %q", err, test.wantErr)
+			}
+			if diff := cmp.Diff(gotUpdate, test.wantUpdate, cmpOpts); diff != "" {
+				t.Errorf("got unexpected update, diff (-got +want): %v", diff)
 			}
-			if !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("UnmarshalListener(%v) = %v want %v", test.resources, gotUpdate, test.wantUpdate)
+			if diff := cmp.Diff(md, test.wantMD, cmpOptsIgnoreDetails); diff != "" {
+				t.Errorf("got unexpected metadata, diff (-got +want): %v", diff)
 			}
 		})
 	}
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index 9bc068fd9d96..0c1e2b285388 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -588,17 +588,27 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 			}(),
 		}
 	)
+	const testVersion = "test-version-rds"
 
 	tests := []struct {
 		name       string
 		resources  []*anypb.Any
 		wantUpdate map[string]RouteConfigUpdate
+		wantMD     UpdateMetadata
 		wantErr    bool
 	}{
 		{
 			name:      "non-routeConfig resource type",
 			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
-			wantErr:   true,
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 		{
 			name: "badly marshaled routeconfig resource",
@@ -608,10 +618,22 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 					Value:   []byte{1, 2, 3, 4},
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
 			wantErr: true,
 		},
 		{
 			name: "empty resource list",
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v2 routeConfig resource",
@@ -628,8 +650,13 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v2ClusterName: {Weight: 1}}}},
 						},
 					},
+					Raw: v2RouteConfig,
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "v3 routeConfig resource",
@@ -646,8 +673,13 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}}},
 						},
 					},
+					Raw: v3RouteConfig,
 				},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
 		},
 		{
 			name:      "multiple routeConfig resources",
@@ -664,6 +696,61 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}}},
 						},
 					},
+					Raw: v3RouteConfig,
+				},
+				v2RouteConfigName: {
+					VirtualHosts: []*VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
+						},
+						{
+							Domains: []string{ldsTarget},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v2ClusterName: {Weight: 1}}}},
+						},
+					},
+					Raw: v2RouteConfig,
+				},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
+		},
+		{
+			// To test that unmarshal keeps processing on errors.
+			name: "good and bad routeConfig resources",
+			resources: []*anypb.Any{
+				v2RouteConfig,
+				{
+					TypeUrl: version.V2RouteConfigURL,
+					Value: func() []byte {
+						rc := &v3routepb.RouteConfiguration{
+							Name: "bad",
+							VirtualHosts: []*v3routepb.VirtualHost{
+								{Domains: []string{ldsTarget},
+									Routes: []*v3routepb.Route{{
+										Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_ConnectMatcher_{}},
+									}}}}}
+						m, _ := proto.Marshal(rc)
+						return m
+					}(),
+				},
+				v3RouteConfig,
+			},
+			wantUpdate: map[string]RouteConfigUpdate{
+				v3RouteConfigName: {
+					VirtualHosts: []*VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
+						},
+						{
+							Domains: []string{ldsTarget},
+							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}}},
+						},
+					},
+					Raw: v3RouteConfig,
 				},
 				v2RouteConfigName: {
 					VirtualHosts: []*VirtualHost{
@@ -676,15 +763,32 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 							Routes:  []*Route{{Prefix: newStringP(""), WeightedClusters: map[string]WeightedCluster{v2ClusterName: {Weight: 1}}}},
 						},
 					},
+					Raw: v2RouteConfig,
 				},
+				"bad": {},
 			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: true,
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, _, err := UnmarshalRouteConfig("", test.resources, nil)
-			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("UnmarshalRouteConfig(%v, %v) = (%v, %v) want (%v, %v)", test.resources, ldsTarget, update, err, test.wantUpdate, test.wantErr)
+			update, md, err := UnmarshalRouteConfig(testVersion, test.resources, nil)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("UnmarshalRouteConfig(), got err: %v, wantErr: %v", err, test.wantErr)
+			}
+			if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" {
+				t.Errorf("got unexpected update, diff (-got +want): %v", diff)
+			}
+			if diff := cmp.Diff(md, test.wantMD, cmpOptsIgnoreDetails); diff != "" {
+				t.Errorf("got unexpected metadata, diff (-got +want): %v", diff)
 			}
 		})
 	}
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index 1f0acad6d1e4..c71b84532315 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -24,7 +24,7 @@ import (
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/version"
@@ -63,7 +63,7 @@ var (
 			},
 		},
 	}
-	marshaledCluster1, _ = proto.Marshal(goodCluster1)
+	marshaledCluster1, _ = ptypes.MarshalAny(goodCluster1)
 	goodCluster2         = &xdspb.Cluster{
 		Name:                 goodClusterName2,
 		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
@@ -77,22 +77,16 @@ var (
 		},
 		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
 	}
-	marshaledCluster2, _ = proto.Marshal(goodCluster2)
+	marshaledCluster2, _ = ptypes.MarshalAny(goodCluster2)
 	goodCDSResponse1     = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ClusterURL,
-				Value:   marshaledCluster1,
-			},
+			marshaledCluster1,
 		},
 		TypeUrl: version.V2ClusterURL,
 	}
 	goodCDSResponse2 = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ClusterURL,
-				Value:   marshaledCluster2,
-			},
+			marshaledCluster2,
 		},
 		TypeUrl: version.V2ClusterURL,
 	}
@@ -106,47 +100,73 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 		name          string
 		cdsResponse   *xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *xdsclient.ClusterUpdate
+		wantUpdate    map[string]xdsclient.ClusterUpdate
+		wantUpdateMD  xdsclient.UpdateMetadata
 		wantUpdateErr bool
 	}{
 		// Badly marshaled CDS response.
 		{
-			name:          "badly-marshaled-response",
-			cdsResponse:   badlyMarshaledCDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "badly-marshaled-response",
+			cdsResponse: badlyMarshaledCDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response does not contain Cluster proto.
 		{
-			name:          "no-cluster-proto-in-response",
-			cdsResponse:   badResourceTypeInLDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "no-cluster-proto-in-response",
+			cdsResponse: badResourceTypeInLDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains no clusters.
 		{
-			name:          "no-cluster",
-			cdsResponse:   &xdspb.DiscoveryResponse{},
-			wantErr:       false,
-			wantUpdate:    nil,
+			name:        "no-cluster",
+			cdsResponse: &xdspb.DiscoveryResponse{},
+			wantErr:     false,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one good cluster we are not interested in.
 		{
-			name:          "one-uninteresting-cluster",
-			cdsResponse:   goodCDSResponse2,
-			wantErr:       false,
-			wantUpdate:    nil,
+			name:        "one-uninteresting-cluster",
+			cdsResponse: goodCDSResponse2,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.ClusterUpdate{
+				goodClusterName2: {ServiceName: serviceName2, Raw: marshaledCluster2},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one cluster and it is good.
 		{
-			name:          "one-good-cluster",
-			cdsResponse:   goodCDSResponse1,
-			wantErr:       false,
-			wantUpdate:    &xdsclient.ClusterUpdate{ServiceName: serviceName1, EnableLRS: true},
+			name:        "one-good-cluster",
+			cdsResponse: goodCDSResponse1,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.ClusterUpdate{
+				goodClusterName1: {ServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 	}
@@ -159,6 +179,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 				responseToHandle: test.cdsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
+				wantUpdateMD:     test.wantUpdateMD,
 				wantUpdateErr:    test.wantUpdateErr,
 			})
 		})
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index fcfa6f7b74a5..b6bc4908120d 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -186,11 +186,8 @@ func (v2c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, stri
 // callback.
 func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalListener(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
-	if err != nil {
-		return err
-	}
 	v2c.parent.NewListeners(update, md)
-	return nil
+	return err
 }
 
 // handleRDSResponse processes an RDS response received from the management
@@ -198,11 +195,8 @@ func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 // invokes the registered watcher callback.
 func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalRouteConfig(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
-	if err != nil {
-		return err
-	}
 	v2c.parent.NewRouteConfigs(update, md)
-	return nil
+	return err
 }
 
 // handleCDSResponse processes an CDS response received from the management
@@ -210,18 +204,12 @@ func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 // callback.
 func (v2c *client) handleCDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalCluster(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
-	if err != nil {
-		return err
-	}
 	v2c.parent.NewClusters(update, md)
-	return nil
+	return err
 }
 
 func (v2c *client) handleEDSResponse(resp *v2xdspb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalEndpoints(resp.GetVersionInfo(), resp.GetResources(), v2c.logger)
-	if err != nil {
-		return err
-	}
 	v2c.parent.NewEndpoints(update, md)
-	return nil
+	return err
 }
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index e79d7bf47f2a..e770324e1b12 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -21,19 +21,14 @@ package v2
 import (
 	"context"
 	"errors"
-	"reflect"
+	"fmt"
 	"testing"
 	"time"
 
-	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
-	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
-	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
 	"github.com/golang/protobuf/proto"
-	anypb "github.com/golang/protobuf/ptypes/any"
-	structpb "github.com/golang/protobuf/ptypes/struct"
+	"github.com/golang/protobuf/ptypes"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpclog"
@@ -44,6 +39,15 @@ import (
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils/fakeserver"
 	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
+	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	structpb "github.com/golang/protobuf/ptypes/struct"
 )
 
 type s struct {
@@ -118,7 +122,7 @@ var (
 			},
 		},
 	}
-	marshaledListener1, _ = proto.Marshal(goodListener1)
+	marshaledListener1, _ = ptypes.MarshalAny(goodListener1)
 	goodListener2         = &xdspb.Listener{
 		Name: goodLDSTarget2,
 		ApiListener: &listenerpb.ApiListener{
@@ -128,7 +132,7 @@ var (
 			},
 		},
 	}
-	marshaledListener2, _     = proto.Marshal(goodListener2)
+	marshaledListener2, _     = ptypes.MarshalAny(goodListener2)
 	noAPIListener             = &xdspb.Listener{Name: goodLDSTarget1}
 	marshaledNoAPIListener, _ = proto.Marshal(noAPIListener)
 	badAPIListener2           = &xdspb.Listener{
@@ -143,19 +147,13 @@ var (
 	badlyMarshaledAPIListener2, _ = proto.Marshal(badAPIListener2)
 	goodLDSResponse1              = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledListener1,
-			},
+			marshaledListener1,
 		},
 		TypeUrl: version.V2ListenerURL,
 	}
 	goodLDSResponse2 = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledListener2,
-			},
+			marshaledListener2,
 		},
 		TypeUrl: version.V2ListenerURL,
 	}
@@ -180,14 +178,8 @@ var (
 	}
 	ldsResponseWithMultipleResources = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledListener2,
-			},
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledListener1,
-			},
+			marshaledListener2,
+			marshaledListener1,
 		},
 		TypeUrl: version.V2ListenerURL,
 	}
@@ -202,14 +194,8 @@ var (
 	}
 	goodBadUglyLDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledListener2,
-			},
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledListener1,
-			},
+			marshaledListener2,
+			marshaledListener1,
 			{
 				TypeUrl: version.V2ListenerURL,
 				Value:   badlyMarshaledAPIListener2,
@@ -238,13 +224,10 @@ var (
 	noVirtualHostsRouteConfig = &xdspb.RouteConfiguration{
 		Name: goodRouteName1,
 	}
-	marshaledNoVirtualHostsRouteConfig, _ = proto.Marshal(noVirtualHostsRouteConfig)
+	marshaledNoVirtualHostsRouteConfig, _ = ptypes.MarshalAny(noVirtualHostsRouteConfig)
 	noVirtualHostsInRDSResponse           = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2RouteConfigURL,
-				Value:   marshaledNoVirtualHostsRouteConfig,
-			},
+			marshaledNoVirtualHostsRouteConfig,
 		},
 		TypeUrl: version.V2RouteConfigURL,
 	}
@@ -279,7 +262,7 @@ var (
 			},
 		},
 	}
-	marshaledGoodRouteConfig1, _ = proto.Marshal(goodRouteConfig1)
+	marshaledGoodRouteConfig1, _ = ptypes.MarshalAny(goodRouteConfig1)
 	goodRouteConfig2             = &xdspb.RouteConfiguration{
 		Name: goodRouteName2,
 		VirtualHosts: []*routepb.VirtualHost{
@@ -311,25 +294,22 @@ var (
 			},
 		},
 	}
-	marshaledGoodRouteConfig2, _ = proto.Marshal(goodRouteConfig2)
+	marshaledGoodRouteConfig2, _ = ptypes.MarshalAny(goodRouteConfig2)
 	goodRDSResponse1             = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2RouteConfigURL,
-				Value:   marshaledGoodRouteConfig1,
-			},
+			marshaledGoodRouteConfig1,
 		},
 		TypeUrl: version.V2RouteConfigURL,
 	}
 	goodRDSResponse2 = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2RouteConfigURL,
-				Value:   marshaledGoodRouteConfig2,
-			},
+			marshaledGoodRouteConfig2,
 		},
 		TypeUrl: version.V2RouteConfigURL,
 	}
+	// An place holder error. When comparing UpdateErrorMetadata, we only check
+	// if error is nil, and don't compare error content.
+	errPlaceHolder = fmt.Errorf("err place holder")
 )
 
 type watchHandleTestcase struct {
@@ -339,6 +319,7 @@ type watchHandleTestcase struct {
 	responseToHandle *xdspb.DiscoveryResponse
 	wantHandleErr    bool
 	wantUpdate       interface{}
+	wantUpdateMD     xdsclient.UpdateMetadata
 	wantUpdateErr    bool
 }
 
@@ -396,15 +377,39 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 
 	type updateErr struct {
 		u   interface{}
+		md  xdsclient.UpdateMetadata
 		err error
 	}
 	gotUpdateCh := testutils.NewChannel()
 
 	v2c, err := newV2Client(&testUpdateReceiver{
-		f: func(rType xdsclient.ResourceType, d map[string]interface{}, _ xdsclient.UpdateMetadata) {
+		f: func(rType xdsclient.ResourceType, d map[string]interface{}, md xdsclient.UpdateMetadata) {
 			if rType == test.rType {
-				if u, ok := d[test.resourceName]; ok {
-					gotUpdateCh.Send(updateErr{u, nil})
+				switch test.rType {
+				case xdsclient.ListenerResource:
+					dd := make(map[string]xdsclient.ListenerUpdate)
+					for n, u := range d {
+						dd[n] = u.(xdsclient.ListenerUpdate)
+					}
+					gotUpdateCh.Send(updateErr{dd, md, nil})
+				case xdsclient.RouteConfigResource:
+					dd := make(map[string]xdsclient.RouteConfigUpdate)
+					for n, u := range d {
+						dd[n] = u.(xdsclient.RouteConfigUpdate)
+					}
+					gotUpdateCh.Send(updateErr{dd, md, nil})
+				case xdsclient.ClusterResource:
+					dd := make(map[string]xdsclient.ClusterUpdate)
+					for n, u := range d {
+						dd[n] = u.(xdsclient.ClusterUpdate)
+					}
+					gotUpdateCh.Send(updateErr{dd, md, nil})
+				case xdsclient.EndpointsResource:
+					dd := make(map[string]xdsclient.EndpointsUpdate)
+					for n, u := range d {
+						dd[n] = u.(xdsclient.EndpointsUpdate)
+					}
+					gotUpdateCh.Send(updateErr{dd, md, nil})
 				}
 			}
 		},
@@ -447,30 +452,25 @@ func testWatchHandle(t *testing.T, test *watchHandleTestcase) {
 		t.Fatalf("v2c.handleRDSResponse() returned err: %v, wantErr: %v", err, test.wantHandleErr)
 	}
 
-	// If the test doesn't expect the callback to be invoked, verify that no
-	// update or error is pushed to the callback.
-	//
-	// Cannot directly compare test.wantUpdate with nil (typed vs non-typed nil:
-	// https://golang.org/doc/faq#nil_error).
-	if c := test.wantUpdate; c == nil || (reflect.ValueOf(c).Kind() == reflect.Ptr && reflect.ValueOf(c).IsNil()) {
-		sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
-		defer sCancel()
-		update, err := gotUpdateCh.Receive(sCtx)
-		if err == context.DeadlineExceeded {
-			return
-		}
-		t.Fatalf("Unexpected update: +%v", update)
-	}
-
-	wantUpdate := reflect.ValueOf(test.wantUpdate).Elem().Interface()
+	wantUpdate := test.wantUpdate
+	cmpOpts := cmp.Options{
+		cmpopts.EquateEmpty(), protocmp.Transform(),
+		cmpopts.IgnoreFields(xdsclient.UpdateMetadata{}, "Timestamp"),
+		cmpopts.IgnoreFields(xdsclient.UpdateErrorMetadata{}, "Timestamp"),
+		cmp.Comparer(func(x, y error) bool { return (x == nil) == (y == nil) }),
+	}
 	uErr, err := gotUpdateCh.Receive(ctx)
 	if err == context.DeadlineExceeded {
 		t.Fatal("Timeout expecting xDS update")
 	}
 	gotUpdate := uErr.(updateErr).u
-	if diff := cmp.Diff(gotUpdate, wantUpdate); diff != "" {
+	if diff := cmp.Diff(gotUpdate, wantUpdate, cmpOpts); diff != "" {
 		t.Fatalf("got update : %+v, want %+v, diff: %s", gotUpdate, wantUpdate, diff)
 	}
+	gotUpdateMD := uErr.(updateErr).md
+	if diff := cmp.Diff(gotUpdateMD, test.wantUpdateMD, cmpOpts); diff != "" {
+		t.Fatalf("got update : %+v, want %+v, diff: %s", gotUpdateMD, test.wantUpdateMD, diff)
+	}
 	gotUpdateErr := uErr.(updateErr).err
 	if (gotUpdateErr != nil) != test.wantUpdateErr {
 		t.Fatalf("got xDS update error {%v}, wantErr: %v", gotUpdateErr, test.wantUpdateErr)
diff --git a/xds/internal/client/v2/eds_test.go b/xds/internal/client/v2/eds_test.go
index 9d04433de746..0990e7ebae0e 100644
--- a/xds/internal/client/v2/eds_test.go
+++ b/xds/internal/client/v2/eds_test.go
@@ -50,27 +50,28 @@ var (
 		},
 		TypeUrl: version.V2EndpointsURL,
 	}
+	marshaledGoodCLA1 = func() *anypb.Any {
+		clab0 := testutils.NewClusterLoadAssignmentBuilder(goodEDSName, nil)
+		clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil)
+		clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil)
+		a, _ := ptypes.MarshalAny(clab0.Build())
+		return a
+	}()
 	goodEDSResponse1 = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			func() *anypb.Any {
-				clab0 := testutils.NewClusterLoadAssignmentBuilder(goodEDSName, nil)
-				clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil)
-				clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil)
-				a, _ := ptypes.MarshalAny(clab0.Build())
-				return a
-			}(),
+			marshaledGoodCLA1,
 		},
 		TypeUrl: version.V2EndpointsURL,
 	}
+	marshaledGoodCLA2 = func() *anypb.Any {
+		clab0 := testutils.NewClusterLoadAssignmentBuilder("not-goodEDSName", nil)
+		clab0.AddLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
+		a, _ := ptypes.MarshalAny(clab0.Build())
+		return a
+	}()
 	goodEDSResponse2 = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
-			func() *anypb.Any {
-				clab0 := testutils.NewClusterLoadAssignmentBuilder("not-goodEDSName", nil)
-				clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil)
-				clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil)
-				a, _ := ptypes.MarshalAny(clab0.Build())
-				return a
-			}(),
+			marshaledGoodCLA2,
 		},
 		TypeUrl: version.V2EndpointsURL,
 	}
@@ -81,31 +82,59 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 		name          string
 		edsResponse   *v2xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *xdsclient.EndpointsUpdate
+		wantUpdate    map[string]xdsclient.EndpointsUpdate
+		wantUpdateMD  xdsclient.UpdateMetadata
 		wantUpdateErr bool
 	}{
 		// Any in resource is badly marshaled.
 		{
-			name:          "badly-marshaled_response",
-			edsResponse:   badlyMarshaledEDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "badly-marshaled_response",
+			edsResponse: badlyMarshaledEDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response doesn't contain resource with the right type.
 		{
-			name:          "no-config-in-response",
-			edsResponse:   badResourceTypeInEDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "no-config-in-response",
+			edsResponse: badResourceTypeInEDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one uninteresting ClusterLoadAssignment.
 		{
-			name:          "one-uninterestring-assignment",
-			edsResponse:   goodEDSResponse2,
-			wantErr:       false,
-			wantUpdate:    nil,
+			name:        "one-uninterestring-assignment",
+			edsResponse: goodEDSResponse2,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.EndpointsUpdate{
+				"not-goodEDSName": {
+					Localities: []xdsclient.Locality{
+						{
+							Endpoints: []xdsclient.Endpoint{{Address: "addr1:314"}},
+							ID:        internal.LocalityID{SubZone: "locality-1"},
+							Priority:  0,
+							Weight:    1,
+						},
+					},
+					Raw: marshaledGoodCLA2,
+				},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one good ClusterLoadAssignment.
@@ -113,22 +142,28 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 			name:        "one-good-assignment",
 			edsResponse: goodEDSResponse1,
 			wantErr:     false,
-			wantUpdate: &xdsclient.EndpointsUpdate{
-				Localities: []xdsclient.Locality{
-					{
-						Endpoints: []xdsclient.Endpoint{{Address: "addr1:314"}},
-						ID:        internal.LocalityID{SubZone: "locality-1"},
-						Priority:  1,
-						Weight:    1,
-					},
-					{
-						Endpoints: []xdsclient.Endpoint{{Address: "addr2:159"}},
-						ID:        internal.LocalityID{SubZone: "locality-2"},
-						Priority:  0,
-						Weight:    1,
+			wantUpdate: map[string]xdsclient.EndpointsUpdate{
+				goodEDSName: {
+					Localities: []xdsclient.Locality{
+						{
+							Endpoints: []xdsclient.Endpoint{{Address: "addr1:314"}},
+							ID:        internal.LocalityID{SubZone: "locality-1"},
+							Priority:  1,
+							Weight:    1,
+						},
+						{
+							Endpoints: []xdsclient.Endpoint{{Address: "addr2:159"}},
+							ID:        internal.LocalityID{SubZone: "locality-2"},
+							Priority:  0,
+							Weight:    1,
+						},
 					},
+					Raw: marshaledGoodCLA1,
 				},
 			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 	}
@@ -140,6 +175,7 @@ func (s) TestEDSHandleResponse(t *testing.T) {
 				responseToHandle: test.edsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
+				wantUpdateMD:     test.wantUpdateMD,
 				wantUpdateErr:    test.wantUpdateErr,
 			})
 		})
diff --git a/xds/internal/client/v2/lds_test.go b/xds/internal/client/v2/lds_test.go
index 28e5633b244a..1f4c980fae5e 100644
--- a/xds/internal/client/v2/lds_test.go
+++ b/xds/internal/client/v2/lds_test.go
@@ -35,77 +35,126 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 		name          string
 		ldsResponse   *v2xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *xdsclient.ListenerUpdate
+		wantUpdate    map[string]xdsclient.ListenerUpdate
+		wantUpdateMD  xdsclient.UpdateMetadata
 		wantUpdateErr bool
 	}{
 		// Badly marshaled LDS response.
 		{
-			name:          "badly-marshaled-response",
-			ldsResponse:   badlyMarshaledLDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "badly-marshaled-response",
+			ldsResponse: badlyMarshaledLDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response does not contain Listener proto.
 		{
-			name:          "no-listener-proto-in-response",
-			ldsResponse:   badResourceTypeInLDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "no-listener-proto-in-response",
+			ldsResponse: badResourceTypeInLDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// No APIListener in the response. Just one test case here for a bad
 		// ApiListener, since the others are covered in
 		// TestGetRouteConfigNameFromListener.
 		{
-			name:          "no-apiListener-in-response",
-			ldsResponse:   noAPIListenerLDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "no-apiListener-in-response",
+			ldsResponse: noAPIListenerLDSResponse,
+			wantErr:     true,
+			wantUpdate: map[string]xdsclient.ListenerUpdate{
+				goodLDSTarget1: {},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one listener and it is good.
 		{
-			name:          "one-good-listener",
-			ldsResponse:   goodLDSResponse1,
-			wantErr:       false,
-			wantUpdate:    &xdsclient.ListenerUpdate{RouteConfigName: goodRouteName1},
+			name:        "one-good-listener",
+			ldsResponse: goodLDSResponse1,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.ListenerUpdate{
+				goodLDSTarget1: {RouteConfigName: goodRouteName1, Raw: marshaledListener1},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains multiple good listeners, including the one we are
 		// interested in.
 		{
-			name:          "multiple-good-listener",
-			ldsResponse:   ldsResponseWithMultipleResources,
-			wantErr:       false,
-			wantUpdate:    &xdsclient.ListenerUpdate{RouteConfigName: goodRouteName1},
+			name:        "multiple-good-listener",
+			ldsResponse: ldsResponseWithMultipleResources,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.ListenerUpdate{
+				goodLDSTarget1: {RouteConfigName: goodRouteName1, Raw: marshaledListener1},
+				goodLDSTarget2: {RouteConfigName: goodRouteName1, Raw: marshaledListener2},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains two good listeners (one interesting and one
 		// uninteresting), and one badly marshaled listener. This will cause a
 		// nack because the uninteresting listener will still be parsed.
 		{
-			name:          "good-bad-ugly-listeners",
-			ldsResponse:   goodBadUglyLDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "good-bad-ugly-listeners",
+			ldsResponse: goodBadUglyLDSResponse,
+			wantErr:     true,
+			wantUpdate: map[string]xdsclient.ListenerUpdate{
+				goodLDSTarget1: {RouteConfigName: goodRouteName1, Raw: marshaledListener1},
+				goodLDSTarget2: {},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one listener, but we are not interested in it.
 		{
-			name:          "one-uninteresting-listener",
-			ldsResponse:   goodLDSResponse2,
-			wantErr:       false,
-			wantUpdate:    nil,
+			name:        "one-uninteresting-listener",
+			ldsResponse: goodLDSResponse2,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.ListenerUpdate{
+				goodLDSTarget2: {RouteConfigName: goodRouteName1, Raw: marshaledListener2},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response constains no resources. This is the case where the server
 		// does not know about the target we are interested in.
 		{
-			name:          "empty-response",
-			ldsResponse:   emptyLDSResponse,
-			wantErr:       false,
-			wantUpdate:    nil,
+			name:        "empty-response",
+			ldsResponse: emptyLDSResponse,
+			wantErr:     false,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 	}
@@ -118,6 +167,7 @@ func (s) TestLDSHandleResponse(t *testing.T) {
 				responseToHandle: test.ldsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
+				wantUpdateMD:     test.wantUpdateMD,
 				wantUpdateErr:    test.wantUpdateErr,
 			})
 		})
diff --git a/xds/internal/client/v2/rds_test.go b/xds/internal/client/v2/rds_test.go
index ed5058836ce6..dd145158b8a9 100644
--- a/xds/internal/client/v2/rds_test.go
+++ b/xds/internal/client/v2/rds_test.go
@@ -49,23 +49,36 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 		name          string
 		rdsResponse   *xdspb.DiscoveryResponse
 		wantErr       bool
-		wantUpdate    *xdsclient.RouteConfigUpdate
+		wantUpdate    map[string]xdsclient.RouteConfigUpdate
+		wantUpdateMD  xdsclient.UpdateMetadata
 		wantUpdateErr bool
 	}{
 		// Badly marshaled RDS response.
 		{
-			name:          "badly-marshaled-response",
-			rdsResponse:   badlyMarshaledRDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "badly-marshaled-response",
+			rdsResponse: badlyMarshaledRDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// Response does not contain RouteConfiguration proto.
 		{
-			name:          "no-route-config-in-response",
-			rdsResponse:   badResourceTypeInRDSResponse,
-			wantErr:       true,
-			wantUpdate:    nil,
+			name:        "no-route-config-in-response",
+			rdsResponse: badResourceTypeInRDSResponse,
+			wantErr:     true,
+			wantUpdate:  nil,
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusNACKed,
+				ErrState: &xdsclient.UpdateErrorMetadata{
+					Err: errPlaceHolder,
+				},
+			},
 			wantUpdateErr: false,
 		},
 		// No VirtualHosts in the response. Just one test case here for a bad
@@ -75,17 +88,40 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 			name:        "no-virtual-hosts-in-response",
 			rdsResponse: noVirtualHostsInRDSResponse,
 			wantErr:     false,
-			wantUpdate: &xdsclient.RouteConfigUpdate{
-				VirtualHosts: nil,
+			wantUpdate: map[string]xdsclient.RouteConfigUpdate{
+				goodRouteName1: {
+					VirtualHosts: nil,
+					Raw:          marshaledNoVirtualHostsRouteConfig,
+				},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
 			},
 			wantUpdateErr: false,
 		},
 		// Response contains one good RouteConfiguration, uninteresting though.
 		{
-			name:          "one-uninteresting-route-config",
-			rdsResponse:   goodRDSResponse2,
-			wantErr:       false,
-			wantUpdate:    nil,
+			name:        "one-uninteresting-route-config",
+			rdsResponse: goodRDSResponse2,
+			wantErr:     false,
+			wantUpdate: map[string]xdsclient.RouteConfigUpdate{
+				goodRouteName2: {
+					VirtualHosts: []*xdsclient.VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
+						},
+						{
+							Domains: []string{goodLDSTarget1},
+							Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{goodClusterName2: {Weight: 1}}}},
+						},
+					},
+					Raw: marshaledGoodRouteConfig2,
+				},
+			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 		// Response contains one good interesting RouteConfiguration.
@@ -93,18 +129,24 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 			name:        "one-good-route-config",
 			rdsResponse: goodRDSResponse1,
 			wantErr:     false,
-			wantUpdate: &xdsclient.RouteConfigUpdate{
-				VirtualHosts: []*xdsclient.VirtualHost{
-					{
-						Domains: []string{uninterestingDomain},
-						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
-					},
-					{
-						Domains: []string{goodLDSTarget1},
-						Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{goodClusterName1: {Weight: 1}}}},
+			wantUpdate: map[string]xdsclient.RouteConfigUpdate{
+				goodRouteName1: {
+					VirtualHosts: []*xdsclient.VirtualHost{
+						{
+							Domains: []string{uninterestingDomain},
+							Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{uninterestingClusterName: {Weight: 1}}}},
+						},
+						{
+							Domains: []string{goodLDSTarget1},
+							Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{goodClusterName1: {Weight: 1}}}},
+						},
 					},
+					Raw: marshaledGoodRouteConfig1,
 				},
 			},
+			wantUpdateMD: xdsclient.UpdateMetadata{
+				Status: xdsclient.ServiceStatusACKed,
+			},
 			wantUpdateErr: false,
 		},
 	}
@@ -116,6 +158,7 @@ func (s) TestRDSHandleResponseWithRouting(t *testing.T) {
 				responseToHandle: test.rdsResponse,
 				wantHandleErr:    test.wantErr,
 				wantUpdate:       test.wantUpdate,
+				wantUpdateMD:     test.wantUpdateMD,
 				wantUpdateErr:    test.wantUpdateErr,
 			})
 		})
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 23bb1bfd478b..55cae56d8cc6 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -186,11 +186,8 @@ func (v3c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, stri
 // callback.
 func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalListener(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
-	if err != nil {
-		return err
-	}
 	v3c.parent.NewListeners(update, md)
-	return nil
+	return err
 }
 
 // handleRDSResponse processes an RDS response received from the management
@@ -198,11 +195,8 @@ func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 // invokes the registered watcher callback.
 func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalRouteConfig(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
-	if err != nil {
-		return err
-	}
 	v3c.parent.NewRouteConfigs(update, md)
-	return nil
+	return err
 }
 
 // handleCDSResponse processes an CDS response received from the management
@@ -210,18 +204,12 @@ func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) erro
 // callback.
 func (v3c *client) handleCDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalCluster(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
-	if err != nil {
-		return err
-	}
 	v3c.parent.NewClusters(update, md)
-	return nil
+	return err
 }
 
 func (v3c *client) handleEDSResponse(resp *v3discoverypb.DiscoveryResponse) error {
 	update, md, err := xdsclient.UnmarshalEndpoints(resp.GetVersionInfo(), resp.GetResources(), v3c.logger)
-	if err != nil {
-		return err
-	}
 	v3c.parent.NewEndpoints(update, md)
-	return nil
+	return err
 }
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index cdecbe29694e..8fc50ea3056c 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -24,6 +24,7 @@ import (
 	"net"
 	"strconv"
 	"strings"
+	"time"
 
 	v1typepb "github.com/cncf/udpa/go/udpa/type/v1"
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
@@ -52,27 +53,30 @@ const transportSocketName = "envoy.transport_sockets.tls"
 // UnmarshalListener processes resources received in an LDS response, validates
 // them, and transforms them into a native struct which contains only fields we
 // are interested in.
-func UnmarshalListener(_ string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, UpdateMetadata, error) {
+func UnmarshalListener(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, UpdateMetadata, error) {
 	update := make(map[string]ListenerUpdate)
-	for _, r := range resources {
-		if !IsListenerResource(r.GetTypeUrl()) {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in LDS response", r.GetTypeUrl())
-		}
-		// TODO: Pass version.TransportAPI instead of relying upon the type URL
-		v2 := r.GetTypeUrl() == version.V2ListenerURL
-		lis := &v3listenerpb.Listener{}
-		if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err)
-		}
-		logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
+	md, err := processAllResources(version, resources, logger, update)
+	return update, md, err
+}
 
-		lu, err := processListener(lis, v2)
-		if err != nil {
-			return nil, UpdateMetadata{}, err
-		}
-		update[lis.GetName()] = *lu
+func unmarshalListenerResource(r *anypb.Any, logger *grpclog.PrefixLogger) (string, ListenerUpdate, error) {
+	if !IsListenerResource(r.GetTypeUrl()) {
+		return "", ListenerUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
 	}
-	return update, UpdateMetadata{}, nil
+	// TODO: Pass version.TransportAPI instead of relying upon the type URL
+	v2 := r.GetTypeUrl() == version.V2ListenerURL
+	lis := &v3listenerpb.Listener{}
+	if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
+		return "", ListenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
+	}
+	logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
+
+	lu, err := processListener(lis, v2)
+	if err != nil {
+		return lis.GetName(), ListenerUpdate{}, err
+	}
+	lu.Raw = r
+	return lis.GetName(), *lu, nil
 }
 
 func processListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUpdate, error) {
@@ -89,31 +93,31 @@ func processClientSideListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUp
 
 	apiLisAny := lis.GetApiListener().GetApiListener()
 	if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
-		return nil, fmt.Errorf("xds: unexpected resource type: %q in LDS response", apiLisAny.GetTypeUrl())
+		return nil, fmt.Errorf("unexpected resource type: %q", apiLisAny.GetTypeUrl())
 	}
 	apiLis := &v3httppb.HttpConnectionManager{}
 	if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
-		return nil, fmt.Errorf("xds: failed to unmarshal api_listner in LDS response: %v", err)
+		return nil, fmt.Errorf("failed to unmarshal api_listner: %v", err)
 	}
 
 	switch apiLis.RouteSpecifier.(type) {
 	case *v3httppb.HttpConnectionManager_Rds:
 		if apiLis.GetRds().GetConfigSource().GetAds() == nil {
-			return nil, fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis)
+			return nil, fmt.Errorf("ConfigSource is not ADS: %+v", lis)
 		}
 		name := apiLis.GetRds().GetRouteConfigName()
 		if name == "" {
-			return nil, fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis)
+			return nil, fmt.Errorf("empty route_config_name: %+v", lis)
 		}
 		update.RouteConfigName = name
 	case *v3httppb.HttpConnectionManager_RouteConfig:
 		// TODO: Add support for specifying the RouteConfiguration inline
 		// in the LDS response.
-		return nil, fmt.Errorf("xds: LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
+		return nil, fmt.Errorf("LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
 	case nil:
-		return nil, fmt.Errorf("xds: no RouteSpecifier in received LDS response: %+v", apiLis)
+		return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis)
 	default:
-		return nil, fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier)
+		return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier)
 	}
 
 	if v2 {
@@ -126,7 +130,7 @@ func processClientSideListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUp
 
 	var err error
 	if update.HTTPFilters, err = processHTTPFilters(apiLis.GetHttpFilters(), false); err != nil {
-		return nil, fmt.Errorf("xds: %v", err)
+		return nil, err
 	}
 
 	return update, nil
@@ -248,28 +252,28 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	// grpc/server?udpa.resource.listening_address=IP:Port.
 	addr := lis.GetAddress()
 	if addr == nil {
-		return nil, fmt.Errorf("xds: no address field in LDS response: %+v", lis)
+		return nil, fmt.Errorf("no address field in LDS response: %+v", lis)
 	}
 	sockAddr := addr.GetSocketAddress()
 	if sockAddr == nil {
-		return nil, fmt.Errorf("xds: no socket_address field in LDS response: %+v", lis)
+		return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis)
 	}
 	host, port, err := getAddressFromName(lis.GetName())
 	if err != nil {
-		return nil, fmt.Errorf("xds: no host:port in name field of LDS response: %+v, error: %v", lis, err)
+		return nil, fmt.Errorf("no host:port in name field of LDS response: %+v, error: %v", lis, err)
 	}
 	if h := sockAddr.GetAddress(); host != h {
-		return nil, fmt.Errorf("xds: socket_address host does not match the one in name. Got %q, want %q", h, host)
+		return nil, fmt.Errorf("socket_address host does not match the one in name. Got %q, want %q", h, host)
 	}
 	if p := strconv.Itoa(int(sockAddr.GetPortValue())); port != p {
-		return nil, fmt.Errorf("xds: socket_address port does not match the one in name. Got %q, want %q", p, port)
+		return nil, fmt.Errorf("socket_address port does not match the one in name. Got %q, want %q", p, port)
 	}
 
 	// Make sure the listener resource contains a single filter chain. We do not
 	// support multiple filter chains and picking the best match from the list.
 	fcs := lis.GetFilterChains()
 	if n := len(fcs); n != 1 {
-		return nil, fmt.Errorf("xds: filter chains count in LDS response does not match expected. Got %d, want 1", n)
+		return nil, fmt.Errorf("filter chains count in LDS response does not match expected. Got %d, want 1", n)
 	}
 	fc := fcs[0]
 
@@ -282,18 +286,18 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 		return &ListenerUpdate{}, nil
 	}
 	if name := ts.GetName(); name != transportSocketName {
-		return nil, fmt.Errorf("xds: transport_socket field has unexpected name: %s", name)
+		return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name)
 	}
 	any := ts.GetTypedConfig()
 	if any == nil || any.TypeUrl != version.V3DownstreamTLSContextURL {
-		return nil, fmt.Errorf("xds: transport_socket field has unexpected typeURL: %s", any.TypeUrl)
+		return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl)
 	}
 	downstreamCtx := &v3tlspb.DownstreamTlsContext{}
 	if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil {
-		return nil, fmt.Errorf("xds: failed to unmarshal DownstreamTlsContext in LDS response: %v", err)
+		return nil, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err)
 	}
 	if downstreamCtx.GetCommonTlsContext() == nil {
-		return nil, errors.New("xds: DownstreamTlsContext in LDS response does not contain a CommonTlsContext")
+		return nil, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext")
 	}
 	sc, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext())
 	if err != nil {
@@ -321,28 +325,30 @@ func getAddressFromName(name string) (host string, port string, err error) {
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
 // configuration resources of interest.
-func UnmarshalRouteConfig(_ string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, UpdateMetadata, error) {
+func UnmarshalRouteConfig(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, UpdateMetadata, error) {
 	update := make(map[string]RouteConfigUpdate)
-	for _, r := range resources {
-		if !IsRouteConfigResource(r.GetTypeUrl()) {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in RDS response", r.GetTypeUrl())
-		}
-		rc := &v3routepb.RouteConfiguration{}
-		if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err)
-		}
-		logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, rc)
+	md, err := processAllResources(version, resources, logger, update)
+	return update, md, err
+}
 
-		// TODO: Pass version.TransportAPI instead of relying upon the type URL
-		v2 := r.GetTypeUrl() == version.V2RouteConfigURL
-		// Use the hostname (resourceName for LDS) to find the routes.
-		u, err := generateRDSUpdateFromRouteConfiguration(rc, logger, v2)
-		if err != nil {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err)
-		}
-		update[rc.GetName()] = u
+func unmarshalRouteConfigResource(r *anypb.Any, logger *grpclog.PrefixLogger) (string, RouteConfigUpdate, error) {
+	if !IsRouteConfigResource(r.GetTypeUrl()) {
+		return "", RouteConfigUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
+	}
+	rc := &v3routepb.RouteConfiguration{}
+	if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
+		return "", RouteConfigUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
+	}
+	logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, rc)
+
+	// TODO: Pass version.TransportAPI instead of relying upon the type URL
+	v2 := r.GetTypeUrl() == version.V2RouteConfigURL
+	u, err := generateRDSUpdateFromRouteConfiguration(rc, logger, v2)
+	if err != nil {
+		return rc.GetName(), RouteConfigUpdate{}, err
 	}
-	return update, UpdateMetadata{}, nil
+	u.Raw = r
+	return rc.GetName(), u, nil
 }
 
 // generateRDSUpdateFromRouteConfiguration checks if the provided
@@ -522,41 +528,43 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger,
 // are interested in.
 func UnmarshalCluster(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ClusterUpdate, UpdateMetadata, error) {
 	update := make(map[string]ClusterUpdate)
-	for _, r := range resources {
-		if !IsClusterResource(r.GetTypeUrl()) {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in CDS response", r.GetTypeUrl())
-		}
+	md, err := processAllResources(version, resources, logger, update)
+	return update, md, err
+}
 
-		cluster := &v3clusterpb.Cluster{}
-		if err := proto.Unmarshal(r.GetValue(), cluster); err != nil {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in CDS response: %v", err)
-		}
-		logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
-		cu, err := validateCluster(cluster)
-		if err != nil {
-			return nil, UpdateMetadata{}, err
-		}
+func unmarshalClusterResource(r *anypb.Any, logger *grpclog.PrefixLogger) (string, ClusterUpdate, error) {
+	if !IsClusterResource(r.GetTypeUrl()) {
+		return "", ClusterUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
+	}
 
-		// If the Cluster message in the CDS response did not contain a
-		// serviceName, we will just use the clusterName for EDS.
-		if cu.ServiceName == "" {
-			cu.ServiceName = cluster.GetName()
-		}
-		logger.Debugf("Resource with name %v, value %+v added to cache", cluster.GetName(), cu)
-		update[cluster.GetName()] = cu
+	cluster := &v3clusterpb.Cluster{}
+	if err := proto.Unmarshal(r.GetValue(), cluster); err != nil {
+		return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
+	}
+	logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
+
+	cu, err := validateCluster(cluster)
+	if err != nil {
+		return cluster.GetName(), ClusterUpdate{}, err
+	}
+	cu.Raw = r
+	// If the Cluster message in the CDS response did not contain a
+	// serviceName, we will just use the clusterName for EDS.
+	if cu.ServiceName == "" {
+		cu.ServiceName = cluster.GetName()
 	}
-	return update, UpdateMetadata{}, nil
+	return cluster.GetName(), cu, nil
 }
 
 func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
 	switch {
 	case cluster.GetType() != v3clusterpb.Cluster_EDS:
-		return emptyUpdate, fmt.Errorf("xds: unexpected cluster type %v in response: %+v", cluster.GetType(), cluster)
+		return emptyUpdate, fmt.Errorf("unexpected cluster type %v in response: %+v", cluster.GetType(), cluster)
 	case cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil:
-		return emptyUpdate, fmt.Errorf("xds: unexpected edsConfig in response: %+v", cluster)
+		return emptyUpdate, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
 	case cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN:
-		return emptyUpdate, fmt.Errorf("xds: unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
+		return emptyUpdate, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
 
 	sc, err := securityConfigFromCluster(cluster)
@@ -582,18 +590,18 @@ func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, e
 		return nil, nil
 	}
 	if name := ts.GetName(); name != transportSocketName {
-		return nil, fmt.Errorf("xds: transport_socket field has unexpected name: %s", name)
+		return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name)
 	}
 	any := ts.GetTypedConfig()
 	if any == nil || any.TypeUrl != version.V3UpstreamTLSContextURL {
-		return nil, fmt.Errorf("xds: transport_socket field has unexpected typeURL: %s", any.TypeUrl)
+		return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl)
 	}
 	upstreamCtx := &v3tlspb.UpstreamTlsContext{}
 	if err := proto.Unmarshal(any.GetValue(), upstreamCtx); err != nil {
-		return nil, fmt.Errorf("xds: failed to unmarshal UpstreamTlsContext in CDS response: %v", err)
+		return nil, fmt.Errorf("failed to unmarshal UpstreamTlsContext in CDS response: %v", err)
 	}
 	if upstreamCtx.GetCommonTlsContext() == nil {
-		return nil, errors.New("xds: UpstreamTlsContext in CDS response does not contain a CommonTlsContext")
+		return nil, errors.New("UpstreamTlsContext in CDS response does not contain a CommonTlsContext")
 	}
 
 	sc, err := securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext())
@@ -649,7 +657,7 @@ func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext) (*Secu
 	case nil:
 		// It is valid for the validation context to be nil on the server side.
 	default:
-		return nil, fmt.Errorf("xds: validation context contains unexpected type: %T", t)
+		return nil, fmt.Errorf("validation context contains unexpected type: %T", t)
 	}
 	return sc, nil
 }
@@ -680,24 +688,27 @@ func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 {
 // fields we are interested in.
 func UnmarshalEndpoints(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]EndpointsUpdate, UpdateMetadata, error) {
 	update := make(map[string]EndpointsUpdate)
-	for _, r := range resources {
-		if !IsEndpointsResource(r.GetTypeUrl()) {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: unexpected resource type: %q in EDS response", r.GetTypeUrl())
-		}
+	md, err := processAllResources(version, resources, logger, update)
+	return update, md, err
+}
 
-		cla := &v3endpointpb.ClusterLoadAssignment{}
-		if err := proto.Unmarshal(r.GetValue(), cla); err != nil {
-			return nil, UpdateMetadata{}, fmt.Errorf("xds: failed to unmarshal resource in EDS response: %v", err)
-		}
-		logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
+func unmarshalEndpointsResource(r *anypb.Any, logger *grpclog.PrefixLogger) (string, EndpointsUpdate, error) {
+	if !IsEndpointsResource(r.GetTypeUrl()) {
+		return "", EndpointsUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
+	}
 
-		u, err := parseEDSRespProto(cla)
-		if err != nil {
-			return nil, UpdateMetadata{}, err
-		}
-		update[cla.GetClusterName()] = u
+	cla := &v3endpointpb.ClusterLoadAssignment{}
+	if err := proto.Unmarshal(r.GetValue(), cla); err != nil {
+		return "", EndpointsUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
 	}
-	return update, UpdateMetadata{}, nil
+	logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
+
+	u, err := parseEDSRespProto(cla)
+	if err != nil {
+		return cla.GetClusterName(), EndpointsUpdate{}, err
+	}
+	u.Raw = r
+	return cla.GetClusterName(), u, nil
 }
 
 func parseAddress(socketAddress *v3corepb.SocketAddress) string {
@@ -769,3 +780,130 @@ func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate,
 	}
 	return ret, nil
 }
+
+// processAllResources unmarshals and validates the resources, populates the
+// provided ret (a map), and returns metadata and error.
+//
+// The type of the resource is determined by the type of ret. E.g.
+// map[string]ListenerUpdate means this is for LDS.
+func processAllResources(version string, resources []*anypb.Any, logger *grpclog.PrefixLogger, ret interface{}) (UpdateMetadata, error) {
+	timestamp := time.Now()
+	md := UpdateMetadata{
+		Version:   version,
+		Timestamp: timestamp,
+	}
+	var topLevelErrors []error
+	perResourceErrors := make(map[string]error)
+
+	for _, r := range resources {
+		switch ret2 := ret.(type) {
+		case map[string]ListenerUpdate:
+			name, update, err := unmarshalListenerResource(r, logger)
+			if err == nil {
+				ret2[name] = update
+				continue
+			}
+			if name == "" {
+				topLevelErrors = append(topLevelErrors, err)
+				continue
+			}
+			perResourceErrors[name] = err
+			// Add place holder in the map so we know this resource name was in
+			// the response.
+			ret2[name] = ListenerUpdate{}
+		case map[string]RouteConfigUpdate:
+			name, update, err := unmarshalRouteConfigResource(r, logger)
+			if err == nil {
+				ret2[name] = update
+				continue
+			}
+			if name == "" {
+				topLevelErrors = append(topLevelErrors, err)
+				continue
+			}
+			perResourceErrors[name] = err
+			// Add place holder in the map so we know this resource name was in
+			// the response.
+			ret2[name] = RouteConfigUpdate{}
+		case map[string]ClusterUpdate:
+			name, update, err := unmarshalClusterResource(r, logger)
+			if err == nil {
+				ret2[name] = update
+				continue
+			}
+			if name == "" {
+				topLevelErrors = append(topLevelErrors, err)
+				continue
+			}
+			perResourceErrors[name] = err
+			// Add place holder in the map so we know this resource name was in
+			// the response.
+			ret2[name] = ClusterUpdate{}
+		case map[string]EndpointsUpdate:
+			name, update, err := unmarshalEndpointsResource(r, logger)
+			if err == nil {
+				ret2[name] = update
+				continue
+			}
+			if name == "" {
+				topLevelErrors = append(topLevelErrors, err)
+				continue
+			}
+			perResourceErrors[name] = err
+			// Add place holder in the map so we know this resource name was in
+			// the response.
+			ret2[name] = EndpointsUpdate{}
+		}
+	}
+
+	if len(topLevelErrors) == 0 && len(perResourceErrors) == 0 {
+		md.Status = ServiceStatusACKed
+		return md, nil
+	}
+
+	var typeStr string
+	switch ret.(type) {
+	case map[string]ListenerUpdate:
+		typeStr = "LDS"
+	case map[string]RouteConfigUpdate:
+		typeStr = "RDS"
+	case map[string]ClusterUpdate:
+		typeStr = "CDS"
+	case map[string]EndpointsUpdate:
+		typeStr = "EDS"
+	}
+
+	md.Status = ServiceStatusNACKed
+	errRet := combineErrors(typeStr, topLevelErrors, perResourceErrors)
+	md.ErrState = &UpdateErrorMetadata{
+		Version:   version,
+		Err:       errRet,
+		Timestamp: timestamp,
+	}
+	return md, errRet
+}
+
+func combineErrors(rType string, topLevelErrors []error, perResourceErrors map[string]error) error {
+	var errStrB strings.Builder
+	errStrB.WriteString(fmt.Sprintf("error parsing %q response: ", rType))
+	if len(topLevelErrors) > 0 {
+		errStrB.WriteString("top level errors: ")
+		for i, err := range topLevelErrors {
+			if i != 0 {
+				errStrB.WriteString(";\n")
+			}
+			errStrB.WriteString(err.Error())
+		}
+	}
+	if len(perResourceErrors) > 0 {
+		var i int
+		for name, err := range perResourceErrors {
+			if i != 0 {
+				errStrB.WriteString(";\n")
+			}
+			i++
+			errStrB.WriteString(fmt.Sprintf("resource %q: %v", name, err.Error()))
+		}
+	}
+	return errors.New(errStrB.String())
+}

From 61f0b5fa7c1c375e2bbf29f2f5f8610d2b5ba956 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 5 Mar 2021 13:31:34 -0800
Subject: [PATCH 376/481] client: implement proper config selector interceptors
 (#4235)

---
 internal/resolver/config_selector.go       | 75 ++++++++++++++++------
 stream.go                                  | 24 +++++--
 xds/internal/resolver/serviceconfig.go     | 17 ++---
 xds/internal/resolver/xds_resolver_test.go | 31 +++++----
 4 files changed, 99 insertions(+), 48 deletions(-)

diff --git a/internal/resolver/config_selector.go b/internal/resolver/config_selector.go
index 1b20c7592e33..5e7f36703d4b 100644
--- a/internal/resolver/config_selector.go
+++ b/internal/resolver/config_selector.go
@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -54,30 +55,66 @@ type RPCConfig struct {
 	Interceptor  ClientInterceptor
 }
 
-// ClientStream will ultimately be a superset of grpc.ClientStream as
-// operations become necessary to support.
+// ClientStream is the same as grpc.ClientStream, but defined here for circular
+// dependency reasons.
 type ClientStream interface {
-	// Done is invoked when the RPC is finished using its connection, or could
-	// not be assigned a connection.  RPC operations may still occur on
-	// ClientStream after done is called, since the interceptor is invoked by
-	// application-layer operations.
-	Done()
+	// Header returns the header metadata received from the server if there
+	// is any. It blocks if the metadata is not ready to read.
+	Header() (metadata.MD, error)
+	// Trailer returns the trailer metadata from the server, if there is any.
+	// It must only be called after stream.CloseAndRecv has returned, or
+	// stream.Recv has returned a non-nil error (including io.EOF).
+	Trailer() metadata.MD
+	// CloseSend closes the send direction of the stream. It closes the stream
+	// when non-nil error is met. It is also not safe to call CloseSend
+	// concurrently with SendMsg.
+	CloseSend() error
+	// Context returns the context for this stream.
+	//
+	// It should not be called until after Header or RecvMsg has returned. Once
+	// called, subsequent client-side retries are disabled.
+	Context() context.Context
+	// SendMsg is generally called by generated code. On error, SendMsg aborts
+	// the stream. If the error was generated by the client, the status is
+	// returned directly; otherwise, io.EOF is returned and the status of
+	// the stream may be discovered using RecvMsg.
+	//
+	// SendMsg blocks until:
+	//   - There is sufficient flow control to schedule m with the transport, or
+	//   - The stream is done, or
+	//   - The stream breaks.
+	//
+	// SendMsg does not wait until the message is received by the server. An
+	// untimely stream closure may result in lost messages. To ensure delivery,
+	// users should ensure the RPC completed successfully using RecvMsg.
+	//
+	// It is safe to have a goroutine calling SendMsg and another goroutine
+	// calling RecvMsg on the same stream at the same time, but it is not safe
+	// to call SendMsg on the same stream in different goroutines. It is also
+	// not safe to call CloseSend concurrently with SendMsg.
+	SendMsg(m interface{}) error
+	// RecvMsg blocks until it receives a message into m or the stream is
+	// done. It returns io.EOF when the stream completes successfully. On
+	// any other error, the stream is aborted and the error contains the RPC
+	// status.
+	//
+	// It is safe to have a goroutine calling SendMsg and another goroutine
+	// calling RecvMsg on the same stream at the same time, but it is not
+	// safe to call RecvMsg on the same stream in different goroutines.
+	RecvMsg(m interface{}) error
 }
 
-// NOPClientStream is a ClientStream that does nothing
-type NOPClientStream struct{}
-
-// Done is a nop.
-func (NOPClientStream) Done() {}
-
-var _ ClientStream = NOPClientStream{}
-
 // ClientInterceptor is an interceptor for gRPC client streams.
 type ClientInterceptor interface {
-	// NewStream can intercept ClientStream calls.  The provided ClientStream
-	// should not be used during NewStream.  RPCInfo.Context should not be used
-	// (will be nil).
-	NewStream(context.Context, RPCInfo, ClientStream) (context.Context, ClientStream, error)
+	// NewStream produces a ClientStream for an RPC which may optionally use
+	// the provided function to produce a stream for delegation.  Note:
+	// RPCInfo.Context should not be used (will be nil).
+	//
+	// done is invoked when the RPC is finished using its connection, or could
+	// not be assigned a connection.  RPC operations may still occur on
+	// ClientStream after done is called, since the interceptor is invoked by
+	// application-layer operations.  done must never be nil when called.
+	NewStream(ctx context.Context, ri RPCInfo, done func(), newStream func(ctx context.Context, done func()) (ClientStream, error)) (ClientStream, error)
 }
 
 // ServerInterceptor is unimplementable; do not use.
diff --git a/stream.go b/stream.go
index ebc74cd030fb..4a9fdb73875b 100644
--- a/stream.go
+++ b/stream.go
@@ -166,7 +166,6 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 			}
 		}()
 	}
-	c := defaultCallInfo()
 	// Provide an opportunity for the first RPC to see the first service config
 	// provided by the resolver.
 	if err := cc.waitForResolvedAddrs(ctx); err != nil {
@@ -175,12 +174,16 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 
 	var mc serviceconfig.MethodConfig
 	var onCommit func()
+	var newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
+		return newClientStreamWithParams(ctx, desc, cc, method, mc, onCommit, done, opts...)
+	}
+
 	rpcInfo := iresolver.RPCInfo{Context: ctx, Method: method}
 	rpcConfig, err := cc.safeConfigSelector.SelectConfig(rpcInfo)
 	if err != nil {
 		return nil, status.Convert(err).Err()
 	}
-	var doneFunc func()
+
 	if rpcConfig != nil {
 		if rpcConfig.Context != nil {
 			ctx = rpcConfig.Context
@@ -189,15 +192,22 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 		onCommit = rpcConfig.OnCommitted
 		if rpcConfig.Interceptor != nil {
 			rpcInfo.Context = nil
-			newCtx, cs, err := rpcConfig.Interceptor.NewStream(ctx, rpcInfo, iresolver.NOPClientStream{})
-			if err != nil {
-				return nil, status.Convert(err).Err()
+			ns := newStream
+			newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
+				cs, err := rpcConfig.Interceptor.NewStream(ctx, rpcInfo, done, ns)
+				if err != nil {
+					return nil, status.Convert(err).Err()
+				}
+				return cs, nil
 			}
-			ctx = newCtx
-			doneFunc = cs.Done
 		}
 	}
 
+	return newStream(ctx, func() {})
+}
+
+func newClientStreamWithParams(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, mc serviceconfig.MethodConfig, onCommit, doneFunc func(), opts ...CallOption) (_ iresolver.ClientStream, err error) {
+	c := defaultCallInfo()
 	if mc.WaitForReady != nil {
 		c.failFast = !*mc.WaitForReady
 	}
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 95025fc8a423..7c1ec853e4cd 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -314,16 +314,13 @@ type interceptorList struct {
 	interceptors []iresolver.ClientInterceptor
 }
 
-func (il *interceptorList) NewStream(ctx context.Context, ri iresolver.RPCInfo, cs iresolver.ClientStream) (context.Context, iresolver.ClientStream, error) {
-	for _, i := range il.interceptors {
-		var err error
-		newCTX, newCS, err := i.NewStream(ctx, ri, cs)
-		if err != nil {
-			cs.Done()
-			return nil, nil, err
+func (il *interceptorList) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) {
+	for i := len(il.interceptors) - 1; i >= 0; i-- {
+		ns := newStream
+		interceptor := il.interceptors[i]
+		newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
+			return interceptor.NewStream(ctx, ri, done, ns)
 		}
-		cs = newCS
-		ctx = newCTX
 	}
-	return ctx, cs, nil
+	return newStream(ctx, func() {})
 }
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index e800d433150f..53ea17042aa5 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -1057,25 +1057,28 @@ type filterInterceptor struct {
 	err  error
 }
 
-func (fi *filterInterceptor) NewStream(ctx context.Context, i iresolver.RPCInfo, cs iresolver.ClientStream) (context.Context, iresolver.ClientStream, error) {
+func (fi *filterInterceptor) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) {
 	*fi.path = append(*fi.path, "newstream:"+fi.s)
 	if fi.err != nil {
-		return nil, nil, fi.err
+		return nil, fi.err
 	}
-	return ctx, &clientStream{cs: cs, path: fi.path, s: fi.s}, nil
+	d := func() {
+		*fi.path = append(*fi.path, "done:"+fi.s)
+		done()
+	}
+	cs, err := newStream(ctx, d)
+	if err != nil {
+		return nil, err
+	}
+	return &clientStream{ClientStream: cs, path: fi.path, s: fi.s}, nil
 }
 
 type clientStream struct {
-	cs   iresolver.ClientStream
+	iresolver.ClientStream
 	path *[]string
 	s    string
 }
 
-func (cs *clientStream) Done() {
-	*cs.path = append(*cs.path, "done:"+cs.s)
-	cs.cs.Done()
-}
-
 type filterCfg struct {
 	httpfilter.FilterConfig
 	s            string
@@ -1249,14 +1252,18 @@ func (s) TestXDSResolverHTTPFilters(t *testing.T) {
 					if err != nil {
 						t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err)
 					}
-					_, cs, err := res.Interceptor.NewStream(context.Background(), iresolver.RPCInfo{}, iresolver.NOPClientStream{})
+					var doneFunc func()
+					_, err = res.Interceptor.NewStream(context.Background(), iresolver.RPCInfo{}, func() {}, func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
+						doneFunc = done
+						return nil, nil
+					})
 					if tc.newStreamErr != "" {
 						if err == nil || !strings.Contains(err.Error(), tc.newStreamErr) {
 							t.Errorf("NewStream(...) = _, %v; want _, Contains(%v)", err, tc.newStreamErr)
 						}
 						if err == nil {
 							res.OnCommitted()
-							cs.Done()
+							doneFunc()
 						}
 						continue
 					}
@@ -1265,7 +1272,7 @@ func (s) TestXDSResolverHTTPFilters(t *testing.T) {
 
 					}
 					res.OnCommitted()
-					cs.Done()
+					doneFunc()
 
 					// Confirm the desired path is found in remainingWant, and remove it.
 					pass := false

From 97798356d2c5c455d63614f2cd9ac3efbcd146dc Mon Sep 17 00:00:00 2001
From: Zach Reyes <39203661+zasweq@users.noreply.github.com>
Date: Mon, 8 Mar 2021 11:33:57 -0500
Subject: [PATCH 377/481] Added check for POST method in http2_server (#4241)

* Added check for POST method in http2_server
---
 internal/transport/http2_server.go   | 15 ++++++
 internal/transport/http_util.go      |  3 ++
 internal/transport/transport_test.go | 78 ++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+)

diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index 0cf1cc320bb2..7c6c89d4f9b2 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -26,6 +26,7 @@ import (
 	"io"
 	"math"
 	"net"
+	"net/http"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -402,6 +403,20 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 		return true
 	}
 	t.maxStreamID = streamID
+	if state.data.httpMethod != http.MethodPost {
+		t.mu.Unlock()
+		if logger.V(logLevel) {
+			logger.Warningf("transport: http2Server.operateHeaders parsed a :method field: %v which should be POST", state.data.httpMethod)
+		}
+		t.controlBuf.put(&cleanupStream{
+			streamID: streamID,
+			rst:      true,
+			rstCode:  http2.ErrCodeProtocol,
+			onWrite:  func() {},
+		})
+		s.cancel()
+		return false
+	}
 	t.activeStreams[streamID] = s
 	if len(t.activeStreams) == 1 {
 		t.idle = time.Time{}
diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go
index 7e41d1183f93..c7dee140cf1a 100644
--- a/internal/transport/http_util.go
+++ b/internal/transport/http_util.go
@@ -111,6 +111,7 @@ type parsedHeaderData struct {
 	timeoutSet bool
 	timeout    time.Duration
 	method     string
+	httpMethod string
 	// key-value metadata map from the peer.
 	mdata          map[string][]string
 	statsTags      []byte
@@ -363,6 +364,8 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) {
 		}
 		d.data.statsTrace = v
 		d.addMetadata(f.Name, string(v))
+	case ":method":
+		d.data.httpMethod = f.Value
 	default:
 		if isReservedHeader(f.Name) && !isWhitelistedHeader(f.Name) {
 			break
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 0058df0d806f..1d8d3ed355df 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -1663,6 +1663,84 @@ func (s) TestReadGivesSameErrorAfterAnyErrorOccurs(t *testing.T) {
 	}
 }
 
+// If the client sends an HTTP/2 request with a :method header with a value other than POST, as specified in
+// the gRPC over HTTP/2 specification, the server should close the stream.
+func (s) TestServerWithClientSendingWrongMethod(t *testing.T) {
+	server := setUpServerOnly(t, 0, &ServerConfig{}, suspended)
+	defer server.stop()
+	// Create a client directly to not couple what you can send to API of http2_client.go.
+	mconn, err := net.Dial("tcp", server.lis.Addr().String())
+	if err != nil {
+		t.Fatalf("Client failed to dial: %v", err)
+	}
+	defer mconn.Close()
+
+	if n, err := mconn.Write(clientPreface); err != nil || n != len(clientPreface) {
+		t.Fatalf("mconn.Write(clientPreface) = %d, %v, want %d, <nil>", n, err, len(clientPreface))
+	}
+
+	framer := http2.NewFramer(mconn, mconn)
+	if err := framer.WriteSettings(); err != nil {
+		t.Fatalf("Error while writing settings: %v", err)
+	}
+
+	// success chan indicates that reader received a RSTStream from server.
+	// An error will be passed on it if any other frame is received.
+	success := testutils.NewChannel()
+
+	// Launch a reader goroutine.
+	go func() {
+		for {
+			frame, err := framer.ReadFrame()
+			if err != nil {
+				return
+			}
+			switch frame := frame.(type) {
+			case *http2.SettingsFrame:
+				// Do nothing. A settings frame is expected from server preface.
+			case *http2.RSTStreamFrame:
+				if frame.Header().StreamID != 1 || http2.ErrCode(frame.ErrCode) != http2.ErrCodeProtocol {
+					// Client only created a single stream, so RST Stream should be for that single stream.
+					t.Errorf("RST stream received with streamID: %d and code %v, want streamID: 1 and code: http.ErrCodeFlowControl", frame.Header().StreamID, http2.ErrCode(frame.ErrCode))
+				}
+				// Records that client successfully received RST Stream frame.
+				success.Send(nil)
+				return
+			default:
+				// The server should send nothing but a single RST Stream frame.
+				success.Send(errors.New("The client received a frame other than RST Stream"))
+			}
+		}
+	}()
+
+	// Done with HTTP/2 setup - now create a stream with a bad method header.
+	var buf bytes.Buffer
+	henc := hpack.NewEncoder(&buf)
+	// Method is required to be POST in a gRPC call.
+	if err := henc.WriteField(hpack.HeaderField{Name: ":method", Value: "PUT"}); err != nil {
+		t.Fatalf("Error while encoding header: %v", err)
+	}
+	// Have the rest of the headers be ok and within the gRPC over HTTP/2 spec.
+	if err := henc.WriteField(hpack.HeaderField{Name: ":path", Value: "foo"}); err != nil {
+		t.Fatalf("Error while encoding header: %v", err)
+	}
+	if err := henc.WriteField(hpack.HeaderField{Name: ":authority", Value: "localhost"}); err != nil {
+		t.Fatalf("Error while encoding header: %v", err)
+	}
+	if err := henc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}); err != nil {
+		t.Fatalf("Error while encoding header: %v", err)
+	}
+
+	if err := framer.WriteHeaders(http2.HeadersFrameParam{StreamID: 1, BlockFragment: buf.Bytes(), EndHeaders: true}); err != nil {
+		t.Fatalf("Error while writing headers: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if e, err := success.Receive(ctx); e != nil || err != nil {
+		t.Fatalf("Error in frame server should send: %v. Error receiving from channel: %v", e, err)
+	}
+}
+
 func (s) TestPingPong1B(t *testing.T) {
 	runPingPongTest(t, 1)
 }

From 19c44c2e0f8e9c5c60b8f703bd56653ac164c168 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 8 Mar 2021 08:44:30 -0800
Subject: [PATCH 378/481] xds: add env var protection for client-side security
 (#4247)

---
 xds/internal/client/cds_test.go | 58 +++++++++++++++++++++++++++++++++
 xds/internal/client/xds.go      | 12 +++++--
 xds/internal/env/env.go         | 16 ++++++---
 3 files changed, 79 insertions(+), 7 deletions(-)

diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 6ad0b0fa88a0..c5f1d76d32c3 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -223,7 +223,65 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 	}
 }
 
+func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
+	// Turn off the env var protection for client-side security.
+	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
+	env.ClientSideSecuritySupport = false
+	defer func() { env.ClientSideSecuritySupport = origClientSideSecurityEnvVar }()
+
+	cluster := &v3clusterpb.Cluster{
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: serviceName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.tls",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: &anypb.Any{
+					TypeUrl: version.V3UpstreamTLSContextURL,
+					Value: func() []byte {
+						tls := &v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "rootInstance",
+										CertificateName: "rootCert",
+									},
+								},
+							},
+						}
+						mtls, _ := proto.Marshal(tls)
+						return mtls
+					}(),
+				},
+			},
+		},
+	}
+	wantUpdate := ClusterUpdate{
+		ServiceName: serviceName,
+		EnableLRS:   false,
+	}
+	gotUpdate, err := validateCluster(cluster)
+	if err != nil {
+		t.Errorf("validateCluster() failed: %v", err)
+	}
+	if diff := cmp.Diff(wantUpdate, gotUpdate); diff != "" {
+		t.Errorf("validateCluster() returned unexpected diff (-want, got):\n%s", diff)
+	}
+}
+
 func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
+	// Turn on the env var protection for client-side security.
+	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
+	env.ClientSideSecuritySupport = true
+	defer func() { env.ClientSideSecuritySupport = origClientSideSecurityEnvVar }()
+
 	const (
 		identityPluginInstance = "identityPluginInstance"
 		identityCertName       = "identityCert"
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 8fc50ea3056c..78dc139e8feb 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -567,10 +567,16 @@ func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 		return emptyUpdate, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
 
-	sc, err := securityConfigFromCluster(cluster)
-	if err != nil {
-		return emptyUpdate, err
+	// Process security configuration received from the control plane iff the
+	// corresponding environment variable is set.
+	var sc *SecurityConfig
+	if env.ClientSideSecuritySupport {
+		var err error
+		if sc, err = securityConfigFromCluster(cluster); err != nil {
+			return emptyUpdate, err
+		}
 	}
+
 	return ClusterUpdate{
 		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
 		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index 065ce6145638..a28d741f3568 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -37,10 +37,11 @@ const (
 	// and kept in variable BootstrapFileName.
 	//
 	// When both bootstrap FileName and FileContent are set, FileName is used.
-	BootstrapFileContentEnv   = "GRPC_XDS_BOOTSTRAP_CONFIG"
-	circuitBreakingSupportEnv = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
-	timeoutSupportEnv         = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
-	faultInjectionSupportEnv  = "GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION"
+	BootstrapFileContentEnv      = "GRPC_XDS_BOOTSTRAP_CONFIG"
+	circuitBreakingSupportEnv    = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
+	timeoutSupportEnv            = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
+	faultInjectionSupportEnv     = "GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION"
+	clientSideSecuritySupportEnv = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"
 )
 
 var (
@@ -67,4 +68,11 @@ var (
 	// FaultInjectionSupport is used to control both fault injection and HTTP
 	// filter support.
 	FaultInjectionSupport = strings.EqualFold(os.Getenv(faultInjectionSupportEnv), "true")
+	// ClientSideSecuritySupport is used to control processing of security
+	// configuration on the client-side.
+	//
+	// Note that there is no env var protection for the server-side because we
+	// have a brand new API on the server-side and users explicitly need to use
+	// the new API to get security integration on the server.
+	ClientSideSecuritySupport = strings.EqualFold(os.Getenv(clientSideSecuritySupportEnv), "true")
 )

From fce74a94bdffb3c990969e32720bf4a131e4f787 Mon Sep 17 00:00:00 2001
From: Fagner Nunes Carvalho <fagnerncarvalho@gmail.com>
Date: Mon, 8 Mar 2021 21:37:15 -0300
Subject: [PATCH 379/481] cleanup: fix typo in Interceptor README (#4249)

---
 examples/features/interceptor/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/features/interceptor/README.md b/examples/features/interceptor/README.md
index 8498fbe11a1c..0f1908be590a 100644
--- a/examples/features/interceptor/README.md
+++ b/examples/features/interceptor/README.md
@@ -113,6 +113,6 @@ handler StreamHandler) error`.
 Refer to client-side stream interceptor section for detailed implementation
 explanation.
 
-To install the unary interceptor for a Server, configure `NewServer` with
+To install the stream interceptor for a Server, configure `NewServer` with
 `ServerOption`
 [`StreamInterceptor`](https://godoc.org/google.golang.org/grpc#StreamInterceptor).

From d5b628860d4e17d106f1964097c0a297659e8110 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 9 Mar 2021 14:03:51 -0800
Subject: [PATCH 380/481] vet: set PATH to HOME/go/bin, and print go version
 (#4254)

---
 .github/workflows/testing.yml | 10 ++++++++--
 vet.sh                        |  3 ++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 91c5d452e875..378e2846676f 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -78,17 +78,22 @@ jobs:
       # detector and Go1.11 (where we run a reduced set of tests).
       - name: Run tests
         if: ${{ matrix.type != 'extras' && matrix.type != 'race' && matrix.type != 'tests111' }}
-        run: go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
+        run: |
+          go version
+          go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Race detector tests
       - name: Run test race
         if: ${{ matrix.TYPE == 'race' }}
-        run: go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
+        run: |
+          go version
+          go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Non-core gRPC tests (examples, interop, etc)
       - name: Run extras tests
         if: ${{ matrix.TYPE == 'extras' }}
         run: |
+          go version
           examples/examples_test.sh
           security/advancedtls/examples/examples_test.sh
           interop/interop_test.sh
@@ -99,6 +104,7 @@ jobs:
       - name: Run Go1.11 tests
         if: ${{ matrix.type == 'tests111' }}
         run: |
+          go version
           tests=$(find ${GITHUB_WORKSPACE} -name '*_test.go' | xargs -n1 dirname | sort -u | sed "s:^${GITHUB_WORKSPACE}:.:" | sed "s:\/$::" | grep -v ^./security | grep -v ^./credentials/sts | grep -v ^./credentials/tls/certprovider | grep -v ^./credentials/xds | grep -v ^./xds )
           echo "Running tests for " ${tests}
           go test -cpu 1,4 -timeout 7m ${tests}
diff --git a/vet.sh b/vet.sh
index 85b32bb6dd5e..dcd939bb3907 100755
--- a/vet.sh
+++ b/vet.sh
@@ -28,7 +28,8 @@ cleanup() {
 }
 trap cleanup EXIT
 
-PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}"
+PATH="${HOME}/go/bin:${GOROOT}/bin:${PATH}"
+go version
 
 if [[ "$1" = "-install" ]]; then
   # Check for module support

From a45f13b160731da4f8b356e0449e9754312e5f1a Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 10 Mar 2021 09:26:23 -0800
Subject: [PATCH 381/481] xds: Support server_listener_resource_name_template 
 (#4233)

---
 xds/internal/client/bootstrap/bootstrap.go    |  16 +-
 .../client/bootstrap/bootstrap_test.go        |  28 +--
 xds/internal/client/client.go                 |  13 ++
 xds/internal/client/lds_test.go               | 123 ++---------
 xds/internal/client/xds.go                    |  30 +--
 .../test/xds_client_integration_test.go       |   7 +-
 .../test/xds_server_integration_test.go       |  17 +-
 xds/internal/testutils/e2e/bootstrap.go       |  16 +-
 xds/server.go                                 |  56 ++++-
 xds/server_test.go                            | 191 +++++++++++++-----
 10 files changed, 263 insertions(+), 234 deletions(-)

diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index c48bb797f36a..fab93f50f4eb 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -79,10 +79,12 @@ type Config struct {
 	// CertProviderConfigs contains a mapping from certificate provider plugin
 	// instance names to parsed buildable configs.
 	CertProviderConfigs map[string]*certprovider.BuildableConfig
-	// ServerResourceNameID contains the value to be used as the id in the
-	// resource name used to fetch the Listener resource on the xDS-enabled gRPC
-	// server.
-	ServerResourceNameID string
+	// ServerListenerResourceNameTemplate is a template for the name of the
+	// Listener resource to subscribe to for a gRPC server. If the token `%s` is
+	// present in the string, it will be replaced with the server's listening
+	// "IP:port" (e.g., "0.0.0.0:8080", "[::]:8080"). For example, a value of
+	// "example/resource/%s" could become "example/resource/0.0.0.0:8080".
+	ServerListenerResourceNameTemplate string
 }
 
 type channelCreds struct {
@@ -145,7 +147,7 @@ func bootstrapConfigFromEnvVariable() ([]byte, error) {
 //        "config": { foo plugin config in JSON }
 //      }
 //    },
-//    "grpc_server_resource_name_id": "grpc/server"
+//    "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s"
 // }
 //
 // Currently, we support exactly one type of credential, which is
@@ -241,8 +243,8 @@ func NewConfig() (*Config, error) {
 				configs[instance] = bc
 			}
 			config.CertProviderConfigs = configs
-		case "grpc_server_resource_name_id":
-			if err := json.Unmarshal(v, &config.ServerResourceNameID); err != nil {
+		case "server_listener_resource_name_template":
+			if err := json.Unmarshal(v, &config.ServerListenerResourceNameTemplate); err != nil {
 				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
 			}
 		}
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index e881594b8770..bdba74099c71 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -240,8 +240,8 @@ func (c *Config) compare(want *Config) error {
 	if diff := cmp.Diff(want.NodeProto, c.NodeProto, cmp.Comparer(proto.Equal)); diff != "" {
 		return fmt.Errorf("config.NodeProto diff (-want, +got):\n%s", diff)
 	}
-	if c.ServerResourceNameID != want.ServerResourceNameID {
-		return fmt.Errorf("config.ServerResourceNameID is %q, want %q", c.ServerResourceNameID, want.ServerResourceNameID)
+	if c.ServerListenerResourceNameTemplate != want.ServerListenerResourceNameTemplate {
+		return fmt.Errorf("config.ServerListenerResourceNameTemplate is %q, want %q", c.ServerListenerResourceNameTemplate, want.ServerListenerResourceNameTemplate)
 	}
 
 	// A vanilla cmp.Equal or cmp.Diff will not produce useful error message
@@ -711,9 +711,9 @@ func TestNewConfigWithCertificateProviders(t *testing.T) {
 	}
 }
 
-func TestNewConfigWithServerResourceNameID(t *testing.T) {
+func TestNewConfigWithServerListenerResourceNameTemplate(t *testing.T) {
 	cancel := setupBootstrapOverride(map[string]string{
-		"badServerResourceNameID": `
+		"badServerListenerResourceNameTemplate:": `
 		{
 			"node": {
 				"id": "ENVOY_NODE_ID",
@@ -727,9 +727,9 @@ func TestNewConfigWithServerResourceNameID(t *testing.T) {
 					{ "type": "google_default" }
 				]
 			}],
-			"grpc_server_resource_name_id": 123456789
+			"server_listener_resource_name_template": 123456789
 		}`,
-		"goodServerResourceNameID": `
+		"goodServerListenerResourceNameTemplate": `
 		{
 			"node": {
 				"id": "ENVOY_NODE_ID",
@@ -743,7 +743,7 @@ func TestNewConfigWithServerResourceNameID(t *testing.T) {
 					{ "type": "google_default" }
 				]
 			}],
-			"grpc_server_resource_name_id": "grpc/server"
+			"server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s"
 		}`,
 	})
 	defer cancel()
@@ -754,17 +754,17 @@ func TestNewConfigWithServerResourceNameID(t *testing.T) {
 		wantErr    bool
 	}{
 		{
-			name:    "badServerResourceNameID",
+			name:    "badServerListenerResourceNameTemplate",
 			wantErr: true,
 		},
 		{
-			name: "goodServerResourceNameID",
+			name: "goodServerListenerResourceNameTemplate",
 			wantConfig: &Config{
-				BalancerName:         "trafficdirector.googleapis.com:443",
-				Creds:                grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
-				TransportAPI:         version.TransportV2,
-				NodeProto:            v2NodeProto,
-				ServerResourceNameID: "grpc/server",
+				BalancerName:                       "trafficdirector.googleapis.com:443",
+				Creds:                              grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
+				TransportAPI:                       version.TransportV2,
+				NodeProto:                          v2NodeProto,
+				ServerListenerResourceNameTemplate: "grpc/server?xds.resource.listening_address=%s",
 			},
 		},
 	}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 21881cc6eae6..a29ea2fe3a12 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -203,6 +203,8 @@ type ListenerUpdate struct {
 	// HTTPFilters is a list of HTTP filters (name, config) from the LDS
 	// response.
 	HTTPFilters []HTTPFilter
+	// InboundListenerCfg contains inbound listener configuration.
+	InboundListenerCfg *InboundListenerConfig
 
 	// Raw is the resource from the xds response.
 	Raw *anypb.Any
@@ -221,6 +223,17 @@ type HTTPFilter struct {
 	Config httpfilter.FilterConfig
 }
 
+// InboundListenerConfig contains information about the inbound listener, i.e
+// the server-side listener.
+type InboundListenerConfig struct {
+	// Address is the local address on which the inbound listener is expected to
+	// accept incoming connections.
+	Address string
+	// Port is the local port on which the inbound listener is expected to
+	// accept incoming connections.
+	Port string
+}
+
 func (lu *ListenerUpdate) String() string {
 	return fmt.Sprintf("{RouteConfigName: %q, SecurityConfig: %+v", lu.RouteConfigName, lu.SecurityCfg)
 }
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 26e79b78d133..c2a7ff4f4f83 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -725,7 +725,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 }
 
 func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
-	const v3LDSTarget = "grpc/server?udpa.resource.listening_address=0.0.0.0:9999"
+	const v3LDSTarget = "grpc/server?xds.resource.listening_address=0.0.0.0:9999"
 
 	var (
 		listenerEmptyTransportSocket = &anypb.Any{
@@ -912,111 +912,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			},
 			wantErr: "no socket_address field in LDS response",
 		},
-		{
-			name: "listener name does not match expected format",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: "foo",
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
-			wantUpdate: map[string]ListenerUpdate{"foo": {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "no host:port in name field of LDS response",
-		},
-		{
-			name: "host mismatch",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "1.2.3.4",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
-			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "socket_address host does not match the one in name",
-		},
-		{
-			name: "port mismatch",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 1234,
-										},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
-			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "socket_address port does not match the one in name",
-		},
 		{
 			name: "unexpected number of filter chains",
 			resources: []*anypb.Any{
@@ -1314,7 +1209,13 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			name:      "empty transport socket",
 			resources: []*anypb.Any{listenerEmptyTransportSocket},
 			wantUpdate: map[string]ListenerUpdate{
-				v3LDSTarget: {Raw: listenerEmptyTransportSocket},
+				v3LDSTarget: {
+					InboundListenerCfg: &InboundListenerConfig{
+						Address: "0.0.0.0",
+						Port:    "9999",
+					},
+					Raw: listenerEmptyTransportSocket,
+				},
 			},
 			wantMD: UpdateMetadata{
 				Status:  ServiceStatusACKed,
@@ -1446,6 +1347,10 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						IdentityInstanceName: "identityPluginInstance",
 						IdentityCertName:     "identityCertName",
 					},
+					InboundListenerCfg: &InboundListenerConfig{
+						Address: "0.0.0.0",
+						Port:    "9999",
+					},
 					Raw: listenerNoValidationContext,
 				},
 			},
@@ -1466,6 +1371,10 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						IdentityCertName:     "identityCertName",
 						RequireClientCert:    true,
 					},
+					InboundListenerCfg: &InboundListenerConfig{
+						Address: "0.0.0.0",
+						Port:    "9999",
+					},
 					Raw: listenerWithValidationContext,
 				},
 			},
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 78dc139e8feb..63e7c2baed33 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -246,10 +246,6 @@ func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool) ([]HTTPFilt
 }
 
 func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
-	// Make sure that an address encoded in the received listener resource, and
-	// that it matches the one specified in the name. Listener names on the
-	// server-side as in the following format:
-	// grpc/server?udpa.resource.listening_address=IP:Port.
 	addr := lis.GetAddress()
 	if addr == nil {
 		return nil, fmt.Errorf("no address field in LDS response: %+v", lis)
@@ -258,15 +254,11 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	if sockAddr == nil {
 		return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis)
 	}
-	host, port, err := getAddressFromName(lis.GetName())
-	if err != nil {
-		return nil, fmt.Errorf("no host:port in name field of LDS response: %+v, error: %v", lis, err)
-	}
-	if h := sockAddr.GetAddress(); host != h {
-		return nil, fmt.Errorf("socket_address host does not match the one in name. Got %q, want %q", h, host)
-	}
-	if p := strconv.Itoa(int(sockAddr.GetPortValue())); port != p {
-		return nil, fmt.Errorf("socket_address port does not match the one in name. Got %q, want %q", p, port)
+	lu := &ListenerUpdate{
+		InboundListenerCfg: &InboundListenerConfig{
+			Address: sockAddr.GetAddress(),
+			Port:    strconv.Itoa(int(sockAddr.GetPortValue())),
+		},
 	}
 
 	// Make sure the listener resource contains a single filter chain. We do not
@@ -283,7 +275,7 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	// xdsCredentials.
 	ts := fc.GetTransportSocket()
 	if ts == nil {
-		return &ListenerUpdate{}, nil
+		return lu, nil
 	}
 	if name := ts.GetName(); name != transportSocketName {
 		return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name)
@@ -310,15 +302,9 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	if sc.RequireClientCert && sc.RootInstanceName == "" {
 		return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set")
 	}
-	return &ListenerUpdate{SecurityCfg: sc}, nil
-}
+	lu.SecurityCfg = sc
 
-func getAddressFromName(name string) (host string, port string, err error) {
-	parts := strings.SplitN(name, "udpa.resource.listening_address=", 2)
-	if len(parts) != 2 {
-		return "", "", fmt.Errorf("udpa.resource_listening_address not found in name: %v", name)
-	}
-	return net.SplitHostPort(parts[1])
+	return lu, nil
 }
 
 // UnmarshalRouteConfig processes resources received in an RDS response,
diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go
index c3ea71acaa35..f950a881e508 100644
--- a/xds/internal/test/xds_client_integration_test.go
+++ b/xds/internal/test/xds_client_integration_test.go
@@ -55,10 +55,9 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
 
 	// Create a bootstrap file in a temporary directory.
 	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:              e2e.TransportV3,
-		NodeID:               nodeID,
-		ServerURI:            fs.Address,
-		ServerResourceNameID: "grpc/server",
+		Version:   e2e.TransportV3,
+		NodeID:    nodeID,
+		ServerURI: fs.Address,
 	})
 	if err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 169daf19d263..3b9156347e8f 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -60,6 +60,9 @@ const (
 	certFile = "cert.pem"
 	keyFile  = "key.pem"
 	rootFile = "ca.pem"
+
+	// Template for server Listener resource name.
+	serverListenerResourceNameTemplate = "grpc/server?xds.resource.listening_address=%s"
 )
 
 func createTmpFile(t *testing.T, src, dst string) {
@@ -148,11 +151,11 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 
 	// Create a bootstrap file in a temporary directory.
 	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:              e2e.TransportV3,
-		NodeID:               nodeID,
-		ServerURI:            fs.Address,
-		CertificateProviders: cpc,
-		ServerResourceNameID: "grpc/server",
+		Version:                            e2e.TransportV3,
+		NodeID:                             nodeID,
+		ServerURI:                          fs.Address,
+		CertificateProviders:               cpc,
+		ServerListenerResourceNameTemplate: serverListenerResourceNameTemplate,
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -211,7 +214,7 @@ func listenerResourceWithoutSecurityConfig(t *testing.T, lis net.Listener) *v3li
 	host, port := hostPortFromListener(t, lis)
 	return &v3listenerpb.Listener{
 		// This needs to match the name we are querying for.
-		Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String()),
+		Name: fmt.Sprintf(serverListenerResourceNameTemplate, lis.Addr().String()),
 		Address: &v3corepb.Address{
 			Address: &v3corepb.Address_SocketAddress{
 				SocketAddress: &v3corepb.SocketAddress{
@@ -237,7 +240,7 @@ func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3liste
 	host, port := hostPortFromListener(t, lis)
 	return &v3listenerpb.Listener{
 		// This needs to match the name we are querying for.
-		Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", lis.Addr().String()),
+		Name: fmt.Sprintf(serverListenerResourceNameTemplate, lis.Addr().String()),
 		Address: &v3corepb.Address{
 			Address: &v3corepb.Address_SocketAddress{
 				SocketAddress: &v3corepb.SocketAddress{
diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go
index 25993f19fc38..c6fd2f9291fe 100644
--- a/xds/internal/testutils/e2e/bootstrap.go
+++ b/xds/internal/testutils/e2e/bootstrap.go
@@ -46,8 +46,8 @@ type BootstrapOptions struct {
 	NodeID string
 	// ServerURI is the address of the management server.
 	ServerURI string
-	// ServerResourceNameID is the Listener resource name to fetch.
-	ServerResourceNameID string
+	// ServerListenerResourceNameTemplate is the Listener resource name to fetch.
+	ServerListenerResourceNameTemplate string
 	// CertificateProviders is the certificate providers configuration.
 	CertificateProviders map[string]json.RawMessage
 }
@@ -79,8 +79,8 @@ func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
 		Node: node{
 			ID: opts.NodeID,
 		},
-		CertificateProviders:     opts.CertificateProviders,
-		GRPCServerResourceNameID: opts.ServerResourceNameID,
+		CertificateProviders:               opts.CertificateProviders,
+		ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate,
 	}
 	switch opts.Version {
 	case TransportV2:
@@ -127,10 +127,10 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) map[string]json.
 }
 
 type bootstrapConfig struct {
-	XdsServers               []server                   `json:"xds_servers,omitempty"`
-	Node                     node                       `json:"node,omitempty"`
-	CertificateProviders     map[string]json.RawMessage `json:"certificate_providers,omitempty"`
-	GRPCServerResourceNameID string                     `json:"grpc_server_resource_name_id,omitempty"`
+	XdsServers                         []server                   `json:"xds_servers,omitempty"`
+	Node                               node                       `json:"node,omitempty"`
+	CertificateProviders               map[string]json.RawMessage `json:"certificate_providers,omitempty"`
+	ServerListenerResourceNameTemplate string                     `json:"server_listener_resource_name_template,omitempty"`
 }
 
 type server struct {
diff --git a/xds/server.go b/xds/server.go
index f1c1e4181b8c..7e7c251bbe3d 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
@@ -220,17 +221,20 @@ func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, err
 	// or not.
 	goodUpdate := grpcsync.NewEvent()
 
-	// The resource_name in the LDS request sent by the xDS-enabled gRPC server
-	// is of the following format:
-	// "/path/to/resource?udpa.resource.listening_address=IP:Port". The
-	// `/path/to/resource` part of the name is sourced from the bootstrap config
-	// field `grpc_server_resource_name_id`. If this field is not specified in
-	// the bootstrap file, we will use a default of `grpc/server`.
-	path := "grpc/server"
-	if cfg := s.xdsC.BootstrapConfig(); cfg != nil && cfg.ServerResourceNameID != "" {
-		path = cfg.ServerResourceNameID
+	// The server listener resource name template from the bootstrap
+	// configuration contains a template for the name of the Listener resource
+	// to subscribe to for a gRPC server. If the token `%s` is present in the
+	// string, it will be replaced with the server's listening "IP:port" (e.g.,
+	// "0.0.0.0:8080", "[::]:8080"). The absence of a template will be treated
+	// as an error since we do not have any default value for this.
+	cfg := s.xdsC.BootstrapConfig()
+	if cfg == nil || cfg.ServerListenerResourceNameTemplate == "" {
+		return nil, errors.New("missing server_listener_resource_name_template in the bootstrap configuration")
+	}
+	name := cfg.ServerListenerResourceNameTemplate
+	if strings.Contains(cfg.ServerListenerResourceNameTemplate, "%s") {
+		name = strings.ReplaceAll(cfg.ServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	}
-	name := fmt.Sprintf("%s?udpa.resource.listening_address=%s", path, lis.Addr().String())
 
 	// Register an LDS watch using our xdsClient, and specify the listening
 	// address as the resource name.
@@ -288,6 +292,38 @@ func (s *GRPCServer) handleListenerUpdate(update listenerUpdate) {
 	}
 	s.logger.Infof("Received update for resource %q: %+v", update.name, update.lds.String())
 
+	// Make sure that the socket address on the received Listener resource
+	// matches the address of the net.Listener passed to us by the user. This
+	// check is done here instead of at the xdsClient layer because of the
+	// following couple of reasons:
+	// - xdsClient cannot know the listening address of every listener in the
+	//   system, and hence cannot perform this check.
+	// - this is a very context-dependent check and only the server has the
+	//   appropriate context to perform this check.
+	//
+	// What this means is that the xdsClient has ACKed a resource which is going
+	// to push the server into a "not serving" state. This is not ideal, but
+	// this is what we have decided to do. See gRPC A36 for more details.
+	// TODO(easwars): Switch to "not serving" if the host:port does not match.
+	lisAddr := update.lw.Listener.Addr().String()
+	addr, port, err := net.SplitHostPort(lisAddr)
+	if err != nil {
+		// This is never expected to return a non-nil error since we have made
+		// sure that the listener is a TCP listener at the beginning of Serve().
+		// This is simply paranoia.
+		s.logger.Warningf("Local listener address %q failed to parse as IP:port: %v", lisAddr, err)
+		return
+	}
+	ilc := update.lds.InboundListenerCfg
+	if ilc == nil {
+		s.logger.Warningf("Missing host:port in Listener updates")
+		return
+	}
+	if ilc.Address != addr || ilc.Port != port {
+		s.logger.Warningf("Received host:port (%s:%d) in Listener update does not match local listening address: %s", ilc.Address, ilc.Port, lisAddr)
+		return
+	}
+
 	if err := s.handleSecurityConfig(update.lds.SecurityCfg, update.lw); err != nil {
 		s.logger.Warningf("Invalid security config update: %v", err)
 		return
diff --git a/xds/server_test.go b/xds/server_test.go
index cde963079262..8b837c4c9597 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -40,9 +41,9 @@ import (
 )
 
 const (
-	defaultTestTimeout       = 5 * time.Second
-	defaultTestShortTimeout  = 10 * time.Millisecond
-	testServerResourceNameID = "/path/to/resource"
+	defaultTestTimeout                     = 5 * time.Second
+	defaultTestShortTimeout                = 10 * time.Millisecond
+	testServerListenerResourceNameTemplate = "/path/to/resource/%s/%s"
 )
 
 type s struct {
@@ -90,6 +91,14 @@ func newFakeGRPCServer() *fakeGRPCServer {
 	}
 }
 
+func splitHostPort(hostport string) (string, string) {
+	addr, port, err := net.SplitHostPort(hostport)
+	if err != nil {
+		panic(fmt.Sprintf("listener address %q does not parse: %v", hostport, err))
+	}
+	return addr, port
+}
+
 func (s) TestNewServer(t *testing.T) {
 	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
 	if err != nil {
@@ -228,11 +237,11 @@ func setupOverrides() (*fakeGRPCServer, *testutils.Channel, *testutils.Channel,
 	newXDSClient = func() (xdsClientInterface, error) {
 		c := fakeclient.NewClient()
 		c.SetBootstrapConfig(&bootstrap.Config{
-			BalancerName:         "dummyBalancer",
-			Creds:                grpc.WithTransportCredentials(insecure.NewCredentials()),
-			NodeProto:            xdstestutils.EmptyNodeProtoV3,
-			ServerResourceNameID: testServerResourceNameID,
-			CertProviderConfigs:  certProviderConfigs,
+			BalancerName:                       "dummyBalancer",
+			Creds:                              grpc.WithTransportCredentials(insecure.NewCredentials()),
+			NodeProto:                          xdstestutils.EmptyNodeProtoV3,
+			ServerListenerResourceNameTemplate: testServerListenerResourceNameTemplate,
+			CertProviderConfigs:                certProviderConfigs,
 		})
 		clientCh.Send(c)
 		return c, nil
@@ -267,10 +276,10 @@ func setupOverridesForXDSCreds(includeCertProviderCfg bool) (*testutils.Channel,
 	newXDSClient = func() (xdsClientInterface, error) {
 		c := fakeclient.NewClient()
 		bc := &bootstrap.Config{
-			BalancerName:         "dummyBalancer",
-			Creds:                grpc.WithTransportCredentials(insecure.NewCredentials()),
-			NodeProto:            xdstestutils.EmptyNodeProtoV3,
-			ServerResourceNameID: testServerResourceNameID,
+			BalancerName:                       "dummyBalancer",
+			Creds:                              grpc.WithTransportCredentials(insecure.NewCredentials()),
+			NodeProto:                          xdstestutils.EmptyNodeProtoV3,
+			ServerListenerResourceNameTemplate: testServerListenerResourceNameTemplate,
 		}
 		if includeCertProviderCfg {
 			bc.CertProviderConfigs = certProviderConfigs
@@ -337,7 +346,7 @@ func (s) TestServeSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -353,10 +362,35 @@ func (s) TestServeSuccess(t *testing.T) {
 
 	// Push a good LDS response, and wait for Serve() to be invoked on the
 	// underlying grpc.Server.
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: "routeconfig"}, nil)
+	addr, port := splitHostPort(lis.Addr().String())
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: addr,
+			Port:    port,
+		},
+	}, nil)
 	if _, err := fs.serveCh.Receive(ctx); err != nil {
 		t.Fatalf("error when waiting for Serve() to be invoked on the grpc.Server")
 	}
+
+	// Push an update to the registered listener watch callback with a Listener
+	// resource whose host:port does not match the actual listening address and
+	// port. Serve() should not return and should continue to use the old state.
+	//
+	// This will change once we add start tracking serving state.
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: "10.20.30.40",
+			Port:    "666",
+		},
+	}, nil)
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Serve() returned after a bad LDS response")
+	}
 }
 
 // TestServeWithStop tests the case where Stop() is called before an LDS update
@@ -399,7 +433,7 @@ func (s) TestServeWithStop(t *testing.T) {
 		server.Stop()
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	if name != wantName {
 		server.Stop()
 		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantName)
@@ -448,39 +482,79 @@ func (s) TestServeBootstrapFailure(t *testing.T) {
 	}
 }
 
-// TestServeBootstrapWithMissingCertProviders tests the case where the bootstrap
-// config does not contain certificate provider configuration, but xdsCreds are
-// passed to the server. Verifies that the call to Serve() fails.
-func (s) TestServeBootstrapWithMissingCertProviders(t *testing.T) {
-	_, _, cleanup := setupOverridesForXDSCreds(false)
-	defer cleanup()
-
-	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
-	if err != nil {
-		t.Fatalf("failed to create xds server credentials: %v", err)
+// TestServeBootstrapConfigInvalid tests the cases where the bootstrap config
+// does not contain expected fields. Verifies that the call to Serve() fails.
+func (s) TestServeBootstrapConfigInvalid(t *testing.T) {
+	tests := []struct {
+		desc            string
+		bootstrapConfig *bootstrap.Config
+	}{
+		{
+			desc:            "bootstrap config is missing",
+			bootstrapConfig: nil,
+		},
+		{
+			desc: "certificate provider config is missing",
+			bootstrapConfig: &bootstrap.Config{
+				BalancerName:                       "dummyBalancer",
+				Creds:                              grpc.WithTransportCredentials(insecure.NewCredentials()),
+				NodeProto:                          xdstestutils.EmptyNodeProtoV3,
+				ServerListenerResourceNameTemplate: testServerListenerResourceNameTemplate,
+			},
+		},
+		{
+			desc: "server_listener_resource_name_template is missing",
+			bootstrapConfig: &bootstrap.Config{
+				BalancerName:        "dummyBalancer",
+				Creds:               grpc.WithTransportCredentials(insecure.NewCredentials()),
+				NodeProto:           xdstestutils.EmptyNodeProtoV3,
+				CertProviderConfigs: certProviderConfigs,
+			},
+		},
 	}
-	server := NewGRPCServer(grpc.Creds(xdsCreds))
-	defer server.Stop()
 
-	lis, err := xdstestutils.LocalTCPListener()
-	if err != nil {
-		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
-	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			// Override the xdsClient creation with one that returns a fake
+			// xdsClient with the specified bootstrap configuration.
+			clientCh := testutils.NewChannel()
+			origNewXDSClient := newXDSClient
+			newXDSClient = func() (xdsClientInterface, error) {
+				c := fakeclient.NewClient()
+				c.SetBootstrapConfig(test.bootstrapConfig)
+				clientCh.Send(c)
+				return c, nil
+			}
+			defer func() { newXDSClient = origNewXDSClient }()
 
-	serveDone := testutils.NewChannel()
-	go func() {
-		err := server.Serve(lis)
-		serveDone.Send(err)
-	}()
+			xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
+			if err != nil {
+				t.Fatalf("failed to create xds server credentials: %v", err)
+			}
+			server := NewGRPCServer(grpc.Creds(xdsCreds))
+			defer server.Stop()
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	v, err := serveDone.Receive(ctx)
-	if err != nil {
-		t.Fatalf("error when waiting for Serve() to exit: %v", err)
-	}
-	if err, ok := v.(error); !ok || err == nil {
-		t.Fatal("Serve() did not exit with error")
+			lis, err := xdstestutils.LocalTCPListener()
+			if err != nil {
+				t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
+			}
+
+			serveDone := testutils.NewChannel()
+			go func() {
+				err := server.Serve(lis)
+				serveDone.Send(err)
+			}()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			v, err := serveDone.Receive(ctx)
+			if err != nil {
+				t.Fatalf("error when waiting for Serve() to exit: %v", err)
+			}
+			if err, ok := v.(error); !ok || err == nil {
+				t.Fatal("Serve() did not exit with error")
+			}
+		})
 	}
 }
 
@@ -556,7 +630,7 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -564,6 +638,7 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 	// Push a good LDS response with security config, and wait for Serve() to be
 	// invoked on the underlying grpc.Server. Also make sure that certificate
 	// providers are not created.
+	addr, port := splitHostPort(lis.Addr().String())
 	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
 		RouteConfigName: "routeconfig",
 		SecurityCfg: &xdsclient.SecurityConfig{
@@ -571,6 +646,10 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 			IdentityInstanceName: "default2",
 			RequireClientCert:    true,
 		},
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: addr,
+			Port:    port,
+		},
 	}, nil)
 	if _, err := fs.serveCh.Receive(ctx); err != nil {
 		t.Fatalf("error when waiting for Serve() to be invoked on the grpc.Server")
@@ -627,21 +706,14 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
 
 	// Push an error to the registered listener watch callback and make sure
 	// that Serve does not return.
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
-		RouteConfigName: "routeconfig",
-		SecurityCfg: &xdsclient.SecurityConfig{
-			RootInstanceName:     "default1",
-			IdentityInstanceName: "default2",
-			RequireClientCert:    true,
-		},
-	}, errors.New("LDS error"))
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, errors.New("LDS error"))
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
 	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
@@ -691,7 +763,7 @@ func (s) TestHandleListenerUpdate_ClosedListener(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := fmt.Sprintf("%s?udpa.resource.listening_address=%s", client.BootstrapConfig().ServerResourceNameID, lis.Addr().String())
+	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -699,9 +771,14 @@ func (s) TestHandleListenerUpdate_ClosedListener(t *testing.T) {
 	// Push a good update to the registered listener watch callback. This will
 	// unblock the xds-enabled server which is waiting for a good listener
 	// update before calling Serve() on the underlying grpc.Server.
+	addr, port := splitHostPort(lis.Addr().String())
 	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
 		RouteConfigName: "routeconfig",
 		SecurityCfg:     &xdsclient.SecurityConfig{IdentityInstanceName: "default2"},
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: addr,
+			Port:    port,
+		},
 	}, nil)
 	if _, err := providerCh.Receive(ctx); err != nil {
 		t.Fatal("error when waiting for certificate provider to be created")
@@ -723,6 +800,10 @@ func (s) TestHandleListenerUpdate_ClosedListener(t *testing.T) {
 	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
 		RouteConfigName: "routeconfig",
 		SecurityCfg:     &xdsclient.SecurityConfig{IdentityInstanceName: "default1"},
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: addr,
+			Port:    port,
+		},
 	}, nil)
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()

From e8930beb0e042eaf70b0a0a0a87dcaf8daffe782 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 10 Mar 2021 21:12:44 -0800
Subject: [PATCH 382/481] xds: Prepare to support filter chains on the server
 (#4222)

---
 credentials/xds/xds.go                     |   7 +-
 credentials/xds/xds_client_test.go         |   8 +-
 credentials/xds/xds_server_test.go         |  61 ++-
 internal/credentials/xds/handshake_info.go |  12 +-
 xds/internal/client/client.go              |  58 ++-
 xds/internal/client/lds_test.go            | 551 ++++++++++++++++++---
 xds/internal/client/xds.go                 |  88 +++-
 xds/internal/server/conn_wrapper.go        | 175 +++++++
 xds/internal/server/listener_wrapper.go    | 200 ++++++++
 xds/server.go                              | 322 +-----------
 xds/server_test.go                         | 183 +++----
 11 files changed, 1140 insertions(+), 525 deletions(-)
 create mode 100644 xds/internal/server/conn_wrapper.go
 create mode 100644 xds/internal/server/listener_wrapper.go

diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go
index ede0806d70db..0243009df644 100644
--- a/credentials/xds/xds.go
+++ b/credentials/xds/xds.go
@@ -216,12 +216,15 @@ func (c *credsImpl) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.Aut
 	// `HandshakeInfo` does not contain the information we are looking for, we
 	// delegate the handshake to the fallback credentials.
 	hiConn, ok := rawConn.(interface {
-		XDSHandshakeInfo() *xdsinternal.HandshakeInfo
+		XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error)
 	})
 	if !ok {
 		return c.fallback.ServerHandshake(rawConn)
 	}
-	hi := hiConn.XDSHandshakeInfo()
+	hi, err := hiConn.XDSHandshakeInfo()
+	if err != nil {
+		return nil, nil, err
+	}
 	if hi.UseFallbackCreds() {
 		return c.fallback.ServerHandshake(rawConn)
 	}
diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index 219d0aefcba6..6d40b0700082 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -41,7 +41,7 @@ import (
 )
 
 const (
-	defaultTestTimeout      = 10 * time.Second
+	defaultTestTimeout      = 1 * time.Second
 	defaultTestShortTimeout = 10 * time.Millisecond
 	defaultTestCertSAN      = "*.test.example.com"
 	authority               = "authority"
@@ -218,7 +218,8 @@ func newTestContextWithHandshakeInfo(parent context.Context, root, identity cert
 	// Creating the HandshakeInfo and adding it to the attributes is very
 	// similar to what the CDS balancer would do when it intercepts calls to
 	// NewSubConn().
-	info := xdsinternal.NewHandshakeInfo(root, identity, sans...)
+	info := xdsinternal.NewHandshakeInfo(root, identity)
+	info.SetAcceptedSANs(sans)
 	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)
 
 	// Moving the attributes from the resolver.Address to the context passed to
@@ -530,7 +531,8 @@ func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	// Create a root provider which will fail the handshake because it does not
 	// use the correct trust roots.
 	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
-	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil, defaultTestCertSAN)
+	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil)
+	handshakeInfo.SetAcceptedSANs([]string{defaultTestCertSAN})
 
 	// We need to repeat most of what newTestContextWithHandshakeInfo() does
 	// here because we need access to the underlying HandshakeInfo so that we
diff --git a/credentials/xds/xds_server_test.go b/credentials/xds/xds_server_test.go
index 68d92b28e286..5c29ba38c286 100644
--- a/credentials/xds/xds_server_test.go
+++ b/credentials/xds/xds_server_test.go
@@ -95,12 +95,13 @@ func (s) TestServerCredsWithoutFallback(t *testing.T) {
 
 type wrapperConn struct {
 	net.Conn
-	xdsHI    *xdsinternal.HandshakeInfo
-	deadline time.Time
+	xdsHI            *xdsinternal.HandshakeInfo
+	deadline         time.Time
+	handshakeInfoErr error
 }
 
-func (wc *wrapperConn) XDSHandshakeInfo() *xdsinternal.HandshakeInfo {
-	return wc.xdsHI
+func (wc *wrapperConn) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) {
+	return wc.xdsHI, wc.handshakeInfoErr
 }
 
 func (wc *wrapperConn) GetDeadline() time.Time {
@@ -166,6 +167,58 @@ func (s) TestServerCredsProviderFailure(t *testing.T) {
 	}
 }
 
+// TestServerCredsHandshake_XDSHandshakeInfoError verifies the case where the
+// call to XDSHandshakeInfo() from the ServerHandshake() method returns an
+// error, and the test verifies that the ServerHandshake() fails with the
+// expected error.
+func (s) TestServerCredsHandshake_XDSHandshakeInfoError(t *testing.T) {
+	opts := ServerOptions{FallbackCreds: &errorCreds{}}
+	creds, err := NewServerCredentials(opts)
+	if err != nil {
+		t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err)
+	}
+
+	// Create a test server which uses the xDS server credentials created above
+	// to perform TLS handshake on incoming connections.
+	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
+		// Create a wrapped conn which returns a nil HandshakeInfo and a non-nil error.
+		conn := newWrappedConn(rawConn, nil, time.Now().Add(defaultTestTimeout))
+		hiErr := errors.New("xdsHandshakeInfo error")
+		conn.handshakeInfoErr = hiErr
+
+		// Invoke the ServerHandshake() method on the xDS credentials and verify
+		// that the error returned by the XDSHandshakeInfo() method on the
+		// wrapped conn is returned here.
+		_, _, err := creds.ServerHandshake(conn)
+		if !errors.Is(err, hiErr) {
+			return handshakeResult{err: fmt.Errorf("ServerHandshake() returned err: %v, wantErr: %v", err, hiErr)}
+		}
+		return handshakeResult{}
+	})
+	defer ts.stop()
+
+	// Dial the test server, but don't trigger the TLS handshake. This will
+	// cause ServerHandshake() to fail.
+	rawConn, err := net.Dial("tcp", ts.address)
+	if err != nil {
+		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
+	}
+	defer rawConn.Close()
+
+	// Read handshake result from the testServer which will return an error if
+	// the handshake succeeded.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	val, err := ts.hsResult.Receive(ctx)
+	if err != nil {
+		t.Fatalf("testServer failed to return handshake result: %v", err)
+	}
+	hsr := val.(handshakeResult)
+	if hsr.err != nil {
+		t.Fatalf("testServer handshake failure: %v", hsr.err)
+	}
+}
+
 // TestServerCredsHandshakeTimeout verifies the case where the client does not
 // send required handshake data before the deadline set on the net.Conn passed
 // to ServerHandshake().
diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go
index 8b2035660639..db0e6542dd23 100644
--- a/internal/credentials/xds/handshake_info.go
+++ b/internal/credentials/xds/handshake_info.go
@@ -217,14 +217,6 @@ func (hi *HandshakeInfo) MatchingSANExists(cert *x509.Certificate) bool {
 
 // NewHandshakeInfo returns a new instance of HandshakeInfo with the given root
 // and identity certificate providers.
-func NewHandshakeInfo(root, identity certprovider.Provider, sans ...string) *HandshakeInfo {
-	acceptedSANs := make(map[string]bool, len(sans))
-	for _, san := range sans {
-		acceptedSANs[san] = true
-	}
-	return &HandshakeInfo{
-		rootProvider:     root,
-		identityProvider: identity,
-		acceptedSANs:     acceptedSANs,
-	}
+func NewHandshakeInfo(root, identity certprovider.Provider) *HandshakeInfo {
+	return &HandshakeInfo{rootProvider: root, identityProvider: identity}
 }
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index a29ea2fe3a12..7e8ffe7b18b3 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -24,6 +24,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"sync"
 	"time"
 
@@ -194,8 +195,6 @@ type ListenerUpdate struct {
 	// RouteConfigName is the route configuration name corresponding to the
 	// target which is being watched through LDS.
 	RouteConfigName string
-	// SecurityCfg contains security configuration sent by the control plane.
-	SecurityCfg *SecurityConfig
 	// MaxStreamDuration contains the HTTP connection manager's
 	// common_http_protocol_options.max_stream_duration field, or zero if
 	// unset.
@@ -232,10 +231,61 @@ type InboundListenerConfig struct {
 	// Port is the local port on which the inbound listener is expected to
 	// accept incoming connections.
 	Port string
+	// FilterChains is the list of filter chains associated with this listener.
+	FilterChains []*FilterChain
+	// DefaultFilterChain is the filter chain to be used when none of the above
+	// filter chains matches an incoming connection.
+	DefaultFilterChain *FilterChain
+}
+
+// FilterChain wraps a set of match criteria and associated security
+// configuration.
+//
+// The actual set filters associated with this filter chain are not captured
+// here, since we do not support these filters on the server yet.
+type FilterChain struct {
+	// Match contains the criteria to use when matching a connection to this
+	// filter chain.
+	Match *FilterChainMatch
+	// SecurityCfg contains transport socket security configuration.
+	SecurityCfg *SecurityConfig
 }
 
-func (lu *ListenerUpdate) String() string {
-	return fmt.Sprintf("{RouteConfigName: %q, SecurityConfig: %+v", lu.RouteConfigName, lu.SecurityCfg)
+// SourceType specifies the connection source IP match type.
+type SourceType int
+
+const (
+	// SourceTypeAny matches connection attempts from any source.
+	SourceTypeAny SourceType = iota
+	// SourceTypeSameOrLoopback matches connection attempts from the same host.
+	SourceTypeSameOrLoopback
+	// SourceTypeExternal matches connection attempts from a different host.
+	SourceTypeExternal
+)
+
+// FilterChainMatch specifies the match criteria for selecting a specific filter
+// chain of a listener, for an incoming connection.
+//
+// The xDS FilterChainMatch proto specifies 8 match criteria. But we only have a
+// subset of those fields here because we explicitly ignore filter chains whose
+// match criteria specifies values for fields like destination_port,
+// server_names, application_protocols, transport_protocol.
+type FilterChainMatch struct {
+	// DestPrefixRanges specifies a set of IP addresses and prefix lengths to
+	// match the destination address of the incoming connection when the
+	// listener is bound to 0.0.0.0/[::]. If this field is empty, the
+	// destination address is ignored.
+	DestPrefixRanges []net.IP
+	// SourceType specifies the connection source IP match type. Can be any,
+	// local or external network.
+	SourceType SourceType
+	// SourcePrefixRanges specifies a set of IP addresses and prefix lengths to
+	// match the source address of the incoming connection. If this field is
+	// empty, the source address is ignored.
+	SourcePrefixRanges []net.IP
+	// SourcePorts specifies a set of ports to match the source port of the
+	// incoming connection. If this field is empty, the source port is ignored.
+	SourcePorts []uint32
 }
 
 // RouteConfigUpdate contains information received in an RDS response, which is
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index c2a7ff4f4f83..3a0f4b5b1991 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -20,6 +20,7 @@ package client
 
 import (
 	"fmt"
+	"net"
 	"strings"
 	"testing"
 	"time"
@@ -149,7 +150,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			return &anypb.Any{
 				TypeUrl: version.V3ListenerURL,
 				Value: func() []byte {
-					mcm, _ := ptypes.MarshalAny(hcm)
+					mcm := marshalAny(hcm)
 					lis := &v3listenerpb.Listener{
 						Name: v3LDSTarget,
 						ApiListener: &v3listenerpb.ApiListener{
@@ -793,6 +794,29 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							},
 						},
 					},
+					DefaultFilterChain: &v3listenerpb.FilterChain{
+						Name: "default-filter-chain-1",
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: &anypb.Any{
+									TypeUrl: version.V3DownstreamTLSContextURL,
+									Value: func() []byte {
+										tls := &v3tlspb.DownstreamTlsContext{
+											CommonTlsContext: &v3tlspb.CommonTlsContext{
+												TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    "defaultIdentityPluginInstance",
+													CertificateName: "defaultIdentityCertName",
+												},
+											},
+										}
+										mtls, _ := proto.Marshal(tls)
+										return mtls
+									}(),
+								},
+							},
+						},
+					},
 				}
 				mLis, _ := proto.Marshal(lis)
 				return mLis
@@ -845,6 +869,36 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 							},
 						},
 					},
+					DefaultFilterChain: &v3listenerpb.FilterChain{
+						Name: "default-filter-chain-1",
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: &anypb.Any{
+									TypeUrl: version.V3DownstreamTLSContextURL,
+									Value: func() []byte {
+										tls := &v3tlspb.DownstreamTlsContext{
+											RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+											CommonTlsContext: &v3tlspb.CommonTlsContext{
+												TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    "defaultIdentityPluginInstance",
+													CertificateName: "defaultIdentityCertName",
+												},
+												ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+													ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+														InstanceName:    "defaultRootPluginInstance",
+														CertificateName: "defaultRootCertName",
+													},
+												},
+											},
+										}
+										mtls, _ := proto.Marshal(tls)
+										return mtls
+									}(),
+								},
+							},
+						},
+					},
 				}
 				mLis, _ := proto.Marshal(lis)
 				return mLis
@@ -912,45 +966,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			},
 			wantErr: "no socket_address field in LDS response",
 		},
-		{
-			name: "unexpected number of filter chains",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{Name: "filter-chain-1"},
-								{Name: "filter-chain-2"},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
-			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "filter chains count in LDS response does not match expected",
-		},
 		{
 			name: "unexpected transport socket name",
 			resources: []*anypb.Any{
@@ -1211,8 +1226,9 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
 					InboundListenerCfg: &InboundListenerConfig{
-						Address: "0.0.0.0",
-						Port:    "9999",
+						Address:      "0.0.0.0",
+						Port:         "9999",
+						FilterChains: []*FilterChain{{Match: &FilterChainMatch{}}},
 					},
 					Raw: listenerEmptyTransportSocket,
 				},
@@ -1343,13 +1359,25 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			resources: []*anypb.Any{listenerNoValidationContext},
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
-					SecurityCfg: &SecurityConfig{
-						IdentityInstanceName: "identityPluginInstance",
-						IdentityCertName:     "identityCertName",
-					},
 					InboundListenerCfg: &InboundListenerConfig{
 						Address: "0.0.0.0",
 						Port:    "9999",
+						FilterChains: []*FilterChain{
+							{
+								Match: &FilterChainMatch{},
+								SecurityCfg: &SecurityConfig{
+									IdentityInstanceName: "identityPluginInstance",
+									IdentityCertName:     "identityCertName",
+								},
+							},
+						},
+						DefaultFilterChain: &FilterChain{
+							Match: &FilterChainMatch{},
+							SecurityCfg: &SecurityConfig{
+								IdentityInstanceName: "defaultIdentityPluginInstance",
+								IdentityCertName:     "defaultIdentityCertName",
+							},
+						},
 					},
 					Raw: listenerNoValidationContext,
 				},
@@ -1364,16 +1392,31 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			resources: []*anypb.Any{listenerWithValidationContext},
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
-					SecurityCfg: &SecurityConfig{
-						RootInstanceName:     "rootPluginInstance",
-						RootCertName:         "rootCertName",
-						IdentityInstanceName: "identityPluginInstance",
-						IdentityCertName:     "identityCertName",
-						RequireClientCert:    true,
-					},
 					InboundListenerCfg: &InboundListenerConfig{
 						Address: "0.0.0.0",
 						Port:    "9999",
+						FilterChains: []*FilterChain{
+							{
+								Match: &FilterChainMatch{},
+								SecurityCfg: &SecurityConfig{
+									RootInstanceName:     "rootPluginInstance",
+									RootCertName:         "rootCertName",
+									IdentityInstanceName: "identityPluginInstance",
+									IdentityCertName:     "identityCertName",
+									RequireClientCert:    true,
+								},
+							},
+						},
+						DefaultFilterChain: &FilterChain{
+							Match: &FilterChainMatch{},
+							SecurityCfg: &SecurityConfig{
+								RootInstanceName:     "defaultRootPluginInstance",
+								RootCertName:         "defaultRootCertName",
+								IdentityInstanceName: "defaultIdentityPluginInstance",
+								IdentityCertName:     "defaultIdentityCertName",
+								RequireClientCert:    true,
+							},
+						},
 					},
 					Raw: listenerWithValidationContext,
 				},
@@ -1404,6 +1447,389 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 	}
 }
 
+func (s) TestGetFilterChain(t *testing.T) {
+	tests := []struct {
+		desc             string
+		inputFilterChain *v3listenerpb.FilterChain
+		wantFilterChain  *FilterChain
+		wantErr          bool
+	}{
+		{
+			desc:             "empty",
+			inputFilterChain: nil,
+		},
+		{
+			desc: "unsupported destination port",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					DestinationPort: &wrapperspb.UInt32Value{
+						Value: 666,
+					},
+				},
+			},
+		},
+		{
+			desc: "unsupported server names",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					ServerNames: []string{"example-server"},
+				},
+			},
+		},
+		{
+			desc: "unsupported transport protocol",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					TransportProtocol: "tls",
+				},
+			},
+		},
+		{
+			desc: "unsupported application protocol",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					ApplicationProtocols: []string{"h2"},
+				},
+			},
+		},
+		{
+			desc: "bad dest address prefix",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "a.b.c.d",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "bad dest prefix length",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "10.1.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 50,
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "dest prefix ranges",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "10.1.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 8,
+							},
+						},
+						{
+							AddressPrefix: "192.168.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 24,
+							},
+						},
+					},
+				},
+			},
+			wantFilterChain: &FilterChain{
+				Match: &FilterChainMatch{
+					DestPrefixRanges: []net.IP{
+						net.IPv4(10, 1, 1, 0),
+						net.IPv4(192, 168, 1, 0),
+					},
+				},
+			},
+		},
+		{
+			desc: "source type local",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+				},
+			},
+			wantFilterChain: &FilterChain{
+				Match: &FilterChainMatch{
+					SourceType: SourceTypeSameOrLoopback,
+				},
+			},
+		},
+		{
+			desc: "source type external",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourceType: v3listenerpb.FilterChainMatch_EXTERNAL,
+				},
+			},
+			wantFilterChain: &FilterChain{
+				Match: &FilterChainMatch{
+					SourceType: SourceTypeExternal,
+				},
+			},
+		},
+		{
+			desc: "source type any",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourceType: v3listenerpb.FilterChainMatch_ANY,
+				},
+			},
+			wantFilterChain: &FilterChain{
+				Match: &FilterChainMatch{
+					SourceType: SourceTypeAny,
+				},
+			},
+		},
+		{
+			desc: "bad source address prefix",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "a.b.c.d",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "bad source prefix length",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "10.1.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 50,
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "source prefix ranges",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "10.1.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 8,
+							},
+						},
+						{
+							AddressPrefix: "192.168.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 24,
+							},
+						},
+					},
+				},
+			},
+			wantFilterChain: &FilterChain{
+				Match: &FilterChainMatch{
+					SourcePrefixRanges: []net.IP{
+						net.IPv4(10, 1, 1, 0),
+						net.IPv4(192, 168, 1, 0),
+					},
+				},
+			},
+		},
+		{
+			desc:             "empty transport socket",
+			inputFilterChain: &v3listenerpb.FilterChain{},
+			wantFilterChain:  &FilterChain{Match: &FilterChainMatch{}},
+		},
+		{
+			desc: "bad transport socket name",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "unsupported-transport-socket-name",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "unexpected url in transport socket",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.UpstreamTlsContext{}),
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "badly marshaled downstream tls context",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3DownstreamTLSContextURL,
+							Value:   []byte{1, 2, 3, 4},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "missing common tls context",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{}),
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "unsupported validation context",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
+									ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
+										Name: "foo-sds-secret",
+									},
+								},
+							},
+						}),
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "no identity and root certificate providers",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    "identityPluginInstance",
+									CertificateName: "identityCertName",
+								},
+							},
+						}),
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "no identity certificate provider with require_client_cert",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{},
+						}),
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "happy case",
+			inputFilterChain: &v3listenerpb.FilterChain{
+				Name: "filter-chain-1",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "10.1.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 8,
+							},
+						},
+					},
+					SourceType: v3listenerpb.FilterChainMatch_EXTERNAL,
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "10.1.1.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: 8,
+							},
+						},
+					},
+					SourcePorts: []uint32{80, 8080},
+				},
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    "identityPluginInstance",
+									CertificateName: "identityCertName",
+								},
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "rootPluginInstance",
+										CertificateName: "rootCertName",
+									},
+								},
+							},
+						}),
+					},
+				},
+			},
+			wantFilterChain: &FilterChain{
+				Match: &FilterChainMatch{
+					DestPrefixRanges:   []net.IP{net.IPv4(10, 1, 1, 0)},
+					SourceType:         SourceTypeExternal,
+					SourcePrefixRanges: []net.IP{net.IPv4(10, 1, 1, 0)},
+					SourcePorts:        []uint32{80, 8080},
+				},
+				SecurityCfg: &SecurityConfig{
+					RootInstanceName:     "rootPluginInstance",
+					RootCertName:         "rootCertName",
+					IdentityInstanceName: "identityPluginInstance",
+					IdentityCertName:     "identityCertName",
+					RequireClientCert:    true,
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotFilterChain, gotErr := getFilterChain(test.inputFilterChain)
+			if (gotErr != nil) != test.wantErr {
+				t.Fatalf("getFilterChain(%+v) returned error: %v, wantErr: %v", test.inputFilterChain, gotErr, test.wantErr)
+			}
+			if diff := cmp.Diff(test.wantFilterChain, gotFilterChain); diff != "" {
+				t.Errorf("getFilterChain(%+v) returned unexpected, diff (-want +got):\n%s", test.inputFilterChain, diff)
+			}
+		})
+	}
+}
+
 type filterConfig struct {
 	httpfilter.FilterConfig
 	Cfg      proto.Message
@@ -1509,11 +1935,7 @@ var customFilterTypedStructConfig = &v1typepb.TypedStruct{
 var wrappedCustomFilterTypedStructConfig *anypb.Any
 
 func init() {
-	var err error
-	wrappedCustomFilterTypedStructConfig, err = ptypes.MarshalAny(customFilterTypedStructConfig)
-	if err != nil {
-		panic(err.Error())
-	}
+	wrappedCustomFilterTypedStructConfig = marshalAny(customFilterTypedStructConfig)
 }
 
 var unknownFilterConfig = &anypb.Any{
@@ -1522,16 +1944,19 @@ var unknownFilterConfig = &anypb.Any{
 }
 
 func wrappedOptionalFilter(name string) *anypb.Any {
-	filter := &v3routepb.FilterConfig{
+	return marshalAny(&v3routepb.FilterConfig{
 		IsOptional: true,
 		Config: &anypb.Any{
 			TypeUrl: name,
 			Value:   []byte{1, 2, 3},
 		},
-	}
-	w, err := ptypes.MarshalAny(filter)
+	})
+}
+
+func marshalAny(m proto.Message) *anypb.Any {
+	a, err := ptypes.MarshalAny(m)
 	if err != nil {
-		panic("error marshalling any: " + err.Error())
+		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
 	}
-	return w
+	return a
 }
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 63e7c2baed33..0305f0a3015f 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -261,13 +261,82 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 		},
 	}
 
-	// Make sure the listener resource contains a single filter chain. We do not
-	// support multiple filter chains and picking the best match from the list.
-	fcs := lis.GetFilterChains()
-	if n := len(fcs); n != 1 {
-		return nil, fmt.Errorf("filter chains count in LDS response does not match expected. Got %d, want 1", n)
+	var filterChains []*FilterChain
+	for _, fc := range lis.GetFilterChains() {
+		filterChain, err := getFilterChain(fc)
+		if err != nil {
+			return nil, err
+		}
+		filterChains = append(filterChains, filterChain)
+	}
+	defaultFilterChain, err := getFilterChain(lis.GetDefaultFilterChain())
+	if err != nil {
+		return nil, err
+	}
+	if len(filterChains) == 0 && defaultFilterChain == nil {
+		return nil, fmt.Errorf("xds: no supported filter chains and no default filter chain")
+	}
+	lu.InboundListenerCfg.FilterChains = filterChains
+	lu.InboundListenerCfg.DefaultFilterChain = defaultFilterChain
+	return lu, nil
+}
+
+// getFilterChain parses the filter chain proto and converts it into the local
+// representation. If fc contains unsupported filter chain match fields, a nil
+// FilterChain object and a nil error are returned. If fc does not parse or
+// contains other invalid data, an non-nil error is returned.
+func getFilterChain(fc *v3listenerpb.FilterChain) (*FilterChain, error) {
+	if fc == nil {
+		return nil, nil
+	}
+
+	// If the match criteria contains unsupported fields, skip the filter chain.
+	fcm := fc.GetFilterChainMatch()
+	if fcm.GetDestinationPort().GetValue() != 0 ||
+		fcm.GetServerNames() != nil ||
+		(fcm.GetTransportProtocol() != "" && fcm.TransportProtocol != "raw_buffer") ||
+		fcm.GetApplicationProtocols() != nil {
+		return nil, nil
+	}
+
+	// Extract the supported match criteria.
+	var dstPrefixRanges []net.IP
+	for _, pr := range fcm.GetPrefixRanges() {
+		cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
+		ip, _, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return nil, fmt.Errorf("xds: failed to parse destination prefix range: %+v", pr)
+		}
+		dstPrefixRanges = append(dstPrefixRanges, ip)
+	}
+	var srcType SourceType
+	switch fcm.GetSourceType() {
+	case v3listenerpb.FilterChainMatch_ANY:
+		srcType = SourceTypeAny
+	case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK:
+		srcType = SourceTypeSameOrLoopback
+	case v3listenerpb.FilterChainMatch_EXTERNAL:
+		srcType = SourceTypeExternal
+	default:
+		return nil, fmt.Errorf("xds: unsupported source type: %v", fcm.GetSourceType())
+	}
+	var srcPrefixRanges []net.IP
+	for _, pr := range fcm.GetSourcePrefixRanges() {
+		cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
+		ip, _, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return nil, fmt.Errorf("xds: failed to parse source prefix range: %+v", pr)
+		}
+		srcPrefixRanges = append(srcPrefixRanges, ip)
+	}
+	filterChain := &FilterChain{
+		Match: &FilterChainMatch{
+			DestPrefixRanges:   dstPrefixRanges,
+			SourceType:         srcType,
+			SourcePrefixRanges: srcPrefixRanges,
+			SourcePorts:        fcm.GetSourcePorts(),
+		},
 	}
-	fc := fcs[0]
 
 	// If the transport_socket field is not specified, it means that the control
 	// plane has not sent us any security config. This is fine and the server
@@ -275,7 +344,7 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	// xdsCredentials.
 	ts := fc.GetTransportSocket()
 	if ts == nil {
-		return lu, nil
+		return filterChain, nil
 	}
 	if name := ts.GetName(); name != transportSocketName {
 		return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name)
@@ -302,9 +371,8 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	if sc.RequireClientCert && sc.RootInstanceName == "" {
 		return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set")
 	}
-	lu.SecurityCfg = sc
-
-	return lu, nil
+	filterChain.SecurityCfg = sc
+	return filterChain, nil
 }
 
 // UnmarshalRouteConfig processes resources received in an RDS response,
diff --git a/xds/internal/server/conn_wrapper.go b/xds/internal/server/conn_wrapper.go
new file mode 100644
index 000000000000..359674417dcf
--- /dev/null
+++ b/xds/internal/server/conn_wrapper.go
@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package server
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+// connWrapper is a thin wrapper around a net.Conn returned by Accept(). It
+// provides the following additional functionality:
+// 1. A way to retrieve the configured deadline. This is required by the
+//    ServerHandshake() method of the xdsCredentials when it attempts to read
+//    key material from the certificate providers.
+// 2. Implements the XDSHandshakeInfo() method used by the xdsCredentials to
+//    retrieve the configured certificate providers.
+// 3. xDS filter_chain matching logic to select appropriate security
+//    configuration for the incoming connection.
+type connWrapper struct {
+	net.Conn
+
+	// A reference fo the listenerWrapper on which this connection was accepted.
+	// Used to access the filter chains during the server-side handshake.
+	parent *listenerWrapper
+
+	// The certificate providers created for this connection.
+	rootProvider, identityProvider certprovider.Provider
+
+	// The connection deadline as configured by the grpc.Server on the rawConn
+	// that is returned by a call to Accept(). This is set to the connection
+	// timeout value configured by the user (or to a default value) before
+	// initiating the transport credential handshake, and set to zero after
+	// completing the HTTP2 handshake.
+	deadlineMu sync.Mutex
+	deadline   time.Time
+}
+
+// SetDeadline makes a copy of the passed in deadline and forwards the call to
+// the underlying rawConn.
+func (c *connWrapper) SetDeadline(t time.Time) error {
+	c.deadlineMu.Lock()
+	c.deadline = t
+	c.deadlineMu.Unlock()
+	return c.Conn.SetDeadline(t)
+}
+
+// GetDeadline returns the configured deadline. This will be invoked by the
+// ServerHandshake() method of the XdsCredentials, which needs a deadline to
+// pass to the certificate provider.
+func (c *connWrapper) GetDeadline() time.Time {
+	c.deadlineMu.Lock()
+	t := c.deadline
+	c.deadlineMu.Unlock()
+	return t
+}
+
+// XDSHandshakeInfo returns a HandshakeInfo with appropriate security
+// configuration for this connection. This method is invoked by the
+// ServerHandshake() method of the XdsCredentials.
+func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) {
+	// Ideally this should never happen, since xdsCredentials are the only ones
+	// which will invoke this method at handshake time. But to be on the safe
+	// side, we avoid acting on the security configuration received from the
+	// control plane when the user has not configured the use of xDS
+	// credentials, by checking the value of this flag.
+	if !c.parent.xdsCredsInUse {
+		return nil, errors.New("user has not configured xDS credentials")
+	}
+
+	fc := c.getMatchingFilterChain()
+	if fc == nil {
+		return nil, errors.New("no matching filter chain for incoming connection")
+	}
+
+	if fc.SecurityCfg == nil {
+		// If the security config is empty, this means that the control plane
+		// did not provide any security configuration and therefore we should
+		// return an empty HandshakeInfo here so that the xdsCreds can use the
+		// configured fallback credentials.
+		return xdsinternal.NewHandshakeInfo(nil, nil), nil
+	}
+
+	cpc := c.parent.xdsC.BootstrapConfig().CertProviderConfigs
+	// Identity provider name is mandatory on the server-side, and this is
+	// enforced when the resource is received at the xdsClient layer.
+	ip, err := buildProviderFunc(cpc, fc.SecurityCfg.IdentityInstanceName, fc.SecurityCfg.IdentityCertName, true, false)
+	if err != nil {
+		return nil, err
+	}
+	// Root provider name is optional and required only when doing mTLS.
+	var rp certprovider.Provider
+	if instance, cert := fc.SecurityCfg.RootInstanceName, fc.SecurityCfg.RootCertName; instance != "" {
+		rp, err = buildProviderFunc(cpc, instance, cert, false, true)
+		if err != nil {
+			return nil, err
+		}
+	}
+	c.identityProvider = ip
+	c.rootProvider = rp
+
+	xdsHI := xdsinternal.NewHandshakeInfo(c.identityProvider, c.rootProvider)
+	xdsHI.SetRequireClientCert(fc.SecurityCfg.RequireClientCert)
+	return xdsHI, nil
+}
+
+// The logic specified in the documentation around the xDS FilterChainMatch
+// proto mentions 8 criteria to match on. gRPC does not support 4 of those, and
+// hence we got rid of them at the time of parsing the received Listener
+// resource. Here we use the remaining 4 criteria to find a matching filter
+// chain: Destination IP address, Source type, Source IP address, Source port.
+func (c *connWrapper) getMatchingFilterChain() *xdsclient.FilterChain {
+	c.parent.mu.RLock()
+	defer c.parent.mu.RUnlock()
+
+	// TODO: Do the filter chain match here and return the best match.
+	// For now, we simply return the first filter_chain in the list or the
+	// default one.
+	if len(c.parent.filterChains) == 0 {
+		return c.parent.defaultFilterChain
+	}
+	return c.parent.filterChains[0]
+}
+
+func (c *connWrapper) Close() error {
+	if c.identityProvider != nil {
+		c.identityProvider.Close()
+	}
+	if c.rootProvider != nil {
+		c.rootProvider.Close()
+	}
+	return c.Conn.Close()
+}
+
+func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
+	cfg, ok := configs[instanceName]
+	if !ok {
+		return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName)
+	}
+	provider, err := cfg.Build(certprovider.BuildOptions{
+		CertName:     certName,
+		WantIdentity: wantIdentity,
+		WantRoot:     wantRoot,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return nil, fmt.Errorf("failed to get security plugin instance (%+v): %v", cfg, err)
+	}
+	return provider, nil
+}
diff --git a/xds/internal/server/listener_wrapper.go b/xds/internal/server/listener_wrapper.go
new file mode 100644
index 000000000000..078b2112581d
--- /dev/null
+++ b/xds/internal/server/listener_wrapper.go
@@ -0,0 +1,200 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package server contains internal server-side functionality used by the public
+// facing xds package.
+package server
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+)
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *listenerWrapper) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p]", p))
+}
+
+// XDSClientInterface wraps the methods on the xdsClient which are required by
+// the listenerWrapper.
+type XDSClientInterface interface {
+	WatchListener(string, func(xdsclient.ListenerUpdate, error)) func()
+	BootstrapConfig() *bootstrap.Config
+}
+
+// ListenerWrapperParams wraps parameters required to create a listenerWrapper.
+type ListenerWrapperParams struct {
+	// Listener is the net.Listener passed by the user that is to be wrapped.
+	Listener net.Listener
+	// ListenerResourceName is the xDS Listener resource to request.
+	ListenerResourceName string
+	// XDSCredsInUse specifies whether or not the user expressed interest to
+	// receive security configuration from the control plane.
+	XDSCredsInUse bool
+	// XDSClient provides the functionality from the xdsClient required here.
+	XDSClient XDSClientInterface
+}
+
+// NewListenerWrapper creates a new listenerWrapper with params. It returns a
+// net.Listener and a channel which is written to, indicating that the former is
+// ready to be passed to grpc.Serve().
+func NewListenerWrapper(params ListenerWrapperParams) (net.Listener, <-chan struct{}) {
+	lw := &listenerWrapper{
+		Listener:      params.Listener,
+		name:          params.ListenerResourceName,
+		xdsCredsInUse: params.XDSCredsInUse,
+		xdsC:          params.XDSClient,
+
+		closed:     grpcsync.NewEvent(),
+		goodUpdate: grpcsync.NewEvent(),
+	}
+	lw.logger = prefixLogger(lw)
+
+	cancelWatch := lw.xdsC.WatchListener(lw.name, lw.handleListenerUpdate)
+	lw.logger.Infof("Watch started on resource name %v", lw.name)
+	lw.cancelWatch = func() {
+		cancelWatch()
+		lw.logger.Infof("Watch cancelled on resource name %v", lw.name)
+	}
+	return lw, lw.goodUpdate.Done()
+}
+
+// listenerWrapper wraps the net.Listener associated with the listening address
+// passed to Serve(). It also contains all other state associated with this
+// particular invocation of Serve().
+type listenerWrapper struct {
+	net.Listener
+	logger *internalgrpclog.PrefixLogger
+
+	// TODO: Maintain serving state of this listener.
+
+	name          string
+	xdsCredsInUse bool
+	xdsC          XDSClientInterface
+	cancelWatch   func()
+
+	// This is used to notify that a good update has been received and that
+	// Serve() can be invoked on the underlying gRPC server. Using an event
+	// instead of a vanilla channel simplifies the update handler as it need not
+	// keep track of whether the received update is the first one or not.
+	goodUpdate *grpcsync.Event
+	// A small race exists in the xdsClient code between the receipt of an xDS
+	// response and the user cancelling the associated watch. In this window,
+	// the registered callback may be invoked after the watch is canceled, and
+	// the user is expected to work around this. This event signifies that the
+	// listener is closed (and hence the watch is cancelled), and we drop any
+	// updates received in the callback if this event has fired.
+	closed *grpcsync.Event
+
+	// Filter chains received as part of the last good update. The reason for
+	// using an rw lock here is that this field will be read by all connections
+	// during their server-side handshake (in the hot path), but writes to this
+	// happen rarely (when we get a Listener resource update).
+	mu                 sync.RWMutex
+	filterChains       []*xdsclient.FilterChain
+	defaultFilterChain *xdsclient.FilterChain
+}
+
+// Accept blocks on an Accept() on the underlying listener, and wraps the
+// returned net.connWrapper with the configured certificate providers.
+func (l *listenerWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	// TODO: Close connections if in "non-serving" state.
+	return &connWrapper{Conn: c, parent: l}, nil
+}
+
+// Close closes the underlying listener. It also cancels the xDS watch
+// registered in Serve() and closes any certificate provider instances created
+// based on security configuration received in the LDS response.
+func (l *listenerWrapper) Close() error {
+	l.closed.Fire()
+	l.Listener.Close()
+	if l.cancelWatch != nil {
+		l.cancelWatch()
+	}
+	return nil
+}
+
+func (l *listenerWrapper) handleListenerUpdate(update xdsclient.ListenerUpdate, err error) {
+	if l.closed.HasFired() {
+		l.logger.Warningf("Resource %q received update: %v with error: %v, after listener was closed", l.name, update, err)
+		return
+	}
+
+	// TODO: Handle resource-not-found errors by moving to not-serving state.
+	if err != nil {
+		// We simply log an error here and hope we get a successful update
+		// in the future. The error could be because of a timeout or an
+		// actual error, like the requested resource not found. In any case,
+		// it is fine for the server to hang indefinitely until Stop() is
+		// called.
+		l.logger.Warningf("Received error for resource %q: %+v", l.name, err)
+		return
+	}
+	l.logger.Infof("Received update for resource %q: %+v", l.name, update)
+
+	// Make sure that the socket address on the received Listener resource
+	// matches the address of the net.Listener passed to us by the user. This
+	// check is done here instead of at the xdsClient layer because of the
+	// following couple of reasons:
+	// - xdsClient cannot know the listening address of every listener in the
+	//   system, and hence cannot perform this check.
+	// - this is a very context-dependent check and only the server has the
+	//   appropriate context to perform this check.
+	//
+	// What this means is that the xdsClient has ACKed a resource which is going
+	// to push the server into a "not serving" state. This is not ideal, but
+	// this is what we have decided to do. See gRPC A36 for more details.
+	// TODO(easwars): Switch to "not serving" if the host:port does not match.
+	lisAddr := l.Listener.Addr().String()
+	addr, port, err := net.SplitHostPort(lisAddr)
+	if err != nil {
+		// This is never expected to return a non-nil error since we have made
+		// sure that the listener is a TCP listener at the beginning of Serve().
+		// This is simply paranoia.
+		l.logger.Warningf("Local listener address %q failed to parse as IP:port: %v", lisAddr, err)
+		return
+	}
+	ilc := update.InboundListenerCfg
+	if ilc == nil {
+		l.logger.Warningf("Missing host:port in Listener updates")
+		return
+	}
+	if ilc.Address != addr || ilc.Port != port {
+		l.logger.Warningf("Received host:port (%s:%d) in Listener update does not match local listening address: %s", ilc.Address, ilc.Port, lisAddr)
+		return
+	}
+
+	l.mu.Lock()
+	l.filterChains = ilc.FilterChains
+	l.defaultFilterChain = ilc.DefaultFilterChain
+	l.mu.Unlock()
+	l.goodUpdate.Fire()
+	// TODO: Move to serving state on receipt of a good response.
+}
diff --git a/xds/server.go b/xds/server.go
index 7e7c251bbe3d..26146aad5e1e 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -25,18 +25,16 @@ import (
 	"net"
 	"strings"
 	"sync"
-	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/server"
 )
 
 const serverPrefix = "[xds-server %p] "
@@ -52,7 +50,6 @@ var (
 
 	// Unexported function to retrieve transport credentials from a gRPC server.
 	grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials)
-	buildProvider      = buildProviderFunc
 	logger             = grpclog.Component("xds")
 )
 
@@ -168,6 +165,7 @@ func (s *GRPCServer) initXDSClient() error {
 // initiated here.
 //
 // Serve will return a non-nil error unless Stop or GracefulStop is called.
+// TODO: Support callback to get notified on serving state changes.
 func (s *GRPCServer) Serve(lis net.Listener) error {
 	s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String())
 	if _, ok := lis.Addr().(*net.TCPAddr); !ok {
@@ -180,78 +178,43 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 		return err
 	}
 
+	cfg := s.xdsC.BootstrapConfig()
+	if cfg == nil {
+		return errors.New("bootstrap configuration is empty")
+	}
+
 	// If xds credentials were specified by the user, but bootstrap configs do
 	// not contain any certificate provider configuration, it is better to fail
 	// right now rather than failing when attempting to create certificate
 	// providers after receiving an LDS response with security configuration.
 	if s.xdsCredsInUse {
-		bc := s.xdsC.BootstrapConfig()
-		if bc == nil || len(bc.CertProviderConfigs) == 0 {
+		if len(cfg.CertProviderConfigs) == 0 {
 			return errors.New("xds: certificate_providers config missing in bootstrap file")
 		}
 	}
 
-	lw, err := s.newListenerWrapper(lis)
-	if lw == nil {
-		// Error returned can be nil (when Stop/GracefulStop() is called). So,
-		// we need to check the returned listenerWrapper instead.
-		return err
-	}
-	return s.gs.Serve(lw)
-}
-
-// newListenerWrapper creates and returns a listenerWrapper, which is a thin
-// wrapper around the passed in listener lis, that can be passed to
-// grpcServer.Serve().
-//
-// It then registers a watch for a Listener resource and blocks until a good
-// response is received or the server is stopped by a call to
-// Stop/GracefulStop().
-func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, error) {
-	lw := &listenerWrapper{
-		Listener: lis,
-		closed:   grpcsync.NewEvent(),
-		xdsHI:    xdsinternal.NewHandshakeInfo(nil, nil),
-	}
-
-	// This is used to notify that a good update has been received and that
-	// Serve() can be invoked on the underlying gRPC server. Using a
-	// grpcsync.Event instead of a vanilla channel simplifies the update handler
-	// as it need not keep track of whether the received update is the first one
-	// or not.
-	goodUpdate := grpcsync.NewEvent()
-
 	// The server listener resource name template from the bootstrap
 	// configuration contains a template for the name of the Listener resource
 	// to subscribe to for a gRPC server. If the token `%s` is present in the
 	// string, it will be replaced with the server's listening "IP:port" (e.g.,
 	// "0.0.0.0:8080", "[::]:8080"). The absence of a template will be treated
 	// as an error since we do not have any default value for this.
-	cfg := s.xdsC.BootstrapConfig()
-	if cfg == nil || cfg.ServerListenerResourceNameTemplate == "" {
-		return nil, errors.New("missing server_listener_resource_name_template in the bootstrap configuration")
+	if cfg.ServerListenerResourceNameTemplate == "" {
+		return errors.New("missing server_listener_resource_name_template in the bootstrap configuration")
 	}
 	name := cfg.ServerListenerResourceNameTemplate
 	if strings.Contains(cfg.ServerListenerResourceNameTemplate, "%s") {
 		name = strings.ReplaceAll(cfg.ServerListenerResourceNameTemplate, "%s", lis.Addr().String())
 	}
 
-	// Register an LDS watch using our xdsClient, and specify the listening
-	// address as the resource name.
-	cancelWatch := s.xdsC.WatchListener(name, func(update xdsclient.ListenerUpdate, err error) {
-		s.handleListenerUpdate(listenerUpdate{
-			lw:         lw,
-			name:       name,
-			lds:        update,
-			err:        err,
-			goodUpdate: goodUpdate,
-		})
+	// Create a listenerWrapper which handles all functionality required by
+	// this particular instance of Serve().
+	lw, goodUpdateCh := server.NewListenerWrapper(server.ListenerWrapperParams{
+		Listener:             lis,
+		ListenerResourceName: name,
+		XDSCredsInUse:        s.xdsCredsInUse,
+		XDSClient:            s.xdsC,
 	})
-	s.logger.Infof("Watch started on resource name %v", name)
-	lw.cancelWatch = func() {
-		cancelWatch()
-		s.logger.Infof("Watch cancelled on resource name %v", name)
-	}
 
 	// Block until a good LDS response is received or the server is stopped.
 	select {
@@ -260,155 +223,10 @@ func (s *GRPCServer) newListenerWrapper(lis net.Listener) (*listenerWrapper, err
 		// need to explicitly close the listener. Cancellation of the xDS watch
 		// is handled by the listenerWrapper.
 		lw.Close()
-		return nil, nil
-	case <-goodUpdate.Done():
-	}
-	return lw, nil
-}
-
-// listenerUpdate wraps the information received from a registered LDS watcher.
-type listenerUpdate struct {
-	lw         *listenerWrapper         // listener associated with this watch
-	name       string                   // resource name being watched
-	lds        xdsclient.ListenerUpdate // received update
-	err        error                    // received error
-	goodUpdate *grpcsync.Event          // event to fire upon a good update
-}
-
-func (s *GRPCServer) handleListenerUpdate(update listenerUpdate) {
-	if update.lw.closed.HasFired() {
-		s.logger.Warningf("Resource %q received update: %v with error: %v, after for listener was closed", update.name, update.lds, update.err)
-		return
-	}
-
-	if update.err != nil {
-		// We simply log an error here and hope we get a successful update
-		// in the future. The error could be because of a timeout or an
-		// actual error, like the requested resource not found. In any case,
-		// it is fine for the server to hang indefinitely until Stop() is
-		// called.
-		s.logger.Warningf("Received error for resource %q: %+v", update.name, update.err)
-		return
-	}
-	s.logger.Infof("Received update for resource %q: %+v", update.name, update.lds.String())
-
-	// Make sure that the socket address on the received Listener resource
-	// matches the address of the net.Listener passed to us by the user. This
-	// check is done here instead of at the xdsClient layer because of the
-	// following couple of reasons:
-	// - xdsClient cannot know the listening address of every listener in the
-	//   system, and hence cannot perform this check.
-	// - this is a very context-dependent check and only the server has the
-	//   appropriate context to perform this check.
-	//
-	// What this means is that the xdsClient has ACKed a resource which is going
-	// to push the server into a "not serving" state. This is not ideal, but
-	// this is what we have decided to do. See gRPC A36 for more details.
-	// TODO(easwars): Switch to "not serving" if the host:port does not match.
-	lisAddr := update.lw.Listener.Addr().String()
-	addr, port, err := net.SplitHostPort(lisAddr)
-	if err != nil {
-		// This is never expected to return a non-nil error since we have made
-		// sure that the listener is a TCP listener at the beginning of Serve().
-		// This is simply paranoia.
-		s.logger.Warningf("Local listener address %q failed to parse as IP:port: %v", lisAddr, err)
-		return
-	}
-	ilc := update.lds.InboundListenerCfg
-	if ilc == nil {
-		s.logger.Warningf("Missing host:port in Listener updates")
-		return
-	}
-	if ilc.Address != addr || ilc.Port != port {
-		s.logger.Warningf("Received host:port (%s:%d) in Listener update does not match local listening address: %s", ilc.Address, ilc.Port, lisAddr)
-		return
-	}
-
-	if err := s.handleSecurityConfig(update.lds.SecurityCfg, update.lw); err != nil {
-		s.logger.Warningf("Invalid security config update: %v", err)
-		return
-	}
-
-	// If we got all the way here, it means the received update was a good one.
-	update.goodUpdate.Fire()
-}
-
-func (s *GRPCServer) handleSecurityConfig(config *xdsclient.SecurityConfig, lw *listenerWrapper) error {
-	// If xdsCredentials are not in use, i.e, the user did not want to get
-	// security configuration from the control plane, we should not be acting on
-	// the received security config here. Doing so poses a security threat.
-	if !s.xdsCredsInUse {
 		return nil
+	case <-goodUpdateCh:
 	}
-
-	// Security config being nil is a valid case where the control plane has
-	// not sent any security configuration. The xdsCredentials implementation
-	// handles this by delegating to its fallback credentials.
-	if config == nil {
-		// We need to explicitly set the fields to nil here since this might be
-		// a case of switching from a good security configuration to an empty
-		// one where fallback credentials are to be used.
-		lw.xdsHI.SetRootCertProvider(nil)
-		lw.xdsHI.SetIdentityCertProvider(nil)
-		lw.xdsHI.SetRequireClientCert(false)
-		return nil
-	}
-
-	cpc := s.xdsC.BootstrapConfig().CertProviderConfigs
-	// Identity provider is mandatory on the server side.
-	identityProvider, err := buildProvider(cpc, config.IdentityInstanceName, config.IdentityCertName, true, false)
-	if err != nil {
-		return err
-	}
-
-	// A root provider is required only when doing mTLS.
-	var rootProvider certprovider.Provider
-	if config.RootInstanceName != "" {
-		rootProvider, err = buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Close the old providers and cache the new ones.
-	lw.providerMu.Lock()
-	if lw.cachedIdentity != nil {
-		lw.cachedIdentity.Close()
-	}
-	if lw.cachedRoot != nil {
-		lw.cachedRoot.Close()
-	}
-	lw.cachedRoot = rootProvider
-	lw.cachedIdentity = identityProvider
-
-	// We set all fields here, even if some of them are nil, since they
-	// could have been non-nil earlier.
-	lw.xdsHI.SetRootCertProvider(rootProvider)
-	lw.xdsHI.SetIdentityCertProvider(identityProvider)
-	lw.xdsHI.SetRequireClientCert(config.RequireClientCert)
-	lw.providerMu.Unlock()
-
-	return nil
-}
-
-func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
-	cfg, ok := configs[instanceName]
-	if !ok {
-		return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName)
-	}
-	provider, err := cfg.Build(certprovider.BuildOptions{
-		CertName:     certName,
-		WantIdentity: wantIdentity,
-		WantRoot:     wantRoot,
-	})
-	if err != nil {
-		// This error is not expected since the bootstrap process parses the
-		// config and makes sure that it is acceptable to the plugin. Still, it
-		// is possible that the plugin parses the config successfully, but its
-		// Build() method errors out.
-		return nil, fmt.Errorf("failed to get security plugin instance (%+v): %v", cfg, err)
-	}
-	return provider, nil
+	return s.gs.Serve(lw)
 }
 
 // Stop stops the underlying gRPC server. It immediately closes all open
@@ -449,105 +267,3 @@ func xdsUnaryInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServ
 func xdsStreamInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
 	return handler(srv, ss)
 }
-
-// listenerWrapper wraps the net.Listener associated with the listening address
-// passed to Serve(). It also contains all other state associated with this
-// particular invocation of Serve().
-type listenerWrapper struct {
-	net.Listener
-	cancelWatch func()
-
-	// A small race exists in the xdsClient code where an xDS response is
-	// received while the user is calling cancel(). In this small window the
-	// registered callback can be called after the watcher is canceled. We avoid
-	// processing updates received in callbacks once the listener is closed, to
-	// make sure that we do not process updates received during this race
-	// window.
-	closed *grpcsync.Event
-
-	// The certificate providers are cached here to that they can be closed when
-	// a new provider is to be created.
-	providerMu     sync.Mutex
-	cachedRoot     certprovider.Provider
-	cachedIdentity certprovider.Provider
-
-	// Wraps all information required by the xds handshaker.
-	xdsHI *xdsinternal.HandshakeInfo
-}
-
-// Accept blocks on an Accept() on the underlying listener, and wraps the
-// returned net.Conn with the configured certificate providers.
-func (l *listenerWrapper) Accept() (net.Conn, error) {
-	c, err := l.Listener.Accept()
-	if err != nil {
-		return nil, err
-	}
-	return &conn{Conn: c, xdsHI: l.xdsHI}, nil
-}
-
-// Close closes the underlying listener. It also cancels the xDS watch
-// registered in Serve() and closes any certificate provider instances created
-// based on security configuration received in the LDS response.
-func (l *listenerWrapper) Close() error {
-	l.closed.Fire()
-	l.Listener.Close()
-	if l.cancelWatch != nil {
-		l.cancelWatch()
-	}
-
-	l.providerMu.Lock()
-	if l.cachedIdentity != nil {
-		l.cachedIdentity.Close()
-		l.cachedIdentity = nil
-	}
-	if l.cachedRoot != nil {
-		l.cachedRoot.Close()
-		l.cachedRoot = nil
-	}
-	l.providerMu.Unlock()
-
-	return nil
-}
-
-// conn is a thin wrapper around a net.Conn returned by Accept().
-type conn struct {
-	net.Conn
-
-	// This is the same HandshakeInfo as stored in the listenerWrapper that
-	// created this conn. The former updates the HandshakeInfo whenever it
-	// receives new security configuration.
-	xdsHI *xdsinternal.HandshakeInfo
-
-	// The connection deadline as configured by the grpc.Server on the rawConn
-	// that is returned by a call to Accept(). This is set to the connection
-	// timeout value configured by the user (or to a default value) before
-	// initiating the transport credential handshake, and set to zero after
-	// completing the HTTP2 handshake.
-	deadlineMu sync.Mutex
-	deadline   time.Time
-}
-
-// SetDeadline makes a copy of the passed in deadline and forwards the call to
-// the underlying rawConn.
-func (c *conn) SetDeadline(t time.Time) error {
-	c.deadlineMu.Lock()
-	c.deadline = t
-	c.deadlineMu.Unlock()
-	return c.Conn.SetDeadline(t)
-}
-
-// GetDeadline returns the configured deadline. This will be invoked by the
-// ServerHandshake() method of the XdsCredentials, which needs a deadline to
-// pass to the certificate provider.
-func (c *conn) GetDeadline() time.Time {
-	c.deadlineMu.Lock()
-	t := c.deadline
-	c.deadlineMu.Unlock()
-	return t
-}
-
-// XDSHandshakeInfo returns a pointer to the HandshakeInfo stored in conn. This
-// will be invoked by the ServerHandshake() method of the XdsCredentials.
-func (c *conn) XDSHandshakeInfo() *xdsinternal.HandshakeInfo {
-	return c.xdsHI
-}
diff --git a/xds/server_test.go b/xds/server_test.go
index 8b837c4c9597..384ef71ba722 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -182,8 +182,14 @@ var (
 )
 
 func init() {
-	fpb1 = &fakeProviderBuilder{name: fakeProvider1Name}
-	fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
+	fpb1 = &fakeProviderBuilder{
+		name:    fakeProvider1Name,
+		buildCh: testutils.NewChannel(),
+	}
+	fpb2 = &fakeProviderBuilder{
+		name:    fakeProvider2Name,
+		buildCh: testutils.NewChannel(),
+	}
 	cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111")
 	cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222")
 	certProviderConfigs = map[string]*certprovider.BuildableConfig{
@@ -197,7 +203,8 @@ func init() {
 // fakeProviderBuilder builds new instances of fakeProvider and interprets the
 // config provided to it as a string.
 type fakeProviderBuilder struct {
-	name string
+	name    string
+	buildCh *testutils.Channel
 }
 
 func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.BuildableConfig, error) {
@@ -206,6 +213,7 @@ func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.Bui
 		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
 	}
 	return certprovider.NewBuildableConfig(b.name, []byte(s), func(certprovider.BuildOptions) certprovider.Provider {
+		b.buildCh.Send(nil)
 		return &fakeProvider{
 			Distributor: certprovider.NewDistributor(),
 			config:      s,
@@ -229,9 +237,9 @@ func (p *fakeProvider) Close() {
 	p.Distributor.Stop()
 }
 
-// setupOverrides sets up overrides for bootstrap config, new xdsClient creation,
-// new gRPC.Server creation, and certificate provider creation.
-func setupOverrides() (*fakeGRPCServer, *testutils.Channel, *testutils.Channel, func()) {
+// setupOverrides sets up overrides for bootstrap config, new xdsClient creation
+// and new gRPC.Server creation.
+func setupOverrides() (*fakeGRPCServer, *testutils.Channel, func()) {
 	clientCh := testutils.NewChannel()
 	origNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) {
@@ -251,18 +259,9 @@ func setupOverrides() (*fakeGRPCServer, *testutils.Channel, *testutils.Channel,
 	origNewGRPCServer := newGRPCServer
 	newGRPCServer = func(opts ...grpc.ServerOption) grpcServerInterface { return fs }
 
-	providerCh := testutils.NewChannel()
-	origBuildProvider := buildProvider
-	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
-		p, err := origBuildProvider(c, id, cert, wi, wr)
-		providerCh.Send(nil)
-		return p, err
-	}
-
-	return fs, clientCh, providerCh, func() {
+	return fs, clientCh, func() {
 		newXDSClient = origNewXDSClient
 		newGRPCServer = origNewGRPCServer
-		buildProvider = origBuildProvider
 	}
 }
 
@@ -270,7 +269,7 @@ func setupOverrides() (*fakeGRPCServer, *testutils.Channel, *testutils.Channel,
 // one. Tests that use xdsCredentials need a real grpc.Server instead of a fake
 // one, because the xDS-enabled server needs to read configured creds from the
 // underlying grpc.Server to confirm whether xdsCreds were configured.
-func setupOverridesForXDSCreds(includeCertProviderCfg bool) (*testutils.Channel, *testutils.Channel, func()) {
+func setupOverridesForXDSCreds(includeCertProviderCfg bool) (*testutils.Channel, func()) {
 	clientCh := testutils.NewChannel()
 	origNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) {
@@ -289,18 +288,7 @@ func setupOverridesForXDSCreds(includeCertProviderCfg bool) (*testutils.Channel,
 		return c, nil
 	}
 
-	providerCh := testutils.NewChannel()
-	origBuildProvider := buildProvider
-	buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
-		p, err := origBuildProvider(c, id, cert, wi, wr)
-		providerCh.Send(nil)
-		return p, err
-	}
-
-	return clientCh, providerCh, func() {
-		newXDSClient = origNewXDSClient
-		buildProvider = origBuildProvider
-	}
+	return clientCh, func() { newXDSClient = origNewXDSClient }
 }
 
 // TestServeSuccess tests the successful case of calling Serve().
@@ -312,7 +300,7 @@ func setupOverridesForXDSCreds(includeCertProviderCfg bool) (*testutils.Channel,
 // 4. Push a good response from the xdsClient, and make sure that Serve() on the
 // 	  underlying grpc.Server is called.
 func (s) TestServeSuccess(t *testing.T) {
-	fs, clientCh, _, cleanup := setupOverrides()
+	fs, clientCh, cleanup := setupOverrides()
 	defer cleanup()
 
 	server := NewGRPCServer()
@@ -397,7 +385,7 @@ func (s) TestServeSuccess(t *testing.T) {
 // is received. This should cause Serve() to exit before calling Serve() on the
 // underlying grpc.Server.
 func (s) TestServeWithStop(t *testing.T) {
-	fs, clientCh, _, cleanup := setupOverrides()
+	fs, clientCh, cleanup := setupOverrides()
 	defer cleanup()
 
 	// Note that we are not deferring the Stop() here since we explicitly call
@@ -596,7 +584,7 @@ func (s) TestServeNewClientFailure(t *testing.T) {
 // server is not configured with xDS credentials. Verifies that the security
 // config received as part of a Listener update is not acted upon.
 func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
-	fs, clientCh, providerCh, cleanup := setupOverrides()
+	fs, clientCh, cleanup := setupOverrides()
 	defer cleanup()
 
 	server := NewGRPCServer()
@@ -641,14 +629,18 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 	addr, port := splitHostPort(lis.Addr().String())
 	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
 		RouteConfigName: "routeconfig",
-		SecurityCfg: &xdsclient.SecurityConfig{
-			RootInstanceName:     "default1",
-			IdentityInstanceName: "default2",
-			RequireClientCert:    true,
-		},
 		InboundListenerCfg: &xdsclient.InboundListenerConfig{
 			Address: addr,
 			Port:    port,
+			FilterChains: []*xdsclient.FilterChain{
+				{
+					SecurityCfg: &xdsclient.SecurityConfig{
+						RootInstanceName:     "default1",
+						IdentityInstanceName: "default2",
+						RequireClientCert:    true,
+					},
+				},
+			},
 		},
 	}, nil)
 	if _, err := fs.serveCh.Receive(ctx); err != nil {
@@ -656,10 +648,8 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 	}
 
 	// Make sure the security configuration is not acted upon.
-	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer sCancel()
-	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Fatalf("certificate provider created when no xDS creds were specified")
+	if err := verifyCertProviderNotCreated(); err != nil {
+		t.Fatal(err)
 	}
 }
 
@@ -667,7 +657,7 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 // server is configured with xDS credentials, but receives a Listener update
 // with an error. Verifies that no certificate providers are created.
 func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
-	clientCh, providerCh, cleanup := setupOverridesForXDSCreds(true)
+	clientCh, cleanup := setupOverridesForXDSCreds(true)
 	defer cleanup()
 
 	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
@@ -713,7 +703,20 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
 
 	// Push an error to the registered listener watch callback and make sure
 	// that Serve does not return.
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, errors.New("LDS error"))
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		RouteConfigName: "routeconfig",
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			FilterChains: []*xdsclient.FilterChain{
+				{
+					SecurityCfg: &xdsclient.SecurityConfig{
+						RootInstanceName:     "default1",
+						IdentityInstanceName: "default2",
+						RequireClientCert:    true,
+					},
+				},
+			},
+		},
+	}, errors.New("LDS error"))
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
 	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
@@ -721,93 +724,21 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
 	}
 
 	// Also make sure that no certificate providers are created.
-	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer sCancel()
-	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Fatalf("certificate provider created when no xDS creds were specified")
+	if err := verifyCertProviderNotCreated(); err != nil {
+		t.Fatal(err)
 	}
 }
 
-func (s) TestHandleListenerUpdate_ClosedListener(t *testing.T) {
-	clientCh, providerCh, cleanup := setupOverridesForXDSCreds(true)
-	defer cleanup()
-
-	xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
-	if err != nil {
-		t.Fatalf("failed to create xds server credentials: %v", err)
-	}
-
-	server := NewGRPCServer(grpc.Creds(xdsCreds))
-	defer server.Stop()
-
-	lis, err := xdstestutils.LocalTCPListener()
-	if err != nil {
-		t.Fatalf("xdstestutils.LocalTCPListener() failed: %v", err)
-	}
-
-	// Call Serve() in a goroutine, and push on a channel when Serve returns.
-	serveDone := testutils.NewChannel()
-	go func() { serveDone.Send(server.Serve(lis)) }()
-
-	// Wait for an xdsClient to be created.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	c, err := clientCh.Receive(ctx)
-	if err != nil {
-		t.Fatalf("error when waiting for new xdsClient to be created: %v", err)
-	}
-	client := c.(*fakeclient.Client)
-
-	// Wait for a listener watch to be registered on the xdsClient.
-	name, err := client.WaitForWatchListener(ctx)
-	if err != nil {
-		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
-	}
-	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
-	if name != wantName {
-		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
-	}
-
-	// Push a good update to the registered listener watch callback. This will
-	// unblock the xds-enabled server which is waiting for a good listener
-	// update before calling Serve() on the underlying grpc.Server.
-	addr, port := splitHostPort(lis.Addr().String())
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
-		RouteConfigName: "routeconfig",
-		SecurityCfg:     &xdsclient.SecurityConfig{IdentityInstanceName: "default2"},
-		InboundListenerCfg: &xdsclient.InboundListenerConfig{
-			Address: addr,
-			Port:    port,
-		},
-	}, nil)
-	if _, err := providerCh.Receive(ctx); err != nil {
-		t.Fatal("error when waiting for certificate provider to be created")
-	}
-
-	// Close the listener passed to Serve(), and wait for the latter to return a
-	// non-nil error.
-	lis.Close()
-	v, err := serveDone.Receive(ctx)
-	if err != nil {
-		t.Fatalf("error when waiting for Serve() to exit: %v", err)
-	}
-	if err, ok := v.(error); !ok || err == nil {
-		t.Fatal("Serve() did not exit with error")
-	}
-
-	// Push another listener update and make sure that no certificate providers
-	// are created.
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
-		RouteConfigName: "routeconfig",
-		SecurityCfg:     &xdsclient.SecurityConfig{IdentityInstanceName: "default1"},
-		InboundListenerCfg: &xdsclient.InboundListenerConfig{
-			Address: addr,
-			Port:    port,
-		},
-	}, nil)
+func verifyCertProviderNotCreated() error {
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
-		t.Fatalf("certificate provider created when no xDS creds were specified")
+	if _, err := fpb1.buildCh.Receive(sCtx); err != context.DeadlineExceeded {
+		return errors.New("certificate provider created when no xDS creds were specified")
 	}
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fpb2.buildCh.Receive(sCtx); err != context.DeadlineExceeded {
+		return errors.New("certificate provider created when no xDS creds were specified")
+	}
+	return nil
 }

From 2f7f1f6c22e925c849e56bbe0823961588f9f9ec Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 11 Mar 2021 12:07:48 -0800
Subject: [PATCH 383/481] rls: update pb.gos after
 https://github.com/grpc/grpc-proto/pull/92 (#4257)

---
 .../internal/proto/grpc_lookup_v1/rls.pb.go   |  73 ++--
 .../proto/grpc_lookup_v1/rls_config.pb.go     | 319 +++++++++++++-----
 2 files changed, 276 insertions(+), 116 deletions(-)

diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index d48a1a6de84d..7741e6649180 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -46,9 +46,15 @@ type RouteLookupRequest struct {
 
 	// Full host name of the target server, e.g. firestore.googleapis.com.
 	// Only set for gRPC requests; HTTP requests must use key_map explicitly.
+	// Deprecated in favor of setting key_map keys with GrpcKeyBuilder.extra_keys.
+	//
+	// Deprecated: Do not use.
 	Server string `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"`
 	// Full path of the request, i.e. "/service/method".
 	// Only set for gRPC requests; HTTP requests must use key_map explicitly.
+	// Deprecated in favor of setting key_map keys with GrpcKeyBuilder.extra_keys.
+	//
+	// Deprecated: Do not use.
 	Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"`
 	// Target type allows the client to specify what kind of target format it
 	// would like from RLS to allow it to find the regional server, e.g. "grpc".
@@ -89,6 +95,7 @@ func (*RouteLookupRequest) Descriptor() ([]byte, []int) {
 	return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0}
 }
 
+// Deprecated: Do not use.
 func (x *RouteLookupRequest) GetServer() string {
 	if x != nil {
 		return x.Server
@@ -96,6 +103,7 @@ func (x *RouteLookupRequest) GetServer() string {
 	return ""
 }
 
+// Deprecated: Do not use.
 func (x *RouteLookupRequest) GetPath() string {
 	if x != nil {
 		return x.Path
@@ -183,40 +191,41 @@ var File_grpc_lookup_v1_rls_proto protoreflect.FileDescriptor
 var file_grpc_lookup_v1_rls_proto_rawDesc = []byte{
 	0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x76, 0x31,
 	0x2f, 0x72, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63,
-	0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x22, 0xe5, 0x01, 0x0a, 0x12, 0x52,
+	0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x22, 0xed, 0x01, 0x0a, 0x12, 0x52,
 	0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74,
-	0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a,
-	0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x47,
-	0x0a, 0x07, 0x6b, 0x65, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x2e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31,
-	0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x2e, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
-	0x06, 0x6b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x1a, 0x39, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x4d, 0x61,
-	0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
-	0x38, 0x01, 0x22, 0x5e, 0x0a, 0x13, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
-	0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72,
-	0x67, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67,
-	0x65, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61,
-	0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72,
-	0x44, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67,
-	0x65, 0x74, 0x32, 0x6e, 0x0a, 0x12, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
-	0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74,
-	0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
+	0x74, 0x12, 0x1a, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x16, 0x0a,
+	0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52,
+	0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f,
+	0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67,
+	0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x47, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x5f, 0x6d, 0x61,
+	0x70, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
 	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f,
-	0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72,
-	0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75,
-	0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x00, 0x42, 0x4d, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f,
-	0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42, 0x08, 0x52, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74,
-	0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61,
-	0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b,
-	0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76,
-	0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4b, 0x65, 0x79, 0x4d,
+	0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x1a,
+	0x39, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
+	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
+	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5e, 0x0a, 0x13, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03,
+	0x28, 0x09, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68,
+	0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01,
+	0x10, 0x02, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x32, 0x6e, 0x0a, 0x12, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x12, 0x58, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x12,
+	0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31,
+	0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75,
+	0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4d, 0x0a, 0x11, 0x69, 0x6f,
+	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42,
+	0x08, 0x52, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67,
+	0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f,
+	0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
 }
 
 var (
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
index 6b0924b335fa..414b74cdb3b5 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go
@@ -50,6 +50,9 @@ type NameMatcher struct {
 	unknownFields protoimpl.UnknownFields
 
 	// The name that will be used in the RLS key_map to refer to this value.
+	// If required_match is true, you may omit this field or set it to an empty
+	// string, in which case the matcher will require a match, but won't update
+	// the key_map.
 	Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
 	// Ordered list of names (headers or query parameter names) that can supply
 	// this value; the first one with a non-empty value is used.
@@ -118,11 +121,17 @@ type GrpcKeyBuilder struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Names []*GrpcKeyBuilder_Name `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"`
+	Names     []*GrpcKeyBuilder_Name    `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"`
+	ExtraKeys *GrpcKeyBuilder_ExtraKeys `protobuf:"bytes,3,opt,name=extra_keys,json=extraKeys,proto3" json:"extra_keys,omitempty"`
 	// Extract keys from all listed headers.
 	// For gRPC, it is an error to specify "required_match" on the NameMatcher
 	// protos.
 	Headers []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"`
+	// You can optionally set one or more specific key/value pairs to be added to
+	// the key_map.  This can be useful to identify which builder built the key,
+	// for example if you are suppressing the actual method, but need to
+	// separately cache and request all the matched methods.
+	ConstantKeys map[string]string `protobuf:"bytes,4,rep,name=constant_keys,json=constantKeys,proto3" json:"constant_keys,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (x *GrpcKeyBuilder) Reset() {
@@ -164,6 +173,13 @@ func (x *GrpcKeyBuilder) GetNames() []*GrpcKeyBuilder_Name {
 	return nil
 }
 
+func (x *GrpcKeyBuilder) GetExtraKeys() *GrpcKeyBuilder_ExtraKeys {
+	if x != nil {
+		return x.ExtraKeys
+	}
+	return nil
+}
+
 func (x *GrpcKeyBuilder) GetHeaders() []*NameMatcher {
 	if x != nil {
 		return x.Headers
@@ -171,6 +187,13 @@ func (x *GrpcKeyBuilder) GetHeaders() []*NameMatcher {
 	return nil
 }
 
+func (x *GrpcKeyBuilder) GetConstantKeys() map[string]string {
+	if x != nil {
+		return x.ConstantKeys
+	}
+	return nil
+}
+
 // An HttpKeyBuilder applies to a given HTTP URL and headers.
 //
 // Path and host patterns use the matching syntax from gRPC transcoding to
@@ -245,6 +268,11 @@ type HttpKeyBuilder struct {
 	// to match.  If a given header appears multiple times in the request we will
 	// report it as a comma-separated string, in standard HTTP fashion.
 	Headers []*NameMatcher `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"`
+	// You can optionally set one or more specific key/value pairs to be added to
+	// the key_map.  This can be useful to identify which builder built the key,
+	// for example if you are suppressing a lot of information from the URL, but
+	// need to separately cache and request URLs with that content.
+	ConstantKeys map[string]string `protobuf:"bytes,5,rep,name=constant_keys,json=constantKeys,proto3" json:"constant_keys,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (x *HttpKeyBuilder) Reset() {
@@ -307,6 +335,13 @@ func (x *HttpKeyBuilder) GetHeaders() []*NameMatcher {
 	return nil
 }
 
+func (x *HttpKeyBuilder) GetConstantKeys() map[string]string {
+	if x != nil {
+		return x.ConstantKeys
+	}
+	return nil
+}
+
 type RouteLookupConfig struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -510,6 +545,75 @@ func (x *GrpcKeyBuilder_Name) GetMethod() string {
 	return ""
 }
 
+// If you wish to include the host, service, or method names as keys in the
+// generated RouteLookupRequest, specify key names to use in the extra_keys
+// submessage. If a key name is empty, no key will be set for that value.
+// If this submessage is specified, the normal host/path fields will be left
+// unset in the RouteLookupRequest. We are deprecating host/path in the
+// RouteLookupRequest, so services should migrate to the ExtraKeys approach.
+type GrpcKeyBuilder_ExtraKeys struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Host    string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"`
+	Service string `protobuf:"bytes,2,opt,name=service,proto3" json:"service,omitempty"`
+	Method  string `protobuf:"bytes,3,opt,name=method,proto3" json:"method,omitempty"`
+}
+
+func (x *GrpcKeyBuilder_ExtraKeys) Reset() {
+	*x = GrpcKeyBuilder_ExtraKeys{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GrpcKeyBuilder_ExtraKeys) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GrpcKeyBuilder_ExtraKeys) ProtoMessage() {}
+
+func (x *GrpcKeyBuilder_ExtraKeys) ProtoReflect() protoreflect.Message {
+	mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GrpcKeyBuilder_ExtraKeys.ProtoReflect.Descriptor instead.
+func (*GrpcKeyBuilder_ExtraKeys) Descriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1, 1}
+}
+
+func (x *GrpcKeyBuilder_ExtraKeys) GetHost() string {
+	if x != nil {
+		return x.Host
+	}
+	return ""
+}
+
+func (x *GrpcKeyBuilder_ExtraKeys) GetService() string {
+	if x != nil {
+		return x.Service
+	}
+	return ""
+}
+
+func (x *GrpcKeyBuilder_ExtraKeys) GetMethod() string {
+	if x != nil {
+		return x.Method
+	}
+	return ""
+}
+
 var File_grpc_lookup_v1_rls_config_proto protoreflect.FileDescriptor
 
 var file_grpc_lookup_v1_rls_config_proto_rawDesc = []byte{
@@ -524,72 +628,101 @@ var file_grpc_lookup_v1_rls_config_proto_rawDesc = []byte{
 	0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x71, 0x75,
 	0x69, 0x72, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
 	0x52, 0x0d, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22,
-	0xbc, 0x01, 0x0a, 0x0e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64,
+	0xf0, 0x03, 0x0a, 0x0e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64,
 	0x65, 0x72, 0x12, 0x39, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
 	0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e,
 	0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65,
-	0x72, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x35, 0x0a,
-	0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e,
-	0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65, 0x61,
-	0x64, 0x65, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07,
+	0x72, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x47, 0x0a,
+	0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e,
+	0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65,
+	0x72, 0x2e, 0x45, 0x78, 0x74, 0x72, 0x61, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x09, 0x65, 0x78, 0x74,
+	0x72, 0x61, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72,
+	0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74,
+	0x63, 0x68, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x55, 0x0a,
+	0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b,
+	0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69,
+	0x6c, 0x64, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x4b, 0x65, 0x79,
+	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74,
+	0x4b, 0x65, 0x79, 0x73, 0x1a, 0x38, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07,
 	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73,
 	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x22, 0xd9,
-	0x01, 0x0a, 0x0e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65,
-	0x72, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72,
-	0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x6f, 0x73, 0x74, 0x50, 0x61,
-	0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70,
-	0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70,
-	0x61, 0x74, 0x68, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x46, 0x0a, 0x10, 0x71,
-	0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18,
-	0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f,
-	0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68,
-	0x65, 0x72, 0x52, 0x0f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74,
-	0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04,
-	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b,
-	0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65,
-	0x72, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x22, 0xa6, 0x04, 0x0a, 0x11, 0x52,
-	0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x12, 0x49, 0x0a, 0x10, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c,
-	0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70,
-	0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70,
-	0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x0f, 0x68, 0x74, 0x74, 0x70,
-	0x4b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x67,
-	0x72, 0x70, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x18,
-	0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f,
-	0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75,
-	0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x62, 0x75,
-	0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
-	0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
-	0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, 0x0a,
-	0x16, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f,
-	0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e,
-	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
-	0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
-	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x32,
-	0x0a, 0x07, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
-	0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41,
-	0x67, 0x65, 0x12, 0x36, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x61, 0x67, 0x65, 0x18,
-	0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
-	0x52, 0x08, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x41, 0x67, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x61,
-	0x63, 0x68, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07,
-	0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x42,
-	0x79, 0x74, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x74, 0x61,
-	0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c,
-	0x69, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x66,
-	0x61, 0x75, 0x6c, 0x74, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74,
-	0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x1b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f,
-	0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74,
-	0x65, 0x67, 0x79, 0x42, 0x53, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
-	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x52, 0x6c, 0x73, 0x43, 0x6f, 0x6e,
-	0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67,
-	0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72,
-	0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c,
-	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0x51,
+	0x0a, 0x09, 0x45, 0x78, 0x74, 0x72, 0x61, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x68,
+	0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12,
+	0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74,
+	0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f,
+	0x64, 0x1a, 0x3f, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x4b, 0x65, 0x79,
+	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
+	0x38, 0x01, 0x22, 0xf1, 0x02, 0x0a, 0x0e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x75,
+	0x69, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x70, 0x61,
+	0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x6f,
+	0x73, 0x74, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61,
+	0x74, 0x68, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x0c, 0x70, 0x61, 0x74, 0x68, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12,
+	0x46, 0x0a, 0x10, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74,
+	0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63,
+	0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d,
+	0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x0f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x50, 0x61, 0x72,
+	0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65,
+	0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+	0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61,
+	0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x55,
+	0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18,
+	0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x75,
+	0x69, 0x6c, 0x64, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x4b, 0x65,
+	0x79, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e,
+	0x74, 0x4b, 0x65, 0x79, 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e,
+	0x74, 0x4b, 0x65, 0x79, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
+	0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa6, 0x04, 0x0a, 0x11, 0x52, 0x6f, 0x75, 0x74, 0x65,
+	0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x49, 0x0a, 0x10,
+	0x68, 0x74, 0x74, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f,
+	0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42,
+	0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x0f, 0x68, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x62,
+	0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x67, 0x72, 0x70, 0x63, 0x5f,
+	0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e,
+	0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65,
+	0x72, 0x52, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65,
+	0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b,
+	0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, 0x0a, 0x16, 0x6c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65,
+	0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61,
+	0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41, 0x67, 0x65, 0x12, 0x36,
+	0x0a, 0x09, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x73, 0x74,
+	0x61, 0x6c, 0x65, 0x41, 0x67, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f,
+	0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03,
+	0x52, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73,
+	0x12, 0x23, 0x0a, 0x0d, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74,
+	0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x61,
+	0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74,
+	0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64,
+	0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4a, 0x04, 0x08, 0x0a,
+	0x10, 0x0b, 0x52, 0x1b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x63,
+	0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x42,
+	0x53, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75,
+	0x70, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x52, 0x6c, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67,
+	0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75,
+	0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -604,30 +737,36 @@ func file_grpc_lookup_v1_rls_config_proto_rawDescGZIP() []byte {
 	return file_grpc_lookup_v1_rls_config_proto_rawDescData
 }
 
-var file_grpc_lookup_v1_rls_config_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_grpc_lookup_v1_rls_config_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
 var file_grpc_lookup_v1_rls_config_proto_goTypes = []interface{}{
-	(*NameMatcher)(nil),         // 0: grpc.lookup.v1.NameMatcher
-	(*GrpcKeyBuilder)(nil),      // 1: grpc.lookup.v1.GrpcKeyBuilder
-	(*HttpKeyBuilder)(nil),      // 2: grpc.lookup.v1.HttpKeyBuilder
-	(*RouteLookupConfig)(nil),   // 3: grpc.lookup.v1.RouteLookupConfig
-	(*GrpcKeyBuilder_Name)(nil), // 4: grpc.lookup.v1.GrpcKeyBuilder.Name
-	(*durationpb.Duration)(nil), // 5: google.protobuf.Duration
+	(*NameMatcher)(nil),              // 0: grpc.lookup.v1.NameMatcher
+	(*GrpcKeyBuilder)(nil),           // 1: grpc.lookup.v1.GrpcKeyBuilder
+	(*HttpKeyBuilder)(nil),           // 2: grpc.lookup.v1.HttpKeyBuilder
+	(*RouteLookupConfig)(nil),        // 3: grpc.lookup.v1.RouteLookupConfig
+	(*GrpcKeyBuilder_Name)(nil),      // 4: grpc.lookup.v1.GrpcKeyBuilder.Name
+	(*GrpcKeyBuilder_ExtraKeys)(nil), // 5: grpc.lookup.v1.GrpcKeyBuilder.ExtraKeys
+	nil,                              // 6: grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntry
+	nil,                              // 7: grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntry
+	(*durationpb.Duration)(nil),      // 8: google.protobuf.Duration
 }
 var file_grpc_lookup_v1_rls_config_proto_depIdxs = []int32{
-	4, // 0: grpc.lookup.v1.GrpcKeyBuilder.names:type_name -> grpc.lookup.v1.GrpcKeyBuilder.Name
-	0, // 1: grpc.lookup.v1.GrpcKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher
-	0, // 2: grpc.lookup.v1.HttpKeyBuilder.query_parameters:type_name -> grpc.lookup.v1.NameMatcher
-	0, // 3: grpc.lookup.v1.HttpKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher
-	2, // 4: grpc.lookup.v1.RouteLookupConfig.http_keybuilders:type_name -> grpc.lookup.v1.HttpKeyBuilder
-	1, // 5: grpc.lookup.v1.RouteLookupConfig.grpc_keybuilders:type_name -> grpc.lookup.v1.GrpcKeyBuilder
-	5, // 6: grpc.lookup.v1.RouteLookupConfig.lookup_service_timeout:type_name -> google.protobuf.Duration
-	5, // 7: grpc.lookup.v1.RouteLookupConfig.max_age:type_name -> google.protobuf.Duration
-	5, // 8: grpc.lookup.v1.RouteLookupConfig.stale_age:type_name -> google.protobuf.Duration
-	9, // [9:9] is the sub-list for method output_type
-	9, // [9:9] is the sub-list for method input_type
-	9, // [9:9] is the sub-list for extension type_name
-	9, // [9:9] is the sub-list for extension extendee
-	0, // [0:9] is the sub-list for field type_name
+	4,  // 0: grpc.lookup.v1.GrpcKeyBuilder.names:type_name -> grpc.lookup.v1.GrpcKeyBuilder.Name
+	5,  // 1: grpc.lookup.v1.GrpcKeyBuilder.extra_keys:type_name -> grpc.lookup.v1.GrpcKeyBuilder.ExtraKeys
+	0,  // 2: grpc.lookup.v1.GrpcKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher
+	6,  // 3: grpc.lookup.v1.GrpcKeyBuilder.constant_keys:type_name -> grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntry
+	0,  // 4: grpc.lookup.v1.HttpKeyBuilder.query_parameters:type_name -> grpc.lookup.v1.NameMatcher
+	0,  // 5: grpc.lookup.v1.HttpKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher
+	7,  // 6: grpc.lookup.v1.HttpKeyBuilder.constant_keys:type_name -> grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntry
+	2,  // 7: grpc.lookup.v1.RouteLookupConfig.http_keybuilders:type_name -> grpc.lookup.v1.HttpKeyBuilder
+	1,  // 8: grpc.lookup.v1.RouteLookupConfig.grpc_keybuilders:type_name -> grpc.lookup.v1.GrpcKeyBuilder
+	8,  // 9: grpc.lookup.v1.RouteLookupConfig.lookup_service_timeout:type_name -> google.protobuf.Duration
+	8,  // 10: grpc.lookup.v1.RouteLookupConfig.max_age:type_name -> google.protobuf.Duration
+	8,  // 11: grpc.lookup.v1.RouteLookupConfig.stale_age:type_name -> google.protobuf.Duration
+	12, // [12:12] is the sub-list for method output_type
+	12, // [12:12] is the sub-list for method input_type
+	12, // [12:12] is the sub-list for extension type_name
+	12, // [12:12] is the sub-list for extension extendee
+	0,  // [0:12] is the sub-list for field type_name
 }
 
 func init() { file_grpc_lookup_v1_rls_config_proto_init() }
@@ -696,6 +835,18 @@ func file_grpc_lookup_v1_rls_config_proto_init() {
 				return nil
 			}
 		}
+		file_grpc_lookup_v1_rls_config_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GrpcKeyBuilder_ExtraKeys); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -703,7 +854,7 @@ func file_grpc_lookup_v1_rls_config_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_grpc_lookup_v1_rls_config_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   5,
+			NumMessages:   8,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

From f168a3cb3bf52b839691623a343b867d9ef7a566 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 11 Mar 2021 14:17:43 -0800
Subject: [PATCH 384/481] c2p: add google-c2p resolver (#4204)

---
 credentials/alts/alts.go                 |  11 +-
 credentials/alts/utils.go                |  94 ---------
 credentials/alts/utils_test.go           |  63 ------
 internal/googlecloud/googlecloud.go      | 128 ++++++++++++
 internal/googlecloud/googlecloud_test.go |  86 ++++++++
 xds/googledirectpath/googlec2p.go        | 179 +++++++++++++++++
 xds/googledirectpath/googlec2p_test.go   | 240 +++++++++++++++++++++++
 xds/googledirectpath/utils.go            |  96 +++++++++
 xds/internal/client/singleton.go         |  44 ++++-
 xds/internal/env/env.go                  |   9 +
 10 files changed, 784 insertions(+), 166 deletions(-)
 create mode 100644 internal/googlecloud/googlecloud.go
 create mode 100644 internal/googlecloud/googlecloud_test.go
 create mode 100644 xds/googledirectpath/googlec2p.go
 create mode 100644 xds/googledirectpath/googlec2p_test.go
 create mode 100644 xds/googledirectpath/utils.go

diff --git a/credentials/alts/alts.go b/credentials/alts/alts.go
index 729c4b43b5fc..4366fbd1f24e 100644
--- a/credentials/alts/alts.go
+++ b/credentials/alts/alts.go
@@ -28,7 +28,6 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync"
 	"time"
 
 	"google.golang.org/grpc/credentials"
@@ -37,6 +36,7 @@ import (
 	"google.golang.org/grpc/credentials/alts/internal/handshaker/service"
 	altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/googlecloud"
 )
 
 const (
@@ -54,7 +54,6 @@ const (
 )
 
 var (
-	once          sync.Once
 	maxRPCVersion = &altspb.RpcProtocolVersions_Version{
 		Major: protocolVersionMaxMajor,
 		Minor: protocolVersionMaxMinor,
@@ -148,10 +147,6 @@ func NewServerCreds(opts *ServerOptions) credentials.TransportCredentials {
 }
 
 func newALTS(side core.Side, accounts []string, hsAddress string) credentials.TransportCredentials {
-	once.Do(func() {
-		vmOnGCP = isRunningOnGCP()
-	})
-
 	if hsAddress == "" {
 		hsAddress = hypervisorHandshakerServiceAddress
 	}
@@ -168,7 +163,7 @@ func newALTS(side core.Side, accounts []string, hsAddress string) credentials.Tr
 
 // ClientHandshake implements the client side handshake protocol.
 func (g *altsTC) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) {
-	if !vmOnGCP {
+	if !googlecloud.OnGCE() {
 		return nil, nil, ErrUntrustedPlatform
 	}
 
@@ -223,7 +218,7 @@ func (g *altsTC) ClientHandshake(ctx context.Context, addr string, rawConn net.C
 
 // ServerHandshake implements the server side ALTS handshaker.
 func (g *altsTC) ServerHandshake(rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) {
-	if !vmOnGCP {
+	if !googlecloud.OnGCE() {
 		return nil, nil, ErrUntrustedPlatform
 	}
 	// Connecting to ALTS handshaker service.
diff --git a/credentials/alts/utils.go b/credentials/alts/utils.go
index 9a300bc19aa3..cbfd056cfb14 100644
--- a/credentials/alts/utils.go
+++ b/credentials/alts/utils.go
@@ -21,14 +21,6 @@ package alts
 import (
 	"context"
 	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"regexp"
-	"runtime"
 	"strings"
 
 	"google.golang.org/grpc/codes"
@@ -36,92 +28,6 @@ import (
 	"google.golang.org/grpc/status"
 )
 
-const (
-	linuxProductNameFile     = "/sys/class/dmi/id/product_name"
-	windowsCheckCommand      = "powershell.exe"
-	windowsCheckCommandArgs  = "Get-WmiObject -Class Win32_BIOS"
-	powershellOutputFilter   = "Manufacturer"
-	windowsManufacturerRegex = ":(.*)"
-)
-
-type platformError string
-
-func (k platformError) Error() string {
-	return fmt.Sprintf("%s is not supported", string(k))
-}
-
-var (
-	// The following two variables will be reassigned in tests.
-	runningOS          = runtime.GOOS
-	manufacturerReader = func() (io.Reader, error) {
-		switch runningOS {
-		case "linux":
-			return os.Open(linuxProductNameFile)
-		case "windows":
-			cmd := exec.Command(windowsCheckCommand, windowsCheckCommandArgs)
-			out, err := cmd.Output()
-			if err != nil {
-				return nil, err
-			}
-
-			for _, line := range strings.Split(strings.TrimSuffix(string(out), "\n"), "\n") {
-				if strings.HasPrefix(line, powershellOutputFilter) {
-					re := regexp.MustCompile(windowsManufacturerRegex)
-					name := re.FindString(line)
-					name = strings.TrimLeft(name, ":")
-					return strings.NewReader(name), nil
-				}
-			}
-
-			return nil, errors.New("cannot determine the machine's manufacturer")
-		default:
-			return nil, platformError(runningOS)
-		}
-	}
-	vmOnGCP bool
-)
-
-// isRunningOnGCP checks whether the local system, without doing a network request is
-// running on GCP.
-func isRunningOnGCP() bool {
-	manufacturer, err := readManufacturer()
-	if os.IsNotExist(err) {
-		return false
-	}
-	if err != nil {
-		log.Fatalf("failure to read manufacturer information: %v", err)
-	}
-	name := string(manufacturer)
-	switch runningOS {
-	case "linux":
-		name = strings.TrimSpace(name)
-		return name == "Google" || name == "Google Compute Engine"
-	case "windows":
-		name = strings.Replace(name, " ", "", -1)
-		name = strings.Replace(name, "\n", "", -1)
-		name = strings.Replace(name, "\r", "", -1)
-		return name == "Google"
-	default:
-		log.Fatal(platformError(runningOS))
-	}
-	return false
-}
-
-func readManufacturer() ([]byte, error) {
-	reader, err := manufacturerReader()
-	if err != nil {
-		return nil, err
-	}
-	if reader == nil {
-		return nil, errors.New("got nil reader")
-	}
-	manufacturer, err := ioutil.ReadAll(reader)
-	if err != nil {
-		return nil, fmt.Errorf("failed reading %v: %v", linuxProductNameFile, err)
-	}
-	return manufacturer, nil
-}
-
 // AuthInfoFromContext extracts the alts.AuthInfo object from the given context,
 // if it exists. This API should be used by gRPC server RPC handlers to get
 // information about the communicating peer. For client-side, use grpc.Peer()
diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go
index 5b54b1d5f77c..2231de3dccc4 100644
--- a/credentials/alts/utils_test.go
+++ b/credentials/alts/utils_test.go
@@ -22,8 +22,6 @@ package alts
 
 import (
 	"context"
-	"io"
-	"os"
 	"strings"
 	"testing"
 	"time"
@@ -42,67 +40,6 @@ const (
 	defaultTestTimeout = 10 * time.Second
 )
 
-func setupManufacturerReader(testOS string, reader func() (io.Reader, error)) func() {
-	tmpOS := runningOS
-	tmpReader := manufacturerReader
-
-	// Set test OS and reader function.
-	runningOS = testOS
-	manufacturerReader = reader
-	return func() {
-		runningOS = tmpOS
-		manufacturerReader = tmpReader
-	}
-
-}
-
-func setup(testOS string, testReader io.Reader) func() {
-	reader := func() (io.Reader, error) {
-		return testReader, nil
-	}
-	return setupManufacturerReader(testOS, reader)
-}
-
-func setupError(testOS string, err error) func() {
-	reader := func() (io.Reader, error) {
-		return nil, err
-	}
-	return setupManufacturerReader(testOS, reader)
-}
-
-func (s) TestIsRunningOnGCP(t *testing.T) {
-	for _, tc := range []struct {
-		description string
-		testOS      string
-		testReader  io.Reader
-		out         bool
-	}{
-		// Linux tests.
-		{"linux: not a GCP platform", "linux", strings.NewReader("not GCP"), false},
-		{"Linux: GCP platform (Google)", "linux", strings.NewReader("Google"), true},
-		{"Linux: GCP platform (Google Compute Engine)", "linux", strings.NewReader("Google Compute Engine"), true},
-		{"Linux: GCP platform (Google Compute Engine) with extra spaces", "linux", strings.NewReader("  Google Compute Engine        "), true},
-		// Windows tests.
-		{"windows: not a GCP platform", "windows", strings.NewReader("not GCP"), false},
-		{"windows: GCP platform (Google)", "windows", strings.NewReader("Google"), true},
-		{"windows: GCP platform (Google) with extra spaces", "windows", strings.NewReader("  Google     "), true},
-	} {
-		reverseFunc := setup(tc.testOS, tc.testReader)
-		if got, want := isRunningOnGCP(), tc.out; got != want {
-			t.Errorf("%v: isRunningOnGCP()=%v, want %v", tc.description, got, want)
-		}
-		reverseFunc()
-	}
-}
-
-func (s) TestIsRunningOnGCPNoProductNameFile(t *testing.T) {
-	reverseFunc := setupError("linux", os.ErrNotExist)
-	if isRunningOnGCP() {
-		t.Errorf("ErrNotExist: isRunningOnGCP()=true, want false")
-	}
-	reverseFunc()
-}
-
 func (s) TestAuthInfoFromContext(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
diff --git a/internal/googlecloud/googlecloud.go b/internal/googlecloud/googlecloud.go
new file mode 100644
index 000000000000..d6c9e03fc4c8
--- /dev/null
+++ b/internal/googlecloud/googlecloud.go
@@ -0,0 +1,128 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package googlecloud contains internal helpful functions for google cloud.
+package googlecloud
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const (
+	linuxProductNameFile     = "/sys/class/dmi/id/product_name"
+	windowsCheckCommand      = "powershell.exe"
+	windowsCheckCommandArgs  = "Get-WmiObject -Class Win32_BIOS"
+	powershellOutputFilter   = "Manufacturer"
+	windowsManufacturerRegex = ":(.*)"
+
+	logPrefix = "[googlecloud]"
+)
+
+var (
+	// The following two variables will be reassigned in tests.
+	runningOS          = runtime.GOOS
+	manufacturerReader = func() (io.Reader, error) {
+		switch runningOS {
+		case "linux":
+			return os.Open(linuxProductNameFile)
+		case "windows":
+			cmd := exec.Command(windowsCheckCommand, windowsCheckCommandArgs)
+			out, err := cmd.Output()
+			if err != nil {
+				return nil, err
+			}
+			for _, line := range strings.Split(strings.TrimSuffix(string(out), "\n"), "\n") {
+				if strings.HasPrefix(line, powershellOutputFilter) {
+					re := regexp.MustCompile(windowsManufacturerRegex)
+					name := re.FindString(line)
+					name = strings.TrimLeft(name, ":")
+					return strings.NewReader(name), nil
+				}
+			}
+			return nil, errors.New("cannot determine the machine's manufacturer")
+		default:
+			return nil, fmt.Errorf("%s is not supported", runningOS)
+		}
+	}
+
+	vmOnGCEOnce sync.Once
+	vmOnGCE     bool
+
+	logger = internalgrpclog.NewPrefixLogger(grpclog.Component("googlecloud"), logPrefix)
+)
+
+// OnGCE returns whether the client is running on GCE.
+//
+// It provides similar functionality as metadata.OnGCE from the cloud library
+// package. We keep this to avoid depending on the cloud library module.
+func OnGCE() bool {
+	vmOnGCEOnce.Do(func() {
+		vmOnGCE = isRunningOnGCE()
+	})
+	return vmOnGCE
+}
+
+// isRunningOnGCE checks whether the local system, without doing a network request is
+// running on GCP.
+func isRunningOnGCE() bool {
+	manufacturer, err := readManufacturer()
+	if err != nil {
+		logger.Infof("failed to read manufacturer %v, returning OnGCE=false", err)
+		return false
+	}
+	name := string(manufacturer)
+	switch runningOS {
+	case "linux":
+		name = strings.TrimSpace(name)
+		return name == "Google" || name == "Google Compute Engine"
+	case "windows":
+		name = strings.Replace(name, " ", "", -1)
+		name = strings.Replace(name, "\n", "", -1)
+		name = strings.Replace(name, "\r", "", -1)
+		return name == "Google"
+	default:
+		return false
+	}
+}
+
+func readManufacturer() ([]byte, error) {
+	reader, err := manufacturerReader()
+	if err != nil {
+		return nil, err
+	}
+	if reader == nil {
+		return nil, errors.New("got nil reader")
+	}
+	manufacturer, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("failed reading %v: %v", linuxProductNameFile, err)
+	}
+	return manufacturer, nil
+}
diff --git a/internal/googlecloud/googlecloud_test.go b/internal/googlecloud/googlecloud_test.go
new file mode 100644
index 000000000000..bd5a42ffab97
--- /dev/null
+++ b/internal/googlecloud/googlecloud_test.go
@@ -0,0 +1,86 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package googlecloud
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+)
+
+func setupManufacturerReader(testOS string, reader func() (io.Reader, error)) func() {
+	tmpOS := runningOS
+	tmpReader := manufacturerReader
+
+	// Set test OS and reader function.
+	runningOS = testOS
+	manufacturerReader = reader
+	return func() {
+		runningOS = tmpOS
+		manufacturerReader = tmpReader
+	}
+}
+
+func setup(testOS string, testReader io.Reader) func() {
+	reader := func() (io.Reader, error) {
+		return testReader, nil
+	}
+	return setupManufacturerReader(testOS, reader)
+}
+
+func setupError(testOS string, err error) func() {
+	reader := func() (io.Reader, error) {
+		return nil, err
+	}
+	return setupManufacturerReader(testOS, reader)
+}
+
+func TestIsRunningOnGCE(t *testing.T) {
+	for _, tc := range []struct {
+		description string
+		testOS      string
+		testReader  io.Reader
+		out         bool
+	}{
+		// Linux tests.
+		{"linux: not a GCP platform", "linux", strings.NewReader("not GCP"), false},
+		{"Linux: GCP platform (Google)", "linux", strings.NewReader("Google"), true},
+		{"Linux: GCP platform (Google Compute Engine)", "linux", strings.NewReader("Google Compute Engine"), true},
+		{"Linux: GCP platform (Google Compute Engine) with extra spaces", "linux", strings.NewReader("  Google Compute Engine        "), true},
+		// Windows tests.
+		{"windows: not a GCP platform", "windows", strings.NewReader("not GCP"), false},
+		{"windows: GCP platform (Google)", "windows", strings.NewReader("Google"), true},
+		{"windows: GCP platform (Google) with extra spaces", "windows", strings.NewReader("  Google     "), true},
+	} {
+		reverseFunc := setup(tc.testOS, tc.testReader)
+		if got, want := isRunningOnGCE(), tc.out; got != want {
+			t.Errorf("%v: isRunningOnGCE()=%v, want %v", tc.description, got, want)
+		}
+		reverseFunc()
+	}
+}
+
+func TestIsRunningOnGCENoProductNameFile(t *testing.T) {
+	reverseFunc := setupError("linux", os.ErrNotExist)
+	if isRunningOnGCE() {
+		t.Errorf("ErrNotExist: isRunningOnGCE()=true, want false")
+	}
+	reverseFunc()
+}
diff --git a/xds/googledirectpath/googlec2p.go b/xds/googledirectpath/googlec2p.go
new file mode 100644
index 000000000000..791a568ecb02
--- /dev/null
+++ b/xds/googledirectpath/googlec2p.go
@@ -0,0 +1,179 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package googledirectpath implements a resolver that configures xds to make
+// cloud to prod directpath connection.
+//
+// It's a combo of DNS and xDS resolvers. It delegates to DNS if
+// - not on GCE, or
+// - xDS bootstrap env var is set (so this client needs to do normal xDS, not
+// direct path, and clients with this scheme is not part of the xDS mesh).
+package googledirectpath
+
+import (
+	"fmt"
+	"time"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/googlecloud"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/resolver"
+	_ "google.golang.org/grpc/xds" // To register xds resolvers and balancers.
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+const (
+	c2pScheme = "google-c2p"
+
+	tdURL          = "directpath-trafficdirector.googleapis.com"
+	httpReqTimeout = 10 * time.Second
+	zoneURL        = "http://metadata.google.internal/computeMetadata/v1/instance/zone"
+	ipv6URL        = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"
+
+	gRPCUserAgentName               = "gRPC Go"
+	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
+	ipv6CapableMetadataName         = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE"
+
+	logPrefix = "[google-c2p-resolver]"
+
+	dnsName, xdsName = "dns", "xds"
+)
+
+type xdsClientInterface interface {
+	Close()
+}
+
+// For overriding in unittests.
+var (
+	onGCE = googlecloud.OnGCE
+
+	newClientWithConfig = func(config *bootstrap.Config) (xdsClientInterface, error) {
+		return xdsclient.NewWithConfig(config)
+	}
+
+	logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
+)
+
+func init() {
+	if env.C2PResolverSupport {
+		resolver.Register(c2pResolverBuilder{})
+	}
+}
+
+type c2pResolverBuilder struct{}
+
+func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
+	if !runDirectPath() {
+		// If not xDS, fallback to DNS.
+		t.Scheme = dnsName
+		return resolver.Get(dnsName).Build(t, cc, opts)
+	}
+
+	// Note that the following calls to getZone() and getIPv6Capable() does I/O,
+	// and has 10 seconds timeout each.
+	//
+	// This should be fine in most of the cases. In certain error cases, this
+	// could block Dial() for up to 10 seconds (each blocking call has its own
+	// goroutine).
+	zoneCh, ipv6CapableCh := make(chan string), make(chan bool)
+	go func() { zoneCh <- getZone(httpReqTimeout) }()
+	go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }()
+
+	balancerName := env.C2PResolverTestOnlyTrafficDirectorURI
+	if balancerName == "" {
+		balancerName = tdURL
+	}
+	config := &bootstrap.Config{
+		BalancerName: balancerName,
+		Creds:        grpc.WithCredentialsBundle(google.NewDefaultCredentials()),
+		TransportAPI: version.TransportV3,
+		NodeProto:    newNode(<-zoneCh, <-ipv6CapableCh),
+	}
+
+	// Create singleton xds client with this config. The xds client will be
+	// used by the xds resolver later.
+	xdsC, err := newClientWithConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to start xDS client: %v", err)
+	}
+
+	// Create and return an xDS resolver.
+	t.Scheme = xdsName
+	xdsR, err := resolver.Get(xdsName).Build(t, cc, opts)
+	if err != nil {
+		xdsC.Close()
+		return nil, err
+	}
+	return &c2pResolver{
+		Resolver: xdsR,
+		client:   xdsC,
+	}, nil
+}
+
+func (c2pResolverBuilder) Scheme() string {
+	return c2pScheme
+}
+
+type c2pResolver struct {
+	resolver.Resolver
+	client xdsClientInterface
+}
+
+func (r *c2pResolver) Close() {
+	r.Resolver.Close()
+	r.client.Close()
+}
+
+var ipv6EnabledMetadata = &structpb.Struct{
+	Fields: map[string]*structpb.Value{
+		ipv6CapableMetadataName: structpb.NewBoolValue(true),
+	},
+}
+
+// newNode makes a copy of defaultNode, and populate it's Metadata and
+// Locality fields.
+func newNode(zone string, ipv6Capable bool) *v3corepb.Node {
+	ret := &v3corepb.Node{
+		// Not all required fields are set in defaultNote. Metadata will be set
+		// if ipv6 is enabled. Locality will be set to the value from metadata.
+		Id:                   "C2P",
+		UserAgentName:        gRPCUserAgentName,
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
+	}
+	ret.Locality = &v3corepb.Locality{Zone: zone}
+	if ipv6Capable {
+		ret.Metadata = ipv6EnabledMetadata
+	}
+	return ret
+}
+
+// runDirectPath returns whether this resolver should use direct path.
+//
+// direct path is enabled if this client is running on GCE, and the normal xDS
+// is not used (bootstrap env vars are not set).
+func runDirectPath() bool {
+	return env.BootstrapFileName == "" && env.BootstrapFileContent == "" && onGCE()
+}
diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go
new file mode 100644
index 000000000000..86ea34599435
--- /dev/null
+++ b/xds/googledirectpath/googlec2p_test.go
@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package googledirectpath
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+type emptyResolver struct {
+	resolver.Resolver
+	scheme string
+}
+
+func (er *emptyResolver) Build(_ resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
+	return er, nil
+}
+
+func (er *emptyResolver) Scheme() string {
+	return er.scheme
+}
+
+func (er *emptyResolver) Close() {}
+
+var (
+	testDNSResolver = &emptyResolver{scheme: "dns"}
+	testXDSResolver = &emptyResolver{scheme: "xds"}
+)
+
+func replaceResolvers() func() {
+	var registerForTesting bool
+	if resolver.Get(c2pScheme) == nil {
+		// If env var to enable c2p is not set, the resolver isn't registered.
+		// Need to register and unregister in defer.
+		registerForTesting = true
+		resolver.Register(&c2pResolverBuilder{})
+	}
+	oldDNS := resolver.Get("dns")
+	resolver.Register(testDNSResolver)
+	oldXDS := resolver.Get("xds")
+	resolver.Register(testXDSResolver)
+	return func() {
+		if oldDNS != nil {
+			resolver.Register(oldDNS)
+		} else {
+			resolver.UnregisterForTesting("dns")
+		}
+		if oldXDS != nil {
+			resolver.Register(oldXDS)
+		} else {
+			resolver.UnregisterForTesting("xds")
+		}
+		if registerForTesting {
+			resolver.UnregisterForTesting(c2pScheme)
+		}
+	}
+}
+
+// Test that when bootstrap env is set, fallback to DNS.
+func TestBuildWithBootstrapEnvSet(t *testing.T) {
+	defer replaceResolvers()()
+	builder := resolver.Get(c2pScheme)
+
+	for i, envP := range []*string{&env.BootstrapFileName, &env.BootstrapFileContent} {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			// Set bootstrap config env var.
+			oldEnv := *envP
+			*envP = "does not matter"
+			defer func() { *envP = oldEnv }()
+
+			// Build should return DNS, not xDS.
+			r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{})
+			if err != nil {
+				t.Fatalf("failed to build resolver: %v", err)
+			}
+			if r != testDNSResolver {
+				t.Fatalf("want dns resolver, got %#v", r)
+			}
+		})
+	}
+}
+
+// Test that when not on GCE, fallback to DNS.
+func TestBuildNotOnGCE(t *testing.T) {
+	defer replaceResolvers()()
+	builder := resolver.Get(c2pScheme)
+
+	oldOnGCE := onGCE
+	onGCE = func() bool { return false }
+	defer func() { onGCE = oldOnGCE }()
+
+	// Build should return DNS, not xDS.
+	r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{})
+	if err != nil {
+		t.Fatalf("failed to build resolver: %v", err)
+	}
+	if r != testDNSResolver {
+		t.Fatalf("want dns resolver, got %#v", r)
+	}
+}
+
+type testXDSClient struct {
+	closed chan struct{}
+}
+
+func (c *testXDSClient) Close() {
+	c.closed <- struct{}{}
+}
+
+// Test that when xDS is built, the client is built with the correct config.
+func TestBuildXDS(t *testing.T) {
+	defer replaceResolvers()()
+	builder := resolver.Get(c2pScheme)
+
+	oldOnGCE := onGCE
+	onGCE = func() bool { return true }
+	defer func() { onGCE = oldOnGCE }()
+
+	const testZone = "test-zone"
+	oldGetZone := getZone
+	getZone = func(time.Duration) string { return testZone }
+	defer func() { getZone = oldGetZone }()
+
+	for _, tt := range []struct {
+		name  string
+		ipv6  bool
+		tdURI string // traffic director URI will be overridden if this is set.
+	}{
+		{name: "ipv6 true", ipv6: true},
+		{name: "ipv6 false", ipv6: false},
+		{name: "override TD URI", ipv6: true, tdURI: "test-uri"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			oldGetIPv6Capability := getIPv6Capable
+			getIPv6Capable = func(time.Duration) bool { return tt.ipv6 }
+			defer func() { getIPv6Capable = oldGetIPv6Capability }()
+
+			if tt.tdURI != "" {
+				oldURI := env.C2PResolverTestOnlyTrafficDirectorURI
+				env.C2PResolverTestOnlyTrafficDirectorURI = tt.tdURI
+				defer func() {
+					env.C2PResolverTestOnlyTrafficDirectorURI = oldURI
+				}()
+			}
+
+			tXDSClient := &testXDSClient{closed: make(chan struct{}, 1)}
+
+			configCh := make(chan *bootstrap.Config, 1)
+			oldNewClient := newClientWithConfig
+			newClientWithConfig = func(config *bootstrap.Config) (xdsClientInterface, error) {
+				configCh <- config
+				return tXDSClient, nil
+			}
+			defer func() { newClientWithConfig = oldNewClient }()
+
+			// Build should return DNS, not xDS.
+			r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{})
+			if err != nil {
+				t.Fatalf("failed to build resolver: %v", err)
+			}
+			rr := r.(*c2pResolver)
+			if rrr := rr.Resolver; rrr != testXDSResolver {
+				t.Fatalf("want xds resolver, got %#v, ", rrr)
+			}
+
+			wantNode := &v3corepb.Node{
+				Id:                   "C2P",
+				Metadata:             nil,
+				Locality:             &v3corepb.Locality{Zone: testZone},
+				UserAgentName:        gRPCUserAgentName,
+				UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+				ClientFeatures:       []string{clientFeatureNoOverprovisioning},
+			}
+			if tt.ipv6 {
+				wantNode.Metadata = &structpb.Struct{
+					Fields: map[string]*structpb.Value{
+						ipv6CapableMetadataName: {
+							Kind: &structpb.Value_BoolValue{BoolValue: true},
+						},
+					},
+				}
+			}
+			wantConfig := &bootstrap.Config{
+				BalancerName: tdURL,
+				TransportAPI: version.TransportV3,
+				NodeProto:    wantNode,
+			}
+			if tt.tdURI != "" {
+				wantConfig.BalancerName = tt.tdURI
+			}
+			cmpOpts := cmp.Options{
+				cmpopts.IgnoreFields(bootstrap.Config{}, "Creds"),
+				protocmp.Transform(),
+			}
+			select {
+			case c := <-configCh:
+				if diff := cmp.Diff(c, wantConfig, cmpOpts); diff != "" {
+					t.Fatalf("%v", diff)
+				}
+			case <-time.After(time.Second):
+				t.Fatalf("timeout waiting for client config")
+			}
+
+			r.Close()
+			select {
+			case <-tXDSClient.closed:
+			case <-time.After(time.Second):
+				t.Fatalf("timeout waiting for client close")
+			}
+		})
+	}
+}
diff --git a/xds/googledirectpath/utils.go b/xds/googledirectpath/utils.go
new file mode 100644
index 000000000000..553b87adf47a
--- /dev/null
+++ b/xds/googledirectpath/utils.go
@@ -0,0 +1,96 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package googledirectpath
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+)
+
+func getFromMetadata(timeout time.Duration, urlStr string) ([]byte, error) {
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil {
+		return nil, err
+	}
+	client := &http.Client{Timeout: timeout}
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    parsedURL,
+		Header: http.Header{"Metadata-Flavor": {"Google"}},
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed communicating with metadata server: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("metadata server returned resp with non-OK: %v", resp)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed reading from metadata server: %w", err)
+	}
+	return body, nil
+}
+
+var (
+	zone     string
+	zoneOnce sync.Once
+)
+
+// Defined as var to be overridden in tests.
+var getZone = func(timeout time.Duration) string {
+	zoneOnce.Do(func() {
+		qualifiedZone, err := getFromMetadata(timeout, zoneURL)
+		if err != nil {
+			logger.Warningf("could not discover instance zone: %v", err)
+			return
+		}
+		i := bytes.LastIndexByte(qualifiedZone, '/')
+		if i == -1 {
+			logger.Warningf("could not parse zone from metadata server: %s", qualifiedZone)
+			return
+		}
+		zone = string(qualifiedZone[i+1:])
+	})
+	return zone
+}
+
+var (
+	ipv6Capable     bool
+	ipv6CapableOnce sync.Once
+)
+
+// Defined as var to be overridden in tests.
+var getIPv6Capable = func(timeout time.Duration) bool {
+	ipv6CapableOnce.Do(func() {
+		_, err := getFromMetadata(timeout, ipv6URL)
+		if err != nil {
+			logger.Warningf("could not discover ipv6 capability: %v", err)
+			return
+		}
+		ipv6Capable = true
+	})
+	return ipv6Capable
+}
diff --git a/xds/internal/client/singleton.go b/xds/internal/client/singleton.go
index 5d92b4146bb3..99f195341acd 100644
--- a/xds/internal/client/singleton.go
+++ b/xds/internal/client/singleton.go
@@ -50,7 +50,14 @@ type Client struct {
 }
 
 // New returns a new xdsClient configured by the bootstrap file specified in env
-// variable GRPC_XDS_BOOTSTRAP.
+// variable GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG.
+//
+// The returned xdsClient is a singleton. This function creates the xds client
+// if it doesn't already exist.
+//
+// Note that the first invocation of New() or NewWithConfig() sets the client
+// singleton. The following calls will return the singleton xds client without
+// checking or using the config.
 func New() (*Client, error) {
 	singletonClient.mu.Lock()
 	defer singletonClient.mu.Unlock()
@@ -76,6 +83,38 @@ func New() (*Client, error) {
 	return singletonClient, nil
 }
 
+// NewWithConfig returns a new xdsClient configured by the given config.
+//
+// The returned xdsClient is a singleton. This function creates the xds client
+// if it doesn't already exist.
+//
+// Note that the first invocation of New() or NewWithConfig() sets the client
+// singleton. The following calls will return the singleton xds client without
+// checking or using the config.
+//
+// This function is internal only, for c2p resolver to use. DO NOT use this
+// elsewhere. Use New() instead.
+func NewWithConfig(config *bootstrap.Config) (*Client, error) {
+	singletonClient.mu.Lock()
+	defer singletonClient.mu.Unlock()
+	// If the client implementation was created, increment ref count and return
+	// the client.
+	if singletonClient.clientImpl != nil {
+		singletonClient.refCount++
+		return singletonClient, nil
+	}
+
+	// Create the new client implementation.
+	c, err := newWithConfig(config, defaultWatchExpiryTimeout)
+	if err != nil {
+		return nil, err
+	}
+
+	singletonClient.clientImpl = c
+	singletonClient.refCount++
+	return singletonClient, nil
+}
+
 // Close closes the client. It does ref count of the xds client implementation,
 // and closes the gRPC connection to the management server when ref count
 // reaches 0.
@@ -92,6 +131,9 @@ func (c *Client) Close() {
 }
 
 // NewWithConfigForTesting is exported for testing only.
+//
+// Note that this function doesn't set the singleton, so that the testing states
+// don't leak.
 func NewWithConfigForTesting(config *bootstrap.Config, watchExpiryTimeout time.Duration) (*Client, error) {
 	cl, err := newWithConfig(config, watchExpiryTimeout)
 	if err != nil {
diff --git a/xds/internal/env/env.go b/xds/internal/env/env.go
index a28d741f3568..035b92b18f13 100644
--- a/xds/internal/env/env.go
+++ b/xds/internal/env/env.go
@@ -42,6 +42,9 @@ const (
 	timeoutSupportEnv            = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
 	faultInjectionSupportEnv     = "GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION"
 	clientSideSecuritySupportEnv = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"
+
+	c2pResolverSupportEnv                    = "GRPC_EXPERIMENTAL_GOOGLE_C2P_RESOLVER"
+	c2pResolverTestOnlyTrafficDirectorURIEnv = "GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI"
 )
 
 var (
@@ -68,6 +71,10 @@ var (
 	// FaultInjectionSupport is used to control both fault injection and HTTP
 	// filter support.
 	FaultInjectionSupport = strings.EqualFold(os.Getenv(faultInjectionSupportEnv), "true")
+	// C2PResolverSupport indicates whether support for C2P resolver is enabled.
+	// This can be enabled by setting the environment variable
+	// "GRPC_EXPERIMENTAL_GOOGLE_C2P_RESOLVER" to "true".
+	C2PResolverSupport = strings.EqualFold(os.Getenv(c2pResolverSupportEnv), "true")
 	// ClientSideSecuritySupport is used to control processing of security
 	// configuration on the client-side.
 	//
@@ -75,4 +82,6 @@ var (
 	// have a brand new API on the server-side and users explicitly need to use
 	// the new API to get security integration on the server.
 	ClientSideSecuritySupport = strings.EqualFold(os.Getenv(clientSideSecuritySupportEnv), "true")
+	// C2PResolverTestOnlyTrafficDirectorURI is the TD URI for testing.
+	C2PResolverTestOnlyTrafficDirectorURI = os.Getenv(c2pResolverTestOnlyTrafficDirectorURIEnv)
 )

From d7737376c30e219e2773ebb143c678bdd91810e4 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 12 Mar 2021 08:38:49 -0800
Subject: [PATCH 385/481] xds: implement fault injection HTTP filter (A33)
 (#4236)

---
 examples/go.sum                             |   5 +
 stream.go                                   |   4 +-
 xds/internal/httpfilter/fault/fault.go      | 295 +++++++++
 xds/internal/httpfilter/fault/fault_test.go | 644 ++++++++++++++++++++
 xds/xds.go                                  |   1 +
 5 files changed, 947 insertions(+), 2 deletions(-)
 create mode 100644 xds/internal/httpfilter/fault/fault.go
 create mode 100644 xds/internal/httpfilter/fault/fault_test.go

diff --git a/examples/go.sum b/examples/go.sum
index c6aa163b01ad..f55a3e5c8968 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -4,6 +4,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
@@ -29,9 +30,11 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -77,7 +80,9 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/stream.go b/stream.go
index 4a9fdb73875b..77d25742cc3d 100644
--- a/stream.go
+++ b/stream.go
@@ -181,7 +181,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 	rpcInfo := iresolver.RPCInfo{Context: ctx, Method: method}
 	rpcConfig, err := cc.safeConfigSelector.SelectConfig(rpcInfo)
 	if err != nil {
-		return nil, status.Convert(err).Err()
+		return nil, toRPCErr(err)
 	}
 
 	if rpcConfig != nil {
@@ -196,7 +196,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 			newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
 				cs, err := rpcConfig.Interceptor.NewStream(ctx, rpcInfo, done, ns)
 				if err != nil {
-					return nil, status.Convert(err).Err()
+					return nil, toRPCErr(err)
 				}
 				return cs, nil
 			}
diff --git a/xds/internal/httpfilter/fault/fault.go b/xds/internal/httpfilter/fault/fault.go
new file mode 100644
index 000000000000..98e05b51590f
--- /dev/null
+++ b/xds/internal/httpfilter/fault/fault.go
@@ -0,0 +1,295 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package fault implements the Envoy Fault Injection HTTP filter.
+package fault
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/grpcrand"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3"
+	fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3"
+	tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+)
+
+const headerAbortHTTPStatus = "x-envoy-fault-abort-request"
+const headerAbortGRPCStatus = "x-envoy-fault-abort-grpc-request"
+const headerAbortPercentage = "x-envoy-fault-abort-request-percentage"
+
+const headerDelayPercentage = "x-envoy-fault-delay-request-percentage"
+const headerDelayDuration = "x-envoy-fault-delay-request"
+
+var statusMap = map[int]codes.Code{
+	400: codes.Internal,
+	401: codes.Unauthenticated,
+	403: codes.PermissionDenied,
+	404: codes.Unimplemented,
+	429: codes.Unavailable,
+	502: codes.Unavailable,
+	503: codes.Unavailable,
+	504: codes.Unavailable,
+}
+
+func init() {
+	if env.FaultInjectionSupport {
+		httpfilter.Register(builder{})
+	}
+}
+
+type builder struct {
+}
+
+type config struct {
+	httpfilter.FilterConfig
+	config *fpb.HTTPFault
+}
+
+func (builder) TypeURLs() []string {
+	return []string{"type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault"}
+}
+
+// Parsing is the same for the base config and the override config.
+func parseConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("fault: nil configuration message provided")
+	}
+	any, ok := cfg.(*anypb.Any)
+	if !ok {
+		return nil, fmt.Errorf("fault: error parsing config %v: unknown type %T", cfg, cfg)
+	}
+	msg := new(fpb.HTTPFault)
+	if err := ptypes.UnmarshalAny(any, msg); err != nil {
+		return nil, fmt.Errorf("fault: error parsing config %v: %v", cfg, err)
+	}
+	return config{config: msg}, nil
+}
+
+func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	return parseConfig(cfg)
+}
+
+func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) {
+	return parseConfig(override)
+}
+
+var _ httpfilter.ClientInterceptorBuilder = builder{}
+
+func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("fault: nil config provided")
+	}
+
+	c, ok := cfg.(config)
+	if !ok {
+		return nil, fmt.Errorf("fault: incorrect config type provided (%T): %v", cfg, cfg)
+	}
+
+	if override != nil {
+		// override completely replaces the listener configuration; but we
+		// still validate the listener config type.
+		c, ok = override.(config)
+		if !ok {
+			return nil, fmt.Errorf("fault: incorrect override config type provided (%T): %v", override, override)
+		}
+	}
+
+	return &interceptor{config: c.config}, nil
+}
+
+type interceptor struct {
+	config *fpb.HTTPFault
+}
+
+var activeFaults uint32 // global active faults; accessed atomically
+
+func (i *interceptor) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) {
+	if maxAF := i.config.GetMaxActiveFaults(); maxAF != nil {
+		defer atomic.AddUint32(&activeFaults, ^uint32(0)) // decrement counter
+		if af := atomic.AddUint32(&activeFaults, 1); af > maxAF.GetValue() {
+			// Would exceed maximum active fault limit.
+			return newStream(ctx, done)
+		}
+	}
+
+	if err := injectDelay(ctx, i.config.GetDelay()); err != nil {
+		return nil, err
+	}
+
+	if err := injectAbort(ctx, i.config.GetAbort()); err != nil {
+		if err == errOKStream {
+			return &okStream{ctx: ctx}, nil
+		}
+		return nil, err
+	}
+	return newStream(ctx, done)
+}
+
+// For overriding in tests
+var randIntn = grpcrand.Intn
+var newTimer = time.NewTimer
+
+func injectDelay(ctx context.Context, delayCfg *cpb.FaultDelay) error {
+	numerator, denominator := splitPct(delayCfg.GetPercentage())
+	var delay time.Duration
+	switch delayType := delayCfg.GetFaultDelaySecifier().(type) {
+	case *cpb.FaultDelay_FixedDelay:
+		delay = delayType.FixedDelay.AsDuration()
+	case *cpb.FaultDelay_HeaderDelay_:
+		md, _ := metadata.FromOutgoingContext(ctx)
+		v := md[headerDelayDuration]
+		if v == nil {
+			// No delay configured for this RPC.
+			return nil
+		}
+		ms, ok := parseIntFromMD(v)
+		if !ok {
+			// Malformed header; no delay.
+			return nil
+		}
+		delay = time.Duration(ms) * time.Millisecond
+		if v := md[headerDelayPercentage]; v != nil {
+			if num, ok := parseIntFromMD(v); ok && num < numerator {
+				numerator = num
+			}
+		}
+	}
+	if delay == 0 || randIntn(denominator) >= numerator {
+		return nil
+	}
+	t := newTimer(delay)
+	select {
+	case <-t.C:
+	case <-ctx.Done():
+		t.Stop()
+		return ctx.Err()
+	}
+	return nil
+}
+
+func injectAbort(ctx context.Context, abortCfg *fpb.FaultAbort) error {
+	numerator, denominator := splitPct(abortCfg.GetPercentage())
+	code := codes.OK
+	okCode := false
+	switch errType := abortCfg.GetErrorType().(type) {
+	case *fpb.FaultAbort_HttpStatus:
+		code, okCode = grpcFromHTTP(int(errType.HttpStatus))
+	case *fpb.FaultAbort_GrpcStatus:
+		code, okCode = sanitizeGRPCCode(codes.Code(errType.GrpcStatus)), true
+	case *fpb.FaultAbort_HeaderAbort_:
+		md, _ := metadata.FromOutgoingContext(ctx)
+		if v := md[headerAbortHTTPStatus]; v != nil {
+			// HTTP status has priority over gRPC status.
+			if httpStatus, ok := parseIntFromMD(v); ok {
+				code, okCode = grpcFromHTTP(httpStatus)
+			}
+		} else if v := md[headerAbortGRPCStatus]; v != nil {
+			if grpcStatus, ok := parseIntFromMD(v); ok {
+				code, okCode = sanitizeGRPCCode(codes.Code(grpcStatus)), true
+			}
+		}
+		if v := md[headerAbortPercentage]; v != nil {
+			if num, ok := parseIntFromMD(v); ok && num < numerator {
+				numerator = num
+			}
+		}
+	}
+	if !okCode || randIntn(denominator) >= numerator {
+		return nil
+	}
+	if code == codes.OK {
+		return errOKStream
+	}
+	return status.Errorf(code, "RPC terminated due to fault injection")
+}
+
+var errOKStream = errors.New("stream terminated early with OK status")
+
+// parseIntFromMD returns the integer in the last header or nil if parsing
+// failed.
+func parseIntFromMD(header []string) (int, bool) {
+	if len(header) == 0 {
+		return 0, false
+	}
+	v, err := strconv.Atoi(header[len(header)-1])
+	return v, err == nil
+}
+
+func splitPct(fp *tpb.FractionalPercent) (num int, den int) {
+	if fp == nil {
+		return 0, 100
+	}
+	num = int(fp.GetNumerator())
+	switch fp.GetDenominator() {
+	case tpb.FractionalPercent_HUNDRED:
+		return num, 100
+	case tpb.FractionalPercent_TEN_THOUSAND:
+		return num, 10 * 1000
+	case tpb.FractionalPercent_MILLION:
+		return num, 1000 * 1000
+	}
+	return num, 100
+}
+
+func grpcFromHTTP(httpStatus int) (codes.Code, bool) {
+	if httpStatus < 200 || httpStatus >= 600 {
+		// Malformed; ignore this fault type.
+		return codes.OK, false
+	}
+	if c := statusMap[httpStatus]; c != codes.OK {
+		// OK = 0/the default for the map.
+		return c, true
+	}
+	// All undefined HTTP status codes convert to Unknown. HTTP status of 200
+	// is "success", but gRPC converts to Unknown due to missing grpc status.
+	return codes.Unknown, true
+}
+
+func sanitizeGRPCCode(c codes.Code) codes.Code {
+	if c > codes.Code(16) {
+		return codes.Unknown
+	}
+	return c
+}
+
+type okStream struct {
+	ctx context.Context
+}
+
+func (*okStream) Header() (metadata.MD, error) { return nil, nil }
+func (*okStream) Trailer() metadata.MD         { return nil }
+func (*okStream) CloseSend() error             { return nil }
+func (o *okStream) Context() context.Context   { return o.ctx }
+func (*okStream) SendMsg(m interface{}) error  { return io.EOF }
+func (*okStream) RecvMsg(m interface{}) error  { return io.EOF }
diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
new file mode 100644
index 000000000000..562cdc0832bf
--- /dev/null
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -0,0 +1,644 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds_test contains e2e tests for xDS use.
+package fault
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/golang/protobuf/ptypes"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpcrand"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/xds/internal/httpfilter"
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3"
+	fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+
+	_ "google.golang.org/grpc/xds/internal/balancer"  // Register the balancers.
+	_ "google.golang.org/grpc/xds/internal/client/v3" // Register the v3 xDS API client.
+	_ "google.golang.org/grpc/xds/internal/resolver"  // Register the xds_resolver.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type testService struct {
+	testpb.TestServiceServer
+}
+
+func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	return &testpb.Empty{}, nil
+}
+
+func (*testService) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error {
+	// End RPC after client does a CloseSend.
+	for {
+		if _, err := stream.Recv(); err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+	}
+}
+
+// clientSetup performs a bunch of steps common to all xDS server tests here:
+// - spin up an xDS management server on a local port
+// - spin up a gRPC server and register the test service on it
+// - create a local TCP listener and start serving on it
+//
+// Returns the following:
+// - the management server: tests use this to configure resources
+// - nodeID expected by the management server: this is set in the Node proto
+//   sent by the xdsClient for queries.
+// - the port the server is listening on
+// - cleanup function to be invoked by the tests when done
+func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
+	// Spin up a xDS management server on a local port.
+	nodeID := uuid.New().String()
+	fs, err := e2e.StartManagementServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a bootstrap file in a temporary directory.
+	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
+		Version:                            e2e.TransportV3,
+		NodeID:                             nodeID,
+		ServerURI:                          fs.Address,
+		ServerListenerResourceNameTemplate: "grpc/server",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Initialize a gRPC server and register the stubServer on it.
+	server := grpc.NewServer()
+	testpb.RegisterTestServiceServer(server, &testService{})
+
+	// Create a local listener and pass it to Serve().
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	go func() {
+		if err := server.Serve(lis); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+
+	return fs, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), func() {
+		fs.Stop()
+		bootstrapCleanup()
+		server.Stop()
+	}
+}
+
+func init() {
+	env.FaultInjectionSupport = true
+	// Manually register to avoid a race between this init and the one that
+	// check the env var to register the fault injection filter.
+	httpfilter.Register(builder{})
+}
+
+func (s) TestFaultInjection_Unary(t *testing.T) {
+	type subcase struct {
+		name   string
+		code   codes.Code
+		repeat int
+		randIn []int           // Intn calls per-repeat (not per-subcase)
+		delays []time.Duration // NewTimer calls per-repeat (not per-subcase)
+		md     metadata.MD
+	}
+	testCases := []struct {
+		name       string
+		cfgs       []*fpb.HTTPFault
+		randOutInc int
+		want       []subcase
+	}{{
+		name: "abort always",
+		cfgs: []*fpb.HTTPFault{{
+			Abort: &fpb.FaultAbort{
+				Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED},
+				ErrorType:  &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)},
+			},
+		}},
+		randOutInc: 5,
+		want: []subcase{{
+			code:   codes.Aborted,
+			randIn: []int{100},
+			repeat: 25,
+		}},
+	}, {
+		name: "abort 10%",
+		cfgs: []*fpb.HTTPFault{{
+			Abort: &fpb.FaultAbort{
+				Percentage: &tpb.FractionalPercent{Numerator: 100000, Denominator: tpb.FractionalPercent_MILLION},
+				ErrorType:  &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)},
+			},
+		}},
+		randOutInc: 50000,
+		want: []subcase{{
+			name:   "[0,10]%",
+			code:   codes.Aborted,
+			randIn: []int{1000000},
+			repeat: 2,
+		}, {
+			name:   "(10,100]%",
+			code:   codes.OK,
+			randIn: []int{1000000},
+			repeat: 18,
+		}, {
+			name:   "[0,10]% again",
+			code:   codes.Aborted,
+			randIn: []int{1000000},
+			repeat: 2,
+		}},
+	}, {
+		name: "delay always",
+		cfgs: []*fpb.HTTPFault{{
+			Delay: &cpb.FaultDelay{
+				Percentage:         &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED},
+				FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)},
+			},
+		}},
+		randOutInc: 5,
+		want: []subcase{{
+			randIn: []int{100},
+			repeat: 25,
+			delays: []time.Duration{time.Second},
+		}},
+	}, {
+		name: "delay 10%",
+		cfgs: []*fpb.HTTPFault{{
+			Delay: &cpb.FaultDelay{
+				Percentage:         &tpb.FractionalPercent{Numerator: 1000, Denominator: tpb.FractionalPercent_TEN_THOUSAND},
+				FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)},
+			},
+		}},
+		randOutInc: 500,
+		want: []subcase{{
+			name:   "[0,10]%",
+			randIn: []int{10000},
+			repeat: 2,
+			delays: []time.Duration{time.Second},
+		}, {
+			name:   "(10,100]%",
+			randIn: []int{10000},
+			repeat: 18,
+		}, {
+			name:   "[0,10]% again",
+			randIn: []int{10000},
+			repeat: 2,
+			delays: []time.Duration{time.Second},
+		}},
+	}, {
+		name: "delay 80%, abort 50%",
+		cfgs: []*fpb.HTTPFault{{
+			Delay: &cpb.FaultDelay{
+				Percentage:         &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED},
+				FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(3 * time.Second)},
+			},
+			Abort: &fpb.FaultAbort{
+				Percentage: &tpb.FractionalPercent{Numerator: 50, Denominator: tpb.FractionalPercent_HUNDRED},
+				ErrorType:  &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Unimplemented)},
+			},
+		}},
+		randOutInc: 5,
+		want: []subcase{{
+			name:   "50% delay and abort",
+			code:   codes.Unimplemented,
+			randIn: []int{100, 100},
+			repeat: 10,
+			delays: []time.Duration{3 * time.Second},
+		}, {
+			name:   "30% delay, no abort",
+			randIn: []int{100, 100},
+			repeat: 6,
+			delays: []time.Duration{3 * time.Second},
+		}, {
+			name:   "20% success",
+			randIn: []int{100, 100},
+			repeat: 4,
+		}, {
+			name:   "50% delay and abort again",
+			code:   codes.Unimplemented,
+			randIn: []int{100, 100},
+			repeat: 10,
+			delays: []time.Duration{3 * time.Second},
+		}},
+	}, {
+		name: "header abort",
+		cfgs: []*fpb.HTTPFault{{
+			Abort: &fpb.FaultAbort{
+				Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED},
+				ErrorType:  &fpb.FaultAbort_HeaderAbort_{},
+			},
+		}},
+		randOutInc: 10,
+		want: []subcase{{
+			name: "30% abort; [0,30]%",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)},
+				headerAbortPercentage: []string{"30"},
+			},
+			code:   codes.DataLoss,
+			randIn: []int{100},
+			repeat: 3,
+		}, {
+			name: "30% abort; (30,60]%",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)},
+				headerAbortPercentage: []string{"30"},
+			},
+			randIn: []int{100},
+			repeat: 3,
+		}, {
+			name: "80% abort; (60,80]%",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)},
+				headerAbortPercentage: []string{"80"},
+			},
+			code:   codes.DataLoss,
+			randIn: []int{100},
+			repeat: 2,
+		}, {
+			name: "cannot exceed percentage in filter",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)},
+				headerAbortPercentage: []string{"100"},
+			},
+			randIn: []int{100},
+			repeat: 2,
+		}, {
+			name: "HTTP Status 404",
+			md: metadata.MD{
+				headerAbortHTTPStatus: []string{"404"},
+				headerAbortPercentage: []string{"100"},
+			},
+			code:   codes.Unimplemented,
+			randIn: []int{100},
+			repeat: 1,
+		}, {
+			name: "HTTP Status 429",
+			md: metadata.MD{
+				headerAbortHTTPStatus: []string{"429"},
+				headerAbortPercentage: []string{"100"},
+			},
+			code:   codes.Unavailable,
+			randIn: []int{100},
+			repeat: 1,
+		}, {
+			name: "HTTP Status 200",
+			md: metadata.MD{
+				headerAbortHTTPStatus: []string{"200"},
+				headerAbortPercentage: []string{"100"},
+			},
+			// No GRPC status, but HTTP Status of 200 translates to Unknown,
+			// per spec in statuscodes.md.
+			code:   codes.Unknown,
+			randIn: []int{100},
+			repeat: 1,
+		}, {
+			name: "gRPC Status OK",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.OK)},
+				headerAbortPercentage: []string{"100"},
+			},
+			// This should be Unimplemented (mismatched request/response
+			// count), per spec in statuscodes.md, but grpc-go currently
+			// returns io.EOF which status.Code() converts to Unknown
+			code:   codes.Unknown,
+			randIn: []int{100},
+			repeat: 1,
+		}, {
+			name: "invalid header results in no abort",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{"error"},
+				headerAbortPercentage: []string{"100"},
+			},
+			repeat: 1,
+		}, {
+			name: "invalid header results in default percentage",
+			md: metadata.MD{
+				headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)},
+				headerAbortPercentage: []string{"error"},
+			},
+			code:   codes.DataLoss,
+			randIn: []int{100},
+			repeat: 1,
+		}},
+	}, {
+		name: "header delay",
+		cfgs: []*fpb.HTTPFault{{
+			Delay: &cpb.FaultDelay{
+				Percentage:         &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED},
+				FaultDelaySecifier: &cpb.FaultDelay_HeaderDelay_{},
+			},
+		}},
+		randOutInc: 10,
+		want: []subcase{{
+			name: "30% delay; [0,30]%",
+			md: metadata.MD{
+				headerDelayDuration:   []string{"2"},
+				headerDelayPercentage: []string{"30"},
+			},
+			randIn: []int{100},
+			delays: []time.Duration{2 * time.Millisecond},
+			repeat: 3,
+		}, {
+			name: "30% delay; (30, 60]%",
+			md: metadata.MD{
+				headerDelayDuration:   []string{"2"},
+				headerDelayPercentage: []string{"30"},
+			},
+			randIn: []int{100},
+			repeat: 3,
+		}, {
+			name: "invalid header results in no delay",
+			md: metadata.MD{
+				headerDelayDuration:   []string{"error"},
+				headerDelayPercentage: []string{"80"},
+			},
+			repeat: 1,
+		}, {
+			name: "invalid header results in default percentage",
+			md: metadata.MD{
+				headerDelayDuration:   []string{"2"},
+				headerDelayPercentage: []string{"error"},
+			},
+			randIn: []int{100},
+			delays: []time.Duration{2 * time.Millisecond},
+			repeat: 1,
+		}, {
+			name: "invalid header results in default percentage",
+			md: metadata.MD{
+				headerDelayDuration:   []string{"2"},
+				headerDelayPercentage: []string{"error"},
+			},
+			randIn: []int{100},
+			repeat: 1,
+		}, {
+			name: "cannot exceed percentage in filter",
+			md: metadata.MD{
+				headerDelayDuration:   []string{"2"},
+				headerDelayPercentage: []string{"100"},
+			},
+			randIn: []int{100},
+			repeat: 1,
+		}},
+	}, {
+		name: "abort then delay filters",
+		cfgs: []*fpb.HTTPFault{{
+			Abort: &fpb.FaultAbort{
+				Percentage: &tpb.FractionalPercent{Numerator: 50, Denominator: tpb.FractionalPercent_HUNDRED},
+				ErrorType:  &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Unimplemented)},
+			},
+		}, {
+			Delay: &cpb.FaultDelay{
+				Percentage:         &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED},
+				FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)},
+			},
+		}},
+		randOutInc: 10,
+		want: []subcase{{
+			name:   "50% delay and abort (abort skips delay)",
+			code:   codes.Unimplemented,
+			randIn: []int{100},
+			repeat: 5,
+		}, {
+			name:   "30% delay, no abort",
+			randIn: []int{100, 100},
+			repeat: 3,
+			delays: []time.Duration{time.Second},
+		}, {
+			name:   "20% success",
+			randIn: []int{100, 100},
+			repeat: 2,
+		}},
+	}}
+
+	fs, nodeID, port, cleanup := clientSetup(t)
+	defer cleanup()
+	resources := e2e.DefaultClientResources("myservice", nodeID, "localhost", port)
+	hcm := new(v3httppb.HttpConnectionManager)
+	err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	routerFilter := hcm.HttpFilters[len(hcm.HttpFilters)-1]
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			defer func() { randIntn = grpcrand.Intn; newTimer = time.NewTimer }()
+			var intnCalls []int
+			var newTimerCalls []time.Duration
+			randOut := 0
+			randIntn = func(n int) int {
+				intnCalls = append(intnCalls, n)
+				return randOut % n
+			}
+
+			newTimer = func(d time.Duration) *time.Timer {
+				newTimerCalls = append(newTimerCalls, d)
+				return time.NewTimer(0)
+			}
+
+			hcm.HttpFilters = nil
+			for i, cfg := range tc.cfgs {
+				hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter(fmt.Sprintf("fault%d", i), cfg))
+			}
+			hcm.HttpFilters = append(hcm.HttpFilters, routerFilter)
+			hcmAny, err := ptypes.MarshalAny(hcm)
+			if err != nil {
+				t.Fatal(err)
+			}
+			resources.Listeners[0].ApiListener.ApiListener = hcmAny
+			resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny}
+
+			if err := fs.Update(resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn and run the test case.
+			cc, err := grpc.Dial("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials()))
+			if err != nil {
+				t.Fatalf("failed to dial local test server: %v", err)
+			}
+			defer cc.Close()
+
+			client := testpb.NewTestServiceClient(cc)
+			count := 0
+			for _, want := range tc.want {
+				t.Run(want.name, func(t *testing.T) {
+					ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+					defer cancel()
+					if want.repeat == 0 {
+						t.Fatalf("invalid repeat count")
+					}
+					for n := 0; n < want.repeat; n++ {
+						intnCalls = nil
+						newTimerCalls = nil
+						ctx = metadata.NewOutgoingContext(ctx, want.md)
+						_, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true))
+						t.Logf("RPC %d: err: %v, intnCalls: %v, newTimerCalls: %v", count, err, intnCalls, newTimerCalls)
+						if status.Code(err) != want.code || !reflect.DeepEqual(intnCalls, want.randIn) || !reflect.DeepEqual(newTimerCalls, want.delays) {
+							t.Errorf("WANTED code: %v, intnCalls: %v, newTimerCalls: %v", want.code, want.randIn, want.delays)
+						}
+						randOut += tc.randOutInc
+						count++
+					}
+				})
+			}
+		})
+	}
+}
+
+func (s) TestFaultInjection_MaxActiveFaults(t *testing.T) {
+	fs, nodeID, port, cleanup := clientSetup(t)
+	defer cleanup()
+	resources := e2e.DefaultClientResources("myservice", nodeID, "localhost", port)
+	hcm := new(v3httppb.HttpConnectionManager)
+	err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() { newTimer = time.NewTimer }()
+	timers := make(chan *time.Timer, 2)
+	newTimer = func(d time.Duration) *time.Timer {
+		t := time.NewTimer(24 * time.Hour) // Will reset to fire.
+		timers <- t
+		return t
+	}
+
+	hcm.HttpFilters = append([]*v3httppb.HttpFilter{
+		e2e.HTTPFilter("fault", &fpb.HTTPFault{
+			MaxActiveFaults: wrapperspb.UInt32(2),
+			Delay: &cpb.FaultDelay{
+				Percentage:         &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED},
+				FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)},
+			},
+		})},
+		hcm.HttpFilters...)
+	hcmAny, err := ptypes.MarshalAny(hcm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resources.Listeners[0].ApiListener.ApiListener = hcmAny
+	resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny}
+
+	if err := fs.Update(resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn
+	cc, err := grpc.Dial("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testpb.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	streams := make(chan testpb.TestService_FullDuplexCallClient)
+	startStream := func() {
+		str, err := client.FullDuplexCall(ctx)
+		if err != nil {
+			t.Error("RPC error:", err)
+		}
+		streams <- str
+	}
+	endStream := func() {
+		str := <-streams
+		str.CloseSend()
+		if _, err := str.Recv(); err != io.EOF {
+			t.Fatal("stream error:", err)
+		}
+	}
+	releaseStream := func() {
+		timer := <-timers
+		timer.Reset(0)
+	}
+
+	// Start three streams; two should delay.
+	go startStream()
+	go startStream()
+	go startStream()
+
+	// End one of the streams.  Ensure the others are blocked on creation.
+	endStream()
+
+	select {
+	case <-streams:
+		t.Errorf("unexpected second stream created before delay expires")
+	case <-time.After(50 * time.Millisecond):
+		// Wait a short time to ensure no other streams were started yet.
+	}
+
+	// Start one more; it should not be blocked.
+	go startStream()
+	endStream()
+
+	// Expire one stream's delay; it should be created.
+	releaseStream()
+	endStream()
+
+	// Another new stream should delay.
+	go startStream()
+	select {
+	case <-streams:
+		t.Errorf("unexpected second stream created before delay expires")
+	case <-time.After(50 * time.Millisecond):
+		// Wait a short time to ensure no other streams were started yet.
+	}
+
+	// Expire both pending timers and end the two streams.
+	releaseStream()
+	releaseStream()
+	endStream()
+	endStream()
+}
diff --git a/xds/xds.go b/xds/xds.go
index 5deafd130a25..17b84ec932cb 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -32,5 +32,6 @@ import (
 	_ "google.golang.org/grpc/xds/internal/balancer"                // Register the balancers.
 	_ "google.golang.org/grpc/xds/internal/client/v2"               // Register the v2 xDS API client.
 	_ "google.golang.org/grpc/xds/internal/client/v3"               // Register the v3 xDS API client.
+	_ "google.golang.org/grpc/xds/internal/httpfilter/fault"        // Register the fault injection filter.
 	_ "google.golang.org/grpc/xds/internal/resolver"                // Register the xds_resolver.
 )

From de3c78e4f1f16ca8e7f8fed6d12dcf3e1de337c1 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 12 Mar 2021 13:23:11 -0800
Subject: [PATCH 386/481] xds: validate 'listener_filters' and
 'use_original_dst' fields (#4258)

---
 xds/internal/client/lds_test.go | 54 +++++++++++++++++++++++++++++++++
 xds/internal/client/xds.go      |  6 ++++
 2 files changed, 60 insertions(+)

diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 3a0f4b5b1991..e2be3b4025ec 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -915,6 +915,60 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 		wantMD     UpdateMetadata
 		wantErr    string
 	}{
+		{
+			name: "non-empty listener filters",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							ListenerFilters: []*v3listenerpb.ListenerFilter{
+								{Name: "listener-filter-1"},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: "unsupported field 'listener_filters'",
+		},
+		{
+			name: "use_original_dst is set",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name:           v3LDSTarget,
+							UseOriginalDst: &wrapperspb.BoolValue{Value: true},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: "unsupported field 'use_original_dst'",
+		},
 		{
 			name: "no address field",
 			resources: []*anypb.Any{
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 0305f0a3015f..8f604d32b8f3 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -246,6 +246,12 @@ func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool) ([]HTTPFilt
 }
 
 func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) {
+	if n := len(lis.ListenerFilters); n != 0 {
+		return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n)
+	}
+	if useOrigDst := lis.GetUseOriginalDst(); useOrigDst != nil && useOrigDst.GetValue() {
+		return nil, errors.New("unsupported field 'use_original_dst' is present and set to true")
+	}
 	addr := lis.GetAddress()
 	if addr == nil {
 		return nil, fmt.Errorf("no address field in LDS response: %+v", lis)

From 21976fa3e38a266811384409bc8b25437cc1ff1d Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 12 Mar 2021 15:19:57 -0800
Subject: [PATCH 387/481] xds: disable fault injection test on 386 (#4264)

---
 xds/internal/httpfilter/fault/fault_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 562cdc0832bf..1ad955146b0c 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -1,3 +1,5 @@
+// +build !386
+
 /*
  *
  * Copyright 2020 gRPC authors.

From 1e7119b13689dac5b8fe0a70118434eff96e997b Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 15 Mar 2021 14:13:13 -0700
Subject: [PATCH 388/481] xds: support all matchers for SANs (#4246)

---
 credentials/xds/xds_client_test.go            |  21 +-
 internal/credentials/xds/handshake_info.go    | 129 +++++--
 .../credentials/xds/handshake_info_test.go    | 304 +++++++++++++++++
 internal/xds/string_matcher.go                | 183 ++++++++++
 internal/xds/string_matcher_test.go           | 316 ++++++++++++++++++
 .../balancer/cdsbalancer/cdsbalancer.go       |   4 +-
 .../cdsbalancer/cdsbalancer_security_test.go  |  37 +-
 xds/internal/client/cds_test.go               | 212 +++++++++++-
 xds/internal/client/client.go                 |  15 +-
 xds/internal/client/xds.go                    |  14 +-
 10 files changed, 1179 insertions(+), 56 deletions(-)
 create mode 100644 internal/credentials/xds/handshake_info_test.go
 create mode 100644 internal/xds/string_matcher.go
 create mode 100644 internal/xds/string_matcher_test.go

diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index 6d40b0700082..82cfa5876acb 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -36,6 +36,7 @@ import (
 	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/testdata"
 )
@@ -43,7 +44,7 @@ import (
 const (
 	defaultTestTimeout      = 1 * time.Second
 	defaultTestShortTimeout = 10 * time.Millisecond
-	defaultTestCertSAN      = "*.test.example.com"
+	defaultTestCertSAN      = "abc.test.example.com"
 	authority               = "authority"
 )
 
@@ -214,12 +215,14 @@ func makeRootProvider(t *testing.T, caPath string) *fakeProvider {
 
 // newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo
 // context value added to it.
-func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sans ...string) context.Context {
+func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sanExactMatch string) context.Context {
 	// Creating the HandshakeInfo and adding it to the attributes is very
 	// similar to what the CDS balancer would do when it intercepts calls to
 	// NewSubConn().
 	info := xdsinternal.NewHandshakeInfo(root, identity)
-	info.SetAcceptedSANs(sans)
+	if sanExactMatch != "" {
+		info.SetSANMatchers([]xds.StringMatcher{xds.StringMatcherForTesting(newStringP(sanExactMatch), nil, nil, nil, nil, false)})
+	}
 	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)
 
 	// Moving the attributes from the resolver.Address to the context passed to
@@ -293,7 +296,7 @@ func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) {
 
 	pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{})
+	ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{}, "")
 	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
 		t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo")
 	}
@@ -330,7 +333,7 @@ func (s) TestClientCredsProviderFailure(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
-			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider)
+			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider, "")
 			if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) {
 				t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
 			}
@@ -372,7 +375,7 @@ func (s) TestClientCredsSuccess(t *testing.T) {
 			desc:          "mTLS with no acceptedSANs specified",
 			handshakeFunc: testServerMutualTLSHandshake,
 			handshakeInfoCtx: func(ctx context.Context) context.Context {
-				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
+				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), "")
 			},
 		},
 	}
@@ -532,7 +535,7 @@ func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	// use the correct trust roots.
 	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
 	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil)
-	handshakeInfo.SetAcceptedSANs([]string{defaultTestCertSAN})
+	handshakeInfo.SetSANMatchers([]xds.StringMatcher{xds.StringMatcherForTesting(newStringP(defaultTestCertSAN), nil, nil, nil, nil, false)})
 
 	// We need to repeat most of what newTestContextWithHandshakeInfo() does
 	// here because we need access to the underlying HandshakeInfo so that we
@@ -584,3 +587,7 @@ func (s) TestClientClone(t *testing.T) {
 		t.Fatal("return value from Clone() doesn't point to new credentials instance")
 	}
 }
+
+func newStringP(s string) *string {
+	return &s
+}
diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go
index db0e6542dd23..ca2e39edd6d1 100644
--- a/internal/credentials/xds/handshake_info.go
+++ b/internal/credentials/xds/handshake_info.go
@@ -25,11 +25,13 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
+	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -64,8 +66,8 @@ type HandshakeInfo struct {
 	mu                sync.Mutex
 	rootProvider      certprovider.Provider
 	identityProvider  certprovider.Provider
-	acceptedSANs      map[string]bool // Only on the client side.
-	requireClientCert bool            // Only on server side.
+	sanMatchers       []xdsinternal.StringMatcher // Only on the client side.
+	requireClientCert bool                        // Only on server side.
 }
 
 // SetRootCertProvider updates the root certificate provider.
@@ -82,13 +84,10 @@ func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider)
 	hi.mu.Unlock()
 }
 
-// SetAcceptedSANs updates the list of accepted SANs.
-func (hi *HandshakeInfo) SetAcceptedSANs(sans []string) {
+// SetSANMatchers updates the list of SAN matchers.
+func (hi *HandshakeInfo) SetSANMatchers(sanMatchers []xdsinternal.StringMatcher) {
 	hi.mu.Lock()
-	hi.acceptedSANs = make(map[string]bool, len(sans))
-	for _, san := range sans {
-		hi.acceptedSANs[san] = true
-	}
+	hi.sanMatchers = sanMatchers
 	hi.mu.Unlock()
 }
 
@@ -112,6 +111,14 @@ func (hi *HandshakeInfo) UseFallbackCreds() bool {
 	return hi.identityProvider == nil && hi.rootProvider == nil
 }
 
+// GetSANMatchersForTesting returns the SAN matchers stored in HandshakeInfo.
+// To be used only for testing purposes.
+func (hi *HandshakeInfo) GetSANMatchersForTesting() []xdsinternal.StringMatcher {
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+	return append([]xdsinternal.StringMatcher{}, hi.sanMatchers...)
+}
+
 // ClientSideTLSConfig constructs a tls.Config to be used in a client-side
 // handshake based on the contents of the HandshakeInfo.
 func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config, error) {
@@ -184,37 +191,113 @@ func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config,
 	return cfg, nil
 }
 
-// MatchingSANExists returns true if the SAN contained in the passed in
-// certificate is present in the list of accepted SANs in the HandshakeInfo.
+// MatchingSANExists returns true if the SANs contained in cert match the
+// criteria enforced by the list of SAN matchers in HandshakeInfo.
 //
-// If the list of accepted SANs in the HandshakeInfo is empty, this function
+// If the list of SAN matchers in the HandshakeInfo is empty, this function
 // returns true for all input certificates.
 func (hi *HandshakeInfo) MatchingSANExists(cert *x509.Certificate) bool {
-	if len(hi.acceptedSANs) == 0 {
+	hi.mu.Lock()
+	defer hi.mu.Unlock()
+	if len(hi.sanMatchers) == 0 {
 		return true
 	}
 
-	var sans []string
 	// SANs can be specified in any of these four fields on the parsed cert.
-	sans = append(sans, cert.DNSNames...)
-	sans = append(sans, cert.EmailAddresses...)
-	for _, ip := range cert.IPAddresses {
-		sans = append(sans, ip.String())
+	for _, san := range cert.DNSNames {
+		if hi.matchSAN(san, true) {
+			return true
+		}
 	}
-	for _, uri := range cert.URIs {
-		sans = append(sans, uri.String())
+	for _, san := range cert.EmailAddresses {
+		if hi.matchSAN(san, false) {
+			return true
+		}
+	}
+	for _, san := range cert.IPAddresses {
+		if hi.matchSAN(san.String(), false) {
+			return true
+		}
+	}
+	for _, san := range cert.URIs {
+		if hi.matchSAN(san.String(), false) {
+			return true
+		}
 	}
+	return false
+}
 
-	hi.mu.Lock()
-	defer hi.mu.Unlock()
-	for _, san := range sans {
-		if hi.acceptedSANs[san] {
+// Caller must hold mu.
+func (hi *HandshakeInfo) matchSAN(san string, isDNS bool) bool {
+	for _, matcher := range hi.sanMatchers {
+		if em := matcher.ExactMatch(); em != "" && isDNS {
+			// This is a special case which is documented in the xDS protos.
+			// If the DNS SAN is a wildcard entry, and the match criteria is
+			// `exact`, then we need to perform DNS wildcard matching
+			// instead of regular string comparison.
+			if dnsMatch(em, san) {
+				return true
+			}
+			continue
+		}
+		if matcher.Match(san) {
 			return true
 		}
 	}
 	return false
 }
 
+// dnsMatch implements a DNS wildcard matching algorithm based on RFC2828 and
+// grpc-java's implementation in `OkHostnameVerifier` class.
+//
+// NOTE: Here the `host` argument is the one from the set of string matchers in
+// the xDS proto and the `san` argument is a DNS SAN from the certificate, and
+// this is the one which can potentially contain a wildcard pattern.
+func dnsMatch(host, san string) bool {
+	// Add trailing "." and turn them into absolute domain names.
+	if !strings.HasSuffix(host, ".") {
+		host += "."
+	}
+	if !strings.HasSuffix(san, ".") {
+		san += "."
+	}
+	// Domain names are case-insensitive.
+	host = strings.ToLower(host)
+	san = strings.ToLower(san)
+
+	// If san does not contain a wildcard, do exact match.
+	if !strings.Contains(san, "*") {
+		return host == san
+	}
+
+	// Wildcard dns matching rules
+	// - '*' is only permitted in the left-most label and must be the only
+	//   character in that label. For example, *.example.com is permitted, while
+	//   *a.example.com, a*.example.com, a*b.example.com, a.*.example.com are
+	//   not permitted.
+	// - '*' matches a single domain name component. For example, *.example.com
+	//   matches test.example.com but does not match sub.test.example.com.
+	// - Wildcard patterns for single-label domain names are not permitted.
+	if san == "*." || !strings.HasPrefix(san, "*.") || strings.Contains(san[1:], "*") {
+		return false
+	}
+	// Optimization: at this point, we know that the san contains a '*' and
+	// is the first domain component of san. So, the host name must be at
+	// least as long as the san to be able to match.
+	if len(host) < len(san) {
+		return false
+	}
+	// Hostname must end with the non-wildcard portion of san.
+	if !strings.HasSuffix(host, san[1:]) {
+		return false
+	}
+	// At this point we know that the hostName and san share the same suffix
+	// (the non-wildcard portion of san). Now, we just need to make sure
+	// that the '*' does not match across domain components.
+	hostPrefix := strings.TrimSuffix(host, san[1:])
+	return !strings.Contains(hostPrefix, ".")
+}
+
 // NewHandshakeInfo returns a new instance of HandshakeInfo with the given root
 // and identity certificate providers.
 func NewHandshakeInfo(root, identity certprovider.Provider) *HandshakeInfo {
diff --git a/internal/credentials/xds/handshake_info_test.go b/internal/credentials/xds/handshake_info_test.go
new file mode 100644
index 000000000000..81906fa758a1
--- /dev/null
+++ b/internal/credentials/xds/handshake_info_test.go
@@ -0,0 +1,304 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"crypto/x509"
+	"net"
+	"net/url"
+	"regexp"
+	"testing"
+
+	xdsinternal "google.golang.org/grpc/internal/xds"
+)
+
+func TestDNSMatch(t *testing.T) {
+	tests := []struct {
+		desc      string
+		host      string
+		pattern   string
+		wantMatch bool
+	}{
+		{
+			desc:      "invalid wildcard 1",
+			host:      "aa.example.com",
+			pattern:   "*a.example.com",
+			wantMatch: false,
+		},
+		{
+			desc:      "invalid wildcard 2",
+			host:      "aa.example.com",
+			pattern:   "a*.example.com",
+			wantMatch: false,
+		},
+		{
+			desc:      "invalid wildcard 3",
+			host:      "abc.example.com",
+			pattern:   "a*c.example.com",
+			wantMatch: false,
+		},
+		{
+			desc:      "wildcard in one of the middle components",
+			host:      "abc.test.example.com",
+			pattern:   "abc.*.example.com",
+			wantMatch: false,
+		},
+		{
+			desc:      "single component wildcard",
+			host:      "a.example.com",
+			pattern:   "*",
+			wantMatch: false,
+		},
+		{
+			desc:      "short host name",
+			host:      "a.com",
+			pattern:   "*.example.com",
+			wantMatch: false,
+		},
+		{
+			desc:      "suffix mismatch",
+			host:      "a.notexample.com",
+			pattern:   "*.example.com",
+			wantMatch: false,
+		},
+		{
+			desc:      "wildcard match across components",
+			host:      "sub.test.example.com",
+			pattern:   "*.example.com.",
+			wantMatch: false,
+		},
+		{
+			desc:      "host doesn't end in period",
+			host:      "test.example.com",
+			pattern:   "test.example.com.",
+			wantMatch: true,
+		},
+		{
+			desc:      "pattern doesn't end in period",
+			host:      "test.example.com.",
+			pattern:   "test.example.com",
+			wantMatch: true,
+		},
+		{
+			desc:      "case insensitive",
+			host:      "TEST.EXAMPLE.COM.",
+			pattern:   "test.example.com.",
+			wantMatch: true,
+		},
+		{
+			desc:      "simple match",
+			host:      "test.example.com",
+			pattern:   "test.example.com",
+			wantMatch: true,
+		},
+		{
+			desc:      "good wildcard",
+			host:      "a.example.com",
+			pattern:   "*.example.com",
+			wantMatch: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotMatch := dnsMatch(test.host, test.pattern)
+			if gotMatch != test.wantMatch {
+				t.Fatalf("dnsMatch(%s, %s) = %v, want %v", test.host, test.pattern, gotMatch, test.wantMatch)
+			}
+		})
+	}
+}
+
+func TestMatchingSANExists_FailureCases(t *testing.T) {
+	url1, err := url.Parse("http://golang.org")
+	if err != nil {
+		t.Fatalf("url.Parse() failed: %v", err)
+	}
+	url2, err := url.Parse("https://github.com/grpc/grpc-go")
+	if err != nil {
+		t.Fatalf("url.Parse() failed: %v", err)
+	}
+	inputCert := &x509.Certificate{
+		DNSNames:       []string{"foo.bar.example.com", "bar.baz.test.com", "*.example.com"},
+		EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"},
+		IPAddresses:    []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")},
+		URIs:           []*url.URL{url1, url2},
+	}
+
+	tests := []struct {
+		desc        string
+		sanMatchers []xdsinternal.StringMatcher
+	}{
+		{
+			desc: "exact match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(newStringP("abcd.test.com"), nil, nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(newStringP("http://golang"), nil, nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(newStringP("HTTP://GOLANG.ORG"), nil, nil, nil, nil, false),
+			},
+		},
+		{
+			desc: "prefix match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, newStringP("i-aint-the-one"), nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, newStringP("FOO.BAR"), nil, nil, nil, false),
+			},
+		},
+		{
+			desc: "suffix match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("i-aint-the-one"), nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("1::68"), nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP(".COM"), nil, nil, false),
+			},
+		},
+		{
+			desc: "regex match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.examples\.com`), false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
+			},
+		},
+		{
+			desc: "contains match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("i-aint-the-one"), nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("2001:db8:1:1::68"), nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, false),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			hi := NewHandshakeInfo(nil, nil)
+			hi.SetSANMatchers(test.sanMatchers)
+
+			if hi.MatchingSANExists(inputCert) {
+				t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v succeeded when expected to fail", inputCert, test.sanMatchers)
+			}
+		})
+	}
+}
+
+func TestMatchingSANExists_Success(t *testing.T) {
+	url1, err := url.Parse("http://golang.org")
+	if err != nil {
+		t.Fatalf("url.Parse() failed: %v", err)
+	}
+	url2, err := url.Parse("https://github.com/grpc/grpc-go")
+	if err != nil {
+		t.Fatalf("url.Parse() failed: %v", err)
+	}
+	inputCert := &x509.Certificate{
+		DNSNames:       []string{"baz.test.com", "*.example.com"},
+		EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"},
+		IPAddresses:    []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")},
+		URIs:           []*url.URL{url1, url2},
+	}
+
+	tests := []struct {
+		desc        string
+		sanMatchers []xdsinternal.StringMatcher
+	}{
+		{
+			desc: "no san matchers",
+		},
+		{
+			desc: "exact match dns wildcard",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(newStringP("abc.example.com"), nil, nil, nil, nil, false),
+			},
+		},
+		{
+			desc: "exact match ignore case",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(newStringP("FOOBAR@EXAMPLE.COM"), nil, nil, nil, nil, true),
+			},
+		},
+		{
+			desc: "prefix match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP(".co.in"), nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, newStringP("baz.test"), nil, nil, nil, false),
+			},
+		},
+		{
+			desc: "prefix match ignore case",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, newStringP("BAZ.test"), nil, nil, nil, true),
+			},
+		},
+		{
+			desc: "suffix  match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("192.168.1.1"), nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("@test.com"), nil, nil, false),
+			},
+		},
+		{
+			desc: "suffix  match ignore case",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("@test.COM"), nil, nil, true),
+			},
+		},
+		{
+			desc: "regex match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("https://github.com/grpc/grpc-java"), nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.test\.com`), false),
+			},
+		},
+		{
+			desc: "contains match",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("2001:68::db8"), nil, false),
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("192.0.0"), nil, false),
+			},
+		},
+		{
+			desc: "contains match ignore case",
+			sanMatchers: []xdsinternal.StringMatcher{
+				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, true),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			hi := NewHandshakeInfo(nil, nil)
+			hi.SetSANMatchers(test.sanMatchers)
+
+			if !hi.MatchingSANExists(inputCert) {
+				t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v failed when expected to succeed", inputCert, test.sanMatchers)
+			}
+		})
+	}
+}
+
+func newStringP(s string) *string {
+	return &s
+}
diff --git a/internal/xds/string_matcher.go b/internal/xds/string_matcher.go
new file mode 100644
index 000000000000..21f15aad1b88
--- /dev/null
+++ b/internal/xds/string_matcher.go
@@ -0,0 +1,183 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds contains types that need to be shared between code under
+// google.golang.org/grpc/xds/... and the rest of gRPC.
+package xds
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+)
+
+// StringMatcher contains match criteria for matching a string, and is an
+// internal representation of the `StringMatcher` proto defined at
+// https://github.com/envoyproxy/envoy/blob/main/api/envoy/type/matcher/v3/string.proto.
+type StringMatcher struct {
+	// Since these match fields are part of a `oneof` in the corresponding xDS
+	// proto, only one of them is expected to be set.
+	exactMatch    *string
+	prefixMatch   *string
+	suffixMatch   *string
+	regexMatch    *regexp.Regexp
+	containsMatch *string
+	// If true, indicates the exact/prefix/suffix/contains matching should be
+	// case insensitive. This has no effect on the regex match.
+	ignoreCase bool
+}
+
+// Match returns true if input matches the criteria in the given StringMatcher.
+func (sm StringMatcher) Match(input string) bool {
+	if sm.ignoreCase {
+		input = strings.ToLower(input)
+	}
+	switch {
+	case sm.exactMatch != nil:
+		return input == *sm.exactMatch
+	case sm.prefixMatch != nil:
+		return strings.HasPrefix(input, *sm.prefixMatch)
+	case sm.suffixMatch != nil:
+		return strings.HasSuffix(input, *sm.suffixMatch)
+	case sm.regexMatch != nil:
+		return sm.regexMatch.MatchString(input)
+	case sm.containsMatch != nil:
+		return strings.Contains(input, *sm.containsMatch)
+	}
+	return false
+}
+
+// StringMatcherFromProto is a helper function to create a StringMatcher from
+// the corresponding StringMatcher proto.
+//
+// Returns a non-nil error if matcherProto is invalid.
+func StringMatcherFromProto(matcherProto *v3matcherpb.StringMatcher) (StringMatcher, error) {
+	if matcherProto == nil {
+		return StringMatcher{}, errors.New("input StringMatcher proto is nil")
+	}
+
+	matcher := StringMatcher{ignoreCase: matcherProto.GetIgnoreCase()}
+	switch mt := matcherProto.GetMatchPattern().(type) {
+	case *v3matcherpb.StringMatcher_Exact:
+		matcher.exactMatch = &mt.Exact
+		if matcher.ignoreCase {
+			*matcher.exactMatch = strings.ToLower(*matcher.exactMatch)
+		}
+	case *v3matcherpb.StringMatcher_Prefix:
+		if matcherProto.GetPrefix() == "" {
+			return StringMatcher{}, errors.New("empty prefix is not allowed in StringMatcher")
+		}
+		matcher.prefixMatch = &mt.Prefix
+		if matcher.ignoreCase {
+			*matcher.prefixMatch = strings.ToLower(*matcher.prefixMatch)
+		}
+	case *v3matcherpb.StringMatcher_Suffix:
+		if matcherProto.GetSuffix() == "" {
+			return StringMatcher{}, errors.New("empty suffix is not allowed in StringMatcher")
+		}
+		matcher.suffixMatch = &mt.Suffix
+		if matcher.ignoreCase {
+			*matcher.suffixMatch = strings.ToLower(*matcher.suffixMatch)
+		}
+	case *v3matcherpb.StringMatcher_SafeRegex:
+		regex := matcherProto.GetSafeRegex().GetRegex()
+		re, err := regexp.Compile(regex)
+		if err != nil {
+			return StringMatcher{}, fmt.Errorf("safe_regex matcher %q is invalid", regex)
+		}
+		matcher.regexMatch = re
+	case *v3matcherpb.StringMatcher_Contains:
+		if matcherProto.GetContains() == "" {
+			return StringMatcher{}, errors.New("empty contains is not allowed in StringMatcher")
+		}
+		matcher.containsMatch = &mt.Contains
+		if matcher.ignoreCase {
+			*matcher.containsMatch = strings.ToLower(*matcher.containsMatch)
+		}
+	default:
+		return StringMatcher{}, fmt.Errorf("unrecognized string matcher: %+v", matcherProto)
+	}
+	return matcher, nil
+}
+
+// StringMatcherForTesting is a helper function to create a StringMatcher based
+// on the given arguments. Intended only for testing purposes.
+func StringMatcherForTesting(exact, prefix, suffix, contains *string, regex *regexp.Regexp, ignoreCase bool) StringMatcher {
+	sm := StringMatcher{
+		exactMatch:    exact,
+		prefixMatch:   prefix,
+		suffixMatch:   suffix,
+		regexMatch:    regex,
+		containsMatch: contains,
+		ignoreCase:    ignoreCase,
+	}
+	if ignoreCase {
+		switch {
+		case sm.exactMatch != nil:
+			*sm.exactMatch = strings.ToLower(*exact)
+		case sm.prefixMatch != nil:
+			*sm.prefixMatch = strings.ToLower(*prefix)
+		case sm.suffixMatch != nil:
+			*sm.suffixMatch = strings.ToLower(*suffix)
+		case sm.containsMatch != nil:
+			*sm.containsMatch = strings.ToLower(*contains)
+		}
+	}
+	return sm
+}
+
+// ExactMatch returns the value of the configured exact match or an empty string
+// if exact match criteria was not specified.
+func (sm StringMatcher) ExactMatch() string {
+	if sm.exactMatch != nil {
+		return *sm.exactMatch
+	}
+	return ""
+}
+
+// Equal returns true if other and sm are equivalent to each other.
+func (sm StringMatcher) Equal(other StringMatcher) bool {
+	if sm.ignoreCase != other.ignoreCase {
+		return false
+	}
+
+	if (sm.exactMatch != nil) != (other.exactMatch != nil) ||
+		(sm.prefixMatch != nil) != (other.prefixMatch != nil) ||
+		(sm.suffixMatch != nil) != (other.suffixMatch != nil) ||
+		(sm.regexMatch != nil) != (other.regexMatch != nil) ||
+		(sm.containsMatch != nil) != (other.containsMatch != nil) {
+		return false
+	}
+
+	switch {
+	case sm.exactMatch != nil:
+		return *sm.exactMatch == *other.exactMatch
+	case sm.prefixMatch != nil:
+		return *sm.prefixMatch == *other.prefixMatch
+	case sm.suffixMatch != nil:
+		return *sm.suffixMatch == *other.suffixMatch
+	case sm.regexMatch != nil:
+		return sm.regexMatch.String() == other.regexMatch.String()
+	case sm.containsMatch != nil:
+		return *sm.containsMatch == *other.containsMatch
+	}
+	return true
+}
diff --git a/internal/xds/string_matcher_test.go b/internal/xds/string_matcher_test.go
new file mode 100644
index 000000000000..7908ac974b23
--- /dev/null
+++ b/internal/xds/string_matcher_test.go
@@ -0,0 +1,316 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"regexp"
+	"testing"
+
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestStringMatcherFromProto(t *testing.T) {
+	tests := []struct {
+		desc        string
+		inputProto  *v3matcherpb.StringMatcher
+		wantMatcher StringMatcher
+		wantErr     bool
+	}{
+		{
+			desc:    "nil proto",
+			wantErr: true,
+		},
+		{
+			desc: "empty prefix",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "empty suffix",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "empty contains",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "invalid regex",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{
+					SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "invalid deprecated regex",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_HiddenEnvoyDeprecatedRegex{},
+			},
+			wantErr: true,
+		},
+		{
+			desc: "happy case exact",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"},
+			},
+			wantMatcher: StringMatcher{exactMatch: newStringP("exact")},
+		},
+		{
+			desc: "happy case exact ignore case",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "EXACT"},
+				IgnoreCase:   true,
+			},
+			wantMatcher: StringMatcher{
+				exactMatch: newStringP("exact"),
+				ignoreCase: true,
+			},
+		},
+		{
+			desc: "happy case prefix",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"},
+			},
+			wantMatcher: StringMatcher{prefixMatch: newStringP("prefix")},
+		},
+		{
+			desc: "happy case prefix ignore case",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "PREFIX"},
+				IgnoreCase:   true,
+			},
+			wantMatcher: StringMatcher{
+				prefixMatch: newStringP("prefix"),
+				ignoreCase:  true,
+			},
+		},
+		{
+			desc: "happy case suffix",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"},
+			},
+			wantMatcher: StringMatcher{suffixMatch: newStringP("suffix")},
+		},
+		{
+			desc: "happy case suffix ignore case",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "SUFFIX"},
+				IgnoreCase:   true,
+			},
+			wantMatcher: StringMatcher{
+				suffixMatch: newStringP("suffix"),
+				ignoreCase:  true,
+			},
+		},
+		{
+			desc: "happy case regex",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{
+					SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"},
+				},
+			},
+			wantMatcher: StringMatcher{regexMatch: regexp.MustCompile("good?regex?")},
+		},
+		{
+			desc: "happy case contains",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"},
+			},
+			wantMatcher: StringMatcher{containsMatch: newStringP("contains")},
+		},
+		{
+			desc: "happy case contains ignore case",
+			inputProto: &v3matcherpb.StringMatcher{
+				MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "CONTAINS"},
+				IgnoreCase:   true,
+			},
+			wantMatcher: StringMatcher{
+				containsMatch: newStringP("contains"),
+				ignoreCase:    true,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotMatcher, err := StringMatcherFromProto(test.inputProto)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("StringMatcherFromProto(%+v) returned err: %v, wantErr: %v", test.inputProto, err, test.wantErr)
+			}
+			if diff := cmp.Diff(gotMatcher, test.wantMatcher, cmp.AllowUnexported(regexp.Regexp{})); diff != "" {
+				t.Fatalf("StringMatcherFromProto(%+v) returned unexpected diff (-got, +want):\n%s", test.inputProto, diff)
+			}
+		})
+	}
+}
+
+func TestMatch(t *testing.T) {
+	var (
+		exactMatcher, _           = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}})
+		prefixMatcher, _          = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}})
+		suffixMatcher, _          = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}})
+		regexMatcher, _           = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}}})
+		containsMatcher, _        = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}})
+		exactMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{
+			MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"},
+			IgnoreCase:   true,
+		})
+		prefixMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{
+			MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"},
+			IgnoreCase:   true,
+		})
+		suffixMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{
+			MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"},
+			IgnoreCase:   true,
+		})
+		containsMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{
+			MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"},
+			IgnoreCase:   true,
+		})
+	)
+
+	tests := []struct {
+		desc      string
+		matcher   StringMatcher
+		input     string
+		wantMatch bool
+	}{
+		{
+			desc:      "exact match success",
+			matcher:   exactMatcher,
+			input:     "exact",
+			wantMatch: true,
+		},
+		{
+			desc:    "exact match failure",
+			matcher: exactMatcher,
+			input:   "not-exact",
+		},
+		{
+			desc:      "exact match success with ignore case",
+			matcher:   exactMatcherIgnoreCase,
+			input:     "EXACT",
+			wantMatch: true,
+		},
+		{
+			desc:    "exact match failure with ignore case",
+			matcher: exactMatcherIgnoreCase,
+			input:   "not-exact",
+		},
+		{
+			desc:      "prefix match success",
+			matcher:   prefixMatcher,
+			input:     "prefixIsHere",
+			wantMatch: true,
+		},
+		{
+			desc:    "prefix match failure",
+			matcher: prefixMatcher,
+			input:   "not-prefix",
+		},
+		{
+			desc:      "prefix match success with ignore case",
+			matcher:   prefixMatcherIgnoreCase,
+			input:     "PREFIXisHere",
+			wantMatch: true,
+		},
+		{
+			desc:    "prefix match failure with ignore case",
+			matcher: prefixMatcherIgnoreCase,
+			input:   "not-PREFIX",
+		},
+		{
+			desc:      "suffix match success",
+			matcher:   suffixMatcher,
+			input:     "hereIsThesuffix",
+			wantMatch: true,
+		},
+		{
+			desc:    "suffix match failure",
+			matcher: suffixMatcher,
+			input:   "suffix-is-not-here",
+		},
+		{
+			desc:      "suffix match success with ignore case",
+			matcher:   suffixMatcherIgnoreCase,
+			input:     "hereIsTheSuFFix",
+			wantMatch: true,
+		},
+		{
+			desc:    "suffix match failure with ignore case",
+			matcher: suffixMatcherIgnoreCase,
+			input:   "SUFFIX-is-not-here",
+		},
+		{
+			desc:      "regex match success",
+			matcher:   regexMatcher,
+			input:     "goodregex",
+			wantMatch: true,
+		},
+		{
+			desc:    "regex match failure",
+			matcher: regexMatcher,
+			input:   "regex-is-not-here",
+		},
+		{
+			desc:      "contains match success",
+			matcher:   containsMatcher,
+			input:     "IScontainsHERE",
+			wantMatch: true,
+		},
+		{
+			desc:    "contains match failure",
+			matcher: containsMatcher,
+			input:   "con-tains-is-not-here",
+		},
+		{
+			desc:      "contains match success with ignore case",
+			matcher:   containsMatcherIgnoreCase,
+			input:     "isCONTAINShere",
+			wantMatch: true,
+		},
+		{
+			desc:    "contains match failure with ignore case",
+			matcher: containsMatcherIgnoreCase,
+			input:   "CON-TAINS-is-not-here",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotMatch := test.matcher.Match(test.input); gotMatch != test.wantMatch {
+				t.Errorf("StringMatcher.Match(%s) returned %v, want %v", test.input, gotMatch, test.wantMatch)
+			}
+		})
+	}
+}
+
+func newStringP(s string) *string {
+	return &s
+}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index e4d349753e13..19e075bd7b5d 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -235,7 +235,7 @@ func (b *cdsBalancer) handleSecurityConfig(config *xdsclient.SecurityConfig) err
 		// one where fallback credentials are to be used.
 		b.xdsHI.SetRootCertProvider(nil)
 		b.xdsHI.SetIdentityCertProvider(nil)
-		b.xdsHI.SetAcceptedSANs(nil)
+		b.xdsHI.SetSANMatchers(nil)
 		return nil
 	}
 
@@ -278,7 +278,7 @@ func (b *cdsBalancer) handleSecurityConfig(config *xdsclient.SecurityConfig) err
 	// could have been non-nil earlier.
 	b.xdsHI.SetRootCertProvider(rootProvider)
 	b.xdsHI.SetIdentityCertProvider(identityProvider)
-	b.xdsHI.SetAcceptedSANs(config.AcceptedSANs)
+	b.xdsHI.SetSANMatchers(config.SubjectAltNameMatchers)
 	return nil
 }
 
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index fee48c262ebe..73459dd64101 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -20,16 +20,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/credentials/local"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	xdscredsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
@@ -41,16 +44,25 @@ const (
 	fakeProvider1Name = "fake-certificate-provider-1"
 	fakeProvider2Name = "fake-certificate-provider-2"
 	fakeConfig        = "my fake config"
+	testSAN           = "test-san"
 )
 
 var (
+	testSANMatchers = []xdsinternal.StringMatcher{
+		xdsinternal.StringMatcherForTesting(newStringP(testSAN), nil, nil, nil, nil, true),
+		xdsinternal.StringMatcherForTesting(nil, newStringP(testSAN), nil, nil, nil, false),
+		xdsinternal.StringMatcherForTesting(nil, nil, newStringP(testSAN), nil, nil, false),
+		xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(testSAN), false),
+		xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP(testSAN), nil, false),
+	}
 	fpb1, fpb2                   *fakeProviderBuilder
 	bootstrapConfig              *bootstrap.Config
 	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
 		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
-			RootInstanceName:     "default1",
-			IdentityInstanceName: "default2",
+			RootInstanceName:       "default1",
+			IdentityInstanceName:   "default2",
+			SubjectAltNameMatchers: testSANMatchers,
 		},
 	}
 	cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{
@@ -61,6 +73,10 @@ var (
 	}
 )
 
+func newStringP(s string) *string {
+	return &s
+}
+
 func init() {
 	fpb1 = &fakeProviderBuilder{name: fakeProvider1Name}
 	fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
@@ -190,7 +206,7 @@ func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xd
 		if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
 			return nil, fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want)
 		}
-		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdsinternal.HandshakeInfo)
+		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo)
 		hi := getHI(gotAddrs[0].Attributes)
 		if hi == nil {
 			return nil, errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes")
@@ -198,6 +214,11 @@ func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xd
 		if gotFallback := hi.UseFallbackCreds(); gotFallback != wantFallback {
 			return nil, fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback)
 		}
+		if !wantFallback {
+			if diff := cmp.Diff(testSANMatchers, hi.GetSANMatchersForTesting(), cmp.AllowUnexported(regexp.Regexp{})); diff != "" {
+				return nil, fmt.Errorf("unexpected diff in the list of SAN matchers (-got, +want):\n%s", diff)
+			}
+		}
 	}
 	return sc, nil
 }
@@ -507,7 +528,7 @@ func (s) TestGoodSecurityConfig(t *testing.T) {
 		if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
 			t.Fatalf("resolver.Address passed to parent ClientConn through UpdateAddresses() has address %q, want %q", got, want)
 		}
-		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdsinternal.HandshakeInfo)
+		getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo)
 		hi := getHI(gotAddrs[0].Attributes)
 		if hi == nil {
 			t.Fatal("resolver.Address passed to parent ClientConn through UpdateAddresses() doesn't contain attributes")
@@ -657,7 +678,8 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	cdsUpdate := xdsclient.ClusterUpdate{
 		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
-			RootInstanceName: "default1",
+			RootInstanceName:       "default1",
+			SubjectAltNameMatchers: testSANMatchers,
 		},
 	}
 	wantCCS := edsCCS(serviceName, nil, false)
@@ -681,7 +703,8 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	cdsUpdate = xdsclient.ClusterUpdate{
 		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
-			RootInstanceName: "default2",
+			RootInstanceName:       "default2",
+			SubjectAltNameMatchers: testSANMatchers,
 		},
 	}
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index c5f1d76d32c3..bac0ef1aeaac 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -19,6 +19,7 @@
 package client
 
 import (
+	"regexp"
 	"testing"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
@@ -31,6 +32,7 @@ import (
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/wrapperspb"
@@ -288,9 +290,14 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		rootPluginInstance     = "rootPluginInstance"
 		rootCertName           = "rootCert"
 		serviceName            = "service"
-		san1                   = "san1"
-		san2                   = "san2"
+		sanExact               = "san-exact"
+		sanPrefix              = "san-prefix"
+		sanSuffix              = "san-suffix"
+		sanRegexBad            = "??"
+		sanRegexGood           = "san?regex?"
+		sanContains            = "san-contains"
 	)
+	var sanRE = regexp.MustCompile(sanRegexGood)
 
 	tests := []struct {
 		name       string
@@ -435,6 +442,182 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "empty-prefix-in-matching-SAN",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+														{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}},
+													},
+												},
+												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    rootPluginInstance,
+													CertificateName: rootCertName,
+												},
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty-suffix-in-matching-SAN",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+														{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}},
+													},
+												},
+												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    rootPluginInstance,
+													CertificateName: rootCertName,
+												},
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty-contains-in-matching-SAN",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+														{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}},
+													},
+												},
+												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    rootPluginInstance,
+													CertificateName: rootCertName,
+												},
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid-regex-in-matching-SAN",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+					ServiceName: serviceName,
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+				TransportSocket: &v3corepb.TransportSocket{
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: &anypb.Any{
+							TypeUrl: version.V3UpstreamTLSContextURL,
+							Value: func() []byte {
+								tls := &v3tlspb.UpstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+														{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}},
+													},
+												},
+												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+													InstanceName:    rootPluginInstance,
+													CertificateName: rootCertName,
+												},
+											},
+										},
+									},
+								}
+								mtls, _ := proto.Marshal(tls)
+								return mtls
+							}(),
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "happy-case-with-no-identity-certs",
 			cluster: &v3clusterpb.Cluster{
@@ -560,8 +743,14 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
 												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
 													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
-														{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: san1}},
-														{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: san2}},
+														{
+															MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact},
+															IgnoreCase:   true,
+														},
+														{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}},
+														{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}},
+														{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}},
+														{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}},
 													},
 												},
 												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -587,7 +776,13 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 					RootCertName:         rootCertName,
 					IdentityInstanceName: identityPluginInstance,
 					IdentityCertName:     identityCertName,
-					AcceptedSANs:         []string{san1, san2},
+					SubjectAltNameMatchers: []xdsinternal.StringMatcher{
+						xdsinternal.StringMatcherForTesting(newStringP(sanExact), nil, nil, nil, nil, true),
+						xdsinternal.StringMatcherForTesting(nil, newStringP(sanPrefix), nil, nil, nil, false),
+						xdsinternal.StringMatcherForTesting(nil, nil, newStringP(sanSuffix), nil, nil, false),
+						xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, sanRE, false),
+						xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP(sanContains), nil, false),
+					},
 				},
 			},
 		},
@@ -596,8 +791,11 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			update, err := validateCluster(test.cluster)
-			if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("validateCluster(%+v) = (%+v, %v), want: (%+v, %v)", test.cluster, update, err, test.wantUpdate, test.wantErr)
+			if (err != nil) != test.wantErr {
+				t.Errorf("validateCluster() returned err %v wantErr %v)", err, test.wantErr)
+			}
+			if diff := cmp.Diff(test.wantUpdate, update, cmpopts.EquateEmpty(), cmp.AllowUnexported(regexp.Regexp{})); diff != "" {
+				t.Errorf("validateCluster() returned unexpected diff (-want, +got):\n%s", diff)
 			}
 		})
 	}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 7e8ffe7b18b3..5c0f38a9782f 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -33,6 +33,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 
@@ -385,11 +386,15 @@ type SecurityConfig struct {
 	// IdentityCertName is the certificate name to be passed to the plugin
 	// (looked up from the bootstrap file) while fetching identity certificates.
 	IdentityCertName string
-	// AcceptedSANs is a list of Subject Alternative Names. During the TLS
-	// handshake, the SAN present in the peer certificate is compared against
-	// this list, and the handshake succeeds only if a match is found. Used only
-	// on the client-side.
-	AcceptedSANs []string
+	// SubjectAltNameMatchers is an optional list of match criteria for SANs
+	// specified on the peer certificate. Used only on the client-side.
+	//
+	// Some intricacies:
+	// - If this field is empty, then any peer certificate is accepted.
+	// - If the peer certificate contains a wildcard DNS SAN, and an `exact`
+	//   matcher is configured, a wildcard DNS match is performed instead of a
+	//   regular string comparison.
+	SubjectAltNameMatchers []xds.StringMatcher
 	// RequireClientCert indicates if the server handshake process expects the
 	// client to present a certificate. Set to true when performing mTLS. Used
 	// only on the server-side.
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 8f604d32b8f3..13b4e7f76d49 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -40,6 +40,7 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
@@ -697,21 +698,24 @@ func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext) (*Secu
 	// those possible values:
 	//  - combined validation context:
 	//    - contains a default validation context which holds the list of
-	//      accepted SANs.
+	//      matchers for accepted SANs.
 	//    - contains certificate provider instance configuration
 	//  - certificate provider instance configuration
 	//    - in this case, we do not get a list of accepted SANs.
 	switch t := common.GetValidationContextType().(type) {
 	case *v3tlspb.CommonTlsContext_CombinedValidationContext:
 		combined := common.GetCombinedValidationContext()
+		var matchers []xds.StringMatcher
 		if def := combined.GetDefaultValidationContext(); def != nil {
-			for _, matcher := range def.GetMatchSubjectAltNames() {
-				// We only support exact matches for now.
-				if exact := matcher.GetExact(); exact != "" {
-					sc.AcceptedSANs = append(sc.AcceptedSANs, exact)
+			for _, m := range def.GetMatchSubjectAltNames() {
+				matcher, err := xds.StringMatcherFromProto(m)
+				if err != nil {
+					return nil, err
 				}
+				matchers = append(matchers, matcher)
 			}
 		}
+		sc.SubjectAltNameMatchers = matchers
 		if pi := combined.GetValidationContextCertificateProviderInstance(); pi != nil {
 			sc.RootInstanceName = pi.GetInstanceName()
 			sc.RootCertName = pi.GetCertificateName()

From 95173a53fe5443098a11acca95a71eae006ecaa9 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 16 Mar 2021 14:05:05 -0700
Subject: [PATCH 389/481] csds: implement CSDS service handler (#4243)

---
 xds/csds/csds.go                     | 313 ++++++++++++
 xds/csds/csds_test.go                | 683 +++++++++++++++++++++++++++
 xds/internal/testutils/e2e/server.go |  10 +-
 3 files changed, 1004 insertions(+), 2 deletions(-)
 create mode 100644 xds/csds/csds.go
 create mode 100644 xds/csds/csds_test.go

diff --git a/xds/csds/csds.go b/xds/csds/csds.go
new file mode 100644
index 000000000000..e803a2d2cec1
--- /dev/null
+++ b/xds/csds/csds.go
@@ -0,0 +1,313 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package csds implements features to dump the status (xDS responses) the
+// xds_client is using.
+//
+// Notice: This package is EXPERIMENTAL and may be changed or removed in a later
+// release.
+package csds
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/client/bootstrap"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	_ "google.golang.org/grpc/xds/internal/client/v2" // Register v2 xds_client.
+	_ "google.golang.org/grpc/xds/internal/client/v3" // Register v3 xds_client.
+)
+
+// xdsClientInterface contains methods from xdsClient.Client which are used by
+// the server. This is useful for overriding in unit tests.
+type xdsClientInterface interface {
+	DumpLDS() (string, map[string]client.UpdateWithMD)
+	DumpRDS() (string, map[string]client.UpdateWithMD)
+	DumpCDS() (string, map[string]client.UpdateWithMD)
+	DumpEDS() (string, map[string]client.UpdateWithMD)
+	BootstrapConfig() *bootstrap.Config
+	Close()
+}
+
+var (
+	logger       = grpclog.Component("xds")
+	newXDSClient = func() (xdsClientInterface, error) {
+		return client.New()
+	}
+)
+
+// ClientStatusDiscoveryServer implementations interface ClientStatusDiscoveryServiceServer.
+type ClientStatusDiscoveryServer struct {
+	// xdsClient will always be the same in practise. But we keep a copy in each
+	// server instance for testing.
+	xdsClient xdsClientInterface
+}
+
+// NewClientStatusDiscoveryServer returns an implementation of the CSDS server that can be
+// registered on a gRPC server.
+func NewClientStatusDiscoveryServer() (*ClientStatusDiscoveryServer, error) {
+	xdsC, err := newXDSClient()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create xds client: %v", err)
+	}
+	return &ClientStatusDiscoveryServer{
+		xdsClient: xdsC,
+	}, nil
+}
+
+// StreamClientStatus implementations interface ClientStatusDiscoveryServiceServer.
+func (s *ClientStatusDiscoveryServer) StreamClientStatus(stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusServer) error {
+	for {
+		req, err := stream.Recv()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		resp, err := s.buildClientStatusRespForReq(req)
+		if err != nil {
+			return err
+		}
+		if err := stream.Send(resp); err != nil {
+			return err
+		}
+	}
+}
+
+// FetchClientStatus implementations interface ClientStatusDiscoveryServiceServer.
+func (s *ClientStatusDiscoveryServer) FetchClientStatus(_ context.Context, req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) {
+	return s.buildClientStatusRespForReq(req)
+}
+
+// buildClientStatusRespForReq fetches the status from the client, and returns
+// the response to be sent back to client.
+//
+// If it returns an error, the error is a status error.
+func (s *ClientStatusDiscoveryServer) buildClientStatusRespForReq(req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) {
+	// Field NodeMatchers is unsupported, by design
+	// https://github.com/grpc/proposal/blob/master/A40-csds-support.md#detail-node-matching.
+	if len(req.NodeMatchers) != 0 {
+		return nil, status.Errorf(codes.InvalidArgument, "node_matchers are not supported, request contains node_matchers: %v", req.NodeMatchers)
+	}
+
+	ret := &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node: nodeProtoToV3(s.xdsClient.BootstrapConfig().NodeProto),
+				XdsConfig: []*v3statuspb.PerXdsConfig{
+					s.buildLDSPerXDSConfig(),
+					s.buildRDSPerXDSConfig(),
+					s.buildCDSPerXDSConfig(),
+					s.buildEDSPerXDSConfig(),
+				},
+			},
+		},
+	}
+	return ret, nil
+}
+
+// Close cleans up the resources.
+func (s *ClientStatusDiscoveryServer) Close() {
+	s.xdsClient.Close()
+}
+
+// nodeProtoToV3 converts the given proto into a v3.Node. n is from bootstrap
+// config, it can be either v2.Node or v3.Node.
+//
+// If n is already a v3.Node, return it.
+// If n is v2.Node, marshal and unmarshal it to v3.
+// Otherwise, return nil.
+//
+// The default case (not v2 or v3) is nil, instead of error, because the
+// resources in the response are more important than the node. The worst case is
+// that the user will receive no Node info, but will still get resources.
+func nodeProtoToV3(n proto.Message) *v3corepb.Node {
+	var node *v3corepb.Node
+	switch nn := n.(type) {
+	case *v3corepb.Node:
+		node = nn
+	case *v2corepb.Node:
+		v2, err := proto.Marshal(nn)
+		if err != nil {
+			logger.Warningf("Failed to marshal node (%v): %v", n, err)
+			break
+		}
+		node = new(v3corepb.Node)
+		if err := proto.Unmarshal(v2, node); err != nil {
+			logger.Warningf("Failed to unmarshal node (%v): %v", v2, err)
+		}
+	default:
+		logger.Warningf("node from bootstrap is %#v, only v2.Node and v3.Node are supported", nn)
+	}
+	return node
+}
+
+func (s *ClientStatusDiscoveryServer) buildLDSPerXDSConfig() *v3statuspb.PerXdsConfig {
+	version, dump := s.xdsClient.DumpLDS()
+	var resources []*v3adminpb.ListenersConfigDump_DynamicListener
+	for name, d := range dump {
+		configDump := &v3adminpb.ListenersConfigDump_DynamicListener{
+			Name:         name,
+			ClientStatus: serviceStatusToProto(d.MD.Status),
+		}
+		if (d.MD.Timestamp != time.Time{}) {
+			configDump.ActiveState = &v3adminpb.ListenersConfigDump_DynamicListenerState{
+				VersionInfo: d.MD.Version,
+				Listener:    d.Raw,
+				LastUpdated: timestamppb.New(d.MD.Timestamp),
+			}
+		}
+		if errState := d.MD.ErrState; errState != nil {
+			configDump.ErrorState = &v3adminpb.UpdateFailureState{
+				LastUpdateAttempt: timestamppb.New(errState.Timestamp),
+				Details:           errState.Err.Error(),
+				VersionInfo:       errState.Version,
+			}
+		}
+		resources = append(resources, configDump)
+	}
+	return &v3statuspb.PerXdsConfig{
+		PerXdsConfig: &v3statuspb.PerXdsConfig_ListenerConfig{
+			ListenerConfig: &v3adminpb.ListenersConfigDump{
+				VersionInfo:      version,
+				DynamicListeners: resources,
+			},
+		},
+	}
+}
+
+func (s *ClientStatusDiscoveryServer) buildRDSPerXDSConfig() *v3statuspb.PerXdsConfig {
+	_, dump := s.xdsClient.DumpRDS()
+	var resources []*v3adminpb.RoutesConfigDump_DynamicRouteConfig
+	for _, d := range dump {
+		configDump := &v3adminpb.RoutesConfigDump_DynamicRouteConfig{
+			VersionInfo:  d.MD.Version,
+			ClientStatus: serviceStatusToProto(d.MD.Status),
+		}
+		if (d.MD.Timestamp != time.Time{}) {
+			configDump.RouteConfig = d.Raw
+			configDump.LastUpdated = timestamppb.New(d.MD.Timestamp)
+		}
+		if errState := d.MD.ErrState; errState != nil {
+			configDump.ErrorState = &v3adminpb.UpdateFailureState{
+				LastUpdateAttempt: timestamppb.New(errState.Timestamp),
+				Details:           errState.Err.Error(),
+				VersionInfo:       errState.Version,
+			}
+		}
+		resources = append(resources, configDump)
+	}
+	return &v3statuspb.PerXdsConfig{
+		PerXdsConfig: &v3statuspb.PerXdsConfig_RouteConfig{
+			RouteConfig: &v3adminpb.RoutesConfigDump{
+				DynamicRouteConfigs: resources,
+			},
+		},
+	}
+}
+
+func (s *ClientStatusDiscoveryServer) buildCDSPerXDSConfig() *v3statuspb.PerXdsConfig {
+	version, dump := s.xdsClient.DumpCDS()
+	var resources []*v3adminpb.ClustersConfigDump_DynamicCluster
+	for _, d := range dump {
+		configDump := &v3adminpb.ClustersConfigDump_DynamicCluster{
+			VersionInfo:  d.MD.Version,
+			ClientStatus: serviceStatusToProto(d.MD.Status),
+		}
+		if (d.MD.Timestamp != time.Time{}) {
+			configDump.Cluster = d.Raw
+			configDump.LastUpdated = timestamppb.New(d.MD.Timestamp)
+		}
+		if errState := d.MD.ErrState; errState != nil {
+			configDump.ErrorState = &v3adminpb.UpdateFailureState{
+				LastUpdateAttempt: timestamppb.New(errState.Timestamp),
+				Details:           errState.Err.Error(),
+				VersionInfo:       errState.Version,
+			}
+		}
+		resources = append(resources, configDump)
+	}
+	return &v3statuspb.PerXdsConfig{
+		PerXdsConfig: &v3statuspb.PerXdsConfig_ClusterConfig{
+			ClusterConfig: &v3adminpb.ClustersConfigDump{
+				VersionInfo:           version,
+				DynamicActiveClusters: resources,
+			},
+		},
+	}
+}
+
+func (s *ClientStatusDiscoveryServer) buildEDSPerXDSConfig() *v3statuspb.PerXdsConfig {
+	_, dump := s.xdsClient.DumpEDS()
+	var resources []*v3adminpb.EndpointsConfigDump_DynamicEndpointConfig
+	for _, d := range dump {
+		configDump := &v3adminpb.EndpointsConfigDump_DynamicEndpointConfig{
+			VersionInfo:  d.MD.Version,
+			ClientStatus: serviceStatusToProto(d.MD.Status),
+		}
+		if (d.MD.Timestamp != time.Time{}) {
+			configDump.EndpointConfig = d.Raw
+			configDump.LastUpdated = timestamppb.New(d.MD.Timestamp)
+		}
+		if errState := d.MD.ErrState; errState != nil {
+			configDump.ErrorState = &v3adminpb.UpdateFailureState{
+				LastUpdateAttempt: timestamppb.New(errState.Timestamp),
+				Details:           errState.Err.Error(),
+				VersionInfo:       errState.Version,
+			}
+		}
+		resources = append(resources, configDump)
+	}
+	return &v3statuspb.PerXdsConfig{
+		PerXdsConfig: &v3statuspb.PerXdsConfig_EndpointConfig{
+			EndpointConfig: &v3adminpb.EndpointsConfigDump{
+				DynamicEndpointConfigs: resources,
+			},
+		},
+	}
+}
+
+func serviceStatusToProto(serviceStatus client.ServiceStatus) v3adminpb.ClientResourceStatus {
+	switch serviceStatus {
+	case client.ServiceStatusUnknown:
+		return v3adminpb.ClientResourceStatus_UNKNOWN
+	case client.ServiceStatusRequested:
+		return v3adminpb.ClientResourceStatus_REQUESTED
+	case client.ServiceStatusNotExist:
+		return v3adminpb.ClientResourceStatus_DOES_NOT_EXIST
+	case client.ServiceStatusACKed:
+		return v3adminpb.ClientResourceStatus_ACKED
+	case client.ServiceStatusNACKed:
+		return v3adminpb.ClientResourceStatus_NACKED
+	default:
+		return v3adminpb.ClientResourceStatus_UNKNOWN
+	}
+}
diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
new file mode 100644
index 000000000000..db919e138fbd
--- /dev/null
+++ b/xds/csds/csds_test.go
@@ -0,0 +1,683 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csds
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+)
+
+const (
+	defaultTestTimeout = 10 * time.Second
+)
+
+type xdsClientInterfaceWithWatch interface {
+	WatchListener(string, func(client.ListenerUpdate, error)) func()
+	WatchRouteConfig(string, func(client.RouteConfigUpdate, error)) func()
+	WatchCluster(string, func(client.ClusterUpdate, error)) func()
+	WatchEndpoints(string, func(client.EndpointsUpdate, error)) func()
+}
+
+var cmpOpts = cmp.Options{
+	cmpopts.EquateEmpty(),
+	cmp.Comparer(func(a, b *timestamppb.Timestamp) bool { return true }),
+	protocmp.IgnoreFields(&v3adminpb.UpdateFailureState{}, "last_update_attempt", "details"),
+	protocmp.SortRepeated(func(a, b *v3adminpb.ListenersConfigDump_DynamicListener) bool {
+		return strings.Compare(a.Name, b.Name) < 0
+	}),
+	protocmp.SortRepeated(func(a, b *v3adminpb.RoutesConfigDump_DynamicRouteConfig) bool {
+		if a.RouteConfig == nil {
+			return false
+		}
+		if b.RouteConfig == nil {
+			return true
+		}
+		var at, bt v3routepb.RouteConfiguration
+		if err := ptypes.UnmarshalAny(a.RouteConfig, &at); err != nil {
+			panic("failed to unmarshal RouteConfig" + err.Error())
+		}
+		if err := ptypes.UnmarshalAny(b.RouteConfig, &bt); err != nil {
+			panic("failed to unmarshal RouteConfig" + err.Error())
+		}
+		return strings.Compare(at.Name, bt.Name) < 0
+	}),
+	protocmp.SortRepeated(func(a, b *v3adminpb.ClustersConfigDump_DynamicCluster) bool {
+		if a.Cluster == nil {
+			return false
+		}
+		if b.Cluster == nil {
+			return true
+		}
+		var at, bt v3clusterpb.Cluster
+		if err := ptypes.UnmarshalAny(a.Cluster, &at); err != nil {
+			panic("failed to unmarshal Cluster" + err.Error())
+		}
+		if err := ptypes.UnmarshalAny(b.Cluster, &bt); err != nil {
+			panic("failed to unmarshal Cluster" + err.Error())
+		}
+		return strings.Compare(at.Name, bt.Name) < 0
+	}),
+	protocmp.SortRepeated(func(a, b *v3adminpb.EndpointsConfigDump_DynamicEndpointConfig) bool {
+		if a.EndpointConfig == nil {
+			return false
+		}
+		if b.EndpointConfig == nil {
+			return true
+		}
+		var at, bt v3endpointpb.ClusterLoadAssignment
+		if err := ptypes.UnmarshalAny(a.EndpointConfig, &at); err != nil {
+			panic("failed to unmarshal Endpoints" + err.Error())
+		}
+		if err := ptypes.UnmarshalAny(b.EndpointConfig, &bt); err != nil {
+			panic("failed to unmarshal Endpoints" + err.Error())
+		}
+		return strings.Compare(at.ClusterName, bt.ClusterName) < 0
+	}),
+	protocmp.IgnoreFields(&v3adminpb.ListenersConfigDump_DynamicListenerState{}, "last_updated"),
+	protocmp.IgnoreFields(&v3adminpb.RoutesConfigDump_DynamicRouteConfig{}, "last_updated"),
+	protocmp.IgnoreFields(&v3adminpb.ClustersConfigDump_DynamicCluster{}, "last_updated"),
+	protocmp.IgnoreFields(&v3adminpb.EndpointsConfigDump_DynamicEndpointConfig{}, "last_updated"),
+	protocmp.Transform(),
+}
+
+var (
+	ldsTargets   = []string{"lds.target.good:0000", "lds.target.good:1111"}
+	listeners    = make([]*v3listenerpb.Listener, len(ldsTargets))
+	listenerAnys = make([]*anypb.Any, len(ldsTargets))
+
+	rdsTargets = []string{"route-config-0", "route-config-1"}
+	routes     = make([]*v3routepb.RouteConfiguration, len(rdsTargets))
+	routeAnys  = make([]*anypb.Any, len(rdsTargets))
+
+	cdsTargets  = []string{"cluster-0", "cluster-1"}
+	clusters    = make([]*v3clusterpb.Cluster, len(cdsTargets))
+	clusterAnys = make([]*anypb.Any, len(cdsTargets))
+
+	edsTargets   = []string{"endpoints-0", "endpoints-1"}
+	endpoints    = make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets))
+	endpointAnys = make([]*anypb.Any, len(edsTargets))
+	ips          = []string{"0.0.0.0", "1.1.1.1"}
+	ports        = []uint32{123, 456}
+)
+
+func init() {
+	for i := range ldsTargets {
+		listeners[i] = e2e.DefaultListener(ldsTargets[i], rdsTargets[i])
+		listenerAnys[i], _ = ptypes.MarshalAny(listeners[i])
+	}
+	for i := range rdsTargets {
+		routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i])
+		routeAnys[i], _ = ptypes.MarshalAny(routes[i])
+	}
+	for i := range cdsTargets {
+		clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i])
+		clusterAnys[i], _ = ptypes.MarshalAny(clusters[i])
+	}
+	for i := range edsTargets {
+		endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i])
+		endpointAnys[i], _ = ptypes.MarshalAny(endpoints[i])
+	}
+}
+
+func TestCSDS(t *testing.T) {
+	const retryCount = 10
+
+	xdsC, mgmServer, nodeID, stream, cleanup := commonSetup(t)
+	defer cleanup()
+
+	for _, target := range ldsTargets {
+		xdsC.WatchListener(target, func(client.ListenerUpdate, error) {})
+	}
+	for _, target := range rdsTargets {
+		xdsC.WatchRouteConfig(target, func(client.RouteConfigUpdate, error) {})
+	}
+	for _, target := range cdsTargets {
+		xdsC.WatchCluster(target, func(client.ClusterUpdate, error) {})
+	}
+	for _, target := range edsTargets {
+		xdsC.WatchEndpoints(target, func(client.EndpointsUpdate, error) {})
+	}
+
+	for i := 0; i < retryCount; i++ {
+		err := checkForRequested(stream)
+		if err == nil {
+			break
+		}
+		if i == retryCount-1 {
+			t.Fatalf("%v", err)
+		}
+		time.Sleep(time.Millisecond * 100)
+	}
+
+	if err := mgmServer.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < retryCount; i++ {
+		err := checkForACKed(stream)
+		if err == nil {
+			break
+		}
+		if i == retryCount-1 {
+			t.Fatalf("%v", err)
+		}
+		time.Sleep(time.Millisecond * 100)
+	}
+
+	const nackResourceIdx = 0
+	if err := mgmServer.Update(e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			{Name: ldsTargets[nackResourceIdx], ApiListener: &v3listenerpb.ApiListener{}}, // 0 will be nacked. 1 will stay the same.
+		},
+		Routes: []*v3routepb.RouteConfiguration{
+			{Name: rdsTargets[nackResourceIdx], VirtualHosts: []*v3routepb.VirtualHost{{
+				Routes: []*v3routepb.Route{{}},
+			}}},
+		},
+		Clusters: []*v3clusterpb.Cluster{
+			{Name: cdsTargets[nackResourceIdx], ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}},
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
+			{ClusterName: edsTargets[nackResourceIdx], Endpoints: []*v3endpointpb.LocalityLbEndpoints{{}}},
+		},
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < retryCount; i++ {
+		err := checkForNACKed(nackResourceIdx, stream)
+		if err == nil {
+			break
+		}
+		if i == retryCount-1 {
+			t.Fatalf("%v", err)
+		}
+		time.Sleep(time.Millisecond * 100)
+	}
+}
+
+func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServer, string, v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient, func()) {
+	t.Helper()
+
+	// Spin up a xDS management server on a local port.
+	nodeID := uuid.New().String()
+	fs, err := e2e.StartManagementServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a bootstrap file in a temporary directory.
+	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
+		Version:   e2e.TransportV3,
+		NodeID:    nodeID,
+		ServerURI: fs.Address,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create xds_client.
+	xdsC, err := client.New()
+	if err != nil {
+		t.Fatalf("failed to create xds client: %v", err)
+	}
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) {
+		return xdsC, nil
+	}
+
+	// Initialize an gRPC server and register CSDS on it.
+	server := grpc.NewServer()
+	csdss, err := NewClientStatusDiscoveryServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v3statuspb.RegisterClientStatusDiscoveryServiceServer(server, csdss)
+	// Create a local listener and pass it to Serve().
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	go func() {
+		if err := server.Serve(lis); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+
+	// Create CSDS client.
+	conn, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("cannot connect to server: %v", err)
+	}
+	c := v3statuspb.NewClientStatusDiscoveryServiceClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	stream, err := c.StreamClientStatus(ctx, grpc.WaitForReady(true))
+	if err != nil {
+		t.Fatalf("cannot get ServerReflectionInfo: %v", err)
+	}
+
+	return xdsC, fs, nodeID, stream, func() {
+		fs.Stop()
+		cancel()
+		conn.Close()
+		server.Stop()
+		csdss.Close()
+		newXDSClient = oldNewXDSClient
+		xdsC.Close()
+		bootstrapCleanup()
+	}
+}
+
+func checkForRequested(stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient) error {
+	if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil {
+		return fmt.Errorf("failed to send request: %v", err)
+	}
+	r, err := stream.Recv()
+	if err != nil {
+		// io.EOF is not ok.
+		return fmt.Errorf("failed to recv response: %v", err)
+	}
+
+	if n := len(r.Config); n != 1 {
+		return fmt.Errorf("got %d configs, want 1: %v", n, proto.MarshalTextString(r))
+	}
+	if n := len(r.Config[0].XdsConfig); n != 4 {
+		return fmt.Errorf("got %d xds configs (one for each type), want 4: %v", n, proto.MarshalTextString(r))
+	}
+	for _, cfg := range r.Config[0].XdsConfig {
+		switch config := cfg.PerXdsConfig.(type) {
+		case *v3statuspb.PerXdsConfig_ListenerConfig:
+			var wantLis []*v3adminpb.ListenersConfigDump_DynamicListener
+			for i := range ldsTargets {
+				wantLis = append(wantLis, &v3adminpb.ListenersConfigDump_DynamicListener{
+					Name:         ldsTargets[i],
+					ClientStatus: v3adminpb.ClientResourceStatus_REQUESTED,
+				})
+			}
+			wantDump := &v3adminpb.ListenersConfigDump{
+				DynamicListeners: wantLis,
+			}
+			if diff := cmp.Diff(config.ListenerConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_RouteConfig:
+			var wantRoutes []*v3adminpb.RoutesConfigDump_DynamicRouteConfig
+			for range rdsTargets {
+				wantRoutes = append(wantRoutes, &v3adminpb.RoutesConfigDump_DynamicRouteConfig{
+					ClientStatus: v3adminpb.ClientResourceStatus_REQUESTED,
+				})
+			}
+			wantDump := &v3adminpb.RoutesConfigDump{
+				DynamicRouteConfigs: wantRoutes,
+			}
+			if diff := cmp.Diff(config.RouteConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_ClusterConfig:
+			var wantCluster []*v3adminpb.ClustersConfigDump_DynamicCluster
+			for range cdsTargets {
+				wantCluster = append(wantCluster, &v3adminpb.ClustersConfigDump_DynamicCluster{
+					ClientStatus: v3adminpb.ClientResourceStatus_REQUESTED,
+				})
+			}
+			wantDump := &v3adminpb.ClustersConfigDump{
+				DynamicActiveClusters: wantCluster,
+			}
+			if diff := cmp.Diff(config.ClusterConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_EndpointConfig:
+			var wantEndpoint []*v3adminpb.EndpointsConfigDump_DynamicEndpointConfig
+			for range cdsTargets {
+				wantEndpoint = append(wantEndpoint, &v3adminpb.EndpointsConfigDump_DynamicEndpointConfig{
+					ClientStatus: v3adminpb.ClientResourceStatus_REQUESTED,
+				})
+			}
+			wantDump := &v3adminpb.EndpointsConfigDump{
+				DynamicEndpointConfigs: wantEndpoint,
+			}
+			if diff := cmp.Diff(config.EndpointConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		default:
+			return fmt.Errorf("unexpected PerXdsConfig: %+v; %v", cfg.PerXdsConfig, protoToJSON(r))
+		}
+	}
+	return nil
+}
+
+func checkForACKed(stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient) error {
+	const wantVersion = "1"
+
+	if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil {
+		return fmt.Errorf("failed to send: %v", err)
+	}
+	r, err := stream.Recv()
+	if err != nil {
+		// io.EOF is not ok.
+		return fmt.Errorf("failed to recv response: %v", err)
+	}
+
+	if n := len(r.Config); n != 1 {
+		return fmt.Errorf("got %d configs, want 1: %v", n, proto.MarshalTextString(r))
+	}
+	if n := len(r.Config[0].XdsConfig); n != 4 {
+		return fmt.Errorf("got %d xds configs (one for each type), want 4: %v", n, proto.MarshalTextString(r))
+	}
+	for _, cfg := range r.Config[0].XdsConfig {
+		switch config := cfg.PerXdsConfig.(type) {
+		case *v3statuspb.PerXdsConfig_ListenerConfig:
+			var wantLis []*v3adminpb.ListenersConfigDump_DynamicListener
+			for i := range ldsTargets {
+				wantLis = append(wantLis, &v3adminpb.ListenersConfigDump_DynamicListener{
+					Name: ldsTargets[i],
+					ActiveState: &v3adminpb.ListenersConfigDump_DynamicListenerState{
+						VersionInfo: wantVersion,
+						Listener:    listenerAnys[i],
+						LastUpdated: nil,
+					},
+					ErrorState:   nil,
+					ClientStatus: v3adminpb.ClientResourceStatus_ACKED,
+				})
+			}
+			wantDump := &v3adminpb.ListenersConfigDump{
+				VersionInfo:      wantVersion,
+				DynamicListeners: wantLis,
+			}
+			if diff := cmp.Diff(config.ListenerConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_RouteConfig:
+			var wantRoutes []*v3adminpb.RoutesConfigDump_DynamicRouteConfig
+			for i := range rdsTargets {
+				wantRoutes = append(wantRoutes, &v3adminpb.RoutesConfigDump_DynamicRouteConfig{
+					VersionInfo:  wantVersion,
+					RouteConfig:  routeAnys[i],
+					LastUpdated:  nil,
+					ClientStatus: v3adminpb.ClientResourceStatus_ACKED,
+				})
+			}
+			wantDump := &v3adminpb.RoutesConfigDump{
+				DynamicRouteConfigs: wantRoutes,
+			}
+			if diff := cmp.Diff(config.RouteConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_ClusterConfig:
+			var wantCluster []*v3adminpb.ClustersConfigDump_DynamicCluster
+			for i := range cdsTargets {
+				wantCluster = append(wantCluster, &v3adminpb.ClustersConfigDump_DynamicCluster{
+					VersionInfo:  wantVersion,
+					Cluster:      clusterAnys[i],
+					LastUpdated:  nil,
+					ClientStatus: v3adminpb.ClientResourceStatus_ACKED,
+				})
+			}
+			wantDump := &v3adminpb.ClustersConfigDump{
+				VersionInfo:           wantVersion,
+				DynamicActiveClusters: wantCluster,
+			}
+			if diff := cmp.Diff(config.ClusterConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_EndpointConfig:
+			var wantEndpoint []*v3adminpb.EndpointsConfigDump_DynamicEndpointConfig
+			for i := range cdsTargets {
+				wantEndpoint = append(wantEndpoint, &v3adminpb.EndpointsConfigDump_DynamicEndpointConfig{
+					VersionInfo:    wantVersion,
+					EndpointConfig: endpointAnys[i],
+					LastUpdated:    nil,
+					ClientStatus:   v3adminpb.ClientResourceStatus_ACKED,
+				})
+			}
+			wantDump := &v3adminpb.EndpointsConfigDump{
+				DynamicEndpointConfigs: wantEndpoint,
+			}
+			if diff := cmp.Diff(config.EndpointConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		default:
+			return fmt.Errorf("unexpected PerXdsConfig: %+v; %v", cfg.PerXdsConfig, protoToJSON(r))
+		}
+	}
+	return nil
+}
+
+func checkForNACKed(nackResourceIdx int, stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient) error {
+	const (
+		ackVersion  = "1"
+		nackVersion = "2"
+	)
+
+	if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil {
+		return fmt.Errorf("failed to send: %v", err)
+	}
+	r, err := stream.Recv()
+	if err != nil {
+		// io.EOF is not ok.
+		return fmt.Errorf("failed to recv response: %v", err)
+	}
+
+	if n := len(r.Config); n != 1 {
+		return fmt.Errorf("got %d configs, want 1: %v", n, proto.MarshalTextString(r))
+	}
+	if n := len(r.Config[0].XdsConfig); n != 4 {
+		return fmt.Errorf("got %d xds configs (one for each type), want 4: %v", n, proto.MarshalTextString(r))
+	}
+	for _, cfg := range r.Config[0].XdsConfig {
+		switch config := cfg.PerXdsConfig.(type) {
+		case *v3statuspb.PerXdsConfig_ListenerConfig:
+			var wantLis []*v3adminpb.ListenersConfigDump_DynamicListener
+			for i := range ldsTargets {
+				configDump := &v3adminpb.ListenersConfigDump_DynamicListener{
+					Name: ldsTargets[i],
+					ActiveState: &v3adminpb.ListenersConfigDump_DynamicListenerState{
+						VersionInfo: ackVersion,
+						Listener:    listenerAnys[i],
+						LastUpdated: nil,
+					},
+					ClientStatus: v3adminpb.ClientResourceStatus_ACKED,
+				}
+				if i == nackResourceIdx {
+					configDump.ClientStatus = v3adminpb.ClientResourceStatus_NACKED
+					configDump.ErrorState = &v3adminpb.UpdateFailureState{
+						Details:     "blahblah",
+						VersionInfo: nackVersion,
+					}
+				}
+				wantLis = append(wantLis, configDump)
+			}
+			wantDump := &v3adminpb.ListenersConfigDump{
+				VersionInfo:      nackVersion,
+				DynamicListeners: wantLis,
+			}
+			if diff := cmp.Diff(config.ListenerConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_RouteConfig:
+			var wantRoutes []*v3adminpb.RoutesConfigDump_DynamicRouteConfig
+			for i := range rdsTargets {
+				configDump := &v3adminpb.RoutesConfigDump_DynamicRouteConfig{
+					VersionInfo:  ackVersion,
+					RouteConfig:  routeAnys[i],
+					LastUpdated:  nil,
+					ClientStatus: v3adminpb.ClientResourceStatus_ACKED,
+				}
+				if i == nackResourceIdx {
+					configDump.ClientStatus = v3adminpb.ClientResourceStatus_NACKED
+					configDump.ErrorState = &v3adminpb.UpdateFailureState{
+						Details:     "blahblah",
+						VersionInfo: nackVersion,
+					}
+				}
+				wantRoutes = append(wantRoutes, configDump)
+			}
+			wantDump := &v3adminpb.RoutesConfigDump{
+				DynamicRouteConfigs: wantRoutes,
+			}
+			if diff := cmp.Diff(config.RouteConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_ClusterConfig:
+			var wantCluster []*v3adminpb.ClustersConfigDump_DynamicCluster
+			for i := range cdsTargets {
+				configDump := &v3adminpb.ClustersConfigDump_DynamicCluster{
+					VersionInfo:  ackVersion,
+					Cluster:      clusterAnys[i],
+					LastUpdated:  nil,
+					ClientStatus: v3adminpb.ClientResourceStatus_ACKED,
+				}
+				if i == nackResourceIdx {
+					configDump.ClientStatus = v3adminpb.ClientResourceStatus_NACKED
+					configDump.ErrorState = &v3adminpb.UpdateFailureState{
+						Details:     "blahblah",
+						VersionInfo: nackVersion,
+					}
+				}
+				wantCluster = append(wantCluster, configDump)
+			}
+			wantDump := &v3adminpb.ClustersConfigDump{
+				VersionInfo:           nackVersion,
+				DynamicActiveClusters: wantCluster,
+			}
+			if diff := cmp.Diff(config.ClusterConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		case *v3statuspb.PerXdsConfig_EndpointConfig:
+			var wantEndpoint []*v3adminpb.EndpointsConfigDump_DynamicEndpointConfig
+			for i := range cdsTargets {
+				configDump := &v3adminpb.EndpointsConfigDump_DynamicEndpointConfig{
+					VersionInfo:    ackVersion,
+					EndpointConfig: endpointAnys[i],
+					LastUpdated:    nil,
+					ClientStatus:   v3adminpb.ClientResourceStatus_ACKED,
+				}
+				if i == nackResourceIdx {
+					configDump.ClientStatus = v3adminpb.ClientResourceStatus_NACKED
+					configDump.ErrorState = &v3adminpb.UpdateFailureState{
+						Details:     "blahblah",
+						VersionInfo: nackVersion,
+					}
+				}
+				wantEndpoint = append(wantEndpoint, configDump)
+			}
+			wantDump := &v3adminpb.EndpointsConfigDump{
+				DynamicEndpointConfigs: wantEndpoint,
+			}
+			if diff := cmp.Diff(config.EndpointConfig, wantDump, cmpOpts); diff != "" {
+				return fmt.Errorf(diff)
+			}
+		default:
+			return fmt.Errorf("unexpected PerXdsConfig: %+v; %v", cfg.PerXdsConfig, protoToJSON(r))
+		}
+	}
+	return nil
+}
+
+func protoToJSON(p proto.Message) string {
+	mm := jsonpb.Marshaler{
+		Indent: "  ",
+	}
+	ret, _ := mm.MarshalToString(p)
+	return ret
+}
+
+func Test_nodeProtoToV3(t *testing.T) {
+	const (
+		testID      = "test-id"
+		testCluster = "test-cluster"
+		testZone    = "test-zone"
+	)
+	tests := []struct {
+		name string
+		n    proto.Message
+		want *v3corepb.Node
+	}{
+		{
+			name: "v3",
+			n: &v3corepb.Node{
+				Id:       testID,
+				Cluster:  testCluster,
+				Locality: &v3corepb.Locality{Zone: testZone},
+			},
+			want: &v3corepb.Node{
+				Id:       testID,
+				Cluster:  testCluster,
+				Locality: &v3corepb.Locality{Zone: testZone},
+			},
+		},
+		{
+			name: "v2",
+			n: &v2corepb.Node{
+				Id:       testID,
+				Cluster:  testCluster,
+				Locality: &v2corepb.Locality{Zone: testZone},
+			},
+			want: &v3corepb.Node{
+				Id:       testID,
+				Cluster:  testCluster,
+				Locality: &v3corepb.Locality{Zone: testZone},
+			},
+		},
+		{
+			name: "not node",
+			n:    &v2corepb.Locality{Zone: testZone},
+			want: nil, // Input is not a node, should return nil.
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := nodeProtoToV3(tt.n)
+			if diff := cmp.Diff(got, tt.want, protocmp.Transform()); diff != "" {
+				t.Errorf("nodeProtoToV3() got unexpected result, diff (-got, +want): %v", diff)
+			}
+		})
+	}
+}
diff --git a/xds/internal/testutils/e2e/server.go b/xds/internal/testutils/e2e/server.go
index cc55c595cae7..9ec2eb0d6f2e 100644
--- a/xds/internal/testutils/e2e/server.go
+++ b/xds/internal/testutils/e2e/server.go
@@ -113,6 +113,10 @@ type UpdateOptions struct {
 	Clusters  []*v3clusterpb.Cluster
 	Routes    []*v3routepb.RouteConfiguration
 	Listeners []*v3listenerpb.Listener
+	// SkipValidation indicates whether we want to skip validation (by not
+	// calling snapshot.Consistent()). It can be useful for negative tests,
+	// where we send updates that the client will NACK.
+	SkipValidation bool
 }
 
 // Update changes the resource snapshot held by the management server, which
@@ -122,8 +126,10 @@ func (s *ManagementServer) Update(opts UpdateOptions) error {
 
 	// Create a snapshot with the passed in resources.
 	snapshot := v3cache.NewSnapshot(strconv.Itoa(s.version), resourceSlice(opts.Endpoints), resourceSlice(opts.Clusters), resourceSlice(opts.Routes), resourceSlice(opts.Listeners), nil /*runtimes*/, nil /*secrets*/)
-	if err := snapshot.Consistent(); err != nil {
-		return fmt.Errorf("failed to create new resource snapshot: %v", err)
+	if !opts.SkipValidation {
+		if err := snapshot.Consistent(); err != nil {
+			return fmt.Errorf("failed to create new resource snapshot: %v", err)
+		}
 	}
 	logger.Infof("Created new resource snapshot...")
 

From 967933baf52a7bd113bfc23cbf4c5d01b8367d5b Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 16 Mar 2021 14:50:07 -0700
Subject: [PATCH 390/481] xds/cdsbalancer: move xds client close to run()
 (#4273)

Otherwise client may be used by run() after closed.
---
 xds/internal/balancer/cdsbalancer/cdsbalancer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 19e075bd7b5d..b991981c14c0 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -391,6 +391,7 @@ func (b *cdsBalancer) run() {
 				b.edsLB.Close()
 				b.edsLB = nil
 			}
+			b.xdsClient.Close()
 			// This is the *ONLY* point of return from this function.
 			b.logger.Infof("Shutdown")
 			return
@@ -493,7 +494,6 @@ func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 // Close closes the cdsBalancer and the underlying edsBalancer.
 func (b *cdsBalancer) Close() {
 	b.closed.Fire()
-	b.xdsClient.Close()
 }
 
 // ccWrapper wraps the balancer.ClientConn passed to the CDS balancer at

From bce1cded4b05db45e02a87b94b75fa5cb07a76a5 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 18 Mar 2021 16:01:39 -0700
Subject: [PATCH 391/481] internal: use strings.Replace instead
 strings.ReplaceAll (#4279)

strings.ReplaceAll is only available after go 1.12. We still support go 1.11.
---
 xds/server.go      | 2 +-
 xds/server_test.go | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/xds/server.go b/xds/server.go
index 26146aad5e1e..805f59b4f5ac 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -204,7 +204,7 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 	}
 	name := cfg.ServerListenerResourceNameTemplate
 	if strings.Contains(cfg.ServerListenerResourceNameTemplate, "%s") {
-		name = strings.ReplaceAll(cfg.ServerListenerResourceNameTemplate, "%s", lis.Addr().String())
+		name = strings.Replace(cfg.ServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)
 	}
 
 	// Create a listenerWrapper which handles all functionality required by
diff --git a/xds/server_test.go b/xds/server_test.go
index 384ef71ba722..f787a129057e 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -334,7 +334,7 @@ func (s) TestServeSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
+	wantName := strings.Replace(testServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -421,7 +421,7 @@ func (s) TestServeWithStop(t *testing.T) {
 		server.Stop()
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
+	wantName := strings.Replace(testServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)
 	if name != wantName {
 		server.Stop()
 		t.Fatalf("LDS watch registered for name %q, wantPrefix %q", name, wantName)
@@ -618,7 +618,7 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
+	wantName := strings.Replace(testServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}
@@ -696,7 +696,7 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error when waiting for a ListenerWatch: %v", err)
 	}
-	wantName := strings.ReplaceAll(testServerListenerResourceNameTemplate, "%s", lis.Addr().String())
+	wantName := strings.Replace(testServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)
 	if name != wantName {
 		t.Fatalf("LDS watch registered for name %q, want %q", name, wantName)
 	}

From f320c793495fc90c222831f1708c18119793e0f8 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Mon, 22 Mar 2021 09:42:11 -0700
Subject: [PATCH 392/481] test: enable fault_injection xds test (#4283)

---
 test/kokoro/xds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index 36a3f4563cfe..e75743dd9e57 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -27,7 +27,7 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh
 # they are added into "all".
 GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
-    --test_case="all,path_matching,header_matching,circuit_breaking,timeout" \
+    --test_case="all,path_matching,header_matching,circuit_breaking,timeout,fault_injection" \
     --project_id=grpc-testing \
     --project_num=830293263384 \
     --source_image=projects/grpc-testing/global/images/xds-test-server-4 \

From d26af8e3916597bde07641df24c3d38ca9b1f5a2 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 22 Mar 2021 15:14:11 -0700
Subject: [PATCH 393/481] admin: implement admin services (#4274)

---
 admin/admin.go                                |  57 ++++++++
 admin/admin_test.go                           |  34 +++++
 admin/test/admin_test.go                      |  38 +++++
 admin/test/utils.go                           | 113 +++++++++++++++
 channelz/service/service.go                   |   2 +-
 internal/admin/admin.go                       |  60 ++++++++
 internal/xds/bootstrap.go                     | 134 ++++++++++++++++++
 {xds/internal => internal/xds}/env/env.go     |   0
 xds/csds/csds_test.go                         |   5 +-
 xds/googledirectpath/googlec2p.go             |   2 +-
 xds/googledirectpath/googlec2p_test.go        |   2 +-
 xds/internal/balancer/edsbalancer/eds_impl.go |   2 +-
 .../balancer/edsbalancer/eds_impl_test.go     |   2 +-
 xds/internal/client/bootstrap/bootstrap.go    |   2 +-
 .../client/bootstrap/bootstrap_test.go        |   2 +-
 xds/internal/client/cds_test.go               |   2 +-
 xds/internal/client/lds_test.go               |   2 +-
 xds/internal/client/rds_test.go               |   2 +-
 xds/internal/client/xds.go                    |   2 +-
 xds/internal/httpfilter/fault/fault.go        |   2 +-
 xds/internal/httpfilter/fault/fault_test.go   |   7 +-
 xds/internal/resolver/serviceconfig.go        |   2 +-
 xds/internal/resolver/xds_resolver_test.go    |   2 +-
 .../test/xds_client_integration_test.go       |   5 +-
 .../test/xds_server_integration_test.go       |   5 +-
 xds/internal/testutils/e2e/bootstrap.go       | 107 --------------
 xds/xds.go                                    |  36 +++++
 27 files changed, 499 insertions(+), 130 deletions(-)
 create mode 100644 admin/admin.go
 create mode 100644 admin/admin_test.go
 create mode 100644 admin/test/admin_test.go
 create mode 100644 admin/test/utils.go
 create mode 100644 internal/admin/admin.go
 create mode 100644 internal/xds/bootstrap.go
 rename {xds/internal => internal/xds}/env/env.go (100%)

diff --git a/admin/admin.go b/admin/admin.go
new file mode 100644
index 000000000000..5212250b7d4e
--- /dev/null
+++ b/admin/admin.go
@@ -0,0 +1,57 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package admin provides a convenient method for registering a collection of
+// administration services to a gRPC server. The services registered are:
+//
+// - Channelz: https://github.com/grpc/proposal/blob/master/A14-channelz.md
+// - CSDS: https://github.com/grpc/proposal/blob/master/A40-csds-support.md
+//
+// Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
+package admin
+
+import (
+	"google.golang.org/grpc"
+	channelzservice "google.golang.org/grpc/channelz/service"
+	internaladmin "google.golang.org/grpc/internal/admin"
+)
+
+func init() {
+	// Add a list of default services to admin here. Optional services, like
+	// CSDS, will be added by other packages.
+	internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) {
+		channelzservice.RegisterChannelzServiceToServer(registrar)
+		return nil, nil
+	})
+}
+
+// Register registers the set of admin services to the given server.
+//
+// The returned cleanup function should be called to clean up the resources
+// allocated for the service handlers after the server is stopped.
+//
+// Note that if `s` is not a *grpc.Server or a *xds.GRPCServer, CSDS will not be
+// registered because CSDS generated code is old and doesn't support interface
+// `grpc.ServiceRegistrar`.
+// https://github.com/envoyproxy/go-control-plane/issues/403
+func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) {
+	return internaladmin.Register(s)
+}
diff --git a/admin/admin_test.go b/admin/admin_test.go
new file mode 100644
index 000000000000..0ee4aade0f3c
--- /dev/null
+++ b/admin/admin_test.go
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package admin_test
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/admin/test"
+	"google.golang.org/grpc/codes"
+)
+
+func TestRegisterNoCSDS(t *testing.T) {
+	test.RunRegisterTests(t, test.ExpectedStatusCodes{
+		ChannelzCode: codes.OK,
+		// CSDS is not registered because xDS isn't imported.
+		CSDSCode: codes.Unimplemented,
+	})
+}
diff --git a/admin/test/admin_test.go b/admin/test/admin_test.go
new file mode 100644
index 000000000000..f0f784bfdf36
--- /dev/null
+++ b/admin/test/admin_test.go
@@ -0,0 +1,38 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// This file has the same content as admin_test.go, difference is that this is
+// in another package, and it imports "xds", so we can test that csds is
+// registered when xds is imported.
+
+package test_test
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/admin/test"
+	"google.golang.org/grpc/codes"
+	_ "google.golang.org/grpc/xds"
+)
+
+func TestRegisterWithCSDS(t *testing.T) {
+	test.RunRegisterTests(t, test.ExpectedStatusCodes{
+		ChannelzCode: codes.OK,
+		CSDSCode:     codes.OK,
+	})
+}
diff --git a/admin/test/utils.go b/admin/test/utils.go
new file mode 100644
index 000000000000..069ffb3cadb7
--- /dev/null
+++ b/admin/test/utils.go
@@ -0,0 +1,113 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package test contains test only functions for package admin. It's used by
+// admin/admin_test.go and admin/test/admin_test.go.
+package test
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/admin"
+	channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	defaultTestTimeout = 10 * time.Second
+)
+
+// ExpectedStatusCodes contains the expected status code for each RPC (can be
+// OK).
+type ExpectedStatusCodes struct {
+	ChannelzCode codes.Code
+	CSDSCode     codes.Code
+}
+
+// RunRegisterTests makes a client, runs the RPCs, and compares the status
+// codes.
+func RunRegisterTests(t *testing.T, ec ExpectedStatusCodes) {
+	nodeID := uuid.New().String()
+	bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{
+		Version:   xds.TransportV3,
+		NodeID:    nodeID,
+		ServerURI: "no.need.for.a.server",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer bootstrapCleanup()
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("cannot create listener: %v", err)
+	}
+
+	server := grpc.NewServer()
+	defer server.Stop()
+	cleanup, err := admin.Register(server)
+	if err != nil {
+		t.Fatalf("failed to register admin: %v", err)
+	}
+	defer cleanup()
+	go func() {
+		server.Serve(lis)
+	}()
+
+	conn, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("cannot connect to server: %v", err)
+	}
+
+	t.Run("channelz", func(t *testing.T) {
+		if err := RunChannelz(conn); status.Code(err) != ec.ChannelzCode {
+			t.Fatalf("%s RPC failed with error %v, want code %v", "channelz", err, ec.ChannelzCode)
+		}
+	})
+	t.Run("csds", func(t *testing.T) {
+		if err := RunCSDS(conn); status.Code(err) != ec.CSDSCode {
+			t.Fatalf("%s RPC failed with error %v, want code %v", "CSDS", err, ec.CSDSCode)
+		}
+	})
+}
+
+// RunChannelz makes a channelz RPC.
+func RunChannelz(conn *grpc.ClientConn) error {
+	c := channelzpb.NewChannelzClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err := c.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{}, grpc.WaitForReady(true))
+	return err
+}
+
+// RunCSDS makes a CSDS RPC.
+func RunCSDS(conn *grpc.ClientConn) error {
+	c := v3statuspb.NewClientStatusDiscoveryServiceClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err := c.FetchClientStatus(ctx, &v3statuspb.ClientStatusRequest{}, grpc.WaitForReady(true))
+	return err
+}
diff --git a/channelz/service/service.go b/channelz/service/service.go
index 4d175fef823d..c60ab604e81b 100644
--- a/channelz/service/service.go
+++ b/channelz/service/service.go
@@ -43,7 +43,7 @@ func init() {
 var logger = grpclog.Component("channelz")
 
 // RegisterChannelzServiceToServer registers the channelz service to the given server.
-func RegisterChannelzServiceToServer(s *grpc.Server) {
+func RegisterChannelzServiceToServer(s grpc.ServiceRegistrar) {
 	channelzgrpc.RegisterChannelzServer(s, newCZServer())
 }
 
diff --git a/internal/admin/admin.go b/internal/admin/admin.go
new file mode 100644
index 000000000000..a9285ee74842
--- /dev/null
+++ b/internal/admin/admin.go
@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package admin contains internal implementation for admin service.
+package admin
+
+import "google.golang.org/grpc"
+
+// services is a map from name to service register functions.
+var services []func(grpc.ServiceRegistrar) (func(), error)
+
+// AddService adds a service to the list of admin services.
+//
+// NOTE: this function must only be called during initialization time (i.e. in
+// an init() function), and is not thread-safe.
+//
+// If multiple services with the same service name are added (e.g. two services
+// for `grpc.channelz.v1.Channelz`), the server will panic on `Register()`.
+func AddService(f func(grpc.ServiceRegistrar) (func(), error)) {
+	services = append(services, f)
+}
+
+// Register registers the set of admin services to the given server.
+func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) {
+	var cleanups []func()
+	for _, f := range services {
+		cleanup, err := f(s)
+		if err != nil {
+			callFuncs(cleanups)
+			return nil, err
+		}
+		if cleanup != nil {
+			cleanups = append(cleanups, cleanup)
+		}
+	}
+	return func() {
+		callFuncs(cleanups)
+	}, nil
+}
+
+func callFuncs(fs []func()) {
+	for _, f := range fs {
+		f()
+	}
+}
diff --git a/internal/xds/bootstrap.go b/internal/xds/bootstrap.go
new file mode 100644
index 000000000000..97ec8e17208e
--- /dev/null
+++ b/internal/xds/bootstrap.go
@@ -0,0 +1,134 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/xds/env"
+)
+
+var logger = grpclog.Component("internal/xds")
+
+// TransportAPI refers to the API version for xDS transport protocol.
+type TransportAPI int
+
+const (
+	// TransportV2 refers to the v2 xDS transport protocol.
+	TransportV2 TransportAPI = iota
+	// TransportV3 refers to the v3 xDS transport protocol.
+	TransportV3
+)
+
+// BootstrapOptions wraps the parameters passed to SetupBootstrapFile.
+type BootstrapOptions struct {
+	// Version is the xDS transport protocol version.
+	Version TransportAPI
+	// NodeID is the node identifier of the gRPC client/server node in the
+	// proxyless service mesh.
+	NodeID string
+	// ServerURI is the address of the management server.
+	ServerURI string
+	// ServerListenerResourceNameTemplate is the Listener resource name to fetch.
+	ServerListenerResourceNameTemplate string
+	// CertificateProviders is the certificate providers configuration.
+	CertificateProviders map[string]json.RawMessage
+}
+
+// SetupBootstrapFile creates a temporary file with bootstrap contents, based on
+// the passed in options, and updates the bootstrap environment variable to
+// point to this file.
+//
+// Returns a cleanup function which will be non-nil if the setup process was
+// completed successfully. It is the responsibility of the caller to invoke the
+// cleanup function at the end of the test.
+func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
+	f, err := ioutil.TempFile("", "test_xds_bootstrap_*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
+	}
+
+	cfg := &bootstrapConfig{
+		XdsServers: []server{
+			{
+				ServerURI: opts.ServerURI,
+				ChannelCreds: []creds{
+					{
+						Type: "insecure",
+					},
+				},
+			},
+		},
+		Node: node{
+			ID: opts.NodeID,
+		},
+		CertificateProviders:               opts.CertificateProviders,
+		ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate,
+	}
+	switch opts.Version {
+	case TransportV2:
+		// TODO: Add any v2 specific fields.
+	case TransportV3:
+		cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "xds_v3")
+	default:
+		return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version)
+	}
+
+	bootstrapContents, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
+	}
+	if err := ioutil.WriteFile(f.Name(), bootstrapContents, 0644); err != nil {
+		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
+	}
+	logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents)
+
+	origBootstrapFileName := env.BootstrapFileName
+	env.BootstrapFileName = f.Name()
+	return func() {
+		os.Remove(f.Name())
+		env.BootstrapFileName = origBootstrapFileName
+	}, nil
+}
+
+type bootstrapConfig struct {
+	XdsServers                         []server                   `json:"xds_servers,omitempty"`
+	Node                               node                       `json:"node,omitempty"`
+	CertificateProviders               map[string]json.RawMessage `json:"certificate_providers,omitempty"`
+	ServerListenerResourceNameTemplate string                     `json:"server_listener_resource_name_template,omitempty"`
+}
+
+type server struct {
+	ServerURI      string   `json:"server_uri,omitempty"`
+	ChannelCreds   []creds  `json:"channel_creds,omitempty"`
+	ServerFeatures []string `json:"server_features,omitempty"`
+}
+
+type creds struct {
+	Type   string      `json:"type,omitempty"`
+	Config interface{} `json:"config,omitempty"`
+}
+
+type node struct {
+	ID string `json:"id,omitempty"`
+}
diff --git a/xds/internal/env/env.go b/internal/xds/env/env.go
similarity index 100%
rename from xds/internal/env/env.go
rename to internal/xds/env/env.go
diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
index db919e138fbd..856894b4e4d4 100644
--- a/xds/csds/csds_test.go
+++ b/xds/csds/csds_test.go
@@ -37,6 +37,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/google/uuid"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
@@ -255,8 +256,8 @@ func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServ
 	}
 
 	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:   e2e.TransportV3,
+	bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{
+		Version:   xds.TransportV3,
 		NodeID:    nodeID,
 		ServerURI: fs.Address,
 	})
diff --git a/xds/googledirectpath/googlec2p.go b/xds/googledirectpath/googlec2p.go
index 791a568ecb02..4ccec4ec4120 100644
--- a/xds/googledirectpath/googlec2p.go
+++ b/xds/googledirectpath/googlec2p.go
@@ -35,11 +35,11 @@ import (
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/googlecloud"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/resolver"
 	_ "google.golang.org/grpc/xds" // To register xds resolvers and balancers.
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/structpb"
 )
diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go
index 86ea34599435..2dd31d754f3f 100644
--- a/xds/googledirectpath/googlec2p_test.go
+++ b/xds/googledirectpath/googlec2p_test.go
@@ -27,9 +27,9 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/structpb"
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 894b9a0152bc..5318a5342e83 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal"
@@ -39,7 +40,6 @@ import (
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
-	"google.golang.org/grpc/xds/internal/env"
 )
 
 // TODO: make this a environment variable?
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 3334376f4a9a..ebaea13cc88a 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -31,12 +31,12 @@ import (
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/testutils"
 )
 
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index fab93f50f4eb..f32c698b4f55 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -35,7 +35,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
-	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index bdba74099c71..501d62102d21 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -36,7 +36,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
-	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index bac0ef1aeaac..104260759b95 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -33,7 +33,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	xdsinternal "google.golang.org/grpc/internal/xds"
-	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 )
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index e2be3b4025ec..df8098df8368 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -31,7 +31,7 @@ import (
 	"github.com/golang/protobuf/ptypes"
 	spb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/durationpb"
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index 0c1e2b285388..2ca01dca9ca2 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -34,7 +34,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 
-	"google.golang.org/grpc/xds/internal/env"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/durationpb"
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 13b4e7f76d49..2791603ce26e 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -41,8 +41,8 @@ import (
 
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
 )
diff --git a/xds/internal/httpfilter/fault/fault.go b/xds/internal/httpfilter/fault/fault.go
index 98e05b51590f..639a08db8e3c 100644
--- a/xds/internal/httpfilter/fault/fault.go
+++ b/xds/internal/httpfilter/fault/fault.go
@@ -33,9 +33,9 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpcrand"
 	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/protobuf/types/known/anypb"
 
diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 1ad955146b0c..61100e8c44f8 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -37,9 +37,10 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
@@ -104,8 +105,8 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
 	}
 
 	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:                            e2e.TransportV3,
+	bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{
+		Version:                            xds.TransportV3,
 		NodeID:                             nodeID,
 		ServerURI:                          fs.Address,
 		ServerListenerResourceNameTemplate: "grpc/server",
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index 7c1ec853e4cd..ef7c37128c13 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -28,10 +28,10 @@ import (
 	"google.golang.org/grpc/codes"
 	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/wrr"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/balancer/clustermanager"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/httpfilter/router"
 )
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 53ea17042aa5..36c7416cb436 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -36,6 +36,7 @@ import (
 	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/wrr"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/status"
@@ -44,7 +45,6 @@ import (
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
-	"google.golang.org/grpc/xds/internal/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/httpfilter/router"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go
index f950a881e508..f97e42af2a0a 100644
--- a/xds/internal/test/xds_client_integration_test.go
+++ b/xds/internal/test/xds_client_integration_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/google/uuid"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 
@@ -54,8 +55,8 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
 	}
 
 	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:   e2e.TransportV3,
+	bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{
+		Version:   xds.TransportV3,
 		NodeID:    nodeID,
 		ServerURI: fs.Address,
 	})
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 3b9156347e8f..f18d4e356aa6 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -38,6 +38,7 @@ import (
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/uuid"
+	xds2 "google.golang.org/grpc/internal/xds"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -150,8 +151,8 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 	cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile))
 
 	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
-		Version:                            e2e.TransportV3,
+	bootstrapCleanup, err := xds2.SetupBootstrapFile(xds2.BootstrapOptions{
+		Version:                            xds2.TransportV3,
 		NodeID:                             nodeID,
 		ServerURI:                          fs.Address,
 		CertificateProviders:               cpc,
diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go
index c6fd2f9291fe..72a1a9900cfe 100644
--- a/xds/internal/testutils/e2e/bootstrap.go
+++ b/xds/internal/testutils/e2e/bootstrap.go
@@ -21,93 +21,8 @@ package e2e
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"os"
-
-	"google.golang.org/grpc/xds/internal/env"
-)
-
-// TransportAPI refers to the API version for xDS transport protocol.
-type TransportAPI int
-
-const (
-	// TransportV2 refers to the v2 xDS transport protocol.
-	TransportV2 TransportAPI = iota
-	// TransportV3 refers to the v3 xDS transport protocol.
-	TransportV3
 )
 
-// BootstrapOptions wraps the parameters passed to SetupBootstrapFile.
-type BootstrapOptions struct {
-	// Version is the xDS transport protocol version.
-	Version TransportAPI
-	// NodeID is the node identifier of the gRPC client/server node in the
-	// proxyless service mesh.
-	NodeID string
-	// ServerURI is the address of the management server.
-	ServerURI string
-	// ServerListenerResourceNameTemplate is the Listener resource name to fetch.
-	ServerListenerResourceNameTemplate string
-	// CertificateProviders is the certificate providers configuration.
-	CertificateProviders map[string]json.RawMessage
-}
-
-// SetupBootstrapFile creates a temporary file with bootstrap contents, based on
-// the passed in options, and updates the bootstrap environment variable to
-// point to this file.
-//
-// Returns a cleanup function which will be non-nil if the setup process was
-// completed successfully. It is the responsibility of the caller to invoke the
-// cleanup function at the end of the test.
-func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
-	f, err := ioutil.TempFile("", "test_xds_bootstrap_*")
-	if err != nil {
-		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
-	}
-
-	cfg := &bootstrapConfig{
-		XdsServers: []server{
-			{
-				ServerURI: opts.ServerURI,
-				ChannelCreds: []creds{
-					{
-						Type: "insecure",
-					},
-				},
-			},
-		},
-		Node: node{
-			ID: opts.NodeID,
-		},
-		CertificateProviders:               opts.CertificateProviders,
-		ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate,
-	}
-	switch opts.Version {
-	case TransportV2:
-		// TODO: Add any v2 specific fields.
-	case TransportV3:
-		cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "xds_v3")
-	default:
-		return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version)
-	}
-
-	bootstrapContents, err := json.MarshalIndent(cfg, "", "  ")
-	if err != nil {
-		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
-	}
-	if err := ioutil.WriteFile(f.Name(), bootstrapContents, 0644); err != nil {
-		return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
-	}
-	logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents)
-
-	origBootstrapFileName := env.BootstrapFileName
-	env.BootstrapFileName = f.Name()
-	return func() {
-		os.Remove(f.Name())
-		env.BootstrapFileName = origBootstrapFileName
-	}, nil
-}
-
 // DefaultFileWatcherConfig is a helper function to create a default certificate
 // provider plugin configuration. The test is expected to have setup the files
 // appropriately before this configuration is used to instantiate providers.
@@ -125,25 +40,3 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) map[string]json.
 		"google_cloud_private_spiffe": json.RawMessage(cfg),
 	}
 }
-
-type bootstrapConfig struct {
-	XdsServers                         []server                   `json:"xds_servers,omitempty"`
-	Node                               node                       `json:"node,omitempty"`
-	CertificateProviders               map[string]json.RawMessage `json:"certificate_providers,omitempty"`
-	ServerListenerResourceNameTemplate string                     `json:"server_listener_resource_name_template,omitempty"`
-}
-
-type server struct {
-	ServerURI      string   `json:"server_uri,omitempty"`
-	ChannelCreds   []creds  `json:"channel_creds,omitempty"`
-	ServerFeatures []string `json:"server_features,omitempty"`
-}
-
-type creds struct {
-	Type   string      `json:"type,omitempty"`
-	Config interface{} `json:"config,omitempty"`
-}
-
-type node struct {
-	ID string `json:"id,omitempty"`
-}
diff --git a/xds/xds.go b/xds/xds.go
index 17b84ec932cb..0af8f3d6d892 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -28,6 +28,13 @@
 package xds
 
 import (
+	"fmt"
+
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+	"google.golang.org/grpc"
+	internaladmin "google.golang.org/grpc/internal/admin"
+	"google.golang.org/grpc/xds/csds"
+
 	_ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin.
 	_ "google.golang.org/grpc/xds/internal/balancer"                // Register the balancers.
 	_ "google.golang.org/grpc/xds/internal/client/v2"               // Register the v2 xDS API client.
@@ -35,3 +42,32 @@ import (
 	_ "google.golang.org/grpc/xds/internal/httpfilter/fault"        // Register the fault injection filter.
 	_ "google.golang.org/grpc/xds/internal/resolver"                // Register the xds_resolver.
 )
+
+func init() {
+	internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) {
+		var grpcServer *grpc.Server
+		switch ss := registrar.(type) {
+		case *grpc.Server:
+			grpcServer = ss
+		case *GRPCServer:
+			sss, ok := ss.gs.(*grpc.Server)
+			if !ok {
+				logger.Warningf("grpc server within xds.GRPCServer is not *grpc.Server, CSDS will not be registered")
+				return nil, nil
+			}
+			grpcServer = sss
+		default:
+			// Returning an error would cause the top level admin.Register() to
+			// fail. Log a warning instead.
+			logger.Warningf("server to register service on is neither a *grpc.Server or a *xds.GRPCServer, CSDS will not be registered")
+			return nil, nil
+		}
+
+		csdss, err := csds.NewClientStatusDiscoveryServer()
+		if err != nil {
+			return nil, fmt.Errorf("failed to create csds server: %v", err)
+		}
+		v3statuspb.RegisterClientStatusDiscoveryServiceServer(grpcServer, csdss)
+		return csdss.Close, nil
+	})
+}

From b331a48e06791ab7595f706af46b8bf9244d1f2e Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 23 Mar 2021 10:42:27 -0700
Subject: [PATCH 394/481] alts: re-add vmOnGCP and once globals for easier
 testing (#4284)

---
 credentials/alts/alts.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/credentials/alts/alts.go b/credentials/alts/alts.go
index 4366fbd1f24e..579adf210c48 100644
--- a/credentials/alts/alts.go
+++ b/credentials/alts/alts.go
@@ -28,6 +28,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
 	"google.golang.org/grpc/credentials"
@@ -54,6 +55,8 @@ const (
 )
 
 var (
+	vmOnGCP       bool
+	once          sync.Once
 	maxRPCVersion = &altspb.RpcProtocolVersions_Version{
 		Major: protocolVersionMaxMajor,
 		Minor: protocolVersionMaxMinor,
@@ -147,6 +150,9 @@ func NewServerCreds(opts *ServerOptions) credentials.TransportCredentials {
 }
 
 func newALTS(side core.Side, accounts []string, hsAddress string) credentials.TransportCredentials {
+	once.Do(func() {
+		vmOnGCP = googlecloud.OnGCE()
+	})
 	if hsAddress == "" {
 		hsAddress = hypervisorHandshakerServiceAddress
 	}
@@ -163,7 +169,7 @@ func newALTS(side core.Side, accounts []string, hsAddress string) credentials.Tr
 
 // ClientHandshake implements the client side handshake protocol.
 func (g *altsTC) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) {
-	if !googlecloud.OnGCE() {
+	if !vmOnGCP {
 		return nil, nil, ErrUntrustedPlatform
 	}
 
@@ -218,7 +224,7 @@ func (g *altsTC) ClientHandshake(ctx context.Context, addr string, rawConn net.C
 
 // ServerHandshake implements the server side ALTS handshaker.
 func (g *altsTC) ServerHandshake(rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) {
-	if !googlecloud.OnGCE() {
+	if !vmOnGCP {
 		return nil, nil, ErrUntrustedPlatform
 	}
 	// Connecting to ALTS handshaker service.

From 46da49ca604aef87498c628719b3408f27f4c6d7 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 23 Mar 2021 13:26:01 -0700
Subject: [PATCH 395/481] xds: use different proto import for grpc services
 (#4285)

---
 admin/test/utils.go | 3 ++-
 xds/csds/csds.go    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/admin/test/utils.go b/admin/test/utils.go
index 069ffb3cadb7..1add8afa824c 100644
--- a/admin/test/utils.go
+++ b/admin/test/utils.go
@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 	"github.com/google/uuid"
 	"google.golang.org/grpc"
@@ -105,7 +106,7 @@ func RunChannelz(conn *grpc.ClientConn) error {
 
 // RunCSDS makes a CSDS RPC.
 func RunCSDS(conn *grpc.ClientConn) error {
-	c := v3statuspb.NewClientStatusDiscoveryServiceClient(conn)
+	c := v3statusgrpc.NewClientStatusDiscoveryServiceClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	_, err := c.FetchClientStatus(ctx, &v3statuspb.ClientStatusRequest{}, grpc.WaitForReady(true))
diff --git a/xds/csds/csds.go b/xds/csds/csds.go
index e803a2d2cec1..73b92e9443ce 100644
--- a/xds/csds/csds.go
+++ b/xds/csds/csds.go
@@ -32,6 +32,7 @@ import (
 	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc/codes"
@@ -83,7 +84,7 @@ func NewClientStatusDiscoveryServer() (*ClientStatusDiscoveryServer, error) {
 }
 
 // StreamClientStatus implementations interface ClientStatusDiscoveryServiceServer.
-func (s *ClientStatusDiscoveryServer) StreamClientStatus(stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusServer) error {
+func (s *ClientStatusDiscoveryServer) StreamClientStatus(stream v3statusgrpc.ClientStatusDiscoveryService_StreamClientStatusServer) error {
 	for {
 		req, err := stream.Recv()
 		if err == io.EOF {

From faf4e1c777f0c306e1632c8efda49f81f8de7646 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 23 Mar 2021 15:19:03 -0700
Subject: [PATCH 396/481] xds: rename proto import to grpc (#4287)

---
 xds/xds.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xds/xds.go b/xds/xds.go
index 0af8f3d6d892..562c5aa82abc 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -30,7 +30,7 @@ package xds
 import (
 	"fmt"
 
-	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+	v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 	"google.golang.org/grpc"
 	internaladmin "google.golang.org/grpc/internal/admin"
 	"google.golang.org/grpc/xds/csds"
@@ -67,7 +67,7 @@ func init() {
 		if err != nil {
 			return nil, fmt.Errorf("failed to create csds server: %v", err)
 		}
-		v3statuspb.RegisterClientStatusDiscoveryServiceServer(grpcServer, csdss)
+		v3statusgrpc.RegisterClientStatusDiscoveryServiceServer(grpcServer, csdss)
 		return csdss.Close, nil
 	})
 }

From 702608ffae4d03a6821b96d3e2311973d34b96dc Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Wed, 24 Mar 2021 10:20:16 -0700
Subject: [PATCH 397/481] xds: enable timeout, circuit breaking, and fault
 injection by default (#4286)

---
 internal/xds/env/env.go | 14 +++++++-------
 xds/csds/csds_test.go   | 11 ++++++-----
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/internal/xds/env/env.go b/internal/xds/env/env.go
index 035b92b18f13..1110722a630b 100644
--- a/internal/xds/env/env.go
+++ b/internal/xds/env/env.go
@@ -61,16 +61,16 @@ var (
 	// When both bootstrap FileName and FileContent are set, FileName is used.
 	BootstrapFileContent = os.Getenv(BootstrapFileContentEnv)
 	// CircuitBreakingSupport indicates whether circuit breaking support is
-	// enabled, which can be done by setting the environment variable
-	// "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING" to "true".
-	CircuitBreakingSupport = strings.EqualFold(os.Getenv(circuitBreakingSupportEnv), "true")
+	// enabled, which can be disabled by setting the environment variable
+	// "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING" to "false".
+	CircuitBreakingSupport = !strings.EqualFold(os.Getenv(circuitBreakingSupportEnv), "false")
 	// TimeoutSupport indicates whether support for max_stream_duration in
-	// route actions is enabled.  This can be enabled by setting the
-	// environment variable "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT" to "true".
-	TimeoutSupport = strings.EqualFold(os.Getenv(timeoutSupportEnv), "true")
+	// route actions is enabled.  This can be disabled by setting the
+	// environment variable "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT" to "false".
+	TimeoutSupport = !strings.EqualFold(os.Getenv(timeoutSupportEnv), "false")
 	// FaultInjectionSupport is used to control both fault injection and HTTP
 	// filter support.
-	FaultInjectionSupport = strings.EqualFold(os.Getenv(faultInjectionSupportEnv), "true")
+	FaultInjectionSupport = !strings.EqualFold(os.Getenv(faultInjectionSupportEnv), "false")
 	// C2PResolverSupport indicates whether support for C2P resolver is enabled.
 	// This can be enabled by setting the environment variable
 	// "GRPC_EXPERIMENTAL_GOOGLE_C2P_RESOLVER" to "true".
diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
index 856894b4e4d4..2993beea0e5d 100644
--- a/xds/csds/csds_test.go
+++ b/xds/csds/csds_test.go
@@ -25,11 +25,6 @@ import (
 	"testing"
 	"time"
 
-	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
-	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
-	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
-	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
@@ -39,14 +34,20 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/client"
+	_ "google.golang.org/grpc/xds/internal/httpfilter/router"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
+	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 )
 

From 80e380eff4edbfdacb4be1ae7d92c772400b2159 Mon Sep 17 00:00:00 2001
From: longxboy <longxboyhi@gmail.com>
Date: Fri, 26 Mar 2021 04:08:24 +0800
Subject: [PATCH 398/481] balancer/base: keep address attributes for pickers
 (#4253)

---
 balancer/base/balancer.go      | 29 +++++++++++-------
 balancer/base/balancer_test.go | 54 +++++++++++++++++++++++++++++++++-
 2 files changed, 72 insertions(+), 11 deletions(-)

diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go
index 383d02ec2bf5..c883efa0bbf5 100644
--- a/balancer/base/balancer.go
+++ b/balancer/base/balancer.go
@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 
+	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/grpclog"
@@ -41,7 +42,7 @@ func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions)
 		cc:            cc,
 		pickerBuilder: bb.pickerBuilder,
 
-		subConns: make(map[resolver.Address]balancer.SubConn),
+		subConns: make(map[resolver.Address]subConnInfo),
 		scStates: make(map[balancer.SubConn]connectivity.State),
 		csEvltr:  &balancer.ConnectivityStateEvaluator{},
 		config:   bb.config,
@@ -57,6 +58,11 @@ func (bb *baseBuilder) Name() string {
 	return bb.name
 }
 
+type subConnInfo struct {
+	subConn balancer.SubConn
+	attrs   *attributes.Attributes
+}
+
 type baseBalancer struct {
 	cc            balancer.ClientConn
 	pickerBuilder PickerBuilder
@@ -64,7 +70,7 @@ type baseBalancer struct {
 	csEvltr *balancer.ConnectivityStateEvaluator
 	state   connectivity.State
 
-	subConns map[resolver.Address]balancer.SubConn // `attributes` is stripped from the keys of this map (the addresses)
+	subConns map[resolver.Address]subConnInfo // `attributes` is stripped from the keys of this map (the addresses)
 	scStates map[balancer.SubConn]connectivity.State
 	picker   balancer.Picker
 	config   Config
@@ -114,7 +120,7 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 		aNoAttrs := a
 		aNoAttrs.Attributes = nil
 		addrsSet[aNoAttrs] = struct{}{}
-		if sc, ok := b.subConns[aNoAttrs]; !ok {
+		if scInfo, ok := b.subConns[aNoAttrs]; !ok {
 			// a is a new address (not existing in b.subConns).
 			//
 			// When creating SubConn, the original address with attributes is
@@ -125,7 +131,7 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 				logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err)
 				continue
 			}
-			b.subConns[aNoAttrs] = sc
+			b.subConns[aNoAttrs] = subConnInfo{subConn: sc, attrs: a.Attributes}
 			b.scStates[sc] = connectivity.Idle
 			sc.Connect()
 		} else {
@@ -135,13 +141,15 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
 			// The SubConn does a reflect.DeepEqual of the new and old
 			// addresses. So this is a noop if the current address is the same
 			// as the old one (including attributes).
-			b.cc.UpdateAddresses(sc, []resolver.Address{a})
+			scInfo.attrs = a.Attributes
+			b.subConns[aNoAttrs] = scInfo
+			b.cc.UpdateAddresses(scInfo.subConn, []resolver.Address{a})
 		}
 	}
-	for a, sc := range b.subConns {
+	for a, scInfo := range b.subConns {
 		// a was removed by resolver.
 		if _, ok := addrsSet[a]; !ok {
-			b.cc.RemoveSubConn(sc)
+			b.cc.RemoveSubConn(scInfo.subConn)
 			delete(b.subConns, a)
 			// Keep the state of this sc in b.scStates until sc's state becomes Shutdown.
 			// The entry will be deleted in UpdateSubConnState.
@@ -184,9 +192,10 @@ func (b *baseBalancer) regeneratePicker() {
 	readySCs := make(map[balancer.SubConn]SubConnInfo)
 
 	// Filter out all ready SCs from full subConn map.
-	for addr, sc := range b.subConns {
-		if st, ok := b.scStates[sc]; ok && st == connectivity.Ready {
-			readySCs[sc] = SubConnInfo{Address: addr}
+	for addr, scInfo := range b.subConns {
+		if st, ok := b.scStates[scInfo.subConn]; ok && st == connectivity.Ready {
+			addr.Attributes = scInfo.attrs
+			readySCs[scInfo.subConn] = SubConnInfo{Address: addr}
 		}
 	}
 	b.picker = b.pickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs})
diff --git a/balancer/base/balancer_test.go b/balancer/base/balancer_test.go
index 03114251a048..f8ff8cf98444 100644
--- a/balancer/base/balancer_test.go
+++ b/balancer/base/balancer_test.go
@@ -23,6 +23,7 @@ import (
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -35,12 +36,24 @@ func (c *testClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewS
 	return c.newSubConn(addrs, opts)
 }
 
+func (c *testClientConn) UpdateState(balancer.State) {}
+
 type testSubConn struct{}
 
 func (sc *testSubConn) UpdateAddresses(addresses []resolver.Address) {}
 
 func (sc *testSubConn) Connect() {}
 
+// testPickBuilder creates balancer.Picker for test.
+type testPickBuilder struct {
+	validate func(info PickerBuildInfo)
+}
+
+func (p *testPickBuilder) Build(info PickerBuildInfo) balancer.Picker {
+	p.validate(info)
+	return nil
+}
+
 func TestBaseBalancerStripAttributes(t *testing.T) {
 	b := (&baseBuilder{}).Build(&testClientConn{
 		newSubConn: func(addrs []resolver.Address, _ balancer.NewSubConnOptions) (balancer.SubConn, error) {
@@ -64,7 +77,46 @@ func TestBaseBalancerStripAttributes(t *testing.T) {
 
 	for addr := range b.subConns {
 		if addr.Attributes != nil {
-			t.Errorf("in b.subConns, got address %+v with nil attributes, want not nil", addr)
+			t.Errorf("in b.subConns, got address %+v with not nil attributes, want nil", addr)
+		}
+	}
+}
+
+func TestBaseBalancerReserveAttributes(t *testing.T) {
+	var v = func(info PickerBuildInfo) {
+		for _, sc := range info.ReadySCs {
+			if sc.Address.Addr == "1.1.1.1" {
+				if sc.Address.Attributes == nil {
+					t.Errorf("in picker.validate, got address %+v with nil attributes, want not nil", sc.Address)
+				}
+				foo, ok := sc.Address.Attributes.Value("foo").(string)
+				if !ok || foo != "2233niang" {
+					t.Errorf("in picker.validate, got address[1.1.1.1] with invalid attributes value %v, want 2233niang", sc.Address.Attributes.Value("foo"))
+				}
+			} else if sc.Address.Addr == "2.2.2.2" {
+				if sc.Address.Attributes != nil {
+					t.Error("in b.subConns, got address[2.2.2.2] with not nil attributes, want nil")
+				}
+			}
 		}
 	}
+	pickBuilder := &testPickBuilder{validate: v}
+	b := (&baseBuilder{pickerBuilder: pickBuilder}).Build(&testClientConn{
+		newSubConn: func(addrs []resolver.Address, _ balancer.NewSubConnOptions) (balancer.SubConn, error) {
+			return &testSubConn{}, nil
+		},
+	}, balancer.BuildOptions{}).(*baseBalancer)
+
+	b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				{Addr: "1.1.1.1", Attributes: attributes.New("foo", "2233niang")},
+				{Addr: "2.2.2.2", Attributes: nil},
+			},
+		},
+	})
+
+	for sc := range b.scStates {
+		b.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready, ConnectionError: nil})
+	}
 }

From 2456c5cff04bb867e220f084bc88034f588c8aa8 Mon Sep 17 00:00:00 2001
From: apolcyn <apolcyn@google.com>
Date: Thu, 25 Mar 2021 20:56:46 -0700
Subject: [PATCH 399/481] Allow using interop client for making Traffic
 Director RPCs (#4291)

---
 interop/client/client.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/interop/client/client.go b/interop/client/client.go
index 8854ed2d76ec..b827f574b8cb 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/interop"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/testdata"
+	_ "google.golang.org/grpc/xds/googledirectpath"
 
 	testgrpc "google.golang.org/grpc/interop/grpc_testing"
 )
@@ -126,7 +127,10 @@ func main() {
 	}
 
 	resolver.SetDefaultScheme("dns")
-	serverAddr := net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort))
+	serverAddr := *serverHost
+	if *serverPort != 0 {
+		serverAddr = net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort))
+	}
 	var opts []grpc.DialOption
 	switch credsChosen {
 	case credsTLS:

From 4a19753e9dfdf7c54c4b44ae419876e94ef3a0cc Mon Sep 17 00:00:00 2001
From: apolcyn <apolcyn@google.com>
Date: Fri, 26 Mar 2021 10:09:12 -0700
Subject: [PATCH 400/481] interop: add a flag to clients to statically
 configure grpclb (#4290)

---
 interop/client/client.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/interop/client/client.go b/interop/client/client.go
index b827f574b8cb..975e0b5d2f3c 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -56,6 +56,7 @@ var (
 	defaultServiceAccount = flag.String("default_service_account", "", "Email of GCE default service account")
 	serverHost            = flag.String("server_host", "localhost", "The server host name")
 	serverPort            = flag.Int("server_port", 10000, "The server port number")
+	serviceConfigJSON     = flag.String("service_config_json", "", "Disables service config lookups and sets the provided string as the default service config.")
 	tlsServerName         = flag.String("server_host_override", "", "The server name use to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.")
 	testCase              = flag.String("test_case", "large_unary",
 		`Configure different test cases. Valid options are:
@@ -187,6 +188,9 @@ func main() {
 			opts = append(opts, grpc.WithPerRPCCredentials(oauth.NewOauthAccess(interop.GetToken(*serviceAccountKeyFile, *oauthScope))))
 		}
 	}
+	if len(*serviceConfigJSON) > 0 {
+		opts = append(opts, grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(*serviceConfigJSON))
+	}
 	opts = append(opts, grpc.WithBlock())
 	conn, err := grpc.Dial(serverAddr, opts...)
 	if err != nil {

From 0028242dbbf8efab46fb0e25cef649ef7bea1730 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 31 Mar 2021 10:36:16 -0700
Subject: [PATCH 401/481] Change version to 1.38.0-dev (#4306)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index c149b22ad8a1..f73faed82920 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.37.0-dev"
+const Version = "1.38.0-dev"

From c72e1c8f7528615e2b5b887d279015abb2b6c659 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 31 Mar 2021 16:30:10 -0700
Subject: [PATCH 402/481] xds/resolver: support inline RDS resource from LDS
 response (#4299)

---
 xds/internal/client/client.go               |  8 +++
 xds/internal/client/lds_test.go             | 53 ++++++++++++++
 xds/internal/client/xds.go                  | 16 +++--
 xds/internal/resolver/watch_service.go      | 39 ++++++++---
 xds/internal/resolver/watch_service_test.go | 78 +++++++++++++++++++++
 5 files changed, 177 insertions(+), 17 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 5c0f38a9782f..d06473b02a6b 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -195,7 +195,15 @@ type UpdateMetadata struct {
 type ListenerUpdate struct {
 	// RouteConfigName is the route configuration name corresponding to the
 	// target which is being watched through LDS.
+	//
+	// Only one of RouteConfigName and InlineRouteConfig is set.
 	RouteConfigName string
+	// InlineRouteConfig is the inline route configuration (RDS response)
+	// returned inside LDS.
+	//
+	// Only one of RouteConfigName and InlineRouteConfig is set.
+	InlineRouteConfig *RouteConfigUpdate
+
 	// MaxStreamDuration contains the HTTP connection manager's
 	// common_http_protocol_options.max_stream_duration field, or zero if
 	// unset.
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index df8098df8368..da0e0f956156 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -54,6 +54,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 		v3LDSTarget       = "lds.target.good:3333"
 		v2RouteConfigName = "v2RouteConfig"
 		v3RouteConfigName = "v3RouteConfig"
+		routeName         = "routeName"
 	)
 
 	var (
@@ -132,6 +133,39 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig},
 			IsOptional: true,
 		}
+		v3LisWithInlineRoute = &anypb.Any{
+			TypeUrl: version.V3ListenerURL,
+			Value: func() []byte {
+				hcm := &v3httppb.HttpConnectionManager{
+					RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+						RouteConfig: &v3routepb.RouteConfiguration{
+							Name: routeName,
+							VirtualHosts: []*v3routepb.VirtualHost{{
+								Domains: []string{v3LDSTarget},
+								Routes: []*v3routepb.Route{{
+									Match: &v3routepb.RouteMatch{
+										PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+									},
+									Action: &v3routepb.Route_Route{
+										Route: &v3routepb.RouteAction{
+											ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+										}}}}}}},
+					},
+					CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
+						MaxStreamDuration: durationpb.New(time.Second),
+					},
+				}
+				mcm := marshalAny(hcm)
+				lis := &v3listenerpb.Listener{
+					Name: v3LDSTarget,
+					ApiListener: &v3listenerpb.ApiListener{
+						ApiListener: mcm,
+					},
+				}
+				mLis, _ := proto.Marshal(lis)
+				return mLis
+			}(),
+		}
 		v3LisWithFilters = func(fs ...*v3httppb.HttpFilter) *anypb.Any {
 			hcm := &v3httppb.HttpConnectionManager{
 				RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
@@ -650,6 +684,25 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 				Version: testVersion,
 			},
 		},
+		{
+			name:      "v3 listener with inline route configuration",
+			resources: []*anypb.Any{v3LisWithInlineRoute},
+			wantUpdate: map[string]ListenerUpdate{
+				v3LDSTarget: {
+					InlineRouteConfig: &RouteConfigUpdate{
+						VirtualHosts: []*VirtualHost{{
+							Domains: []string{v3LDSTarget},
+							Routes:  []*Route{{Prefix: newStringP("/"), WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}}},
+						}}},
+					MaxStreamDuration: time.Second,
+					Raw:               v3LisWithInlineRoute,
+				},
+			},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusACKed,
+				Version: testVersion,
+			},
+		},
 		{
 			name:      "multiple listener resources",
 			resources: []*anypb.Any{v2Lis, v3LisWithFilters()},
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 2791603ce26e..fc1112e180bc 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -72,7 +72,7 @@ func unmarshalListenerResource(r *anypb.Any, logger *grpclog.PrefixLogger) (stri
 	}
 	logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
 
-	lu, err := processListener(lis, v2)
+	lu, err := processListener(lis, logger, v2)
 	if err != nil {
 		return lis.GetName(), ListenerUpdate{}, err
 	}
@@ -80,16 +80,16 @@ func unmarshalListenerResource(r *anypb.Any, logger *grpclog.PrefixLogger) (stri
 	return lis.GetName(), *lu, nil
 }
 
-func processListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUpdate, error) {
+func processListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger, v2 bool) (*ListenerUpdate, error) {
 	if lis.GetApiListener() != nil {
-		return processClientSideListener(lis, v2)
+		return processClientSideListener(lis, logger, v2)
 	}
 	return processServerSideListener(lis)
 }
 
 // processClientSideListener checks if the provided Listener proto meets
 // the expected criteria. If so, it returns a non-empty routeConfigName.
-func processClientSideListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUpdate, error) {
+func processClientSideListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger, v2 bool) (*ListenerUpdate, error) {
 	update := &ListenerUpdate{}
 
 	apiLisAny := lis.GetApiListener().GetApiListener()
@@ -112,9 +112,11 @@ func processClientSideListener(lis *v3listenerpb.Listener, v2 bool) (*ListenerUp
 		}
 		update.RouteConfigName = name
 	case *v3httppb.HttpConnectionManager_RouteConfig:
-		// TODO: Add support for specifying the RouteConfiguration inline
-		// in the LDS response.
-		return nil, fmt.Errorf("LDS response contains RDS config inline. Not supported for now: %+v", apiLis)
+		routeU, err := generateRDSUpdateFromRouteConfiguration(apiLis.GetRouteConfig(), logger, v2)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err)
+		}
+		update.InlineRouteConfig = &routeU
 	case nil:
 		return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis)
 	default:
diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go
index 913ac4ced15c..42ede988300c 100644
--- a/xds/internal/resolver/watch_service.go
+++ b/xds/internal/resolver/watch_service.go
@@ -110,6 +110,22 @@ func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, er
 		httpFilterConfig:  update.HTTPFilters,
 	}
 
+	if update.InlineRouteConfig != nil {
+		// If there was an RDS watch, cancel it.
+		w.rdsName = ""
+		if w.rdsCancel != nil {
+			w.rdsCancel()
+			w.rdsCancel = nil
+		}
+
+		// Handle the inline RDS update as if it's from an RDS watch.
+		w.updateVirtualHostsFromRDS(*update.InlineRouteConfig)
+		return
+	}
+
+	// RDS name from update is not an empty string, need RDS to fetch the
+	// routes.
+
 	if w.rdsName == update.RouteConfigName {
 		// If the new RouteConfigName is same as the previous, don't cancel and
 		// restart the RDS watch.
@@ -126,6 +142,18 @@ func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, er
 	w.rdsCancel = w.c.WatchRouteConfig(update.RouteConfigName, w.handleRDSResp)
 }
 
+func (w *serviceUpdateWatcher) updateVirtualHostsFromRDS(update xdsclient.RouteConfigUpdate) {
+	matchVh := findBestMatchingVirtualHost(w.serviceName, update.VirtualHosts)
+	if matchVh == nil {
+		// No matching virtual host found.
+		w.serviceCb(serviceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName))
+		return
+	}
+
+	w.lastUpdate.virtualHost = matchVh
+	w.serviceCb(w.lastUpdate, nil)
+}
+
 func (w *serviceUpdateWatcher) handleRDSResp(update xdsclient.RouteConfigUpdate, err error) {
 	w.logger.Infof("received RDS update: %+v, err: %v", update, err)
 	w.mu.Lock()
@@ -142,16 +170,7 @@ func (w *serviceUpdateWatcher) handleRDSResp(update xdsclient.RouteConfigUpdate,
 		w.serviceCb(serviceUpdate{}, err)
 		return
 	}
-
-	matchVh := findBestMatchingVirtualHost(w.serviceName, update.VirtualHosts)
-	if matchVh == nil {
-		// No matching virtual host found.
-		w.serviceCb(serviceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName))
-		return
-	}
-
-	w.lastUpdate.virtualHost = matchVh
-	w.serviceCb(w.lastUpdate, nil)
+	w.updateVirtualHostsFromRDS(update)
 }
 
 func (w *serviceUpdateWatcher) close() {
diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go
index 705a3d35ae1b..2bfe3e984d3c 100644
--- a/xds/internal/resolver/watch_service_test.go
+++ b/xds/internal/resolver/watch_service_test.go
@@ -356,3 +356,81 @@ func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) {
 		t.Fatalf("wait for cancel route watch failed: %v, want nil", err)
 	}
 }
+
+// TestServiceWatchInlineRDS covers the cases switching between:
+// - LDS update contains RDS name to watch
+// - LDS update contains inline RDS resource
+func (s) TestServiceWatchInlineRDS(t *testing.T) {
+	serviceUpdateCh := testutils.NewChannel()
+	xdsC := fakeclient.NewClient()
+	cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) {
+		serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err})
+	}, nil)
+	defer cancelWatch()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// First LDS update is LDS with RDS name to watch.
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+	wantUpdate := serviceUpdate{virtualHost: &xdsclient.VirtualHost{Domains: []string{"target"}, Routes: []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}}}}
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch LDS resp to a LDS with inline RDS resource
+	wantVirtualHosts2 := &xdsclient.VirtualHost{Domains: []string{"target"},
+		Routes: []*xdsclient.Route{{
+			Path:             newStringP(""),
+			WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}},
+		}},
+	}
+	wantUpdate2 := serviceUpdate{virtualHost: wantVirtualHosts2}
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{InlineRouteConfig: &xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{wantVirtualHosts2},
+	}}, nil)
+	// This inline RDS resource should cause the RDS watch to be canceled.
+	if err := xdsC.WaitForCancelRouteConfigWatch(ctx); err != nil {
+		t.Fatalf("wait for cancel route watch failed: %v, want nil", err)
+	}
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch LDS update back to LDS with RDS name to watch.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{
+			{
+				Domains: []string{targetStr},
+				Routes:  []*xdsclient.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsclient.WeightedCluster{cluster: {Weight: 1}}}},
+			},
+		},
+	}, nil)
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch LDS resp to a LDS with inline RDS resource again.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{InlineRouteConfig: &xdsclient.RouteConfigUpdate{
+		VirtualHosts: []*xdsclient.VirtualHost{wantVirtualHosts2},
+	}}, nil)
+	// This inline RDS resource should cause the RDS watch to be canceled.
+	if err := xdsC.WaitForCancelRouteConfigWatch(ctx); err != nil {
+		t.Fatalf("wait for cancel route watch failed: %v, want nil", err)
+	}
+	if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil {
+		t.Fatal(err)
+	}
+}

From f6bb3972ed15a0aaf47730344c47e9840bb5dbba Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 31 Mar 2021 16:58:24 -0700
Subject: [PATCH 403/481] xds: filter chain matching logic for server-side
 (#4281)

---
 xds/internal/client/client.go                 |   56 +-
 xds/internal/client/filter_chain.go           |  552 +++++++
 xds/internal/client/filter_chain_test.go      | 1316 +++++++++++++++++
 xds/internal/client/lds_test.go               |  844 ++++-------
 xds/internal/client/xds.go                    |  110 +-
 xds/internal/server/conn_wrapper.go           |   36 +-
 xds/internal/server/listener_wrapper.go       |  143 +-
 xds/internal/server/listener_wrapper_test.go  |  335 +++++
 .../test/xds_server_integration_test.go       |   19 +
 xds/server_test.go                            |   65 +-
 10 files changed, 2655 insertions(+), 821 deletions(-)
 create mode 100644 xds/internal/client/filter_chain.go
 create mode 100644 xds/internal/client/filter_chain_test.go
 create mode 100644 xds/internal/server/listener_wrapper_test.go

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index d06473b02a6b..37eaae79f2d5 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -24,7 +24,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"sync"
 	"time"
 
@@ -241,60 +240,7 @@ type InboundListenerConfig struct {
 	// accept incoming connections.
 	Port string
 	// FilterChains is the list of filter chains associated with this listener.
-	FilterChains []*FilterChain
-	// DefaultFilterChain is the filter chain to be used when none of the above
-	// filter chains matches an incoming connection.
-	DefaultFilterChain *FilterChain
-}
-
-// FilterChain wraps a set of match criteria and associated security
-// configuration.
-//
-// The actual set filters associated with this filter chain are not captured
-// here, since we do not support these filters on the server yet.
-type FilterChain struct {
-	// Match contains the criteria to use when matching a connection to this
-	// filter chain.
-	Match *FilterChainMatch
-	// SecurityCfg contains transport socket security configuration.
-	SecurityCfg *SecurityConfig
-}
-
-// SourceType specifies the connection source IP match type.
-type SourceType int
-
-const (
-	// SourceTypeAny matches connection attempts from any source.
-	SourceTypeAny SourceType = iota
-	// SourceTypeSameOrLoopback matches connection attempts from the same host.
-	SourceTypeSameOrLoopback
-	// SourceTypeExternal matches connection attempts from a different host.
-	SourceTypeExternal
-)
-
-// FilterChainMatch specifies the match criteria for selecting a specific filter
-// chain of a listener, for an incoming connection.
-//
-// The xDS FilterChainMatch proto specifies 8 match criteria. But we only have a
-// subset of those fields here because we explicitly ignore filter chains whose
-// match criteria specifies values for fields like destination_port,
-// server_names, application_protocols, transport_protocol.
-type FilterChainMatch struct {
-	// DestPrefixRanges specifies a set of IP addresses and prefix lengths to
-	// match the destination address of the incoming connection when the
-	// listener is bound to 0.0.0.0/[::]. If this field is empty, the
-	// destination address is ignored.
-	DestPrefixRanges []net.IP
-	// SourceType specifies the connection source IP match type. Can be any,
-	// local or external network.
-	SourceType SourceType
-	// SourcePrefixRanges specifies a set of IP addresses and prefix lengths to
-	// match the source address of the incoming connection. If this field is
-	// empty, the source address is ignored.
-	SourcePrefixRanges []net.IP
-	// SourcePorts specifies a set of ports to match the source port of the
-	// incoming connection. If this field is empty, the source port is ignored.
-	SourcePorts []uint32
+	FilterChains *FilterChainManager
 }
 
 // RouteConfigUpdate contains information received in an RDS response, which is
diff --git a/xds/internal/client/filter_chain.go b/xds/internal/client/filter_chain.go
new file mode 100644
index 000000000000..8e24ab858230
--- /dev/null
+++ b/xds/internal/client/filter_chain.go
@@ -0,0 +1,552 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"errors"
+	"fmt"
+	"net"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	"github.com/golang/protobuf/proto"
+
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+// Represents a wildcard IP prefix. Go stdlib `Contains()` method works for both
+// v4 and v6 addresses when used on this wildcard address.
+const emptyAddrMapKey = "0.0.0.0/0"
+
+var (
+	// Parsed wildcard IP prefix.
+	_, zeroIP, _ = net.ParseCIDR("0.0.0.0/0")
+)
+
+// FilterChain captures information from within a FilterChain message in a
+// Listener resource.
+//
+// Currently, this simply contains the security configuration found in the
+// 'transport_socket' field of the filter chain. The actual set of filters
+// associated with this filter chain are not captured here, since we do not
+// support these filters on the server-side yet.
+type FilterChain struct {
+	// SecurityCfg contains transport socket security configuration.
+	SecurityCfg *SecurityConfig
+}
+
+// SourceType specifies the connection source IP match type.
+type SourceType int
+
+const (
+	// SourceTypeAny matches connection attempts from any source.
+	SourceTypeAny SourceType = iota
+	// SourceTypeSameOrLoopback matches connection attempts from the same host.
+	SourceTypeSameOrLoopback
+	// SourceTypeExternal matches connection attempts from a different host.
+	SourceTypeExternal
+)
+
+// FilterChainManager contains all the match criteria specified through all
+// filter chains in a single Listener resource. It also contains the default
+// filter chain specified in the Listener resource. It provides two important
+// pieces of functionality:
+// 1. Validate the filter chains in an incoming Listener resource to make sure
+//    that there aren't filter chains which contain the same match criteria.
+// 2. As part of performing the above validation, it builds an internal data
+//    structure which will if used to look up the matching filter chain at
+//    connection time.
+//
+// The logic specified in the documentation around the xDS FilterChainMatch
+// proto mentions 8 criteria to match on. gRPC does not support 4 of those, and
+// we ignore filter chains which contain any of these unsupported fields at
+// parsing time. Here we use the remaining 4 criteria to find a matching filter
+// chain in the following order:
+// Destination IP address, Source type, Source IP address, Source port.
+// TODO: Ignore chains with unsupported fields *only* at connection time.
+type FilterChainManager struct {
+	// Destination prefix is the first match criteria that we support.
+	// Therefore, this multi-stage map is indexed on destination prefixes
+	// specified in the match criteria.
+	// Unspecified destination prefix matches end up as a wildcard entry here
+	// with a key of 0.0.0.0/0.
+	dstPrefixMap map[string]*destPrefixEntry
+
+	// At connection time, we do not have the actual destination prefix to match
+	// on. We only have the real destination address of the incoming connection.
+	// This means that we cannot use the above map at connection time. This list
+	// contains the map entries from the above map that we can use at connection
+	// time to find matching destination prefixes in O(n) time.
+	//
+	// TODO: Implement LC-trie to support logarithmic time lookups. If that
+	// involves too much time/effort, sort this slice based on the netmask size.
+	dstPrefixes []*destPrefixEntry
+
+	def   *FilterChain // Default filter chain, if specified.
+	fcCnt int          // Count of supported filter chains, for validation.
+}
+
+// destPrefixEntry is the value type of the map indexed on destination prefixes.
+type destPrefixEntry struct {
+	net *net.IPNet // The actual destination prefix.
+	// For each specified source type in the filter chain match criteria, this
+	// array points to the set of specified source prefixes.
+	// Unspecified source type matches end up as a wildcard entry here with an
+	// index of 0, which actually represents the source type `ANY`.
+	srcTypeArr [3]*sourcePrefixes
+}
+
+// sourcePrefixes contains source prefix related information specified in the
+// match criteria. These are pointed to by the array of source types.
+type sourcePrefixes struct {
+	// These are very similar to the 'dstPrefixMap' and 'dstPrefixes' field of
+	// FilterChainManager. Go there for more info.
+	srcPrefixMap map[string]*sourcePrefixEntry
+	srcPrefixes  []*sourcePrefixEntry
+}
+
+// sourcePrefixEntry contains match criteria per source prefix.
+type sourcePrefixEntry struct {
+	net *net.IPNet // The actual source prefix.
+	// Mapping from source ports specified in the match criteria to the actual
+	// filter chain. Unspecified source port matches en up as a wildcard entry
+	// here with a key of 0.
+	srcPortMap map[int]*FilterChain
+}
+
+// NewFilterChainManager parses the received Listener resource and builds a
+// FilterChainManager. Returns a non-nil error on validation failures.
+//
+// This function is only exported so that tests outside of this package can
+// create a FilterChainManager.
+func NewFilterChainManager(lis *v3listenerpb.Listener) (*FilterChainManager, error) {
+	// Parse all the filter chains and build the internal data structures.
+	fci := &FilterChainManager{dstPrefixMap: make(map[string]*destPrefixEntry)}
+	if err := fci.addFilterChains(lis.GetFilterChains()); err != nil {
+		return nil, err
+	}
+
+	// Retrieve the default filter chain. The match criteria specified on the
+	// default filter chain is never used. The default filter chain simply gets
+	// used when none of the other filter chains match.
+	var def *FilterChain
+	if dfc := lis.GetDefaultFilterChain(); dfc != nil {
+		var err error
+		if def, err = filterChainFromProto(dfc); err != nil {
+			return nil, err
+		}
+	}
+	fci.def = def
+
+	// If there are no supported filter chains and no default filter chain, we
+	// fail here. This will call the Listener resource to be NACK'ed.
+	if fci.fcCnt == 0 && fci.def == nil {
+		return nil, fmt.Errorf("no supported filter chains and no default filter chain")
+	}
+	return fci, nil
+}
+
+// addFilterChains parses the filter chains in fcs and adds the required
+// internal data structures corresponding to the match criteria.
+func (fci *FilterChainManager) addFilterChains(fcs []*v3listenerpb.FilterChain) error {
+	for _, fc := range fcs {
+		// Skip filter chains with unsupported match fields/criteria.
+		fcm := fc.GetFilterChainMatch()
+		if fcm.GetDestinationPort().GetValue() != 0 ||
+			fcm.GetServerNames() != nil ||
+			(fcm.GetTransportProtocol() != "" && fcm.TransportProtocol != "raw_buffer") ||
+			fcm.GetApplicationProtocols() != nil {
+			continue
+		}
+
+		// Extract the supported match criteria, which will be used by
+		// successive addFilterChainsForXxx() functions.
+		var dstPrefixes []*net.IPNet
+		for _, pr := range fcm.GetPrefixRanges() {
+			cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
+			_, ipnet, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("failed to parse destination prefix range: %+v", pr)
+			}
+			dstPrefixes = append(dstPrefixes, ipnet)
+		}
+
+		var srcType SourceType
+		switch fcm.GetSourceType() {
+		case v3listenerpb.FilterChainMatch_ANY:
+			srcType = SourceTypeAny
+		case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK:
+			srcType = SourceTypeSameOrLoopback
+		case v3listenerpb.FilterChainMatch_EXTERNAL:
+			srcType = SourceTypeExternal
+		default:
+			return fmt.Errorf("unsupported source type: %v", fcm.GetSourceType())
+		}
+
+		var srcPrefixes []*net.IPNet
+		for _, pr := range fcm.GetSourcePrefixRanges() {
+			cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
+			_, ipnet, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("failed to parse source prefix range: %+v", pr)
+			}
+			srcPrefixes = append(srcPrefixes, ipnet)
+		}
+
+		var srcPorts []int
+		for _, port := range fcm.GetSourcePorts() {
+			srcPorts = append(srcPorts, int(port))
+		}
+
+		// Build the internal representation of the filter chain match fields.
+		if err := fci.addFilterChainsForDestPrefixes(dstPrefixes, srcType, srcPrefixes, srcPorts, fc); err != nil {
+			return err
+		}
+		fci.fcCnt++
+	}
+
+	// Build the source and dest prefix slices used by Lookup().
+	for _, dstPrefix := range fci.dstPrefixMap {
+		fci.dstPrefixes = append(fci.dstPrefixes, dstPrefix)
+		for _, st := range dstPrefix.srcTypeArr {
+			if st == nil {
+				continue
+			}
+			for _, srcPrefix := range st.srcPrefixMap {
+				st.srcPrefixes = append(st.srcPrefixes, srcPrefix)
+			}
+		}
+	}
+	return nil
+}
+
+// addFilterChainsForDestPrefixes adds destination prefixes to the internal data
+// structures and delegates control to addFilterChainsForSourceType to continue
+// building the internal data structure.
+func (fci *FilterChainManager) addFilterChainsForDestPrefixes(dstPrefixes []*net.IPNet, srcType SourceType, srcPrefixes []*net.IPNet, srcPorts []int, fc *v3listenerpb.FilterChain) error {
+	if len(dstPrefixes) == 0 {
+		// Use the wildcard IP when destination prefix is unspecified.
+		if fci.dstPrefixMap[emptyAddrMapKey] == nil {
+			fci.dstPrefixMap[emptyAddrMapKey] = &destPrefixEntry{net: zeroIP}
+		}
+		return fci.addFilterChainsForSourceType(fci.dstPrefixMap[emptyAddrMapKey], srcType, srcPrefixes, srcPorts, fc)
+	}
+	for _, prefix := range dstPrefixes {
+		p := prefix.String()
+		if fci.dstPrefixMap[p] == nil {
+			fci.dstPrefixMap[p] = &destPrefixEntry{net: prefix}
+		}
+		if err := fci.addFilterChainsForSourceType(fci.dstPrefixMap[p], srcType, srcPrefixes, srcPorts, fc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// addFilterChainsForSourceType adds source types to the internal data
+// structures and delegates control to addFilterChainsForSourcePrefixes to
+// continue building the internal data structure.
+func (fci *FilterChainManager) addFilterChainsForSourceType(dstEntry *destPrefixEntry, srcType SourceType, srcPrefixes []*net.IPNet, srcPorts []int, fc *v3listenerpb.FilterChain) error {
+	st := int(srcType)
+	if dstEntry.srcTypeArr[st] == nil {
+		dstEntry.srcTypeArr[st] = &sourcePrefixes{srcPrefixMap: make(map[string]*sourcePrefixEntry)}
+	}
+	return fci.addFilterChainsForSourcePrefixes(dstEntry.srcTypeArr[st].srcPrefixMap, srcPrefixes, srcPorts, fc)
+}
+
+// addFilterChainsForSourcePrefixes adds source prefixes to the internal data
+// structures and delegates control to addFilterChainsForSourcePorts to continue
+// building the internal data structure.
+func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map[string]*sourcePrefixEntry, srcPrefixes []*net.IPNet, srcPorts []int, fc *v3listenerpb.FilterChain) error {
+	if len(srcPrefixes) == 0 {
+		// Use the wildcard IP when source prefix is unspecified.
+		if srcPrefixMap[emptyAddrMapKey] == nil {
+			srcPrefixMap[emptyAddrMapKey] = &sourcePrefixEntry{
+				net:        zeroIP,
+				srcPortMap: make(map[int]*FilterChain),
+			}
+		}
+		return fci.addFilterChainsForSourcePorts(srcPrefixMap[emptyAddrMapKey], srcPorts, fc)
+	}
+	for _, prefix := range srcPrefixes {
+		p := prefix.String()
+		if srcPrefixMap[p] == nil {
+			srcPrefixMap[p] = &sourcePrefixEntry{
+				net:        prefix,
+				srcPortMap: make(map[int]*FilterChain),
+			}
+		}
+		if err := fci.addFilterChainsForSourcePorts(srcPrefixMap[p], srcPorts, fc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// addFilterChainsForSourcePorts adds source ports to the internal data
+// structures and completes the process of building the internal data structure.
+// It is here that we determine if there are multiple filter chains with
+// overlapping matching rules.
+func (fci *FilterChainManager) addFilterChainsForSourcePorts(srcEntry *sourcePrefixEntry, srcPorts []int, fcProto *v3listenerpb.FilterChain) error {
+	fc, err := filterChainFromProto(fcProto)
+	if err != nil {
+		return err
+	}
+
+	if len(srcPorts) == 0 {
+		// Use the wildcard port '0', when source ports are unspecified.
+		if curFC := srcEntry.srcPortMap[0]; curFC != nil {
+			return errors.New("multiple filter chains with overlapping matching rules are defined")
+		}
+		srcEntry.srcPortMap[0] = fc
+		return nil
+	}
+	for _, port := range srcPorts {
+		if curFC := srcEntry.srcPortMap[port]; curFC != nil {
+			return errors.New("multiple filter chains with overlapping matching rules are defined")
+		}
+		srcEntry.srcPortMap[port] = fc
+	}
+	return nil
+}
+
+// filterChainFromProto extracts the relevant information from the FilterChain
+// proto and stores it in our internal representation. Currently, we only
+// process the security configuration stored in the transport_socket field.
+func filterChainFromProto(fc *v3listenerpb.FilterChain) (*FilterChain, error) {
+	// If the transport_socket field is not specified, it means that the control
+	// plane has not sent us any security config. This is fine and the server
+	// will use the fallback credentials configured as part of the
+	// xdsCredentials.
+	ts := fc.GetTransportSocket()
+	if ts == nil {
+		return &FilterChain{}, nil
+	}
+	if name := ts.GetName(); name != transportSocketName {
+		return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name)
+	}
+	any := ts.GetTypedConfig()
+	if any == nil || any.TypeUrl != version.V3DownstreamTLSContextURL {
+		return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl)
+	}
+	downstreamCtx := &v3tlspb.DownstreamTlsContext{}
+	if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err)
+	}
+	if downstreamCtx.GetCommonTlsContext() == nil {
+		return nil, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext")
+	}
+	sc, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext())
+	if err != nil {
+		return nil, err
+	}
+	if sc.IdentityInstanceName == "" {
+		return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name")
+	}
+	sc.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue()
+	if sc.RequireClientCert && sc.RootInstanceName == "" {
+		return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set")
+	}
+	return &FilterChain{SecurityCfg: sc}, nil
+}
+
+// FilterChainLookupParams wraps parameters to be passed to Lookup.
+type FilterChainLookupParams struct {
+	// IsUnspecified indicates whether the server is listening on a wildcard
+	// address, "0.0.0.0" for IPv4 and "::" for IPv6. Only when this is set to
+	// true, do we consider the destination prefixes specified in the filter
+	// chain match criteria.
+	IsUnspecifiedListener bool
+	// DestAddr is the local address of an incoming connection.
+	DestAddr net.IP
+	// SourceAddr is the remote address of an incoming connection.
+	SourceAddr net.IP
+	// SourcePort is the remote port of an incoming connection.
+	SourcePort int
+}
+
+// Lookup returns the most specific matching filter chain to be used for an
+// incoming connection on the server side.
+//
+// Returns a non-nil error if no matching filter chain could be found or
+// multiple matching filter chains were found, and in both cases, the incoming
+// connection must be dropped.
+func (fci *FilterChainManager) Lookup(params FilterChainLookupParams) (*FilterChain, error) {
+	dstPrefixes := filterByDestinationPrefixes(fci.dstPrefixes, params.IsUnspecifiedListener, params.DestAddr)
+	if len(dstPrefixes) == 0 {
+		if fci.def != nil {
+			return fci.def, nil
+		}
+		return nil, fmt.Errorf("no matching filter chain based on destination prefix match for %+v", params)
+	}
+
+	srcType := SourceTypeExternal
+	if params.SourceAddr.Equal(params.DestAddr) || params.SourceAddr.IsLoopback() {
+		srcType = SourceTypeSameOrLoopback
+	}
+	srcPrefixes := filterBySourceType(dstPrefixes, srcType)
+	if len(srcPrefixes) == 0 {
+		if fci.def != nil {
+			return fci.def, nil
+		}
+		return nil, fmt.Errorf("no matching filter chain based on source type match for %+v", params)
+	}
+	srcPrefixEntry, err := filterBySourcePrefixes(srcPrefixes, params.SourceAddr)
+	if err != nil {
+		return nil, err
+	}
+	if fc := filterBySourcePorts(srcPrefixEntry, params.SourcePort); fc != nil {
+		return fc, nil
+	}
+	if fci.def != nil {
+		return fci.def, nil
+	}
+	return nil, fmt.Errorf("no matching filter chain after all match criteria for %+v", params)
+}
+
+// filterByDestinationPrefixes is the first stage of the filter chain
+// matching algorithm. It takes the complete set of configured filter chain
+// matchers and returns the most specific matchers based on the destination
+// prefix match criteria (the prefixes which match the most number of bits).
+func filterByDestinationPrefixes(dstPrefixes []*destPrefixEntry, isUnspecified bool, dstAddr net.IP) []*destPrefixEntry {
+	if !isUnspecified {
+		// Destination prefix matchers are considered only when the listener is
+		// bound to the wildcard address.
+		return dstPrefixes
+	}
+	var (
+		matchingDstPrefixes []*destPrefixEntry
+		maxSubnetMatch      int
+	)
+	for _, prefix := range dstPrefixes {
+		if !prefix.net.Contains(dstAddr) {
+			continue
+		}
+		matchSize, _ := prefix.net.Mask.Size()
+		if matchSize < maxSubnetMatch {
+			continue
+		}
+		if matchSize > maxSubnetMatch {
+			maxSubnetMatch = matchSize
+			matchingDstPrefixes = make([]*destPrefixEntry, 0, 1)
+		}
+		matchingDstPrefixes = append(matchingDstPrefixes, prefix)
+	}
+	return matchingDstPrefixes
+}
+
+// filterBySourceType is the second stage of the matching algorithm. It
+// trims the filter chains based on the most specific source type match.
+func filterBySourceType(dstPrefixes []*destPrefixEntry, srcType SourceType) []*sourcePrefixes {
+	var (
+		srcPrefixes      []*sourcePrefixes
+		bestSrcTypeMatch int
+	)
+	for _, prefix := range dstPrefixes {
+		var (
+			srcPrefix *sourcePrefixes
+			match     int
+		)
+		switch srcType {
+		case SourceTypeExternal:
+			match = int(SourceTypeExternal)
+			srcPrefix = prefix.srcTypeArr[match]
+		case SourceTypeSameOrLoopback:
+			match = int(SourceTypeSameOrLoopback)
+			srcPrefix = prefix.srcTypeArr[match]
+		}
+		if srcPrefix == nil {
+			match = int(SourceTypeAny)
+			srcPrefix = prefix.srcTypeArr[match]
+		}
+		if match < bestSrcTypeMatch {
+			continue
+		}
+		if match > bestSrcTypeMatch {
+			bestSrcTypeMatch = match
+			srcPrefixes = make([]*sourcePrefixes, 0)
+		}
+		if srcPrefix != nil {
+			// The source type array always has 3 entries, but these could be
+			// nil if the appropriate source type match was not specified.
+			srcPrefixes = append(srcPrefixes, srcPrefix)
+		}
+	}
+	return srcPrefixes
+}
+
+// filterBySourcePrefixes is the third stage of the filter chain matching
+// algorithm. It trims the filter chains based on the source prefix. At most one
+// filter chain with the most specific match progress to the next stage.
+func filterBySourcePrefixes(srcPrefixes []*sourcePrefixes, srcAddr net.IP) (*sourcePrefixEntry, error) {
+	var (
+		matchingSrcPrefixes []*sourcePrefixEntry
+		maxSubnetMatch      int
+	)
+	for _, sp := range srcPrefixes {
+		for _, prefix := range sp.srcPrefixes {
+			if !prefix.net.Contains(srcAddr) {
+				continue
+			}
+			matchSize, _ := prefix.net.Mask.Size()
+			if matchSize < maxSubnetMatch {
+				continue
+			}
+			if matchSize > maxSubnetMatch {
+				maxSubnetMatch = matchSize
+				matchingSrcPrefixes = make([]*sourcePrefixEntry, 0, 1)
+			}
+			matchingSrcPrefixes = append(matchingSrcPrefixes, prefix)
+		}
+	}
+	if len(matchingSrcPrefixes) == 0 {
+		// Finding no match is not an error condition. The caller will end up
+		// using the default filter chain if one was configured.
+		return nil, nil
+	}
+	// We expect at most a single matching source prefix entry at this point. If
+	// we have multiple entries here, and some of their source port matchers had
+	// wildcard entries, we could be left with more than one matching filter
+	// chain and hence would have been flagged as an invalid configuration at
+	// config validation time.
+	if len(matchingSrcPrefixes) != 1 {
+		return nil, errors.New("multiple matching filter chains")
+	}
+	return matchingSrcPrefixes[0], nil
+}
+
+// filterBySourcePorts is the last stage of the filter chain matching
+// algorithm. It trims the filter chains based on the source ports.
+func filterBySourcePorts(spe *sourcePrefixEntry, srcPort int) *FilterChain {
+	if spe == nil {
+		return nil
+	}
+	// A match could be a wildcard match (this happens when the match
+	// criteria does not specify source ports) or a specific port match (this
+	// happens when the match criteria specifies a set of ports and the source
+	// port of the incoming connection matches one of the specified ports). The
+	// latter is considered to be a more specific match.
+	if fc := spe.srcPortMap[srcPort]; fc != nil {
+		return fc
+	}
+	if fc := spe.srcPortMap[0]; fc != nil {
+		return fc
+	}
+	return nil
+}
diff --git a/xds/internal/client/filter_chain_test.go b/xds/internal/client/filter_chain_test.go
new file mode 100644
index 000000000000..ba0ed944160d
--- /dev/null
+++ b/xds/internal/client/filter_chain_test.go
@@ -0,0 +1,1316 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package client
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	"google.golang.org/grpc/xds/internal/version"
+)
+
+func TestNewFilterChainImpl_Failure_BadMatchFields(t *testing.T) {
+	tests := []struct {
+		desc string
+		lis  *v3listenerpb.Listener
+	}{
+		{
+			desc: "unsupported destination port field",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}},
+					},
+				},
+			},
+		},
+		{
+			desc: "unsupported server names field",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{ServerNames: []string{"example-server"}},
+					},
+				},
+			},
+		},
+		{
+			desc: "unsupported transport protocol ",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{TransportProtocol: "tls"},
+					},
+				},
+			},
+		},
+		{
+			desc: "unsupported application protocol field",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{ApplicationProtocols: []string{"h2"}},
+					},
+				},
+			},
+		},
+		{
+			desc: "bad dest address prefix",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{{AddressPrefix: "a.b.c.d"}}},
+					},
+				},
+			},
+		},
+		{
+			desc: "bad dest prefix length",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.1.1.0", 50)}},
+					},
+				},
+			},
+		},
+		{
+			desc: "bad source address prefix",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{{AddressPrefix: "a.b.c.d"}}},
+					},
+				},
+			},
+		},
+		{
+			desc: "bad source prefix length",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.1.1.0", 50)}},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if fci, err := NewFilterChainManager(test.lis); err == nil {
+				t.Fatalf("NewFilterChainManager() returned %v when expected to fail", fci)
+			}
+		})
+	}
+}
+
+func TestNewFilterChainImpl_Failure_OverlappingMatchingRules(t *testing.T) {
+	tests := []struct {
+		desc string
+		lis  *v3listenerpb.Listener
+	}{
+		{
+			desc: "matching destination prefixes with no other matchers",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16), cidrRangeFromAddressAndPrefixLen("10.0.0.0", 0)},
+						},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.2.2", 16)},
+						},
+					},
+				},
+			},
+		},
+		{
+			desc: "matching source type",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_ANY},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_EXTERNAL},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_EXTERNAL},
+					},
+				},
+			},
+		},
+		{
+			desc: "matching source prefixes",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16), cidrRangeFromAddressAndPrefixLen("10.0.0.0", 0)},
+						},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.2.2", 16)},
+						},
+					},
+				},
+			},
+		},
+		{
+			desc: "matching source ports",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}},
+					},
+				},
+			},
+		},
+	}
+
+	const wantErr = "multiple filter chains with overlapping matching rules are defined"
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			_, err := NewFilterChainManager(test.lis)
+			if err == nil || !strings.Contains(err.Error(), wantErr) {
+				t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: %s", err, wantErr)
+			}
+		})
+	}
+}
+
+func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
+	tests := []struct {
+		desc    string
+		lis     *v3listenerpb.Listener
+		wantErr string
+	}{
+		{
+			desc:    "no filter chains",
+			lis:     &v3listenerpb.Listener{},
+			wantErr: "no supported filter chains and no default filter chain",
+		},
+		{
+			desc: "unexpected transport socket name",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{Name: "unsupported-transport-socket-name"},
+					},
+				},
+			},
+			wantErr: "transport_socket field has unexpected name",
+		},
+		{
+			desc: "unexpected transport socket URL",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.UpstreamTlsContext{}),
+							},
+						},
+					},
+				},
+			},
+			wantErr: "transport_socket field has unexpected typeURL",
+		},
+		{
+			desc: "badly marshaled transport socket",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: &anypb.Any{
+									TypeUrl: version.V3DownstreamTLSContextURL,
+									Value:   []byte{1, 2, 3, 4},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantErr: "failed to unmarshal DownstreamTlsContext in LDS response",
+		},
+		{
+			desc: "missing CommonTlsContext",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{}),
+							},
+						},
+					},
+				},
+			},
+			wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext",
+		},
+		{
+			desc: "unsupported validation context in transport socket",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
+											ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
+												Name: "foo-sds-secret",
+											},
+										},
+									},
+								}),
+							},
+						},
+					},
+				},
+			},
+			wantErr: "validation context contains unexpected type",
+		},
+		{
+			desc: "no root certificate provider with require_client_cert",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    "identityPluginInstance",
+											CertificateName: "identityCertName",
+										},
+									},
+								}),
+							},
+						},
+					},
+				},
+			},
+			wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set",
+		},
+		{
+			desc: "no identity certificate provider",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{},
+								}),
+							},
+						},
+					},
+				},
+			},
+			wantErr: "security configuration on the server-side does not contain identity certificate provider instance name",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			_, err := NewFilterChainManager(test.lis)
+			if err == nil || !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: %s", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
+	tests := []struct {
+		desc   string
+		lis    *v3listenerpb.Listener
+		wantFC *FilterChainManager
+	}{
+		{
+			desc: "empty transport socket",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def:   &FilterChain{},
+				fcCnt: 1,
+			},
+		},
+		{
+			desc: "no validation context",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    "identityPluginInstance",
+											CertificateName: "identityCertName",
+										},
+									},
+								}),
+							},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{
+					TransportSocket: &v3corepb.TransportSocket{
+						Name: "envoy.transport_sockets.tls",
+						ConfigType: &v3corepb.TransportSocket_TypedConfig{
+							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								CommonTlsContext: &v3tlspb.CommonTlsContext{
+									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "defaultIdentityPluginInstance",
+										CertificateName: "defaultIdentityCertName",
+									},
+								},
+							}),
+						},
+					},
+				},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {
+												SecurityCfg: &SecurityConfig{
+													IdentityInstanceName: "identityPluginInstance",
+													IdentityCertName:     "identityCertName",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def: &FilterChain{
+					SecurityCfg: &SecurityConfig{
+						IdentityInstanceName: "defaultIdentityPluginInstance",
+						IdentityCertName:     "defaultIdentityCertName",
+					},
+				},
+				fcCnt: 1,
+			},
+		},
+		{
+			desc: "validation context with certificate provider",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    "identityPluginInstance",
+											CertificateName: "identityCertName",
+										},
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+												InstanceName:    "rootPluginInstance",
+												CertificateName: "rootCertName",
+											},
+										},
+									},
+								}),
+							},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{
+					Name: "default-filter-chain-1",
+					TransportSocket: &v3corepb.TransportSocket{
+						Name: "envoy.transport_sockets.tls",
+						ConfigType: &v3corepb.TransportSocket_TypedConfig{
+							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+								CommonTlsContext: &v3tlspb.CommonTlsContext{
+									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "defaultIdentityPluginInstance",
+										CertificateName: "defaultIdentityCertName",
+									},
+									ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    "defaultRootPluginInstance",
+											CertificateName: "defaultRootCertName",
+										},
+									},
+								},
+							}),
+						},
+					},
+				},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {
+												SecurityCfg: &SecurityConfig{
+													RootInstanceName:     "rootPluginInstance",
+													RootCertName:         "rootCertName",
+													IdentityInstanceName: "identityPluginInstance",
+													IdentityCertName:     "identityCertName",
+													RequireClientCert:    true,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def: &FilterChain{
+					SecurityCfg: &SecurityConfig{
+						RootInstanceName:     "defaultRootPluginInstance",
+						RootCertName:         "defaultRootCertName",
+						IdentityInstanceName: "defaultIdentityPluginInstance",
+						IdentityCertName:     "defaultIdentityCertName",
+						RequireClientCert:    true,
+					},
+				},
+				fcCnt: 1,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotFC, err := NewFilterChainManager(test.lis)
+			if err != nil {
+				t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err)
+			}
+			if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{})) {
+				t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC)
+			}
+		})
+	}
+}
+
+func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
+	tests := []struct {
+		desc   string
+		lis    *v3listenerpb.Listener
+		wantFC *FilterChainManager
+	}{
+		{
+			desc: "multiple destination prefixes",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"10.0.0.0/8": {
+						net: ipNetFromCIDR("10.0.0.0/8"),
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def:   &FilterChain{},
+				fcCnt: 3,
+			},
+		},
+		{
+			desc: "multiple source types",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							SourceType:   v3listenerpb.FilterChainMatch_EXTERNAL,
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							nil,
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+						srcTypeArr: [3]*sourcePrefixes{
+							nil,
+							nil,
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def:   &FilterChain{},
+				fcCnt: 2,
+			},
+		},
+		{
+			desc: "multiple source prefixes",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:       []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"10.0.0.0/8": {
+										net: ipNetFromCIDR("10.0.0.0/8"),
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"192.168.0.0/16": {
+										net: ipNetFromCIDR("192.168.0.0/16"),
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def:   &FilterChain{},
+				fcCnt: 2,
+			},
+		},
+		{
+			desc: "multiple source ports",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:       []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							SourceType:         v3listenerpb.FilterChainMatch_EXTERNAL,
+							SourcePorts:        []uint32{1, 2, 3},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											1: {},
+											2: {},
+											3: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+						srcTypeArr: [3]*sourcePrefixes{
+							nil,
+							nil,
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"192.168.0.0/16": {
+										net: ipNetFromCIDR("192.168.0.0/16"),
+										srcPortMap: map[int]*FilterChain{
+											1: {},
+											2: {},
+											3: {},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				def:   &FilterChain{},
+				fcCnt: 2,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotFC, err := NewFilterChainManager(test.lis)
+			if err != nil {
+				t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err)
+			}
+			if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{})) {
+				t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC)
+			}
+		})
+	}
+}
+
+func TestLookup_Failures(t *testing.T) {
+	tests := []struct {
+		desc    string
+		lis     *v3listenerpb.Listener
+		params  FilterChainLookupParams
+		wantErr string
+	}{
+		{
+			desc: "no destination prefix match",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
+					},
+				},
+			},
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(10, 1, 1, 1),
+			},
+			wantErr: "no matching filter chain based on destination prefix match",
+		},
+		{
+			desc: "no source type match",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							SourceType:   v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+						},
+					},
+				},
+			},
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 100, 1),
+				SourceAddr:            net.IPv4(192, 168, 100, 2),
+			},
+			wantErr: "no matching filter chain based on source type match",
+		},
+		{
+			desc: "no source prefix match",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)},
+							SourceType:         v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+						},
+					},
+				},
+			},
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 100, 1),
+				SourceAddr:            net.IPv4(192, 168, 100, 1),
+			},
+			wantErr: "no matching filter chain after all match criteria",
+		},
+		{
+			desc: "multiple matching filter chains",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							SourcePorts:  []uint32{1},
+						},
+					},
+				},
+			},
+			params: FilterChainLookupParams{
+				// IsUnspecified is not set. This means that the destination
+				// prefix matchers will be ignored.
+				DestAddr:   net.IPv4(192, 168, 100, 1),
+				SourceAddr: net.IPv4(192, 168, 100, 1),
+				SourcePort: 1,
+			},
+			wantErr: "multiple matching filter chains",
+		},
+		{
+			desc: "no default filter chain",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}},
+					},
+				},
+			},
+			params: FilterChainLookupParams{
+				DestAddr:   net.IPv4(192, 168, 100, 1),
+				SourceAddr: net.IPv4(192, 168, 100, 1),
+				SourcePort: 80,
+			},
+			wantErr: "no matching filter chain after all match criteria",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			fci, err := NewFilterChainManager(test.lis)
+			if err != nil {
+				t.Fatalf("NewFilterChainManager() failed: %v", err)
+			}
+			fc, err := fci.Lookup(test.params)
+			if err == nil || !strings.Contains(err.Error(), test.wantErr) {
+				t.Fatalf("FilterChainManager.Lookup(%v) = (%v, %v) want (nil, %s)", test.params, fc, err, test.wantErr)
+			}
+		})
+	}
+}
+
+func TestLookup_Successes(t *testing.T) {
+	lisWithDefaultChain := &v3listenerpb.Listener{
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "instance1"},
+							},
+						}),
+					},
+				},
+			},
+		},
+		// A default filter chain with an empty transport socket.
+		DefaultFilterChain: &v3listenerpb.FilterChain{
+			TransportSocket: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "default"},
+						},
+					}),
+				},
+			},
+		},
+	}
+	lisWithoutDefaultChain := &v3listenerpb.Listener{
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				TransportSocket: transportSocketWithInstanceName("wildcard"),
+			},
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)},
+					SourceType:   v3listenerpb.FilterChainMatch_EXTERNAL,
+				},
+				TransportSocket: transportSocketWithInstanceName("any-destination-prefix"),
+			},
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
+				TransportSocket:  transportSocketWithInstanceName("specific-destination-prefix-wildcard-source-type"),
+			},
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)},
+					SourceType:   v3listenerpb.FilterChainMatch_EXTERNAL,
+				},
+				TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-specific-source-type"),
+			},
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges:       []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)},
+					SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.92.1", 24)},
+					SourceType:         v3listenerpb.FilterChainMatch_EXTERNAL,
+				},
+				TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-specific-source-type-specific-source-prefix"),
+			},
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges:       []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)},
+					SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.92.1", 24)},
+					SourceType:         v3listenerpb.FilterChainMatch_EXTERNAL,
+					SourcePorts:        []uint32{80},
+				},
+				TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port"),
+			},
+		},
+	}
+
+	tests := []struct {
+		desc   string
+		lis    *v3listenerpb.Listener
+		params FilterChainLookupParams
+		wantFC *FilterChain
+	}{
+		{
+			desc: "default filter chain",
+			lis:  lisWithDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(10, 1, 1, 1),
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "default"}},
+		},
+		{
+			desc: "wildcard destination match",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(10, 1, 1, 1),
+				SourceAddr:            net.IPv4(10, 1, 1, 1),
+				SourcePort:            1,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "wildcard"}},
+		},
+		{
+			desc: "ANY destination match",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(10, 1, 1, 1),
+				SourceAddr:            net.IPv4(10, 1, 1, 2),
+				SourcePort:            1,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "any-destination-prefix"}},
+		},
+		{
+			desc: "specific destination and wildcard source type match",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 100, 1),
+				SourceAddr:            net.IPv4(192, 168, 100, 1),
+				SourcePort:            80,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-wildcard-source-type"}},
+		},
+		{
+			desc: "specific destination and source type match",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 1, 1),
+				SourceAddr:            net.IPv4(10, 1, 1, 1),
+				SourcePort:            80,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type"}},
+		},
+		{
+			desc: "specific destination source type and source prefix",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 1, 1),
+				SourceAddr:            net.IPv4(192, 168, 92, 100),
+				SourcePort:            70,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type-specific-source-prefix"}},
+		},
+		{
+			desc: "specific destination source type source prefix and source port",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 1, 1),
+				SourceAddr:            net.IPv4(192, 168, 92, 100),
+				SourcePort:            80,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port"}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			fci, err := NewFilterChainManager(test.lis)
+			if err != nil {
+				t.Fatalf("NewFilterChainManager() failed: %v", err)
+			}
+			gotFC, err := fci.Lookup(test.params)
+			if err != nil {
+				t.Fatalf("FilterChainManager.Lookup(%v) failed: %v", test.params, err)
+			}
+			if !cmp.Equal(gotFC, test.wantFC) {
+				t.Fatalf("FilterChainManager.Lookup(%v) = %v, want %v", test.params, gotFC, test.wantFC)
+			}
+		})
+	}
+}
+
+// The Equal() methods defined below help with using cmp.Equal() on these types
+// which contain all unexported fields.
+
+func (fci *FilterChainManager) Equal(other *FilterChainManager) bool {
+	if (fci == nil) != (other == nil) {
+		return false
+	}
+	if fci == nil {
+		return true
+	}
+	switch {
+	case !cmp.Equal(fci.dstPrefixMap, other.dstPrefixMap):
+		return false
+	case !cmp.Equal(fci.def, other.def):
+		return false
+	case fci.fcCnt != other.fcCnt:
+		return false
+	}
+	return true
+}
+
+func (dpe *destPrefixEntry) Equal(other *destPrefixEntry) bool {
+	if (dpe == nil) != (other == nil) {
+		return false
+	}
+	if dpe == nil {
+		return true
+	}
+	if !cmp.Equal(dpe.net, other.net) {
+		return false
+	}
+	for i, st := range dpe.srcTypeArr {
+		if !cmp.Equal(st, other.srcTypeArr[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+func (sp *sourcePrefixes) Equal(other *sourcePrefixes) bool {
+	if (sp == nil) != (other == nil) {
+		return false
+	}
+	if sp == nil {
+		return true
+	}
+	return cmp.Equal(sp.srcPrefixMap, other.srcPrefixMap)
+}
+
+func (spe *sourcePrefixEntry) Equal(other *sourcePrefixEntry) bool {
+	if (spe == nil) != (other == nil) {
+		return false
+	}
+	if spe == nil {
+		return true
+	}
+	switch {
+	case !cmp.Equal(spe.net, other.net):
+		return false
+	case !cmp.Equal(spe.srcPortMap, other.srcPortMap):
+		return false
+	}
+	return true
+}
+
+// The String() methods defined below help with debugging test failures as the
+// regular %v or %+v formatting directives do not expands pointer fields inside
+// structs, and these types have a lot of pointers pointing to other structs.
+func (fci *FilterChainManager) String() string {
+	if fci == nil {
+		return ""
+	}
+
+	var sb strings.Builder
+	if fci.dstPrefixMap != nil {
+		sb.WriteString("destination_prefix_map: map {\n")
+		for k, v := range fci.dstPrefixMap {
+			sb.WriteString(fmt.Sprintf("%q: %v\n", k, v))
+		}
+		sb.WriteString("}\n")
+	}
+	if fci.dstPrefixes != nil {
+		sb.WriteString("destination_prefixes: [")
+		for _, p := range fci.dstPrefixes {
+			sb.WriteString(fmt.Sprintf("%v ", p))
+		}
+		sb.WriteString("]")
+	}
+	if fci.def != nil {
+		sb.WriteString(fmt.Sprintf("default_filter_chain: %+v ", fci.def))
+	}
+	sb.WriteString(fmt.Sprintf("filter_chain_count: %d ", fci.fcCnt))
+	return sb.String()
+}
+
+func (dpe *destPrefixEntry) String() string {
+	if dpe == nil {
+		return ""
+	}
+	var sb strings.Builder
+	if dpe.net != nil {
+		sb.WriteString(fmt.Sprintf("destination_prefix: %s ", dpe.net.String()))
+	}
+	sb.WriteString("source_types_array: [")
+	for _, st := range dpe.srcTypeArr {
+		sb.WriteString(fmt.Sprintf("%v ", st))
+	}
+	sb.WriteString("]")
+	return sb.String()
+}
+
+func (sp *sourcePrefixes) String() string {
+	if sp == nil {
+		return ""
+	}
+	var sb strings.Builder
+	if sp.srcPrefixMap != nil {
+		sb.WriteString("source_prefix_map: map {")
+		for k, v := range sp.srcPrefixMap {
+			sb.WriteString(fmt.Sprintf("%q: %v ", k, v))
+		}
+		sb.WriteString("}")
+	}
+	if sp.srcPrefixes != nil {
+		sb.WriteString("source_prefixes: [")
+		for _, p := range sp.srcPrefixes {
+			sb.WriteString(fmt.Sprintf("%v ", p))
+		}
+		sb.WriteString("]")
+	}
+	return sb.String()
+}
+
+func (spe *sourcePrefixEntry) String() string {
+	if spe == nil {
+		return ""
+	}
+	var sb strings.Builder
+	if spe.net != nil {
+		sb.WriteString(fmt.Sprintf("source_prefix: %s ", spe.net.String()))
+	}
+	if spe.srcPortMap != nil {
+		sb.WriteString("source_ports_map: map {")
+		for k, v := range spe.srcPortMap {
+			sb.WriteString(fmt.Sprintf("%d: %+v ", k, v))
+		}
+		sb.WriteString("}")
+	}
+	return sb.String()
+}
+
+func (f *FilterChain) String() string {
+	if f == nil || f.SecurityCfg == nil {
+		return ""
+	}
+	return fmt.Sprintf("security_config: %v", f.SecurityCfg)
+}
+
+func ipNetFromCIDR(cidr string) *net.IPNet {
+	_, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		panic(err)
+	}
+	return ipnet
+}
+
+func transportSocketWithInstanceName(name string) *v3corepb.TransportSocket {
+	return &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: name},
+				},
+			}),
+		},
+	}
+}
+
+func cidrRangeFromAddressAndPrefixLen(address string, len int) *v3corepb.CidrRange {
+	return &v3corepb.CidrRange{
+		AddressPrefix: address,
+		PrefixLen: &wrapperspb.UInt32Value{
+			Value: uint32(len),
+		},
+	}
+}
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index da0e0f956156..42421f66b2d1 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -20,7 +20,6 @@ package client
 
 import (
 	"fmt"
-	"net"
 	"strings"
 	"testing"
 	"time"
@@ -782,181 +781,132 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 	const v3LDSTarget = "grpc/server?xds.resource.listening_address=0.0.0.0:9999"
 
 	var (
-		listenerEmptyTransportSocket = &anypb.Any{
-			TypeUrl: version.V3ListenerURL,
-			Value: func() []byte {
-				lis := &v3listenerpb.Listener{
-					Name: v3LDSTarget,
-					Address: &v3corepb.Address{
-						Address: &v3corepb.Address_SocketAddress{
-							SocketAddress: &v3corepb.SocketAddress{
-								Address: "0.0.0.0",
-								PortSpecifier: &v3corepb.SocketAddress_PortValue{
-									PortValue: 9999,
-								},
-							},
-						},
-					},
-					FilterChains: []*v3listenerpb.FilterChain{
-						{
-							Name: "filter-chain-1",
+		listenerEmptyTransportSocket = marshalAny(&v3listenerpb.Listener{
+			Name: v3LDSTarget,
+			Address: &v3corepb.Address{
+				Address: &v3corepb.Address_SocketAddress{
+					SocketAddress: &v3corepb.SocketAddress{
+						Address: "0.0.0.0",
+						PortSpecifier: &v3corepb.SocketAddress_PortValue{
+							PortValue: 9999,
 						},
 					},
-				}
-				mLis, _ := proto.Marshal(lis)
-				return mLis
-			}(),
-		}
-		listenerNoValidationContext = &anypb.Any{
-			TypeUrl: version.V3ListenerURL,
-			Value: func() []byte {
-				lis := &v3listenerpb.Listener{
-					Name: v3LDSTarget,
-					Address: &v3corepb.Address{
-						Address: &v3corepb.Address_SocketAddress{
-							SocketAddress: &v3corepb.SocketAddress{
-								Address: "0.0.0.0",
-								PortSpecifier: &v3corepb.SocketAddress_PortValue{
-									PortValue: 9999,
-								},
-							},
+				},
+			},
+			FilterChains: []*v3listenerpb.FilterChain{
+				{
+					Name: "filter-chain-1",
+				},
+			},
+		})
+		listenerNoValidationContext = marshalAny(&v3listenerpb.Listener{
+			Name: v3LDSTarget,
+			Address: &v3corepb.Address{
+				Address: &v3corepb.Address_SocketAddress{
+					SocketAddress: &v3corepb.SocketAddress{
+						Address: "0.0.0.0",
+						PortSpecifier: &v3corepb.SocketAddress_PortValue{
+							PortValue: 9999,
 						},
 					},
-					FilterChains: []*v3listenerpb.FilterChain{
-						{
-							Name: "filter-chain-1",
-							TransportSocket: &v3corepb.TransportSocket{
-								Name: "envoy.transport_sockets.tls",
-								ConfigType: &v3corepb.TransportSocket_TypedConfig{
-									TypedConfig: &anypb.Any{
-										TypeUrl: version.V3DownstreamTLSContextURL,
-										Value: func() []byte {
-											tls := &v3tlspb.DownstreamTlsContext{
-												CommonTlsContext: &v3tlspb.CommonTlsContext{
-													TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-														InstanceName:    "identityPluginInstance",
-														CertificateName: "identityCertName",
-													},
-												},
-											}
-											mtls, _ := proto.Marshal(tls)
-											return mtls
-										}(),
+				},
+			},
+			FilterChains: []*v3listenerpb.FilterChain{
+				{
+					Name: "filter-chain-1",
+					TransportSocket: &v3corepb.TransportSocket{
+						Name: "envoy.transport_sockets.tls",
+						ConfigType: &v3corepb.TransportSocket_TypedConfig{
+							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								CommonTlsContext: &v3tlspb.CommonTlsContext{
+									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "identityPluginInstance",
+										CertificateName: "identityCertName",
 									},
 								},
-							},
+							}),
 						},
 					},
-					DefaultFilterChain: &v3listenerpb.FilterChain{
-						Name: "default-filter-chain-1",
-						TransportSocket: &v3corepb.TransportSocket{
-							Name: "envoy.transport_sockets.tls",
-							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: &anypb.Any{
-									TypeUrl: version.V3DownstreamTLSContextURL,
-									Value: func() []byte {
-										tls := &v3tlspb.DownstreamTlsContext{
-											CommonTlsContext: &v3tlspb.CommonTlsContext{
-												TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    "defaultIdentityPluginInstance",
-													CertificateName: "defaultIdentityCertName",
-												},
-											},
-										}
-										mtls, _ := proto.Marshal(tls)
-										return mtls
-									}(),
+				},
+			},
+			DefaultFilterChain: &v3listenerpb.FilterChain{
+				Name: "default-filter-chain-1",
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    "defaultIdentityPluginInstance",
+									CertificateName: "defaultIdentityCertName",
 								},
 							},
-						},
+						}),
 					},
-				}
-				mLis, _ := proto.Marshal(lis)
-				return mLis
-			}(),
-		}
-		listenerWithValidationContext = &anypb.Any{
-			TypeUrl: version.V3ListenerURL,
-			Value: func() []byte {
-				lis := &v3listenerpb.Listener{
-					Name: v3LDSTarget,
-					Address: &v3corepb.Address{
-						Address: &v3corepb.Address_SocketAddress{
-							SocketAddress: &v3corepb.SocketAddress{
-								Address: "0.0.0.0",
-								PortSpecifier: &v3corepb.SocketAddress_PortValue{
-									PortValue: 9999,
-								},
-							},
+				},
+			},
+		})
+		listenerWithValidationContext = marshalAny(&v3listenerpb.Listener{
+			Name: v3LDSTarget,
+			Address: &v3corepb.Address{
+				Address: &v3corepb.Address_SocketAddress{
+					SocketAddress: &v3corepb.SocketAddress{
+						Address: "0.0.0.0",
+						PortSpecifier: &v3corepb.SocketAddress_PortValue{
+							PortValue: 9999,
 						},
 					},
-					FilterChains: []*v3listenerpb.FilterChain{
-						{
-							Name: "filter-chain-1",
-							TransportSocket: &v3corepb.TransportSocket{
-								Name: "envoy.transport_sockets.tls",
-								ConfigType: &v3corepb.TransportSocket_TypedConfig{
-									TypedConfig: &anypb.Any{
-										TypeUrl: version.V3DownstreamTLSContextURL,
-										Value: func() []byte {
-											tls := &v3tlspb.DownstreamTlsContext{
-												RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-												CommonTlsContext: &v3tlspb.CommonTlsContext{
-													TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-														InstanceName:    "identityPluginInstance",
-														CertificateName: "identityCertName",
-													},
-													ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-														ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-															InstanceName:    "rootPluginInstance",
-															CertificateName: "rootCertName",
-														},
-													},
-												},
-											}
-											mtls, _ := proto.Marshal(tls)
-											return mtls
-										}(),
+				},
+			},
+			FilterChains: []*v3listenerpb.FilterChain{
+				{
+					Name: "filter-chain-1",
+					TransportSocket: &v3corepb.TransportSocket{
+						Name: "envoy.transport_sockets.tls",
+						ConfigType: &v3corepb.TransportSocket_TypedConfig{
+							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+								CommonTlsContext: &v3tlspb.CommonTlsContext{
+									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "identityPluginInstance",
+										CertificateName: "identityCertName",
+									},
+									ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    "rootPluginInstance",
+											CertificateName: "rootCertName",
+										},
 									},
 								},
-							},
+							}),
 						},
 					},
-					DefaultFilterChain: &v3listenerpb.FilterChain{
-						Name: "default-filter-chain-1",
-						TransportSocket: &v3corepb.TransportSocket{
-							Name: "envoy.transport_sockets.tls",
-							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: &anypb.Any{
-									TypeUrl: version.V3DownstreamTLSContextURL,
-									Value: func() []byte {
-										tls := &v3tlspb.DownstreamTlsContext{
-											RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-											CommonTlsContext: &v3tlspb.CommonTlsContext{
-												TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    "defaultIdentityPluginInstance",
-													CertificateName: "defaultIdentityCertName",
-												},
-												ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-													ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-														InstanceName:    "defaultRootPluginInstance",
-														CertificateName: "defaultRootCertName",
-													},
-												},
-											},
-										}
-										mtls, _ := proto.Marshal(tls)
-										return mtls
-									}(),
+				},
+			},
+			DefaultFilterChain: &v3listenerpb.FilterChain{
+				Name: "default-filter-chain-1",
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    "defaultIdentityPluginInstance",
+									CertificateName: "defaultIdentityCertName",
+								},
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    "defaultRootPluginInstance",
+										CertificateName: "defaultRootCertName",
+									},
 								},
 							},
-						},
+						}),
 					},
-				}
-				mLis, _ := proto.Marshal(lis)
-				return mLis
-			}(),
-		}
+				},
+			},
+		})
 	)
 
 	const testVersion = "test-version-lds-server"
@@ -1073,6 +1023,92 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			},
 			wantErr: "no socket_address field in LDS response",
 		},
+		{
+			name: "no filter chains and no default filter chain",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: "no supported filter chains and no default filter chain",
+		},
+		{
+			name: "overlapping filter chain match criteria",
+			resources: []*anypb.Any{
+				{
+					TypeUrl: version.V3ListenerURL,
+					Value: func() []byte {
+						lis := &v3listenerpb.Listener{
+							Name: v3LDSTarget,
+							Address: &v3corepb.Address{
+								Address: &v3corepb.Address_SocketAddress{
+									SocketAddress: &v3corepb.SocketAddress{
+										Address: "0.0.0.0",
+										PortSpecifier: &v3corepb.SocketAddress_PortValue{
+											PortValue: 9999,
+										},
+									},
+								},
+							},
+							FilterChains: []*v3listenerpb.FilterChain{
+								{
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}},
+								},
+								{
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{},
+								},
+								{
+									FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}},
+								},
+							},
+						}
+						mLis, _ := proto.Marshal(lis)
+						return mLis
+					}(),
+				},
+			},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD: UpdateMetadata{
+				Status:  ServiceStatusNACKed,
+				Version: testVersion,
+				ErrState: &UpdateErrorMetadata{
+					Version: testVersion,
+					Err:     errPlaceHolder,
+				},
+			},
+			wantErr: "multiple filter chains with overlapping matching rules are defined",
+		},
 		{
 			name: "unexpected transport socket name",
 			resources: []*anypb.Any{
@@ -1333,9 +1369,28 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 			wantUpdate: map[string]ListenerUpdate{
 				v3LDSTarget: {
 					InboundListenerCfg: &InboundListenerConfig{
-						Address:      "0.0.0.0",
-						Port:         "9999",
-						FilterChains: []*FilterChain{{Match: &FilterChainMatch{}}},
+						Address: "0.0.0.0",
+						Port:    "9999",
+						FilterChains: &FilterChainManager{
+							dstPrefixMap: map[string]*destPrefixEntry{
+								"0.0.0.0/0": {
+									net: zeroIP,
+									srcTypeArr: [3]*sourcePrefixes{
+										{
+											srcPrefixMap: map[string]*sourcePrefixEntry{
+												"0.0.0.0/0": {
+													net: zeroIP,
+													srcPortMap: map[int]*FilterChain{
+														0: {},
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+							fcCnt: 1,
+						},
 					},
 					Raw: listenerEmptyTransportSocket,
 				},
@@ -1469,21 +1524,36 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					InboundListenerCfg: &InboundListenerConfig{
 						Address: "0.0.0.0",
 						Port:    "9999",
-						FilterChains: []*FilterChain{
-							{
-								Match: &FilterChainMatch{},
-								SecurityCfg: &SecurityConfig{
-									IdentityInstanceName: "identityPluginInstance",
-									IdentityCertName:     "identityCertName",
+						FilterChains: &FilterChainManager{
+							dstPrefixMap: map[string]*destPrefixEntry{
+								"0.0.0.0/0": {
+									net: zeroIP,
+									srcTypeArr: [3]*sourcePrefixes{
+										{
+											srcPrefixMap: map[string]*sourcePrefixEntry{
+												"0.0.0.0/0": {
+													net: zeroIP,
+													srcPortMap: map[int]*FilterChain{
+														0: {
+															SecurityCfg: &SecurityConfig{
+																IdentityInstanceName: "identityPluginInstance",
+																IdentityCertName:     "identityCertName",
+															},
+														},
+													},
+												},
+											},
+										},
+									},
 								},
 							},
-						},
-						DefaultFilterChain: &FilterChain{
-							Match: &FilterChainMatch{},
-							SecurityCfg: &SecurityConfig{
-								IdentityInstanceName: "defaultIdentityPluginInstance",
-								IdentityCertName:     "defaultIdentityCertName",
+							def: &FilterChain{
+								SecurityCfg: &SecurityConfig{
+									IdentityInstanceName: "defaultIdentityPluginInstance",
+									IdentityCertName:     "defaultIdentityCertName",
+								},
 							},
+							fcCnt: 1,
 						},
 					},
 					Raw: listenerNoValidationContext,
@@ -1502,27 +1572,42 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 					InboundListenerCfg: &InboundListenerConfig{
 						Address: "0.0.0.0",
 						Port:    "9999",
-						FilterChains: []*FilterChain{
-							{
-								Match: &FilterChainMatch{},
+						FilterChains: &FilterChainManager{
+							dstPrefixMap: map[string]*destPrefixEntry{
+								"0.0.0.0/0": {
+									net: zeroIP,
+									srcTypeArr: [3]*sourcePrefixes{
+										{
+											srcPrefixMap: map[string]*sourcePrefixEntry{
+												"0.0.0.0/0": {
+													net: zeroIP,
+													srcPortMap: map[int]*FilterChain{
+														0: {
+															SecurityCfg: &SecurityConfig{
+																RootInstanceName:     "rootPluginInstance",
+																RootCertName:         "rootCertName",
+																IdentityInstanceName: "identityPluginInstance",
+																IdentityCertName:     "identityCertName",
+																RequireClientCert:    true,
+															},
+														},
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+							def: &FilterChain{
 								SecurityCfg: &SecurityConfig{
-									RootInstanceName:     "rootPluginInstance",
-									RootCertName:         "rootCertName",
-									IdentityInstanceName: "identityPluginInstance",
-									IdentityCertName:     "identityCertName",
+									RootInstanceName:     "defaultRootPluginInstance",
+									RootCertName:         "defaultRootCertName",
+									IdentityInstanceName: "defaultIdentityPluginInstance",
+									IdentityCertName:     "defaultIdentityCertName",
 									RequireClientCert:    true,
 								},
 							},
-						},
-						DefaultFilterChain: &FilterChain{
-							Match: &FilterChainMatch{},
-							SecurityCfg: &SecurityConfig{
-								RootInstanceName:     "defaultRootPluginInstance",
-								RootCertName:         "defaultRootCertName",
-								IdentityInstanceName: "defaultIdentityPluginInstance",
-								IdentityCertName:     "defaultIdentityCertName",
-								RequireClientCert:    true,
-							},
+							fcCnt: 1,
 						},
 					},
 					Raw: listenerWithValidationContext,
@@ -1554,389 +1639,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 	}
 }
 
-func (s) TestGetFilterChain(t *testing.T) {
-	tests := []struct {
-		desc             string
-		inputFilterChain *v3listenerpb.FilterChain
-		wantFilterChain  *FilterChain
-		wantErr          bool
-	}{
-		{
-			desc:             "empty",
-			inputFilterChain: nil,
-		},
-		{
-			desc: "unsupported destination port",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					DestinationPort: &wrapperspb.UInt32Value{
-						Value: 666,
-					},
-				},
-			},
-		},
-		{
-			desc: "unsupported server names",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					ServerNames: []string{"example-server"},
-				},
-			},
-		},
-		{
-			desc: "unsupported transport protocol",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					TransportProtocol: "tls",
-				},
-			},
-		},
-		{
-			desc: "unsupported application protocol",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					ApplicationProtocols: []string{"h2"},
-				},
-			},
-		},
-		{
-			desc: "bad dest address prefix",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "a.b.c.d",
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "bad dest prefix length",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "10.1.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 50,
-							},
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "dest prefix ranges",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "10.1.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 8,
-							},
-						},
-						{
-							AddressPrefix: "192.168.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 24,
-							},
-						},
-					},
-				},
-			},
-			wantFilterChain: &FilterChain{
-				Match: &FilterChainMatch{
-					DestPrefixRanges: []net.IP{
-						net.IPv4(10, 1, 1, 0),
-						net.IPv4(192, 168, 1, 0),
-					},
-				},
-			},
-		},
-		{
-			desc: "source type local",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
-				},
-			},
-			wantFilterChain: &FilterChain{
-				Match: &FilterChainMatch{
-					SourceType: SourceTypeSameOrLoopback,
-				},
-			},
-		},
-		{
-			desc: "source type external",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					SourceType: v3listenerpb.FilterChainMatch_EXTERNAL,
-				},
-			},
-			wantFilterChain: &FilterChain{
-				Match: &FilterChainMatch{
-					SourceType: SourceTypeExternal,
-				},
-			},
-		},
-		{
-			desc: "source type any",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					SourceType: v3listenerpb.FilterChainMatch_ANY,
-				},
-			},
-			wantFilterChain: &FilterChain{
-				Match: &FilterChainMatch{
-					SourceType: SourceTypeAny,
-				},
-			},
-		},
-		{
-			desc: "bad source address prefix",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					SourcePrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "a.b.c.d",
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "bad source prefix length",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					SourcePrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "10.1.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 50,
-							},
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "source prefix ranges",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					SourcePrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "10.1.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 8,
-							},
-						},
-						{
-							AddressPrefix: "192.168.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 24,
-							},
-						},
-					},
-				},
-			},
-			wantFilterChain: &FilterChain{
-				Match: &FilterChainMatch{
-					SourcePrefixRanges: []net.IP{
-						net.IPv4(10, 1, 1, 0),
-						net.IPv4(192, 168, 1, 0),
-					},
-				},
-			},
-		},
-		{
-			desc:             "empty transport socket",
-			inputFilterChain: &v3listenerpb.FilterChain{},
-			wantFilterChain:  &FilterChain{Match: &FilterChainMatch{}},
-		},
-		{
-			desc: "bad transport socket name",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "unsupported-transport-socket-name",
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "unexpected url in transport socket",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.UpstreamTlsContext{}),
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "badly marshaled downstream tls context",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3DownstreamTLSContextURL,
-							Value:   []byte{1, 2, 3, 4},
-						},
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "missing common tls context",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{}),
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "unsupported validation context",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
-							CommonTlsContext: &v3tlspb.CommonTlsContext{
-								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
-									ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
-										Name: "foo-sds-secret",
-									},
-								},
-							},
-						}),
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "no identity and root certificate providers",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
-							RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-							CommonTlsContext: &v3tlspb.CommonTlsContext{
-								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-									InstanceName:    "identityPluginInstance",
-									CertificateName: "identityCertName",
-								},
-							},
-						}),
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "no identity certificate provider with require_client_cert",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
-							CommonTlsContext: &v3tlspb.CommonTlsContext{},
-						}),
-					},
-				},
-			},
-			wantErr: true,
-		},
-		{
-			desc: "happy case",
-			inputFilterChain: &v3listenerpb.FilterChain{
-				Name: "filter-chain-1",
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "10.1.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 8,
-							},
-						},
-					},
-					SourceType: v3listenerpb.FilterChainMatch_EXTERNAL,
-					SourcePrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "10.1.1.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: 8,
-							},
-						},
-					},
-					SourcePorts: []uint32{80, 8080},
-				},
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
-							RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-							CommonTlsContext: &v3tlspb.CommonTlsContext{
-								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-									InstanceName:    "identityPluginInstance",
-									CertificateName: "identityCertName",
-								},
-								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-										InstanceName:    "rootPluginInstance",
-										CertificateName: "rootCertName",
-									},
-								},
-							},
-						}),
-					},
-				},
-			},
-			wantFilterChain: &FilterChain{
-				Match: &FilterChainMatch{
-					DestPrefixRanges:   []net.IP{net.IPv4(10, 1, 1, 0)},
-					SourceType:         SourceTypeExternal,
-					SourcePrefixRanges: []net.IP{net.IPv4(10, 1, 1, 0)},
-					SourcePorts:        []uint32{80, 8080},
-				},
-				SecurityCfg: &SecurityConfig{
-					RootInstanceName:     "rootPluginInstance",
-					RootCertName:         "rootCertName",
-					IdentityInstanceName: "identityPluginInstance",
-					IdentityCertName:     "identityCertName",
-					RequireClientCert:    true,
-				},
-			},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.desc, func(t *testing.T) {
-			gotFilterChain, gotErr := getFilterChain(test.inputFilterChain)
-			if (gotErr != nil) != test.wantErr {
-				t.Fatalf("getFilterChain(%+v) returned error: %v, wantErr: %v", test.inputFilterChain, gotErr, test.wantErr)
-			}
-			if diff := cmp.Diff(test.wantFilterChain, gotFilterChain); diff != "" {
-				t.Errorf("getFilterChain(%+v) returned unexpected, diff (-want +got):\n%s", test.inputFilterChain, diff)
-			}
-		})
-	}
-}
-
 type filterConfig struct {
 	httpfilter.FilterConfig
 	Cfg      proto.Message
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index fc1112e180bc..9b957399983c 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -270,120 +270,14 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 		},
 	}
 
-	var filterChains []*FilterChain
-	for _, fc := range lis.GetFilterChains() {
-		filterChain, err := getFilterChain(fc)
-		if err != nil {
-			return nil, err
-		}
-		filterChains = append(filterChains, filterChain)
-	}
-	defaultFilterChain, err := getFilterChain(lis.GetDefaultFilterChain())
+	fcMgr, err := NewFilterChainManager(lis)
 	if err != nil {
 		return nil, err
 	}
-	if len(filterChains) == 0 && defaultFilterChain == nil {
-		return nil, fmt.Errorf("xds: no supported filter chains and no default filter chain")
-	}
-	lu.InboundListenerCfg.FilterChains = filterChains
-	lu.InboundListenerCfg.DefaultFilterChain = defaultFilterChain
+	lu.InboundListenerCfg.FilterChains = fcMgr
 	return lu, nil
 }
 
-// getFilterChain parses the filter chain proto and converts it into the local
-// representation. If fc contains unsupported filter chain match fields, a nil
-// FilterChain object and a nil error are returned. If fc does not parse or
-// contains other invalid data, an non-nil error is returned.
-func getFilterChain(fc *v3listenerpb.FilterChain) (*FilterChain, error) {
-	if fc == nil {
-		return nil, nil
-	}
-
-	// If the match criteria contains unsupported fields, skip the filter chain.
-	fcm := fc.GetFilterChainMatch()
-	if fcm.GetDestinationPort().GetValue() != 0 ||
-		fcm.GetServerNames() != nil ||
-		(fcm.GetTransportProtocol() != "" && fcm.TransportProtocol != "raw_buffer") ||
-		fcm.GetApplicationProtocols() != nil {
-		return nil, nil
-	}
-
-	// Extract the supported match criteria.
-	var dstPrefixRanges []net.IP
-	for _, pr := range fcm.GetPrefixRanges() {
-		cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
-		ip, _, err := net.ParseCIDR(cidr)
-		if err != nil {
-			return nil, fmt.Errorf("xds: failed to parse destination prefix range: %+v", pr)
-		}
-		dstPrefixRanges = append(dstPrefixRanges, ip)
-	}
-	var srcType SourceType
-	switch fcm.GetSourceType() {
-	case v3listenerpb.FilterChainMatch_ANY:
-		srcType = SourceTypeAny
-	case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK:
-		srcType = SourceTypeSameOrLoopback
-	case v3listenerpb.FilterChainMatch_EXTERNAL:
-		srcType = SourceTypeExternal
-	default:
-		return nil, fmt.Errorf("xds: unsupported source type: %v", fcm.GetSourceType())
-	}
-	var srcPrefixRanges []net.IP
-	for _, pr := range fcm.GetSourcePrefixRanges() {
-		cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
-		ip, _, err := net.ParseCIDR(cidr)
-		if err != nil {
-			return nil, fmt.Errorf("xds: failed to parse source prefix range: %+v", pr)
-		}
-		srcPrefixRanges = append(srcPrefixRanges, ip)
-	}
-	filterChain := &FilterChain{
-		Match: &FilterChainMatch{
-			DestPrefixRanges:   dstPrefixRanges,
-			SourceType:         srcType,
-			SourcePrefixRanges: srcPrefixRanges,
-			SourcePorts:        fcm.GetSourcePorts(),
-		},
-	}
-
-	// If the transport_socket field is not specified, it means that the control
-	// plane has not sent us any security config. This is fine and the server
-	// will use the fallback credentials configured as part of the
-	// xdsCredentials.
-	ts := fc.GetTransportSocket()
-	if ts == nil {
-		return filterChain, nil
-	}
-	if name := ts.GetName(); name != transportSocketName {
-		return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name)
-	}
-	any := ts.GetTypedConfig()
-	if any == nil || any.TypeUrl != version.V3DownstreamTLSContextURL {
-		return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl)
-	}
-	downstreamCtx := &v3tlspb.DownstreamTlsContext{}
-	if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err)
-	}
-	if downstreamCtx.GetCommonTlsContext() == nil {
-		return nil, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext")
-	}
-	sc, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext())
-	if err != nil {
-		return nil, err
-	}
-	if sc.IdentityInstanceName == "" {
-		return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name")
-	}
-	sc.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue()
-	if sc.RequireClientCert && sc.RootInstanceName == "" {
-		return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set")
-	}
-	filterChain.SecurityCfg = sc
-	return filterChain, nil
-}
-
 // UnmarshalRouteConfig processes resources received in an RDS response,
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
diff --git a/xds/internal/server/conn_wrapper.go b/xds/internal/server/conn_wrapper.go
index 359674417dcf..a92a9ddb038f 100644
--- a/xds/internal/server/conn_wrapper.go
+++ b/xds/internal/server/conn_wrapper.go
@@ -42,8 +42,10 @@ import (
 type connWrapper struct {
 	net.Conn
 
+	// The specific filter chain picked for handling this connection.
+	filterChain *xdsclient.FilterChain
+
 	// A reference fo the listenerWrapper on which this connection was accepted.
-	// Used to access the filter chains during the server-side handshake.
 	parent *listenerWrapper
 
 	// The certificate providers created for this connection.
@@ -90,12 +92,7 @@ func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) {
 		return nil, errors.New("user has not configured xDS credentials")
 	}
 
-	fc := c.getMatchingFilterChain()
-	if fc == nil {
-		return nil, errors.New("no matching filter chain for incoming connection")
-	}
-
-	if fc.SecurityCfg == nil {
+	if c.filterChain.SecurityCfg == nil {
 		// If the security config is empty, this means that the control plane
 		// did not provide any security configuration and therefore we should
 		// return an empty HandshakeInfo here so that the xdsCreds can use the
@@ -106,13 +103,14 @@ func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) {
 	cpc := c.parent.xdsC.BootstrapConfig().CertProviderConfigs
 	// Identity provider name is mandatory on the server-side, and this is
 	// enforced when the resource is received at the xdsClient layer.
-	ip, err := buildProviderFunc(cpc, fc.SecurityCfg.IdentityInstanceName, fc.SecurityCfg.IdentityCertName, true, false)
+	secCfg := c.filterChain.SecurityCfg
+	ip, err := buildProviderFunc(cpc, secCfg.IdentityInstanceName, secCfg.IdentityCertName, true, false)
 	if err != nil {
 		return nil, err
 	}
 	// Root provider name is optional and required only when doing mTLS.
 	var rp certprovider.Provider
-	if instance, cert := fc.SecurityCfg.RootInstanceName, fc.SecurityCfg.RootCertName; instance != "" {
+	if instance, cert := secCfg.RootInstanceName, secCfg.RootCertName; instance != "" {
 		rp, err = buildProviderFunc(cpc, instance, cert, false, true)
 		if err != nil {
 			return nil, err
@@ -122,28 +120,10 @@ func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) {
 	c.rootProvider = rp
 
 	xdsHI := xdsinternal.NewHandshakeInfo(c.identityProvider, c.rootProvider)
-	xdsHI.SetRequireClientCert(fc.SecurityCfg.RequireClientCert)
+	xdsHI.SetRequireClientCert(secCfg.RequireClientCert)
 	return xdsHI, nil
 }
 
-// The logic specified in the documentation around the xDS FilterChainMatch
-// proto mentions 8 criteria to match on. gRPC does not support 4 of those, and
-// hence we got rid of them at the time of parsing the received Listener
-// resource. Here we use the remaining 4 criteria to find a matching filter
-// chain: Destination IP address, Source type, Source IP address, Source port.
-func (c *connWrapper) getMatchingFilterChain() *xdsclient.FilterChain {
-	c.parent.mu.RLock()
-	defer c.parent.mu.RUnlock()
-
-	// TODO: Do the filter chain match here and return the best match.
-	// For now, we simply return the first filter_chain in the list or the
-	// default one.
-	if len(c.parent.filterChains) == 0 {
-		return c.parent.defaultFilterChain
-	}
-	return c.parent.filterChains[0]
-}
-
 func (c *connWrapper) Close() error {
 	if c.identityProvider != nil {
 		c.identityProvider.Close()
diff --git a/xds/internal/server/listener_wrapper.go b/xds/internal/server/listener_wrapper.go
index 078b2112581d..c65736540920 100644
--- a/xds/internal/server/listener_wrapper.go
+++ b/xds/internal/server/listener_wrapper.go
@@ -24,18 +24,32 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"time"
 
+	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/grpclog"
+	internalbackoff "google.golang.org/grpc/internal/backoff"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 )
 
-var logger = grpclog.Component("xds")
+var (
+	logger = grpclog.Component("xds")
+
+	// Backoff strategy for temporary errors received from Accept(). If this
+	// needs to be configurable, we can inject it through ListenerWrapperParams.
+	bs = internalbackoff.Exponential{Config: backoff.Config{
+		BaseDelay:  5 * time.Millisecond,
+		Multiplier: 2.0,
+		MaxDelay:   1 * time.Second,
+	}}
+	backoffFunc = bs.Backoff
+)
 
 func prefixLogger(p *listenerWrapper) *internalgrpclog.PrefixLogger {
-	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p]", p))
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p] ", p))
 }
 
 // XDSClientInterface wraps the methods on the xdsClient which are required by
@@ -61,18 +75,26 @@ type ListenerWrapperParams struct {
 // NewListenerWrapper creates a new listenerWrapper with params. It returns a
 // net.Listener and a channel which is written to, indicating that the former is
 // ready to be passed to grpc.Serve().
+//
+// Only TCP listeners are supported.
 func NewListenerWrapper(params ListenerWrapperParams) (net.Listener, <-chan struct{}) {
 	lw := &listenerWrapper{
-		Listener:      params.Listener,
-		name:          params.ListenerResourceName,
-		xdsCredsInUse: params.XDSCredsInUse,
-		xdsC:          params.XDSClient,
+		Listener:          params.Listener,
+		name:              params.ListenerResourceName,
+		xdsCredsInUse:     params.XDSCredsInUse,
+		xdsC:              params.XDSClient,
+		isUnspecifiedAddr: params.Listener.Addr().(*net.TCPAddr).IP.IsUnspecified(),
 
 		closed:     grpcsync.NewEvent(),
 		goodUpdate: grpcsync.NewEvent(),
 	}
 	lw.logger = prefixLogger(lw)
 
+	// Serve() verifies that Addr() returns a valid TCPAddr. So, it is safe to
+	// ignore the error from SplitHostPort().
+	lisAddr := lw.Listener.Addr().String()
+	lw.addr, lw.port, _ = net.SplitHostPort(lisAddr)
+
 	cancelWatch := lw.xdsC.WatchListener(lw.name, lw.handleListenerUpdate)
 	lw.logger.Infof("Watch started on resource name %v", lw.name)
 	lw.cancelWatch = func() {
@@ -96,6 +118,13 @@ type listenerWrapper struct {
 	xdsC          XDSClientInterface
 	cancelWatch   func()
 
+	// Set to true if the listener is bound to the IP_ANY address (which is
+	// "0.0.0.0" for IPv4 and "::" for IPv6).
+	isUnspecifiedAddr bool
+	// Listening address and port. Used to validate the socket address in the
+	// Listener resource received from the control plane.
+	addr, port string
+
 	// This is used to notify that a good update has been received and that
 	// Serve() can be invoked on the underlying gRPC server. Using an event
 	// instead of a vanilla channel simplifies the update handler as it need not
@@ -113,20 +142,80 @@ type listenerWrapper struct {
 	// using an rw lock here is that this field will be read by all connections
 	// during their server-side handshake (in the hot path), but writes to this
 	// happen rarely (when we get a Listener resource update).
-	mu                 sync.RWMutex
-	filterChains       []*xdsclient.FilterChain
-	defaultFilterChain *xdsclient.FilterChain
+	mu           sync.RWMutex
+	filterChains *xdsclient.FilterChainManager
 }
 
 // Accept blocks on an Accept() on the underlying listener, and wraps the
 // returned net.connWrapper with the configured certificate providers.
 func (l *listenerWrapper) Accept() (net.Conn, error) {
-	c, err := l.Listener.Accept()
-	if err != nil {
-		return nil, err
+	var retries int
+	for {
+		conn, err := l.Listener.Accept()
+		if err != nil {
+			// Temporary() method is implemented by certain error types returned
+			// from the net package, and it is useful for us to not shutdown the
+			// server in these conditions. The listen queue being full is one
+			// such case.
+			if ne, ok := err.(interface{ Temporary() bool }); !ok || !ne.Temporary() {
+				return nil, err
+			}
+			retries++
+			timer := time.NewTimer(backoffFunc(retries))
+			select {
+			case <-timer.C:
+			case <-l.closed.Done():
+				timer.Stop()
+				// Continuing here will cause us to call Accept() again
+				// which will return a non-temporary error.
+				continue
+			}
+			continue
+		}
+		// Reset retries after a successful Accept().
+		retries = 0
+
+		// TODO: Close connections if in "non-serving" state
+
+		// Since the net.Conn represents an incoming connection, the source and
+		// destination address can be retrieved from the local address and
+		// remote address of the net.Conn respectively.
+		destAddr, ok1 := conn.LocalAddr().(*net.TCPAddr)
+		srcAddr, ok2 := conn.RemoteAddr().(*net.TCPAddr)
+		if !ok1 || !ok2 {
+			// If the incoming connection is not a TCP connection, which is
+			// really unexpected since we check whether the provided listener is
+			// a TCP listener in Serve(), we return an error which would cause
+			// us to stop serving.
+			return nil, fmt.Errorf("received connection with non-TCP address (local: %T, remote %T)", conn.LocalAddr(), conn.RemoteAddr())
+		}
+
+		l.mu.RLock()
+		fc, err := l.filterChains.Lookup(xdsclient.FilterChainLookupParams{
+			IsUnspecifiedListener: l.isUnspecifiedAddr,
+			DestAddr:              destAddr.IP,
+			SourceAddr:            srcAddr.IP,
+			SourcePort:            srcAddr.Port,
+		})
+		l.mu.RUnlock()
+		if err != nil {
+			// When a matching filter chain is not found, we close the
+			// connection right away, but do not return an error back to
+			// `grpc.Serve()` from where this Accept() was invoked. Returning an
+			// error to `grpc.Serve()` causes the server to shutdown. If we want
+			// to avoid the server from shutting down, we would need to return
+			// an error type which implements the `Temporary() bool` method,
+			// which is invoked by `grpc.Serve()` to see if the returned error
+			// represents a temporary condition. In the case of a temporary
+			// error, `grpc.Serve()` method sleeps for a small duration and
+			// therefore ends up blocking all connection attempts during that
+			// time frame, which is also not ideal for an error like this.
+			l.logger.Warningf("connection from %s to %s failed to find any matching filter chain", conn.RemoteAddr().String(), conn.LocalAddr().String())
+			conn.Close()
+			continue
+		}
+		return &connWrapper{Conn: conn, filterChain: fc, parent: l}, nil
 	}
-	// TODO: Close connections if in "non-serving" state.
-	return &connWrapper{Conn: c, parent: l}, nil
 }
 
 // Close closes the underlying listener. It also cancels the xDS watch
@@ -168,32 +257,18 @@ func (l *listenerWrapper) handleListenerUpdate(update xdsclient.ListenerUpdate,
 	// - this is a very context-dependent check and only the server has the
 	//   appropriate context to perform this check.
 	//
-	// What this means is that the xdsClient has ACKed a resource which is going
-	// to push the server into a "not serving" state. This is not ideal, but
-	// this is what we have decided to do. See gRPC A36 for more details.
-	// TODO(easwars): Switch to "not serving" if the host:port does not match.
-	lisAddr := l.Listener.Addr().String()
-	addr, port, err := net.SplitHostPort(lisAddr)
-	if err != nil {
-		// This is never expected to return a non-nil error since we have made
-		// sure that the listener is a TCP listener at the beginning of Serve().
-		// This is simply paranoia.
-		l.logger.Warningf("Local listener address %q failed to parse as IP:port: %v", lisAddr, err)
-		return
-	}
+	// What this means is that the xdsClient has ACKed a resource which can push
+	// the server into a "not serving" state. This is not ideal, but this is
+	// what we have decided to do. See gRPC A36 for more details.
 	ilc := update.InboundListenerCfg
-	if ilc == nil {
-		l.logger.Warningf("Missing host:port in Listener updates")
-		return
-	}
-	if ilc.Address != addr || ilc.Port != port {
-		l.logger.Warningf("Received host:port (%s:%d) in Listener update does not match local listening address: %s", ilc.Address, ilc.Port, lisAddr)
+	if ilc.Address != l.addr || ilc.Port != l.port {
+		// TODO: Switch to "not serving" if the host:port does not match.
+		l.logger.Warningf("Received host:port (%s:%d) in Listener update does not match local listening address: (%s:%s", ilc.Address, ilc.Port, l.addr, l.port)
 		return
 	}
 
 	l.mu.Lock()
 	l.filterChains = ilc.FilterChains
-	l.defaultFilterChain = ilc.DefaultFilterChain
 	l.mu.Unlock()
 	l.goodUpdate.Fire()
 	// TODO: Move to serving state on receipt of a good response.
diff --git a/xds/internal/server/listener_wrapper_test.go b/xds/internal/server/listener_wrapper_test.go
new file mode 100644
index 000000000000..220be0e08ae7
--- /dev/null
+++ b/xds/internal/server/listener_wrapper_test.go
@@ -0,0 +1,335 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package server
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+)
+
+const (
+	fakeListenerHost         = "0.0.0.0"
+	fakeListenerPort         = 50051
+	testListenerResourceName = "lds.target.1.2.3.4:1111"
+	defaultTestTimeout       = 1 * time.Second
+	defaultTestShortTimeout  = 10 * time.Millisecond
+)
+
+var listenerWithFilterChains = &v3listenerpb.Listener{
+	FilterChains: []*v3listenerpb.FilterChain{
+		{
+			FilterChainMatch: &v3listenerpb.FilterChainMatch{
+				PrefixRanges: []*v3corepb.CidrRange{
+					{
+						AddressPrefix: "192.168.0.0",
+						PrefixLen: &wrapperspb.UInt32Value{
+							Value: uint32(16),
+						},
+					},
+				},
+				SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+				SourcePrefixRanges: []*v3corepb.CidrRange{
+					{
+						AddressPrefix: "192.168.0.0",
+						PrefixLen: &wrapperspb.UInt32Value{
+							Value: uint32(16),
+						},
+					},
+				},
+				SourcePorts: []uint32{80},
+			},
+			TransportSocket: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName:    "identityPluginInstance",
+								CertificateName: "identityCertName",
+							},
+						},
+					}),
+				},
+			},
+		},
+	},
+}
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type tempError struct{}
+
+func (tempError) Error() string {
+	return "listenerWrapper test temporary error"
+}
+
+func (tempError) Temporary() bool {
+	return true
+}
+
+// connAndErr wraps a net.Conn and an error.
+type connAndErr struct {
+	conn net.Conn
+	err  error
+}
+
+// fakeListener allows the user to inject conns returned by Accept().
+type fakeListener struct {
+	acceptCh chan connAndErr
+	closeCh  *testutils.Channel
+}
+
+func (fl *fakeListener) Accept() (net.Conn, error) {
+	cne := <-fl.acceptCh
+	return cne.conn, cne.err
+}
+
+func (fl *fakeListener) Close() error {
+	fl.closeCh.Send(nil)
+	return nil
+}
+
+func (fl *fakeListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.IPv4(0, 0, 0, 0),
+		Port: fakeListenerPort,
+	}
+}
+
+// fakeConn overrides LocalAddr, RemoteAddr and Close methods.
+type fakeConn struct {
+	net.Conn
+	local, remote net.Addr
+	closeCh       *testutils.Channel
+}
+
+func (fc *fakeConn) LocalAddr() net.Addr {
+	return fc.local
+}
+
+func (fc *fakeConn) RemoteAddr() net.Addr {
+	return fc.remote
+}
+
+func (fc *fakeConn) Close() error {
+	fc.closeCh.Send(nil)
+	return nil
+}
+
+func newListenerWrapper(t *testing.T) (*listenerWrapper, <-chan struct{}, *fakeclient.Client, *fakeListener, func()) {
+	t.Helper()
+
+	// Create a listener wrapper with a fake listener and fake xdsClient and
+	// verify that it extracts the host and port from the passed in listener.
+	lis := &fakeListener{
+		acceptCh: make(chan connAndErr, 1),
+		closeCh:  testutils.NewChannel(),
+	}
+	xdsC := fakeclient.NewClient()
+	lParams := ListenerWrapperParams{
+		Listener:             lis,
+		ListenerResourceName: testListenerResourceName,
+		XDSClient:            xdsC,
+	}
+	l, readyCh := NewListenerWrapper(lParams)
+	if l == nil {
+		t.Fatalf("NewListenerWrapper(%+v) returned nil", lParams)
+	}
+	lw, ok := l.(*listenerWrapper)
+	if !ok {
+		t.Fatalf("NewListenerWrapper(%+v) returned listener of type %T want *listenerWrapper", lParams, l)
+	}
+	if lw.addr != fakeListenerHost || lw.port != strconv.Itoa(fakeListenerPort) {
+		t.Fatalf("listenerWrapper has host:port %s:%s, want %s:%d", lw.addr, lw.port, fakeListenerHost, fakeListenerPort)
+	}
+	return lw, readyCh, xdsC, lis, func() { l.Close() }
+}
+
+func (s) TestNewListenerWrapper(t *testing.T) {
+	_, readyCh, xdsC, _, cleanup := newListenerWrapper(t)
+	defer cleanup()
+
+	// Verify that the listener wrapper registers a listener watch for the
+	// expected Listener resource name.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	name, err := xdsC.WaitForWatchListener(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for a watch on a Listener resource: %v", err)
+	}
+	if name != testListenerResourceName {
+		t.Fatalf("listenerWrapper registered a watch on %s, want %s", name, testListenerResourceName)
+	}
+
+	// Push an error to the listener update handler.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, errors.New("bad listener update"))
+	timer := time.NewTimer(defaultTestShortTimeout)
+	select {
+	case <-timer.C:
+		timer.Stop()
+	case <-readyCh:
+		t.Fatalf("ready channel written to after receipt of a bad Listener update")
+	}
+
+	// Push an update whose address does not match the address to which our
+	// listener is bound, and verify that the ready channel is not written to.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: "10.0.0.1",
+			Port:    "50051",
+		}}, nil)
+	timer = time.NewTimer(defaultTestShortTimeout)
+	select {
+	case <-timer.C:
+		timer.Stop()
+	case <-readyCh:
+		t.Fatalf("ready channel written to after receipt of a bad Listener update")
+	}
+
+	// Push a good update, and verify that the ready channel is written to.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address: fakeListenerHost,
+			Port:    strconv.Itoa(fakeListenerPort),
+		}}, nil)
+	select {
+	case <-ctx.Done():
+		t.Fatalf("timeout waiting for the ready channel to be written to after receipt of a good Listener update")
+	case <-readyCh:
+	}
+}
+
+func (s) TestListenerWrapper_Accept(t *testing.T) {
+	boCh := testutils.NewChannel()
+	origBackoffFunc := backoffFunc
+	backoffFunc = func(v int) time.Duration {
+		boCh.Send(v)
+		return 0
+	}
+	defer func() { backoffFunc = origBackoffFunc }()
+
+	lw, readyCh, xdsC, lis, cleanup := newListenerWrapper(t)
+	defer cleanup()
+
+	// Push a good update with a filter chain which accepts local connections on
+	// 192.168.0.0/16 subnet and port 80.
+	fcm, err := xdsclient.NewFilterChainManager(listenerWithFilterChains)
+	if err != nil {
+		t.Fatalf("xdsclient.NewFilterChainManager() failed with error: %v", err)
+	}
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
+		InboundListenerCfg: &xdsclient.InboundListenerConfig{
+			Address:      fakeListenerHost,
+			Port:         strconv.Itoa(fakeListenerPort),
+			FilterChains: fcm,
+		}}, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	select {
+	case <-ctx.Done():
+		t.Fatalf("timeout waiting for the ready channel to be written to after receipt of a good Listener update")
+	case <-readyCh:
+	}
+
+	// Push a non-temporary error into Accept().
+	nonTempErr := errors.New("a non-temporary error")
+	lis.acceptCh <- connAndErr{err: nonTempErr}
+	if _, err := lw.Accept(); err != nonTempErr {
+		t.Fatalf("listenerWrapper.Accept() returned error: %v, want: %v", err, nonTempErr)
+	}
+
+	// Invoke Accept() in a goroutine since we expect it to swallow:
+	// 1. temporary errors returned from the underlying listener
+	// 2. errors related to finding a matching filter chain for the incoming
+	// 	  connection.
+	errCh := testutils.NewChannel()
+	go func() {
+		conn, err := lw.Accept()
+		if err != nil {
+			errCh.Send(err)
+			return
+		}
+		if _, ok := conn.(*connWrapper); !ok {
+			errCh.Send(errors.New("listenerWrapper.Accept() returned a Conn of type %T, want *connWrapper"))
+			return
+		}
+		errCh.Send(nil)
+	}()
+
+	// Push a temporary error into Accept() and verify that it backs off.
+	lis.acceptCh <- connAndErr{err: tempError{}}
+	if _, err := boCh.Receive(ctx); err != nil {
+		t.Fatalf("error when waiting for Accept() to backoff on temporary errors: %v", err)
+	}
+
+	// Push a fakeConn which does not match any filter chains configured on the
+	// received Listener resource. Verify that the conn is closed.
+	fc := &fakeConn{
+		local:   &net.TCPAddr{IP: net.IPv4(192, 168, 1, 2), Port: 79},
+		remote:  &net.TCPAddr{IP: net.IPv4(10, 1, 1, 1), Port: 80},
+		closeCh: testutils.NewChannel(),
+	}
+	lis.acceptCh <- connAndErr{conn: fc}
+	if _, err := fc.closeCh.Receive(ctx); err != nil {
+		t.Fatalf("error when waiting for conn to be closed on no filter chain match: %v", err)
+	}
+
+	// Push a fakeConn which matches the filter chains configured on the
+	// received Listener resource. Verify that Accept() returns.
+	fc = &fakeConn{
+		local:   &net.TCPAddr{IP: net.IPv4(192, 168, 1, 2)},
+		remote:  &net.TCPAddr{IP: net.IPv4(192, 168, 1, 2), Port: 80},
+		closeCh: testutils.NewChannel(),
+	}
+	lis.acceptCh <- connAndErr{conn: fc}
+	if _, err := errCh.Receive(ctx); err != nil {
+		t.Fatalf("error when waiting for Accept() to return the conn on filter chain match: %v", err)
+	}
+}
+
+func marshalAny(m proto.Message) *anypb.Any {
+	a, err := ptypes.MarshalAny(m)
+	if err != nil {
+		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
+	}
+	return a
+}
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index f18d4e356aa6..7606a35218ca 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -252,6 +252,25 @@ func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3liste
 		FilterChains: []*v3listenerpb.FilterChain{
 			{
 				Name: "filter-chain-1",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "0.0.0.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "0.0.0.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+				},
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
diff --git a/xds/server_test.go b/xds/server_test.go
index f787a129057e..2a6677a3ccbb 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -28,6 +28,12 @@ import (
 	"testing"
 	"time"
 
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	anypb "github.com/golang/protobuf/ptypes/any"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
@@ -626,21 +632,35 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 	// Push a good LDS response with security config, and wait for Serve() to be
 	// invoked on the underlying grpc.Server. Also make sure that certificate
 	// providers are not created.
+	fcm, err := xdsclient.NewFilterChainManager(&v3listenerpb.Listener{
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				TransportSocket: &v3corepb.TransportSocket{
+					Name: "envoy.transport_sockets.tls",
+					ConfigType: &v3corepb.TransportSocket_TypedConfig{
+						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    "identityPluginInstance",
+									CertificateName: "identityCertName",
+								},
+							},
+						}),
+					},
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("xdsclient.NewFilterChainManager() failed with error: %v", err)
+	}
 	addr, port := splitHostPort(lis.Addr().String())
 	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
 		RouteConfigName: "routeconfig",
 		InboundListenerCfg: &xdsclient.InboundListenerConfig{
-			Address: addr,
-			Port:    port,
-			FilterChains: []*xdsclient.FilterChain{
-				{
-					SecurityCfg: &xdsclient.SecurityConfig{
-						RootInstanceName:     "default1",
-						IdentityInstanceName: "default2",
-						RequireClientCert:    true,
-					},
-				},
-			},
+			Address:      addr,
+			Port:         port,
+			FilterChains: fcm,
 		},
 	}, nil)
 	if _, err := fs.serveCh.Receive(ctx); err != nil {
@@ -703,20 +723,7 @@ func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) {
 
 	// Push an error to the registered listener watch callback and make sure
 	// that Serve does not return.
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
-		RouteConfigName: "routeconfig",
-		InboundListenerCfg: &xdsclient.InboundListenerConfig{
-			FilterChains: []*xdsclient.FilterChain{
-				{
-					SecurityCfg: &xdsclient.SecurityConfig{
-						RootInstanceName:     "default1",
-						IdentityInstanceName: "default2",
-						RequireClientCert:    true,
-					},
-				},
-			},
-		},
-	}, errors.New("LDS error"))
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, errors.New("LDS error"))
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
 	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
@@ -742,3 +749,11 @@ func verifyCertProviderNotCreated() error {
 	}
 	return nil
 }
+
+func marshalAny(m proto.Message) *anypb.Any {
+	a, err := ptypes.MarshalAny(m)
+	if err != nil {
+		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
+	}
+	return a
+}

From db816235452978bb98c6d18ac03ce643e9ab13fc Mon Sep 17 00:00:00 2001
From: Zach Reyes <39203661+zasweq@users.noreply.github.com>
Date: Thu, 1 Apr 2021 14:41:47 -0400
Subject: [PATCH 404/481] xds: Add fields to cluster update (#4277)

* Added support for more fields in CDS response
---
 xds/internal/client/cds_test.go | 58 ++++++++++++++++++++++++-------
 xds/internal/client/client.go   | 21 ++++++++++++
 xds/internal/client/xds.go      | 61 +++++++++++++++++++++++----------
 3 files changed, 110 insertions(+), 30 deletions(-)

diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 104260759b95..3fb889db9486 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -53,7 +53,7 @@ func (s) TestValidateCluster_Failure(t *testing.T) {
 		wantErr    bool
 	}{
 		{
-			name: "non-eds-cluster-type",
+			name: "non-supported-cluster-type-static",
 			cluster: &v3clusterpb.Cluster{
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
@@ -68,6 +68,22 @@ func (s) TestValidateCluster_Failure(t *testing.T) {
 			wantUpdate: emptyUpdate,
 			wantErr:    true,
 		},
+		{
+			name: "non-supported-cluster-type-original-dst",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_ORIGINAL_DST},
+				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+					EdsConfig: &v3corepb.ConfigSource{
+						ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+							Ads: &v3corepb.AggregatedConfigSource{},
+						},
+					},
+				},
+				LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
+			},
+			wantUpdate: emptyUpdate,
+			wantErr:    true,
+		},
 		{
 			name: "no-eds-config",
 			cluster: &v3clusterpb.Cluster{
@@ -107,8 +123,8 @@ func (s) TestValidateCluster_Failure(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if update, err := validateCluster(test.cluster); err == nil {
-				t.Errorf("validateCluster(%+v) = %v, wanted error", test.cluster, update)
+			if update, err := validateClusterAndConstructClusterUpdate(test.cluster); err == nil {
+				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, wanted error", test.cluster, update)
 			}
 		})
 	}
@@ -120,6 +136,24 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		cluster    *v3clusterpb.Cluster
 		wantUpdate ClusterUpdate
 	}{
+		{
+			name: "happy-case-logical-dns",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS},
+				LbPolicy:             v3clusterpb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
+		},
+		{
+			name: "happy-case-aggregate-v3",
+			cluster: &v3clusterpb.Cluster{
+				ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{
+					ClusterType: &v3clusterpb.Cluster_CustomClusterType{Name: "envoy.clusters.aggregate"},
+				},
+				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+			},
+			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeAggregate},
+		},
 		{
 			name: "happy-case-no-service-name-no-lrs",
 			cluster: &v3clusterpb.Cluster{
@@ -214,12 +248,12 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 	defer func() { env.CircuitBreakingSupport = origCircuitBreakingSupport }()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := validateCluster(test.cluster)
+			update, err := validateClusterAndConstructClusterUpdate(test.cluster)
 			if err != nil {
-				t.Errorf("validateCluster(%+v) failed: %v", test.cluster, err)
+				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err)
 			}
 			if !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("validateCluster(%+v) = %v, want: %v", test.cluster, update, test.wantUpdate)
+				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, want: %v", test.cluster, update, test.wantUpdate)
 			}
 		})
 	}
@@ -269,12 +303,12 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 		ServiceName: serviceName,
 		EnableLRS:   false,
 	}
-	gotUpdate, err := validateCluster(cluster)
+	gotUpdate, err := validateClusterAndConstructClusterUpdate(cluster)
 	if err != nil {
-		t.Errorf("validateCluster() failed: %v", err)
+		t.Errorf("validateClusterAndConstructClusterUpdate() failed: %v", err)
 	}
 	if diff := cmp.Diff(wantUpdate, gotUpdate); diff != "" {
-		t.Errorf("validateCluster() returned unexpected diff (-want, got):\n%s", diff)
+		t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want, got):\n%s", diff)
 	}
 }
 
@@ -790,12 +824,12 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			update, err := validateCluster(test.cluster)
+			update, err := validateClusterAndConstructClusterUpdate(test.cluster)
 			if (err != nil) != test.wantErr {
-				t.Errorf("validateCluster() returned err %v wantErr %v)", err, test.wantErr)
+				t.Errorf("validateClusterAndConstructClusterUpdate() returned err %v wantErr %v)", err, test.wantErr)
 			}
 			if diff := cmp.Diff(test.wantUpdate, update, cmpopts.EquateEmpty(), cmp.AllowUnexported(regexp.Regexp{})); diff != "" {
-				t.Errorf("validateCluster() returned unexpected diff (-want, +got):\n%s", diff)
+				t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want, +got):\n%s", diff)
 			}
 		})
 	}
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 37eaae79f2d5..2daceede5398 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -355,9 +355,26 @@ type SecurityConfig struct {
 	RequireClientCert bool
 }
 
+// ClusterType is the type of cluster from a received CDS response.
+type ClusterType int
+
+const (
+	// ClusterTypeEDS represents the EDS cluster type, which will delegate endpoint
+	// discovery to the management server.
+	ClusterTypeEDS ClusterType = iota
+	// ClusterTypeLogicalDNS represents the Logical DNS cluster type, which essentially
+	// maps to the gRPC behavior of using the DNS resolver with pick_first LB policy.
+	ClusterTypeLogicalDNS
+	// ClusterTypeAggregate represents the Aggregate Cluster type, which provides a
+	// prioritized list of clusters to use. It is used for failover between clusters
+	// with a different configuration.
+	ClusterTypeAggregate
+)
+
 // ClusterUpdate contains information from a received CDS response, which is of
 // interest to the registered CDS watcher.
 type ClusterUpdate struct {
+	ClusterType ClusterType
 	// ServiceName is the service name corresponding to the clusterName which
 	// is being watched for through CDS.
 	ServiceName string
@@ -370,6 +387,10 @@ type ClusterUpdate struct {
 
 	// Raw is the resource from the xds response.
 	Raw *anypb.Any
+
+	// PrioritizedClusterNames is used only for cluster type aggregate. It represents
+	// a prioritized list of cluster names.
+	PrioritizedClusterNames []string
 }
 
 // OverloadDropConfig contains the config to drop overloads.
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 9b957399983c..571fff670daa 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -32,6 +32,7 @@ import (
 	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
@@ -499,30 +500,52 @@ func unmarshalClusterResource(r *anypb.Any, logger *grpclog.PrefixLogger) (strin
 		return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
 	}
 	logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
-
-	cu, err := validateCluster(cluster)
+	cu, err := validateClusterAndConstructClusterUpdate(cluster)
 	if err != nil {
 		return cluster.GetName(), ClusterUpdate{}, err
 	}
 	cu.Raw = r
-	// If the Cluster message in the CDS response did not contain a
-	// serviceName, we will just use the clusterName for EDS.
-	if cu.ServiceName == "" {
-		cu.ServiceName = cluster.GetName()
-	}
+
 	return cluster.GetName(), cu, nil
 }
 
-func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
+func clusterTypeFromCluster(cluster *v3clusterpb.Cluster) (ClusterType, string, []string, error) {
+	if cluster.GetType() == v3clusterpb.Cluster_EDS {
+		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
+			return 0, "", nil, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
+		}
+		// If the Cluster message in the CDS response did not contain a
+		// serviceName, we will just use the clusterName for EDS.
+		if cluster.GetEdsClusterConfig().GetServiceName() == "" {
+			return ClusterTypeEDS, cluster.GetName(), nil, nil
+		}
+		return ClusterTypeEDS, cluster.GetEdsClusterConfig().GetServiceName(), nil, nil
+	}
+
+	if cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS {
+		return ClusterTypeLogicalDNS, cluster.GetName(), nil, nil
+	}
+
+	if cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate" {
+		// Loop through ClusterConfig here to get cluster names.
+		clusters := &v3aggregateclusterpb.ClusterConfig{}
+		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
+			return 0, "", nil, fmt.Errorf("failed to unmarshal resource: %v", err)
+		}
+		return ClusterTypeAggregate, cluster.GetName(), clusters.Clusters, nil
+	}
+	return 0, "", nil, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+}
+
+func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
-	switch {
-	case cluster.GetType() != v3clusterpb.Cluster_EDS:
-		return emptyUpdate, fmt.Errorf("unexpected cluster type %v in response: %+v", cluster.GetType(), cluster)
-	case cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil:
-		return emptyUpdate, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
-	case cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN:
+	if cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN {
 		return emptyUpdate, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
+	clusterType, serviceName, prioritizedClusters, err := clusterTypeFromCluster(cluster)
+	if err != nil {
+		return emptyUpdate, err
+	}
 
 	// Process security configuration received from the control plane iff the
 	// corresponding environment variable is set.
@@ -535,10 +558,12 @@ func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
 	}
 
 	return ClusterUpdate{
-		ServiceName: cluster.GetEdsClusterConfig().GetServiceName(),
-		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
-		SecurityCfg: sc,
-		MaxRequests: circuitBreakersFromCluster(cluster),
+		ClusterType:             clusterType,
+		ServiceName:             serviceName,
+		EnableLRS:               cluster.GetLrsServer().GetSelf() != nil,
+		SecurityCfg:             sc,
+		MaxRequests:             circuitBreakersFromCluster(cluster),
+		PrioritizedClusterNames: prioritizedClusters,
 	}, nil
 }
 

From 5730f8d113ee31f14709a787572c4a3f3af5d3dd Mon Sep 17 00:00:00 2001
From: ZhenLian <zhenlian@google.com>
Date: Fri, 2 Apr 2021 11:19:22 -0700
Subject: [PATCH 405/481] Invoke Go Vet Check in Sub-modules (#4302)

* Invoke Go Vet Check in Sub-modules
---
 test/tools/tools.go     |  7 +++----
 test/tools/tools_vet.go | 21 +++++++++++++++++++++
 vet.sh                  | 24 ++++++++++++++----------
 3 files changed, 38 insertions(+), 14 deletions(-)
 create mode 100644 test/tools/tools_vet.go

diff --git a/test/tools/tools.go b/test/tools/tools.go
index 511dc2534462..3a614f765f50 100644
--- a/test/tools/tools.go
+++ b/test/tools/tools.go
@@ -18,10 +18,9 @@
  *
  */
 
-// This package exists to cause `go mod` and `go get` to believe these tools
-// are dependencies, even though they are not runtime dependencies of any grpc
-// package.  This means they will appear in our `go.mod` file, but will not be
-// a part of the build.
+// This file is not intended to be compiled.  Because some of these imports are
+// not actual go packages, we use a build constraint at the top of this file to
+// prevent tools from inspecting the imports.
 
 package tools
 
diff --git a/test/tools/tools_vet.go b/test/tools/tools_vet.go
new file mode 100644
index 000000000000..06ab2fd10be2
--- /dev/null
+++ b/test/tools/tools_vet.go
@@ -0,0 +1,21 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package tools is used to pin specific versions of external tools in this
+// module's go.mod that gRPC uses for internal testing.
+package tools
diff --git a/vet.sh b/vet.sh
index dcd939bb3907..1a0dbd7ee5ab 100755
--- a/vet.sh
+++ b/vet.sh
@@ -105,12 +105,6 @@ git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' ':(exclude)*.
 # TODO: Remove when we drop Go 1.10 support
 go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go
 
-# - gofmt, goimports, golint (with exceptions for generated code), go vet.
-gofmt -s -d -l . 2>&1 | fail_on_output
-goimports -l . 2>&1 | not grep -vE "\.pb\.go"
-golint ./... 2>&1 | not grep -vE "/testv3\.pb\.go:"
-go vet -all ./...
-
 misspell -error .
 
 # - Check that generated proto files are up to date.
@@ -120,12 +114,22 @@ if [[ -z "${VET_SKIP_PROTO}" ]]; then
     (git status; git --no-pager diff; exit 1)
 fi
 
-# - Check that our modules are tidy.
-if go help mod >& /dev/null; then
-  find . -name 'go.mod' | xargs -IXXX bash -c 'cd $(dirname XXX); go mod tidy'
+# - gofmt, goimports, golint (with exceptions for generated code), go vet,
+# go mod tidy.
+# Perform these checks on each module inside gRPC.
+for MOD_FILE in $(find . -name 'go.mod'); do
+  MOD_DIR=$(dirname ${MOD_FILE})
+  pushd ${MOD_DIR}
+  go vet -all ./... | fail_on_output
+  gofmt -s -d -l . 2>&1 | fail_on_output
+  goimports -l . 2>&1 | not grep -vE "\.pb\.go"
+  golint ./... 2>&1 | not grep -vE "/testv3\.pb\.go:"
+
+  go mod tidy
   git status --porcelain 2>&1 | fail_on_output || \
     (git status; git --no-pager diff; exit 1)
-fi
+  popd
+done
 
 # - Collection of static analysis checks
 #

From 8892a7b247c0aef5059175bacee30f2b055aac88 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 5 Apr 2021 13:56:00 -0700
Subject: [PATCH 406/481] [xds_interop_client_admin] xds/interop: register
 admin services and reflection (#4307)

---
 interop/xds/client/client.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index 5b755272d3e7..b028ec79228e 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -31,9 +31,11 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/admin"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/reflection"
 	"google.golang.org/grpc/status"
 	_ "google.golang.org/grpc/xds"
 
@@ -370,6 +372,12 @@ func main() {
 	defer s.Stop()
 	testgrpc.RegisterLoadBalancerStatsServiceServer(s, &statsService{})
 	testgrpc.RegisterXdsUpdateClientConfigureServiceServer(s, &configureService{})
+	reflection.Register(s)
+	cleanup, err := admin.Register(s)
+	if err != nil {
+		logger.Fatalf("failed to register admin: %v", err)
+	}
+	defer cleanup()
 	go s.Serve(lis)
 
 	clients := make([]testgrpc.TestServiceClient, *numChannels)

From 777b228b599fd383aafd29155c35741d617b564c Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 6 Apr 2021 10:55:19 -0700
Subject: [PATCH 407/481] xds: fix service request counter flaky test (#4324)

---
 xds/internal/client/requests_counter.go      | 4 ++++
 xds/internal/client/requests_counter_test.go | 6 +++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/xds/internal/client/requests_counter.go b/xds/internal/client/requests_counter.go
index 7ef18345ed6c..f033e1920991 100644
--- a/xds/internal/client/requests_counter.go
+++ b/xds/internal/client/requests_counter.go
@@ -56,6 +56,10 @@ func GetServiceRequestsCounter(serviceName string) *ServiceRequestsCounter {
 // StartRequest starts a request for a service, incrementing its number of
 // requests by 1. Returns an error if the max number of requests is exceeded.
 func (c *ServiceRequestsCounter) StartRequest(max uint32) error {
+	// Note that during race, the limits could be exceeded. This is allowed:
+	// "Since the implementation is eventually consistent, races between threads
+	// may allow limits to be potentially exceeded."
+	// https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking#arch-overview-circuit-break.
 	if atomic.LoadUint32(&c.numRequests) >= max {
 		return fmt.Errorf("max requests %v exceeded on service %v", max, c.ServiceName)
 	}
diff --git a/xds/internal/client/requests_counter_test.go b/xds/internal/client/requests_counter_test.go
index fe532724d14e..2dc336d1c1d5 100644
--- a/xds/internal/client/requests_counter_test.go
+++ b/xds/internal/client/requests_counter_test.go
@@ -91,7 +91,11 @@ func testCounter(t *testing.T, test counterTest) {
 	if test.expectedErrors == 0 && loadedError != nil {
 		t.Errorf("error starting request: %v", loadedError.(error))
 	}
-	if successes != test.expectedSuccesses || errors != test.expectedErrors {
+	// We allow the limits to be exceeded during races.
+	//
+	// But we should never over-limit, so this test fails if there are less
+	// successes than expected.
+	if successes < test.expectedSuccesses || errors > test.expectedErrors {
 		t.Errorf("unexpected number of (successes, errors), expected (%v, %v), encountered (%v, %v)", test.expectedSuccesses, test.expectedErrors, successes, errors)
 	}
 }

From 9a10f357871cf04dbc16b064b993e81e66c660f7 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 6 Apr 2021 13:11:49 -0700
Subject: [PATCH 408/481] balancergroup: fix leak child balancer not closed
 (#4308)

---
 .../balancer/balancergroup/balancergroup.go   |  7 +++--
 .../balancergroup/balancergroup_test.go       | 30 +++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/xds/internal/balancer/balancergroup/balancergroup.go
index 2ec576a4b572..5b6d42a25e44 100644
--- a/xds/internal/balancer/balancergroup/balancergroup.go
+++ b/xds/internal/balancer/balancergroup/balancergroup.go
@@ -479,6 +479,10 @@ func (bg *BalancerGroup) Close() {
 	}
 	bg.incomingMu.Unlock()
 
+	// Clear(true) runs clear function to close sub-balancers in cache. It
+	// must be called out of outgoing mutex.
+	bg.balancerCache.Clear(true)
+
 	bg.outgoingMu.Lock()
 	if bg.outgoingStarted {
 		bg.outgoingStarted = false
@@ -487,9 +491,6 @@ func (bg *BalancerGroup) Close() {
 		}
 	}
 	bg.outgoingMu.Unlock()
-	// Clear(true) runs clear function to close sub-balancers in cache. It
-	// must be called out of outgoing mutex.
-	bg.balancerCache.Clear(true)
 }
 
 const (
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index 0ad4bf8df10f..ab6ac3913cbb 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -938,6 +938,36 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
 	}
 }
 
+// After removing a sub-balancer, it will be kept in cache. Make sure that this
+// sub-balancer's Close is called when the balancer group is closed.
+func (s) TestBalancerGroup_CloseStopsBalancerInCache(t *testing.T) {
+	const balancerName = "stub-TestBalancerGroup_check_close"
+	closed := make(chan struct{})
+	stub.Register(balancerName, stub.BalancerFuncs{Close: func(_ *stub.BalancerData) {
+		close(closed)
+	}})
+	builder := balancer.Get(balancerName)
+
+	defer replaceDefaultSubBalancerCloseTimeout(time.Second)()
+	gator, bg, _, _ := initBalancerGroupForCachingTest(t)
+
+	// Add balancer, and remove
+	gator.Add(testBalancerIDs[2], 1)
+	bg.Add(testBalancerIDs[2], builder)
+	gator.Remove(testBalancerIDs[2])
+	bg.Remove(testBalancerIDs[2])
+
+	// Immediately close balancergroup, before the cache timeout.
+	bg.Close()
+
+	// Make sure the removed child balancer is closed eventually.
+	select {
+	case <-closed:
+	case <-time.After(time.Second * 2):
+		t.Fatalf("timeout waiting for the child balancer in cache to be closed")
+	}
+}
+
 // TestBalancerGroupBuildOptions verifies that the balancer.BuildOptions passed
 // to the balancergroup at creation time is passed to child policies.
 func (s) TestBalancerGroupBuildOptions(t *testing.T) {

From 004ef8ade68b267f285c82e955a2f663c9a591be Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 6 Apr 2021 13:47:15 -0700
Subject: [PATCH 409/481] xds/clusterimpl: fix picker update race after
 balancer is closed (#4318)

---
 .../balancer/clusterimpl/balancer_test.go     | 45 +++++++++++++++++++
 .../balancer/clusterimpl/clusterimpl.go       | 25 ++++++++++-
 2 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index 3e6ac0fd2900..6d9b7a5082f6 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -324,3 +324,48 @@ func TestDropCircuitBreaking(t *testing.T) {
 		t.Fatalf("got unexpected drop reports, diff (-got, +want): %v", diff)
 	}
 }
+
+// TestPickerUpdateAfterClose covers the case that cluster_impl wants to update
+// picker after it's closed. Because picker updates are sent in the run()
+// goroutine.
+func TestPickerUpdateAfterClose(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
+	builder := balancer.Get(clusterImplName)
+	cc := testutils.NewTestClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+
+	var maxRequest uint32 = 50
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: testBackendAddrs,
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:               testClusterName,
+			EDSServiceName:        testServiceName,
+			MaxConcurrentRequests: &maxRequest,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		b.Close()
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	// Send SubConn state changes to trigger picker updates. Balancer will
+	// closed in a defer.
+	sc1 := <-cc.NewSubConnCh
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This close will race with the SubConn state update.
+	b.Close()
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("unexpected picker update after balancer is closed")
+	case <-time.After(time.Millisecond * 10):
+	}
+}
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index 4e4af5a02b45..4435f9e65a03 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -26,6 +26,7 @@ package clusterimpl
 import (
 	"encoding/json"
 	"fmt"
+	"sync"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/buffer"
@@ -90,8 +91,19 @@ type xdsClientInterface interface {
 
 type clusterImplBalancer struct {
 	balancer.ClientConn
-	bOpts  balancer.BuildOptions
+
+	// mu guarantees mutual exclusion between Close() and handling of picker
+	// update to the parent ClientConn in run(). It's to make sure that the
+	// run() goroutine doesn't send picker update to parent after the balancer
+	// is closed.
+	//
+	// It's only used by the run() goroutine, but not the other exported
+	// functions. Because the exported functions are guaranteed to be
+	// synchronized with Close().
+	mu     sync.Mutex
 	closed *grpcsync.Event
+
+	bOpts  balancer.BuildOptions
 	logger *grpclog.PrefixLogger
 	xdsC   xdsClientInterface
 
@@ -274,12 +286,15 @@ func (cib *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balanc
 }
 
 func (cib *clusterImplBalancer) Close() {
+	cib.mu.Lock()
+	cib.closed.Fire()
+	cib.mu.Unlock()
+
 	if cib.childLB != nil {
 		cib.childLB.Close()
 		cib.childLB = nil
 	}
 	cib.xdsC.Close()
-	cib.closed.Fire()
 	cib.logger.Infof("Shutdown")
 }
 
@@ -301,6 +316,11 @@ func (cib *clusterImplBalancer) run() {
 		select {
 		case update := <-cib.pickerUpdateCh.Get():
 			cib.pickerUpdateCh.Load()
+			cib.mu.Lock()
+			if cib.closed.HasFired() {
+				cib.mu.Unlock()
+				return
+			}
 			switch u := update.(type) {
 			case balancer.State:
 				cib.childState = u
@@ -322,6 +342,7 @@ func (cib *clusterImplBalancer) run() {
 					})
 				}
 			}
+			cib.mu.Unlock()
 		case <-cib.closed.Done():
 			return
 		}

From 493d388ad24c7a3e957f552a1a15dccdd1c9124b Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 6 Apr 2021 15:09:00 -0700
Subject: [PATCH 410/481] xds/csds: update proto imports to separate grpc from
 non-grpc symbols (#4326)

---
 xds/csds/csds_test.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
index 2993beea0e5d..867d74e5b25b 100644
--- a/xds/csds/csds_test.go
+++ b/xds/csds/csds_test.go
@@ -49,6 +49,7 @@ import (
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+	v3statuspbgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 )
 
 const (
@@ -246,7 +247,7 @@ func TestCSDS(t *testing.T) {
 	}
 }
 
-func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServer, string, v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient, func()) {
+func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServer, string, v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient, func()) {
 	t.Helper()
 
 	// Spin up a xDS management server on a local port.
@@ -281,7 +282,7 @@ func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServ
 	if err != nil {
 		t.Fatal(err)
 	}
-	v3statuspb.RegisterClientStatusDiscoveryServiceServer(server, csdss)
+	v3statuspbgrpc.RegisterClientStatusDiscoveryServiceServer(server, csdss)
 	// Create a local listener and pass it to Serve().
 	lis, err := testutils.LocalTCPListener()
 	if err != nil {
@@ -298,7 +299,7 @@ func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServ
 	if err != nil {
 		t.Fatalf("cannot connect to server: %v", err)
 	}
-	c := v3statuspb.NewClientStatusDiscoveryServiceClient(conn)
+	c := v3statuspbgrpc.NewClientStatusDiscoveryServiceClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	stream, err := c.StreamClientStatus(ctx, grpc.WaitForReady(true))
 	if err != nil {
@@ -317,7 +318,7 @@ func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServ
 	}
 }
 
-func checkForRequested(stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient) error {
+func checkForRequested(stream v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient) error {
 	if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil {
 		return fmt.Errorf("failed to send request: %v", err)
 	}
@@ -395,7 +396,7 @@ func checkForRequested(stream v3statuspb.ClientStatusDiscoveryService_StreamClie
 	return nil
 }
 
-func checkForACKed(stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient) error {
+func checkForACKed(stream v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient) error {
 	const wantVersion = "1"
 
 	if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil {
@@ -492,7 +493,7 @@ func checkForACKed(stream v3statuspb.ClientStatusDiscoveryService_StreamClientSt
 	return nil
 }
 
-func checkForNACKed(nackResourceIdx int, stream v3statuspb.ClientStatusDiscoveryService_StreamClientStatusClient) error {
+func checkForNACKed(nackResourceIdx int, stream v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient) error {
 	const (
 		ackVersion  = "1"
 		nackVersion = "2"

From 1895da54b012305f2628e3feee697937149aac57 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 8 Apr 2021 11:34:02 -0700
Subject: [PATCH 411/481] xds/resolver: fix panic when two LDS updates are
 receives without RDS in between (#4327)

Also confirmed that the LDS updates shouldn't trigger state update without the
RDS.
---
 xds/internal/resolver/watch_service.go     | 10 +++++-
 xds/internal/resolver/xds_resolver_test.go | 42 ++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go
index 42ede988300c..7667592ccd6f 100644
--- a/xds/internal/resolver/watch_service.go
+++ b/xds/internal/resolver/watch_service.go
@@ -132,7 +132,15 @@ func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, er
 		//
 		// If the route name did change, then we must wait until the first RDS
 		// update before reporting this LDS config.
-		w.serviceCb(w.lastUpdate, nil)
+		if w.lastUpdate.virtualHost != nil {
+			// We want to send an update with the new fields from the new LDS
+			// (e.g. max stream duration), and old fields from the the previous
+			// RDS.
+			//
+			// But note that this should only happen when virtual host is set,
+			// which means an RDS was received.
+			w.serviceCb(w.lastUpdate, nil)
+		}
 		return
 	}
 	w.rdsName = update.RouteConfigName
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 36c7416cb436..82355eecc70b 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -1030,6 +1030,48 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	}
 }
 
+// TestXDSResolverMultipleLDSUpdates tests the case where two LDS updates with
+// the same RDS name to watch are received without an RDS in between. Those LDS
+// updates shouldn't trigger service config update.
+//
+// This test case also makes sure the resolver doesn't panic.
+func (s) TestXDSResolverMultipleLDSUpdates(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	xdsR, tcc, cancel := testSetup(t, setupOpts{
+		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
+	})
+	defer func() {
+		cancel()
+		xdsR.Close()
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	waitForWatchListener(ctx, t, xdsC, targetStr)
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
+	waitForWatchRouteConfig(ctx, t, xdsC, routeStr)
+	defer replaceRandNumGenerator(0)()
+
+	// Send a new LDS update, with the same fields.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, HTTPFilters: routerFilterList}, nil)
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer cancel()
+	// Should NOT trigger a state update.
+	gotState, err := tcc.stateCh.Receive(ctx)
+	if err == nil {
+		t.Fatalf("ClientConn.UpdateState received %v, want timeout error", gotState)
+	}
+
+	// Send a new LDS update, with the same RDS name, but different fields.
+	xdsC.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second, HTTPFilters: routerFilterList}, nil)
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer cancel()
+	gotState, err = tcc.stateCh.Receive(ctx)
+	if err == nil {
+		t.Fatalf("ClientConn.UpdateState received %v, want timeout error", gotState)
+	}
+}
+
 type filterBuilder struct {
 	httpfilter.Filter // embedded as we do not need to implement registry / parsing in this test.
 	path              *[]string

From c7a203dcb5c97bf4cc7fd79b905b044ab14a5fbc Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 8 Apr 2021 14:31:20 -0700
Subject: [PATCH 412/481] xds/interop: move header/path matching to all (#4325)

---
 test/kokoro/xds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index e75743dd9e57..65b35e3acac6 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -27,7 +27,7 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh
 # they are added into "all".
 GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
-    --test_case="all,path_matching,header_matching,circuit_breaking,timeout,fault_injection" \
+    --test_case="all,circuit_breaking,timeout,fault_injection" \
     --project_id=grpc-testing \
     --project_num=830293263384 \
     --source_image=projects/grpc-testing/global/images/xds-test-server-4 \

From 69f6f5a51249d3a9f4b6a9262167ddd984599cdc Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 8 Apr 2021 15:52:49 -0700
Subject: [PATCH 413/481] xds: add support for unsupported filter matchers
 (#4315)

---
 xds/internal/client/filter_chain.go      | 216 ++++++++++++++---------
 xds/internal/client/filter_chain_test.go | 173 ++++++++++++++++--
 xds/internal/client/lds_test.go          |   3 -
 3 files changed, 291 insertions(+), 101 deletions(-)

diff --git a/xds/internal/client/filter_chain.go b/xds/internal/client/filter_chain.go
index 8e24ab858230..73ffa18347e8 100644
--- a/xds/internal/client/filter_chain.go
+++ b/xds/internal/client/filter_chain.go
@@ -74,12 +74,17 @@ const (
 //    connection time.
 //
 // The logic specified in the documentation around the xDS FilterChainMatch
-// proto mentions 8 criteria to match on. gRPC does not support 4 of those, and
-// we ignore filter chains which contain any of these unsupported fields at
-// parsing time. Here we use the remaining 4 criteria to find a matching filter
-// chain in the following order:
-// Destination IP address, Source type, Source IP address, Source port.
-// TODO: Ignore chains with unsupported fields *only* at connection time.
+// proto mentions 8 criteria to match on.
+// The following order applies:
+//
+// 1. Destination port.
+// 2. Destination IP address.
+// 3. Server name (e.g. SNI for TLS protocol),
+// 4. Transport protocol.
+// 5. Application protocols (e.g. ALPN for TLS protocol).
+// 6. Source type (e.g. any, local or external network).
+// 7. Source IP address.
+// 8. Source port.
 type FilterChainManager struct {
 	// Destination prefix is the first match criteria that we support.
 	// Therefore, this multi-stage map is indexed on destination prefixes
@@ -98,20 +103,29 @@ type FilterChainManager struct {
 	// involves too much time/effort, sort this slice based on the netmask size.
 	dstPrefixes []*destPrefixEntry
 
-	def   *FilterChain // Default filter chain, if specified.
-	fcCnt int          // Count of supported filter chains, for validation.
+	def *FilterChain // Default filter chain, if specified.
 }
 
 // destPrefixEntry is the value type of the map indexed on destination prefixes.
 type destPrefixEntry struct {
 	net *net.IPNet // The actual destination prefix.
+	// We need to keep track of the transport protocols seen as part of the
+	// config validation (and internal structure building) phase. The only two
+	// values that we support are empty string and "raw_buffer", with the latter
+	// taking preference. Once we have seen one filter chain with "raw_buffer",
+	// we can drop everything other filter chain with an empty transport
+	// protocol.
+	rawBufferSeen bool
 	// For each specified source type in the filter chain match criteria, this
 	// array points to the set of specified source prefixes.
 	// Unspecified source type matches end up as a wildcard entry here with an
 	// index of 0, which actually represents the source type `ANY`.
-	srcTypeArr [3]*sourcePrefixes
+	srcTypeArr sourceTypesArray
 }
 
+// An array for the fixed number of source types that we have.
+type sourceTypesArray [3]*sourcePrefixes
+
 // sourcePrefixes contains source prefix related information specified in the
 // match criteria. These are pointed to by the array of source types.
 type sourcePrefixes struct {
@@ -141,6 +155,24 @@ func NewFilterChainManager(lis *v3listenerpb.Listener) (*FilterChainManager, err
 	if err := fci.addFilterChains(lis.GetFilterChains()); err != nil {
 		return nil, err
 	}
+	// Build the source and dest prefix slices used by Lookup().
+	fcSeen := false
+	for _, dstPrefix := range fci.dstPrefixMap {
+		fci.dstPrefixes = append(fci.dstPrefixes, dstPrefix)
+		for _, st := range dstPrefix.srcTypeArr {
+			if st == nil {
+				continue
+			}
+			for _, srcPrefix := range st.srcPrefixMap {
+				st.srcPrefixes = append(st.srcPrefixes, srcPrefix)
+				for _, fc := range srcPrefix.srcPortMap {
+					if fc != nil {
+						fcSeen = true
+					}
+				}
+			}
+		}
+	}
 
 	// Retrieve the default filter chain. The match criteria specified on the
 	// default filter chain is never used. The default filter chain simply gets
@@ -156,7 +188,7 @@ func NewFilterChainManager(lis *v3listenerpb.Listener) (*FilterChainManager, err
 
 	// If there are no supported filter chains and no default filter chain, we
 	// fail here. This will call the Listener resource to be NACK'ed.
-	if fci.fcCnt == 0 && fci.def == nil {
+	if !fcSeen && fci.def == nil {
 		return nil, fmt.Errorf("no supported filter chains and no default filter chain")
 	}
 	return fci, nil
@@ -166,114 +198,133 @@ func NewFilterChainManager(lis *v3listenerpb.Listener) (*FilterChainManager, err
 // internal data structures corresponding to the match criteria.
 func (fci *FilterChainManager) addFilterChains(fcs []*v3listenerpb.FilterChain) error {
 	for _, fc := range fcs {
-		// Skip filter chains with unsupported match fields/criteria.
 		fcm := fc.GetFilterChainMatch()
-		if fcm.GetDestinationPort().GetValue() != 0 ||
-			fcm.GetServerNames() != nil ||
-			(fcm.GetTransportProtocol() != "" && fcm.TransportProtocol != "raw_buffer") ||
-			fcm.GetApplicationProtocols() != nil {
+		if fcm.GetDestinationPort().GetValue() != 0 {
+			// Destination port is the first match criteria and we do not
+			// support filter chains which contains this match criteria.
+			logger.Warningf("Dropping filter chain %+v since it contains unsupported destination_port match field", fc)
 			continue
 		}
 
-		// Extract the supported match criteria, which will be used by
-		// successive addFilterChainsForXxx() functions.
-		var dstPrefixes []*net.IPNet
-		for _, pr := range fcm.GetPrefixRanges() {
-			cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
-			_, ipnet, err := net.ParseCIDR(cidr)
-			if err != nil {
-				return fmt.Errorf("failed to parse destination prefix range: %+v", pr)
-			}
-			dstPrefixes = append(dstPrefixes, ipnet)
-		}
-
-		var srcType SourceType
-		switch fcm.GetSourceType() {
-		case v3listenerpb.FilterChainMatch_ANY:
-			srcType = SourceTypeAny
-		case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK:
-			srcType = SourceTypeSameOrLoopback
-		case v3listenerpb.FilterChainMatch_EXTERNAL:
-			srcType = SourceTypeExternal
-		default:
-			return fmt.Errorf("unsupported source type: %v", fcm.GetSourceType())
-		}
-
-		var srcPrefixes []*net.IPNet
-		for _, pr := range fcm.GetSourcePrefixRanges() {
-			cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
-			_, ipnet, err := net.ParseCIDR(cidr)
-			if err != nil {
-				return fmt.Errorf("failed to parse source prefix range: %+v", pr)
-			}
-			srcPrefixes = append(srcPrefixes, ipnet)
-		}
-
-		var srcPorts []int
-		for _, port := range fcm.GetSourcePorts() {
-			srcPorts = append(srcPorts, int(port))
-		}
-
 		// Build the internal representation of the filter chain match fields.
-		if err := fci.addFilterChainsForDestPrefixes(dstPrefixes, srcType, srcPrefixes, srcPorts, fc); err != nil {
+		if err := fci.addFilterChainsForDestPrefixes(fc); err != nil {
 			return err
 		}
-		fci.fcCnt++
 	}
 
-	// Build the source and dest prefix slices used by Lookup().
-	for _, dstPrefix := range fci.dstPrefixMap {
-		fci.dstPrefixes = append(fci.dstPrefixes, dstPrefix)
-		for _, st := range dstPrefix.srcTypeArr {
-			if st == nil {
-				continue
-			}
-			for _, srcPrefix := range st.srcPrefixMap {
-				st.srcPrefixes = append(st.srcPrefixes, srcPrefix)
-			}
-		}
-	}
 	return nil
 }
 
-// addFilterChainsForDestPrefixes adds destination prefixes to the internal data
-// structures and delegates control to addFilterChainsForSourceType to continue
-// building the internal data structure.
-func (fci *FilterChainManager) addFilterChainsForDestPrefixes(dstPrefixes []*net.IPNet, srcType SourceType, srcPrefixes []*net.IPNet, srcPorts []int, fc *v3listenerpb.FilterChain) error {
+func (fci *FilterChainManager) addFilterChainsForDestPrefixes(fc *v3listenerpb.FilterChain) error {
+	var dstPrefixes []*net.IPNet
+	for _, pr := range fc.GetFilterChainMatch().GetPrefixRanges() {
+		cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
+		_, ipnet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return fmt.Errorf("failed to parse destination prefix range: %+v", pr)
+		}
+		dstPrefixes = append(dstPrefixes, ipnet)
+	}
+
 	if len(dstPrefixes) == 0 {
 		// Use the wildcard IP when destination prefix is unspecified.
 		if fci.dstPrefixMap[emptyAddrMapKey] == nil {
 			fci.dstPrefixMap[emptyAddrMapKey] = &destPrefixEntry{net: zeroIP}
 		}
-		return fci.addFilterChainsForSourceType(fci.dstPrefixMap[emptyAddrMapKey], srcType, srcPrefixes, srcPorts, fc)
+		return fci.addFilterChainsForServerNames(fci.dstPrefixMap[emptyAddrMapKey], fc)
 	}
 	for _, prefix := range dstPrefixes {
 		p := prefix.String()
 		if fci.dstPrefixMap[p] == nil {
 			fci.dstPrefixMap[p] = &destPrefixEntry{net: prefix}
 		}
-		if err := fci.addFilterChainsForSourceType(fci.dstPrefixMap[p], srcType, srcPrefixes, srcPorts, fc); err != nil {
+		if err := fci.addFilterChainsForServerNames(fci.dstPrefixMap[p], fc); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
+func (fci *FilterChainManager) addFilterChainsForServerNames(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error {
+	// Filter chains specifying server names in their match criteria always fail
+	// a match at connection time. So, these filter chains can be dropped now.
+	if len(fc.GetFilterChainMatch().GetServerNames()) != 0 {
+		logger.Warningf("Dropping filter chain %+v since it contains unsupported server_names match field", fc)
+		return nil
+	}
+
+	return fci.addFilterChainsForTransportProtocols(dstEntry, fc)
+}
+
+func (fci *FilterChainManager) addFilterChainsForTransportProtocols(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error {
+	tp := fc.GetFilterChainMatch().GetTransportProtocol()
+	switch {
+	case tp != "" && tp != "raw_buffer":
+		// Only allow filter chains with transport protocol set to empty string
+		// or "raw_buffer".
+		logger.Warningf("Dropping filter chain %+v since it contains unsupported value for transport_protocols match field", fc)
+		return nil
+	case tp == "" && dstEntry.rawBufferSeen:
+		// If we have already seen filter chains with transport protocol set to
+		// "raw_buffer", we can drop filter chains with transport protocol set
+		// to empty string, since the former takes precedence.
+		logger.Warningf("Dropping filter chain %+v since it contains unsupported value for transport_protocols match field", fc)
+		return nil
+	case tp != "" && !dstEntry.rawBufferSeen:
+		// This is the first "raw_buffer" that we are seeing. Set the bit and
+		// reset the source types array which might contain entries for filter
+		// chains with transport protocol set to empty string.
+		dstEntry.rawBufferSeen = true
+		dstEntry.srcTypeArr = sourceTypesArray{}
+	}
+	return fci.addFilterChainsForApplicationProtocols(dstEntry, fc)
+}
+
+func (fci *FilterChainManager) addFilterChainsForApplicationProtocols(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error {
+	if len(fc.GetFilterChainMatch().GetApplicationProtocols()) != 0 {
+		logger.Warningf("Dropping filter chain %+v since it contains unsupported application_protocols match field", fc)
+		return nil
+	}
+	return fci.addFilterChainsForSourceType(dstEntry, fc)
+}
+
 // addFilterChainsForSourceType adds source types to the internal data
 // structures and delegates control to addFilterChainsForSourcePrefixes to
 // continue building the internal data structure.
-func (fci *FilterChainManager) addFilterChainsForSourceType(dstEntry *destPrefixEntry, srcType SourceType, srcPrefixes []*net.IPNet, srcPorts []int, fc *v3listenerpb.FilterChain) error {
+func (fci *FilterChainManager) addFilterChainsForSourceType(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error {
+	var srcType SourceType
+	switch st := fc.GetFilterChainMatch().GetSourceType(); st {
+	case v3listenerpb.FilterChainMatch_ANY:
+		srcType = SourceTypeAny
+	case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK:
+		srcType = SourceTypeSameOrLoopback
+	case v3listenerpb.FilterChainMatch_EXTERNAL:
+		srcType = SourceTypeExternal
+	default:
+		return fmt.Errorf("unsupported source type: %v", st)
+	}
+
 	st := int(srcType)
 	if dstEntry.srcTypeArr[st] == nil {
 		dstEntry.srcTypeArr[st] = &sourcePrefixes{srcPrefixMap: make(map[string]*sourcePrefixEntry)}
 	}
-	return fci.addFilterChainsForSourcePrefixes(dstEntry.srcTypeArr[st].srcPrefixMap, srcPrefixes, srcPorts, fc)
+	return fci.addFilterChainsForSourcePrefixes(dstEntry.srcTypeArr[st].srcPrefixMap, fc)
 }
 
 // addFilterChainsForSourcePrefixes adds source prefixes to the internal data
 // structures and delegates control to addFilterChainsForSourcePorts to continue
 // building the internal data structure.
-func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map[string]*sourcePrefixEntry, srcPrefixes []*net.IPNet, srcPorts []int, fc *v3listenerpb.FilterChain) error {
+func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map[string]*sourcePrefixEntry, fc *v3listenerpb.FilterChain) error {
+	var srcPrefixes []*net.IPNet
+	for _, pr := range fc.GetFilterChainMatch().GetSourcePrefixRanges() {
+		cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue())
+		_, ipnet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return fmt.Errorf("failed to parse source prefix range: %+v", pr)
+		}
+		srcPrefixes = append(srcPrefixes, ipnet)
+	}
+
 	if len(srcPrefixes) == 0 {
 		// Use the wildcard IP when source prefix is unspecified.
 		if srcPrefixMap[emptyAddrMapKey] == nil {
@@ -282,7 +333,7 @@ func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map
 				srcPortMap: make(map[int]*FilterChain),
 			}
 		}
-		return fci.addFilterChainsForSourcePorts(srcPrefixMap[emptyAddrMapKey], srcPorts, fc)
+		return fci.addFilterChainsForSourcePorts(srcPrefixMap[emptyAddrMapKey], fc)
 	}
 	for _, prefix := range srcPrefixes {
 		p := prefix.String()
@@ -292,7 +343,7 @@ func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map
 				srcPortMap: make(map[int]*FilterChain),
 			}
 		}
-		if err := fci.addFilterChainsForSourcePorts(srcPrefixMap[p], srcPorts, fc); err != nil {
+		if err := fci.addFilterChainsForSourcePorts(srcPrefixMap[p], fc); err != nil {
 			return err
 		}
 	}
@@ -303,7 +354,12 @@ func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map
 // structures and completes the process of building the internal data structure.
 // It is here that we determine if there are multiple filter chains with
 // overlapping matching rules.
-func (fci *FilterChainManager) addFilterChainsForSourcePorts(srcEntry *sourcePrefixEntry, srcPorts []int, fcProto *v3listenerpb.FilterChain) error {
+func (fci *FilterChainManager) addFilterChainsForSourcePorts(srcEntry *sourcePrefixEntry, fcProto *v3listenerpb.FilterChain) error {
+	var srcPorts []int
+	for _, port := range fcProto.GetFilterChainMatch().GetSourcePorts() {
+		srcPorts = append(srcPorts, int(port))
+	}
+
 	fc, err := filterChainFromProto(fcProto)
 	if err != nil {
 		return err
diff --git a/xds/internal/client/filter_chain_test.go b/xds/internal/client/filter_chain_test.go
index ba0ed944160d..ec100c561c96 100644
--- a/xds/internal/client/filter_chain_test.go
+++ b/xds/internal/client/filter_chain_test.go
@@ -402,8 +402,7 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 						},
 					},
 				},
-				def:   &FilterChain{},
-				fcCnt: 1,
+				def: &FilterChain{},
 			},
 		},
 		{
@@ -471,7 +470,6 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 						IdentityCertName:     "defaultIdentityCertName",
 					},
 				},
-				fcCnt: 1,
 			},
 		},
 		{
@@ -560,7 +558,6 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 						RequireClientCert:    true,
 					},
 				},
-				fcCnt: 1,
 			},
 		},
 	}
@@ -648,8 +645,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						},
 					},
 				},
-				def:   &FilterChain{},
-				fcCnt: 3,
+				def: &FilterChain{},
 			},
 		},
 		{
@@ -704,8 +700,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						},
 					},
 				},
-				def:   &FilterChain{},
-				fcCnt: 2,
+				def: &FilterChain{},
 			},
 		},
 		{
@@ -757,8 +752,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						},
 					},
 				},
-				def:   &FilterChain{},
-				fcCnt: 2,
+				def: &FilterChain{},
 			},
 		},
 		{
@@ -818,8 +812,126 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						},
 					},
 				},
-				def:   &FilterChain{},
-				fcCnt: 2,
+				def: &FilterChain{},
+			},
+		},
+		{
+			desc: "some chains have unsupported fields",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:      []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)},
+							TransportProtocol: "raw_buffer",
+						},
+					},
+					{
+						// This chain will be dropped in favor of the above
+						// filter chain because they both have the same
+						// destination prefix, but this one has an empty
+						// transport protocol while the above chain has the more
+						// preferred "raw_buffer".
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:       []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)},
+							TransportProtocol:  "",
+							SourceType:         v3listenerpb.FilterChainMatch_EXTERNAL,
+							SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 16)},
+						},
+					},
+					{
+						// This chain will be dropped for unsupported server
+						// names.
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.1", 32)},
+							ServerNames:  []string{"foo", "bar"},
+						},
+					},
+					{
+						// This chain will be dropped for unsupported transport
+						// protocol.
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:      []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.2", 32)},
+							TransportProtocol: "not-raw-buffer",
+						},
+					},
+					{
+						// This chain will be dropped for unsupported
+						// application protocol.
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:         []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.3", 32)},
+							ApplicationProtocols: []string{"h2"},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					"0.0.0.0/0": {
+						net: zeroIP,
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"10.0.0.0/8": {
+						net: ipNetFromCIDR("10.0.0.0/8"),
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									"0.0.0.0/0": {
+										net: zeroIP,
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"192.168.100.1/32": {
+						net:        ipNetFromCIDR("192.168.100.1/32"),
+						srcTypeArr: [3]*sourcePrefixes{},
+					},
+					"192.168.100.2/32": {
+						net:        ipNetFromCIDR("192.168.100.2/32"),
+						srcTypeArr: [3]*sourcePrefixes{},
+					},
+					"192.168.100.3/32": {
+						net:        ipNetFromCIDR("192.168.100.3/32"),
+						srcTypeArr: [3]*sourcePrefixes{},
+					},
+				},
+				def: &FilterChain{},
 			},
 		},
 	}
@@ -931,12 +1043,40 @@ func TestLookup_Failures(t *testing.T) {
 				},
 			},
 			params: FilterChainLookupParams{
-				DestAddr:   net.IPv4(192, 168, 100, 1),
-				SourceAddr: net.IPv4(192, 168, 100, 1),
-				SourcePort: 80,
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 100, 1),
+				SourceAddr:            net.IPv4(192, 168, 100, 1),
+				SourcePort:            80,
 			},
 			wantErr: "no matching filter chain after all match criteria",
 		},
+		{
+			desc: "most specific match dropped for unsupported field",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						// This chain will be picked in the destination prefix
+						// stage, but will be dropped at the server names stage.
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.1", 32)},
+							ServerNames:  []string{"foo"},
+						},
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.0", 16)},
+						},
+					},
+				},
+			},
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.IPv4(192, 168, 100, 1),
+				SourceAddr:            net.IPv4(192, 168, 100, 1),
+				SourcePort:            80,
+			},
+			wantErr: "no matching filter chain based on source type match",
+		},
 	}
 
 	for _, test := range tests {
@@ -1142,8 +1282,6 @@ func (fci *FilterChainManager) Equal(other *FilterChainManager) bool {
 		return false
 	case !cmp.Equal(fci.def, other.def):
 		return false
-	case fci.fcCnt != other.fcCnt:
-		return false
 	}
 	return true
 }
@@ -1218,7 +1356,6 @@ func (fci *FilterChainManager) String() string {
 	if fci.def != nil {
 		sb.WriteString(fmt.Sprintf("default_filter_chain: %+v ", fci.def))
 	}
-	sb.WriteString(fmt.Sprintf("filter_chain_count: %d ", fci.fcCnt))
 	return sb.String()
 }
 
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 42421f66b2d1..76ea543b2d94 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -1389,7 +1389,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 									},
 								},
 							},
-							fcCnt: 1,
 						},
 					},
 					Raw: listenerEmptyTransportSocket,
@@ -1553,7 +1552,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 									IdentityCertName:     "defaultIdentityCertName",
 								},
 							},
-							fcCnt: 1,
 						},
 					},
 					Raw: listenerNoValidationContext,
@@ -1607,7 +1605,6 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 									RequireClientCert:    true,
 								},
 							},
-							fcCnt: 1,
 						},
 					},
 					Raw: listenerWithValidationContext,

From 2df4370b332809e4daf1e2109b2389500e64c1c0 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 8 Apr 2021 16:02:52 -0700
Subject: [PATCH 414/481] examples: update xds examples for PSM security
 (#4256)

---
 examples/features/xds/client/main.go | 66 +++++++-------------
 examples/features/xds/server/main.go | 93 ++++++++++------------------
 2 files changed, 53 insertions(+), 106 deletions(-)

diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go
index b1daa1cae9c8..97918faa2245 100644
--- a/examples/features/xds/client/main.go
+++ b/examples/features/xds/client/main.go
@@ -16,78 +16,56 @@
  *
  */
 
-// Package main implements a client for Greeter service.
+// Binary main implements a client for Greeter service using gRPC's client-side
+// support for xDS APIs.
 package main
 
 import (
 	"context"
 	"flag"
-	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 
 	_ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers.
 )
 
-const (
-	defaultTarget = "localhost:50051"
-	defaultName   = "world"
+var (
+	target   = flag.String("target", "xds:///localhost:50051", "uri of the Greeter Server, e.g. 'xds:///helloworld-service:8080'")
+	name     = flag.String("name", "world", "name you wished to be greeted by the server")
+	xdsCreds = flag.Bool("xds_creds", false, "whether the server should use xDS APIs to receive security configuration")
 )
 
-var help = flag.Bool("help", false, "Print usage information")
-
-func init() {
-	flag.Usage = func() {
-		fmt.Fprintf(flag.CommandLine.Output(), `
-Usage: client [name [target]]
-
-  name
-        The name you wish to be greeted by. Defaults to %q
-  target
-        The URI of the server, e.g. "xds:///helloworld-service". Defaults to %q
-`, defaultName, defaultTarget)
-
-		flag.PrintDefaults()
-	}
-}
-
 func main() {
 	flag.Parse()
-	if *help {
-		flag.Usage()
-		return
-	}
-	args := flag.Args()
-
-	if len(args) > 2 {
-		flag.Usage()
-		return
-	}
 
-	name := defaultName
-	if len(args) > 0 {
-		name = args[0]
+	if !strings.HasPrefix(*target, "xds:///") {
+		log.Fatalf("-target must use a URI with scheme set to 'xds'")
 	}
 
-	target := defaultTarget
-	if len(args) > 1 {
-		target = args[1]
+	creds := insecure.NewCredentials()
+	if *xdsCreds {
+		log.Println("Using xDS credentials...")
+		var err error
+		if creds, err = xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}); err != nil {
+			log.Fatalf("failed to create client-side xDS credentials: %v", err)
+		}
 	}
-
-	// Set up a connection to the server.
-	conn, err := grpc.Dial(target, grpc.WithInsecure())
+	conn, err := grpc.Dial(*target, grpc.WithTransportCredentials(creds))
 	if err != nil {
-		log.Fatalf("did not connect: %v", err)
+		log.Fatalf("grpc.Dial(%s) failed: %v", *target, err)
 	}
 	defer conn.Close()
-	c := pb.NewGreeterClient(conn)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	r, err := c.SayHello(ctx, &pb.HelloRequest{Name: name})
+	c := pb.NewGreeterClient(conn)
+	r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name})
 	if err != nil {
 		log.Fatalf("could not greet: %v", err)
 	}
diff --git a/examples/features/xds/server/main.go b/examples/features/xds/server/main.go
index 7e0815645e5a..0367060f4b5d 100644
--- a/examples/features/xds/server/main.go
+++ b/examples/features/xds/server/main.go
@@ -16,7 +16,8 @@
  *
  */
 
-// Package main starts Greeter service that will response with the hostname.
+// Binary server demonstrated gRPC's support for xDS APIs on the server-side. It
+// exposes the Greeter service that will response with the hostname.
 package main
 
 import (
@@ -27,36 +28,29 @@ import (
 	"math/rand"
 	"net"
 	"os"
-	"strconv"
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"
 	"google.golang.org/grpc/health"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
-	"google.golang.org/grpc/reflection"
+	"google.golang.org/grpc/xds"
 )
 
-var help = flag.Bool("help", false, "Print usage information")
-
-const (
-	defaultPort = 50051
+var (
+	port     = flag.Int("port", 50051, "the port to serve Greeter service requests on. Health service will be served on `port+1`")
+	xdsCreds = flag.Bool("xds_creds", false, "whether the server should use xDS APIs to receive security configuration")
 )
 
-// server is used to implement helloworld.GreeterServer.
+// server implements helloworld.GreeterServer interface.
 type server struct {
 	pb.UnimplementedGreeterServer
-
 	serverName string
 }
 
-func newServer(serverName string) *server {
-	return &server{
-		serverName: serverName,
-	}
-}
-
-// SayHello implements helloworld.GreeterServer
+// SayHello implements helloworld.GreeterServer interface.
 func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
 	log.Printf("Received: %v", in.GetName())
 	return &pb.HelloReply{Message: "Hello " + in.GetName() + ", from " + s.serverName}, nil
@@ -72,65 +66,40 @@ func determineHostname() string {
 	return hostname
 }
 
-func init() {
-	flag.Usage = func() {
-		fmt.Fprintf(flag.CommandLine.Output(), `
-Usage: server [port [hostname]]
-
-  port
-        The listen port. Defaults to %d
-  hostname
-        The name clients will see in greet responses. Defaults to the machine's hostname
-`, defaultPort)
-
-		flag.PrintDefaults()
-	}
-}
-
 func main() {
 	flag.Parse()
-	if *help {
-		flag.Usage()
-		return
-	}
-	args := flag.Args()
 
-	if len(args) > 2 {
-		flag.Usage()
-		return
+	greeterPort := fmt.Sprintf(":%d", *port)
+	greeterLis, err := net.Listen("tcp4", greeterPort)
+	if err != nil {
+		log.Fatalf("net.Listen(tcp4, %q) failed: %v", greeterPort, err)
 	}
 
-	port := defaultPort
-	if len(args) > 0 {
+	creds := insecure.NewCredentials()
+	if *xdsCreds {
+		log.Println("Using xDS credentials...")
 		var err error
-		port, err = strconv.Atoi(args[0])
-		if err != nil {
-			log.Printf("Invalid port number: %v", err)
-			flag.Usage()
-			return
+		if creds, err = xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}); err != nil {
+			log.Fatalf("failed to create server-side xDS credentials: %v", err)
 		}
 	}
 
-	var hostname string
-	if len(args) > 1 {
-		hostname = args[1]
-	}
-	if hostname == "" {
-		hostname = determineHostname()
-	}
+	greeterServer := xds.NewGRPCServer(grpc.Creds(creds))
+	pb.RegisterGreeterServer(greeterServer, &server{serverName: determineHostname()})
 
-	lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
+	healthPort := fmt.Sprintf(":%d", *port+1)
+	healthLis, err := net.Listen("tcp4", healthPort)
 	if err != nil {
-		log.Fatalf("failed to listen: %v", err)
+		log.Fatalf("net.Listen(tcp4, %q) failed: %v", healthPort, err)
 	}
-	s := grpc.NewServer()
-	pb.RegisterGreeterServer(s, newServer(hostname))
-
-	reflection.Register(s)
+	grpcServer := grpc.NewServer()
 	healthServer := health.NewServer()
 	healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)
-	healthpb.RegisterHealthServer(s, healthServer)
+	healthpb.RegisterHealthServer(grpcServer, healthServer)
 
-	log.Printf("serving on %s, hostname %s", lis.Addr(), hostname)
-	s.Serve(lis)
+	log.Printf("Serving GreeterService on %s and HealthService on %s", greeterLis.Addr().String(), healthLis.Addr().String())
+	go func() {
+		greeterServer.Serve(greeterLis)
+	}()
+	grpcServer.Serve(healthLis)
 }

From 1d1bbb55b381f39fbe93edbb1d0fd96a6b1ecaef Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 8 Apr 2021 16:11:44 -0700
Subject: [PATCH 415/481] weightedtarget: handle updating child policy name
 (#4309)

---
 .../balancer/weightedtarget/weightedtarget.go | 10 +++++
 .../weightedtarget_config_test.go             | 28 ++++++-------
 .../weightedtarget/weightedtarget_test.go     | 40 +++++++++++++++++++
 3 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index 02b199258cd2..ac1aaecd8e51 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -115,6 +115,16 @@ func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnStat
 			w.bg.Add(name, balancer.Get(newT.ChildPolicy.Name))
 			// Not trigger a state/picker update. Wait for the new sub-balancer
 			// to send its updates.
+		} else if newT.ChildPolicy.Name != oldT.ChildPolicy.Name {
+			// If the child policy name is differet, remove from balancer group
+			// and re-add.
+			w.stateAggregator.Remove(name)
+			w.bg.Remove(name)
+			w.stateAggregator.Add(name, newT.Weight)
+			w.bg.Add(name, balancer.Get(newT.ChildPolicy.Name))
+			// Trigger a state/picker update, because we don't want `ClientConn`
+			// to pick this sub-balancer anymore.
+			rebuildStateAndPicker = true
 		} else if newT.Weight != oldT.Weight {
 			// If this is an existing sub-balancer, update weight if necessary.
 			w.stateAggregator.UpdateWeight(name, newT.Weight)
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
index 2208117f60e1..92dff8f5fbfc 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
@@ -24,7 +24,8 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/balancer"
 	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
-	_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer"
+
+	_ "google.golang.org/grpc/xds/internal/balancer/lrs" // Register LRS balancer, so we can use it as child policy in the config tests.
 )
 
 const (
@@ -32,24 +33,23 @@ const (
   "targets": {
 	"cluster_1" : {
 	  "weight":75,
-	  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+	  "childPolicy":[{"lrs_experimental":{"clusterName":"cluster_1","lrsLoadReportingServerName":"lrs.server","locality":{"zone":"test-zone-1"}}}]
 	},
 	"cluster_2" : {
 	  "weight":25,
-	  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+	  "childPolicy":[{"lrs_experimental":{"clusterName":"cluster_2","lrsLoadReportingServerName":"lrs.server","locality":{"zone":"test-zone-2"}}}]
 	}
   }
 }`
-
-	cdsName = "cds_experimental"
+	lrsBalancerName = "lrs_experimental"
 )
 
 var (
-	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
-	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
-	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
-	cdsConfigJSON2  = `{"cluster":"cluster_2"}`
-	cdsConfig2, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON2))
+	lrsConfigParser = balancer.Get(lrsBalancerName).(balancer.ConfigParser)
+	lrsConfigJSON1  = `{"clusterName":"cluster_1","lrsLoadReportingServerName":"lrs.server","locality":{"zone":"test-zone-1"}}`
+	lrsConfig1, _   = lrsConfigParser.ParseConfig([]byte(lrsConfigJSON1))
+	lrsConfigJSON2  = `{"clusterName":"cluster_2","lrsLoadReportingServerName":"lrs.server","locality":{"zone":"test-zone-2"}}`
+	lrsConfig2, _   = lrsConfigParser.ParseConfig([]byte(lrsConfigJSON2))
 )
 
 func Test_parseConfig(t *testing.T) {
@@ -73,15 +73,15 @@ func Test_parseConfig(t *testing.T) {
 					"cluster_1": {
 						Weight: 75,
 						ChildPolicy: &internalserviceconfig.BalancerConfig{
-							Name:   cdsName,
-							Config: cdsConfig1,
+							Name:   lrsBalancerName,
+							Config: lrsConfig1,
 						},
 					},
 					"cluster_2": {
 						Weight: 25,
 						ChildPolicy: &internalserviceconfig.BalancerConfig{
-							Name:   cdsName,
-							Config: cdsConfig2,
+							Name:   lrsBalancerName,
+							Config: lrsConfig2,
 						},
 					},
 				},
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
index 7f9e566ca5b5..131f89832c79 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_test.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
@@ -215,6 +215,46 @@ func TestWeightedTarget(t *testing.T) {
 	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
 		t.Fatalf("want %v, got %v", want, err)
 	}
+
+	// Replace child policy of "cluster_1" to "round_robin".
+	config3, err := wtbParser.ParseConfig([]byte(`{"targets":{"cluster_2":{"weight":1,"childPolicy":[{"round_robin":""}]}}}`))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddr4 := resolver.Address{Addr: testBackendAddrStrs[0], Attributes: nil}
+	if err := wtb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{
+			hierarchy.Set(wantAddr4, []string{"cluster_2"}),
+		}},
+		BalancerConfig: config3,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	addr4 := <-cc.NewSubConnAddrsCh
+	if want := []resolver.Address{
+		hierarchy.Set(wantAddr4, []string{}),
+	}; !cmp.Equal(addr4, want, cmp.AllowUnexported(attributes.Attributes{})) {
+		t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr4, want, cmp.AllowUnexported(attributes.Attributes{})))
+	}
+
+	// Send subconn state change.
+	sc4 := <-cc.NewSubConnCh
+	wtb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	wtb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Test pick with one backend.
+	p3 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p3.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc4, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc4)
+		}
+	}
 }
 
 func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {

From d6abfb459860721299c6f0bc7ffcbed5f9feebe4 Mon Sep 17 00:00:00 2001
From: Aliaksandr Mianzhynski <amenzhinsky@gmail.com>
Date: Sat, 10 Apr 2021 02:30:59 +0300
Subject: [PATCH 416/481] cmd/protoc-gen-go-grpc: add protoc and
 protoc-gen-go-grpc versions to top comment (#4313)

---
 .../grpc_lb_v1/load_balancer_grpc.pb.go       |  4 ++++
 .../proto/grpc_lookup_v1/rls_grpc.pb.go       |  4 ++++
 channelz/grpc_channelz_v1/channelz_grpc.pb.go |  4 ++++
 cmd/protoc-gen-go-grpc/grpc.go                | 21 ++++++++++++++++++-
 .../proto/grpc_gcp/handshaker_grpc.pb.go      |  4 ++++
 .../meshca/internal/v1/meshca_grpc.pb.go      |  4 ++++
 examples/features/proto/echo/echo_grpc.pb.go  |  4 ++++
 .../helloworld/helloworld_grpc.pb.go          |  4 ++++
 .../routeguide/route_guide_grpc.pb.go         |  4 ++++
 health/grpc_health_v1/health_grpc.pb.go       |  4 ++++
 .../grpc_testing/benchmark_service_grpc.pb.go |  4 ++++
 .../report_qps_scenario_service_grpc.pb.go    |  4 ++++
 interop/grpc_testing/test_grpc.pb.go          |  4 ++++
 .../grpc_testing/worker_service_grpc.pb.go    |  4 ++++
 profiling/proto/service_grpc.pb.go            |  4 ++++
 .../reflection_grpc.pb.go                     |  4 ++++
 reflection/grpc_testing/test_grpc.pb.go       |  4 ++++
 stress/grpc_testing/metrics_grpc.pb.go        |  4 ++++
 test/grpc_testing/test_grpc.pb.go             |  4 ++++
 19 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
index d56b77cca634..50cc9da4a907 100644
--- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
+++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/lb/v1/load_balancer.proto
 
 package grpc_lb_v1
 
diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
index b469089ed570..39d79e13343e 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/lookup/v1/rls.proto
 
 package grpc_lookup_v1
 
diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
index 051d1ac440c7..ee425c219940 100644
--- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go
+++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/channelz/v1/channelz.proto
 
 package grpc_channelz_v1
 
diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go
index 1e787344ebcc..24ad747cc91f 100644
--- a/cmd/protoc-gen-go-grpc/grpc.go
+++ b/cmd/protoc-gen-go-grpc/grpc.go
@@ -24,7 +24,6 @@ import (
 	"strings"
 
 	"google.golang.org/protobuf/compiler/protogen"
-
 	"google.golang.org/protobuf/types/descriptorpb"
 )
 
@@ -43,6 +42,14 @@ func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated
 	filename := file.GeneratedFilenamePrefix + "_grpc.pb.go"
 	g := gen.NewGeneratedFile(filename, file.GoImportPath)
 	g.P("// Code generated by protoc-gen-go-grpc. DO NOT EDIT.")
+	g.P("// versions:")
+	g.P("// - protoc-gen-go-grpc v", version)
+	g.P("// - protoc             ", protocVersion(gen))
+	if file.Proto.GetOptions().GetDeprecated() {
+		g.P("// ", file.Desc.Path(), " is a deprecated file.")
+	} else {
+		g.P("// source: ", file.Desc.Path())
+	}
 	g.P()
 	g.P("package ", file.GoPackageName)
 	g.P()
@@ -50,6 +57,18 @@ func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated
 	return g
 }
 
+func protocVersion(gen *protogen.Plugin) string {
+	v := gen.Request.GetCompilerVersion()
+	if v == nil {
+		return "(unknown)"
+	}
+	var suffix string
+	if s := v.GetSuffix(); s != "" {
+		suffix = "-" + s
+	}
+	return fmt.Sprintf("v%d.%d.%d%s", v.GetMajor(), v.GetMinor(), v.GetPatch(), suffix)
+}
+
 // generateFileContent generates the gRPC service definitions, excluding the package statement.
 func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile) {
 	if len(file.Services) == 0 {
diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
index efdbd13fa304..a02c4582815d 100644
--- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
+++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/gcp/handshaker.proto
 
 package grpc_gcp
 
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
index e53a61598aba..4663ff1ef35c 100644
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: istio/google/security/meshca/v1/meshca.proto
 
 package google_security_meshca_v1
 
diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go
index 052087dae369..e1d24b1e8309 100644
--- a/examples/features/proto/echo/echo_grpc.pb.go
+++ b/examples/features/proto/echo/echo_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: examples/features/proto/echo/echo.proto
 
 package echo
 
diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go
index 39a0301c16b2..ae27dfa3cfee 100644
--- a/examples/helloworld/helloworld/helloworld_grpc.pb.go
+++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: examples/helloworld/helloworld/helloworld.proto
 
 package helloworld
 
diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go
index 66860e63c476..efa7c28ce6f5 100644
--- a/examples/route_guide/routeguide/route_guide_grpc.pb.go
+++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: examples/route_guide/routeguide/route_guide.proto
 
 package routeguide
 
diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go
index 386d16ce62d1..bdc3ae284e7a 100644
--- a/health/grpc_health_v1/health_grpc.pb.go
+++ b/health/grpc_health_v1/health_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/health/v1/health.proto
 
 package grpc_health_v1
 
diff --git a/interop/grpc_testing/benchmark_service_grpc.pb.go b/interop/grpc_testing/benchmark_service_grpc.pb.go
index 1dcba4587d29..f4e4436e97e8 100644
--- a/interop/grpc_testing/benchmark_service_grpc.pb.go
+++ b/interop/grpc_testing/benchmark_service_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/testing/benchmark_service.proto
 
 package grpc_testing
 
diff --git a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
index b0fe8c8f5ee5..4bf3fce68ab1 100644
--- a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
+++ b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/testing/report_qps_scenario_service.proto
 
 package grpc_testing
 
diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go
index ad5310aed623..137a1e98ce6e 100644
--- a/interop/grpc_testing/test_grpc.pb.go
+++ b/interop/grpc_testing/test_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/testing/test.proto
 
 package grpc_testing
 
diff --git a/interop/grpc_testing/worker_service_grpc.pb.go b/interop/grpc_testing/worker_service_grpc.pb.go
index cc49b22b9261..a97366df09a2 100644
--- a/interop/grpc_testing/worker_service_grpc.pb.go
+++ b/interop/grpc_testing/worker_service_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: grpc/testing/worker_service.proto
 
 package grpc_testing
 
diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go
index bfdcc69bffb8..a0656149bda1 100644
--- a/profiling/proto/service_grpc.pb.go
+++ b/profiling/proto/service_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: profiling/proto/service.proto
 
 package proto
 
diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
index c2b7429a06b0..7d05c14ebd89 100644
--- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
+++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: reflection/grpc_reflection_v1alpha/reflection.proto
 
 package grpc_reflection_v1alpha
 
diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go
index 76ec8d52d684..235b5d82484b 100644
--- a/reflection/grpc_testing/test_grpc.pb.go
+++ b/reflection/grpc_testing/test_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: reflection/grpc_testing/test.proto
 
 package grpc_testing
 
diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go
index 2ece03255630..0730fad49a46 100644
--- a/stress/grpc_testing/metrics_grpc.pb.go
+++ b/stress/grpc_testing/metrics_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: stress/grpc_testing/metrics.proto
 
 package grpc_testing
 
diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go
index ab3b68a92bcc..76b3935620ca 100644
--- a/test/grpc_testing/test_grpc.pb.go
+++ b/test/grpc_testing/test_grpc.pb.go
@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.1.0
+// - protoc             v3.14.0
+// source: test/grpc_testing/test.proto
 
 package grpc_testing
 

From fab5982df20a27885393f866db267ee7b35808d2 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 9 Apr 2021 16:49:25 -0700
Subject: [PATCH 417/481] xds: server-side listener network filter validation
 (#4312)

---
 internal/testutils/marshal_any.go             |   36 +
 xds/internal/client/filter_chain_test.go      |   25 +-
 xds/internal/client/lds_test.go               | 1305 +++++++----------
 xds/internal/client/xds.go                    |   62 +
 xds/internal/httpfilter/httpfilter.go         |    3 -
 xds/internal/httpfilter/router/router.go      |   17 +-
 .../test/xds_server_integration_test.go       |   46 +-
 7 files changed, 655 insertions(+), 839 deletions(-)
 create mode 100644 internal/testutils/marshal_any.go

diff --git a/internal/testutils/marshal_any.go b/internal/testutils/marshal_any.go
new file mode 100644
index 000000000000..9ddef6de15d6
--- /dev/null
+++ b/internal/testutils/marshal_any.go
@@ -0,0 +1,36 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package testutils
+
+import (
+	"fmt"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/protobuf/types/known/anypb"
+)
+
+// MarshalAny is a convenience function to marshal protobuf messages into any
+// protos. It will panic if the marshaling fails.
+func MarshalAny(m proto.Message) *anypb.Any {
+	a, err := ptypes.MarshalAny(m)
+	if err != nil {
+		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
+	}
+	return a
+}
diff --git a/xds/internal/client/filter_chain_test.go b/xds/internal/client/filter_chain_test.go
index ec100c561c96..e66f518828ac 100644
--- a/xds/internal/client/filter_chain_test.go
+++ b/xds/internal/client/filter_chain_test.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -247,7 +248,7 @@ func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.UpstreamTlsContext{}),
+								TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{}),
 							},
 						},
 					},
@@ -282,7 +283,7 @@ func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{}),
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{}),
 							},
 						},
 					},
@@ -298,7 +299,7 @@ func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 									CommonTlsContext: &v3tlspb.CommonTlsContext{
 										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
 											ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
@@ -322,7 +323,7 @@ func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 									CommonTlsContext: &v3tlspb.CommonTlsContext{
 										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -346,7 +347,7 @@ func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 									CommonTlsContext: &v3tlspb.CommonTlsContext{},
 								}),
 							},
@@ -413,7 +414,7 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 									CommonTlsContext: &v3tlspb.CommonTlsContext{
 										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 											InstanceName:    "identityPluginInstance",
@@ -429,7 +430,7 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 					TransportSocket: &v3corepb.TransportSocket{
 						Name: "envoy.transport_sockets.tls",
 						ConfigType: &v3corepb.TransportSocket_TypedConfig{
-							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 								CommonTlsContext: &v3tlspb.CommonTlsContext{
 									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 										InstanceName:    "defaultIdentityPluginInstance",
@@ -480,7 +481,7 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 						TransportSocket: &v3corepb.TransportSocket{
 							Name: "envoy.transport_sockets.tls",
 							ConfigType: &v3corepb.TransportSocket_TypedConfig{
-								TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 									CommonTlsContext: &v3tlspb.CommonTlsContext{
 										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -504,7 +505,7 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 					TransportSocket: &v3corepb.TransportSocket{
 						Name: "envoy.transport_sockets.tls",
 						ConfigType: &v3corepb.TransportSocket_TypedConfig{
-							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 								RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 								CommonTlsContext: &v3tlspb.CommonTlsContext{
 									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -1101,7 +1102,7 @@ func TestLookup_Successes(t *testing.T) {
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+						TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 							CommonTlsContext: &v3tlspb.CommonTlsContext{
 								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "instance1"},
 							},
@@ -1115,7 +1116,7 @@ func TestLookup_Successes(t *testing.T) {
 			TransportSocket: &v3corepb.TransportSocket{
 				Name: "envoy.transport_sockets.tls",
 				ConfigType: &v3corepb.TransportSocket_TypedConfig{
-					TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+					TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 						CommonTlsContext: &v3tlspb.CommonTlsContext{
 							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "default"},
 						},
@@ -1434,7 +1435,7 @@ func transportSocketWithInstanceName(name string) *v3corepb.TransportSocket {
 	return &v3corepb.TransportSocket{
 		Name: "envoy.transport_sockets.tls",
 		ConfigType: &v3corepb.TransportSocket_TypedConfig{
-			TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+			TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 				CommonTlsContext: &v3tlspb.CommonTlsContext{
 					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: name},
 				},
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 76ea543b2d94..21718a4edc5e 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -27,13 +27,14 @@ import (
 	v1typepb "github.com/cncf/udpa/go/udpa/type/v1"
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
 	spb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
-	"google.golang.org/protobuf/types/known/durationpb"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
@@ -54,13 +55,14 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 		v2RouteConfigName = "v2RouteConfig"
 		v3RouteConfigName = "v3RouteConfig"
 		routeName         = "routeName"
+		testVersion       = "test-version-lds-client"
 	)
 
 	var (
-		v2Lis = &anypb.Any{
-			TypeUrl: version.V2ListenerURL,
-			Value: func() []byte {
-				cm := &v2httppb.HttpConnectionManager{
+		v2Lis = testutils.MarshalAny(&v2xdspb.Listener{
+			Name: v2LDSTarget,
+			ApiListener: &v2listenerpb.ApiListener{
+				ApiListener: testutils.MarshalAny(&v2httppb.HttpConnectionManager{
 					RouteSpecifier: &v2httppb.HttpConnectionManager_Rds{
 						Rds: &v2httppb.Rds{
 							ConfigSource: &v2corepb.ConfigSource{
@@ -69,21 +71,9 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 							RouteConfigName: v2RouteConfigName,
 						},
 					},
-				}
-				mcm, _ := proto.Marshal(cm)
-				lis := &v2xdspb.Listener{
-					Name: v2LDSTarget,
-					ApiListener: &v2listenerpb.ApiListener{
-						ApiListener: &anypb.Any{
-							TypeUrl: version.V2HTTPConnManagerURL,
-							Value:   mcm,
-						},
-					},
-				}
-				mLis, _ := proto.Marshal(lis)
-				return mLis
-			}(),
-		}
+				}),
+			},
+		})
 		customFilter = &v3httppb.HttpFilter{
 			Name:       "customFilter",
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig},
@@ -132,10 +122,10 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig},
 			IsOptional: true,
 		}
-		v3LisWithInlineRoute = &anypb.Any{
-			TypeUrl: version.V3ListenerURL,
-			Value: func() []byte {
-				hcm := &v3httppb.HttpConnectionManager{
+		v3LisWithInlineRoute = testutils.MarshalAny(&v3listenerpb.Listener{
+			Name: v3LDSTarget,
+			ApiListener: &v3listenerpb.ApiListener{
+				ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
 					RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
 						RouteConfig: &v3routepb.RouteConfiguration{
 							Name: routeName,
@@ -153,50 +143,40 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 					CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
 						MaxStreamDuration: durationpb.New(time.Second),
 					},
-				}
-				mcm := marshalAny(hcm)
-				lis := &v3listenerpb.Listener{
-					Name: v3LDSTarget,
-					ApiListener: &v3listenerpb.ApiListener{
-						ApiListener: mcm,
-					},
-				}
-				mLis, _ := proto.Marshal(lis)
-				return mLis
-			}(),
-		}
+				}),
+			},
+		})
 		v3LisWithFilters = func(fs ...*v3httppb.HttpFilter) *anypb.Any {
-			hcm := &v3httppb.HttpConnectionManager{
-				RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-					Rds: &v3httppb.Rds{
-						ConfigSource: &v3corepb.ConfigSource{
-							ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-						},
-						RouteConfigName: v3RouteConfigName,
-					},
-				},
-				CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
-					MaxStreamDuration: durationpb.New(time.Second),
+			return testutils.MarshalAny(&v3listenerpb.Listener{
+				Name: v3LDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: testutils.MarshalAny(
+						&v3httppb.HttpConnectionManager{
+							RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+								Rds: &v3httppb.Rds{
+									ConfigSource: &v3corepb.ConfigSource{
+										ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+									},
+									RouteConfigName: v3RouteConfigName,
+								},
+							},
+							CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
+								MaxStreamDuration: durationpb.New(time.Second),
+							},
+							HttpFilters: fs,
+						}),
 				},
-				HttpFilters: fs,
-			}
-			return &anypb.Any{
-				TypeUrl: version.V3ListenerURL,
-				Value: func() []byte {
-					mcm := marshalAny(hcm)
-					lis := &v3listenerpb.Listener{
-						Name: v3LDSTarget,
-						ApiListener: &v3listenerpb.ApiListener{
-							ApiListener: mcm,
-						},
-					}
-					mLis, _ := proto.Marshal(lis)
-					return mLis
-				}(),
-			}
+			})
+		}
+		errMD = UpdateMetadata{
+			Status:  ServiceStatusNACKed,
+			Version: testVersion,
+			ErrState: &UpdateErrorMetadata{
+				Version: testVersion,
+				Err:     errPlaceHolder,
+			},
 		}
 	)
-	const testVersion = "test-version-lds-client"
 
 	tests := []struct {
 		name       string
@@ -209,15 +189,8 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 		{
 			name:      "non-listener resource",
 			resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:    errMD,
+			wantErr:   true,
 		},
 		{
 			name: "badly marshaled listener resource",
@@ -240,181 +213,80 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 				},
 			},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name: "wrong type in apiListener",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							ApiListener: &v3listenerpb.ApiListener{
-								ApiListener: &anypb.Any{
-									TypeUrl: version.V2ListenerURL,
-									Value: func() []byte {
-										cm := &v3httppb.HttpConnectionManager{
-											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-												Rds: &v3httppb.Rds{
-													ConfigSource: &v3corepb.ConfigSource{
-														ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-													},
-													RouteConfigName: v3RouteConfigName,
-												},
-											},
-										}
-										mcm, _ := proto.Marshal(cm)
-										return mcm
-									}(),
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name: v3LDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: testutils.MarshalAny(&v2xdspb.Listener{}),
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name: "empty httpConnMgr in apiListener",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							ApiListener: &v3listenerpb.ApiListener{
-								ApiListener: &anypb.Any{
-									TypeUrl: version.V2ListenerURL,
-									Value: func() []byte {
-										cm := &v3httppb.HttpConnectionManager{
-											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-												Rds: &v3httppb.Rds{},
-											},
-										}
-										mcm, _ := proto.Marshal(cm)
-										return mcm
-									}(),
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name: v3LDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
+						RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+							Rds: &v3httppb.Rds{},
+						},
+					}),
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name: "scopedRoutes routeConfig in apiListener",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							ApiListener: &v3listenerpb.ApiListener{
-								ApiListener: &anypb.Any{
-									TypeUrl: version.V2ListenerURL,
-									Value: func() []byte {
-										cm := &v3httppb.HttpConnectionManager{
-											RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
-										}
-										mcm, _ := proto.Marshal(cm)
-										return mcm
-									}(),
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name: v3LDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: &anypb.Any{
+						TypeUrl: version.V2ListenerURL,
+						Value: func() []byte {
+							cm := &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+							}
+							mcm, _ := proto.Marshal(cm)
+							return mcm
+						}(),
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name: "rds.ConfigSource in apiListener is not ADS",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							ApiListener: &v3listenerpb.ApiListener{
-								ApiListener: &anypb.Any{
-									TypeUrl: version.V2ListenerURL,
-									Value: func() []byte {
-										cm := &v3httppb.HttpConnectionManager{
-											RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-												Rds: &v3httppb.Rds{
-													ConfigSource: &v3corepb.ConfigSource{
-														ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{
-															Path: "/some/path",
-														},
-													},
-													RouteConfigName: v3RouteConfigName,
-												},
-											},
-										}
-										mcm, _ := proto.Marshal(cm)
-										return mcm
-									}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name: v3LDSTarget,
+				ApiListener: &v3listenerpb.ApiListener{
+					ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
+						RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+							Rds: &v3httppb.Rds{
+								ConfigSource: &v3corepb.ConfigSource{
+									ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{
+										Path: "/some/path",
+									},
 								},
+								RouteConfigName: v3RouteConfigName,
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					}),
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name: "empty resource list",
@@ -507,15 +379,8 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			name:       "v3 with two filters with same name",
 			resources:  []*anypb.Any{v3LisWithFilters(customFilter, customFilter)},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name:      "v3 with two filters - same type different name",
@@ -544,15 +409,8 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			name:       "v3 with server-only filter",
 			resources:  []*anypb.Any{v3LisWithFilters(serverOnlyCustomFilter)},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name:      "v3 with optional server-only filter",
@@ -592,43 +450,22 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			name:       "v3 with err filter",
 			resources:  []*anypb.Any{v3LisWithFilters(errFilter)},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name:       "v3 with optional err filter",
 			resources:  []*anypb.Any{v3LisWithFilters(errOptionalFilter)},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name:       "v3 with unknown filter",
 			resources:  []*anypb.Any{v3LisWithFilters(unknownFilter)},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: true,
+			wantMD:     errMD,
+			wantErr:    true,
 		},
 		{
 			name:      "v3 with unknown filter (optional)",
@@ -745,14 +582,7 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 				v3LDSTarget: {RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, Raw: v3LisWithFilters()},
 				"bad":       {},
 			},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
+			wantMD:  errMD,
 			wantErr: true,
 		},
 	}
@@ -778,46 +608,51 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 }
 
 func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
-	const v3LDSTarget = "grpc/server?xds.resource.listening_address=0.0.0.0:9999"
+	const (
+		v3LDSTarget = "grpc/server?xds.resource.listening_address=0.0.0.0:9999"
+		testVersion = "test-version-lds-server"
+	)
 
 	var (
-		listenerEmptyTransportSocket = marshalAny(&v3listenerpb.Listener{
-			Name: v3LDSTarget,
-			Address: &v3corepb.Address{
-				Address: &v3corepb.Address_SocketAddress{
-					SocketAddress: &v3corepb.SocketAddress{
-						Address: "0.0.0.0",
-						PortSpecifier: &v3corepb.SocketAddress_PortValue{
-							PortValue: 9999,
-						},
+		emptyValidNetworkFilters = []*v3listenerpb.Filter{
+			{
+				Name: "filter-1",
+				ConfigType: &v3listenerpb.Filter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
+				},
+			},
+		}
+		localSocketAddress = &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: "0.0.0.0",
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: 9999,
 					},
 				},
 			},
+		}
+		listenerEmptyTransportSocket = testutils.MarshalAny(&v3listenerpb.Listener{
+			Name:    v3LDSTarget,
+			Address: localSocketAddress,
 			FilterChains: []*v3listenerpb.FilterChain{
 				{
-					Name: "filter-chain-1",
+					Name:    "filter-chain-1",
+					Filters: emptyValidNetworkFilters,
 				},
 			},
 		})
-		listenerNoValidationContext = marshalAny(&v3listenerpb.Listener{
-			Name: v3LDSTarget,
-			Address: &v3corepb.Address{
-				Address: &v3corepb.Address_SocketAddress{
-					SocketAddress: &v3corepb.SocketAddress{
-						Address: "0.0.0.0",
-						PortSpecifier: &v3corepb.SocketAddress_PortValue{
-							PortValue: 9999,
-						},
-					},
-				},
-			},
+		listenerNoValidationContext = testutils.MarshalAny(&v3listenerpb.Listener{
+			Name:    v3LDSTarget,
+			Address: localSocketAddress,
 			FilterChains: []*v3listenerpb.FilterChain{
 				{
-					Name: "filter-chain-1",
+					Name:    "filter-chain-1",
+					Filters: emptyValidNetworkFilters,
 					TransportSocket: &v3corepb.TransportSocket{
 						Name: "envoy.transport_sockets.tls",
 						ConfigType: &v3corepb.TransportSocket_TypedConfig{
-							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 								CommonTlsContext: &v3tlspb.CommonTlsContext{
 									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 										InstanceName:    "identityPluginInstance",
@@ -830,11 +665,12 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 				},
 			},
 			DefaultFilterChain: &v3listenerpb.FilterChain{
-				Name: "default-filter-chain-1",
+				Name:    "default-filter-chain-1",
+				Filters: emptyValidNetworkFilters,
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+						TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 							CommonTlsContext: &v3tlspb.CommonTlsContext{
 								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 									InstanceName:    "defaultIdentityPluginInstance",
@@ -846,25 +682,17 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 				},
 			},
 		})
-		listenerWithValidationContext = marshalAny(&v3listenerpb.Listener{
-			Name: v3LDSTarget,
-			Address: &v3corepb.Address{
-				Address: &v3corepb.Address_SocketAddress{
-					SocketAddress: &v3corepb.SocketAddress{
-						Address: "0.0.0.0",
-						PortSpecifier: &v3corepb.SocketAddress_PortValue{
-							PortValue: 9999,
-						},
-					},
-				},
-			},
+		listenerWithValidationContext = testutils.MarshalAny(&v3listenerpb.Listener{
+			Name:    v3LDSTarget,
+			Address: localSocketAddress,
 			FilterChains: []*v3listenerpb.FilterChain{
 				{
-					Name: "filter-chain-1",
+					Name:    "filter-chain-1",
+					Filters: emptyValidNetworkFilters,
 					TransportSocket: &v3corepb.TransportSocket{
 						Name: "envoy.transport_sockets.tls",
 						ConfigType: &v3corepb.TransportSocket_TypedConfig{
-							TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+							TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 								RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 								CommonTlsContext: &v3tlspb.CommonTlsContext{
 									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -884,11 +712,12 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 				},
 			},
 			DefaultFilterChain: &v3listenerpb.FilterChain{
-				Name: "default-filter-chain-1",
+				Name:    "default-filter-chain-1",
+				Filters: emptyValidNetworkFilters,
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+						TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 							RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 							CommonTlsContext: &v3tlspb.CommonTlsContext{
 								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
@@ -907,10 +736,16 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 				},
 			},
 		})
+		errMD = UpdateMetadata{
+			Status:  ServiceStatusNACKed,
+			Version: testVersion,
+			ErrState: &UpdateErrorMetadata{
+				Version: testVersion,
+				Err:     errPlaceHolder,
+			},
+		}
 	)
 
-	const testVersion = "test-version-lds-server"
-
 	tests := []struct {
 		name       string
 		resources  []*anypb.Any
@@ -920,448 +755,366 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 	}{
 		{
 			name: "non-empty listener filters",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							ListenerFilters: []*v3listenerpb.ListenerFilter{
-								{Name: "listener-filter-1"},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name: v3LDSTarget,
+				ListenerFilters: []*v3listenerpb.ListenerFilter{
+					{Name: "listener-filter-1"},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "unsupported field 'listener_filters'",
+			wantMD:     errMD,
+			wantErr:    "unsupported field 'listener_filters'",
 		},
 		{
 			name: "use_original_dst is set",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name:           v3LDSTarget,
-							UseOriginalDst: &wrapperspb.BoolValue{Value: true},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:           v3LDSTarget,
+				UseOriginalDst: &wrapperspb.BoolValue{Value: true},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "unsupported field 'use_original_dst'",
+			wantMD:     errMD,
+			wantErr:    "unsupported field 'use_original_dst'",
 		},
 		{
-			name: "no address field",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
-			},
+			name:       "no address field",
+			resources:  []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{Name: v3LDSTarget})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "no address field in LDS response",
+			wantMD:     errMD,
+			wantErr:    "no address field in LDS response",
 		},
 		{
 			name: "no socket address field",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name:    v3LDSTarget,
-							Address: &v3corepb.Address{},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: &v3corepb.Address{},
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "no socket_address field in LDS response",
+		},
+		{
+			name: "no filter chains and no default filter chain",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}},
+						Filters:          emptyValidNetworkFilters,
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
+			wantMD:     errMD,
+			wantErr:    "no supported filter chains and no default filter chain",
+		},
+		{
+			name: "missing http connection manager network filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+					},
 				},
-			},
-			wantErr: "no socket_address field in LDS response",
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "missing HttpConnectionManager filter",
 		},
 		{
-			name: "no filter chains and no default filter chain",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}},
+			name: "missing filter name in http filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+						Filters: []*v3listenerpb.Filter{
+							{
+								ConfigType: &v3listenerpb.Filter_TypedConfig{
+									TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
 								},
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "no supported filter chains and no default filter chain",
+			wantMD:     errMD,
+			wantErr:    "missing name field in filter",
 		},
 		{
-			name: "overlapping filter chain match criteria",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
+			name: "duplicate filter names in http filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+						Filters: []*v3listenerpb.Filter{
+							{
+								Name: "name",
+								ConfigType: &v3listenerpb.Filter_TypedConfig{
+									TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
 								},
 							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}},
-								},
-								{
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{},
-								},
-								{
-									FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}},
+							{
+								Name: "name",
+								ConfigType: &v3listenerpb.Filter_TypedConfig{
+									TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
 								},
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "multiple filter chains with overlapping matching rules are defined",
+			wantMD:     errMD,
+			wantErr:    "duplicate filter name",
 		},
 		{
-			name: "unexpected transport socket name",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "unsupported-transport-socket-name",
-									},
-								},
+			name: "unsupported oneof in typed config of http filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+						Filters: []*v3listenerpb.Filter{
+							{
+								Name:       "name",
+								ConfigType: &v3listenerpb.Filter_ConfigDiscovery{},
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
+			wantMD:     errMD,
+			wantErr:    "unsupported config_type",
+		},
+		{
+			name: "overlapping filter chain match criteria",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}},
+						Filters:          emptyValidNetworkFilters,
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{},
+						Filters:          emptyValidNetworkFilters,
+					},
+					{
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}},
+						Filters:          emptyValidNetworkFilters,
+					},
 				},
-			},
-			wantErr: "transport_socket field has unexpected name",
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "multiple filter chains with overlapping matching rules are defined",
 		},
 		{
-			name: "unexpected transport socket typedConfig URL",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
+			name: "unsupported network filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+						Filters: []*v3listenerpb.Filter{
+							{
+								Name: "name",
+								ConfigType: &v3listenerpb.Filter_TypedConfig{
+									TypedConfig: testutils.MarshalAny(&v3httppb.LocalReplyConfig{}),
 								},
 							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3UpstreamTLSContextURL,
-											},
-										},
-									},
-								},
-							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "transport_socket field has unexpected typeURL",
+			wantMD:     errMD,
+			wantErr:    "unsupported network filter",
 		},
 		{
-			name: "badly marshaled transport socket",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
+			name: "badly marshaled network filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+						Filters: []*v3listenerpb.Filter{
+							{
+								Name: "name",
+								ConfigType: &v3listenerpb.Filter_TypedConfig{
+									TypedConfig: &anypb.Any{
+										TypeUrl: version.V3HTTPConnManagerURL,
+										Value:   []byte{1, 2, 3, 4},
 									},
 								},
 							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value:   []byte{1, 2, 3, 4},
+						},
+					},
+				},
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "failed unmarshaling of network filter",
+		},
+		{
+			name: "client only http filter inside the network filter",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "filter-chain-1",
+						Filters: []*v3listenerpb.Filter{
+							{
+								Name: "hcm",
+								ConfigType: &v3listenerpb.Filter_TypedConfig{
+									TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
+										HttpFilters: []*v3httppb.HttpFilter{
+											{
+												Name:       "clientOnlyCustomFilter",
+												ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig},
 											},
 										},
-									},
+									}),
 								},
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
+			wantMD:     errMD,
+			wantErr:    "not supported server-side",
+		},
+		{
+			name: "unexpected transport socket name",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "unsupported-transport-socket-name",
+						},
+					},
 				},
-			},
-			wantErr: "failed to unmarshal DownstreamTlsContext in LDS response",
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "transport_socket field has unexpected name",
 		},
 		{
-			name: "missing CommonTlsContext",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
+			name: "unexpected transport socket typedConfig URL",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{}),
 							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value: func() []byte {
-													tls := &v3tlspb.DownstreamTlsContext{}
-													mtls, _ := proto.Marshal(tls)
-													return mtls
-												}(),
-											},
-										},
-									},
+						},
+					},
+				},
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "transport_socket field has unexpected typeURL",
+		},
+		{
+			name: "badly marshaled transport socket",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: &anypb.Any{
+									TypeUrl: version.V3DownstreamTLSContextURL,
+									Value:   []byte{1, 2, 3, 4},
 								},
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
+			wantMD:     errMD,
+			wantErr:    "failed to unmarshal DownstreamTlsContext in LDS response",
+		},
+		{
+			name: "missing CommonTlsContext",
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{}),
+							},
+						},
+					},
 				},
-			},
-			wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext",
+			})},
+			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
+			wantMD:     errMD,
+			wantErr:    "DownstreamTlsContext in LDS response does not contain a CommonTlsContext",
 		},
 		{
 			name: "unsupported validation context in transport socket",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value: func() []byte {
-													tls := &v3tlspb.DownstreamTlsContext{
-														CommonTlsContext: &v3tlspb.CommonTlsContext{
-															ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
-																ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
-																	Name: "foo-sds-secret",
-																},
-															},
-														},
-													}
-													mtls, _ := proto.Marshal(tls)
-													return mtls
-												}(),
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
+											ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
+												Name: "foo-sds-secret",
 											},
 										},
 									},
-								},
+								}),
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "validation context contains unexpected type",
+			wantMD:     errMD,
+			wantErr:    "validation context contains unexpected type",
 		},
 		{
 			name:      "empty transport socket",
@@ -1401,119 +1154,57 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 		},
 		{
 			name: "no identity and root certificate providers",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
+									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+									CommonTlsContext: &v3tlspb.CommonTlsContext{
+										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    "identityPluginInstance",
+											CertificateName: "identityCertName",
 										},
 									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value: func() []byte {
-													tls := &v3tlspb.DownstreamTlsContext{
-														RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-														CommonTlsContext: &v3tlspb.CommonTlsContext{
-															TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-																InstanceName:    "identityPluginInstance",
-																CertificateName: "identityCertName",
-															},
-														},
-													}
-													mtls, _ := proto.Marshal(tls)
-													return mtls
-												}(),
-											},
-										},
-									},
-								},
+								}),
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set",
+			wantMD:     errMD,
+			wantErr:    "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set",
 		},
 		{
 			name: "no identity certificate provider with require_client_cert",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: v3LDSTarget,
-							Address: &v3corepb.Address{
-								Address: &v3corepb.Address_SocketAddress{
-									SocketAddress: &v3corepb.SocketAddress{
-										Address: "0.0.0.0",
-										PortSpecifier: &v3corepb.SocketAddress_PortValue{
-											PortValue: 9999,
-										},
-									},
-								},
-							},
-							FilterChains: []*v3listenerpb.FilterChain{
-								{
-									Name: "filter-chain-1",
-									TransportSocket: &v3corepb.TransportSocket{
-										Name: "envoy.transport_sockets.tls",
-										ConfigType: &v3corepb.TransportSocket_TypedConfig{
-											TypedConfig: &anypb.Any{
-												TypeUrl: version.V3DownstreamTLSContextURL,
-												Value: func() []byte {
-													tls := &v3tlspb.DownstreamTlsContext{
-														CommonTlsContext: &v3tlspb.CommonTlsContext{},
-													}
-													mtls, _ := proto.Marshal(tls)
-													return mtls
-												}(),
-											},
-										},
-									},
-								},
+			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
+				Name:    v3LDSTarget,
+				Address: localSocketAddress,
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name:    "filter-chain-1",
+						Filters: emptyValidNetworkFilters,
+						TransportSocket: &v3corepb.TransportSocket{
+							Name: "envoy.transport_sockets.tls",
+							ConfigType: &v3corepb.TransportSocket_TypedConfig{
+								TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
+									CommonTlsContext: &v3tlspb.CommonTlsContext{},
+								}),
 							},
-						}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
+						},
+					},
 				},
-			},
+			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
-			wantMD: UpdateMetadata{
-				Status:  ServiceStatusNACKed,
-				Version: testVersion,
-				ErrState: &UpdateErrorMetadata{
-					Version: testVersion,
-					Err:     errPlaceHolder,
-				},
-			},
-			wantErr: "security configuration on the server-side does not contain identity certificate provider instance name",
+			wantMD:     errMD,
+			wantErr:    "security configuration on the server-side does not contain identity certificate provider instance name",
 		},
 		{
 			name:      "happy case with no validation context",
@@ -1619,6 +1310,10 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			oldFI := env.FaultInjectionSupport
+			env.FaultInjectionSupport = true
+			defer func() { env.FaultInjectionSupport = oldFI }()
+
 			gotUpdate, md, err := UnmarshalListener(testVersion, test.resources, nil)
 			if (err != nil) != (test.wantErr != "") {
 				t.Fatalf("UnmarshalListener(), got err: %v, wantErr: %v", err, test.wantErr)
@@ -1741,7 +1436,7 @@ var customFilterTypedStructConfig = &v1typepb.TypedStruct{
 var wrappedCustomFilterTypedStructConfig *anypb.Any
 
 func init() {
-	wrappedCustomFilterTypedStructConfig = marshalAny(customFilterTypedStructConfig)
+	wrappedCustomFilterTypedStructConfig = testutils.MarshalAny(customFilterTypedStructConfig)
 }
 
 var unknownFilterConfig = &anypb.Any{
@@ -1750,7 +1445,7 @@ var unknownFilterConfig = &anypb.Any{
 }
 
 func wrappedOptionalFilter(name string) *anypb.Any {
-	return marshalAny(&v3routepb.FilterConfig{
+	return testutils.MarshalAny(&v3routepb.FilterConfig{
 		IsOptional: true,
 		Config: &anypb.Any{
 			TypeUrl: name,
@@ -1758,11 +1453,3 @@ func wrappedOptionalFilter(name string) *anypb.Any {
 		},
 	})
 }
-
-func marshalAny(m proto.Message) *anypb.Any {
-	a, err := ptypes.MarshalAny(m)
-	if err != nil {
-		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
-	}
-	return a
-}
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 571fff670daa..c0caf5cceb57 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -270,6 +270,13 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 			Port:    strconv.Itoa(int(sockAddr.GetPortValue())),
 		},
 	}
+	chains := lis.GetFilterChains()
+	if def := lis.GetDefaultFilterChain(); def != nil {
+		chains = append(chains, def)
+	}
+	if err := validateNetworkFilterChains(chains); err != nil {
+		return nil, err
+	}
 
 	fcMgr, err := NewFilterChainManager(lis)
 	if err != nil {
@@ -279,6 +286,61 @@ func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, err
 	return lu, nil
 }
 
+func validateNetworkFilterChains(filterChains []*v3listenerpb.FilterChain) error {
+	for _, filterChain := range filterChains {
+		seenNames := make(map[string]bool, len(filterChain.GetFilters()))
+		seenHCM := false
+		for _, filter := range filterChain.GetFilters() {
+			name := filter.GetName()
+			if name == "" {
+				return fmt.Errorf("filter chain {%+v} is missing name field in filter: {%+v}", filterChain, filter)
+			}
+			if seenNames[name] {
+				return fmt.Errorf("filter chain {%+v} has duplicate filter name %q", filterChain, name)
+			}
+			seenNames[name] = true
+
+			// Network filters have a oneof field named `config_type` where we
+			// only support `TypedConfig` variant.
+			switch typ := filter.GetConfigType().(type) {
+			case *v3listenerpb.Filter_TypedConfig:
+				// The typed_config field has an `anypb.Any` proto which could
+				// directly contain the serialized bytes of the actual filter
+				// configuration, or it could be encoded as a `TypedStruct`.
+				// TODO: Add support for `TypedStruct`.
+				tc := filter.GetTypedConfig()
+
+				// The only network filter that we currently support is the v3
+				// HttpConnectionManager. So, we can directly check the type_url
+				// and unmarshal the config.
+				// TODO: Implement a registry of supported network filters (like
+				// we have for HTTP filters), when we have to support network
+				// filters other than HttpConnectionManager.
+				if tc.GetTypeUrl() != version.V3HTTPConnManagerURL {
+					return fmt.Errorf("filter chain {%+v} has unsupported network filter %q in filter {%+v}", filterChain, tc.GetTypeUrl(), filter)
+				}
+				hcm := &v3httppb.HttpConnectionManager{}
+				if err := ptypes.UnmarshalAny(tc, hcm); err != nil {
+					return fmt.Errorf("filter chain {%+v} failed unmarshaling of network filter {%+v}: %v", filterChain, filter, err)
+				}
+				// We currently don't support HTTP filters on the server-side.
+				// We will be adding support for it in the future. So, we want
+				// to make sure that the http_filters configuration is valid.
+				if _, err := processHTTPFilters(hcm.GetHttpFilters(), true); err != nil {
+					return err
+				}
+				seenHCM = true
+			default:
+				return fmt.Errorf("filter chain {%+v} has unsupported config_type %T in filter %s", filterChain, typ, filter.GetName())
+			}
+		}
+		if !seenHCM {
+			return fmt.Errorf("filter chain {%+v} missing HttpConnectionManager filter", filterChain)
+		}
+	}
+	return nil
+}
+
 // UnmarshalRouteConfig processes resources received in an RDS response,
 // validates them, and transforms them into a native struct which contains only
 // fields we are interested in. The provided hostname determines the route
diff --git a/xds/internal/httpfilter/httpfilter.go b/xds/internal/httpfilter/httpfilter.go
index 6650241fab71..1f5f005e9bd2 100644
--- a/xds/internal/httpfilter/httpfilter.go
+++ b/xds/internal/httpfilter/httpfilter.go
@@ -65,9 +65,6 @@ type ClientInterceptorBuilder interface {
 
 // ServerInterceptorBuilder constructs a Server Interceptor.  If this type is
 // implemented by a Filter, it is capable of working on a server.
-//
-// Server side filters are not currently supported, but this interface is
-// defined for clarity.
 type ServerInterceptorBuilder interface {
 	// BuildServerInterceptor uses the FilterConfigs produced above to produce
 	// an HTTP filter interceptor for servers.  config will always be non-nil,
diff --git a/xds/internal/httpfilter/router/router.go b/xds/internal/httpfilter/router/router.go
index 26e3acb5a4f4..b0f9d9d9a1e9 100644
--- a/xds/internal/httpfilter/router/router.go
+++ b/xds/internal/httpfilter/router/router.go
@@ -73,7 +73,10 @@ func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.Fil
 	return config{}, nil
 }
 
-var _ httpfilter.ClientInterceptorBuilder = builder{}
+var (
+	_ httpfilter.ClientInterceptorBuilder = builder{}
+	_ httpfilter.ServerInterceptorBuilder = builder{}
+)
 
 func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) {
 	if _, ok := cfg.(config); !ok {
@@ -88,6 +91,18 @@ func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (ir
 	return nil, nil
 }
 
+func (builder) BuildServerInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ServerInterceptor, error) {
+	if _, ok := cfg.(config); !ok {
+		return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg)
+	}
+	if override != nil {
+		return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override)
+	}
+	// The gRPC router is currently unimplemented on the server side. So we
+	// return a nil HTTPFilter, which will not be invoked.
+	return nil, nil
+}
+
 // The gRPC router filter does not currently support any configuration.  Verify
 // type only.
 type config struct {
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 7606a35218ca..9c2e6daebfec 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -33,27 +33,29 @@ import (
 	"strconv"
 	"testing"
 
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/uuid"
-	xds2 "google.golang.org/grpc/internal/xds"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
-	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/status"
-	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/testdata"
 	"google.golang.org/grpc/xds"
-	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 	"google.golang.org/grpc/xds/internal/version"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 )
 
 const (
@@ -151,8 +153,8 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 	cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile))
 
 	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := xds2.SetupBootstrapFile(xds2.BootstrapOptions{
-		Version:                            xds2.TransportV3,
+	bootstrapCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
+		Version:                            xdsinternal.TransportV3,
 		NodeID:                             nodeID,
 		ServerURI:                          fs.Address,
 		CertificateProviders:               cpc,
@@ -175,7 +177,7 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 	testpb.RegisterTestServiceServer(server, &testService{})
 
 	// Create a local listener and pass it to Serve().
-	lis, err := testutils.LocalTCPListener()
+	lis, err := xdstestutils.LocalTCPListener()
 	if err != nil {
 		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
 	}
@@ -229,6 +231,14 @@ func listenerResourceWithoutSecurityConfig(t *testing.T, lis net.Listener) *v3li
 		FilterChains: []*v3listenerpb.FilterChain{
 			{
 				Name: "filter-chain-1",
+				Filters: []*v3listenerpb.Filter{
+					{
+						Name: "filter-1",
+						ConfigType: &v3listenerpb.Filter_TypedConfig{
+							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
+						},
+					},
+				},
 			},
 		},
 	}
@@ -271,6 +281,14 @@ func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3liste
 						},
 					},
 				},
+				Filters: []*v3listenerpb.Filter{
+					{
+						Name: "filter-1",
+						ConfigType: &v3listenerpb.Filter_TypedConfig{
+							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
+						},
+					},
+				},
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{

From 950ddd3c37fc38deaf95f3a27b5883af4776a679 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 12 Apr 2021 09:56:37 -0700
Subject: [PATCH 418/481] xds/google_default_creds: handshake based on cluster
 name in address attributes (#4310)

---
 credentials/google/google.go                  |  13 +-
 credentials/google/google_test.go             | 132 ++++++++++++++++++
 credentials/google/xds.go                     |  90 ++++++++++++
 internal/credentials/xds/handshake_cluster.go |  42 ++++++
 .../balancer/clusterimpl/balancer_test.go     |  91 ++++++++++++
 .../balancer/clusterimpl/clusterimpl.go       |  44 +++++-
 xds/internal/balancer/edsbalancer/eds.go      |   4 +
 xds/internal/balancer/edsbalancer/eds_impl.go |  33 ++++-
 .../balancer/edsbalancer/eds_impl_test.go     |  71 ++++++++++
 xds/internal/balancer/edsbalancer/eds_test.go |  71 ++++++++++
 10 files changed, 584 insertions(+), 7 deletions(-)
 create mode 100644 credentials/google/google_test.go
 create mode 100644 credentials/google/xds.go
 create mode 100644 internal/credentials/xds/handshake_cluster.go

diff --git a/credentials/google/google.go b/credentials/google/google.go
index 7f3e240e475b..265d193c7c37 100644
--- a/credentials/google/google.go
+++ b/credentials/google/google.go
@@ -99,6 +99,15 @@ func (c *creds) PerRPCCredentials() credentials.PerRPCCredentials {
 	return c.perRPCCreds
 }
 
+var (
+	newTLS = func() credentials.TransportCredentials {
+		return credentials.NewTLS(nil)
+	}
+	newALTS = func() credentials.TransportCredentials {
+		return alts.NewClientCreds(alts.DefaultClientOptions())
+	}
+)
+
 // NewWithMode should make a copy of Bundle, and switch mode. Modifying the
 // existing Bundle may cause races.
 func (c *creds) NewWithMode(mode string) (credentials.Bundle, error) {
@@ -110,11 +119,11 @@ func (c *creds) NewWithMode(mode string) (credentials.Bundle, error) {
 	// Create transport credentials.
 	switch mode {
 	case internal.CredsBundleModeFallback:
-		newCreds.transportCreds = credentials.NewTLS(nil)
+		newCreds.transportCreds = newClusterTransportCreds(newTLS(), newALTS())
 	case internal.CredsBundleModeBackendFromBalancer, internal.CredsBundleModeBalancer:
 		// Only the clients can use google default credentials, so we only need
 		// to create new ALTS client creds here.
-		newCreds.transportCreds = alts.NewClientCreds(alts.DefaultClientOptions())
+		newCreds.transportCreds = newALTS()
 	default:
 		return nil, fmt.Errorf("unsupported mode: %v", mode)
 	}
diff --git a/credentials/google/google_test.go b/credentials/google/google_test.go
new file mode 100644
index 000000000000..c20445811202
--- /dev/null
+++ b/credentials/google/google_test.go
@@ -0,0 +1,132 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package google
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/resolver"
+)
+
+type testCreds struct {
+	credentials.TransportCredentials
+	typ string
+}
+
+func (c *testCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return nil, &testAuthInfo{typ: c.typ}, nil
+}
+
+func (c *testCreds) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return nil, &testAuthInfo{typ: c.typ}, nil
+}
+
+type testAuthInfo struct {
+	typ string
+}
+
+func (t *testAuthInfo) AuthType() string {
+	return t.typ
+}
+
+var (
+	testTLS  = &testCreds{typ: "tls"}
+	testALTS = &testCreds{typ: "alts"}
+
+	contextWithHandshakeInfo = internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
+)
+
+func overrideNewCredsFuncs() func() {
+	oldNewTLS := newTLS
+	newTLS = func() credentials.TransportCredentials {
+		return testTLS
+	}
+	oldNewALTS := newALTS
+	newALTS = func() credentials.TransportCredentials {
+		return testALTS
+	}
+	return func() {
+		newTLS = oldNewTLS
+		newALTS = oldNewALTS
+	}
+}
+
+// TestClientHandshakeBasedOnClusterName that by default (without switching
+// modes), ClientHandshake does either tls or alts base on the cluster name in
+// attributes.
+func TestClientHandshakeBasedOnClusterName(t *testing.T) {
+	defer overrideNewCredsFuncs()()
+	for bundleTyp, tc := range map[string]credentials.Bundle{
+		"defaultCreds": NewDefaultCredentials(),
+		"computeCreds": NewComputeEngineCredentials(),
+	} {
+		tests := []struct {
+			name    string
+			ctx     context.Context
+			wantTyp string
+		}{
+			{
+				name:    "no cluster name",
+				ctx:     context.Background(),
+				wantTyp: "tls",
+			},
+			{
+				name: "with non-CFE cluster name",
+				ctx: contextWithHandshakeInfo(context.Background(), credentials.ClientHandshakeInfo{
+					Attributes: xdsinternal.SetHandshakeClusterName(resolver.Address{}, "lalala").Attributes,
+				}),
+				// non-CFE backends should use alts.
+				wantTyp: "alts",
+			},
+			{
+				name: "with CFE cluster name",
+				ctx: contextWithHandshakeInfo(context.Background(), credentials.ClientHandshakeInfo{
+					Attributes: xdsinternal.SetHandshakeClusterName(resolver.Address{}, cfeClusterName).Attributes,
+				}),
+				// CFE should use tls.
+				wantTyp: "tls",
+			},
+		}
+		for _, tt := range tests {
+			t.Run(bundleTyp+" "+tt.name, func(t *testing.T) {
+				_, info, err := tc.TransportCredentials().ClientHandshake(tt.ctx, "", nil)
+				if err != nil {
+					t.Fatalf("ClientHandshake failed: %v", err)
+				}
+				if gotType := info.AuthType(); gotType != tt.wantTyp {
+					t.Fatalf("unexpected authtype: %v, want: %v", gotType, tt.wantTyp)
+				}
+
+				_, infoServer, err := tc.TransportCredentials().ServerHandshake(nil)
+				if err != nil {
+					t.Fatalf("ClientHandshake failed: %v", err)
+				}
+				// ServerHandshake should always do TLS.
+				if gotType := infoServer.AuthType(); gotType != "tls" {
+					t.Fatalf("unexpected server authtype: %v, want: %v", gotType, "tls")
+				}
+			})
+		}
+	}
+}
diff --git a/credentials/google/xds.go b/credentials/google/xds.go
new file mode 100644
index 000000000000..22997ce2532c
--- /dev/null
+++ b/credentials/google/xds.go
@@ -0,0 +1,90 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package google
+
+import (
+	"context"
+	"net"
+
+	"google.golang.org/grpc/credentials"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+)
+
+const cfeClusterName = "google-cfe"
+
+// clusterTransportCreds is a combo of TLS + ALTS.
+//
+// On the client, ClientHandshake picks TLS or ALTS based on address attributes.
+// - if attributes has cluster name
+//   - if cluster name is "google_cfe", use TLS
+//   - otherwise, use ALTS
+// - else, do TLS
+//
+// On the server, ServerHandshake always does TLS.
+type clusterTransportCreds struct {
+	tls  credentials.TransportCredentials
+	alts credentials.TransportCredentials
+}
+
+func newClusterTransportCreds(tls, alts credentials.TransportCredentials) *clusterTransportCreds {
+	return &clusterTransportCreds{
+		tls:  tls,
+		alts: alts,
+	}
+}
+
+func (c *clusterTransportCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	chi := credentials.ClientHandshakeInfoFromContext(ctx)
+	if chi.Attributes == nil {
+		return c.tls.ClientHandshake(ctx, authority, rawConn)
+	}
+	cn, ok := xdsinternal.GetHandshakeClusterName(chi.Attributes)
+	if !ok || cn == cfeClusterName {
+		return c.tls.ClientHandshake(ctx, authority, rawConn)
+	}
+	// If attributes have cluster name, and cluster name is not cfe, it's a
+	// backend address, use ALTS.
+	return c.alts.ClientHandshake(ctx, authority, rawConn)
+}
+
+func (c *clusterTransportCreds) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return c.tls.ServerHandshake(conn)
+}
+
+func (c *clusterTransportCreds) Info() credentials.ProtocolInfo {
+	// TODO: this always returns tls.Info now, because we don't have a cluster
+	// name to check when this method is called. This method doesn't affect
+	// anything important now. We may want to revisit this if it becomes more
+	// important later.
+	return c.tls.Info()
+}
+
+func (c *clusterTransportCreds) Clone() credentials.TransportCredentials {
+	return &clusterTransportCreds{
+		tls:  c.tls.Clone(),
+		alts: c.alts.Clone(),
+	}
+}
+
+func (c *clusterTransportCreds) OverrideServerName(s string) error {
+	if err := c.tls.OverrideServerName(s); err != nil {
+		return err
+	}
+	return c.alts.OverrideServerName(s)
+}
diff --git a/internal/credentials/xds/handshake_cluster.go b/internal/credentials/xds/handshake_cluster.go
new file mode 100644
index 000000000000..cb059bd6669a
--- /dev/null
+++ b/internal/credentials/xds/handshake_cluster.go
@@ -0,0 +1,42 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/resolver"
+)
+
+// handshakeClusterNameKey is the type used as the key to store cluster name in
+// the Attributes field of resolver.Address.
+type handshakeClusterNameKey struct{}
+
+// SetHandshakeClusterName returns a copy of addr in which the Attributes field
+// is updated with the cluster name.
+func SetHandshakeClusterName(addr resolver.Address, clusterName string) resolver.Address {
+	addr.Attributes = addr.Attributes.WithValues(handshakeClusterNameKey{}, clusterName)
+	return addr
+}
+
+// GetHandshakeClusterName returns cluster name stored in attr.
+func GetHandshakeClusterName(attr *attributes.Attributes) (string, bool) {
+	v := attr.Value(handshakeClusterNameKey{})
+	name, ok := v.(string)
+	return name, ok
+}
diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index 6d9b7a5082f6..7fb31ab7affa 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal/client/load"
@@ -369,3 +370,93 @@ func TestPickerUpdateAfterClose(t *testing.T) {
 	case <-time.After(time.Millisecond * 10):
 	}
 }
+
+// TestClusterNameInAddressAttributes covers the case that cluster name is
+// attached to the subconn address attributes.
+func TestClusterNameInAddressAttributes(t *testing.T) {
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
+	builder := balancer.Get(clusterImplName)
+	cc := testutils.NewTestClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: testBackendAddrs,
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:        testClusterName,
+			EDSServiceName: testServiceName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 10; i++ {
+		_, err := p0.Pick(balancer.PickInfo{})
+		if err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("picker.Pick, got _,%v, want Err=%v", err, balancer.ErrNoSubConnAvailable)
+		}
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendAddrs[0].Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	cn, ok := xdsinternal.GetHandshakeClusterName(addrs1[0].Attributes)
+	if !ok || cn != testClusterName {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName)
+	}
+
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	p1 := <-cc.NewPickerCh
+	const rpcCount = 20
+	for i := 0; i < rpcCount; i++ {
+		gotSCSt, err := p1.Pick(balancer.PickInfo{})
+		if err != nil || !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1)
+		}
+		if gotSCSt.Done != nil {
+			gotSCSt.Done(balancer.DoneInfo{})
+		}
+	}
+
+	const testClusterName2 = "test-cluster-2"
+	var addr2 = resolver.Address{Addr: "2.2.2.2"}
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{addr2},
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:        testClusterName2,
+			EDSServiceName: testServiceName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, addr2.Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	// New addresses should have the new cluster name.
+	cn2, ok := xdsinternal.GetHandshakeClusterName(addrs2[0].Attributes)
+	if !ok || cn2 != testClusterName2 {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2)
+	}
+}
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index 4435f9e65a03..0cc8d0d82f5c 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -30,8 +30,10 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/buffer"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -110,11 +112,13 @@ type clusterImplBalancer struct {
 	config           *lbConfig
 	childLB          balancer.Balancer
 	cancelLoadReport func()
-	clusterName      string
 	edsServiceName   string
 	lrsServerName    string
 	loadWrapper      *loadstore.Wrapper
 
+	clusterNameMu sync.Mutex
+	clusterName   string
+
 	// childState/drops/requestCounter can only be accessed in run(). And run()
 	// is the only goroutine that sends picker to the parent ClientConn. All
 	// requests to update picker need to be sent to pickerUpdateCh.
@@ -132,9 +136,11 @@ func (cib *clusterImplBalancer) updateLoadStore(newConfig *lbConfig) error {
 
 	// ClusterName is different, restart. ClusterName is from ClusterName and
 	// EdsServiceName.
-	if cib.clusterName != newConfig.Cluster {
+	clusterName := cib.getClusterName()
+	if clusterName != newConfig.Cluster {
 		updateLoadClusterAndService = true
-		cib.clusterName = newConfig.Cluster
+		cib.setClusterName(newConfig.Cluster)
+		clusterName = newConfig.Cluster
 	}
 	if cib.edsServiceName != newConfig.EDSServiceName {
 		updateLoadClusterAndService = true
@@ -149,7 +155,7 @@ func (cib *clusterImplBalancer) updateLoadStore(newConfig *lbConfig) error {
 		// On the other hand, this will almost never happen. Each LRS policy
 		// shouldn't get updated config. The parent should do a graceful switch
 		// when the clusterName or serviceName is changed.
-		cib.loadWrapper.UpdateClusterAndService(cib.clusterName, cib.edsServiceName)
+		cib.loadWrapper.UpdateClusterAndService(clusterName, cib.edsServiceName)
 	}
 
 	// Check if it's necessary to restart load report.
@@ -305,6 +311,36 @@ func (cib *clusterImplBalancer) UpdateState(state balancer.State) {
 	cib.pickerUpdateCh.Put(state)
 }
 
+func (cib *clusterImplBalancer) setClusterName(n string) {
+	cib.clusterNameMu.Lock()
+	defer cib.clusterNameMu.Unlock()
+	cib.clusterName = n
+}
+
+func (cib *clusterImplBalancer) getClusterName() string {
+	cib.clusterNameMu.Lock()
+	defer cib.clusterNameMu.Unlock()
+	return cib.clusterName
+}
+
+func (cib *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	clusterName := cib.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+	}
+	return cib.ClientConn.NewSubConn(newAddrs, opts)
+}
+
+func (cib *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {
+	clusterName := cib.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+	}
+	cib.ClientConn.UpdateAddresses(sc, newAddrs)
+}
+
 type dropConfigs struct {
 	drops           []*dropper
 	requestCounter  *xdsclient.ServiceRequestsCounter
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 423df7aed95e..de724701df94 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -116,6 +116,9 @@ type edsBalancerImplInterface interface {
 	// updateServiceRequestsConfig updates the service requests counter to the
 	// one for the given service name.
 	updateServiceRequestsConfig(serviceName string, max *uint32)
+	// updateClusterName updates the cluster name that will be attached to the
+	// address attributes.
+	updateClusterName(name string)
 	// close closes the eds balancer.
 	close()
 }
@@ -250,6 +253,7 @@ func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
 		// This is OK for now, because we don't actually expect edsServiceName
 		// to change. Fix this (a bigger change) will happen later.
 		x.lsw.updateServiceName(x.edsServiceName)
+		x.edsImpl.updateClusterName(x.edsServiceName)
 	}
 
 	// Restart load reporting when the loadReportServer name has changed.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 5318a5342e83..94f643d3355e 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/base"
@@ -104,6 +105,9 @@ type edsBalancerImpl struct {
 	innerState             balancer.State // The state of the picker without drop support.
 	serviceRequestsCounter *client.ServiceRequestsCounter
 	serviceRequestCountMax uint32
+
+	clusterNameMu sync.Mutex
+	clusterName   string
 }
 
 // newEDSBalancerImpl create a new edsBalancerImpl.
@@ -444,6 +448,18 @@ func (edsImpl *edsBalancerImpl) updateServiceRequestsConfig(serviceName string,
 	edsImpl.pickerMu.Unlock()
 }
 
+func (edsImpl *edsBalancerImpl) updateClusterName(name string) {
+	edsImpl.clusterNameMu.Lock()
+	defer edsImpl.clusterNameMu.Unlock()
+	edsImpl.clusterName = name
+}
+
+func (edsImpl *edsBalancerImpl) getClusterName() string {
+	edsImpl.clusterNameMu.Lock()
+	defer edsImpl.clusterNameMu.Unlock()
+	return edsImpl.clusterName
+}
+
 // updateState first handles priority, and then wraps picker in a drop picker
 // before forwarding the update.
 func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.State) {
@@ -479,8 +495,23 @@ type edsBalancerWrapperCC struct {
 }
 
 func (ebwcc *edsBalancerWrapperCC) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
-	return ebwcc.parent.newSubConn(ebwcc.priority, addrs, opts)
+	clusterName := ebwcc.parent.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+	}
+	return ebwcc.parent.newSubConn(ebwcc.priority, newAddrs, opts)
 }
+
+func (ebwcc *edsBalancerWrapperCC) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {
+	clusterName := ebwcc.parent.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+	}
+	ebwcc.ClientConn.UpdateAddresses(sc, newAddrs)
+}
+
 func (ebwcc *edsBalancerWrapperCC) UpdateState(state balancer.State) {
 	ebwcc.parent.enqueueChildBalancerStateUpdate(ebwcc.priority, state)
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index ebaea13cc88a..79332dfe1fd3 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -26,6 +26,7 @@ import (
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -933,3 +934,73 @@ func (s) TestEDS_LoadReportDisabled(t *testing.T) {
 		p1.Pick(balancer.PickInfo{})
 	}
 }
+
+// TestEDS_ClusterNameInAddressAttributes covers the case that cluster name is
+// attached to the subconn address attributes.
+func (s) TestEDS_ClusterNameInAddressAttributes(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
+	edsb.enqueueChildBalancerStateUpdate = edsb.updateState
+
+	const clusterName1 = "cluster-name-1"
+	edsb.updateClusterName(clusterName1)
+
+	// One locality with one backend.
+	clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build()))
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	cn, ok := xdsinternal.GetHandshakeClusterName(addrs1[0].Attributes)
+	if !ok || cn != clusterName1 {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, clusterName1)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
+
+	// Pick with only the first backend.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p1.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1)
+		}
+	}
+
+	// Change cluster name.
+	const clusterName2 = "cluster-name-2"
+	edsb.updateClusterName(clusterName2)
+
+	// Change backend.
+	clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
+	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[1:2], nil)
+	edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build()))
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	// New addresses should have the new cluster name.
+	cn2, ok := xdsinternal.GetHandshakeClusterName(addrs2[0].Attributes)
+	if !ok || cn2 != clusterName2 {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, clusterName1)
+	}
+
+	sc2 := <-cc.NewSubConnCh
+	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
+	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
+
+	// Test roundrobin with two subconns.
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, _ := p2.Pick(balancer.PickInfo{})
+		if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
+			t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2)
+		}
+	}
+}
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 5fe1f2ef6b90..65b74a1b8af5 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -117,6 +117,7 @@ type fakeEDSBalancer struct {
 	edsUpdate          *testutils.Channel
 	serviceName        *testutils.Channel
 	serviceRequestMax  *testutils.Channel
+	clusterName        *testutils.Channel
 }
 
 func (f *fakeEDSBalancer) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
@@ -138,6 +139,10 @@ func (f *fakeEDSBalancer) updateServiceRequestsConfig(serviceName string, max *u
 	f.serviceRequestMax.Send(max)
 }
 
+func (f *fakeEDSBalancer) updateClusterName(name string) {
+	f.clusterName.Send(name)
+}
+
 func (f *fakeEDSBalancer) close() {}
 
 func (f *fakeEDSBalancer) waitForChildPolicy(ctx context.Context, wantPolicy *loadBalancingConfig) error {
@@ -207,6 +212,18 @@ func (f *fakeEDSBalancer) waitForCountMaxUpdate(ctx context.Context, want *uint3
 	return fmt.Errorf("got countMax %+v, want %+v", got, want)
 }
 
+func (f *fakeEDSBalancer) waitForClusterNameUpdate(ctx context.Context, wantClusterName string) error {
+	val, err := f.clusterName.Receive(ctx)
+	if err != nil {
+		return err
+	}
+	gotServiceName := val.(string)
+	if gotServiceName != wantClusterName {
+		return fmt.Errorf("got clusterName %v, want %v", gotServiceName, wantClusterName)
+	}
+	return nil
+}
+
 func newFakeEDSBalancer(cc balancer.ClientConn) edsBalancerImplInterface {
 	return &fakeEDSBalancer{
 		cc:                 cc,
@@ -215,6 +232,7 @@ func newFakeEDSBalancer(cc balancer.ClientConn) edsBalancerImplInterface {
 		edsUpdate:          testutils.NewChannelWithSize(10),
 		serviceName:        testutils.NewChannelWithSize(10),
 		serviceRequestMax:  testutils.NewChannelWithSize(10),
+		clusterName:        testutils.NewChannelWithSize(10),
 	}
 }
 
@@ -657,6 +675,59 @@ func (s) TestCounterUpdate(t *testing.T) {
 	}
 }
 
+// TestClusterNameUpdateInAddressAttributes verifies that cluster name update in
+// edsImpl is triggered with the update from a new service config.
+func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
+	edsLBCh := testutils.NewChannel()
+	xdsC, cleanup := setup(edsLBCh)
+	defer cleanup()
+
+	builder := balancer.Get(edsName)
+	edsB := builder.Build(newNoopTestClientConn(), balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}})
+	if edsB == nil {
+		t.Fatalf("builder.Build(%s) failed and returned nil", edsName)
+	}
+	defer edsB.Close()
+
+	// Update should trigger counter update with provided service name.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{
+			EDSServiceName: "foobar-1",
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotCluster, err := xdsC.WaitForWatchEDS(ctx)
+	if err != nil || gotCluster != "foobar-1" {
+		t.Fatalf("unexpected EDS watch: %v, %v", gotCluster, err)
+	}
+	edsI := edsB.(*edsBalancer).edsImpl.(*fakeEDSBalancer)
+	if err := edsI.waitForClusterNameUpdate(ctx, "foobar-1"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update should trigger counter update with provided service name.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{
+			EDSServiceName: "foobar-2",
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := xdsC.WaitForCancelEDSWatch(ctx); err != nil {
+		t.Fatalf("failed to wait for EDS cancel: %v", err)
+	}
+	gotCluster2, err := xdsC.WaitForWatchEDS(ctx)
+	if err != nil || gotCluster2 != "foobar-2" {
+		t.Fatalf("unexpected EDS watch: %v, %v", gotCluster2, err)
+	}
+	if err := edsI.waitForClusterNameUpdate(ctx, "foobar-2"); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func (s) TestBalancerConfigParsing(t *testing.T) {
 	const testEDSName = "eds.service"
 	var testLRSName = "lrs.server"

From 7a6ab591158c9c43b13b229a5d0a6471abfbeca6 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 13 Apr 2021 11:47:25 -0700
Subject: [PATCH 419/481] multiple: go mod tidy to make vet happy (#4337)

---
 examples/go.sum               |  5 -----
 go.sum                        |  3 ---
 security/authorization/go.sum | 11 -----------
 3 files changed, 19 deletions(-)

diff --git a/examples/go.sum b/examples/go.sum
index f55a3e5c8968..1984770a80b3 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -17,14 +17,12 @@ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -62,7 +60,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
@@ -71,12 +68,10 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
diff --git a/go.sum b/go.sum
index bb25cd49156d..24d2976abbaf 100644
--- a/go.sum
+++ b/go.sum
@@ -1,10 +1,8 @@
 cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -43,7 +41,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
diff --git a/security/authorization/go.sum b/security/authorization/go.sum
index a953711e01e6..3c7ea6cf47fe 100644
--- a/security/authorization/go.sum
+++ b/security/authorization/go.sum
@@ -14,14 +14,11 @@ github.com/envoyproxy/go-control-plane v0.9.5 h1:lRJIqDD8yjV1YyPRqecMdytjDLs2fTX
 github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -36,7 +33,6 @@ github.com/google/cel-spec v0.4.0/go.mod h1:2pBM5cU4UKjbPDXBgwWkiwBsVgnxknuEJ7C5
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -49,7 +45,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -58,11 +53,9 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -76,18 +69,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171 h1:xes2Q2k+d/+YNXVw0FpZkIDJiaux4OVrRKXRAzH6A0U=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=

From c229922995e2c1af095282ef4d17abcd7300ecaf Mon Sep 17 00:00:00 2001
From: apolcyn <apolcyn@google.com>
Date: Tue, 13 Apr 2021 13:06:05 -0700
Subject: [PATCH 420/481] client: propagate connection error causes to RPC
 statuses (#4311)

---
 clientconn.go                        |  4 +--
 internal/transport/http2_client.go   | 47 +++++++++++++++-------------
 internal/transport/keepalive_test.go | 33 +++++++++----------
 internal/transport/transport.go      |  2 +-
 internal/transport/transport_test.go | 42 ++++++++++++-------------
 test/end2end_test.go                 | 47 ++++++++++++++++++++++++++++
 6 files changed, 114 insertions(+), 61 deletions(-)

diff --git a/clientconn.go b/clientconn.go
index 77a08fd33bf8..0db796ccbd66 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -1197,7 +1197,7 @@ func (ac *addrConn) resetTransport() {
 		ac.mu.Lock()
 		if ac.state == connectivity.Shutdown {
 			ac.mu.Unlock()
-			newTr.Close()
+			newTr.Close(fmt.Errorf("reached connectivity state: SHUTDOWN"))
 			return
 		}
 		ac.curAddr = addr
@@ -1329,7 +1329,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 	select {
 	case <-time.After(time.Until(connectDeadline)):
 		// We didn't get the preface in time.
-		newTr.Close()
+		newTr.Close(fmt.Errorf("failed to receive server preface within timeout"))
 		channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr)
 		return nil, nil, errors.New("timed out waiting for server handshake")
 	case <-prefaceReceived:
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index d5bbe720db54..ce0166012d7d 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -347,12 +347,14 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 	// Send connection preface to server.
 	n, err := t.conn.Write(clientPreface)
 	if err != nil {
-		t.Close()
-		return nil, connectionErrorf(true, err, "transport: failed to write client preface: %v", err)
+		err = connectionErrorf(true, err, "transport: failed to write client preface: %v", err)
+		t.Close(err)
+		return nil, err
 	}
 	if n != len(clientPreface) {
-		t.Close()
-		return nil, connectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface))
+		err = connectionErrorf(true, nil, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface))
+		t.Close(err)
+		return nil, err
 	}
 	var ss []http2.Setting
 
@@ -370,14 +372,16 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 	}
 	err = t.framer.fr.WriteSettings(ss...)
 	if err != nil {
-		t.Close()
-		return nil, connectionErrorf(true, err, "transport: failed to write initial settings frame: %v", err)
+		err = connectionErrorf(true, err, "transport: failed to write initial settings frame: %v", err)
+		t.Close(err)
+		return nil, err
 	}
 	// Adjust the connection flow control window if needed.
 	if delta := uint32(icwz - defaultWindowSize); delta > 0 {
 		if err := t.framer.fr.WriteWindowUpdate(0, delta); err != nil {
-			t.Close()
-			return nil, connectionErrorf(true, err, "transport: failed to write window update: %v", err)
+			err = connectionErrorf(true, err, "transport: failed to write window update: %v", err)
+			t.Close(err)
+			return nil, err
 		}
 	}
 
@@ -845,12 +849,12 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2.
 // This method blocks until the addrConn that initiated this transport is
 // re-connected. This happens because t.onClose() begins reconnect logic at the
 // addrConn level and blocks until the addrConn is successfully connected.
-func (t *http2Client) Close() error {
+func (t *http2Client) Close(err error) {
 	t.mu.Lock()
 	// Make sure we only Close once.
 	if t.state == closing {
 		t.mu.Unlock()
-		return nil
+		return
 	}
 	// Call t.onClose before setting the state to closing to prevent the client
 	// from attempting to create new streams ASAP.
@@ -866,13 +870,13 @@ func (t *http2Client) Close() error {
 	t.mu.Unlock()
 	t.controlBuf.finish()
 	t.cancel()
-	err := t.conn.Close()
+	t.conn.Close()
 	if channelz.IsOn() {
 		channelz.RemoveEntry(t.channelzID)
 	}
 	// Notify all active streams.
 	for _, s := range streams {
-		t.closeStream(s, ErrConnClosing, false, http2.ErrCodeNo, status.New(codes.Unavailable, ErrConnClosing.Desc), nil, false)
+		t.closeStream(s, err, false, http2.ErrCodeNo, status.New(codes.Unavailable, err.Error()), nil, false)
 	}
 	if t.statsHandler != nil {
 		connEnd := &stats.ConnEnd{
@@ -880,7 +884,6 @@ func (t *http2Client) Close() error {
 		}
 		t.statsHandler.HandleConn(t.ctx, connEnd)
 	}
-	return err
 }
 
 // GracefulClose sets the state to draining, which prevents new streams from
@@ -899,7 +902,7 @@ func (t *http2Client) GracefulClose() {
 	active := len(t.activeStreams)
 	t.mu.Unlock()
 	if active == 0 {
-		t.Close()
+		t.Close(ErrConnClosing)
 		return
 	}
 	t.controlBuf.put(&incomingGoAway{})
@@ -1147,7 +1150,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 	id := f.LastStreamID
 	if id > 0 && id%2 != 1 {
 		t.mu.Unlock()
-		t.Close()
+		t.Close(connectionErrorf(true, nil, "received goaway with non-zero even-numbered numbered stream id: %v", id))
 		return
 	}
 	// A client can receive multiple GoAways from the server (see
@@ -1165,7 +1168,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 		// If there are multiple GoAways the first one should always have an ID greater than the following ones.
 		if id > t.prevGoAwayID {
 			t.mu.Unlock()
-			t.Close()
+			t.Close(connectionErrorf(true, nil, "received goaway with stream id: %v, which exceeds stream id of previous goaway: %v", id, t.prevGoAwayID))
 			return
 		}
 	default:
@@ -1195,7 +1198,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 	active := len(t.activeStreams)
 	t.mu.Unlock()
 	if active == 0 {
-		t.Close()
+		t.Close(connectionErrorf(true, nil, "received goaway and there are no active streams"))
 	}
 }
 
@@ -1313,7 +1316,8 @@ func (t *http2Client) reader() {
 	// Check the validity of server preface.
 	frame, err := t.framer.fr.ReadFrame()
 	if err != nil {
-		t.Close() // this kicks off resetTransport, so must be last before return
+		err = connectionErrorf(true, err, "error reading server preface: %v", err)
+		t.Close(err) // this kicks off resetTransport, so must be last before return
 		return
 	}
 	t.conn.SetReadDeadline(time.Time{}) // reset deadline once we get the settings frame (we didn't time out, yay!)
@@ -1322,7 +1326,8 @@ func (t *http2Client) reader() {
 	}
 	sf, ok := frame.(*http2.SettingsFrame)
 	if !ok {
-		t.Close() // this kicks off resetTransport, so must be last before return
+		// this kicks off resetTransport, so must be last before return
+		t.Close(connectionErrorf(true, nil, "initial http2 frame from server is not a settings frame: %T", frame))
 		return
 	}
 	t.onPrefaceReceipt()
@@ -1358,7 +1363,7 @@ func (t *http2Client) reader() {
 				continue
 			} else {
 				// Transport error.
-				t.Close()
+				t.Close(connectionErrorf(true, err, "error reading from server: %v", err))
 				return
 			}
 		}
@@ -1417,7 +1422,7 @@ func (t *http2Client) keepalive() {
 				continue
 			}
 			if outstandingPing && timeoutLeft <= 0 {
-				t.Close()
+				t.Close(connectionErrorf(true, nil, "keepalive ping failed to receive ACK within timeout"))
 				return
 			}
 			t.mu.Lock()
diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go
index c8f177fecf1b..1f6603cd759e 100644
--- a/internal/transport/keepalive_test.go
+++ b/internal/transport/keepalive_test.go
@@ -24,6 +24,7 @@ package transport
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"testing"
@@ -47,7 +48,7 @@ func (s) TestMaxConnectionIdle(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -86,7 +87,7 @@ func (s) TestMaxConnectionIdleBusyClient(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -122,7 +123,7 @@ func (s) TestMaxConnectionAge(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -169,7 +170,7 @@ func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -228,7 +229,7 @@ func (s) TestKeepaliveServerWithResponsiveClient(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -257,7 +258,7 @@ func (s) TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) {
 		PermitWithoutStream: true,
 	}}, connCh)
 	defer cancel()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 
 	conn, ok := <-connCh
 	if !ok {
@@ -288,7 +289,7 @@ func (s) TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) {
 		Timeout: 1 * time.Second,
 	}}, connCh)
 	defer cancel()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 
 	conn, ok := <-connCh
 	if !ok {
@@ -317,7 +318,7 @@ func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) {
 		Timeout: 1 * time.Second,
 	}}, connCh)
 	defer cancel()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 
 	conn, ok := <-connCh
 	if !ok {
@@ -352,7 +353,7 @@ func (s) TestKeepaliveClientStaysHealthyWithResponsiveServer(t *testing.T) {
 			PermitWithoutStream: true,
 		}})
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -391,7 +392,7 @@ func (s) TestKeepaliveClientFrequency(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions)
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -436,7 +437,7 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions)
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -480,7 +481,7 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions)
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -530,7 +531,7 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientNoRPC(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions)
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -564,7 +565,7 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) {
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions)
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -604,7 +605,7 @@ func (s) TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient(t *testing.T
 	}
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions)
 	defer func() {
-		client.Close()
+		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()
 		cancel()
 	}()
@@ -658,7 +659,7 @@ func (s) TestTCPUserTimeout(t *testing.T) {
 			},
 		)
 		defer func() {
-			client.Close()
+			client.Close(fmt.Errorf("closed manually by test"))
 			server.stop()
 			cancel()
 		}()
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index 5cf7c5f80fe1..068f4d0e5023 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -622,7 +622,7 @@ type ClientTransport interface {
 	// Close tears down this transport. Once it returns, the transport
 	// should not be accessed any more. The caller must make sure this
 	// is called only once.
-	Close() error
+	Close(err error)
 
 	// GracefulClose starts to tear down the transport: the transport will stop
 	// accepting new RPCs and NewStream will return error. Once all streams are
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 1d8d3ed355df..c3830a8fd0b1 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -481,7 +481,7 @@ func (s) TestInflightStreamClosing(t *testing.T) {
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer cancel()
 	defer server.stop()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -550,7 +550,7 @@ func (s) TestClientSendAndReceive(t *testing.T) {
 	if recvErr != io.EOF {
 		t.Fatalf("Error: %v; want <EOF>", recvErr)
 	}
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	server.stop()
 }
 
@@ -560,7 +560,7 @@ func (s) TestClientErrorNotify(t *testing.T) {
 	go server.stop()
 	// ct.reader should detect the error and activate ct.Error().
 	<-ct.Error()
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 }
 
 func performOneRPC(ct ClientTransport) {
@@ -597,7 +597,7 @@ func (s) TestClientMix(t *testing.T) {
 	}(s)
 	go func(ct ClientTransport) {
 		<-ct.Error()
-		ct.Close()
+		ct.Close(fmt.Errorf("closed manually by test"))
 	}(ct)
 	for i := 0; i < 1000; i++ {
 		time.Sleep(10 * time.Millisecond)
@@ -636,7 +636,7 @@ func (s) TestLargeMessage(t *testing.T) {
 		}()
 	}
 	wg.Wait()
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	server.stop()
 }
 
@@ -653,7 +653,7 @@ func (s) TestLargeMessageWithDelayRead(t *testing.T) {
 	server, ct, cancel := setUpWithOptions(t, 0, sc, delayRead, co)
 	defer cancel()
 	defer server.stop()
-	defer ct.Close()
+	defer ct.Close(fmt.Errorf("closed manually by test"))
 	server.mu.Lock()
 	ready := server.ready
 	server.mu.Unlock()
@@ -831,7 +831,7 @@ func (s) TestLargeMessageSuspension(t *testing.T) {
 	if _, err := s.Read(make([]byte, 8)); err.Error() != expectedErr.Error() {
 		t.Fatalf("Read got %v of type %T, want %v", err, err, expectedErr)
 	}
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	server.stop()
 }
 
@@ -841,7 +841,7 @@ func (s) TestMaxStreams(t *testing.T) {
 	}
 	server, ct, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer cancel()
-	defer ct.Close()
+	defer ct.Close(fmt.Errorf("closed manually by test"))
 	defer server.stop()
 	callHdr := &CallHdr{
 		Host:   "localhost",
@@ -901,7 +901,7 @@ func (s) TestMaxStreams(t *testing.T) {
 	// Close the first stream created so that the new stream can finally be created.
 	ct.CloseStream(s, nil)
 	<-done
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	<-ct.writerDone
 	if ct.maxConcurrentStreams != 1 {
 		t.Fatalf("ct.maxConcurrentStreams: %d, want 1", ct.maxConcurrentStreams)
@@ -960,7 +960,7 @@ func (s) TestServerContextCanceledOnClosedConnection(t *testing.T) {
 		sc.mu.Unlock()
 		break
 	}
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	select {
 	case <-ss.Context().Done():
 		if ss.Context().Err() != context.Canceled {
@@ -980,7 +980,7 @@ func (s) TestClientConnDecoupledFromApplicationRead(t *testing.T) {
 	server, client, cancel := setUpWithOptions(t, 0, &ServerConfig{}, notifyCall, connectOptions)
 	defer cancel()
 	defer server.stop()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 
 	waitWhileTrue(t, func() (bool, error) {
 		server.mu.Lock()
@@ -1069,7 +1069,7 @@ func (s) TestServerConnDecoupledFromApplicationRead(t *testing.T) {
 	server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{})
 	defer cancel()
 	defer server.stop()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 	waitWhileTrue(t, func() (bool, error) {
 		server.mu.Lock()
 		defer server.mu.Unlock()
@@ -1302,7 +1302,7 @@ func (s) TestClientWithMisbehavedServer(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Error while creating client transport: %v", err)
 	}
-	defer ct.Close()
+	defer ct.Close(fmt.Errorf("closed manually by test"))
 	str, err := ct.NewStream(connectCtx, &CallHdr{})
 	if err != nil {
 		t.Fatalf("Error while creating stream: %v", err)
@@ -1345,7 +1345,7 @@ func (s) TestEncodingRequiredStatus(t *testing.T) {
 	if !testutils.StatusErrEqual(s.Status().Err(), encodingTestStatus.Err()) {
 		t.Fatalf("stream with status %v, want %v", s.Status(), encodingTestStatus)
 	}
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	server.stop()
 }
 
@@ -1367,7 +1367,7 @@ func (s) TestInvalidHeaderField(t *testing.T) {
 	if se, ok := status.FromError(err); !ok || se.Code() != codes.Internal || !strings.Contains(err.Error(), expectedInvalidHeaderField) {
 		t.Fatalf("Read got error %v, want error with code %s and contains %q", err, codes.Internal, expectedInvalidHeaderField)
 	}
-	ct.Close()
+	ct.Close(fmt.Errorf("closed manually by test"))
 	server.stop()
 }
 
@@ -1375,7 +1375,7 @@ func (s) TestHeaderChanClosedAfterReceivingAnInvalidHeader(t *testing.T) {
 	server, ct, cancel := setUp(t, 0, math.MaxUint32, invalidHeaderField)
 	defer cancel()
 	defer server.stop()
-	defer ct.Close()
+	defer ct.Close(fmt.Errorf("closed manually by test"))
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	s, err := ct.NewStream(ctx, &CallHdr{Host: "localhost", Method: "foo"})
@@ -1481,7 +1481,7 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig)
 	server, client, cancel := setUpWithOptions(t, 0, sc, pingpong, co)
 	defer cancel()
 	defer server.stop()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 	waitWhileTrue(t, func() (bool, error) {
 		server.mu.Lock()
 		defer server.mu.Unlock()
@@ -1563,7 +1563,7 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig)
 	}
 	// Close down both server and client so that their internals can be read without data
 	// races.
-	client.Close()
+	client.Close(fmt.Errorf("closed manually by test"))
 	st.Close()
 	<-st.readerDone
 	<-st.writerDone
@@ -1762,7 +1762,7 @@ func runPingPongTest(t *testing.T, msgSize int) {
 	server, client, cancel := setUp(t, 0, 0, pingpong)
 	defer cancel()
 	defer server.stop()
-	defer client.Close()
+	defer client.Close(fmt.Errorf("closed manually by test"))
 	waitWhileTrue(t, func() (bool, error) {
 		server.mu.Lock()
 		defer server.mu.Unlock()
@@ -1850,7 +1850,7 @@ func (s) TestHeaderTblSize(t *testing.T) {
 
 	server, ct, cancel := setUp(t, 0, math.MaxUint32, normal)
 	defer cancel()
-	defer ct.Close()
+	defer ct.Close(fmt.Errorf("closed manually by test"))
 	defer server.stop()
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -1969,7 +1969,7 @@ func (s) TestClientHandshakeInfo(t *testing.T) {
 	if err != nil {
 		t.Fatalf("NewClientTransport(): %v", err)
 	}
-	defer tr.Close()
+	defer tr.Close(fmt.Errorf("closed manually by test"))
 
 	wantAttr := attributes.New(testAttrKey, testAttrVal)
 	if gotAttr := creds.attr; !cmp.Equal(gotAttr, wantAttr, cmp.AllowUnexported(attributes.Attributes{})) {
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 902e94241048..746df1745116 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -1333,6 +1333,53 @@ func testConcurrentServerStopAndGoAway(t *testing.T, e env) {
 	awaitNewConnLogOutput()
 }
 
+func (s) TestDetailedConnectionCloseErrorPropagatesToRpcError(t *testing.T) {
+	rpcStartedOnServer := make(chan struct{})
+	rpcDoneOnClient := make(chan struct{})
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			close(rpcStartedOnServer)
+			<-rpcDoneOnClient
+			return status.Error(codes.Internal, "arbitrary status")
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// The precise behavior of this test is subject to raceyness around the timing of when TCP packets
+	// are sent from client to server, and when we tell the server to stop, so we need to account for both
+	// of these possible error messages:
+	// 1) If the call to ss.S.Stop() causes the server's sockets to close while there's still in-fight
+	//    data from the client on the TCP connection, then the kernel can send an RST back to the client (also
+	//    see https://stackoverflow.com/questions/33053507/econnreset-in-send-linux-c). Note that while this
+	//    condition is expected to be rare due to the rpcStartedOnServer synchronization, in theory it should
+	//    be possible, e.g. if the client sends a BDP ping at the right time.
+	// 2) If, for example, the call to ss.S.Stop() happens after the RPC headers have been received at the
+	//    server, then the TCP connection can shutdown gracefully when the server's socket closes.
+	const possibleConnResetMsg = "connection reset by peer"
+	const possibleEOFMsg = "error reading from server: EOF"
+	// Start an RPC. Then, while the RPC is still being accepted or handled at the server, abruptly
+	// stop the server, killing the connection. The RPC error message should include details about the specific
+	// connection error that was encountered.
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall = _, %v, want _, <nil>", ss.Client, err)
+	}
+	// Block until the RPC has been started on the server. This ensures that the ClientConn will find a healthy
+	// connection for the RPC to go out on initially, and that the TCP connection will shut down strictly after
+	// the RPC has been started on it.
+	<-rpcStartedOnServer
+	ss.S.Stop()
+	if _, err := stream.Recv(); err == nil || (!strings.Contains(err.Error(), possibleConnResetMsg) && !strings.Contains(err.Error(), possibleEOFMsg)) {
+		t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: %q OR %q", stream, err, possibleConnResetMsg, possibleEOFMsg)
+	}
+	close(rpcDoneOnClient)
+}
+
 func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) {
 	for _, e := range listTestEnv() {
 		if e.name == "handler-tls" {

From 6fafb9193bde04c61d75a2da9de53c4d029748b4 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 13 Apr 2021 15:31:34 -0700
Subject: [PATCH 421/481] xds: support unspecified and wildcard filter chain
 prefixes (#4333)

---
 xds/internal/client/filter_chain.go           |  85 +++--
 xds/internal/client/filter_chain_test.go      | 301 +++++++++++++++---
 xds/internal/client/lds_test.go               |  18 +-
 .../test/xds_server_integration_test.go       |  97 ++++--
 4 files changed, 374 insertions(+), 127 deletions(-)

diff --git a/xds/internal/client/filter_chain.go b/xds/internal/client/filter_chain.go
index 73ffa18347e8..66d26d03b634 100644
--- a/xds/internal/client/filter_chain.go
+++ b/xds/internal/client/filter_chain.go
@@ -30,13 +30,22 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
-// Represents a wildcard IP prefix. Go stdlib `Contains()` method works for both
-// v4 and v6 addresses when used on this wildcard address.
-const emptyAddrMapKey = "0.0.0.0/0"
-
-var (
-	// Parsed wildcard IP prefix.
-	_, zeroIP, _ = net.ParseCIDR("0.0.0.0/0")
+const (
+	// Used as the map key for unspecified prefixes. The actual value of this
+	// key is immaterial.
+	unspecifiedPrefixMapKey = "unspecified"
+
+	// An unspecified destination or source prefix should be considered a less
+	// specific match than a wildcard prefix, `0.0.0.0/0` or `::/0`. Also, an
+	// unspecified prefix should match most v4 and v6 addresses compared to the
+	// wildcard prefixes which match only a specific network (v4 or v6).
+	//
+	// We use these constants when looking up the most specific prefix match. A
+	// wildcard prefix will match 0 bits, and to make sure that a wildcard
+	// prefix is considered a more specific match than an unspecified prefix, we
+	// use a value of -1 for the latter.
+	noPrefixMatch          = -2
+	unspecifiedPrefixMatch = -1
 )
 
 // FilterChain captures information from within a FilterChain message in a
@@ -108,7 +117,8 @@ type FilterChainManager struct {
 
 // destPrefixEntry is the value type of the map indexed on destination prefixes.
 type destPrefixEntry struct {
-	net *net.IPNet // The actual destination prefix.
+	// The actual destination prefix. Set to nil for unspecified prefixes.
+	net *net.IPNet
 	// We need to keep track of the transport protocols seen as part of the
 	// config validation (and internal structure building) phase. The only two
 	// values that we support are empty string and "raw_buffer", with the latter
@@ -137,7 +147,8 @@ type sourcePrefixes struct {
 
 // sourcePrefixEntry contains match criteria per source prefix.
 type sourcePrefixEntry struct {
-	net *net.IPNet // The actual source prefix.
+	// The actual destination prefix. Set to nil for unspecified prefixes.
+	net *net.IPNet
 	// Mapping from source ports specified in the match criteria to the actual
 	// filter chain. Unspecified source port matches en up as a wildcard entry
 	// here with a key of 0.
@@ -227,11 +238,12 @@ func (fci *FilterChainManager) addFilterChainsForDestPrefixes(fc *v3listenerpb.F
 	}
 
 	if len(dstPrefixes) == 0 {
-		// Use the wildcard IP when destination prefix is unspecified.
-		if fci.dstPrefixMap[emptyAddrMapKey] == nil {
-			fci.dstPrefixMap[emptyAddrMapKey] = &destPrefixEntry{net: zeroIP}
+		// Use the unspecified entry when destination prefix is unspecified, and
+		// set the `net` field to nil.
+		if fci.dstPrefixMap[unspecifiedPrefixMapKey] == nil {
+			fci.dstPrefixMap[unspecifiedPrefixMapKey] = &destPrefixEntry{}
 		}
-		return fci.addFilterChainsForServerNames(fci.dstPrefixMap[emptyAddrMapKey], fc)
+		return fci.addFilterChainsForServerNames(fci.dstPrefixMap[unspecifiedPrefixMapKey], fc)
 	}
 	for _, prefix := range dstPrefixes {
 		p := prefix.String()
@@ -326,14 +338,14 @@ func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map
 	}
 
 	if len(srcPrefixes) == 0 {
-		// Use the wildcard IP when source prefix is unspecified.
-		if srcPrefixMap[emptyAddrMapKey] == nil {
-			srcPrefixMap[emptyAddrMapKey] = &sourcePrefixEntry{
-				net:        zeroIP,
+		// Use the unspecified entry when destination prefix is unspecified, and
+		// set the `net` field to nil.
+		if srcPrefixMap[unspecifiedPrefixMapKey] == nil {
+			srcPrefixMap[unspecifiedPrefixMapKey] = &sourcePrefixEntry{
 				srcPortMap: make(map[int]*FilterChain),
 			}
 		}
-		return fci.addFilterChainsForSourcePorts(srcPrefixMap[emptyAddrMapKey], fc)
+		return fci.addFilterChainsForSourcePorts(srcPrefixMap[unspecifiedPrefixMapKey], fc)
 	}
 	for _, prefix := range srcPrefixes {
 		p := prefix.String()
@@ -486,15 +498,22 @@ func filterByDestinationPrefixes(dstPrefixes []*destPrefixEntry, isUnspecified b
 		// bound to the wildcard address.
 		return dstPrefixes
 	}
-	var (
-		matchingDstPrefixes []*destPrefixEntry
-		maxSubnetMatch      int
-	)
+
+	var matchingDstPrefixes []*destPrefixEntry
+	maxSubnetMatch := noPrefixMatch
 	for _, prefix := range dstPrefixes {
-		if !prefix.net.Contains(dstAddr) {
+		if prefix.net != nil && !prefix.net.Contains(dstAddr) {
+			// Skip prefixes which don't match.
 			continue
 		}
-		matchSize, _ := prefix.net.Mask.Size()
+		// For unspecified prefixes, since we do not store a real net.IPNet
+		// inside prefix, we do not perform a match. Instead we simply set
+		// the matchSize to -1, which is less than the matchSize (0) for a
+		// wildcard prefix.
+		matchSize := unspecifiedPrefixMatch
+		if prefix.net != nil {
+			matchSize, _ = prefix.net.Mask.Size()
+		}
 		if matchSize < maxSubnetMatch {
 			continue
 		}
@@ -551,16 +570,22 @@ func filterBySourceType(dstPrefixes []*destPrefixEntry, srcType SourceType) []*s
 // algorithm. It trims the filter chains based on the source prefix. At most one
 // filter chain with the most specific match progress to the next stage.
 func filterBySourcePrefixes(srcPrefixes []*sourcePrefixes, srcAddr net.IP) (*sourcePrefixEntry, error) {
-	var (
-		matchingSrcPrefixes []*sourcePrefixEntry
-		maxSubnetMatch      int
-	)
+	var matchingSrcPrefixes []*sourcePrefixEntry
+	maxSubnetMatch := noPrefixMatch
 	for _, sp := range srcPrefixes {
 		for _, prefix := range sp.srcPrefixes {
-			if !prefix.net.Contains(srcAddr) {
+			if prefix.net != nil && !prefix.net.Contains(srcAddr) {
+				// Skip prefixes which don't match.
 				continue
 			}
-			matchSize, _ := prefix.net.Mask.Size()
+			// For unspecified prefixes, since we do not store a real net.IPNet
+			// inside prefix, we do not perform a match. Instead we simply set
+			// the matchSize to -1, which is less than the matchSize (0) for a
+			// wildcard prefix.
+			matchSize := unspecifiedPrefixMatch
+			if prefix.net != nil {
+				matchSize, _ = prefix.net.Mask.Size()
+			}
 			if matchSize < maxSubnetMatch {
 				continue
 			}
diff --git a/xds/internal/client/filter_chain_test.go b/xds/internal/client/filter_chain_test.go
index e66f518828ac..afb0c81fda14 100644
--- a/xds/internal/client/filter_chain_test.go
+++ b/xds/internal/client/filter_chain_test.go
@@ -35,6 +35,8 @@ import (
 	"google.golang.org/grpc/xds/internal/version"
 )
 
+// TestNewFilterChainImpl_Failure_BadMatchFields verifies cases where we have a
+// single filter chain with match criteria that contains unsupported fields.
 func TestNewFilterChainImpl_Failure_BadMatchFields(t *testing.T) {
 	tests := []struct {
 		desc string
@@ -131,6 +133,8 @@ func TestNewFilterChainImpl_Failure_BadMatchFields(t *testing.T) {
 	}
 }
 
+// TestNewFilterChainImpl_Failure_OverlappingMatchingRules verifies cases where
+// there are multiple filter chains and they have overlapping match rules.
 func TestNewFilterChainImpl_Failure_OverlappingMatchingRules(t *testing.T) {
 	tests := []struct {
 		desc string
@@ -218,6 +222,8 @@ func TestNewFilterChainImpl_Failure_OverlappingMatchingRules(t *testing.T) {
 	}
 }
 
+// TestNewFilterChainImpl_Failure_BadSecurityConfig verifies cases where the
+// security configuration in the filter chain is invalid.
 func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 	tests := []struct {
 		desc    string
@@ -369,6 +375,8 @@ func TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) {
 	}
 }
 
+// TestNewFilterChainImpl_Success_SecurityConfig verifies cases where the
+// security configuration in the filter chain contains valid data.
 func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 	tests := []struct {
 		desc   string
@@ -387,13 +395,11 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -444,13 +450,11 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {
 												SecurityCfg: &SecurityConfig{
@@ -526,13 +530,11 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {
 												SecurityCfg: &SecurityConfig{
@@ -576,6 +578,153 @@ func TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) {
 	}
 }
 
+// TestNewFilterChainImpl_Success_UnsupportedMatchFields verifies cases where
+// there are multiple filter chains, and one of them is valid while the other
+// contains unsupported match fields. These configurations should lead to
+// success at config validation time and the filter chains which contains
+// unsupported match fields will be skipped at lookup time.
+func TestNewFilterChainImpl_Success_UnsupportedMatchFields(t *testing.T) {
+	unspecifiedEntry := &destPrefixEntry{
+		srcTypeArr: [3]*sourcePrefixes{
+			{
+				srcPrefixMap: map[string]*sourcePrefixEntry{
+					unspecifiedPrefixMapKey: {
+						srcPortMap: map[int]*FilterChain{
+							0: {},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		desc   string
+		lis    *v3listenerpb.Listener
+		wantFC *FilterChainManager
+	}{
+		{
+			desc: "unsupported destination port",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "good-chain",
+					},
+					{
+						Name: "unsupported-destination-port",
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:    []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							DestinationPort: &wrapperspb.UInt32Value{Value: 666},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					unspecifiedPrefixMapKey: unspecifiedEntry,
+				},
+				def: &FilterChain{},
+			},
+		},
+		{
+			desc: "unsupported server names",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "good-chain",
+					},
+					{
+						Name: "unsupported-server-names",
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							ServerNames:  []string{"example-server"},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					unspecifiedPrefixMapKey: unspecifiedEntry,
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+					},
+				},
+				def: &FilterChain{},
+			},
+		},
+		{
+			desc: "unsupported transport protocol",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "good-chain",
+					},
+					{
+						Name: "unsupported-transport-protocol",
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:      []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							TransportProtocol: "tls",
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					unspecifiedPrefixMapKey: unspecifiedEntry,
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+					},
+				},
+				def: &FilterChain{},
+			},
+		},
+		{
+			desc: "unsupported application protocol",
+			lis: &v3listenerpb.Listener{
+				FilterChains: []*v3listenerpb.FilterChain{
+					{
+						Name: "good-chain",
+					},
+					{
+						Name: "unsupported-application-protocol",
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges:         []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)},
+							ApplicationProtocols: []string{"h2"},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{},
+			},
+			wantFC: &FilterChainManager{
+				dstPrefixMap: map[string]*destPrefixEntry{
+					unspecifiedPrefixMapKey: unspecifiedEntry,
+					"192.168.0.0/16": {
+						net: ipNetFromCIDR("192.168.2.2/16"),
+					},
+				},
+				def: &FilterChain{},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotFC, err := NewFilterChainManager(test.lis)
+			if err != nil {
+				t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err)
+			}
+			if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{})) {
+				t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC)
+			}
+		})
+	}
+}
+
+// TestNewFilterChainImpl_Success_AllCombinations verifies different
+// combinations of the supported match criteria.
 func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 	tests := []struct {
 		desc   string
@@ -587,8 +736,23 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 			lis: &v3listenerpb.Listener{
 				FilterChains: []*v3listenerpb.FilterChain{
 					{
+						// Unspecified destination prefix.
 						FilterChainMatch: &v3listenerpb.FilterChainMatch{},
 					},
+					{
+						// v4 wildcard destination prefix.
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)},
+							SourceType:   v3listenerpb.FilterChainMatch_EXTERNAL,
+						},
+					},
+					{
+						// v6 wildcard destination prefix.
+						FilterChainMatch: &v3listenerpb.FilterChainMatch{
+							PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("::", 0)},
+							SourceType:   v3listenerpb.FilterChainMatch_EXTERNAL,
+						},
+					},
 					{
 						FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
 					},
@@ -600,13 +764,43 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
+					unspecifiedPrefixMapKey: {
+						srcTypeArr: [3]*sourcePrefixes{
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									unspecifiedPrefixMapKey: {
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
 					"0.0.0.0/0": {
-						net: zeroIP,
+						net: ipNetFromCIDR("0.0.0.0/0"),
+						srcTypeArr: [3]*sourcePrefixes{
+							nil,
+							nil,
+							{
+								srcPrefixMap: map[string]*sourcePrefixEntry{
+									unspecifiedPrefixMapKey: {
+										srcPortMap: map[int]*FilterChain{
+											0: {},
+										},
+									},
+								},
+							},
+						},
+					},
+					"::/0": {
+						net: ipNetFromCIDR("::/0"),
 						srcTypeArr: [3]*sourcePrefixes{
+							nil,
+							nil,
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -620,8 +814,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -635,8 +828,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -667,14 +859,12 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							nil,
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -690,8 +880,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 							nil,
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -722,8 +911,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
@@ -776,13 +964,11 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											1: {},
 											2: {},
@@ -874,13 +1060,11 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 			},
 			wantFC: &FilterChainManager{
 				dstPrefixMap: map[string]*destPrefixEntry{
-					"0.0.0.0/0": {
-						net: zeroIP,
+					unspecifiedPrefixMapKey: {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -894,8 +1078,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -909,8 +1092,7 @@ func TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) {
 						srcTypeArr: [3]*sourcePrefixes{
 							{
 								srcPrefixMap: map[string]*sourcePrefixEntry{
-									"0.0.0.0/0": {
-										net: zeroIP,
+									unspecifiedPrefixMapKey: {
 										srcPortMap: map[int]*FilterChain{
 											0: {},
 										},
@@ -1128,18 +1310,24 @@ func TestLookup_Successes(t *testing.T) {
 	lisWithoutDefaultChain := &v3listenerpb.Listener{
 		FilterChains: []*v3listenerpb.FilterChain{
 			{
-				TransportSocket: transportSocketWithInstanceName("wildcard"),
+				TransportSocket: transportSocketWithInstanceName("unspecified-dest-and-source-prefix"),
 			},
 			{
 				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)},
-					SourceType:   v3listenerpb.FilterChainMatch_EXTERNAL,
+					PrefixRanges:       []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)},
+					SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)},
 				},
-				TransportSocket: transportSocketWithInstanceName("any-destination-prefix"),
+				TransportSocket: transportSocketWithInstanceName("wildcard-prefixes-v4"),
+			},
+			{
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("::", 0)},
+				},
+				TransportSocket: transportSocketWithInstanceName("wildcard-source-prefix-v6"),
 			},
 			{
 				FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}},
-				TransportSocket:  transportSocketWithInstanceName("specific-destination-prefix-wildcard-source-type"),
+				TransportSocket:  transportSocketWithInstanceName("specific-destination-prefix-unspecified-source-type"),
 			},
 			{
 				FilterChainMatch: &v3listenerpb.FilterChainMatch{
@@ -1184,26 +1372,37 @@ func TestLookup_Successes(t *testing.T) {
 			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "default"}},
 		},
 		{
-			desc: "wildcard destination match",
+			desc: "unspecified destination match",
 			lis:  lisWithoutDefaultChain,
 			params: FilterChainLookupParams{
 				IsUnspecifiedListener: true,
-				DestAddr:              net.IPv4(10, 1, 1, 1),
+				DestAddr:              net.ParseIP("2001:68::db8"),
 				SourceAddr:            net.IPv4(10, 1, 1, 1),
 				SourcePort:            1,
 			},
-			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "wildcard"}},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "unspecified-dest-and-source-prefix"}},
 		},
 		{
-			desc: "ANY destination match",
+			desc: "wildcard destination match v4",
 			lis:  lisWithoutDefaultChain,
 			params: FilterChainLookupParams{
 				IsUnspecifiedListener: true,
 				DestAddr:              net.IPv4(10, 1, 1, 1),
-				SourceAddr:            net.IPv4(10, 1, 1, 2),
+				SourceAddr:            net.IPv4(10, 1, 1, 1),
+				SourcePort:            1,
+			},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "wildcard-prefixes-v4"}},
+		},
+		{
+			desc: "wildcard source match v6",
+			lis:  lisWithoutDefaultChain,
+			params: FilterChainLookupParams{
+				IsUnspecifiedListener: true,
+				DestAddr:              net.ParseIP("2001:68::1"),
+				SourceAddr:            net.ParseIP("2001:68::2"),
 				SourcePort:            1,
 			},
-			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "any-destination-prefix"}},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "wildcard-source-prefix-v6"}},
 		},
 		{
 			desc: "specific destination and wildcard source type match",
@@ -1214,7 +1413,7 @@ func TestLookup_Successes(t *testing.T) {
 				SourceAddr:            net.IPv4(192, 168, 100, 1),
 				SourcePort:            80,
 			},
-			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-wildcard-source-type"}},
+			wantFC: &FilterChain{SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-unspecified-source-type"}},
 		},
 		{
 			desc: "specific destination and source type match",
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 21718a4edc5e..21e94557b3e9 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -1126,13 +1126,11 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						Port:    "9999",
 						FilterChains: &FilterChainManager{
 							dstPrefixMap: map[string]*destPrefixEntry{
-								"0.0.0.0/0": {
-									net: zeroIP,
+								unspecifiedPrefixMapKey: {
 									srcTypeArr: [3]*sourcePrefixes{
 										{
 											srcPrefixMap: map[string]*sourcePrefixEntry{
-												"0.0.0.0/0": {
-													net: zeroIP,
+												unspecifiedPrefixMapKey: {
 													srcPortMap: map[int]*FilterChain{
 														0: {},
 													},
@@ -1216,13 +1214,11 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						Port:    "9999",
 						FilterChains: &FilterChainManager{
 							dstPrefixMap: map[string]*destPrefixEntry{
-								"0.0.0.0/0": {
-									net: zeroIP,
+								unspecifiedPrefixMapKey: {
 									srcTypeArr: [3]*sourcePrefixes{
 										{
 											srcPrefixMap: map[string]*sourcePrefixEntry{
-												"0.0.0.0/0": {
-													net: zeroIP,
+												unspecifiedPrefixMapKey: {
 													srcPortMap: map[int]*FilterChain{
 														0: {
 															SecurityCfg: &SecurityConfig{
@@ -1263,13 +1259,11 @@ func (s) TestUnmarshalListener_ServerSide(t *testing.T) {
 						Port:    "9999",
 						FilterChains: &FilterChainManager{
 							dstPrefixMap: map[string]*destPrefixEntry{
-								"0.0.0.0/0": {
-									net: zeroIP,
+								unspecifiedPrefixMapKey: {
 									srcTypeArr: [3]*sourcePrefixes{
 										{
 											srcPrefixMap: map[string]*sourcePrefixEntry{
-												"0.0.0.0/0": {
-													net: zeroIP,
+												unspecifiedPrefixMapKey: {
 													srcPortMap: map[int]*FilterChain{
 														0: {
 															SecurityCfg: &SecurityConfig{
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 9c2e6daebfec..e31bba772ed9 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -33,29 +33,26 @@ import (
 	"strconv"
 	"testing"
 
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/uuid"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/status"
+	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/testdata"
 	"google.golang.org/grpc/xds"
-	"google.golang.org/grpc/xds/internal/testutils/e2e"
-	"google.golang.org/grpc/xds/internal/version"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
-	xdscreds "google.golang.org/grpc/credentials/xds"
-	xdsinternal "google.golang.org/grpc/internal/xds"
-	testpb "google.golang.org/grpc/test/grpc_testing"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
 )
 
 const (
@@ -248,6 +245,24 @@ func listenerResourceWithoutSecurityConfig(t *testing.T, lis net.Listener) *v3li
 // configuration pointing to the use of the file_watcher certificate provider
 // plugin, and name and address fields matching the passed in net.Listener.
 func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3listenerpb.Listener {
+	transportSocket := &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
+				RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName: "google_cloud_private_spiffe",
+					},
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "google_cloud_private_spiffe",
+						},
+					},
+				},
+			}),
+		},
+	}
 	host, port := hostPortFromListener(t, lis)
 	return &v3listenerpb.Listener{
 		// This needs to match the name we are querying for.
@@ -261,7 +276,7 @@ func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3liste
 					}}}},
 		FilterChains: []*v3listenerpb.FilterChain{
 			{
-				Name: "filter-chain-1",
+				Name: "v4-wildcard",
 				FilterChainMatch: &v3listenerpb.FilterChainMatch{
 					PrefixRanges: []*v3corepb.CidrRange{
 						{
@@ -289,26 +304,40 @@ func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3liste
 						},
 					},
 				},
-				TransportSocket: &v3corepb.TransportSocket{
-					Name: "envoy.transport_sockets.tls",
-					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3DownstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.DownstreamTlsContext{
-									RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-											InstanceName: "google_cloud_private_spiffe",
-										},
-										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-												InstanceName: "google_cloud_private_spiffe",
-											}}}}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						}}}}},
+				TransportSocket: transportSocket,
+			},
+			{
+				Name: "v6-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "::",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "::",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+				},
+				Filters: []*v3listenerpb.Filter{
+					{
+						Name: "filter-1",
+						ConfigType: &v3listenerpb.Filter_TypedConfig{
+							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
+						},
+					},
+				},
+				TransportSocket: transportSocket,
+			},
+		},
 	}
 }
 

From 87eb5b7502493f758e76c4d09430c0049a81a557 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 13 Apr 2021 16:19:17 -0700
Subject: [PATCH 422/481] credentials/google: remove unnecessary dependency on
 xds protos (#4339)

---
 credentials/credentials.go                    | 24 ++-------
 credentials/google/google_test.go             | 12 ++---
 credentials/google/xds.go                     |  4 +-
 credentials/sts/sts_test.go                   |  4 +-
 credentials/xds/xds_client_test.go            |  8 ++-
 internal/credentials/credentials.go           | 49 +++++++++++++++++++
 internal/internal.go                          |  6 ---
 internal/transport/http2_client.go            | 14 +++---
 ...ke_cluster.go => xds_handshake_cluster.go} | 12 ++---
 .../balancer/clusterimpl/balancer_test.go     |  6 +--
 .../balancer/clusterimpl/clusterimpl.go       |  6 +--
 xds/internal/balancer/edsbalancer/eds_impl.go | 15 +++---
 .../balancer/edsbalancer/eds_impl_test.go     | 14 +++---
 13 files changed, 96 insertions(+), 78 deletions(-)
 create mode 100644 internal/credentials/credentials.go
 rename internal/{credentials/xds/handshake_cluster.go => xds_handshake_cluster.go} (75%)

diff --git a/credentials/credentials.go b/credentials/credentials.go
index e69562e78786..7eee7e4ec126 100644
--- a/credentials/credentials.go
+++ b/credentials/credentials.go
@@ -30,7 +30,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc/attributes"
-	"google.golang.org/grpc/internal"
+	icredentials "google.golang.org/grpc/internal/credentials"
 )
 
 // PerRPCCredentials defines the common interface for the credentials which need to
@@ -188,15 +188,12 @@ type RequestInfo struct {
 	AuthInfo AuthInfo
 }
 
-// requestInfoKey is a struct to be used as the key when attaching a RequestInfo to a context object.
-type requestInfoKey struct{}
-
 // RequestInfoFromContext extracts the RequestInfo from the context if it exists.
 //
 // This API is experimental.
 func RequestInfoFromContext(ctx context.Context) (ri RequestInfo, ok bool) {
-	ri, ok = ctx.Value(requestInfoKey{}).(RequestInfo)
-	return
+	ri, ok = icredentials.RequestInfoFromContext(ctx).(RequestInfo)
+	return ri, ok
 }
 
 // ClientHandshakeInfo holds data to be passed to ClientHandshake. This makes
@@ -211,16 +208,12 @@ type ClientHandshakeInfo struct {
 	Attributes *attributes.Attributes
 }
 
-// clientHandshakeInfoKey is a struct used as the key to store
-// ClientHandshakeInfo in a context.
-type clientHandshakeInfoKey struct{}
-
 // ClientHandshakeInfoFromContext returns the ClientHandshakeInfo struct stored
 // in ctx.
 //
 // This API is experimental.
 func ClientHandshakeInfoFromContext(ctx context.Context) ClientHandshakeInfo {
-	chi, _ := ctx.Value(clientHandshakeInfoKey{}).(ClientHandshakeInfo)
+	chi, _ := icredentials.ClientHandshakeInfoFromContext(ctx).(ClientHandshakeInfo)
 	return chi
 }
 
@@ -249,15 +242,6 @@ func CheckSecurityLevel(ai AuthInfo, level SecurityLevel) error {
 	return nil
 }
 
-func init() {
-	internal.NewRequestInfoContext = func(ctx context.Context, ri RequestInfo) context.Context {
-		return context.WithValue(ctx, requestInfoKey{}, ri)
-	}
-	internal.NewClientHandshakeInfoContext = func(ctx context.Context, chi ClientHandshakeInfo) context.Context {
-		return context.WithValue(ctx, clientHandshakeInfoKey{}, chi)
-	}
-}
-
 // ChannelzSecurityInfo defines the interface that security protocols should implement
 // in order to provide security info to channelz.
 //
diff --git a/credentials/google/google_test.go b/credentials/google/google_test.go
index c20445811202..fee51f945016 100644
--- a/credentials/google/google_test.go
+++ b/credentials/google/google_test.go
@@ -25,7 +25,7 @@ import (
 
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	icredentials "google.golang.org/grpc/internal/credentials"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -53,8 +53,6 @@ func (t *testAuthInfo) AuthType() string {
 var (
 	testTLS  = &testCreds{typ: "tls"}
 	testALTS = &testCreds{typ: "alts"}
-
-	contextWithHandshakeInfo = internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
 )
 
 func overrideNewCredsFuncs() func() {
@@ -93,16 +91,16 @@ func TestClientHandshakeBasedOnClusterName(t *testing.T) {
 			},
 			{
 				name: "with non-CFE cluster name",
-				ctx: contextWithHandshakeInfo(context.Background(), credentials.ClientHandshakeInfo{
-					Attributes: xdsinternal.SetHandshakeClusterName(resolver.Address{}, "lalala").Attributes,
+				ctx: icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{
+					Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, "lalala").Attributes,
 				}),
 				// non-CFE backends should use alts.
 				wantTyp: "alts",
 			},
 			{
 				name: "with CFE cluster name",
-				ctx: contextWithHandshakeInfo(context.Background(), credentials.ClientHandshakeInfo{
-					Attributes: xdsinternal.SetHandshakeClusterName(resolver.Address{}, cfeClusterName).Attributes,
+				ctx: icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{
+					Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, cfeClusterName).Attributes,
 				}),
 				// CFE should use tls.
 				wantTyp: "tls",
diff --git a/credentials/google/xds.go b/credentials/google/xds.go
index 22997ce2532c..588c685e2592 100644
--- a/credentials/google/xds.go
+++ b/credentials/google/xds.go
@@ -23,7 +23,7 @@ import (
 	"net"
 
 	"google.golang.org/grpc/credentials"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/internal"
 )
 
 const cfeClusterName = "google-cfe"
@@ -54,7 +54,7 @@ func (c *clusterTransportCreds) ClientHandshake(ctx context.Context, authority s
 	if chi.Attributes == nil {
 		return c.tls.ClientHandshake(ctx, authority, rawConn)
 	}
-	cn, ok := xdsinternal.GetHandshakeClusterName(chi.Attributes)
+	cn, ok := internal.GetXDSHandshakeClusterName(chi.Attributes)
 	if !ok || cn == cfeClusterName {
 		return c.tls.ClientHandshake(ctx, authority, rawConn)
 	}
diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go
index ac680e001112..8738c424d087 100644
--- a/credentials/sts/sts_test.go
+++ b/credentials/sts/sts_test.go
@@ -37,7 +37,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/internal"
+	icredentials "google.golang.org/grpc/internal/credentials"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 )
@@ -104,7 +104,7 @@ func createTestContext(ctx context.Context, s credentials.SecurityLevel) context
 		Method:   "testInfo",
 		AuthInfo: auth,
 	}
-	return internal.NewRequestInfoContext.(func(context.Context, credentials.RequestInfo) context.Context)(ctx, ri)
+	return icredentials.NewRequestInfoContext(ctx, ri)
 }
 
 // errReader implements the io.Reader interface and returns an error from the
diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index 82cfa5876acb..8859946ef5fa 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -32,7 +32,7 @@ import (
 
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/internal"
+	icredentials "google.golang.org/grpc/internal/credentials"
 	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
@@ -228,8 +228,7 @@ func newTestContextWithHandshakeInfo(parent context.Context, root, identity cert
 	// Moving the attributes from the resolver.Address to the context passed to
 	// the handshaker is done in the transport layer. Since we directly call the
 	// handshaker in these tests, we need to do the same here.
-	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
-	return contextWithHandshakeInfo(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
+	return icredentials.NewClientHandshakeInfoContext(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 }
 
 // compareAuthInfo compares the AuthInfo received on the client side after a
@@ -541,8 +540,7 @@ func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	// here because we need access to the underlying HandshakeInfo so that we
 	// can update it before the next call to ClientHandshake().
 	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, handshakeInfo)
-	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
-	ctx = contextWithHandshakeInfo(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
+	ctx = icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
 		t.Fatal("ClientHandshake() succeeded when expected to fail")
 	}
diff --git a/internal/credentials/credentials.go b/internal/credentials/credentials.go
new file mode 100644
index 000000000000..32c9b59033cd
--- /dev/null
+++ b/internal/credentials/credentials.go
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package credentials
+
+import (
+	"context"
+)
+
+// requestInfoKey is a struct to be used as the key to store RequestInfo in a
+// context.
+type requestInfoKey struct{}
+
+// NewRequestInfoContext creates a context with ri.
+func NewRequestInfoContext(ctx context.Context, ri interface{}) context.Context {
+	return context.WithValue(ctx, requestInfoKey{}, ri)
+}
+
+// RequestInfoFromContext extracts the RequestInfo from ctx.
+func RequestInfoFromContext(ctx context.Context) interface{} {
+	return ctx.Value(requestInfoKey{})
+}
+
+// clientHandshakeInfoKey is a struct used as the key to store
+// ClientHandshakeInfo in a context.
+type clientHandshakeInfoKey struct{}
+
+// ClientHandshakeInfoFromContext extracts the ClientHandshakeInfo from ctx.
+func ClientHandshakeInfoFromContext(ctx context.Context) interface{} {
+	return ctx.Value(clientHandshakeInfoKey{})
+}
+
+// NewClientHandshakeInfoContext creates a context with chi.
+func NewClientHandshakeInfoContext(ctx context.Context, chi interface{}) context.Context {
+	return context.WithValue(ctx, clientHandshakeInfoKey{}, chi)
+}
diff --git a/internal/internal.go b/internal/internal.go
index 1e2834c70f67..2a3243bd701a 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -38,12 +38,6 @@ var (
 	// KeepaliveMinPingTime is the minimum ping interval.  This must be 10s by
 	// default, but tests may wish to set it lower for convenience.
 	KeepaliveMinPingTime = 10 * time.Second
-	// NewRequestInfoContext creates a new context based on the argument context attaching
-	// the passed in RequestInfo to the new context.
-	NewRequestInfoContext interface{} // func(context.Context, credentials.RequestInfo) context.Context
-	// NewClientHandshakeInfoContext returns a copy of the input context with
-	// the passed in ClientHandshakeInfo struct added to it.
-	NewClientHandshakeInfoContext interface{} // func(context.Context, credentials.ClientHandshakeInfo) context.Context
 	// ParseServiceConfigForTesting is for creating a fake
 	// ClientConn for resolver testing only
 	ParseServiceConfigForTesting interface{} // func(string) *serviceconfig.ParseResult
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index ce0166012d7d..892317cc6fca 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -32,15 +32,14 @@ import (
 
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
-	"google.golang.org/grpc/internal/grpcutil"
-	imetadata "google.golang.org/grpc/internal/metadata"
-	"google.golang.org/grpc/internal/transport/networktype"
-
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/channelz"
+	icredentials "google.golang.org/grpc/internal/credentials"
+	"google.golang.org/grpc/internal/grpcutil"
+	imetadata "google.golang.org/grpc/internal/metadata"
 	"google.golang.org/grpc/internal/syscall"
+	"google.golang.org/grpc/internal/transport/networktype"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
@@ -238,8 +237,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts
 		// Attributes field of resolver.Address, which is shoved into connectCtx
 		// and passed to the credential handshaker. This makes it possible for
 		// address specific arbitrary data to reach the credential handshaker.
-		contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
-		connectCtx = contextWithHandshakeInfo(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
+		connectCtx = icredentials.NewClientHandshakeInfoContext(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 		conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.ServerName, conn)
 		if err != nil {
 			return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err)
@@ -458,7 +456,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 		Method:   callHdr.Method,
 		AuthInfo: t.authInfo,
 	}
-	ctxWithRequestInfo := internal.NewRequestInfoContext.(func(context.Context, credentials.RequestInfo) context.Context)(ctx, ri)
+	ctxWithRequestInfo := icredentials.NewRequestInfoContext(ctx, ri)
 	authData, err := t.getTrAuthData(ctxWithRequestInfo, aud)
 	if err != nil {
 		return nil, err
diff --git a/internal/credentials/xds/handshake_cluster.go b/internal/xds_handshake_cluster.go
similarity index 75%
rename from internal/credentials/xds/handshake_cluster.go
rename to internal/xds_handshake_cluster.go
index cb059bd6669a..3677c3f04f84 100644
--- a/internal/credentials/xds/handshake_cluster.go
+++ b/internal/xds_handshake_cluster.go
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2021 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +12,9 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
-package xds
+package internal
 
 import (
 	"google.golang.org/grpc/attributes"
@@ -27,15 +25,15 @@ import (
 // the Attributes field of resolver.Address.
 type handshakeClusterNameKey struct{}
 
-// SetHandshakeClusterName returns a copy of addr in which the Attributes field
+// SetXDSHandshakeClusterName returns a copy of addr in which the Attributes field
 // is updated with the cluster name.
-func SetHandshakeClusterName(addr resolver.Address, clusterName string) resolver.Address {
+func SetXDSHandshakeClusterName(addr resolver.Address, clusterName string) resolver.Address {
 	addr.Attributes = addr.Attributes.WithValues(handshakeClusterNameKey{}, clusterName)
 	return addr
 }
 
-// GetHandshakeClusterName returns cluster name stored in attr.
-func GetHandshakeClusterName(attr *attributes.Attributes) (string, bool) {
+// GetXDSHandshakeClusterName returns cluster name stored in attr.
+func GetXDSHandshakeClusterName(attr *attributes.Attributes) (string, bool) {
 	v := attr.Value(handshakeClusterNameKey{})
 	name, ok := v.(string)
 	return name, ok
diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index 7fb31ab7affa..f002c6a239d7 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -29,7 +29,7 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/internal"
 	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal/client/load"
@@ -414,7 +414,7 @@ func TestClusterNameInAddressAttributes(t *testing.T) {
 	if got, want := addrs1[0].Addr, testBackendAddrs[0].Addr; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	cn, ok := xdsinternal.GetHandshakeClusterName(addrs1[0].Attributes)
+	cn, ok := internal.GetXDSHandshakeClusterName(addrs1[0].Attributes)
 	if !ok || cn != testClusterName {
 		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName)
 	}
@@ -455,7 +455,7 @@ func TestClusterNameInAddressAttributes(t *testing.T) {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	// New addresses should have the new cluster name.
-	cn2, ok := xdsinternal.GetHandshakeClusterName(addrs2[0].Attributes)
+	cn2, ok := internal.GetXDSHandshakeClusterName(addrs2[0].Attributes)
 	if !ok || cn2 != testClusterName2 {
 		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2)
 	}
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index 0cc8d0d82f5c..cc87ba0cc898 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -29,8 +29,8 @@ import (
 	"sync"
 
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/buffer"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/resolver"
@@ -327,7 +327,7 @@ func (cib *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balanc
 	clusterName := cib.getClusterName()
 	newAddrs := make([]resolver.Address, len(addrs))
 	for i, addr := range addrs {
-		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName)
 	}
 	return cib.ClientConn.NewSubConn(newAddrs, opts)
 }
@@ -336,7 +336,7 @@ func (cib *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, addrs []res
 	clusterName := cib.getClusterName()
 	newAddrs := make([]resolver.Address, len(addrs))
 	for i, addr := range addrs {
-		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName)
 	}
 	cib.ClientConn.UpdateAddresses(sc, newAddrs)
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go
index 94f643d3355e..63b75caae8f9 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl.go
@@ -23,19 +23,18 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
-
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/base"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/balancer/weightedroundrobin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/xds/internal"
+	xdsi "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 	"google.golang.org/grpc/xds/internal/client"
@@ -58,7 +57,7 @@ type localityConfig struct {
 type balancerGroupWithConfig struct {
 	bg              *balancergroup.BalancerGroup
 	stateAggregator *weightedaggregator.Aggregator
-	configs         map[internal.LocalityID]*localityConfig
+	configs         map[xdsi.LocalityID]*localityConfig
 }
 
 // edsBalancerImpl does load balancing based on the EDS responses. Note that it
@@ -261,7 +260,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
 			bgwc = &balancerGroupWithConfig{
 				bg:              balancergroup.New(ccPriorityWrapper, edsImpl.buildOpts, stateAggregator, edsImpl.loadReporter, edsImpl.logger),
 				stateAggregator: stateAggregator,
-				configs:         make(map[internal.LocalityID]*localityConfig),
+				configs:         make(map[xdsi.LocalityID]*localityConfig),
 			}
 			edsImpl.priorityToLocalities[priority] = bgwc
 			priorityChanged = true
@@ -295,7 +294,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroup
 	// newLocalitiesSet contains all names of localities in the new EDS response
 	// for the same priority. It's used to delete localities that are removed in
 	// the new EDS response.
-	newLocalitiesSet := make(map[internal.LocalityID]struct{})
+	newLocalitiesSet := make(map[xdsi.LocalityID]struct{})
 	var rebuildStateAndPicker bool
 	for _, locality := range newLocalities {
 		// One balancer for each locality.
@@ -498,7 +497,7 @@ func (ebwcc *edsBalancerWrapperCC) NewSubConn(addrs []resolver.Address, opts bal
 	clusterName := ebwcc.parent.getClusterName()
 	newAddrs := make([]resolver.Address, len(addrs))
 	for i, addr := range addrs {
-		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName)
 	}
 	return ebwcc.parent.newSubConn(ebwcc.priority, newAddrs, opts)
 }
@@ -507,7 +506,7 @@ func (ebwcc *edsBalancerWrapperCC) UpdateAddresses(sc balancer.SubConn, addrs []
 	clusterName := ebwcc.parent.getClusterName()
 	newAddrs := make([]resolver.Address, len(addrs))
 	for i, addr := range addrs {
-		newAddrs[i] = xdsinternal.SetHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName)
 	}
 	ebwcc.ClientConn.UpdateAddresses(sc, newAddrs)
 }
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 79332dfe1fd3..7e793c034a84 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -26,14 +26,14 @@ import (
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/internal"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/xds/env"
-	"google.golang.org/grpc/xds/internal"
+	xdsinternal "google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 	"google.golang.org/grpc/xds/internal/client"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -834,7 +834,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	edsb.updateServiceRequestsConfig(testServiceName, &maxRequestsTemp)
 	defer client.ClearCounterForTesting(testServiceName)
 
-	backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID)
+	backendToBalancerID := make(map[balancer.SubConn]xdsinternal.LocalityID)
 
 	const testDropCategory = "test-drop"
 	// Two localities, each with one backend.
@@ -844,7 +844,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	sc1 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc1, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc1, connectivity.Ready)
-	locality1 := internal.LocalityID{SubZone: testSubZones[0]}
+	locality1 := xdsinternal.LocalityID{SubZone: testSubZones[0]}
 	backendToBalancerID[sc1] = locality1
 
 	// Add the second locality later to make sure sc2 belongs to the second
@@ -855,7 +855,7 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	sc2 := <-cc.NewSubConnCh
 	edsb.handleSubConnStateChange(sc2, connectivity.Connecting)
 	edsb.handleSubConnStateChange(sc2, connectivity.Ready)
-	locality2 := internal.LocalityID{SubZone: testSubZones[1]}
+	locality2 := xdsinternal.LocalityID{SubZone: testSubZones[1]}
 	backendToBalancerID[sc2] = locality2
 
 	// Test roundrobin with two subconns.
@@ -954,7 +954,7 @@ func (s) TestEDS_ClusterNameInAddressAttributes(t *testing.T) {
 	if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
-	cn, ok := xdsinternal.GetHandshakeClusterName(addrs1[0].Attributes)
+	cn, ok := internal.GetXDSHandshakeClusterName(addrs1[0].Attributes)
 	if !ok || cn != clusterName1 {
 		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, clusterName1)
 	}
@@ -986,7 +986,7 @@ func (s) TestEDS_ClusterNameInAddressAttributes(t *testing.T) {
 		t.Fatalf("sc is created with addr %v, want %v", got, want)
 	}
 	// New addresses should have the new cluster name.
-	cn2, ok := xdsinternal.GetHandshakeClusterName(addrs2[0].Attributes)
+	cn2, ok := internal.GetXDSHandshakeClusterName(addrs2[0].Attributes)
 	if !ok || cn2 != clusterName2 {
 		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, clusterName1)
 	}

From 1a870aec2ff99bb682d5e200763c9124185eafca Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 15 Apr 2021 15:08:03 -0700
Subject: [PATCH 423/481] xds/clusterimpl: trigger re-resolution on subconn
 transient_failure (#4314)

---
 .../balancer/clusterimpl/balancer_test.go     | 95 +++++++++++++++++++
 .../balancer/clusterimpl/clusterimpl.go       | 12 +++
 xds/internal/testutils/balancer.go            | 18 ++--
 3 files changed, 119 insertions(+), 6 deletions(-)

diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index f002c6a239d7..d3789ffddca3 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -32,6 +32,7 @@ import (
 	"google.golang.org/grpc/internal"
 	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
 	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
@@ -55,6 +56,13 @@ var (
 	}
 )
 
+func subConnFromPicker(p balancer.Picker) func() balancer.SubConn {
+	return func() balancer.SubConn {
+		scst, _ := p.Pick(balancer.PickInfo{})
+		return scst.SubConn
+	}
+}
+
 func init() {
 	newRandomWRR = testutils.NewTestWRR
 }
@@ -62,6 +70,7 @@ func init() {
 // TestDropByCategory verifies that the balancer correctly drops the picks, and
 // that the drops are reported.
 func TestDropByCategory(t *testing.T) {
+	defer client.ClearCounterForTesting(testClusterName)
 	xdsC := fakeclient.NewClient()
 	oldNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
@@ -219,6 +228,7 @@ func TestDropByCategory(t *testing.T) {
 // TestDropCircuitBreaking verifies that the balancer correctly drops the picks
 // due to circuit breaking, and that the drops are reported.
 func TestDropCircuitBreaking(t *testing.T) {
+	defer client.ClearCounterForTesting(testClusterName)
 	xdsC := fakeclient.NewClient()
 	oldNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
@@ -330,6 +340,7 @@ func TestDropCircuitBreaking(t *testing.T) {
 // picker after it's closed. Because picker updates are sent in the run()
 // goroutine.
 func TestPickerUpdateAfterClose(t *testing.T) {
+	defer client.ClearCounterForTesting(testClusterName)
 	xdsC := fakeclient.NewClient()
 	oldNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
@@ -374,6 +385,7 @@ func TestPickerUpdateAfterClose(t *testing.T) {
 // TestClusterNameInAddressAttributes covers the case that cluster name is
 // attached to the subconn address attributes.
 func TestClusterNameInAddressAttributes(t *testing.T) {
+	defer client.ClearCounterForTesting(testClusterName)
 	xdsC := fakeclient.NewClient()
 	oldNewXDSClient := newXDSClient
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
@@ -460,3 +472,86 @@ func TestClusterNameInAddressAttributes(t *testing.T) {
 		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2)
 	}
 }
+
+// TestReResolution verifies that when a SubConn turns transient failure,
+// re-resolution is triggered.
+func TestReResolution(t *testing.T) {
+	defer client.ClearCounterForTesting(testClusterName)
+	xdsC := fakeclient.NewClient()
+	oldNewXDSClient := newXDSClient
+	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
+	defer func() { newXDSClient = oldNewXDSClient }()
+
+	builder := balancer.Get(clusterImplName)
+	cc := testutils.NewTestClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: testBackendAddrs,
+		},
+		BalancerConfig: &lbConfig{
+			Cluster:        testClusterName,
+			EDSServiceName: testServiceName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	p0 := <-cc.NewPickerCh
+	for i := 0; i < 10; i++ {
+		_, err := p0.Pick(balancer.PickInfo{})
+		if err != balancer.ErrNoSubConnAvailable {
+			t.Fatalf("picker.Pick, got _,%v, want Err=%v", err, balancer.ErrNoSubConnAvailable)
+		}
+	}
+
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	// This should get the transient failure picker.
+	p1 := <-cc.NewPickerCh
+	for i := 0; i < 10; i++ {
+		_, err := p1.Pick(balancer.PickInfo{})
+		if err == nil {
+			t.Fatalf("picker.Pick, got _,%v, want not nil", err)
+		}
+	}
+
+	// The transient failure should trigger a re-resolution.
+	select {
+	case <-cc.ResolveNowCh:
+	case <-time.After(defaultTestTimeout):
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	p2 := <-cc.NewPickerCh
+	want := []balancer.SubConn{sc1}
+	if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil {
+		t.Fatalf("want %v, got %v", want, err)
+	}
+
+	b.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+	// This should get the transient failure picker.
+	p3 := <-cc.NewPickerCh
+	for i := 0; i < 10; i++ {
+		_, err := p3.Pick(balancer.PickInfo{})
+		if err == nil {
+			t.Fatalf("picker.Pick, got _,%v, want not nil", err)
+		}
+	}
+
+	// The transient failure should trigger a re-resolution.
+	select {
+	case <-cc.ResolveNowCh:
+	case <-time.After(defaultTestTimeout):
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+}
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index cc87ba0cc898..7d4f7695daf4 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -29,6 +29,7 @@ import (
 	"sync"
 
 	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
@@ -286,6 +287,17 @@ func (cib *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balanc
 		return
 	}
 
+	// Trigger re-resolution when a SubConn turns transient failure. This is
+	// necessary for the LogicalDNS in cluster_resolver policy to re-resolve.
+	//
+	// Note that this happens not only for the addresses from DNS, but also for
+	// EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent
+	// knows). The parent priority policy is configured to ignore re-resolution
+	// signal from the EDS children.
+	if s.ConnectivityState == connectivity.TransientFailure {
+		cib.ClientConn.ResolveNow(resolver.ResolveNowOptions{})
+	}
+
 	if cib.childLB != nil {
 		cib.childLB.UpdateSubConnState(sc, s)
 	}
diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go
index dab84a84e072..820d5bf3a1ac 100644
--- a/xds/internal/testutils/balancer.go
+++ b/xds/internal/testutils/balancer.go
@@ -76,8 +76,9 @@ type TestClientConn struct {
 	RemoveSubConnCh        chan balancer.SubConn   // the last 10 subconn removed.
 	UpdateAddressesAddrsCh chan []resolver.Address // last updated address via UpdateAddresses().
 
-	NewPickerCh chan balancer.Picker    // the last picker updated.
-	NewStateCh  chan connectivity.State // the last state.
+	NewPickerCh  chan balancer.Picker            // the last picker updated.
+	NewStateCh   chan connectivity.State         // the last state.
+	ResolveNowCh chan resolver.ResolveNowOptions // the last ResolveNow().
 
 	subConnIdx int
 }
@@ -92,8 +93,9 @@ func NewTestClientConn(t *testing.T) *TestClientConn {
 		RemoveSubConnCh:        make(chan balancer.SubConn, 10),
 		UpdateAddressesAddrsCh: make(chan []resolver.Address, 1),
 
-		NewPickerCh: make(chan balancer.Picker, 1),
-		NewStateCh:  make(chan connectivity.State, 1),
+		NewPickerCh:  make(chan balancer.Picker, 1),
+		NewStateCh:   make(chan connectivity.State, 1),
+		ResolveNowCh: make(chan resolver.ResolveNowOptions, 1),
 	}
 }
 
@@ -151,8 +153,12 @@ func (tcc *TestClientConn) UpdateState(bs balancer.State) {
 }
 
 // ResolveNow panics.
-func (tcc *TestClientConn) ResolveNow(resolver.ResolveNowOptions) {
-	panic("not implemented")
+func (tcc *TestClientConn) ResolveNow(o resolver.ResolveNowOptions) {
+	select {
+	case <-tcc.ResolveNowCh:
+	default:
+	}
+	tcc.ResolveNowCh <- o
 }
 
 // Target panics.

From 41676e61b1d576484cf2c0315a25fe2c9438c769 Mon Sep 17 00:00:00 2001
From: lzhfromustc <43191155+lzhfromustc@users.noreply.github.com>
Date: Mon, 19 Apr 2021 12:49:37 -0400
Subject: [PATCH 424/481] Fix goroutine leaks (#4214)

---
 clientconn_state_transition_test.go              | 16 ++++++----------
 .../alts/internal/handshaker/handshaker_test.go  |  7 +++++--
 credentials/local/local_test.go                  |  2 ++
 internal/transport/keepalive_test.go             |  2 +-
 internal/transport/proxy_test.go                 |  2 +-
 test/end2end_test.go                             |  2 --
 6 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/clientconn_state_transition_test.go b/clientconn_state_transition_test.go
index 0c58131a1c6f..cd1213fb4fd1 100644
--- a/clientconn_state_transition_test.go
+++ b/clientconn_state_transition_test.go
@@ -210,7 +210,8 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) {
 	}
 	defer lis.Close()
 
-	sawReady := make(chan struct{})
+	sawReady := make(chan struct{}, 1)
+	defer close(sawReady)
 
 	// Launch the server.
 	go func() {
@@ -250,7 +251,7 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) {
 			t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want)
 		case seen := <-stateNotifications:
 			if seen == connectivity.Ready {
-				close(sawReady)
+				sawReady <- struct{}{}
 			}
 			if seen != want[i] {
 				t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen)
@@ -378,7 +379,8 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) {
 	defer lis2.Close()
 
 	server1Done := make(chan struct{})
-	sawReady := make(chan struct{})
+	sawReady := make(chan struct{}, 1)
+	defer close(sawReady)
 
 	// Launch server 1.
 	go func() {
@@ -400,12 +402,6 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) {
 
 		conn.Close()
 
-		_, err = lis1.Accept()
-		if err != nil {
-			t.Error(err)
-			return
-		}
-
 		close(server1Done)
 	}()
 
@@ -430,7 +426,7 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) {
 			t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want)
 		case seen := <-stateNotifications:
 			if seen == connectivity.Ready {
-				close(sawReady)
+				sawReady <- struct{}{}
 			}
 			if seen != want[i] {
 				t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen)
diff --git a/credentials/alts/internal/handshaker/handshaker_test.go b/credentials/alts/internal/handshaker/handshaker_test.go
index bf516dc53c87..14a0721054f2 100644
--- a/credentials/alts/internal/handshaker/handshaker_test.go
+++ b/credentials/alts/internal/handshaker/handshaker_test.go
@@ -21,6 +21,7 @@ package handshaker
 import (
 	"bytes"
 	"context"
+	"errors"
 	"testing"
 	"time"
 
@@ -163,7 +164,8 @@ func (s) TestClientHandshake(t *testing.T) {
 			go func() {
 				_, context, err := chs.ClientHandshake(ctx)
 				if err == nil && context == nil {
-					panic("expected non-nil ALTS context")
+					errc <- errors.New("expected non-nil ALTS context")
+					return
 				}
 				errc <- err
 				chs.Close()
@@ -219,7 +221,8 @@ func (s) TestServerHandshake(t *testing.T) {
 			go func() {
 				_, context, err := shs.ServerHandshake(ctx)
 				if err == nil && context == nil {
-					panic("expected non-nil ALTS context")
+					errc <- errors.New("expected non-nil ALTS context")
+					return
 				}
 				errc <- err
 				shs.Close()
diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go
index 00ae39f07e56..47f8dbb4ec85 100644
--- a/credentials/local/local_test.go
+++ b/credentials/local/local_test.go
@@ -131,11 +131,13 @@ func serverHandle(hs serverHandshake, done chan testServerHandleResult, lis net.
 	serverRawConn, err := lis.Accept()
 	if err != nil {
 		done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed to accept connection. Error: %v", err)}
+		return
 	}
 	serverAuthInfo, err := hs(serverRawConn)
 	if err != nil {
 		serverRawConn.Close()
 		done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed while handshake. Error: %v", err)}
+		return
 	}
 	done <- testServerHandleResult{authInfo: serverAuthInfo, err: nil}
 }
diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go
index 1f6603cd759e..d684f5827103 100644
--- a/internal/transport/keepalive_test.go
+++ b/internal/transport/keepalive_test.go
@@ -193,7 +193,7 @@ func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) {
 
 	// We read from the net.Conn till we get an error, which is expected when
 	// the server closes the connection as part of the keepalive logic.
-	errCh := make(chan error)
+	errCh := make(chan error, 1)
 	go func() {
 		b := make([]byte, 24)
 		for {
diff --git a/internal/transport/proxy_test.go b/internal/transport/proxy_test.go
index a2f1aa438546..5434e9ed5c01 100644
--- a/internal/transport/proxy_test.go
+++ b/internal/transport/proxy_test.go
@@ -119,7 +119,7 @@ func testHTTPConnect(t *testing.T, proxyURLModify func(*url.URL) *url.URL, proxy
 
 	msg := []byte{4, 3, 5, 2}
 	recvBuf := make([]byte, len(msg))
-	done := make(chan error)
+	done := make(chan error, 1)
 	go func() {
 		in, err := blis.Accept()
 		if err != nil {
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 746df1745116..832ac8bd7180 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -6504,10 +6504,8 @@ func (s) TestDisabledIOBuffers(t *testing.T) {
 		t.Fatalf("Failed to create listener: %v", err)
 	}
 
-	done := make(chan struct{})
 	go func() {
 		s.Serve(lis)
-		close(done)
 	}()
 	defer s.Stop()
 	dctx, dcancel := context.WithTimeout(context.Background(), 5*time.Second)

From 1c598a11a4e503e1cfd500999c040e72072dc16b Mon Sep 17 00:00:00 2001
From: Zach Reyes <39203661+zasweq@users.noreply.github.com>
Date: Tue, 20 Apr 2021 13:20:09 -0400
Subject: [PATCH 425/481] Move exponential backoff to DNS resolver from
 resolver.ClientConn (#4270)

---
 balancer_conn_wrappers_test.go             |  90 -------
 dialoptions.go                             |  17 +-
 internal/resolver/dns/dns_resolver.go      |  39 ++-
 internal/resolver/dns/dns_resolver_test.go | 268 ++++++++++++++++++++-
 resolver/resolver.go                       |   2 +-
 resolver_conn_wrapper.go                   |  63 +----
 resolver_conn_wrapper_test.go              |  85 -------
 test/balancer_test.go                      |  14 +-
 xds/internal/resolver/xds_resolver_test.go |   3 +-
 9 files changed, 301 insertions(+), 280 deletions(-)
 delete mode 100644 balancer_conn_wrappers_test.go

diff --git a/balancer_conn_wrappers_test.go b/balancer_conn_wrappers_test.go
deleted file mode 100644
index 935d11d1d391..000000000000
--- a/balancer_conn_wrappers_test.go
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package grpc
-
-import (
-	"fmt"
-	"net"
-	"testing"
-
-	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/balancer/roundrobin"
-	"google.golang.org/grpc/internal/balancer/stub"
-	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/resolver/manual"
-)
-
-// TestBalancerErrorResolverPolling injects balancer errors and verifies
-// ResolveNow is called on the resolver with the appropriate backoff strategy
-// being consulted between ResolveNow calls.
-func (s) TestBalancerErrorResolverPolling(t *testing.T) {
-	// The test balancer will return ErrBadResolverState iff the
-	// ClientConnState contains no addresses.
-	bf := stub.BalancerFuncs{
-		UpdateClientConnState: func(_ *stub.BalancerData, s balancer.ClientConnState) error {
-			if len(s.ResolverState.Addresses) == 0 {
-				return balancer.ErrBadResolverState
-			}
-			return nil
-		},
-	}
-	const balName = "BalancerErrorResolverPolling"
-	stub.Register(balName, bf)
-
-	testResolverErrorPolling(t,
-		func(r *manual.Resolver) {
-			// No addresses so the balancer will fail.
-			r.CC.UpdateState(resolver.State{})
-		}, func(r *manual.Resolver) {
-			// UpdateState will block if ResolveNow is being called (which blocks on
-			// rn), so call it in a goroutine.  Include some address so the balancer
-			// will be happy.
-			go r.CC.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "x"}}})
-		},
-		WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, balName)))
-}
-
-// TestRoundRobinZeroAddressesResolverPolling reports no addresses to the round
-// robin balancer and verifies ResolveNow is called on the resolver with the
-// appropriate backoff strategy being consulted between ResolveNow calls.
-func (s) TestRoundRobinZeroAddressesResolverPolling(t *testing.T) {
-	// We need to start a real server or else the connecting loop will call
-	// ResolveNow after every iteration, even after a valid resolver result is
-	// returned.
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("Error while listening. Err: %v", err)
-	}
-	defer lis.Close()
-	s := NewServer()
-	defer s.Stop()
-	go s.Serve(lis)
-
-	testResolverErrorPolling(t,
-		func(r *manual.Resolver) {
-			// No addresses so the balancer will fail.
-			r.CC.UpdateState(resolver.State{})
-		}, func(r *manual.Resolver) {
-			// UpdateState will block if ResolveNow is being called (which
-			// blocks on rn), so call it in a goroutine.  Include a valid
-			// address so the balancer will be happy.
-			go r.CC.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
-		},
-		WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, roundrobin.Name)))
-}
diff --git a/dialoptions.go b/dialoptions.go
index e7f86e6d7c81..7a497237bbd3 100644
--- a/dialoptions.go
+++ b/dialoptions.go
@@ -66,11 +66,7 @@ type dialOptions struct {
 	minConnectTimeout           func() time.Duration
 	defaultServiceConfig        *ServiceConfig // defaultServiceConfig is parsed from defaultServiceConfigRawJSON.
 	defaultServiceConfigRawJSON *string
-	// This is used by ccResolverWrapper to backoff between successive calls to
-	// resolver.ResolveNow(). The user will have no need to configure this, but
-	// we need to be able to configure this in tests.
-	resolveNowBackoff func(int) time.Duration
-	resolvers         []resolver.Builder
+	resolvers                   []resolver.Builder
 }
 
 // DialOption configures how we set up the connection.
@@ -596,7 +592,6 @@ func defaultDialOptions() dialOptions {
 			ReadBufferSize:  defaultReadBufSize,
 			UseProxy:        true,
 		},
-		resolveNowBackoff: internalbackoff.DefaultExponential.Backoff,
 	}
 }
 
@@ -611,16 +606,6 @@ func withMinConnectDeadline(f func() time.Duration) DialOption {
 	})
 }
 
-// withResolveNowBackoff specifies the function that clientconn uses to backoff
-// between successive calls to resolver.ResolveNow().
-//
-// For testing purpose only.
-func withResolveNowBackoff(f func(int) time.Duration) DialOption {
-	return newFuncDialOption(func(o *dialOptions) {
-		o.resolveNowBackoff = f
-	})
-}
-
 // WithResolvers allows a list of resolver implementations to be registered
 // locally with the ClientConn without needing to be globally registered via
 // resolver.Register.  They will be matched against the scheme used for the
diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go
index 304235566589..9d86460ab6fb 100644
--- a/internal/resolver/dns/dns_resolver.go
+++ b/internal/resolver/dns/dns_resolver.go
@@ -34,6 +34,7 @@ import (
 
 	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/envconfig"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/resolver"
@@ -46,6 +47,9 @@ var EnableSRVLookups = false
 
 var logger = grpclog.Component("dns")
 
+// A global to stub out in tests.
+var newTimer = time.NewTimer
+
 func init() {
 	resolver.Register(NewBuilder())
 }
@@ -143,7 +147,6 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts
 
 	d.wg.Add(1)
 	go d.watcher()
-	d.ResolveNow(resolver.ResolveNowOptions{})
 	return d, nil
 }
 
@@ -201,28 +204,38 @@ func (d *dnsResolver) Close() {
 
 func (d *dnsResolver) watcher() {
 	defer d.wg.Done()
+	backoffIndex := 1
 	for {
-		select {
-		case <-d.ctx.Done():
-			return
-		case <-d.rn:
-		}
-
 		state, err := d.lookup()
 		if err != nil {
+			// Report error to the underlying grpc.ClientConn.
 			d.cc.ReportError(err)
 		} else {
-			d.cc.UpdateState(*state)
+			err = d.cc.UpdateState(*state)
 		}
 
-		// Sleep to prevent excessive re-resolutions. Incoming resolution requests
-		// will be queued in d.rn.
-		t := time.NewTimer(minDNSResRate)
+		var timer *time.Timer
+		if err == nil {
+			// Success resolving, wait for the next ResolveNow. However, also wait 30 seconds at the very least
+			// to prevent constantly re-resolving.
+			backoffIndex = 1
+			timer = time.NewTimer(minDNSResRate)
+			select {
+			case <-d.ctx.Done():
+				timer.Stop()
+				return
+			case <-d.rn:
+			}
+		} else {
+			// Poll on an error found in DNS Resolver or an error received from ClientConn.
+			timer = newTimer(backoff.DefaultExponential.Backoff(backoffIndex))
+			backoffIndex++
+		}
 		select {
-		case <-t.C:
 		case <-d.ctx.Done():
-			t.Stop()
+			timer.Stop()
 			return
+		case <-timer.C:
 		}
 	}
 }
diff --git a/internal/resolver/dns/dns_resolver_test.go b/internal/resolver/dns/dns_resolver_test.go
index 1c8469a275a7..52067e39cc68 100644
--- a/internal/resolver/dns/dns_resolver_test.go
+++ b/internal/resolver/dns/dns_resolver_test.go
@@ -30,9 +30,11 @@ import (
 	"testing"
 	"time"
 
+	"google.golang.org/grpc/balancer"
 	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
 	"google.golang.org/grpc/internal/envconfig"
 	"google.golang.org/grpc/internal/leakcheck"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 )
@@ -47,7 +49,8 @@ func TestMain(m *testing.M) {
 }
 
 const (
-	txtBytesLimit = 255
+	txtBytesLimit      = 255
+	defaultTestTimeout = 10 * time.Second
 )
 
 type testClientConn struct {
@@ -57,13 +60,17 @@ type testClientConn struct {
 	state               resolver.State
 	updateStateCalls    int
 	errChan             chan error
+	updateStateErr      error
 }
 
-func (t *testClientConn) UpdateState(s resolver.State) {
+func (t *testClientConn) UpdateState(s resolver.State) error {
 	t.m1.Lock()
 	defer t.m1.Unlock()
 	t.state = s
 	t.updateStateCalls++
+	// This error determines whether DNS Resolver actually decides to exponentially backoff or not.
+	// This can be any error.
+	return t.updateStateErr
 }
 
 func (t *testClientConn) getState() (resolver.State, int) {
@@ -669,6 +676,13 @@ func TestResolve(t *testing.T) {
 
 func testDNSResolver(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(_ time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	tests := []struct {
 		target   string
 		addrWant []resolver.Address
@@ -736,12 +750,151 @@ func testDNSResolver(t *testing.T) {
 	}
 }
 
+// DNS Resolver immediately starts polling on an error from grpc. This should continue until the ClientConn doesn't
+// send back an error from updating the DNS Resolver's state.
+func TestDNSResolverExponentialBackoff(t *testing.T) {
+	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	timerChan := testutils.NewChannel()
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, allows this test to call timer immediately.
+		t := time.NewTimer(time.Hour)
+		timerChan.Send(t)
+		return t
+	}
+	tests := []struct {
+		name     string
+		target   string
+		addrWant []resolver.Address
+		scWant   string
+	}{
+		{
+			"happy case default port",
+			"foo.bar.com",
+			[]resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}},
+			generateSC("foo.bar.com"),
+		},
+		{
+			"happy case specified port",
+			"foo.bar.com:1234",
+			[]resolver.Address{{Addr: "1.2.3.4:1234"}, {Addr: "5.6.7.8:1234"}},
+			generateSC("foo.bar.com"),
+		},
+		{
+			"happy case another default port",
+			"srv.ipv4.single.fake",
+			[]resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}},
+			generateSC("srv.ipv4.single.fake"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			b := NewBuilder()
+			cc := &testClientConn{target: test.target}
+			// Cause ClientConn to return an error.
+			cc.updateStateErr = balancer.ErrBadResolverState
+			r, err := b.Build(resolver.Target{Endpoint: test.target}, cc, resolver.BuildOptions{})
+			if err != nil {
+				t.Fatalf("Error building resolver for target %v: %v", test.target, err)
+			}
+			var state resolver.State
+			var cnt int
+			for i := 0; i < 2000; i++ {
+				state, cnt = cc.getState()
+				if cnt > 0 {
+					break
+				}
+				time.Sleep(time.Millisecond)
+			}
+			if cnt == 0 {
+				t.Fatalf("UpdateState not called after 2s; aborting")
+			}
+			if !reflect.DeepEqual(test.addrWant, state.Addresses) {
+				t.Errorf("Resolved addresses of target: %q = %+v, want %+v", test.target, state.Addresses, test.addrWant)
+			}
+			sc := scFromState(state)
+			if test.scWant != sc {
+				t.Errorf("Resolved service config of target: %q = %+v, want %+v", test.target, sc, test.scWant)
+			}
+			ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer ctxCancel()
+			// Cause timer to go off 10 times, and see if it calls updateState() correctly.
+			for i := 0; i < 10; i++ {
+				timer, err := timerChan.Receive(ctx)
+				if err != nil {
+					t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
+				}
+				timerPointer := timer.(*time.Timer)
+				timerPointer.Reset(0)
+			}
+			// Poll to see if DNS Resolver updated state the correct number of times, which allows time for the DNS Resolver to call
+			// ClientConn update state.
+			deadline := time.Now().Add(defaultTestTimeout)
+			for {
+				cc.m1.Lock()
+				got := cc.updateStateCalls
+				cc.m1.Unlock()
+				if got == 11 {
+					break
+				}
+
+				if time.Now().After(deadline) {
+					t.Fatalf("Exponential backoff is not working as expected - should update state 11 times instead of %d", got)
+				}
+
+				time.Sleep(time.Millisecond)
+			}
+
+			// Update resolver.ClientConn to not return an error anymore - this should stop it from backing off.
+			cc.updateStateErr = nil
+			timer, err := timerChan.Receive(ctx)
+			if err != nil {
+				t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
+			}
+			timerPointer := timer.(*time.Timer)
+			timerPointer.Reset(0)
+			// Poll to see if DNS Resolver updated state the correct number of times, which allows time for the DNS Resolver to call
+			// ClientConn update state the final time. The DNS Resolver should then stop polling.
+			deadline = time.Now().Add(defaultTestTimeout)
+			for {
+				cc.m1.Lock()
+				got := cc.updateStateCalls
+				cc.m1.Unlock()
+				if got == 12 {
+					break
+				}
+
+				if time.Now().After(deadline) {
+					t.Fatalf("Exponential backoff is not working as expected - should stop backing off at 12 total UpdateState calls instead of %d", got)
+				}
+
+				_, err := timerChan.ReceiveOrFail()
+				if err {
+					t.Fatalf("Should not poll again after Client Conn stops returning error.")
+				}
+
+				time.Sleep(time.Millisecond)
+			}
+			r.Close()
+		})
+	}
+}
+
 func testDNSResolverWithSRV(t *testing.T) {
 	EnableSRVLookups = true
 	defer func() {
 		EnableSRVLookups = false
 	}()
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(_ time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	tests := []struct {
 		target      string
 		addrWant    []resolver.Address
@@ -855,6 +1008,13 @@ func mutateTbl(target string) func() {
 
 func testDNSResolveNow(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(_ time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	tests := []struct {
 		target   string
 		addrWant []resolver.Address
@@ -926,6 +1086,13 @@ const colonDefaultPort = ":" + defaultPort
 
 func testIPResolver(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(_ time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	tests := []struct {
 		target string
 		want   []resolver.Address
@@ -975,6 +1142,13 @@ func testIPResolver(t *testing.T) {
 
 func TestResolveFunc(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	tests := []struct {
 		addr string
 		want error
@@ -1013,6 +1187,13 @@ func TestResolveFunc(t *testing.T) {
 
 func TestDisableServiceConfig(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	tests := []struct {
 		target               string
 		scWant               string
@@ -1059,6 +1240,13 @@ func TestDisableServiceConfig(t *testing.T) {
 
 func TestTXTError(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	defer func(v bool) { envconfig.TXTErrIgnore = v }(envconfig.TXTErrIgnore)
 	for _, ignore := range []bool{false, true} {
 		envconfig.TXTErrIgnore = ignore
@@ -1090,6 +1278,13 @@ func TestTXTError(t *testing.T) {
 }
 
 func TestDNSResolverRetry(t *testing.T) {
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 	b := NewBuilder()
 	target := "ipv4.single.fake"
 	cc := &testClientConn{target: target}
@@ -1144,6 +1339,13 @@ func TestDNSResolverRetry(t *testing.T) {
 
 func TestCustomAuthority(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 
 	tests := []struct {
 		authority     string
@@ -1251,6 +1453,13 @@ func TestCustomAuthority(t *testing.T) {
 // requests are made.
 func TestRateLimitedResolve(t *testing.T) {
 	defer leakcheck.Check(t)
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, will protect from triggering exponential backoff.
+		return time.NewTimer(time.Hour)
+	}
 
 	const dnsResRate = 10 * time.Millisecond
 	dc := replaceDNSResRate(dnsResRate)
@@ -1347,21 +1556,66 @@ func TestRateLimitedResolve(t *testing.T) {
 	}
 }
 
+// DNS Resolver immediately starts polling on an error. This will cause the re-resolution to return another error.
+// Thus, test that it constantly sends errors to the grpc.ClientConn.
 func TestReportError(t *testing.T) {
 	const target = "notfoundaddress"
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimer = nt
+	}(newTimer)
+	timerChan := testutils.NewChannel()
+	newTimer = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, allows this test to call timer immediately.
+		t := time.NewTimer(time.Hour)
+		timerChan.Send(t)
+		return t
+	}
 	cc := &testClientConn{target: target, errChan: make(chan error)}
+	totalTimesCalledError := 0
 	b := NewBuilder()
 	r, err := b.Build(resolver.Target{Endpoint: target}, cc, resolver.BuildOptions{})
 	if err != nil {
-		t.Fatalf("%v\n", err)
+		t.Fatalf("Error building resolver for target %v: %v", target, err)
+	}
+	// Should receive first error.
+	err = <-cc.errChan
+	if !strings.Contains(err.Error(), "hostLookup error") {
+		t.Fatalf(`ReportError(err=%v) called; want err contains "hostLookupError"`, err)
 	}
+	totalTimesCalledError++
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	timer, err := timerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
+	}
+	timerPointer := timer.(*time.Timer)
+	timerPointer.Reset(0)
 	defer r.Close()
-	select {
-	case err := <-cc.errChan:
+
+	// Cause timer to go off 10 times, and see if it matches DNS Resolver updating Error.
+	for i := 0; i < 10; i++ {
+		// Should call ReportError().
+		err = <-cc.errChan
 		if !strings.Contains(err.Error(), "hostLookup error") {
 			t.Fatalf(`ReportError(err=%v) called; want err contains "hostLookupError"`, err)
 		}
-	case <-time.After(time.Second):
-		t.Fatalf("did not receive error after 1s")
+		totalTimesCalledError++
+		timer, err := timerChan.Receive(ctx)
+		if err != nil {
+			t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
+		}
+		timerPointer := timer.(*time.Timer)
+		timerPointer.Reset(0)
+	}
+
+	if totalTimesCalledError != 11 {
+		t.Errorf("ReportError() not called 11 times, instead called %d times.", totalTimesCalledError)
+	}
+	// Clean up final watcher iteration.
+	<-cc.errChan
+	_, err = timerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
 	}
 }
diff --git a/resolver/resolver.go b/resolver/resolver.go
index e9fa8e33d923..6a9d234a597a 100644
--- a/resolver/resolver.go
+++ b/resolver/resolver.go
@@ -181,7 +181,7 @@ type State struct {
 // gRPC to add new methods to this interface.
 type ClientConn interface {
 	// UpdateState updates the state of the ClientConn appropriately.
-	UpdateState(State)
+	UpdateState(State) error
 	// ReportError notifies the ClientConn that the Resolver encountered an
 	// error.  The ClientConn will notify the load balancer and begin calling
 	// ResolveNow on the Resolver with exponential backoff.
diff --git a/resolver_conn_wrapper.go b/resolver_conn_wrapper.go
index f2d81968f9ec..4118de571ab5 100644
--- a/resolver_conn_wrapper.go
+++ b/resolver_conn_wrapper.go
@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"strings"
 	"sync"
-	"time"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/credentials"
@@ -40,9 +39,6 @@ type ccResolverWrapper struct {
 	resolver   resolver.Resolver
 	done       *grpcsync.Event
 	curState   resolver.State
-
-	pollingMu sync.Mutex
-	polling   chan struct{}
 }
 
 // newCCResolverWrapper uses the resolver.Builder to build a Resolver and
@@ -93,59 +89,19 @@ func (ccr *ccResolverWrapper) close() {
 	ccr.resolverMu.Unlock()
 }
 
-// poll begins or ends asynchronous polling of the resolver based on whether
-// err is ErrBadResolverState.
-func (ccr *ccResolverWrapper) poll(err error) {
-	ccr.pollingMu.Lock()
-	defer ccr.pollingMu.Unlock()
-	if err != balancer.ErrBadResolverState {
-		// stop polling
-		if ccr.polling != nil {
-			close(ccr.polling)
-			ccr.polling = nil
-		}
-		return
-	}
-	if ccr.polling != nil {
-		// already polling
-		return
-	}
-	p := make(chan struct{})
-	ccr.polling = p
-	go func() {
-		for i := 0; ; i++ {
-			ccr.resolveNow(resolver.ResolveNowOptions{})
-			t := time.NewTimer(ccr.cc.dopts.resolveNowBackoff(i))
-			select {
-			case <-p:
-				t.Stop()
-				return
-			case <-ccr.done.Done():
-				// Resolver has been closed.
-				t.Stop()
-				return
-			case <-t.C:
-				select {
-				case <-p:
-					return
-				default:
-				}
-				// Timer expired; re-resolve.
-			}
-		}
-	}()
-}
-
-func (ccr *ccResolverWrapper) UpdateState(s resolver.State) {
+func (ccr *ccResolverWrapper) UpdateState(s resolver.State) error {
 	if ccr.done.HasFired() {
-		return
+		return nil
 	}
 	channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending update to cc: %v", s)
 	if channelz.IsOn() {
 		ccr.addChannelzTraceEvent(s)
 	}
 	ccr.curState = s
-	ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil))
+	if err := ccr.cc.updateResolverState(ccr.curState, nil); err == balancer.ErrBadResolverState {
+		return balancer.ErrBadResolverState
+	}
+	return nil
 }
 
 func (ccr *ccResolverWrapper) ReportError(err error) {
@@ -153,7 +109,7 @@ func (ccr *ccResolverWrapper) ReportError(err error) {
 		return
 	}
 	channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: reporting error to cc: %v", err)
-	ccr.poll(ccr.cc.updateResolverState(resolver.State{}, err))
+	ccr.cc.updateResolverState(resolver.State{}, err)
 }
 
 // NewAddress is called by the resolver implementation to send addresses to gRPC.
@@ -166,7 +122,7 @@ func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) {
 		ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig})
 	}
 	ccr.curState.Addresses = addrs
-	ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil))
+	ccr.cc.updateResolverState(ccr.curState, nil)
 }
 
 // NewServiceConfig is called by the resolver implementation to send service
@@ -183,14 +139,13 @@ func (ccr *ccResolverWrapper) NewServiceConfig(sc string) {
 	scpr := parseServiceConfig(sc)
 	if scpr.Err != nil {
 		channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err)
-		ccr.poll(balancer.ErrBadResolverState)
 		return
 	}
 	if channelz.IsOn() {
 		ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: scpr})
 	}
 	ccr.curState.ServiceConfig = scpr
-	ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil))
+	ccr.cc.updateResolverState(ccr.curState, nil)
 }
 
 func (ccr *ccResolverWrapper) ParseServiceConfig(scJSON string) *serviceconfig.ParseResult {
diff --git a/resolver_conn_wrapper_test.go b/resolver_conn_wrapper_test.go
index f13a408937b1..81c5b9ea874f 100644
--- a/resolver_conn_wrapper_test.go
+++ b/resolver_conn_wrapper_test.go
@@ -67,62 +67,6 @@ func (s) TestDialParseTargetUnknownScheme(t *testing.T) {
 	}
 }
 
-func testResolverErrorPolling(t *testing.T, badUpdate func(*manual.Resolver), goodUpdate func(*manual.Resolver), dopts ...DialOption) {
-	boIter := make(chan int)
-	resolverBackoff := func(v int) time.Duration {
-		boIter <- v
-		return 0
-	}
-
-	r := manual.NewBuilderWithScheme("whatever")
-	rn := make(chan struct{})
-	defer func() { close(rn) }()
-	r.ResolveNowCallback = func(resolver.ResolveNowOptions) { rn <- struct{}{} }
-
-	defaultDialOptions := []DialOption{
-		WithInsecure(),
-		WithResolvers(r),
-		withResolveNowBackoff(resolverBackoff),
-	}
-	cc, err := Dial(r.Scheme()+":///test.server", append(defaultDialOptions, dopts...)...)
-	if err != nil {
-		t.Fatalf("Dial(_, _) = _, %v; want _, nil", err)
-	}
-	defer cc.Close()
-	badUpdate(r)
-
-	panicAfter := time.AfterFunc(5*time.Second, func() { panic("timed out polling resolver") })
-	defer panicAfter.Stop()
-
-	// Ensure ResolveNow is called, then Backoff with the right parameter, several times
-	for i := 0; i < 7; i++ {
-		<-rn
-		if v := <-boIter; v != i {
-			t.Errorf("Backoff call %v uses value %v", i, v)
-		}
-	}
-
-	// UpdateState will block if ResolveNow is being called (which blocks on
-	// rn), so call it in a goroutine.
-	goodUpdate(r)
-
-	// Wait awhile to ensure ResolveNow and Backoff stop being called when the
-	// state is OK (i.e. polling was cancelled).
-	for {
-		t := time.NewTimer(50 * time.Millisecond)
-		select {
-		case <-rn:
-			// ClientConn is still calling ResolveNow
-			<-boIter
-			time.Sleep(5 * time.Millisecond)
-			continue
-		case <-t.C:
-			// ClientConn stopped calling ResolveNow; success
-		}
-		break
-	}
-}
-
 const happyBalancerName = "happy balancer"
 
 func init() {
@@ -136,35 +80,6 @@ func init() {
 	stub.Register(happyBalancerName, bf)
 }
 
-// TestResolverErrorPolling injects resolver errors and verifies ResolveNow is
-// called with the appropriate backoff strategy being consulted between
-// ResolveNow calls.
-func (s) TestResolverErrorPolling(t *testing.T) {
-	testResolverErrorPolling(t, func(r *manual.Resolver) {
-		r.CC.ReportError(errors.New("res err"))
-	}, func(r *manual.Resolver) {
-		// UpdateState will block if ResolveNow is being called (which blocks on
-		// rn), so call it in a goroutine.
-		go r.CC.UpdateState(resolver.State{})
-	},
-		WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, happyBalancerName)))
-}
-
-// TestServiceConfigErrorPolling injects a service config error and verifies
-// ResolveNow is called with the appropriate backoff strategy being consulted
-// between ResolveNow calls.
-func (s) TestServiceConfigErrorPolling(t *testing.T) {
-	testResolverErrorPolling(t, func(r *manual.Resolver) {
-		badsc := r.CC.ParseServiceConfig("bad config")
-		r.UpdateState(resolver.State{ServiceConfig: badsc})
-	}, func(r *manual.Resolver) {
-		// UpdateState will block if ResolveNow is being called (which blocks on
-		// rn), so call it in a goroutine.
-		go r.CC.UpdateState(resolver.State{})
-	},
-		WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, happyBalancerName)))
-}
-
 // TestResolverErrorInBuild makes the resolver.Builder call into the ClientConn
 // during the Build call. We use two separate mutexes in the code which make
 // sure there is no data race in this code path, and also that there is no
diff --git a/test/balancer_test.go b/test/balancer_test.go
index bc22036dbac3..a6a8f726afa8 100644
--- a/test/balancer_test.go
+++ b/test/balancer_test.go
@@ -37,7 +37,6 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/balancer/stub"
 	"google.golang.org/grpc/internal/balancerload"
-	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpcutil"
 	imetadata "google.golang.org/grpc/internal/metadata"
 	"google.golang.org/grpc/internal/stubserver"
@@ -698,10 +697,7 @@ func (s) TestEmptyAddrs(t *testing.T) {
 
 	// Initialize pickfirst client
 	pfr := manual.NewBuilderWithScheme("whatever")
-	pfrnCalled := grpcsync.NewEvent()
-	pfr.ResolveNowCallback = func(resolver.ResolveNowOptions) {
-		pfrnCalled.Fire()
-	}
+
 	pfr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
 
 	pfcc, err := grpc.DialContext(ctx, pfr.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(pfr))
@@ -718,16 +714,10 @@ func (s) TestEmptyAddrs(t *testing.T) {
 
 	// Remove all addresses.
 	pfr.UpdateState(resolver.State{})
-	// Wait for a ResolveNow call on the pick first client's resolver.
-	<-pfrnCalled.Done()
 
 	// Initialize roundrobin client
 	rrr := manual.NewBuilderWithScheme("whatever")
 
-	rrrnCalled := grpcsync.NewEvent()
-	rrr.ResolveNowCallback = func(resolver.ResolveNowOptions) {
-		rrrnCalled.Fire()
-	}
 	rrr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
 
 	rrcc, err := grpc.DialContext(ctx, rrr.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(rrr),
@@ -745,8 +735,6 @@ func (s) TestEmptyAddrs(t *testing.T) {
 
 	// Remove all addresses.
 	rrr.UpdateState(resolver.State{})
-	// Wait for a ResolveNow call on the round robin client's resolver.
-	<-rrrnCalled.Done()
 
 	// Confirm several new RPCs succeed on pick first.
 	for i := 0; i < 10; i++ {
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 82355eecc70b..b3c2006b72be 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -88,8 +88,9 @@ type testClientConn struct {
 	errorCh *testutils.Channel
 }
 
-func (t *testClientConn) UpdateState(s resolver.State) {
+func (t *testClientConn) UpdateState(s resolver.State) error {
 	t.stateCh.Send(s)
+	return nil
 }
 
 func (t *testClientConn) ReportError(err error) {

From 970aa0928304dec8dbf2bc11ee0dd49ad16c8f30 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 21 Apr 2021 10:11:28 -0700
Subject: [PATCH 426/481] xds/balancers: export balancer names and config
 structs (#4334)

---
 .../balancer/clusterimpl/balancer_test.go     |  44 +++---
 .../balancer/clusterimpl/clusterimpl.go       |  19 +--
 xds/internal/balancer/clusterimpl/config.go   |  27 ++--
 .../balancer/clusterimpl/config_test.go       |  14 +-
 xds/internal/balancer/clusterimpl/picker.go   |   2 +-
 xds/internal/balancer/edsbalancer/eds_test.go |   2 +-
 xds/internal/balancer/lrs/balancer.go         |  23 +--
 xds/internal/balancer/lrs/balancer_test.go    |  14 +-
 xds/internal/balancer/lrs/config.go           |  24 +--
 xds/internal/balancer/lrs/config_test.go      |  10 +-
 xds/internal/balancer/priority/balancer.go    |  12 +-
 .../balancer/priority/balancer_test.go        | 148 +++++++++---------
 xds/internal/balancer/priority/config.go      |  18 ++-
 xds/internal/balancer/priority/config_test.go |   6 +-
 .../balancer/weightedtarget/weightedtarget.go |   9 +-
 .../weightedtarget/weightedtarget_config.go   |  29 ++--
 .../weightedtarget_config_test.go             |   6 +-
 .../weightedtarget/weightedtarget_test.go     |   2 +-
 18 files changed, 208 insertions(+), 201 deletions(-)

diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index d3789ffddca3..f421f2281b59 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -76,7 +76,7 @@ func TestDropByCategory(t *testing.T) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 	defer func() { newXDSClient = oldNewXDSClient }()
 
-	builder := balancer.Get(clusterImplName)
+	builder := balancer.Get(Name)
 	cc := testutils.NewTestClientConn(t)
 	b := builder.Build(cc, balancer.BuildOptions{})
 	defer b.Close()
@@ -90,11 +90,11 @@ func TestDropByCategory(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
-			Cluster:                    testClusterName,
-			EDSServiceName:             testServiceName,
-			LRSLoadReportingServerName: newString(testLRSServerName),
-			DropCategories: []dropCategory{{
+		BalancerConfig: &LBConfig{
+			Cluster:                 testClusterName,
+			EDSServiceName:          testServiceName,
+			LoadReportingServerName: newString(testLRSServerName),
+			DropCategories: []DropConfig{{
 				Category:           dropReason,
 				RequestsPerMillion: million * dropNumerator / dropDenominator,
 			}},
@@ -177,11 +177,11 @@ func TestDropByCategory(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
-			Cluster:                    testClusterName,
-			EDSServiceName:             testServiceName,
-			LRSLoadReportingServerName: newString(testLRSServerName),
-			DropCategories: []dropCategory{{
+		BalancerConfig: &LBConfig{
+			Cluster:                 testClusterName,
+			EDSServiceName:          testServiceName,
+			LoadReportingServerName: newString(testLRSServerName),
+			DropCategories: []DropConfig{{
 				Category:           dropReason2,
 				RequestsPerMillion: million * dropNumerator2 / dropDenominator2,
 			}},
@@ -234,7 +234,7 @@ func TestDropCircuitBreaking(t *testing.T) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 	defer func() { newXDSClient = oldNewXDSClient }()
 
-	builder := balancer.Get(clusterImplName)
+	builder := balancer.Get(Name)
 	cc := testutils.NewTestClientConn(t)
 	b := builder.Build(cc, balancer.BuildOptions{})
 	defer b.Close()
@@ -244,11 +244,11 @@ func TestDropCircuitBreaking(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
-			Cluster:                    testClusterName,
-			EDSServiceName:             testServiceName,
-			LRSLoadReportingServerName: newString(testLRSServerName),
-			MaxConcurrentRequests:      &maxRequest,
+		BalancerConfig: &LBConfig{
+			Cluster:                 testClusterName,
+			EDSServiceName:          testServiceName,
+			LoadReportingServerName: newString(testLRSServerName),
+			MaxConcurrentRequests:   &maxRequest,
 			ChildPolicy: &internalserviceconfig.BalancerConfig{
 				Name: roundrobin.Name,
 			},
@@ -346,7 +346,7 @@ func TestPickerUpdateAfterClose(t *testing.T) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 	defer func() { newXDSClient = oldNewXDSClient }()
 
-	builder := balancer.Get(clusterImplName)
+	builder := balancer.Get(Name)
 	cc := testutils.NewTestClientConn(t)
 	b := builder.Build(cc, balancer.BuildOptions{})
 
@@ -355,7 +355,7 @@ func TestPickerUpdateAfterClose(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
+		BalancerConfig: &LBConfig{
 			Cluster:               testClusterName,
 			EDSServiceName:        testServiceName,
 			MaxConcurrentRequests: &maxRequest,
@@ -391,7 +391,7 @@ func TestClusterNameInAddressAttributes(t *testing.T) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 	defer func() { newXDSClient = oldNewXDSClient }()
 
-	builder := balancer.Get(clusterImplName)
+	builder := balancer.Get(Name)
 	cc := testutils.NewTestClientConn(t)
 	b := builder.Build(cc, balancer.BuildOptions{})
 	defer b.Close()
@@ -400,7 +400,7 @@ func TestClusterNameInAddressAttributes(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
+		BalancerConfig: &LBConfig{
 			Cluster:        testClusterName,
 			EDSServiceName: testServiceName,
 			ChildPolicy: &internalserviceconfig.BalancerConfig{
@@ -451,7 +451,7 @@ func TestClusterNameInAddressAttributes(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: []resolver.Address{addr2},
 		},
-		BalancerConfig: &lbConfig{
+		BalancerConfig: &LBConfig{
 			Cluster:        testClusterName2,
 			EDSServiceName: testServiceName,
 			ChildPolicy: &internalserviceconfig.BalancerConfig{
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index 7d4f7695daf4..56664e391ac6 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -42,7 +42,8 @@ import (
 )
 
 const (
-	clusterImplName        = "xds_cluster_impl_experimental"
+	// Name is the name of the cluster_impl balancer.
+	Name                   = "xds_cluster_impl_experimental"
 	defaultRequestCountMax = 1024
 )
 
@@ -78,7 +79,7 @@ func (clusterImplBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions)
 }
 
 func (clusterImplBB) Name() string {
-	return clusterImplName
+	return Name
 }
 
 func (clusterImplBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
@@ -110,7 +111,7 @@ type clusterImplBalancer struct {
 	logger *grpclog.PrefixLogger
 	xdsC   xdsClientInterface
 
-	config           *lbConfig
+	config           *LBConfig
 	childLB          balancer.Balancer
 	cancelLoadReport func()
 	edsServiceName   string
@@ -132,11 +133,11 @@ type clusterImplBalancer struct {
 
 // updateLoadStore checks the config for load store, and decides whether it
 // needs to restart the load reporting stream.
-func (cib *clusterImplBalancer) updateLoadStore(newConfig *lbConfig) error {
+func (cib *clusterImplBalancer) updateLoadStore(newConfig *LBConfig) error {
 	var updateLoadClusterAndService bool
 
 	// ClusterName is different, restart. ClusterName is from ClusterName and
-	// EdsServiceName.
+	// EDSServiceName.
 	clusterName := cib.getClusterName()
 	if clusterName != newConfig.Cluster {
 		updateLoadClusterAndService = true
@@ -161,11 +162,11 @@ func (cib *clusterImplBalancer) updateLoadStore(newConfig *lbConfig) error {
 
 	// Check if it's necessary to restart load report.
 	var newLRSServerName string
-	if newConfig.LRSLoadReportingServerName != nil {
-		newLRSServerName = *newConfig.LRSLoadReportingServerName
+	if newConfig.LoadReportingServerName != nil {
+		newLRSServerName = *newConfig.LoadReportingServerName
 	}
 	if cib.lrsServerName != newLRSServerName {
-		// LrsLoadReportingServerName is different, load should be report to a
+		// LoadReportingServerName is different, load should be report to a
 		// different server, restart.
 		cib.lrsServerName = newLRSServerName
 		if cib.cancelLoadReport != nil {
@@ -188,7 +189,7 @@ func (cib *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState
 		return nil
 	}
 
-	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
 	}
diff --git a/xds/internal/balancer/clusterimpl/config.go b/xds/internal/balancer/clusterimpl/config.go
index 548ab34bce4d..51ff654f6eb5 100644
--- a/xds/internal/balancer/clusterimpl/config.go
+++ b/xds/internal/balancer/clusterimpl/config.go
@@ -25,32 +25,33 @@ import (
 	"google.golang.org/grpc/serviceconfig"
 )
 
-type dropCategory struct {
+// DropConfig contains the category, and drop ratio.
+type DropConfig struct {
 	Category           string
 	RequestsPerMillion uint32
 }
 
-// lbConfig is the balancer config for weighted_target.
-type lbConfig struct {
-	serviceconfig.LoadBalancingConfig
+// LBConfig is the balancer config for cluster_impl balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
 
-	Cluster                    string
-	EDSServiceName             string
-	LRSLoadReportingServerName *string
-	MaxConcurrentRequests      *uint32
-	DropCategories             []dropCategory
-	ChildPolicy                *internalserviceconfig.BalancerConfig
+	Cluster                 string                                `json:"cluster,omitempty"`
+	EDSServiceName          string                                `json:"edsServiceName,omitempty"`
+	LoadReportingServerName *string                               `json:"lrsLoadReportingServerName,omitempty"`
+	MaxConcurrentRequests   *uint32                               `json:"maxConcurrentRequests,omitempty"`
+	DropCategories          []DropConfig                          `json:"dropCategories,omitempty"`
+	ChildPolicy             *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
 }
 
-func parseConfig(c json.RawMessage) (*lbConfig, error) {
-	var cfg lbConfig
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
 	if err := json.Unmarshal(c, &cfg); err != nil {
 		return nil, err
 	}
 	return &cfg, nil
 }
 
-func equalDropCategories(a, b []dropCategory) bool {
+func equalDropCategories(a, b []DropConfig) bool {
 	if len(a) != len(b) {
 		return false
 	}
diff --git a/xds/internal/balancer/clusterimpl/config_test.go b/xds/internal/balancer/clusterimpl/config_test.go
index 89696981e2a0..ccb0c5e74d90 100644
--- a/xds/internal/balancer/clusterimpl/config_test.go
+++ b/xds/internal/balancer/clusterimpl/config_test.go
@@ -87,7 +87,7 @@ func TestParseConfig(t *testing.T) {
 	tests := []struct {
 		name    string
 		js      string
-		want    *lbConfig
+		want    *LBConfig
 		wantErr bool
 	}{
 		{
@@ -105,12 +105,12 @@ func TestParseConfig(t *testing.T) {
 		{
 			name: "OK",
 			js:   testJSONConfig,
-			want: &lbConfig{
-				Cluster:                    "test_cluster",
-				EDSServiceName:             "test-eds",
-				LRSLoadReportingServerName: newString("lrs_server"),
-				MaxConcurrentRequests:      newUint32(123),
-				DropCategories: []dropCategory{
+			want: &LBConfig{
+				Cluster:                 "test_cluster",
+				EDSServiceName:          "test-eds",
+				LoadReportingServerName: newString("lrs_server"),
+				MaxConcurrentRequests:   newUint32(123),
+				DropCategories: []DropConfig{
 					{Category: "drop-1", RequestsPerMillion: 314},
 					{Category: "drop-2", RequestsPerMillion: 159},
 				},
diff --git a/xds/internal/balancer/clusterimpl/picker.go b/xds/internal/balancer/clusterimpl/picker.go
index 6e9d27911534..87faba13a746 100644
--- a/xds/internal/balancer/clusterimpl/picker.go
+++ b/xds/internal/balancer/clusterimpl/picker.go
@@ -47,7 +47,7 @@ func gcd(a, b uint32) uint32 {
 	return a
 }
 
-func newDropper(c dropCategory) *dropper {
+func newDropper(c DropConfig) *dropper {
 	w := newRandomWRR()
 	gcdv := gcd(c.RequestsPerMillion, million)
 	// Return true for RequestPerMillion, false for the rest.
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 65b74a1b8af5..544d0a301672 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -818,7 +818,7 @@ func (s) TestBalancerConfigParsing(t *testing.T) {
 			},
 		},
 		{
-			// json with no lrs server name, LrsLoadReportingServerName should
+			// json with no lrs server name, LoadReportingServerName should
 			// be nil (not an empty string).
 			name: "no-lrs-server-name",
 			js: json.RawMessage(`
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index ab9ee7109db1..460788eb53c3 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -37,7 +37,8 @@ func init() {
 
 var newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
 
-const lrsBalancerName = "lrs_experimental"
+// Name is the name of the LRS balancer.
+const Name = "lrs_experimental"
 
 type lrsBB struct{}
 
@@ -60,7 +61,7 @@ func (l *lrsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balanc
 }
 
 func (l *lrsBB) Name() string {
-	return lrsBalancerName
+	return Name
 }
 
 func (l *lrsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
@@ -74,12 +75,12 @@ type lrsBalancer struct {
 	logger *grpclog.PrefixLogger
 	client *xdsClientWrapper
 
-	config *lbConfig
+	config *LBConfig
 	lb     balancer.Balancer // The sub balancer.
 }
 
 func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
 	}
@@ -182,21 +183,21 @@ func newXDSClientWrapper(c xdsClientInterface) *xdsClientWrapper {
 
 // update checks the config and xdsclient, and decides whether it needs to
 // restart the load reporting stream.
-func (w *xdsClientWrapper) update(newConfig *lbConfig) error {
+func (w *xdsClientWrapper) update(newConfig *LBConfig) error {
 	var (
 		restartLoadReport           bool
 		updateLoadClusterAndService bool
 	)
 
 	// ClusterName is different, restart. ClusterName is from ClusterName and
-	// EdsServiceName.
+	// EDSServiceName.
 	if w.clusterName != newConfig.ClusterName {
 		updateLoadClusterAndService = true
 		w.clusterName = newConfig.ClusterName
 	}
-	if w.edsServiceName != newConfig.EdsServiceName {
+	if w.edsServiceName != newConfig.EDSServiceName {
 		updateLoadClusterAndService = true
-		w.edsServiceName = newConfig.EdsServiceName
+		w.edsServiceName = newConfig.EDSServiceName
 	}
 
 	if updateLoadClusterAndService {
@@ -211,11 +212,11 @@ func (w *xdsClientWrapper) update(newConfig *lbConfig) error {
 		w.loadWrapper.UpdateClusterAndService(w.clusterName, w.edsServiceName)
 	}
 
-	if w.lrsServerName != newConfig.LrsLoadReportingServerName {
-		// LrsLoadReportingServerName is different, load should be report to a
+	if w.lrsServerName != newConfig.LoadReportingServerName {
+		// LoadReportingServerName is different, load should be report to a
 		// different server, restart.
 		restartLoadReport = true
-		w.lrsServerName = newConfig.LrsLoadReportingServerName
+		w.lrsServerName = newConfig.LoadReportingServerName
 	}
 
 	if restartLoadReport {
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index 0b575b112104..b115860bf16d 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -49,7 +49,7 @@ var (
 )
 
 // TestLoadReporting verifies that the lrs balancer starts the loadReport
-// stream when the lbConfig passed to it contains a valid value for the LRS
+// stream when the LBConfig passed to it contains a valid value for the LRS
 // server (empty string).
 func TestLoadReporting(t *testing.T) {
 	xdsC := fakeclient.NewClient()
@@ -57,7 +57,7 @@ func TestLoadReporting(t *testing.T) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 	defer func() { newXDSClient = oldNewXDSClient }()
 
-	builder := balancer.Get(lrsBalancerName)
+	builder := balancer.Get(Name)
 	cc := testutils.NewTestClientConn(t)
 	lrsB := builder.Build(cc, balancer.BuildOptions{})
 	defer lrsB.Close()
@@ -66,11 +66,11 @@ func TestLoadReporting(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
-			ClusterName:                testClusterName,
-			EdsServiceName:             testServiceName,
-			LrsLoadReportingServerName: testLRSServerName,
-			Locality:                   testLocality,
+		BalancerConfig: &LBConfig{
+			ClusterName:             testClusterName,
+			EDSServiceName:          testServiceName,
+			LoadReportingServerName: testLRSServerName,
+			Locality:                testLocality,
 			ChildPolicy: &internalserviceconfig.BalancerConfig{
 				Name: roundrobin.Name,
 			},
diff --git a/xds/internal/balancer/lrs/config.go b/xds/internal/balancer/lrs/config.go
index 3d39961401b5..9e8fc1045d4d 100644
--- a/xds/internal/balancer/lrs/config.go
+++ b/xds/internal/balancer/lrs/config.go
@@ -27,25 +27,27 @@ import (
 	"google.golang.org/grpc/xds/internal"
 )
 
-type lbConfig struct {
-	serviceconfig.LoadBalancingConfig
-	ClusterName                string
-	EdsServiceName             string
-	LrsLoadReportingServerName string
-	Locality                   *internal.LocalityID
-	ChildPolicy                *internalserviceconfig.BalancerConfig
+// LBConfig is the balancer config for lrs balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	ClusterName             string                                `json:"clusterName,omitempty"`
+	EDSServiceName          string                                `json:"edsServiceName,omitempty"`
+	LoadReportingServerName string                                `json:"lrsLoadReportingServerName,omitempty"`
+	Locality                *internal.LocalityID                  `json:"locality,omitempty"`
+	ChildPolicy             *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
 }
 
-func parseConfig(c json.RawMessage) (*lbConfig, error) {
-	var cfg lbConfig
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
 	if err := json.Unmarshal(c, &cfg); err != nil {
 		return nil, err
 	}
 	if cfg.ClusterName == "" {
 		return nil, fmt.Errorf("required ClusterName is not set in %+v", cfg)
 	}
-	if cfg.LrsLoadReportingServerName == "" {
-		return nil, fmt.Errorf("required LrsLoadReportingServerName is not set in %+v", cfg)
+	if cfg.LoadReportingServerName == "" {
+		return nil, fmt.Errorf("required LoadReportingServerName is not set in %+v", cfg)
 	}
 	if cfg.Locality == nil {
 		return nil, fmt.Errorf("required Locality is not set in %+v", cfg)
diff --git a/xds/internal/balancer/lrs/config_test.go b/xds/internal/balancer/lrs/config_test.go
index f49430569fed..c460cd008fff 100644
--- a/xds/internal/balancer/lrs/config_test.go
+++ b/xds/internal/balancer/lrs/config_test.go
@@ -37,7 +37,7 @@ func TestParseConfig(t *testing.T) {
 	tests := []struct {
 		name    string
 		js      string
-		want    *lbConfig
+		want    *LBConfig
 		wantErr bool
 	}{
 		{
@@ -95,10 +95,10 @@ func TestParseConfig(t *testing.T) {
   "childPolicy":[{"round_robin":{}}]
 }
 			`,
-			want: &lbConfig{
-				ClusterName:                testClusterName,
-				EdsServiceName:             testServiceName,
-				LrsLoadReportingServerName: testLRSServerName,
+			want: &LBConfig{
+				ClusterName:             testClusterName,
+				EDSServiceName:          testServiceName,
+				LoadReportingServerName: testLRSServerName,
 				Locality: &xdsinternal.LocalityID{
 					Region:  "test-region",
 					Zone:    "test-zone",
diff --git a/xds/internal/balancer/priority/balancer.go b/xds/internal/balancer/priority/balancer.go
index 6c4ff08378ec..d5c99b0b9146 100644
--- a/xds/internal/balancer/priority/balancer.go
+++ b/xds/internal/balancer/priority/balancer.go
@@ -24,6 +24,7 @@
 package priority
 
 import (
+	"encoding/json"
 	"fmt"
 	"sync"
 	"time"
@@ -34,10 +35,12 @@ import (
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/hierarchy"
 	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
 )
 
-const priorityBalancerName = "priority_experimental"
+// Name is the name of the priority balancer.
+const Name = "priority_experimental"
 
 func init() {
 	balancer.Register(priorityBB{})
@@ -60,11 +63,14 @@ func (priorityBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) bal
 	go b.run()
 	b.logger.Infof("Created")
 	return b
+}
 
+func (b priorityBB) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(s)
 }
 
 func (priorityBB) Name() string {
-	return priorityBalancerName
+	return Name
 }
 
 // timerWrapper wraps a timer with a boolean. So that when a race happens
@@ -102,7 +108,7 @@ type priorityBalancer struct {
 }
 
 func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
 	}
diff --git a/xds/internal/balancer/priority/balancer_test.go b/xds/internal/balancer/priority/balancer_test.go
index be14231dcb3f..b15ea303d78f 100644
--- a/xds/internal/balancer/priority/balancer_test.go
+++ b/xds/internal/balancer/priority/balancer_test.go
@@ -83,7 +83,7 @@ func subConnFromPicker(t *testing.T, p balancer.Picker) func() balancer.SubConn
 // Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0.
 func (s) TestPriority_HighPriorityReady(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -95,8 +95,8 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -132,8 +132,8 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
@@ -162,8 +162,8 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -190,7 +190,7 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 // down, use 2; remove 2, use 1.
 func (s) TestPriority_SwitchPriority(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -202,8 +202,8 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -269,8 +269,8 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
@@ -328,8 +328,8 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -373,7 +373,7 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 // use 0.
 func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -385,8 +385,8 @@ func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -468,7 +468,7 @@ func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) {
 // Init 0 and 1; 0 and 1 both down; add 2, use 2.
 func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -480,8 +480,8 @@ func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -534,8 +534,8 @@ func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
@@ -579,7 +579,7 @@ func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) {
 	// defer time.Sleep(10 * time.Millisecond)
 
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -592,8 +592,8 @@ func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
@@ -695,7 +695,7 @@ func (s) TestPriority_InitTimeout(t *testing.T) {
 	}()()
 
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -707,8 +707,8 @@ func (s) TestPriority_InitTimeout(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -765,7 +765,7 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 	}()()
 
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -777,8 +777,8 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -808,7 +808,7 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: nil,
 		},
-		BalancerConfig: &lbConfig{
+		BalancerConfig: &LBConfig{
 			Children:   nil,
 			Priorities: nil,
 		},
@@ -838,8 +838,8 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[3]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -882,8 +882,8 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
@@ -933,7 +933,7 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 // will be used.
 func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -945,8 +945,8 @@ func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -980,8 +980,8 @@ func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -1032,7 +1032,7 @@ func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
 	defaultPriorityInitTimeout = testPriorityInitTimeout
 
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1043,8 +1043,8 @@ func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
@@ -1058,7 +1058,7 @@ func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: nil,
 		},
-		BalancerConfig: &lbConfig{
+		BalancerConfig: &LBConfig{
 			Children:   nil,
 			Priorities: nil,
 		},
@@ -1075,7 +1075,7 @@ func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
 // Init a(p0) and b(p1); a(p0) is up, use a; move b to p0, a to p1, use b.
 func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1087,8 +1087,8 @@ func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -1124,8 +1124,8 @@ func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -1176,7 +1176,7 @@ func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
 // Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b.
 func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1188,8 +1188,8 @@ func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -1240,8 +1240,8 @@ func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -1276,7 +1276,7 @@ func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
 // Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b.
 func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1288,8 +1288,8 @@ func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
@@ -1338,8 +1338,8 @@ func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
@@ -1384,7 +1384,7 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 	}()()
 
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1395,8 +1395,8 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
@@ -1426,7 +1426,7 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 	// be different.
 	if err := pb.UpdateClientConnState(balancer.ClientConnState{
 		ResolverState:  resolver.State{},
-		BalancerConfig: &lbConfig{},
+		BalancerConfig: &LBConfig{},
 	}); err != nil {
 		t.Fatalf("failed to update ClientConn state: %v", err)
 	}
@@ -1454,8 +1454,8 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
@@ -1487,7 +1487,7 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 // Init 0; 0 is up, use 0; change 0's policy, 0 is used.
 func (s) TestPriority_ChildPolicyChange(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1498,8 +1498,8 @@ func (s) TestPriority_ChildPolicyChange(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
@@ -1533,8 +1533,8 @@ func (s) TestPriority_ChildPolicyChange(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: testRRBalancerName}},
 			},
 			Priorities: []string{"child-0"},
@@ -1587,7 +1587,7 @@ func init() {
 // by acquiring a locked mutex.
 func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) {
 	cc := testutils.NewTestClientConn(t)
-	bb := balancer.Get(priorityBalancerName)
+	bb := balancer.Get(Name)
 	pb := bb.Build(cc, balancer.BuildOptions{})
 	defer pb.Close()
 
@@ -1598,8 +1598,8 @@ func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) {
 				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
 			},
 		},
-		BalancerConfig: &lbConfig{
-			Children: map[string]*child{
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
 				"child-0": {&internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}},
 			},
 			Priorities: []string{"child-0"},
diff --git a/xds/internal/balancer/priority/config.go b/xds/internal/balancer/priority/config.go
index da085908c71d..7704f21d13bd 100644
--- a/xds/internal/balancer/priority/config.go
+++ b/xds/internal/balancer/priority/config.go
@@ -26,24 +26,26 @@ import (
 	"google.golang.org/grpc/serviceconfig"
 )
 
-type child struct {
-	Config *internalserviceconfig.BalancerConfig
+// Child is a child of priority balancer.
+type Child struct {
+	Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"`
 }
 
-type lbConfig struct {
-	serviceconfig.LoadBalancingConfig
+// LBConfig represents priority balancer's config.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
 
 	// Children is a map from the child balancer names to their configs. Child
 	// names can be found in field Priorities.
-	Children map[string]*child
+	Children map[string]*Child `json:"children,omitempty"`
 	// Priorities is a list of child balancer names. They are sorted from
 	// highest priority to low. The type/config for each child can be found in
 	// field Children, with the balancer name as the key.
-	Priorities []string
+	Priorities []string `json:"priorities,omitempty"`
 }
 
-func parseConfig(c json.RawMessage) (*lbConfig, error) {
-	var cfg lbConfig
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
 	if err := json.Unmarshal(c, &cfg); err != nil {
 		return nil, err
 	}
diff --git a/xds/internal/balancer/priority/config_test.go b/xds/internal/balancer/priority/config_test.go
index 15c4069dd1e7..189aa1c91ca4 100644
--- a/xds/internal/balancer/priority/config_test.go
+++ b/xds/internal/balancer/priority/config_test.go
@@ -30,7 +30,7 @@ func TestParseConfig(t *testing.T) {
 	tests := []struct {
 		name    string
 		js      string
-		want    *lbConfig
+		want    *LBConfig
 		wantErr bool
 	}{
 		{
@@ -69,8 +69,8 @@ func TestParseConfig(t *testing.T) {
   }
 }
 			`,
-			want: &lbConfig{
-				Children: map[string]*child{
+			want: &LBConfig{
+				Children: map[string]*Child{
 					"child-1": {
 						&internalserviceconfig.BalancerConfig{
 							Name: roundrobin.Name,
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index ac1aaecd8e51..a210816332b0 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -33,7 +33,8 @@ import (
 	"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
 )
 
-const weightedTargetName = "weighted_target_experimental"
+// Name is the name of the weighted_target balancer.
+const Name = "weighted_target_experimental"
 
 // newRandomWRR is the WRR constructor used to pick sub-pickers from
 // sub-balancers. It's to be modified in tests.
@@ -57,7 +58,7 @@ func (wt *weightedTargetBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOp
 }
 
 func (wt *weightedTargetBB) Name() string {
-	return weightedTargetName
+	return Name
 }
 
 func (wt *weightedTargetBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
@@ -75,14 +76,14 @@ type weightedTargetBalancer struct {
 	bg              *balancergroup.BalancerGroup
 	stateAggregator *weightedaggregator.Aggregator
 
-	targets map[string]target
+	targets map[string]Target
 }
 
 // UpdateClientConnState takes the new targets in balancer group,
 // creates/deletes sub-balancers and sends them update. Addresses are split into
 // groups based on hierarchy path.
 func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
-	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
 	}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config.go b/xds/internal/balancer/weightedtarget/weightedtarget_config.go
index 747ce918bc68..52090cd67b02 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_config.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_config.go
@@ -25,30 +25,23 @@ import (
 	"google.golang.org/grpc/serviceconfig"
 )
 
-type target struct {
+// Target represents one target with the weight and the child policy.
+type Target struct {
 	// Weight is the weight of the child policy.
-	Weight uint32
+	Weight uint32 `json:"weight,omitempty"`
 	// ChildPolicy is the child policy and it's config.
-	ChildPolicy *internalserviceconfig.BalancerConfig
+	ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
 }
 
-// lbConfig is the balancer config for weighted_target. The proto representation
-// is:
-//
-// message WeightedTargetConfig {
-//   message Target {
-//     uint32 weight = 1;
-//     repeated LoadBalancingConfig child_policy = 2;
-//   }
-//   map<string, Target> targets = 1;
-// }
-type lbConfig struct {
-	serviceconfig.LoadBalancingConfig
-	Targets map[string]target
+// LBConfig is the balancer config for weighted_target.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	Targets map[string]Target `json:"targets,omitempty"`
 }
 
-func parseConfig(c json.RawMessage) (*lbConfig, error) {
-	var cfg lbConfig
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
 	if err := json.Unmarshal(c, &cfg); err != nil {
 		return nil, err
 	}
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
index 92dff8f5fbfc..57cad5c85d0f 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
@@ -56,7 +56,7 @@ func Test_parseConfig(t *testing.T) {
 	tests := []struct {
 		name    string
 		js      string
-		want    *lbConfig
+		want    *LBConfig
 		wantErr bool
 	}{
 		{
@@ -68,8 +68,8 @@ func Test_parseConfig(t *testing.T) {
 		{
 			name: "OK",
 			js:   testJSONConfig,
-			want: &lbConfig{
-				Targets: map[string]target{
+			want: &LBConfig{
+				Targets: map[string]Target{
 					"cluster_1": {
 						Weight: 75,
 						ChildPolicy: &internalserviceconfig.BalancerConfig{
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
index 131f89832c79..eeebab733d61 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_test.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
@@ -103,7 +103,7 @@ func init() {
 	for i := 0; i < testBackendAddrsCount; i++ {
 		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
 	}
-	wtbBuilder = balancer.Get(weightedTargetName)
+	wtbBuilder = balancer.Get(Name)
 	wtbParser = wtbBuilder.(balancer.ConfigParser)
 
 	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond

From 671707bdf3bfa85f176f07810de5100d0109776b Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 21 Apr 2021 14:06:54 -0700
Subject: [PATCH 427/481] internal: fix symbol undefined build failure (#4353)

Caused by git merge
---
 xds/internal/balancer/clusterimpl/balancer_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index f421f2281b59..0ff27894ebd5 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -482,7 +482,7 @@ func TestReResolution(t *testing.T) {
 	newXDSClient = func() (xdsClientInterface, error) { return xdsC, nil }
 	defer func() { newXDSClient = oldNewXDSClient }()
 
-	builder := balancer.Get(clusterImplName)
+	builder := balancer.Get(Name)
 	cc := testutils.NewTestClientConn(t)
 	b := builder.Build(cc, balancer.BuildOptions{})
 	defer b.Close()
@@ -491,7 +491,7 @@ func TestReResolution(t *testing.T) {
 		ResolverState: resolver.State{
 			Addresses: testBackendAddrs,
 		},
-		BalancerConfig: &lbConfig{
+		BalancerConfig: &LBConfig{
 			Cluster:        testClusterName,
 			EDSServiceName: testServiceName,
 			ChildPolicy: &internalserviceconfig.BalancerConfig{

From 6f35bbbfb82de348a1537774af2ffd706cd3bb12 Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Wed, 21 Apr 2021 17:27:51 -0700
Subject: [PATCH 428/481] test: enable xDS CSDS test (#4354)

---
 test/kokoro/xds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh
index 65b35e3acac6..f9cb7dab7332 100755
--- a/test/kokoro/xds.sh
+++ b/test/kokoro/xds.sh
@@ -27,7 +27,7 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh
 # they are added into "all".
 GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
   python3 grpc/tools/run_tests/run_xds_tests.py \
-    --test_case="all,circuit_breaking,timeout,fault_injection" \
+    --test_case="all,circuit_breaking,timeout,fault_injection,csds" \
     --project_id=grpc-testing \
     --project_num=830293263384 \
     --source_image=projects/grpc-testing/global/images/xds-test-server-4 \

From f2783f271924fd379910c91fb62aae1dbfad83bd Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Thu, 22 Apr 2021 18:08:53 +0200
Subject: [PATCH 429/481] Run emulated linux arm64 tests (#4344)

---
 .github/workflows/testing.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 378e2846676f..348800443705 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -53,12 +53,22 @@ jobs:
             goversion: 1.13
           - type: tests111
             goversion: 1.11  # Keep until interop tests no longer require Go1.11
+          - type: arm64
+            goversion: 1.15
 
     steps:
       # Setup the environment.
       - name: Setup GOARCH=386
         if: ${{ matrix.type == '386' }}
         run: echo "GOARCH=386" >> $GITHUB_ENV
+      - name: Setup GOARCH=arm64
+        if: ${{ matrix.type == 'arm64' }}
+        run: echo "GOARCH=arm64" >> $GITHUB_ENV
+      - name: Setup qemu emulator
+        if: ${{ matrix.type == 'arm64' }}
+        # setup qemu-user-static emulator and register it with binfmt_misc so that aarch64 binaries
+        # are automatically executed using qemu.
+        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
       - name: Setup RETRY
         if: ${{ matrix.type == 'retry' }}
         run: echo "GRPC_GO_RETRY=on" >> $GITHUB_ENV

From 7276af6dd73483d9edfedbef778c831f044736eb Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 22 Apr 2021 10:45:24 -0700
Subject: [PATCH 430/481] client: fix leaked addrConn struct when addresses are
 updated (#4347)

---
 balancer_conn_wrappers.go | 4 ++--
 clientconn.go             | 7 +++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go
index 41061d6d3dc5..4cc7f9159b16 100644
--- a/balancer_conn_wrappers.go
+++ b/balancer_conn_wrappers.go
@@ -205,7 +205,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) {
 	acbw.mu.Lock()
 	defer acbw.mu.Unlock()
 	if len(addrs) <= 0 {
-		acbw.ac.tearDown(errConnDrain)
+		acbw.ac.cc.removeAddrConn(acbw.ac, errConnDrain)
 		return
 	}
 	if !acbw.ac.tryUpdateAddrs(addrs) {
@@ -220,7 +220,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) {
 		acbw.ac.acbw = nil
 		acbw.ac.mu.Unlock()
 		acState := acbw.ac.getState()
-		acbw.ac.tearDown(errConnDrain)
+		acbw.ac.cc.removeAddrConn(acbw.ac, errConnDrain)
 
 		if acState == connectivity.Shutdown {
 			return
diff --git a/clientconn.go b/clientconn.go
index 0db796ccbd66..d57e54b4dc52 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -1446,10 +1446,9 @@ func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) {
 }
 
 // tearDown starts to tear down the addrConn.
-// TODO(zhaoq): Make this synchronous to avoid unbounded memory consumption in
-// some edge cases (e.g., the caller opens and closes many addrConn's in a
-// tight loop.
-// tearDown doesn't remove ac from ac.cc.conns.
+//
+// Note that tearDown doesn't remove ac from ac.cc.conns, so the addrConn struct
+// will leak. In most cases, call cc.removeAddrConn() instead.
 func (ac *addrConn) tearDown(err error) {
 	ac.mu.Lock()
 	if ac.state == connectivity.Shutdown {

From f02863c306d287e05bcb796035b38fd956db1576 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 22 Apr 2021 14:58:58 -0700
Subject: [PATCH 431/481] xds: specify "h2" as the alpn in xds creds (#4361)

---
 internal/credentials/xds/handshake_info.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go
index ca2e39edd6d1..6789a4cf2e51 100644
--- a/internal/credentials/xds/handshake_info.go
+++ b/internal/credentials/xds/handshake_info.go
@@ -138,7 +138,10 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config,
 	// Currently the Go stdlib does complete verification of the cert (which
 	// includes hostname verification) or none. We are forced to go with the
 	// latter and perform the normal cert validation ourselves.
-	cfg := &tls.Config{InsecureSkipVerify: true}
+	cfg := &tls.Config{
+		InsecureSkipVerify: true,
+		NextProtos:         []string{"h2"},
+	}
 
 	km, err := rootProv.KeyMaterial(ctx)
 	if err != nil {
@@ -159,7 +162,10 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config,
 // ServerSideTLSConfig constructs a tls.Config to be used in a server-side
 // handshake based on the contents of the HandshakeInfo.
 func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) {
-	cfg := &tls.Config{ClientAuth: tls.NoClientCert}
+	cfg := &tls.Config{
+		ClientAuth: tls.NoClientCert,
+		NextProtos: []string{"h2"},
+	}
 	hi.mu.Lock()
 	// On the server side, identityProvider is mandatory. RootProvider is
 	// optional based on whether the server is doing TLS or mTLS.

From 74fe6eaa41706a8451df3c03a0b131c70f71773d Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Thu, 22 Apr 2021 14:59:51 -0700
Subject: [PATCH 432/481] github: testing action workflow improvements and
 update to test Go1.16 (#4358)

---
 .github/workflows/testing.yml                 | 89 +++++++++----------
 .../tls/certprovider/pemfile/builder_test.go  |  2 +
 .../tls/certprovider/pemfile/watcher_test.go  |  2 +
 credentials/xds/xds_client_test.go            |  2 +
 credentials/xds/xds_server_test.go            |  2 +
 .../advancedtls_integration_test.go           |  2 +
 security/advancedtls/advancedtls_test.go      |  2 +
 security/authorization/engine/engine_test.go  |  2 +
 security/authorization/engine/util_test.go    |  2 +
 xds/csds/csds_test.go                         |  2 +
 xds/googledirectpath/googlec2p_test.go        |  2 +
 xds/googledirectpath/utils.go                 |  4 +-
 .../balancergroup/balancergroup_test.go       |  2 +
 .../balancer/balancergroup/testutils_test.go  |  2 +
 .../cdsbalancer/cdsbalancer_security_test.go  |  2 +
 .../balancer/cdsbalancer/cdsbalancer_test.go  |  2 +
 .../balancer/clusterimpl/balancer_test.go     |  2 +
 .../balancer/clusterimpl/config_test.go       |  2 +
 .../clustermanager/clustermanager_test.go     |  2 +
 .../balancer/clustermanager/config_test.go    |  2 +
 .../edsbalancer/eds_impl_priority_test.go     |  2 +
 .../balancer/edsbalancer/eds_impl_test.go     |  2 +
 xds/internal/balancer/edsbalancer/eds_test.go |  2 +
 .../balancer/edsbalancer/util_test.go         |  2 +
 .../balancer/edsbalancer/xds_lrs_test.go      |  2 +
 xds/internal/balancer/lrs/balancer_test.go    |  2 +
 xds/internal/balancer/lrs/config_test.go      |  2 +
 xds/internal/balancer/orca/orca_test.go       |  2 +
 .../balancer/priority/balancer_test.go        |  2 +
 xds/internal/balancer/priority/config_test.go |  2 +
 xds/internal/balancer/priority/utils_test.go  |  2 +
 .../weightedtarget_config_test.go             |  2 +
 .../weightedtarget/weightedtarget_test.go     |  2 +
 .../client/bootstrap/bootstrap_test.go        |  2 +
 xds/internal/client/cds_test.go               |  2 +
 xds/internal/client/client_test.go            |  2 +
 xds/internal/client/eds_test.go               |  2 +
 xds/internal/client/filter_chain_test.go      |  2 +
 xds/internal/client/lds_test.go               |  2 +
 xds/internal/client/load/store_test.go        |  2 +
 xds/internal/client/rds_test.go               |  2 +
 xds/internal/client/requests_counter_test.go  |  2 +
 xds/internal/client/tests/client_test.go      |  2 +
 xds/internal/client/tests/dump_test.go        |  2 +
 xds/internal/client/tests/loadreport_test.go  |  2 +
 xds/internal/client/v2/ack_test.go            |  2 +
 xds/internal/client/v2/cds_test.go            |  2 +
 xds/internal/client/v2/client_test.go         |  2 +
 xds/internal/client/v2/eds_test.go            |  2 +
 xds/internal/client/v2/lds_test.go            |  2 +
 xds/internal/client/v2/rds_test.go            |  2 +
 xds/internal/client/watchers_cluster_test.go  |  2 +
 .../client/watchers_endpoints_test.go         |  2 +
 xds/internal/client/watchers_listener_test.go |  2 +
 xds/internal/client/watchers_route_test.go    |  2 +
 xds/internal/httpfilter/fault/fault_test.go   |  1 +
 xds/internal/internal_test.go                 |  2 +
 xds/internal/resolver/matcher_header_test.go  |  2 +
 xds/internal/resolver/matcher_path_test.go    |  2 +
 xds/internal/resolver/matcher_test.go         |  2 +
 xds/internal/resolver/serviceconfig_test.go   |  2 +
 xds/internal/resolver/watch_service_test.go   |  2 +
 xds/internal/resolver/xds_resolver_test.go    |  2 +
 xds/internal/server/listener_wrapper_test.go  |  2 +
 .../test/xds_client_integration_test.go       |  1 +
 xds/internal/test/xds_integration_test.go     |  1 +
 .../test/xds_server_integration_test.go       |  1 +
 xds/internal/testutils/balancer_test.go       |  2 +
 xds/server_test.go                            |  2 +
 69 files changed, 175 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 348800443705..b6277ea3065f 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -37,71 +37,77 @@ jobs:
     strategy:
       matrix:
         include:
-          - type: vet
-            goversion: 1.15
-          - type: race
-            goversion: 1.15
-          - type: 386
-            goversion: 1.15
-          - type: retry
-            goversion: 1.15
+          - type: vet+tests
+            goversion: 1.16
+
+          - type: tests
+            goversion: 1.16
+            testflags: -race
+
+          - type: tests
+            goversion: 1.16
+            grpcenv: GRPC_GO_RETRY=on
+
           - type: extras
-            goversion: 1.15
+            goversion: 1.16
+
           - type: tests
-            goversion: 1.14
+            goversion: 1.16
+            goarch: 386
+
+          - type: tests
+            goversion: 1.16
+            goarch: arm64
+
           - type: tests
-            goversion: 1.13
-          - type: tests111
-            goversion: 1.11  # Keep until interop tests no longer require Go1.11
-          - type: arm64
             goversion: 1.15
 
+          - type: tests
+            goversion: 1.14
+
+          - type: tests  # Keep until interop tests no longer require Go1.11
+            goversion: 1.11
+
     steps:
       # Setup the environment.
-      - name: Setup GOARCH=386
-        if: ${{ matrix.type == '386' }}
-        run: echo "GOARCH=386" >> $GITHUB_ENV
-      - name: Setup GOARCH=arm64
-        if: ${{ matrix.type == 'arm64' }}
-        run: echo "GOARCH=arm64" >> $GITHUB_ENV
+      - name: Setup GOARCH
+        if: matrix.goarch != ''
+        run: echo "GOARCH=${{ matrix.goarch }}" >> $GITHUB_ENV
+
       - name: Setup qemu emulator
-        if: ${{ matrix.type == 'arm64' }}
+        if: matrix.goarch == 'arm64'
         # setup qemu-user-static emulator and register it with binfmt_misc so that aarch64 binaries
         # are automatically executed using qemu.
         run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
-      - name: Setup RETRY
-        if: ${{ matrix.type == 'retry' }}
-        run: echo "GRPC_GO_RETRY=on" >> $GITHUB_ENV
+
+      - name: Setup GRPC environment
+        if: matrix.grpcenv != ''
+        run: echo "${{ matrix.grpcenv }}" >> $GITHUB_ENV
+
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.goversion }}
+
       - name: Checkout repo
         uses: actions/checkout@v2
 
       # Only run vet for 'vet' runs.
       - name: Run vet.sh
-        if: ${{ matrix.type == 'vet' }}
+        if: startsWith(matrix.type, 'vet')
         run: ./vet.sh -install && ./vet.sh
 
-      # Main tests run for everything except when testing "extras", the race
-      # detector and Go1.11 (where we run a reduced set of tests).
+      # Main tests run for everything except when testing "extras"
+      # (where we run a reduced set of tests).
       - name: Run tests
-        if: ${{ matrix.type != 'extras' && matrix.type != 'race' && matrix.type != 'tests111' }}
-        run: |
-          go version
-          go test -cpu 1,4 -timeout 7m google.golang.org/grpc/...
-
-      # Race detector tests
-      - name: Run test race
-        if: ${{ matrix.TYPE == 'race' }}
+        if: contains(matrix.type, 'tests')
         run: |
           go version
-          go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/...
+          go test ${{ matrix.testflags }} -cpu 1,4 -timeout 7m google.golang.org/grpc/...
 
       # Non-core gRPC tests (examples, interop, etc)
       - name: Run extras tests
-        if: ${{ matrix.TYPE == 'extras' }}
+        if: matrix.type == 'extras'
         run: |
           go version
           examples/examples_test.sh
@@ -109,12 +115,3 @@ jobs:
           interop/interop_test.sh
           cd ${GITHUB_WORKSPACE}/security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/...
           cd ${GITHUB_WORKSPACE}/security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/...
-
-      # Reduced set of tests for Go 1.11
-      - name: Run Go1.11 tests
-        if: ${{ matrix.type == 'tests111' }}
-        run: |
-          go version
-          tests=$(find ${GITHUB_WORKSPACE} -name '*_test.go' | xargs -n1 dirname | sort -u | sed "s:^${GITHUB_WORKSPACE}:.:" | sed "s:\/$::" | grep -v ^./security | grep -v ^./credentials/sts | grep -v ^./credentials/tls/certprovider | grep -v ^./credentials/xds | grep -v ^./xds )
-          echo "Running tests for " ${tests}
-          go test -cpu 1,4 -timeout 7m ${tests}
diff --git a/credentials/tls/certprovider/pemfile/builder_test.go b/credentials/tls/certprovider/pemfile/builder_test.go
index bef00e10c19d..2e49289ff899 100644
--- a/credentials/tls/certprovider/pemfile/builder_test.go
+++ b/credentials/tls/certprovider/pemfile/builder_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/credentials/tls/certprovider/pemfile/watcher_test.go b/credentials/tls/certprovider/pemfile/watcher_test.go
index e43cf7358eca..8b772245525e 100644
--- a/credentials/tls/certprovider/pemfile/watcher_test.go
+++ b/credentials/tls/certprovider/pemfile/watcher_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index 8859946ef5fa..2c882be8a549 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/credentials/xds/xds_server_test.go b/credentials/xds/xds_server_test.go
index 5c29ba38c286..65f7e8ffa3b9 100644
--- a/credentials/xds/xds_server_test.go
+++ b/credentials/xds/xds_server_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go
index 4bb9e645b0a1..2fabe8f17a8d 100644
--- a/security/advancedtls/advancedtls_integration_test.go
+++ b/security/advancedtls/advancedtls_integration_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go
index 64da81a1700c..827cf031ef6f 100644
--- a/security/advancedtls/advancedtls_test.go
+++ b/security/advancedtls/advancedtls_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/security/authorization/engine/engine_test.go b/security/authorization/engine/engine_test.go
index c159c4bd5c21..e56f218e5e2b 100644
--- a/security/authorization/engine/engine_test.go
+++ b/security/authorization/engine/engine_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2020 gRPC authors.
  *
diff --git a/security/authorization/engine/util_test.go b/security/authorization/engine/util_test.go
index e766fbf3ffe0..43514296d83a 100644
--- a/security/authorization/engine/util_test.go
+++ b/security/authorization/engine/util_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
index 867d74e5b25b..04a71a7d1e6c 100644
--- a/xds/csds/csds_test.go
+++ b/xds/csds/csds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go
index 2dd31d754f3f..524bb82e0f39 100644
--- a/xds/googledirectpath/googlec2p_test.go
+++ b/xds/googledirectpath/googlec2p_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/googledirectpath/utils.go b/xds/googledirectpath/utils.go
index 553b87adf47a..600441979785 100644
--- a/xds/googledirectpath/utils.go
+++ b/xds/googledirectpath/utils.go
@@ -41,7 +41,7 @@ func getFromMetadata(timeout time.Duration, urlStr string) ([]byte, error) {
 	}
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("failed communicating with metadata server: %w", err)
+		return nil, fmt.Errorf("failed communicating with metadata server: %v", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
@@ -49,7 +49,7 @@ func getFromMetadata(timeout time.Duration, urlStr string) ([]byte, error) {
 	}
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed reading from metadata server: %w", err)
+		return nil, fmt.Errorf("failed reading from metadata server: %v", err)
 	}
 	return body, nil
 }
diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go
index ab6ac3913cbb..1ba9195ab1d0 100644
--- a/xds/internal/balancer/balancergroup/balancergroup_test.go
+++ b/xds/internal/balancer/balancergroup/balancergroup_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2019 gRPC authors.
  *
diff --git a/xds/internal/balancer/balancergroup/testutils_test.go b/xds/internal/balancer/balancergroup/testutils_test.go
index 1429fa87b3f2..8c0543083ab3 100644
--- a/xds/internal/balancer/balancergroup/testutils_test.go
+++ b/xds/internal/balancer/balancergroup/testutils_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 73459dd64101..5c746cfa163c 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2020 gRPC authors.
  *
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 9c7bc2362ab7..4476a1532d05 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2019 gRPC authors.
  *
diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go
index 0ff27894ebd5..d1da371c27f3 100644
--- a/xds/internal/balancer/clusterimpl/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/balancer_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/clusterimpl/config_test.go b/xds/internal/balancer/clusterimpl/config_test.go
index ccb0c5e74d90..f83155b5339a 100644
--- a/xds/internal/balancer/clusterimpl/config_test.go
+++ b/xds/internal/balancer/clusterimpl/config_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/clustermanager/clustermanager_test.go b/xds/internal/balancer/clustermanager/clustermanager_test.go
index a40d954ad64f..42c53648553e 100644
--- a/xds/internal/balancer/clustermanager/clustermanager_test.go
+++ b/xds/internal/balancer/clustermanager/clustermanager_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/clustermanager/config_test.go b/xds/internal/balancer/clustermanager/config_test.go
index 3328ba1d300f..f591f5ad32d8 100644
--- a/xds/internal/balancer/clustermanager/config_test.go
+++ b/xds/internal/balancer/clustermanager/config_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
index 7696feb5bd04..51b35f22f09d 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 7e793c034a84..c5e3071d10d8 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2019 gRPC authors.
  *
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 544d0a301672..5c9e5f0b1d53 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/balancer/edsbalancer/util_test.go b/xds/internal/balancer/edsbalancer/util_test.go
index 748aeffe2bb9..b94905d49889 100644
--- a/xds/internal/balancer/edsbalancer/util_test.go
+++ b/xds/internal/balancer/edsbalancer/util_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2019 gRPC authors.
  *
diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
index 9f93e0b42f08..8b7aab657667 100644
--- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go
+++ b/xds/internal/balancer/edsbalancer/xds_lrs_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go
index b115860bf16d..f91937385a92 100644
--- a/xds/internal/balancer/lrs/balancer_test.go
+++ b/xds/internal/balancer/lrs/balancer_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/balancer/lrs/config_test.go b/xds/internal/balancer/lrs/config_test.go
index c460cd008fff..35118298b18e 100644
--- a/xds/internal/balancer/lrs/config_test.go
+++ b/xds/internal/balancer/lrs/config_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/orca/orca_test.go b/xds/internal/balancer/orca/orca_test.go
index d7a44134e22b..ff02b3c16087 100644
--- a/xds/internal/balancer/orca/orca_test.go
+++ b/xds/internal/balancer/orca/orca_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  * Copyright 2019 gRPC authors.
  *
diff --git a/xds/internal/balancer/priority/balancer_test.go b/xds/internal/balancer/priority/balancer_test.go
index b15ea303d78f..d546216123d1 100644
--- a/xds/internal/balancer/priority/balancer_test.go
+++ b/xds/internal/balancer/priority/balancer_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/internal/balancer/priority/config_test.go b/xds/internal/balancer/priority/config_test.go
index 189aa1c91ca4..f3a09fe3a32e 100644
--- a/xds/internal/balancer/priority/config_test.go
+++ b/xds/internal/balancer/priority/config_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/priority/utils_test.go b/xds/internal/balancer/priority/utils_test.go
index c80a89b080f9..a4b1a5285102 100644
--- a/xds/internal/balancer/priority/utils_test.go
+++ b/xds/internal/balancer/priority/utils_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
index 57cad5c85d0f..351a13553e4e 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
index eeebab733d61..b792c28c6ab0 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget_test.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go
index 501d62102d21..f62ed2b54245 100644
--- a/xds/internal/client/bootstrap/bootstrap_test.go
+++ b/xds/internal/client/bootstrap/bootstrap_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 3fb889db9486..bb1117ec5349 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 8275ea60e0dc..69930557b26e 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/eds_test.go b/xds/internal/client/eds_test.go
index daa5d6525e19..9d6a3113b0c3 100644
--- a/xds/internal/client/eds_test.go
+++ b/xds/internal/client/eds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/filter_chain_test.go b/xds/internal/client/filter_chain_test.go
index afb0c81fda14..c68e22286763 100644
--- a/xds/internal/client/filter_chain_test.go
+++ b/xds/internal/client/filter_chain_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 21e94557b3e9..9fb27987e36b 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/load/store_test.go b/xds/internal/client/load/store_test.go
index 46568591f9e4..e7db4e26b176 100644
--- a/xds/internal/client/load/store_test.go
+++ b/xds/internal/client/load/store_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index 2ca01dca9ca2..cde40ee80dfe 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/requests_counter_test.go b/xds/internal/client/requests_counter_test.go
index 2dc336d1c1d5..30892fc747a0 100644
--- a/xds/internal/client/requests_counter_test.go
+++ b/xds/internal/client/requests_counter_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/tests/client_test.go b/xds/internal/client/tests/client_test.go
index f5a57fbcd218..755f0e05ea45 100644
--- a/xds/internal/client/tests/client_test.go
+++ b/xds/internal/client/tests/client_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/tests/dump_test.go b/xds/internal/client/tests/dump_test.go
index 58220866eb19..de3fcade47e9 100644
--- a/xds/internal/client/tests/dump_test.go
+++ b/xds/internal/client/tests/dump_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/internal/client/tests/loadreport_test.go b/xds/internal/client/tests/loadreport_test.go
index af145e7f2a92..b1ec37294771 100644
--- a/xds/internal/client/tests/loadreport_test.go
+++ b/xds/internal/client/tests/loadreport_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/v2/ack_test.go b/xds/internal/client/v2/ack_test.go
index 813d8baa79d9..53c8cef189d5 100644
--- a/xds/internal/client/v2/ack_test.go
+++ b/xds/internal/client/v2/ack_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index c71b84532315..b56ae6108bbe 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index e770324e1b12..1e464405eeaf 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/v2/eds_test.go b/xds/internal/client/v2/eds_test.go
index 0990e7ebae0e..7eba32f5c605 100644
--- a/xds/internal/client/v2/eds_test.go
+++ b/xds/internal/client/v2/eds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/v2/lds_test.go b/xds/internal/client/v2/lds_test.go
index 1f4c980fae5e..22fa35d5e51e 100644
--- a/xds/internal/client/v2/lds_test.go
+++ b/xds/internal/client/v2/lds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/client/v2/rds_test.go b/xds/internal/client/v2/rds_test.go
index dd145158b8a9..12495428bf95 100644
--- a/xds/internal/client/v2/rds_test.go
+++ b/xds/internal/client/v2/rds_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/watchers_cluster_test.go b/xds/internal/client/watchers_cluster_test.go
index fdef0cf61649..2d10c7f43b5f 100644
--- a/xds/internal/client/watchers_cluster_test.go
+++ b/xds/internal/client/watchers_cluster_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/watchers_endpoints_test.go b/xds/internal/client/watchers_endpoints_test.go
index b79397414d4a..bff4544d2679 100644
--- a/xds/internal/client/watchers_endpoints_test.go
+++ b/xds/internal/client/watchers_endpoints_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/watchers_listener_test.go b/xds/internal/client/watchers_listener_test.go
index bf3a122da075..fdd4ebd163fa 100644
--- a/xds/internal/client/watchers_listener_test.go
+++ b/xds/internal/client/watchers_listener_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/client/watchers_route_test.go b/xds/internal/client/watchers_route_test.go
index 5f44e5493330..41640b85b574 100644
--- a/xds/internal/client/watchers_route_test.go
+++ b/xds/internal/client/watchers_route_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 61100e8c44f8..20de9b9a697a 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -1,3 +1,4 @@
+// +build go1.12
 // +build !386
 
 /*
diff --git a/xds/internal/internal_test.go b/xds/internal/internal_test.go
index 903b9db23c48..9240d0a89d58 100644
--- a/xds/internal/internal_test.go
+++ b/xds/internal/internal_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/resolver/matcher_header_test.go b/xds/internal/resolver/matcher_header_test.go
index fb87cc5dd329..c83c3ec3923c 100644
--- a/xds/internal/resolver/matcher_header_test.go
+++ b/xds/internal/resolver/matcher_header_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/resolver/matcher_path_test.go b/xds/internal/resolver/matcher_path_test.go
index 263a049108e4..7b0d296fc324 100644
--- a/xds/internal/resolver/matcher_path_test.go
+++ b/xds/internal/resolver/matcher_path_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/resolver/matcher_test.go b/xds/internal/resolver/matcher_test.go
index 7657b87bf45f..5c8dca5c9e5b 100644
--- a/xds/internal/resolver/matcher_test.go
+++ b/xds/internal/resolver/matcher_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go
index 1e253841e801..7fe8218160f5 100644
--- a/xds/internal/resolver/serviceconfig_test.go
+++ b/xds/internal/resolver/serviceconfig_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go
index 2bfe3e984d3c..421e5345a9d2 100644
--- a/xds/internal/resolver/watch_service_test.go
+++ b/xds/internal/resolver/watch_service_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index b3c2006b72be..8ec29af9ebf2 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2019 gRPC authors.
diff --git a/xds/internal/server/listener_wrapper_test.go b/xds/internal/server/listener_wrapper_test.go
index 220be0e08ae7..8b5b5c3851de 100644
--- a/xds/internal/server/listener_wrapper_test.go
+++ b/xds/internal/server/listener_wrapper_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2021 gRPC authors.
diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go
index f97e42af2a0a..39b3add77fbc 100644
--- a/xds/internal/test/xds_client_integration_test.go
+++ b/xds/internal/test/xds_client_integration_test.go
@@ -1,3 +1,4 @@
+// +build go1.12
 // +build !386
 
 /*
diff --git a/xds/internal/test/xds_integration_test.go b/xds/internal/test/xds_integration_test.go
index ae306ae7864e..13ab5e351a52 100644
--- a/xds/internal/test/xds_integration_test.go
+++ b/xds/internal/test/xds_integration_test.go
@@ -1,3 +1,4 @@
+// +build go1.12
 // +build !386
 
 /*
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index e31bba772ed9..d5b9b8dd20a2 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -1,3 +1,4 @@
+// +build go1.12
 // +build !386
 
 /*
diff --git a/xds/internal/testutils/balancer_test.go b/xds/internal/testutils/balancer_test.go
index 4891eb9cdadf..83393dcd1e98 100644
--- a/xds/internal/testutils/balancer_test.go
+++ b/xds/internal/testutils/balancer_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.
diff --git a/xds/server_test.go b/xds/server_test.go
index 2a6677a3ccbb..41767bc1555a 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -1,3 +1,5 @@
+// +build go1.12
+
 /*
  *
  * Copyright 2020 gRPC authors.

From e158e3e82cbac01ba513de4b0982b35b1fcc6183 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 23 Apr 2021 13:15:21 -0700
Subject: [PATCH 433/481] xds/lrs: server name is not required to be non-empty
 (#4356)

---
 xds/internal/balancer/lrs/balancer.go    |  8 ++++----
 xds/internal/balancer/lrs/config.go      |  3 ---
 xds/internal/balancer/lrs/config_test.go | 15 ---------------
 3 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index 460788eb53c3..e062fa234363 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -168,7 +168,7 @@ type xdsClientWrapper struct {
 	cancelLoadReport func()
 	clusterName      string
 	edsServiceName   string
-	lrsServerName    string
+	lrsServerName    *string
 	// loadWrapper is a wrapper with loadOriginal, with clusterName and
 	// edsServiceName. It's used children to report loads.
 	loadWrapper *loadstore.Wrapper
@@ -212,11 +212,11 @@ func (w *xdsClientWrapper) update(newConfig *LBConfig) error {
 		w.loadWrapper.UpdateClusterAndService(w.clusterName, w.edsServiceName)
 	}
 
-	if w.lrsServerName != newConfig.LoadReportingServerName {
+	if w.lrsServerName == nil || *w.lrsServerName != newConfig.LoadReportingServerName {
 		// LoadReportingServerName is different, load should be report to a
 		// different server, restart.
 		restartLoadReport = true
-		w.lrsServerName = newConfig.LoadReportingServerName
+		w.lrsServerName = &newConfig.LoadReportingServerName
 	}
 
 	if restartLoadReport {
@@ -226,7 +226,7 @@ func (w *xdsClientWrapper) update(newConfig *LBConfig) error {
 		}
 		var loadStore *load.Store
 		if w.c != nil {
-			loadStore, w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName)
+			loadStore, w.cancelLoadReport = w.c.ReportLoad(*w.lrsServerName)
 		}
 		w.loadWrapper.UpdateLoadStore(loadStore)
 	}
diff --git a/xds/internal/balancer/lrs/config.go b/xds/internal/balancer/lrs/config.go
index 9e8fc1045d4d..e0e30bbb8821 100644
--- a/xds/internal/balancer/lrs/config.go
+++ b/xds/internal/balancer/lrs/config.go
@@ -46,9 +46,6 @@ func parseConfig(c json.RawMessage) (*LBConfig, error) {
 	if cfg.ClusterName == "" {
 		return nil, fmt.Errorf("required ClusterName is not set in %+v", cfg)
 	}
-	if cfg.LoadReportingServerName == "" {
-		return nil, fmt.Errorf("required LoadReportingServerName is not set in %+v", cfg)
-	}
 	if cfg.Locality == nil {
 		return nil, fmt.Errorf("required Locality is not set in %+v", cfg)
 	}
diff --git a/xds/internal/balancer/lrs/config_test.go b/xds/internal/balancer/lrs/config_test.go
index 35118298b18e..eaf902ac535d 100644
--- a/xds/internal/balancer/lrs/config_test.go
+++ b/xds/internal/balancer/lrs/config_test.go
@@ -53,21 +53,6 @@ func TestParseConfig(t *testing.T) {
     "subZone": "test-sub-zone"
   },
   "childPolicy":[{"round_robin":{}}]
-}
-			`,
-			wantErr: true,
-		},
-		{
-			name: "no LRS server name",
-			js: `{
-  "clusterName": "test-cluster",
-  "edsServiceName": "test-eds-service",
-  "locality": {
-    "region": "test-region",
-    "zone": "test-zone",
-    "subZone": "test-sub-zone"
-  },
-  "childPolicy":[{"round_robin":{}}]
 }
 			`,
 			wantErr: true,

From 9572fd6faeaee33fe295ce3a79eab729d05bb349 Mon Sep 17 00:00:00 2001
From: apolcyn <apolcyn@google.com>
Date: Fri, 23 Apr 2021 17:26:26 -0700
Subject: [PATCH 434/481] client: include details about GOAWAYs in status
 messages (#4316)

---
 internal/transport/http2_client.go   | 16 ++++--
 internal/transport/keepalive_test.go | 10 ++--
 internal/transport/transport.go      |  5 +-
 test/end2end_test.go                 | 80 ++++++++++++++++++++++++++++
 4 files changed, 101 insertions(+), 10 deletions(-)

diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 892317cc6fca..48c5e52edae9 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -115,6 +115,9 @@ type http2Client struct {
 	// goAwayReason records the http2.ErrCode and debug data received with the
 	// GoAway frame.
 	goAwayReason GoAwayReason
+	// goAwayDebugMessage contains a detailed human readable string about a
+	// GoAway frame, useful for error messages.
+	goAwayDebugMessage string
 	// A condition variable used to signal when the keepalive goroutine should
 	// go dormant. The condition for dormancy is based on the number of active
 	// streams and the `PermitWithoutStream` keepalive client parameter. And
@@ -872,6 +875,12 @@ func (t *http2Client) Close(err error) {
 	if channelz.IsOn() {
 		channelz.RemoveEntry(t.channelzID)
 	}
+	// Append info about previous goaways if there were any, since this may be important
+	// for understanding the root cause for this connection to be closed.
+	_, goAwayDebugMessage := t.GetGoAwayReason()
+	if len(goAwayDebugMessage) > 0 {
+		err = fmt.Errorf("closing transport due to: %v, received prior goaway: %v", err, goAwayDebugMessage)
+	}
 	// Notify all active streams.
 	for _, s := range streams {
 		t.closeStream(s, err, false, http2.ErrCodeNo, status.New(codes.Unavailable, err.Error()), nil, false)
@@ -1146,7 +1155,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 		}
 	}
 	id := f.LastStreamID
-	if id > 0 && id%2 != 1 {
+	if id > 0 && id%2 == 0 {
 		t.mu.Unlock()
 		t.Close(connectionErrorf(true, nil, "received goaway with non-zero even-numbered numbered stream id: %v", id))
 		return
@@ -1212,12 +1221,13 @@ func (t *http2Client) setGoAwayReason(f *http2.GoAwayFrame) {
 			t.goAwayReason = GoAwayTooManyPings
 		}
 	}
+	t.goAwayDebugMessage = fmt.Sprintf("code: %s, debug data: %v", f.ErrCode, string(f.DebugData()))
 }
 
-func (t *http2Client) GetGoAwayReason() GoAwayReason {
+func (t *http2Client) GetGoAwayReason() (GoAwayReason, string) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return t.goAwayReason
+	return t.goAwayReason, t.goAwayDebugMessage
 }
 
 func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) {
diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go
index d684f5827103..571cacca7e91 100644
--- a/internal/transport/keepalive_test.go
+++ b/internal/transport/keepalive_test.go
@@ -69,7 +69,7 @@ func (s) TestMaxConnectionIdle(t *testing.T) {
 		if !timeout.Stop() {
 			<-timeout.C
 		}
-		if reason := client.GetGoAwayReason(); reason != GoAwayNoReason {
+		if reason, _ := client.GetGoAwayReason(); reason != GoAwayNoReason {
 			t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason)
 		}
 	case <-timeout.C:
@@ -143,7 +143,7 @@ func (s) TestMaxConnectionAge(t *testing.T) {
 		if !timeout.Stop() {
 			<-timeout.C
 		}
-		if reason := client.GetGoAwayReason(); reason != GoAwayNoReason {
+		if reason, _ := client.GetGoAwayReason(); reason != GoAwayNoReason {
 			t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason)
 		}
 	case <-timeout.C:
@@ -403,7 +403,7 @@ func (s) TestKeepaliveClientFrequency(t *testing.T) {
 		if !timeout.Stop() {
 			<-timeout.C
 		}
-		if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings {
+		if reason, _ := client.GetGoAwayReason(); reason != GoAwayTooManyPings {
 			t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings)
 		}
 	case <-timeout.C:
@@ -448,7 +448,7 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) {
 		if !timeout.Stop() {
 			<-timeout.C
 		}
-		if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings {
+		if reason, _ := client.GetGoAwayReason(); reason != GoAwayTooManyPings {
 			t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings)
 		}
 	case <-timeout.C:
@@ -498,7 +498,7 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) {
 		if !timeout.Stop() {
 			<-timeout.C
 		}
-		if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings {
+		if reason, _ := client.GetGoAwayReason(); reason != GoAwayTooManyPings {
 			t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings)
 		}
 	case <-timeout.C:
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index 068f4d0e5023..6cc1031fd92f 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -656,8 +656,9 @@ type ClientTransport interface {
 	// HTTP/2).
 	GoAway() <-chan struct{}
 
-	// GetGoAwayReason returns the reason why GoAway frame was received.
-	GetGoAwayReason() GoAwayReason
+	// GetGoAwayReason returns the reason why GoAway frame was received, along
+	// with a human readable string with debug info.
+	GetGoAwayReason() (GoAwayReason, string)
 
 	// RemoteAddr returns the remote network address.
 	RemoteAddr() net.Addr
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 832ac8bd7180..1baf2e347d15 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -55,12 +55,14 @@ import (
 	"google.golang.org/grpc/health"
 	healthgrpc "google.golang.org/grpc/health/grpc_health_v1"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/transport"
+	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/resolver"
@@ -1380,6 +1382,84 @@ func (s) TestDetailedConnectionCloseErrorPropagatesToRpcError(t *testing.T) {
 	close(rpcDoneOnClient)
 }
 
+func (s) TestDetailedGoawayErrorOnGracefulClosePropagatesToRPCError(t *testing.T) {
+	rpcDoneOnClient := make(chan struct{})
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			<-rpcDoneOnClient
+			return status.Error(codes.Internal, "arbitrary status")
+		},
+	}
+	sopts := []grpc.ServerOption{
+		grpc.KeepaliveParams(keepalive.ServerParameters{
+			MaxConnectionAge:      time.Millisecond * 100,
+			MaxConnectionAgeGrace: time.Millisecond,
+		}),
+	}
+	if err := ss.Start(sopts); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall = _, %v, want _, <nil>", ss.Client, err)
+	}
+	const expectedErrorMessageSubstring = "received prior goaway: code: NO_ERROR"
+	_, err = stream.Recv()
+	close(rpcDoneOnClient)
+	if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) {
+		t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: %q", stream, err, expectedErrorMessageSubstring)
+	}
+}
+
+func (s) TestDetailedGoawayErrorOnAbruptClosePropagatesToRPCError(t *testing.T) {
+	// set the min keepalive time very low so that this test can take
+	// a reasonable amount of time
+	prev := internal.KeepaliveMinPingTime
+	internal.KeepaliveMinPingTime = time.Millisecond
+	defer func() { internal.KeepaliveMinPingTime = prev }()
+
+	rpcDoneOnClient := make(chan struct{})
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			<-rpcDoneOnClient
+			return status.Error(codes.Internal, "arbitrary status")
+		},
+	}
+	sopts := []grpc.ServerOption{
+		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime: time.Second * 1000, /* arbitrary, large value */
+		}),
+	}
+	dopts := []grpc.DialOption{
+		grpc.WithKeepaliveParams(keepalive.ClientParameters{
+			Time:                time.Millisecond,   /* should trigger "too many pings" error quickly */
+			Timeout:             time.Second * 1000, /* arbitrary, large value */
+			PermitWithoutStream: false,
+		}),
+	}
+	if err := ss.Start(sopts, dopts...); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall = _, %v, want _, <nil>", ss.Client, err)
+	}
+	const expectedErrorMessageSubstring = "received prior goaway: code: ENHANCE_YOUR_CALM, debug data: too_many_pings"
+	_, err = stream.Recv()
+	close(rpcDoneOnClient)
+	if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) {
+		t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: |%v|", stream, err, expectedErrorMessageSubstring)
+	}
+}
+
 func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) {
 	for _, e := range listTestEnv() {
 		if e.name == "handler-tls" {

From 52a707c0dafe4ac6c0443c3d83dfdeeb9b828684 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 26 Apr 2021 14:29:06 -0700
Subject: [PATCH 435/481] xds: serving mode changes outlined in gRFC A36
 (#4328)

---
 internal/internal.go                          |   5 +
 server.go                                     |  79 +++--
 xds/internal/server/listener_wrapper.go       | 101 ++++--
 xds/internal/test/xds_integration_test.go     |   3 +-
 .../test/xds_server_serving_mode_test.go      | 297 ++++++++++++++++++
 xds/server.go                                 |  85 ++++-
 xds/server_options.go                         |  84 +++++
 xds/server_test.go                            |  42 ++-
 xds/xds.go                                    |   5 +
 9 files changed, 639 insertions(+), 62 deletions(-)
 create mode 100644 xds/internal/test/xds_server_serving_mode_test.go
 create mode 100644 xds/server_options.go

diff --git a/internal/internal.go b/internal/internal.go
index 2a3243bd701a..1b596bf3579f 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -59,6 +59,11 @@ var (
 	// gRPC server. An xDS-enabled server needs to know what type of credentials
 	// is configured on the underlying gRPC server. This is set by server.go.
 	GetServerCredentials interface{} // func (*grpc.Server) credentials.TransportCredentials
+	// DrainServerTransports initiates a graceful close of existing connections
+	// on a gRPC server accepted on the provided listener address. An
+	// xDS-enabled server invokes this method on a grpc.Server when a particular
+	// listener moves to "not-serving" mode.
+	DrainServerTransports interface{} // func(*grpc.Server, string)
 )
 
 // HealthChecker defines the signature of the client-side LB channel health checking function.
diff --git a/server.go b/server.go
index 7a2aa28a1147..b2793ab00b53 100644
--- a/server.go
+++ b/server.go
@@ -57,12 +57,22 @@ import (
 const (
 	defaultServerMaxReceiveMessageSize = 1024 * 1024 * 4
 	defaultServerMaxSendMessageSize    = math.MaxInt32
+
+	// Server transports are tracked in a map which is keyed on listener
+	// address. For regular gRPC traffic, connections are accepted in Serve()
+	// through a call to Accept(), and we use the actual listener address as key
+	// when we add it to the map. But for connections received through
+	// ServeHTTP(), we do not have a listener and hence use this dummy value.
+	listenerAddressForServeHTTP = "listenerAddressForServeHTTP"
 )
 
 func init() {
 	internal.GetServerCredentials = func(srv *Server) credentials.TransportCredentials {
 		return srv.opts.creds
 	}
+	internal.DrainServerTransports = func(srv *Server, addr string) {
+		srv.drainServerTransports(addr)
+	}
 }
 
 var statusOK = status.New(codes.OK, "")
@@ -107,9 +117,12 @@ type serverWorkerData struct {
 type Server struct {
 	opts serverOptions
 
-	mu       sync.Mutex // guards following
-	lis      map[net.Listener]bool
-	conns    map[transport.ServerTransport]bool
+	mu  sync.Mutex // guards following
+	lis map[net.Listener]bool
+	// conns contains all active server transports. It is a map keyed on a
+	// listener address with the value being the set of active transports
+	// belonging to that listener.
+	conns    map[string]map[transport.ServerTransport]bool
 	serve    bool
 	drain    bool
 	cv       *sync.Cond              // signaled when connections close for GracefulStop
@@ -519,7 +532,7 @@ func NewServer(opt ...ServerOption) *Server {
 	s := &Server{
 		lis:      make(map[net.Listener]bool),
 		opts:     opts,
-		conns:    make(map[transport.ServerTransport]bool),
+		conns:    make(map[string]map[transport.ServerTransport]bool),
 		services: make(map[string]*serviceInfo),
 		quit:     grpcsync.NewEvent(),
 		done:     grpcsync.NewEvent(),
@@ -778,7 +791,7 @@ func (s *Server) Serve(lis net.Listener) error {
 		// s.conns before this conn can be added.
 		s.serveWG.Add(1)
 		go func() {
-			s.handleRawConn(rawConn)
+			s.handleRawConn(lis.Addr().String(), rawConn)
 			s.serveWG.Done()
 		}()
 	}
@@ -786,7 +799,7 @@ func (s *Server) Serve(lis net.Listener) error {
 
 // handleRawConn forks a goroutine to handle a just-accepted connection that
 // has not had any I/O performed on it yet.
-func (s *Server) handleRawConn(rawConn net.Conn) {
+func (s *Server) handleRawConn(lisAddr string, rawConn net.Conn) {
 	if s.quit.HasFired() {
 		rawConn.Close()
 		return
@@ -814,15 +827,24 @@ func (s *Server) handleRawConn(rawConn net.Conn) {
 	}
 
 	rawConn.SetDeadline(time.Time{})
-	if !s.addConn(st) {
+	if !s.addConn(lisAddr, st) {
 		return
 	}
 	go func() {
 		s.serveStreams(st)
-		s.removeConn(st)
+		s.removeConn(lisAddr, st)
 	}()
 }
 
+func (s *Server) drainServerTransports(addr string) {
+	s.mu.Lock()
+	conns := s.conns[addr]
+	for st := range conns {
+		st.Drain()
+	}
+	s.mu.Unlock()
+}
+
 // newHTTP2Transport sets up a http/2 transport (using the
 // gRPC http2 server transport in transport/http2_server.go).
 func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) transport.ServerTransport {
@@ -924,10 +946,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if !s.addConn(st) {
+	if !s.addConn(listenerAddressForServeHTTP, st) {
 		return
 	}
-	defer s.removeConn(st)
+	defer s.removeConn(listenerAddressForServeHTTP, st)
 	s.serveStreams(st)
 }
 
@@ -955,7 +977,7 @@ func (s *Server) traceInfo(st transport.ServerTransport, stream *transport.Strea
 	return trInfo
 }
 
-func (s *Server) addConn(st transport.ServerTransport) bool {
+func (s *Server) addConn(addr string, st transport.ServerTransport) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.conns == nil {
@@ -967,15 +989,28 @@ func (s *Server) addConn(st transport.ServerTransport) bool {
 		// immediately.
 		st.Drain()
 	}
-	s.conns[st] = true
+
+	if s.conns[addr] == nil {
+		// Create a map entry if this is the first connection on this listener.
+		s.conns[addr] = make(map[transport.ServerTransport]bool)
+	}
+	s.conns[addr][st] = true
 	return true
 }
 
-func (s *Server) removeConn(st transport.ServerTransport) {
+func (s *Server) removeConn(addr string, st transport.ServerTransport) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.conns != nil {
-		delete(s.conns, st)
+
+	conns := s.conns[addr]
+	if conns != nil {
+		delete(conns, st)
+		if len(conns) == 0 {
+			// If the last connection for this address is being removed, also
+			// remove the map entry corresponding to the address. This is used
+			// in GracefulStop() when waiting for all connections to be closed.
+			delete(s.conns, addr)
+		}
 		s.cv.Broadcast()
 	}
 }
@@ -1639,7 +1674,7 @@ func (s *Server) Stop() {
 	s.mu.Lock()
 	listeners := s.lis
 	s.lis = nil
-	st := s.conns
+	conns := s.conns
 	s.conns = nil
 	// interrupt GracefulStop if Stop and GracefulStop are called concurrently.
 	s.cv.Broadcast()
@@ -1648,8 +1683,10 @@ func (s *Server) Stop() {
 	for lis := range listeners {
 		lis.Close()
 	}
-	for c := range st {
-		c.Close()
+	for _, cs := range conns {
+		for st := range cs {
+			st.Close()
+		}
 	}
 	if s.opts.numServerWorkers > 0 {
 		s.stopServerWorkers()
@@ -1686,8 +1723,10 @@ func (s *Server) GracefulStop() {
 	}
 	s.lis = nil
 	if !s.drain {
-		for st := range s.conns {
-			st.Drain()
+		for _, conns := range s.conns {
+			for st := range conns {
+				st.Drain()
+			}
 		}
 		s.drain = true
 	}
diff --git a/xds/internal/server/listener_wrapper.go b/xds/internal/server/listener_wrapper.go
index c65736540920..17f31f28f576 100644
--- a/xds/internal/server/listener_wrapper.go
+++ b/xds/internal/server/listener_wrapper.go
@@ -48,6 +48,42 @@ var (
 	backoffFunc = bs.Backoff
 )
 
+// ServingMode indicates the current mode of operation of the server.
+//
+// This API exactly mirrors the one in the public xds package. We have to
+// redefine it here to avoid a cyclic dependency.
+type ServingMode int
+
+const (
+	// ServingModeStarting indicates that the serving is starting up.
+	ServingModeStarting ServingMode = iota
+	// ServingModeServing indicates the the server contains all required xDS
+	// configuration is serving RPCs.
+	ServingModeServing
+	// ServingModeNotServing indicates that the server is not accepting new
+	// connections. Existing connections will be closed gracefully, allowing
+	// in-progress RPCs to complete. A server enters this mode when it does not
+	// contain the required xDS configuration to serve RPCs.
+	ServingModeNotServing
+)
+
+func (s ServingMode) String() string {
+	switch s {
+	case ServingModeNotServing:
+		return "not-serving"
+	case ServingModeServing:
+		return "serving"
+	default:
+		return "starting"
+	}
+}
+
+// ServingModeCallback is the callback that users can register to get notified
+// about the server's serving mode changes. The callback is invoked with the
+// address of the listener and its new mode. The err parameter is set to a
+// non-nil error if the server has transitioned into not-serving mode.
+type ServingModeCallback func(addr net.Addr, mode ServingMode, err error)
+
 func prefixLogger(p *listenerWrapper) *internalgrpclog.PrefixLogger {
 	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p] ", p))
 }
@@ -70,6 +106,8 @@ type ListenerWrapperParams struct {
 	XDSCredsInUse bool
 	// XDSClient provides the functionality from the xdsClient required here.
 	XDSClient XDSClientInterface
+	// ModeCallback is the callback to invoke when the serving mode changes.
+	ModeCallback ServingModeCallback
 }
 
 // NewListenerWrapper creates a new listenerWrapper with params. It returns a
@@ -83,6 +121,7 @@ func NewListenerWrapper(params ListenerWrapperParams) (net.Listener, <-chan stru
 		name:              params.ListenerResourceName,
 		xdsCredsInUse:     params.XDSCredsInUse,
 		xdsC:              params.XDSClient,
+		modeCallback:      params.ModeCallback,
 		isUnspecifiedAddr: params.Listener.Addr().(*net.TCPAddr).IP.IsUnspecified(),
 
 		closed:     grpcsync.NewEvent(),
@@ -111,12 +150,11 @@ type listenerWrapper struct {
 	net.Listener
 	logger *internalgrpclog.PrefixLogger
 
-	// TODO: Maintain serving state of this listener.
-
 	name          string
 	xdsCredsInUse bool
 	xdsC          XDSClientInterface
 	cancelWatch   func()
+	modeCallback  ServingModeCallback
 
 	// Set to true if the listener is bound to the IP_ANY address (which is
 	// "0.0.0.0" for IPv4 and "::" for IPv6).
@@ -138,11 +176,14 @@ type listenerWrapper struct {
 	// updates received in the callback if this event has fired.
 	closed *grpcsync.Event
 
-	// Filter chains received as part of the last good update. The reason for
-	// using an rw lock here is that this field will be read by all connections
-	// during their server-side handshake (in the hot path), but writes to this
-	// happen rarely (when we get a Listener resource update).
-	mu           sync.RWMutex
+	// mu guards access to the current serving mode and the filter chains. The
+	// reason for using an rw lock here is that these fields are read in
+	// Accept() for all incoming connections, but writes happen rarely (when we
+	// get a Listener resource update).
+	mu sync.RWMutex
+	// Current serving mode.
+	mode ServingMode
+	// Filter chains received as part of the last good update.
 	filterChains *xdsclient.FilterChainManager
 }
 
@@ -175,8 +216,6 @@ func (l *listenerWrapper) Accept() (net.Conn, error) {
 		// Reset retries after a successful Accept().
 		retries = 0
 
-		// TODO: Close connections if in "non-serving" state
-
 		// Since the net.Conn represents an incoming connection, the source and
 		// destination address can be retrieved from the local address and
 		// remote address of the net.Conn respectively.
@@ -191,6 +230,17 @@ func (l *listenerWrapper) Accept() (net.Conn, error) {
 		}
 
 		l.mu.RLock()
+		if l.mode == ServingModeNotServing {
+			// Close connections as soon as we accept them when we are in
+			// "not-serving" mode. Since we accept a net.Listener from the user
+			// in Serve(), we cannot close the listener when we move to
+			// "not-serving". Closing the connection immediately upon accepting
+			// is one of the other ways to implement the "not-serving" mode as
+			// outlined in gRFC A36.
+			l.mu.RUnlock()
+			conn.Close()
+			continue
+		}
 		fc, err := l.filterChains.Lookup(xdsclient.FilterChainLookupParams{
 			IsUnspecifiedListener: l.isUnspecifiedAddr,
 			DestAddr:              destAddr.IP,
@@ -236,14 +286,13 @@ func (l *listenerWrapper) handleListenerUpdate(update xdsclient.ListenerUpdate,
 		return
 	}
 
-	// TODO: Handle resource-not-found errors by moving to not-serving state.
 	if err != nil {
-		// We simply log an error here and hope we get a successful update
-		// in the future. The error could be because of a timeout or an
-		// actual error, like the requested resource not found. In any case,
-		// it is fine for the server to hang indefinitely until Stop() is
-		// called.
 		l.logger.Warningf("Received error for resource %q: %+v", l.name, err)
+		if xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound {
+			l.switchMode(nil, ServingModeNotServing, err)
+		}
+		// For errors which are anything other than "resource-not-found", we
+		// continue to use the old configuration.
 		return
 	}
 	l.logger.Infof("Received update for resource %q: %+v", l.name, update)
@@ -258,18 +307,26 @@ func (l *listenerWrapper) handleListenerUpdate(update xdsclient.ListenerUpdate,
 	//   appropriate context to perform this check.
 	//
 	// What this means is that the xdsClient has ACKed a resource which can push
-	// the server into a "not serving" state. This is not ideal, but this is
+	// the server into a "not serving" mode. This is not ideal, but this is
 	// what we have decided to do. See gRPC A36 for more details.
 	ilc := update.InboundListenerCfg
 	if ilc.Address != l.addr || ilc.Port != l.port {
-		// TODO: Switch to "not serving" if the host:port does not match.
-		l.logger.Warningf("Received host:port (%s:%d) in Listener update does not match local listening address: (%s:%s", ilc.Address, ilc.Port, l.addr, l.port)
+		l.switchMode(nil, ServingModeNotServing, fmt.Errorf("address (%s:%s) in Listener update does not match listening address: (%s:%s)", ilc.Address, ilc.Port, l.addr, l.port))
 		return
 	}
 
-	l.mu.Lock()
-	l.filterChains = ilc.FilterChains
-	l.mu.Unlock()
+	l.switchMode(ilc.FilterChains, ServingModeServing, nil)
 	l.goodUpdate.Fire()
-	// TODO: Move to serving state on receipt of a good response.
+}
+
+func (l *listenerWrapper) switchMode(fcs *xdsclient.FilterChainManager, newMode ServingMode, err error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	l.filterChains = fcs
+	l.mode = newMode
+	if l.modeCallback != nil {
+		l.modeCallback(l.Listener.Addr(), newMode, err)
+	}
+	l.logger.Warningf("Listener %q entering mode: %q due to error: %v", l.Addr(), newMode, err)
 }
diff --git a/xds/internal/test/xds_integration_test.go b/xds/internal/test/xds_integration_test.go
index 13ab5e351a52..1c4b73ac58f8 100644
--- a/xds/internal/test/xds_integration_test.go
+++ b/xds/internal/test/xds_integration_test.go
@@ -32,7 +32,8 @@ import (
 )
 
 const (
-	defaultTestTimeout = 10 * time.Second
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 100 * time.Millisecond
 )
 
 type s struct {
diff --git a/xds/internal/test/xds_server_serving_mode_test.go b/xds/internal/test/xds_server_serving_mode_test.go
new file mode 100644
index 000000000000..0055bc7be508
--- /dev/null
+++ b/xds/internal/test/xds_server_serving_mode_test.go
@@ -0,0 +1,297 @@
+// +build go1.13
+// +build !386
+
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package xds_test contains e2e tests for xDS use.
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"path"
+	"sync"
+	"testing"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	"github.com/google/uuid"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+	"google.golang.org/grpc/xds"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
+)
+
+// A convenience typed used to keep track of mode changes on multiple listeners.
+type modeTracker struct {
+	mu       sync.Mutex
+	modes    map[string]xds.ServingMode
+	updateCh *testutils.Channel
+}
+
+func newModeTracker() *modeTracker {
+	return &modeTracker{
+		modes:    make(map[string]xds.ServingMode),
+		updateCh: testutils.NewChannel(),
+	}
+}
+
+func (mt *modeTracker) updateMode(addr net.Addr, mode xds.ServingMode) {
+	mt.mu.Lock()
+	defer mt.mu.Unlock()
+
+	mt.modes[addr.String()] = mode
+	mt.updateCh.Send(nil)
+}
+
+func (mt *modeTracker) getMode(addr net.Addr) xds.ServingMode {
+	mt.mu.Lock()
+	defer mt.mu.Unlock()
+	return mt.modes[addr.String()]
+}
+
+func (mt *modeTracker) waitForUpdate(ctx context.Context) error {
+	_, err := mt.updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("error when waiting for a mode change update: %v", err)
+	}
+	return nil
+}
+
+// TestServerSideXDS_ServingModeChanges tests the serving mode functionality in
+// xDS enabled gRPC servers. It verifies that appropriate mode changes happen in
+// the server, and also verifies behavior of clientConns under these modes.
+func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
+	// Spin up a xDS management server on a local port.
+	nodeID := uuid.New().String()
+	fs, err := e2e.StartManagementServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fs.Stop()
+
+	// Create certificate and key files in a temporary directory and generate
+	// certificate provider configuration for a file_watcher plugin.
+	tmpdir := createTmpDirWithFiles(t, "testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile))
+
+	// Create a bootstrap file in a temporary directory.
+	bsCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
+		Version:                            xdsinternal.TransportV3,
+		NodeID:                             nodeID,
+		ServerURI:                          fs.Address,
+		CertificateProviders:               cpc,
+		ServerListenerResourceNameTemplate: serverListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer bsCleanup()
+
+	// Configure xDS credentials to be used on the server-side.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a server option to get notified about serving mode changes.
+	modeTracker := newModeTracker()
+	modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
+		t.Logf("serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
+		modeTracker.updateMode(addr, args.Mode)
+	})
+
+	// Initialize an xDS-enabled gRPC server and register the stubServer on it.
+	server := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt)
+	defer server.Stop()
+	testpb.RegisterTestServiceServer(server, &testService{})
+
+	// Create two local listeners and pass it to Serve().
+	lis1, err := xdstestutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	lis2, err := xdstestutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	go func() {
+		if err := server.Serve(lis1); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+	go func() {
+		if err := server.Serve(lis2); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+
+	// Setup the fake management server to respond with Listener resources that
+	// we are interested in.
+	listener1 := listenerResourceWithoutSecurityConfig(t, lis1)
+	listener2 := listenerResourceWithoutSecurityConfig(t, lis2)
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener1, listener2},
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Wait for both listeners to move to "serving" mode.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := waitForModeChange(ctx, modeTracker, lis1.Addr(), xds.ServingModeServing); err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForModeChange(ctx, modeTracker, lis2.Addr(), xds.ServingModeServing); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn to the first listener and make a successful RPCs.
+	cc1, err := grpc.DialContext(ctx, lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc1.Close()
+
+	client1 := testpb.NewTestServiceClient(cc1)
+	if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// Create a ClientConn to the second listener and make a successful RPCs.
+	cc2, err := grpc.DialContext(ctx, lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc2.Close()
+
+	client2 := testpb.NewTestServiceClient(cc2)
+	if _, err := client2.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// Update the management server to remove the second listener resource. This should
+	// push the only the second listener into "not-serving" mode.
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener1},
+	}); err != nil {
+		t.Error(err)
+	}
+	if err := waitForModeChange(ctx, modeTracker, lis2.Addr(), xds.ServingModeNotServing); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure cc1 is still in READY state, while cc2 has moved out of READY.
+	if s := cc1.GetState(); s != connectivity.Ready {
+		t.Fatalf("clientConn1 state is %s, want %s", s, connectivity.Ready)
+	}
+	if !cc2.WaitForStateChange(ctx, connectivity.Ready) {
+		t.Fatal("clientConn2 failed to move out of READY")
+	}
+
+	// Make sure RPCs succeed on cc1 and fail on cc2.
+	if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+	if _, err := client2.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
+	}
+
+	// Update the management server to remove the first listener resource as
+	// well. This should push the first listener into "not-serving" mode. Second
+	// listener is already in "not-serving" mode.
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{},
+	}); err != nil {
+		t.Error(err)
+	}
+	if err := waitForModeChange(ctx, modeTracker, lis1.Addr(), xds.ServingModeNotServing); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure cc1 has moved out of READY.
+	if !cc1.WaitForStateChange(ctx, connectivity.Ready) {
+		t.Fatal("clientConn1 failed to move out of READY")
+	}
+
+	// Make sure RPCs fail on both.
+	if _, err := client1.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
+	}
+	if _, err := client2.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
+	}
+
+	// Make sure new connection attempts to "not-serving" servers fail. We use a
+	// short timeout since we expect this to fail.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	_, err = grpc.DialContext(sCtx, lis1.Addr().String(), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err == nil {
+		t.Fatal("successfully created clientConn to a server in \"not-serving\" state")
+	}
+
+	// Update the management server with both listener resources.
+	if err := fs.Update(e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{listener1, listener2},
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Wait for both listeners to move to "serving" mode.
+	if err := waitForModeChange(ctx, modeTracker, lis1.Addr(), xds.ServingModeServing); err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForModeChange(ctx, modeTracker, lis2.Addr(), xds.ServingModeServing); err != nil {
+		t.Fatal(err)
+	}
+
+	// The clientConns created earlier should be able to make RPCs now.
+	if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+	if _, err := client2.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+func waitForModeChange(ctx context.Context, modeTracker *modeTracker, addr net.Addr, wantMode xds.ServingMode) error {
+	for {
+		if gotMode := modeTracker.getMode(addr); gotMode == wantMode {
+			return nil
+		}
+		if err := modeTracker.waitForUpdate(ctx); err != nil {
+			return err
+		}
+	}
+}
diff --git a/xds/server.go b/xds/server.go
index 805f59b4f5ac..3a2b629ae986 100644
--- a/xds/server.go
+++ b/xds/server.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/buffer"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -48,9 +49,9 @@ var (
 		return grpc.NewServer(opts...)
 	}
 
-	// Unexported function to retrieve transport credentials from a gRPC server.
-	grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials)
-	logger             = grpclog.Component("xds")
+	grpcGetServerCreds    = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials)
+	drainServerTransports = internal.DrainServerTransports.(func(*grpc.Server, string))
+	logger                = grpclog.Component("xds")
 )
 
 func prefixLogger(p *GRPCServer) *internalgrpclog.PrefixLogger {
@@ -78,16 +79,12 @@ type grpcServerInterface interface {
 // communication with a management server using xDS APIs. It implements the
 // grpc.ServiceRegistrar interface and can be passed to service registration
 // functions in IDL generated code.
-//
-// Experimental
-//
-// Notice: This type is EXPERIMENTAL and may be changed or removed in a
-// later release.
 type GRPCServer struct {
 	gs            grpcServerInterface
 	quit          *grpcsync.Event
 	logger        *internalgrpclog.PrefixLogger
 	xdsCredsInUse bool
+	opts          *serverOptions
 
 	// clientMu is used only in initXDSClient(), which is called at the
 	// beginning of Serve(), where we have to decide if we have to create a
@@ -99,11 +96,6 @@ type GRPCServer struct {
 // NewGRPCServer creates an xDS-enabled gRPC server using the passed in opts.
 // The underlying gRPC server has no service registered and has not started to
 // accept requests yet.
-//
-// Experimental
-//
-// Notice: This API is EXPERIMENTAL and may be changed or removed in a later
-// release.
 func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer {
 	newOpts := []grpc.ServerOption{
 		grpc.ChainUnaryInterceptor(xdsUnaryInterceptor),
@@ -113,6 +105,7 @@ func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer {
 	s := &GRPCServer{
 		gs:   newGRPCServer(newOpts...),
 		quit: grpcsync.NewEvent(),
+		opts: handleServerOptions(opts),
 	}
 	s.logger = prefixLogger(s)
 	s.logger.Infof("Created xds.GRPCServer")
@@ -133,6 +126,18 @@ func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer {
 	return s
 }
 
+// handleServerOptions iterates through the list of server options passed in by
+// the user, and handles the xDS server specific options.
+func handleServerOptions(opts []grpc.ServerOption) *serverOptions {
+	so := &serverOptions{}
+	for _, opt := range opts {
+		if o, ok := opt.(serverOption); ok {
+			o.applyServerOption(so)
+		}
+	}
+	return so
+}
+
 // RegisterService registers a service and its implementation to the underlying
 // gRPC server. It is called from the IDL generated code. This must be called
 // before invoking Serve.
@@ -165,7 +170,6 @@ func (s *GRPCServer) initXDSClient() error {
 // initiated here.
 //
 // Serve will return a non-nil error unless Stop or GracefulStop is called.
-// TODO: Support callback to get notified on serving state changes.
 func (s *GRPCServer) Serve(lis net.Listener) error {
 	s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String())
 	if _, ok := lis.Addr().(*net.TCPAddr); !ok {
@@ -207,6 +211,11 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 		name = strings.Replace(cfg.ServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)
 	}
 
+	modeUpdateCh := buffer.NewUnbounded()
+	go func() {
+		s.handleServingModeChanges(modeUpdateCh)
+	}()
+
 	// Create a listenerWrapper which handles all functionality required by
 	// this particular instance of Serve().
 	lw, goodUpdateCh := server.NewListenerWrapper(server.ListenerWrapperParams{
@@ -214,6 +223,13 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 		ListenerResourceName: name,
 		XDSCredsInUse:        s.xdsCredsInUse,
 		XDSClient:            s.xdsC,
+		ModeCallback: func(addr net.Addr, mode server.ServingMode, err error) {
+			modeUpdateCh.Put(&modeChangeArgs{
+				addr: addr,
+				mode: mode,
+				err:  err,
+			})
+		},
 	})
 
 	// Block until a good LDS response is received or the server is stopped.
@@ -229,6 +245,47 @@ func (s *GRPCServer) Serve(lis net.Listener) error {
 	return s.gs.Serve(lw)
 }
 
+// modeChangeArgs wraps argument required for invoking mode change callback.
+type modeChangeArgs struct {
+	addr net.Addr
+	mode server.ServingMode
+	err  error
+}
+
+// handleServingModeChanges runs as a separate goroutine, spawned from Serve().
+// It reads a channel on to which mode change arguments are pushed, and in turn
+// invokes the user registered callback. It also calls an internal method on the
+// underlying grpc.Server to gracefully close existing connections, if the
+// listener moved to a "not-serving" mode.
+func (s *GRPCServer) handleServingModeChanges(updateCh *buffer.Unbounded) {
+	for {
+		select {
+		case <-s.quit.Done():
+			return
+		case u := <-updateCh.Get():
+			updateCh.Load()
+			args := u.(*modeChangeArgs)
+			if args.mode == ServingModeNotServing {
+				// We type assert our underlying gRPC server to the real
+				// grpc.Server here before trying to initiate the drain
+				// operation. This approach avoids performing the same type
+				// assertion in the grpc package which provides the
+				// implementation for internal.GetServerCredentials, and allows
+				// us to use a fake gRPC server in tests.
+				if gs, ok := s.gs.(*grpc.Server); ok {
+					drainServerTransports(gs, args.addr.String())
+				}
+			}
+			if s.opts.modeCallback != nil {
+				s.opts.modeCallback(args.addr, ServingModeChangeArgs{
+					Mode: args.mode,
+					Err:  args.err,
+				})
+			}
+		}
+	}
+}
+
 // Stop stops the underlying gRPC server. It immediately closes all open
 // connections. It cancels all active RPCs on the server side and the
 // corresponding pending RPCs on the client side will get notified by connection
diff --git a/xds/server_options.go b/xds/server_options.go
new file mode 100644
index 000000000000..44b7b374fd00
--- /dev/null
+++ b/xds/server_options.go
@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds
+
+import (
+	"net"
+
+	"google.golang.org/grpc"
+	iserver "google.golang.org/grpc/xds/internal/server"
+)
+
+// ServingModeCallback returns a grpc.ServerOption which allows users to
+// register a callback to get notified about serving mode changes.
+func ServingModeCallback(cb ServingModeCallbackFunc) grpc.ServerOption {
+	return &smcOption{cb: cb}
+}
+
+type serverOption interface {
+	applyServerOption(*serverOptions)
+}
+
+// smcOption is a server option containing a callback to be invoked when the
+// serving mode changes.
+type smcOption struct {
+	// Embedding the empty server option makes it safe to pass it to
+	// grpc.NewServer().
+	grpc.EmptyServerOption
+	cb ServingModeCallbackFunc
+}
+
+func (s *smcOption) applyServerOption(o *serverOptions) {
+	o.modeCallback = s.cb
+}
+
+type serverOptions struct {
+	modeCallback ServingModeCallbackFunc
+}
+
+// ServingMode indicates the current mode of operation of the server.
+type ServingMode = iserver.ServingMode
+
+const (
+	// ServingModeServing indicates the the server contains all required xDS
+	// configuration is serving RPCs.
+	ServingModeServing = iserver.ServingModeServing
+	// ServingModeNotServing indicates that the server is not accepting new
+	// connections. Existing connections will be closed gracefully, allowing
+	// in-progress RPCs to complete. A server enters this mode when it does not
+	// contain the required xDS configuration to serve RPCs.
+	ServingModeNotServing = iserver.ServingModeNotServing
+)
+
+// ServingModeCallbackFunc is the callback that users can register to get
+// notified about the server's serving mode changes. The callback is invoked
+// with the address of the listener and its new mode.
+//
+// Users must not perform any blocking operations in this callback.
+type ServingModeCallbackFunc func(addr net.Addr, args ServingModeChangeArgs)
+
+// ServingModeChangeArgs wraps the arguments passed to the serving mode callback
+// function.
+type ServingModeChangeArgs struct {
+	// Mode is the new serving mode of the server listener.
+	Mode ServingMode
+	// Err is set to a non-nil error if the server has transitioned into
+	// not-serving mode.
+	Err error
+}
diff --git a/xds/server_test.go b/xds/server_test.go
index 41767bc1555a..3fb3bcd3818b 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -311,7 +311,14 @@ func (s) TestServeSuccess(t *testing.T) {
 	fs, clientCh, cleanup := setupOverrides()
 	defer cleanup()
 
-	server := NewGRPCServer()
+	// Create a new xDS-enabled gRPC server and pass it a server option to get
+	// notified about serving mode changes.
+	modeChangeCh := testutils.NewChannel()
+	modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) {
+		t.Logf("server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err)
+		modeChangeCh.Send(args.Mode)
+	})
+	server := NewGRPCServer(modeChangeOption)
 	defer server.Stop()
 
 	lis, err := xdstestutils.LocalTCPListener()
@@ -349,13 +356,22 @@ func (s) TestServeSuccess(t *testing.T) {
 
 	// Push an error to the registered listener watch callback and make sure
 	// that Serve does not return.
-	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, errors.New("LDS error"))
+	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{}, xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "LDS resource not found"))
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
 	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("Serve() returned after a bad LDS response")
 	}
 
+	// Make sure the serving mode changes appropriately.
+	v, err := modeChangeCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for serving mode to change: %v", err)
+	}
+	if mode := v.(ServingMode); mode != ServingModeNotServing {
+		t.Fatalf("server mode is %q, want %q", mode, ServingModeNotServing)
+	}
+
 	// Push a good LDS response, and wait for Serve() to be invoked on the
 	// underlying grpc.Server.
 	addr, port := splitHostPort(lis.Addr().String())
@@ -370,11 +386,18 @@ func (s) TestServeSuccess(t *testing.T) {
 		t.Fatalf("error when waiting for Serve() to be invoked on the grpc.Server")
 	}
 
+	// Make sure the serving mode changes appropriately.
+	v, err = modeChangeCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for serving mode to change: %v", err)
+	}
+	if mode := v.(ServingMode); mode != ServingModeServing {
+		t.Fatalf("server mode is %q, want %q", mode, ServingModeServing)
+	}
+
 	// Push an update to the registered listener watch callback with a Listener
 	// resource whose host:port does not match the actual listening address and
-	// port. Serve() should not return and should continue to use the old state.
-	//
-	// This will change once we add start tracking serving state.
+	// port. This will push the listener to "not-serving" mode.
 	client.InvokeWatchListenerCallback(xdsclient.ListenerUpdate{
 		RouteConfigName: "routeconfig",
 		InboundListenerCfg: &xdsclient.InboundListenerConfig{
@@ -387,6 +410,15 @@ func (s) TestServeSuccess(t *testing.T) {
 	if _, err := serveDone.Receive(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("Serve() returned after a bad LDS response")
 	}
+
+	// Make sure the serving mode changes appropriately.
+	v, err = modeChangeCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("error when waiting for serving mode to change: %v", err)
+	}
+	if mode := v.(ServingMode); mode != ServingModeNotServing {
+		t.Fatalf("server mode is %q, want %q", mode, ServingModeNotServing)
+	}
 }
 
 // TestServeWithStop tests the case where Stop() is called before an LDS update
diff --git a/xds/xds.go b/xds/xds.go
index 562c5aa82abc..23c88903f40b 100644
--- a/xds/xds.go
+++ b/xds/xds.go
@@ -25,6 +25,11 @@
 //
 // See https://github.com/grpc/grpc-go/tree/master/examples/features/xds for
 // example.
+//
+// Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
 package xds
 
 import (

From 145f12a95b19d2a2f926176cd63fe5645b376186 Mon Sep 17 00:00:00 2001
From: Joshua Humphries <jh@fullstory.com>
Date: Tue, 27 Apr 2021 16:15:08 -0400
Subject: [PATCH 436/481] reflection: accept interface instead of grpc.Server
 struct in Register() (#4340)

---
 reflection/serverreflection.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/reflection/serverreflection.go b/reflection/serverreflection.go
index d2696168b10c..82a5ba7f2444 100644
--- a/reflection/serverreflection.go
+++ b/reflection/serverreflection.go
@@ -54,9 +54,19 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+// GRPCServer is the interface provided by a gRPC server. It is implemented by
+// *grpc.Server, but could also be implemented by other concrete types. It acts
+// as a registry, for accumulating the services exposed by the server.
+type GRPCServer interface {
+	grpc.ServiceRegistrar
+	GetServiceInfo() map[string]grpc.ServiceInfo
+}
+
+var _ GRPCServer = (*grpc.Server)(nil)
+
 type serverReflectionServer struct {
 	rpb.UnimplementedServerReflectionServer
-	s *grpc.Server
+	s GRPCServer
 
 	initSymbols  sync.Once
 	serviceNames []string
@@ -64,7 +74,7 @@ type serverReflectionServer struct {
 }
 
 // Register registers the server reflection service on the given gRPC server.
-func Register(s *grpc.Server) {
+func Register(s GRPCServer) {
 	rpb.RegisterServerReflectionServer(s, &serverReflectionServer{
 		s: s,
 	})

From 7c5e73795d163c13e616aa53066f9e1d845275dd Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 27 Apr 2021 13:37:48 -0700
Subject: [PATCH 437/481] xds/cds: add separate fields for cluster name and eds
 service name (#4352)

---
 .../balancer/cdsbalancer/cdsbalancer.go       |  3 +-
 .../cdsbalancer/cdsbalancer_security_test.go  | 14 +--
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 16 ++--
 xds/internal/balancer/edsbalancer/config.go   |  8 +-
 xds/internal/balancer/edsbalancer/eds.go      | 56 ++++++++----
 .../balancer/edsbalancer/eds_impl_test.go     | 11 +--
 xds/internal/balancer/edsbalancer/eds_test.go | 22 +++--
 .../edsbalancer/load_store_wrapper.go         | 88 -------------------
 xds/internal/client/cds_test.go               | 75 +++++++++++-----
 xds/internal/client/client.go                 | 15 ++--
 xds/internal/client/client_test.go            |  4 +-
 xds/internal/client/v2/cds_test.go            |  4 +-
 xds/internal/client/watchers_cluster_test.go  | 16 ++--
 xds/internal/client/xds.go                    | 75 +++++++---------
 14 files changed, 190 insertions(+), 217 deletions(-)
 delete mode 100644 xds/internal/balancer/edsbalancer/load_store_wrapper.go

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index b991981c14c0..c97e92bcd02f 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -340,7 +340,8 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
 	}
 	lbCfg := &edsbalancer.EDSConfig{
-		EDSServiceName:        update.cds.ServiceName,
+		ClusterName:           update.cds.ClusterName,
+		EDSServiceName:        update.cds.EDSServiceName,
 		MaxConcurrentRequests: update.cds.MaxRequests,
 	}
 	if update.cds.EnableLRS {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 5c746cfa163c..18a2298c5258 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -60,7 +60,7 @@ var (
 	fpb1, fpb2                   *fakeProviderBuilder
 	bootstrapConfig              *bootstrap.Config
 	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default1",
 			IdentityInstanceName:   "default2",
@@ -68,7 +68,7 @@ var (
 		},
 	}
 	cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName: "not-default",
 		},
@@ -256,7 +256,7 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -312,7 +312,7 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup. No security config is
 	// passed to the CDS balancer as part of this update.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -572,7 +572,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	// an update which contains bad security config. So, we expect the CDS
 	// balancer to forward this error to the EDS balancer and eventually the
 	// channel needs to be put in a bad state.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -678,7 +678,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default1",
 			SubjectAltNameMatchers: testSANMatchers,
@@ -703,7 +703,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 
 	// Push another update with a new security configuration.
 	cdsUpdate = xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default2",
 			SubjectAltNameMatchers: testSANMatchers,
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 4476a1532d05..13b17306df8b 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -197,7 +197,7 @@ func cdsCCS(cluster string) balancer.ClientConnState {
 // cdsBalancer to the edsBalancer.
 func edsCCS(service string, countMax *uint32, enableLRS bool) balancer.ClientConnState {
 	lbCfg := &edsbalancer.EDSConfig{
-		EDSServiceName:        service,
+		ClusterName:           service,
 		MaxConcurrentRequests: countMax,
 	}
 	if enableLRS {
@@ -354,12 +354,12 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 	}{
 		{
 			name:      "happy-case-with-lrs",
-			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
+			cdsUpdate: xdsclient.ClusterUpdate{ClusterName: serviceName, EnableLRS: true},
 			wantCCS:   edsCCS(serviceName, nil, true),
 		},
 		{
 			name:      "happy-case-without-lrs",
-			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
+			cdsUpdate: xdsclient.ClusterUpdate{ClusterName: serviceName},
 			wantCCS:   edsCCS(serviceName, nil, false),
 		},
 	}
@@ -427,7 +427,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -512,7 +512,7 @@ func (s) TestResolverError(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -561,7 +561,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -596,7 +596,7 @@ func (s) TestCircuitBreaking(t *testing.T) {
 	// will trigger the watch handler on the CDS balancer, which will update
 	// the service's counter with the new max requests.
 	var maxRequests uint32 = 1
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName, MaxRequests: &maxRequests}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName, MaxRequests: &maxRequests}
 	wantCCS := edsCCS(serviceName, &maxRequests, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -629,7 +629,7 @@ func (s) TestClose(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
diff --git a/xds/internal/balancer/edsbalancer/config.go b/xds/internal/balancer/edsbalancer/config.go
index 11c1338c81f7..d1583e2bf276 100644
--- a/xds/internal/balancer/edsbalancer/config.go
+++ b/xds/internal/balancer/edsbalancer/config.go
@@ -35,8 +35,10 @@ type EDSConfig struct {
 	// FallBackPolicy represents the load balancing config for the
 	// fallback.
 	FallBackPolicy *loadBalancingConfig
-	// Name to use in EDS query.  If not present, defaults to the server
-	// name from the target URI.
+	// ClusterName is the cluster name.
+	ClusterName string
+	// EDSServiceName is the name to use in EDS query. If not set, use
+	// ClusterName.
 	EDSServiceName string
 	// MaxConcurrentRequests is the max number of concurrent request allowed for
 	// this service. If unset, default value 1024 is used.
@@ -59,6 +61,7 @@ type EDSConfig struct {
 type edsConfigJSON struct {
 	ChildPolicy                []*loadBalancingConfig
 	FallbackPolicy             []*loadBalancingConfig
+	ClusterName                string
 	EDSServiceName             string
 	MaxConcurrentRequests      *uint32
 	LRSLoadReportingServerName *string
@@ -73,6 +76,7 @@ func (l *EDSConfig) UnmarshalJSON(data []byte) error {
 		return err
 	}
 
+	l.ClusterName = configJSON.ClusterName
 	l.EDSServiceName = configJSON.EDSServiceName
 	l.MaxConcurrentRequests = configJSON.MaxConcurrentRequests
 	l.LrsLoadReportingServerName = configJSON.LRSLoadReportingServerName
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index de724701df94..1b5191aa1037 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -68,7 +69,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
-		lsw:               &loadStoreWrapper{},
+		loadWrapper:       loadstore.NewWrapper(),
 		config:            &EDSConfig{},
 	}
 	x.logger = prefixLogger(x)
@@ -80,7 +81,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 	}
 
 	x.xdsClient = client
-	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.lsw, x.logger)
+	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.loadWrapper, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
@@ -138,14 +139,14 @@ type edsBalancer struct {
 	xdsClientUpdate   chan *edsUpdate
 	childPolicyUpdate *buffer.Unbounded
 
-	xdsClient xdsClientInterface
-	lsw       *loadStoreWrapper
-	config    *EDSConfig // may change when passed a different service config
-	edsImpl   edsBalancerImplInterface
+	xdsClient   xdsClientInterface
+	loadWrapper *loadstore.Wrapper
+	config      *EDSConfig // may change when passed a different service config
+	edsImpl     edsBalancerImplInterface
 
-	// edsServiceName is the edsServiceName currently being watched, not
-	// necessary the edsServiceName from service config.
+	clusterName          string
 	edsServiceName       string
+	edsToWatch           string // this is edsServiceName if it's set, otherwise, it's clusterName.
 	cancelEndpointsWatch func()
 	loadReportServer     *string // LRS is disabled if loadReporterServer is nil.
 	cancelLoadReport     func()
@@ -241,10 +242,35 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 // handleServiceConfigUpdate applies the service config update, watching a new
 // EDS service name and restarting LRS stream, as required.
 func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
-	// Restart EDS watch when the edsServiceName has changed.
+	var updateLoadClusterAndService bool
+	if x.clusterName != config.ClusterName {
+		updateLoadClusterAndService = true
+		x.clusterName = config.ClusterName
+		x.edsImpl.updateClusterName(x.clusterName)
+	}
 	if x.edsServiceName != config.EDSServiceName {
+		updateLoadClusterAndService = true
 		x.edsServiceName = config.EDSServiceName
+	}
+
+	// If EDSServiceName is set, use it to watch EDS. Otherwise, use the cluster
+	// name.
+	newEDSToWatch := config.EDSServiceName
+	if newEDSToWatch == "" {
+		newEDSToWatch = config.ClusterName
+	}
+	var restartEDSWatch bool
+	if x.edsToWatch != newEDSToWatch {
+		restartEDSWatch = true
+		x.edsToWatch = newEDSToWatch
+	}
+
+	// Restart EDS watch when the eds name has changed.
+	if restartEDSWatch {
 		x.startEndpointsWatch()
+	}
+
+	if updateLoadClusterAndService {
 		// TODO: this update for the LRS service name is too early. It should
 		// only apply to the new EDS response. But this is applied to the RPCs
 		// before the new EDS response. To fully fix this, the EDS balancer
@@ -252,14 +278,13 @@ func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
 		//
 		// This is OK for now, because we don't actually expect edsServiceName
 		// to change. Fix this (a bigger change) will happen later.
-		x.lsw.updateServiceName(x.edsServiceName)
-		x.edsImpl.updateClusterName(x.edsServiceName)
+		x.loadWrapper.UpdateClusterAndService(x.clusterName, x.edsServiceName)
 	}
 
 	// Restart load reporting when the loadReportServer name has changed.
 	if !equalStringPointers(x.loadReportServer, config.LrsLoadReportingServerName) {
 		loadStore := x.startLoadReport(config.LrsLoadReportingServerName)
-		x.lsw.updateLoadStore(loadStore)
+		x.loadWrapper.UpdateLoadStore(loadStore)
 	}
 
 	return nil
@@ -273,14 +298,15 @@ func (x *edsBalancer) startEndpointsWatch() {
 	if x.cancelEndpointsWatch != nil {
 		x.cancelEndpointsWatch()
 	}
-	cancelEDSWatch := x.xdsClient.WatchEndpoints(x.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
+	edsToWatch := x.edsToWatch
+	cancelEDSWatch := x.xdsClient.WatchEndpoints(edsToWatch, func(update xdsclient.EndpointsUpdate, err error) {
 		x.logger.Infof("Watch update from xds-client %p, content: %+v", x.xdsClient, update)
 		x.handleEDSUpdate(update, err)
 	})
-	x.logger.Infof("Watch started on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
+	x.logger.Infof("Watch started on resource name %v with xds-client %p", edsToWatch, x.xdsClient)
 	x.cancelEndpointsWatch = func() {
 		cancelEDSWatch()
-		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
+		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", edsToWatch, x.xdsClient)
 	}
 }
 
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index c5e3071d10d8..3cfc620a2400 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -820,9 +821,9 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
 	loadStore := load.NewStore()
-	lsWrapper := &loadStoreWrapper{}
-	lsWrapper.updateServiceName(testClusterNames[0])
-	lsWrapper.updateLoadStore(loadStore)
+	lsWrapper := loadstore.NewWrapper()
+	lsWrapper.UpdateClusterAndService(testClusterNames[0], "")
+	lsWrapper.UpdateLoadStore(loadStore)
 
 	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, lsWrapper, nil)
@@ -913,8 +914,8 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 // TestEDS_LoadReportDisabled covers the case that LRS is disabled. It makes
 // sure the EDS implementation isn't broken (doesn't panic).
 func (s) TestEDS_LoadReportDisabled(t *testing.T) {
-	lsWrapper := &loadStoreWrapper{}
-	lsWrapper.updateServiceName(testClusterNames[0])
+	lsWrapper := loadstore.NewWrapper()
+	lsWrapper.UpdateClusterAndService(testClusterNames[0], "")
 	// Not calling lsWrapper.updateLoadStore(loadStore) because LRS is disabled.
 
 	cc := testutils.NewTestClientConn(t)
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 5c9e5f0b1d53..b986654bc409 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -586,7 +586,7 @@ func verifyExpectedRequests(ctx context.Context, fc *fakeclient.Client, resource
 			if err := fc.WaitForCancelEDSWatch(ctx); err != nil {
 				return fmt.Errorf("timed out when expecting resource %q", name)
 			}
-			return nil
+			continue
 		}
 
 		resName, err := fc.WaitForWatchEDS(ctx)
@@ -615,6 +615,18 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}
 	defer edsB.Close()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// If eds service name is not set, should watch for cluster name.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{ClusterName: "cluster-1"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyExpectedRequests(ctx, xdsC, "cluster-1"); err != nil {
+		t.Fatal(err)
+	}
+
 	// Update with an non-empty edsServiceName should trigger an EDS watch for
 	// the same.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
@@ -622,9 +634,7 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if err := verifyExpectedRequests(ctx, xdsC, "foobar-1"); err != nil {
+	if err := verifyExpectedRequests(ctx, xdsC, "", "foobar-1"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -694,7 +704,7 @@ func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			EDSServiceName: "foobar-1",
+			ClusterName: "foobar-1",
 		},
 	}); err != nil {
 		t.Fatal(err)
@@ -713,7 +723,7 @@ func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			EDSServiceName: "foobar-2",
+			ClusterName: "foobar-2",
 		},
 	}); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/balancer/edsbalancer/load_store_wrapper.go b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
deleted file mode 100644
index 18904e47a42e..000000000000
--- a/xds/internal/balancer/edsbalancer/load_store_wrapper.go
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package edsbalancer
-
-import (
-	"sync"
-
-	"google.golang.org/grpc/xds/internal/client/load"
-)
-
-type loadStoreWrapper struct {
-	mu      sync.RWMutex
-	service string
-	// Both store and perCluster will be nil if load reporting is disabled (EDS
-	// response doesn't have LRS server name). Note that methods on Store and
-	// perCluster all handle nil, so there's no need to check nil before calling
-	// them.
-	store      *load.Store
-	perCluster load.PerClusterReporter
-}
-
-func (lsw *loadStoreWrapper) updateServiceName(service string) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if lsw.service == service {
-		return
-	}
-	lsw.service = service
-	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
-}
-
-func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if store == lsw.store {
-		return
-	}
-	lsw.store = store
-	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
-}
-
-func (lsw *loadStoreWrapper) CallStarted(locality string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallStarted(locality)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallFinished(locality, err)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallServerLoad(locality, name, val)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallDropped(category string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallDropped(category)
-	}
-}
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index bb1117ec5349..0fa2b402ffc9 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -28,12 +28,14 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/testutils"
 	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
@@ -45,7 +47,7 @@ const (
 	serviceName = "service"
 )
 
-var emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false}
+var emptyUpdate = ClusterUpdate{ClusterName: clusterName, EnableLRS: false}
 
 func (s) TestValidateCluster_Failure(t *testing.T) {
 	tests := []struct {
@@ -141,24 +143,35 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		{
 			name: "happy-case-logical-dns",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS},
 				LbPolicy:             v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
 		},
 		{
 			name: "happy-case-aggregate-v3",
 			cluster: &v3clusterpb.Cluster{
+				Name: clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{
-					ClusterType: &v3clusterpb.Cluster_CustomClusterType{Name: "envoy.clusters.aggregate"},
+					ClusterType: &v3clusterpb.Cluster_CustomClusterType{
+						Name: "envoy.clusters.aggregate",
+						TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{
+							Clusters: []string{"a", "b", "c"},
+						}),
+					},
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeAggregate},
+			wantUpdate: ClusterUpdate{
+				ClusterName: clusterName, EnableLRS: false, ClusterType: ClusterTypeAggregate,
+				PrioritizedClusterNames: []string{"a", "b", "c"},
+			},
 		},
 		{
 			name: "happy-case-no-service-name-no-lrs",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -174,6 +187,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		{
 			name: "happy-case-no-lrs",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -185,7 +199,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: false},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: false},
 		},
 		{
 			name: "happiest-case",
@@ -207,7 +221,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: true},
 		},
 		{
 			name: "happiest-case-with-circuitbreakers",
@@ -241,7 +255,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
 		},
 	}
 
@@ -254,8 +268,8 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 			if err != nil {
 				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err)
 			}
-			if !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, want: %v", test.cluster, update, test.wantUpdate)
+			if diff := cmp.Diff(update, test.wantUpdate, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) got diff: %v (-got, +want)", test.cluster, diff)
 			}
 		})
 	}
@@ -268,6 +282,7 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 	defer func() { env.ClientSideSecuritySupport = origClientSideSecurityEnvVar }()
 
 	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
 		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 			EdsConfig: &v3corepb.ConfigSource{
@@ -302,8 +317,9 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 		},
 	}
 	wantUpdate := ClusterUpdate{
-		ServiceName: serviceName,
-		EnableLRS:   false,
+		ClusterName:    clusterName,
+		EDSServiceName: serviceName,
+		EnableLRS:      false,
 	}
 	gotUpdate, err := validateClusterAndConstructClusterUpdate(cluster)
 	if err != nil {
@@ -325,6 +341,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		identityCertName       = "identityCert"
 		rootPluginInstance     = "rootPluginInstance"
 		rootCertName           = "rootCert"
+		clusterName            = "cluster"
 		serviceName            = "service"
 		sanExact               = "san-exact"
 		sanPrefix              = "san-prefix"
@@ -657,6 +674,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-no-identity-certs",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -691,8 +709,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ServiceName: serviceName,
-				EnableLRS:   false,
+				ClusterName:    clusterName,
+				EDSServiceName: serviceName,
+				EnableLRS:      false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName: rootPluginInstance,
 					RootCertName:     rootCertName,
@@ -702,6 +721,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-validation-context-provider-instance",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -740,8 +760,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ServiceName: serviceName,
-				EnableLRS:   false,
+				ClusterName:    clusterName,
+				EDSServiceName: serviceName,
+				EnableLRS:      false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName:     rootPluginInstance,
 					RootCertName:         rootCertName,
@@ -753,6 +774,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-combined-validation-context",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -805,8 +827,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ServiceName: serviceName,
-				EnableLRS:   false,
+				ClusterName:    clusterName,
+				EDSServiceName: serviceName,
+				EnableLRS:      false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName:     rootPluginInstance,
 					RootCertName:         rootCertName,
@@ -968,7 +991,8 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v2ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ServiceName: v2Service, EnableLRS: true,
+					ClusterName:    v2ClusterName,
+					EDSServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 			},
@@ -982,7 +1006,8 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v3ClusterName: {
-					ServiceName: v3Service, EnableLRS: true,
+					ClusterName:    v3ClusterName,
+					EDSServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 			},
@@ -996,11 +1021,13 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v2ClusterAny, v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ServiceName: v2Service, EnableLRS: true,
+					ClusterName:    v2ClusterName,
+					EDSServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 				v3ClusterName: {
-					ServiceName: v3Service, EnableLRS: true,
+					ClusterName:    v3ClusterName,
+					EDSServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 			},
@@ -1030,11 +1057,13 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ServiceName: v2Service, EnableLRS: true,
+					ClusterName:    v2ClusterName,
+					EDSServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 				v3ClusterName: {
-					ServiceName: v3Service, EnableLRS: true,
+					ClusterName:    v3ClusterName,
+					EDSServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 				"bad": {},
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 2daceede5398..4067536dda24 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -375,22 +375,23 @@ const (
 // interest to the registered CDS watcher.
 type ClusterUpdate struct {
 	ClusterType ClusterType
-	// ServiceName is the service name corresponding to the clusterName which
-	// is being watched for through CDS.
-	ServiceName string
+	// ClusterName is the clusterName being watched for through CDS.
+	ClusterName string
+	// EDSServiceName is an optional name for EDS. If it's not set, the balancer
+	// should watch ClusterName for the EDS resources.
+	EDSServiceName string
 	// EnableLRS indicates whether or not load should be reported through LRS.
 	EnableLRS bool
 	// SecurityCfg contains security configuration sent by the control plane.
 	SecurityCfg *SecurityConfig
 	// MaxRequests for circuit breaking, if any (otherwise nil).
 	MaxRequests *uint32
-
-	// Raw is the resource from the xds response.
-	Raw *anypb.Any
-
 	// PrioritizedClusterNames is used only for cluster type aggregate. It represents
 	// a prioritized list of cluster names.
 	PrioritizedClusterNames []string
+
+	// Raw is the resource from the xds response.
+	Raw *anypb.Any
 }
 
 // OverloadDropConfig contains the config to drop overloads.
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 69930557b26e..d57bc20e2c2c 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -185,13 +185,13 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
-	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate2}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate2); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index b56ae6108bbe..d868beb1831b 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -151,7 +151,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			cdsResponse: goodCDSResponse2,
 			wantErr:     false,
 			wantUpdate: map[string]xdsclient.ClusterUpdate{
-				goodClusterName2: {ServiceName: serviceName2, Raw: marshaledCluster2},
+				goodClusterName2: {ClusterName: goodClusterName2, EDSServiceName: serviceName2, Raw: marshaledCluster2},
 			},
 			wantUpdateMD: xdsclient.UpdateMetadata{
 				Status: xdsclient.ServiceStatusACKed,
@@ -164,7 +164,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			cdsResponse: goodCDSResponse1,
 			wantErr:     false,
 			wantUpdate: map[string]xdsclient.ClusterUpdate{
-				goodClusterName1: {ServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
+				goodClusterName1: {ClusterName: goodClusterName1, EDSServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
 			},
 			wantUpdateMD: xdsclient.UpdateMetadata{
 				Status: xdsclient.ServiceStatusACKed,
diff --git a/xds/internal/client/watchers_cluster_test.go b/xds/internal/client/watchers_cluster_test.go
index 2d10c7f43b5f..c9837cd51978 100644
--- a/xds/internal/client/watchers_cluster_test.go
+++ b/xds/internal/client/watchers_cluster_test.go
@@ -64,7 +64,7 @@ func (s) TestClusterWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -128,7 +128,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		}
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
@@ -200,8 +200,8 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
-	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	wantUpdate1 := ClusterUpdate{ClusterName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
@@ -245,7 +245,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	}, UpdateMetadata{})
@@ -345,7 +345,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	}, UpdateMetadata{})
@@ -402,8 +402,8 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
-	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	wantUpdate1 := ClusterUpdate{ClusterName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index c0caf5cceb57..f64d22ce67d3 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -571,42 +571,9 @@ func unmarshalClusterResource(r *anypb.Any, logger *grpclog.PrefixLogger) (strin
 	return cluster.GetName(), cu, nil
 }
 
-func clusterTypeFromCluster(cluster *v3clusterpb.Cluster) (ClusterType, string, []string, error) {
-	if cluster.GetType() == v3clusterpb.Cluster_EDS {
-		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
-			return 0, "", nil, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
-		}
-		// If the Cluster message in the CDS response did not contain a
-		// serviceName, we will just use the clusterName for EDS.
-		if cluster.GetEdsClusterConfig().GetServiceName() == "" {
-			return ClusterTypeEDS, cluster.GetName(), nil, nil
-		}
-		return ClusterTypeEDS, cluster.GetEdsClusterConfig().GetServiceName(), nil, nil
-	}
-
-	if cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS {
-		return ClusterTypeLogicalDNS, cluster.GetName(), nil, nil
-	}
-
-	if cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate" {
-		// Loop through ClusterConfig here to get cluster names.
-		clusters := &v3aggregateclusterpb.ClusterConfig{}
-		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
-			return 0, "", nil, fmt.Errorf("failed to unmarshal resource: %v", err)
-		}
-		return ClusterTypeAggregate, cluster.GetName(), clusters.Clusters, nil
-	}
-	return 0, "", nil, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
-}
-
 func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
-	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
 	if cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN {
-		return emptyUpdate, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
-	}
-	clusterType, serviceName, prioritizedClusters, err := clusterTypeFromCluster(cluster)
-	if err != nil {
-		return emptyUpdate, err
+		return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
 
 	// Process security configuration received from the control plane iff the
@@ -615,18 +582,40 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu
 	if env.ClientSideSecuritySupport {
 		var err error
 		if sc, err = securityConfigFromCluster(cluster); err != nil {
-			return emptyUpdate, err
+			return ClusterUpdate{}, err
 		}
 	}
 
-	return ClusterUpdate{
-		ClusterType:             clusterType,
-		ServiceName:             serviceName,
-		EnableLRS:               cluster.GetLrsServer().GetSelf() != nil,
-		SecurityCfg:             sc,
-		MaxRequests:             circuitBreakersFromCluster(cluster),
-		PrioritizedClusterNames: prioritizedClusters,
-	}, nil
+	ret := ClusterUpdate{
+		ClusterName: cluster.GetName(),
+		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
+		SecurityCfg: sc,
+		MaxRequests: circuitBreakersFromCluster(cluster),
+	}
+
+	// Validate and set cluster type from the response.
+	switch {
+	case cluster.GetType() == v3clusterpb.Cluster_EDS:
+		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
+			return ClusterUpdate{}, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
+		}
+		ret.ClusterType = ClusterTypeEDS
+		ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName()
+		return ret, nil
+	case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS:
+		ret.ClusterType = ClusterTypeLogicalDNS
+		return ret, nil
+	case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate":
+		clusters := &v3aggregateclusterpb.ClusterConfig{}
+		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
+			return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
+		}
+		ret.ClusterType = ClusterTypeAggregate
+		ret.PrioritizedClusterNames = clusters.Clusters
+		return ret, nil
+	default:
+		return ClusterUpdate{}, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+	}
 }
 
 // securityConfigFromCluster extracts the relevant security configuration from

From 24d03d9f769106b3c96b4145244ce682999d3d88 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 27 Apr 2021 15:22:25 -0700
Subject: [PATCH 438/481] xds/priority: add ignore reresolution boolean to
 config (#4275)

---
 xds/internal/balancer/priority/balancer.go    |   6 +-
 .../balancer/priority/balancer_child.go       |  20 +-
 .../balancer/priority/balancer_test.go        | 266 ++++++++++++++----
 xds/internal/balancer/priority/config.go      |   3 +-
 xds/internal/balancer/priority/config_test.go |   9 +-
 .../balancer/priority/ignore_resolve_now.go   |  73 +++++
 .../priority/ignore_resolve_now_test.go       | 106 +++++++
 7 files changed, 417 insertions(+), 66 deletions(-)
 create mode 100644 xds/internal/balancer/priority/ignore_resolve_now.go
 create mode 100644 xds/internal/balancer/priority/ignore_resolve_now_test.go

diff --git a/xds/internal/balancer/priority/balancer.go b/xds/internal/balancer/priority/balancer.go
index d5c99b0b9146..e12a5068737f 100644
--- a/xds/internal/balancer/priority/balancer.go
+++ b/xds/internal/balancer/priority/balancer.go
@@ -131,7 +131,7 @@ func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) err
 			// the balancer isn't built, because this child can be a low
 			// priority. If necessary, it will be built when syncing priorities.
 			cb := newChildBalancer(name, b, bb)
-			cb.updateConfig(newSubConfig.Config.Config, resolver.State{
+			cb.updateConfig(newSubConfig, resolver.State{
 				Addresses:     addressesSplit[name],
 				ServiceConfig: s.ResolverState.ServiceConfig,
 				Attributes:    s.ResolverState.Attributes,
@@ -146,13 +146,13 @@ func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) err
 		// rebuild, rebuild will happen when syncing priorities.
 		if currentChild.bb.Name() != bb.Name() {
 			currentChild.stop()
-			currentChild.bb = bb
+			currentChild.updateBuilder(bb)
 		}
 
 		// Update config and address, but note that this doesn't send the
 		// updates to child balancer (the child balancer might not be built, if
 		// it's a low priority).
-		currentChild.updateConfig(newSubConfig.Config.Config, resolver.State{
+		currentChild.updateConfig(newSubConfig, resolver.State{
 			Addresses:     addressesSplit[name],
 			ServiceConfig: s.ResolverState.ServiceConfig,
 			Attributes:    s.ResolverState.Attributes,
diff --git a/xds/internal/balancer/priority/balancer_child.go b/xds/internal/balancer/priority/balancer_child.go
index d012ad4e4593..600705da01af 100644
--- a/xds/internal/balancer/priority/balancer_child.go
+++ b/xds/internal/balancer/priority/balancer_child.go
@@ -29,10 +29,11 @@ import (
 type childBalancer struct {
 	name   string
 	parent *priorityBalancer
-	bb     balancer.Builder
+	bb     *ignoreResolveNowBalancerBuilder
 
-	config serviceconfig.LoadBalancingConfig
-	rState resolver.State
+	ignoreReresolutionRequests bool
+	config                     serviceconfig.LoadBalancingConfig
+	rState                     resolver.State
 
 	started bool
 	state   balancer.State
@@ -44,7 +45,7 @@ func newChildBalancer(name string, parent *priorityBalancer, bb balancer.Builder
 	return &childBalancer{
 		name:    name,
 		parent:  parent,
-		bb:      bb,
+		bb:      newIgnoreResolveNowBalancerBuilder(bb, false),
 		started: false,
 		// Start with the connecting state and picker with re-pick error, so
 		// that when a priority switch causes this child picked before it's
@@ -56,10 +57,16 @@ func newChildBalancer(name string, parent *priorityBalancer, bb balancer.Builder
 	}
 }
 
+// updateBuilder updates builder for the child, but doesn't build.
+func (cb *childBalancer) updateBuilder(bb balancer.Builder) {
+	cb.bb = newIgnoreResolveNowBalancerBuilder(bb, cb.ignoreReresolutionRequests)
+}
+
 // updateConfig sets childBalancer's config and state, but doesn't send update to
 // the child balancer.
-func (cb *childBalancer) updateConfig(config serviceconfig.LoadBalancingConfig, rState resolver.State) {
-	cb.config = config
+func (cb *childBalancer) updateConfig(child *Child, rState resolver.State) {
+	cb.ignoreReresolutionRequests = child.IgnoreReresolutionRequests
+	cb.config = child.Config.Config
 	cb.rState = rState
 }
 
@@ -76,6 +83,7 @@ func (cb *childBalancer) start() {
 
 // sendUpdate sends the addresses and config to the child balancer.
 func (cb *childBalancer) sendUpdate() {
+	cb.bb.updateIgnoreResolveNow(cb.ignoreReresolutionRequests)
 	// TODO: return and aggregate the returned error in the parent.
 	err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{
 		ResolverState:  cb.rState,
diff --git a/xds/internal/balancer/priority/balancer_test.go b/xds/internal/balancer/priority/balancer_test.go
index d546216123d1..61e3dee94d9a 100644
--- a/xds/internal/balancer/priority/balancer_test.go
+++ b/xds/internal/balancer/priority/balancer_test.go
@@ -21,6 +21,7 @@
 package priority
 
 import (
+	"context"
 	"fmt"
 	"testing"
 	"time"
@@ -99,8 +100,8 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -136,9 +137,9 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1", "child-2"},
 		},
@@ -166,8 +167,8 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -206,8 +207,8 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -273,9 +274,9 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1", "child-2"},
 		},
@@ -332,8 +333,8 @@ func (s) TestPriority_SwitchPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -389,8 +390,8 @@ func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -484,8 +485,8 @@ func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -538,9 +539,9 @@ func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1", "child-2"},
 		},
@@ -596,9 +597,9 @@ func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-2": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1", "child-2"},
 		},
@@ -711,8 +712,8 @@ func (s) TestPriority_InitTimeout(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -781,8 +782,8 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -842,8 +843,8 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -886,7 +887,7 @@ func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -949,8 +950,8 @@ func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -984,8 +985,8 @@ func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -1047,7 +1048,7 @@ func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1091,8 +1092,8 @@ func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -1128,8 +1129,8 @@ func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-1", "child-0"},
 		},
@@ -1192,8 +1193,8 @@ func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -1244,8 +1245,8 @@ func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-1", "child-0"},
 		},
@@ -1292,8 +1293,8 @@ func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
-				"child-1": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0", "child-1"},
 		},
@@ -1342,7 +1343,7 @@ func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1399,7 +1400,7 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1458,7 +1459,7 @@ func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1502,7 +1503,7 @@ func (s) TestPriority_ChildPolicyChange(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1537,7 +1538,7 @@ func (s) TestPriority_ChildPolicyChange(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: testRRBalancerName}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: testRRBalancerName}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1602,7 +1603,7 @@ func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) {
 		},
 		BalancerConfig: &LBConfig{
 			Children: map[string]*Child{
-				"child-0": {&internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}},
+				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}},
 			},
 			Priorities: []string{"child-0"},
 		},
@@ -1618,3 +1619,164 @@ func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) {
 		}
 	}
 }
+
+// When the child policy's configured to ignore reresolution requests, the
+// ResolveNow() calls from this child should be all ignored.
+func (s) TestPriority_IgnoreReresolutionRequest(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(Name)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// One children, with priorities [0], with one backend, reresolution is
+	// ignored.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
+				"child-0": {
+					Config:                     &internalserviceconfig.BalancerConfig{Name: resolveNowBalancerName},
+					IgnoreReresolutionRequests: true,
+				},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	// This is the balancer.ClientConn that the inner resolverNowBalancer is
+	// built with.
+	balancerCCI, err := resolveNowBalancerCCCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout waiting for ClientConn from balancer builder")
+	}
+	balancerCC := balancerCCI.(balancer.ClientConn)
+
+	// Since IgnoreReresolutionRequests was set to true, all ResolveNow() calls
+	// should be ignored.
+	for i := 0; i < 5; i++ {
+		balancerCC.ResolveNow(resolver.ResolveNowOptions{})
+	}
+	select {
+	case <-cc.ResolveNowCh:
+		t.Fatalf("got unexpected ResolveNow() call")
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// Send another update to set IgnoreReresolutionRequests to false.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+			},
+		},
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
+				"child-0": {
+					Config:                     &internalserviceconfig.BalancerConfig{Name: resolveNowBalancerName},
+					IgnoreReresolutionRequests: false,
+				},
+			},
+			Priorities: []string{"child-0"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Call ResolveNow() on the CC, it should be forwarded.
+	balancerCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-time.After(time.Second):
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+
+}
+
+// When the child policy's configured to ignore reresolution requests, the
+// ResolveNow() calls from this child should be all ignored, from the other
+// children are forwarded.
+func (s) TestPriority_IgnoreReresolutionRequestTwoChildren(t *testing.T) {
+	cc := testutils.NewTestClientConn(t)
+	bb := balancer.Get(Name)
+	pb := bb.Build(cc, balancer.BuildOptions{})
+	defer pb.Close()
+
+	// One children, with priorities [0, 1], each with one backend.
+	// Reresolution is ignored for p0.
+	if err := pb.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses: []resolver.Address{
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
+				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
+			},
+		},
+		BalancerConfig: &LBConfig{
+			Children: map[string]*Child{
+				"child-0": {
+					Config:                     &internalserviceconfig.BalancerConfig{Name: resolveNowBalancerName},
+					IgnoreReresolutionRequests: true,
+				},
+				"child-1": {
+					Config: &internalserviceconfig.BalancerConfig{Name: resolveNowBalancerName},
+				},
+			},
+			Priorities: []string{"child-0", "child-1"},
+		},
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	// This is the balancer.ClientConn from p0.
+	balancerCCI0, err := resolveNowBalancerCCCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout waiting for ClientConn from balancer builder 0")
+	}
+	balancerCC0 := balancerCCI0.(balancer.ClientConn)
+
+	// Set p0 to transient failure, p1 will be started.
+	addrs0 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	sc0 := <-cc.NewSubConnCh
+	pb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
+
+	// This is the balancer.ClientConn from p1.
+	ctx1, cancel1 := context.WithTimeout(context.Background(), time.Second)
+	defer cancel1()
+	balancerCCI1, err := resolveNowBalancerCCCh.Receive(ctx1)
+	if err != nil {
+		t.Fatalf("timeout waiting for ClientConn from balancer builder 1")
+	}
+	balancerCC1 := balancerCCI1.(balancer.ClientConn)
+
+	// Since IgnoreReresolutionRequests was set to true for p0, ResolveNow()
+	// from p0 should all be ignored.
+	for i := 0; i < 5; i++ {
+		balancerCC0.ResolveNow(resolver.ResolveNowOptions{})
+	}
+	select {
+	case <-cc.ResolveNowCh:
+		t.Fatalf("got unexpected ResolveNow() call")
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// But IgnoreReresolutionRequests was false for p1, ResolveNow() from p1
+	// should be forwarded.
+	balancerCC1.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-time.After(time.Second):
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+}
diff --git a/xds/internal/balancer/priority/config.go b/xds/internal/balancer/priority/config.go
index 7704f21d13bd..c9cb16e323f0 100644
--- a/xds/internal/balancer/priority/config.go
+++ b/xds/internal/balancer/priority/config.go
@@ -28,7 +28,8 @@ import (
 
 // Child is a child of priority balancer.
 type Child struct {
-	Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"`
+	Config                     *internalserviceconfig.BalancerConfig `json:"config,omitempty"`
+	IgnoreReresolutionRequests bool
 }
 
 // LBConfig represents priority balancer's config.
diff --git a/xds/internal/balancer/priority/config_test.go b/xds/internal/balancer/priority/config_test.go
index f3a09fe3a32e..42498bfa2b74 100644
--- a/xds/internal/balancer/priority/config_test.go
+++ b/xds/internal/balancer/priority/config_test.go
@@ -65,7 +65,7 @@ func TestParseConfig(t *testing.T) {
 			js: `{
   "priorities": ["child-1", "child-2", "child-3"],
   "children": {
-    "child-1": {"config": [{"round_robin":{}}]},
+    "child-1": {"config": [{"round_robin":{}}], "ignoreReresolutionRequests": true},
     "child-2": {"config": [{"round_robin":{}}]},
     "child-3": {"config": [{"round_robin":{}}]}
   }
@@ -74,17 +74,18 @@ func TestParseConfig(t *testing.T) {
 			want: &LBConfig{
 				Children: map[string]*Child{
 					"child-1": {
-						&internalserviceconfig.BalancerConfig{
+						Config: &internalserviceconfig.BalancerConfig{
 							Name: roundrobin.Name,
 						},
+						IgnoreReresolutionRequests: true,
 					},
 					"child-2": {
-						&internalserviceconfig.BalancerConfig{
+						Config: &internalserviceconfig.BalancerConfig{
 							Name: roundrobin.Name,
 						},
 					},
 					"child-3": {
-						&internalserviceconfig.BalancerConfig{
+						Config: &internalserviceconfig.BalancerConfig{
 							Name: roundrobin.Name,
 						},
 					},
diff --git a/xds/internal/balancer/priority/ignore_resolve_now.go b/xds/internal/balancer/priority/ignore_resolve_now.go
new file mode 100644
index 000000000000..9a9f4777269a
--- /dev/null
+++ b/xds/internal/balancer/priority/ignore_resolve_now.go
@@ -0,0 +1,73 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"sync/atomic"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/resolver"
+)
+
+type ignoreResolveNowBalancerBuilder struct {
+	balancer.Builder
+	ignoreResolveNow *uint32
+}
+
+// If `ignore` is true, all `ResolveNow()` from the balancer built from this
+// builder will be ignored.
+//
+// `ignore` can be updated later by `updateIgnoreResolveNow`, and the update
+// will be propagated to all the old and new balancers built with this.
+func newIgnoreResolveNowBalancerBuilder(bb balancer.Builder, ignore bool) *ignoreResolveNowBalancerBuilder {
+	ret := &ignoreResolveNowBalancerBuilder{
+		Builder:          bb,
+		ignoreResolveNow: new(uint32),
+	}
+	ret.updateIgnoreResolveNow(ignore)
+	return ret
+}
+
+func (irnbb *ignoreResolveNowBalancerBuilder) updateIgnoreResolveNow(b bool) {
+	if b {
+		atomic.StoreUint32(irnbb.ignoreResolveNow, 1)
+		return
+	}
+	atomic.StoreUint32(irnbb.ignoreResolveNow, 0)
+
+}
+
+func (irnbb *ignoreResolveNowBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	return irnbb.Builder.Build(&ignoreResolveNowClientConn{
+		ClientConn:       cc,
+		ignoreResolveNow: irnbb.ignoreResolveNow,
+	}, opts)
+}
+
+type ignoreResolveNowClientConn struct {
+	balancer.ClientConn
+	ignoreResolveNow *uint32
+}
+
+func (i ignoreResolveNowClientConn) ResolveNow(o resolver.ResolveNowOptions) {
+	if atomic.LoadUint32(i.ignoreResolveNow) != 0 {
+		return
+	}
+	i.ClientConn.ResolveNow(o)
+}
diff --git a/xds/internal/balancer/priority/ignore_resolve_now_test.go b/xds/internal/balancer/priority/ignore_resolve_now_test.go
new file mode 100644
index 000000000000..452753de6c7b
--- /dev/null
+++ b/xds/internal/balancer/priority/ignore_resolve_now_test.go
@@ -0,0 +1,106 @@
+// +build go1.12
+
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	grpctestutils "google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/internal/testutils"
+)
+
+const resolveNowBalancerName = "test-resolve-now-balancer"
+
+var resolveNowBalancerCCCh = grpctestutils.NewChannel()
+
+type resolveNowBalancerBuilder struct {
+	balancer.Builder
+}
+
+func (r *resolveNowBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	resolveNowBalancerCCCh.Send(cc)
+	return r.Builder.Build(cc, opts)
+}
+
+func (r *resolveNowBalancerBuilder) Name() string {
+	return resolveNowBalancerName
+}
+
+func init() {
+	balancer.Register(&resolveNowBalancerBuilder{
+		Builder: balancer.Get(roundrobin.Name),
+	})
+}
+
+func (s) TestIgnoreResolveNowBalancerBuilder(t *testing.T) {
+	resolveNowBB := balancer.Get(resolveNowBalancerName)
+	// Create a build wrapper, but will not ignore ResolveNow().
+	ignoreResolveNowBB := newIgnoreResolveNowBalancerBuilder(resolveNowBB, false)
+
+	cc := testutils.NewTestClientConn(t)
+	tb := ignoreResolveNowBB.Build(cc, balancer.BuildOptions{})
+	defer tb.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	// This is the balancer.ClientConn that the inner resolverNowBalancer is
+	// built with.
+	balancerCCI, err := resolveNowBalancerCCCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout waiting for ClientConn from balancer builder")
+	}
+	balancerCC := balancerCCI.(balancer.ClientConn)
+
+	// Call ResolveNow() on the CC, it should be forwarded.
+	balancerCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-time.After(time.Second):
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+
+	// Update ignoreResolveNow to true, call ResolveNow() on the CC, they should
+	// all be ignored.
+	ignoreResolveNowBB.updateIgnoreResolveNow(true)
+	for i := 0; i < 5; i++ {
+		balancerCC.ResolveNow(resolver.ResolveNowOptions{})
+	}
+	select {
+	case <-cc.ResolveNowCh:
+		t.Fatalf("got unexpected ResolveNow() call")
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// Update ignoreResolveNow to false, new ResolveNow() calls should be
+	// forwarded.
+	ignoreResolveNowBB.updateIgnoreResolveNow(false)
+	balancerCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-time.After(time.Second):
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+}

From b602d17e459c0e4d64e24b6d07875f58d5f40f0e Mon Sep 17 00:00:00 2001
From: irfan sharif <irfanmahmoudsharif@gmail.com>
Date: Wed, 28 Apr 2021 13:05:50 -0400
Subject: [PATCH 439/481] metadata: reduce memory footprint in
 FromOutgoingContext (#4360)

When Looking at memory profiles for cockroachdb/cockroach, we observed
that the intermediate metadata.MD array constructed to iterate over
appended metadata escaped to the heap. Fortunately, this is easily
rectifiable.

  go build -gcflags '-m' google.golang.org/grpc/metadata
  ...
  google.golang.org/grpc/metadata/metadata.go:198:13: make([]MD, 0, len(raw.added) + 1) escapes to heap
---
 metadata/metadata.go | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/metadata/metadata.go b/metadata/metadata.go
index cf6d1b94781c..e4cbea917498 100644
--- a/metadata/metadata.go
+++ b/metadata/metadata.go
@@ -75,13 +75,9 @@ func Pairs(kv ...string) MD {
 		panic(fmt.Sprintf("metadata: Pairs got the odd number of input pairs for metadata: %d", len(kv)))
 	}
 	md := MD{}
-	var key string
-	for i, s := range kv {
-		if i%2 == 0 {
-			key = strings.ToLower(s)
-			continue
-		}
-		md[key] = append(md[key], s)
+	for i := 0; i < len(kv); i += 2 {
+		key := strings.ToLower(kv[i])
+		md[key] = append(md[key], kv[i+1])
 	}
 	return md
 }
@@ -195,12 +191,18 @@ func FromOutgoingContext(ctx context.Context) (MD, bool) {
 		return nil, false
 	}
 
-	mds := make([]MD, 0, len(raw.added)+1)
-	mds = append(mds, raw.md)
-	for _, vv := range raw.added {
-		mds = append(mds, Pairs(vv...))
+	out := raw.md.Copy()
+	for _, added := range raw.added {
+		if len(added)%2 == 1 {
+			panic(fmt.Sprintf("metadata: FromOutgoingContext got an odd number of input pairs for metadata: %d", len(added)))
+		}
+
+		for i := 0; i < len(added); i += 2 {
+			key := strings.ToLower(added[i])
+			out[key] = append(out[key], added[i+1])
+		}
 	}
-	return Join(mds...), ok
+	return out, ok
 }
 
 type rawMD struct {

From 91d8f0c916d76f2a5aac9e846cd7ffcb838db769 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 28 Apr 2021 18:11:45 -0700
Subject: [PATCH 440/481] serviceconfig: support marshalling BalancerConfig to
 JSON (#4368)

---
 internal/serviceconfig/serviceconfig.go      | 16 ++++++
 internal/serviceconfig/serviceconfig_test.go | 53 ++++++++++++++++++--
 2 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go
index bd4b8875f1a7..c0634d152c2e 100644
--- a/internal/serviceconfig/serviceconfig.go
+++ b/internal/serviceconfig/serviceconfig.go
@@ -46,6 +46,22 @@ type BalancerConfig struct {
 
 type intermediateBalancerConfig []map[string]json.RawMessage
 
+// MarshalJSON implements the json.Marshaler interface.
+//
+// It marshals the balancer and config into a length-1 slice
+// ([]map[string]config).
+func (bc *BalancerConfig) MarshalJSON() ([]byte, error) {
+	if bc.Config == nil {
+		// If config is nil, return empty config `{}`.
+		return []byte(fmt.Sprintf(`[{%q: %v}]`, bc.Name, "{}")), nil
+	}
+	c, err := json.Marshal(bc.Config)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(fmt.Sprintf(`[{%q: %s}]`, bc.Name, c)), nil
+}
+
 // UnmarshalJSON implements the json.Unmarshaler interface.
 //
 // ServiceConfig contains a list of loadBalancingConfigs, each with a name and
diff --git a/internal/serviceconfig/serviceconfig_test.go b/internal/serviceconfig/serviceconfig_test.go
index b8abaae027ef..770ee2efeb83 100644
--- a/internal/serviceconfig/serviceconfig_test.go
+++ b/internal/serviceconfig/serviceconfig_test.go
@@ -29,16 +29,18 @@ import (
 )
 
 type testBalancerConfigType struct {
-	externalserviceconfig.LoadBalancingConfig
+	externalserviceconfig.LoadBalancingConfig `json:"-"`
+
+	Check bool `json:"check"`
 }
 
-var testBalancerConfig = testBalancerConfigType{}
+var testBalancerConfig = testBalancerConfigType{Check: true}
 
 const (
 	testBalancerBuilderName          = "test-bb"
 	testBalancerBuilderNotParserName = "test-bb-not-parser"
 
-	testBalancerConfigJSON = `{"test-balancer-config":"true"}`
+	testBalancerConfigJSON = `{"check":true}`
 )
 
 type testBalancerBuilder struct {
@@ -133,3 +135,48 @@ func TestBalancerConfigUnmarshalJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestBalancerConfigMarshalJSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		bc       BalancerConfig
+		wantJSON string
+	}{
+		{
+			name: "OK",
+			bc: BalancerConfig{
+				Name:   testBalancerBuilderName,
+				Config: testBalancerConfig,
+			},
+			wantJSON: fmt.Sprintf(`[{"test-bb": {"check":true}}]`),
+		},
+		{
+			name: "OK config is nil",
+			bc: BalancerConfig{
+				Name:   testBalancerBuilderNotParserName,
+				Config: nil, // nil should be marshalled to an empty config "{}".
+			},
+			wantJSON: fmt.Sprintf(`[{"test-bb-not-parser": {}}]`),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b, err := tt.bc.MarshalJSON()
+			if err != nil {
+				t.Fatalf("failed to marshal: %v", err)
+			}
+
+			if str := string(b); str != tt.wantJSON {
+				t.Fatalf("got str %q, want %q", str, tt.wantJSON)
+			}
+
+			var bc BalancerConfig
+			if err := bc.UnmarshalJSON(b); err != nil {
+				t.Errorf("failed to mnmarshal: %v", err)
+			}
+			if !cmp.Equal(bc, tt.bc) {
+				t.Errorf("diff: %v", cmp.Diff(bc, tt.bc))
+			}
+		})
+	}
+}

From c3b66015bd51d33d3e0a75ea5086defcb9d05e64 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 29 Apr 2021 11:56:50 -0700
Subject: [PATCH 441/481] xds/circuit_breaking: use cluster name as key, not
 EDS service name (#4372)

---
 xds/internal/balancer/edsbalancer/eds.go      | 2 +-
 xds/internal/balancer/edsbalancer/eds_test.go | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 1b5191aa1037..09e8b28748a3 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -219,7 +219,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			x.logger.Warningf("failed to update xDS client: %v", err)
 		}
 
-		x.edsImpl.updateServiceRequestsConfig(cfg.EDSServiceName, cfg.MaxConcurrentRequests)
+		x.edsImpl.updateServiceRequestsConfig(cfg.ClusterName, cfg.MaxConcurrentRequests)
 
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index b986654bc409..37ecb1acfc69 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -354,6 +354,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			ChildPolicy:    lbCfgA,
+			ClusterName:    testEDSClusterName,
 			EDSServiceName: testServiceName,
 		},
 	}); err != nil {
@@ -367,7 +368,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgA); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
+	if err := edsLB.waitForCounterUpdate(ctx, testEDSClusterName); err != nil {
 		t.Fatal(err)
 	}
 	if err := edsLB.waitForCountMaxUpdate(ctx, nil); err != nil {
@@ -382,6 +383,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			ChildPolicy:           lbCfgB,
+			ClusterName:           testEDSClusterName,
 			EDSServiceName:        testServiceName,
 			MaxConcurrentRequests: &testCountMax,
 		},
@@ -391,7 +393,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgB); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
+	if err := edsLB.waitForCounterUpdate(ctx, testEDSClusterName); err != nil {
 		// Counter is updated even though the service name didn't change. The
 		// eds_impl will compare the service names, and skip if it didn't change.
 		t.Fatal(err)
@@ -670,7 +672,7 @@ func (s) TestCounterUpdate(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			EDSServiceName:        "foobar-1",
+			ClusterName:           "foobar-1",
 			MaxConcurrentRequests: &testCountMax,
 		},
 	}); err != nil {

From aa3ef8fb8ff6c92134743e780cf659eaa7eeccbc Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 29 Apr 2021 12:17:56 -0700
Subject: [PATCH 442/481] internal: regenerate proto (#4373)

---
 .../internal/proto/grpc_lookup_v1/rls.pb.go   | 151 +++++++++++++-----
 1 file changed, 111 insertions(+), 40 deletions(-)

diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
index 7741e6649180..9de23aae645c 100644
--- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
+++ b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go
@@ -39,6 +39,56 @@ const (
 // of the legacy proto package is being used.
 const _ = proto.ProtoPackageIsVersion4
 
+// Possible reasons for making a request.
+type RouteLookupRequest_Reason int32
+
+const (
+	RouteLookupRequest_REASON_UNKNOWN RouteLookupRequest_Reason = 0 // Unused
+	RouteLookupRequest_REASON_MISS    RouteLookupRequest_Reason = 1 // No data available in local cache
+	RouteLookupRequest_REASON_STALE   RouteLookupRequest_Reason = 2 // Data in local cache is stale
+)
+
+// Enum value maps for RouteLookupRequest_Reason.
+var (
+	RouteLookupRequest_Reason_name = map[int32]string{
+		0: "REASON_UNKNOWN",
+		1: "REASON_MISS",
+		2: "REASON_STALE",
+	}
+	RouteLookupRequest_Reason_value = map[string]int32{
+		"REASON_UNKNOWN": 0,
+		"REASON_MISS":    1,
+		"REASON_STALE":   2,
+	}
+)
+
+func (x RouteLookupRequest_Reason) Enum() *RouteLookupRequest_Reason {
+	p := new(RouteLookupRequest_Reason)
+	*p = x
+	return p
+}
+
+func (x RouteLookupRequest_Reason) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (RouteLookupRequest_Reason) Descriptor() protoreflect.EnumDescriptor {
+	return file_grpc_lookup_v1_rls_proto_enumTypes[0].Descriptor()
+}
+
+func (RouteLookupRequest_Reason) Type() protoreflect.EnumType {
+	return &file_grpc_lookup_v1_rls_proto_enumTypes[0]
+}
+
+func (x RouteLookupRequest_Reason) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use RouteLookupRequest_Reason.Descriptor instead.
+func (RouteLookupRequest_Reason) EnumDescriptor() ([]byte, []int) {
+	return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0, 0}
+}
+
 type RouteLookupRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -59,6 +109,8 @@ type RouteLookupRequest struct {
 	// Target type allows the client to specify what kind of target format it
 	// would like from RLS to allow it to find the regional server, e.g. "grpc".
 	TargetType string `protobuf:"bytes,3,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"`
+	// Reason for making this request.
+	Reason RouteLookupRequest_Reason `protobuf:"varint,5,opt,name=reason,proto3,enum=grpc.lookup.v1.RouteLookupRequest_Reason" json:"reason,omitempty"`
 	// Map of key values extracted via key builders for the gRPC or HTTP request.
 	KeyMap map[string]string `protobuf:"bytes,4,rep,name=key_map,json=keyMap,proto3" json:"key_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
@@ -118,6 +170,13 @@ func (x *RouteLookupRequest) GetTargetType() string {
 	return ""
 }
 
+func (x *RouteLookupRequest) GetReason() RouteLookupRequest_Reason {
+	if x != nil {
+		return x.Reason
+	}
+	return RouteLookupRequest_REASON_UNKNOWN
+}
+
 func (x *RouteLookupRequest) GetKeyMap() map[string]string {
 	if x != nil {
 		return x.KeyMap
@@ -191,41 +250,49 @@ var File_grpc_lookup_v1_rls_proto protoreflect.FileDescriptor
 var file_grpc_lookup_v1_rls_proto_rawDesc = []byte{
 	0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x76, 0x31,
 	0x2f, 0x72, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63,
-	0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x22, 0xed, 0x01, 0x0a, 0x12, 0x52,
+	0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x22, 0xf1, 0x02, 0x0a, 0x12, 0x52,
 	0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
 	0x74, 0x12, 0x1a, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
 	0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x16, 0x0a,
 	0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52,
 	0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f,
 	0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67,
-	0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x47, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x5f, 0x6d, 0x61,
-	0x70, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
-	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f,
-	0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4b, 0x65, 0x79, 0x4d,
-	0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x1a,
-	0x39, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5e, 0x0a, 0x13, 0x52, 0x6f,
-	0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68,
-	0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01,
-	0x10, 0x02, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x32, 0x6e, 0x0a, 0x12, 0x52, 0x6f,
-	0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
-	0x12, 0x58, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x12,
-	0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31,
-	0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75,
+	0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x41, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f,
+	0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x61, 0x73, 0x6f,
+	0x6e, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x47, 0x0a, 0x07, 0x6b, 0x65, 0x79,
+	0x5f, 0x6d, 0x61, 0x70, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x67, 0x72, 0x70,
+	0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74,
+	0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4b,
+	0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6b, 0x65, 0x79, 0x4d,
+	0x61, 0x70, 0x1a, 0x39, 0x0a, 0x0b, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
+	0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3f, 0x0a,
+	0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x41, 0x53, 0x4f,
+	0x4e, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x52,
+	0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x49, 0x53, 0x53, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c,
+	0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x4c, 0x45, 0x10, 0x02, 0x22, 0x5e,
+	0x0a, 0x13, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73,
+	0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12,
+	0x1f, 0x0a, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61,
+	0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x32, 0x6e,
+	0x0a, 0x12, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75,
 	0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4d, 0x0a, 0x11, 0x69, 0x6f,
-	0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42,
-	0x08, 0x52, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f,
-	0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67,
-	0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f,
-	0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f,
+	0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4d,
+	0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
+	0x2e, 0x76, 0x31, 0x42, 0x08, 0x52, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a,
+	0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f,
+	0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67,
+	0x72, 0x70, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -240,21 +307,24 @@ func file_grpc_lookup_v1_rls_proto_rawDescGZIP() []byte {
 	return file_grpc_lookup_v1_rls_proto_rawDescData
 }
 
+var file_grpc_lookup_v1_rls_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
 var file_grpc_lookup_v1_rls_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
 var file_grpc_lookup_v1_rls_proto_goTypes = []interface{}{
-	(*RouteLookupRequest)(nil),  // 0: grpc.lookup.v1.RouteLookupRequest
-	(*RouteLookupResponse)(nil), // 1: grpc.lookup.v1.RouteLookupResponse
-	nil,                         // 2: grpc.lookup.v1.RouteLookupRequest.KeyMapEntry
+	(RouteLookupRequest_Reason)(0), // 0: grpc.lookup.v1.RouteLookupRequest.Reason
+	(*RouteLookupRequest)(nil),     // 1: grpc.lookup.v1.RouteLookupRequest
+	(*RouteLookupResponse)(nil),    // 2: grpc.lookup.v1.RouteLookupResponse
+	nil,                            // 3: grpc.lookup.v1.RouteLookupRequest.KeyMapEntry
 }
 var file_grpc_lookup_v1_rls_proto_depIdxs = []int32{
-	2, // 0: grpc.lookup.v1.RouteLookupRequest.key_map:type_name -> grpc.lookup.v1.RouteLookupRequest.KeyMapEntry
-	0, // 1: grpc.lookup.v1.RouteLookupService.RouteLookup:input_type -> grpc.lookup.v1.RouteLookupRequest
-	1, // 2: grpc.lookup.v1.RouteLookupService.RouteLookup:output_type -> grpc.lookup.v1.RouteLookupResponse
-	2, // [2:3] is the sub-list for method output_type
-	1, // [1:2] is the sub-list for method input_type
-	1, // [1:1] is the sub-list for extension type_name
-	1, // [1:1] is the sub-list for extension extendee
-	0, // [0:1] is the sub-list for field type_name
+	0, // 0: grpc.lookup.v1.RouteLookupRequest.reason:type_name -> grpc.lookup.v1.RouteLookupRequest.Reason
+	3, // 1: grpc.lookup.v1.RouteLookupRequest.key_map:type_name -> grpc.lookup.v1.RouteLookupRequest.KeyMapEntry
+	1, // 2: grpc.lookup.v1.RouteLookupService.RouteLookup:input_type -> grpc.lookup.v1.RouteLookupRequest
+	2, // 3: grpc.lookup.v1.RouteLookupService.RouteLookup:output_type -> grpc.lookup.v1.RouteLookupResponse
+	3, // [3:4] is the sub-list for method output_type
+	2, // [2:3] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
 }
 
 func init() { file_grpc_lookup_v1_rls_proto_init() }
@@ -293,13 +363,14 @@ func file_grpc_lookup_v1_rls_proto_init() {
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_grpc_lookup_v1_rls_proto_rawDesc,
-			NumEnums:      0,
+			NumEnums:      1,
 			NumMessages:   3,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
 		GoTypes:           file_grpc_lookup_v1_rls_proto_goTypes,
 		DependencyIndexes: file_grpc_lookup_v1_rls_proto_depIdxs,
+		EnumInfos:         file_grpc_lookup_v1_rls_proto_enumTypes,
 		MessageInfos:      file_grpc_lookup_v1_rls_proto_msgTypes,
 	}.Build()
 	File_grpc_lookup_v1_rls_proto = out.File

From 28078834f35b944281662807d8ec071645c37307 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Thu, 29 Apr 2021 21:44:26 -0700
Subject: [PATCH 443/481] grpc: call balancer.Close() before returning from
 ccBalancerWrapper.close() (#4364)

---
 balancer_conn_wrappers.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go
index 4cc7f9159b16..38b48fcdc5b4 100644
--- a/balancer_conn_wrappers.go
+++ b/balancer_conn_wrappers.go
@@ -69,10 +69,10 @@ func (ccb *ccBalancerWrapper) watcher() {
 		select {
 		case t := <-ccb.scBuffer.Get():
 			ccb.scBuffer.Load()
+			ccb.balancerMu.Lock()
 			if ccb.done.HasFired() {
 				break
 			}
-			ccb.balancerMu.Lock()
 			su := t.(*scStateUpdate)
 			ccb.balancer.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state, ConnectionError: su.err})
 			ccb.balancerMu.Unlock()
@@ -80,7 +80,6 @@ func (ccb *ccBalancerWrapper) watcher() {
 		}
 
 		if ccb.done.HasFired() {
-			ccb.balancer.Close()
 			ccb.mu.Lock()
 			scs := ccb.subConns
 			ccb.subConns = nil
@@ -95,6 +94,9 @@ func (ccb *ccBalancerWrapper) watcher() {
 }
 
 func (ccb *ccBalancerWrapper) close() {
+	ccb.balancerMu.Lock()
+	defer ccb.balancerMu.Unlock()
+	ccb.balancer.Close()
 	ccb.done.Fire()
 }
 
@@ -119,13 +121,19 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co
 func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error {
 	ccb.balancerMu.Lock()
 	defer ccb.balancerMu.Unlock()
+	if ccb.done.HasFired() {
+		return nil
+	}
 	return ccb.balancer.UpdateClientConnState(*ccs)
 }
 
 func (ccb *ccBalancerWrapper) resolverError(err error) {
 	ccb.balancerMu.Lock()
+	defer ccb.balancerMu.Unlock()
+	if ccb.done.HasFired() {
+		return
+	}
 	ccb.balancer.ResolverError(err)
-	ccb.balancerMu.Unlock()
 }
 
 func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {

From b418de839e738968aa8f845584efd0d34da4bae8 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 30 Apr 2021 11:53:31 -0700
Subject: [PATCH 444/481] xds/eds: restart EDS watch after previous was
 canceled (#4378)

---
 xds/internal/balancer/edsbalancer/eds.go      |  5 +++-
 xds/internal/balancer/edsbalancer/eds_test.go | 23 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 09e8b28748a3..a5f653f737aa 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -314,10 +314,12 @@ func (x *edsBalancer) cancelWatch() {
 	x.loadReportServer = nil
 	if x.cancelLoadReport != nil {
 		x.cancelLoadReport()
+		x.cancelLoadReport = nil
 	}
-	x.edsServiceName = ""
 	if x.cancelEndpointsWatch != nil {
+		x.edsToWatch = ""
 		x.cancelEndpointsWatch()
+		x.cancelEndpointsWatch = nil
 	}
 }
 
@@ -331,6 +333,7 @@ func (x *edsBalancer) startLoadReport(loadReportServer *string) *load.Store {
 	x.loadReportServer = loadReportServer
 	if x.cancelLoadReport != nil {
 		x.cancelLoadReport()
+		x.cancelLoadReport = nil
 	}
 	if loadReportServer == nil {
 		return nil
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 37ecb1acfc69..3fe66098973c 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -512,6 +512,18 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("eds impl expecting empty update, got %v", err)
 	}
+
+	// An update with the same service name should not trigger a new watch.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := xdsC.WaitForWatchEDS(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("got unexpected new EDS watch")
+	}
 }
 
 // TestErrorFromResolver verifies that resolver errors are handled correctly.
@@ -577,6 +589,17 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
 	}
+
+	// An update with the same service name should trigger a new watch, because
+	// the previous watch was canceled.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
+		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
+	}
 }
 
 // Given a list of resource names, verifies that EDS requests for the same are

From ebd6aba6754d073a696e5727158cd0c917ce1019 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Mon, 3 May 2021 15:16:49 -0700
Subject: [PATCH 445/481] Revert "xds/cds: add separate fields for cluster name
 and eds service name" (#4382)

This reverts PRs #4352 (and two follow up fixes #4372 #4378).
Because the xds interop tests were flaky.

Revert before the branch cut.
---
 .../balancer/cdsbalancer/cdsbalancer.go       |  3 +-
 .../cdsbalancer/cdsbalancer_security_test.go  | 14 +--
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 16 ++--
 xds/internal/balancer/edsbalancer/config.go   |  8 +-
 xds/internal/balancer/edsbalancer/eds.go      | 63 ++++---------
 .../balancer/edsbalancer/eds_impl_test.go     | 11 ++-
 xds/internal/balancer/edsbalancer/eds_test.go | 53 ++---------
 .../edsbalancer/load_store_wrapper.go         | 88 +++++++++++++++++++
 xds/internal/client/cds_test.go               | 75 +++++-----------
 xds/internal/client/client.go                 | 15 ++--
 xds/internal/client/client_test.go            |  4 +-
 xds/internal/client/v2/cds_test.go            |  4 +-
 xds/internal/client/watchers_cluster_test.go  | 16 ++--
 xds/internal/client/xds.go                    | 75 +++++++++-------
 14 files changed, 222 insertions(+), 223 deletions(-)
 create mode 100644 xds/internal/balancer/edsbalancer/load_store_wrapper.go

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index c97e92bcd02f..b991981c14c0 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -340,8 +340,7 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
 	}
 	lbCfg := &edsbalancer.EDSConfig{
-		ClusterName:           update.cds.ClusterName,
-		EDSServiceName:        update.cds.EDSServiceName,
+		EDSServiceName:        update.cds.ServiceName,
 		MaxConcurrentRequests: update.cds.MaxRequests,
 	}
 	if update.cds.EnableLRS {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 18a2298c5258..5c746cfa163c 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -60,7 +60,7 @@ var (
 	fpb1, fpb2                   *fakeProviderBuilder
 	bootstrapConfig              *bootstrap.Config
 	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
-		ClusterName: serviceName,
+		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default1",
 			IdentityInstanceName:   "default2",
@@ -68,7 +68,7 @@ var (
 		},
 	}
 	cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{
-		ClusterName: serviceName,
+		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName: "not-default",
 		},
@@ -256,7 +256,7 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -312,7 +312,7 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup. No security config is
 	// passed to the CDS balancer as part of this update.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -572,7 +572,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	// an update which contains bad security config. So, we expect the CDS
 	// balancer to forward this error to the EDS balancer and eventually the
 	// channel needs to be put in a bad state.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -678,7 +678,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{
-		ClusterName: serviceName,
+		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default1",
 			SubjectAltNameMatchers: testSANMatchers,
@@ -703,7 +703,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 
 	// Push another update with a new security configuration.
 	cdsUpdate = xdsclient.ClusterUpdate{
-		ClusterName: serviceName,
+		ServiceName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default2",
 			SubjectAltNameMatchers: testSANMatchers,
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 13b17306df8b..4476a1532d05 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -197,7 +197,7 @@ func cdsCCS(cluster string) balancer.ClientConnState {
 // cdsBalancer to the edsBalancer.
 func edsCCS(service string, countMax *uint32, enableLRS bool) balancer.ClientConnState {
 	lbCfg := &edsbalancer.EDSConfig{
-		ClusterName:           service,
+		EDSServiceName:        service,
 		MaxConcurrentRequests: countMax,
 	}
 	if enableLRS {
@@ -354,12 +354,12 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 	}{
 		{
 			name:      "happy-case-with-lrs",
-			cdsUpdate: xdsclient.ClusterUpdate{ClusterName: serviceName, EnableLRS: true},
+			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
 			wantCCS:   edsCCS(serviceName, nil, true),
 		},
 		{
 			name:      "happy-case-without-lrs",
-			cdsUpdate: xdsclient.ClusterUpdate{ClusterName: serviceName},
+			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
 			wantCCS:   edsCCS(serviceName, nil, false),
 		},
 	}
@@ -427,7 +427,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -512,7 +512,7 @@ func (s) TestResolverError(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -561,7 +561,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -596,7 +596,7 @@ func (s) TestCircuitBreaking(t *testing.T) {
 	// will trigger the watch handler on the CDS balancer, which will update
 	// the service's counter with the new max requests.
 	var maxRequests uint32 = 1
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName, MaxRequests: &maxRequests}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName, MaxRequests: &maxRequests}
 	wantCCS := edsCCS(serviceName, &maxRequests, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -629,7 +629,7 @@ func (s) TestClose(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
diff --git a/xds/internal/balancer/edsbalancer/config.go b/xds/internal/balancer/edsbalancer/config.go
index d1583e2bf276..11c1338c81f7 100644
--- a/xds/internal/balancer/edsbalancer/config.go
+++ b/xds/internal/balancer/edsbalancer/config.go
@@ -35,10 +35,8 @@ type EDSConfig struct {
 	// FallBackPolicy represents the load balancing config for the
 	// fallback.
 	FallBackPolicy *loadBalancingConfig
-	// ClusterName is the cluster name.
-	ClusterName string
-	// EDSServiceName is the name to use in EDS query. If not set, use
-	// ClusterName.
+	// Name to use in EDS query.  If not present, defaults to the server
+	// name from the target URI.
 	EDSServiceName string
 	// MaxConcurrentRequests is the max number of concurrent request allowed for
 	// this service. If unset, default value 1024 is used.
@@ -61,7 +59,6 @@ type EDSConfig struct {
 type edsConfigJSON struct {
 	ChildPolicy                []*loadBalancingConfig
 	FallbackPolicy             []*loadBalancingConfig
-	ClusterName                string
 	EDSServiceName             string
 	MaxConcurrentRequests      *uint32
 	LRSLoadReportingServerName *string
@@ -76,7 +73,6 @@ func (l *EDSConfig) UnmarshalJSON(data []byte) error {
 		return err
 	}
 
-	l.ClusterName = configJSON.ClusterName
 	l.EDSServiceName = configJSON.EDSServiceName
 	l.MaxConcurrentRequests = configJSON.MaxConcurrentRequests
 	l.LrsLoadReportingServerName = configJSON.LRSLoadReportingServerName
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index a5f653f737aa..de724701df94 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -25,7 +25,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -69,7 +68,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
-		loadWrapper:       loadstore.NewWrapper(),
+		lsw:               &loadStoreWrapper{},
 		config:            &EDSConfig{},
 	}
 	x.logger = prefixLogger(x)
@@ -81,7 +80,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 	}
 
 	x.xdsClient = client
-	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.loadWrapper, x.logger)
+	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.lsw, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
@@ -139,14 +138,14 @@ type edsBalancer struct {
 	xdsClientUpdate   chan *edsUpdate
 	childPolicyUpdate *buffer.Unbounded
 
-	xdsClient   xdsClientInterface
-	loadWrapper *loadstore.Wrapper
-	config      *EDSConfig // may change when passed a different service config
-	edsImpl     edsBalancerImplInterface
+	xdsClient xdsClientInterface
+	lsw       *loadStoreWrapper
+	config    *EDSConfig // may change when passed a different service config
+	edsImpl   edsBalancerImplInterface
 
-	clusterName          string
+	// edsServiceName is the edsServiceName currently being watched, not
+	// necessary the edsServiceName from service config.
 	edsServiceName       string
-	edsToWatch           string // this is edsServiceName if it's set, otherwise, it's clusterName.
 	cancelEndpointsWatch func()
 	loadReportServer     *string // LRS is disabled if loadReporterServer is nil.
 	cancelLoadReport     func()
@@ -219,7 +218,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			x.logger.Warningf("failed to update xDS client: %v", err)
 		}
 
-		x.edsImpl.updateServiceRequestsConfig(cfg.ClusterName, cfg.MaxConcurrentRequests)
+		x.edsImpl.updateServiceRequestsConfig(cfg.EDSServiceName, cfg.MaxConcurrentRequests)
 
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
@@ -242,35 +241,10 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 // handleServiceConfigUpdate applies the service config update, watching a new
 // EDS service name and restarting LRS stream, as required.
 func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
-	var updateLoadClusterAndService bool
-	if x.clusterName != config.ClusterName {
-		updateLoadClusterAndService = true
-		x.clusterName = config.ClusterName
-		x.edsImpl.updateClusterName(x.clusterName)
-	}
+	// Restart EDS watch when the edsServiceName has changed.
 	if x.edsServiceName != config.EDSServiceName {
-		updateLoadClusterAndService = true
 		x.edsServiceName = config.EDSServiceName
-	}
-
-	// If EDSServiceName is set, use it to watch EDS. Otherwise, use the cluster
-	// name.
-	newEDSToWatch := config.EDSServiceName
-	if newEDSToWatch == "" {
-		newEDSToWatch = config.ClusterName
-	}
-	var restartEDSWatch bool
-	if x.edsToWatch != newEDSToWatch {
-		restartEDSWatch = true
-		x.edsToWatch = newEDSToWatch
-	}
-
-	// Restart EDS watch when the eds name has changed.
-	if restartEDSWatch {
 		x.startEndpointsWatch()
-	}
-
-	if updateLoadClusterAndService {
 		// TODO: this update for the LRS service name is too early. It should
 		// only apply to the new EDS response. But this is applied to the RPCs
 		// before the new EDS response. To fully fix this, the EDS balancer
@@ -278,13 +252,14 @@ func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
 		//
 		// This is OK for now, because we don't actually expect edsServiceName
 		// to change. Fix this (a bigger change) will happen later.
-		x.loadWrapper.UpdateClusterAndService(x.clusterName, x.edsServiceName)
+		x.lsw.updateServiceName(x.edsServiceName)
+		x.edsImpl.updateClusterName(x.edsServiceName)
 	}
 
 	// Restart load reporting when the loadReportServer name has changed.
 	if !equalStringPointers(x.loadReportServer, config.LrsLoadReportingServerName) {
 		loadStore := x.startLoadReport(config.LrsLoadReportingServerName)
-		x.loadWrapper.UpdateLoadStore(loadStore)
+		x.lsw.updateLoadStore(loadStore)
 	}
 
 	return nil
@@ -298,15 +273,14 @@ func (x *edsBalancer) startEndpointsWatch() {
 	if x.cancelEndpointsWatch != nil {
 		x.cancelEndpointsWatch()
 	}
-	edsToWatch := x.edsToWatch
-	cancelEDSWatch := x.xdsClient.WatchEndpoints(edsToWatch, func(update xdsclient.EndpointsUpdate, err error) {
+	cancelEDSWatch := x.xdsClient.WatchEndpoints(x.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
 		x.logger.Infof("Watch update from xds-client %p, content: %+v", x.xdsClient, update)
 		x.handleEDSUpdate(update, err)
 	})
-	x.logger.Infof("Watch started on resource name %v with xds-client %p", edsToWatch, x.xdsClient)
+	x.logger.Infof("Watch started on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
 	x.cancelEndpointsWatch = func() {
 		cancelEDSWatch()
-		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", edsToWatch, x.xdsClient)
+		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
 	}
 }
 
@@ -314,12 +288,10 @@ func (x *edsBalancer) cancelWatch() {
 	x.loadReportServer = nil
 	if x.cancelLoadReport != nil {
 		x.cancelLoadReport()
-		x.cancelLoadReport = nil
 	}
+	x.edsServiceName = ""
 	if x.cancelEndpointsWatch != nil {
-		x.edsToWatch = ""
 		x.cancelEndpointsWatch()
-		x.cancelEndpointsWatch = nil
 	}
 }
 
@@ -333,7 +305,6 @@ func (x *edsBalancer) startLoadReport(loadReportServer *string) *load.Store {
 	x.loadReportServer = loadReportServer
 	if x.cancelLoadReport != nil {
 		x.cancelLoadReport()
-		x.cancelLoadReport = nil
 	}
 	if loadReportServer == nil {
 		return nil
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index 3cfc620a2400..c5e3071d10d8 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -29,7 +29,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/internal"
-	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -821,9 +820,9 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
 	loadStore := load.NewStore()
-	lsWrapper := loadstore.NewWrapper()
-	lsWrapper.UpdateClusterAndService(testClusterNames[0], "")
-	lsWrapper.UpdateLoadStore(loadStore)
+	lsWrapper := &loadStoreWrapper{}
+	lsWrapper.updateServiceName(testClusterNames[0])
+	lsWrapper.updateLoadStore(loadStore)
 
 	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, lsWrapper, nil)
@@ -914,8 +913,8 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 // TestEDS_LoadReportDisabled covers the case that LRS is disabled. It makes
 // sure the EDS implementation isn't broken (doesn't panic).
 func (s) TestEDS_LoadReportDisabled(t *testing.T) {
-	lsWrapper := loadstore.NewWrapper()
-	lsWrapper.UpdateClusterAndService(testClusterNames[0], "")
+	lsWrapper := &loadStoreWrapper{}
+	lsWrapper.updateServiceName(testClusterNames[0])
 	// Not calling lsWrapper.updateLoadStore(loadStore) because LRS is disabled.
 
 	cc := testutils.NewTestClientConn(t)
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 3fe66098973c..5c9e5f0b1d53 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -354,7 +354,6 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			ChildPolicy:    lbCfgA,
-			ClusterName:    testEDSClusterName,
 			EDSServiceName: testServiceName,
 		},
 	}); err != nil {
@@ -368,7 +367,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgA); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsLB.waitForCounterUpdate(ctx, testEDSClusterName); err != nil {
+	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
 		t.Fatal(err)
 	}
 	if err := edsLB.waitForCountMaxUpdate(ctx, nil); err != nil {
@@ -383,7 +382,6 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			ChildPolicy:           lbCfgB,
-			ClusterName:           testEDSClusterName,
 			EDSServiceName:        testServiceName,
 			MaxConcurrentRequests: &testCountMax,
 		},
@@ -393,7 +391,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgB); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsLB.waitForCounterUpdate(ctx, testEDSClusterName); err != nil {
+	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
 		// Counter is updated even though the service name didn't change. The
 		// eds_impl will compare the service names, and skip if it didn't change.
 		t.Fatal(err)
@@ -512,18 +510,6 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("eds impl expecting empty update, got %v", err)
 	}
-
-	// An update with the same service name should not trigger a new watch.
-	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
-	}); err != nil {
-		t.Fatal(err)
-	}
-	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer sCancel()
-	if _, err := xdsC.WaitForWatchEDS(sCtx); err != context.DeadlineExceeded {
-		t.Fatal("got unexpected new EDS watch")
-	}
 }
 
 // TestErrorFromResolver verifies that resolver errors are handled correctly.
@@ -589,17 +575,6 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
 	}
-
-	// An update with the same service name should trigger a new watch, because
-	// the previous watch was canceled.
-	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
-	}); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
-		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
-	}
 }
 
 // Given a list of resource names, verifies that EDS requests for the same are
@@ -611,7 +586,7 @@ func verifyExpectedRequests(ctx context.Context, fc *fakeclient.Client, resource
 			if err := fc.WaitForCancelEDSWatch(ctx); err != nil {
 				return fmt.Errorf("timed out when expecting resource %q", name)
 			}
-			continue
+			return nil
 		}
 
 		resName, err := fc.WaitForWatchEDS(ctx)
@@ -640,18 +615,6 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}
 	defer edsB.Close()
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	// If eds service name is not set, should watch for cluster name.
-	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
-		BalancerConfig: &EDSConfig{ClusterName: "cluster-1"},
-	}); err != nil {
-		t.Fatal(err)
-	}
-	if err := verifyExpectedRequests(ctx, xdsC, "cluster-1"); err != nil {
-		t.Fatal(err)
-	}
-
 	// Update with an non-empty edsServiceName should trigger an EDS watch for
 	// the same.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
@@ -659,7 +622,9 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	if err := verifyExpectedRequests(ctx, xdsC, "", "foobar-1"); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := verifyExpectedRequests(ctx, xdsC, "foobar-1"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -695,7 +660,7 @@ func (s) TestCounterUpdate(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			ClusterName:           "foobar-1",
+			EDSServiceName:        "foobar-1",
 			MaxConcurrentRequests: &testCountMax,
 		},
 	}); err != nil {
@@ -729,7 +694,7 @@ func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			ClusterName: "foobar-1",
+			EDSServiceName: "foobar-1",
 		},
 	}); err != nil {
 		t.Fatal(err)
@@ -748,7 +713,7 @@ func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			ClusterName: "foobar-2",
+			EDSServiceName: "foobar-2",
 		},
 	}); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/balancer/edsbalancer/load_store_wrapper.go b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
new file mode 100644
index 000000000000..18904e47a42e
--- /dev/null
+++ b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
@@ -0,0 +1,88 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package edsbalancer
+
+import (
+	"sync"
+
+	"google.golang.org/grpc/xds/internal/client/load"
+)
+
+type loadStoreWrapper struct {
+	mu      sync.RWMutex
+	service string
+	// Both store and perCluster will be nil if load reporting is disabled (EDS
+	// response doesn't have LRS server name). Note that methods on Store and
+	// perCluster all handle nil, so there's no need to check nil before calling
+	// them.
+	store      *load.Store
+	perCluster load.PerClusterReporter
+}
+
+func (lsw *loadStoreWrapper) updateServiceName(service string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if lsw.service == service {
+		return
+	}
+	lsw.service = service
+	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
+}
+
+func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
+}
+
+func (lsw *loadStoreWrapper) CallStarted(locality string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallStarted(locality)
+	}
+}
+
+func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallFinished(locality, err)
+	}
+}
+
+func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallServerLoad(locality, name, val)
+	}
+}
+
+func (lsw *loadStoreWrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallDropped(category)
+	}
+}
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 0fa2b402ffc9..bb1117ec5349 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -28,14 +28,12 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"google.golang.org/grpc/internal/testutils"
 	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
@@ -47,7 +45,7 @@ const (
 	serviceName = "service"
 )
 
-var emptyUpdate = ClusterUpdate{ClusterName: clusterName, EnableLRS: false}
+var emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false}
 
 func (s) TestValidateCluster_Failure(t *testing.T) {
 	tests := []struct {
@@ -143,35 +141,24 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		{
 			name: "happy-case-logical-dns",
 			cluster: &v3clusterpb.Cluster{
-				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS},
 				LbPolicy:             v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ClusterName: clusterName, EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
+			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
 		},
 		{
 			name: "happy-case-aggregate-v3",
 			cluster: &v3clusterpb.Cluster{
-				Name: clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{
-					ClusterType: &v3clusterpb.Cluster_CustomClusterType{
-						Name: "envoy.clusters.aggregate",
-						TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{
-							Clusters: []string{"a", "b", "c"},
-						}),
-					},
+					ClusterType: &v3clusterpb.Cluster_CustomClusterType{Name: "envoy.clusters.aggregate"},
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{
-				ClusterName: clusterName, EnableLRS: false, ClusterType: ClusterTypeAggregate,
-				PrioritizedClusterNames: []string{"a", "b", "c"},
-			},
+			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeAggregate},
 		},
 		{
 			name: "happy-case-no-service-name-no-lrs",
 			cluster: &v3clusterpb.Cluster{
-				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -187,7 +174,6 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		{
 			name: "happy-case-no-lrs",
 			cluster: &v3clusterpb.Cluster{
-				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -199,7 +185,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: false},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: false},
 		},
 		{
 			name: "happiest-case",
@@ -221,7 +207,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: true},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
 		},
 		{
 			name: "happiest-case-with-circuitbreakers",
@@ -255,7 +241,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
+			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
 		},
 	}
 
@@ -268,8 +254,8 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 			if err != nil {
 				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err)
 			}
-			if diff := cmp.Diff(update, test.wantUpdate, cmpopts.EquateEmpty()); diff != "" {
-				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) got diff: %v (-got, +want)", test.cluster, diff)
+			if !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
+				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, want: %v", test.cluster, update, test.wantUpdate)
 			}
 		})
 	}
@@ -282,7 +268,6 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 	defer func() { env.ClientSideSecuritySupport = origClientSideSecurityEnvVar }()
 
 	cluster := &v3clusterpb.Cluster{
-		Name:                 clusterName,
 		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 			EdsConfig: &v3corepb.ConfigSource{
@@ -317,9 +302,8 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 		},
 	}
 	wantUpdate := ClusterUpdate{
-		ClusterName:    clusterName,
-		EDSServiceName: serviceName,
-		EnableLRS:      false,
+		ServiceName: serviceName,
+		EnableLRS:   false,
 	}
 	gotUpdate, err := validateClusterAndConstructClusterUpdate(cluster)
 	if err != nil {
@@ -341,7 +325,6 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		identityCertName       = "identityCert"
 		rootPluginInstance     = "rootPluginInstance"
 		rootCertName           = "rootCert"
-		clusterName            = "cluster"
 		serviceName            = "service"
 		sanExact               = "san-exact"
 		sanPrefix              = "san-prefix"
@@ -674,7 +657,6 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-no-identity-certs",
 			cluster: &v3clusterpb.Cluster{
-				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -709,9 +691,8 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ClusterName:    clusterName,
-				EDSServiceName: serviceName,
-				EnableLRS:      false,
+				ServiceName: serviceName,
+				EnableLRS:   false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName: rootPluginInstance,
 					RootCertName:     rootCertName,
@@ -721,7 +702,6 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-validation-context-provider-instance",
 			cluster: &v3clusterpb.Cluster{
-				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -760,9 +740,8 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ClusterName:    clusterName,
-				EDSServiceName: serviceName,
-				EnableLRS:      false,
+				ServiceName: serviceName,
+				EnableLRS:   false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName:     rootPluginInstance,
 					RootCertName:         rootCertName,
@@ -774,7 +753,6 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-combined-validation-context",
 			cluster: &v3clusterpb.Cluster{
-				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -827,9 +805,8 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ClusterName:    clusterName,
-				EDSServiceName: serviceName,
-				EnableLRS:      false,
+				ServiceName: serviceName,
+				EnableLRS:   false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName:     rootPluginInstance,
 					RootCertName:         rootCertName,
@@ -991,8 +968,7 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v2ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ClusterName:    v2ClusterName,
-					EDSServiceName: v2Service, EnableLRS: true,
+					ServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 			},
@@ -1006,8 +982,7 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v3ClusterName: {
-					ClusterName:    v3ClusterName,
-					EDSServiceName: v3Service, EnableLRS: true,
+					ServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 			},
@@ -1021,13 +996,11 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v2ClusterAny, v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ClusterName:    v2ClusterName,
-					EDSServiceName: v2Service, EnableLRS: true,
+					ServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 				v3ClusterName: {
-					ClusterName:    v3ClusterName,
-					EDSServiceName: v3Service, EnableLRS: true,
+					ServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 			},
@@ -1057,13 +1030,11 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ClusterName:    v2ClusterName,
-					EDSServiceName: v2Service, EnableLRS: true,
+					ServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 				v3ClusterName: {
-					ClusterName:    v3ClusterName,
-					EDSServiceName: v3Service, EnableLRS: true,
+					ServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 				"bad": {},
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 4067536dda24..2daceede5398 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -375,23 +375,22 @@ const (
 // interest to the registered CDS watcher.
 type ClusterUpdate struct {
 	ClusterType ClusterType
-	// ClusterName is the clusterName being watched for through CDS.
-	ClusterName string
-	// EDSServiceName is an optional name for EDS. If it's not set, the balancer
-	// should watch ClusterName for the EDS resources.
-	EDSServiceName string
+	// ServiceName is the service name corresponding to the clusterName which
+	// is being watched for through CDS.
+	ServiceName string
 	// EnableLRS indicates whether or not load should be reported through LRS.
 	EnableLRS bool
 	// SecurityCfg contains security configuration sent by the control plane.
 	SecurityCfg *SecurityConfig
 	// MaxRequests for circuit breaking, if any (otherwise nil).
 	MaxRequests *uint32
-	// PrioritizedClusterNames is used only for cluster type aggregate. It represents
-	// a prioritized list of cluster names.
-	PrioritizedClusterNames []string
 
 	// Raw is the resource from the xds response.
 	Raw *anypb.Any
+
+	// PrioritizedClusterNames is used only for cluster type aggregate. It represents
+	// a prioritized list of cluster names.
+	PrioritizedClusterNames []string
 }
 
 // OverloadDropConfig contains the config to drop overloads.
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index d57bc20e2c2c..69930557b26e 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -185,13 +185,13 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
-	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
+	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate2}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate2); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index d868beb1831b..b56ae6108bbe 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -151,7 +151,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			cdsResponse: goodCDSResponse2,
 			wantErr:     false,
 			wantUpdate: map[string]xdsclient.ClusterUpdate{
-				goodClusterName2: {ClusterName: goodClusterName2, EDSServiceName: serviceName2, Raw: marshaledCluster2},
+				goodClusterName2: {ServiceName: serviceName2, Raw: marshaledCluster2},
 			},
 			wantUpdateMD: xdsclient.UpdateMetadata{
 				Status: xdsclient.ServiceStatusACKed,
@@ -164,7 +164,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			cdsResponse: goodCDSResponse1,
 			wantErr:     false,
 			wantUpdate: map[string]xdsclient.ClusterUpdate{
-				goodClusterName1: {ClusterName: goodClusterName1, EDSServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
+				goodClusterName1: {ServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
 			},
 			wantUpdateMD: xdsclient.UpdateMetadata{
 				Status: xdsclient.ServiceStatusACKed,
diff --git a/xds/internal/client/watchers_cluster_test.go b/xds/internal/client/watchers_cluster_test.go
index c9837cd51978..2d10c7f43b5f 100644
--- a/xds/internal/client/watchers_cluster_test.go
+++ b/xds/internal/client/watchers_cluster_test.go
@@ -64,7 +64,7 @@ func (s) TestClusterWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -128,7 +128,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		}
 	}
 
-	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
@@ -200,8 +200,8 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ClusterUpdate{ClusterName: testEDSName + "1"}
-	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
+	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
@@ -245,7 +245,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	}, UpdateMetadata{})
@@ -345,7 +345,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
+	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	}, UpdateMetadata{})
@@ -402,8 +402,8 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ClusterUpdate{ClusterName: testEDSName + "1"}
-	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
+	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index f64d22ce67d3..c0caf5cceb57 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -571,9 +571,42 @@ func unmarshalClusterResource(r *anypb.Any, logger *grpclog.PrefixLogger) (strin
 	return cluster.GetName(), cu, nil
 }
 
+func clusterTypeFromCluster(cluster *v3clusterpb.Cluster) (ClusterType, string, []string, error) {
+	if cluster.GetType() == v3clusterpb.Cluster_EDS {
+		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
+			return 0, "", nil, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
+		}
+		// If the Cluster message in the CDS response did not contain a
+		// serviceName, we will just use the clusterName for EDS.
+		if cluster.GetEdsClusterConfig().GetServiceName() == "" {
+			return ClusterTypeEDS, cluster.GetName(), nil, nil
+		}
+		return ClusterTypeEDS, cluster.GetEdsClusterConfig().GetServiceName(), nil, nil
+	}
+
+	if cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS {
+		return ClusterTypeLogicalDNS, cluster.GetName(), nil, nil
+	}
+
+	if cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate" {
+		// Loop through ClusterConfig here to get cluster names.
+		clusters := &v3aggregateclusterpb.ClusterConfig{}
+		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
+			return 0, "", nil, fmt.Errorf("failed to unmarshal resource: %v", err)
+		}
+		return ClusterTypeAggregate, cluster.GetName(), clusters.Clusters, nil
+	}
+	return 0, "", nil, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+}
+
 func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
+	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
 	if cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN {
-		return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
+		return emptyUpdate, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
+	}
+	clusterType, serviceName, prioritizedClusters, err := clusterTypeFromCluster(cluster)
+	if err != nil {
+		return emptyUpdate, err
 	}
 
 	// Process security configuration received from the control plane iff the
@@ -582,40 +615,18 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu
 	if env.ClientSideSecuritySupport {
 		var err error
 		if sc, err = securityConfigFromCluster(cluster); err != nil {
-			return ClusterUpdate{}, err
+			return emptyUpdate, err
 		}
 	}
 
-	ret := ClusterUpdate{
-		ClusterName: cluster.GetName(),
-		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
-		SecurityCfg: sc,
-		MaxRequests: circuitBreakersFromCluster(cluster),
-	}
-
-	// Validate and set cluster type from the response.
-	switch {
-	case cluster.GetType() == v3clusterpb.Cluster_EDS:
-		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
-			return ClusterUpdate{}, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
-		}
-		ret.ClusterType = ClusterTypeEDS
-		ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName()
-		return ret, nil
-	case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS:
-		ret.ClusterType = ClusterTypeLogicalDNS
-		return ret, nil
-	case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate":
-		clusters := &v3aggregateclusterpb.ClusterConfig{}
-		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
-			return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
-		}
-		ret.ClusterType = ClusterTypeAggregate
-		ret.PrioritizedClusterNames = clusters.Clusters
-		return ret, nil
-	default:
-		return ClusterUpdate{}, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
-	}
+	return ClusterUpdate{
+		ClusterType:             clusterType,
+		ServiceName:             serviceName,
+		EnableLRS:               cluster.GetLrsServer().GetSelf() != nil,
+		SecurityCfg:             sc,
+		MaxRequests:             circuitBreakersFromCluster(cluster),
+		PrioritizedClusterNames: prioritizedClusters,
+	}, nil
 }
 
 // securityConfigFromCluster extracts the relevant security configuration from

From 75497df97f8bc9d5ec905d6e6b283a207eb3e9f0 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 4 May 2021 14:38:47 -0700
Subject: [PATCH 446/481] meshca: remove meshca certificate provider
 implementation (#4385)

---
 .../tls/certprovider/meshca/builder.go        | 165 -------
 .../tls/certprovider/meshca/builder_test.go   | 177 -------
 credentials/tls/certprovider/meshca/config.go | 310 ------------
 .../tls/certprovider/meshca/config_test.go    | 375 --------------
 .../meshca/internal/v1/meshca.pb.go           | 276 -----------
 .../meshca/internal/v1/meshca_grpc.pb.go      | 110 -----
 .../tls/certprovider/meshca/logging.go        |  36 --
 credentials/tls/certprovider/meshca/plugin.go | 289 -----------
 .../tls/certprovider/meshca/plugin_test.go    | 459 ------------------
 regenerate.sh                                 |  10 -
 xds/go113.go                                  |  25 -
 11 files changed, 2232 deletions(-)
 delete mode 100644 credentials/tls/certprovider/meshca/builder.go
 delete mode 100644 credentials/tls/certprovider/meshca/builder_test.go
 delete mode 100644 credentials/tls/certprovider/meshca/config.go
 delete mode 100644 credentials/tls/certprovider/meshca/config_test.go
 delete mode 100644 credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
 delete mode 100644 credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
 delete mode 100644 credentials/tls/certprovider/meshca/logging.go
 delete mode 100644 credentials/tls/certprovider/meshca/plugin.go
 delete mode 100644 credentials/tls/certprovider/meshca/plugin_test.go
 delete mode 100644 xds/go113.go

diff --git a/credentials/tls/certprovider/meshca/builder.go b/credentials/tls/certprovider/meshca/builder.go
deleted file mode 100644
index 4b8af7c9b3c5..000000000000
--- a/credentials/tls/certprovider/meshca/builder.go
+++ /dev/null
@@ -1,165 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package meshca
-
-import (
-	"crypto/x509"
-	"encoding/json"
-	"fmt"
-	"sync"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/credentials/sts"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/internal/backoff"
-)
-
-const pluginName = "mesh_ca"
-
-// For overriding in unit tests.
-var (
-	grpcDialFunc = grpc.Dial
-	backoffFunc  = backoff.DefaultExponential.Backoff
-)
-
-func init() {
-	certprovider.Register(newPluginBuilder())
-}
-
-func newPluginBuilder() *pluginBuilder {
-	return &pluginBuilder{clients: make(map[ccMapKey]*refCountedCC)}
-}
-
-// Key for the map containing ClientConns to the MeshCA server. Only the server
-// name and the STS options (which is used to create call creds) from the plugin
-// configuration determine if two configs can share the same ClientConn. Hence
-// only those form the key to this map.
-type ccMapKey struct {
-	name    string
-	stsOpts sts.Options
-}
-
-// refCountedCC wraps a grpc.ClientConn to MeshCA along with a reference count.
-type refCountedCC struct {
-	cc     *grpc.ClientConn
-	refCnt int
-}
-
-// pluginBuilder is an implementation of the certprovider.Builder interface,
-// which builds certificate provider instances to get certificates signed from
-// the MeshCA.
-type pluginBuilder struct {
-	// A collection of ClientConns to the MeshCA server along with a reference
-	// count. Provider instances whose config point to the same server name will
-	// end up sharing the ClientConn.
-	mu      sync.Mutex
-	clients map[ccMapKey]*refCountedCC
-}
-
-// ParseConfig parses the configuration to be passed to the MeshCA plugin
-// implementation. Expects the config to be a json.RawMessage which contains a
-// serialized JSON representation of the meshca_experimental.GoogleMeshCaConfig
-// proto message.
-//
-// Takes care of sharing the ClientConn to the MeshCA server among
-// different plugin instantiations.
-func (b *pluginBuilder) ParseConfig(c interface{}) (*certprovider.BuildableConfig, error) {
-	data, ok := c.(json.RawMessage)
-	if !ok {
-		return nil, fmt.Errorf("meshca: unsupported config type: %T", c)
-	}
-	cfg, err := pluginConfigFromJSON(data)
-	if err != nil {
-		return nil, err
-	}
-	return certprovider.NewBuildableConfig(pluginName, cfg.canonical(), func(opts certprovider.BuildOptions) certprovider.Provider {
-		return b.buildFromConfig(cfg, opts)
-	}), nil
-}
-
-// buildFromConfig builds a certificate provider instance for the given config
-// and options. Provider instances are shared wherever possible.
-func (b *pluginBuilder) buildFromConfig(cfg *pluginConfig, opts certprovider.BuildOptions) certprovider.Provider {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
-	ccmk := ccMapKey{
-		name:    cfg.serverURI,
-		stsOpts: cfg.stsOpts,
-	}
-	rcc, ok := b.clients[ccmk]
-	if !ok {
-		// STS call credentials take care of exchanging a locally provisioned
-		// JWT token for an access token which will be accepted by the MeshCA.
-		callCreds, err := sts.NewCredentials(cfg.stsOpts)
-		if err != nil {
-			logger.Errorf("sts.NewCredentials() failed: %v", err)
-			return nil
-		}
-
-		// MeshCA is a public endpoint whose certificate is Web-PKI compliant.
-		// So, we just need to use the system roots to authenticate the MeshCA.
-		cp, err := x509.SystemCertPool()
-		if err != nil {
-			logger.Errorf("x509.SystemCertPool() failed: %v", err)
-			return nil
-		}
-		transportCreds := credentials.NewClientTLSFromCert(cp, "")
-
-		cc, err := grpcDialFunc(cfg.serverURI, grpc.WithTransportCredentials(transportCreds), grpc.WithPerRPCCredentials(callCreds))
-		if err != nil {
-			logger.Errorf("grpc.Dial(%s) failed: %v", cfg.serverURI, err)
-			return nil
-		}
-
-		rcc = &refCountedCC{cc: cc}
-		b.clients[ccmk] = rcc
-	}
-	rcc.refCnt++
-
-	p := newProviderPlugin(providerParams{
-		cc:      rcc.cc,
-		cfg:     cfg,
-		opts:    opts,
-		backoff: backoffFunc,
-		doneFunc: func() {
-			// The plugin implementation will invoke this function when it is
-			// being closed, and here we take care of closing the ClientConn
-			// when there are no more plugins using it. We need to acquire the
-			// lock before accessing the rcc from the enclosing function.
-			b.mu.Lock()
-			defer b.mu.Unlock()
-			rcc.refCnt--
-			if rcc.refCnt == 0 {
-				logger.Infof("Closing grpc.ClientConn to %s", ccmk.name)
-				rcc.cc.Close()
-				delete(b.clients, ccmk)
-			}
-		},
-	})
-	return p
-}
-
-// Name returns the MeshCA plugin name.
-func (b *pluginBuilder) Name() string {
-	return pluginName
-}
diff --git a/credentials/tls/certprovider/meshca/builder_test.go b/credentials/tls/certprovider/meshca/builder_test.go
deleted file mode 100644
index 79035d008d9e..000000000000
--- a/credentials/tls/certprovider/meshca/builder_test.go
+++ /dev/null
@@ -1,177 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package meshca
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"testing"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	"google.golang.org/grpc/internal/testutils"
-)
-
-func overrideHTTPFuncs() func() {
-	// Directly override the functions which are used to read the zone and
-	// audience instead of overriding the http.Client.
-	origReadZone := readZoneFunc
-	readZoneFunc = func(httpDoer) string { return "test-zone" }
-	origReadAudience := readAudienceFunc
-	readAudienceFunc = func(httpDoer) string { return "test-audience" }
-	return func() {
-		readZoneFunc = origReadZone
-		readAudienceFunc = origReadAudience
-	}
-}
-
-func (s) TestBuildSameConfig(t *testing.T) {
-	defer overrideHTTPFuncs()()
-
-	// We will attempt to create `cnt` number of providers. So we create a
-	// channel of the same size here, even though we expect only one ClientConn
-	// to be pushed into this channel. This makes sure that even if more than
-	// one ClientConn ends up being created, the Build() call does not block.
-	const cnt = 5
-	ccChan := testutils.NewChannelWithSize(cnt)
-
-	// Override the dial func to dial a dummy MeshCA endpoint, and also push the
-	// returned ClientConn on a channel to be inspected by the test.
-	origDialFunc := grpcDialFunc
-	grpcDialFunc = func(string, ...grpc.DialOption) (*grpc.ClientConn, error) {
-		cc, err := grpc.Dial("dummy-meshca-endpoint", grpc.WithInsecure())
-		ccChan.Send(cc)
-		return cc, err
-	}
-	defer func() { grpcDialFunc = origDialFunc }()
-
-	// Parse a good config to generate a stable config which will be passed to
-	// invocations of Build().
-	builder := newPluginBuilder()
-	buildableConfig, err := builder.ParseConfig(goodConfigFullySpecified)
-	if err != nil {
-		t.Fatalf("builder.ParseConfig(%q) failed: %v", goodConfigFullySpecified, err)
-	}
-
-	// Create multiple providers with the same config. All these providers must
-	// end up sharing the same ClientConn.
-	providers := []certprovider.Provider{}
-	for i := 0; i < cnt; i++ {
-		p, err := buildableConfig.Build(certprovider.BuildOptions{})
-		if err != nil {
-			t.Fatalf("Build(%+v) failed: %v", buildableConfig, err)
-		}
-		providers = append(providers, p)
-	}
-
-	// Make sure only one ClientConn is created.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	val, err := ccChan.Receive(ctx)
-	if err != nil {
-		t.Fatalf("Failed to create ClientConn: %v", err)
-	}
-	testCC := val.(*grpc.ClientConn)
-
-	// Attempt to read the second ClientConn should timeout.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer cancel()
-	if _, err := ccChan.Receive(ctx); err != context.DeadlineExceeded {
-		t.Fatal("Builder created more than one ClientConn")
-	}
-
-	for _, p := range providers {
-		p.Close()
-	}
-
-	for {
-		state := testCC.GetState()
-		if state == connectivity.Shutdown {
-			break
-		}
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		if !testCC.WaitForStateChange(ctx, state) {
-			t.Fatalf("timeout waiting for clientConn state to change from %s", state)
-		}
-	}
-}
-
-func (s) TestBuildDifferentConfig(t *testing.T) {
-	defer overrideHTTPFuncs()()
-
-	// We will attempt to create two providers with different configs. So we
-	// expect two ClientConns to be pushed on to this channel.
-	const cnt = 2
-	ccChan := testutils.NewChannelWithSize(cnt)
-
-	// Override the dial func to dial a dummy MeshCA endpoint, and also push the
-	// returned ClientConn on a channel to be inspected by the test.
-	origDialFunc := grpcDialFunc
-	grpcDialFunc = func(string, ...grpc.DialOption) (*grpc.ClientConn, error) {
-		cc, err := grpc.Dial("dummy-meshca-endpoint", grpc.WithInsecure())
-		ccChan.Send(cc)
-		return cc, err
-	}
-	defer func() { grpcDialFunc = origDialFunc }()
-
-	builder := newPluginBuilder()
-	providers := []certprovider.Provider{}
-	for i := 0; i < cnt; i++ {
-		// Copy the good test config and modify the serverURI to make sure that
-		// a new provider is created for the config.
-		inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, fmt.Sprintf("test-mesh-ca:%d", i)))
-		buildableConfig, err := builder.ParseConfig(inputConfig)
-		if err != nil {
-			t.Fatalf("builder.ParseConfig(%q) failed: %v", inputConfig, err)
-		}
-
-		p, err := buildableConfig.Build(certprovider.BuildOptions{})
-		if err != nil {
-			t.Fatalf("Build(%+v) failed: %v", buildableConfig, err)
-		}
-		providers = append(providers, p)
-	}
-
-	// Make sure two ClientConns are created.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	for i := 0; i < cnt; i++ {
-		if _, err := ccChan.Receive(ctx); err != nil {
-			t.Fatalf("Failed to create ClientConn: %v", err)
-		}
-	}
-
-	// Close the first provider, and attempt to read key material from the
-	// second provider. The call to read key material should timeout, but it
-	// should not return certprovider.errProviderClosed.
-	providers[0].Close()
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
-	defer cancel()
-	if _, err := providers[1].KeyMaterial(ctx); err != context.DeadlineExceeded {
-		t.Fatalf("provider.KeyMaterial(ctx) = %v, want contextDeadlineExceeded", err)
-	}
-
-	// Close the second provider to make sure that the leakchecker is happy.
-	providers[1].Close()
-}
diff --git a/credentials/tls/certprovider/meshca/config.go b/credentials/tls/certprovider/meshca/config.go
deleted file mode 100644
index c0772b3bb7ea..000000000000
--- a/credentials/tls/certprovider/meshca/config.go
+++ /dev/null
@@ -1,310 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package meshca
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/http/httputil"
-	"path"
-	"strings"
-	"time"
-
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	"github.com/golang/protobuf/ptypes"
-	"google.golang.org/protobuf/encoding/protojson"
-	"google.golang.org/protobuf/types/known/durationpb"
-
-	"google.golang.org/grpc/credentials/sts"
-)
-
-const (
-	// GKE metadata server endpoint.
-	mdsBaseURI        = "http://metadata.google.internal/"
-	mdsRequestTimeout = 5 * time.Second
-
-	// The following are default values used in the interaction with MeshCA.
-	defaultMeshCaEndpoint    = "meshca.googleapis.com"
-	defaultCallTimeout       = 10 * time.Second
-	defaultCertLifetimeSecs  = 86400 // 24h in seconds
-	defaultCertGraceTimeSecs = 43200 // 12h in seconds
-	defaultKeyTypeRSA        = "RSA"
-	defaultKeySize           = 2048
-
-	// The following are default values used in the interaction with STS or
-	// Secure Token Service, which is used to exchange the JWT token for an
-	// access token.
-	defaultSTSEndpoint        = "securetoken.googleapis.com"
-	defaultCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"
-	defaultRequestedTokenType = "urn:ietf:params:oauth:token-type:access_token"
-	defaultSubjectTokenType   = "urn:ietf:params:oauth:token-type:jwt"
-)
-
-// For overriding in unit tests.
-var (
-	makeHTTPDoer     = makeHTTPClient
-	readZoneFunc     = readZone
-	readAudienceFunc = readAudience
-)
-
-type pluginConfig struct {
-	serverURI     string
-	stsOpts       sts.Options
-	callTimeout   time.Duration
-	certLifetime  time.Duration
-	certGraceTime time.Duration
-	keyType       string
-	keySize       int
-	location      string
-}
-
-// Type of key to be embedded in CSRs sent to the MeshCA.
-const (
-	keyTypeUnknown = 0
-	keyTypeRSA     = 1
-)
-
-// pluginConfigFromJSON parses the provided config in JSON.
-//
-// For certain values missing in the config, we use default values defined at
-// the top of this file.
-//
-// If the location field or STS audience field is missing, we try talking to the
-// GKE Metadata server and try to infer these values. If this attempt does not
-// succeed, we let those fields have empty values.
-func pluginConfigFromJSON(data json.RawMessage) (*pluginConfig, error) {
-	// This anonymous struct corresponds to the expected JSON config.
-	cfgJSON := &struct {
-		Server              json.RawMessage `json:"server,omitempty"`               // Expect a v3corepb.ApiConfigSource
-		CertificateLifetime json.RawMessage `json:"certificate_lifetime,omitempty"` // Expect a durationpb.Duration
-		RenewalGracePeriod  json.RawMessage `json:"renewal_grace_period,omitempty"` // Expect a durationpb.Duration
-		KeyType             int             `json:"key_type,omitempty"`
-		KeySize             int             `json:"key_size,omitempty"`
-		Location            string          `json:"location,omitempty"`
-	}{}
-	if err := json.Unmarshal(data, cfgJSON); err != nil {
-		return nil, fmt.Errorf("meshca: failed to unmarshal config: %v", err)
-	}
-
-	// Further unmarshal fields represented as json.RawMessage in the above
-	// anonymous struct, and use default values if not specified.
-	server := &v3corepb.ApiConfigSource{}
-	if cfgJSON.Server != nil {
-		if err := protojson.Unmarshal(cfgJSON.Server, server); err != nil {
-			return nil, fmt.Errorf("meshca: protojson.Unmarshal(%+v) failed: %v", cfgJSON.Server, err)
-		}
-	}
-	certLifetime := &durationpb.Duration{Seconds: defaultCertLifetimeSecs}
-	if cfgJSON.CertificateLifetime != nil {
-		if err := protojson.Unmarshal(cfgJSON.CertificateLifetime, certLifetime); err != nil {
-			return nil, fmt.Errorf("meshca: protojson.Unmarshal(%+v) failed: %v", cfgJSON.CertificateLifetime, err)
-		}
-	}
-	certGraceTime := &durationpb.Duration{Seconds: defaultCertGraceTimeSecs}
-	if cfgJSON.RenewalGracePeriod != nil {
-		if err := protojson.Unmarshal(cfgJSON.RenewalGracePeriod, certGraceTime); err != nil {
-			return nil, fmt.Errorf("meshca: protojson.Unmarshal(%+v) failed: %v", cfgJSON.RenewalGracePeriod, err)
-		}
-	}
-
-	if api := server.GetApiType(); api != v3corepb.ApiConfigSource_GRPC {
-		return nil, fmt.Errorf("meshca: server has apiType %s, want %s", api, v3corepb.ApiConfigSource_GRPC)
-	}
-
-	pc := &pluginConfig{
-		certLifetime:  certLifetime.AsDuration(),
-		certGraceTime: certGraceTime.AsDuration(),
-	}
-	gs := server.GetGrpcServices()
-	if l := len(gs); l != 1 {
-		return nil, fmt.Errorf("meshca: number of gRPC services in config is %d, expected 1", l)
-	}
-	grpcService := gs[0]
-	googGRPC := grpcService.GetGoogleGrpc()
-	if googGRPC == nil {
-		return nil, errors.New("meshca: missing google gRPC service in config")
-	}
-	pc.serverURI = googGRPC.GetTargetUri()
-	if pc.serverURI == "" {
-		pc.serverURI = defaultMeshCaEndpoint
-	}
-
-	callCreds := googGRPC.GetCallCredentials()
-	if len(callCreds) == 0 {
-		return nil, errors.New("meshca: missing call credentials in config")
-	}
-	var stsCallCreds *v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService
-	for _, cc := range callCreds {
-		if stsCallCreds = cc.GetStsService(); stsCallCreds != nil {
-			break
-		}
-	}
-	if stsCallCreds == nil {
-		return nil, errors.New("meshca: missing STS call credentials in config")
-	}
-	if stsCallCreds.GetSubjectTokenPath() == "" {
-		return nil, errors.New("meshca: missing subjectTokenPath in STS call credentials config")
-	}
-	pc.stsOpts = makeStsOptsWithDefaults(stsCallCreds)
-
-	var err error
-	if pc.callTimeout, err = ptypes.Duration(grpcService.GetTimeout()); err != nil {
-		pc.callTimeout = defaultCallTimeout
-	}
-	switch cfgJSON.KeyType {
-	case keyTypeUnknown, keyTypeRSA:
-		pc.keyType = defaultKeyTypeRSA
-	default:
-		return nil, fmt.Errorf("meshca: unsupported key type: %s, only support RSA keys", pc.keyType)
-	}
-	pc.keySize = cfgJSON.KeySize
-	if pc.keySize == 0 {
-		pc.keySize = defaultKeySize
-	}
-	pc.location = cfgJSON.Location
-	if pc.location == "" {
-		pc.location = readZoneFunc(makeHTTPDoer())
-	}
-
-	return pc, nil
-}
-
-func (pc *pluginConfig) canonical() []byte {
-	return []byte(fmt.Sprintf("%s:%s:%s:%s:%s:%s:%d:%s", pc.serverURI, pc.stsOpts, pc.callTimeout, pc.certLifetime, pc.certGraceTime, pc.keyType, pc.keySize, pc.location))
-}
-
-func makeStsOptsWithDefaults(stsCallCreds *v3corepb.GrpcService_GoogleGrpc_CallCredentials_StsService) sts.Options {
-	opts := sts.Options{
-		TokenExchangeServiceURI: stsCallCreds.GetTokenExchangeServiceUri(),
-		Resource:                stsCallCreds.GetResource(),
-		Audience:                stsCallCreds.GetAudience(),
-		Scope:                   stsCallCreds.GetScope(),
-		RequestedTokenType:      stsCallCreds.GetRequestedTokenType(),
-		SubjectTokenPath:        stsCallCreds.GetSubjectTokenPath(),
-		SubjectTokenType:        stsCallCreds.GetSubjectTokenType(),
-		ActorTokenPath:          stsCallCreds.GetActorTokenPath(),
-		ActorTokenType:          stsCallCreds.GetActorTokenType(),
-	}
-
-	// Use sane defaults for unspecified fields.
-	if opts.TokenExchangeServiceURI == "" {
-		opts.TokenExchangeServiceURI = defaultSTSEndpoint
-	}
-	if opts.Audience == "" {
-		opts.Audience = readAudienceFunc(makeHTTPDoer())
-	}
-	if opts.Scope == "" {
-		opts.Scope = defaultCloudPlatformScope
-	}
-	if opts.RequestedTokenType == "" {
-		opts.RequestedTokenType = defaultRequestedTokenType
-	}
-	if opts.SubjectTokenType == "" {
-		opts.SubjectTokenType = defaultSubjectTokenType
-	}
-	return opts
-}
-
-// httpDoer wraps the single method on the http.Client type that we use. This
-// helps with overriding in unit tests.
-type httpDoer interface {
-	Do(req *http.Request) (*http.Response, error)
-}
-
-func makeHTTPClient() httpDoer {
-	return &http.Client{Timeout: mdsRequestTimeout}
-}
-
-func readMetadata(client httpDoer, uriPath string) (string, error) {
-	req, err := http.NewRequest("GET", mdsBaseURI+uriPath, nil)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Add("Metadata-Flavor", "Google")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		dump, err := httputil.DumpRequestOut(req, false)
-		if err != nil {
-			logger.Warningf("Failed to dump HTTP request: %v", err)
-		}
-		logger.Warningf("Request %q returned status %v", dump, resp.StatusCode)
-	}
-	return string(body), err
-}
-
-func readZone(client httpDoer) string {
-	zoneURI := "computeMetadata/v1/instance/zone"
-	data, err := readMetadata(client, zoneURI)
-	if err != nil {
-		logger.Warningf("GET %s failed: %v", path.Join(mdsBaseURI, zoneURI), err)
-		return ""
-	}
-
-	// The output returned by the metadata server looks like this:
-	// projects/<PROJECT-NUMBER>/zones/<ZONE>
-	parts := strings.Split(data, "/")
-	if len(parts) == 0 {
-		logger.Warningf("GET %s returned {%s}, does not match expected format {projects/<PROJECT-NUMBER>/zones/<ZONE>}", path.Join(mdsBaseURI, zoneURI))
-		return ""
-	}
-	return parts[len(parts)-1]
-}
-
-// readAudience constructs the audience field to be used in the STS request, if
-// it is not specified in the plugin configuration.
-//
-// "identitynamespace:{TRUST_DOMAIN}:{GKE_CLUSTER_URL}" is the format of the
-// audience field. When workload identity is enabled on a GCP project, a default
-// trust domain is created whose value is "{PROJECT_ID}.svc.id.goog". The format
-// of the GKE_CLUSTER_URL is:
-// https://container.googleapis.com/v1/projects/{PROJECT_ID}/zones/{ZONE}/clusters/{CLUSTER_NAME}.
-func readAudience(client httpDoer) string {
-	projURI := "computeMetadata/v1/project/project-id"
-	project, err := readMetadata(client, projURI)
-	if err != nil {
-		logger.Warningf("GET %s failed: %v", path.Join(mdsBaseURI, projURI), err)
-		return ""
-	}
-	trustDomain := fmt.Sprintf("%s.svc.id.goog", project)
-
-	clusterURI := "computeMetadata/v1/instance/attributes/cluster-name"
-	cluster, err := readMetadata(client, clusterURI)
-	if err != nil {
-		logger.Warningf("GET %s failed: %v", path.Join(mdsBaseURI, clusterURI), err)
-		return ""
-	}
-	zone := readZoneFunc(client)
-	clusterURL := fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/zones/%s/clusters/%s", project, zone, cluster)
-	audience := fmt.Sprintf("identitynamespace:%s:%s", trustDomain, clusterURL)
-	return audience
-}
diff --git a/credentials/tls/certprovider/meshca/config_test.go b/credentials/tls/certprovider/meshca/config_test.go
deleted file mode 100644
index 5deb484f341c..000000000000
--- a/credentials/tls/certprovider/meshca/config_test.go
+++ /dev/null
@@ -1,375 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package meshca
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-
-	"google.golang.org/grpc/internal/grpctest"
-	"google.golang.org/grpc/internal/testutils"
-)
-
-const (
-	testProjectID  = "test-project-id"
-	testGKECluster = "test-gke-cluster"
-	testGCEZone    = "test-zone"
-)
-
-type s struct {
-	grpctest.Tester
-}
-
-func Test(t *testing.T) {
-	grpctest.RunSubTests(t, s{})
-}
-
-var (
-	goodConfigFormatStr = `
-	{
-		"server": {
-			"api_type": 2,
-			"grpc_services": [
-				{
-					"googleGrpc": {
-						"target_uri": %q,
-						"call_credentials": [
-							{
-								"access_token": "foo"
-							},
-							{
-								"sts_service": {
-									"token_exchange_service_uri": "http://test-sts",
-									"resource": "test-resource",
-									"audience": "test-audience",
-									"scope": "test-scope",
-									"requested_token_type": "test-requested-token-type",
-									"subject_token_path": "test-subject-token-path",
-									"subject_token_type": "test-subject-token-type",
-									"actor_token_path": "test-actor-token-path",
-									"actor_token_type": "test-actor-token-type"
-								}
-							}
-						]
-					},
-					"timeout": "10s"
-				}
-			]
-		},
-		"certificate_lifetime": "86400s",
-		"renewal_grace_period":  "43200s",
-		"key_type": 1,
-		"key_size": 2048,
-		"location": "us-west1-b"
-	}`
-	goodConfigWithDefaults = json.RawMessage(`
-	{
-		"server": {
-			"api_type": 2,
-			"grpc_services": [
-				{
-					"googleGrpc": {
-						"call_credentials": [
-							{
-								"sts_service": {
-									"subject_token_path": "test-subject-token-path"
-								}
-							}
-						]
-					},
-					"timeout": "10s"
-				}
-			]
-		}
-	}`)
-)
-
-var goodConfigFullySpecified = json.RawMessage(fmt.Sprintf(goodConfigFormatStr, "test-meshca"))
-
-// verifyReceivedRequest reads the HTTP request received by the fake client
-// (exposed through a channel), and verifies that it matches the expected
-// request.
-func verifyReceivedRequest(fc *testutils.FakeHTTPClient, wantURI string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	val, err := fc.ReqChan.Receive(ctx)
-	if err != nil {
-		return err
-	}
-	gotReq := val.(*http.Request)
-	if gotURI := gotReq.URL.String(); gotURI != wantURI {
-		return fmt.Errorf("request contains URL %q want %q", gotURI, wantURI)
-	}
-	if got, want := gotReq.Header.Get("Metadata-Flavor"), "Google"; got != want {
-		return fmt.Errorf("request contains flavor %q want %q", got, want)
-	}
-	return nil
-}
-
-// TestParseConfigSuccessFullySpecified tests the case where the config is fully
-// specified and no defaults are required.
-func (s) TestParseConfigSuccessFullySpecified(t *testing.T) {
-	wantConfig := "test-meshca:http://test-sts:test-resource:test-audience:test-scope:test-requested-token-type:test-subject-token-path:test-subject-token-type:test-actor-token-path:test-actor-token-type:10s:24h0m0s:12h0m0s:RSA:2048:us-west1-b"
-
-	cfg, err := pluginConfigFromJSON(goodConfigFullySpecified)
-	if err != nil {
-		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", goodConfigFullySpecified, err)
-	}
-	gotConfig := cfg.canonical()
-	if diff := cmp.Diff(wantConfig, string(gotConfig)); diff != "" {
-		t.Errorf("pluginConfigFromJSON(%q) returned config does not match expected (-want +got):\n%s", string(goodConfigFullySpecified), diff)
-	}
-}
-
-// TestParseConfigSuccessWithDefaults tests cases where the config is not fully
-// specified, and we end up using some sane defaults.
-func (s) TestParseConfigSuccessWithDefaults(t *testing.T) {
-	wantConfig := fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s",
-		"meshca.googleapis.com",      // Mesh CA Server URI.
-		"securetoken.googleapis.com", // STS Server URI.
-		"",                           // STS Resource Name.
-		"identitynamespace:test-project-id.svc.id.goog:https://container.googleapis.com/v1/projects/test-project-id/zones/test-zone/clusters/test-gke-cluster", // STS Audience.
-		"https://www.googleapis.com/auth/cloud-platform", // STS Scope.
-		"urn:ietf:params:oauth:token-type:access_token",  // STS requested token type.
-		"test-subject-token-path",                        // STS subject token path.
-		"urn:ietf:params:oauth:token-type:jwt",           // STS subject token type.
-		"",                                               // STS actor token path.
-		"",                                               // STS actor token type.
-		"10s",                                            // Call timeout.
-		"24h0m0s",                                        // Cert life time.
-		"12h0m0s",                                        // Cert grace time.
-		"RSA",                                            // Key type
-		"2048",                                           // Key size
-		"test-zone",                                      // Zone
-	)
-
-	// We expect the config parser to make four HTTP requests and receive four
-	// responses. Hence we setup the request and response channels in the fake
-	// client with appropriate buffer size.
-	fc := &testutils.FakeHTTPClient{
-		ReqChan:  testutils.NewChannelWithSize(4),
-		RespChan: testutils.NewChannelWithSize(4),
-	}
-	// Set up the responses to be delivered to the config parser by the fake
-	// client. The config parser expects responses with project_id,
-	// gke_cluster_id and gce_zone. The zone is read twice, once as part of
-	// reading the STS audience and once to get location metadata.
-	fc.RespChan.Send(&http.Response{
-		Status:     "200 OK",
-		StatusCode: http.StatusOK,
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte(testProjectID))),
-	})
-	fc.RespChan.Send(&http.Response{
-		Status:     "200 OK",
-		StatusCode: http.StatusOK,
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte(testGKECluster))),
-	})
-	fc.RespChan.Send(&http.Response{
-		Status:     "200 OK",
-		StatusCode: http.StatusOK,
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("projects/%s/zones/%s", testProjectID, testGCEZone)))),
-	})
-	fc.RespChan.Send(&http.Response{
-		Status:     "200 OK",
-		StatusCode: http.StatusOK,
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("projects/%s/zones/%s", testProjectID, testGCEZone)))),
-	})
-	// Override the http.Client with our fakeClient.
-	origMakeHTTPDoer := makeHTTPDoer
-	makeHTTPDoer = func() httpDoer { return fc }
-	defer func() { makeHTTPDoer = origMakeHTTPDoer }()
-
-	// Spawn a goroutine to verify the HTTP requests sent out as part of the
-	// config parsing.
-	errCh := make(chan error, 1)
-	go func() {
-		if err := verifyReceivedRequest(fc, "http://metadata.google.internal/computeMetadata/v1/project/project-id"); err != nil {
-			errCh <- err
-			return
-		}
-		if err := verifyReceivedRequest(fc, "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name"); err != nil {
-			errCh <- err
-			return
-		}
-		if err := verifyReceivedRequest(fc, "http://metadata.google.internal/computeMetadata/v1/instance/zone"); err != nil {
-			errCh <- err
-			return
-		}
-		errCh <- nil
-	}()
-
-	cfg, err := pluginConfigFromJSON(goodConfigWithDefaults)
-	if err != nil {
-		t.Fatalf("pluginConfigFromJSON(%q) failed: %v", goodConfigWithDefaults, err)
-	}
-	gotConfig := cfg.canonical()
-	if diff := cmp.Diff(wantConfig, string(gotConfig)); diff != "" {
-		t.Errorf("builder.ParseConfig(%q) returned config does not match expected (-want +got):\n%s", goodConfigWithDefaults, diff)
-	}
-
-	if err := <-errCh; err != nil {
-		t.Fatal(err)
-	}
-}
-
-// TestParseConfigFailureCases tests several invalid configs which all result in
-// config parsing failures.
-func (s) TestParseConfigFailureCases(t *testing.T) {
-	tests := []struct {
-		desc        string
-		inputConfig json.RawMessage
-		wantErr     string
-	}{
-		{
-			desc:        "invalid JSON",
-			inputConfig: json.RawMessage(`bad bad json`),
-			wantErr:     "failed to unmarshal config",
-		},
-		{
-			desc: "bad apiType",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 1
-				}
-			}`),
-			wantErr: "server has apiType REST, want GRPC",
-		},
-		{
-			desc: "no grpc services",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 2
-				}
-			}`),
-			wantErr: "number of gRPC services in config is 0, expected 1",
-		},
-		{
-			desc: "too many grpc services",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 2,
-					"grpc_services": [{}, {}]
-				}
-			}`),
-			wantErr: "number of gRPC services in config is 2, expected 1",
-		},
-		{
-			desc: "missing google grpc service",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 2,
-					"grpc_services": [
-						{
-							"envoyGrpc": {}
-						}
-					]
-				}
-			}`),
-			wantErr: "missing google gRPC service in config",
-		},
-		{
-			desc: "missing call credentials",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 2,
-					"grpc_services": [
-						{
-							"googleGrpc": {
-								"target_uri": "foo"
-							}
-						}
-					]
-				}
-			}`),
-			wantErr: "missing call credentials in config",
-		},
-		{
-			desc: "missing STS call credentials",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 2,
-					"grpc_services": [
-						{
-							"googleGrpc": {
-								"target_uri": "foo",
-								"call_credentials": [
-									{
-										"access_token": "foo"
-									}
-								]
-							}
-						}
-					]
-				}
-			}`),
-			wantErr: "missing STS call credentials in config",
-		},
-		{
-			desc: "with no defaults",
-			inputConfig: json.RawMessage(`
-			{
-				"server": {
-					"api_type": 2,
-					"grpc_services": [
-						{
-							"googleGrpc": {
-								"target_uri": "foo",
-								"call_credentials": [
-									{
-										"sts_service": {}
-									}
-								]
-							}
-						}
-					]
-				}
-			}`),
-			wantErr: "missing subjectTokenPath in STS call credentials config",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.desc, func(t *testing.T) {
-			cfg, err := pluginConfigFromJSON(test.inputConfig)
-			if err == nil {
-				t.Fatalf("pluginConfigFromJSON(%q) = %v, expected to return error (%v)", test.inputConfig, string(cfg.canonical()), test.wantErr)
-
-			}
-			if !strings.Contains(err.Error(), test.wantErr) {
-				t.Fatalf("builder.ParseConfig(%q) = (%v), want error (%v)", test.inputConfig, err, test.wantErr)
-			}
-		})
-	}
-}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
deleted file mode 100644
index 387f8c55abc0..000000000000
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go
+++ /dev/null
@@ -1,276 +0,0 @@
-// Copyright 2019 Istio Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.14.0
-// source: istio/google/security/meshca/v1/meshca.proto
-
-package google_security_meshca_v1
-
-import (
-	proto "github.com/golang/protobuf/proto"
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	durationpb "google.golang.org/protobuf/types/known/durationpb"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
-// Certificate request message.
-type MeshCertificateRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The request ID must be a valid UUID with the exception that zero UUID is
-	// not supported (00000000-0000-0000-0000-000000000000).
-	RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
-	// PEM-encoded certificate request.
-	Csr string `protobuf:"bytes,2,opt,name=csr,proto3" json:"csr,omitempty"`
-	// Optional: requested certificate validity period.
-	Validity *durationpb.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"` // Reserved 4
-}
-
-func (x *MeshCertificateRequest) Reset() {
-	*x = MeshCertificateRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *MeshCertificateRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*MeshCertificateRequest) ProtoMessage() {}
-
-func (x *MeshCertificateRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use MeshCertificateRequest.ProtoReflect.Descriptor instead.
-func (*MeshCertificateRequest) Descriptor() ([]byte, []int) {
-	return file_istio_google_security_meshca_v1_meshca_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *MeshCertificateRequest) GetRequestId() string {
-	if x != nil {
-		return x.RequestId
-	}
-	return ""
-}
-
-func (x *MeshCertificateRequest) GetCsr() string {
-	if x != nil {
-		return x.Csr
-	}
-	return ""
-}
-
-func (x *MeshCertificateRequest) GetValidity() *durationpb.Duration {
-	if x != nil {
-		return x.Validity
-	}
-	return nil
-}
-
-// Certificate response message.
-type MeshCertificateResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// PEM-encoded certificate chain.
-	// Leaf cert is element '0'. Root cert is element 'n'.
-	CertChain []string `protobuf:"bytes,1,rep,name=cert_chain,json=certChain,proto3" json:"cert_chain,omitempty"`
-}
-
-func (x *MeshCertificateResponse) Reset() {
-	*x = MeshCertificateResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *MeshCertificateResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*MeshCertificateResponse) ProtoMessage() {}
-
-func (x *MeshCertificateResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_istio_google_security_meshca_v1_meshca_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use MeshCertificateResponse.ProtoReflect.Descriptor instead.
-func (*MeshCertificateResponse) Descriptor() ([]byte, []int) {
-	return file_istio_google_security_meshca_v1_meshca_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *MeshCertificateResponse) GetCertChain() []string {
-	if x != nil {
-		return x.CertChain
-	}
-	return nil
-}
-
-var File_istio_google_security_meshca_v1_meshca_proto protoreflect.FileDescriptor
-
-var file_istio_google_security_meshca_v1_meshca_proto_rawDesc = []byte{
-	0x0a, 0x2c, 0x69, 0x73, 0x74, 0x69, 0x6f, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x73,
-	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2f, 0x76,
-	0x31, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19,
-	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e,
-	0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
-	0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x80, 0x01, 0x0a, 0x16, 0x4d, 0x65,
-	0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f,
-	0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x73, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x03, 0x63, 0x73, 0x72, 0x12, 0x35, 0x0a, 0x08, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74,
-	0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x52, 0x08, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74, 0x79, 0x22, 0x38, 0x0a, 0x17,
-	0x4d, 0x65, 0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f,
-	0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x63, 0x65, 0x72,
-	0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x32, 0x96, 0x01, 0x0a, 0x16, 0x4d, 0x65, 0x73, 0x68, 0x43,
-	0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
-	0x65, 0x12, 0x7c, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69,
-	0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x31, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
-	0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e,
-	0x76, 0x31, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61,
-	0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
-	0x6c, 0x65, 0x2e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x6d, 0x65, 0x73, 0x68,
-	0x63, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66,
-	0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42,
-	0x2e, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x73, 0x65,
-	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x63, 0x61, 0x2e, 0x76, 0x31,
-	0x42, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62,
-	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-}
-
-var (
-	file_istio_google_security_meshca_v1_meshca_proto_rawDescOnce sync.Once
-	file_istio_google_security_meshca_v1_meshca_proto_rawDescData = file_istio_google_security_meshca_v1_meshca_proto_rawDesc
-)
-
-func file_istio_google_security_meshca_v1_meshca_proto_rawDescGZIP() []byte {
-	file_istio_google_security_meshca_v1_meshca_proto_rawDescOnce.Do(func() {
-		file_istio_google_security_meshca_v1_meshca_proto_rawDescData = protoimpl.X.CompressGZIP(file_istio_google_security_meshca_v1_meshca_proto_rawDescData)
-	})
-	return file_istio_google_security_meshca_v1_meshca_proto_rawDescData
-}
-
-var file_istio_google_security_meshca_v1_meshca_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
-var file_istio_google_security_meshca_v1_meshca_proto_goTypes = []interface{}{
-	(*MeshCertificateRequest)(nil),  // 0: google.security.meshca.v1.MeshCertificateRequest
-	(*MeshCertificateResponse)(nil), // 1: google.security.meshca.v1.MeshCertificateResponse
-	(*durationpb.Duration)(nil),     // 2: google.protobuf.Duration
-}
-var file_istio_google_security_meshca_v1_meshca_proto_depIdxs = []int32{
-	2, // 0: google.security.meshca.v1.MeshCertificateRequest.validity:type_name -> google.protobuf.Duration
-	0, // 1: google.security.meshca.v1.MeshCertificateService.CreateCertificate:input_type -> google.security.meshca.v1.MeshCertificateRequest
-	1, // 2: google.security.meshca.v1.MeshCertificateService.CreateCertificate:output_type -> google.security.meshca.v1.MeshCertificateResponse
-	2, // [2:3] is the sub-list for method output_type
-	1, // [1:2] is the sub-list for method input_type
-	1, // [1:1] is the sub-list for extension type_name
-	1, // [1:1] is the sub-list for extension extendee
-	0, // [0:1] is the sub-list for field type_name
-}
-
-func init() { file_istio_google_security_meshca_v1_meshca_proto_init() }
-func file_istio_google_security_meshca_v1_meshca_proto_init() {
-	if File_istio_google_security_meshca_v1_meshca_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_istio_google_security_meshca_v1_meshca_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*MeshCertificateRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_istio_google_security_meshca_v1_meshca_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*MeshCertificateResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_istio_google_security_meshca_v1_meshca_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   2,
-			NumExtensions: 0,
-			NumServices:   1,
-		},
-		GoTypes:           file_istio_google_security_meshca_v1_meshca_proto_goTypes,
-		DependencyIndexes: file_istio_google_security_meshca_v1_meshca_proto_depIdxs,
-		MessageInfos:      file_istio_google_security_meshca_v1_meshca_proto_msgTypes,
-	}.Build()
-	File_istio_google_security_meshca_v1_meshca_proto = out.File
-	file_istio_google_security_meshca_v1_meshca_proto_rawDesc = nil
-	file_istio_google_security_meshca_v1_meshca_proto_goTypes = nil
-	file_istio_google_security_meshca_v1_meshca_proto_depIdxs = nil
-}
diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
deleted file mode 100644
index 4663ff1ef35c..000000000000
--- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go
+++ /dev/null
@@ -1,110 +0,0 @@
-// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
-// versions:
-// - protoc-gen-go-grpc v1.1.0
-// - protoc             v3.14.0
-// source: istio/google/security/meshca/v1/meshca.proto
-
-package google_security_meshca_v1
-
-import (
-	context "context"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
-)
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-// Requires gRPC-Go v1.32.0 or later.
-const _ = grpc.SupportPackageIsVersion7
-
-// MeshCertificateServiceClient is the client API for MeshCertificateService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type MeshCertificateServiceClient interface {
-	// Using provided CSR, returns a signed certificate that represents a GCP
-	// service account identity.
-	CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error)
-}
-
-type meshCertificateServiceClient struct {
-	cc grpc.ClientConnInterface
-}
-
-func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient {
-	return &meshCertificateServiceClient{cc}
-}
-
-func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) {
-	out := new(MeshCertificateResponse)
-	err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// MeshCertificateServiceServer is the server API for MeshCertificateService service.
-// All implementations must embed UnimplementedMeshCertificateServiceServer
-// for forward compatibility
-type MeshCertificateServiceServer interface {
-	// Using provided CSR, returns a signed certificate that represents a GCP
-	// service account identity.
-	CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error)
-	mustEmbedUnimplementedMeshCertificateServiceServer()
-}
-
-// UnimplementedMeshCertificateServiceServer must be embedded to have forward compatible implementations.
-type UnimplementedMeshCertificateServiceServer struct {
-}
-
-func (UnimplementedMeshCertificateServiceServer) CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented")
-}
-func (UnimplementedMeshCertificateServiceServer) mustEmbedUnimplementedMeshCertificateServiceServer() {
-}
-
-// UnsafeMeshCertificateServiceServer may be embedded to opt out of forward compatibility for this service.
-// Use of this interface is not recommended, as added methods to MeshCertificateServiceServer will
-// result in compilation errors.
-type UnsafeMeshCertificateServiceServer interface {
-	mustEmbedUnimplementedMeshCertificateServiceServer()
-}
-
-func RegisterMeshCertificateServiceServer(s grpc.ServiceRegistrar, srv MeshCertificateServiceServer) {
-	s.RegisterService(&MeshCertificateService_ServiceDesc, srv)
-}
-
-func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(MeshCertificateRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-// MeshCertificateService_ServiceDesc is the grpc.ServiceDesc for MeshCertificateService service.
-// It's only intended for direct use with grpc.RegisterService,
-// and not to be introspected or modified (even as a copy)
-var MeshCertificateService_ServiceDesc = grpc.ServiceDesc{
-	ServiceName: "google.security.meshca.v1.MeshCertificateService",
-	HandlerType: (*MeshCertificateServiceServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "CreateCertificate",
-			Handler:    _MeshCertificateService_CreateCertificate_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "istio/google/security/meshca/v1/meshca.proto",
-}
diff --git a/credentials/tls/certprovider/meshca/logging.go b/credentials/tls/certprovider/meshca/logging.go
deleted file mode 100644
index ae20059c4f72..000000000000
--- a/credentials/tls/certprovider/meshca/logging.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package meshca
-
-import (
-	"fmt"
-
-	"google.golang.org/grpc/grpclog"
-	internalgrpclog "google.golang.org/grpc/internal/grpclog"
-)
-
-const prefix = "[%p] "
-
-var logger = grpclog.Component("meshca")
-
-func prefixLogger(p *providerPlugin) *internalgrpclog.PrefixLogger {
-	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
-}
diff --git a/credentials/tls/certprovider/meshca/plugin.go b/credentials/tls/certprovider/meshca/plugin.go
deleted file mode 100644
index ab1958ac1fd0..000000000000
--- a/credentials/tls/certprovider/meshca/plugin.go
+++ /dev/null
@@ -1,289 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Package meshca provides an implementation of the Provider interface which
-// communicates with MeshCA to get certificates signed.
-package meshca
-
-import (
-	"context"
-	"crypto"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-	"time"
-
-	durationpb "github.com/golang/protobuf/ptypes/duration"
-	"github.com/google/uuid"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	meshgrpc "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
-	meshpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
-	"google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/metadata"
-)
-
-// In requests sent to the MeshCA, we add a metadata header with this key and
-// the value being the GCE zone in which the workload is running in.
-const locationMetadataKey = "x-goog-request-params"
-
-// For overriding from unit tests.
-var newDistributorFunc = func() distributor { return certprovider.NewDistributor() }
-
-// distributor wraps the methods on certprovider.Distributor which are used by
-// the plugin. This is very useful in tests which need to know exactly when the
-// plugin updates its key material.
-type distributor interface {
-	KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error)
-	Set(km *certprovider.KeyMaterial, err error)
-	Stop()
-}
-
-// providerPlugin is an implementation of the certprovider.Provider interface,
-// which gets certificates signed by communicating with the MeshCA.
-type providerPlugin struct {
-	distributor // Holds the key material.
-	cancel      context.CancelFunc
-	cc          *grpc.ClientConn          // Connection to MeshCA server.
-	cfg         *pluginConfig             // Plugin configuration.
-	opts        certprovider.BuildOptions // Key material options.
-	logger      *grpclog.PrefixLogger     // Plugin instance specific prefix.
-	backoff     func(int) time.Duration   // Exponential backoff.
-	doneFunc    func()                    // Notify the builder when done.
-}
-
-// providerParams wraps params passed to the provider plugin at creation time.
-type providerParams struct {
-	// This ClientConn to the MeshCA server is owned by the builder.
-	cc       *grpc.ClientConn
-	cfg      *pluginConfig
-	opts     certprovider.BuildOptions
-	backoff  func(int) time.Duration
-	doneFunc func()
-}
-
-func newProviderPlugin(params providerParams) *providerPlugin {
-	ctx, cancel := context.WithCancel(context.Background())
-	p := &providerPlugin{
-		cancel:      cancel,
-		cc:          params.cc,
-		cfg:         params.cfg,
-		opts:        params.opts,
-		backoff:     params.backoff,
-		doneFunc:    params.doneFunc,
-		distributor: newDistributorFunc(),
-	}
-	p.logger = prefixLogger((p))
-	p.logger.Infof("plugin created")
-	go p.run(ctx)
-	return p
-}
-
-func (p *providerPlugin) Close() {
-	p.logger.Infof("plugin closed")
-	p.Stop() // Stop the embedded distributor.
-	p.cancel()
-	p.doneFunc()
-}
-
-// run is a long running goroutine which periodically sends out CSRs to the
-// MeshCA, and updates the underlying Distributor with the new key material.
-func (p *providerPlugin) run(ctx context.Context) {
-	// We need to start fetching key material right away. The next attempt will
-	// be triggered by the timer firing.
-	for {
-		certValidity, err := p.updateKeyMaterial(ctx)
-		if err != nil {
-			return
-		}
-
-		// We request a certificate with the configured validity duration (which
-		// is usually twice as much as the grace period). But the server is free
-		// to return a certificate with whatever validity time it deems right.
-		refreshAfter := p.cfg.certGraceTime
-		if refreshAfter > certValidity {
-			// The default value of cert grace time is half that of the default
-			// cert validity time. So here, when we have to use a non-default
-			// cert life time, we will set the grace time again to half that of
-			// the validity time.
-			refreshAfter = certValidity / 2
-		}
-		timer := time.NewTimer(refreshAfter)
-		select {
-		case <-ctx.Done():
-			return
-		case <-timer.C:
-		}
-	}
-}
-
-// updateKeyMaterial generates a CSR and attempts to get it signed from the
-// MeshCA. It retries with an exponential backoff till it succeeds or the
-// deadline specified in ctx expires. Once it gets the CSR signed from the
-// MeshCA, it updates the Distributor with the new key material.
-//
-// It returns the amount of time the new certificate is valid for.
-func (p *providerPlugin) updateKeyMaterial(ctx context.Context) (time.Duration, error) {
-	client := meshgrpc.NewMeshCertificateServiceClient(p.cc)
-	retries := 0
-	for {
-		if ctx.Err() != nil {
-			return 0, ctx.Err()
-		}
-
-		if retries != 0 {
-			bi := p.backoff(retries)
-			p.logger.Warningf("Backing off for %s before attempting the next CreateCertificate() request", bi)
-			timer := time.NewTimer(bi)
-			select {
-			case <-timer.C:
-			case <-ctx.Done():
-				return 0, ctx.Err()
-			}
-		}
-		retries++
-
-		privKey, err := rsa.GenerateKey(rand.Reader, p.cfg.keySize)
-		if err != nil {
-			p.logger.Warningf("RSA key generation failed: %v", err)
-			continue
-		}
-		// We do not set any fields in the CSR (we use an empty
-		// x509.CertificateRequest as the template) because the MeshCA discards
-		// them anyways, and uses the workload identity from the access token
-		// that we present (as part of the STS call creds).
-		csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, crypto.PrivateKey(privKey))
-		if err != nil {
-			p.logger.Warningf("CSR creation failed: %v", err)
-			continue
-		}
-		csrPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})
-
-		// Send out the CSR with a call timeout and location metadata, as
-		// specified in the plugin configuration.
-		req := &meshpb.MeshCertificateRequest{
-			RequestId: uuid.New().String(),
-			Csr:       string(csrPEM),
-			Validity:  &durationpb.Duration{Seconds: int64(p.cfg.certLifetime / time.Second)},
-		}
-		p.logger.Debugf("Sending CreateCertificate() request: %v", req)
-
-		callCtx, ctxCancel := context.WithTimeout(context.Background(), p.cfg.callTimeout)
-		callCtx = metadata.NewOutgoingContext(callCtx, metadata.Pairs(locationMetadataKey, p.cfg.location))
-		resp, err := client.CreateCertificate(callCtx, req)
-		if err != nil {
-			p.logger.Warningf("CreateCertificate request failed: %v", err)
-			ctxCancel()
-			continue
-		}
-		ctxCancel()
-
-		// The returned cert chain must contain more than one cert. Leaf cert is
-		// element '0', while root cert is element 'n', and the intermediate
-		// entries form the chain from the root to the leaf.
-		certChain := resp.GetCertChain()
-		if l := len(certChain); l <= 1 {
-			p.logger.Errorf("Received certificate chain contains %d certificates, need more than one", l)
-			continue
-		}
-
-		// We need to explicitly parse the PEM cert contents as an
-		// x509.Certificate to read the certificate validity period. We use this
-		// to decide when to refresh the cert. Even though the call to
-		// tls.X509KeyPair actually parses the PEM contents into an
-		// x509.Certificate, it does not store that in the `Leaf` field. See:
-		// https://golang.org/pkg/crypto/tls/#X509KeyPair.
-		identity, intermediates, roots, err := parseCertChain(certChain)
-		if err != nil {
-			p.logger.Errorf(err.Error())
-			continue
-		}
-		_, err = identity.Verify(x509.VerifyOptions{
-			Intermediates: intermediates,
-			Roots:         roots,
-		})
-		if err != nil {
-			p.logger.Errorf("Certificate verification failed for return certChain: %v", err)
-			continue
-		}
-
-		key := x509.MarshalPKCS1PrivateKey(privKey)
-		keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: key})
-		certPair, err := tls.X509KeyPair([]byte(certChain[0]), keyPEM)
-		if err != nil {
-			p.logger.Errorf("Failed to create x509 key pair: %v", err)
-			continue
-		}
-
-		// At this point, the received response has been deemed good.
-		retries = 0
-
-		// All certs signed by the MeshCA roll up to the same root. And treating
-		// the last element of the returned chain as the root is the only
-		// supported option to get the root certificate. So, we ignore the
-		// options specified in the call to Build(), which contain certificate
-		// name and whether the caller is interested in identity or root cert.
-		p.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{certPair}, Roots: roots}, nil)
-		return time.Until(identity.NotAfter), nil
-	}
-}
-
-// ParseCertChain parses the result returned by the MeshCA which consists of a
-// list of PEM encoded certs. The first element in the list is the leaf or
-// identity cert, while the last element is the root, and everything in between
-// form the chain of trust.
-//
-// Caller needs to make sure that certChain has at least two elements.
-func parseCertChain(certChain []string) (*x509.Certificate, *x509.CertPool, *x509.CertPool, error) {
-	identity, err := parseCert([]byte(certChain[0]))
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	intermediates := x509.NewCertPool()
-	for _, cert := range certChain[1 : len(certChain)-1] {
-		i, err := parseCert([]byte(cert))
-		if err != nil {
-			return nil, nil, nil, err
-		}
-		intermediates.AddCert(i)
-	}
-
-	roots := x509.NewCertPool()
-	root, err := parseCert([]byte(certChain[len(certChain)-1]))
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	roots.AddCert(root)
-
-	return identity, intermediates, roots, nil
-}
-
-func parseCert(certPEM []byte) (*x509.Certificate, error) {
-	block, _ := pem.Decode(certPEM)
-	if block == nil {
-		return nil, fmt.Errorf("failed to decode received PEM data: %v", certPEM)
-	}
-	return x509.ParseCertificate(block.Bytes)
-}
diff --git a/credentials/tls/certprovider/meshca/plugin_test.go b/credentials/tls/certprovider/meshca/plugin_test.go
deleted file mode 100644
index 51f545d6a0e3..000000000000
--- a/credentials/tls/certprovider/meshca/plugin_test.go
+++ /dev/null
@@ -1,459 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package meshca
-
-import (
-	"context"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/json"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"math/big"
-	"net"
-	"reflect"
-	"testing"
-	"time"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/tls/certprovider"
-	meshgrpc "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
-	meshpb "google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1"
-	"google.golang.org/grpc/internal/testutils"
-)
-
-const (
-	// Used when waiting for something that is expected to *not* happen.
-	defaultTestShortTimeout = 10 * time.Millisecond
-	defaultTestTimeout      = 5 * time.Second
-	defaultTestCertLife     = time.Hour
-	shortTestCertLife       = 2 * time.Second
-	maxErrCount             = 2
-)
-
-// fakeCA provides a very simple fake implementation of the certificate signing
-// service as exported by the MeshCA.
-type fakeCA struct {
-	meshgrpc.UnimplementedMeshCertificateServiceServer
-
-	withErrors    bool // Whether the CA returns errors to begin with.
-	withShortLife bool // Whether to create certs with short lifetime
-
-	ccChan  *testutils.Channel // Channel to get notified about CreateCertificate calls.
-	errors  int                // Error count.
-	key     *rsa.PrivateKey    // Private key of CA.
-	cert    *x509.Certificate  // Signing certificate.
-	certPEM []byte             // PEM encoding of signing certificate.
-}
-
-// Returns a new instance of the fake Mesh CA. It generates a new RSA key and a
-// self-signed certificate which will be used to sign CSRs received in incoming
-// requests.
-// withErrors controls whether the fake returns errors before succeeding, while
-// withShortLife controls whether the fake returns certs with very small
-// lifetimes (to test plugin refresh behavior). Every time a CreateCertificate()
-// call succeeds, an event is pushed on the ccChan.
-func newFakeMeshCA(ccChan *testutils.Channel, withErrors, withShortLife bool) (*fakeCA, error) {
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return nil, fmt.Errorf("RSA key generation failed: %v", err)
-	}
-
-	now := time.Now()
-	tmpl := &x509.Certificate{
-		Subject:               pkix.Name{CommonName: "my-fake-ca"},
-		SerialNumber:          big.NewInt(10),
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.Add(time.Hour),
-		KeyUsage:              x509.KeyUsageCertSign,
-		IsCA:                  true,
-		BasicConstraintsValid: true,
-	}
-	certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
-	if err != nil {
-		return nil, fmt.Errorf("x509.CreateCertificate(%v) failed: %v", tmpl, err)
-	}
-	// The PEM encoding of the self-signed certificate is stored because we need
-	// to return a chain of certificates in the response, starting with the
-	// client certificate and ending in the root.
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-	cert, err := x509.ParseCertificate(certDER)
-	if err != nil {
-		return nil, fmt.Errorf("x509.ParseCertificate(%v) failed: %v", certDER, err)
-	}
-
-	return &fakeCA{
-		withErrors:    withErrors,
-		withShortLife: withShortLife,
-		ccChan:        ccChan,
-		key:           key,
-		cert:          cert,
-		certPEM:       certPEM,
-	}, nil
-}
-
-// CreateCertificate helps implement the MeshCA service.
-//
-// If the fakeMeshCA was created with `withErrors` set to true, the first
-// `maxErrCount` number of RPC return errors. Subsequent requests are signed and
-// returned without error.
-func (f *fakeCA) CreateCertificate(ctx context.Context, req *meshpb.MeshCertificateRequest) (*meshpb.MeshCertificateResponse, error) {
-	if f.withErrors {
-		if f.errors < maxErrCount {
-			f.errors++
-			return nil, errors.New("fake Mesh CA error")
-
-		}
-	}
-
-	csrPEM := []byte(req.GetCsr())
-	block, _ := pem.Decode(csrPEM)
-	if block == nil {
-		return nil, fmt.Errorf("failed to decode received CSR: %v", csrPEM)
-	}
-	csr, err := x509.ParseCertificateRequest(block.Bytes)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse received CSR: %v", csrPEM)
-	}
-
-	// By default, we create certs which are valid for an hour. But if
-	// `withShortLife` is set, we create certs which are valid only for a couple
-	// of seconds.
-	now := time.Now()
-	notBefore, notAfter := now.Add(-defaultTestCertLife), now.Add(defaultTestCertLife)
-	if f.withShortLife {
-		notBefore, notAfter = now.Add(-shortTestCertLife), now.Add(shortTestCertLife)
-	}
-	tmpl := &x509.Certificate{
-		Subject:      pkix.Name{CommonName: "signed-cert"},
-		SerialNumber: big.NewInt(10),
-		NotBefore:    notBefore,
-		NotAfter:     notAfter,
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-	}
-	certDER, err := x509.CreateCertificate(rand.Reader, tmpl, f.cert, csr.PublicKey, f.key)
-	if err != nil {
-		return nil, fmt.Errorf("x509.CreateCertificate(%v) failed: %v", tmpl, err)
-	}
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	// Push to ccChan to indicate that the RPC is processed.
-	f.ccChan.Send(nil)
-
-	certChain := []string{
-		string(certPEM),   // Signed certificate corresponding to CSR
-		string(f.certPEM), // Root certificate
-	}
-	return &meshpb.MeshCertificateResponse{CertChain: certChain}, nil
-}
-
-// opts wraps the options to be passed to setup.
-type opts struct {
-	// Whether the CA returns certs with short lifetime. Used to test client refresh.
-	withShortLife bool
-	// Whether the CA returns errors to begin with. Used to test client backoff.
-	withbackoff bool
-}
-
-// events wraps channels which indicate different events.
-type events struct {
-	// Pushed to when the plugin dials the MeshCA.
-	dialDone *testutils.Channel
-	// Pushed to when CreateCertifcate() succeeds on the MeshCA.
-	createCertDone *testutils.Channel
-	// Pushed to when the plugin updates the distributor with new key material.
-	keyMaterialDone *testutils.Channel
-	// Pushed to when the client backs off after a failed CreateCertificate().
-	backoffDone *testutils.Channel
-}
-
-// setup performs tasks common to all tests in this file.
-func setup(t *testing.T, o opts) (events, string, func()) {
-	t.Helper()
-
-	// Create a fake MeshCA which pushes events on the passed channel for
-	// successful RPCs.
-	createCertDone := testutils.NewChannel()
-	fs, err := newFakeMeshCA(createCertDone, o.withbackoff, o.withShortLife)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create a gRPC server and register the fake MeshCA on it.
-	server := grpc.NewServer()
-	meshgrpc.RegisterMeshCertificateServiceServer(server, fs)
-
-	// Start a net.Listener on a local port, and pass it to the gRPC server
-	// created above and start serving.
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-	addr := lis.Addr().String()
-	go server.Serve(lis)
-
-	// Override the plugin's dial function and perform a blocking dial. Also
-	// push on dialDone once the dial is complete so that test can block on this
-	// event before verifying other things.
-	dialDone := testutils.NewChannel()
-	origDialFunc := grpcDialFunc
-	grpcDialFunc = func(uri string, _ ...grpc.DialOption) (*grpc.ClientConn, error) {
-		if uri != addr {
-			t.Fatalf("plugin dialing MeshCA at %s, want %s", uri, addr)
-		}
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		cc, err := grpc.DialContext(ctx, uri, grpc.WithInsecure(), grpc.WithBlock())
-		if err != nil {
-			t.Fatalf("grpc.DialContext(%s) failed: %v", addr, err)
-		}
-		dialDone.Send(nil)
-		return cc, nil
-	}
-
-	// Override the plugin's newDistributorFunc and return a wrappedDistributor
-	// which allows the test to be notified whenever the plugin pushes new key
-	// material into the distributor.
-	origDistributorFunc := newDistributorFunc
-	keyMaterialDone := testutils.NewChannel()
-	d := newWrappedDistributor(keyMaterialDone)
-	newDistributorFunc = func() distributor { return d }
-
-	// Override the plugin's backoff function to perform no real backoff, but
-	// push on a channel so that the test can verifiy that backoff actually
-	// happened.
-	backoffDone := testutils.NewChannelWithSize(maxErrCount)
-	origBackoffFunc := backoffFunc
-	if o.withbackoff {
-		// Override the plugin's backoff function with this, so that we can verify
-		// that a backoff actually was triggered.
-		backoffFunc = func(v int) time.Duration {
-			backoffDone.Send(v)
-			return 0
-		}
-	}
-
-	// Return all the channels, and a cancel function to undo all the overrides.
-	e := events{
-		dialDone:        dialDone,
-		createCertDone:  createCertDone,
-		keyMaterialDone: keyMaterialDone,
-		backoffDone:     backoffDone,
-	}
-	done := func() {
-		server.Stop()
-		grpcDialFunc = origDialFunc
-		newDistributorFunc = origDistributorFunc
-		backoffFunc = origBackoffFunc
-	}
-	return e, addr, done
-}
-
-// wrappedDistributor wraps a distributor and pushes on a channel whenever new
-// key material is pushed to the distributor.
-type wrappedDistributor struct {
-	*certprovider.Distributor
-	kmChan *testutils.Channel
-}
-
-func newWrappedDistributor(kmChan *testutils.Channel) *wrappedDistributor {
-	return &wrappedDistributor{
-		kmChan:      kmChan,
-		Distributor: certprovider.NewDistributor(),
-	}
-}
-
-func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) {
-	wd.Distributor.Set(km, err)
-	wd.kmChan.Send(nil)
-}
-
-// TestCreateCertificate verifies the simple case where the MeshCA server
-// returns a good certificate.
-func (s) TestCreateCertificate(t *testing.T) {
-	e, addr, cancel := setup(t, opts{})
-	defer cancel()
-
-	// Set the MeshCA targetURI to point to our fake MeshCA.
-	inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, addr))
-
-	// Lookup MeshCA plugin builder, parse config and start the plugin.
-	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s) failed: %v", pluginName, string(inputConfig), err)
-	}
-	defer prov.Close()
-
-	// Wait till the plugin dials the MeshCA server.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := e.dialDone.Receive(ctx); err != nil {
-		t.Fatal("timeout waiting for plugin to dial MeshCA")
-	}
-
-	// Wait till the plugin makes a CreateCertificate() call.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := e.createCertDone.Receive(ctx); err != nil {
-		t.Fatal("timeout waiting for plugin to make CreateCertificate RPC")
-	}
-
-	// We don't really care about the exact key material returned here. All we
-	// care about is whether we get any key material at all, and that we don't
-	// get any errors.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err = prov.KeyMaterial(ctx); err != nil {
-		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
-	}
-}
-
-// TestCreateCertificateWithBackoff verifies the case where the MeshCA server
-// returns errors initially and then returns a good certificate. The test makes
-// sure that the client backs off when the server returns errors.
-func (s) TestCreateCertificateWithBackoff(t *testing.T) {
-	e, addr, cancel := setup(t, opts{withbackoff: true})
-	defer cancel()
-
-	// Set the MeshCA targetURI to point to our fake MeshCA.
-	inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, addr))
-
-	// Lookup MeshCA plugin builder, parse config and start the plugin.
-	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s) failed: %v", pluginName, string(inputConfig), err)
-	}
-	defer prov.Close()
-
-	// Wait till the plugin dials the MeshCA server.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := e.dialDone.Receive(ctx); err != nil {
-		t.Fatal("timeout waiting for plugin to dial MeshCA")
-	}
-
-	// Making the CreateCertificateRPC involves generating the keys, creating
-	// the CSR etc which seem to take reasonable amount of time. And in this
-	// test, the first two attempts will fail. Hence we give it a reasonable
-	// deadline here.
-	ctx, cancel = context.WithTimeout(context.Background(), 3*defaultTestTimeout)
-	defer cancel()
-	if _, err := e.createCertDone.Receive(ctx); err != nil {
-		t.Fatal("timeout waiting for plugin to make CreateCertificate RPC")
-	}
-
-	// The first `maxErrCount` calls to CreateCertificate end in failure, and
-	// should lead to a backoff.
-	for i := 0; i < maxErrCount; i++ {
-		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-		defer cancel()
-		if _, err := e.backoffDone.Receive(ctx); err != nil {
-			t.Fatalf("plugin failed to backoff after error from fake server: %v", err)
-		}
-	}
-
-	// We don't really care about the exact key material returned here. All we
-	// care about is whether we get any key material at all, and that we don't
-	// get any errors.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err = prov.KeyMaterial(ctx); err != nil {
-		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
-	}
-}
-
-// TestCreateCertificateWithRefresh verifies the case where the MeshCA returns a
-// certificate with a really short lifetime, and makes sure that the plugin
-// refreshes the cert in time.
-func (s) TestCreateCertificateWithRefresh(t *testing.T) {
-	e, addr, cancel := setup(t, opts{withShortLife: true})
-	defer cancel()
-
-	// Set the MeshCA targetURI to point to our fake MeshCA.
-	inputConfig := json.RawMessage(fmt.Sprintf(goodConfigFormatStr, addr))
-
-	// Lookup MeshCA plugin builder, parse config and start the plugin.
-	prov, err := certprovider.GetProvider(pluginName, inputConfig, certprovider.BuildOptions{})
-	if err != nil {
-		t.Fatalf("GetProvider(%s, %s) failed: %v", pluginName, string(inputConfig), err)
-	}
-	defer prov.Close()
-
-	// Wait till the plugin dials the MeshCA server.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := e.dialDone.Receive(ctx); err != nil {
-		t.Fatal("timeout waiting for plugin to dial MeshCA")
-	}
-
-	// Wait till the plugin makes a CreateCertificate() call.
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if _, err := e.createCertDone.Receive(ctx); err != nil {
-		t.Fatal("timeout waiting for plugin to make CreateCertificate RPC")
-	}
-
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	km1, err := prov.KeyMaterial(ctx)
-	if err != nil {
-		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
-	}
-
-	// At this point, we have read the first key material, and since the
-	// returned key material has a really short validity period, we expect the
-	// key material to be refreshed quite soon. We drain the channel on which
-	// the event corresponding to setting of new key material is pushed. This
-	// enables us to block on the same channel, waiting for refreshed key
-	// material.
-	// Since we do not expect this call to block, it is OK to pass the
-	// background context.
-	e.keyMaterialDone.Receive(context.Background())
-
-	// Wait for the next call to CreateCertificate() to refresh the certificate
-	// returned earlier.
-	ctx, cancel = context.WithTimeout(context.Background(), 2*shortTestCertLife)
-	defer cancel()
-	if _, err := e.keyMaterialDone.Receive(ctx); err != nil {
-		t.Fatalf("CreateCertificate() RPC not made: %v", err)
-	}
-
-	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	km2, err := prov.KeyMaterial(ctx)
-	if err != nil {
-		t.Fatalf("provider.KeyMaterial(ctx) failed: %v", err)
-	}
-
-	// TODO(easwars): Remove all references to reflect.DeepEqual and use
-	// cmp.Equal instead. Currently, the later panics because x509.Certificate
-	// type defines an Equal method, but does not check for nil. This has been
-	// fixed in
-	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
-	// but this is only available starting go1.14. So, once we remove support
-	// for go1.13, we can make the switch.
-	if reflect.DeepEqual(km1, km2) {
-		t.Error("certificate refresh did not happen in the background")
-	}
-}
diff --git a/regenerate.sh b/regenerate.sh
index fc6725b89f84..dfd3226a1d96 100755
--- a/regenerate.sh
+++ b/regenerate.sh
@@ -48,11 +48,6 @@ mkdir -p ${WORKDIR}/googleapis/google/rpc
 echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
 curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > ${WORKDIR}/googleapis/google/rpc/code.proto
 
-# Pull in the MeshCA service proto.
-mkdir -p ${WORKDIR}/istio/istio/google/security/meshca/v1
-echo "curl https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto"
-curl --silent https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto > ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
-
 mkdir -p ${WORKDIR}/out
 
 # Generates sources without the embed requirement
@@ -76,7 +71,6 @@ SOURCES=(
   ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto
   ${WORKDIR}/grpc-proto/grpc/testing/*.proto
   ${WORKDIR}/grpc-proto/grpc/core/*.proto
-  ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto
 )
 
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
@@ -122,8 +116,4 @@ mv ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_s
 mv ${WORKDIR}/out/grpc/testing/*.pb.go interop/grpc_testing/
 mv ${WORKDIR}/out/grpc/core/*.pb.go interop/grpc_testing/core/
 
-# istio/google/security/meshca/v1/meshca.proto does not have a go_package option.
-mkdir -p ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/
-mv ${WORKDIR}/out/istio/google/security/meshca/v1/* ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/
-
 cp -R ${WORKDIR}/out/google.golang.org/grpc/* .
diff --git a/xds/go113.go b/xds/go113.go
deleted file mode 100644
index 40f82cde5c1e..000000000000
--- a/xds/go113.go
+++ /dev/null
@@ -1,25 +0,0 @@
-// +build go1.13
-
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package xds
-
-import (
-	_ "google.golang.org/grpc/credentials/tls/certprovider/meshca" // Register the MeshCA certificate provider plugin.
-)

From 11bd77660dba95e270659c6a5077507ef37a8c41 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 4 May 2021 14:51:32 -0700
Subject: [PATCH 447/481] xds: work around xdsclient race in fault injection
 test (#4377)

---
 xds/internal/httpfilter/fault/fault_test.go   | 54 +++++++++----------
 xds/internal/testutils/e2e/clientresources.go |  6 +--
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 20de9b9a697a..65f616f44ea7 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -465,15 +465,8 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 
 	fs, nodeID, port, cleanup := clientSetup(t)
 	defer cleanup()
-	resources := e2e.DefaultClientResources("myservice", nodeID, "localhost", port)
-	hcm := new(v3httppb.HttpConnectionManager)
-	err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	routerFilter := hcm.HttpFilters[len(hcm.HttpFilters)-1]
 
-	for _, tc := range testCases {
+	for tcNum, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			defer func() { randIntn = grpcrand.Intn; newTimer = time.NewTimer }()
 			var intnCalls []int
@@ -489,6 +482,15 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 				return time.NewTimer(0)
 			}
 
+			serviceName := fmt.Sprintf("myservice%d", tcNum)
+			resources := e2e.DefaultClientResources(serviceName, nodeID, "localhost", port)
+			hcm := new(v3httppb.HttpConnectionManager)
+			err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm)
+			if err != nil {
+				t.Fatal(err)
+			}
+			routerFilter := hcm.HttpFilters[len(hcm.HttpFilters)-1]
+
 			hcm.HttpFilters = nil
 			for i, cfg := range tc.cfgs {
 				hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter(fmt.Sprintf("fault%d", i), cfg))
@@ -506,7 +508,7 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 			}
 
 			// Create a ClientConn and run the test case.
-			cc, err := grpc.Dial("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials()))
+			cc, err := grpc.Dial("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()))
 			if err != nil {
 				t.Fatalf("failed to dial local test server: %v", err)
 			}
@@ -515,25 +517,23 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 			client := testpb.NewTestServiceClient(cc)
 			count := 0
 			for _, want := range tc.want {
-				t.Run(want.name, func(t *testing.T) {
-					ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-					defer cancel()
-					if want.repeat == 0 {
-						t.Fatalf("invalid repeat count")
-					}
-					for n := 0; n < want.repeat; n++ {
-						intnCalls = nil
-						newTimerCalls = nil
-						ctx = metadata.NewOutgoingContext(ctx, want.md)
-						_, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true))
-						t.Logf("RPC %d: err: %v, intnCalls: %v, newTimerCalls: %v", count, err, intnCalls, newTimerCalls)
-						if status.Code(err) != want.code || !reflect.DeepEqual(intnCalls, want.randIn) || !reflect.DeepEqual(newTimerCalls, want.delays) {
-							t.Errorf("WANTED code: %v, intnCalls: %v, newTimerCalls: %v", want.code, want.randIn, want.delays)
-						}
-						randOut += tc.randOutInc
-						count++
+				ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+				defer cancel()
+				if want.repeat == 0 {
+					t.Fatalf("invalid repeat count")
+				}
+				for n := 0; n < want.repeat; n++ {
+					intnCalls = nil
+					newTimerCalls = nil
+					ctx = metadata.NewOutgoingContext(ctx, want.md)
+					_, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true))
+					t.Logf("%v: RPC %d: err: %v, intnCalls: %v, newTimerCalls: %v", want.name, count, err, intnCalls, newTimerCalls)
+					if status.Code(err) != want.code || !reflect.DeepEqual(intnCalls, want.randIn) || !reflect.DeepEqual(newTimerCalls, want.delays) {
+						t.Fatalf("WANTED code: %v, intnCalls: %v, newTimerCalls: %v", want.code, want.randIn, want.delays)
 					}
-				})
+					randOut += tc.randOutInc
+					count++
+				}
 			}
 		})
 	}
diff --git a/xds/internal/testutils/e2e/clientresources.go b/xds/internal/testutils/e2e/clientresources.go
index 79424b13b918..86b34861e3db 100644
--- a/xds/internal/testutils/e2e/clientresources.go
+++ b/xds/internal/testutils/e2e/clientresources.go
@@ -45,9 +45,9 @@ func any(m proto.Message) *anypb.Any {
 // DefaultClientResources returns a set of resources (LDS, RDS, CDS, EDS) for a
 // client to generically connect to one server.
 func DefaultClientResources(target, nodeID, host string, port uint32) UpdateOptions {
-	const routeConfigName = "route"
-	const clusterName = "cluster"
-	const endpointsName = "endpoints"
+	routeConfigName := "route-" + target
+	clusterName := "cluster-" + target
+	endpointsName := "endpoints-" + target
 
 	return UpdateOptions{
 		NodeID:    nodeID,

From 79e55d64442716d4082d373540eac78b018e81c4 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 4 May 2021 15:06:43 -0700
Subject: [PATCH 448/481] xds: use SendContext() to fail in time when the
 channel is full (#4386)

---
 xds/internal/test/xds_server_serving_mode_test.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/xds/internal/test/xds_server_serving_mode_test.go b/xds/internal/test/xds_server_serving_mode_test.go
index 0055bc7be508..c1f9634b16b0 100644
--- a/xds/internal/test/xds_server_serving_mode_test.go
+++ b/xds/internal/test/xds_server_serving_mode_test.go
@@ -59,12 +59,17 @@ func newModeTracker() *modeTracker {
 	}
 }
 
-func (mt *modeTracker) updateMode(addr net.Addr, mode xds.ServingMode) {
+func (mt *modeTracker) updateMode(ctx context.Context, addr net.Addr, mode xds.ServingMode) {
 	mt.mu.Lock()
 	defer mt.mu.Unlock()
 
 	mt.modes[addr.String()] = mode
-	mt.updateCh.Send(nil)
+	// Sometimes we could get state updates which are not expected by the test.
+	// Using `Send()` here would block in that case and cause the whole test to
+	// hang and will eventually only timeout when the `-timeout` passed to `go
+	// test` elapses. Using `SendContext()` here instead fails the test within a
+	// reasonable timeout.
+	mt.updateCh.SendContext(ctx, nil)
 }
 
 func (mt *modeTracker) getMode(addr net.Addr) xds.ServingMode {
@@ -120,10 +125,12 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	}
 
 	// Create a server option to get notified about serving mode changes.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 	modeTracker := newModeTracker()
 	modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
 		t.Logf("serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
-		modeTracker.updateMode(addr, args.Mode)
+		modeTracker.updateMode(ctx, addr, args.Mode)
 	})
 
 	// Initialize an xDS-enabled gRPC server and register the stubServer on it.
@@ -164,8 +171,6 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	}
 
 	// Wait for both listeners to move to "serving" mode.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
 	if err := waitForModeChange(ctx, modeTracker, lis1.Addr(), xds.ServingModeServing); err != nil {
 		t.Fatal(err)
 	}

From 4f3aa7cfa157c38bd5c2da7f4568614f815ab4ad Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 4 May 2021 15:29:58 -0700
Subject: [PATCH 449/481] xds: optimize fault injection filter with empty
 config (#4367)

---
 xds/internal/httpfilter/fault/fault.go      |  7 ++++++-
 xds/internal/httpfilter/fault/fault_test.go | 22 +++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/xds/internal/httpfilter/fault/fault.go b/xds/internal/httpfilter/fault/fault.go
index 639a08db8e3c..ee2ed9fd4922 100644
--- a/xds/internal/httpfilter/fault/fault.go
+++ b/xds/internal/httpfilter/fault/fault.go
@@ -125,7 +125,12 @@ func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (ir
 		}
 	}
 
-	return &interceptor{config: c.config}, nil
+	icfg := c.config
+	if (icfg.GetMaxActiveFaults() != nil && icfg.GetMaxActiveFaults().GetValue() == 0) ||
+		(icfg.GetDelay() == nil && icfg.GetAbort() == nil) {
+		return nil, nil
+	}
+	return &interceptor{config: icfg}, nil
 }
 
 type interceptor struct {
diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 65f616f44ea7..624bc9f23461 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -161,6 +161,28 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 		randOutInc int
 		want       []subcase
 	}{{
+		name: "max faults zero",
+		cfgs: []*fpb.HTTPFault{{
+			MaxActiveFaults: wrapperspb.UInt32(0),
+			Abort: &fpb.FaultAbort{
+				Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED},
+				ErrorType:  &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)},
+			},
+		}},
+		randOutInc: 5,
+		want: []subcase{{
+			code:   codes.OK,
+			repeat: 25,
+		}},
+	}, {
+		name:       "no abort or delay",
+		cfgs:       []*fpb.HTTPFault{{}},
+		randOutInc: 5,
+		want: []subcase{{
+			code:   codes.OK,
+			repeat: 25,
+		}},
+	}, {
 		name: "abort always",
 		cfgs: []*fpb.HTTPFault{{
 			Abort: &fpb.FaultAbort{

From 0fc0397d779d96879d7b903c3fa1b9bd53e490e3 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 4 May 2021 16:54:57 -0700
Subject: [PATCH 450/481] xds: actually close stuff in cds/eds `Close()`
 (#4381)

---
 xds/internal/balancer/cdsbalancer/cdsbalancer.go | 8 ++++----
 xds/internal/balancer/edsbalancer/eds.go         | 6 +++++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index b991981c14c0..c072af45e00b 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -78,6 +78,7 @@ func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.
 		bOpts:       opts,
 		updateCh:    buffer.NewUnbounded(),
 		closed:      grpcsync.NewEvent(),
+		done:        grpcsync.NewEvent(),
 		cancelWatch: func() {}, // No-op at this point.
 		xdsHI:       xdsinternal.NewHandshakeInfo(nil, nil),
 	}
@@ -181,6 +182,7 @@ type cdsBalancer struct {
 	clusterToWatch string
 	logger         *grpclog.PrefixLogger
 	closed         *grpcsync.Event
+	done           *grpcsync.Event
 
 	// The certificate providers are cached here to that they can be closed when
 	// a new provider is to be created.
@@ -380,9 +382,6 @@ func (b *cdsBalancer) run() {
 			case *watchUpdate:
 				b.handleWatchUpdate(update)
 			}
-
-		// Close results in cancellation of the CDS watch and closing of the
-		// underlying edsBalancer and is the only way to exit this goroutine.
 		case <-b.closed.Done():
 			b.cancelWatch()
 			b.cancelWatch = func() {}
@@ -392,8 +391,8 @@ func (b *cdsBalancer) run() {
 				b.edsLB = nil
 			}
 			b.xdsClient.Close()
-			// This is the *ONLY* point of return from this function.
 			b.logger.Infof("Shutdown")
+			b.done.Fire()
 			return
 		}
 	}
@@ -494,6 +493,7 @@ func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 // Close closes the cdsBalancer and the underlying edsBalancer.
 func (b *cdsBalancer) Close() {
 	b.closed.Fire()
+	<-b.done.Done()
 }
 
 // ccWrapper wraps the balancer.ClientConn passed to the CDS balancer at
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index de724701df94..d1a226e98987 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -65,6 +65,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 	x := &edsBalancer{
 		cc:                cc,
 		closed:            grpcsync.NewEvent(),
+		done:              grpcsync.NewEvent(),
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
@@ -130,6 +131,7 @@ type edsBalancerImplInterface interface {
 type edsBalancer struct {
 	cc     balancer.ClientConn
 	closed *grpcsync.Event
+	done   *grpcsync.Event
 	logger *grpclog.PrefixLogger
 
 	// edsBalancer continuously monitors the channels below, and will handle
@@ -170,6 +172,8 @@ func (x *edsBalancer) run() {
 			x.cancelWatch()
 			x.xdsClient.Close()
 			x.edsImpl.close()
+			x.logger.Infof("Shutdown")
+			x.done.Fire()
 			return
 		}
 	}
@@ -379,7 +383,7 @@ func (x *edsBalancer) enqueueChildBalancerState(p priorityType, s balancer.State
 
 func (x *edsBalancer) Close() {
 	x.closed.Fire()
-	x.logger.Infof("Shutdown")
+	<-x.done.Done()
 }
 
 // equalStringPointers returns true if

From 40b25c5b2c2d1b06d5f5d750d759294c6037d995 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 5 May 2021 12:34:15 -0700
Subject: [PATCH 451/481] xds: set correct order of certificate providers in
 handshake info (#4350)

---
 xds/csds/csds_test.go                         |   4 +-
 .../balancer/cdsbalancer/cdsbalancer.go       |   9 +-
 .../balancer/clustermanager/clustermanager.go |   1 +
 xds/internal/client/transport_helper.go       |   2 +-
 xds/internal/httpfilter/fault/fault_test.go   |  16 +-
 xds/internal/server/conn_wrapper.go           |   2 +-
 .../test/xds_client_integration_test.go       |   8 +-
 .../test/xds_server_integration_test.go       | 452 +++++++++---------
 .../test/xds_server_serving_mode_test.go      |  37 +-
 xds/internal/testutils/e2e/bootstrap.go       |   9 +-
 xds/internal/testutils/e2e/clientresources.go | 241 +++++++++-
 11 files changed, 498 insertions(+), 283 deletions(-)

diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
index 04a71a7d1e6c..018f770494b1 100644
--- a/xds/csds/csds_test.go
+++ b/xds/csds/csds_test.go
@@ -149,7 +149,7 @@ var (
 
 func init() {
 	for i := range ldsTargets {
-		listeners[i] = e2e.DefaultListener(ldsTargets[i], rdsTargets[i])
+		listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
 		listenerAnys[i], _ = ptypes.MarshalAny(listeners[i])
 	}
 	for i := range rdsTargets {
@@ -157,7 +157,7 @@ func init() {
 		routeAnys[i], _ = ptypes.MarshalAny(routes[i])
 	}
 	for i := range cdsTargets {
-		clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i])
+		clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone)
 		clusterAnys[i], _ = ptypes.MarshalAny(clusters[i])
 	}
 	for i := range edsTargets {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index c072af45e00b..bf1519bb8ce0 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -391,6 +391,12 @@ func (b *cdsBalancer) run() {
 				b.edsLB = nil
 			}
 			b.xdsClient.Close()
+			if b.cachedRoot != nil {
+				b.cachedRoot.Close()
+			}
+			if b.cachedIdentity != nil {
+				b.cachedIdentity.Close()
+			}
 			b.logger.Infof("Shutdown")
 			b.done.Fire()
 			return
@@ -490,7 +496,8 @@ func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Sub
 	b.updateCh.Put(&scUpdate{subConn: sc, state: state})
 }
 
-// Close closes the cdsBalancer and the underlying edsBalancer.
+// Close cancels the CDS watch, closes the child policy and closes the
+// cdsBalancer.
 func (b *cdsBalancer) Close() {
 	b.closed.Fire()
 	<-b.done.Done()
diff --git a/xds/internal/balancer/clustermanager/clustermanager.go b/xds/internal/balancer/clustermanager/clustermanager.go
index 1e4dee7f5d3a..b4ae3710cd27 100644
--- a/xds/internal/balancer/clustermanager/clustermanager.go
+++ b/xds/internal/balancer/clustermanager/clustermanager.go
@@ -132,6 +132,7 @@ func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnStat
 func (b *bal) Close() {
 	b.stateAggregator.close()
 	b.bg.Close()
+	b.logger.Infof("Shutdown")
 }
 
 const prefix = "[xds-cluster-manager-lb %p] "
diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go
index b286a61d638b..671e5b3220f1 100644
--- a/xds/internal/client/transport_helper.go
+++ b/xds/internal/client/transport_helper.go
@@ -297,7 +297,7 @@ func (t *TransportHelper) sendExisting(stream grpc.ClientStream) bool {
 
 	for rType, s := range t.watchMap {
 		if err := t.vClient.SendRequest(stream, mapToSlice(s), rType, "", "", ""); err != nil {
-			t.logger.Errorf("ADS request failed: %v", err)
+			t.logger.Warningf("ADS request failed: %v", err)
 			return false
 		}
 	}
diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 624bc9f23461..6aeea8a8a782 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -505,7 +505,13 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 			}
 
 			serviceName := fmt.Sprintf("myservice%d", tcNum)
-			resources := e2e.DefaultClientResources(serviceName, nodeID, "localhost", port)
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       "localhost",
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelNone,
+			})
 			hcm := new(v3httppb.HttpConnectionManager)
 			err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm)
 			if err != nil {
@@ -564,7 +570,13 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 func (s) TestFaultInjection_MaxActiveFaults(t *testing.T) {
 	fs, nodeID, port, cleanup := clientSetup(t)
 	defer cleanup()
-	resources := e2e.DefaultClientResources("myservice", nodeID, "localhost", port)
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "myservice",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
 	hcm := new(v3httppb.HttpConnectionManager)
 	err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm)
 	if err != nil {
diff --git a/xds/internal/server/conn_wrapper.go b/xds/internal/server/conn_wrapper.go
index a92a9ddb038f..a02d75b21445 100644
--- a/xds/internal/server/conn_wrapper.go
+++ b/xds/internal/server/conn_wrapper.go
@@ -119,7 +119,7 @@ func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) {
 	c.identityProvider = ip
 	c.rootProvider = rp
 
-	xdsHI := xdsinternal.NewHandshakeInfo(c.identityProvider, c.rootProvider)
+	xdsHI := xdsinternal.NewHandshakeInfo(c.rootProvider, c.identityProvider)
 	xdsHI.SetRequireClientCert(secCfg.RequireClientCert)
 	return xdsHI, nil
 }
diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go
index 39b3add77fbc..2e0e03aa3aca 100644
--- a/xds/internal/test/xds_client_integration_test.go
+++ b/xds/internal/test/xds_client_integration_test.go
@@ -92,7 +92,13 @@ func (s) TestClientSideXDS(t *testing.T) {
 	fs, nodeID, port, cleanup := clientSetup(t)
 	defer cleanup()
 
-	resources := e2e.DefaultClientResources("myservice", nodeID, "localhost", port)
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "myservice",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
 	if err := fs.Update(resources); err != nil {
 		t.Fatal(err)
 	}
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index d5b9b8dd20a2..5e266cdc5fb6 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -26,6 +26,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -34,26 +35,21 @@ import (
 	"strconv"
 	"testing"
 
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/uuid"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
-	xdscreds "google.golang.org/grpc/credentials/xds"
-	"google.golang.org/grpc/internal/testutils"
-	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/status"
-	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/testdata"
 	"google.golang.org/grpc/xds"
-	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
+
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	testpb "google.golang.org/grpc/test/grpc_testing"
+	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 )
 
 const (
@@ -62,8 +58,7 @@ const (
 	keyFile  = "key.pem"
 	rootFile = "ca.pem"
 
-	// Template for server Listener resource name.
-	serverListenerResourceNameTemplate = "grpc/server?xds.resource.listening_address=%s"
+	xdsServiceName = "my-service"
 )
 
 func createTmpFile(t *testing.T, src, dst string) {
@@ -77,7 +72,6 @@ func createTmpFile(t *testing.T, src, dst string) {
 		t.Fatalf("ioutil.WriteFile(%q) failed: %v", dst, err)
 	}
 	t.Logf("Wrote file at: %s", dst)
-	t.Logf("%s", string(data))
 }
 
 // createTempDirWithFiles creates a temporary directory under the system default
@@ -138,17 +132,29 @@ func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
 func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, func()) {
 	t.Helper()
 
-	// Spin up a xDS management server on a local port.
+	// Turn on the env var protection for client-side security.
+	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
+	env.ClientSideSecuritySupport = true
+
+	// Spin up an xDS management server on a local port.
 	nodeID := uuid.New().String()
 	fs, err := e2e.StartManagementServer()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Create certificate and key files in a temporary directory and generate
-	// certificate provider configuration for a file_watcher plugin.
-	tmpdir := createTmpDirWithFiles(t, "testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
-	cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile))
+	// Create a directory to hold certs and key files used on the server side.
+	serverDir := createTmpDirWithFiles(t, "testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+
+	// Create a directory to hold certs and key files used on the client side.
+	clientDir := createTmpDirWithFiles(t, "testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
+
+	// Create certificate providers section of the bootstrap config with entries
+	// for both the client and server sides.
+	cpc := map[string]json.RawMessage{
+		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
+		e2e.ClientSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)),
+	}
 
 	// Create a bootstrap file in a temporary directory.
 	bootstrapCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
@@ -156,7 +162,7 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 		NodeID:                             nodeID,
 		ServerURI:                          fs.Address,
 		CertificateProviders:               cpc,
-		ServerListenerResourceNameTemplate: serverListenerResourceNameTemplate,
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -190,217 +196,74 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 		fs.Stop()
 		bootstrapCleanup()
 		server.Stop()
+		env.ClientSideSecuritySupport = origClientSideSecurityEnvVar
 	}
 }
 
-func hostPortFromListener(t *testing.T, lis net.Listener) (string, uint32) {
-	t.Helper()
-
+func hostPortFromListener(lis net.Listener) (string, uint32, error) {
 	host, p, err := net.SplitHostPort(lis.Addr().String())
 	if err != nil {
-		t.Fatalf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err)
+		return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err)
 	}
 	port, err := strconv.ParseInt(p, 10, 32)
 	if err != nil {
-		t.Fatalf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err)
-	}
-	return host, uint32(port)
-
-}
-
-// listenerResourceWithoutSecurityConfig returns a listener resource with no
-// security configuration, and name and address fields matching the passed in
-// net.Listener.
-func listenerResourceWithoutSecurityConfig(t *testing.T, lis net.Listener) *v3listenerpb.Listener {
-	host, port := hostPortFromListener(t, lis)
-	return &v3listenerpb.Listener{
-		// This needs to match the name we are querying for.
-		Name: fmt.Sprintf(serverListenerResourceNameTemplate, lis.Addr().String()),
-		Address: &v3corepb.Address{
-			Address: &v3corepb.Address_SocketAddress{
-				SocketAddress: &v3corepb.SocketAddress{
-					Address: host,
-					PortSpecifier: &v3corepb.SocketAddress_PortValue{
-						PortValue: port,
-					},
-				},
-			},
-		},
-		FilterChains: []*v3listenerpb.FilterChain{
-			{
-				Name: "filter-chain-1",
-				Filters: []*v3listenerpb.Filter{
-					{
-						Name: "filter-1",
-						ConfigType: &v3listenerpb.Filter_TypedConfig{
-							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
-						},
-					},
-				},
-			},
-		},
+		return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err)
 	}
+	return host, uint32(port), nil
 }
 
-// listenerResourceWithSecurityConfig returns a listener resource with security
-// configuration pointing to the use of the file_watcher certificate provider
-// plugin, and name and address fields matching the passed in net.Listener.
-func listenerResourceWithSecurityConfig(t *testing.T, lis net.Listener) *v3listenerpb.Listener {
-	transportSocket := &v3corepb.TransportSocket{
-		Name: "envoy.transport_sockets.tls",
-		ConfigType: &v3corepb.TransportSocket_TypedConfig{
-			TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
-				RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
-				CommonTlsContext: &v3tlspb.CommonTlsContext{
-					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-						InstanceName: "google_cloud_private_spiffe",
-					},
-					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-							InstanceName: "google_cloud_private_spiffe",
-						},
-					},
-				},
-			}),
-		},
-	}
-	host, port := hostPortFromListener(t, lis)
-	return &v3listenerpb.Listener{
-		// This needs to match the name we are querying for.
-		Name: fmt.Sprintf(serverListenerResourceNameTemplate, lis.Addr().String()),
-		Address: &v3corepb.Address{
-			Address: &v3corepb.Address_SocketAddress{
-				SocketAddress: &v3corepb.SocketAddress{
-					Address: host,
-					PortSpecifier: &v3corepb.SocketAddress_PortValue{
-						PortValue: port,
-					}}}},
-		FilterChains: []*v3listenerpb.FilterChain{
-			{
-				Name: "v4-wildcard",
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "0.0.0.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: uint32(0),
-							},
-						},
-					},
-					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
-					SourcePrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "0.0.0.0",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: uint32(0),
-							},
-						},
-					},
-				},
-				Filters: []*v3listenerpb.Filter{
-					{
-						Name: "filter-1",
-						ConfigType: &v3listenerpb.Filter_TypedConfig{
-							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
-						},
-					},
-				},
-				TransportSocket: transportSocket,
-			},
-			{
-				Name: "v6-wildcard",
-				FilterChainMatch: &v3listenerpb.FilterChainMatch{
-					PrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "::",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: uint32(0),
-							},
-						},
-					},
-					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
-					SourcePrefixRanges: []*v3corepb.CidrRange{
-						{
-							AddressPrefix: "::",
-							PrefixLen: &wrapperspb.UInt32Value{
-								Value: uint32(0),
-							},
-						},
-					},
-				},
-				Filters: []*v3listenerpb.Filter{
-					{
-						Name: "filter-1",
-						ConfigType: &v3listenerpb.Filter_TypedConfig{
-							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
-						},
-					},
-				},
-				TransportSocket: transportSocket,
-			},
-		},
-	}
-}
-
-// TestServerSideXDS_Fallback is an e2e test where xDS is enabled on the
-// server-side and xdsCredentials are configured for security. The control plane
-// does not provide any security configuration and therefore the xdsCredentials
-// uses fallback credentials, which in this case is insecure creds.
+// TestServerSideXDS_Fallback is an e2e test which verifies xDS credentials
+// fallback functionality.
+//
+// The following sequence of events happen as part of this test:
+// - An xDS-enabled gRPC server is created and xDS credentials are configured.
+// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//   credentials are configured.
+// - Control plane is configured to not send any security configuration to both
+//   the client and the server. This results in both of them using the
+//   configured fallback credentials (which is insecure creds in this case).
 func (s) TestServerSideXDS_Fallback(t *testing.T) {
 	fs, nodeID, lis, cleanup := commonSetup(t)
 	defer cleanup()
 
-	// Setup the fake management server to respond with a Listener resource that
-	// does not contain any security configuration. This should force the
-	// server-side xdsCredentials to use fallback.
-	listener := listenerResourceWithoutSecurityConfig(t, lis)
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
-		Listeners: []*v3listenerpb.Listener{listener},
-	}); err != nil {
-		t.Error(err)
-	}
-
-	// Create a ClientConn and make a successful RPC.
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	// Grab the host and port of the server and create client side xDS resources
+	// corresponding to it. This contains default resources with no security
+	// configuration in the Cluster resources.
+	host, port, err := hostPortFromListener(lis)
 	if err != nil {
-		t.Fatalf("failed to dial local test server: %v", err)
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
 	}
-	defer cc.Close()
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: xdsServiceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
 
-	client := testpb.NewTestServiceClient(cc)
-	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
-		t.Fatalf("rpc EmptyCall() failed: %v", err)
-	}
-}
+	// Create an inbound xDS listener resource for the server side that does not
+	// contain any security configuration. This should force the server-side
+	// xdsCredentials to use fallback.
+	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone)
+	resources.Listeners = append(resources.Listeners, inboundLis)
 
-// TestServerSideXDS_FileWatcherCerts is an e2e test where xDS is enabled on the
-// server-side and xdsCredentials are configured for security. The control plane
-// sends security configuration pointing to the use of the file_watcher plugin,
-// and we verify that a client connecting with TLS creds is able to successfully
-// make an RPC.
-func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
-	fs, nodeID, lis, cleanup := commonSetup(t)
-	defer cleanup()
+	// Setup the management server with client and server-side resources.
+	if err := fs.Update(resources); err != nil {
+		t.Fatal(err)
+	}
 
-	// Setup the fake management server to respond with a Listener resource with
-	// security configuration pointing to the file watcher plugin and requiring
-	// mTLS.
-	listener := listenerResourceWithSecurityConfig(t, lis)
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
-		Listeners: []*v3listenerpb.Listener{listener},
-	}); err != nil {
-		t.Error(err)
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	// Create a ClientConn with TLS creds and make a successful RPC.
-	clientCreds := createClientTLSCredentials(t)
+	// Create a ClientConn with the xds scheme and make a successful RPC.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(clientCreds))
+	cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(creds))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
@@ -412,55 +275,178 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
 	}
 }
 
+// TestServerSideXDS_FileWatcherCerts is an e2e test which verifies xDS
+// credentials with file watcher certificate provider.
+//
+// The following sequence of events happen as part of this test:
+// - An xDS-enabled gRPC server is created and xDS credentials are configured.
+// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//   credentials are configured.
+// - Control plane is configured to send security configuration to both the
+//   client and the server, pointing to the file watcher certificate provider.
+//   We verify both TLS and mTLS scenarios.
+func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
+	tests := []struct {
+		name     string
+		secLevel e2e.SecurityLevel
+	}{
+		{
+			name:     "tls",
+			secLevel: e2e.SecurityLevelTLS,
+		},
+		{
+			name:     "mtls",
+			secLevel: e2e.SecurityLevelMTLS,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fs, nodeID, lis, cleanup := commonSetup(t)
+			defer cleanup()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: xdsServiceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   test.secLevel,
+			})
+
+			// Create an inbound xDS listener resource for the server side that
+			// contains security configuration pointing to the file watcher
+			// plugin.
+			inboundLis := e2e.DefaultServerListener(host, port, test.secLevel)
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			// Setup the management server with client and server resources.
+			if err := fs.Update(resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+				FallbackCreds: insecure.NewCredentials(),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(creds))
+			if err != nil {
+				t.Fatalf("failed to dial local test server: %v", err)
+			}
+			defer cc.Close()
+
+			client := testpb.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+				t.Fatalf("rpc EmptyCall() failed: %v", err)
+			}
+		})
+	}
+}
+
 // TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on
 // the server-side and xdsCredentials are configured for security. The control
 // plane initially does not any security configuration. This forces the
 // xdsCredentials to use fallback creds, which is this case is insecure creds.
 // We verify that a client connecting with TLS creds is not able to successfully
-// make an RPC. The control plan then sends a listener resource with security
+// make an RPC. The control plane then sends a listener resource with security
 // configuration pointing to the use of the file_watcher plugin and we verify
 // that the same client is now able to successfully make an RPC.
 func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
 	fs, nodeID, lis, cleanup := commonSetup(t)
 	defer cleanup()
 
-	// Setup the fake management server to respond with a Listener resource that
-	// does not contain any security configuration. This should force the
-	// server-side xdsCredentials to use fallback.
-	listener := listenerResourceWithoutSecurityConfig(t, lis)
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
-		Listeners: []*v3listenerpb.Listener{listener},
-	}); err != nil {
-		t.Error(err)
+	// Grab the host and port of the server and create client side xDS resources
+	// corresponding to it. This contains default resources with no security
+	// configuration in the Cluster resource. This should force the xDS
+	// credentials on the client to use its fallback.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
 	}
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: xdsServiceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
 
-	// Create a ClientConn with TLS creds. This should fail since the server is
-	// using fallback credentials which in this case in insecure creds.
-	clientCreds := createClientTLSCredentials(t)
+	// Create an inbound xDS listener resource for the server side that does not
+	// contain any security configuration. This should force the xDS credentials
+	// on server to use its fallback.
+	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone)
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	// Setup the management server with client and server-side resources.
+	if err := fs.Update(resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	xdsCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn with the xds scheme and make a successful RPC.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(clientCreds))
+	xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(xdsCreds))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
-	defer cc.Close()
+	defer xdsCC.Close()
+
+	client := testpb.NewTestServiceClient(xdsCC)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// Create a ClientConn with TLS creds. This should fail since the server is
+	// using fallback credentials which in this case in insecure creds.
+	tlsCreds := createClientTLSCredentials(t)
+	tlsCC, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer tlsCC.Close()
 
 	// We don't set 'waitForReady` here since we want this call to failfast.
-	client := testpb.NewTestServiceClient(cc)
-	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Code() != codes.Unavailable {
+	client = testpb.NewTestServiceClient(tlsCC)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
 		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
 	}
 
-	// Setup the fake management server to respond with a Listener resource with
-	// security configuration pointing to the file watcher plugin and requiring
-	// mTLS.
-	listener = listenerResourceWithSecurityConfig(t, lis)
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
-		Listeners: []*v3listenerpb.Listener{listener},
-	}); err != nil {
-		t.Error(err)
+	// Switch server and client side resources with ones that contain required
+	// security configuration for mTLS with a file watcher certificate provider.
+	resources = e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: xdsServiceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS)
+	resources.Listeners = append(resources.Listeners, inboundLis)
+	if err := fs.Update(resources); err != nil {
+		t.Fatal(err)
 	}
 
 	// Make another RPC with `waitForReady` set and expect this to succeed.
diff --git a/xds/internal/test/xds_server_serving_mode_test.go b/xds/internal/test/xds_server_serving_mode_test.go
index c1f9634b16b0..484a5b5ab741 100644
--- a/xds/internal/test/xds_server_serving_mode_test.go
+++ b/xds/internal/test/xds_server_serving_mode_test.go
@@ -24,6 +24,7 @@ package xds_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"path"
@@ -98,10 +99,13 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	}
 	defer fs.Stop()
 
-	// Create certificate and key files in a temporary directory and generate
-	// certificate provider configuration for a file_watcher plugin.
-	tmpdir := createTmpDirWithFiles(t, "testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
-	cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile))
+	// Create a directory to hold certs and key files used on the server side.
+	serverDir := createTmpDirWithFiles(t, "testServerSideServingMode*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+
+	// Create certificate providers section of the bootstrap config.
+	cpc := map[string]json.RawMessage{
+		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
+	}
 
 	// Create a bootstrap file in a temporary directory.
 	bsCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
@@ -109,7 +113,7 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 		NodeID:                             nodeID,
 		ServerURI:                          fs.Address,
 		CertificateProviders:               cpc,
-		ServerListenerResourceNameTemplate: serverListenerResourceNameTemplate,
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -159,15 +163,24 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 		}
 	}()
 
-	// Setup the fake management server to respond with Listener resources that
-	// we are interested in.
-	listener1 := listenerResourceWithoutSecurityConfig(t, lis1)
-	listener2 := listenerResourceWithoutSecurityConfig(t, lis2)
-	if err := fs.Update(e2e.UpdateOptions{
+	// Setup the management server to respond with server-side Listener
+	// resources for both listeners.
+	host1, port1, err := hostPortFromListener(lis1)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	listener1 := e2e.DefaultServerListener(host1, port1, e2e.SecurityLevelNone)
+	host2, port2, err := hostPortFromListener(lis2)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	listener2 := e2e.DefaultServerListener(host2, port2, e2e.SecurityLevelNone)
+	resources := e2e.UpdateOptions{
 		NodeID:    nodeID,
 		Listeners: []*v3listenerpb.Listener{listener1, listener2},
-	}); err != nil {
-		t.Error(err)
+	}
+	if err := fs.Update(resources); err != nil {
+		t.Fatal(err)
 	}
 
 	// Wait for both listeners to move to "serving" mode.
diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go
index 72a1a9900cfe..99702032f817 100644
--- a/xds/internal/testutils/e2e/bootstrap.go
+++ b/xds/internal/testutils/e2e/bootstrap.go
@@ -26,8 +26,8 @@ import (
 // DefaultFileWatcherConfig is a helper function to create a default certificate
 // provider plugin configuration. The test is expected to have setup the files
 // appropriately before this configuration is used to instantiate providers.
-func DefaultFileWatcherConfig(certPath, keyPath, caPath string) map[string]json.RawMessage {
-	cfg := fmt.Sprintf(`{
+func DefaultFileWatcherConfig(certPath, keyPath, caPath string) json.RawMessage {
+	return json.RawMessage(fmt.Sprintf(`{
 			"plugin_name": "file_watcher",
 			"config": {
 				"certificate_file": %q,
@@ -35,8 +35,5 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) map[string]json.
 				"ca_certificate_file": %q,
 				"refresh_interval": "600s"
 			}
-		}`, certPath, keyPath, caPath)
-	return map[string]json.RawMessage{
-		"google_cloud_private_spiffe": json.RawMessage(cfg),
-	}
+		}`, certPath, keyPath, caPath))
 }
diff --git a/xds/internal/testutils/e2e/clientresources.go b/xds/internal/testutils/e2e/clientresources.go
index 86b34861e3db..b521db950558 100644
--- a/xds/internal/testutils/e2e/clientresources.go
+++ b/xds/internal/testutils/e2e/clientresources.go
@@ -19,9 +19,11 @@
 package e2e
 
 import (
+	"fmt"
+
 	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/internal/testutils"
 
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -30,37 +32,73 @@ import (
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-	anypb "github.com/golang/protobuf/ptypes/any"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 )
 
-func any(m proto.Message) *anypb.Any {
-	a, err := ptypes.MarshalAny(m)
-	if err != nil {
-		panic("error marshalling any: " + err.Error())
-	}
-	return a
+const (
+	// ServerListenerResourceNameTemplate is the Listener resource name template
+	// used on the server side.
+	ServerListenerResourceNameTemplate = "grpc/server?xds.resource.listening_address=%s"
+	// ClientSideCertProviderInstance is the certificate provider instance name
+	// used in the Cluster resource on the client side.
+	ClientSideCertProviderInstance = "client-side-certificate-provider-instance"
+	// ServerSideCertProviderInstance is the certificate provider instance name
+	// used in the Listener resource on the server side.
+	ServerSideCertProviderInstance = "server-side-certificate-provider-instance"
+)
+
+// SecurityLevel allows the test to control the security level to be used in the
+// resource returned by this package.
+type SecurityLevel int
+
+const (
+	// SecurityLevelNone is used when no security configuration is required.
+	SecurityLevelNone SecurityLevel = iota
+	// SecurityLevelTLS is used when security configuration corresponding to TLS
+	// is required. Only the server presents an identity certificate in this
+	// configuration.
+	SecurityLevelTLS
+	// SecurityLevelMTLS is used when security ocnfiguration corresponding to
+	// mTLS is required. Both client and server present identity certificates in
+	// this configuration.
+	SecurityLevelMTLS
+)
+
+// ResourceParams wraps the arguments to be passed to DefaultClientResources.
+type ResourceParams struct {
+	// DialTarget is the client's dial target. This is used as the name of the
+	// Listener resource.
+	DialTarget string
+	// NodeID is the id of the xdsClient to which this update is to be pushed.
+	NodeID string
+	// Host is the host of the default Endpoint resource.
+	Host string
+	// port is the port of the default Endpoint resource.
+	Port uint32
+	// SecLevel controls the security configuration in the Cluster resource.
+	SecLevel SecurityLevel
 }
 
 // DefaultClientResources returns a set of resources (LDS, RDS, CDS, EDS) for a
 // client to generically connect to one server.
-func DefaultClientResources(target, nodeID, host string, port uint32) UpdateOptions {
-	routeConfigName := "route-" + target
-	clusterName := "cluster-" + target
-	endpointsName := "endpoints-" + target
-
+func DefaultClientResources(params ResourceParams) UpdateOptions {
+	routeConfigName := "route-" + params.DialTarget
+	clusterName := "cluster-" + params.DialTarget
+	endpointsName := "endpoints-" + params.DialTarget
 	return UpdateOptions{
-		NodeID:    nodeID,
-		Listeners: []*v3listenerpb.Listener{DefaultListener(target, routeConfigName)},
-		Routes:    []*v3routepb.RouteConfiguration{DefaultRouteConfig(routeConfigName, target, clusterName)},
-		Clusters:  []*v3clusterpb.Cluster{DefaultCluster(clusterName, endpointsName)},
-		Endpoints: []*v3endpointpb.ClusterLoadAssignment{DefaultEndpoint(endpointsName, host, port)},
+		NodeID:    params.NodeID,
+		Listeners: []*v3listenerpb.Listener{DefaultClientListener(params.DialTarget, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{DefaultCluster(clusterName, endpointsName, params.SecLevel)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{DefaultEndpoint(endpointsName, params.Host, params.Port)},
 	}
 }
 
-// DefaultListener returns a basic xds Listener resource.
-func DefaultListener(target, routeName string) *v3listenerpb.Listener {
-	hcm := any(&v3httppb.HttpConnectionManager{
+// DefaultClientListener returns a basic xds Listener resource to be used on
+// the client side.
+func DefaultClientListener(target, routeName string) *v3listenerpb.Listener {
+	hcm := testutils.MarshalAny(&v3httppb.HttpConnectionManager{
 		RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{
 			ConfigSource: &v3corepb.ConfigSource{
 				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
@@ -82,12 +120,130 @@ func DefaultListener(target, routeName string) *v3listenerpb.Listener {
 	}
 }
 
+// DefaultServerListener returns a basic xds Listener resource to be used on
+// the server side.
+func DefaultServerListener(host string, port uint32, secLevel SecurityLevel) *v3listenerpb.Listener {
+	var tlsContext *v3tlspb.DownstreamTlsContext
+	switch secLevel {
+	case SecurityLevelNone:
+	case SecurityLevelTLS:
+		tlsContext = &v3tlspb.DownstreamTlsContext{
+			CommonTlsContext: &v3tlspb.CommonTlsContext{
+				TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+					InstanceName: ServerSideCertProviderInstance,
+				},
+			},
+		}
+	case SecurityLevelMTLS:
+		tlsContext = &v3tlspb.DownstreamTlsContext{
+			RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
+			CommonTlsContext: &v3tlspb.CommonTlsContext{
+				TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+					InstanceName: ServerSideCertProviderInstance,
+				},
+				ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+					ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName: ServerSideCertProviderInstance,
+					},
+				},
+			},
+		}
+	}
+
+	var ts *v3corepb.TransportSocket
+	if tlsContext != nil {
+		ts = &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.tls",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: testutils.MarshalAny(tlsContext),
+			},
+		}
+	}
+	return &v3listenerpb.Listener{
+		Name: fmt.Sprintf(ServerListenerResourceNameTemplate, fmt.Sprintf("%s:%d", host, port)),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "v4-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "0.0.0.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "0.0.0.0",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+				},
+				Filters: []*v3listenerpb.Filter{
+					{
+						Name: "filter-1",
+						ConfigType: &v3listenerpb.Filter_TypedConfig{
+							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
+						},
+					},
+				},
+				TransportSocket: ts,
+			},
+			{
+				Name: "v6-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "::",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{
+						{
+							AddressPrefix: "::",
+							PrefixLen: &wrapperspb.UInt32Value{
+								Value: uint32(0),
+							},
+						},
+					},
+				},
+				Filters: []*v3listenerpb.Filter{
+					{
+						Name: "filter-1",
+						ConfigType: &v3listenerpb.Filter_TypedConfig{
+							TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}),
+						},
+					},
+				},
+				TransportSocket: ts,
+			},
+		},
+	}
+}
+
 // HTTPFilter constructs an xds HttpFilter with the provided name and config.
 func HTTPFilter(name string, config proto.Message) *v3httppb.HttpFilter {
 	return &v3httppb.HttpFilter{
 		Name: name,
 		ConfigType: &v3httppb.HttpFilter_TypedConfig{
-			TypedConfig: any(config),
+			TypedConfig: testutils.MarshalAny(config),
 		},
 	}
 }
@@ -109,8 +265,36 @@ func DefaultRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.Rou
 }
 
 // DefaultCluster returns a basic xds Cluster resource.
-func DefaultCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster {
-	return &v3clusterpb.Cluster{
+func DefaultCluster(clusterName, edsServiceName string, secLevel SecurityLevel) *v3clusterpb.Cluster {
+	var tlsContext *v3tlspb.UpstreamTlsContext
+	switch secLevel {
+	case SecurityLevelNone:
+	case SecurityLevelTLS:
+		tlsContext = &v3tlspb.UpstreamTlsContext{
+			CommonTlsContext: &v3tlspb.CommonTlsContext{
+				ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+					ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName: ClientSideCertProviderInstance,
+					},
+				},
+			},
+		}
+	case SecurityLevelMTLS:
+		tlsContext = &v3tlspb.UpstreamTlsContext{
+			CommonTlsContext: &v3tlspb.CommonTlsContext{
+				ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+					ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName: ClientSideCertProviderInstance,
+					},
+				},
+				TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+					InstanceName: ClientSideCertProviderInstance,
+				},
+			},
+		}
+	}
+
+	cluster := &v3clusterpb.Cluster{
 		Name:                 clusterName,
 		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
@@ -123,6 +307,15 @@ func DefaultCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster {
 		},
 		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 	}
+	if tlsContext != nil {
+		cluster.TransportSocket = &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.tls",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: testutils.MarshalAny(tlsContext),
+			},
+		}
+	}
+	return cluster
 }
 
 // DefaultEndpoint returns a basic xds Endpoint resource.

From d426aa5f2e5e809639b45d9619416ce22e56319a Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Wed, 5 May 2021 13:37:13 -0700
Subject: [PATCH 452/481] test: extend the xDS interop tests timeout to 360
 mins (#4380)

---
 test/kokoro/xds.cfg    | 2 +-
 test/kokoro/xds_v3.cfg | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/kokoro/xds.cfg b/test/kokoro/xds.cfg
index d1a078217b84..a1e4ed0bb5e6 100644
--- a/test/kokoro/xds.cfg
+++ b/test/kokoro/xds.cfg
@@ -2,7 +2,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc-go/test/kokoro/xds.sh"
-timeout_mins: 120
+timeout_mins: 360
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
diff --git a/test/kokoro/xds_v3.cfg b/test/kokoro/xds_v3.cfg
index c4c8aad9e6f2..1991efd325d3 100644
--- a/test/kokoro/xds_v3.cfg
+++ b/test/kokoro/xds_v3.cfg
@@ -2,7 +2,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc-go/test/kokoro/xds_v3.sh"
-timeout_mins: 120
+timeout_mins: 360
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"

From d2d6bdae07e844b8a3502dcaf00dc7b1b5519a59 Mon Sep 17 00:00:00 2001
From: Mikhail Mazurskiy <126021+ash2k@users.noreply.github.com>
Date: Fri, 7 May 2021 02:40:54 +1000
Subject: [PATCH 453/481] server: add ForceServerCodec() to set a custom
 encoding.Codec on the server (#4205)

---
 rpc_util.go          |  4 +--
 server.go            | 29 ++++++++++++++++++++++
 test/end2end_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/rpc_util.go b/rpc_util.go
index c0a1208f2f30..c8ae0e4444c7 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -429,9 +429,9 @@ func (o ContentSubtypeCallOption) before(c *callInfo) error {
 }
 func (o ContentSubtypeCallOption) after(c *callInfo, attempt *csAttempt) {}
 
-// ForceCodec returns a CallOption that will set the given Codec to be
+// ForceCodec returns a CallOption that will set codec to be
 // used for all request and response messages for a call. The result of calling
-// String() will be used as the content-subtype in a case-insensitive manner.
+// Name() will be used as the content-subtype in a case-insensitive manner.
 //
 // See Content-Type on
 // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
diff --git a/server.go b/server.go
index b2793ab00b53..d2bc884277a4 100644
--- a/server.go
+++ b/server.go
@@ -279,6 +279,35 @@ func CustomCodec(codec Codec) ServerOption {
 	})
 }
 
+// ForceServerCodec returns a ServerOption that sets a codec for message
+// marshaling and unmarshaling.
+//
+// This will override any lookups by content-subtype for Codecs registered
+// with RegisterCodec.
+//
+// See Content-Type on
+// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
+// more details. Also see the documentation on RegisterCodec and
+// CallContentSubtype for more details on the interaction between encoding.Codec
+// and content-subtype.
+//
+// This function is provided for advanced users; prefer to register codecs
+// using encoding.RegisterCodec.
+// The server will automatically use registered codecs based on the incoming
+// requests' headers. See also
+// https://github.com/grpc/grpc-go/blob/master/Documentation/encoding.md#using-a-codec.
+// Will be supported throughout 1.x.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
+func ForceServerCodec(codec encoding.Codec) ServerOption {
+	return newFuncServerOption(func(o *serverOptions) {
+		o.codec = codec
+	})
+}
+
 // RPCCompressor returns a ServerOption that sets a compressor for outbound
 // messages.  For backward compatibility, all outbound messages will be sent
 // using this compressor, regardless of incoming message compression.  By
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 1baf2e347d15..86fc480669ce 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -5284,6 +5284,37 @@ func (s) TestGRPCMethod(t *testing.T) {
 	}
 }
 
+func (s) TestForceServerCodec(t *testing.T) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	codec := &countingProtoCodec{}
+	if err := ss.Start([]grpc.ServerOption{grpc.ForceServerCodec(codec)}); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+
+	unmarshalCount := atomic.LoadInt32(&codec.unmarshalCount)
+	const wantUnmarshalCount = 1
+	if unmarshalCount != wantUnmarshalCount {
+		t.Fatalf("protoCodec.unmarshalCount = %d; want %d", unmarshalCount, wantUnmarshalCount)
+	}
+	marshalCount := atomic.LoadInt32(&codec.marshalCount)
+	const wantMarshalCount = 1
+	if marshalCount != wantMarshalCount {
+		t.Fatalf("protoCodec.marshalCount = %d; want %d", marshalCount, wantMarshalCount)
+	}
+}
+
 func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) {
 	const mdkey = "somedata"
 
@@ -5653,6 +5684,33 @@ func (c *errCodec) Name() string {
 	return "Fermat's near-miss."
 }
 
+type countingProtoCodec struct {
+	marshalCount   int32
+	unmarshalCount int32
+}
+
+func (p *countingProtoCodec) Marshal(v interface{}) ([]byte, error) {
+	atomic.AddInt32(&p.marshalCount, 1)
+	vv, ok := v.(proto.Message)
+	if !ok {
+		return nil, fmt.Errorf("failed to marshal, message is %T, want proto.Message", v)
+	}
+	return proto.Marshal(vv)
+}
+
+func (p *countingProtoCodec) Unmarshal(data []byte, v interface{}) error {
+	atomic.AddInt32(&p.unmarshalCount, 1)
+	vv, ok := v.(proto.Message)
+	if !ok {
+		return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v)
+	}
+	return proto.Unmarshal(data, vv)
+}
+
+func (*countingProtoCodec) Name() string {
+	return "proto"
+}
+
 func (s) TestEncodeDoesntPanic(t *testing.T) {
 	for _, e := range listTestEnv() {
 		testEncodeDoesntPanic(t, e)

From cb396472c2f78e923dc0b28565c9d704291196f8 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 6 May 2021 13:28:27 -0700
Subject: [PATCH 454/481] Revert "grpc: call balancer.Close() before returning
 from ccBalancerWrapper.close()" (#4391)

This reverts commit 28078834f35b944281662807d8ec071645c37307.
---
 balancer_conn_wrappers.go | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go
index 38b48fcdc5b4..4cc7f9159b16 100644
--- a/balancer_conn_wrappers.go
+++ b/balancer_conn_wrappers.go
@@ -69,10 +69,10 @@ func (ccb *ccBalancerWrapper) watcher() {
 		select {
 		case t := <-ccb.scBuffer.Get():
 			ccb.scBuffer.Load()
-			ccb.balancerMu.Lock()
 			if ccb.done.HasFired() {
 				break
 			}
+			ccb.balancerMu.Lock()
 			su := t.(*scStateUpdate)
 			ccb.balancer.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state, ConnectionError: su.err})
 			ccb.balancerMu.Unlock()
@@ -80,6 +80,7 @@ func (ccb *ccBalancerWrapper) watcher() {
 		}
 
 		if ccb.done.HasFired() {
+			ccb.balancer.Close()
 			ccb.mu.Lock()
 			scs := ccb.subConns
 			ccb.subConns = nil
@@ -94,9 +95,6 @@ func (ccb *ccBalancerWrapper) watcher() {
 }
 
 func (ccb *ccBalancerWrapper) close() {
-	ccb.balancerMu.Lock()
-	defer ccb.balancerMu.Unlock()
-	ccb.balancer.Close()
 	ccb.done.Fire()
 }
 
@@ -121,19 +119,13 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co
 func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error {
 	ccb.balancerMu.Lock()
 	defer ccb.balancerMu.Unlock()
-	if ccb.done.HasFired() {
-		return nil
-	}
 	return ccb.balancer.UpdateClientConnState(*ccs)
 }
 
 func (ccb *ccBalancerWrapper) resolverError(err error) {
 	ccb.balancerMu.Lock()
-	defer ccb.balancerMu.Unlock()
-	if ccb.done.HasFired() {
-		return
-	}
 	ccb.balancer.ResolverError(err)
+	ccb.balancerMu.Unlock()
 }
 
 func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {

From c7ea734087dbbcdb22137ab3b7d8b16842b080bf Mon Sep 17 00:00:00 2001
From: Zach Reyes <39203661+zasweq@users.noreply.github.com>
Date: Fri, 7 May 2021 08:28:34 -0400
Subject: [PATCH 455/481] dns: fix flaky TestRateLimitedResolve (#4387)

* Rewrote TestRateLimitedResolve in dns resolver test to get rid of flakiness.
---
 internal/resolver/dns/dns_resolver.go      |  10 +-
 internal/resolver/dns/dns_resolver_test.go | 141 +++++++++++++--------
 2 files changed, 92 insertions(+), 59 deletions(-)

diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go
index 9d86460ab6fb..03825bbe7b56 100644
--- a/internal/resolver/dns/dns_resolver.go
+++ b/internal/resolver/dns/dns_resolver.go
@@ -47,8 +47,12 @@ var EnableSRVLookups = false
 
 var logger = grpclog.Component("dns")
 
-// A global to stub out in tests.
-var newTimer = time.NewTimer
+// Globals to stub out in tests. TODO: Perhaps these two can be combined into a
+// single variable for testing the resolver?
+var (
+	newTimer           = time.NewTimer
+	newTimerDNSResRate = time.NewTimer
+)
 
 func init() {
 	resolver.Register(NewBuilder())
@@ -219,7 +223,7 @@ func (d *dnsResolver) watcher() {
 			// Success resolving, wait for the next ResolveNow. However, also wait 30 seconds at the very least
 			// to prevent constantly re-resolving.
 			backoffIndex = 1
-			timer = time.NewTimer(minDNSResRate)
+			timer = newTimerDNSResRate(minDNSResRate)
 			select {
 			case <-d.ctx.Done():
 				timer.Stop()
diff --git a/internal/resolver/dns/dns_resolver_test.go b/internal/resolver/dns/dns_resolver_test.go
index 52067e39cc68..73749eca44b0 100644
--- a/internal/resolver/dns/dns_resolver_test.go
+++ b/internal/resolver/dns/dns_resolver_test.go
@@ -43,14 +43,15 @@ func TestMain(m *testing.M) {
 	// Set a non-zero duration only for tests which are actually testing that
 	// feature.
 	replaceDNSResRate(time.Duration(0)) // No nead to clean up since we os.Exit
-	replaceNetFunc(nil)                 // No nead to clean up since we os.Exit
+	overrideDefaultResolver(false)      // No nead to clean up since we os.Exit
 	code := m.Run()
 	os.Exit(code)
 }
 
 const (
-	txtBytesLimit      = 255
-	defaultTestTimeout = 10 * time.Second
+	txtBytesLimit           = 255
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
 )
 
 type testClientConn struct {
@@ -106,12 +107,12 @@ type testResolver struct {
 	// A write to this channel is made when this resolver receives a resolution
 	// request. Tests can rely on reading from this channel to be notified about
 	// resolution requests instead of sleeping for a predefined period of time.
-	ch chan struct{}
+	lookupHostCh *testutils.Channel
 }
 
 func (tr *testResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
-	if tr.ch != nil {
-		tr.ch <- struct{}{}
+	if tr.lookupHostCh != nil {
+		tr.lookupHostCh.Send(nil)
 	}
 	return hostLookup(host)
 }
@@ -124,9 +125,17 @@ func (*testResolver) LookupTXT(ctx context.Context, host string) ([]string, erro
 	return txtLookup(host)
 }
 
-func replaceNetFunc(ch chan struct{}) func() {
+// overrideDefaultResolver overrides the defaultResolver used by the code with
+// an instance of the testResolver. pushOnLookup controls whether the
+// testResolver created here pushes lookupHost events on its channel.
+func overrideDefaultResolver(pushOnLookup bool) func() {
 	oldResolver := defaultResolver
-	defaultResolver = &testResolver{ch: ch}
+
+	var lookupHostCh *testutils.Channel
+	if pushOnLookup {
+		lookupHostCh = testutils.NewChannel()
+	}
+	defaultResolver = &testResolver{lookupHostCh: lookupHostCh}
 
 	return func() {
 		defaultResolver = oldResolver
@@ -1451,23 +1460,33 @@ func TestCustomAuthority(t *testing.T) {
 // requests. It sets the re-resolution rate to a small value and repeatedly
 // calls ResolveNow() and ensures only the expected number of resolution
 // requests are made.
+
 func TestRateLimitedResolve(t *testing.T) {
 	defer leakcheck.Check(t)
 	defer func(nt func(d time.Duration) *time.Timer) {
 		newTimer = nt
 	}(newTimer)
 	newTimer = func(d time.Duration) *time.Timer {
-		// Will never fire on its own, will protect from triggering exponential backoff.
+		// Will never fire on its own, will protect from triggering exponential
+		// backoff.
 		return time.NewTimer(time.Hour)
 	}
+	defer func(nt func(d time.Duration) *time.Timer) {
+		newTimerDNSResRate = nt
+	}(newTimerDNSResRate)
 
-	const dnsResRate = 10 * time.Millisecond
-	dc := replaceDNSResRate(dnsResRate)
-	defer dc()
+	timerChan := testutils.NewChannel()
+	newTimerDNSResRate = func(d time.Duration) *time.Timer {
+		// Will never fire on its own, allows this test to call timer
+		// immediately.
+		t := time.NewTimer(time.Hour)
+		timerChan.Send(t)
+		return t
+	}
 
 	// Create a new testResolver{} for this test because we want the exact count
 	// of the number of times the resolver was invoked.
-	nc := replaceNetFunc(make(chan struct{}))
+	nc := overrideDefaultResolver(true)
 	defer nc()
 
 	target := "foo.bar.com"
@@ -1490,55 +1509,65 @@ func TestRateLimitedResolve(t *testing.T) {
 		t.Fatalf("delegate resolver returned unexpected type: %T\n", tr)
 	}
 
-	// Observe the time before unblocking the lookupHost call.  The 100ms rate
-	// limiting timer will begin immediately after that.  This means the next
-	// resolution could happen less than 100ms if we read the time *after*
-	// receiving from tr.ch
-	start := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
 
 	// Wait for the first resolution request to be done. This happens as part
-	// of the first iteration of the for loop in watcher() because we call
-	// ResolveNow in Build.
-	<-tr.ch
-
-	// Here we start a couple of goroutines. One repeatedly calls ResolveNow()
-	// until asked to stop, and the other waits for two resolution requests to be
-	// made to our testResolver and stops the former. We measure the start and
-	// end times, and expect the duration elapsed to be in the interval
-	// {wantCalls*dnsResRate, wantCalls*dnsResRate}
-	done := make(chan struct{})
-	go func() {
-		for {
-			select {
-			case <-done:
-				return
-			default:
-				r.ResolveNow(resolver.ResolveNowOptions{})
-				time.Sleep(1 * time.Millisecond)
-			}
-		}
-	}()
+	// of the first iteration of the for loop in watcher().
+	if _, err := tr.lookupHostCh.Receive(ctx); err != nil {
+		t.Fatalf("Timed out waiting for lookup() call.")
+	}
 
-	gotCalls := 0
-	const wantCalls = 3
-	min, max := wantCalls*dnsResRate, (wantCalls+1)*dnsResRate
-	tMax := time.NewTimer(max)
-	for gotCalls != wantCalls {
-		select {
-		case <-tr.ch:
-			gotCalls++
-		case <-tMax.C:
-			t.Fatalf("Timed out waiting for %v calls after %v; got %v", wantCalls, max, gotCalls)
-		}
+	// Call Resolve Now 100 times, shouldn't continue onto next iteration of
+	// watcher, thus shouldn't lookup again.
+	for i := 0; i <= 100; i++ {
+		r.ResolveNow(resolver.ResolveNowOptions{})
+	}
+
+	continueCtx, continueCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer continueCancel()
+
+	if _, err := tr.lookupHostCh.Receive(continueCtx); err == nil {
+		t.Fatalf("Should not have looked up again as DNS Min Res Rate timer has not gone off.")
+	}
+
+	// Make the DNSMinResRate timer fire immediately (by receiving it, then
+	// resetting to 0), this will unblock the resolver which is currently
+	// blocked on the DNS Min Res Rate timer going off, which will allow it to
+	// continue to the next iteration of the watcher loop.
+	timer, err := timerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
 	}
-	close(done)
-	elapsed := time.Since(start)
+	timerPointer := timer.(*time.Timer)
+	timerPointer.Reset(0)
 
-	if gotCalls != wantCalls {
-		t.Fatalf("resolve count mismatch for target: %q = %+v, want %+v\n", target, gotCalls, wantCalls)
+	// Now that DNS Min Res Rate timer has gone off, it should lookup again.
+	if _, err := tr.lookupHostCh.Receive(ctx); err != nil {
+		t.Fatalf("Timed out waiting for lookup() call.")
 	}
-	if elapsed < min {
-		t.Fatalf("elapsed time: %v, wanted it to be between {%v and %v}", elapsed, min, max)
+
+	// Resolve Now 1000 more times, shouldn't lookup again as DNS Min Res Rate
+	// timer has not gone off.
+	for i := 0; i < 1000; i++ {
+		r.ResolveNow(resolver.ResolveNowOptions{})
+	}
+
+	if _, err = tr.lookupHostCh.Receive(continueCtx); err == nil {
+		t.Fatalf("Should not have looked up again as DNS Min Res Rate timer has not gone off.")
+	}
+
+	// Make the DNSMinResRate timer fire immediately again.
+	timer, err = timerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Error receiving timer from mock NewTimer call: %v", err)
+	}
+	timerPointer = timer.(*time.Timer)
+	timerPointer.Reset(0)
+
+	// Now that DNS Min Res Rate timer has gone off, it should lookup again.
+	if _, err = tr.lookupHostCh.Receive(ctx); err != nil {
+		t.Fatalf("Timed out waiting for lookup() call.")
 	}
 
 	wantAddrs := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}

From b6f206b84f739768a1c75c1c83fe50ed75845245 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 7 May 2021 11:17:26 -0700
Subject: [PATCH 456/481] grpc: improve docs on StreamDesc (#4397)

---
 stream.go | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/stream.go b/stream.go
index 77d25742cc3d..1f3e70d2c440 100644
--- a/stream.go
+++ b/stream.go
@@ -52,14 +52,20 @@ import (
 // of the RPC.
 type StreamHandler func(srv interface{}, stream ServerStream) error
 
-// StreamDesc represents a streaming RPC service's method specification.
+// StreamDesc represents a streaming RPC service's method specification.  Used
+// on the server when registering services and on the client when initiating
+// new streams.
 type StreamDesc struct {
-	StreamName string
-	Handler    StreamHandler
-
-	// At least one of these is true.
-	ServerStreams bool
-	ClientStreams bool
+	// StreamName and Handler are only used when registering handlers on a
+	// server.
+	StreamName string        // the name of the method excluding the service
+	Handler    StreamHandler // the handler called for the method
+
+	// ServerStreams and ClientStreams are used for registering handlers on a
+	// server as well as defining RPC behavior when passed to NewClientStream
+	// and ClientConn.NewStream.  At least one must be true.
+	ServerStreams bool // indicates the server can perform streaming sends
+	ClientStreams bool // indicates the client can perform streaming sends
 }
 
 // Stream defines the common interface a client or server stream has to satisfy.

From 0ab423af82154f9466b48cfece8043314e7114d4 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 7 May 2021 11:55:48 -0700
Subject: [PATCH 457/481] test: fix flaky GoAwayThenClose (#4394)

In this test, we
1. make a streaming RPC on a connection
1. graceful stop it to send a GOAWAY
1. hard stop it, so the client will create a connection to another server

Before this fix, 2 and 3 can happen too soon, so the RPC in 1 would fail
and then transparent retry (because the stream is unprocessed by the
server in that case).

This retry attempt could pick the new connection, and then the RPC would
block until timeout.

After this streaming RPC fails, we make unary RPCs with the same
deadline (note: deadline not timeout) as the streaming RPC and expect
them to succeed. But they will also fail due to timeout.

The fix is to make a round-trip on the streaming RPC first, to make sure
it actually goes on the first connection.
---
 test/end2end_test.go | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/test/end2end_test.go b/test/end2end_test.go
index 86fc480669ce..a63dfa1409fd 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -6921,6 +6921,10 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 			return &testpb.SimpleResponse{}, nil
 		},
 		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+			if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+				t.Errorf("unexpected error from send: %v", err)
+				return err
+			}
 			// Wait forever.
 			_, err := stream.Recv()
 			if err == nil {
@@ -6954,12 +6958,19 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 
 	client := testpb.NewTestServiceClient(cc)
 
-	// Should go on connection 1. We use a long-lived RPC because it will cause GracefulStop to send GO_AWAY, but the
-	// connection doesn't get closed until the server stops and the client receives.
+	// We make a streaming RPC and do an one-message-round-trip to make sure
+	// it's created on connection 1.
+	//
+	// We use a long-lived RPC because it will cause GracefulStop to send
+	// GO_AWAY, but the connection doesn't get closed until the server stops and
+	// the client receives the error.
 	stream, err := client.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("FullDuplexCall(_) = _, %v; want _, nil", err)
 	}
+	if _, err = stream.Recv(); err != nil {
+		t.Fatalf("unexpected error from first recv: %v", err)
+	}
 
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{
 		{Addr: lis1.Addr().String()},
@@ -6976,8 +6987,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	s1.Stop()
 
 	// Wait for client to close.
-	_, err = stream.Recv()
-	if err == nil {
+	if _, err = stream.Recv(); err == nil {
 		t.Fatal("expected the stream to die, but got a successful Recv")
 	}
 

From 0439465fe2b4020767d9aab1bc3055e492c14089 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 7 May 2021 11:57:56 -0700
Subject: [PATCH 458/481] xds_resolver: fix flaky
 Test/XDSResolverDelayedOnCommitted (#4393)

Before this change, if two xds client updates came too close together, the second one could replace the first one.  The fix is to wait for the effects of the first update before sending the second update.  I injected a synthetic delay into handling the updates from the channel to reproduce this flake 100%, and confirmed this change fixes it.

As part of this change I also noticed that we're actually calling the context cancellation function twice via defers, and never the cancel function from the test setup, so I fixed that, too.
---
 xds/internal/resolver/xds_resolver_test.go | 85 +++++++++-------------
 1 file changed, 34 insertions(+), 51 deletions(-)

diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 8ec29af9ebf2..eca561f369c5 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -289,10 +289,8 @@ func (s) TestXDSResolverBadServiceUpdate(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -317,10 +315,8 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -407,7 +403,7 @@ func (s) TestXDSResolverGoodServiceUpdate(t *testing.T) {
 		defer cancel()
 		gotState, err := tcc.stateCh.Receive(ctx)
 		if err != nil {
-			t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+			t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 		}
 		rState := gotState.(resolver.State)
 		if err := rState.ServiceConfig.Err; err != nil {
@@ -475,7 +471,7 @@ func (s) TestXDSResolverRemovedWithRPCs(t *testing.T) {
 
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -498,7 +494,7 @@ func (s) TestXDSResolverRemovedWithRPCs(t *testing.T) {
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
 
 	if _, err = tcc.stateCh.Receive(ctx); err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 
 	// "Finish the RPC"; this could cause a panic if the resolver doesn't
@@ -544,7 +540,7 @@ func (s) TestXDSResolverRemovedResource(t *testing.T) {
 
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -577,7 +573,7 @@ func (s) TestXDSResolverRemovedResource(t *testing.T) {
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{}, suErr)
 
 	if gotState, err = tcc.stateCh.Receive(ctx); err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState = gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -602,7 +598,7 @@ func (s) TestXDSResolverRemovedResource(t *testing.T) {
 
 	// In the meantime, an empty ServiceConfig update should have been sent.
 	if gotState, err = tcc.stateCh.Receive(ctx); err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState = gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -621,10 +617,8 @@ func (s) TestXDSResolverWRR(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -651,7 +645,7 @@ func (s) TestXDSResolverWRR(t *testing.T) {
 
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -684,10 +678,8 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -722,7 +714,7 @@ func (s) TestXDSResolverMaxStreamDuration(t *testing.T) {
 
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -789,10 +781,8 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -813,7 +803,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -860,6 +850,8 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 			},
 		},
 	}, nil)
+	tcc.stateCh.Receive(ctx) // Ignore the first update.
+
 	xdsC.InvokeWatchRouteConfigCallback(xdsclient.RouteConfigUpdate{
 		VirtualHosts: []*xdsclient.VirtualHost{
 			{
@@ -869,10 +861,9 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 		},
 	}, nil)
 
-	tcc.stateCh.Receive(ctx) // Ignore the first update
 	gotState, err = tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState = gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -910,7 +901,7 @@ func (s) TestXDSResolverDelayedOnCommitted(t *testing.T) {
 	}, nil)
 	gotState, err = tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState = gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -939,10 +930,8 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -971,7 +960,7 @@ func (s) TestXDSResolverGoodUpdateAfterError(t *testing.T) {
 	}, nil)
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	if err := rState.ServiceConfig.Err; err != nil {
@@ -995,10 +984,8 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -1019,7 +1006,7 @@ func (s) TestXDSResolverResourceNotFoundError(t *testing.T) {
 	defer cancel()
 	gotState, err := tcc.stateCh.Receive(ctx)
 	if err != nil {
-		t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+		t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 	}
 	rState := gotState.(resolver.State)
 	wantParsedConfig := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)("{}")
@@ -1043,10 +1030,8 @@ func (s) TestXDSResolverMultipleLDSUpdates(t *testing.T) {
 	xdsR, tcc, cancel := testSetup(t, setupOpts{
 		xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 	})
-	defer func() {
-		cancel()
-		xdsR.Close()
-	}()
+	defer xdsR.Close()
+	defer cancel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
@@ -1220,10 +1205,8 @@ func (s) TestXDSResolverHTTPFilters(t *testing.T) {
 			xdsR, tcc, cancel := testSetup(t, setupOpts{
 				xdsClientFunc: func() (xdsClientInterface, error) { return xdsC, nil },
 			})
-			defer func() {
-				cancel()
-				xdsR.Close()
-			}()
+			defer xdsR.Close()
+			defer cancel()
 
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
@@ -1265,7 +1248,7 @@ func (s) TestXDSResolverHTTPFilters(t *testing.T) {
 
 			gotState, err := tcc.stateCh.Receive(ctx)
 			if err != nil {
-				t.Fatalf("ClientConn.UpdateState returned error: %v", err)
+				t.Fatalf("Error waiting for UpdateState to be called: %v", err)
 			}
 			rState := gotState.(resolver.State)
 			if err := rState.ServiceConfig.Err; err != nil {

From aff517ba8a8ded7306801c3b95f1f7f480c1268b Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 7 May 2021 14:35:48 -0700
Subject: [PATCH 459/481] xds: make e2e tests use a single management server
 instance (#4399)

---
 .../test/xds_client_integration_test.go       |  41 +----
 xds/internal/test/xds_integration_test.go     | 161 ++++++++++++++++++
 .../test/xds_server_integration_test.go       | 157 +++--------------
 .../test/xds_server_serving_mode_test.go      |  49 +-----
 xds/internal/testutils/e2e/server.go          |   1 -
 5 files changed, 203 insertions(+), 206 deletions(-)

diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go
index 2e0e03aa3aca..713331b325e0 100644
--- a/xds/internal/test/xds_client_integration_test.go
+++ b/xds/internal/test/xds_client_integration_test.go
@@ -23,13 +23,12 @@ package xds_test
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"testing"
 
-	"github.com/google/uuid"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 
@@ -37,34 +36,13 @@ import (
 )
 
 // clientSetup performs a bunch of steps common to all xDS client tests here:
-// - spin up an xDS management server on a local port
 // - spin up a gRPC server and register the test service on it
 // - create a local TCP listener and start serving on it
 //
 // Returns the following:
-// - the management server: tests use this to configure resources
-// - nodeID expected by the management server: this is set in the Node proto
-//   sent by the xdsClient for queries.
 // - the port the server is listening on
 // - cleanup function to be invoked by the tests when done
-func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
-	// Spin up a xDS management server on a local port.
-	nodeID := uuid.New().String()
-	fs, err := e2e.StartManagementServer()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{
-		Version:   xds.TransportV3,
-		NodeID:    nodeID,
-		ServerURI: fs.Address,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
+func clientSetup(t *testing.T) (uint32, func()) {
 	// Initialize a gRPC server and register the stubServer on it.
 	server := grpc.NewServer()
 	testpb.RegisterTestServiceServer(server, &testService{})
@@ -81,30 +59,29 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
 		}
 	}()
 
-	return fs, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), func() {
-		fs.Stop()
-		bootstrapCleanup()
+	return uint32(lis.Addr().(*net.TCPAddr).Port), func() {
 		server.Stop()
 	}
 }
 
 func (s) TestClientSideXDS(t *testing.T) {
-	fs, nodeID, port, cleanup := clientSetup(t)
+	port, cleanup := clientSetup(t)
 	defer cleanup()
 
+	serviceName := xdsServiceName + "-client-side-xds"
 	resources := e2e.DefaultClientResources(e2e.ResourceParams{
-		DialTarget: "myservice",
-		NodeID:     nodeID,
+		DialTarget: serviceName,
+		NodeID:     xdsClientNodeID,
 		Host:       "localhost",
 		Port:       port,
 		SecLevel:   e2e.SecurityLevelNone,
 	})
-	if err := fs.Update(resources); err != nil {
+	if err := managementServer.Update(resources); err != nil {
 		t.Fatal(err)
 	}
 
 	// Create a ClientConn and make a successful RPC.
-	cc, err := grpc.Dial("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
diff --git a/xds/internal/test/xds_integration_test.go b/xds/internal/test/xds_integration_test.go
index 1c4b73ac58f8..a41fec929762 100644
--- a/xds/internal/test/xds_integration_test.go
+++ b/xds/internal/test/xds_integration_test.go
@@ -24,10 +24,26 @@ package xds_test
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/leakcheck"
+	"google.golang.org/grpc/internal/xds/env"
+	"google.golang.org/grpc/testdata"
+	"google.golang.org/grpc/xds/internal/testutils/e2e"
+
+	xdsinternal "google.golang.org/grpc/internal/xds"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 )
 
@@ -51,3 +67,148 @@ type testService struct {
 func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 	return &testpb.Empty{}, nil
 }
+
+var (
+	// Globals corresponding to the single instance of the xDS management server
+	// which is spawned for all the tests in this package.
+	managementServer *e2e.ManagementServer
+	xdsClientNodeID  string
+)
+
+// TestMain sets up an xDS management server, runs all tests, and stops the
+// management server.
+func TestMain(m *testing.M) {
+	// The management server is started and stopped from here, but the leakcheck
+	// runs after every individual test. So, we need to skip the goroutine which
+	// spawns the management server and is blocked on the call to `Serve()`.
+	leakcheck.RegisterIgnoreGoroutine("e2e.StartManagementServer")
+
+	cancel, err := setupManagementServer()
+	if err != nil {
+		log.Printf("setupManagementServer() failed: %v", err)
+		os.Exit(1)
+	}
+
+	code := m.Run()
+	cancel()
+	os.Exit(code)
+}
+
+func createTmpFile(src, dst string) error {
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		return fmt.Errorf("ioutil.ReadFile(%q) failed: %v", src, err)
+	}
+	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
+		return fmt.Errorf("ioutil.WriteFile(%q) failed: %v", dst, err)
+	}
+	return nil
+}
+
+// createTempDirWithFiles creates a temporary directory under the system default
+// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
+// rootSrc files are creates appropriate files under the newly create tempDir.
+// Returns the name of the created tempDir.
+func createTmpDirWithFiles(dirSuffix, certSrc, keySrc, rootSrc string) (string, error) {
+	// Create a temp directory. Passing an empty string for the first argument
+	// uses the system temp directory.
+	dir, err := ioutil.TempDir("", dirSuffix)
+	if err != nil {
+		return "", fmt.Errorf("ioutil.TempDir() failed: %v", err)
+	}
+
+	if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil {
+		return "", err
+	}
+	if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil {
+		return "", err
+	}
+	if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil {
+		return "", err
+	}
+	return dir, nil
+}
+
+// createClientTLSCredentials creates client-side TLS transport credentials.
+func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err)
+	}
+	b, err := ioutil.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("ioutil.ReadFile(x509/server_ca_cert.pem) failed: %v", err)
+	}
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(b) {
+		t.Fatal("failed to append certificates")
+	}
+	return credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      roots,
+		ServerName:   "x.test.example.com",
+	})
+}
+
+// setupManagement server performs the following:
+// - spin up an xDS management server on a local port
+// - set up certificates for consumption by the file_watcher plugin
+// - sets up the global variables which refer to this management server and the
+//   nodeID to be used when talking to this management server.
+//
+// Returns a function to be invoked by the caller to stop the management server.
+func setupManagementServer() (func(), error) {
+	// Turn on the env var protection for client-side security.
+	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
+	env.ClientSideSecuritySupport = true
+
+	// Spin up an xDS management server on a local port.
+	var err error
+	managementServer, err = e2e.StartManagementServer()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a directory to hold certs and key files used on the server side.
+	serverDir, err := createTmpDirWithFiles("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	if err != nil {
+		managementServer.Stop()
+		return nil, err
+	}
+
+	// Create a directory to hold certs and key files used on the client side.
+	clientDir, err := createTmpDirWithFiles("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
+	if err != nil {
+		managementServer.Stop()
+		return nil, err
+	}
+
+	// Create certificate providers section of the bootstrap config with entries
+	// for both the client and server sides.
+	cpc := map[string]json.RawMessage{
+		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
+		e2e.ClientSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)),
+	}
+
+	// Create a bootstrap file in a temporary directory.
+	xdsClientNodeID = uuid.New().String()
+	bootstrapCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
+		Version:                            xdsinternal.TransportV3,
+		NodeID:                             xdsClientNodeID,
+		ServerURI:                          managementServer.Address,
+		CertificateProviders:               cpc,
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		managementServer.Stop()
+		return nil, err
+	}
+
+	return func() {
+		managementServer.Stop()
+		bootstrapCleanup()
+		env.ClientSideSecuritySupport = origClientSideSecurityEnvVar
+	}, nil
+}
diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go
index 5e266cdc5fb6..6511a6134cf8 100644
--- a/xds/internal/test/xds_server_integration_test.go
+++ b/xds/internal/test/xds_server_integration_test.go
@@ -24,30 +24,19 @@ package xds_test
 
 import (
 	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net"
-	"os"
-	"path"
 	"strconv"
 	"testing"
 
-	"github.com/google/uuid"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/testdata"
 	"google.golang.org/grpc/xds"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 
 	xdscreds "google.golang.org/grpc/credentials/xds"
-	xdsinternal "google.golang.org/grpc/internal/xds"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 )
@@ -61,113 +50,17 @@ const (
 	xdsServiceName = "my-service"
 )
 
-func createTmpFile(t *testing.T, src, dst string) {
-	t.Helper()
-
-	data, err := ioutil.ReadFile(src)
-	if err != nil {
-		t.Fatalf("ioutil.ReadFile(%q) failed: %v", src, err)
-	}
-	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
-		t.Fatalf("ioutil.WriteFile(%q) failed: %v", dst, err)
-	}
-	t.Logf("Wrote file at: %s", dst)
-}
-
-// createTempDirWithFiles creates a temporary directory under the system default
-// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
-// rootSrc files are creates appropriate files under the newly create tempDir.
-// Returns the name of the created tempDir.
-func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc string) string {
-	t.Helper()
-
-	// Create a temp directory. Passing an empty string for the first argument
-	// uses the system temp directory.
-	dir, err := ioutil.TempDir("", dirSuffix)
-	if err != nil {
-		t.Fatalf("ioutil.TempDir() failed: %v", err)
-	}
-	t.Logf("Using tmpdir: %s", dir)
-
-	createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile))
-	createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile))
-	createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile))
-	return dir
-}
-
-// createClientTLSCredentials creates client-side TLS transport credentials.
-func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
-	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem"))
-	if err != nil {
-		t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err)
-	}
-	b, err := ioutil.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
-	if err != nil {
-		t.Fatalf("ioutil.ReadFile(x509/server_ca_cert.pem) failed: %v", err)
-	}
-	roots := x509.NewCertPool()
-	if !roots.AppendCertsFromPEM(b) {
-		t.Fatal("failed to append certificates")
-	}
-	return credentials.NewTLS(&tls.Config{
-		Certificates: []tls.Certificate{cert},
-		RootCAs:      roots,
-		ServerName:   "x.test.example.com",
-	})
-}
-
-// commonSetup performs a bunch of steps common to all xDS server tests here:
-// - spin up an xDS management server on a local port
-// - set up certificates for consumption by the file_watcher plugin
+// setupGRPCServer performs the following:
 // - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and
 //   register the test service on it
 // - create a local TCP listener and start serving on it
 //
 // Returns the following:
-// - the management server: tests use this to configure resources
-// - nodeID expected by the management server: this is set in the Node proto
-//   sent by the xdsClient used on the xDS-enabled gRPC server
 // - local listener on which the xDS-enabled gRPC server is serving on
 // - cleanup function to be invoked by the tests when done
-func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, func()) {
+func setupGRPCServer(t *testing.T) (net.Listener, func()) {
 	t.Helper()
 
-	// Turn on the env var protection for client-side security.
-	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
-	env.ClientSideSecuritySupport = true
-
-	// Spin up an xDS management server on a local port.
-	nodeID := uuid.New().String()
-	fs, err := e2e.StartManagementServer()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create a directory to hold certs and key files used on the server side.
-	serverDir := createTmpDirWithFiles(t, "testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
-
-	// Create a directory to hold certs and key files used on the client side.
-	clientDir := createTmpDirWithFiles(t, "testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
-
-	// Create certificate providers section of the bootstrap config with entries
-	// for both the client and server sides.
-	cpc := map[string]json.RawMessage{
-		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
-		e2e.ClientSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)),
-	}
-
-	// Create a bootstrap file in a temporary directory.
-	bootstrapCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
-		Version:                            xdsinternal.TransportV3,
-		NodeID:                             nodeID,
-		ServerURI:                          fs.Address,
-		CertificateProviders:               cpc,
-		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Configure xDS credentials to be used on the server-side.
 	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
 		FallbackCreds: insecure.NewCredentials(),
@@ -192,11 +85,8 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun
 		}
 	}()
 
-	return fs, nodeID, lis, func() {
-		fs.Stop()
-		bootstrapCleanup()
+	return lis, func() {
 		server.Stop()
-		env.ClientSideSecuritySupport = origClientSideSecurityEnvVar
 	}
 }
 
@@ -223,7 +113,7 @@ func hostPortFromListener(lis net.Listener) (string, uint32, error) {
 //   the client and the server. This results in both of them using the
 //   configured fallback credentials (which is insecure creds in this case).
 func (s) TestServerSideXDS_Fallback(t *testing.T) {
-	fs, nodeID, lis, cleanup := commonSetup(t)
+	lis, cleanup := setupGRPCServer(t)
 	defer cleanup()
 
 	// Grab the host and port of the server and create client side xDS resources
@@ -233,9 +123,10 @@ func (s) TestServerSideXDS_Fallback(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to retrieve host and port of server: %v", err)
 	}
+	serviceName := xdsServiceName + "-fallback"
 	resources := e2e.DefaultClientResources(e2e.ResourceParams{
-		DialTarget: xdsServiceName,
-		NodeID:     nodeID,
+		DialTarget: serviceName,
+		NodeID:     xdsClientNodeID,
 		Host:       host,
 		Port:       port,
 		SecLevel:   e2e.SecurityLevelNone,
@@ -248,7 +139,7 @@ func (s) TestServerSideXDS_Fallback(t *testing.T) {
 	resources.Listeners = append(resources.Listeners, inboundLis)
 
 	// Setup the management server with client and server-side resources.
-	if err := fs.Update(resources); err != nil {
+	if err := managementServer.Update(resources); err != nil {
 		t.Fatal(err)
 	}
 
@@ -263,7 +154,7 @@ func (s) TestServerSideXDS_Fallback(t *testing.T) {
 	// Create a ClientConn with the xds scheme and make a successful RPC.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(creds))
+	cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
@@ -271,7 +162,7 @@ func (s) TestServerSideXDS_Fallback(t *testing.T) {
 
 	client := testpb.NewTestServiceClient(cc)
 	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
-		t.Fatalf("rpc EmptyCall() failed: %v", err)
+		t.Errorf("rpc EmptyCall() failed: %v", err)
 	}
 }
 
@@ -301,7 +192,7 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			fs, nodeID, lis, cleanup := commonSetup(t)
+			lis, cleanup := setupGRPCServer(t)
 			defer cleanup()
 
 			// Grab the host and port of the server and create client side xDS
@@ -314,9 +205,10 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
 			// Create xDS resources to be consumed on the client side. This
 			// includes the listener, route configuration, cluster (with
 			// security configuration) and endpoint resources.
+			serviceName := xdsServiceName + "-file-watcher-certs-" + test.name
 			resources := e2e.DefaultClientResources(e2e.ResourceParams{
-				DialTarget: xdsServiceName,
-				NodeID:     nodeID,
+				DialTarget: serviceName,
+				NodeID:     xdsClientNodeID,
 				Host:       host,
 				Port:       port,
 				SecLevel:   test.secLevel,
@@ -329,7 +221,7 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
 			resources.Listeners = append(resources.Listeners, inboundLis)
 
 			// Setup the management server with client and server resources.
-			if err := fs.Update(resources); err != nil {
+			if err := managementServer.Update(resources); err != nil {
 				t.Fatal(err)
 			}
 
@@ -344,7 +236,7 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
 			// Create a ClientConn with the xds scheme and make an RPC.
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
-			cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(creds))
+			cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds))
 			if err != nil {
 				t.Fatalf("failed to dial local test server: %v", err)
 			}
@@ -367,7 +259,7 @@ func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
 // configuration pointing to the use of the file_watcher plugin and we verify
 // that the same client is now able to successfully make an RPC.
 func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
-	fs, nodeID, lis, cleanup := commonSetup(t)
+	lis, cleanup := setupGRPCServer(t)
 	defer cleanup()
 
 	// Grab the host and port of the server and create client side xDS resources
@@ -378,9 +270,10 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to retrieve host and port of server: %v", err)
 	}
+	serviceName := xdsServiceName + "-security-config-change"
 	resources := e2e.DefaultClientResources(e2e.ResourceParams{
-		DialTarget: xdsServiceName,
-		NodeID:     nodeID,
+		DialTarget: serviceName,
+		NodeID:     xdsClientNodeID,
 		Host:       host,
 		Port:       port,
 		SecLevel:   e2e.SecurityLevelNone,
@@ -393,7 +286,7 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
 	resources.Listeners = append(resources.Listeners, inboundLis)
 
 	// Setup the management server with client and server-side resources.
-	if err := fs.Update(resources); err != nil {
+	if err := managementServer.Update(resources); err != nil {
 		t.Fatal(err)
 	}
 
@@ -408,7 +301,7 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
 	// Create a ClientConn with the xds scheme and make a successful RPC.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-	xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(xdsCreds))
+	xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
 	}
@@ -437,15 +330,15 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
 	// Switch server and client side resources with ones that contain required
 	// security configuration for mTLS with a file watcher certificate provider.
 	resources = e2e.DefaultClientResources(e2e.ResourceParams{
-		DialTarget: xdsServiceName,
-		NodeID:     nodeID,
+		DialTarget: serviceName,
+		NodeID:     xdsClientNodeID,
 		Host:       host,
 		Port:       port,
 		SecLevel:   e2e.SecurityLevelMTLS,
 	})
 	inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS)
 	resources.Listeners = append(resources.Listeners, inboundLis)
-	if err := fs.Update(resources); err != nil {
+	if err := managementServer.Update(resources); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/xds/internal/test/xds_server_serving_mode_test.go b/xds/internal/test/xds_server_serving_mode_test.go
index 484a5b5ab741..664f0b85759a 100644
--- a/xds/internal/test/xds_server_serving_mode_test.go
+++ b/xds/internal/test/xds_server_serving_mode_test.go
@@ -24,22 +24,18 @@ package xds_test
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net"
-	"path"
 	"sync"
 	"testing"
 
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	"github.com/google/uuid"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials/insecure"
 	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/testutils"
-	xdsinternal "google.golang.org/grpc/internal/xds"
 	testpb "google.golang.org/grpc/test/grpc_testing"
 	"google.golang.org/grpc/xds"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
@@ -91,35 +87,6 @@ func (mt *modeTracker) waitForUpdate(ctx context.Context) error {
 // xDS enabled gRPC servers. It verifies that appropriate mode changes happen in
 // the server, and also verifies behavior of clientConns under these modes.
 func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
-	// Spin up a xDS management server on a local port.
-	nodeID := uuid.New().String()
-	fs, err := e2e.StartManagementServer()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer fs.Stop()
-
-	// Create a directory to hold certs and key files used on the server side.
-	serverDir := createTmpDirWithFiles(t, "testServerSideServingMode*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
-
-	// Create certificate providers section of the bootstrap config.
-	cpc := map[string]json.RawMessage{
-		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
-	}
-
-	// Create a bootstrap file in a temporary directory.
-	bsCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
-		Version:                            xdsinternal.TransportV3,
-		NodeID:                             nodeID,
-		ServerURI:                          fs.Address,
-		CertificateProviders:               cpc,
-		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer bsCleanup()
-
 	// Configure xDS credentials to be used on the server-side.
 	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
 		FallbackCreds: insecure.NewCredentials(),
@@ -176,10 +143,10 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	}
 	listener2 := e2e.DefaultServerListener(host2, port2, e2e.SecurityLevelNone)
 	resources := e2e.UpdateOptions{
-		NodeID:    nodeID,
+		NodeID:    xdsClientNodeID,
 		Listeners: []*v3listenerpb.Listener{listener1, listener2},
 	}
-	if err := fs.Update(resources); err != nil {
+	if err := managementServer.Update(resources); err != nil {
 		t.Fatal(err)
 	}
 
@@ -217,8 +184,8 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 
 	// Update the management server to remove the second listener resource. This should
 	// push the only the second listener into "not-serving" mode.
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
+	if err := managementServer.Update(e2e.UpdateOptions{
+		NodeID:    xdsClientNodeID,
 		Listeners: []*v3listenerpb.Listener{listener1},
 	}); err != nil {
 		t.Error(err)
@@ -246,8 +213,8 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	// Update the management server to remove the first listener resource as
 	// well. This should push the first listener into "not-serving" mode. Second
 	// listener is already in "not-serving" mode.
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
+	if err := managementServer.Update(e2e.UpdateOptions{
+		NodeID:    xdsClientNodeID,
 		Listeners: []*v3listenerpb.Listener{},
 	}); err != nil {
 		t.Error(err)
@@ -279,8 +246,8 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	}
 
 	// Update the management server with both listener resources.
-	if err := fs.Update(e2e.UpdateOptions{
-		NodeID:    nodeID,
+	if err := managementServer.Update(e2e.UpdateOptions{
+		NodeID:    xdsClientNodeID,
 		Listeners: []*v3listenerpb.Listener{listener1, listener2},
 	}); err != nil {
 		t.Error(err)
diff --git a/xds/internal/testutils/e2e/server.go b/xds/internal/testutils/e2e/server.go
index 9ec2eb0d6f2e..4a71a5054d7d 100644
--- a/xds/internal/testutils/e2e/server.go
+++ b/xds/internal/testutils/e2e/server.go
@@ -147,7 +147,6 @@ func (s *ManagementServer) Stop() {
 		s.cancel()
 	}
 	s.gs.Stop()
-	logger.Infof("Stopped the xDS management server...")
 }
 
 // resourceSlice accepts a slice of any type of proto messages and returns a

From 328b1d171a65d7e855bcd7bb5cb1f973c7e6f5d2 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 7 May 2021 14:37:52 -0700
Subject: [PATCH 460/481] transport: allow InTapHandle to return status errors
 (#4365)

---
 internal/transport/controlbuf.go   |  32 ++++++
 internal/transport/http2_server.go |  39 ++++---
 server.go                          |   5 +
 tap/tap.go                         |  16 +--
 test/end2end_test.go               | 167 +++++++++++++++++------------
 5 files changed, 163 insertions(+), 96 deletions(-)

diff --git a/internal/transport/controlbuf.go b/internal/transport/controlbuf.go
index 40ef23923fda..f63a0137622a 100644
--- a/internal/transport/controlbuf.go
+++ b/internal/transport/controlbuf.go
@@ -20,13 +20,17 @@ package transport
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"runtime"
+	"strconv"
 	"sync"
 	"sync/atomic"
 
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
+	"google.golang.org/grpc/internal/grpcutil"
+	"google.golang.org/grpc/status"
 )
 
 var updateHeaderTblSize = func(e *hpack.Encoder, v uint32) {
@@ -128,6 +132,14 @@ type cleanupStream struct {
 
 func (c *cleanupStream) isTransportResponseFrame() bool { return c.rst } // Results in a RST_STREAM
 
+type earlyAbortStream struct {
+	streamID       uint32
+	contentSubtype string
+	status         *status.Status
+}
+
+func (*earlyAbortStream) isTransportResponseFrame() bool { return false }
+
 type dataFrame struct {
 	streamID  uint32
 	endStream bool
@@ -749,6 +761,24 @@ func (l *loopyWriter) cleanupStreamHandler(c *cleanupStream) error {
 	return nil
 }
 
+func (l *loopyWriter) earlyAbortStreamHandler(eas *earlyAbortStream) error {
+	if l.side == clientSide {
+		return errors.New("earlyAbortStream not handled on client")
+	}
+
+	headerFields := []hpack.HeaderField{
+		{Name: ":status", Value: "200"},
+		{Name: "content-type", Value: grpcutil.ContentType(eas.contentSubtype)},
+		{Name: "grpc-status", Value: strconv.Itoa(int(eas.status.Code()))},
+		{Name: "grpc-message", Value: encodeGrpcMessage(eas.status.Message())},
+	}
+
+	if err := l.writeHeader(eas.streamID, true, headerFields, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (l *loopyWriter) incomingGoAwayHandler(*incomingGoAway) error {
 	if l.side == clientSide {
 		l.draining = true
@@ -787,6 +817,8 @@ func (l *loopyWriter) handle(i interface{}) error {
 		return l.registerStreamHandler(i)
 	case *cleanupStream:
 		return l.cleanupStreamHandler(i)
+	case *earlyAbortStream:
+		return l.earlyAbortStreamHandler(i)
 	case *incomingGoAway:
 		return l.incomingGoAwayHandler(i)
 	case *dataFrame:
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index 7c6c89d4f9b2..11be5599cd47 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -356,26 +356,6 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 	if state.data.statsTrace != nil {
 		s.ctx = stats.SetIncomingTrace(s.ctx, state.data.statsTrace)
 	}
-	if t.inTapHandle != nil {
-		var err error
-		info := &tap.Info{
-			FullMethodName: state.data.method,
-		}
-		s.ctx, err = t.inTapHandle(s.ctx, info)
-		if err != nil {
-			if logger.V(logLevel) {
-				logger.Warningf("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err)
-			}
-			t.controlBuf.put(&cleanupStream{
-				streamID: s.id,
-				rst:      true,
-				rstCode:  http2.ErrCodeRefusedStream,
-				onWrite:  func() {},
-			})
-			s.cancel()
-			return false
-		}
-	}
 	t.mu.Lock()
 	if t.state != reachable {
 		t.mu.Unlock()
@@ -417,6 +397,25 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 		s.cancel()
 		return false
 	}
+	if t.inTapHandle != nil {
+		var err error
+		if s.ctx, err = t.inTapHandle(s.ctx, &tap.Info{FullMethodName: state.data.method}); err != nil {
+			t.mu.Unlock()
+			if logger.V(logLevel) {
+				logger.Infof("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err)
+			}
+			stat, ok := status.FromError(err)
+			if !ok {
+				stat = status.New(codes.PermissionDenied, err.Error())
+			}
+			t.controlBuf.put(&earlyAbortStream{
+				streamID:       s.id,
+				contentSubtype: s.contentSubtype,
+				status:         stat,
+			})
+			return false
+		}
+	}
 	t.activeStreams[streamID] = s
 	if len(t.activeStreams) == 1 {
 		t.idle = time.Time{}
diff --git a/server.go b/server.go
index d2bc884277a4..0a151dee4fcb 100644
--- a/server.go
+++ b/server.go
@@ -418,6 +418,11 @@ func ChainStreamInterceptor(interceptors ...StreamServerInterceptor) ServerOptio
 
 // InTapHandle returns a ServerOption that sets the tap handle for all the server
 // transport to be created. Only one can be installed.
+//
+// Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a
+// later release.
 func InTapHandle(h tap.ServerInHandle) ServerOption {
 	return newFuncServerOption(func(o *serverOptions) {
 		if o.inTapHandle != nil {
diff --git a/tap/tap.go b/tap/tap.go
index caea1ebed6e3..dbf34e6bb5f5 100644
--- a/tap/tap.go
+++ b/tap/tap.go
@@ -37,16 +37,16 @@ type Info struct {
 	// TODO: More to be added.
 }
 
-// ServerInHandle defines the function which runs before a new stream is created
-// on the server side. If it returns a non-nil error, the stream will not be
-// created and a RST_STREAM will be sent back to the client with REFUSED_STREAM.
-// The client will receive an RPC error "code = Unavailable, desc = stream
-// terminated by RST_STREAM with error code: REFUSED_STREAM".
+// ServerInHandle defines the function which runs before a new stream is
+// created on the server side. If it returns a non-nil error, the stream will
+// not be created and an error will be returned to the client.  If the error
+// returned is a status error, that status code and message will be used,
+// otherwise PermissionDenied will be the code and err.Error() will be the
+// message.
 //
 // It's intended to be used in situations where you don't want to waste the
-// resources to accept the new stream (e.g. rate-limiting). And the content of
-// the error will be ignored and won't be sent back to the client. For other
-// general usages, please use interceptors.
+// resources to accept the new stream (e.g. rate-limiting). For other general
+// usages, please use interceptors.
 //
 // Note that it is executed in the per-connection I/O goroutine(s) instead of
 // per-RPC goroutine. Therefore, users should NOT have any
diff --git a/test/end2end_test.go b/test/end2end_test.go
index a63dfa1409fd..861a2f29623d 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -2507,10 +2507,13 @@ type myTap struct {
 
 func (t *myTap) handle(ctx context.Context, info *tap.Info) (context.Context, error) {
 	if info != nil {
-		if info.FullMethodName == "/grpc.testing.TestService/EmptyCall" {
+		switch info.FullMethodName {
+		case "/grpc.testing.TestService/EmptyCall":
 			t.cnt++
-		} else if info.FullMethodName == "/grpc.testing.TestService/UnaryCall" {
+		case "/grpc.testing.TestService/UnaryCall":
 			return nil, fmt.Errorf("tap error")
+		case "/grpc.testing.TestService/FullDuplexCall":
+			return nil, status.Errorf(codes.FailedPrecondition, "test custom error")
 		}
 	}
 	return ctx, nil
@@ -2550,8 +2553,15 @@ func testTap(t *testing.T, e env) {
 		ResponseSize: 45,
 		Payload:      payload,
 	}
-	if _, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.Unavailable {
-		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, %s", err, codes.Unavailable)
+	if _, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.PermissionDenied {
+		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, %s", err, codes.PermissionDenied)
+	}
+	str, err := tc.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("Unexpected error creating stream: %v", err)
+	}
+	if _, err := str.Recv(); status.Code(err) != codes.FailedPrecondition {
+		t.Fatalf("FullDuplexCall Recv() = _, %v, want _, %s", err, codes.FailedPrecondition)
 	}
 }
 
@@ -3639,66 +3649,77 @@ func testMalformedHTTP2Metadata(t *testing.T, e env) {
 	}
 }
 
+// Tests that the client transparently retries correctly when receiving a
+// RST_STREAM with code REFUSED_STREAM.
 func (s) TestTransparentRetry(t *testing.T) {
-	for _, e := range listTestEnv() {
-		if e.name == "handler-tls" {
-			// Fails with RST_STREAM / FLOW_CONTROL_ERROR
-			continue
-		}
-		testTransparentRetry(t, e)
-	}
-}
-
-// This test makes sure RPCs are retried times when they receive a RST_STREAM
-// with the REFUSED_STREAM error code, which the InTapHandle provokes.
-func testTransparentRetry(t *testing.T, e env) {
-	te := newTest(t, e)
-	attempts := 0
-	successAttempt := 2
-	te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) {
-		attempts++
-		if attempts < successAttempt {
-			return nil, errors.New("not now")
-		}
-		return ctx, nil
-	}
-	te.startServer(&testServer{security: e.security})
-	defer te.tearDown()
-
-	cc := te.clientConn()
-	tsc := testpb.NewTestServiceClient(cc)
 	testCases := []struct {
-		successAttempt int
-		failFast       bool
-		errCode        codes.Code
+		failFast bool
+		errCode  codes.Code
 	}{{
-		successAttempt: 1,
+		// success attempt: 1, (stream ID 1)
 	}, {
-		successAttempt: 2,
+		// success attempt: 2, (stream IDs 3, 5)
 	}, {
-		successAttempt: 3,
-		errCode:        codes.Unavailable,
+		// no success attempt (stream IDs 7, 9)
+		errCode: codes.Unavailable,
 	}, {
-		successAttempt: 1,
-		failFast:       true,
+		// success attempt: 1 (stream ID 11),
+		failFast: true,
 	}, {
-		successAttempt: 2,
-		failFast:       true,
+		// success attempt: 2 (stream IDs 13, 15),
+		failFast: true,
 	}, {
-		successAttempt: 3,
-		failFast:       true,
-		errCode:        codes.Unavailable,
+		// no success attempt (stream IDs 17, 19)
+		failFast: true,
+		errCode:  codes.Unavailable,
 	}}
-	for _, tc := range testCases {
-		attempts = 0
-		successAttempt = tc.successAttempt
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-		_, err := tsc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!tc.failFast))
-		cancel()
-		if status.Code(err) != tc.errCode {
-			t.Errorf("%+v: tsc.EmptyCall(_, _) = _, %v, want _, Code=%v", tc, err, tc.errCode)
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Failed to listen. Err: %v", err)
+	}
+	defer lis.Close()
+	server := &httpServer{
+		headerFields: [][]string{{
+			":status", "200",
+			"content-type", "application/grpc",
+			"grpc-status", "0",
+		}},
+		refuseStream: func(i uint32) bool {
+			switch i {
+			case 1, 5, 11, 15: // these stream IDs succeed
+				return false
+			}
+			return true // these are refused
+		},
+	}
+	server.start(t, lis)
+	cc, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("failed to dial due to err: %v", err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	client := testpb.NewTestServiceClient(cc)
+
+	for i, tc := range testCases {
+		stream, err := client.FullDuplexCall(ctx)
+		if err != nil {
+			t.Fatalf("error creating stream due to err: %v", err)
+		}
+		code := func(err error) codes.Code {
+			if err == io.EOF {
+				return codes.OK
+			}
+			return status.Code(err)
 		}
+		if _, err := stream.Recv(); code(err) != tc.errCode {
+			t.Fatalf("%v: stream.Recv() = _, %v, want error code: %v", i, err, tc.errCode)
+		}
+
 	}
 }
 
@@ -7191,6 +7212,7 @@ func (s) TestHTTPHeaderFrameErrorHandlingMoreThanTwoHeaders(t *testing.T) {
 
 type httpServer struct {
 	headerFields [][]string
+	refuseStream func(uint32) bool
 }
 
 func (s *httpServer) writeHeader(framer *http2.Framer, sid uint32, headerFields []string, endStream bool) error {
@@ -7238,24 +7260,33 @@ func (s *httpServer) start(t *testing.T, lis net.Listener) {
 		writer.Flush() // necessary since client is expecting preface before declaring connection fully setup.
 
 		var sid uint32
-		// Read frames until a header is received.
+		// Loop until conn is closed and framer returns io.EOF
 		for {
-			frame, err := framer.ReadFrame()
-			if err != nil {
-				t.Errorf("Error at server-side while reading frame. Err: %v", err)
-				return
-			}
-			if hframe, ok := frame.(*http2.HeadersFrame); ok {
-				sid = hframe.Header().StreamID
-				break
+			// Read frames until a header is received.
+			for {
+				frame, err := framer.ReadFrame()
+				if err != nil {
+					if err != io.EOF {
+						t.Errorf("Error at server-side while reading frame. Err: %v", err)
+					}
+					return
+				}
+				if hframe, ok := frame.(*http2.HeadersFrame); ok {
+					sid = hframe.Header().StreamID
+					if s.refuseStream == nil || !s.refuseStream(sid) {
+						break
+					}
+					framer.WriteRSTStream(sid, http2.ErrCodeRefusedStream)
+					writer.Flush()
+				}
 			}
-		}
-		for i, headers := range s.headerFields {
-			if err = s.writeHeader(framer, sid, headers, i == len(s.headerFields)-1); err != nil {
-				t.Errorf("Error at server-side while writing headers. Err: %v", err)
-				return
+			for i, headers := range s.headerFields {
+				if err = s.writeHeader(framer, sid, headers, i == len(s.headerFields)-1); err != nil {
+					t.Errorf("Error at server-side while writing headers. Err: %v", err)
+					return
+				}
+				writer.Flush()
 			}
-			writer.Flush()
 		}
 	}()
 }

From c15291b0f5929ab8cf659269a11e8aa79cb71788 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Fri, 7 May 2021 15:24:10 -0700
Subject: [PATCH 461/481] client: initialize safe config selector when creating
 ClientConn (#4398)

---
 clientconn.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clientconn.go b/clientconn.go
index d57e54b4dc52..24109264f557 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -143,6 +143,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		firstResolveEvent: grpcsync.NewEvent(),
 	}
 	cc.retryThrottler.Store((*retryThrottler)(nil))
+	cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil})
 	cc.ctx, cc.cancel = context.WithCancel(context.Background())
 
 	for _, opt := range opts {

From 12a377b1e4c9f1960bd25f47b9156d9dbd732ed0 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 7 May 2021 15:42:59 -0700
Subject: [PATCH 462/481] xds: nack route configuration with regexes that don't
 compile (#4388)

---
 xds/internal/client/client.go    |  25 ++++---
 xds/internal/client/rds_test.go  | 113 +++++++++++++++++++++++++++----
 xds/internal/client/xds.go       |  15 +++-
 xds/internal/resolver/matcher.go |  15 +---
 4 files changed, 128 insertions(+), 40 deletions(-)

diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 2daceede5398..603632801b04 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -24,6 +24,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"sync"
 	"time"
 
@@ -271,7 +272,9 @@ type VirtualHost struct {
 // Route is both a specification of how to match a request as well as an
 // indication of the action to take upon match.
 type Route struct {
-	Path, Prefix, Regex *string
+	Path   *string
+	Prefix *string
+	Regex  *regexp.Regexp
 	// Indicates if prefix/path matching should be case insensitive. The default
 	// is false (case sensitive).
 	CaseInsensitive bool
@@ -304,20 +307,20 @@ type WeightedCluster struct {
 
 // HeaderMatcher represents header matchers.
 type HeaderMatcher struct {
-	Name         string      `json:"name"`
-	InvertMatch  *bool       `json:"invertMatch,omitempty"`
-	ExactMatch   *string     `json:"exactMatch,omitempty"`
-	RegexMatch   *string     `json:"regexMatch,omitempty"`
-	PrefixMatch  *string     `json:"prefixMatch,omitempty"`
-	SuffixMatch  *string     `json:"suffixMatch,omitempty"`
-	RangeMatch   *Int64Range `json:"rangeMatch,omitempty"`
-	PresentMatch *bool       `json:"presentMatch,omitempty"`
+	Name         string
+	InvertMatch  *bool
+	ExactMatch   *string
+	RegexMatch   *regexp.Regexp
+	PrefixMatch  *string
+	SuffixMatch  *string
+	RangeMatch   *Int64Range
+	PresentMatch *bool
 }
 
 // Int64Range is a range for header range match.
 type Int64Range struct {
-	Start int64 `json:"start"`
-	End   int64 `json:"end"`
+	Start int64
+	End   int64
 }
 
 // SecurityConfig contains the security configuration received as part of the
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index cde40ee80dfe..a4aaf03e4ae0 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -22,24 +22,26 @@ package client
 
 import (
 	"fmt"
+	"regexp"
 	"testing"
 	"time"
 
-	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	v2routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
-	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
-	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/golang/protobuf/proto"
-	anypb "github.com/golang/protobuf/ptypes/any"
-	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/durationpb"
+
+	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v2routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	anypb "github.com/golang/protobuf/ptypes/any"
+	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 )
 
 func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) {
@@ -915,6 +917,51 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			}},
 			wantErr: false,
 		},
+		{
+			name: "good with regex matchers",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/a/"}},
+						Headers: []*v3routepb.HeaderMatcher{
+							{
+								Name:                 "th",
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "tv"}},
+							},
+						},
+						RuntimeFraction: &v3corepb.RuntimeFractionalPercent{
+							DefaultValue: &v3typepb.FractionalPercent{
+								Numerator:   1,
+								Denominator: v3typepb.FractionalPercent_HUNDRED,
+							},
+						},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{
+								WeightedClusters: &v3routepb.WeightedCluster{
+									Clusters: []*v3routepb.WeightedCluster_ClusterWeight{
+										{Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}},
+										{Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}},
+									},
+									TotalWeight: &wrapperspb.UInt32Value{Value: 100},
+								}}}},
+				},
+			},
+			wantRoutes: []*Route{{
+				Regex: func() *regexp.Regexp { return regexp.MustCompile("/a/") }(),
+				Headers: []*HeaderMatcher{
+					{
+						Name:        "th",
+						InvertMatch: newBoolP(false),
+						RegexMatch:  func() *regexp.Regexp { return regexp.MustCompile("tv") }(),
+					},
+				},
+				Fraction:         newUInt32P(10000),
+				WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}},
+			}},
+			wantErr: false,
+		},
 		{
 			name: "query is ignored",
 			routes: []*v3routepb.Route{
@@ -960,6 +1007,44 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "bad regex in path specifier",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"}},
+						Headers: []*v3routepb.HeaderMatcher{
+							{
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "tv"},
+							},
+						},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "bad regex in header specifier",
+			routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"},
+						Headers: []*v3routepb.HeaderMatcher{
+							{
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "??"}},
+							},
+						},
+					},
+					Action: &v3routepb.Route_Route{
+						Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}},
+					},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "unrecognized header match specifier",
 			routes: []*v3routepb.Route{
@@ -1063,7 +1148,7 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 	}
 
 	cmpOpts := []cmp.Option{
-		cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}),
+		cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}, regexp.Regexp{}),
 		cmpopts.EquateEmpty(),
 		cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string {
 			return fmt.Sprint(fc)
@@ -1074,17 +1159,15 @@ func (s) TestRoutesProtoToSlice(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			oldFI := env.FaultInjectionSupport
 			env.FaultInjectionSupport = !tt.disableFI
+			defer func() { env.FaultInjectionSupport = oldFI }()
 
 			got, err := routesProtoToSlice(tt.routes, nil, false)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr)
-				return
+				t.Fatalf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr)
 			}
-			if !cmp.Equal(got, tt.wantRoutes, cmpOpts...) {
-				t.Errorf("routesProtoToSlice() got = %v, want %v, diff: %v", got, tt.wantRoutes, cmp.Diff(got, tt.wantRoutes, cmpOpts...))
+			if diff := cmp.Diff(got, tt.wantRoutes, cmpOpts...); diff != "" {
+				t.Fatalf("routesProtoToSlice() returned unexpected diff (-got +want):\n%s", diff)
 			}
-
-			env.FaultInjectionSupport = oldFI
 		})
 	}
 }
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index c0caf5cceb57..55bcc2e936ca 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -437,7 +438,12 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger,
 		case *v3routepb.RouteMatch_Path:
 			route.Path = &pt.Path
 		case *v3routepb.RouteMatch_SafeRegex:
-			route.Regex = &pt.SafeRegex.Regex
+			regex := pt.SafeRegex.GetRegex()
+			re, err := regexp.Compile(regex)
+			if err != nil {
+				return nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex)
+			}
+			route.Regex = re
 		default:
 			return nil, fmt.Errorf("route %+v has an unrecognized path specifier: %+v", r, pt)
 		}
@@ -452,7 +458,12 @@ func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger,
 			case *v3routepb.HeaderMatcher_ExactMatch:
 				header.ExactMatch = &ht.ExactMatch
 			case *v3routepb.HeaderMatcher_SafeRegexMatch:
-				header.RegexMatch = &ht.SafeRegexMatch.Regex
+				regex := ht.SafeRegexMatch.GetRegex()
+				re, err := regexp.Compile(regex)
+				if err != nil {
+					return nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex)
+				}
+				header.RegexMatch = re
 			case *v3routepb.HeaderMatcher_RangeMatch:
 				header.RangeMatch = &Int64Range{
 					Start: ht.RangeMatch.Start,
diff --git a/xds/internal/resolver/matcher.go b/xds/internal/resolver/matcher.go
index b7b5f3db0e3e..06456a585573 100644
--- a/xds/internal/resolver/matcher.go
+++ b/xds/internal/resolver/matcher.go
@@ -20,7 +20,6 @@ package resolver
 
 import (
 	"fmt"
-	"regexp"
 	"strings"
 
 	"google.golang.org/grpc/internal/grpcrand"
@@ -34,11 +33,7 @@ func routeToMatcher(r *xdsclient.Route) (*compositeMatcher, error) {
 	var pathMatcher pathMatcherInterface
 	switch {
 	case r.Regex != nil:
-		re, err := regexp.Compile(*r.Regex)
-		if err != nil {
-			return nil, fmt.Errorf("failed to compile regex %q", *r.Regex)
-		}
-		pathMatcher = newPathRegexMatcher(re)
+		pathMatcher = newPathRegexMatcher(r.Regex)
 	case r.Path != nil:
 		pathMatcher = newPathExactMatcher(*r.Path, r.CaseInsensitive)
 	case r.Prefix != nil:
@@ -53,12 +48,8 @@ func routeToMatcher(r *xdsclient.Route) (*compositeMatcher, error) {
 		switch {
 		case h.ExactMatch != nil && *h.ExactMatch != "":
 			matcherT = newHeaderExactMatcher(h.Name, *h.ExactMatch)
-		case h.RegexMatch != nil && *h.RegexMatch != "":
-			re, err := regexp.Compile(*h.RegexMatch)
-			if err != nil {
-				return nil, fmt.Errorf("failed to compile regex %q, skipping this matcher", *h.RegexMatch)
-			}
-			matcherT = newHeaderRegexMatcher(h.Name, re)
+		case h.RegexMatch != nil:
+			matcherT = newHeaderRegexMatcher(h.Name, h.RegexMatch)
 		case h.PrefixMatch != nil && *h.PrefixMatch != "":
 			matcherT = newHeaderPrefixMatcher(h.Name, *h.PrefixMatch)
 		case h.SuffixMatch != nil && *h.SuffixMatch != "":

From 98c895f7e06adc82ad030c4f90bcada672f523a2 Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Mon, 10 May 2021 09:35:55 -0700
Subject: [PATCH 463/481] cleanup: use testutils.MarshalAny in more places
 (#4404)

---
 channelz/service/func_linux.go               |  54 ++-
 xds/csds/csds_test.go                        |  15 +-
 xds/internal/client/cds_test.go              | 370 +++++++------------
 xds/internal/client/eds_test.go              |  68 ++--
 xds/internal/client/lds_test.go              |  39 +-
 xds/internal/client/rds_test.go              |  53 +--
 xds/internal/client/tests/dump_test.go       |  51 +--
 xds/internal/client/v2/cds_test.go           |  10 +-
 xds/internal/client/v2/client_test.go        |  66 ++--
 xds/internal/client/v2/eds_test.go           |  23 +-
 xds/internal/httpfilter/fault/fault_test.go  |  17 +-
 xds/internal/server/listener_wrapper_test.go |  14 +-
 xds/server_test.go                           |  13 +-
 13 files changed, 276 insertions(+), 517 deletions(-)

diff --git a/channelz/service/func_linux.go b/channelz/service/func_linux.go
index ce38a921b974..2e52d5f5a98f 100644
--- a/channelz/service/func_linux.go
+++ b/channelz/service/func_linux.go
@@ -25,6 +25,7 @@ import (
 	durpb "github.com/golang/protobuf/ptypes/duration"
 	channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1"
 	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/testutils"
 )
 
 func convertToPtypesDuration(sec int64, usec int64) *durpb.Duration {
@@ -34,41 +35,32 @@ func convertToPtypesDuration(sec int64, usec int64) *durpb.Duration {
 func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOption {
 	var opts []*channelzpb.SocketOption
 	if skopts.Linger != nil {
-		additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionLinger{
-			Active:   skopts.Linger.Onoff != 0,
-			Duration: convertToPtypesDuration(int64(skopts.Linger.Linger), 0),
+		opts = append(opts, &channelzpb.SocketOption{
+			Name: "SO_LINGER",
+			Additional: testutils.MarshalAny(&channelzpb.SocketOptionLinger{
+				Active:   skopts.Linger.Onoff != 0,
+				Duration: convertToPtypesDuration(int64(skopts.Linger.Linger), 0),
+			}),
 		})
-		if err == nil {
-			opts = append(opts, &channelzpb.SocketOption{
-				Name:       "SO_LINGER",
-				Additional: additional,
-			})
-		}
 	}
 	if skopts.RecvTimeout != nil {
-		additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionTimeout{
-			Duration: convertToPtypesDuration(int64(skopts.RecvTimeout.Sec), int64(skopts.RecvTimeout.Usec)),
+		opts = append(opts, &channelzpb.SocketOption{
+			Name: "SO_RCVTIMEO",
+			Additional: testutils.MarshalAny(&channelzpb.SocketOptionTimeout{
+				Duration: convertToPtypesDuration(int64(skopts.RecvTimeout.Sec), int64(skopts.RecvTimeout.Usec)),
+			}),
 		})
-		if err == nil {
-			opts = append(opts, &channelzpb.SocketOption{
-				Name:       "SO_RCVTIMEO",
-				Additional: additional,
-			})
-		}
 	}
 	if skopts.SendTimeout != nil {
-		additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionTimeout{
-			Duration: convertToPtypesDuration(int64(skopts.SendTimeout.Sec), int64(skopts.SendTimeout.Usec)),
+		opts = append(opts, &channelzpb.SocketOption{
+			Name: "SO_SNDTIMEO",
+			Additional: testutils.MarshalAny(&channelzpb.SocketOptionTimeout{
+				Duration: convertToPtypesDuration(int64(skopts.SendTimeout.Sec), int64(skopts.SendTimeout.Usec)),
+			}),
 		})
-		if err == nil {
-			opts = append(opts, &channelzpb.SocketOption{
-				Name:       "SO_SNDTIMEO",
-				Additional: additional,
-			})
-		}
 	}
 	if skopts.TCPInfo != nil {
-		additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionTcpInfo{
+		additional := testutils.MarshalAny(&channelzpb.SocketOptionTcpInfo{
 			TcpiState:       uint32(skopts.TCPInfo.State),
 			TcpiCaState:     uint32(skopts.TCPInfo.Ca_state),
 			TcpiRetransmits: uint32(skopts.TCPInfo.Retransmits),
@@ -99,12 +91,10 @@ func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOptio
 			TcpiAdvmss:       skopts.TCPInfo.Advmss,
 			TcpiReordering:   skopts.TCPInfo.Reordering,
 		})
-		if err == nil {
-			opts = append(opts, &channelzpb.SocketOption{
-				Name:       "TCP_INFO",
-				Additional: additional,
-			})
-		}
+		opts = append(opts, &channelzpb.SocketOption{
+			Name:       "TCP_INFO",
+			Additional: additional,
+		})
 	}
 	return opts
 }
diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go
index 018f770494b1..6cf88f6d3942 100644
--- a/xds/csds/csds_test.go
+++ b/xds/csds/csds_test.go
@@ -34,10 +34,11 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/google/uuid"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/xds/internal/client"
 	_ "google.golang.org/grpc/xds/internal/httpfilter/router"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xtestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -150,19 +151,19 @@ var (
 func init() {
 	for i := range ldsTargets {
 		listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
-		listenerAnys[i], _ = ptypes.MarshalAny(listeners[i])
+		listenerAnys[i] = testutils.MarshalAny(listeners[i])
 	}
 	for i := range rdsTargets {
 		routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i])
-		routeAnys[i], _ = ptypes.MarshalAny(routes[i])
+		routeAnys[i] = testutils.MarshalAny(routes[i])
 	}
 	for i := range cdsTargets {
 		clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone)
-		clusterAnys[i], _ = ptypes.MarshalAny(clusters[i])
+		clusterAnys[i] = testutils.MarshalAny(clusters[i])
 	}
 	for i := range edsTargets {
 		endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i])
-		endpointAnys[i], _ = ptypes.MarshalAny(endpoints[i])
+		endpointAnys[i] = testutils.MarshalAny(endpoints[i])
 	}
 }
 
@@ -286,9 +287,9 @@ func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServ
 	}
 	v3statuspbgrpc.RegisterClientStatusDiscoveryServiceServer(server, csdss)
 	// Create a local listener and pass it to Serve().
-	lis, err := testutils.LocalTCPListener()
+	lis, err := xtestutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+		t.Fatalf("xtestutils.LocalTCPListener() failed: %v", err)
 	}
 	go func() {
 		if err := server.Serve(lis); err != nil {
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index bb1117ec5349..627229de7ad0 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -30,10 +30,10 @@ import (
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
-	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/testutils"
 	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
@@ -281,23 +281,16 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 		TransportSocket: &v3corepb.TransportSocket{
 			Name: "envoy.transport_sockets.tls",
 			ConfigType: &v3corepb.TransportSocket_TypedConfig{
-				TypedConfig: &anypb.Any{
-					TypeUrl: version.V3UpstreamTLSContextURL,
-					Value: func() []byte {
-						tls := &v3tlspb.UpstreamTlsContext{
-							CommonTlsContext: &v3tlspb.CommonTlsContext{
-								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-										InstanceName:    "rootInstance",
-										CertificateName: "rootCert",
-									},
-								},
+				TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+					CommonTlsContext: &v3tlspb.CommonTlsContext{
+						ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+							ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName:    "rootInstance",
+								CertificateName: "rootCert",
 							},
-						}
-						mtls, _ := proto.Marshal(tls)
-						return mtls
-					}(),
-				},
+						},
+					},
+				}),
 			},
 		},
 	}
@@ -427,22 +420,15 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
-											ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
-												Name: "foo-sds-secret",
-											},
-										},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{
+									ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{
+										Name: "foo-sds-secret",
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -463,16 +449,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{},
+						}),
 					},
 				},
 			},
@@ -493,30 +472,23 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
-											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
-												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
-													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
-														{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}},
-													},
-												},
-												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    rootPluginInstance,
-													CertificateName: rootCertName,
-												},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+									CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+										DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+											MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+												{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}},
 											},
 										},
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    rootPluginInstance,
+											CertificateName: rootCertName,
+										},
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -537,30 +509,23 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
-											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
-												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
-													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
-														{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}},
-													},
-												},
-												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    rootPluginInstance,
-													CertificateName: rootCertName,
-												},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+									CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+										DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+											MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+												{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}},
 											},
 										},
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    rootPluginInstance,
+											CertificateName: rootCertName,
+										},
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -581,30 +546,23 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
-											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
-												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
-													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
-														{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}},
-													},
-												},
-												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    rootPluginInstance,
-													CertificateName: rootCertName,
-												},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+									CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+										DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+											MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+												{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}},
 											},
 										},
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    rootPluginInstance,
+											CertificateName: rootCertName,
+										},
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -625,30 +583,23 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 				TransportSocket: &v3corepb.TransportSocket{
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
-											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
-												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
-													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
-														{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}},
-													},
-												},
-												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    rootPluginInstance,
-													CertificateName: rootCertName,
-												},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+									CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+										DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+											MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+												{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}},
 											},
 										},
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    rootPluginInstance,
+											CertificateName: rootCertName,
+										},
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -670,23 +621,16 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-												InstanceName:    rootPluginInstance,
-												CertificateName: rootCertName,
-											},
-										},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    rootPluginInstance,
+										CertificateName: rootCertName,
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -715,27 +659,20 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-											InstanceName:    identityPluginInstance,
-											CertificateName: identityCertName,
-										},
-										ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
-											ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-												InstanceName:    rootPluginInstance,
-												CertificateName: rootCertName,
-											},
-										},
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    identityPluginInstance,
+									CertificateName: identityCertName,
+								},
+								ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+									ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName:    rootPluginInstance,
+										CertificateName: rootCertName,
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -766,41 +703,34 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: &anypb.Any{
-							TypeUrl: version.V3UpstreamTLSContextURL,
-							Value: func() []byte {
-								tls := &v3tlspb.UpstreamTlsContext{
-									CommonTlsContext: &v3tlspb.CommonTlsContext{
-										TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-											InstanceName:    identityPluginInstance,
-											CertificateName: identityCertName,
-										},
-										ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
-											CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
-												DefaultValidationContext: &v3tlspb.CertificateValidationContext{
-													MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
-														{
-															MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact},
-															IgnoreCase:   true,
-														},
-														{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}},
-														{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}},
-														{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}},
-														{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}},
-													},
-												},
-												ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
-													InstanceName:    rootPluginInstance,
-													CertificateName: rootCertName,
+						TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{
+							CommonTlsContext: &v3tlspb.CommonTlsContext{
+								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName:    identityPluginInstance,
+									CertificateName: identityCertName,
+								},
+								ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+									CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+										DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+											MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+												{
+													MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact},
+													IgnoreCase:   true,
 												},
+												{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}},
+												{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}},
+												{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}},
+												{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}},
 											},
 										},
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName:    rootPluginInstance,
+											CertificateName: rootCertName,
+										},
 									},
-								}
-								mtls, _ := proto.Marshal(tls)
-								return mtls
-							}(),
-						},
+								},
+							},
+						}),
 					},
 				},
 			},
@@ -845,7 +775,7 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 		v3Service     = "v2Service"
 	)
 	var (
-		v2Cluster = &v2xdspb.Cluster{
+		v2ClusterAny = testutils.MarshalAny(&v2xdspb.Cluster{
 			Name:                 v2ClusterName,
 			ClusterDiscoveryType: &v2xdspb.Cluster_Type{Type: v2xdspb.Cluster_EDS},
 			EdsClusterConfig: &v2xdspb.Cluster_EdsClusterConfig{
@@ -862,16 +792,9 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 					Self: &v2corepb.SelfConfigSource{},
 				},
 			},
-		}
-		v2ClusterAny = &anypb.Any{
-			TypeUrl: version.V2ClusterURL,
-			Value: func() []byte {
-				mcl, _ := proto.Marshal(v2Cluster)
-				return mcl
-			}(),
-		}
+		})
 
-		v3Cluster = &v3clusterpb.Cluster{
+		v3ClusterAny = testutils.MarshalAny(&v3clusterpb.Cluster{
 			Name:                 v3ClusterName,
 			ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 			EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
@@ -888,14 +811,7 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 					Self: &v3corepb.SelfConfigSource{},
 				},
 			},
-		}
-		v3ClusterAny = &anypb.Any{
-			TypeUrl: version.V3ClusterURL,
-			Value: func() []byte {
-				mcl, _ := proto.Marshal(v3Cluster)
-				return mcl
-			}(),
-		}
+		})
 	)
 	const testVersion = "test-version-cds"
 
@@ -940,17 +856,10 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 		{
 			name: "bad cluster resource",
 			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3ClusterURL,
-					Value: func() []byte {
-						cl := &v3clusterpb.Cluster{
-							Name:                 "test",
-							ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
-						}
-						mcl, _ := proto.Marshal(cl)
-						return mcl
-					}(),
-				},
+				testutils.MarshalAny(&v3clusterpb.Cluster{
+					Name:                 "test",
+					ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
+				}),
 			},
 			wantUpdate: map[string]ClusterUpdate{"test": {}},
 			wantMD: UpdateMetadata{
@@ -1014,18 +923,11 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			name: "good and bad clusters",
 			resources: []*anypb.Any{
 				v2ClusterAny,
-				{
-					// bad cluster resource
-					TypeUrl: version.V3ClusterURL,
-					Value: func() []byte {
-						cl := &v3clusterpb.Cluster{
-							Name:                 "bad",
-							ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
-						}
-						mcl, _ := proto.Marshal(cl)
-						return mcl
-					}(),
-				},
+				// bad cluster resource
+				testutils.MarshalAny(&v3clusterpb.Cluster{
+					Name:                 "bad",
+					ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC},
+				}),
 				v3ClusterAny,
 			},
 			wantUpdate: map[string]ClusterUpdate{
diff --git a/xds/internal/client/eds_test.go b/xds/internal/client/eds_test.go
index 9d6a3113b0c3..467f25269cdf 100644
--- a/xds/internal/client/eds_test.go
+++ b/xds/internal/client/eds_test.go
@@ -29,10 +29,10 @@ import (
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
-	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/version"
 )
@@ -122,23 +122,18 @@ func (s) TestEDSParseRespProto(t *testing.T) {
 }
 
 func (s) TestUnmarshalEndpoints(t *testing.T) {
-	var v3EndpointsAny = &anypb.Any{
-		TypeUrl: version.V3EndpointsURL,
-		Value: func() []byte {
-			clab0 := newClaBuilder("test", nil)
-			clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{
-				Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY},
-				Weight: []uint32{271},
-			})
-			clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{
-				Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING},
-				Weight: []uint32{828},
-			})
-			e := clab0.Build()
-			me, _ := proto.Marshal(e)
-			return me
-		}(),
-	}
+	var v3EndpointsAny = testutils.MarshalAny(func() *v3endpointpb.ClusterLoadAssignment {
+		clab0 := newClaBuilder("test", nil)
+		clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{
+			Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY},
+			Weight: []uint32{271},
+		})
+		clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{
+			Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING},
+			Weight: []uint32{828},
+		})
+		return clab0.Build()
+	}())
 	const testVersion = "test-version-eds"
 
 	tests := []struct {
@@ -181,19 +176,12 @@ func (s) TestUnmarshalEndpoints(t *testing.T) {
 		},
 		{
 			name: "bad endpoints resource",
-			resources: []*anypb.Any{
-				{
-					TypeUrl: version.V3EndpointsURL,
-					Value: func() []byte {
-						clab0 := newClaBuilder("test", nil)
-						clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
-						clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
-						e := clab0.Build()
-						me, _ := proto.Marshal(e)
-						return me
-					}(),
-				},
-			},
+			resources: []*anypb.Any{testutils.MarshalAny(func() *v3endpointpb.ClusterLoadAssignment {
+				clab0 := newClaBuilder("test", nil)
+				clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
+				clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
+				return clab0.Build()
+			}())},
 			wantUpdate: map[string]EndpointsUpdate{"test": {}},
 			wantMD: UpdateMetadata{
 				Status:  ServiceStatusNACKed,
@@ -246,18 +234,12 @@ func (s) TestUnmarshalEndpoints(t *testing.T) {
 			name: "good and bad endpoints",
 			resources: []*anypb.Any{
 				v3EndpointsAny,
-				{
-					// bad endpoints resource
-					TypeUrl: version.V3EndpointsURL,
-					Value: func() []byte {
-						clab0 := newClaBuilder("bad", nil)
-						clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
-						clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
-						e := clab0.Build()
-						me, _ := proto.Marshal(e)
-						return me
-					}(),
-				},
+				testutils.MarshalAny(func() *v3endpointpb.ClusterLoadAssignment {
+					clab0 := newClaBuilder("bad", nil)
+					clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
+					clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil)
+					return clab0.Build()
+				}()),
 			},
 			wantUpdate: map[string]EndpointsUpdate{
 				"test": {
diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go
index 9fb27987e36b..ad9af4c885a2 100644
--- a/xds/internal/client/lds_test.go
+++ b/xds/internal/client/lds_test.go
@@ -251,16 +251,9 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{
 				Name: v3LDSTarget,
 				ApiListener: &v3listenerpb.ApiListener{
-					ApiListener: &anypb.Any{
-						TypeUrl: version.V2ListenerURL,
-						Value: func() []byte {
-							cm := &v3httppb.HttpConnectionManager{
-								RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
-							}
-							mcm, _ := proto.Marshal(cm)
-							return mcm
-						}(),
-					},
+					ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
+						RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+					}),
 				},
 			})},
 			wantUpdate: map[string]ListenerUpdate{v3LDSTarget: {}},
@@ -558,25 +551,13 @@ func (s) TestUnmarshalListener_ClientSide(t *testing.T) {
 			name: "good and bad listener resources",
 			resources: []*anypb.Any{
 				v2Lis,
-				{
-					TypeUrl: version.V3ListenerURL,
-					Value: func() []byte {
-						lis := &v3listenerpb.Listener{
-							Name: "bad",
-							ApiListener: &v3listenerpb.ApiListener{
-								ApiListener: &anypb.Any{
-									TypeUrl: version.V2ListenerURL,
-									Value: func() []byte {
-										cm := &v3httppb.HttpConnectionManager{
-											RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
-										}
-										mcm, _ := proto.Marshal(cm)
-										return mcm
-									}()}}}
-						mLis, _ := proto.Marshal(lis)
-						return mLis
-					}(),
-				},
+				testutils.MarshalAny(&v3listenerpb.Listener{
+					Name: "bad",
+					ApiListener: &v3listenerpb.ApiListener{
+						ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
+							RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+						}),
+					}}),
 				v3LisWithFilters(),
 			},
 			wantUpdate: map[string]ListenerUpdate{
diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go
index a4aaf03e4ae0..10745e9e97da 100644
--- a/xds/internal/client/rds_test.go
+++ b/xds/internal/client/rds_test.go
@@ -26,9 +26,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 	"google.golang.org/grpc/xds/internal/version"
@@ -541,17 +541,10 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 				},
 			},
 		}
-		v2RouteConfig = &anypb.Any{
-			TypeUrl: version.V2RouteConfigURL,
-			Value: func() []byte {
-				rc := &v2xdspb.RouteConfiguration{
-					Name:         v2RouteConfigName,
-					VirtualHosts: v2VirtualHost,
-				}
-				m, _ := proto.Marshal(rc)
-				return m
-			}(),
-		}
+		v2RouteConfig = testutils.MarshalAny(&v2xdspb.RouteConfiguration{
+			Name:         v2RouteConfigName,
+			VirtualHosts: v2VirtualHost,
+		})
 		v3VirtualHost = []*v3routepb.VirtualHost{
 			{
 				Domains: []string{uninterestingDomain},
@@ -580,17 +573,10 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 				},
 			},
 		}
-		v3RouteConfig = &anypb.Any{
-			TypeUrl: version.V2RouteConfigURL,
-			Value: func() []byte {
-				rc := &v3routepb.RouteConfiguration{
-					Name:         v3RouteConfigName,
-					VirtualHosts: v3VirtualHost,
-				}
-				m, _ := proto.Marshal(rc)
-				return m
-			}(),
-		}
+		v3RouteConfig = testutils.MarshalAny(&v3routepb.RouteConfiguration{
+			Name:         v3RouteConfigName,
+			VirtualHosts: v3VirtualHost,
+		})
 	)
 	const testVersion = "test-version-rds"
 
@@ -726,20 +712,13 @@ func (s) TestUnmarshalRouteConfig(t *testing.T) {
 			name: "good and bad routeConfig resources",
 			resources: []*anypb.Any{
 				v2RouteConfig,
-				{
-					TypeUrl: version.V2RouteConfigURL,
-					Value: func() []byte {
-						rc := &v3routepb.RouteConfiguration{
-							Name: "bad",
-							VirtualHosts: []*v3routepb.VirtualHost{
-								{Domains: []string{ldsTarget},
-									Routes: []*v3routepb.Route{{
-										Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_ConnectMatcher_{}},
-									}}}}}
-						m, _ := proto.Marshal(rc)
-						return m
-					}(),
-				},
+				testutils.MarshalAny(&v3routepb.RouteConfiguration{
+					Name: "bad",
+					VirtualHosts: []*v3routepb.VirtualHost{
+						{Domains: []string{ldsTarget},
+							Routes: []*v3routepb.Route{{
+								Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_ConnectMatcher_{}},
+							}}}}}),
 				v3RouteConfig,
 			},
 			wantUpdate: map[string]RouteConfigUpdate{
diff --git a/xds/internal/client/tests/dump_test.go b/xds/internal/client/tests/dump_test.go
index de3fcade47e9..815850973e3d 100644
--- a/xds/internal/client/tests/dump_test.go
+++ b/xds/internal/client/tests/dump_test.go
@@ -30,7 +30,6 @@ import (
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-	"github.com/golang/protobuf/ptypes"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/protobuf/testing/protocmp"
@@ -39,6 +38,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
@@ -58,29 +58,22 @@ func (s) TestLDSConfigDump(t *testing.T) {
 		listenersT := &v3listenerpb.Listener{
 			Name: ldsTargets[i],
 			ApiListener: &v3listenerpb.ApiListener{
-				ApiListener: func() *anypb.Any {
-					mcm, _ := ptypes.MarshalAny(&v3httppb.HttpConnectionManager{
-						RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
-							Rds: &v3httppb.Rds{
-								ConfigSource: &v3corepb.ConfigSource{
-									ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
-								},
-								RouteConfigName: routeConfigNames[i],
+				ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{
+					RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+						Rds: &v3httppb.Rds{
+							ConfigSource: &v3corepb.ConfigSource{
+								ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
 							},
+							RouteConfigName: routeConfigNames[i],
 						},
-						CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
-							MaxStreamDuration: durationpb.New(time.Second),
-						},
-					})
-					return mcm
-				}(),
+					},
+					CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{
+						MaxStreamDuration: durationpb.New(time.Second),
+					},
+				}),
 			},
 		}
-		anyT, err := ptypes.MarshalAny(listenersT)
-		if err != nil {
-			t.Fatalf("failed to marshal proto to any: %v", err)
-		}
-		listenerRaws[ldsTargets[i]] = anyT
+		listenerRaws[ldsTargets[i]] = testutils.MarshalAny(listenersT)
 	}
 
 	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
@@ -190,11 +183,7 @@ func (s) TestRDSConfigDump(t *testing.T) {
 			},
 		}
 
-		anyT, err := ptypes.MarshalAny(routeConfigT)
-		if err != nil {
-			t.Fatalf("failed to marshal proto to any: %v", err)
-		}
-		routeRaws[rdsTargets[i]] = anyT
+		routeRaws[rdsTargets[i]] = testutils.MarshalAny(routeConfigT)
 	}
 
 	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
@@ -304,11 +293,7 @@ func (s) TestCDSConfigDump(t *testing.T) {
 			},
 		}
 
-		anyT, err := ptypes.MarshalAny(clusterT)
-		if err != nil {
-			t.Fatalf("failed to marshal proto to any: %v", err)
-		}
-		clusterRaws[cdsTargets[i]] = anyT
+		clusterRaws[cdsTargets[i]] = testutils.MarshalAny(clusterT)
 	}
 
 	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
@@ -404,11 +389,7 @@ func (s) TestEDSConfigDump(t *testing.T) {
 		clab0.AddLocality(localityNames[i], 1, 1, []string{addrs[i]}, nil)
 		claT := clab0.Build()
 
-		anyT, err := ptypes.MarshalAny(claT)
-		if err != nil {
-			t.Fatalf("failed to marshal proto to any: %v", err)
-		}
-		endpointRaws[edsTargets[i]] = anyT
+		endpointRaws[edsTargets[i]] = testutils.MarshalAny(claT)
 	}
 
 	client, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index b56ae6108bbe..e627860d2a9f 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -26,8 +26,8 @@ import (
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
-	"github.com/golang/protobuf/ptypes"
 	anypb "github.com/golang/protobuf/ptypes/any"
+	"google.golang.org/grpc/internal/testutils"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/version"
 )
@@ -65,8 +65,8 @@ var (
 			},
 		},
 	}
-	marshaledCluster1, _ = ptypes.MarshalAny(goodCluster1)
-	goodCluster2         = &xdspb.Cluster{
+	marshaledCluster1 = testutils.MarshalAny(goodCluster1)
+	goodCluster2      = &xdspb.Cluster{
 		Name:                 goodClusterName2,
 		ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS},
 		EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{
@@ -79,8 +79,8 @@ var (
 		},
 		LbPolicy: xdspb.Cluster_ROUND_ROBIN,
 	}
-	marshaledCluster2, _ = ptypes.MarshalAny(goodCluster2)
-	goodCDSResponse1     = &xdspb.DiscoveryResponse{
+	marshaledCluster2 = testutils.MarshalAny(goodCluster2)
+	goodCDSResponse1  = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			marshaledCluster1,
 		},
diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go
index 1e464405eeaf..371375f3ee5c 100644
--- a/xds/internal/client/v2/client_test.go
+++ b/xds/internal/client/v2/client_test.go
@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc"
@@ -114,30 +113,24 @@ var (
 			},
 		},
 	}
-	marshaledConnMgr1, _ = proto.Marshal(goodHTTPConnManager1)
-	goodListener1        = &xdspb.Listener{
+	marshaledConnMgr1 = testutils.MarshalAny(goodHTTPConnManager1)
+	goodListener1     = &xdspb.Listener{
 		Name: goodLDSTarget1,
 		ApiListener: &listenerpb.ApiListener{
-			ApiListener: &anypb.Any{
-				TypeUrl: httpConnManagerURL,
-				Value:   marshaledConnMgr1,
-			},
+			ApiListener: marshaledConnMgr1,
 		},
 	}
-	marshaledListener1, _ = ptypes.MarshalAny(goodListener1)
-	goodListener2         = &xdspb.Listener{
+	marshaledListener1 = testutils.MarshalAny(goodListener1)
+	goodListener2      = &xdspb.Listener{
 		Name: goodLDSTarget2,
 		ApiListener: &listenerpb.ApiListener{
-			ApiListener: &anypb.Any{
-				TypeUrl: httpConnManagerURL,
-				Value:   marshaledConnMgr1,
-			},
+			ApiListener: marshaledConnMgr1,
 		},
 	}
-	marshaledListener2, _     = ptypes.MarshalAny(goodListener2)
-	noAPIListener             = &xdspb.Listener{Name: goodLDSTarget1}
-	marshaledNoAPIListener, _ = proto.Marshal(noAPIListener)
-	badAPIListener2           = &xdspb.Listener{
+	marshaledListener2     = testutils.MarshalAny(goodListener2)
+	noAPIListener          = &xdspb.Listener{Name: goodLDSTarget1}
+	marshaledNoAPIListener = testutils.MarshalAny(noAPIListener)
+	badAPIListener2        = &xdspb.Listener{
 		Name: goodLDSTarget2,
 		ApiListener: &listenerpb.ApiListener{
 			ApiListener: &anypb.Any{
@@ -170,13 +163,8 @@ var (
 		TypeUrl: version.V2ListenerURL,
 	}
 	badResourceTypeInLDSResponse = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: httpConnManagerURL,
-				Value:   marshaledConnMgr1,
-			},
-		},
-		TypeUrl: version.V2ListenerURL,
+		Resources: []*anypb.Any{marshaledConnMgr1},
+		TypeUrl:   version.V2ListenerURL,
 	}
 	ldsResponseWithMultipleResources = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
@@ -186,13 +174,8 @@ var (
 		TypeUrl: version.V2ListenerURL,
 	}
 	noAPIListenerLDSResponse = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: version.V2ListenerURL,
-				Value:   marshaledNoAPIListener,
-			},
-		},
-		TypeUrl: version.V2ListenerURL,
+		Resources: []*anypb.Any{marshaledNoAPIListener},
+		TypeUrl:   version.V2ListenerURL,
 	}
 	goodBadUglyLDSResponse = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
@@ -215,19 +198,14 @@ var (
 		TypeUrl: version.V2RouteConfigURL,
 	}
 	badResourceTypeInRDSResponse = &xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: httpConnManagerURL,
-				Value:   marshaledConnMgr1,
-			},
-		},
-		TypeUrl: version.V2RouteConfigURL,
+		Resources: []*anypb.Any{marshaledConnMgr1},
+		TypeUrl:   version.V2RouteConfigURL,
 	}
 	noVirtualHostsRouteConfig = &xdspb.RouteConfiguration{
 		Name: goodRouteName1,
 	}
-	marshaledNoVirtualHostsRouteConfig, _ = ptypes.MarshalAny(noVirtualHostsRouteConfig)
-	noVirtualHostsInRDSResponse           = &xdspb.DiscoveryResponse{
+	marshaledNoVirtualHostsRouteConfig = testutils.MarshalAny(noVirtualHostsRouteConfig)
+	noVirtualHostsInRDSResponse        = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			marshaledNoVirtualHostsRouteConfig,
 		},
@@ -264,8 +242,8 @@ var (
 			},
 		},
 	}
-	marshaledGoodRouteConfig1, _ = ptypes.MarshalAny(goodRouteConfig1)
-	goodRouteConfig2             = &xdspb.RouteConfiguration{
+	marshaledGoodRouteConfig1 = testutils.MarshalAny(goodRouteConfig1)
+	goodRouteConfig2          = &xdspb.RouteConfiguration{
 		Name: goodRouteName2,
 		VirtualHosts: []*routepb.VirtualHost{
 			{
@@ -296,8 +274,8 @@ var (
 			},
 		},
 	}
-	marshaledGoodRouteConfig2, _ = ptypes.MarshalAny(goodRouteConfig2)
-	goodRDSResponse1             = &xdspb.DiscoveryResponse{
+	marshaledGoodRouteConfig2 = testutils.MarshalAny(goodRouteConfig2)
+	goodRDSResponse1          = &xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
 			marshaledGoodRouteConfig1,
 		},
diff --git a/xds/internal/client/v2/eds_test.go b/xds/internal/client/v2/eds_test.go
index 7eba32f5c605..08e75d373017 100644
--- a/xds/internal/client/v2/eds_test.go
+++ b/xds/internal/client/v2/eds_test.go
@@ -25,11 +25,11 @@ import (
 	"time"
 
 	v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	"github.com/golang/protobuf/ptypes"
 	anypb "github.com/golang/protobuf/ptypes/any"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/xds/internal"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xtestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -44,20 +44,14 @@ var (
 		TypeUrl: version.V2EndpointsURL,
 	}
 	badResourceTypeInEDSResponse = &v2xdspb.DiscoveryResponse{
-		Resources: []*anypb.Any{
-			{
-				TypeUrl: httpConnManagerURL,
-				Value:   marshaledConnMgr1,
-			},
-		},
-		TypeUrl: version.V2EndpointsURL,
+		Resources: []*anypb.Any{marshaledConnMgr1},
+		TypeUrl:   version.V2EndpointsURL,
 	}
 	marshaledGoodCLA1 = func() *anypb.Any {
-		clab0 := testutils.NewClusterLoadAssignmentBuilder(goodEDSName, nil)
+		clab0 := xtestutils.NewClusterLoadAssignmentBuilder(goodEDSName, nil)
 		clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil)
 		clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil)
-		a, _ := ptypes.MarshalAny(clab0.Build())
-		return a
+		return testutils.MarshalAny(clab0.Build())
 	}()
 	goodEDSResponse1 = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
@@ -66,10 +60,9 @@ var (
 		TypeUrl: version.V2EndpointsURL,
 	}
 	marshaledGoodCLA2 = func() *anypb.Any {
-		clab0 := testutils.NewClusterLoadAssignmentBuilder("not-goodEDSName", nil)
+		clab0 := xtestutils.NewClusterLoadAssignmentBuilder("not-goodEDSName", nil)
 		clab0.AddLocality("locality-1", 1, 0, []string{"addr1:314"}, nil)
-		a, _ := ptypes.MarshalAny(clab0.Build())
-		return a
+		return testutils.MarshalAny(clab0.Build())
 	}()
 	goodEDSResponse2 = &v2xdspb.DiscoveryResponse{
 		Resources: []*anypb.Any{
diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go
index 6aeea8a8a782..c132e912f92a 100644
--- a/xds/internal/httpfilter/fault/fault_test.go
+++ b/xds/internal/httpfilter/fault/fault_test.go
@@ -38,12 +38,13 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/xds/internal/httpfilter"
-	"google.golang.org/grpc/xds/internal/testutils"
+	xtestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/testutils/e2e"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
@@ -121,9 +122,9 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) {
 	testpb.RegisterTestServiceServer(server, &testService{})
 
 	// Create a local listener and pass it to Serve().
-	lis, err := testutils.LocalTCPListener()
+	lis, err := xtestutils.LocalTCPListener()
 	if err != nil {
-		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+		t.Fatalf("xtestutils.LocalTCPListener() failed: %v", err)
 	}
 
 	go func() {
@@ -524,10 +525,7 @@ func (s) TestFaultInjection_Unary(t *testing.T) {
 				hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter(fmt.Sprintf("fault%d", i), cfg))
 			}
 			hcm.HttpFilters = append(hcm.HttpFilters, routerFilter)
-			hcmAny, err := ptypes.MarshalAny(hcm)
-			if err != nil {
-				t.Fatal(err)
-			}
+			hcmAny := testutils.MarshalAny(hcm)
 			resources.Listeners[0].ApiListener.ApiListener = hcmAny
 			resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny}
 
@@ -600,10 +598,7 @@ func (s) TestFaultInjection_MaxActiveFaults(t *testing.T) {
 			},
 		})},
 		hcm.HttpFilters...)
-	hcmAny, err := ptypes.MarshalAny(hcm)
-	if err != nil {
-		t.Fatal(err)
-	}
+	hcmAny := testutils.MarshalAny(hcm)
 	resources.Listeners[0].ApiListener.ApiListener = hcmAny
 	resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny}
 
diff --git a/xds/internal/server/listener_wrapper_test.go b/xds/internal/server/listener_wrapper_test.go
index 8b5b5c3851de..b22f647a93ca 100644
--- a/xds/internal/server/listener_wrapper_test.go
+++ b/xds/internal/server/listener_wrapper_test.go
@@ -23,7 +23,6 @@ package server
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net"
 	"strconv"
 	"testing"
@@ -32,9 +31,6 @@ import (
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
-	anypb "github.com/golang/protobuf/ptypes/any"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
@@ -76,7 +72,7 @@ var listenerWithFilterChains = &v3listenerpb.Listener{
 			TransportSocket: &v3corepb.TransportSocket{
 				Name: "envoy.transport_sockets.tls",
 				ConfigType: &v3corepb.TransportSocket_TypedConfig{
-					TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+					TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 						CommonTlsContext: &v3tlspb.CommonTlsContext{
 							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 								InstanceName:    "identityPluginInstance",
@@ -327,11 +323,3 @@ func (s) TestListenerWrapper_Accept(t *testing.T) {
 		t.Fatalf("error when waiting for Accept() to return the conn on filter chain match: %v", err)
 	}
 }
-
-func marshalAny(m proto.Message) *anypb.Any {
-	a, err := ptypes.MarshalAny(m)
-	if err != nil {
-		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
-	}
-	return a
-}
diff --git a/xds/server_test.go b/xds/server_test.go
index 3fb3bcd3818b..e16ac36b01f2 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -33,9 +33,6 @@ import (
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	"github.com/golang/protobuf/proto"
-	"github.com/golang/protobuf/ptypes"
-	anypb "github.com/golang/protobuf/ptypes/any"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
@@ -672,7 +669,7 @@ func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) {
 				TransportSocket: &v3corepb.TransportSocket{
 					Name: "envoy.transport_sockets.tls",
 					ConfigType: &v3corepb.TransportSocket_TypedConfig{
-						TypedConfig: marshalAny(&v3tlspb.DownstreamTlsContext{
+						TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{
 							CommonTlsContext: &v3tlspb.CommonTlsContext{
 								TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
 									InstanceName:    "identityPluginInstance",
@@ -783,11 +780,3 @@ func verifyCertProviderNotCreated() error {
 	}
 	return nil
 }
-
-func marshalAny(m proto.Message) *anypb.Any {
-	a, err := ptypes.MarshalAny(m)
-	if err != nil {
-		panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err))
-	}
-	return a
-}

From b1940e15f6778067675e2192d8947608e8a20e32 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Mon, 10 May 2021 10:11:31 -0700
Subject: [PATCH 464/481] xds: register resources at the mgmt server before
 requesting them (#4406)

---
 .../test/xds_server_serving_mode_test.go      | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/xds/internal/test/xds_server_serving_mode_test.go b/xds/internal/test/xds_server_serving_mode_test.go
index 664f0b85759a..414a559b0982 100644
--- a/xds/internal/test/xds_server_serving_mode_test.go
+++ b/xds/internal/test/xds_server_serving_mode_test.go
@@ -119,17 +119,6 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
 	}
 
-	go func() {
-		if err := server.Serve(lis1); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-	go func() {
-		if err := server.Serve(lis2); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-
 	// Setup the management server to respond with server-side Listener
 	// resources for both listeners.
 	host1, port1, err := hostPortFromListener(lis1)
@@ -150,6 +139,17 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	go func() {
+		if err := server.Serve(lis1); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+	go func() {
+		if err := server.Serve(lis2); err != nil {
+			t.Errorf("Serve() failed: %v", err)
+		}
+	}()
+
 	// Wait for both listeners to move to "serving" mode.
 	if err := waitForModeChange(ctx, modeTracker, lis1.Addr(), xds.ServingModeServing); err != nil {
 		t.Fatal(err)

From 5f95ad62331add45bbf5ee167b67cadc72e1d322 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Tue, 11 May 2021 10:39:31 -0700
Subject: [PATCH 465/481] xds: workaround to deflake xds e2e tests (#4413)

---
 xds/internal/test/xds_integration_test.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/xds/internal/test/xds_integration_test.go b/xds/internal/test/xds_integration_test.go
index a41fec929762..c2bb6bc18f67 100644
--- a/xds/internal/test/xds_integration_test.go
+++ b/xds/internal/test/xds_integration_test.go
@@ -83,6 +83,11 @@ func TestMain(m *testing.M) {
 	// spawns the management server and is blocked on the call to `Serve()`.
 	leakcheck.RegisterIgnoreGoroutine("e2e.StartManagementServer")
 
+	// Remove this once https://github.com/envoyproxy/go-control-plane/pull/430
+	// is merged. For more information about this goroutine leak, see:
+	// https://github.com/envoyproxy/go-control-plane/issues/429.
+	leakcheck.RegisterIgnoreGoroutine("(*server).StreamHandler")
+
 	cancel, err := setupManagementServer()
 	if err != nil {
 		log.Printf("setupManagementServer() failed: %v", err)

From 81b8cca6a9d92794be3e789b179e798aa1bc3209 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Tue, 11 May 2021 15:28:46 -0700
Subject: [PATCH 466/481] Change version to 1.39.0-dev (#4420)

---
 version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.go b/version.go
index f73faed82920..4e26aec6ac18 100644
--- a/version.go
+++ b/version.go
@@ -19,4 +19,4 @@
 package grpc
 
 // Version is the current grpc version.
-const Version = "1.38.0-dev"
+const Version = "1.39.0-dev"

From 62adda2ece5ec803c824c5009b83cea86de5030d Mon Sep 17 00:00:00 2001
From: Doug Fawley <dfawley@google.com>
Date: Tue, 11 May 2021 17:05:16 -0700
Subject: [PATCH 467/481] client: fix ForceCodec to set content-type header
 appropriately (#4401)

---
 rpc_util.go          | 19 ++++++++++++----
 test/end2end_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/rpc_util.go b/rpc_util.go
index c8ae0e4444c7..6db356fa56a7 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -429,9 +429,10 @@ func (o ContentSubtypeCallOption) before(c *callInfo) error {
 }
 func (o ContentSubtypeCallOption) after(c *callInfo, attempt *csAttempt) {}
 
-// ForceCodec returns a CallOption that will set codec to be
-// used for all request and response messages for a call. The result of calling
-// Name() will be used as the content-subtype in a case-insensitive manner.
+// ForceCodec returns a CallOption that will set codec to be used for all
+// request and response messages for a call. The result of calling Name() will
+// be used as the content-subtype after converting to lowercase, unless
+// CallContentSubtype is also used.
 //
 // See Content-Type on
 // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
@@ -853,7 +854,17 @@ func toRPCErr(err error) error {
 // setCallInfoCodec should only be called after CallOptions have been applied.
 func setCallInfoCodec(c *callInfo) error {
 	if c.codec != nil {
-		// codec was already set by a CallOption; use it.
+		// codec was already set by a CallOption; use it, but set the content
+		// subtype if it is not set.
+		if c.contentSubtype == "" {
+			// c.codec is a baseCodec to hide the difference between grpc.Codec and
+			// encoding.Codec (Name vs. String method name).  We only support
+			// setting content subtype from encoding.Codec to avoid a behavior
+			// change with the deprecated version.
+			if ec, ok := c.codec.(encoding.Codec); ok {
+				c.contentSubtype = strings.ToLower(ec.Name())
+			}
+		}
 		return nil
 	}
 
diff --git a/test/end2end_test.go b/test/end2end_test.go
index 861a2f29623d..eb91d09afdf0 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -5293,7 +5293,7 @@ func (s) TestGRPCMethod(t *testing.T) {
 	}
 	defer ss.Stop()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 
 	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
@@ -5305,6 +5305,55 @@ func (s) TestGRPCMethod(t *testing.T) {
 	}
 }
 
+// renameProtoCodec is an encoding.Codec wrapper that allows customizing the
+// Name() of another codec.
+type renameProtoCodec struct {
+	encoding.Codec
+	name string
+}
+
+func (r *renameProtoCodec) Name() string { return r.name }
+
+// TestForceCodecName confirms that the ForceCodec call option sets the subtype
+// in the content-type header according to the Name() of the codec provided.
+func (s) TestForceCodecName(t *testing.T) {
+	wantContentTypeCh := make(chan []string, 1)
+	defer close(wantContentTypeCh)
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			md, ok := metadata.FromIncomingContext(ctx)
+			if !ok {
+				return nil, status.Errorf(codes.Internal, "no metadata in context")
+			}
+			if got, want := md["content-type"], <-wantContentTypeCh; !reflect.DeepEqual(got, want) {
+				return nil, status.Errorf(codes.Internal, "got content-type=%q; want [%q]", got, want)
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start([]grpc.ServerOption{grpc.ForceServerCodec(encoding.GetCodec("proto"))}); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	codec := &renameProtoCodec{Codec: encoding.GetCodec("proto"), name: "some-test-name"}
+	wantContentTypeCh <- []string{"application/grpc+some-test-name"}
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodec(codec)); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+
+	// Confirm the name is converted to lowercase before transmitting.
+	codec.name = "aNoTHeRNaME"
+	wantContentTypeCh <- []string{"application/grpc+anothername"}
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodec(codec)); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+}
+
 func (s) TestForceServerCodec(t *testing.T) {
 	ss := &stubserver.StubServer{
 		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
@@ -5317,7 +5366,7 @@ func (s) TestForceServerCodec(t *testing.T) {
 	}
 	defer ss.Stop()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 
 	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {

From a95a5c3bacecea965def0addd986b3ef709f6e27 Mon Sep 17 00:00:00 2001
From: James Protzman <JNProtzman@gmail.com>
Date: Wed, 12 May 2021 11:49:07 -0400
Subject: [PATCH 468/481] transport: remove decodeState from client to reduce
 allocations (#3313)

---
 internal/transport/http2_client.go | 117 ++++++++++++++++++++++++++---
 internal/transport/http_util.go    |  34 ++++++---
 2 files changed, 130 insertions(+), 21 deletions(-)

diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index 48c5e52edae9..64ebd4a167f3 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -1254,11 +1254,97 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 		return
 	}
 
-	state := &decodeState{}
-	// Initialize isGRPC value to be !initialHeader, since if a gRPC Response-Headers has already been received, then it means that the peer is speaking gRPC and we are in gRPC mode.
-	state.data.isGRPC = !initialHeader
-	if h2code, err := state.decodeHeader(frame); err != nil {
-		t.closeStream(s, err, true, h2code, status.Convert(err), nil, endStream)
+	// frame.Truncated is set to true when framer detects that the current header
+	// list size hits MaxHeaderListSize limit.
+	if frame.Truncated {
+		se := status.New(codes.Internal, "peer header list size exceeded limit")
+		t.closeStream(s, se.Err(), true, http2.ErrCodeFrameSize, se, nil, endStream)
+		return
+	}
+
+	var (
+		// If a gRPC Response-Headers has already been received, then it means
+		// that the peer is speaking gRPC and we are in gRPC mode.
+		isGRPC         = !initialHeader
+		mdata          = make(map[string][]string)
+		contentTypeErr string
+		grpcMessage    string
+		statusGen      *status.Status
+
+		httpStatus string
+		rawStatus  string
+		// headerError is set if an error is encountered while parsing the headers
+		headerError string
+	)
+
+	for _, hf := range frame.Fields {
+		switch hf.Name {
+		case "content-type":
+			if _, validContentType := grpcutil.ContentSubtype(hf.Value); !validContentType {
+				contentTypeErr = fmt.Sprintf("transport: received the unexpected content-type %q", hf.Value)
+				break
+			}
+			mdata[hf.Name] = append(mdata[hf.Name], hf.Value)
+			isGRPC = true
+		case "grpc-encoding":
+			s.recvCompress = hf.Value
+		case "grpc-status":
+			rawStatus = hf.Value
+		case "grpc-message":
+			grpcMessage = decodeGrpcMessage(hf.Value)
+		case "grpc-status-details-bin":
+			var err error
+			statusGen, err = decodeGRPCStatusDetails(hf.Value)
+			if err != nil {
+				headerError = fmt.Sprintf("transport: malformed grpc-status-details-bin: %v", err)
+			}
+		case ":status":
+			httpStatus = hf.Value
+		default:
+			if isReservedHeader(hf.Name) && !isWhitelistedHeader(hf.Name) {
+				break
+			}
+			v, err := decodeMetadataHeader(hf.Name, hf.Value)
+			if err != nil {
+				headerError = fmt.Sprintf("transport: malformed %s: %v", hf.Name, err)
+				logger.Warningf("Failed to decode metadata header (%q, %q): %v", hf.Name, hf.Value, err)
+				break
+			}
+			mdata[hf.Name] = append(mdata[hf.Name], v)
+		}
+	}
+
+	if !isGRPC {
+		var (
+			code           = codes.Internal // when header does not include HTTP status, return INTERNAL
+			httpStatusCode int
+		)
+
+		if httpStatus != "" {
+			c, err := strconv.ParseInt(httpStatus, 10, 32)
+			if err != nil {
+				se := status.New(codes.Internal, fmt.Sprintf("transport: malformed http-status: %v", err))
+				t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream)
+				return
+			}
+			httpStatusCode = int(c)
+
+			var ok bool
+			code, ok = HTTPStatusConvTab[httpStatusCode]
+			if !ok {
+				code = codes.Unknown
+			}
+		}
+
+		// Verify the HTTP response is a 200.
+		se := status.New(code, constructHTTPErrMsg(&httpStatusCode, contentTypeErr))
+		t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream)
+		return
+	}
+
+	if headerError != "" {
+		se := status.New(codes.Internal, headerError)
+		t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream)
 		return
 	}
 
@@ -1293,9 +1379,8 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 			// These values can be set without any synchronization because
 			// stream goroutine will read it only after seeing a closed
 			// headerChan which we'll close after setting this.
-			s.recvCompress = state.data.encoding
-			if len(state.data.mdata) > 0 {
-				s.header = state.data.mdata
+			if len(mdata) > 0 {
+				s.header = mdata
 			}
 		} else {
 			// HEADERS frame block carries a Trailers-Only.
@@ -1308,9 +1393,23 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 		return
 	}
 
+	if statusGen == nil {
+		rawStatusCode := codes.Unknown
+		if rawStatus != "" {
+			code, err := strconv.ParseInt(rawStatus, 10, 32)
+			if err != nil {
+				se := status.New(codes.Internal, fmt.Sprintf("transport: malformed grpc-status: %v", err))
+				t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream)
+				return
+			}
+			rawStatusCode = codes.Code(uint32(code))
+		}
+		statusGen = status.New(rawStatusCode, grpcMessage)
+	}
+
 	// if client received END_STREAM from server while stream was still active, send RST_STREAM
 	rst := s.getState() == streamActive
-	t.closeStream(s, io.EOF, rst, http2.ErrCodeNo, state.status(), state.data.mdata, true)
+	t.closeStream(s, io.EOF, rst, http2.ErrCodeNo, statusGen, mdata, true)
 }
 
 // reader runs as a separate goroutine in charge of reading data from network
diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go
index c7dee140cf1a..2771e224f7bf 100644
--- a/internal/transport/http_util.go
+++ b/internal/transport/http_util.go
@@ -180,14 +180,6 @@ func isWhitelistedHeader(hdr string) bool {
 	}
 }
 
-func (d *decodeState) status() *status.Status {
-	if d.data.statusGen == nil {
-		// No status-details were provided; generate status using code/msg.
-		d.data.statusGen = status.New(codes.Code(int32(*(d.data.rawStatusCode))), d.data.rawStatusMsg)
-	}
-	return d.data.statusGen
-}
-
 const binHdrSuffix = "-bin"
 
 func encodeBinHeader(v []byte) string {
@@ -217,6 +209,18 @@ func decodeMetadataHeader(k, v string) (string, error) {
 	return v, nil
 }
 
+func decodeGRPCStatusDetails(rawDetails string) (*status.Status, error) {
+	v, err := decodeBinHeader(rawDetails)
+	if err != nil {
+		return nil, err
+	}
+	st := &spb.Status{}
+	if err = proto.Unmarshal(v, st); err != nil {
+		return nil, err
+	}
+	return status.FromProto(st), nil
+}
+
 func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) (http2.ErrCode, error) {
 	// frame.Truncated is set to true when framer detects that the current header
 	// list size hits MaxHeaderListSize limit.
@@ -271,18 +275,24 @@ func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) (http2.ErrCode
 // constructErrMsg constructs error message to be returned in HTTP fallback mode.
 // Format: HTTP status code and its corresponding message + content-type error message.
 func (d *decodeState) constructHTTPErrMsg() string {
+	return constructHTTPErrMsg(d.data.httpStatus, d.data.contentTypeErr)
+}
+
+// constructErrMsg constructs error message to be returned in HTTP fallback mode.
+// Format: HTTP status code and its corresponding message + content-type error message.
+func constructHTTPErrMsg(httpStatus *int, contentTypeErr string) string {
 	var errMsgs []string
 
-	if d.data.httpStatus == nil {
+	if httpStatus == nil {
 		errMsgs = append(errMsgs, "malformed header: missing HTTP status")
 	} else {
-		errMsgs = append(errMsgs, fmt.Sprintf("%s: HTTP status code %d", http.StatusText(*(d.data.httpStatus)), *d.data.httpStatus))
+		errMsgs = append(errMsgs, fmt.Sprintf("%s: HTTP status code %d", http.StatusText(*(httpStatus)), *httpStatus))
 	}
 
-	if d.data.contentTypeErr == "" {
+	if contentTypeErr == "" {
 		errMsgs = append(errMsgs, "transport: missing content-type field")
 	} else {
-		errMsgs = append(errMsgs, d.data.contentTypeErr)
+		errMsgs = append(errMsgs, contentTypeErr)
 	}
 
 	return strings.Join(errMsgs, "; ")

From aa59641d5da52eaa3728c4624e16a3ac76688c39 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 12 May 2021 10:17:13 -0700
Subject: [PATCH 469/481] interop: use credentials.NewTLS() when possible
 (#4390)

---
 interop/client/client.go | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/interop/client/client.go b/interop/client/client.go
index 975e0b5d2f3c..7b9339d7b614 100644
--- a/interop/client/client.go
+++ b/interop/client/client.go
@@ -20,7 +20,10 @@
 package main
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"flag"
+	"io/ioutil"
 	"net"
 	"strconv"
 
@@ -57,7 +60,7 @@ var (
 	serverHost            = flag.String("server_host", "localhost", "The server host name")
 	serverPort            = flag.Int("server_port", 10000, "The server port number")
 	serviceConfigJSON     = flag.String("service_config_json", "", "Disables service config lookups and sets the provided string as the default service config.")
-	tlsServerName         = flag.String("server_host_override", "", "The server name use to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.")
+	tlsServerName         = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.")
 	testCase              = flag.String("test_case", "large_unary",
 		`Configure different test cases. Valid options are:
         empty_unary : empty (zero bytes) request and response;
@@ -135,22 +138,25 @@ func main() {
 	var opts []grpc.DialOption
 	switch credsChosen {
 	case credsTLS:
-		var sn string
-		if *tlsServerName != "" {
-			sn = *tlsServerName
-		}
-		var creds credentials.TransportCredentials
+		var roots *x509.CertPool
 		if *testCA {
-			var err error
 			if *caFile == "" {
 				*caFile = testdata.Path("ca.pem")
 			}
-			creds, err = credentials.NewClientTLSFromFile(*caFile, sn)
+			b, err := ioutil.ReadFile(*caFile)
 			if err != nil {
-				logger.Fatalf("Failed to create TLS credentials %v", err)
+				logger.Fatalf("Failed to read root certificate file %q: %v", *caFile, err)
+			}
+			roots = x509.NewCertPool()
+			if !roots.AppendCertsFromPEM(b) {
+				logger.Fatalf("Failed to append certificates: %s", string(b))
 			}
+		}
+		var creds credentials.TransportCredentials
+		if *tlsServerName != "" {
+			creds = credentials.NewClientTLSFromCert(roots, *tlsServerName)
 		} else {
-			creds = credentials.NewClientTLSFromCert(nil, sn)
+			creds = credentials.NewTLS(&tls.Config{RootCAs: roots})
 		}
 		opts = append(opts, grpc.WithTransportCredentials(creds))
 	case credsALTS:

From 8bf65c69b99ed9e1106c07c1f5d2f42f312b7ec5 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 12 May 2021 10:18:50 -0700
Subject: [PATCH 470/481] xds: use same format while registering and watching
 resources (#4422)

---
 xds/internal/testutils/e2e/clientresources.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/xds/internal/testutils/e2e/clientresources.go b/xds/internal/testutils/e2e/clientresources.go
index b521db950558..7c8311a51cc3 100644
--- a/xds/internal/testutils/e2e/clientresources.go
+++ b/xds/internal/testutils/e2e/clientresources.go
@@ -20,6 +20,8 @@ package e2e
 
 import (
 	"fmt"
+	"net"
+	"strconv"
 
 	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	"github.com/golang/protobuf/proto"
@@ -160,7 +162,7 @@ func DefaultServerListener(host string, port uint32, secLevel SecurityLevel) *v3
 		}
 	}
 	return &v3listenerpb.Listener{
-		Name: fmt.Sprintf(ServerListenerResourceNameTemplate, fmt.Sprintf("%s:%d", host, port)),
+		Name: fmt.Sprintf(ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
 		Address: &v3corepb.Address{
 			Address: &v3corepb.Address_SocketAddress{
 				SocketAddress: &v3corepb.SocketAddress{

From 45e60095da54baad1e7ae28391941b64a40477e5 Mon Sep 17 00:00:00 2001
From: Zach Reyes <39203661+zasweq@users.noreply.github.com>
Date: Wed, 12 May 2021 17:28:49 -0400
Subject: [PATCH 471/481] xds: add support for aggregate clusters (#4332)

Add support for aggregate clusters in CDS Balancer
---
 .../balancer/cdsbalancer/cdsbalancer.go       |   5 +
 .../cdsbalancer/cdsbalancer_security_test.go  |   2 +-
 .../balancer/cdsbalancer/cdsbalancer_test.go  |  14 +-
 .../balancer/cdsbalancer/cluster_handler.go   | 273 +++++++
 .../cdsbalancer/cluster_handler_test.go       | 676 ++++++++++++++++++
 xds/internal/testutils/fakeclient/client.go   |  42 +-
 6 files changed, 992 insertions(+), 20 deletions(-)
 create mode 100644 xds/internal/balancer/cdsbalancer/cluster_handler.go
 create mode 100644 xds/internal/balancer/cdsbalancer/cluster_handler_test.go

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index bf1519bb8ce0..9b987c00c2ca 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -151,6 +151,11 @@ type ccUpdate struct {
 	err         error
 }
 
+type clusterHandlerUpdate struct {
+	chu []xdsclient.ClusterUpdate
+	err error
+}
+
 // scUpdate wraps a subConn update received from gRPC. This is directly passed
 // on to the edsBalancer.
 type scUpdate struct {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 5c746cfa163c..d1074f2a1c45 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -640,7 +640,7 @@ func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
 	// registered watch should not be cancelled.
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+	if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 }
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index 4476a1532d05..e93df5a10aab 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -399,7 +399,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// registered watch should not be cancelled.
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+	if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// The CDS balancer has not yet created an EDS balancer. So, this resolver
@@ -438,7 +438,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// Make sure the registered watch is not cancelled.
 	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+	if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// Make sure the error is forwarded to the EDS balancer.
@@ -453,7 +453,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// request cluster resource is not found. We should continue to watch it.
 	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+	if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a resource-not-found-error")
 	}
 	// Make sure the error is forwarded to the EDS balancer.
@@ -485,7 +485,7 @@ func (s) TestResolverError(t *testing.T) {
 	// registered watch should not be cancelled.
 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+	if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// The CDS balancer has not yet created an EDS balancer. So, this resolver
@@ -523,7 +523,7 @@ func (s) TestResolverError(t *testing.T) {
 	// Make sure the registered watch is not cancelled.
 	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
-	if err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
+	if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
 		t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
 	}
 	// Make sure the error is forwarded to the EDS balancer.
@@ -535,7 +535,7 @@ func (s) TestResolverError(t *testing.T) {
 	resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error")
 	cdsB.ResolverError(resourceErr)
 	// Make sure the registered watch is cancelled.
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
+	if _, err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatalf("want watch to be canceled, watchForCancel failed: %v", err)
 	}
 	// Make sure the error is forwarded to the EDS balancer.
@@ -642,7 +642,7 @@ func (s) TestClose(t *testing.T) {
 
 	// Make sure that the cluster watch registered by the CDS balancer is
 	// cancelled.
-	if err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
+	if _, err := xdsC.WaitForCancelClusterWatch(ctx); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/xds/internal/balancer/cdsbalancer/cluster_handler.go b/xds/internal/balancer/cdsbalancer/cluster_handler.go
new file mode 100644
index 000000000000..2dafb212f4c9
--- /dev/null
+++ b/xds/internal/balancer/cdsbalancer/cluster_handler.go
@@ -0,0 +1,273 @@
+/*
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"errors"
+	"sync"
+
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+)
+
+var errNotReceivedUpdate = errors.New("tried to construct a cluster update on a cluster that has not received an update")
+
+// clusterHandler will be given a name representing a cluster. It will then
+// update the CDS policy constantly with a list of Clusters to pass down to
+// XdsClusterResolverLoadBalancingPolicyConfig in a stream like fashion.
+type clusterHandler struct {
+	// A mutex to protect entire tree of clusters.
+	clusterMutex    sync.Mutex
+	root            *clusterNode
+	rootClusterName string
+
+	// A way to ping CDS Balancer about any updates or errors to a Node in the
+	// tree. This will either get called from this handler constructing an
+	// update or from a child with an error. Capacity of one as the only update
+	// CDS Balancer cares about is the most recent update.
+	updateChannel chan clusterHandlerUpdate
+
+	xdsClient xdsClientInterface
+}
+
+func (ch *clusterHandler) updateRootCluster(rootClusterName string) {
+	ch.clusterMutex.Lock()
+	defer ch.clusterMutex.Unlock()
+	if ch.root == nil {
+		// Construct a root node on first update.
+		ch.root = createClusterNode(rootClusterName, ch.xdsClient, ch)
+		ch.rootClusterName = rootClusterName
+		return
+	}
+	// Check if root cluster was changed. If it was, delete old one and start
+	// new one, if not do nothing.
+	if rootClusterName != ch.rootClusterName {
+		ch.root.delete()
+		ch.root = createClusterNode(rootClusterName, ch.xdsClient, ch)
+		ch.rootClusterName = rootClusterName
+	}
+}
+
+// This function tries to construct a cluster update to send to CDS.
+func (ch *clusterHandler) constructClusterUpdate() {
+	// If there was an error received no op, as this simply means one of the
+	// children hasn't received an update yet.
+	if clusterUpdate, err := ch.root.constructClusterUpdate(); err == nil {
+		// For a ClusterUpdate, the only update CDS cares about is the most
+		// recent one, so opportunistically drain the update channel before
+		// sending the new update.
+		select {
+		case <-ch.updateChannel:
+		default:
+		}
+		ch.updateChannel <- clusterHandlerUpdate{chu: clusterUpdate, err: nil}
+	}
+}
+
+// close() is meant to be called by CDS when the CDS balancer is closed, and it
+// cancels the watches for every cluster in the cluster tree.
+func (ch *clusterHandler) close() {
+	ch.clusterMutex.Lock()
+	defer ch.clusterMutex.Unlock()
+	ch.root.delete()
+	ch.root = nil
+	ch.rootClusterName = ""
+}
+
+// This logically represents a cluster. This handles all the logic for starting
+// and stopping a cluster watch, handling any updates, and constructing a list
+// recursively for the ClusterHandler.
+type clusterNode struct {
+	// A way to cancel the watch for the cluster.
+	cancelFunc func()
+
+	// A list of children, as the Node can be an aggregate Cluster.
+	children []*clusterNode
+
+	// A ClusterUpdate in order to build a list of cluster updates for CDS to
+	// send down to child XdsClusterResolverLoadBalancingPolicy.
+	clusterUpdate xdsclient.ClusterUpdate
+
+	// This boolean determines whether this Node has received an update or not.
+	// This isn't the best practice, but this will protect a list of Cluster
+	// Updates from being constructed if a cluster in the tree has not received
+	// an update yet.
+	receivedUpdate bool
+
+	clusterHandler *clusterHandler
+}
+
+// CreateClusterNode creates a cluster node from a given clusterName. This will
+// also start the watch for that cluster.
+func createClusterNode(clusterName string, xdsClient xdsClientInterface, topLevelHandler *clusterHandler) *clusterNode {
+	c := &clusterNode{
+		clusterHandler: topLevelHandler,
+	}
+	// Communicate with the xds client here.
+	c.cancelFunc = xdsClient.WatchCluster(clusterName, c.handleResp)
+	return c
+}
+
+// This function cancels the cluster watch on the cluster and all of it's
+// children.
+func (c *clusterNode) delete() {
+	c.cancelFunc()
+	for _, child := range c.children {
+		child.delete()
+	}
+}
+
+// Construct cluster update (potentially a list of ClusterUpdates) for a node.
+func (c *clusterNode) constructClusterUpdate() ([]xdsclient.ClusterUpdate, error) {
+	// If the cluster has not yet received an update, the cluster update is not
+	// yet ready.
+	if !c.receivedUpdate {
+		return nil, errNotReceivedUpdate
+	}
+
+	// Base case - LogicalDNS or EDS. Both of these cluster types will be tied
+	// to a single ClusterUpdate.
+	if c.clusterUpdate.ClusterType != xdsclient.ClusterTypeAggregate {
+		return []xdsclient.ClusterUpdate{c.clusterUpdate}, nil
+	}
+
+	// If an aggregate construct a list by recursively calling down to all of
+	// it's children.
+	var childrenUpdates []xdsclient.ClusterUpdate
+	for _, child := range c.children {
+		childUpdateList, err := child.constructClusterUpdate()
+		if err != nil {
+			return nil, err
+		}
+		childrenUpdates = append(childrenUpdates, childUpdateList...)
+	}
+	return childrenUpdates, nil
+}
+
+// handleResp handles a xds response for a particular cluster. This function
+// also handles any logic with regards to any child state that may have changed.
+// At the end of the handleResp(), the clusterUpdate will be pinged in certain
+// situations to try and construct an update to send back to CDS.
+func (c *clusterNode) handleResp(clusterUpdate xdsclient.ClusterUpdate, err error) {
+	c.clusterHandler.clusterMutex.Lock()
+	defer c.clusterHandler.clusterMutex.Unlock()
+	if err != nil { // Write this error for run() to pick up in CDS LB policy.
+		// For a ClusterUpdate, the only update CDS cares about is the most
+		// recent one, so opportunistically drain the update channel before
+		// sending the new update.
+		select {
+		case <-c.clusterHandler.updateChannel:
+		default:
+		}
+		c.clusterHandler.updateChannel <- clusterHandlerUpdate{chu: nil, err: err}
+		return
+	}
+
+	// deltaInClusterUpdateFields determines whether there was a delta in the
+	// clusterUpdate fields (forgetting the children). This will be used to help
+	// determine whether to pingClusterHandler at the end of this callback or
+	// not.
+	deltaInClusterUpdateFields := clusterUpdate.ServiceName != c.clusterUpdate.ServiceName || clusterUpdate.ClusterType != c.clusterUpdate.ClusterType
+	c.receivedUpdate = true
+	c.clusterUpdate = clusterUpdate
+
+	// If the cluster was a leaf node, if the cluster update received had change
+	// in the cluster update then the overall cluster update would change and
+	// there is a possibility for the overall update to build so ping cluster
+	// handler to return. Also, if there was any children from previously,
+	// delete the children, as the cluster type is no longer an aggregate
+	// cluster.
+	if clusterUpdate.ClusterType != xdsclient.ClusterTypeAggregate {
+		for _, child := range c.children {
+			child.delete()
+		}
+		c.children = nil
+		if deltaInClusterUpdateFields {
+			c.clusterHandler.constructClusterUpdate()
+		}
+		return
+	}
+
+	// Aggregate cluster handling.
+	newChildren := make(map[string]bool)
+	for _, childName := range clusterUpdate.PrioritizedClusterNames {
+		newChildren[childName] = true
+	}
+
+	// These booleans help determine whether this callback will ping the overall
+	// clusterHandler to try and construct an update to send back to CDS. This
+	// will be determined by whether there would be a change in the overall
+	// clusterUpdate for the whole tree (ex. change in clusterUpdate for current
+	// cluster or a deleted child) and also if there's even a possibility for
+	// the update to build (ex. if a child is created and a watch is started,
+	// that child hasn't received an update yet due to the mutex lock on this
+	// callback).
+	var createdChild, deletedChild bool
+
+	// This map will represent the current children of the cluster. It will be
+	// first added to in order to represent the new children. It will then have
+	// any children deleted that are no longer present. Then, from the cluster
+	// update received, will be used to construct the new child list.
+	mapCurrentChildren := make(map[string]*clusterNode)
+	for _, child := range c.children {
+		mapCurrentChildren[child.clusterUpdate.ServiceName] = child
+	}
+
+	// Add and construct any new child nodes.
+	for child := range newChildren {
+		if _, inChildrenAlready := mapCurrentChildren[child]; !inChildrenAlready {
+			createdChild = true
+			mapCurrentChildren[child] = createClusterNode(child, c.clusterHandler.xdsClient, c.clusterHandler)
+		}
+	}
+
+	// Delete any child nodes no longer in the aggregate cluster's children.
+	for child := range mapCurrentChildren {
+		if _, stillAChild := newChildren[child]; !stillAChild {
+			deletedChild = true
+			mapCurrentChildren[child].delete()
+			delete(mapCurrentChildren, child)
+		}
+	}
+
+	// The order of the children list matters, so use the clusterUpdate from
+	// xdsclient as the ordering, and use that logical ordering for the new
+	// children list. This will be a mixture of child nodes which are all
+	// already constructed in the mapCurrentChildrenMap.
+	var children = make([]*clusterNode, 0, len(clusterUpdate.PrioritizedClusterNames))
+
+	for _, orderedChild := range clusterUpdate.PrioritizedClusterNames {
+		// The cluster's already have watches started for them in xds client, so
+		// you can use these pointers to construct the new children list, you
+		// just have to put them in the correct order using the original cluster
+		// update.
+		currentChild := mapCurrentChildren[orderedChild]
+		children = append(children, currentChild)
+	}
+
+	c.children = children
+
+	// If the cluster is an aggregate cluster, if this callback created any new
+	// child cluster nodes, then there's no possibility for a full cluster
+	// update to successfully build, as those created children will not have
+	// received an update yet. However, if there was simply a child deleted,
+	// then there is a possibility that it will have a full cluster update to
+	// build and also will have a changed overall cluster update from the
+	// deleted child.
+	if deletedChild && !createdChild {
+		c.clusterHandler.constructClusterUpdate()
+	}
+}
diff --git a/xds/internal/balancer/cdsbalancer/cluster_handler_test.go b/xds/internal/balancer/cdsbalancer/cluster_handler_test.go
new file mode 100644
index 000000000000..96fca8a2696a
--- /dev/null
+++ b/xds/internal/balancer/cdsbalancer/cluster_handler_test.go
@@ -0,0 +1,676 @@
+// +build go1.12
+
+/*
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	xdsclient "google.golang.org/grpc/xds/internal/client"
+	"google.golang.org/grpc/xds/internal/testutils/fakeclient"
+)
+
+const (
+	edsService              = "EDS Service"
+	logicalDNSService       = "Logical DNS Service"
+	edsService2             = "EDS Service 2"
+	logicalDNSService2      = "Logical DNS Service 2"
+	aggregateClusterService = "Aggregate Cluster Service"
+)
+
+// setupTests creates a clusterHandler with a fake xds client for control over
+// xds client.
+func setupTests(t *testing.T) (*clusterHandler, *fakeclient.Client) {
+	xdsC := fakeclient.NewClient()
+	ch := &clusterHandler{
+		xdsClient: xdsC,
+		// This is will be how the update channel is created in cds. It will be
+		// a separate channel to the buffer.Unbounded. This channel will also be
+		// read from to test any cluster updates.
+		updateChannel: make(chan clusterHandlerUpdate, 1),
+	}
+	return ch, xdsC
+}
+
+// Simplest case: the cluster handler receives a cluster name, handler starts a
+// watch for that cluster, xds client returns that it is a Leaf Node (EDS or
+// LogicalDNS), not a tree, so expectation that update is written to buffer
+// which will be read by CDS LB.
+func (s) TestSuccessCaseLeafNode(t *testing.T) {
+	tests := []struct {
+		name          string
+		clusterName   string
+		clusterUpdate xdsclient.ClusterUpdate
+	}{
+		{name: "test-update-root-cluster-EDS-success",
+			clusterName: edsService,
+			clusterUpdate: xdsclient.ClusterUpdate{
+				ClusterType: xdsclient.ClusterTypeEDS,
+				ServiceName: edsService,
+			}},
+		{
+			name:        "test-update-root-cluster-Logical-DNS-success",
+			clusterName: logicalDNSService,
+			clusterUpdate: xdsclient.ClusterUpdate{
+				ClusterType: xdsclient.ClusterTypeLogicalDNS,
+				ServiceName: logicalDNSService,
+			}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ch, fakeClient := setupTests(t)
+			// When you first update the root cluster, it should hit the code
+			// path which will start a cluster node for that root. Updating the
+			// root cluster logically represents a ping from a ClientConn.
+			ch.updateRootCluster(test.clusterName)
+			// Starting a cluster node involves communicating with the
+			// xdsClient, telling it to watch a cluster.
+			ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer ctxCancel()
+			gotCluster, err := fakeClient.WaitForWatchCluster(ctx)
+			if err != nil {
+				t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+			}
+			if gotCluster != test.clusterName {
+				t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, test.clusterName)
+			}
+			// Invoke callback with xds client with a certain clusterUpdate. Due
+			// to this cluster update filling out the whole cluster tree, as the
+			// cluster is of a root type (EDS or Logical DNS) and not an
+			// aggregate cluster, this should trigger the ClusterHandler to
+			// write to the update buffer to update the CDS policy.
+			fakeClient.InvokeWatchClusterCallback(test.clusterUpdate, nil)
+			select {
+			case chu := <-ch.updateChannel:
+				if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{test.clusterUpdate}); diff != "" {
+					t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
+				}
+			case <-ctx.Done():
+				t.Fatal("Timed out waiting for update from update channel.")
+			}
+			// Close the clusterHandler. This is meant to be called when the CDS
+			// Balancer is closed, and the call should cancel the watch for this
+			// cluster.
+			ch.close()
+			clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx)
+			if err != nil {
+				t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+			}
+			if clusterNameDeleted != test.clusterName {
+				t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService)
+			}
+		})
+	}
+}
+
+// The cluster handler receives a cluster name, handler starts a watch for that
+// cluster, xds client returns that it is a Leaf Node (EDS or LogicalDNS), not a
+// tree, so expectation that first update is written to buffer which will be
+// read by CDS LB. Then, send a new cluster update that is different, with the
+// expectation that it is also written to the update buffer to send back to CDS.
+func (s) TestSuccessCaseLeafNodeThenNewUpdate(t *testing.T) {
+	tests := []struct {
+		name             string
+		clusterName      string
+		clusterUpdate    xdsclient.ClusterUpdate
+		newClusterUpdate xdsclient.ClusterUpdate
+	}{
+		{name: "test-update-root-cluster-then-new-update-EDS-success",
+			clusterName: edsService,
+			clusterUpdate: xdsclient.ClusterUpdate{
+				ClusterType: xdsclient.ClusterTypeEDS,
+				ServiceName: edsService,
+			},
+			newClusterUpdate: xdsclient.ClusterUpdate{
+				ClusterType: xdsclient.ClusterTypeEDS,
+				ServiceName: edsService2,
+			},
+		},
+		{
+			name:        "test-update-root-cluster-then-new-update-Logical-DNS-success",
+			clusterName: logicalDNSService,
+			clusterUpdate: xdsclient.ClusterUpdate{
+				ClusterType: xdsclient.ClusterTypeLogicalDNS,
+				ServiceName: logicalDNSService,
+			},
+			newClusterUpdate: xdsclient.ClusterUpdate{
+				ClusterType: xdsclient.ClusterTypeLogicalDNS,
+				ServiceName: logicalDNSService2,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ch, fakeClient := setupTests(t)
+			ch.updateRootCluster(test.clusterName)
+			ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer ctxCancel()
+			_, err := fakeClient.WaitForWatchCluster(ctx)
+			if err != nil {
+				t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+			}
+			fakeClient.InvokeWatchClusterCallback(test.clusterUpdate, nil)
+			select {
+			case <-ch.updateChannel:
+			case <-ctx.Done():
+				t.Fatal("Timed out waiting for update from updateChannel.")
+			}
+
+			// Check that sending the same cluster update does not induce a
+			// update to be written to update buffer.
+			fakeClient.InvokeWatchClusterCallback(test.clusterUpdate, nil)
+			shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+			defer shouldNotHappenCtxCancel()
+			select {
+			case <-ch.updateChannel:
+				t.Fatal("Should not have written an update to update buffer, as cluster update did not change.")
+			case <-shouldNotHappenCtx.Done():
+			}
+
+			// Above represents same thing as the simple
+			// TestSuccessCaseLeafNode, extra behavior + validation (clusterNode
+			// which is a leaf receives a changed clusterUpdate, which should
+			// ping clusterHandler, which should then write to the update
+			// buffer).
+			fakeClient.InvokeWatchClusterCallback(test.newClusterUpdate, nil)
+			select {
+			case chu := <-ch.updateChannel:
+				if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{test.newClusterUpdate}); diff != "" {
+					t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
+				}
+			case <-ctx.Done():
+				t.Fatal("Timed out waiting for update from updateChannel.")
+			}
+		})
+	}
+}
+
+// TestUpdateRootClusterAggregateSuccess tests the case where an aggregate
+// cluster is a root pointing to two child clusters one of type EDS and the
+// other of type LogicalDNS. This test will then send cluster updates for both
+// the children, and at the end there should be a successful clusterUpdate
+// written to the update buffer to send back to CDS.
+func (s) TestUpdateRootClusterAggregateSuccess(t *testing.T) {
+	ch, fakeClient := setupTests(t)
+	ch.updateRootCluster(aggregateClusterService)
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	gotCluster, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != aggregateClusterService {
+		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, aggregateClusterService)
+	}
+
+	// The xdsClient telling the clusterNode that the cluster type is an
+	// aggregate cluster which will cause a lot of downstream behavior. For a
+	// cluster type that isn't an aggregate, the behavior is simple. The
+	// clusterNode will simply get a successful update, which will then ping the
+	// clusterHandler which will successfully build an update to send to the CDS
+	// policy. In the aggregate cluster case, the handleResp callback must also
+	// start watches for the aggregate cluster's children. The ping to the
+	// clusterHandler at the end of handleResp should be a no-op, as neither the
+	// EDS or LogicalDNS child clusters have received an update yet.
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType:             xdsclient.ClusterTypeAggregate,
+		ServiceName:             aggregateClusterService,
+		PrioritizedClusterNames: []string{edsService, logicalDNSService},
+	}, nil)
+
+	// xds client should be called to start a watch for one of the child
+	// clusters of the aggregate. The order of the children in the update
+	// written to the buffer to send to CDS matters, however there is no
+	// guarantee on the order it will start the watches of the children.
+	gotCluster, err = fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != edsService {
+		if gotCluster != logicalDNSService {
+			t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, edsService)
+		}
+	}
+
+	// xds client should then be called to start a watch for the second child
+	// cluster.
+	gotCluster, err = fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != edsService {
+		if gotCluster != logicalDNSService {
+			t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, logicalDNSService)
+		}
+	}
+
+	// The handleResp() call on the root aggregate cluster should not ping the
+	// cluster handler to try and construct an update, as the handleResp()
+	// callback knows that when a child is created, it cannot possibly build a
+	// successful update yet. Thus, there should be nothing in the update
+	// channel.
+
+	shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer shouldNotHappenCtxCancel()
+
+	select {
+	case <-ch.updateChannel:
+		t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update")
+	case <-shouldNotHappenCtx.Done():
+	}
+
+	// Send callback for the EDS child cluster.
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeEDS,
+		ServiceName: edsService,
+	}, nil)
+
+	// EDS child cluster will ping the Cluster Handler, to try an update, which
+	// still won't successfully build as the LogicalDNS child of the root
+	// aggregate cluster has not yet received and handled an update.
+	select {
+	case <-ch.updateChannel:
+		t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update")
+	case <-shouldNotHappenCtx.Done():
+	}
+
+	// Invoke callback for Logical DNS child cluster.
+
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeLogicalDNS,
+		ServiceName: logicalDNSService,
+	}, nil)
+
+	// Will Ping Cluster Handler, which will finally successfully build an
+	// update as all nodes in the tree of clusters have received an update.
+	// Since this cluster is an aggregate cluster comprised of two children, the
+	// returned update should be length 2, as the xds cluster resolver LB policy
+	// only cares about the full list of LogicalDNS and EDS clusters
+	// representing the base nodes of the tree of clusters. This list should be
+	// ordered as per the cluster update.
+	select {
+	case chu := <-ch.updateChannel:
+		if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{{
+			ClusterType: xdsclient.ClusterTypeEDS,
+			ServiceName: edsService,
+		}, {
+			ClusterType: xdsclient.ClusterTypeLogicalDNS,
+			ServiceName: logicalDNSService,
+		}}); diff != "" {
+			t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
+		}
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.")
+	}
+}
+
+// TestUpdateRootClusterAggregateThenChangeChild tests the scenario where you
+// have an aggregate cluster with an EDS child and a LogicalDNS child, then you
+// change one of the children and send an update for the changed child. This
+// should write a new update to the update buffer to send back to CDS.
+func (s) TestUpdateRootClusterAggregateThenChangeChild(t *testing.T) {
+	// This initial code is the same as the test for the aggregate success case,
+	// except without validations. This will get this test to the point where it
+	// can change one of the children.
+	ch, fakeClient := setupTests(t)
+	ch.updateRootCluster(aggregateClusterService)
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	_, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType:             xdsclient.ClusterTypeAggregate,
+		ServiceName:             aggregateClusterService,
+		PrioritizedClusterNames: []string{edsService, logicalDNSService},
+	}, nil)
+	fakeClient.WaitForWatchCluster(ctx)
+	fakeClient.WaitForWatchCluster(ctx)
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeEDS,
+		ServiceName: edsService,
+	}, nil)
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeLogicalDNS,
+		ServiceName: logicalDNSService,
+	}, nil)
+
+	select {
+	case <-ch.updateChannel:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.")
+	}
+
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType:             xdsclient.ClusterTypeAggregate,
+		ServiceName:             aggregateClusterService,
+		PrioritizedClusterNames: []string{edsService, logicalDNSService2},
+	}, nil)
+
+	// The cluster update let's the aggregate cluster know that it's children
+	// are now edsService and logicalDNSService2, which implies that the
+	// aggregateCluster lost it's old logicalDNSService child. Thus, the
+	// logicalDNSService child should be deleted.
+	clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+	}
+	if clusterNameDeleted != logicalDNSService {
+		t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService)
+	}
+
+	// The handleResp() callback should then start a watch for
+	// logicalDNSService2.
+	clusterNameCreated, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if clusterNameCreated != logicalDNSService2 {
+		t.Fatalf("xdsClient.WatchCDS called for cluster %v, want: %v", clusterNameCreated, logicalDNSService2)
+	}
+
+	// handleResp() should try and send an update here, but it will fail as
+	// logicalDNSService2 has not yet received an update.
+	shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer shouldNotHappenCtxCancel()
+	select {
+	case <-ch.updateChannel:
+		t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update")
+	case <-shouldNotHappenCtx.Done():
+	}
+
+	// Invoke a callback for the new logicalDNSService2 - this will fill out the
+	// tree with successful updates.
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeLogicalDNS,
+		ServiceName: logicalDNSService2,
+	}, nil)
+
+	// Behavior: This update make every node in the tree of cluster have
+	// received an update. Thus, at the end of this callback, when you ping the
+	// clusterHandler to try and construct an update, the update should now
+	// successfully be written to update buffer to send back to CDS. This new
+	// update should contain the new child of LogicalDNS2.
+
+	select {
+	case chu := <-ch.updateChannel:
+		if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{{
+			ClusterType: xdsclient.ClusterTypeEDS,
+			ServiceName: edsService,
+		}, {
+			ClusterType: xdsclient.ClusterTypeLogicalDNS,
+			ServiceName: logicalDNSService2,
+		}}); diff != "" {
+			t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
+		}
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.")
+	}
+}
+
+// TestUpdateRootClusterAggregateThenChangeRootToEDS tests the situation where
+// you have a fully updated aggregate cluster (where AggregateCluster success
+// test gets you) as the root cluster, then you update that root cluster to a
+// cluster of type EDS.
+func (s) TestUpdateRootClusterAggregateThenChangeRootToEDS(t *testing.T) {
+	// This initial code is the same as the test for the aggregate success case,
+	// except without validations. This will get this test to the point where it
+	// can update the root cluster to one of type EDS.
+	ch, fakeClient := setupTests(t)
+	ch.updateRootCluster(aggregateClusterService)
+
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	_, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType:             xdsclient.ClusterTypeAggregate,
+		ServiceName:             aggregateClusterService,
+		PrioritizedClusterNames: []string{edsService, logicalDNSService},
+	}, nil)
+	fakeClient.WaitForWatchCluster(ctx)
+	fakeClient.WaitForWatchCluster(ctx)
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeEDS,
+		ServiceName: edsService,
+	}, nil)
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeLogicalDNS,
+		ServiceName: logicalDNSService,
+	}, nil)
+
+	select {
+	case <-ch.updateChannel:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.")
+	}
+
+	// Changes the root aggregate cluster to a EDS cluster. This should delete
+	// the root aggregate cluster and all of it's children by successfully
+	// canceling the watches for them.
+	ch.updateRootCluster(edsService2)
+
+	// Reads from the cancel channel, should first be type Aggregate, then EDS
+	// then Logical DNS.
+	clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+	}
+	if clusterNameDeleted != aggregateClusterService {
+		t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService)
+	}
+
+	clusterNameDeleted, err = fakeClient.WaitForCancelClusterWatch(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+	}
+	if clusterNameDeleted != edsService {
+		t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService)
+	}
+
+	clusterNameDeleted, err = fakeClient.WaitForCancelClusterWatch(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+	}
+	if clusterNameDeleted != logicalDNSService {
+		t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService)
+	}
+
+	// After deletion, it should start a watch for the EDS Cluster. The behavior
+	// for this EDS Cluster receiving an update from xds client and then
+	// successfully writing an update to send back to CDS is already tested in
+	// the updateEDS success case.
+	gotCluster, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != edsService2 {
+		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, edsService2)
+	}
+}
+
+// TestHandleRespInvokedWithError tests that when handleResp is invoked with an
+// error, that the error is successfully written to the update buffer.
+func (s) TestHandleRespInvokedWithError(t *testing.T) {
+	ch, fakeClient := setupTests(t)
+	ch.updateRootCluster(edsService)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	_, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, errors.New("some error"))
+	select {
+	case chu := <-ch.updateChannel:
+		if chu.err.Error() != "some error" {
+			t.Fatalf("Did not receive the expected error, instead received: %v", chu.err.Error())
+		}
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for update from update channel.")
+	}
+}
+
+// TestSwitchClusterNodeBetweenLeafAndAggregated tests having an existing
+// cluster node switch between a leaf and an aggregated cluster. When the
+// cluster switches from a leaf to an aggregated cluster, it should add
+// children, and when it switches back to a leaf, it should delete those new
+// children and also successfully write a cluster update to the update buffer.
+func (s) TestSwitchClusterNodeBetweenLeafAndAggregated(t *testing.T) {
+	// Getting the test to the point where there's a root cluster which is a eds
+	// leaf.
+	ch, fakeClient := setupTests(t)
+	ch.updateRootCluster(edsService2)
+	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer ctxCancel()
+	_, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeEDS,
+		ServiceName: edsService2,
+	}, nil)
+	select {
+	case <-ch.updateChannel:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for update from update channel.")
+	}
+	// Switch the cluster to an aggregate cluster, this should cause two new
+	// child watches to be created.
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType:             xdsclient.ClusterTypeAggregate,
+		ServiceName:             edsService2,
+		PrioritizedClusterNames: []string{edsService, logicalDNSService},
+	}, nil)
+
+	// xds client should be called to start a watch for one of the child
+	// clusters of the aggregate. The order of the children in the update
+	// written to the buffer to send to CDS matters, however there is no
+	// guarantee on the order it will start the watches of the children.
+	gotCluster, err := fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != edsService {
+		if gotCluster != logicalDNSService {
+			t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, edsService)
+		}
+	}
+
+	// xds client should then be called to start a watch for the second child
+	// cluster.
+	gotCluster, err = fakeClient.WaitForWatchCluster(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
+	}
+	if gotCluster != edsService {
+		if gotCluster != logicalDNSService {
+			t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, logicalDNSService)
+		}
+	}
+
+	// After starting a watch for the second child cluster, there should be no
+	// more watches started on the xds client.
+	shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer shouldNotHappenCtxCancel()
+	gotCluster, err = fakeClient.WaitForWatchCluster(shouldNotHappenCtx)
+	if err == nil {
+		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, no more watches should be started.", gotCluster)
+	}
+
+	// The handleResp() call on the root aggregate cluster should not ping the
+	// cluster handler to try and construct an update, as the handleResp()
+	// callback knows that when a child is created, it cannot possibly build a
+	// successful update yet. Thus, there should be nothing in the update
+	// channel.
+
+	shouldNotHappenCtx, shouldNotHappenCtxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer shouldNotHappenCtxCancel()
+
+	select {
+	case <-ch.updateChannel:
+		t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update")
+	case <-shouldNotHappenCtx.Done():
+	}
+
+	// Switch the cluster back to an EDS Cluster. This should cause the two
+	// children to be deleted.
+	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
+		ClusterType: xdsclient.ClusterTypeEDS,
+		ServiceName: edsService2,
+	}, nil)
+
+	// Should delete the two children (no guarantee of ordering deleted, which
+	// is ok), then successfully write an update to the update buffer as the
+	// full cluster tree has received updates.
+	clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+	}
+	// No guarantee of ordering, so one of the children should be deleted first.
+	if clusterNameDeleted != edsService {
+		if clusterNameDeleted != logicalDNSService {
+			t.Fatalf("xdsClient.CancelCDS called for cluster %v, want either: %v or: %v", clusterNameDeleted, edsService, logicalDNSService)
+		}
+	}
+	// Then the other child should be deleted.
+	clusterNameDeleted, err = fakeClient.WaitForCancelClusterWatch(ctx)
+	if err != nil {
+		t.Fatalf("xdsClient.CancelCDS failed with error: %v", err)
+	}
+	if clusterNameDeleted != edsService {
+		if clusterNameDeleted != logicalDNSService {
+			t.Fatalf("xdsClient.CancelCDS called for cluster %v, want either: %v or: %v", clusterNameDeleted, edsService, logicalDNSService)
+		}
+	}
+
+	// After cancelling a watch for the second child cluster, there should be no
+	// more watches cancelled on the xds client.
+	shouldNotHappenCtx, shouldNotHappenCtxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer shouldNotHappenCtxCancel()
+	gotCluster, err = fakeClient.WaitForCancelClusterWatch(shouldNotHappenCtx)
+	if err == nil {
+		t.Fatalf("xdsClient.WatchCDS called for cluster: %v, no more watches should be cancelled.", gotCluster)
+	}
+
+	// Then an update should successfully be written to the update buffer.
+	select {
+	case chu := <-ch.updateChannel:
+		if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{{
+			ClusterType: xdsclient.ClusterTypeEDS,
+			ServiceName: edsService2,
+		}}); diff != "" {
+			t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
+		}
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for update from update channel.")
+	}
+}
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index 0978125b8aeb..eb4c659e505a 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -45,10 +45,10 @@ type Client struct {
 	loadStore    *load.Store
 	bootstrapCfg *bootstrap.Config
 
-	ldsCb func(xdsclient.ListenerUpdate, error)
-	rdsCb func(xdsclient.RouteConfigUpdate, error)
-	cdsCb func(xdsclient.ClusterUpdate, error)
-	edsCb func(xdsclient.EndpointsUpdate, error)
+	ldsCb  func(xdsclient.ListenerUpdate, error)
+	rdsCb  func(xdsclient.RouteConfigUpdate, error)
+	cdsCbs map[string]func(xdsclient.ClusterUpdate, error)
+	edsCb  func(xdsclient.EndpointsUpdate, error)
 }
 
 // WatchListener registers a LDS watch.
@@ -121,10 +121,13 @@ func (xdsC *Client) WaitForCancelRouteConfigWatch(ctx context.Context) error {
 
 // WatchCluster registers a CDS watch.
 func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsclient.ClusterUpdate, error)) func() {
-	xdsC.cdsCb = callback
+	// Due to the tree like structure of aggregate clusters, there can be multiple callbacks persisted for each cluster
+	// node. However, the client doesn't care about the parent child relationship between the nodes, only that it invokes
+	// the right callback for a particular cluster.
+	xdsC.cdsCbs[clusterName] = callback
 	xdsC.cdsWatchCh.Send(clusterName)
 	return func() {
-		xdsC.cdsCancelCh.Send(nil)
+		xdsC.cdsCancelCh.Send(clusterName)
 	}
 }
 
@@ -143,14 +146,28 @@ func (xdsC *Client) WaitForWatchCluster(ctx context.Context) (string, error) {
 // Not thread safe with WatchCluster. Only call this after
 // WaitForWatchCluster.
 func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.ClusterUpdate, err error) {
-	xdsC.cdsCb(update, err)
+	// Keeps functionality with previous usage of this, if single callback call that callback.
+	if len(xdsC.cdsCbs) == 1 {
+		var clusterName string
+		for cluster := range xdsC.cdsCbs {
+			clusterName = cluster
+		}
+		xdsC.cdsCbs[clusterName](update, err)
+	} else {
+		// Have what callback you call with the update determined by the service name in the ClusterUpdate. Left up to the
+		// caller to make sure the cluster update matches with a persisted callback.
+		xdsC.cdsCbs[update.ServiceName](update, err)
+	}
 }
 
 // WaitForCancelClusterWatch waits for a CDS watch to be cancelled  and returns
 // context.DeadlineExceeded otherwise.
-func (xdsC *Client) WaitForCancelClusterWatch(ctx context.Context) error {
-	_, err := xdsC.cdsCancelCh.Receive(ctx)
-	return err
+func (xdsC *Client) WaitForCancelClusterWatch(ctx context.Context) (string, error) {
+	clusterNameReceived, err := xdsC.cdsCancelCh.Receive(ctx)
+	if err != nil {
+		return "", err
+	}
+	return clusterNameReceived.(string), err
 }
 
 // WatchEndpoints registers an EDS watch for provided clusterName.
@@ -251,14 +268,15 @@ func NewClientWithName(name string) *Client {
 		name:         name,
 		ldsWatchCh:   testutils.NewChannel(),
 		rdsWatchCh:   testutils.NewChannel(),
-		cdsWatchCh:   testutils.NewChannel(),
+		cdsWatchCh:   testutils.NewChannelWithSize(10),
 		edsWatchCh:   testutils.NewChannel(),
 		ldsCancelCh:  testutils.NewChannel(),
 		rdsCancelCh:  testutils.NewChannel(),
-		cdsCancelCh:  testutils.NewChannel(),
+		cdsCancelCh:  testutils.NewChannelWithSize(10),
 		edsCancelCh:  testutils.NewChannel(),
 		loadReportCh: testutils.NewChannel(),
 		closeCh:      testutils.NewChannel(),
 		loadStore:    load.NewStore(),
+		cdsCbs:       make(map[string]func(xdsclient.ClusterUpdate, error)),
 	}
 }

From 9cb99a52111e9b67165d498ec2c322774b54a5f1 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 12 May 2021 15:48:16 -0700
Subject: [PATCH 472/481] xds: pretty print xDS updates and service config
 (#4405)

---
 internal/pretty/pretty.go                     | 82 +++++++++++++++++++
 .../balancer/cdsbalancer/cdsbalancer.go       |  5 +-
 .../balancer/clusterimpl/clusterimpl.go       |  2 +
 .../balancer/clustermanager/clustermanager.go |  3 +-
 xds/internal/balancer/edsbalancer/eds.go      |  5 +-
 xds/internal/balancer/lrs/balancer.go         |  2 +
 xds/internal/balancer/priority/balancer.go    |  2 +
 .../balancer/weightedtarget/weightedtarget.go |  2 +
 xds/internal/client/bootstrap/bootstrap.go    |  3 +-
 xds/internal/client/callback.go               | 10 ++-
 xds/internal/client/v2/client.go              |  5 +-
 xds/internal/client/v2/loadreport.go          |  7 +-
 xds/internal/client/v3/client.go              |  5 +-
 xds/internal/client/v3/loadreport.go          |  7 +-
 xds/internal/client/watchers.go               | 10 ++-
 xds/internal/client/xds.go                    |  9 +-
 xds/internal/resolver/serviceconfig.go        |  6 +-
 xds/internal/resolver/watch_service.go        |  5 +-
 xds/internal/resolver/xds_resolver.go         |  5 +-
 19 files changed, 140 insertions(+), 35 deletions(-)
 create mode 100644 internal/pretty/pretty.go

diff --git a/internal/pretty/pretty.go b/internal/pretty/pretty.go
new file mode 100644
index 000000000000..0177af4b5114
--- /dev/null
+++ b/internal/pretty/pretty.go
@@ -0,0 +1,82 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package pretty defines helper functions to pretty-print structs for logging.
+package pretty
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/golang/protobuf/jsonpb"
+	protov1 "github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/encoding/protojson"
+	protov2 "google.golang.org/protobuf/proto"
+)
+
+const jsonIndent = "  "
+
+// ToJSON marshals the input into a json string.
+//
+// If marshal fails, it falls back to fmt.Sprintf("%+v").
+func ToJSON(e interface{}) string {
+	switch ee := e.(type) {
+	case protov1.Message:
+		mm := jsonpb.Marshaler{Indent: jsonIndent}
+		ret, err := mm.MarshalToString(ee)
+		if err != nil {
+			// This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2
+			// messages are not imported, and this will fail because the message
+			// is not found.
+			return fmt.Sprintf("%+v", ee)
+		}
+		return ret
+	case protov2.Message:
+		mm := protojson.MarshalOptions{
+			Multiline: true,
+			Indent:    jsonIndent,
+		}
+		ret, err := mm.Marshal(ee)
+		if err != nil {
+			// This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2
+			// messages are not imported, and this will fail because the message
+			// is not found.
+			return fmt.Sprintf("%+v", ee)
+		}
+		return string(ret)
+	default:
+		ret, err := json.MarshalIndent(ee, "", jsonIndent)
+		if err != nil {
+			return fmt.Sprintf("%+v", ee)
+		}
+		return string(ret)
+	}
+}
+
+// FormatJSON formats the input json bytes with indentation.
+//
+// If Indent fails, it returns the unchanged input as string.
+func FormatJSON(b []byte) string {
+	var out bytes.Buffer
+	err := json.Indent(&out, b, "", jsonIndent)
+	if err != nil {
+		return string(b)
+	}
+	return out.String()
+}
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 9b987c00c2ca..1134cf72570f 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -31,6 +31,7 @@ import (
 	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
@@ -318,7 +319,7 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		return
 	}
 
-	b.logger.Infof("Watch update from xds-client %p, content: %+v", b.xdsClient, update.cds)
+	b.logger.Infof("Watch update from xds-client %p, content: %+v", b.xdsClient, pretty.ToJSON(update.cds))
 
 	// Process the security config from the received update before building the
 	// child policy or forwarding the update to it. We do this because the child
@@ -466,7 +467,7 @@ func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) erro
 		return errBalancerClosed
 	}
 
-	b.logger.Infof("Received update from resolver, balancer config: %+v", state.BalancerConfig)
+	b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(state.BalancerConfig))
 	// The errors checked here should ideally never happen because the
 	// ServiceConfig in this case is prepared by the xdsResolver and is not
 	// something that is received on the wire.
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index 56664e391ac6..4bd29901d760 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/internal/buffer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/loadstore"
@@ -189,6 +190,7 @@ func (cib *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState
 		return nil
 	}
 
+	cib.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
 	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
diff --git a/xds/internal/balancer/clustermanager/clustermanager.go b/xds/internal/balancer/clustermanager/clustermanager.go
index b4ae3710cd27..c00a9a16f458 100644
--- a/xds/internal/balancer/clustermanager/clustermanager.go
+++ b/xds/internal/balancer/clustermanager/clustermanager.go
@@ -27,6 +27,7 @@ import (
 	"google.golang.org/grpc/grpclog"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
@@ -115,7 +116,7 @@ func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error {
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
 	}
-	b.logger.Infof("update with config %+v, resolver state %+v", s.BalancerConfig, s.ResolverState)
+	b.logger.Infof("update with config %+v, resolver state %+v", pretty.ToJSON(s.BalancerConfig), s.ResolverState)
 
 	b.updateChildren(s, newConfig)
 	return nil
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index d1a226e98987..5191dbd30db8 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/pretty"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -211,7 +212,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 	case *subConnStateUpdate:
 		x.edsImpl.handleSubConnStateChange(u.sc, u.state.ConnectivityState)
 	case *balancer.ClientConnState:
-		x.logger.Infof("Receive update from resolver, balancer config: %+v", u.BalancerConfig)
+		x.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(u.BalancerConfig))
 		cfg, _ := u.BalancerConfig.(*EDSConfig)
 		if cfg == nil {
 			// service config parsing failed. should never happen.
@@ -278,7 +279,7 @@ func (x *edsBalancer) startEndpointsWatch() {
 		x.cancelEndpointsWatch()
 	}
 	cancelEDSWatch := x.xdsClient.WatchEndpoints(x.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
-		x.logger.Infof("Watch update from xds-client %p, content: %+v", x.xdsClient, update)
+		x.logger.Infof("Watch update from xds-client %p, content: %+v", x.xdsClient, pretty.ToJSON(update))
 		x.handleEDSUpdate(update, err)
 	})
 	x.logger.Infof("Watch started on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go
index e062fa234363..0642c54ed111 100644
--- a/xds/internal/balancer/lrs/balancer.go
+++ b/xds/internal/balancer/lrs/balancer.go
@@ -25,6 +25,7 @@ import (
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
@@ -80,6 +81,7 @@ type lrsBalancer struct {
 }
 
 func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
 	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
diff --git a/xds/internal/balancer/priority/balancer.go b/xds/internal/balancer/priority/balancer.go
index e12a5068737f..d760a58fa4ec 100644
--- a/xds/internal/balancer/priority/balancer.go
+++ b/xds/internal/balancer/priority/balancer.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
@@ -108,6 +109,7 @@ type priorityBalancer struct {
 }
 
 func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
 	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go
index a210816332b0..89f5ec660a04 100644
--- a/xds/internal/balancer/weightedtarget/weightedtarget.go
+++ b/xds/internal/balancer/weightedtarget/weightedtarget.go
@@ -26,6 +26,7 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/internal/wrr"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/serviceconfig"
@@ -83,6 +84,7 @@ type weightedTargetBalancer struct {
 // creates/deletes sub-balancers and sends them update. Addresses are split into
 // groups based on hierarchy path.
 func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	w.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
 	newConfig, ok := s.BalancerConfig.(*LBConfig)
 	if !ok {
 		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go
index f32c698b4f55..a3fb5c0816b4 100644
--- a/xds/internal/client/bootstrap/bootstrap.go
+++ b/xds/internal/client/bootstrap/bootstrap.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal/version"
 )
@@ -270,7 +271,7 @@ func NewConfig() (*Config, error) {
 	if err := config.updateNodeProto(); err != nil {
 		return nil, err
 	}
-	logger.Infof("Bootstrap config for creating xds-client: %+v", config)
+	logger.Infof("Bootstrap config for creating xds-client: %v", pretty.ToJSON(config))
 	return config, nil
 }
 
diff --git a/xds/internal/client/callback.go b/xds/internal/client/callback.go
index da8e2f62d6c0..ac6f0151a5f9 100644
--- a/xds/internal/client/callback.go
+++ b/xds/internal/client/callback.go
@@ -18,6 +18,8 @@
 
 package client
 
+import "google.golang.org/grpc/internal/pretty"
+
 type watcherInfoWithUpdate struct {
 	wi     *watchInfo
 	update interface{}
@@ -104,7 +106,7 @@ func (c *clientImpl) NewListeners(updates map[string]ListenerUpdate, metadata Up
 				wi.newUpdate(update)
 			}
 			// Sync cache.
-			c.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, update)
+			c.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(update))
 			c.ldsCache[name] = update
 			c.ldsMD[name] = metadata
 		}
@@ -163,7 +165,7 @@ func (c *clientImpl) NewRouteConfigs(updates map[string]RouteConfigUpdate, metad
 				wi.newUpdate(update)
 			}
 			// Sync cache.
-			c.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, update)
+			c.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(update))
 			c.rdsCache[name] = update
 			c.rdsMD[name] = metadata
 		}
@@ -205,7 +207,7 @@ func (c *clientImpl) NewClusters(updates map[string]ClusterUpdate, metadata Upda
 				wi.newUpdate(update)
 			}
 			// Sync cache.
-			c.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, update)
+			c.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(update))
 			c.cdsCache[name] = update
 			c.cdsMD[name] = metadata
 		}
@@ -264,7 +266,7 @@ func (c *clientImpl) NewEndpoints(updates map[string]EndpointsUpdate, metadata U
 				wi.newUpdate(update)
 			}
 			// Sync cache.
-			c.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, update)
+			c.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(update))
 			c.edsCache[name] = update
 			c.edsMD[name] = metadata
 		}
diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go
index b6bc4908120d..b91482d13cf9 100644
--- a/xds/internal/client/v2/client.go
+++ b/xds/internal/client/v2/client.go
@@ -27,6 +27,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/version"
 
@@ -125,7 +126,7 @@ func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rTyp
 	if err := stream.Send(req); err != nil {
 		return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err)
 	}
-	v2c.logger.Debugf("ADS request sent: %v", req)
+	v2c.logger.Debugf("ADS request sent: %v", pretty.ToJSON(req))
 	return nil
 }
 
@@ -143,7 +144,7 @@ func (v2c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
 		return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err)
 	}
 	v2c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
-	v2c.logger.Debugf("ADS response received: %v", resp)
+	v2c.logger.Debugf("ADS response received: %v", pretty.ToJSON(resp))
 	return resp, nil
 }
 
diff --git a/xds/internal/client/v2/loadreport.go b/xds/internal/client/v2/loadreport.go
index 69405fcd9ad3..17ea8c8d4c43 100644
--- a/xds/internal/client/v2/loadreport.go
+++ b/xds/internal/client/v2/loadreport.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/xds/internal/client/load"
 
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
@@ -57,7 +58,7 @@ func (v2c *client) SendFirstLoadStatsRequest(s grpc.ClientStream) error {
 	node.ClientFeatures = append(node.ClientFeatures, clientFeatureLRSSendAllClusters)
 
 	req := &lrspb.LoadStatsRequest{Node: node}
-	v2c.logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
+	v2c.logger.Infof("lrs: sending init LoadStatsRequest: %v", pretty.ToJSON(req))
 	return stream.Send(req)
 }
 
@@ -71,7 +72,7 @@ func (v2c *client) HandleLoadStatsResponse(s grpc.ClientStream) ([]string, time.
 	if err != nil {
 		return nil, 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
 	}
-	v2c.logger.Infof("lrs: received first LoadStatsResponse: %+v", resp)
+	v2c.logger.Infof("lrs: received first LoadStatsResponse: %+v", pretty.ToJSON(resp))
 
 	interval, err := ptypes.Duration(resp.GetLoadReportingInterval())
 	if err != nil {
@@ -149,6 +150,6 @@ func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data)
 	}
 
 	req := &lrspb.LoadStatsRequest{ClusterStats: clusterStats}
-	v2c.logger.Infof("lrs: sending LRS loads: %+v", req)
+	v2c.logger.Infof("lrs: sending LRS loads: %+v", pretty.ToJSON(req))
 	return stream.Send(req)
 }
diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go
index 55cae56d8cc6..200da2ac7d73 100644
--- a/xds/internal/client/v3/client.go
+++ b/xds/internal/client/v3/client.go
@@ -28,6 +28,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/version"
 
@@ -125,7 +126,7 @@ func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rTyp
 	if err := stream.Send(req); err != nil {
 		return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err)
 	}
-	v3c.logger.Debugf("ADS request sent: %v", req)
+	v3c.logger.Debugf("ADS request sent: %v", pretty.ToJSON(req))
 	return nil
 }
 
@@ -143,7 +144,7 @@ func (v3c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) {
 		return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err)
 	}
 	v3c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl())
-	v3c.logger.Debugf("ADS response received: %+v", resp)
+	v3c.logger.Debugf("ADS response received: %+v", pretty.ToJSON(resp))
 	return resp, nil
 }
 
diff --git a/xds/internal/client/v3/loadreport.go b/xds/internal/client/v3/loadreport.go
index 74e18632aa07..1de0ccf57503 100644
--- a/xds/internal/client/v3/loadreport.go
+++ b/xds/internal/client/v3/loadreport.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/xds/internal/client/load"
 
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -57,7 +58,7 @@ func (v3c *client) SendFirstLoadStatsRequest(s grpc.ClientStream) error {
 	node.ClientFeatures = append(node.ClientFeatures, clientFeatureLRSSendAllClusters)
 
 	req := &lrspb.LoadStatsRequest{Node: node}
-	v3c.logger.Infof("lrs: sending init LoadStatsRequest: %v", req)
+	v3c.logger.Infof("lrs: sending init LoadStatsRequest: %v", pretty.ToJSON(req))
 	return stream.Send(req)
 }
 
@@ -71,7 +72,7 @@ func (v3c *client) HandleLoadStatsResponse(s grpc.ClientStream) ([]string, time.
 	if err != nil {
 		return nil, 0, fmt.Errorf("lrs: failed to receive first response: %v", err)
 	}
-	v3c.logger.Infof("lrs: received first LoadStatsResponse: %+v", resp)
+	v3c.logger.Infof("lrs: received first LoadStatsResponse: %+v", pretty.ToJSON(resp))
 
 	interval, err := ptypes.Duration(resp.GetLoadReportingInterval())
 	if err != nil {
@@ -148,6 +149,6 @@ func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data)
 	}
 
 	req := &lrspb.LoadStatsRequest{ClusterStats: clusterStats}
-	v3c.logger.Infof("lrs: sending LRS loads: %+v", req)
+	v3c.logger.Infof("lrs: sending LRS loads: %+v", pretty.ToJSON(req))
 	return stream.Send(req)
 }
diff --git a/xds/internal/client/watchers.go b/xds/internal/client/watchers.go
index 9fafe5a60f83..446f5cca9973 100644
--- a/xds/internal/client/watchers.go
+++ b/xds/internal/client/watchers.go
@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"google.golang.org/grpc/internal/pretty"
 )
 
 type watchInfoState int
@@ -161,22 +163,22 @@ func (c *clientImpl) watch(wi *watchInfo) (cancel func()) {
 	switch wi.rType {
 	case ListenerResource:
 		if v, ok := c.ldsCache[resourceName]; ok {
-			c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, v)
+			c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v))
 			wi.newUpdate(v)
 		}
 	case RouteConfigResource:
 		if v, ok := c.rdsCache[resourceName]; ok {
-			c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, v)
+			c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v))
 			wi.newUpdate(v)
 		}
 	case ClusterResource:
 		if v, ok := c.cdsCache[resourceName]; ok {
-			c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, v)
+			c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v))
 			wi.newUpdate(v)
 		}
 	case EndpointsResource:
 		if v, ok := c.edsCache[resourceName]; ok {
-			c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, v)
+			c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v))
 			wi.newUpdate(v)
 		}
 	}
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 55bcc2e936ca..cf705587e24a 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -39,6 +39,7 @@ import (
 	v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"google.golang.org/grpc/internal/grpclog"
@@ -72,7 +73,7 @@ func unmarshalListenerResource(r *anypb.Any, logger *grpclog.PrefixLogger) (stri
 	if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
 		return "", ListenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
 	}
-	logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis)
+	logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, pretty.ToJSON(lis))
 
 	lu, err := processListener(lis, logger, v2)
 	if err != nil {
@@ -360,7 +361,7 @@ func unmarshalRouteConfigResource(r *anypb.Any, logger *grpclog.PrefixLogger) (s
 	if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
 		return "", RouteConfigUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
 	}
-	logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, rc)
+	logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, pretty.ToJSON(rc))
 
 	// TODO: Pass version.TransportAPI instead of relying upon the type URL
 	v2 := r.GetTypeUrl() == version.V2RouteConfigURL
@@ -572,7 +573,7 @@ func unmarshalClusterResource(r *anypb.Any, logger *grpclog.PrefixLogger) (strin
 	if err := proto.Unmarshal(r.GetValue(), cluster); err != nil {
 		return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
 	}
-	logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster)
+	logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, pretty.ToJSON(cluster))
 	cu, err := validateClusterAndConstructClusterUpdate(cluster)
 	if err != nil {
 		return cluster.GetName(), ClusterUpdate{}, err
@@ -765,7 +766,7 @@ func unmarshalEndpointsResource(r *anypb.Any, logger *grpclog.PrefixLogger) (str
 	if err := proto.Unmarshal(r.GetValue(), cla); err != nil {
 		return "", EndpointsUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
 	}
-	logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla)
+	logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, pretty.ToJSON(cla))
 
 	u, err := parseEDSRespProto(cla)
 	if err != nil {
diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go
index ef7c37128c13..49ee8970d7d8 100644
--- a/xds/internal/resolver/serviceconfig.go
+++ b/xds/internal/resolver/serviceconfig.go
@@ -76,7 +76,7 @@ func (r *xdsResolver) pruneActiveClusters() {
 // serviceConfigJSON produces a service config in JSON format representing all
 // the clusters referenced in activeClusters.  This includes clusters with zero
 // references, so they must be pruned first.
-func serviceConfigJSON(activeClusters map[string]*clusterInfo) (string, error) {
+func serviceConfigJSON(activeClusters map[string]*clusterInfo) ([]byte, error) {
 	// Generate children (all entries in activeClusters).
 	children := make(map[string]xdsChildConfig)
 	for cluster := range activeClusters {
@@ -93,9 +93,9 @@ func serviceConfigJSON(activeClusters map[string]*clusterInfo) (string, error) {
 
 	bs, err := json.Marshal(sc)
 	if err != nil {
-		return "", fmt.Errorf("failed to marshal json: %v", err)
+		return nil, fmt.Errorf("failed to marshal json: %v", err)
 	}
-	return string(bs), nil
+	return bs, nil
 }
 
 type virtualHost struct {
diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go
index 7667592ccd6f..f29fb32832e0 100644
--- a/xds/internal/resolver/watch_service.go
+++ b/xds/internal/resolver/watch_service.go
@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
 
@@ -82,7 +83,7 @@ type serviceUpdateWatcher struct {
 }
 
 func (w *serviceUpdateWatcher) handleLDSResp(update xdsclient.ListenerUpdate, err error) {
-	w.logger.Infof("received LDS update: %+v, err: %v", update, err)
+	w.logger.Infof("received LDS update: %+v, err: %v", pretty.ToJSON(update), err)
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	if w.closed {
@@ -163,7 +164,7 @@ func (w *serviceUpdateWatcher) updateVirtualHostsFromRDS(update xdsclient.RouteC
 }
 
 func (w *serviceUpdateWatcher) handleRDSResp(update xdsclient.RouteConfigUpdate, err error) {
-	w.logger.Infof("received RDS update: %+v, err: %v", update, err)
+	w.logger.Infof("received RDS update: %+v, err: %v", pretty.ToJSON(update), err)
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	if w.closed {
diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go
index d8c09db69b5a..e995fa9fa8fe 100644
--- a/xds/internal/resolver/xds_resolver.go
+++ b/xds/internal/resolver/xds_resolver.go
@@ -26,6 +26,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/internal/grpclog"
 	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
 
@@ -171,11 +172,11 @@ func (r *xdsResolver) sendNewServiceConfig(cs *configSelector) bool {
 		r.cc.ReportError(err)
 		return false
 	}
-	r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc)
+	r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, pretty.FormatJSON(sc))
 
 	// Send the update to the ClientConn.
 	state := iresolver.SetConfigSelector(resolver.State{
-		ServiceConfig: r.cc.ParseServiceConfig(sc),
+		ServiceConfig: r.cc.ParseServiceConfig(string(sc)),
 	}, cs)
 	r.cc.UpdateState(state)
 	return true

From 397adad6a0d1d12ddd9b7f0101e902da274c15c8 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 12 May 2021 15:52:15 -0700
Subject: [PATCH 473/481] update go.mod and go.sum to point to latest
 go-control-plane (#4425)

---
 examples/go.mod                      |  4 +--
 examples/go.sum                      | 39 ++++++++++++++----------
 go.mod                               | 10 +++----
 go.sum                               | 44 ++++++++++++++++++++++------
 security/advancedtls/examples/go.mod |  2 +-
 security/advancedtls/examples/go.sum | 34 +++++++++++++--------
 security/advancedtls/go.mod          |  2 +-
 security/advancedtls/go.sum          | 34 +++++++++++++--------
 8 files changed, 110 insertions(+), 59 deletions(-)

diff --git a/examples/go.mod b/examples/go.mod
index 18c67afed969..143a8fc03f57 100644
--- a/examples/go.mod
+++ b/examples/go.mod
@@ -3,10 +3,10 @@ module google.golang.org/grpc/examples
 go 1.11
 
 require (
-	github.com/golang/protobuf v1.4.2
+	github.com/golang/protobuf v1.4.3
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98
-	google.golang.org/grpc v1.31.0
+	google.golang.org/grpc v1.36.0
 	google.golang.org/protobuf v1.25.0
 )
 
diff --git a/examples/go.sum b/examples/go.sum
index 1984770a80b3..a5b967336903 100644
--- a/examples/go.sum
+++ b/examples/go.sum
@@ -1,26 +1,31 @@
 cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed h1:OZmjad4L3H8ncOIR8rnb5MREYqG8ixi5+WbeUsquF0c=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0 h1:dulLQAYQFYtG5MTplgNGHWuV2D+OBD+Z8lmDBmbLg+s=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -28,39 +33,43 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -77,7 +86,7 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/go.mod b/go.mod
index b177cfa66df5..6eed9370b462 100644
--- a/go.mod
+++ b/go.mod
@@ -4,14 +4,14 @@ go 1.11
 
 require (
 	github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403
-	github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d
+	github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/protobuf v1.4.2
+	github.com/golang/protobuf v1.4.3
 	github.com/google/go-cmp v0.5.0
 	github.com/google/uuid v1.1.2
-	golang.org/x/net v0.0.0-20190311183353-d8887717615a
-	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
-	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
+	golang.org/x/net v0.0.0-20200822124328-c89045814202
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
 	google.golang.org/protobuf v1.25.0
 )
diff --git a/go.sum b/go.sum
index 24d2976abbaf..51fd1436e38f 100644
--- a/go.sum
+++ b/go.sum
@@ -1,32 +1,42 @@
-cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed h1:OZmjad4L3H8ncOIR8rnb5MREYqG8ixi5+WbeUsquF0c=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0 h1:dulLQAYQFYtG5MTplgNGHWuV2D+OBD+Z8lmDBmbLg+s=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -35,49 +45,64 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -90,7 +115,8 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/security/advancedtls/examples/go.mod b/security/advancedtls/examples/go.mod
index 936aa476893b..f141d4c05c81 100644
--- a/security/advancedtls/examples/go.mod
+++ b/security/advancedtls/examples/go.mod
@@ -3,7 +3,7 @@ module google.golang.org/grpc/security/advancedtls/examples
 go 1.15
 
 require (
-	google.golang.org/grpc v1.33.1
+	google.golang.org/grpc v1.36.0
 	google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b
 	google.golang.org/grpc/security/advancedtls v0.0.0-20201112215255-90f1b3ee835b
 )
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
index 519267dbc278..c24d070fa7f1 100644
--- a/security/advancedtls/examples/go.sum
+++ b/security/advancedtls/examples/go.sum
@@ -1,20 +1,24 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -22,35 +26,39 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -67,5 +75,5 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod
index be35029503da..90dee4a46fd0 100644
--- a/security/advancedtls/go.mod
+++ b/security/advancedtls/go.mod
@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/google/go-cmp v0.5.1 // indirect
-	google.golang.org/grpc v1.31.0
+	google.golang.org/grpc v1.36.0
 	google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b
 )
 
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 519267dbc278..c24d070fa7f1 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,20 +1,24 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -22,35 +26,39 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -67,5 +75,5 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From a712a738897ceebf3b6690d722006b61013572e0 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 12 May 2021 16:25:07 -0700
Subject: [PATCH 474/481] xds/cds: add separate fields for cluster name and eds
 service name (#4414)

---
 .../balancer/cdsbalancer/cdsbalancer.go       |  3 +-
 .../cdsbalancer/cdsbalancer_security_test.go  | 14 +--
 .../balancer/cdsbalancer/cdsbalancer_test.go  | 16 ++--
 .../balancer/cdsbalancer/cluster_handler.go   |  4 +-
 .../cdsbalancer/cluster_handler_test.go       | 50 +++++------
 xds/internal/balancer/edsbalancer/config.go   |  8 +-
 xds/internal/balancer/edsbalancer/eds.go      | 63 +++++++++----
 .../balancer/edsbalancer/eds_impl_test.go     | 11 +--
 xds/internal/balancer/edsbalancer/eds_test.go | 53 +++++++++--
 .../edsbalancer/load_store_wrapper.go         | 88 -------------------
 xds/internal/client/cds_test.go               | 74 +++++++++++-----
 xds/internal/client/client.go                 | 15 ++--
 xds/internal/client/client_test.go            |  4 +-
 xds/internal/client/v2/cds_test.go            |  4 +-
 xds/internal/client/watchers_cluster_test.go  | 16 ++--
 xds/internal/client/xds.go                    | 75 +++++++---------
 xds/internal/testutils/fakeclient/client.go   |  2 +-
 17 files changed, 250 insertions(+), 250 deletions(-)
 delete mode 100644 xds/internal/balancer/edsbalancer/load_store_wrapper.go

diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 1134cf72570f..0e8b83481ac9 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -348,7 +348,8 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
 		b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName)
 	}
 	lbCfg := &edsbalancer.EDSConfig{
-		EDSServiceName:        update.cds.ServiceName,
+		ClusterName:           update.cds.ClusterName,
+		EDSServiceName:        update.cds.EDSServiceName,
 		MaxConcurrentRequests: update.cds.MaxRequests,
 	}
 	if update.cds.EnableLRS {
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index d1074f2a1c45..30c0d9105ed3 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -60,7 +60,7 @@ var (
 	fpb1, fpb2                   *fakeProviderBuilder
 	bootstrapConfig              *bootstrap.Config
 	cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default1",
 			IdentityInstanceName:   "default2",
@@ -68,7 +68,7 @@ var (
 		},
 	}
 	cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName: "not-default",
 		},
@@ -256,7 +256,7 @@ func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -312,7 +312,7 @@ func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup. No security config is
 	// passed to the CDS balancer as part of this update.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -572,7 +572,7 @@ func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
 	// an update which contains bad security config. So, we expect the CDS
 	// balancer to forward this error to the EDS balancer and eventually the
 	// channel needs to be put in a bad state.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
 	}
@@ -678,7 +678,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
 	cdsUpdate := xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default1",
 			SubjectAltNameMatchers: testSANMatchers,
@@ -703,7 +703,7 @@ func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
 
 	// Push another update with a new security configuration.
 	cdsUpdate = xdsclient.ClusterUpdate{
-		ServiceName: serviceName,
+		ClusterName: serviceName,
 		SecurityCfg: &xdsclient.SecurityConfig{
 			RootInstanceName:       "default2",
 			SubjectAltNameMatchers: testSANMatchers,
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
index e93df5a10aab..fba3ee1531e9 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
@@ -197,7 +197,7 @@ func cdsCCS(cluster string) balancer.ClientConnState {
 // cdsBalancer to the edsBalancer.
 func edsCCS(service string, countMax *uint32, enableLRS bool) balancer.ClientConnState {
 	lbCfg := &edsbalancer.EDSConfig{
-		EDSServiceName:        service,
+		ClusterName:           service,
 		MaxConcurrentRequests: countMax,
 	}
 	if enableLRS {
@@ -354,12 +354,12 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
 	}{
 		{
 			name:      "happy-case-with-lrs",
-			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
+			cdsUpdate: xdsclient.ClusterUpdate{ClusterName: serviceName, EnableLRS: true},
 			wantCCS:   edsCCS(serviceName, nil, true),
 		},
 		{
 			name:      "happy-case-without-lrs",
-			cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName},
+			cdsUpdate: xdsclient.ClusterUpdate{ClusterName: serviceName},
 			wantCCS:   edsCCS(serviceName, nil, false),
 		},
 	}
@@ -427,7 +427,7 @@ func (s) TestHandleClusterUpdateError(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -512,7 +512,7 @@ func (s) TestResolverError(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
 		t.Fatal(err)
@@ -561,7 +561,7 @@ func (s) TestUpdateSubConnState(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -596,7 +596,7 @@ func (s) TestCircuitBreaking(t *testing.T) {
 	// will trigger the watch handler on the CDS balancer, which will update
 	// the service's counter with the new max requests.
 	var maxRequests uint32 = 1
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName, MaxRequests: &maxRequests}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName, MaxRequests: &maxRequests}
 	wantCCS := edsCCS(serviceName, &maxRequests, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
@@ -629,7 +629,7 @@ func (s) TestClose(t *testing.T) {
 	// create a new EDS balancer. The fake EDS balancer created above will be
 	// returned to the CDS balancer, because we have overridden the
 	// newEDSBalancer function as part of test setup.
-	cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName}
+	cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
 	wantCCS := edsCCS(serviceName, nil, false)
 	ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer ctxCancel()
diff --git a/xds/internal/balancer/cdsbalancer/cluster_handler.go b/xds/internal/balancer/cdsbalancer/cluster_handler.go
index 2dafb212f4c9..4c13b55594da 100644
--- a/xds/internal/balancer/cdsbalancer/cluster_handler.go
+++ b/xds/internal/balancer/cdsbalancer/cluster_handler.go
@@ -180,7 +180,7 @@ func (c *clusterNode) handleResp(clusterUpdate xdsclient.ClusterUpdate, err erro
 	// clusterUpdate fields (forgetting the children). This will be used to help
 	// determine whether to pingClusterHandler at the end of this callback or
 	// not.
-	deltaInClusterUpdateFields := clusterUpdate.ServiceName != c.clusterUpdate.ServiceName || clusterUpdate.ClusterType != c.clusterUpdate.ClusterType
+	deltaInClusterUpdateFields := clusterUpdate.ClusterName != c.clusterUpdate.ClusterName || clusterUpdate.ClusterType != c.clusterUpdate.ClusterType
 	c.receivedUpdate = true
 	c.clusterUpdate = clusterUpdate
 
@@ -223,7 +223,7 @@ func (c *clusterNode) handleResp(clusterUpdate xdsclient.ClusterUpdate, err erro
 	// update received, will be used to construct the new child list.
 	mapCurrentChildren := make(map[string]*clusterNode)
 	for _, child := range c.children {
-		mapCurrentChildren[child.clusterUpdate.ServiceName] = child
+		mapCurrentChildren[child.clusterUpdate.ClusterName] = child
 	}
 
 	// Add and construct any new child nodes.
diff --git a/xds/internal/balancer/cdsbalancer/cluster_handler_test.go b/xds/internal/balancer/cdsbalancer/cluster_handler_test.go
index 96fca8a2696a..049cdf55ee6a 100644
--- a/xds/internal/balancer/cdsbalancer/cluster_handler_test.go
+++ b/xds/internal/balancer/cdsbalancer/cluster_handler_test.go
@@ -64,14 +64,14 @@ func (s) TestSuccessCaseLeafNode(t *testing.T) {
 			clusterName: edsService,
 			clusterUpdate: xdsclient.ClusterUpdate{
 				ClusterType: xdsclient.ClusterTypeEDS,
-				ServiceName: edsService,
+				ClusterName: edsService,
 			}},
 		{
 			name:        "test-update-root-cluster-Logical-DNS-success",
 			clusterName: logicalDNSService,
 			clusterUpdate: xdsclient.ClusterUpdate{
 				ClusterType: xdsclient.ClusterTypeLogicalDNS,
-				ServiceName: logicalDNSService,
+				ClusterName: logicalDNSService,
 			}},
 	}
 
@@ -138,11 +138,11 @@ func (s) TestSuccessCaseLeafNodeThenNewUpdate(t *testing.T) {
 			clusterName: edsService,
 			clusterUpdate: xdsclient.ClusterUpdate{
 				ClusterType: xdsclient.ClusterTypeEDS,
-				ServiceName: edsService,
+				ClusterName: edsService,
 			},
 			newClusterUpdate: xdsclient.ClusterUpdate{
 				ClusterType: xdsclient.ClusterTypeEDS,
-				ServiceName: edsService2,
+				ClusterName: edsService2,
 			},
 		},
 		{
@@ -150,11 +150,11 @@ func (s) TestSuccessCaseLeafNodeThenNewUpdate(t *testing.T) {
 			clusterName: logicalDNSService,
 			clusterUpdate: xdsclient.ClusterUpdate{
 				ClusterType: xdsclient.ClusterTypeLogicalDNS,
-				ServiceName: logicalDNSService,
+				ClusterName: logicalDNSService,
 			},
 			newClusterUpdate: xdsclient.ClusterUpdate{
 				ClusterType: xdsclient.ClusterTypeLogicalDNS,
-				ServiceName: logicalDNSService2,
+				ClusterName: logicalDNSService2,
 			},
 		},
 	}
@@ -235,7 +235,7 @@ func (s) TestUpdateRootClusterAggregateSuccess(t *testing.T) {
 	// EDS or LogicalDNS child clusters have received an update yet.
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType:             xdsclient.ClusterTypeAggregate,
-		ServiceName:             aggregateClusterService,
+		ClusterName:             aggregateClusterService,
 		PrioritizedClusterNames: []string{edsService, logicalDNSService},
 	}, nil)
 
@@ -283,7 +283,7 @@ func (s) TestUpdateRootClusterAggregateSuccess(t *testing.T) {
 	// Send callback for the EDS child cluster.
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeEDS,
-		ServiceName: edsService,
+		ClusterName: edsService,
 	}, nil)
 
 	// EDS child cluster will ping the Cluster Handler, to try an update, which
@@ -299,7 +299,7 @@ func (s) TestUpdateRootClusterAggregateSuccess(t *testing.T) {
 
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeLogicalDNS,
-		ServiceName: logicalDNSService,
+		ClusterName: logicalDNSService,
 	}, nil)
 
 	// Will Ping Cluster Handler, which will finally successfully build an
@@ -313,10 +313,10 @@ func (s) TestUpdateRootClusterAggregateSuccess(t *testing.T) {
 	case chu := <-ch.updateChannel:
 		if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{{
 			ClusterType: xdsclient.ClusterTypeEDS,
-			ServiceName: edsService,
+			ClusterName: edsService,
 		}, {
 			ClusterType: xdsclient.ClusterTypeLogicalDNS,
-			ServiceName: logicalDNSService,
+			ClusterName: logicalDNSService,
 		}}); diff != "" {
 			t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
 		}
@@ -345,18 +345,18 @@ func (s) TestUpdateRootClusterAggregateThenChangeChild(t *testing.T) {
 
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType:             xdsclient.ClusterTypeAggregate,
-		ServiceName:             aggregateClusterService,
+		ClusterName:             aggregateClusterService,
 		PrioritizedClusterNames: []string{edsService, logicalDNSService},
 	}, nil)
 	fakeClient.WaitForWatchCluster(ctx)
 	fakeClient.WaitForWatchCluster(ctx)
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeEDS,
-		ServiceName: edsService,
+		ClusterName: edsService,
 	}, nil)
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeLogicalDNS,
-		ServiceName: logicalDNSService,
+		ClusterName: logicalDNSService,
 	}, nil)
 
 	select {
@@ -367,7 +367,7 @@ func (s) TestUpdateRootClusterAggregateThenChangeChild(t *testing.T) {
 
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType:             xdsclient.ClusterTypeAggregate,
-		ServiceName:             aggregateClusterService,
+		ClusterName:             aggregateClusterService,
 		PrioritizedClusterNames: []string{edsService, logicalDNSService2},
 	}, nil)
 
@@ -407,7 +407,7 @@ func (s) TestUpdateRootClusterAggregateThenChangeChild(t *testing.T) {
 	// tree with successful updates.
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeLogicalDNS,
-		ServiceName: logicalDNSService2,
+		ClusterName: logicalDNSService2,
 	}, nil)
 
 	// Behavior: This update make every node in the tree of cluster have
@@ -420,10 +420,10 @@ func (s) TestUpdateRootClusterAggregateThenChangeChild(t *testing.T) {
 	case chu := <-ch.updateChannel:
 		if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{{
 			ClusterType: xdsclient.ClusterTypeEDS,
-			ServiceName: edsService,
+			ClusterName: edsService,
 		}, {
 			ClusterType: xdsclient.ClusterTypeLogicalDNS,
-			ServiceName: logicalDNSService2,
+			ClusterName: logicalDNSService2,
 		}}); diff != "" {
 			t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
 		}
@@ -452,18 +452,18 @@ func (s) TestUpdateRootClusterAggregateThenChangeRootToEDS(t *testing.T) {
 
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType:             xdsclient.ClusterTypeAggregate,
-		ServiceName:             aggregateClusterService,
+		ClusterName:             aggregateClusterService,
 		PrioritizedClusterNames: []string{edsService, logicalDNSService},
 	}, nil)
 	fakeClient.WaitForWatchCluster(ctx)
 	fakeClient.WaitForWatchCluster(ctx)
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeEDS,
-		ServiceName: edsService,
+		ClusterName: edsService,
 	}, nil)
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeLogicalDNS,
-		ServiceName: logicalDNSService,
+		ClusterName: logicalDNSService,
 	}, nil)
 
 	select {
@@ -556,7 +556,7 @@ func (s) TestSwitchClusterNodeBetweenLeafAndAggregated(t *testing.T) {
 	}
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeEDS,
-		ServiceName: edsService2,
+		ClusterName: edsService2,
 	}, nil)
 	select {
 	case <-ch.updateChannel:
@@ -567,7 +567,7 @@ func (s) TestSwitchClusterNodeBetweenLeafAndAggregated(t *testing.T) {
 	// child watches to be created.
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType:             xdsclient.ClusterTypeAggregate,
-		ServiceName:             edsService2,
+		ClusterName:             edsService2,
 		PrioritizedClusterNames: []string{edsService, logicalDNSService},
 	}, nil)
 
@@ -625,7 +625,7 @@ func (s) TestSwitchClusterNodeBetweenLeafAndAggregated(t *testing.T) {
 	// children to be deleted.
 	fakeClient.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{
 		ClusterType: xdsclient.ClusterTypeEDS,
-		ServiceName: edsService2,
+		ClusterName: edsService2,
 	}, nil)
 
 	// Should delete the two children (no guarantee of ordering deleted, which
@@ -666,7 +666,7 @@ func (s) TestSwitchClusterNodeBetweenLeafAndAggregated(t *testing.T) {
 	case chu := <-ch.updateChannel:
 		if diff := cmp.Diff(chu.chu, []xdsclient.ClusterUpdate{{
 			ClusterType: xdsclient.ClusterTypeEDS,
-			ServiceName: edsService2,
+			ClusterName: edsService2,
 		}}); diff != "" {
 			t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff)
 		}
diff --git a/xds/internal/balancer/edsbalancer/config.go b/xds/internal/balancer/edsbalancer/config.go
index 11c1338c81f7..d1583e2bf276 100644
--- a/xds/internal/balancer/edsbalancer/config.go
+++ b/xds/internal/balancer/edsbalancer/config.go
@@ -35,8 +35,10 @@ type EDSConfig struct {
 	// FallBackPolicy represents the load balancing config for the
 	// fallback.
 	FallBackPolicy *loadBalancingConfig
-	// Name to use in EDS query.  If not present, defaults to the server
-	// name from the target URI.
+	// ClusterName is the cluster name.
+	ClusterName string
+	// EDSServiceName is the name to use in EDS query. If not set, use
+	// ClusterName.
 	EDSServiceName string
 	// MaxConcurrentRequests is the max number of concurrent request allowed for
 	// this service. If unset, default value 1024 is used.
@@ -59,6 +61,7 @@ type EDSConfig struct {
 type edsConfigJSON struct {
 	ChildPolicy                []*loadBalancingConfig
 	FallbackPolicy             []*loadBalancingConfig
+	ClusterName                string
 	EDSServiceName             string
 	MaxConcurrentRequests      *uint32
 	LRSLoadReportingServerName *string
@@ -73,6 +76,7 @@ func (l *EDSConfig) UnmarshalJSON(data []byte) error {
 		return err
 	}
 
+	l.ClusterName = configJSON.ClusterName
 	l.EDSServiceName = configJSON.EDSServiceName
 	l.MaxConcurrentRequests = configJSON.MaxConcurrentRequests
 	l.LrsLoadReportingServerName = configJSON.LRSLoadReportingServerName
diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go
index 5191dbd30db8..98b1dbaedd46 100644
--- a/xds/internal/balancer/edsbalancer/eds.go
+++ b/xds/internal/balancer/edsbalancer/eds.go
@@ -26,6 +26,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -70,7 +71,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 		grpcUpdate:        make(chan interface{}),
 		xdsClientUpdate:   make(chan *edsUpdate),
 		childPolicyUpdate: buffer.NewUnbounded(),
-		lsw:               &loadStoreWrapper{},
+		loadWrapper:       loadstore.NewWrapper(),
 		config:            &EDSConfig{},
 	}
 	x.logger = prefixLogger(x)
@@ -82,7 +83,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOp
 	}
 
 	x.xdsClient = client
-	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.lsw, x.logger)
+	x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.loadWrapper, x.logger)
 	x.logger.Infof("Created")
 	go x.run()
 	return x
@@ -141,14 +142,14 @@ type edsBalancer struct {
 	xdsClientUpdate   chan *edsUpdate
 	childPolicyUpdate *buffer.Unbounded
 
-	xdsClient xdsClientInterface
-	lsw       *loadStoreWrapper
-	config    *EDSConfig // may change when passed a different service config
-	edsImpl   edsBalancerImplInterface
+	xdsClient   xdsClientInterface
+	loadWrapper *loadstore.Wrapper
+	config      *EDSConfig // may change when passed a different service config
+	edsImpl     edsBalancerImplInterface
 
-	// edsServiceName is the edsServiceName currently being watched, not
-	// necessary the edsServiceName from service config.
+	clusterName          string
 	edsServiceName       string
+	edsToWatch           string // this is edsServiceName if it's set, otherwise, it's clusterName.
 	cancelEndpointsWatch func()
 	loadReportServer     *string // LRS is disabled if loadReporterServer is nil.
 	cancelLoadReport     func()
@@ -223,7 +224,7 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 			x.logger.Warningf("failed to update xDS client: %v", err)
 		}
 
-		x.edsImpl.updateServiceRequestsConfig(cfg.EDSServiceName, cfg.MaxConcurrentRequests)
+		x.edsImpl.updateServiceRequestsConfig(cfg.ClusterName, cfg.MaxConcurrentRequests)
 
 		// We will update the edsImpl with the new child policy, if we got a
 		// different one.
@@ -246,10 +247,35 @@ func (x *edsBalancer) handleGRPCUpdate(update interface{}) {
 // handleServiceConfigUpdate applies the service config update, watching a new
 // EDS service name and restarting LRS stream, as required.
 func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
-	// Restart EDS watch when the edsServiceName has changed.
+	var updateLoadClusterAndService bool
+	if x.clusterName != config.ClusterName {
+		updateLoadClusterAndService = true
+		x.clusterName = config.ClusterName
+		x.edsImpl.updateClusterName(x.clusterName)
+	}
 	if x.edsServiceName != config.EDSServiceName {
+		updateLoadClusterAndService = true
 		x.edsServiceName = config.EDSServiceName
+	}
+
+	// If EDSServiceName is set, use it to watch EDS. Otherwise, use the cluster
+	// name.
+	newEDSToWatch := config.EDSServiceName
+	if newEDSToWatch == "" {
+		newEDSToWatch = config.ClusterName
+	}
+	var restartEDSWatch bool
+	if x.edsToWatch != newEDSToWatch {
+		restartEDSWatch = true
+		x.edsToWatch = newEDSToWatch
+	}
+
+	// Restart EDS watch when the eds name has changed.
+	if restartEDSWatch {
 		x.startEndpointsWatch()
+	}
+
+	if updateLoadClusterAndService {
 		// TODO: this update for the LRS service name is too early. It should
 		// only apply to the new EDS response. But this is applied to the RPCs
 		// before the new EDS response. To fully fix this, the EDS balancer
@@ -257,14 +283,13 @@ func (x *edsBalancer) handleServiceConfigUpdate(config *EDSConfig) error {
 		//
 		// This is OK for now, because we don't actually expect edsServiceName
 		// to change. Fix this (a bigger change) will happen later.
-		x.lsw.updateServiceName(x.edsServiceName)
-		x.edsImpl.updateClusterName(x.edsServiceName)
+		x.loadWrapper.UpdateClusterAndService(x.clusterName, x.edsServiceName)
 	}
 
 	// Restart load reporting when the loadReportServer name has changed.
 	if !equalStringPointers(x.loadReportServer, config.LrsLoadReportingServerName) {
 		loadStore := x.startLoadReport(config.LrsLoadReportingServerName)
-		x.lsw.updateLoadStore(loadStore)
+		x.loadWrapper.UpdateLoadStore(loadStore)
 	}
 
 	return nil
@@ -278,14 +303,15 @@ func (x *edsBalancer) startEndpointsWatch() {
 	if x.cancelEndpointsWatch != nil {
 		x.cancelEndpointsWatch()
 	}
-	cancelEDSWatch := x.xdsClient.WatchEndpoints(x.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) {
+	edsToWatch := x.edsToWatch
+	cancelEDSWatch := x.xdsClient.WatchEndpoints(edsToWatch, func(update xdsclient.EndpointsUpdate, err error) {
 		x.logger.Infof("Watch update from xds-client %p, content: %+v", x.xdsClient, pretty.ToJSON(update))
 		x.handleEDSUpdate(update, err)
 	})
-	x.logger.Infof("Watch started on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
+	x.logger.Infof("Watch started on resource name %v with xds-client %p", edsToWatch, x.xdsClient)
 	x.cancelEndpointsWatch = func() {
 		cancelEDSWatch()
-		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", x.edsServiceName, x.xdsClient)
+		x.logger.Infof("Watch cancelled on resource name %v with xds-client %p", edsToWatch, x.xdsClient)
 	}
 }
 
@@ -293,10 +319,12 @@ func (x *edsBalancer) cancelWatch() {
 	x.loadReportServer = nil
 	if x.cancelLoadReport != nil {
 		x.cancelLoadReport()
+		x.cancelLoadReport = nil
 	}
-	x.edsServiceName = ""
 	if x.cancelEndpointsWatch != nil {
+		x.edsToWatch = ""
 		x.cancelEndpointsWatch()
+		x.cancelEndpointsWatch = nil
 	}
 }
 
@@ -310,6 +338,7 @@ func (x *edsBalancer) startLoadReport(loadReportServer *string) *load.Store {
 	x.loadReportServer = loadReportServer
 	if x.cancelLoadReport != nil {
 		x.cancelLoadReport()
+		x.cancelLoadReport = nil
 	}
 	if loadReportServer == nil {
 		return nil
diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go
index c5e3071d10d8..3cfc620a2400 100644
--- a/xds/internal/balancer/edsbalancer/eds_impl_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/xds/internal/balancer/loadstore"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/roundrobin"
@@ -820,9 +821,9 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 	// implements the LoadStore() method to return the underlying load.Store to
 	// be used.
 	loadStore := load.NewStore()
-	lsWrapper := &loadStoreWrapper{}
-	lsWrapper.updateServiceName(testClusterNames[0])
-	lsWrapper.updateLoadStore(loadStore)
+	lsWrapper := loadstore.NewWrapper()
+	lsWrapper.UpdateClusterAndService(testClusterNames[0], "")
+	lsWrapper.UpdateLoadStore(loadStore)
 
 	cc := testutils.NewTestClientConn(t)
 	edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, lsWrapper, nil)
@@ -913,8 +914,8 @@ func (s) TestEDS_LoadReport(t *testing.T) {
 // TestEDS_LoadReportDisabled covers the case that LRS is disabled. It makes
 // sure the EDS implementation isn't broken (doesn't panic).
 func (s) TestEDS_LoadReportDisabled(t *testing.T) {
-	lsWrapper := &loadStoreWrapper{}
-	lsWrapper.updateServiceName(testClusterNames[0])
+	lsWrapper := loadstore.NewWrapper()
+	lsWrapper.UpdateClusterAndService(testClusterNames[0], "")
 	// Not calling lsWrapper.updateLoadStore(loadStore) because LRS is disabled.
 
 	cc := testutils.NewTestClientConn(t)
diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go
index 5c9e5f0b1d53..3fe66098973c 100644
--- a/xds/internal/balancer/edsbalancer/eds_test.go
+++ b/xds/internal/balancer/edsbalancer/eds_test.go
@@ -354,6 +354,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			ChildPolicy:    lbCfgA,
+			ClusterName:    testEDSClusterName,
 			EDSServiceName: testServiceName,
 		},
 	}); err != nil {
@@ -367,7 +368,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgA); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
+	if err := edsLB.waitForCounterUpdate(ctx, testEDSClusterName); err != nil {
 		t.Fatal(err)
 	}
 	if err := edsLB.waitForCountMaxUpdate(ctx, nil); err != nil {
@@ -382,6 +383,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
 			ChildPolicy:           lbCfgB,
+			ClusterName:           testEDSClusterName,
 			EDSServiceName:        testServiceName,
 			MaxConcurrentRequests: &testCountMax,
 		},
@@ -391,7 +393,7 @@ func (s) TestConfigChildPolicyUpdate(t *testing.T) {
 	if err := edsLB.waitForChildPolicy(ctx, lbCfgB); err != nil {
 		t.Fatal(err)
 	}
-	if err := edsLB.waitForCounterUpdate(ctx, testServiceName); err != nil {
+	if err := edsLB.waitForCounterUpdate(ctx, testEDSClusterName); err != nil {
 		// Counter is updated even though the service name didn't change. The
 		// eds_impl will compare the service names, and skip if it didn't change.
 		t.Fatal(err)
@@ -510,6 +512,18 @@ func (s) TestErrorFromXDSClientUpdate(t *testing.T) {
 	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("eds impl expecting empty update, got %v", err)
 	}
+
+	// An update with the same service name should not trigger a new watch.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := xdsC.WaitForWatchEDS(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("got unexpected new EDS watch")
+	}
 }
 
 // TestErrorFromResolver verifies that resolver errors are handled correctly.
@@ -575,6 +589,17 @@ func (s) TestErrorFromResolver(t *testing.T) {
 	if err := edsLB.waitForEDSResponse(ctx, xdsclient.EndpointsUpdate{}); err != nil {
 		t.Fatalf("EDS impl got unexpected EDS response: %v", err)
 	}
+
+	// An update with the same service name should trigger a new watch, because
+	// the previous watch was canceled.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{EDSServiceName: testServiceName},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
+		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
+	}
 }
 
 // Given a list of resource names, verifies that EDS requests for the same are
@@ -586,7 +611,7 @@ func verifyExpectedRequests(ctx context.Context, fc *fakeclient.Client, resource
 			if err := fc.WaitForCancelEDSWatch(ctx); err != nil {
 				return fmt.Errorf("timed out when expecting resource %q", name)
 			}
-			return nil
+			continue
 		}
 
 		resName, err := fc.WaitForWatchEDS(ctx)
@@ -615,6 +640,18 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}
 	defer edsB.Close()
 
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// If eds service name is not set, should watch for cluster name.
+	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &EDSConfig{ClusterName: "cluster-1"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyExpectedRequests(ctx, xdsC, "cluster-1"); err != nil {
+		t.Fatal(err)
+	}
+
 	// Update with an non-empty edsServiceName should trigger an EDS watch for
 	// the same.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
@@ -622,9 +659,7 @@ func (s) TestClientWatchEDS(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
-	defer cancel()
-	if err := verifyExpectedRequests(ctx, xdsC, "foobar-1"); err != nil {
+	if err := verifyExpectedRequests(ctx, xdsC, "", "foobar-1"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -660,7 +695,7 @@ func (s) TestCounterUpdate(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			EDSServiceName:        "foobar-1",
+			ClusterName:           "foobar-1",
 			MaxConcurrentRequests: &testCountMax,
 		},
 	}); err != nil {
@@ -694,7 +729,7 @@ func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			EDSServiceName: "foobar-1",
+			ClusterName: "foobar-1",
 		},
 	}); err != nil {
 		t.Fatal(err)
@@ -713,7 +748,7 @@ func (s) TestClusterNameUpdateInAddressAttributes(t *testing.T) {
 	// Update should trigger counter update with provided service name.
 	if err := edsB.UpdateClientConnState(balancer.ClientConnState{
 		BalancerConfig: &EDSConfig{
-			EDSServiceName: "foobar-2",
+			ClusterName: "foobar-2",
 		},
 	}); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/balancer/edsbalancer/load_store_wrapper.go b/xds/internal/balancer/edsbalancer/load_store_wrapper.go
deleted file mode 100644
index 18904e47a42e..000000000000
--- a/xds/internal/balancer/edsbalancer/load_store_wrapper.go
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- *
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package edsbalancer
-
-import (
-	"sync"
-
-	"google.golang.org/grpc/xds/internal/client/load"
-)
-
-type loadStoreWrapper struct {
-	mu      sync.RWMutex
-	service string
-	// Both store and perCluster will be nil if load reporting is disabled (EDS
-	// response doesn't have LRS server name). Note that methods on Store and
-	// perCluster all handle nil, so there's no need to check nil before calling
-	// them.
-	store      *load.Store
-	perCluster load.PerClusterReporter
-}
-
-func (lsw *loadStoreWrapper) updateServiceName(service string) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if lsw.service == service {
-		return
-	}
-	lsw.service = service
-	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
-}
-
-func (lsw *loadStoreWrapper) updateLoadStore(store *load.Store) {
-	lsw.mu.Lock()
-	defer lsw.mu.Unlock()
-	if store == lsw.store {
-		return
-	}
-	lsw.store = store
-	lsw.perCluster = lsw.store.PerCluster(lsw.service, "")
-}
-
-func (lsw *loadStoreWrapper) CallStarted(locality string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallStarted(locality)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallFinished(locality string, err error) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallFinished(locality, err)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallServerLoad(locality, name string, val float64) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallServerLoad(locality, name, val)
-	}
-}
-
-func (lsw *loadStoreWrapper) CallDropped(category string) {
-	lsw.mu.RLock()
-	defer lsw.mu.RUnlock()
-	if lsw.perCluster != nil {
-		lsw.perCluster.CallDropped(category)
-	}
-}
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 627229de7ad0..bb526116bda6 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -28,6 +28,7 @@ import (
 	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
 	anypb "github.com/golang/protobuf/ptypes/any"
@@ -45,7 +46,7 @@ const (
 	serviceName = "service"
 )
 
-var emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false}
+var emptyUpdate = ClusterUpdate{ClusterName: clusterName, EnableLRS: false}
 
 func (s) TestValidateCluster_Failure(t *testing.T) {
 	tests := []struct {
@@ -141,24 +142,35 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		{
 			name: "happy-case-logical-dns",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS},
 				LbPolicy:             v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EnableLRS: false, ClusterType: ClusterTypeLogicalDNS},
 		},
 		{
 			name: "happy-case-aggregate-v3",
 			cluster: &v3clusterpb.Cluster{
+				Name: clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{
-					ClusterType: &v3clusterpb.Cluster_CustomClusterType{Name: "envoy.clusters.aggregate"},
+					ClusterType: &v3clusterpb.Cluster_CustomClusterType{
+						Name: "envoy.clusters.aggregate",
+						TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{
+							Clusters: []string{"a", "b", "c"},
+						}),
+					},
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: "", EnableLRS: false, ClusterType: ClusterTypeAggregate},
+			wantUpdate: ClusterUpdate{
+				ClusterName: clusterName, EnableLRS: false, ClusterType: ClusterTypeAggregate,
+				PrioritizedClusterNames: []string{"a", "b", "c"},
+			},
 		},
 		{
 			name: "happy-case-no-service-name-no-lrs",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -174,6 +186,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 		{
 			name: "happy-case-no-lrs",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -185,7 +198,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 				},
 				LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: false},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: false},
 		},
 		{
 			name: "happiest-case",
@@ -207,7 +220,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: true},
 		},
 		{
 			name: "happiest-case-with-circuitbreakers",
@@ -241,7 +254,7 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 					},
 				},
 			},
-			wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
+			wantUpdate: ClusterUpdate{ClusterName: clusterName, EDSServiceName: serviceName, EnableLRS: true, MaxRequests: func() *uint32 { i := uint32(512); return &i }()},
 		},
 	}
 
@@ -254,8 +267,8 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 			if err != nil {
 				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err)
 			}
-			if !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) {
-				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, want: %v", test.cluster, update, test.wantUpdate)
+			if diff := cmp.Diff(update, test.wantUpdate, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("validateClusterAndConstructClusterUpdate(%+v) got diff: %v (-got, +want)", test.cluster, diff)
 			}
 		})
 	}
@@ -268,6 +281,7 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 	defer func() { env.ClientSideSecuritySupport = origClientSideSecurityEnvVar }()
 
 	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
 		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 			EdsConfig: &v3corepb.ConfigSource{
@@ -295,8 +309,9 @@ func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) {
 		},
 	}
 	wantUpdate := ClusterUpdate{
-		ServiceName: serviceName,
-		EnableLRS:   false,
+		ClusterName:    clusterName,
+		EDSServiceName: serviceName,
+		EnableLRS:      false,
 	}
 	gotUpdate, err := validateClusterAndConstructClusterUpdate(cluster)
 	if err != nil {
@@ -318,6 +333,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		identityCertName       = "identityCert"
 		rootPluginInstance     = "rootPluginInstance"
 		rootCertName           = "rootCert"
+		clusterName            = "cluster"
 		serviceName            = "service"
 		sanExact               = "san-exact"
 		sanPrefix              = "san-prefix"
@@ -608,6 +624,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-no-identity-certs",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -635,8 +652,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ServiceName: serviceName,
-				EnableLRS:   false,
+				ClusterName:    clusterName,
+				EDSServiceName: serviceName,
+				EnableLRS:      false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName: rootPluginInstance,
 					RootCertName:     rootCertName,
@@ -646,6 +664,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-validation-context-provider-instance",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -677,8 +696,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ServiceName: serviceName,
-				EnableLRS:   false,
+				ClusterName:    clusterName,
+				EDSServiceName: serviceName,
+				EnableLRS:      false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName:     rootPluginInstance,
 					RootCertName:         rootCertName,
@@ -690,6 +710,7 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 		{
 			name: "happy-case-with-combined-validation-context",
 			cluster: &v3clusterpb.Cluster{
+				Name:                 clusterName,
 				ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
 				EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
 					EdsConfig: &v3corepb.ConfigSource{
@@ -735,8 +756,9 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 				},
 			},
 			wantUpdate: ClusterUpdate{
-				ServiceName: serviceName,
-				EnableLRS:   false,
+				ClusterName:    clusterName,
+				EDSServiceName: serviceName,
+				EnableLRS:      false,
 				SecurityCfg: &SecurityConfig{
 					RootInstanceName:     rootPluginInstance,
 					RootCertName:         rootCertName,
@@ -877,7 +899,8 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v2ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ServiceName: v2Service, EnableLRS: true,
+					ClusterName:    v2ClusterName,
+					EDSServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 			},
@@ -891,7 +914,8 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v3ClusterName: {
-					ServiceName: v3Service, EnableLRS: true,
+					ClusterName:    v3ClusterName,
+					EDSServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 			},
@@ -905,11 +929,13 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			resources: []*anypb.Any{v2ClusterAny, v3ClusterAny},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ServiceName: v2Service, EnableLRS: true,
+					ClusterName:    v2ClusterName,
+					EDSServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 				v3ClusterName: {
-					ServiceName: v3Service, EnableLRS: true,
+					ClusterName:    v3ClusterName,
+					EDSServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 			},
@@ -932,11 +958,13 @@ func (s) TestUnmarshalCluster(t *testing.T) {
 			},
 			wantUpdate: map[string]ClusterUpdate{
 				v2ClusterName: {
-					ServiceName: v2Service, EnableLRS: true,
+					ClusterName:    v2ClusterName,
+					EDSServiceName: v2Service, EnableLRS: true,
 					Raw: v2ClusterAny,
 				},
 				v3ClusterName: {
-					ServiceName: v3Service, EnableLRS: true,
+					ClusterName:    v3ClusterName,
+					EDSServiceName: v3Service, EnableLRS: true,
 					Raw: v3ClusterAny,
 				},
 				"bad": {},
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 603632801b04..05093653c075 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -378,22 +378,23 @@ const (
 // interest to the registered CDS watcher.
 type ClusterUpdate struct {
 	ClusterType ClusterType
-	// ServiceName is the service name corresponding to the clusterName which
-	// is being watched for through CDS.
-	ServiceName string
+	// ClusterName is the clusterName being watched for through CDS.
+	ClusterName string
+	// EDSServiceName is an optional name for EDS. If it's not set, the balancer
+	// should watch ClusterName for the EDS resources.
+	EDSServiceName string
 	// EnableLRS indicates whether or not load should be reported through LRS.
 	EnableLRS bool
 	// SecurityCfg contains security configuration sent by the control plane.
 	SecurityCfg *SecurityConfig
 	// MaxRequests for circuit breaking, if any (otherwise nil).
 	MaxRequests *uint32
-
-	// Raw is the resource from the xds response.
-	Raw *anypb.Any
-
 	// PrioritizedClusterNames is used only for cluster type aggregate. It represents
 	// a prioritized list of cluster names.
 	PrioritizedClusterNames []string
+
+	// Raw is the resource from the xds response.
+	Raw *anypb.Any
 }
 
 // OverloadDropConfig contains the config to drop overloads.
diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go
index 69930557b26e..d57bc20e2c2c 100644
--- a/xds/internal/client/client_test.go
+++ b/xds/internal/client/client_test.go
@@ -185,13 +185,13 @@ func (s) TestWatchCallAnotherWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
 	}
 
-	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate2}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate2); err != nil {
 		t.Fatal(err)
diff --git a/xds/internal/client/v2/cds_test.go b/xds/internal/client/v2/cds_test.go
index e627860d2a9f..ba7f627b25ff 100644
--- a/xds/internal/client/v2/cds_test.go
+++ b/xds/internal/client/v2/cds_test.go
@@ -151,7 +151,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			cdsResponse: goodCDSResponse2,
 			wantErr:     false,
 			wantUpdate: map[string]xdsclient.ClusterUpdate{
-				goodClusterName2: {ServiceName: serviceName2, Raw: marshaledCluster2},
+				goodClusterName2: {ClusterName: goodClusterName2, EDSServiceName: serviceName2, Raw: marshaledCluster2},
 			},
 			wantUpdateMD: xdsclient.UpdateMetadata{
 				Status: xdsclient.ServiceStatusACKed,
@@ -164,7 +164,7 @@ func (s) TestCDSHandleResponse(t *testing.T) {
 			cdsResponse: goodCDSResponse1,
 			wantErr:     false,
 			wantUpdate: map[string]xdsclient.ClusterUpdate{
-				goodClusterName1: {ServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
+				goodClusterName1: {ClusterName: goodClusterName1, EDSServiceName: serviceName1, EnableLRS: true, Raw: marshaledCluster1},
 			},
 			wantUpdateMD: xdsclient.UpdateMetadata{
 				Status: xdsclient.ServiceStatusACKed,
diff --git a/xds/internal/client/watchers_cluster_test.go b/xds/internal/client/watchers_cluster_test.go
index 2d10c7f43b5f..c9837cd51978 100644
--- a/xds/internal/client/watchers_cluster_test.go
+++ b/xds/internal/client/watchers_cluster_test.go
@@ -64,7 +64,7 @@ func (s) TestClusterWatch(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	if err := verifyClusterUpdate(ctx, clusterUpdateCh, wantUpdate); err != nil {
 		t.Fatal(err)
@@ -128,7 +128,7 @@ func (s) TestClusterTwoWatchSameResourceName(t *testing.T) {
 		}
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{testCDSName: wantUpdate}, UpdateMetadata{})
 	for i := 0; i < count; i++ {
 		if err := verifyClusterUpdate(ctx, clusterUpdateChs[i], wantUpdate); err != nil {
@@ -200,8 +200,8 @@ func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
-	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	wantUpdate1 := ClusterUpdate{ClusterName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
@@ -245,7 +245,7 @@ func (s) TestClusterWatchAfterCache(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	}, UpdateMetadata{})
@@ -345,7 +345,7 @@ func (s) TestClusterWatchExpiryTimerStop(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate := ClusterUpdate{ServiceName: testEDSName}
+	wantUpdate := ClusterUpdate{ClusterName: testEDSName}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName: wantUpdate,
 	}, UpdateMetadata{})
@@ -402,8 +402,8 @@ func (s) TestClusterResourceRemoved(t *testing.T) {
 		t.Fatalf("want new watch to start, got error %v", err)
 	}
 
-	wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"}
-	wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"}
+	wantUpdate1 := ClusterUpdate{ClusterName: testEDSName + "1"}
+	wantUpdate2 := ClusterUpdate{ClusterName: testEDSName + "2"}
 	client.NewClusters(map[string]ClusterUpdate{
 		testCDSName + "1": wantUpdate1,
 		testCDSName + "2": wantUpdate2,
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index cf705587e24a..781eeb47a062 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -583,42 +583,9 @@ func unmarshalClusterResource(r *anypb.Any, logger *grpclog.PrefixLogger) (strin
 	return cluster.GetName(), cu, nil
 }
 
-func clusterTypeFromCluster(cluster *v3clusterpb.Cluster) (ClusterType, string, []string, error) {
-	if cluster.GetType() == v3clusterpb.Cluster_EDS {
-		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
-			return 0, "", nil, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
-		}
-		// If the Cluster message in the CDS response did not contain a
-		// serviceName, we will just use the clusterName for EDS.
-		if cluster.GetEdsClusterConfig().GetServiceName() == "" {
-			return ClusterTypeEDS, cluster.GetName(), nil, nil
-		}
-		return ClusterTypeEDS, cluster.GetEdsClusterConfig().GetServiceName(), nil, nil
-	}
-
-	if cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS {
-		return ClusterTypeLogicalDNS, cluster.GetName(), nil, nil
-	}
-
-	if cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate" {
-		// Loop through ClusterConfig here to get cluster names.
-		clusters := &v3aggregateclusterpb.ClusterConfig{}
-		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
-			return 0, "", nil, fmt.Errorf("failed to unmarshal resource: %v", err)
-		}
-		return ClusterTypeAggregate, cluster.GetName(), clusters.Clusters, nil
-	}
-	return 0, "", nil, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
-}
-
 func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
-	emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false}
 	if cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN {
-		return emptyUpdate, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
-	}
-	clusterType, serviceName, prioritizedClusters, err := clusterTypeFromCluster(cluster)
-	if err != nil {
-		return emptyUpdate, err
+		return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
 	}
 
 	// Process security configuration received from the control plane iff the
@@ -627,18 +594,40 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu
 	if env.ClientSideSecuritySupport {
 		var err error
 		if sc, err = securityConfigFromCluster(cluster); err != nil {
-			return emptyUpdate, err
+			return ClusterUpdate{}, err
 		}
 	}
 
-	return ClusterUpdate{
-		ClusterType:             clusterType,
-		ServiceName:             serviceName,
-		EnableLRS:               cluster.GetLrsServer().GetSelf() != nil,
-		SecurityCfg:             sc,
-		MaxRequests:             circuitBreakersFromCluster(cluster),
-		PrioritizedClusterNames: prioritizedClusters,
-	}, nil
+	ret := ClusterUpdate{
+		ClusterName: cluster.GetName(),
+		EnableLRS:   cluster.GetLrsServer().GetSelf() != nil,
+		SecurityCfg: sc,
+		MaxRequests: circuitBreakersFromCluster(cluster),
+	}
+
+	// Validate and set cluster type from the response.
+	switch {
+	case cluster.GetType() == v3clusterpb.Cluster_EDS:
+		if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil {
+			return ClusterUpdate{}, fmt.Errorf("unexpected edsConfig in response: %+v", cluster)
+		}
+		ret.ClusterType = ClusterTypeEDS
+		ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName()
+		return ret, nil
+	case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS:
+		ret.ClusterType = ClusterTypeLogicalDNS
+		return ret, nil
+	case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate":
+		clusters := &v3aggregateclusterpb.ClusterConfig{}
+		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
+			return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
+		}
+		ret.ClusterType = ClusterTypeAggregate
+		ret.PrioritizedClusterNames = clusters.Clusters
+		return ret, nil
+	default:
+		return ClusterUpdate{}, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+	}
 }
 
 // securityConfigFromCluster extracts the relevant security configuration from
diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go
index eb4c659e505a..f85f0ecbf3f3 100644
--- a/xds/internal/testutils/fakeclient/client.go
+++ b/xds/internal/testutils/fakeclient/client.go
@@ -156,7 +156,7 @@ func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.ClusterUpdate, e
 	} else {
 		// Have what callback you call with the update determined by the service name in the ClusterUpdate. Left up to the
 		// caller to make sure the cluster update matches with a persisted callback.
-		xdsC.cdsCbs[update.ServiceName](update, err)
+		xdsC.cdsCbs[update.ClusterName](update, err)
 	}
 }
 

From 6fea90d7a884ad070a4f04863521eaf43e6c5d11 Mon Sep 17 00:00:00 2001
From: Mayank Singhal <mayanksinghal308@gmail.com>
Date: Thu, 13 May 2021 05:45:47 +0530
Subject: [PATCH 475/481] benchmark: do not allow addition of values lower than
 the minimum allowed in histogram stats

---
 benchmark/stats/histogram.go | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/benchmark/stats/histogram.go b/benchmark/stats/histogram.go
index f038d26ed0aa..461135f0125c 100644
--- a/benchmark/stats/histogram.go
+++ b/benchmark/stats/histogram.go
@@ -118,10 +118,6 @@ func (h *Histogram) PrintWithUnit(w io.Writer, unit float64) {
 	}
 
 	maxBucketDigitLen := len(strconv.FormatFloat(h.Buckets[len(h.Buckets)-1].LowBound, 'f', 6, 64))
-	if maxBucketDigitLen < 3 {
-		// For "inf".
-		maxBucketDigitLen = 3
-	}
 	maxCountDigitLen := len(strconv.FormatInt(h.Count, 10))
 	percentMulti := 100 / float64(h.Count)
 
@@ -131,9 +127,9 @@ func (h *Histogram) PrintWithUnit(w io.Writer, unit float64) {
 		if i+1 < len(h.Buckets) {
 			fmt.Fprintf(w, "%*f)", maxBucketDigitLen, h.Buckets[i+1].LowBound/unit)
 		} else {
-			fmt.Fprintf(w, "%*s)", maxBucketDigitLen, "inf")
+			upperBound := float64(h.opts.MinValue) + (b.LowBound-float64(h.opts.MinValue))*(1.0+h.opts.GrowthFactor)
+			fmt.Fprintf(w, "%*f)", maxBucketDigitLen, upperBound/unit)
 		}
-
 		accCount += b.Count
 		fmt.Fprintf(w, "  %*d  %5.1f%%  %5.1f%%", maxCountDigitLen, b.Count, float64(b.Count)*percentMulti, float64(accCount)*percentMulti)
 
@@ -188,6 +184,9 @@ func (h *Histogram) Add(value int64) error {
 
 func (h *Histogram) findBucket(value int64) (int, error) {
 	delta := float64(value - h.opts.MinValue)
+	if delta < 0 {
+		return 0, fmt.Errorf("no bucket for value: %d", value)
+	}
 	var b int
 	if delta >= h.opts.BaseBucketSize {
 		// b = log_{1+growthFactor} (delta / baseBucketSize) + 1

From a16b156e990b0fb4100a4694e1c6dda779b08f77 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Wed, 12 May 2021 17:43:29 -0700
Subject: [PATCH 476/481] internal: fix flaky test
 KeepaliveClientStaysHealthyWithResponsiveServer (#4427)

Server should allow `NoStream`, otherwise there's a small chance
(5/1000) the connection will be closed due to `too many pings`.
---
 internal/transport/keepalive_test.go | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go
index 571cacca7e91..c4021925f325 100644
--- a/internal/transport/keepalive_test.go
+++ b/internal/transport/keepalive_test.go
@@ -346,12 +346,19 @@ func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) {
 // responds to keepalive pings, and makes sure than a client transport stays
 // healthy without any active streams.
 func (s) TestKeepaliveClientStaysHealthyWithResponsiveServer(t *testing.T) {
-	server, client, cancel := setUpWithOptions(t, 0, &ServerConfig{}, normal, ConnectOptions{
-		KeepaliveParams: keepalive.ClientParameters{
-			Time:                1 * time.Second,
-			Timeout:             1 * time.Second,
-			PermitWithoutStream: true,
-		}})
+	server, client, cancel := setUpWithOptions(t, 0,
+		&ServerConfig{
+			KeepalivePolicy: keepalive.EnforcementPolicy{
+				PermitWithoutStream: true,
+			},
+		},
+		normal,
+		ConnectOptions{
+			KeepaliveParams: keepalive.ClientParameters{
+				Time:                1 * time.Second,
+				Timeout:             1 * time.Second,
+				PermitWithoutStream: true,
+			}})
 	defer func() {
 		client.Close(fmt.Errorf("closed manually by test"))
 		server.stop()

From dc77d7ffe311f78f2e577572d984af3c0a8df82b Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Wed, 12 May 2021 18:03:52 -0700
Subject: [PATCH 477/481] xds: revert a workaround made in #4413 (#4428)

---
 xds/internal/test/xds_integration_test.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/xds/internal/test/xds_integration_test.go b/xds/internal/test/xds_integration_test.go
index c2bb6bc18f67..a41fec929762 100644
--- a/xds/internal/test/xds_integration_test.go
+++ b/xds/internal/test/xds_integration_test.go
@@ -83,11 +83,6 @@ func TestMain(m *testing.M) {
 	// spawns the management server and is blocked on the call to `Serve()`.
 	leakcheck.RegisterIgnoreGoroutine("e2e.StartManagementServer")
 
-	// Remove this once https://github.com/envoyproxy/go-control-plane/pull/430
-	// is merged. For more information about this goroutine leak, see:
-	// https://github.com/envoyproxy/go-control-plane/issues/429.
-	leakcheck.RegisterIgnoreGoroutine("(*server).StreamHandler")
-
 	cancel, err := setupManagementServer()
 	if err != nil {
 		log.Printf("setupManagementServer() failed: %v", err)

From 71a1ca6c7f859658e44f0073fb754c4698216202 Mon Sep 17 00:00:00 2001
From: Easwar Swaminathan <easwars@google.com>
Date: Fri, 14 May 2021 11:13:26 -0700
Subject: [PATCH 478/481] interop/xds: support xds credentials in interop
 client (#4436)

---
 interop/xds/client/client.go | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go
index b028ec79228e..92999b2193ed 100644
--- a/interop/xds/client/client.go
+++ b/interop/xds/client/client.go
@@ -32,6 +32,8 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/admin"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
@@ -175,6 +177,7 @@ var (
 	rpcTimeout      = flag.Duration("rpc_timeout", 20*time.Second, "Per RPC timeout")
 	server          = flag.String("server", "localhost:8080", "Address of server to connect to")
 	statsPort       = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service")
+	secureMode      = flag.Bool("secure_mode", false, "If true, retrieve security configuration from the management server. Else, use insecure credentials.")
 
 	rpcCfgs atomic.Value
 
@@ -375,14 +378,23 @@ func main() {
 	reflection.Register(s)
 	cleanup, err := admin.Register(s)
 	if err != nil {
-		logger.Fatalf("failed to register admin: %v", err)
+		logger.Fatalf("Failed to register admin: %v", err)
 	}
 	defer cleanup()
 	go s.Serve(lis)
 
+	creds := insecure.NewCredentials()
+	if *secureMode {
+		var err error
+		creds, err = xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+		if err != nil {
+			logger.Fatalf("Failed to create xDS credentials: %v", err)
+		}
+	}
+
 	clients := make([]testgrpc.TestServiceClient, *numChannels)
 	for i := 0; i < *numChannels; i++ {
-		conn, err := grpc.Dial(*server, grpc.WithInsecure())
+		conn, err := grpc.Dial(*server, grpc.WithTransportCredentials(creds))
 		if err != nil {
 			logger.Fatalf("Fail to dial: %v", err)
 		}

From b759b408e84fd5e990073fdaa28cd24d8eb2adad Mon Sep 17 00:00:00 2001
From: Zach Reyes <39203661+zasweq@users.noreply.github.com>
Date: Fri, 14 May 2021 17:02:10 -0400
Subject: [PATCH 479/481] xds: moved shared matchers to internal/xds (#4441)

* Moved shared matchers to internal/xds
---
 credentials/xds/xds_client_test.go            |   6 +-
 internal/credentials/xds/handshake_info.go    |  12 +-
 .../credentials/xds/handshake_info_test.go    | 100 ++++----
 internal/xds/bootstrap.go                     |   2 +
 internal/xds/matcher/matcher_header.go        | 224 ++++++++++++++++++
 .../xds/matcher}/matcher_header_test.go       |  36 +--
 internal/xds/{ => matcher}/string_matcher.go  |   4 +-
 .../xds/{ => matcher}/string_matcher_test.go  |   2 +-
 .../cdsbalancer/cdsbalancer_security_test.go  |  14 +-
 xds/internal/client/cds_test.go               |  14 +-
 xds/internal/client/client.go                 |   4 +-
 xds/internal/client/xds.go                    |   6 +-
 xds/internal/resolver/matcher.go              |  25 +-
 xds/internal/resolver/matcher_header.go       | 188 ---------------
 xds/internal/resolver/matcher_test.go         |  17 +-
 15 files changed, 347 insertions(+), 307 deletions(-)
 create mode 100644 internal/xds/matcher/matcher_header.go
 rename {xds/internal/resolver => internal/xds/matcher}/matcher_header_test.go (88%)
 rename internal/xds/{ => matcher}/string_matcher.go (98%)
 rename internal/xds/{ => matcher}/string_matcher_test.go (99%)
 delete mode 100644 xds/internal/resolver/matcher_header.go

diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go
index 2c882be8a549..07a65b6e3cf2 100644
--- a/credentials/xds/xds_client_test.go
+++ b/credentials/xds/xds_client_test.go
@@ -38,7 +38,7 @@ import (
 	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
-	"google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/testdata"
 )
@@ -223,7 +223,7 @@ func newTestContextWithHandshakeInfo(parent context.Context, root, identity cert
 	// NewSubConn().
 	info := xdsinternal.NewHandshakeInfo(root, identity)
 	if sanExactMatch != "" {
-		info.SetSANMatchers([]xds.StringMatcher{xds.StringMatcherForTesting(newStringP(sanExactMatch), nil, nil, nil, nil, false)})
+		info.SetSANMatchers([]matcher.StringMatcher{matcher.StringMatcherForTesting(newStringP(sanExactMatch), nil, nil, nil, nil, false)})
 	}
 	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)
 
@@ -536,7 +536,7 @@ func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	// use the correct trust roots.
 	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
 	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil)
-	handshakeInfo.SetSANMatchers([]xds.StringMatcher{xds.StringMatcherForTesting(newStringP(defaultTestCertSAN), nil, nil, nil, nil, false)})
+	handshakeInfo.SetSANMatchers([]matcher.StringMatcher{matcher.StringMatcherForTesting(newStringP(defaultTestCertSAN), nil, nil, nil, nil, false)})
 
 	// We need to repeat most of what newTestContextWithHandshakeInfo() does
 	// here because we need access to the underlying HandshakeInfo so that we
diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go
index 6789a4cf2e51..6ef43cc89fa3 100644
--- a/internal/credentials/xds/handshake_info.go
+++ b/internal/credentials/xds/handshake_info.go
@@ -31,7 +31,7 @@ import (
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
-	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -66,8 +66,8 @@ type HandshakeInfo struct {
 	mu                sync.Mutex
 	rootProvider      certprovider.Provider
 	identityProvider  certprovider.Provider
-	sanMatchers       []xdsinternal.StringMatcher // Only on the client side.
-	requireClientCert bool                        // Only on server side.
+	sanMatchers       []matcher.StringMatcher // Only on the client side.
+	requireClientCert bool                    // Only on server side.
 }
 
 // SetRootCertProvider updates the root certificate provider.
@@ -85,7 +85,7 @@ func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider)
 }
 
 // SetSANMatchers updates the list of SAN matchers.
-func (hi *HandshakeInfo) SetSANMatchers(sanMatchers []xdsinternal.StringMatcher) {
+func (hi *HandshakeInfo) SetSANMatchers(sanMatchers []matcher.StringMatcher) {
 	hi.mu.Lock()
 	hi.sanMatchers = sanMatchers
 	hi.mu.Unlock()
@@ -113,10 +113,10 @@ func (hi *HandshakeInfo) UseFallbackCreds() bool {
 
 // GetSANMatchersForTesting returns the SAN matchers stored in HandshakeInfo.
 // To be used only for testing purposes.
-func (hi *HandshakeInfo) GetSANMatchersForTesting() []xdsinternal.StringMatcher {
+func (hi *HandshakeInfo) GetSANMatchersForTesting() []matcher.StringMatcher {
 	hi.mu.Lock()
 	defer hi.mu.Unlock()
-	return append([]xdsinternal.StringMatcher{}, hi.sanMatchers...)
+	return append([]matcher.StringMatcher{}, hi.sanMatchers...)
 }
 
 // ClientSideTLSConfig constructs a tls.Config to be used in a client-side
diff --git a/internal/credentials/xds/handshake_info_test.go b/internal/credentials/xds/handshake_info_test.go
index 81906fa758a1..91257a1925da 100644
--- a/internal/credentials/xds/handshake_info_test.go
+++ b/internal/credentials/xds/handshake_info_test.go
@@ -25,7 +25,7 @@ import (
 	"regexp"
 	"testing"
 
-	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/matcher"
 )
 
 func TestDNSMatch(t *testing.T) {
@@ -143,45 +143,45 @@ func TestMatchingSANExists_FailureCases(t *testing.T) {
 
 	tests := []struct {
 		desc        string
-		sanMatchers []xdsinternal.StringMatcher
+		sanMatchers []matcher.StringMatcher
 	}{
 		{
 			desc: "exact match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(newStringP("abcd.test.com"), nil, nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(newStringP("http://golang"), nil, nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(newStringP("HTTP://GOLANG.ORG"), nil, nil, nil, nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(newStringP("abcd.test.com"), nil, nil, nil, nil, false),
+				matcher.StringMatcherForTesting(newStringP("http://golang"), nil, nil, nil, nil, false),
+				matcher.StringMatcherForTesting(newStringP("HTTP://GOLANG.ORG"), nil, nil, nil, nil, false),
 			},
 		},
 		{
 			desc: "prefix match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, newStringP("i-aint-the-one"), nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, newStringP("FOO.BAR"), nil, nil, nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, newStringP("i-aint-the-one"), nil, nil, nil, false),
+				matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
+				matcher.StringMatcherForTesting(nil, newStringP("FOO.BAR"), nil, nil, nil, false),
 			},
 		},
 		{
 			desc: "suffix match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("i-aint-the-one"), nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("1::68"), nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP(".COM"), nil, nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, newStringP("i-aint-the-one"), nil, nil, false),
+				matcher.StringMatcherForTesting(nil, nil, newStringP("1::68"), nil, nil, false),
+				matcher.StringMatcherForTesting(nil, nil, newStringP(".COM"), nil, nil, false),
 			},
 		},
 		{
 			desc: "regex match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.examples\.com`), false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.examples\.com`), false),
+				matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
 			},
 		},
 		{
 			desc: "contains match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("i-aint-the-one"), nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("2001:db8:1:1::68"), nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("i-aint-the-one"), nil, false),
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("2001:db8:1:1::68"), nil, false),
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, false),
 			},
 		},
 	}
@@ -216,73 +216,73 @@ func TestMatchingSANExists_Success(t *testing.T) {
 
 	tests := []struct {
 		desc        string
-		sanMatchers []xdsinternal.StringMatcher
+		sanMatchers []matcher.StringMatcher
 	}{
 		{
 			desc: "no san matchers",
 		},
 		{
 			desc: "exact match dns wildcard",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(newStringP("abc.example.com"), nil, nil, nil, nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
+				matcher.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false),
+				matcher.StringMatcherForTesting(newStringP("abc.example.com"), nil, nil, nil, nil, false),
 			},
 		},
 		{
 			desc: "exact match ignore case",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(newStringP("FOOBAR@EXAMPLE.COM"), nil, nil, nil, nil, true),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(newStringP("FOOBAR@EXAMPLE.COM"), nil, nil, nil, nil, true),
 			},
 		},
 		{
 			desc: "prefix match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP(".co.in"), nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, newStringP("baz.test"), nil, nil, nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, newStringP(".co.in"), nil, nil, false),
+				matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false),
+				matcher.StringMatcherForTesting(nil, newStringP("baz.test"), nil, nil, nil, false),
 			},
 		},
 		{
 			desc: "prefix match ignore case",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, newStringP("BAZ.test"), nil, nil, nil, true),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, newStringP("BAZ.test"), nil, nil, nil, true),
 			},
 		},
 		{
 			desc: "suffix  match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("192.168.1.1"), nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("@test.com"), nil, nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
+				matcher.StringMatcherForTesting(nil, nil, newStringP("192.168.1.1"), nil, nil, false),
+				matcher.StringMatcherForTesting(nil, nil, newStringP("@test.com"), nil, nil, false),
 			},
 		},
 		{
 			desc: "suffix  match ignore case",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, newStringP("@test.COM"), nil, nil, true),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, newStringP("@test.COM"), nil, nil, true),
 			},
 		},
 		{
 			desc: "regex match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("https://github.com/grpc/grpc-java"), nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.test\.com`), false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("https://github.com/grpc/grpc-java"), nil, false),
+				matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false),
+				matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.test\.com`), false),
 			},
 		},
 		{
 			desc: "contains match",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("2001:68::db8"), nil, false),
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("192.0.0"), nil, false),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false),
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("2001:68::db8"), nil, false),
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("192.0.0"), nil, false),
 			},
 		},
 		{
 			desc: "contains match ignore case",
-			sanMatchers: []xdsinternal.StringMatcher{
-				xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, true),
+			sanMatchers: []matcher.StringMatcher{
+				matcher.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, true),
 			},
 		},
 	}
diff --git a/internal/xds/bootstrap.go b/internal/xds/bootstrap.go
index 97ec8e17208e..a3f80d8f2496 100644
--- a/internal/xds/bootstrap.go
+++ b/internal/xds/bootstrap.go
@@ -16,6 +16,8 @@
  *
  */
 
+// Package xds contains types that need to be shared between code under
+// google.golang.org/grpc/xds/... and the rest of gRPC.
 package xds
 
 import (
diff --git a/internal/xds/matcher/matcher_header.go b/internal/xds/matcher/matcher_header.go
new file mode 100644
index 000000000000..f9c0322179e8
--- /dev/null
+++ b/internal/xds/matcher/matcher_header.go
@@ -0,0 +1,224 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package matcher
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"google.golang.org/grpc/metadata"
+)
+
+// HeaderMatcherInterface is an interface for header matchers. These are
+// documented in (EnvoyProxy link here?). These matchers will match on different
+// aspects of HTTP header name/value pairs.
+type HeaderMatcherInterface interface {
+	Match(metadata.MD) bool
+	String() string
+}
+
+// mdValuesFromOutgoingCtx retrieves metadata from context. If there are
+// multiple values, the values are concatenated with "," (comma and no space).
+//
+// All header matchers only match against the comma-concatenated string.
+func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) {
+	vs, ok := md[key]
+	if !ok {
+		return "", false
+	}
+	return strings.Join(vs, ","), true
+}
+
+// HeaderExactMatcher matches on an exact match of the value of the header.
+type HeaderExactMatcher struct {
+	key   string
+	exact string
+}
+
+// NewHeaderExactMatcher returns a new HeaderExactMatcher.
+func NewHeaderExactMatcher(key, exact string) *HeaderExactMatcher {
+	return &HeaderExactMatcher{key: key, exact: exact}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// HeaderExactMatcher.
+func (hem *HeaderExactMatcher) Match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hem.key)
+	if !ok {
+		return false
+	}
+	return v == hem.exact
+}
+
+func (hem *HeaderExactMatcher) String() string {
+	return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact)
+}
+
+// HeaderRegexMatcher matches on whether the entire request header value matches
+// the regex.
+type HeaderRegexMatcher struct {
+	key string
+	re  *regexp.Regexp
+}
+
+// NewHeaderRegexMatcher returns a new HeaderRegexMatcher.
+func NewHeaderRegexMatcher(key string, re *regexp.Regexp) *HeaderRegexMatcher {
+	return &HeaderRegexMatcher{key: key, re: re}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// HeaderRegexMatcher.
+func (hrm *HeaderRegexMatcher) Match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
+	if !ok {
+		return false
+	}
+	return hrm.re.MatchString(v)
+}
+
+func (hrm *HeaderRegexMatcher) String() string {
+	return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String())
+}
+
+// HeaderRangeMatcher matches on whether the request header value is within the
+// range. The header value must be an integer in base 10 notation.
+type HeaderRangeMatcher struct {
+	key        string
+	start, end int64 // represents [start, end).
+}
+
+// NewHeaderRangeMatcher returns a new HeaderRangeMatcher.
+func NewHeaderRangeMatcher(key string, start, end int64) *HeaderRangeMatcher {
+	return &HeaderRangeMatcher{key: key, start: start, end: end}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// HeaderRangeMatcher.
+func (hrm *HeaderRangeMatcher) Match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
+	if !ok {
+		return false
+	}
+	if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end {
+		return true
+	}
+	return false
+}
+
+func (hrm *HeaderRangeMatcher) String() string {
+	return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end)
+}
+
+// HeaderPresentMatcher will match based on whether the header is present in the
+// whole request.
+type HeaderPresentMatcher struct {
+	key     string
+	present bool
+}
+
+// NewHeaderPresentMatcher returns a new HeaderPresentMatcher.
+func NewHeaderPresentMatcher(key string, present bool) *HeaderPresentMatcher {
+	return &HeaderPresentMatcher{key: key, present: present}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// HeaderPresentMatcher.
+func (hpm *HeaderPresentMatcher) Match(md metadata.MD) bool {
+	vs, ok := mdValuesFromOutgoingCtx(md, hpm.key)
+	present := ok && len(vs) > 0
+	return present == hpm.present
+}
+
+func (hpm *HeaderPresentMatcher) String() string {
+	return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present)
+}
+
+// HeaderPrefixMatcher matches on whether the prefix of the header value matches
+// the prefix passed into this struct.
+type HeaderPrefixMatcher struct {
+	key    string
+	prefix string
+}
+
+// NewHeaderPrefixMatcher returns a new HeaderPrefixMatcher.
+func NewHeaderPrefixMatcher(key string, prefix string) *HeaderPrefixMatcher {
+	return &HeaderPrefixMatcher{key: key, prefix: prefix}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// HeaderPrefixMatcher.
+func (hpm *HeaderPrefixMatcher) Match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hpm.key)
+	if !ok {
+		return false
+	}
+	return strings.HasPrefix(v, hpm.prefix)
+}
+
+func (hpm *HeaderPrefixMatcher) String() string {
+	return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix)
+}
+
+// HeaderSuffixMatcher matches on whether the suffix of the header value matches
+// the suffix passed into this struct.
+type HeaderSuffixMatcher struct {
+	key    string
+	suffix string
+}
+
+// NewHeaderSuffixMatcher returns a new HeaderSuffixMatcher.
+func NewHeaderSuffixMatcher(key string, suffix string) *HeaderSuffixMatcher {
+	return &HeaderSuffixMatcher{key: key, suffix: suffix}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// HeaderSuffixMatcher.
+func (hsm *HeaderSuffixMatcher) Match(md metadata.MD) bool {
+	v, ok := mdValuesFromOutgoingCtx(md, hsm.key)
+	if !ok {
+		return false
+	}
+	return strings.HasSuffix(v, hsm.suffix)
+}
+
+func (hsm *HeaderSuffixMatcher) String() string {
+	return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix)
+}
+
+// InvertMatcher inverts the match result of the underlying header matcher.
+type InvertMatcher struct {
+	m HeaderMatcherInterface
+}
+
+// NewInvertMatcher returns a new InvertMatcher.
+func NewInvertMatcher(m HeaderMatcherInterface) *InvertMatcher {
+	return &InvertMatcher{m: m}
+}
+
+// Match returns whether the passed in HTTP Headers match according to the
+// InvertMatcher.
+func (i *InvertMatcher) Match(md metadata.MD) bool {
+	return !i.m.Match(md)
+}
+
+func (i *InvertMatcher) String() string {
+	return fmt.Sprintf("invert{%s}", i.m)
+}
diff --git a/xds/internal/resolver/matcher_header_test.go b/internal/xds/matcher/matcher_header_test.go
similarity index 88%
rename from xds/internal/resolver/matcher_header_test.go
rename to internal/xds/matcher/matcher_header_test.go
index c83c3ec3923c..911e7bcfaca1 100644
--- a/xds/internal/resolver/matcher_header_test.go
+++ b/internal/xds/matcher/matcher_header_test.go
@@ -18,7 +18,7 @@
  *
  */
 
-package resolver
+package matcher
 
 import (
 	"regexp"
@@ -66,8 +66,8 @@ func TestHeaderExactMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hem := newHeaderExactMatcher(tt.key, tt.exact)
-			if got := hem.match(tt.md); got != tt.want {
+			hem := NewHeaderExactMatcher(tt.key, tt.exact)
+			if got := hem.Match(tt.md); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}
 		})
@@ -112,8 +112,8 @@ func TestHeaderRegexMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hrm := newHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr))
-			if got := hrm.match(tt.md); got != tt.want {
+			hrm := NewHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr))
+			if got := hrm.Match(tt.md); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}
 		})
@@ -159,8 +159,8 @@ func TestHeaderRangeMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hrm := newHeaderRangeMatcher(tt.key, tt.start, tt.end)
-			if got := hrm.match(tt.md); got != tt.want {
+			hrm := NewHeaderRangeMatcher(tt.key, tt.start, tt.end)
+			if got := hrm.Match(tt.md); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}
 		})
@@ -206,8 +206,8 @@ func TestHeaderPresentMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hpm := newHeaderPresentMatcher(tt.key, tt.present)
-			if got := hpm.match(tt.md); got != tt.want {
+			hpm := NewHeaderPresentMatcher(tt.key, tt.present)
+			if got := hpm.Match(tt.md); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}
 		})
@@ -252,8 +252,8 @@ func TestHeaderPrefixMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hpm := newHeaderPrefixMatcher(tt.key, tt.prefix)
-			if got := hpm.match(tt.md); got != tt.want {
+			hpm := NewHeaderPrefixMatcher(tt.key, tt.prefix)
+			if got := hpm.Match(tt.md); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}
 		})
@@ -298,8 +298,8 @@ func TestHeaderSuffixMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hsm := newHeaderSuffixMatcher(tt.key, tt.suffix)
-			if got := hsm.match(tt.md); got != tt.want {
+			hsm := NewHeaderSuffixMatcher(tt.key, tt.suffix)
+			if got := hsm.Match(tt.md); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}
 		})
@@ -309,24 +309,24 @@ func TestHeaderSuffixMatcherMatch(t *testing.T) {
 func TestInvertMatcherMatch(t *testing.T) {
 	tests := []struct {
 		name string
-		m    headerMatcherInterface
+		m    HeaderMatcherInterface
 		md   metadata.MD
 	}{
 		{
 			name: "true->false",
-			m:    newHeaderExactMatcher("th", "tv"),
+			m:    NewHeaderExactMatcher("th", "tv"),
 			md:   metadata.Pairs("th", "tv"),
 		},
 		{
 			name: "false->true",
-			m:    newHeaderExactMatcher("th", "abc"),
+			m:    NewHeaderExactMatcher("th", "abc"),
 			md:   metadata.Pairs("th", "tv"),
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := newInvertMatcher(tt.m).match(tt.md)
-			want := !tt.m.match(tt.md)
+			got := NewInvertMatcher(tt.m).Match(tt.md)
+			want := !tt.m.Match(tt.md)
 			if got != want {
 				t.Errorf("match() = %v, want %v", got, want)
 			}
diff --git a/internal/xds/string_matcher.go b/internal/xds/matcher/string_matcher.go
similarity index 98%
rename from internal/xds/string_matcher.go
rename to internal/xds/matcher/string_matcher.go
index 21f15aad1b88..d7df6a1e2b40 100644
--- a/internal/xds/string_matcher.go
+++ b/internal/xds/matcher/string_matcher.go
@@ -16,9 +16,9 @@
  *
  */
 
-// Package xds contains types that need to be shared between code under
+// Package matcher contains types that need to be shared between code under
 // google.golang.org/grpc/xds/... and the rest of gRPC.
-package xds
+package matcher
 
 import (
 	"errors"
diff --git a/internal/xds/string_matcher_test.go b/internal/xds/matcher/string_matcher_test.go
similarity index 99%
rename from internal/xds/string_matcher_test.go
rename to internal/xds/matcher/string_matcher_test.go
index 7908ac974b23..b634aa041963 100644
--- a/internal/xds/string_matcher_test.go
+++ b/internal/xds/matcher/string_matcher_test.go
@@ -16,7 +16,7 @@
  *
  */
 
-package xds
+package matcher
 
 import (
 	"regexp"
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
index 30c0d9105ed3..52b1a05f1362 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go
@@ -34,7 +34,7 @@ import (
 	"google.golang.org/grpc/internal"
 	xdscredsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/testutils"
-	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/resolver"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 	"google.golang.org/grpc/xds/internal/client/bootstrap"
@@ -50,12 +50,12 @@ const (
 )
 
 var (
-	testSANMatchers = []xdsinternal.StringMatcher{
-		xdsinternal.StringMatcherForTesting(newStringP(testSAN), nil, nil, nil, nil, true),
-		xdsinternal.StringMatcherForTesting(nil, newStringP(testSAN), nil, nil, nil, false),
-		xdsinternal.StringMatcherForTesting(nil, nil, newStringP(testSAN), nil, nil, false),
-		xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(testSAN), false),
-		xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP(testSAN), nil, false),
+	testSANMatchers = []matcher.StringMatcher{
+		matcher.StringMatcherForTesting(newStringP(testSAN), nil, nil, nil, nil, true),
+		matcher.StringMatcherForTesting(nil, newStringP(testSAN), nil, nil, nil, false),
+		matcher.StringMatcherForTesting(nil, nil, newStringP(testSAN), nil, nil, false),
+		matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(testSAN), false),
+		matcher.StringMatcherForTesting(nil, nil, nil, newStringP(testSAN), nil, false),
 	}
 	fpb1, fpb2                   *fakeProviderBuilder
 	bootstrapConfig              *bootstrap.Config
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index bb526116bda6..7d8cf6a8670b 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -35,8 +35,8 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/internal/testutils"
-	xdsinternal "google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/internal/xds/env"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/xds/internal/version"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 )
@@ -764,12 +764,12 @@ func (s) TestValidateClusterWithSecurityConfig(t *testing.T) {
 					RootCertName:         rootCertName,
 					IdentityInstanceName: identityPluginInstance,
 					IdentityCertName:     identityCertName,
-					SubjectAltNameMatchers: []xdsinternal.StringMatcher{
-						xdsinternal.StringMatcherForTesting(newStringP(sanExact), nil, nil, nil, nil, true),
-						xdsinternal.StringMatcherForTesting(nil, newStringP(sanPrefix), nil, nil, nil, false),
-						xdsinternal.StringMatcherForTesting(nil, nil, newStringP(sanSuffix), nil, nil, false),
-						xdsinternal.StringMatcherForTesting(nil, nil, nil, nil, sanRE, false),
-						xdsinternal.StringMatcherForTesting(nil, nil, nil, newStringP(sanContains), nil, false),
+					SubjectAltNameMatchers: []matcher.StringMatcher{
+						matcher.StringMatcherForTesting(newStringP(sanExact), nil, nil, nil, nil, true),
+						matcher.StringMatcherForTesting(nil, newStringP(sanPrefix), nil, nil, nil, false),
+						matcher.StringMatcherForTesting(nil, nil, newStringP(sanSuffix), nil, nil, false),
+						matcher.StringMatcherForTesting(nil, nil, nil, nil, sanRE, false),
+						matcher.StringMatcherForTesting(nil, nil, nil, newStringP(sanContains), nil, false),
 					},
 				},
 			},
diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go
index 05093653c075..8888e6214297 100644
--- a/xds/internal/client/client.go
+++ b/xds/internal/client/client.go
@@ -33,7 +33,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/xds/internal/client/load"
 	"google.golang.org/grpc/xds/internal/httpfilter"
 
@@ -351,7 +351,7 @@ type SecurityConfig struct {
 	// - If the peer certificate contains a wildcard DNS SAN, and an `exact`
 	//   matcher is configured, a wildcard DNS match is performed instead of a
 	//   regular string comparison.
-	SubjectAltNameMatchers []xds.StringMatcher
+	SubjectAltNameMatchers []matcher.StringMatcher
 	// RequireClientCert indicates if the server handshake process expects the
 	// client to present a certificate. Set to true when performing mTLS. Used
 	// only on the server-side.
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index 781eeb47a062..e9fe205a7ce2 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -40,10 +40,10 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
 	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/internal/xds"
 	"google.golang.org/grpc/internal/xds/env"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/httpfilter"
@@ -689,10 +689,10 @@ func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext) (*Secu
 	switch t := common.GetValidationContextType().(type) {
 	case *v3tlspb.CommonTlsContext_CombinedValidationContext:
 		combined := common.GetCombinedValidationContext()
-		var matchers []xds.StringMatcher
+		var matchers []matcher.StringMatcher
 		if def := combined.GetDefaultValidationContext(); def != nil {
 			for _, m := range def.GetMatchSubjectAltNames() {
-				matcher, err := xds.StringMatcherFromProto(m)
+				matcher, err := matcher.StringMatcherFromProto(m)
 				if err != nil {
 					return nil, err
 				}
diff --git a/xds/internal/resolver/matcher.go b/xds/internal/resolver/matcher.go
index 06456a585573..e329944e1db1 100644
--- a/xds/internal/resolver/matcher.go
+++ b/xds/internal/resolver/matcher.go
@@ -25,6 +25,7 @@ import (
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpcutil"
 	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/metadata"
 	xdsclient "google.golang.org/grpc/xds/internal/client"
 )
@@ -42,27 +43,27 @@ func routeToMatcher(r *xdsclient.Route) (*compositeMatcher, error) {
 		return nil, fmt.Errorf("illegal route: missing path_matcher")
 	}
 
-	var headerMatchers []headerMatcherInterface
+	var headerMatchers []matcher.HeaderMatcherInterface
 	for _, h := range r.Headers {
-		var matcherT headerMatcherInterface
+		var matcherT matcher.HeaderMatcherInterface
 		switch {
 		case h.ExactMatch != nil && *h.ExactMatch != "":
-			matcherT = newHeaderExactMatcher(h.Name, *h.ExactMatch)
+			matcherT = matcher.NewHeaderExactMatcher(h.Name, *h.ExactMatch)
 		case h.RegexMatch != nil:
-			matcherT = newHeaderRegexMatcher(h.Name, h.RegexMatch)
+			matcherT = matcher.NewHeaderRegexMatcher(h.Name, h.RegexMatch)
 		case h.PrefixMatch != nil && *h.PrefixMatch != "":
-			matcherT = newHeaderPrefixMatcher(h.Name, *h.PrefixMatch)
+			matcherT = matcher.NewHeaderPrefixMatcher(h.Name, *h.PrefixMatch)
 		case h.SuffixMatch != nil && *h.SuffixMatch != "":
-			matcherT = newHeaderSuffixMatcher(h.Name, *h.SuffixMatch)
+			matcherT = matcher.NewHeaderSuffixMatcher(h.Name, *h.SuffixMatch)
 		case h.RangeMatch != nil:
-			matcherT = newHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End)
+			matcherT = matcher.NewHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End)
 		case h.PresentMatch != nil:
-			matcherT = newHeaderPresentMatcher(h.Name, *h.PresentMatch)
+			matcherT = matcher.NewHeaderPresentMatcher(h.Name, *h.PresentMatch)
 		default:
 			return nil, fmt.Errorf("illegal route: missing header_match_specifier")
 		}
 		if h.InvertMatch != nil && *h.InvertMatch {
-			matcherT = newInvertMatcher(matcherT)
+			matcherT = matcher.NewInvertMatcher(matcherT)
 		}
 		headerMatchers = append(headerMatchers, matcherT)
 	}
@@ -77,11 +78,11 @@ func routeToMatcher(r *xdsclient.Route) (*compositeMatcher, error) {
 // compositeMatcher.match returns true if all matchers return true.
 type compositeMatcher struct {
 	pm  pathMatcherInterface
-	hms []headerMatcherInterface
+	hms []matcher.HeaderMatcherInterface
 	fm  *fractionMatcher
 }
 
-func newCompositeMatcher(pm pathMatcherInterface, hms []headerMatcherInterface, fm *fractionMatcher) *compositeMatcher {
+func newCompositeMatcher(pm pathMatcherInterface, hms []matcher.HeaderMatcherInterface, fm *fractionMatcher) *compositeMatcher {
 	return &compositeMatcher{pm: pm, hms: hms, fm: fm}
 }
 
@@ -107,7 +108,7 @@ func (a *compositeMatcher) match(info iresolver.RPCInfo) bool {
 		}
 	}
 	for _, m := range a.hms {
-		if !m.match(md) {
+		if !m.Match(md) {
 			return false
 		}
 	}
diff --git a/xds/internal/resolver/matcher_header.go b/xds/internal/resolver/matcher_header.go
deleted file mode 100644
index 05a92788d7bf..000000000000
--- a/xds/internal/resolver/matcher_header.go
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- *
- * Copyright 2020 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package resolver
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"google.golang.org/grpc/metadata"
-)
-
-type headerMatcherInterface interface {
-	match(metadata.MD) bool
-	String() string
-}
-
-// mdValuesFromOutgoingCtx retrieves metadata from context. If there are
-// multiple values, the values are concatenated with "," (comma and no space).
-//
-// All header matchers only match against the comma-concatenated string.
-func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) {
-	vs, ok := md[key]
-	if !ok {
-		return "", false
-	}
-	return strings.Join(vs, ","), true
-}
-
-type headerExactMatcher struct {
-	key   string
-	exact string
-}
-
-func newHeaderExactMatcher(key, exact string) *headerExactMatcher {
-	return &headerExactMatcher{key: key, exact: exact}
-}
-
-func (hem *headerExactMatcher) match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hem.key)
-	if !ok {
-		return false
-	}
-	return v == hem.exact
-}
-
-func (hem *headerExactMatcher) String() string {
-	return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact)
-}
-
-type headerRegexMatcher struct {
-	key string
-	re  *regexp.Regexp
-}
-
-func newHeaderRegexMatcher(key string, re *regexp.Regexp) *headerRegexMatcher {
-	return &headerRegexMatcher{key: key, re: re}
-}
-
-func (hrm *headerRegexMatcher) match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
-	if !ok {
-		return false
-	}
-	return hrm.re.MatchString(v)
-}
-
-func (hrm *headerRegexMatcher) String() string {
-	return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String())
-}
-
-type headerRangeMatcher struct {
-	key        string
-	start, end int64 // represents [start, end).
-}
-
-func newHeaderRangeMatcher(key string, start, end int64) *headerRangeMatcher {
-	return &headerRangeMatcher{key: key, start: start, end: end}
-}
-
-func (hrm *headerRangeMatcher) match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
-	if !ok {
-		return false
-	}
-	if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end {
-		return true
-	}
-	return false
-}
-
-func (hrm *headerRangeMatcher) String() string {
-	return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end)
-}
-
-type headerPresentMatcher struct {
-	key     string
-	present bool
-}
-
-func newHeaderPresentMatcher(key string, present bool) *headerPresentMatcher {
-	return &headerPresentMatcher{key: key, present: present}
-}
-
-func (hpm *headerPresentMatcher) match(md metadata.MD) bool {
-	vs, ok := mdValuesFromOutgoingCtx(md, hpm.key)
-	present := ok && len(vs) > 0
-	return present == hpm.present
-}
-
-func (hpm *headerPresentMatcher) String() string {
-	return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present)
-}
-
-type headerPrefixMatcher struct {
-	key    string
-	prefix string
-}
-
-func newHeaderPrefixMatcher(key string, prefix string) *headerPrefixMatcher {
-	return &headerPrefixMatcher{key: key, prefix: prefix}
-}
-
-func (hpm *headerPrefixMatcher) match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hpm.key)
-	if !ok {
-		return false
-	}
-	return strings.HasPrefix(v, hpm.prefix)
-}
-
-func (hpm *headerPrefixMatcher) String() string {
-	return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix)
-}
-
-type headerSuffixMatcher struct {
-	key    string
-	suffix string
-}
-
-func newHeaderSuffixMatcher(key string, suffix string) *headerSuffixMatcher {
-	return &headerSuffixMatcher{key: key, suffix: suffix}
-}
-
-func (hsm *headerSuffixMatcher) match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hsm.key)
-	if !ok {
-		return false
-	}
-	return strings.HasSuffix(v, hsm.suffix)
-}
-
-func (hsm *headerSuffixMatcher) String() string {
-	return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix)
-}
-
-type invertMatcher struct {
-	m headerMatcherInterface
-}
-
-func newInvertMatcher(m headerMatcherInterface) *invertMatcher {
-	return &invertMatcher{m: m}
-}
-
-func (i *invertMatcher) match(md metadata.MD) bool {
-	return !i.m.match(md)
-}
-
-func (i *invertMatcher) String() string {
-	return fmt.Sprintf("invert{%s}", i.m)
-}
diff --git a/xds/internal/resolver/matcher_test.go b/xds/internal/resolver/matcher_test.go
index 5c8dca5c9e5b..6f599b82da2a 100644
--- a/xds/internal/resolver/matcher_test.go
+++ b/xds/internal/resolver/matcher_test.go
@@ -27,6 +27,7 @@ import (
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/internal/grpcutil"
 	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -34,14 +35,14 @@ func TestAndMatcherMatch(t *testing.T) {
 	tests := []struct {
 		name string
 		pm   pathMatcherInterface
-		hm   headerMatcherInterface
+		hm   matcher.HeaderMatcherInterface
 		info iresolver.RPCInfo
 		want bool
 	}{
 		{
 			name: "both match",
 			pm:   newPathExactMatcher("/a/b", false),
-			hm:   newHeaderExactMatcher("th", "tv"),
+			hm:   matcher.NewHeaderExactMatcher("th", "tv"),
 			info: iresolver.RPCInfo{
 				Method:  "/a/b",
 				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
@@ -51,7 +52,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		{
 			name: "both match with path case insensitive",
 			pm:   newPathExactMatcher("/A/B", true),
-			hm:   newHeaderExactMatcher("th", "tv"),
+			hm:   matcher.NewHeaderExactMatcher("th", "tv"),
 			info: iresolver.RPCInfo{
 				Method:  "/a/b",
 				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
@@ -61,7 +62,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		{
 			name: "only one match",
 			pm:   newPathExactMatcher("/a/b", false),
-			hm:   newHeaderExactMatcher("th", "tv"),
+			hm:   matcher.NewHeaderExactMatcher("th", "tv"),
 			info: iresolver.RPCInfo{
 				Method:  "/z/y",
 				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
@@ -71,7 +72,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		{
 			name: "both not match",
 			pm:   newPathExactMatcher("/z/y", false),
-			hm:   newHeaderExactMatcher("th", "abc"),
+			hm:   matcher.NewHeaderExactMatcher("th", "abc"),
 			info: iresolver.RPCInfo{
 				Method:  "/a/b",
 				Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")),
@@ -81,7 +82,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		{
 			name: "fake header",
 			pm:   newPathPrefixMatcher("/", false),
-			hm:   newHeaderExactMatcher("content-type", "fake"),
+			hm:   matcher.NewHeaderExactMatcher("content-type", "fake"),
 			info: iresolver.RPCInfo{
 				Method: "/a/b",
 				Context: grpcutil.WithExtraMetadata(context.Background(), metadata.Pairs(
@@ -93,7 +94,7 @@ func TestAndMatcherMatch(t *testing.T) {
 		{
 			name: "binary header",
 			pm:   newPathPrefixMatcher("/", false),
-			hm:   newHeaderPresentMatcher("t-bin", true),
+			hm:   matcher.NewHeaderPresentMatcher("t-bin", true),
 			info: iresolver.RPCInfo{
 				Method: "/a/b",
 				Context: grpcutil.WithExtraMetadata(
@@ -107,7 +108,7 @@ func TestAndMatcherMatch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			a := newCompositeMatcher(tt.pm, []headerMatcherInterface{tt.hm}, nil)
+			a := newCompositeMatcher(tt.pm, []matcher.HeaderMatcherInterface{tt.hm}, nil)
 			if got := a.match(tt.info); got != tt.want {
 				t.Errorf("match() = %v, want %v", got, tt.want)
 			}

From 50c071e9b5431dcb90be089c7159efc63edff4cb Mon Sep 17 00:00:00 2001
From: Zeke Lu <lvzecai@gmail.com>
Date: Sat, 15 May 2021 05:09:26 +0800
Subject: [PATCH 480/481] example: correct the default value for
 server_host_override (#4407)

---
 examples/features/encryption/README.md | 4 ++--
 examples/route_guide/client/client.go  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/features/encryption/README.md b/examples/features/encryption/README.md
index a00188d66a2d..2afca1d785f5 100644
--- a/examples/features/encryption/README.md
+++ b/examples/features/encryption/README.md
@@ -42,8 +42,8 @@ configure TLS and create the server credential using
 On client side, we provide the path to the "ca_cert.pem" to configure TLS and create
 the client credential using
 [`credentials.NewClientTLSFromFile`](https://godoc.org/google.golang.org/grpc/credentials#NewClientTLSFromFile).
-Note that we override the server name with "x.test.youtube.com", as the server
-certificate is valid for *.test.youtube.com but not localhost. It is solely for
+Note that we override the server name with "x.test.example.com", as the server
+certificate is valid for *.test.example.com but not localhost. It is solely for
 the convenience of making an example.
 
 Once the credentials have been created at both sides, we can start the server
diff --git a/examples/route_guide/client/client.go b/examples/route_guide/client/client.go
index 172f10fb308b..f18c10af8b14 100644
--- a/examples/route_guide/client/client.go
+++ b/examples/route_guide/client/client.go
@@ -40,7 +40,7 @@ var (
 	tls                = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP")
 	caFile             = flag.String("ca_file", "", "The file containing the CA root cert file")
 	serverAddr         = flag.String("server_addr", "localhost:10000", "The server address in the format of host:port")
-	serverHostOverride = flag.String("server_host_override", "x.test.youtube.com", "The server name used to verify the hostname returned by the TLS handshake")
+	serverHostOverride = flag.String("server_host_override", "x.test.example.com", "The server name used to verify the hostname returned by the TLS handshake")
 )
 
 // printFeature gets the feature for the given point.

From a12250e98f973530f34191d39f840ae435f00a91 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 14 May 2021 15:20:45 -0700
Subject: [PATCH 481/481] xds/cds: add env var for aggregated and DNS cluster
 (#4440)

---
 internal/xds/env/env.go         | 20 +++++++++++++++-----
 xds/internal/client/cds_test.go |  6 ++++++
 xds/internal/client/xds.go      |  8 +++++++-
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/internal/xds/env/env.go b/internal/xds/env/env.go
index 1110722a630b..db9ac93b968c 100644
--- a/internal/xds/env/env.go
+++ b/internal/xds/env/env.go
@@ -37,11 +37,13 @@ const (
 	// and kept in variable BootstrapFileName.
 	//
 	// When both bootstrap FileName and FileContent are set, FileName is used.
-	BootstrapFileContentEnv      = "GRPC_XDS_BOOTSTRAP_CONFIG"
+	BootstrapFileContentEnv = "GRPC_XDS_BOOTSTRAP_CONFIG"
+
 	circuitBreakingSupportEnv    = "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING"
 	timeoutSupportEnv            = "GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"
 	faultInjectionSupportEnv     = "GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION"
 	clientSideSecuritySupportEnv = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"
+	aggregateAndDNSSupportEnv    = "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER"
 
 	c2pResolverSupportEnv                    = "GRPC_EXPERIMENTAL_GOOGLE_C2P_RESOLVER"
 	c2pResolverTestOnlyTrafficDirectorURIEnv = "GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI"
@@ -60,6 +62,7 @@ var (
 	//
 	// When both bootstrap FileName and FileContent are set, FileName is used.
 	BootstrapFileContent = os.Getenv(BootstrapFileContentEnv)
+
 	// CircuitBreakingSupport indicates whether circuit breaking support is
 	// enabled, which can be disabled by setting the environment variable
 	// "GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING" to "false".
@@ -71,10 +74,6 @@ var (
 	// FaultInjectionSupport is used to control both fault injection and HTTP
 	// filter support.
 	FaultInjectionSupport = !strings.EqualFold(os.Getenv(faultInjectionSupportEnv), "false")
-	// C2PResolverSupport indicates whether support for C2P resolver is enabled.
-	// This can be enabled by setting the environment variable
-	// "GRPC_EXPERIMENTAL_GOOGLE_C2P_RESOLVER" to "true".
-	C2PResolverSupport = strings.EqualFold(os.Getenv(c2pResolverSupportEnv), "true")
 	// ClientSideSecuritySupport is used to control processing of security
 	// configuration on the client-side.
 	//
@@ -82,6 +81,17 @@ var (
 	// have a brand new API on the server-side and users explicitly need to use
 	// the new API to get security integration on the server.
 	ClientSideSecuritySupport = strings.EqualFold(os.Getenv(clientSideSecuritySupportEnv), "true")
+	// AggregateAndDNSSupportEnv indicates whether processing of aggregated
+	// cluster and DNS cluster is enabled, which can be enabled by setting the
+	// environment variable
+	// "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER" to
+	// "true".
+	AggregateAndDNSSupportEnv = strings.EqualFold(os.Getenv(aggregateAndDNSSupportEnv), "true")
+
+	// C2PResolverSupport indicates whether support for C2P resolver is enabled.
+	// This can be enabled by setting the environment variable
+	// "GRPC_EXPERIMENTAL_GOOGLE_C2P_RESOLVER" to "true".
+	C2PResolverSupport = strings.EqualFold(os.Getenv(c2pResolverSupportEnv), "true")
 	// C2PResolverTestOnlyTrafficDirectorURI is the TD URI for testing.
 	C2PResolverTestOnlyTrafficDirectorURI = os.Getenv(c2pResolverTestOnlyTrafficDirectorURIEnv)
 )
diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go
index 7d8cf6a8670b..5209105b430a 100644
--- a/xds/internal/client/cds_test.go
+++ b/xds/internal/client/cds_test.go
@@ -124,6 +124,9 @@ func (s) TestValidateCluster_Failure(t *testing.T) {
 		},
 	}
 
+	oldAggregateAndDNSSupportEnv := env.AggregateAndDNSSupportEnv
+	env.AggregateAndDNSSupportEnv = true
+	defer func() { env.CircuitBreakingSupport = oldAggregateAndDNSSupportEnv }()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			if update, err := validateClusterAndConstructClusterUpdate(test.cluster); err == nil {
@@ -261,6 +264,9 @@ func (s) TestValidateCluster_Success(t *testing.T) {
 	origCircuitBreakingSupport := env.CircuitBreakingSupport
 	env.CircuitBreakingSupport = true
 	defer func() { env.CircuitBreakingSupport = origCircuitBreakingSupport }()
+	oldAggregateAndDNSSupportEnv := env.AggregateAndDNSSupportEnv
+	env.AggregateAndDNSSupportEnv = true
+	defer func() { env.CircuitBreakingSupport = oldAggregateAndDNSSupportEnv }()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			update, err := validateClusterAndConstructClusterUpdate(test.cluster)
diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go
index e9fe205a7ce2..7b4f4048ea58 100644
--- a/xds/internal/client/xds.go
+++ b/xds/internal/client/xds.go
@@ -615,9 +615,15 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu
 		ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName()
 		return ret, nil
 	case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS:
+		if !env.AggregateAndDNSSupportEnv {
+			return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+		}
 		ret.ClusterType = ClusterTypeLogicalDNS
 		return ret, nil
 	case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate":
+		if !env.AggregateAndDNSSupportEnv {
+			return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+		}
 		clusters := &v3aggregateclusterpb.ClusterConfig{}
 		if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
 			return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
@@ -626,7 +632,7 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu
 		ret.PrioritizedClusterNames = clusters.Clusters
 		return ret, nil
 	default:
-		return ClusterUpdate{}, fmt.Errorf("unexpected cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
+		return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
 	}
 }